diff --git a/.cargo/config.toml b/.cargo/config.toml index 847ca2408..5d9d7c734 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,4 +1,4 @@ [env] # Debug DataFusion/Lance projection uses more than the default 2MiB test-thread -# stack, which overflowed canonical event import on Linux CI. +# stack when importing trajectories on Linux CI. RUST_MIN_STACK = "8388608" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1e8804de5..699fc4fcc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,27 +33,8 @@ jobs: - name: Rust formatting run: just fmt-check-rust - rust-clippy-workspace: - name: Clippy / Workspace - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Setup Build Environment - uses: ./.github/actions/setup-build-env - with: - install-nextest: "false" - components: "clippy" - - - uses: Swatinem/rust-cache@v2 - with: - shared-key: ci-clippy-workspace - - - name: Workspace Clippy - run: just clippy-deny - - rust-clippy-pchronicle: - name: Clippy / pChronicle Feature Contracts + rust-clippy: + name: Clippy / pChronicle runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -68,8 +49,8 @@ jobs: with: shared-key: ci-clippy-pchronicle - - name: Strict pChronicle Clippy - run: just clippy-pchronicle-panics + - name: pChronicle and CLI Clippy + run: just clippy-deny - name: pChronicle storage feature contracts run: just clippy-pchronicle-features @@ -94,70 +75,13 @@ jobs: - name: pChronicle Web Clippy run: just clippy-pchronicle-web - python-lint: - name: Python Lint - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Setup Build Environment - uses: ./.github/actions/setup-build-env - with: - install-rust: "false" - install-nextest: "false" - components: "" - - - name: Ruff - run: just lint-py - rust-test: - name: Rust Test / ${{ matrix.platform }} / ${{ matrix.shard }} + name: Rust Test / ${{ matrix.os }} runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - include: - - platform: Linux - os: ubuntu-latest - shard: core - packages: >- - persisting-agentctl - persisting-events - persisting-overlaynet - event_control: true - smoke_pchronicle: false - - platform: Linux - os: ubuntu-latest - shard: gateway - packages: persisting-gateway - event_control: false - smoke_pchronicle: false - - platform: Linux - os: ubuntu-latest - shard: pchronicle - packages: >- - persisting-pchronicle - persisting-pchronicle-cli - event_control: false - smoke_pchronicle: true - - platform: macOS - os: macos-latest - shard: capture - packages: >- - persisting-agentctl - persisting-events - persisting-gateway - persisting-overlaynet - event_control: false - smoke_pchronicle: false - - platform: macOS - os: macos-latest - shard: pchronicle - packages: >- - persisting-pchronicle - persisting-pchronicle-cli - event_control: false - smoke_pchronicle: true + os: [ubuntu-latest, macos-latest] steps: - uses: actions/checkout@v4 @@ -168,17 +92,12 @@ jobs: - uses: Swatinem/rust-cache@v2 with: - shared-key: ci-rust-${{ matrix.shard }} - - - name: Run ${{ matrix.shard }} tests - run: just ci-nextest ${{ matrix.packages }} + shared-key: ci-rust-pchronicle - - name: Test shared event control contract - if: matrix.event_control - run: just test-events-control + - name: Run pChronicle and CLI tests + run: just test pchronicle - name: Smoke pChronicle CLI - if: matrix.smoke_pchronicle run: just smoke-pchronicle-cli rust-proptest: @@ -219,7 +138,10 @@ jobs: shared-key: ci-pchronicle-web workspaces: pchronicle-web -> target - - name: Build and test Web crate + - name: Build Web assets + run: just chronicle-web-build + + - name: Test Web crate run: just test-pchronicle-web s3-contract: @@ -312,8 +234,8 @@ jobs: kill "$(cat "$RUNNER_TEMP/minio.pid")" || true fi - python-test: - name: Python Test + python: + name: Python Lint and Test runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -328,6 +250,9 @@ jobs: - name: Sync Python deps run: just py-dev + - name: Ruff + run: just lint-py + - name: Pytest run: just test-py @@ -360,15 +285,13 @@ jobs: if: always() needs: - format - - rust-clippy-workspace - - rust-clippy-pchronicle + - rust-clippy - rust-clippy-pchronicle-web - - python-lint - rust-test - rust-proptest - pchronicle-web-test - s3-contract - - python-test + - python - examples runs-on: ubuntu-latest steps: diff --git a/.github/workflows/pchronicle-benchmark.yml b/.github/workflows/pchronicle-benchmark.yml index 08e20edbf..4f26dc949 100644 --- a/.github/workflows/pchronicle-benchmark.yml +++ b/.github/workflows/pchronicle-benchmark.yml @@ -91,34 +91,6 @@ jobs: --target-dir "${GITHUB_WORKSPACE}/target/pchronicle-benchmark-ci" \ --output "${RUNNER_TEMP}/pchronicle-benchmark/candidate" - - name: Upload benchmark data - if: always() - uses: actions/upload-artifact@v4 - with: - name: pchronicle-benchmark-part-${{ github.run_id }} - path: ${{ runner.temp }}/pchronicle-benchmark - retention-days: 30 - if-no-files-found: warn - - compare: - name: Criterion + hyperfine report - needs: benchmark - if: always() - runs-on: ubuntu-latest - timeout-minutes: 10 - steps: - - name: Checkout candidate - uses: actions/checkout@v4 - with: - fetch-depth: 1 - - - name: Download benchmark data - uses: actions/download-artifact@v4 - with: - pattern: pchronicle-benchmark-part-* - path: ${{ runner.temp }}/pchronicle-benchmark - merge-multiple: true - - name: Compare and render report page shell: bash run: | diff --git a/.gitignore b/.gitignore index 5ee29b208..758b6c7a2 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,8 @@ __pycache__/ crates/persisting-engine/ds/ # Local review, benchmark, and runtime artifacts +.pchronicle-import-wal/ +import.log /data/ /tmp/ diff --git a/AGENTS.md b/AGENTS.md index bf714448b..b78a14b94 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -26,9 +26,9 @@ This exclusion covers, in particular: workflows The default active scope is pChronicle, its trajectory CLI and Web UI, and -the Gateway, Control, events, and OverlayNet libraries it depends on. +the data import, storage, conversion, and query pipeline they depend on. `persisting-dlcapt` is a separate standalone component; excluding it -does not exclude Gateway trajectory capture or pChronicle capture storage. +does not exclude pChronicle trajectory import or storage. Enter an excluded subsystem only when: @@ -44,7 +44,7 @@ required. Do not use incidental cleanup as a reason to broaden scope. Use `just test` as the default validation command. It runs Rust tests through `cargo nextest` in debug mode (for faster iteration) and then runs the Python test suite. To limit Rust coverage to one package, pass its Cargo package name -positionally, for example `just test persisting-agentctl`. The `pchronicle` +positionally, for example `just test persisting-pchronicle-cli`. The `pchronicle` alias also runs `persisting-pchronicle-cli` (same as the CI pchronicle shard); use `just test pchronicle-cli` for the CLI crate alone. Use a direct `cargo test` invocation only when doctests or an explicitly documented special diff --git a/Cargo.lock b/Cargo.lock index 1a15eb697..75c753540 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -958,15 +958,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "bincode" -version = "1.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" -dependencies = [ - "serde", -] - [[package]] name = "bit-set" version = "0.8.0" @@ -2329,15 +2320,6 @@ dependencies = [ "sqlparser", ] -[[package]] -name = "defmt" -version = "0.3.100" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0963443817029b2024136fc4dd07a5107eb8f977eaf18fcd1fdeb11306b64ad" -dependencies = [ - "defmt 1.1.1", -] - [[package]] name = "defmt" version = "1.1.1" @@ -3016,7 +2998,7 @@ dependencies = [ "io-uring", "itoa", "libc", - "lru 0.18.4", + "lru", "memmap2", "moka", "prost", @@ -3093,8 +3075,6 @@ version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ - "allocator-api2", - "equivalent", "foldhash 0.1.5", ] @@ -3136,16 +3116,6 @@ dependencies = [ "stable_deref_trait", ] -[[package]] -name = "heapless" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25ba4bd83f9415b58b4ed8dc5714c76e626a105be4646c02630ad730ad3b5aa4" -dependencies = [ - "hash32", - "stable_deref_trait", -] - [[package]] name = "heck" version = "0.5.0" @@ -3757,7 +3727,7 @@ version = "0.2.35" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "668b7183bd07af9a4885f5c35b0cc5c83c4607a913c16b7e17291832910d2dcc" dependencies = [ - "defmt 1.1.1", + "defmt", "jiff-core", "jiff-static", "jiff-tzdb-platform", @@ -3776,7 +3746,7 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7feca88439efe53da3754500c1851dedf3cb36c524dd5cf8225cc0794de95d09" dependencies = [ - "defmt 1.1.1", + "defmt", ] [[package]] @@ -4689,15 +4659,6 @@ dependencies = [ "tracing-subscriber", ] -[[package]] -name = "lru" -version = "0.12.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" -dependencies = [ - "hashbrown 0.15.5", -] - [[package]] name = "lru" version = "0.18.4" @@ -4741,12 +4702,6 @@ dependencies = [ "twox-hash", ] -[[package]] -name = "managed" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ca88d725a0a943b096803bd34e73a4437208b6077654cc4ecb2947a5f91618d" - [[package]] name = "matchers" version = "0.2.0" @@ -5683,41 +5638,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" -[[package]] -name = "opentelemetry" -version = "0.27.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab70038c28ed37b97d8ed414b6429d343a8bbf44c9f79ec854f3a643029ba6d7" -dependencies = [ - "futures-core", - "futures-sink", - "js-sys", - "pin-project-lite", - "thiserror 1.0.69", - "tracing", -] - -[[package]] -name = "opentelemetry_sdk" -version = "0.27.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "231e9d6ceef9b0b2546ddf52335785ce41252bc7474ee8ba05bfad277be13ab8" -dependencies = [ - "async-trait", - "futures-channel", - "futures-executor", - "futures-util", - "glob", - "opentelemetry", - "percent-encoding", - "rand 0.8.6", - "serde_json", - "thiserror 1.0.69", - "tokio", - "tokio-stream", - "tracing", -] - [[package]] name = "option-ext" version = "0.2.0" @@ -5856,17 +5776,6 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df202b0b0f5b8e389955afd5f27b007b00fb948162953f1db9c70d2c7e3157d7" -[[package]] -name = "persisting-agentctl" -version = "0.3.0" -dependencies = [ - "anyhow", - "ipnet", - "proptest", - "serde", - "serde_json", -] - [[package]] name = "persisting-dlcapt" version = "0.3.0" @@ -5892,76 +5801,6 @@ dependencies = [ "tracing-subscriber", ] -[[package]] -name = "persisting-events" -version = "0.3.0" -dependencies = [ - "anyhow", - "async-trait", - "persisting-agentctl", - "proptest", - "serde", - "serde_json", - "thiserror 2.0.18", - "tokio", -] - -[[package]] -name = "persisting-gateway" -version = "0.3.0" -dependencies = [ - "anyhow", - "async-trait", - "axum", - "base64 0.22.1", - "blake3", - "bytes", - "chrono", - "dashmap", - "dirs", - "futures-util", - "hostname", - "http-body-util", - "persisting-agentctl", - "persisting-events", - "persisting-overlaynet", - "persisting-pchronicle", - "proptest", - "pulsing-actor", - "regex", - "reqwest 0.12.28", - "serde", - "serde_json", - "serde_yaml", - "tempfile", - "tokio", - "tokio-stream", - "toml", - "tracing", - "url", -] - -[[package]] -name = "persisting-overlaynet" -version = "0.3.0" -dependencies = [ - "anyhow", - "async-trait", - "axum", - "futures-util", - "http-body-util", - "hyper", - "hyper-util", - "persisting-agentctl", - "proptest", - "reqwest 0.12.28", - "serde", - "smoltcp", - "thiserror 2.0.18", - "tokio", - "tracing", -] - [[package]] name = "persisting-pchronicle" version = "0.3.0" @@ -5989,8 +5828,6 @@ dependencies = [ "libc", "object_store", "opendal 0.57.0", - "persisting-agentctl", - "persisting-events", "proptest", "serde", "serde_json", @@ -6023,9 +5860,6 @@ dependencies = [ "include_dir", "lance", "libc", - "persisting-agentctl", - "persisting-events", - "persisting-gateway", "persisting-pchronicle", "reqwest 0.12.28", "serde", @@ -6264,17 +6098,6 @@ dependencies = [ "syn", ] -[[package]] -name = "probing-memtable" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6b153e11fbce92a05d067c191f88b14d3162c15d0f90679619b38435465b548" -dependencies = [ - "libc", - "memmap2", - "xxhash-rust", -] - [[package]] name = "proc-macro-crate" version = "3.5.0" @@ -6410,38 +6233,6 @@ dependencies = [ "thiserror 2.0.18", ] -[[package]] -name = "pulsing-actor" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a751c49db41d362084cb011c77391c1b0e857d11f4e6a839f6452c722a090a6" -dependencies = [ - "anyhow", - "async-trait", - "bincode", - "bytes", - "dashmap", - "futures", - "http-body-util", - "hyper", - "hyper-util", - "lru 0.12.5", - "opentelemetry", - "opentelemetry_sdk", - "probing-memtable", - "rand 0.9.4", - "serde", - "serde_json", - "thiserror 2.0.18", - "tokio", - "tokio-stream", - "tokio-util", - "tracing", - "tracing-opentelemetry", - "tracing-subscriber", - "uuid", -] - [[package]] name = "quick-error" version = "1.2.3" @@ -6569,7 +6360,6 @@ version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" dependencies = [ - "libc", "rand_chacha 0.3.1", "rand_core 0.6.4", ] @@ -7089,7 +6879,7 @@ version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "421400d13ccfd26dfa5858199c30a5d76f9c54e0dba7575273025b43c5175dbb" dependencies = [ - "heapless 0.8.0", + "heapless", "num-traits", "smallvec", ] @@ -7624,21 +7414,6 @@ version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" -[[package]] -name = "smoltcp" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f73d40463bba65efc9adc6370b56df76d563cc46e2482bba58351b4afb7535e" -dependencies = [ - "bitflags 1.3.2", - "byteorder", - "cfg-if 1.0.4", - "defmt 0.3.100", - "heapless 0.9.3", - "log", - "managed", -] - [[package]] name = "snafu" version = "0.9.0" @@ -8080,7 +7855,6 @@ dependencies = [ "bytes", "futures-core", "futures-sink", - "futures-util", "pin-project-lite", "tokio", ] @@ -8306,24 +8080,6 @@ dependencies = [ "tracing-core", ] -[[package]] -name = "tracing-opentelemetry" -version = "0.28.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97a971f6058498b5c0f1affa23e7ea202057a7301dbff68e968b2d578bcbd053" -dependencies = [ - "js-sys", - "once_cell", - "opentelemetry", - "opentelemetry_sdk", - "smallvec", - "tracing", - "tracing-core", - "tracing-log", - "tracing-subscriber", - "web-time", -] - [[package]] name = "tracing-serde" version = "0.2.0" @@ -8489,7 +8245,6 @@ checksum = "ddd74a9687298c6858e9b88ec8935ec45d22e8fd5e6394fa1bd4e99a87789c76" dependencies = [ "getrandom 0.4.2", "js-sys", - "rand 0.10.1", "serde_core", "sha1_smol", "wasm-bindgen", diff --git a/Cargo.toml b/Cargo.toml index 1078f8258..2295487d5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,18 +3,14 @@ version = "0.3.0" edition = "2024" authors = ["reiase"] license = "Apache-2.0" -description = "Durable Agent trajectory capture, storage, and query infrastructure" +description = "Durable Agent trajectory processing, storage, and query infrastructure" [workspace] resolver = "3" members = [ - "crates/persisting-agentctl", - "crates/persisting-events", - "crates/persisting-gateway", "crates/persisting-dlcapt", "crates/persisting-pchronicle", "crates/persisting-pchronicle-cli", - "crates/persisting-overlaynet", ] default-members = [ @@ -30,17 +26,11 @@ blake3 = "1" bytes = "1" chrono = "0.4" clap = "4" -dashmap = "6" datafusion = { version = "54.1.0", default-features = false } dirs = "6" fs2 = "0.4" futures = "0.3" -futures-util = { version = "0.3", default-features = false } -hostname = "0.4" http-body-util = "0.1" -hyper = "1" -hyper-util = { version = "0.1", default-features = false } -ipnet = "2" lance = { version = "11.0.0", default-features = false } lance-arrow = "11.0.0" lance-core = "11.0.0" @@ -62,19 +52,13 @@ opendal = { version = "0.57.0", default-features = true, features = [ "services-tos", ] } object_store = "0.13.2" -persisting-agentctl = { path = "crates/persisting-agentctl" } -persisting-events = { path = "crates/persisting-events" } -persisting-gateway = { path = "crates/persisting-gateway" } -persisting-overlaynet = { path = "crates/persisting-overlaynet" } persisting-pchronicle = { path = "crates/persisting-pchronicle", default-features = false } -pulsing-actor = { version = "0.1.2", default-features = false } proptest = "1" regex = "1" reqwest = { version = "0.12", default-features = false } serde = "1" serde_json = "1" serde_yaml = "0.9" -smoltcp = { version = "0.13.1", default-features = false } tempfile = "3" thiserror = "2" tokio = { version = "1", default-features = false } @@ -129,7 +113,7 @@ codegen-units = 8 panic = "abort" [patch.crates-io] -# Lance 9.0.1 invokes prost-build from six build scripts. This compatibility +# Lance invokes prost-build from its build scripts. This compatibility # adapter preserves Prost's code generation API but parses .proto files with # pure-Rust protox, avoiding both a system protoc and vendored protobuf-src. prost-build = { path = "vendor/prost-build-protox" } diff --git a/NOTICE b/NOTICE index 92fc7ae05..3b1314fc5 100644 --- a/NOTICE +++ b/NOTICE @@ -9,14 +9,6 @@ Version 2.0. See the repository root `LICENSE` file. Third-party notices =================== -agentgateway test fixtures --------------------------- - -Regression fixtures under `crates/persisting-gateway/tests/fixtures/` may -include materials originally published under Apache-2.0 by the agentgateway -project. Those files remain under their upstream Apache-2.0 terms as noted in -`crates/persisting-gateway/tests/fixtures/README.md`. - dldb ---- diff --git a/README.md b/README.md index f074f7573..713410c88 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ Persisting logo -Persisting contains **pChronicle**, which captures, browses, queries, exchanges, +Persisting contains **pChronicle**, which imports, browses, queries, exchanges, and serves durable Agent trajectory Datasets. pChronicle @@ -41,16 +41,12 @@ The onboarding flow creates a temporary example Dataset—no source checkout required. `pchronicle import` accepts ATIF, ACTF, and OpenAI Messages; `pchronicle serve` starts a loopback-only, read-only Dataset UI and API. -Gateway capture can write trajectory events into pChronicle Datasets. See the -[Gateway guide](https://deeplink-org.github.io/Persisting/pchronicle/guides/serve-gateway/). - ## Current maturity | Capability | Status | |---|---| | pChronicle local/S3 catalog, bounded SQL, analysis, find, import/export | Implemented | | pChronicle loopback-only read API and embedded Web UI | Implemented | -| Gateway capture and cooperative proxy policy | Implemented | | Queue and document Search | Separate stable capabilities | | Tensor Memory / TTAS | Experimental | @@ -70,10 +66,6 @@ Latest nightly pChronicle benchmark: `71dcf3c4d39e` on `linux/x86_64` (2026-10-0 |---|---:|---:| | `criterion/atif_conversion/parse_corpus` | `latency_median_ns` | 5.199e+06 ns | | `criterion/atif_conversion/roundtrip_corpus` | `latency_median_ns` | 6.936e+06 ns | -| `criterion/projection_cpu/events_to_storyline_corpus` | `latency_median_ns` | 3.327e+05 ns | -| `system/projection_pipeline/event_append` | `initial_append_ms` | 64.546 ms | -| `system/projection_pipeline/projection_build` | `build_ms` | 5191.079 ms | -| `system/projection_pipeline/projection_incremental` | `sync_ms` | 41.371 ms | | `system/lance_vs_json/lifecycle` | `cold_query_ms` | 3007.575 ms | | `system/lance_vs_json/lifecycle` | `get_storyline_full_ms` | 9.927 ms | | `system/lance_vs_json/lifecycle` | `replace_storyline_ms` | 42.698 ms | @@ -83,7 +75,6 @@ Latest nightly pChronicle benchmark: `71dcf3c4d39e` on `linux/x86_64` (2026-10-0 | `system/json_streaming_ndjson/json_streaming` | `p95_ms` | 13.217 ms | | `system/json_streaming_ndjson/json_streaming` | `rows_s` | 3.021e+05 ops/s | | `system/json_streaming_ndjson/json_streaming` | `process_peak_rss_mib` | 47.5 MiB | -| `hyperfine/projection_pipeline` | `wall_median_seconds` | 5.407 s | | `hyperfine/lance_vs_json` | `wall_median_seconds` | 40.222 s | [Open the complete benchmark run](https://github.com/DeepLink-org/Persisting/actions/runs/36975082961). diff --git a/benchmark/README.md b/benchmark/README.md index aea55dedf..df9895323 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -1,25 +1,10 @@ # Persisting benchmarks -**仓库级黑盒与微基准入口：Gateway、pChronicle，以及独立的 Queue 吞吐压测。** +**仓库级黑盒与微基准入口：pChronicle，以及独立的 Queue 吞吐压测。** 拥有可复现的压测脚本与报告契约。不拥有被测组件的产品行为；Queue 子系统的语义以 Queue 文档为准，这里只保留既有压测入口。 -## Gateway - -Gateway 黑盒压测位于 [`gateway/`](gateway/README.md)，使用确定性的本地 Echo upstream， -同时测量转发、Typed LLM capture、WAL 和 Lance durable append： - -```bash -just benchmark-gateway - -# 回放 examples/data，生成可人工检查的 request/response/capture bundle -just benchmark-gateway-replay -``` - -结果包含吞吐、p50/p95/p99 延迟、Echo 直连基线，以及基于 canonical manifest 的 -持久化事件数量校验。 - ## pChronicle 统一 Criterion + hyperfine 套件，输出 raw JSON、Markdown、HTML 和 Bencher 投影： diff --git a/benchmark/gateway/.gitignore b/benchmark/gateway/.gitignore deleted file mode 100644 index fbca22537..000000000 --- a/benchmark/gateway/.gitignore +++ /dev/null @@ -1 +0,0 @@ -results/ diff --git a/benchmark/gateway/README.md b/benchmark/gateway/README.md deleted file mode 100644 index 3d85af107..000000000 --- a/benchmark/gateway/README.md +++ /dev/null @@ -1,120 +0,0 @@ -# Gateway benchmark - -**使用本地 `pchronicle echo` 测量完整 Gateway 路径：HTTP 转发、Typed LLM capture、WAL 与 Lance durable append。** - -拥有黑盒压测、并发 sweep 和 example-data replay。不访问外部模型服务，也不隐式 -`cargo build`；首次运行或代码变更后请先显式构建 release binary。 - -默认先压测 Echo 直连作为本机 HTTP 基线，再以相同并发压测 Gateway。Gateway 进程会优雅 -退出，脚本随后逐个读取 capture session 的 canonical manifest，确认 published event 总数 -等于成功请求数的两倍（请求和响应各一条），因此性能结果同时具备回归测试的持久化数量 -约束。之后再通过真实 SQL 查询核对 `llm.request` / `llm.response` 数量，确保后台小文件合并 -把 live fragment 数控制在可查询范围；manifest 校验本身不会扫描 Lance 历史版本。 - -增量合并只约束当前快照引用的 live fragment。为保护并发 snapshot reader，历史 Lance -version/file 不会在写入路径立即删除；磁盘回收仍由显式 maintenance/vacuum 按保留期执行。 -在线布局采用 leveled compaction：8 个微小 fragment 形成一个 L0 segment，连续 8 个同层 -sealed segment 自动晋升到下一层。因此 visible segment 数随数据量近似对数增长，而不是 -持续按事件数线性增长；JSON 结果中的 `max_segment_level` 可用于观察是否发生层级晋升。 - -## Run - -从仓库根目录运行： - -```bash -cargo build --release --locked -p persisting-pchronicle-cli --bin pchronicle -just benchmark-gateway - -# 最多 30 秒、32 并发、1 KiB payload、2048 个请求 -just benchmark-gateway 30 32 1024 0 16 2048 - -# 按并发度扫描吞吐峰值和饱和点（默认 1/2/4/8/16/32） -just benchmark-gateway-sweep -``` - -也可以使用已有 release binary 直接运行： - -```bash -bash benchmark/gateway/run.sh \ - --duration 10 \ - --warmup 1 \ - --concurrency 16 \ - --sessions 16 \ - --requests 1024 \ - --payload-bytes 256 -``` - -可通过 `PERSISTING_PCHRONICLE_BIN` 指定 binary。常用选项： - -- `--skip-baseline`：只测 Gateway。 -- `--sessions N`：固定 capture session 数，默认 16；它与并发度独立。 -- `--requests N`：每个测量阶段的请求上限，默认 1024；`duration` 仍是时间上限。 -- `--min-rps N`：设置机器相关的最低吞吐门槛；默认只检查请求和持久化正确性。 -- `--output FILE`：指定 JSON 结果文件。 -- `--keep-artifacts`：保留配置、日志、WAL 和 Dataset。 - -通过 `just` 运行时，可以使用与 regression 一致的环境变量保留原始产物： - -```bash -PERSISTING_KEEP_TEST_ARTIFACTS=1 just benchmark-gateway -``` - -命令结束时会打印 `Gateway benchmark artifacts: <目录>`。其中 `dataset/` 是完整的 -Lance capture Dataset，`gateway-state/` 包含 WAL，`logs/` 包含进程日志和聚合后的 -`capture-counts.jsonl`。例如，把完整事件导出成人工可读的 JSONL： - -```bash -benchmark_artifacts=/tmp/persisting-gateway-benchmark.example -target/release/pchronicle query "$benchmark_artifacts/dataset" \ - 'SELECT seq, kind, session_id, model, call_id, payload_json FROM dataset.events ORDER BY session_id, seq' \ - --format jsonl \ - --output "$benchmark_artifacts/logs/events.jsonl" -``` - -压测器是 closed-loop 模型：每个 worker 最多只有一个在途请求，所以 -`concurrency` 就是最大在途请求数。JSON 中的 `estimated_mean_in_flight` 用吞吐和 -平均延迟按 Little's law 估算实际在途数；它接近 `max_in_flight` 时，表示发压端已经 -把该并发窗口持续占满。判断服务饱和点应使用 sweep：吞吐不再上升、延迟显著上升的第一个 -并发档位，而不是只观察一个高并发数字。sweep 会在所有档位保持 `sessions` -不变，避免把「在途请求数」和「Lance Dataset 数」两个变量混在一起。 -默认使用有限 request budget：请求阶段的 RPS/延迟表示 Gateway 转发能力，随后的 -优雅退出会等待 capture drain，并仍严格校验每个请求的 request/response 两条事件都已 -持久化。这避免用无限 closed-loop 负载必然打满 best-effort capture queue，却把排队时间误认为 -HTTP 转发延迟。 -JSON 中的 `shutdown_drain_seconds` 是停止发送后等待 apply、Lance publication、合并与 -优雅退出的时间；`events_per_drain_second` 单独表示这次 burst 的 durable capture drain 能力。 - -默认结果写入 `benchmark/gateway/results/latest.json`，该目录不会提交到 Git。 -sweep 结果分别写入 `benchmark/gateway/results/sweep-c.json`。 - -## 回放 example data 并人工检查 - -下面的命令读取 `examples/data/` 中的 ATIF、ACTF 和 OpenAI Messages 样例，把每个可 -回放的 agent/inference step 转换成 Chat Completions 请求，经过真实 Gateway 和本地 Echo， -最后将输入、HTTP 响应和 canonical capture 汇总到一个长期保留的 review bundle： - -```bash -just benchmark-gateway-replay -``` - -默认输出根目录是 `benchmark/gateway/results/replay-review/`。每次运行创建独立的 UTC -时间戳子目录，并把该目录写入 `latest.txt`，不会覆盖以前的 review。也可以指定输入和输出： - -```bash -just benchmark-gateway-replay examples/data /tmp/gateway-replay-review -``` - -建议先打开 bundle 中的 `REVIEW.md` 看每个 case 的来源、参考回答、Echo 回答和自动检查； -需要逐字段核对时再查看 `review.jsonl`。后者把 source record、实际 Gateway request/response、 -解析后的 canonical request/response 以及检查结果 join 在同一行。`captured-events.jsonl` 是 -未经 join 的完整事件导出，`dataset/` 和 `gateway-state/` 分别保留 Lance Dataset 与 WAL。 - -这里的 source response 只作为人工 review 的参考，不参与相等性断言。本地 Echo 固定返回 -最后一条 user message，因此这个 replay 验证的是 Gateway wire handling、事件配对和 capture -保真度，不声称能够复现原始模型回答。 - -## Links - -- [Gateway architecture](../../docs/src/en/pchronicle/guides/serve-gateway.md) -- [`persisting-gateway`](../../crates/persisting-gateway/README.md) -- [Regression tests](../../tests/regression/README.md) diff --git a/benchmark/gateway/benchmark.py b/benchmark/gateway/benchmark.py deleted file mode 100755 index 7fa8d1715..000000000 --- a/benchmark/gateway/benchmark.py +++ /dev/null @@ -1,628 +0,0 @@ -#!/usr/bin/env python3 -"""Black-box throughput benchmark for Gateway forwarding plus durable capture.""" - -from __future__ import annotations - -import argparse -import concurrent.futures -import http.client -import json -import os -import platform -import shutil -import signal -import statistics -import subprocess -import sys -import tempfile -import threading -import time -import urllib.parse -import uuid -from dataclasses import asdict, dataclass -from datetime import UTC, datetime -from pathlib import Path -from typing import TextIO - -SCENARIO_DIR = Path(__file__).resolve().parent -REPO_ROOT = SCENARIO_DIR.parents[1] -sys.path.insert(0, str(REPO_ROOT / "tests" / "regression")) - -from gateway_harness import ( # noqa: E402 - require_subcommand, - resolve_binary, - stop_process, - wait_http, - wait_logged_url, - without_proxy_environment, -) - - -@dataclass -class LoadResult: - elapsed_seconds: float - requests: int - errors: int - requests_per_second: float - latency_ms_mean: float - latency_ms_p50: float - latency_ms_p95: float - latency_ms_p99: float - latency_ms_max: float - estimated_mean_in_flight: float - error_samples: list[str] - - -def percentile(values: list[float], quantile: float) -> float: - if not values: - return 0.0 - ordered = sorted(values) - index = max(0, min(len(ordered) - 1, int(len(ordered) * quantile + 0.999999) - 1)) - return ordered[index] - - -def one_worker( - base_url: str, - start: threading.Event, - deadline: list[float], - request_budget: list[int] | None, - request_budget_lock: threading.Lock, - worker: int, - concurrency: int, - sessions: int, - session_prefix: str, - payload: str, -) -> tuple[list[float], int, list[str]]: - parsed = urllib.parse.urlparse(base_url) - if parsed.scheme != "http" or parsed.hostname is None or parsed.port is None: - raise ValueError(f"benchmark target must be an explicit HTTP URL: {base_url}") - path = f"{parsed.path.rstrip('/')}/v1/chat/completions" - body = json.dumps( - { - "model": "gateway-benchmark", - "messages": [{"role": "user", "content": payload}], - "stream": False, - }, - separators=(",", ":"), - ).encode() - headers = { - "authorization": "Bearer benchmark-local", - "content-type": "application/json", - } - latencies: list[float] = [] - error_count = 0 - errors: list[str] = [] - connection: http.client.HTTPConnection | None = None - request_index = 0 - start.wait() - while time.monotonic() < deadline[0]: - if request_budget is not None: - with request_budget_lock: - if request_budget[0] <= 0: - break - request_budget[0] -= 1 - request_started = time.perf_counter() - try: - if connection is None: - connection = http.client.HTTPConnection(parsed.hostname, parsed.port, timeout=15) - # Keep storage topology constant across a concurrency sweep. Advancing - # by `concurrency` makes all workers collectively round-robin over the - # configured session set without changing its cardinality. - session = (worker + request_index * concurrency) % sessions - headers["x-persisting-benchmark-session"] = f"{session_prefix}-{session}" - request_index += 1 - connection.request("POST", path, body=body, headers=headers) - response = connection.getresponse() - response_body = response.read() - if response.status != 200: - raise RuntimeError(f"HTTP {response.status}: {response_body[:160]!r}") - document = json.loads(response_body) - content = document["choices"][0]["message"]["content"] - if content != payload: - raise RuntimeError("echo response content mismatch") - latencies.append((time.perf_counter() - request_started) * 1000.0) - except Exception as error: # benchmark must report transport and protocol failures - error_count += 1 - if len(errors) < 10: - errors.append(f"worker {worker}: {error}") - if connection is not None: - connection.close() - connection = None - if connection is not None: - connection.close() - return latencies, error_count, errors - - -def run_load( - base_url: str, - *, - duration: float, - concurrency: int, - sessions: int, - max_requests: int | None, - session_prefix: str, - payload: str, -) -> LoadResult: - start = threading.Event() - deadline = [0.0] - request_budget = [max_requests] if max_requests is not None else None - request_budget_lock = threading.Lock() - started = time.perf_counter() - with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as executor: - futures = [ - executor.submit( - one_worker, - base_url, - start, - deadline, - request_budget, - request_budget_lock, - worker, - concurrency, - sessions, - session_prefix, - payload, - ) - for worker in range(concurrency) - ] - deadline[0] = time.monotonic() + duration - start.set() - worker_results = [future.result() for future in futures] - elapsed = time.perf_counter() - started - latencies = [latency for worker, _, _ in worker_results for latency in worker] - error_count = sum(count for _, count, _ in worker_results) - errors = [error for _, _, worker_errors in worker_results for error in worker_errors][:10] - return LoadResult( - elapsed_seconds=elapsed, - requests=len(latencies), - errors=error_count, - requests_per_second=len(latencies) / elapsed if elapsed else 0.0, - latency_ms_mean=statistics.fmean(latencies) if latencies else 0.0, - latency_ms_p50=percentile(latencies, 0.50), - latency_ms_p95=percentile(latencies, 0.95), - latency_ms_p99=percentile(latencies, 0.99), - latency_ms_max=max(latencies, default=0.0), - # Little's law: throughput * residence time estimates how many requests - # were resident in the measured path on average. In this closed-loop - # driver, a value near `concurrency` means the client kept the path full. - estimated_mean_in_flight=( - (len(latencies) / elapsed) * statistics.fmean(latencies) / 1000.0 - if elapsed and latencies - else 0.0 - ), - error_samples=errors, - ) - - -def write_configs(work_dir: Path, echo_url: str) -> tuple[Path, Path]: - warehouse = work_dir / "warehouse.toml" - gateway = work_dir / "gateway.toml" - warehouse.write_text( - f'''default_dataset = "captures" - -[[datasets]] -name = "captures" -uri = "{work_dir / "dataset"}" -''', - encoding="utf-8", - ) - gateway.write_text( - f'''listen = "127.0.0.1:0" -admin_listen = "127.0.0.1:0" -agent_id = "gateway-benchmark" -session_header = "x-persisting-benchmark-session" -capture_level = "full" - -[[models]] -name = "gateway-benchmark" -provider = "openai" -upstream = "{echo_url}/v1" -''', - encoding="utf-8", - ) - return warehouse, gateway - - -def captured_manifest_stats(dataset: Path, agent_id: str, prefix: str) -> dict[str, int]: - """Count published capture rows without opening every Lance version. - - The epoch-fenced manifest is the canonical visibility pointer for an event - log. Reading one manifest at a time keeps file-descriptor use constant even - when a benchmark produces many sessions and Lance versions. - """ - agent_dir = dataset / agent_id - if not agent_dir.is_dir(): - raise RuntimeError(f"capture agent directory is missing: {agent_dir}") - - session_prefix = f"{prefix}-" - sessions = sorted( - path - for path in agent_dir.iterdir() - if path.is_dir() and path.name.startswith(session_prefix) - ) - if not sessions: - raise RuntimeError(f"no capture sessions found for prefix {prefix!r}") - - published_events = 0 - visible_segments = 0 - max_segment_level = 0 - min_revision: int | None = None - max_revision: int | None = None - for session in sessions: - manifest_path = session / "events.lance" / "_manifest.json" - try: - manifest = json.loads(manifest_path.read_text(encoding="utf-8")) - except FileNotFoundError as error: - raise RuntimeError(f"capture manifest is missing: {manifest_path}") from error - except (OSError, json.JSONDecodeError) as error: - raise RuntimeError(f"cannot read capture manifest {manifest_path}: {error}") from error - - if manifest.get("schema_version") != 1: - raise RuntimeError( - f"unsupported capture manifest schema in {manifest_path}: " - f"{manifest.get('schema_version')!r}" - ) - revision = manifest.get("revision") - segments = manifest.get("segments") - if ( - not isinstance(revision, int) - or isinstance(revision, bool) - or revision < 0 - or not isinstance(segments, list) - ): - raise RuntimeError(f"invalid capture manifest metadata: {manifest_path}") - - for segment in segments: - rows = segment.get("rows") if isinstance(segment, dict) else None - level = segment.get("level", 0) if isinstance(segment, dict) else None - if not isinstance(rows, int) or isinstance(rows, bool) or rows < 0: - raise RuntimeError(f"invalid segment row count in {manifest_path}: {rows!r}") - if not isinstance(level, int) or isinstance(level, bool) or level < 0: - raise RuntimeError(f"invalid segment level in {manifest_path}: {level!r}") - published_events += rows - max_segment_level = max(max_segment_level, level) - visible_segments += len(segments) - min_revision = revision if min_revision is None else min(min_revision, revision) - max_revision = revision if max_revision is None else max(max_revision, revision) - - return { - "sessions": len(sessions), - "visible_segments": visible_segments, - "max_segment_level": max_segment_level, - "published_events": published_events, - "min_manifest_revision": min_revision or 0, - "max_manifest_revision": max_revision or 0, - } - - -def captured_event_counts( - pchronicle: Path, dataset: Path, prefix: str, output: Path -) -> dict[str, int]: - """Exercise the real query path after storage-fragment maintenance.""" - query = ( - "SELECT kind, COUNT(*) AS count FROM dataset.events " - f"WHERE session_id LIKE '{prefix}%' GROUP BY kind ORDER BY kind" - ) - subprocess.run( - [ - str(pchronicle), - "query", - str(dataset), - query, - "--format", - "jsonl", - "--output", - str(output), - ], - cwd=REPO_ROOT, - check=True, - ) - counts: dict[str, int] = {} - for line in output.read_text(encoding="utf-8").splitlines(): - row = json.loads(line) - counts[str(row["kind"])] = int(row["count"]) - return counts - - -def benchmark(args: argparse.Namespace, work_dir: Path, pchronicle: Path) -> dict[str, object]: - logs = work_dir / "logs" - dataset = work_dir / "dataset" - state = work_dir / "gateway-state" - logs.mkdir() - dataset.mkdir() - state.mkdir() - payload = "x" * args.payload_bytes - run_token = uuid.uuid4().hex - measured_prefix = f"gateway-bench-{run_token}" - processes: list[tuple[str, subprocess.Popen[bytes]]] = [] - handles: list[TextIO] = [] - serve_process: subprocess.Popen[bytes] | None = None - try: - echo_log = (logs / "echo.log").open("w", encoding="utf-8") - handles.append(echo_log) - echo_process = subprocess.Popen( - [str(pchronicle), "echo", "--listen", "127.0.0.1:0", "--encoding", "plain"], - cwd=REPO_ROOT, - stdout=echo_log, - stderr=subprocess.STDOUT, - ) - processes.append(("pChronicle Echo", echo_process)) - echo_url = wait_logged_url( - logs / "echo.log", "pChronicle Echo: ", echo_process, "pChronicle Echo" - ) - wait_http(f"{echo_url}/health", echo_process, "pChronicle Echo") - - baseline: LoadResult | None = None - with without_proxy_environment(): - if not args.skip_baseline: - if args.warmup > 0: - warmup = run_load( - echo_url, - duration=args.warmup, - concurrency=args.concurrency, - sessions=args.sessions, - max_requests=min(args.requests, 64), - session_prefix=f"baseline-warmup-{run_token}", - payload=payload, - ) - if warmup.errors: - raise RuntimeError(f"Echo baseline warmup failed: {warmup.error_samples}") - baseline = run_load( - echo_url, - duration=args.duration, - concurrency=args.concurrency, - sessions=args.sessions, - max_requests=args.requests, - session_prefix=f"baseline-{run_token}", - payload=payload, - ) - if baseline.errors: - raise RuntimeError(f"Echo baseline failed: {baseline.error_samples}") - - warehouse, gateway = write_configs(work_dir, echo_url) - serve_log = (logs / "serve.log").open("w", encoding="utf-8") - handles.append(serve_log) - serve_process = subprocess.Popen( - [ - str(pchronicle), - "serve", - "--config", - str(warehouse), - "--listen", - "127.0.0.1:0", - "--gateway", - str(gateway), - "--gateway-dataset", - "captures", - "--gateway-state", - str(state), - ], - cwd=REPO_ROOT, - stdout=serve_log, - stderr=subprocess.STDOUT, - ) - processes.append(("pChronicle serve", serve_process)) - gateway_url = wait_logged_url( - logs / "serve.log", "pChronicle Gateway: ", serve_process, "pChronicle Gateway" - ) - admin_url = wait_logged_url( - logs / "serve.log", - "pChronicle Gateway admin: ", - serve_process, - "pChronicle Gateway admin", - ) - wait_http(f"{admin_url}/admin/status", serve_process, "pChronicle Gateway admin") - - with without_proxy_environment(): - if args.warmup > 0: - warmup = run_load( - gateway_url, - duration=args.warmup, - concurrency=args.concurrency, - sessions=args.sessions, - max_requests=min(args.requests, 64), - session_prefix=f"gateway-warmup-{run_token}", - payload=payload, - ) - if warmup.errors: - raise RuntimeError(f"Gateway warmup failed: {warmup.error_samples}") - gateway_result = run_load( - gateway_url, - duration=args.duration, - concurrency=args.concurrency, - sessions=args.sessions, - max_requests=args.requests, - session_prefix=measured_prefix, - payload=payload, - ) - if gateway_result.errors: - raise RuntimeError(f"Gateway load failed: {gateway_result.error_samples}") - if gateway_result.requests_per_second < args.min_rps: - raise RuntimeError( - f"Gateway throughput {gateway_result.requests_per_second:.2f} req/s is below " - f"--min-rps {args.min_rps:.2f}" - ) - - drain_started = time.perf_counter() - stop_process(serve_process, label="pChronicle serve", require_success=True) - shutdown_drain_seconds = time.perf_counter() - drain_started - serve_process = None - capture = captured_manifest_stats(dataset, "gateway-benchmark", measured_prefix) - expected_requests = gateway_result.requests - expected_events = expected_requests * 2 - if capture["published_events"] != expected_events: - raise RuntimeError( - "durable capture count mismatch: " - f"requests={expected_requests}, expected_events={expected_events}, " - f"capture={json.dumps(capture, sort_keys=True)}" - ) - event_counts = captured_event_counts( - pchronicle, dataset, measured_prefix, logs / "capture-counts.jsonl" - ) - if ( - event_counts.get("llm.request", 0) != expected_requests - or event_counts.get("llm.response", 0) != expected_requests - ): - raise RuntimeError( - "durable capture kind mismatch: " - f"requests={expected_requests}, counts={json.dumps(event_counts, sort_keys=True)}" - ) - - comparison = None - if baseline is not None: - comparison = { - "throughput_ratio": ( - gateway_result.requests_per_second / baseline.requests_per_second - if baseline.requests_per_second - else None - ), - "p95_latency_ratio": ( - gateway_result.latency_ms_p95 / baseline.latency_ms_p95 - if baseline.latency_ms_p95 - else None - ), - } - return { - "schema_version": 2, - "generated_at": datetime.now(UTC).isoformat(), - "system": { - "platform": platform.platform(), - "python": platform.python_version(), - "cpu_count": os.cpu_count(), - }, - "config": { - "duration_seconds": args.duration, - "request_limit": args.requests, - "warmup_seconds": args.warmup, - "concurrency": args.concurrency, - "load_model": "closed_loop", - "max_in_flight": args.concurrency, - "sessions": args.sessions, - "payload_bytes": args.payload_bytes, - "capture_level": "full", - }, - "echo_baseline": asdict(baseline) if baseline is not None else None, - "gateway": asdict(gateway_result), - "comparison": comparison, - "durable_capture": { - "expected_requests": expected_requests, - "expected_events": expected_events, - "shutdown_drain_seconds": shutdown_drain_seconds, - "events_per_drain_second": ( - expected_events / shutdown_drain_seconds - if shutdown_drain_seconds - else None - ), - **capture, - "event_counts": event_counts, - "validated": True, - }, - } - finally: - stop_process(serve_process, label="pChronicle serve") - for label, process in reversed(processes): - stop_process(process, label=label) - for handle in handles: - handle.close() - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--duration", type=float, default=10.0) - parser.add_argument("--warmup", type=float, default=1.0) - parser.add_argument("--concurrency", type=int, default=16) - parser.add_argument( - "--requests", - type=int, - default=1024, - help="maximum requests per measured phase; duration remains a time ceiling", - ) - parser.add_argument( - "--sessions", - type=int, - default=16, - help="fixed capture-session cardinality (independent of concurrency)", - ) - parser.add_argument("--payload-bytes", type=int, default=256) - parser.add_argument("--min-rps", type=float, default=0.0) - parser.add_argument("--skip-baseline", action="store_true") - parser.add_argument("--keep-artifacts", action="store_true") - parser.add_argument("--output", type=Path) - args = parser.parse_args() - if args.duration <= 0 or args.warmup < 0: - parser.error("duration must be positive and warmup must be non-negative") - if args.concurrency <= 0 or args.sessions <= 0 or args.requests <= 0: - parser.error("concurrency, sessions, and requests must be positive") - if args.payload_bytes < 0 or args.min_rps < 0: - parser.error("payload-bytes and min-rps cannot be negative") - return args - - -def interrupt_on_sigterm(_signum: int, _frame: object) -> None: - raise KeyboardInterrupt - - -def main() -> None: - args = parse_args() - signal.signal(signal.SIGTERM, interrupt_on_sigterm) - pchronicle = resolve_binary( - "PERSISTING_PCHRONICLE_BIN", - REPO_ROOT / "target" / "release" / "pchronicle", - "cargo build --release --locked -p persisting-pchronicle-cli --bin pchronicle", - ) - for subcommand in ["echo", "serve", "query"]: - require_subcommand( - pchronicle, - subcommand, - "cargo build --release --locked -p persisting-pchronicle-cli --bin pchronicle", - ) - work_dir = Path(tempfile.mkdtemp(prefix="persisting-gateway-benchmark.")) - success = False - try: - result = benchmark(args, work_dir, pchronicle) - output = args.output or SCENARIO_DIR / "results" / "latest.json" - output = output.expanduser().resolve() - output.parent.mkdir(parents=True, exist_ok=True) - temporary = output.with_suffix(f"{output.suffix}.tmp") - temporary.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8") - temporary.replace(output) - gateway = result["gateway"] - baseline = result["echo_baseline"] - print( - "Gateway: " - f"{gateway['requests_per_second']:.2f} req/s, " - f"mean={gateway['latency_ms_mean']:.2f} ms, " - f"p50={gateway['latency_ms_p50']:.2f} ms, " - f"p95={gateway['latency_ms_p95']:.2f} ms, " - f"p99={gateway['latency_ms_p99']:.2f} ms, " - f"requests={gateway['requests']}" - ) - print( - "Load: closed-loop, " - f"max_in_flight={args.concurrency}, " - f"estimated_mean_in_flight={gateway['estimated_mean_in_flight']:.2f}" - ) - if baseline is not None: - print( - "Echo baseline: " - f"{baseline['requests_per_second']:.2f} req/s, " - f"p95={baseline['latency_ms_p95']:.2f} ms" - ) - print(f"Durable capture: {json.dumps(result['durable_capture'], sort_keys=True)}") - capture = result["durable_capture"] - print( - "Capture drain: " - f"{capture['shutdown_drain_seconds']:.2f} s, " - f"{capture['events_per_drain_second']:.2f} events/s" - ) - print(f"Result: {output}") - success = True - finally: - if args.keep_artifacts or not success: - print(f"Gateway benchmark artifacts: {work_dir}", file=sys.stderr) - else: - shutil.rmtree(work_dir) - - -if __name__ == "__main__": - main() diff --git a/benchmark/gateway/replay.py b/benchmark/gateway/replay.py deleted file mode 100644 index 90acc6ffb..000000000 --- a/benchmark/gateway/replay.py +++ /dev/null @@ -1,668 +0,0 @@ -#!/usr/bin/env python3 -"""Replay the example trajectory data through Gateway into a reviewable bundle.""" - -from __future__ import annotations - -import argparse -import copy -import http.client -import json -import os -import re -import signal -import subprocess -import sys -import time -import urllib.parse -from collections import defaultdict -from dataclasses import asdict, dataclass -from datetime import UTC, datetime -from pathlib import Path -from typing import Any, TextIO - -SCENARIO_DIR = Path(__file__).resolve().parent -REPO_ROOT = SCENARIO_DIR.parents[1] -sys.path.insert(0, str(REPO_ROOT / "tests" / "regression")) - -from gateway_harness import ( # noqa: E402 - append_jsonl, - require_subcommand, - resolve_binary, - stop_process, - wait_http, - wait_logged_url, - without_proxy_environment, -) - - -@dataclass(frozen=True) -class ReplayCase: - case_id: str - source_file: str - source_format: str - source_record_id: str - session_id: str - model: str - messages: list[dict[str, Any]] - reference_response: Any - source_record: dict[str, Any] - - -def source_label(path: Path) -> str: - try: - return str(path.relative_to(REPO_ROOT)) - except ValueError: - return str(path) - - -def safe_id(value: object) -> str: - normalized = re.sub(r"[^A-Za-z0-9_.-]+", "-", str(value)).strip("-.") - return normalized or "unknown" - - -def message_text(content: Any) -> str: - if isinstance(content, str): - return content - if isinstance(content, list): - parts: list[str] = [] - for part in content: - if isinstance(part, str): - parts.append(part) - elif isinstance(part, dict): - text = part.get("text", part.get("input_text")) - if isinstance(text, str): - parts.append(text) - return "".join(parts) - if isinstance(content, dict): - text = content.get("text") - return text if isinstance(text, str) else "" - return "" - - -def last_user_text(messages: list[dict[str, Any]]) -> str: - for message in reversed(messages): - if message.get("role") == "user": - return message_text(message.get("content")) - return "" - - -def openai_cases(path: Path, document: list[Any]) -> list[ReplayCase]: - cases: list[ReplayCase] = [] - for index, row in enumerate(document): - if not isinstance(row, dict) or not isinstance(row.get("messages"), list): - raise ValueError(f"invalid OpenAI Messages row {index} in {path}") - record_id = str(row.get("id", index)) - session = safe_id(row.get("session_id", record_id)) - model = str(row.get("agent_model", "example-openai-model")) - cases.append( - ReplayCase( - case_id=f"openai-{safe_id(record_id)}", - source_file=source_label(path), - source_format="openai-messages", - source_record_id=record_id, - session_id=f"gateway-replay-openai-{session}", - model=model, - messages=copy.deepcopy(row["messages"]), - reference_response=copy.deepcopy(row.get("response")), - source_record=copy.deepcopy(row), - ) - ) - return cases - - -def tool_call_message(tool_calls: Any) -> list[dict[str, Any]]: - if not isinstance(tool_calls, list): - return [] - converted: list[dict[str, Any]] = [] - for index, call in enumerate(tool_calls): - if not isinstance(call, dict): - continue - call_id = str(call.get("tool_call_id", call.get("id", f"call-{index}"))) - name = str(call.get("function_name", call.get("name", "unknown_tool"))) - arguments = call.get("arguments", call.get("input", {})) - converted.append( - { - "id": call_id, - "type": "function", - "function": { - "name": name, - "arguments": json.dumps(arguments, ensure_ascii=False, sort_keys=True), - }, - } - ) - return converted - - -def append_tool_results(history: list[dict[str, Any]], tool_calls: Any, observations: Any) -> None: - if isinstance(tool_calls, list): - for index, call in enumerate(tool_calls): - if not isinstance(call, dict) or "result" not in call: - continue - call_id = str(call.get("tool_call_id", call.get("id", f"call-{index}"))) - history.append( - { - "role": "tool", - "tool_call_id": call_id, - "content": json.dumps(call["result"], ensure_ascii=False, sort_keys=True), - } - ) - if isinstance(observations, list): - for index, observation in enumerate(observations): - if not isinstance(observation, dict): - continue - call_id = str(observation.get("tool_use_id", f"observation-{index}")) - history.append( - { - "role": "tool", - "tool_call_id": call_id, - "content": message_text(observation.get("content")), - } - ) - - -def atif_cases(path: Path, document: dict[str, Any]) -> list[ReplayCase]: - session = safe_id(document.get("session_id", document.get("trajectory_id", path.stem))) - agent = document.get("agent") if isinstance(document.get("agent"), dict) else {} - model = str(agent.get("model_name", "example-atif-model")) - history: list[dict[str, Any]] = [] - cases: list[ReplayCase] = [] - for index, step in enumerate(document.get("steps", [])): - if not isinstance(step, dict): - raise ValueError(f"invalid ATIF step {index} in {path}") - source = step.get("source") - message = message_text(step.get("message")) - if source == "user": - history.append({"role": "user", "content": message}) - continue - if source != "agent": - continue - step_id = str(step.get("step_id", index)) - cases.append( - ReplayCase( - case_id=f"atif-{session}-step-{safe_id(step_id)}", - source_file=source_label(path), - source_format="atif", - source_record_id=step_id, - session_id=f"gateway-replay-atif-{session}", - model=str(step.get("model_name", model)), - messages=copy.deepcopy(history), - reference_response={ - "message": step.get("message"), - "tool_calls": copy.deepcopy(step.get("tool_calls", [])), - }, - source_record=copy.deepcopy(step), - ) - ) - assistant: dict[str, Any] = {"role": "assistant", "content": message} - converted_calls = tool_call_message(step.get("tool_calls")) - if converted_calls: - assistant["tool_calls"] = converted_calls - history.append(assistant) - append_tool_results(history, step.get("tool_calls"), None) - return cases - - -def actf_cases(path: Path, document: dict[str, Any]) -> list[ReplayCase]: - task_id = safe_id(document.get("task_id", path.stem)) - attempts = document.get("attempts") - if not isinstance(attempts, dict): - raise ValueError(f"ACTF attempts must be an object in {path}") - cases: list[ReplayCase] = [] - for attempt_id, attempt in sorted(attempts.items(), key=lambda item: str(item[0])): - if not isinstance(attempt, dict): - raise ValueError(f"invalid ACTF attempt {attempt_id} in {path}") - trajectory = attempt.get("trajectory") - if not isinstance(trajectory, dict) or not isinstance(trajectory.get("steps"), list): - raise ValueError(f"ACTF attempt {attempt_id} has no trajectory steps in {path}") - session = f"gateway-replay-actf-{task_id}-attempt-{safe_id(attempt_id)}" - history: list[dict[str, Any]] = [] - current_system: str | None = None - for index, step in enumerate(trajectory["steps"]): - if not isinstance(step, dict): - raise ValueError(f"invalid ACTF step {index} in {path}") - system_prompt = message_text(step.get("system_prompt")) - if system_prompt and system_prompt != current_system: - history.append({"role": "system", "content": system_prompt}) - current_system = system_prompt - user_content = message_text(step.get("user_content")) - if user_content: - history.append({"role": "user", "content": user_content}) - assistant_content = step.get("assistant_content") - if not isinstance(assistant_content, dict): - assistant_content = {"content": message_text(assistant_content)} - step_id = str(step.get("step_id", index)) - cases.append( - ReplayCase( - case_id=( - f"actf-{task_id}-attempt-{safe_id(attempt_id)}-step-{safe_id(step_id)}" - ), - source_file=source_label(path), - source_format="actf", - source_record_id=f"attempt-{attempt_id}/step-{step_id}", - session_id=session, - model="example-actf-model", - messages=copy.deepcopy(history), - reference_response=copy.deepcopy(assistant_content), - source_record=copy.deepcopy(step), - ) - ) - assistant: dict[str, Any] = { - "role": "assistant", - "content": message_text(assistant_content.get("content")), - } - converted_calls = tool_call_message(assistant_content.get("tool_calls")) - if converted_calls: - assistant["tool_calls"] = converted_calls - history.append(assistant) - append_tool_results(history, None, step.get("observation")) - return cases - - -def load_cases(data_dir: Path) -> list[ReplayCase]: - cases: list[ReplayCase] = [] - json_paths = sorted(data_dir.rglob("*.json")) - if not json_paths: - raise ValueError(f"no JSON example data found under {data_dir}") - for path in json_paths: - document = json.loads(path.read_text(encoding="utf-8")) - if isinstance(document, list): - cases.extend(openai_cases(path, document)) - elif isinstance(document, dict) and str(document.get("schema_version", "")).startswith( - "ATIF-" - ): - cases.extend(atif_cases(path, document)) - elif isinstance(document, dict) and "attempts" in document: - cases.extend(actf_cases(path, document)) - else: - raise ValueError(f"unsupported example JSON format: {path}") - if not cases: - raise ValueError(f"example data produced no replayable Gateway calls: {data_dir}") - case_ids = [case.case_id for case in cases] - if len(case_ids) != len(set(case_ids)): - raise ValueError("example data produced duplicate replay case IDs") - return cases - - -def write_configs(run_dir: Path, echo_url: str, cases: list[ReplayCase]) -> tuple[Path, Path]: - warehouse = run_dir / "warehouse.toml" - gateway = run_dir / "gateway.toml" - warehouse.write_text( - 'default_dataset = "captures"\n\n' - '[[datasets]]\nname = "captures"\n' - f"uri = {json.dumps(str(run_dir / 'dataset'))}\n", - encoding="utf-8", - ) - routes = [] - for model in sorted({case.model for case in cases}): - routes.append( - "[[models]]\n" - f"name = {json.dumps(model)}\n" - 'provider = "openai"\n' - f"upstream = {json.dumps(echo_url + '/v1')}\n" - ) - gateway.write_text( - 'listen = "127.0.0.1:0"\n' - 'admin_listen = "127.0.0.1:0"\n' - 'agent_id = "gateway-example-replay"\n' - 'capture_level = "full"\n\n' - + "\n".join(routes), - encoding="utf-8", - ) - return warehouse, gateway - - -def send_case(gateway_url: str, case: ReplayCase) -> dict[str, Any]: - parsed = urllib.parse.urlparse(gateway_url) - if parsed.scheme != "http" or parsed.hostname is None or parsed.port is None: - raise ValueError(f"Gateway must publish an explicit HTTP URL: {gateway_url}") - request = {"model": case.model, "messages": case.messages, "stream": False} - body = json.dumps(request, ensure_ascii=False, separators=(",", ":")).encode() - headers = { - "authorization": "Bearer gateway-example-replay", - "content-type": "application/json", - "x-persisting-session-id": case.session_id, - } - started = time.perf_counter() - connection = http.client.HTTPConnection(parsed.hostname, parsed.port, timeout=15) - try: - connection.request("POST", "/v1/chat/completions", body=body, headers=headers) - response = connection.getresponse() - response_body = response.read() - finally: - connection.close() - latency_ms = (time.perf_counter() - started) * 1000 - try: - response_json = json.loads(response_body) - except json.JSONDecodeError: - response_json = None - result = { - "case_id": case.case_id, - "session_id": case.session_id, - "http_status": response.status, - "latency_ms": latency_ms, - "request": request, - "response": response_json, - "response_body": response_body.decode("utf-8", errors="replace"), - } - if response.status != 200 or not isinstance(response_json, dict): - raise RuntimeError( - f"Gateway replay failed for {case.case_id}: HTTP {response.status} " - f"body={result['response_body'][:500]!r}" - ) - return result - - -def query_events(pchronicle: Path, dataset: Path, output: Path) -> list[dict[str, Any]]: - subprocess.run( - [ - str(pchronicle), - "query", - str(dataset), - ( - "SELECT seq, kind, session_id, model, call_id, payload_json " - "FROM dataset.events ORDER BY session_id, seq" - ), - "--format", - "jsonl", - "--output", - str(output), - ], - cwd=REPO_ROOT, - check=True, - ) - return [json.loads(line) for line in output.read_text(encoding="utf-8").splitlines()] - - -def response_text(result: dict[str, Any]) -> str: - try: - return str(result["response"]["choices"][0]["message"]["content"] or "") - except (KeyError, IndexError, TypeError): - return "" - - -def reference_text(reference: Any) -> str: - if not isinstance(reference, dict): - return message_text(reference) - for field in ["content", "message", "reasoning_content"]: - text = message_text(reference.get(field)) - if text: - return text - return "" - - -def captured_payload(event: dict[str, Any]) -> dict[str, Any]: - payload = event.get("payload_json") - if not isinstance(payload, str): - raise ValueError(f"captured event has no payload_json: {event}") - parsed = json.loads(payload) - if not isinstance(parsed, dict): - raise ValueError(f"captured payload is not an object: {event}") - return parsed - - -def build_review( - run_dir: Path, - data_dir: Path, - cases: list[ReplayCase], - results: list[dict[str, Any]], - events: list[dict[str, Any]], -) -> None: - results_by_case = {result["case_id"]: result for result in results} - requests_by_session: dict[str, list[dict[str, Any]]] = defaultdict(list) - responses_by_session: dict[str, list[dict[str, Any]]] = defaultdict(list) - for event in events: - kind = event.get("kind") - if kind == "llm.request": - requests_by_session[str(event.get("session_id"))].append(event) - elif kind in {"llm.response", "llm.response.stream"}: - responses_by_session[str(event.get("session_id"))].append(event) - - session_offsets: dict[str, int] = defaultdict(int) - review_path = run_dir / "review.jsonl" - checks_passed = 0 - markdown_rows: list[str] = [] - for case in cases: - offset = session_offsets[case.session_id] - session_offsets[case.session_id] += 1 - captured_requests = requests_by_session.get(case.session_id, []) - captured_responses = responses_by_session.get(case.session_id, []) - request_event = captured_requests[offset] if offset < len(captured_requests) else None - response_event = captured_responses[offset] if offset < len(captured_responses) else None - result = results_by_case[case.case_id] - actual = response_text(result) - expected_echo = last_user_text(case.messages) - request_capture = None if request_event is None else captured_payload(request_event) - response_capture = None if response_event is None else captured_payload(response_event) - checks = { - "http_200": result["http_status"] == 200, - "echo_matches_last_user": actual == expected_echo, - "captured_request": request_event is not None, - "captured_response": response_event is not None, - "capture_call_id_matches": ( - request_event is not None - and response_event is not None - and request_event.get("call_id") == response_event.get("call_id") - ), - "captured_request_wire_matches": ( - request_capture is not None - and request_capture.get("payload", {}).get("http", {}).get("request_body") - == result["request"] - ), - "captured_response_wire_matches": ( - response_capture is not None - and response_capture.get("payload", {}).get("http", {}).get("response_body") - == result["response"] - ), - } - passed = all(checks.values()) - checks_passed += int(passed) - append_jsonl( - review_path, - { - "case": asdict(case), - "gateway_result": result, - "expected_echo_response": expected_echo, - "captured_request": request_capture, - "captured_response": response_capture, - "checks": checks, - "passed": passed, - }, - ) - short_actual = actual.replace("\n", "\\n")[:64].replace("|", "\\|") - short_reference = ( - reference_text(case.reference_response) - .replace("\n", "\\n")[:64] - .replace("|", "\\|") - ) - markdown_rows.append( - f"| `{case.case_id}` | `{case.source_format}` | {len(case.messages)} | " - f"{'PASS' if passed else 'FAIL'} | {short_reference} | {short_actual} |" - ) - - expected_events = len(cases) * 2 - global_checks = { - "exact_event_count": len(events) == expected_events, - "only_request_response_events": all( - event.get("kind") in {"llm.request", "llm.response"} for event in events - ), - } - summary = { - "schema_version": 1, - "generated_at": datetime.now(UTC).isoformat(), - "source_directory": str(data_dir), - "output_directory": str(run_dir), - "cases": len(cases), - "expected_events": expected_events, - "captured_events": len(events), - "passed": checks_passed, - "failed": len(cases) - checks_passed, - "global_checks": global_checks, - } - (run_dir / "summary.json").write_text( - json.dumps(summary, ensure_ascii=False, indent=2, sort_keys=True) + "\n", - encoding="utf-8", - ) - (run_dir / "REVIEW.md").write_text( - "# Gateway example-data replay review\n\n" - f"Source: `{data_dir}` \n" - f"Cases: {len(cases)} \n" - f"Canonical events: {len(events)}/{expected_events} \n" - f"Automated checks: {checks_passed}/{len(cases)} passed\n\n" - "The recorded source response is reference material only. The local Echo upstream " - "deterministically returns the last user message, so this run reviews Gateway wire " - "handling and capture fidelity rather than model-output equivalence.\n\n" - "## Files\n\n" - "- `inputs.jsonl`: source record plus the replayed Chat Completions messages.\n" - "- `client-results.jsonl`: exact HTTP request and response seen by the driver.\n" - "- `captured-events.jsonl`: canonical events exported from the Lance Dataset.\n" - "- `review.jsonl`: one joined record per call, including parsed canonical payloads.\n" - "- `dataset/`: durable Lance capture; `gateway-state/`: Gateway WAL/state.\n" - "- `logs/`: Echo and Gateway process logs.\n\n" - "## Calls\n\n" - "| Case | Source format | Request messages | Checks | Reference response | Echo output |\n" - "|---|---:|---:|---:|---|---|\n" - + "\n".join(markdown_rows) - + "\n", - encoding="utf-8", - ) - if checks_passed != len(cases) or not all(global_checks.values()): - raise RuntimeError( - f"replay review checks failed: cases={checks_passed}/{len(cases)} " - f"global={global_checks}; see {review_path}" - ) - - -def run_replay(run_dir: Path, data_dir: Path, pchronicle: Path) -> None: - cases = load_cases(data_dir) - logs = run_dir / "logs" - dataset = run_dir / "dataset" - state = run_dir / "gateway-state" - logs.mkdir() - dataset.mkdir() - state.mkdir() - for case in cases: - append_jsonl(run_dir / "inputs.jsonl", asdict(case)) - - processes: list[tuple[str, subprocess.Popen[bytes]]] = [] - handles: list[TextIO] = [] - serve_process: subprocess.Popen[bytes] | None = None - try: - echo_log = (logs / "echo.log").open("w", encoding="utf-8") - handles.append(echo_log) - echo_process = subprocess.Popen( - [str(pchronicle), "echo", "--listen", "127.0.0.1:0", "--encoding", "plain"], - cwd=REPO_ROOT, - stdout=echo_log, - stderr=subprocess.STDOUT, - ) - processes.append(("pChronicle Echo", echo_process)) - echo_url = wait_logged_url( - logs / "echo.log", "pChronicle Echo: ", echo_process, "pChronicle Echo" - ) - wait_http(f"{echo_url}/health", echo_process, "pChronicle Echo") - - warehouse, gateway = write_configs(run_dir, echo_url, cases) - serve_log = (logs / "serve.log").open("w", encoding="utf-8") - handles.append(serve_log) - serve_process = subprocess.Popen( - [ - str(pchronicle), - "serve", - "--config", - str(warehouse), - "--listen", - "127.0.0.1:0", - "--gateway", - str(gateway), - "--gateway-dataset", - "captures", - "--gateway-state", - str(state), - ], - cwd=REPO_ROOT, - stdout=serve_log, - stderr=subprocess.STDOUT, - ) - processes.append(("pChronicle serve", serve_process)) - gateway_url = wait_logged_url( - logs / "serve.log", "pChronicle Gateway: ", serve_process, "pChronicle Gateway" - ) - admin_url = wait_logged_url( - logs / "serve.log", - "pChronicle Gateway admin: ", - serve_process, - "pChronicle Gateway admin", - ) - wait_http(f"{admin_url}/admin/status", serve_process, "pChronicle Gateway admin") - - results: list[dict[str, Any]] = [] - with without_proxy_environment(): - for case in cases: - result = send_case(gateway_url, case) - results.append(result) - append_jsonl(run_dir / "client-results.jsonl", result) - - stop_process(serve_process, label="pChronicle serve", require_success=True) - serve_process = None - events = query_events(pchronicle, dataset, run_dir / "captured-events.jsonl") - build_review(run_dir, data_dir, cases, results, events) - finally: - stop_process(serve_process, label="pChronicle serve") - for label, process in reversed(processes): - stop_process(process, label=label) - for handle in handles: - handle.close() - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--data", type=Path, default=REPO_ROOT / "examples" / "data") - parser.add_argument( - "--output", - type=Path, - default=SCENARIO_DIR / "results" / "replay-review", - help="parent directory for timestamped review bundles", - ) - return parser.parse_args() - - -def interrupt_on_sigterm(_signum: int, _frame: object) -> None: - raise KeyboardInterrupt - - -def main() -> None: - args = parse_args() - signal.signal(signal.SIGTERM, interrupt_on_sigterm) - data_dir = args.data.expanduser().resolve() - if not data_dir.is_dir(): - raise ValueError(f"example data directory does not exist: {data_dir}") - output_root = args.output.expanduser().resolve() - output_root.mkdir(parents=True, exist_ok=True) - run_id = datetime.now(UTC).strftime("%Y%m%dT%H%M%S.%fZ") - run_dir = output_root / run_id - run_dir.mkdir() - (output_root / "latest.txt").write_text(str(run_dir) + "\n", encoding="utf-8") - print(f"Gateway example replay review: {run_dir}", flush=True) - - pchronicle = resolve_binary( - "PERSISTING_PCHRONICLE_BIN", - REPO_ROOT / "target" / "release" / "pchronicle", - "cargo build --release --locked -p persisting-pchronicle-cli --bin pchronicle", - ) - for subcommand in ["echo", "serve", "query"]: - require_subcommand( - pchronicle, - subcommand, - "cargo build --release --locked -p persisting-pchronicle-cli --bin pchronicle", - ) - run_replay(run_dir, data_dir, pchronicle) - summary = json.loads((run_dir / "summary.json").read_text(encoding="utf-8")) - print( - f"PASS Gateway example replay: cases={summary['cases']} " - f"events={summary['captured_events']} review={run_dir / 'REVIEW.md'}", - flush=True, - ) - - -if __name__ == "__main__": - main() diff --git a/benchmark/gateway/replay.sh b/benchmark/gateway/replay.sh deleted file mode 100755 index a7f1e3194..000000000 --- a/benchmark/gateway/replay.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -scenario_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" -exec python3 "$scenario_dir/replay.py" "$@" diff --git a/benchmark/gateway/run.sh b/benchmark/gateway/run.sh deleted file mode 100755 index 37beb8849..000000000 --- a/benchmark/gateway/run.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -scenario_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" -exec python3 "$scenario_dir/benchmark.py" "$@" diff --git a/benchmark/langfuse-pchronicle-review/README.md b/benchmark/langfuse-pchronicle-review/README.md deleted file mode 100644 index 063594192..000000000 --- a/benchmark/langfuse-pchronicle-review/README.md +++ /dev/null @@ -1,132 +0,0 @@ -# Langfuse → pChronicle backend feasibility probe - -**Review-only probe: replace Langfuse's ClickHouse analytics backend with -pChronicle, without changing Langfuse's public API or pChronicle storage -formats.** - -Owns the adapter experiment, deterministic fixture, and recorded verdicts. -Postgres, Redis, and object storage remain unchanged. TTAS, Queue/Sampler, -Search, and `persisting-dlcapt` are outside this probe. - -The original recorded verdict and evidence are in [REPORT.zh.md](REPORT.zh.md). -The lazy-catalog retest is in -[RETEST-2026-08-12.zh.md](RETEST-2026-08-12.zh.md), with machine-readable -results in -[recorded-results-2026-08-12.json](recorded-results-2026-08-12.json). The -Server source-routing implementation and release measurements are in -[SERVER-ACCELERATION-2026-08-12.zh.md](SERVER-ACCELERATION-2026-08-12.zh.md), -with machine-readable results in -[server-acceleration-results-2026-08-12.json](server-acceleration-results-2026-08-12.json). The -original run remains in -[recorded-results-2026-08-11.json](recorded-results-2026-08-11.json). - -Pinned review baselines: - -- Langfuse: `d18a59ad663ffc7c04afc61354186c141b3ec0f3` (v4.7.1) -- Persisting: `94531cf903e5abc336de347588fb1858e9d52b6a` plus the local, - uncommitted catalog/query-engine work present during the review -- ClickHouse: 25.12.x; the recorded run used 25.12.11.4 - -## Workload - -The default deterministic fixture contains two projects and: - -- 100,000 trace/span events, including 100 duplicate logical versions -- 10,000 scores -- 2,000 dataset-run items -- 1,000 blob-storage log rows -- metadata, tags, tool names, large I/O values, and 12-digit decimal costs - -Trace IDs map to pChronicle Run IDs. Rows without a trace use synthetic Run -IDs. `project_id` maps to `agent_id`; the complete logical row is retained in -the event payload. The default 200 traces plus synthetic Runs create 210 Lance -event datasets. - -The executable defines the review-only `LangfuseAnalyticsBackend` contract and -implements append, point/list/aggregate reads, JSON streaming, flush, and -health. Update and delete methods deliberately return unsupported errors. An -implementation that hides those errors behind a new mutable projection engine -would exceed the bounded-integration decision rule. - -## Run - -Use an empty temporary directory; the fixture is about 238 MB. - -```bash -PROBE_DIR="$(mktemp -d /tmp/langfuse-pchronicle-review.XXXXXX)" -PCHRONICLE_LANGFUSE_WORKDIR="$PROBE_DIR" \ - cargo run --release -p persisting-pchronicle \ - --example langfuse_backend_feasibility -``` - -The command writes: - -- `$PROBE_DIR/logical_rows.jsonl` — shared deterministic input -- `$PROBE_DIR/pchronicle-store/` — pChronicle data -- `$PROBE_DIR/pchronicle-report.json` — measurements and capability failures - -The load phase samples visibility after every acknowledged single-row append, -so `visibility_p95_ms` is an observed percentile rather than a final-row spot -check. - -Useful scale controls are `PCHRONICLE_LANGFUSE_EVENTS`, -`PCHRONICLE_LANGFUSE_SCORES`, `PCHRONICLE_LANGFUSE_DATASET_RUN_ITEMS`, -`PCHRONICLE_LANGFUSE_BLOB_LOG_ROWS`, `PCHRONICLE_LANGFUSE_TRACES`, -`PCHRONICLE_LANGFUSE_PRELOAD_BATCH_ROWS`, and -`PCHRONICLE_LANGFUSE_LOAD_SECONDS`. - -Start an isolated ClickHouse 25.12 instance. One Docker example is: - -```bash -docker run --rm --name langfuse-pchronicle-review \ - -e CLICKHOUSE_PASSWORD=review-only \ - -p 127.0.0.1:18123:8123 \ - clickhouse/clickhouse-server:25.12 -``` - -In another shell, run the same fixture: - -```bash -python3 benchmark/langfuse-pchronicle-review/clickhouse_baseline.py \ - --url http://127.0.0.1:18123 \ - --user default \ - --password review-only \ - --fixture "$PROBE_DIR/logical_rows.jsonl" \ - --output "$PROBE_DIR/clickhouse-report.json" -``` - -The script sends credentials only in ClickHouse authentication headers. It -drops and recreates only the disposable database -`langfuse_pchronicle_review`. Its schema mirrors the relevant Langfuse v4 -patterns: `ReplacingMergeTree`, project/time/trace ordering, full/core event -tables with an incremental materialized view, text and skipping indexes, -async inserts with acknowledgement, and separate score/dataset/blob tables. -It samples visibility after every acknowledged 10-row load batch. - -```bash -cargo test -p persisting-pchronicle --test langfuse_backend_faults -- --nocapture -cargo test -p persisting-pchronicle --test production_scale --test query_engine -cargo test -p persisting-pchronicle store::catalog -``` - -These cover SIGKILL after an acknowledged append, writer fencing, pinned -snapshots during append, independent Run isolation, maintenance/restart, the -read-only query boundary, and catalog refresh behavior. - -## Acceptance gates - -- No acknowledged loss and no duplicate logical rows -- Visibility P95 ≤ 2 s; update/delete visibility ≤ 5 s -- Point P95 ≤ 500 ms; list/facet P95 ≤ 1 s -- Dashboard, full-text, and export first-byte P95 ≤ 2 s -- No query class more than 5× slower than the ClickHouse baseline -- Catalog cold start ≤ 30 s and RSS ≤ 2 GiB -- Exact project isolation and full Langfuse v4 update/delete/export semantics - -Isolation, durability, mutation, or public-semantic failure is an immediate -NO-GO. A localized performance-only miss may be conditional. - -## Links - -- [pChronicle design](../../docs/src/pchronicle/design/index.md) -- [`persisting-pchronicle`](../../crates/persisting-pchronicle/README.md) diff --git a/benchmark/langfuse-pchronicle-review/REPORT.zh.md b/benchmark/langfuse-pchronicle-review/REPORT.zh.md deleted file mode 100644 index cda11d2f6..000000000 --- a/benchmark/langfuse-pchronicle-review/REPORT.zh.md +++ /dev/null @@ -1,205 +0,0 @@ -# Langfuse ClickHouse → pChronicle 存储后端可行性评审 - -评审日期：2026-08-11 - -> [KNOWN, HIGH] 2026-08-12 的 lazy-catalog 优化已显著改变性能结论；性能章节请以 -> [RETEST-2026-08-12.zh.md](RETEST-2026-08-12.zh.md) 为准。本报告的 full-v4 -> 功能兼容结论保留为历史基线。 - -## 先看最强支持理由 - -[COMPUTED, HIGH] 如果目标被收窄为“低吞吐、追加式、按轨迹回放的原始事件库”，pChronicle 在本次 100k event PoC 中是可用的：113,000 行预载及 30 行 10 eps 在线写入均无 ack 后丢失；写入可见延迟 P95 为 4.13 ms；项目交叉点查为 0 行；SIGKILL-after-ack、writer fencing、固定快照、maintenance 后重启均通过；cold catalog 2.10 s，进程 RSS 1.04 GiB。 - -[INFERRED, HIGH] 这说明 pChronicle 适合承接 Langfuse 的 canonical/cold trajectory、回放和证据归档角色，也说明“完全不适合 Langfuse 数据”这个更强的否定命题不成立。 - -## 结论 - -**[INFERRED, HIGH] NO-GO：在钉住的代码状态和已批准的验收标准下，不应以 pChronicle 直接替换 Langfuse 的 ClickHouse 后端。** - -[COMPUTED, HIGH] 直接替换触发了四个立即否决项：重复逻辑版本仍作为物理行暴露；`bookmarked/public` 更新不支持；trace/project/retention 删除不支持；Langfuse dashboard、FTS、去重和 Parquet export 所依赖的关键 ClickHouse 语义不支持。 - -[COMPUTED, HIGH] 即使只看已可表达的查询，最终记录跑中 pChronicle point/list P95 分别是 ClickHouse 的 82.02×/190.13×，超过“不慢于 5×”门槛。第二次完整 pChronicle 跑的 point P95 为 527.94 ms，说明 500 ms 的绝对 point 门槛也不稳定。 - -[INFERRED, HIGH] 补齐这些缺口至少需要新的 latest-version/tombstone 投影、typed OLAP projections、ClickHouse 查询语义兼容层、多租户网络服务与 catalog pruning；这已是一个新的多租户 OLAP 产品，而不是有界适配器，因此违反本次评审的停止条件。 - -[INFERRED, HIGH] 推荐保留 ClickHouse 作为 hot OLAP 和 Langfuse API/UI 的查询后端，把 pChronicle 放在旁路，作为追加式 canonical/cold trajectory store。Postgres、Redis 和现有对象存储保持不变。 - -## 评审范围与基线 - -[KNOWN, HIGH] 本评审只讨论替换 ClickHouse。Langfuse 的 Postgres 事务数据、Redis 队列/缓存、S3/对象存储不在替换范围内。Langfuse 官方架构也将这些职责分开：[Architecture](https://langfuse.com/handbook/product-engineering/architecture)、[ClickHouse infrastructure](https://langfuse.com/self-hosting/deployment/infrastructure/clickhouse)。 - -[KNOWN, HIGH] 钉住的基线为： - -- Persisting `94531cf903e5abc336de347588fb1858e9d52b6a`，加评审时已存在、未提交的 catalog/query-engine 实验代码； -- Langfuse `d18a59ad663ffc7c04afc61354186c141b3ec0f3`，工作区干净； -- ClickHouse `25.12.11.4`； -- Langfuse v4 语义。v4 对 ClickHouse 的版本和表结构要求可参见官方 [v3 → v4 upgrade guide](https://langfuse.com/self-hosting/upgrade/upgrade-guides/upgrade-v3-to-v4)。 - -[KNOWN, HIGH] Persisting 的 TTAS、Queue/Sampler、Search 和独立 `persisting-dlcapt` 不在本评审范围内。FTS 只检查现有非 Search 的 DataFusion/Lance 路径，没有借助另一个搜索子系统掩盖后端缺口。 - -[KNOWN, HIGH] 新增的实现与证据文件是： - -- `crates/persisting-pchronicle/examples/langfuse_backend_feasibility.rs`：有界 backend contract、fixture、语义/性能/资源/maintenance 探针； -- `crates/persisting-pchronicle/tests/langfuse_backend_faults.rs`：ack 后强制终止进程再恢复读取； -- `benchmark/langfuse-pchronicle-review/clickhouse_baseline.py`：相同 fixture 的 ClickHouse 25.12 对照； -- `benchmark/langfuse-pchronicle-review/recorded-results-2026-08-11.json`：最终机器可读结果； -- `benchmark/langfuse-pchronicle-review/README.md`：复现命令。 - -## 数据映射与 workload - -[KNOWN, HIGH] PoC 使用以下映射；完整 Langfuse 逻辑行同时保留在 `payload_json` 中： - -| Langfuse | pChronicle | -|---|---| -| `project_id` | `agent_id` | -| `trace_id` | Run / `root_session_id` | -| `span_id` | `call_id` | -| `parent_span_id` | `parent_call_id` | -| 无 trace 的 score/dataset/blob 行 | synthetic Run | - -[COMPUTED, HIGH] 两个后端读取的是同一份 238,470,862-byte JSONL，SHA-256 为 `1b2ef8ed2f51e6d85a9907bbbebbdafbc4c961d8c4368b2a20b737f94cfc4c90`。它包含两个 project、100,000 events（100 个重复逻辑版本、99,900 个 distinct logical IDs）、10,000 scores、2,000 dataset-run items 和 1,000 blob-log rows；另含 metadata、tags、tool names、65 KiB I/O 和 12 位小数 cost。200 条 trace 加 synthetic Runs 在 pChronicle 中形成 210 个 source datasets。 - -[KNOWN, HIGH] 在线阶段持续 3 秒、目标 10 eps，并运行一个并发查询；查询基准每类重复 7 次。该规模对应已批准的 team self-hosted/dev 验收，不代表生产容量。 - -## 验收结果 - -[COMPUTED, HIGH] 下表采用最终记录跑；原始摘要在 `recorded-results-2026-08-11.json`。 - -| Gate / workload | pChronicle | ClickHouse | 判定 | -|---|---:|---:|---| -| 113k 预载 ack loss | 0 | 0 | `[COMPUTED, HIGH]` 通过 | -| 10 eps 在线阶段 | 10.26 eps | 9.86 eps | `[COMPUTED, HIGH]` 通过 | -| 在线 ack loss | 0 / 30 | 0 / 30 | `[COMPUTED, HIGH]` 通过 | -| 可见延迟 P95 | 4.13 ms | 6.51 ms | `[COMPUTED, HIGH]` 通过 ≤2 s | -| 重复版本后的物理/逻辑 event 行 | 100,000 / 99,900 | 99,900 / 99,900 | `[COMPUTED, HIGH]` pChronicle 失败 | -| point P95 | 395.87 ms | 4.83 ms | `[COMPUTED, HIGH]` 绝对值通过，82.02× 相对门槛失败 | -| list P95 | 564.82 ms | 2.97 ms | `[COMPUTED, HIGH]` 绝对值通过，190.13× 相对门槛失败 | -| simple group/facet P95 | 806.70 ms（model group） | 3.47 ms（tag facet） | `[COMPUTED, MED]` 都过 1 s；查询并非完全同构，不用其比值作单独否决 | -| 并发查询 P95 | 350.41 ms | 13.96 ms | `[COMPUTED, HIGH]` pChronicle 为 25.10× | -| dashboard | ClickHouse curried quantile/`WITH FILL` 不支持 | 4.53 ms | `[COMPUTED, HIGH]` 失败 | -| FTS | `hasAllTokens` 不支持 | 3.47 ms | `[COMPUTED, HIGH]` 失败 | -| JSON export TTFB | 429.09 ms / 50k 行 | 68.77 ms / 10k 行 | `[COMPUTED, MED]` 都过 2 s；行数不同，不比较倍数 | -| Parquet export TTFB | 不支持 | 28.27 ms | `[COMPUTED, HIGH]` 失败 | -| update flags | 不支持 | 25.32 ms | `[COMPUTED, HIGH]` 失败 | -| trace/project/retention delete | 全部不支持 | 8.98 / 5.96 / 7.64 ms | `[COMPUTED, HIGH]` 失败 | -| catalog cold start | 2.10 s | N/A（常驻服务） | `[COMPUTED, HIGH]` 通过 ≤30 s | -| RSS / resident memory | 1,117,782,016 B | 1,422,721,024 B | `[COMPUTED, MED]` 两者采集边界不同；pChronicle 通过 ≤2 GiB | -| maintenance + restart visibility | 38.36 ms + 4.16 s，可见 | N/A | `[COMPUTED, HIGH]` 通过本次故障探针 | -| cross-project negative point | 0 行 | 0 行 | `[COMPUTED, HIGH]` 数据映射级通过 | - -[COMPUTED, HIGH] bulk append 吞吐为 pChronicle 9,374.59 rows/s、ClickHouse 73,449.48 rows/s，ClickHouse 高 7.83×；两者都足以覆盖本次 10 eps 目标，因此吞吐本身不是立即否决项。 - -[COMPUTED, HIGH] 另一轮相同 fixture 的完整跑中，pChronicle point/list P95 为 527.94/666.71 ms，ClickHouse 为 4.18/4.00 ms。跨跑波动会改变 point 是否略过 500 ms，但不会改变远超 5× 相对门槛的结论。 - -## 语义兼容矩阵 - -| Langfuse v4 能力 | 当前 pChronicle | 结论 | -|---|---|---| -| acknowledged append / batch | 原生 append 可用，PoC 零 ack loss | `[COMPUTED, HIGH]` 可适配 | -| latest-version dedup | 明确 at-least-once；重复 `event_id` 是合法物理事实 | `[KNOWN, HIGH]` 不兼容 | -| `events_full` wide typed schema | 只有少量索引列和完整 JSON payload | `[KNOWN, HIGH]` 不兼容 | -| `events_core` truncated projection/MV | 无对应 typed incremental projection | `[KNOWN, HIGH]` 不兼容 | -| scores / dataset-run items / blob log | PoC 只能以 generic event + synthetic Run 保存 | `[COMPUTED, HIGH]` 存得下，不等于 API/查询兼容 | -| `bookmarked` / `public` 更新 | adapter 明确返回 unsupported | `[COMPUTED, HIGH]` 不兼容 | -| trace/project/retention 删除 | 无逻辑 tombstone/delete contract | `[KNOWN, HIGH]` 不兼容 | -| point/list | 可由少量索引列表达 | `[COMPUTED, HIGH]` 功能可做，性能门槛失败 | -| tags/metadata/tools/cost facets | 主要埋在 JSON；无 Array/Map/Decimal 等同构列 | `[KNOWN, HIGH]` 不兼容 | -| FTS | 非 Search 路径不支持 Langfuse token SQL | `[COMPUTED, HIGH]` 不兼容 | -| dashboards/session aggregates | `argMaxIf`、`sumMap`、curried quantile、`WITH FILL` 等不兼容 | `[COMPUTED, HIGH]` 不兼容 | -| JSONEachRow export | 可流式输出 generic event JSONL | `[COMPUTED, HIGH]` 格式/列语义仍不等同 | -| Parquet export | public query adapter 不支持 | `[COMPUTED, HIGH]` 不兼容 | -| Langfuse migrations/direct schema assumptions | 无 ClickHouse-compatible schema/migration surface | `[KNOWN, HIGH]` 不兼容 | - -## 静态代码证据 - -[KNOWN, HIGH] Langfuse `events_full` 是 wide typed OLAP 表：`Array`/`Map`/`Decimal(18,12)`、materialized cost、bloom/minmax/text indexes、`ReplacingMergeTree(event_ts, is_deleted)`，且主键/排序键以 project、minute、trace、span、time 为中心。见相邻 Langfuse 仓库的 `packages/shared/clickhouse/migrations/unclustered/0039_create_events_full.up.sql:1`。 - -[KNOWN, HIGH] `events_core` 由增量 materialized view 从 full 表生成，并将 I/O 和 metadata 值截断到 200 字符，承担轻量列表查询。见相邻 Langfuse 仓库的 `packages/shared/clickhouse/migrations/unclustered/0041_create_events_core_mv.up.sql:1`。 - -[KNOWN, HIGH] Langfuse writer 为 events、scores、dataset-run items 和 blob log 提供 batch/retry/async acknowledged inserts；client 强制 `async_insert=1`、`wait_for_async_insert=1`。见相邻 Langfuse 仓库的 `worker/src/services/ClickhouseWriter/index.ts:35` 和 `packages/shared/src/server/clickhouse/client.ts:190`。 - -[KNOWN, HIGH] Langfuse repository 会同步更新 full/core 的 `bookmarked/public`，并按 project+trace、project、project+time 执行删除；同时提供 JSON/raw text/Parquet 流式 export。见相邻 Langfuse 仓库的 `packages/shared/src/server/repositories/events.ts:1948`、`:2246`、`:2334`、`:2400`、`:3139`。 - -[KNOWN, HIGH] query builder 使用 `argMaxIf`、`sumMap`、`groupUniqArray*` 和 `LIMIT 1 BY`，FTS 使用 `tokens` + `hasAllTokens` 再叠加精确 predicate。见相邻 Langfuse 仓库的 `packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts:522`、`:710`、`:1437` 和 `packages/shared/src/server/queries/clickhouse-sql/fts.ts:74`。 - -[COMPUTED, HIGH] 对 `web/`、`worker/`、`packages/shared/` 下 `.ts/.tsx` 的词法扫描得到 423 个文件含 ClickHouse 大小写变体；`LIMIT 1 BY` 53 次、`argMaxIf` 35 次、`sumMap` 38 次、`hasAllTokens` 16 次、`WITH FILL` 21 次。这些是包含测试/注释的 occurrence counts，不是迁移工期估算，但足以反驳“只替换一个数据库 client 即可”的假设。 - -[KNOWN, HIGH] pChronicle 的 canonical Lance writer 自述为 at-least-once、append-only，重复 `event_id` 每次都成为物理行；重维护不在 ingestion path。见 `crates/persisting-pchronicle/src/store/events/mod.rs`。 - -[KNOWN, HIGH] pChronicle `EventRow` 只有 seq、标识/时间、kind/source、agent/session/call/trace/model 等索引字段，其他内容进入 `payload_json`。见 `crates/persisting-pchronicle/src/store/event_row.rs:7`。 - -[KNOWN, HIGH] query engine 是 read-only，只允许单条 `SELECT/VALUES/DESCRIBE/EXPLAIN`。见 `crates/persisting-pchronicle/src/store/query_engine.rs:93`、`:569`。 - -[KNOWN, HIGH] 当前实验 catalog 会在 snapshot discovery 时打开每个 source，并对 event source 执行 `SELECT * FROM events ORDER BY seq`、collect 后物化 normalized MemTables。见 `crates/persisting-pchronicle/src/store/catalog/`。 - -[INFERRED, HIGH] Langfuse 的主访问模式是 project/time 范围 OLAP；当前 pChronicle 以 Run 为物理单元并在 catalog cold path 打开、归一化每个 source。210 sources 尚可在 2.10 s 打开，但该机制没有给出 source 数量增长时的稳定上界，因此不能从本次小规模结果外推到长期多租户项目。 - -## SQL 兼容探针 - -[COMPUTED, HIGH] 当前 DataFusion 路径无法 plan：`approx_top_k`、`argMax`、`arrayJoin`、`hasAllTokens`、`JSONExtractString`、`LIMIT 1 BY`、curried `quantile`、`sumMap`、`WITH FILL`。 - -[COMPUTED, HIGH] `FINAL` 和 `PREWHERE` 文本在探针中没有报 parser/planner error。 - -[INFERRED, HIGH] 这不证明语义兼容：`FINAL` 可能被当作 alias/no-op 接受，而 `PREWHERE` 能被解析也不意味着拥有 ClickHouse 的存储级 prewhere 和 latest-version 行为。因此二者没有记为通过。 - -## ClickHouse best-practices 检查 - -[KNOWN, HIGH] 已逐项检查该仓库 skill 中的 28 条规则。对本评审有直接影响的结论如下： - -- [KNOWN, HIGH] 按 `schema-pk-plan-before-creation`、`schema-pk-prioritize-filters`、`schema-pk-filter-on-orderby`、`schema-pk-cardinality-order`，对照 schema 保留了 Langfuse 的 project/time/trace 优先顺序；这正是 point/list/facet 快的关键结构之一。 -- [KNOWN, HIGH] 按 `schema-types-native-types`、`schema-types-lowcardinality`、`schema-types-minimize-bitwidth`、`schema-json-when-to-use`，常用过滤/聚合字段使用 native Array/Map/Decimal/LowCardinality；JSON 只作为完整 payload，不替代 typed projection。pChronicle 当前恰好缺这层 projection。 -- [KNOWN, HIGH] 按 `schema-partition-lifecycle`、`schema-partition-low-cardinality`、`schema-partition-query-tradeoffs`、`schema-partition-start-without`，复核了 Langfuse 的月分区与 retention 模型；本 PoC 没有为 pChronicle 创造第二套分区系统。 -- [KNOWN, HIGH] 按 `insert-batch-size`、`insert-async-small-batches`，ClickHouse 对照使用 batch 和 `async_insert=1, wait_for_async_insert=1`。`insert-format-native` 建议更高效的 Native 格式，但对照保留 Langfuse 实际 `JSONEachRow`，避免用更优但非现状的 producer 美化基线。 -- [KNOWN, HIGH] 按 `insert-optimize-avoid-final`、`insert-mutation-avoid-update`、`insert-mutation-avoid-delete`，频繁 mutation/`FINAL` 应慎用；但 Langfuse 已公开要求 bookmarks/public/delete/retention 语义，不能以“最佳实践”名义删掉。可行的替代仍需版本行+tombstone+latest projection，而当前 pChronicle 没有。 -- [KNOWN, HIGH] 按 `query-index-skipping-indices`、`query-mv-incremental`，对照保留 text/bloom/minmax indexes 与 full→core 增量 MV。`query-mv-refreshable` 不适用于这条 ingestion path。 -- [KNOWN, HIGH] `query-join-filter-before`、`query-join-consider-alternatives`、`query-join-use-any`、`query-join-choose-algorithm`、`query-join-null-handling` 已检查；本次 point/list/facet 对照没有 join，故它们不是判定因素。 -- [KNOWN, HIGH] `schema-types-avoid-nullable` 和 `schema-types-enum` 已检查；Langfuse schema 的 nullable 时间/版本字段和 LowCardinality strings 属于现有兼容约束，本 PoC 不借机重构它们。 - -## 故障、隔离与安全 - -[COMPUTED, HIGH] `langfuse_backend_faults` 通过 2/2：子进程 append 并输出 ack 后被强制终止，新的 process/runtime 仍读到唯一目标行。 - -[COMPUTED, HIGH] targeted regression tests 通过：`production_scale` 5 passed、1 ignored；`query_engine` 14 passed；`store::catalog` filter 6 passed。覆盖 writer fencing、固定 snapshot 在 append 期间不漂移、独立 Runs、maintenance/restart 和 read-only SQL boundary。 - -[KNOWN, HIGH] 当前 pChronicle server 是 loopback-only single-user UI，并明确因为没有 authentication 而拒绝 non-loopback bind；它提供 query/export/maintenance routes，但没有供 Langfuse web/worker 使用的 canonical authenticated network append service。见 `crates/persisting-pchronicle-server/src/lib.rs:1`、`:155`、`:168`。 - -[COMPUTED, HIGH] PoC 中 `project_id → agent_id` 且 cross-project negative point 为 0，证明当前测试 adapter 没有在该查询中串项目。 - -[INFERRED, HIGH] 这不等于服务级 tenant isolation：共享 catalog 能看见多个项目，本身没有 Langfuse project RBAC/row policy。若暴露任意 SQL endpoint，调用者可能绕过 adapter 的 project predicate。 - -[INFERRED, HIGH] 因此未来旁路 dual-write 只能使用服务端固定 endpoint、强认证（mTLS 或短期 service token）、每请求显式 project scope、最小化 SQL surface、传输/静态加密和无 payload/credential 日志；不得允许 tenant 自定义出站 host。此次实现没有新增生产网络端点或扩大权限。 - -## 为什么不是“再补几个函数” - -[INFERRED, HIGH] 满足 full Langfuse v4 至少要新增以下五个耦合层： - -1. event version、logical tombstone、latest-row projection，以及 update/delete/retention 的一致性契约； -2. `events_full/events_core/scores/dataset_run_items/blob_log` 的 typed projections、索引、迁移和生命周期； -3. dashboard、sessions、facets、FTS、export 对应的 SQL/执行语义和差分测试； -4. web/worker 可用的 authenticated network write/query service 与 tenant authorization； -5. 跨大量 Runs 的 durable metadata、source pruning、增量 catalog，而不是 cold path 全打开和全 normalize。 - -[INFERRED, HIGH] 这些层彼此依赖：没有 version projection 就无法正确聚合；没有 tenant-aware service 就不能部署；没有 typed/pruned projection 就无法达到现有查询延迟。它们构成产品级扩展，不属于批准的 bounded integration。 - -## 推荐落点 - -[INFERRED, HIGH] 建议采用并行角色，而非替换角色： - -- ClickHouse 继续承载 Langfuse hot OLAP、lists/facets/dashboards/FTS、更新删除以及 JSON/Parquet export； -- pChronicle 接收原始、追加式 trajectory 副本，服务 replay、审计证据、冷归档、离线研究； -- dual-write 失败必须有独立重试/回补水位，不能让 pChronicle ack 替代现有 ClickHouse ack； -- project deletion/retention 必须在两端有可审计的 lifecycle 协议后，才可处理真实租户数据。 - -[INFERRED, MED] 一个后续有界 pilot 可以只做“单 project、原始 event dual-write + trace replay”，不接管 dashboards、FTS、facets、updates/deletes，也不宣称它是 Langfuse analytics backend。 - -## 限制与未知边界 - -[KNOWN, HIGH] 记录环境为 Apple M4、24 GiB RAM、macOS 26.5；pChronicle 是本地 release binary，ClickHouse 是本地 Docker container。没有隔离所有后台负载，因此数值不是可发布的容量基准。 - -[KNOWN, HIGH] 本次只有单进程、一个并发查询、200 traces/210 sources、两次主要完整跑；没有 HA、远端对象存储故障、长期 compaction、百万级 Runs、多日 retention 或真实租户 payload。 - -[KNOWN, HIGH] ClickHouse PoC schema 是 Langfuse v4 相关模式的代表性子集，不是完整部署；full API/UI 没有做 E2E，因为静态兼容矩阵已触发 hard NO-GO。该限制不会反转 mutation/dedup/SQL/export 的确定性失败，但会限制对所有边缘查询的覆盖度。 - -[KNOWN, HIGH] pChronicle catalog/query-engine 基于评审时未提交的实验代码。结论只对上文钉住的 snapshot 有效，后续实现变化必须重跑同一套 fixture、fault tests 和语义矩阵。 - -[RULES I BROKE]: none diff --git a/benchmark/langfuse-pchronicle-review/RETEST-2026-08-12.zh.md b/benchmark/langfuse-pchronicle-review/RETEST-2026-08-12.zh.md deleted file mode 100644 index 5db05dfef..000000000 --- a/benchmark/langfuse-pchronicle-review/RETEST-2026-08-12.zh.md +++ /dev/null @@ -1,86 +0,0 @@ -# pChronicle 优化后复测（2026-08-12） - -## 结论 - -**[INFERRED, HIGH] 本次 lazy catalog / source-pruning 优化有效；按“serverless Lite tracing backend”范围，性能与可靠性可以进入 Conditional GO 原型阶段。** - -[COMPUTED, HIGH] 相同 113,000-row fixture 下，catalog cold start 从 2,104.04 ms 降到 35.04 ms，warm point/list/simple aggregate P95 分别改善 34.8%/49.9%/63.2%。 - -[COMPUTED, HIGH] 当 adapter 能把 Run 映射成 exact `_file_` predicate 时，catalog point P95 为 3.01 ms；此前记录的 ClickHouse point P95 为 4.83 ms。这个本地结果说明 per-Run Lance format 不是 point path 的主要性能障碍。 - -[COMPUTED, HIGH] project-path pruning 后 list P95 为 167.17 ms，满足 Lite 场景 1 s 绝对门槛，但仍是 ClickHouse 记录值的 56.3×。跨 Run list/aggregate 仍是主要差距。 - -[KNOWN, HIGH] full Langfuse v4 harness 仍会报告 unsupported mutation/delete/FTS/dashboard/Parquet 和 duplicate-version 语义。这些能力本次没有实现；若 Lite profile 明确关闭它们，就不应把它们解读成性能回归。 - -## 可比性 - -[COMPUTED, HIGH] 新旧 pChronicle 与 ClickHouse 使用相同 fixture：238,470,862 bytes、113,000 rows，SHA-256 `1b2ef8ed2f51e6d85a9907bbbebbdafbc4c961d8c4368b2a20b737f94cfc4c90`。 - -[KNOWN, HIGH] Persisting HEAD 仍为 `94531cf903e5abc336de347588fb1858e9d52b6a`；本次测量包含 2026-08-12 工作区内未提交的 lazy catalog/query changes。关键文件 SHA 已写入 `recorded-results-2026-08-12.json`。 - -[KNOWN, HIGH] ClickHouse 未重新启动；因 fixture SHA 未变，复用 2026-08-11 在同一机器记录的 ClickHouse 25.12.11.4 baseline，避免额外容器噪声。 - -## 结果 - -| Metric | 旧 pChronicle | 新 pChronicle | 变化 | -|---|---:|---:|---:| -| catalog cold start | 2,104.04 ms | 35.04 ms | `[COMPUTED, HIGH]` -98.3% | -| catalog 后 RSS | 1,117,782,016 B | 316,669,952 B | `[COMPUTED, HIGH]` -71.7% | -| 新版执行 unpruned queries 后 RSS | N/A | 624,885,760 B | `[COMPUTED, HIGH]` 仍低于旧 eager catalog 44.1% | -| warm point P95 | 395.87 ms | 258.13 ms | `[COMPUTED, HIGH]` -34.8% | -| warm list P95 | 564.82 ms | 282.80 ms | `[COMPUTED, HIGH]` -49.9% | -| simple model aggregate P95 | 806.70 ms | 296.71 ms | `[COMPUTED, HIGH]` -63.2% | -| concurrent query P95 @10 eps | 350.41 ms | 243.45 ms | `[COMPUTED, HIGH]` -30.5% | -| JSON export TTFB / 50k rows | 429.09 ms | 336.69 ms | `[COMPUTED, HIGH]` -21.5% | -| append throughput | 9,374.59 rows/s | 9,185.15 rows/s | `[COMPUTED, MED]` -2.0%，视为无改善/运行噪声 | -| visibility P95 | 4.13 ms | 5.46 ms | `[COMPUTED, HIGH]` 仍远低于 2 s | - -[COMPUTED, HIGH] 当前代码三次完整跑的 catalog cold start 为 35.04–35.36 ms，warm point 为 258.13–262.03 ms，aggregate 为 292.39–299.31 ms；这些核心改善具有重复性。warm list 为 282.80–352.99 ms，波动较大但仍明显好于旧版 564.82–666.71 ms。 - -## Lazy/pruning 路径 - -| Path | Catalog/open | Query P95 | End-to-end approximation | -|---|---:|---:|---:| -| fresh global point，210 sources | 10.84 ms | 302.94 ms | `[COMPUTED, HIGH]` 313.77 ms | -| fresh global list，210 sources | 12.47 ms | 326.68 ms | `[COMPUTED, HIGH]` 339.16 ms | -| direct Run point | 0.51 ms | 2.14 ms | `[COMPUTED, HIGH]` 2.64 ms | -| exact `_file_` catalog point | 12.34 ms | 3.01 ms | `[COMPUTED, HIGH]` 15.35 ms | -| project `_file_ LIKE` list | 12.02 ms | 167.17 ms | `[COMPUTED, HIGH]` 179.20 ms | - -[INFERRED, HIGH] 结果支持此前的因果判断：主要瓶颈是跨 Run fan-out，而不是 Lance 文件格式。exact source 被选中后，catalog provider 能在 resolve 前排除其他 209 个 source，point latency 即降到毫秒级。 - -[INFERRED, HIGH] Langfuse Lite adapter 必须把已知 project/trace/Run 映射成 `_file_` predicate 或直接 Run URI；只传 `agent_id/event_id` 会触发全 catalog source resolution，丢失本次优化的大部分收益。 - -[COMPUTED, HIGH] restart 的 append+catalog 为 51.76 ms，随后首次全局 visibility query 为 344.02 ms，端到端 395.78 ms且目标行可见。该拆分避免把 lazy catalog 的快速构建误报成完整 restart latency。 - -## Durability 与回归测试 - -[COMPUTED, HIGH] 所有 targeted tests 通过： - -- `langfuse_backend_faults`: 2 passed； -- `production_scale`: 5 passed，1 ignored stress test； -- `query_engine`: 14 passed； -- catalog unit tests: 9 passed，包括 lazy resolution 与 source pruning； -- raw event Lance/row unit tests: 21 passed； -- `persisting-pchronicle-server --lib`: 16 passed。 - -[KNOWN, HIGH] 唯一重复出现的 warning 是 macOS linker 的 `__eh_frame section too large`；没有测试因此失败。 - -## Rules checked - -- [KNOWN, HIGH] Per `schema-pk-prioritize-filters` / `schema-pk-filter-on-orderby`：exact `_file_` 相当于先使用物理 source key；这是新版 point path 的关键合规点。 -- [KNOWN, HIGH] Per `schema-partition-low-cardinality`：per-Run source 仍是高基数布局；lazy resolution 缓解 metadata/open cost，但 project-wide query 仍随选中 Run 数增长。 -- [KNOWN, HIGH] Per `query-index-skipping-indices`：source-level pruning 已有效；source 内 `event_id/time` scalar index 仍可进一步降低 scan。 -- [KNOWN, HIGH] Per `schema-types-native-types` / `schema-json-when-to-use`：当前 timestamp/known Langfuse fields 仍偏 UTF-8/opaque JSON；这更影响跨 Run sort/aggregate，而非已路由的 point。 -- [KNOWN, HIGH] Per `query-mv-incremental`：Lite profile 若不做 dashboards，可暂不引入完整 OLAP MV；若未来需要 project-wide analytics，应增加小型 typed projection，而不是重新 eager-normalize catalog。 -- [KNOWN, HIGH] Per `insert-batch-size` / `insert-async-small-batches`：本次优化未改善 append throughput；per-Run commit/fragment batching 仍是独立优化方向。 - -## 判定 - -[INFERRED, HIGH] 对明确缩减能力的 `PCHRONICLE_LITE`：性能、cold start、内存、visibility 和 durability 达到本次 dev-scale absolute gates，可以继续实现 adapter prototype。 - -[INFERRED, HIGH] 对完整 ClickHouse-compatible backend：项目级查询相对 ClickHouse 的差距和未实现语义仍存在，因此旧的 full-v4 NO-GO 不被本次性能优化推翻。 - -[INFERRED, HIGH] 下一优先级不是替换存储格式，而是让 adapter 强制走 project/Run source pruning，并增加 project-level run index；之后再测 1/10/100/1000 Runs 的规模曲线。 - -[RULES I BROKE]: none diff --git a/benchmark/langfuse-pchronicle-review/SERVER-ACCELERATION-2026-08-12.zh.md b/benchmark/langfuse-pchronicle-review/SERVER-ACCELERATION-2026-08-12.zh.md deleted file mode 100644 index 003f1e006..000000000 --- a/benchmark/langfuse-pchronicle-review/SERVER-ACCELERATION-2026-08-12.zh.md +++ /dev/null @@ -1,70 +0,0 @@ -# pChronicle Server source-routing 加速复测 - -日期：2026-08-12 - -## 结论 - -本轮 Server 加速把可唯一定位的 warm point query 从 pChronicle 原来的 `258.13ms` 降到 -HTTP P95 `4.72ms`，与此前 ClickHouse `4.83ms` 基线处于同一数量级。项目 list 和 aggregate -分别降到 `151.05ms` 和 `165.41ms`，绝对延迟可用，但仍约为 ClickHouse 的 `50.8x` 和 -`47.7x`。 - -因此 source-routing index 已经消除了 point query 的主要 fan-out 成本；项目级扫描仍需要 -访问 106 个 source，剩余差距主要来自多 source scan/merge/sort，而不是 Catalog 初始化。 - -## 实现边界 - -- `DatasetCatalogSnapshot`、`CatalogDataset`、`DiscoveredSource` 和持久化格式未改变。 -- `CatalogRuntime` 同代持有可重建的 `ServerAcceleration`。 -- Run、event identity、event partition 分别 lazy single-flight 构建。 -- value 使用每代随机 keyed 64-bit fingerprint，collision 只扩大 source 候选；原 SQL 谓词 - 负责最终过滤。 -- 只改写简单单表查询的顶层必要等值/`IN` 条件；不确定时回退原 SQL。 -- 索引构建使用 Arrow stream，并限制为 100 万行、100 万 distinct value。 -- refresh 原子发布新 snapshot/engine/空索引，旧索引不会跨 generation 复用。 - -## 113k 行 release 结果 - -| 场景 | 结果 | -|---|---:| -| Catalog cold（3 次中位数） | `37.15ms` | -| cold identity point，含首次 identity index | `134.26–166.17ms`，中位数 `159.45ms` | -| Catalog + cold identity point（中位数相加） | `196.60ms` | -| warm point HTTP P50 / P95 | `4.15 / 4.72ms` | -| 首次 project list，含 partition index | `312.94ms` | -| warm project list HTTP P50 / P95 | `147.24 / 151.05ms` | -| warm project aggregate HTTP P50 / P95 | `158.63 / 165.41ms` | - -Warm 数字各采样 20 次。point 命中 1 个 source；project-a list/aggregate 命中 106 个 source。 - -## 与 ClickHouse 基线比较 - -| 场景 | pChronicle Server P95 | ClickHouse P95 | 比率 | -|---|---:|---:|---:| -| point | `4.72ms` | `4.83ms` | `0.98x` | -| project list | `151.05ms` | `2.97ms` | `50.84x` | -| project aggregate | `165.41ms` | `3.47ms` | `47.72x` | - -这里 pChronicle 包含 Axum 和 loopback HTTP 开销，ClickHouse 数字复用 2026-08-11 的相同 -fixture 基线，并非同轮并发 A/B。因此 point 的约 2% 差异只能解释为接近持平，不能解释为 -严格引擎胜负。 - -## 内存说明 - -partition index 记录 113,031 行、211 个 source、213 个 distinct fingerprint；identity index -记录相同行数和 113,132 个 distinct fingerprint。fresh process 中 Catalog 后 RSS 为 -`26.2MiB`，完成 identity index 与 point query 后为 `101.9MiB`。 - -这个 RSS 增量同时包含 211 个 LazySource 的首次解析、Lance/DataFusion scan buffer、索引和 -allocator retention，不能当作索引对象自身大小。两级 lazy index 的意义是：只运行项目列表 -时不会建立 113k-value identity map；超出硬限制时索引不会发布，查询回退原 fan-out 路径。 - -## 验证 - -- `cargo test -p persisting-pchronicle-server --lib`：21 passed。 -- `cargo clippy -p persisting-pchronicle-server --all-targets -- -D warnings`：passed。 -- 真实双 `events.lance` 测试验证 point/project 路由、结果等价和两级 lazy build。 -- Server 集成测试验证 `_file_` 注入、quoted alias、显式 `_file_` 保留以及 refresh 清空索引。 - -机器可读数据见 -[`server-acceleration-results-2026-08-12.json`](server-acceleration-results-2026-08-12.json)。 diff --git a/benchmark/langfuse-pchronicle-review/clickhouse_baseline.py b/benchmark/langfuse-pchronicle-review/clickhouse_baseline.py deleted file mode 100644 index a0eb15f20..000000000 --- a/benchmark/langfuse-pchronicle-review/clickhouse_baseline.py +++ /dev/null @@ -1,647 +0,0 @@ -#!/usr/bin/env python3 -"""Run the Langfuse-shaped fixture against an isolated ClickHouse 25.12 server. - -The script uses only the ClickHouse HTTP API, never prints credentials, and -writes one JSON report. It is intended for the disposable database named -``langfuse_pchronicle_review``. -""" - -from __future__ import annotations - -import argparse -import json -import math -import pathlib -import threading -import time -import urllib.error -import urllib.parse -import urllib.request -from typing import Any, Iterable - -DATABASE = "langfuse_pchronicle_review" -LOAD_ROWS_PER_SECOND = 10 - -COLUMNS = """ - row_kind LowCardinality(String), - project_id String, - trace_id String, - span_id String, - parent_span_id String, - logical_id String, - event_ts DateTime64(3), - start_time DateTime64(3), - name LowCardinality(String), - event_type LowCardinality(String), - session_id String, - user_id String, - environment LowCardinality(String), - tags Array(String), - metadata_names Array(String), - metadata_values Array(String), - tool_names Array(String), - model LowCardinality(String), - input String CODEC(ZSTD(3)), - output String CODEC(ZSTD(3)), - usage_input UInt64, - usage_output UInt64, - total_cost Decimal(38, 12), - version UInt32, - is_deleted Bool, - bookmarked Bool, - public Bool, - dataset_id String, - dataset_run_id String, - storage_run_id String, - payload_json String CODEC(ZSTD(3)) -""" - -COLUMN_NAMES = [ - "row_kind", - "project_id", - "trace_id", - "span_id", - "parent_span_id", - "logical_id", - "event_ts", - "start_time", - "name", - "event_type", - "session_id", - "user_id", - "environment", - "tags", - "metadata_names", - "metadata_values", - "tool_names", - "model", - "input", - "output", - "usage_input", - "usage_output", - "total_cost", - "version", - "is_deleted", - "bookmarked", - "public", - "dataset_id", - "dataset_run_id", - "storage_run_id", - "payload_json", -] - - -class ClickHouseHttp: - def __init__(self, url: str, user: str, password: str, timeout: float = 120.0): - self.url = url.rstrip("/") - self.timeout = timeout - self.headers = { - "X-ClickHouse-User": user, - "X-ClickHouse-Key": password, - } - - def request( - self, - query: str, - body: bytes | None = None, - settings: dict[str, str | int] | None = None, - ) -> bytes: - params = dict(settings or {}) - if body is not None: - params["query"] = query - data = body - else: - data = query.encode("utf-8") - url = self.url - if params: - url += "?" + urllib.parse.urlencode(params) - request = urllib.request.Request(url, data=data, headers=self.headers, method="POST") - try: - with urllib.request.urlopen(request, timeout=self.timeout) as response: - return response.read() - except urllib.error.HTTPError as error: - detail = error.read().decode("utf-8", errors="replace") - raise RuntimeError(f"ClickHouse HTTP {error.code}: {detail[:1000]}") from error - - def json_query(self, query: str) -> dict[str, Any]: - document = self.request(query.rstrip().rstrip(";") + " FORMAT JSON") - return json.loads(document) - - def scalar(self, query: str) -> Any: - result = self.json_query(query) - if len(result.get("data", [])) != 1 or len(result["data"][0]) != 1: - raise RuntimeError(f"expected one scalar result for query: {query}") - return next(iter(result["data"][0].values())) - - def stream(self, query: str, output_format: str) -> dict[str, Any]: - sql = query.rstrip().rstrip(";") + f" FORMAT {output_format}" - request = urllib.request.Request( - self.url, - data=sql.encode("utf-8"), - headers=self.headers, - method="POST", - ) - started = time.perf_counter() - with urllib.request.urlopen(request, timeout=self.timeout) as response: - first = response.read(1) - first_byte = time.perf_counter() - remainder = response.read() - finished = time.perf_counter() - return { - "bytes": len(first) + len(remainder), - "first_byte_ms": (first_byte - started) * 1000.0, - "elapsed_ms": (finished - started) * 1000.0, - } - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser() - parser.add_argument("--url", required=True, help="ClickHouse HTTP URL") - parser.add_argument("--user", default="default") - parser.add_argument("--password", default="") - parser.add_argument("--fixture", required=True, type=pathlib.Path) - parser.add_argument("--output", required=True, type=pathlib.Path) - parser.add_argument("--load-seconds", type=int, default=3) - parser.add_argument("--insert-batch-rows", type=int, default=10_000) - return parser.parse_args() - - -def execute_ddl(client: ClickHouseHttp) -> None: - client.request(f"DROP DATABASE IF EXISTS {DATABASE}") - client.request(f"CREATE DATABASE {DATABASE}") - client.request( - f""" - CREATE TABLE {DATABASE}.staging ({COLUMNS}) - ENGINE = MergeTree - ORDER BY (row_kind, project_id, event_ts, logical_id) - """ - ) - client.request( - f""" - CREATE TABLE {DATABASE}.events_full ( - {COLUMNS}, - INDEX idx_span_id span_id TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_trace_id trace_id TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_user_id user_id TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_session_id session_id TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_fts_input lower(input) TYPE text(tokenizer = splitByNonAlpha), - INDEX idx_fts_output lower(output) TYPE text(tokenizer = splitByNonAlpha), - INDEX idx_fts_metadata_values metadata_values TYPE text(tokenizer = splitByNonAlpha) - ) - ENGINE = ReplacingMergeTree(event_ts, is_deleted) - PARTITION BY toYYYYMM(start_time) - PRIMARY KEY (project_id, toStartOfMinute(start_time), xxHash32(trace_id)) - ORDER BY (project_id, toStartOfMinute(start_time), xxHash32(trace_id), span_id, start_time) - SAMPLE BY xxHash32(trace_id) - SETTINGS index_granularity_bytes = '64Mi', enable_full_text_index = 1 - """ - ) - client.request( - f""" - CREATE TABLE {DATABASE}.events_core ( - {COLUMNS}, - INDEX idx_trace_id trace_id TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_fts_metadata_values metadata_values TYPE text(tokenizer = splitByNonAlpha) - ) - ENGINE = ReplacingMergeTree(event_ts, is_deleted) - PARTITION BY toYYYYMM(start_time) - PRIMARY KEY (project_id, toStartOfMinute(start_time), xxHash32(trace_id)) - ORDER BY (project_id, toStartOfMinute(start_time), xxHash32(trace_id), span_id, start_time) - SAMPLE BY xxHash32(trace_id) - SETTINGS enable_full_text_index = 1 - """ - ) - projected = [] - for column in COLUMN_NAMES: - if column == "input": - projected.append("leftUTF8(input, 200) AS input") - elif column == "output": - projected.append("leftUTF8(output, 200) AS output") - elif column == "metadata_values": - projected.append( - "arrayMap(value -> leftUTF8(value, 200), metadata_values) AS metadata_values" - ) - else: - projected.append(column) - client.request( - f""" - CREATE MATERIALIZED VIEW {DATABASE}.events_core_mv TO {DATABASE}.events_core AS - SELECT {", ".join(projected)} FROM {DATABASE}.events_full - """ - ) - for table, order_by in [ - ("scores", "(project_id, name, event_ts, logical_id)"), - ("dataset_run_items_rmt", "(project_id, dataset_id, dataset_run_id, logical_id)"), - ("blob_storage_file_log", "(project_id, event_ts, logical_id)"), - ( - "mutation_events", - "(project_id, toStartOfMinute(start_time), trace_id, span_id, start_time)", - ), - ]: - client.request( - f""" - CREATE TABLE {DATABASE}.{table} ({COLUMNS}) - ENGINE = ReplacingMergeTree(event_ts, is_deleted) - PARTITION BY toYYYYMM(start_time) - ORDER BY {order_by} - """ - ) - - -def batches(lines: Iterable[bytes], size: int) -> Iterable[bytes]: - batch: list[bytes] = [] - for line in lines: - if line.strip(): - batch.append(line) - if len(batch) == size: - yield b"".join(batch) - batch = [] - if batch: - yield b"".join(batch) - - -def load_fixture(client: ClickHouseHttp, fixture: pathlib.Path, batch_rows: int) -> dict[str, Any]: - started = time.perf_counter() - inserted = 0 - insert_latencies: list[float] = [] - with fixture.open("rb") as source: - for body in batches(source, batch_rows): - row_count = body.count(b"\n") - insert_started = time.perf_counter() - client.request( - f"INSERT INTO {DATABASE}.staging FORMAT JSONEachRow", - body=body, - settings={ - "async_insert": 1, - "wait_for_async_insert": 1, - "input_format_skip_unknown_fields": 1, - }, - ) - insert_latencies.append((time.perf_counter() - insert_started) * 1000.0) - inserted += row_count - elapsed = time.perf_counter() - started - return { - "acknowledged_rows": inserted, - "elapsed_ms": elapsed * 1000.0, - "rows_per_second": inserted / elapsed, - "batch_ack_p95_ms": percentile(insert_latencies, 0.95), - } - - -def populate_tables(client: ClickHouseHttp) -> None: - columns = ", ".join(COLUMN_NAMES) - for table, row_kind in [ - ("events_full", "event"), - ("scores", "score"), - ("dataset_run_items_rmt", "dataset_run_item"), - ("blob_storage_file_log", "blob_storage_file_log"), - ]: - client.request( - f"INSERT INTO {DATABASE}.{table} ({columns}) " - f"SELECT {columns} FROM {DATABASE}.staging WHERE row_kind = '{row_kind}'" - ) - client.request( - f"INSERT INTO {DATABASE}.mutation_events ({columns}) " - f"SELECT {columns} FROM {DATABASE}.staging WHERE row_kind = 'event'" - ) - - -def measure_query(client: ClickHouseHttp, sql: str, repetitions: int = 7) -> dict[str, Any]: - samples: list[float] = [] - result_rows = 0 - for _ in range(repetitions): - started = time.perf_counter() - result = client.json_query(sql) - samples.append((time.perf_counter() - started) * 1000.0) - result_rows = int(result.get("rows", len(result.get("data", [])))) - return { - "repetitions": repetitions, - "result_rows": result_rows, - "p50_ms": percentile(samples, 0.50), - "p95_ms": percentile(samples, 0.95), - "max_ms": max(samples), - } - - -def run_load_phase( - client: ClickHouseHttp, fixture: pathlib.Path, load_seconds: int -) -> dict[str, Any]: - with fixture.open("r", encoding="utf-8") as source: - template = json.loads(next(source)) - query_samples: list[float] = [] - query_errors: list[str] = [] - stop = threading.Event() - - def query_loop() -> None: - while not stop.is_set(): - started = time.perf_counter() - try: - client.scalar( - f"SELECT count() FROM {DATABASE}.events_core WHERE project_id = 'project-a'" - ) - query_samples.append((time.perf_counter() - started) * 1000.0) - except Exception as error: # recorded and surfaced in the report - query_errors.append(str(error)[:500]) - return - stop.wait(0.25) - - thread = threading.Thread(target=query_loop, name="clickhouse-review-query", daemon=True) - thread.start() - ack_samples: list[float] = [] - visibility_samples: list[float] = [] - load_rows = 0 - phase_started = time.perf_counter() - load_elapsed = 0.0 - try: - for second in range(load_seconds): - rows = [] - for offset in range(LOAD_ROWS_PER_SECOND): - index = second * LOAD_ROWS_PER_SECOND + offset - row = dict(template) - row["logical_id"] = f"project-a-clickhouse-load-{index:04d}" - row["trace_id"] = "project-a-clickhouse-load-trace" - row["span_id"] = f"project-a-clickhouse-load-span-{index:04d}" - row["storage_run_id"] = row["trace_id"] - row["event_ts"] = f"2026-01-01 00:05:{index // 10:02d}.{offset * 100:03d}" - row["start_time"] = row["event_ts"] - row["payload_json"] = json.dumps(row, ensure_ascii=False, separators=(",", ":")) - rows.append(json.dumps(row, ensure_ascii=False, separators=(",", ":"))) - target = phase_started + second + 1 - remaining = target - time.perf_counter() - if remaining > 0: - time.sleep(remaining) - body = ("\n".join(rows) + "\n").encode("utf-8") - started = time.perf_counter() - client.request( - f"INSERT INTO {DATABASE}.events_full FORMAT JSONEachRow", - body=body, - settings={ - "async_insert": 1, - "wait_for_async_insert": 1, - "input_format_skip_unknown_fields": 1, - }, - ) - ack_samples.append((time.perf_counter() - started) * 1000.0) - load_rows += len(rows) - visibility_started = time.perf_counter() - visible = int( - client.scalar( - f"SELECT count() FROM {DATABASE}.events_full " - "WHERE logical_id LIKE 'project-a-clickhouse-load-%'" - ) - ) - if visible != load_rows: - raise RuntimeError( - f"acknowledged ClickHouse rows not visible: expected {load_rows}, got {visible}" - ) - visibility_samples.append((time.perf_counter() - visibility_started) * 1000.0) - load_elapsed = time.perf_counter() - phase_started - finally: - stop.set() - thread.join(timeout=10) - visible = int( - client.scalar( - f"SELECT count() FROM {DATABASE}.events_full " - "WHERE logical_id LIKE 'project-a-clickhouse-load-%'" - ) - ) - return { - "rows": load_rows, - "elapsed_ms": load_elapsed * 1000.0, - "effective_rows_per_second": load_rows / load_elapsed, - "visible_rows": visible, - "zero_acknowledged_loss": visible == load_rows, - "ack_p50_ms": percentile(ack_samples, 0.50), - "ack_p95_ms": percentile(ack_samples, 0.95), - "visibility_p50_ms": percentile(visibility_samples, 0.50), - "visibility_p95_ms": percentile(visibility_samples, 0.95), - "concurrent_query_p95_ms": percentile(query_samples, 0.95), - "concurrent_query_errors": query_errors, - } - - -def run_mutations(client: ClickHouseHttp) -> dict[str, Any]: - update_started = time.perf_counter() - for table in ("events_full", "events_core"): - client.request( - f"ALTER TABLE {DATABASE}.{table} UPDATE bookmarked = true, public = true " - "WHERE project_id = 'project-a' AND logical_id = 'project-a-event-00000000' " - "SETTINGS mutations_sync = 2" - ) - update_ms = (time.perf_counter() - update_started) * 1000.0 - updated_full = int( - client.scalar( - f"SELECT count() FROM {DATABASE}.events_full WHERE project_id = 'project-a' " - "AND logical_id = 'project-a-event-00000000' AND bookmarked AND public" - ) - ) - updated_core = int( - client.scalar( - f"SELECT count() FROM {DATABASE}.events_core WHERE project_id = 'project-a' " - "AND logical_id = 'project-a-event-00000000' AND bookmarked AND public" - ) - ) - - trace_started = time.perf_counter() - client.request( - f"DELETE FROM {DATABASE}.mutation_events " - "WHERE project_id = 'project-a' AND trace_id = 'project-a-trace-0000' " - "SETTINGS mutations_sync = 2" - ) - trace_delete_ms = (time.perf_counter() - trace_started) * 1000.0 - trace_remaining = int( - client.scalar( - f"SELECT count() FROM {DATABASE}.mutation_events " - "WHERE project_id = 'project-a' AND trace_id = 'project-a-trace-0000'" - ) - ) - - retention_started = time.perf_counter() - client.request( - f"DELETE FROM {DATABASE}.mutation_events " - "WHERE project_id = 'project-a' AND start_time < '2026-01-01 00:00:30.000' " - "SETTINGS mutations_sync = 2" - ) - retention_delete_ms = (time.perf_counter() - retention_started) * 1000.0 - retention_remaining = int( - client.scalar( - f"SELECT count() FROM {DATABASE}.mutation_events " - "WHERE project_id = 'project-a' AND start_time < '2026-01-01 00:00:30.000'" - ) - ) - - project_started = time.perf_counter() - client.request( - f"DELETE FROM {DATABASE}.mutation_events WHERE project_id = 'project-b' " - "SETTINGS mutations_sync = 2" - ) - project_delete_ms = (time.perf_counter() - project_started) * 1000.0 - project_remaining = int( - client.scalar( - f"SELECT count() FROM {DATABASE}.mutation_events WHERE project_id = 'project-b'" - ) - ) - return { - "update_flags": { - "elapsed_ms": update_ms, - "events_full_visible": updated_full > 0, - "events_core_visible": updated_core > 0, - }, - "delete_trace": { - "elapsed_ms": trace_delete_ms, - "remaining_rows": trace_remaining, - }, - "delete_retention": { - "elapsed_ms": retention_delete_ms, - "remaining_rows": retention_remaining, - }, - "delete_project": { - "elapsed_ms": project_delete_ms, - "remaining_rows": project_remaining, - }, - } - - -def percentile(samples: list[float], quantile: float) -> float: - if not samples: - return 0.0 - ordered = sorted(samples) - rank = math.ceil((len(ordered) - 1) * quantile) - return ordered[min(rank, len(ordered) - 1)] - - -def main() -> None: - args = parse_args() - if args.load_seconds <= 0 or args.insert_batch_rows <= 0: - raise SystemExit("load seconds and insert batch rows must be positive") - if not args.fixture.is_file(): - raise SystemExit(f"fixture does not exist: {args.fixture}") - if args.output.exists(): - raise SystemExit(f"refusing to overwrite report: {args.output}") - - client = ClickHouseHttp(args.url, args.user, args.password) - version = str(client.scalar("SELECT version()")) - execute_ddl(client) - insert_metrics = load_fixture(client, args.fixture, args.insert_batch_rows) - populate_started = time.perf_counter() - populate_tables(client) - populate_ms = (time.perf_counter() - populate_started) * 1000.0 - - counts = { - table: int(client.scalar(f"SELECT count() FROM {DATABASE}.{table}")) - for table in ( - "staging", - "events_full", - "events_core", - "scores", - "dataset_run_items_rmt", - "blob_storage_file_log", - ) - } - event_distinct_ids = int( - client.scalar(f"SELECT uniqExact(logical_id) FROM {DATABASE}.events_full") - ) - point_sql = f""" - SELECT logical_id, event_ts, event_type, trace_id, span_id - FROM {DATABASE}.events_full - WHERE project_id = 'project-a' - AND start_time >= '2026-01-01 00:00:00.000' - AND trace_id = 'project-a-trace-0000' - AND span_id = 'project-a-span-00000000' - ORDER BY event_ts DESC LIMIT 1 - """ - list_sql = f""" - SELECT logical_id, event_ts, event_type, trace_id, span_id - FROM {DATABASE}.events_core - WHERE project_id = 'project-a' - AND start_time >= '2026-01-01 00:00:00.000' - ORDER BY start_time DESC LIMIT 100 - """ - facet_sql = f""" - SELECT arrayJoin(tags) AS tag, count() AS rows - FROM {DATABASE}.events_core - WHERE project_id = 'project-a' - GROUP BY tag ORDER BY rows DESC LIMIT 20 - """ - dashboard_sql = f""" - SELECT toStartOfInterval(start_time, INTERVAL 10 SECOND) AS bucket, - quantile(0.95)(usage_input) AS p95_input, - sum(total_cost) AS total_cost - FROM {DATABASE}.events_core - WHERE project_id = 'project-a' - AND start_time >= '2026-01-01 00:00:00.000' - AND start_time < '2026-01-01 00:02:00.000' - GROUP BY bucket ORDER BY bucket - WITH FILL FROM toDateTime('2026-01-01 00:00:00') - TO toDateTime('2026-01-01 00:02:00') - STEP INTERVAL 10 SECOND - """ - fts_sql = f""" - SELECT logical_id - FROM {DATABASE}.events_full - WHERE project_id = 'project-a' - AND hasAllTokens(lower(input), - arraySlice(arrayDistinct(tokens(lower('needle-token'))), 1, 64)) - AND positionCaseInsensitiveUTF8(input, 'needle-token') > 0 - LIMIT 100 - SETTINGS enable_full_text_index = 1 - """ - score_sql = f""" - SELECT name, count() AS rows - FROM {DATABASE}.scores - WHERE project_id = 'project-a' - GROUP BY name ORDER BY rows DESC - """ - query_metrics = { - "point": measure_query(client, point_sql), - "list": measure_query(client, list_sql), - "facet": measure_query(client, facet_sql), - "dashboard": measure_query(client, dashboard_sql), - "full_text": measure_query(client, fts_sql), - "scores": measure_query(client, score_sql), - } - cross_project_point_rows = int( - client.scalar( - f"SELECT count() FROM {DATABASE}.events_full " - "WHERE project_id = 'project-a' AND logical_id = 'project-b-event-00000500'" - ) - ) - export_query = ( - f"SELECT * FROM {DATABASE}.events_full " - "WHERE project_id = 'project-a' ORDER BY start_time LIMIT 10000" - ) - exports = { - "json_each_row": client.stream(export_query, "JSONEachRow"), - "parquet": client.stream(export_query, "Parquet"), - } - load_phase = run_load_phase(client, args.fixture, args.load_seconds) - mutations = run_mutations(client) - memory_resident = client.scalar( - "SELECT value FROM system.asynchronous_metrics WHERE metric = 'MemoryResident'" - ) - - report = { - "probe": "langfuse-clickhouse-baseline", - "clickhouse_version": version, - "database": DATABASE, - "fixture": str(args.fixture), - "insert": insert_metrics, - "populate_ms": populate_ms, - "counts": counts, - "event_physical_rows": counts["events_full"], - "event_distinct_logical_ids": event_distinct_ids, - "duplicate_versions_present": counts["events_full"] > event_distinct_ids, - "cross_project_point_rows": cross_project_point_rows, - "query_latency": query_metrics, - "exports": exports, - "load_phase": load_phase, - "mutations": mutations, - "memory_resident_bytes": int(memory_resident), - } - args.output.parent.mkdir(parents=True, exist_ok=True) - args.output.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") - print(json.dumps(report, indent=2, sort_keys=True)) - - -if __name__ == "__main__": - main() diff --git a/benchmark/langfuse-pchronicle-review/recorded-results-2026-08-11.json b/benchmark/langfuse-pchronicle-review/recorded-results-2026-08-11.json deleted file mode 100644 index 82fe3602c..000000000 --- a/benchmark/langfuse-pchronicle-review/recorded-results-2026-08-11.json +++ /dev/null @@ -1,146 +0,0 @@ -{ - "status": "experimental-local-review", - "recorded_at": "2026-08-11", - "verdict": "NO-GO_DIRECT_REPLACEMENT", - "baselines": { - "persisting": "94531cf903e5abc336de347588fb1858e9d52b6a plus local uncommitted catalog/query-engine work", - "langfuse": "d18a59ad663ffc7c04afc61354186c141b3ec0f3", - "clickhouse": "25.12.11.4" - }, - "environment": { - "kernel": "Darwin 25.5.0 arm64", - "macos": "26.5", - "cpu": "Apple M4", - "memory_bytes": 25769803776, - "rustc": "1.97.1 (8bab26f4f 2026-07-14)", - "python": "3.14.5", - "docker_server": "29.6.2" - }, - "fixture": { - "sha256": "1b2ef8ed2f51e6d85a9907bbbebbdafbc4c961d8c4368b2a20b737f94cfc4c90", - "bytes": 238470862, - "rows": 113000, - "events": 100000, - "event_distinct_logical_ids": 99900, - "scores": 10000, - "dataset_run_items": 2000, - "blob_log_rows": 1000, - "projects": 2, - "traces": 200, - "pchronicle_sources": 210 - }, - "pchronicle": { - "append": { - "acknowledged_rows": 113000, - "elapsed_ms": 12053.854958, - "rows_per_second": 9374.594301468946, - "zero_acknowledged_loss": true - }, - "load_phase": { - "rows": 30, - "effective_rows_per_second": 10.263871301317753, - "ack_p95_ms": 80.864458, - "visibility_p95_ms": 4.1275, - "concurrent_query_p95_ms": 350.409292, - "visible_rows": 30, - "zero_acknowledged_loss": true, - "pinned_count_after_append": 113000, - "refreshed_count": 113030 - }, - "query_p95_ms": { - "point": 395.870541, - "list": 564.816625, - "simple_model_aggregate": 806.700875 - }, - "catalog": { - "cold_start_ms": 2104.038375, - "rss_after_bytes": 1117782016, - "rss_before_bytes": 308281344 - }, - "maintenance_restart": { - "fragments_removed": 3, - "maintenance_ms": 38.360458, - "restart_append_and_catalog_ms": 4160.961459, - "restart_row_visible": true - }, - "json_export": { - "rows": 50000, - "bytes": 96902959, - "first_byte_ms": 429.086209, - "elapsed_ms": 585.346084 - }, - "semantic_results": { - "event_physical_rows": 100000, - "event_distinct_logical_ids": 99900, - "duplicate_versions_remain_physical": true, - "cross_project_point_rows": 0, - "update_flags": "unsupported", - "delete_trace": "unsupported", - "delete_project": "unsupported", - "delete_retention": "unsupported", - "parquet_export": "unsupported" - } - }, - "clickhouse": { - "append": { - "acknowledged_rows": 113000, - "elapsed_ms": 1538.472415995784, - "rows_per_second": 73449.48068299306 - }, - "load_phase": { - "rows": 30, - "effective_rows_per_second": 9.857884896590003, - "ack_p95_ms": 107.69920799066313, - "visibility_p95_ms": 6.510250008432195, - "concurrent_query_p95_ms": 13.96033400669694, - "visible_rows": 30, - "zero_acknowledged_loss": true - }, - "query_p95_ms": { - "point": 4.826582997338846, - "list": 2.9707499779760838, - "facet": 3.466208989266306, - "dashboard": 4.525625001406297, - "full_text": 3.4713750064838678, - "scores": 2.282792003825307 - }, - "exports": { - "json_first_byte_ms": 68.76837500021793, - "parquet_first_byte_ms": 28.269458998693153 - }, - "mutations_ms": { - "update_flags": 25.31504098442383, - "delete_trace": 8.981124992715195, - "delete_project": 5.9625419962685555, - "delete_retention": 7.64183298451826 - }, - "semantic_results": { - "event_physical_rows": 99900, - "event_distinct_logical_ids": 99900, - "duplicate_versions_present": false, - "cross_project_point_rows": 0 - }, - "memory_resident_bytes": 1422721024 - }, - "relative_results": { - "clickhouse_append_throughput_over_pchronicle": 7.83, - "pchronicle_point_p95_over_clickhouse": 82.02, - "pchronicle_list_p95_over_clickhouse": 190.13, - "pchronicle_simple_aggregate_over_clickhouse_tag_facet_not_semantically_identical": 232.73, - "pchronicle_concurrent_query_p95_over_clickhouse": 25.10 - }, - "same_fixture_repeat_observations": { - "pchronicle_append_rows_per_second": [9374.594301468946, 9594.449551771359], - "pchronicle_point_p95_ms": [395.870541, 527.936125], - "pchronicle_list_p95_ms": [564.816625, 666.709167], - "clickhouse_append_rows_per_second": [73449.48068299306, 73661.05953160369], - "clickhouse_point_p95_ms": [4.183750017546117, 4.826582997338846], - "clickhouse_list_p95_ms": [2.9707499779760838, 3.9958749839570373] - }, - "tests": { - "langfuse_backend_faults": "2 passed; 0 failed", - "production_scale": "5 passed; 0 failed; 1 ignored", - "query_engine": "14 passed; 0 failed", - "store_catalog_filter": "6 passed; 0 failed; 182 filtered" - } -} diff --git a/benchmark/langfuse-pchronicle-review/recorded-results-2026-08-12.json b/benchmark/langfuse-pchronicle-review/recorded-results-2026-08-12.json deleted file mode 100644 index abd9fa7ab..000000000 --- a/benchmark/langfuse-pchronicle-review/recorded-results-2026-08-12.json +++ /dev/null @@ -1,133 +0,0 @@ -{ - "status": "experimental-local-retest", - "recorded_at": "2026-08-12", - "scope": "pChronicle lazy catalog and source-pruning retest", - "baseline": { - "persisting_head": "94531cf903e5abc336de347588fb1858e9d52b6a", - "state": "head plus local uncommitted pChronicle catalog/query changes", - "previous_results": "recorded-results-2026-08-11.json", - "clickhouse_version": "25.12.11.4" - }, - "source_sha256": { - "catalog.rs": "f5066a96a0c8a8eb062b50b205b4f81a2272cf98385f17708fe496cea787872a", - "raw_event_datafusion.rs": "e4f2a19b1e62ffaa9e7538430d7b033d72bfca4d5ef0ccb816a8fe59f9dbba78", - "raw_event_lance.rs": "42089b5d10c966360f9a9beb72db74222a7d3fec499f579ccdd732916f544ce2", - "query_engine.rs": "40c95f0bc10de90cd04f97fc81cd5a0410321b56ad001df4c487b332042c26ea", - "storyline_datafusion.rs": "3f56eacf94735364b2b190313973510f6b0bcf8b04726c65054097de8f40e71f", - "langfuse_backend_feasibility.rs": "8c822ab13cdf7c949e3b93c0eab6b1c7335a560b3e24720165e7eaa4ad12f1b8" - }, - "fixture": { - "sha256": "1b2ef8ed2f51e6d85a9907bbbebbdafbc4c961d8c4368b2a20b737f94cfc4c90", - "bytes": 238470862, - "rows": 113000, - "events": 100000, - "event_distinct_logical_ids": 99900, - "scores": 10000, - "dataset_run_items": 2000, - "blob_log_rows": 1000, - "projects": 2, - "traces": 200, - "pchronicle_sources": 210 - }, - "pchronicle": { - "append": { - "acknowledged_rows": 113000, - "elapsed_ms": 12302.462166, - "rows_per_second": 9185.15322179126, - "zero_acknowledged_loss": true - }, - "catalog": { - "cold_start_ms": 35.041416999999996, - "rss_before_bytes": 308871168, - "rss_after_catalog_bytes": 316669952, - "rss_after_unpruned_queries_bytes": 624885760, - "sources": 210 - }, - "warm_query_p95_ms": { - "point": 258.13295899999997, - "list": 282.798583, - "simple_model_aggregate": 296.706625 - }, - "fresh_global": { - "point_catalog_setup_ms": 10.836791999999999, - "point_p95_ms": 302.93704199999996, - "list_catalog_setup_ms": 12.47275, - "list_p95_ms": 326.682458 - }, - "catalog_pruning": { - "direct_run_open_ms": 0.5066660000000001, - "direct_run_point_p95_ms": 2.1355, - "exact_file_catalog_setup_ms": 12.337958, - "exact_file_point_p95_ms": 3.0130410000000003, - "project_file_catalog_setup_ms": 12.02425, - "project_file_list_p95_ms": 167.17120799999998 - }, - "load_phase": { - "rows": 30, - "effective_rows_per_second": 9.582487175227183, - "ack_p95_ms": 27.738083, - "visibility_p95_ms": 5.46075, - "concurrent_query_p95_ms": 243.45462500000002, - "visible_rows": 30, - "zero_acknowledged_loss": true, - "pinned_count_after_append": 113000, - "refreshed_count": 113030 - }, - "json_export": { - "rows": 50000, - "bytes": 96902959, - "first_byte_ms": 336.693958, - "elapsed_ms": 503.48304200000007 - }, - "maintenance_restart": { - "maintenance_ms": 37.512542, - "fragments_removed": 3, - "restart_append_and_catalog_ms": 51.761459, - "restart_visibility_ms": 344.021583, - "restart_end_to_end_ms": 395.78304199999997, - "restart_row_visible": true - }, - "semantics": { - "event_physical_rows": 100000, - "event_distinct_logical_ids": 99900, - "duplicate_versions_remain_physical": true, - "cross_project_point_rows": 0 - } - }, - "same_code_repeat_ranges": { - "catalog_cold_start_ms": [35.041416999999996, 35.360584], - "append_rows_per_second": [9105.865426522427, 9239.77754263836], - "warm_point_p95_ms": [258.13295899999997, 262.034875], - "warm_list_p95_ms": [282.798583, 352.99425], - "simple_model_aggregate_p95_ms": [292.394042, 299.305875], - "exact_file_point_p95_ms": [2.978792, 3.0130410000000003], - "project_file_list_p95_ms": [166.093667, 167.17120799999998] - }, - "comparison_to_2026_08_11_pct": { - "catalog_cold_start_reduction": 98.33456378855257, - "cold_catalog_rss_reduction": 71.66979362101313, - "old_eager_catalog_vs_new_post_query_rss_reduction": 44.09591932457786, - "warm_point_latency_reduction": 34.79359228197787, - "warm_list_latency_reduction": 49.930903149318596, - "simple_model_aggregate_latency_reduction": 63.219746724583636, - "concurrent_query_latency_reduction": 30.52278276912816, - "append_throughput_reduction": 2.020792298691799 - }, - "comparison_to_clickhouse_2026_08_11": { - "clickhouse_point_p95_ms": 4.826582997338846, - "exact_file_point_ratio": 0.6242596473864122, - "clickhouse_list_p95_ms": 2.9707499779760838, - "project_file_list_ratio": 56.27239223742773, - "warm_global_point_ratio": 53.481512519793505, - "warm_global_list_ratio": 95.19433984567944 - }, - "tests": { - "langfuse_backend_faults": "2 passed; 0 failed", - "production_scale": "5 passed; 0 failed; 1 ignored", - "query_engine": "14 passed; 0 failed", - "catalog": "9 passed; 0 failed", - "raw_event_lance_and_rows": "21 passed; 0 failed", - "pchronicle_server_lib": "16 passed; 0 failed" - }, - "artifact_directory": "/tmp/langfuse-pchronicle-retest-final.n9fgAW" -} diff --git a/benchmark/langfuse-pchronicle-review/server-acceleration-results-2026-08-12.json b/benchmark/langfuse-pchronicle-review/server-acceleration-results-2026-08-12.json deleted file mode 100644 index b311f49d8..000000000 --- a/benchmark/langfuse-pchronicle-review/server-acceleration-results-2026-08-12.json +++ /dev/null @@ -1,80 +0,0 @@ -{ - "recorded_at": "2026-08-12", - "scope": "pChronicle release Server source-routing acceleration", - "fixture": { - "storage": "/tmp/langfuse-pchronicle-retest-final.n9fgAW/pchronicle-store", - "catalog_sources": 211, - "indexed_event_rows": 113031 - }, - "routing_indexes": { - "event_partition": { - "rows": 113031, - "sources": 211, - "distinct_fingerprints": 213, - "project_a_candidate_sources": 106 - }, - "event_identity": { - "rows": 113031, - "sources": 211, - "distinct_fingerprints": 113132 - } - }, - "cold": { - "catalog_ms": 37.149, - "identity_point_ms_repeats": [166.173, 159.452, 134.261], - "identity_point_median_ms": 159.452, - "catalog_plus_identity_point_ms": 196.601, - "first_project_list_ms": 312.941 - }, - "warm_http_p95_ms": { - "point": 4.724, - "project_list": 151.047, - "project_aggregate": 165.406 - }, - "warm_http_p50_ms": { - "point": 4.149, - "project_list": 147.244, - "project_aggregate": 158.632 - }, - "warm_samples": { - "point": 20, - "project_list": 20, - "project_aggregate": 20 - }, - "process_rss_kib": { - "startup": 13456, - "after_catalog": 26880, - "after_partition_index_and_query": 129344, - "after_partition_then_identity_index_and_queries": 216928, - "fresh_after_catalog_identity_index_and_query": 104384, - "note": "RSS includes lazy source resolution, Lance/DataFusion buffers and allocator retention; it is not the routing-index heap size" - }, - "comparison": { - "previous_pchronicle_p95_ms": { - "global_point": 258.132959, - "global_list": 282.798583, - "global_aggregate": 296.706625 - }, - "improvement_vs_previous_pchronicle_pct": { - "warm_point": 98.170, - "warm_project_list": 46.588, - "warm_project_aggregate": 44.253 - }, - "clickhouse_2026_08_11_p95_ms": { - "point": 4.826583, - "list": 2.970750, - "aggregate": 3.466209 - }, - "pchronicle_to_clickhouse_ratio": { - "warm_point": 0.978746, - "warm_project_list": 50.844737, - "warm_project_aggregate": 47.719569 - } - }, - "methodology": { - "build": "cargo build --release -p persisting-cli", - "server": "target/release/persisting chronicle serve ... --listen 127.0.0.1:19877", - "client": "curl over loopback HTTP/1.1", - "caveat": "pChronicle measurements include Axum and loopback HTTP; reused ClickHouse baseline was not rerun concurrently" - } -} diff --git a/benchmark/pchronicle/README.md b/benchmark/pchronicle/README.md index 52d585574..590c29f3a 100644 --- a/benchmark/pchronicle/README.md +++ b/benchmark/pchronicle/README.md @@ -8,10 +8,7 @@ The guarded paths are: - ATIF parse and round-trip conversion; -- canonical events → Storyline projection; - Storyline split into and reconstruction from the three relational tables; -- canonical Lance event append; -- full and incremental Storyline projection, including verification; - Storyline point reads, replacement, indexed SQL, and group-by queries; - projected JSON streaming latency, throughput, allocations, and peak RSS. @@ -58,9 +55,9 @@ array. Every leaf has one stable JSONPath address: { "measurements": { "system": { - "projection_pipeline": { - "projection_incremental": { - "sync_ms": { + "lance_vs_json": { + "lifecycle": { + "cold_query_ms": { "value": 42.1, "unit": "ms", "direction": "lower", @@ -79,7 +76,7 @@ escaping. Read a value with: ```bash python3 benchmark/pchronicle/bench.py jsonpath-get \ --document target/pchronicle-benchmark/current/raw-report.json \ - --path '$["measurements"]["system"]["projection_pipeline"]["projection_incremental"]["sync_ms"]' + --path '$["measurements"]["system"]["lance_vs_json"]["lifecycle"]["cold_query_ms"]' ``` Future benchmark producers can insert a new leaf with `jsonpath-set`. Duplicate @@ -115,5 +112,5 @@ unbounded repository history. ## Links -- [pChronicle design](../../docs/src/pchronicle/design/index.md) +- [pChronicle design](../../docs/src/en/pchronicle/design/index.md) - [`persisting-pchronicle`](../../crates/persisting-pchronicle/README.md) diff --git a/benchmark/pchronicle/bench.py b/benchmark/pchronicle/bench.py index 4b636f765..fcd65068f 100644 --- a/benchmark/pchronicle/bench.py +++ b/benchmark/pchronicle/bench.py @@ -22,7 +22,6 @@ import sys from typing import Any - SCHEMA = "pchronicle-benchmark" README_START = "" README_END = "" @@ -82,9 +81,10 @@ def command_output(command: list[str], cwd: pathlib.Path) -> str: def environment(repo: pathlib.Path, suite: str) -> dict[str, Any]: - dirty = subprocess.run( - ["git", "diff", "--quiet", "--ignore-submodules", "--"], cwd=repo - ).returncode != 0 + dirty = ( + subprocess.run(["git", "diff", "--quiet", "--ignore-submodules", "--"], cwd=repo).returncode + != 0 + ) return { "suite": suite, "git_commit": command_output(["git", "rev-parse", "HEAD"], repo), @@ -149,9 +149,7 @@ def jsonpath_segments(path: str) -> list[str]: while position < len(path): matched = JSONPATH_TOKEN_RE.match(path, position) if matched is None: - raise ValueError( - "benchmark JSONPath supports only bracketed name selectors: " + path - ) + raise ValueError("benchmark JSONPath supports only bracketed name selectors: " + path) key = json.loads(matched.group("key")) if not isinstance(key, str) or not key: raise ValueError(f"JSONPath name selector must be a non-empty string: {path}") @@ -169,9 +167,7 @@ def jsonpath_get(document: dict[str, Any], path: str) -> Any: return current -def jsonpath_set( - document: dict[str, Any], path: str, value: Any, *, replace: bool = False -) -> None: +def jsonpath_set(document: dict[str, Any], path: str, value: Any, *, replace: bool = False) -> None: segments = jsonpath_segments(path) if not segments: raise ValueError("the benchmark JSONPath root cannot be replaced") @@ -190,9 +186,7 @@ def jsonpath_set( def store_metrics(document: dict[str, Any], metrics: list[dict[str, Any]]) -> None: for item in metrics: path = metric_jsonpath(item["case"], item["metric"]) - measurement = { - key: value for key, value in item.items() if key not in {"case", "metric"} - } + measurement = {key: value for key, value in item.items() if key not in {"case", "metric"}} jsonpath_set(document, path, measurement) @@ -249,9 +243,7 @@ def run_criterion( { "CARGO_TARGET_DIR": str(target_dir), "PCHRONICLE_CRITERION_SAMPLES": str(config["criterion_samples"]), - "PCHRONICLE_CRITERION_MEASUREMENT_MS": str( - config["criterion_measurement_ms"] - ), + "PCHRONICLE_CRITERION_MEASUREMENT_MS": str(config["criterion_measurement_ms"]), "PCHRONICLE_CRITERION_WARMUP_MS": str(config["criterion_warmup_ms"]), } ) @@ -290,12 +282,8 @@ def run_criterion( "ns", "lower", "criterion", - float(confidence["lower_bound"]) - if "lower_bound" in confidence - else None, - float(confidence["upper_bound"]) - if "upper_bound" in confidence - else None, + float(confidence["lower_bound"]) if "lower_bound" in confidence else None, + float(confidence["upper_bound"]) if "upper_bound" in confidence else None, ) ) if not metrics: @@ -488,18 +476,6 @@ def run_suite(args: argparse.Namespace) -> None: config["hyperfine_runs"], ) ) - if bench_target_exists(repo, "projection_pipeline"): - projection = build_bench_executable(repo, target_dir, "projection_pipeline", output) - metrics.extend( - run_system_scenario( - repo, - output, - projection, - "projection_pipeline", - common, - config["hyperfine_runs"], - ) - ) for shape in ("ndjson", "array"): metrics.extend( run_system_scenario( @@ -762,14 +738,12 @@ def html_report( th:first-child,td:first-child{{text-align:left}} th{{background:#f5f7fa}}

pChronicle benchmark

{verdict}{len(regressions)} regressions · {len(improvements)} improvements
-{''.join(sections)}
Complete Markdown report
{html.escape(markdown)}
+{"".join(sections)}
Complete Markdown report
{html.escape(markdown)}
""" def update_readme(args: argparse.Namespace) -> None: - report = read_report( - pathlib.Path(args.report), getattr(args, "report_jsonpath", None) - ) + report = read_report(pathlib.Path(args.report), getattr(args, "report_jsonpath", None)) readme = pathlib.Path(args.readme) content = readme.read_text(encoding="utf-8") if README_START not in content or README_END not in content: @@ -777,15 +751,7 @@ def update_readme(args: argparse.Namespace) -> None: run = report["run"] priorities = [ metric_jsonpath("criterion/atif_conversion/parse_corpus", "latency_median_ns"), - metric_jsonpath( - "criterion/atif_conversion/roundtrip_corpus", "latency_median_ns" - ), - metric_jsonpath( - "criterion/projection_cpu/events_to_storyline_corpus", "latency_median_ns" - ), - metric_jsonpath("system/projection_pipeline/event_append", "initial_append_ms"), - metric_jsonpath("system/projection_pipeline/projection_build", "build_ms"), - metric_jsonpath("system/projection_pipeline/projection_incremental", "sync_ms"), + metric_jsonpath("criterion/atif_conversion/roundtrip_corpus", "latency_median_ns"), metric_jsonpath("system/lance_vs_json/lifecycle", "cold_query_ms"), metric_jsonpath("system/lance_vs_json/lifecycle", "get_storyline_full_ms"), metric_jsonpath("system/lance_vs_json/lifecycle", "replace_storyline_ms"), @@ -794,10 +760,7 @@ def update_readme(args: argparse.Namespace) -> None: metric_jsonpath("system/lance_vs_json/summary", "lance_over_json"), metric_jsonpath("system/json_streaming_ndjson/json_streaming", "p95_ms"), metric_jsonpath("system/json_streaming_ndjson/json_streaming", "rows_s"), - metric_jsonpath( - "system/json_streaming_ndjson/json_streaming", "process_peak_rss_mib" - ), - metric_jsonpath("hyperfine/projection_pipeline", "wall_median_seconds"), + metric_jsonpath("system/json_streaming_ndjson/json_streaming", "process_peak_rss_mib"), metric_jsonpath("hyperfine/lance_vs_json", "wall_median_seconds"), ] selected = [] diff --git a/benchmark/pchronicle/test_bench.py b/benchmark/pchronicle/test_bench.py index b7b5d5966..50f1d6227 100644 --- a/benchmark/pchronicle/test_bench.py +++ b/benchmark/pchronicle/test_bench.py @@ -4,7 +4,6 @@ import tempfile import unittest - MODULE_PATH = pathlib.Path(__file__).with_name("bench.py") SPEC = importlib.util.spec_from_file_location("pchronicle_bench", MODULE_PATH) bench = importlib.util.module_from_spec(SPEC) diff --git a/crates/persisting-agentctl/Cargo.toml b/crates/persisting-agentctl/Cargo.toml deleted file mode 100644 index 8d1f8b6f1..000000000 --- a/crates/persisting-agentctl/Cargo.toml +++ /dev/null @@ -1,16 +0,0 @@ -[package] -name = "persisting-agentctl" -version.workspace = true -edition.workspace = true -authors.workspace = true -license.workspace = true -description = "Agent control contracts, policies, wire protocol, and client SDK" - -[dependencies] -anyhow.workspace = true -ipnet.workspace = true -serde = { workspace = true, features = ["derive"] } -serde_json.workspace = true - -[dev-dependencies] -proptest.workspace = true diff --git a/crates/persisting-agentctl/README.md b/crates/persisting-agentctl/README.md deleted file mode 100644 index 26eb1e6c2..000000000 --- a/crates/persisting-agentctl/README.md +++ /dev/null @@ -1,68 +0,0 @@ -# persisting-agentctl - -**Agent control contracts, policies, the versioned AgentCtl v1 protocol, and its -synchronous client SDK.** - -Owns the runtime control state machine, the wire protocol, and -`AgentCtlClient`. AgentCtl is an optional, cooperative channel between a control server -and a Run-local runtime client. It is not a sandbox, does not discover -processes or external effects, and is never enforcement evidence by itself. - -Integrations provide the Run-scoped server and credential injection. OverlayNet and -Gateway apply policy decisions; they do not own this protocol. - -```text -Requested -> Allowed / Denied -> Applied / Failed -``` - -- `ControlRequest` is a typed resource request (currently network or model). -- `ControlController` evaluates policy and returns the authorization transition. -- `ControlMachine` validates transitions and retains the state/history. -- `protocol` is the dependency-light request/response schema shared with - the control server. -- `AgentCtlClient` discovers the authenticated Unix endpoint from the - environment and drives Session creation, periodic state synchronization, and - checkpoint quiescence. - -An `Applied { effect: Deny }` state means the driver successfully blocked an -operation. It does not mean that a proxy-based driver is non-bypassable. - -The protocol has two requests: `Hello` authenticates and opens a Session; -`Sync` exchanges the client's current state for the control server's current directive. -Clients report `active`, `idle`, or `quiesced { checkpoint_id }`. The server replies -with `continue`, `quiesce { checkpoint_id, deadline_unix_ms? }`, or -`shutdown { reason? }`. A checkpoint succeeds only after every Session frozen -into that checkpoint reports the matching quiesced state. - -Integrations provide four Run-local variables: `PERSISTING_AGENTCTL_ENDPOINT`, -`PERSISTING_AGENTCTL_TOKEN`, `PERSISTING_AGENTCTL_VERSION` (exactly `1`), and -`PERSISTING_AGENTCTL_TRANSPORT` (currently `unix`). New integrations use only -`PERSISTING_AGENTCTL_*`. A future interactive login or terminal will use a -separately authorized Debug protocol; terminal byte streams, PTY resize, and -signals will not enlarge this compact Control protocol. - -See [`src/protocol.rs`](src/protocol.rs) for the complete wire contract, JSON -examples, state semantics, typed errors, and safety boundary. - -## Develop - -```rust -use persisting_agentctl::{AgentCtlClient, AgentCtlClientConfig, AgentState}; - -let Some(config) = AgentCtlClientConfig::from_current_environment("worker-1")? else { - return Ok(()); // no control endpoint configured -}; -let mut client = AgentCtlClient::new(config); -let directive = client.connect()?; -let directive = client.sync(AgentState::Active)?; -# Ok::<(), anyhow::Error>(()) -``` - -```bash -just test persisting-agentctl -# or: just test-crate agentctl -``` - -## Links - -- [System architecture](../../docs/src/en/system-design/architecture.md) diff --git a/crates/persisting-agentctl/src/client.rs b/crates/persisting-agentctl/src/client.rs deleted file mode 100644 index e674e1289..000000000 --- a/crates/persisting-agentctl/src/client.rs +++ /dev/null @@ -1,241 +0,0 @@ -//! Synchronous client for the cooperative, low-frequency AgentCtl protocol. - -use crate::{ - AGENTCTL_ENDPOINT_ENV, AGENTCTL_MAX_FRAME_BYTES, AGENTCTL_TOKEN_ENV, AGENTCTL_TRANSPORT_ENV, - AGENTCTL_VERSION, AGENTCTL_VERSION_ENV, AgentDirective, AgentErrorCode, AgentRequest, - AgentResponse, AgentState, -}; -use anyhow::{Context, bail}; -use std::collections::BTreeMap; -use std::fmt; -use std::io::{BufRead, Read, Write}; -use std::path::{Path, PathBuf}; - -/// Configuration discovered from one Run's injected AgentCtl environment. -pub struct AgentCtlClientConfig { - /// Run-local Unix socket path. - pub endpoint: PathBuf, - /// Run-scoped authentication token. - pub token: String, - /// Stable non-empty identity for this runtime client. - pub client_id: String, -} - -impl AgentCtlClientConfig { - /// Discover AgentCtl from the current process environment. - pub fn from_current_environment(client_id: impl Into) -> anyhow::Result> { - Self::from_environment(&std::env::vars().collect(), client_id) - } - - /// Discover AgentCtl from an explicit environment projection. - pub fn from_environment( - environment: &BTreeMap, - client_id: impl Into, - ) -> anyhow::Result> { - let Some(endpoint) = environment.get(AGENTCTL_ENDPOINT_ENV) else { - return Ok(None); - }; - let transport = environment - .get(AGENTCTL_TRANSPORT_ENV) - .map(String::as_str) - .unwrap_or("unix"); - if transport != "unix" { - bail!("unsupported AgentCtl transport {transport:?}"); - } - let version = environment - .get(AGENTCTL_VERSION_ENV) - .context("AgentCtl endpoint is present without a version")? - .parse::() - .context("parse AgentCtl version")?; - if version != AGENTCTL_VERSION { - bail!( - "AgentCtl version mismatch: client expects {}, environment advertises {}", - AGENTCTL_VERSION, - version - ); - } - let client_id = client_id.into(); - if client_id.trim().is_empty() { - bail!("AgentCtl client_id must be non-empty"); - } - Ok(Some(Self { - endpoint: endpoint.into(), - token: environment - .get(AGENTCTL_TOKEN_ENV) - .context("AgentCtl endpoint is present without an auth token")? - .clone(), - client_id, - })) - } -} - -/// A machine-readable error returned by the AgentCtl server. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct AgentCtlResponseError { - /// Stable category used for client control flow. - pub code: AgentErrorCode, - /// Human-readable diagnostic context. - pub message: String, -} - -impl fmt::Display for AgentCtlResponseError { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - let code = match self.code { - AgentErrorCode::InvalidRequest => "invalid_request", - AgentErrorCode::Unauthorized => "unauthorized", - AgentErrorCode::VersionMismatch => "version_mismatch", - AgentErrorCode::Conflict => "conflict", - }; - write!(formatter, "AgentCtl {code}: {}", self.message) - } -} - -impl std::error::Error for AgentCtlResponseError {} - -/// One runtime client's AgentCtl Session. -pub struct AgentCtlClient { - config: AgentCtlClientConfig, - session_id: Option, - sync_interval_ms: Option, -} - -impl AgentCtlClient { - /// Construct a disconnected client. - pub fn new(config: AgentCtlClientConfig) -> Self { - Self { - config, - session_id: None, - sync_interval_ms: None, - } - } - - /// Return the current Session identifier after a successful connection. - pub fn session_id(&self) -> Option<&str> { - self.session_id.as_deref() - } - - /// Return the server-recommended synchronization interval after connection. - pub fn sync_interval_ms(&self) -> Option { - self.sync_interval_ms - } - - /// Authenticate, create a Session, and return the current directive. - pub fn connect(&mut self) -> anyhow::Result { - let response = exchange( - &self.config.endpoint, - &AgentRequest::Hello { - version: AGENTCTL_VERSION, - token: self.config.token.clone(), - client_id: self.config.client_id.clone(), - }, - )?; - match response { - AgentResponse::Welcome { - session_id, - sync_interval_ms, - directive, - } => { - self.session_id = Some(session_id); - self.sync_interval_ms = Some(sync_interval_ms); - Ok(directive) - } - response => unexpected("welcome", response), - } - } - - /// Report cooperative state and return the server's current directive. - pub fn sync(&mut self, state: AgentState) -> anyhow::Result { - let session_id = self - .session_id - .clone() - .context("AgentCtl client is not connected")?; - match exchange( - &self.config.endpoint, - &AgentRequest::Sync { - version: AGENTCTL_VERSION, - session_id, - state, - }, - )? { - AgentResponse::Synced { directive } => Ok(directive), - response => unexpected("sync acknowledgement", response), - } - } -} - -fn exchange(path: &Path, request: &AgentRequest) -> anyhow::Result { - let mut stream = std::os::unix::net::UnixStream::connect(path) - .with_context(|| format!("connect AgentCtl endpoint {}", path.display()))?; - stream.set_read_timeout(Some(std::time::Duration::from_secs(2)))?; - stream.set_write_timeout(Some(std::time::Duration::from_secs(2)))?; - serde_json::to_writer(&mut stream, request)?; - stream.write_all(b"\n")?; - let mut frame = Vec::new(); - std::io::BufReader::new(stream) - .by_ref() - .take((AGENTCTL_MAX_FRAME_BYTES + 1) as u64) - .read_until(b'\n', &mut frame)?; - if frame.len() > AGENTCTL_MAX_FRAME_BYTES { - bail!("AgentCtl response exceeds {AGENTCTL_MAX_FRAME_BYTES} bytes"); - } - let response: AgentResponse = serde_json::from_slice(&frame)?; - if let AgentResponse::Error { code, message } = response { - return Err(AgentCtlResponseError { code, message }.into()); - } - Ok(response) -} - -fn unexpected(expected: &str, response: AgentResponse) -> anyhow::Result { - bail!("expected AgentCtl {expected}, got {response:?}") -} - -#[cfg(test)] -mod tests { - use super::*; - - fn environment() -> BTreeMap { - BTreeMap::from([ - (AGENTCTL_ENDPOINT_ENV.into(), "/tmp/agentctl.sock".into()), - (AGENTCTL_TOKEN_ENV.into(), "secret".into()), - (AGENTCTL_VERSION_ENV.into(), "1".into()), - (AGENTCTL_TRANSPORT_ENV.into(), "unix".into()), - ]) - } - - #[test] - fn config_reads_only_agentctl_v1_environment() { - let config = AgentCtlClientConfig::from_environment(&environment(), "worker-1") - .unwrap() - .unwrap(); - - assert_eq!(config.client_id, "worker-1"); - assert_eq!(config.token, "secret"); - assert_eq!(config.endpoint, PathBuf::from("/tmp/agentctl.sock")); - } - - #[test] - fn legacy_agent_abi_environment_is_not_discovered() { - let environment = BTreeMap::from([( - "PERSISTING_AGENT_ABI_ENDPOINT".into(), - "/tmp/legacy.sock".into(), - )]); - - assert!( - AgentCtlClientConfig::from_environment(&environment, "worker-1") - .unwrap() - .is_none() - ); - } - - #[test] - fn response_error_keeps_machine_code_and_human_message() { - let error = AgentCtlResponseError { - code: AgentErrorCode::Conflict, - message: "busy".into(), - }; - - assert_eq!(error.code, AgentErrorCode::Conflict); - assert_eq!(error.message, "busy"); - assert_eq!(error.to_string(), "AgentCtl conflict: busy"); - } -} diff --git a/crates/persisting-agentctl/src/lib.rs b/crates/persisting-agentctl/src/lib.rs deleted file mode 100644 index 6b560013f..000000000 --- a/crates/persisting-agentctl/src/lib.rs +++ /dev/null @@ -1,934 +0,0 @@ -//! Shared Agent control contracts, policies, wire protocol, and client SDK. -//! -//! Drivers such as OverlayNet and Capture submit typed resources to a -//! [`ControlController`]. Authorization is represented as a state transition; -//! the driver then records whether the authorized operation was applied or -//! failed. [`AgentCtlClient`] implements the optional cooperative AgentCtl -//! protocol. Supervisor messages are shared wire contracts for integrations. - -mod client; -pub mod protocol; -mod runtime; -mod supervisor; - -pub use client::{AgentCtlClient, AgentCtlClientConfig, AgentCtlResponseError}; -use ipnet::IpNet; -pub use protocol::*; -pub use runtime::*; -pub use runtime::{AccessEffect as ControlEffect, AccessReason as ControlReason}; -use serde::{Deserialize, Serialize}; -use std::net::IpAddr; -use std::str::FromStr; -pub use supervisor::*; - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum ControlState { - Requested, - Allowed, - Denied, - Applied { effect: ControlEffect }, - Failed { effect: ControlEffect }, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct ControlTransition { - pub from: ControlState, - pub to: ControlState, - pub reason: ControlReason, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ControlMachine { - state: ControlState, - history: Vec, -} - -impl Default for ControlMachine { - fn default() -> Self { - Self::new() - } -} - -impl ControlMachine { - pub fn new() -> Self { - Self { - state: ControlState::Requested, - history: Vec::new(), - } - } - - pub fn state(&self) -> ControlState { - self.state - } - - pub fn history(&self) -> &[ControlTransition] { - &self.history - } - - pub fn authorize( - &mut self, - controller: &dyn ControlController, - request: ControlRequest<'_>, - ) -> anyhow::Result { - if self.state != ControlState::Requested { - anyhow::bail!( - "control request was already authorized from {:?}", - self.state - ); - } - let transition = controller.authorize(request); - if transition.from != ControlState::Requested - || !matches!(transition.to, ControlState::Allowed | ControlState::Denied) - { - anyhow::bail!( - "controller returned invalid authorization transition {:?} -> {:?}", - transition.from, - transition.to - ); - } - self.commit(transition) - } - - pub fn applied(&mut self) -> anyhow::Result { - let transition = self - .history - .last() - .ok_or_else(|| anyhow::anyhow!("control request has not been authorized"))? - .applied()?; - self.commit(transition) - } - - pub fn failed(&mut self) -> anyhow::Result { - let transition = self - .history - .last() - .ok_or_else(|| anyhow::anyhow!("control request has not been authorized"))? - .failed()?; - self.commit(transition) - } - - fn commit(&mut self, transition: ControlTransition) -> anyhow::Result { - if transition.from != self.state { - anyhow::bail!( - "control state mismatch: current {:?}, transition starts at {:?}", - self.state, - transition.from - ); - } - self.state = transition.to; - self.history.push(transition.clone()); - Ok(transition) - } -} - -impl ControlTransition { - pub fn allowed(reason: ControlReason) -> Self { - Self { - from: ControlState::Requested, - to: ControlState::Allowed, - reason, - } - } - - pub fn denied(reason: ControlReason) -> Self { - Self { - from: ControlState::Requested, - to: ControlState::Denied, - reason, - } - } - - pub fn is_allowed(&self) -> bool { - matches!( - self.to, - ControlState::Allowed - | ControlState::Applied { - effect: ControlEffect::Allow - } - ) - } - - pub fn applied(&self) -> anyhow::Result { - self.follow(true) - } - - pub fn failed(&self) -> anyhow::Result { - self.follow(false) - } - - fn follow(&self, applied: bool) -> anyhow::Result { - let effect = match self.to { - ControlState::Allowed => ControlEffect::Allow, - ControlState::Denied => ControlEffect::Deny, - state => anyhow::bail!( - "control transition {state:?} cannot be applied; authorization must happen first" - ), - }; - let to = if applied { - ControlState::Applied { effect } - } else { - ControlState::Failed { effect } - }; - Ok(Self { - from: self.to, - to, - reason: self.reason, - }) - } -} - -pub enum ControlRequest<'a> { - Network { - policy: &'a NetworkGuard, - request: &'a NetworkAccessRequest, - }, - Model { - policy: &'a ModelAccessPolicy, - request: &'a ModelCallRequest, - }, -} - -pub trait ControlController: Send + Sync { - fn authorize(&self, request: ControlRequest<'_>) -> ControlTransition; -} - -#[derive(Debug, Default)] -pub struct PolicyControlController; - -impl ControlController for PolicyControlController { - fn authorize(&self, request: ControlRequest<'_>) -> ControlTransition { - match request { - ControlRequest::Network { policy, request } => authorize_network(policy, request), - ControlRequest::Model { policy, request } => authorize_model(policy, request), - } - } -} - -fn authorize_network(policy: &NetworkGuard, request: &NetworkAccessRequest) -> ControlTransition { - if policy.denied(request) { - return ControlTransition::denied(ControlReason::ExplicitlyDenied); - } - if policy.is_trusted(&request.host) { - return ControlTransition::allowed(ControlReason::TrustedLocal); - } - match policy.capability() { - NetworkCapability::Ambient => ControlTransition::allowed(ControlReason::AmbientNetwork), - NetworkCapability::Deny => ControlTransition::denied(ControlReason::NetworkDenied), - NetworkCapability::AllowList { .. } => match policy.evaluate(request) { - Ok(()) => ControlTransition::allowed(ControlReason::NetworkAllowList), - Err(NetworkMatchFailure::Empty) => { - ControlTransition::denied(ControlReason::NetworkAllowListEmpty) - } - Err(NetworkMatchFailure::Host) => { - ControlTransition::denied(ControlReason::HostNotAllowed) - } - Err(NetworkMatchFailure::Port) => { - ControlTransition::denied(ControlReason::PortNotAllowed) - } - Err(NetworkMatchFailure::Transport) => { - ControlTransition::denied(ControlReason::TransportNotAllowed) - } - Err(NetworkMatchFailure::ResolvedAddress) => { - ControlTransition::denied(ControlReason::ResolvedAddressNotAllowed) - } - }, - NetworkCapability::Policy { - default_action: NetworkDefaultAction::Allow, - .. - } => ControlTransition::allowed(ControlReason::AmbientNetwork), - NetworkCapability::Policy { - default_action: NetworkDefaultAction::Deny, - .. - } => match policy.evaluate(request) { - Ok(()) => ControlTransition::allowed(ControlReason::NetworkAllowList), - Err(NetworkMatchFailure::Empty) => { - ControlTransition::denied(ControlReason::NetworkAllowListEmpty) - } - Err(NetworkMatchFailure::Host) => { - ControlTransition::denied(ControlReason::HostNotAllowed) - } - Err(NetworkMatchFailure::Port) => { - ControlTransition::denied(ControlReason::PortNotAllowed) - } - Err(NetworkMatchFailure::Transport) => { - ControlTransition::denied(ControlReason::TransportNotAllowed) - } - Err(NetworkMatchFailure::ResolvedAddress) => { - ControlTransition::denied(ControlReason::ResolvedAddressNotAllowed) - } - }, - } -} - -fn authorize_model(policy: &ModelAccessPolicy, request: &ModelCallRequest) -> ControlTransition { - let model_allowed = policy.allowed_models.iter().any(|pattern| { - model_matches(pattern, &request.client_model) - || model_matches(pattern, &request.upstream_model) - }); - if !model_allowed { - return ControlTransition::denied(ControlReason::ModelNotAllowed); - } - if !policy.allowed_providers.is_empty() - && !policy - .allowed_providers - .iter() - .any(|provider| provider.eq_ignore_ascii_case(&request.provider)) - { - return ControlTransition::denied(ControlReason::ProviderNotAllowed); - } - ControlTransition::allowed(ControlReason::ModelAllowed) -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum NetworkHostRule { - Exact(String), - WildcardSuffix(String), - Ip(IpAddr), - Cidr(IpNet), -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct NetworkRule { - pub host: NetworkHostRule, - pub ports: Vec, - pub transports: Vec, - pub allow_private_ips: bool, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum NetworkMatchFailure { - Empty, - Host, - Port, - Transport, - ResolvedAddress, -} - -#[derive(Debug, Clone)] -pub struct NetworkGuard { - capability: NetworkCapability, - rules: Vec, - deny_rules: Vec, - trusted_hosts: Vec, -} - -impl NetworkGuard { - pub fn compile( - capability: NetworkCapability, - trusted_hosts: impl IntoIterator, - ) -> anyhow::Result { - let rules = match &capability { - NetworkCapability::AllowList { hosts, rules } => { - let mut compiled = hosts - .iter() - .map(|host| parse_network_rule(host)) - .collect::>>()?; - compiled.extend( - rules - .iter() - .map(compile_network_rule) - .collect::>>()?, - ); - compiled - } - NetworkCapability::Policy { allow, .. } => allow - .iter() - .map(compile_network_rule) - .collect::>>()?, - _ => Vec::new(), - }; - let deny_rules = match &capability { - NetworkCapability::Policy { deny, .. } => deny - .iter() - .map(compile_network_rule) - .collect::>>()?, - _ => Vec::new(), - }; - Ok(Self { - capability, - rules, - deny_rules, - trusted_hosts: trusted_hosts - .into_iter() - .map(|host| normalize_host(&host)) - .collect(), - }) - } - - pub fn capability(&self) -> &NetworkCapability { - &self.capability - } - - pub fn rules(&self) -> &[NetworkRule] { - &self.rules - } - - fn evaluate(&self, request: &NetworkAccessRequest) -> Result<(), NetworkMatchFailure> { - evaluate_rules(&self.rules, request, true) - } - - fn denied(&self, request: &NetworkAccessRequest) -> bool { - evaluate_rules(&self.deny_rules, request, false).is_ok() - } - - fn is_trusted(&self, host: &str) -> bool { - let host = normalize_host(host); - self.trusted_hosts.iter().any(|trusted| trusted == &host) - } -} - -fn evaluate_rules( - rules: &[NetworkRule], - request: &NetworkAccessRequest, - enforce_resolved_address_safety: bool, -) -> Result<(), NetworkMatchFailure> { - if rules.is_empty() { - return Err(NetworkMatchFailure::Empty); - } - let host_rules: Vec<&NetworkRule> = rules - .iter() - .filter(|rule| rule.matches_host(&request.host, request.resolved_ip)) - .collect(); - if host_rules.is_empty() { - return Err(if request.resolved_ip.is_some() { - NetworkMatchFailure::ResolvedAddress - } else { - NetworkMatchFailure::Host - }); - } - let port_rules: Vec<&NetworkRule> = host_rules - .into_iter() - .filter(|rule| rule.matches_port(request.port)) - .collect(); - if port_rules.is_empty() { - return Err(NetworkMatchFailure::Port); - } - let transport_rules: Vec<&NetworkRule> = port_rules - .into_iter() - .filter(|rule| rule.matches_transport(request.transport)) - .collect(); - if transport_rules.is_empty() { - return Err(NetworkMatchFailure::Transport); - } - if enforce_resolved_address_safety - && request.resolved_ip.is_some() - && !transport_rules - .iter() - .any(|rule| rule.allows_resolved_address(request.resolved_ip)) - { - return Err(NetworkMatchFailure::ResolvedAddress); - } - Ok(()) -} - -impl NetworkRule { - fn matches_host(&self, host: &str, resolved_ip: Option) -> bool { - let host = normalize_host(host); - let host_ip = IpAddr::from_str(&host).ok(); - match &self.host { - NetworkHostRule::Exact(allowed) => host == *allowed, - NetworkHostRule::WildcardSuffix(suffix) => { - host.ends_with(suffix) - && host.len() > suffix.len() - && host.as_bytes()[host.len() - suffix.len() - 1] == b'.' - } - NetworkHostRule::Ip(allowed) => { - host_ip == Some(*allowed) || resolved_ip == Some(*allowed) - } - NetworkHostRule::Cidr(network) => host_ip - .or(resolved_ip) - .is_some_and(|ip| network.contains(&ip)), - } - } - - fn matches_port(&self, port: Option) -> bool { - self.ports.is_empty() || port.is_some_and(|port| self.ports.contains(&port)) - } - - fn matches_transport(&self, transport: NetworkTransport) -> bool { - self.transports.is_empty() || self.transports.contains(&transport) - } - - fn allows_resolved_address(&self, resolved_ip: Option) -> bool { - let Some(ip) = resolved_ip else { - return true; - }; - match &self.host { - NetworkHostRule::Ip(allowed) => ip == *allowed, - NetworkHostRule::Cidr(network) => network.contains(&ip), - NetworkHostRule::Exact(_) | NetworkHostRule::WildcardSuffix(_) => { - is_public_egress_ip(ip) || (self.allow_private_ips && is_opt_in_private_ip(ip)) - } - } - } -} - -fn is_opt_in_private_ip(ip: IpAddr) -> bool { - match ip { - IpAddr::V4(ip) => ip.is_private() || ip.is_loopback(), - IpAddr::V6(ip) => { - if let Some(mapped) = ip.to_ipv4_mapped() { - return is_opt_in_private_ip(IpAddr::V4(mapped)); - } - ip.is_loopback() || ip.segments()[0] & 0xfe00 == 0xfc00 - } - } -} - -pub fn normalize_host(host: &str) -> String { - host.trim() - .trim_matches(|character| character == '[' || character == ']') - .to_ascii_lowercase() - .trim_end_matches('.') - .to_string() -} - -pub fn parse_network_rule(raw: &str) -> anyhow::Result { - compile_network_rule(&NetworkAccessRule { - host: raw.to_string(), - ports: Vec::new(), - transports: Vec::new(), - allow_private_ips: false, - }) -} - -fn compile_network_rule(rule: &NetworkAccessRule) -> anyhow::Result { - if rule.ports.contains(&0) { - anyhow::bail!("network rule ports must not contain zero"); - } - let mut ports = rule.ports.clone(); - ports.sort_unstable(); - ports.dedup(); - let mut transports = rule.transports.clone(); - transports.dedup(); - - let host = parse_network_host_rule(&rule.host)?; - Ok(NetworkRule { - host, - ports, - transports, - allow_private_ips: rule.allow_private_ips, - }) -} - -fn parse_network_host_rule(raw: &str) -> anyhow::Result { - let entry = raw.trim(); - if entry.is_empty() { - anyhow::bail!("network allowlist entry must not be empty"); - } - if entry.contains("://") || entry.contains(']') || entry.contains('[') { - anyhow::bail!( - "network allowlist entry `{entry}` must be a hostname, `*.suffix`, IP, or CIDR" - ); - } - if entry.contains('/') && IpNet::from_str(entry).is_err() { - anyhow::bail!( - "network allowlist entry `{entry}` must be a hostname, `*.suffix`, IP, or CIDR" - ); - } - if let Some((host, port)) = entry.rsplit_once(':') - && !host.is_empty() - && port.chars().all(|character| character.is_ascii_digit()) - && !entry.contains('/') - && host.parse::().is_err() - && !host.contains(':') - { - anyhow::bail!("network allowlist entry `{entry}` must not include a port"); - } - if let Some(suffix) = entry.strip_prefix("*.") { - let suffix = normalize_host(suffix); - if suffix.is_empty() || suffix.contains('*') || suffix.parse::().is_ok() { - anyhow::bail!("invalid wildcard network allowlist entry `{entry}`"); - } - return Ok(NetworkHostRule::WildcardSuffix(suffix)); - } - if entry.contains('*') { - anyhow::bail!("only leading `*.suffix` host wildcards are supported"); - } - if let Ok(network) = IpNet::from_str(entry) { - if entry.contains('/') { - return Ok(NetworkHostRule::Cidr(network)); - } - return Ok(NetworkHostRule::Ip(network.addr())); - } - if let Ok(ip) = IpAddr::from_str(entry) { - return Ok(NetworkHostRule::Ip(ip)); - } - let host = normalize_host(entry); - if host.is_empty() || host.contains(':') { - anyhow::bail!("invalid network allowlist hostname `{entry}`"); - } - Ok(NetworkHostRule::Exact(host)) -} - -pub fn host_matches(host: &str, rules: &[NetworkRule]) -> bool { - rules.iter().any(|rule| rule.matches_host(host, None)) -} - -/// Whether an address is suitable as the destination of a hostname rule -/// without an explicit private-address opt-in. -pub fn is_public_egress_ip(ip: IpAddr) -> bool { - match ip { - IpAddr::V4(ip) => { - let octets = ip.octets(); - octets[0] != 0 - && !ip.is_loopback() - && !ip.is_private() - && !ip.is_link_local() - && !ip.is_multicast() - && !ip.is_broadcast() - && !(octets[0] == 100 && (64..=127).contains(&octets[1])) - && !(octets[0] == 192 && octets[1] == 0 && octets[2] == 0) - && !(octets[0] == 192 && octets[1] == 0 && octets[2] == 2) - && !(octets[0] == 198 && (18..=19).contains(&octets[1])) - && !(octets[0] == 198 && octets[1] == 51 && octets[2] == 100) - && !(octets[0] == 203 && octets[1] == 0 && octets[2] == 113) - && octets[0] < 240 - } - IpAddr::V6(ip) => { - if let Some(mapped) = ip.to_ipv4_mapped() { - return is_public_egress_ip(IpAddr::V4(mapped)); - } - let segments = ip.segments(); - // Only globally-routed unicast space is eligible. This also - // rejects IPv4-compatible, site-local, link-local, ULA, and other - // special-purpose prefixes before considering narrower carveouts. - (segments[0] & 0xe000 == 0x2000) - && !(segments[0] == 0x2001 && segments[1] <= 0x01ff) - && !(segments[0] == 0x2001 && segments[1] == 0x0db8) - && segments[0] != 0x2002 - && !(segments[0] == 0x3fff && segments[1] & 0xf000 == 0) - } - } -} - -fn model_matches(pattern: &str, model: &str) -> bool { - if pattern == "*" { - return true; - } - if let Some(prefix) = pattern.strip_suffix('*') { - return !prefix.is_empty() && model.starts_with(prefix); - } - if let Some(suffix) = pattern.strip_prefix('*') { - return !suffix.is_empty() && model.ends_with(suffix); - } - pattern == model -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::NetworkTransport; - - fn network_request(host: &str) -> NetworkAccessRequest { - NetworkAccessRequest { - run_id: None, - attempt_id: None, - storyline_id: None, - host: host.into(), - port: Some(443), - transport: NetworkTransport::TcpTunnel, - resolved_ip: None, - } - } - - #[test] - fn authorization_is_a_control_transition() { - let controller = PolicyControlController; - let guard = NetworkGuard::compile(NetworkCapability::Deny, Vec::new()).unwrap(); - let transition = controller.authorize(ControlRequest::Network { - policy: &guard, - request: &network_request("example.com"), - }); - assert_eq!(transition.from, ControlState::Requested); - assert_eq!(transition.to, ControlState::Denied); - assert!(!transition.is_allowed()); - assert_eq!( - transition.applied().unwrap().to, - ControlState::Applied { - effect: ControlEffect::Deny - } - ); - } - - #[test] - fn allowed_operation_can_transition_to_applied_or_failed() { - let transition = ControlTransition::allowed(ControlReason::TrustedLocal); - assert_eq!( - transition.applied().unwrap().to, - ControlState::Applied { - effect: ControlEffect::Allow - } - ); - assert_eq!( - transition.failed().unwrap().to, - ControlState::Failed { - effect: ControlEffect::Allow - } - ); - } - - #[test] - fn machine_owns_state_and_transition_history() { - let controller = PolicyControlController; - let guard = NetworkGuard::compile(NetworkCapability::Ambient, Vec::new()).unwrap(); - let request = network_request("example.com"); - let mut machine = ControlMachine::new(); - let decision = machine - .authorize( - &controller, - ControlRequest::Network { - policy: &guard, - request: &request, - }, - ) - .unwrap(); - assert!(decision.is_allowed()); - assert_eq!(machine.state(), ControlState::Allowed); - machine.applied().unwrap(); - assert_eq!( - machine.state(), - ControlState::Applied { - effect: ControlEffect::Allow - } - ); - assert_eq!(machine.history().len(), 2); - let encoded = serde_json::to_string(&machine).unwrap(); - let restored: ControlMachine = serde_json::from_str(&encoded).unwrap(); - assert_eq!(restored.state(), machine.state()); - assert_eq!(restored.history(), machine.history()); - assert!( - machine - .authorize( - &controller, - ControlRequest::Network { - policy: &guard, - request: &request, - }, - ) - .is_err() - ); - } - - #[test] - fn policy_controller_handles_network_and_model_resources() { - let controller = PolicyControlController; - let guard = NetworkGuard::compile( - NetworkCapability::AllowList { - hosts: vec!["*.example.com".into(), "10.0.0.0/8".into()], - rules: Vec::new(), - }, - Vec::new(), - ) - .unwrap(); - assert!( - controller - .authorize(ControlRequest::Network { - policy: &guard, - request: &network_request("api.example.com"), - }) - .is_allowed() - ); - - let policy = ModelAccessPolicy { - allowed_models: vec!["claude-*".into()], - allowed_providers: vec!["anthropic".into()], - }; - let request = ModelCallRequest { - run_id: None, - attempt_id: None, - storyline_id: None, - call_id: "call-1".into(), - client_model: "claude-sonnet".into(), - upstream_model: "claude-sonnet".into(), - provider: "anthropic".into(), - protocol: "messages".into(), - upstream_host: "api.anthropic.com".into(), - }; - assert!( - controller - .authorize(ControlRequest::Model { - policy: &policy, - request: &request, - }) - .is_allowed() - ); - } - - #[test] - fn loopback_is_only_trusted_when_explicitly_configured() { - let controller = PolicyControlController; - let denied = NetworkGuard::compile(NetworkCapability::Deny, Vec::new()).unwrap(); - assert!( - !controller - .authorize(ControlRequest::Network { - policy: &denied, - request: &network_request("127.0.0.1"), - }) - .is_allowed() - ); - - let trusted = NetworkGuard::compile(NetworkCapability::Deny, ["127.0.0.1".into()]).unwrap(); - let transition = controller.authorize(ControlRequest::Network { - policy: &trusted, - request: &network_request("127.0.0.1"), - }); - assert!(transition.is_allowed()); - assert_eq!(transition.reason, ControlReason::TrustedLocal); - } - - #[test] - fn structured_rules_constrain_port_transport_and_resolved_address() { - let controller = PolicyControlController; - let guard = NetworkGuard::compile( - NetworkCapability::AllowList { - hosts: Vec::new(), - rules: vec![NetworkAccessRule { - host: "api.example.com".into(), - ports: vec![443], - transports: vec![NetworkTransport::Https], - allow_private_ips: false, - }], - }, - Vec::new(), - ) - .unwrap(); - let decide = |port, transport, resolved_ip| { - controller.authorize(ControlRequest::Network { - policy: &guard, - request: &NetworkAccessRequest { - run_id: None, - attempt_id: None, - storyline_id: None, - host: "api.example.com".into(), - port: Some(port), - transport, - resolved_ip, - }, - }) - }; - - assert!( - decide( - 443, - NetworkTransport::Https, - Some("93.184.216.34".parse().unwrap()) - ) - .is_allowed() - ); - assert_eq!( - decide( - 8443, - NetworkTransport::Https, - Some("93.184.216.34".parse().unwrap()) - ) - .reason, - ControlReason::PortNotAllowed - ); - assert_eq!( - decide( - 443, - NetworkTransport::TcpTunnel, - Some("93.184.216.34".parse().unwrap()) - ) - .reason, - ControlReason::TransportNotAllowed - ); - assert_eq!( - decide( - 443, - NetworkTransport::Https, - Some("127.0.0.1".parse().unwrap()) - ) - .reason, - ControlReason::ResolvedAddressNotAllowed - ); - } - - #[test] - fn explicit_ip_and_private_opt_in_allow_private_destinations() { - let controller = PolicyControlController; - for rule in [ - NetworkAccessRule { - host: "127.0.0.1".into(), - ports: vec![8080], - transports: vec![NetworkTransport::Http], - allow_private_ips: false, - }, - NetworkAccessRule { - host: "local.example".into(), - ports: vec![8080], - transports: vec![NetworkTransport::Http], - allow_private_ips: true, - }, - ] { - let host = rule.host.clone(); - let guard = NetworkGuard::compile( - NetworkCapability::AllowList { - hosts: Vec::new(), - rules: vec![rule], - }, - Vec::new(), - ) - .unwrap(); - let transition = controller.authorize(ControlRequest::Network { - policy: &guard, - request: &NetworkAccessRequest { - run_id: None, - attempt_id: None, - storyline_id: None, - host, - port: Some(8080), - transport: NetworkTransport::Http, - resolved_ip: Some("127.0.0.1".parse().unwrap()), - }, - }); - assert!(transition.is_allowed()); - } - } - - #[test] - fn address_classifier_rejects_local_and_metadata_ranges() { - for address in [ - "0.0.0.0", - "0.1.2.3", - "10.0.0.1", - "100.64.0.1", - "127.0.0.1", - "169.254.169.254", - "192.0.2.1", - "198.18.0.1", - "198.51.100.1", - "203.0.113.1", - "224.0.0.1", - "240.0.0.1", - "255.255.255.255", - "::1", - "::ffff:127.0.0.1", - "fc00::1", - "fe80::1", - "::192.0.2.1", - "2001::1", - "2001:db8::1", - "2002::1", - "3fff::1", - ] { - assert!(!is_public_egress_ip(address.parse().unwrap()), "{address}"); - } - assert!(is_public_egress_ip("1.1.1.1".parse().unwrap())); - assert!(is_public_egress_ip("8.8.8.8".parse().unwrap())); - assert!(is_public_egress_ip("2001:4860:4860::8888".parse().unwrap())); - assert!(is_public_egress_ip("2606:4700:4700::1111".parse().unwrap())); - } - - #[test] - fn private_opt_in_does_not_include_link_local_or_special_addresses() { - assert!(is_opt_in_private_ip("10.0.0.1".parse().unwrap())); - assert!(is_opt_in_private_ip("127.0.0.1".parse().unwrap())); - assert!(is_opt_in_private_ip("fc00::1".parse().unwrap())); - assert!(!is_opt_in_private_ip("169.254.169.254".parse().unwrap())); - assert!(!is_opt_in_private_ip("224.0.0.1".parse().unwrap())); - assert!(!is_opt_in_private_ip("fe80::1".parse().unwrap())); - } -} diff --git a/crates/persisting-agentctl/src/protocol.rs b/crates/persisting-agentctl/src/protocol.rs deleted file mode 100644 index 01605b95b..000000000 --- a/crates/persisting-agentctl/src/protocol.rs +++ /dev/null @@ -1,438 +0,0 @@ -//! AgentCtl v1 Control-plane wire contract. -//! -//! AgentCtl is an optional, cooperative channel between a control server and runtime -//! clients inside one Run. A client authenticates once with [`AgentRequest::Hello`] -//! and then periodically exchanges its [`AgentState`] for the server's current -//! [`AgentDirective`] through [`AgentRequest::Sync`]. Each bounded, -//! newline-delimited JSON connection carries exactly one request and one -//! response. -//! -//! # Session lifecycle and liveness -//! -//! `Hello` requires version 1, the Run-scoped token, and a non-empty -//! `client_id` that is unique among live Sessions. `Welcome` returns an opaque -//! Session ID, the recommended Sync interval, and the directive that the -//! client must observe before admitting work. A successful `Sync` refreshes -//! Session liveness and returns the latest directive. -//! -//! Outside a checkpoint, the server considers a Session stale after it misses -//! three recommended Sync intervals. A later `Hello` may replace a stale -//! Session. Live duplicates and capacity conflicts are rejected. Staleness is -//! diagnostic and lifecycle state; it is not proof that the client process -//! stopped. -//! -//! # Checkpoint protocol -//! -//! When the server publishes [`AgentDirective::Quiesce`], every runtime Session -//! that was live at checkpoint start must stop accepting work, drain in-flight -//! work, and report [`AgentState::Quiesced`] with the same checkpoint ID. -//! The server freezes that participant set, rejects every new or replacement -//! `Hello`, and never expires a participant until the checkpoint completes or -//! is abandoned. Each checkpoint attempt requires a fresh matching report; -//! an acknowledgement retained from an earlier attempt cannot satisfy it. -//! -//! A new `Quiesced` report whose ID does not match the active checkpoint is a -//! conflict. Repeating the exact accepted report is idempotent, including -//! after the server publishes `Continue` or `Shutdown`, so a still-quiesced client -//! can learn that new directive. Reporting `Active` or `Idle` while draining -//! removes any current acknowledgement. Clients remain quiesced until they -//! observe [`AgentDirective::Continue`] or [`AgentDirective::Shutdown`]. A -//! missing, late, or ambiguous acknowledgement fails the checkpoint rather -//! than producing an unsafe success. -//! -//! # Errors -//! -//! [`AgentErrorCode::InvalidRequest`] covers malformed frames and values; -//! [`AgentErrorCode::Unauthorized`] covers an invalid Run token or Session ID; -//! [`AgentErrorCode::VersionMismatch`] covers every non-v1 request; and -//! [`AgentErrorCode::Conflict`] covers live duplicates, capacity, checkpoint -//! admission, and state that conflicts with the active checkpoint. The -//! accompanying message is diagnostic text, not a machine contract. -//! -//! # Safety boundary -//! -//! Client states are Agent declarations. They are not enforcement evidence, -//! an authoritative process inventory, or proof that unreported external -//! effects do not exist. Integrations obtain authoritative process facts from the -//! execution provider. -//! -//! # Wire examples -//! -//! A client opens a Session: -//! -//! ```json -//! {"type":"hello","version":1,"token":"run-secret","client_id":"worker-1"} -//! ``` -//! -//! It then reports state and receives the current directive: -//! -//! ```json -//! {"type":"sync","version":1,"session_id":"session-1","state":{"kind":"active"}} -//! ``` -//! -//! ```json -//! {"type":"synced","directive":{"kind":"quiesce","checkpoint_id":"checkpoint-7"}} -//! ``` -//! -//! # Debug plane -//! -//! A future interactive terminal belongs to a separately authorized Debug -//! protocol and Session. PTY input, output, signals, and window size are not -//! extensions of the Control request, response, state, or directive enums. - -use serde::{Deserialize, Serialize}; - -/// The first public AgentCtl Control protocol version. -pub const AGENTCTL_VERSION: u32 = 1; -/// Maximum size of one newline-delimited request or response frame. -pub const AGENTCTL_MAX_FRAME_BYTES: usize = 1024 * 1024; - -/// Environment variable containing the Run-local AgentCtl endpoint. -pub const AGENTCTL_ENDPOINT_ENV: &str = "PERSISTING_AGENTCTL_ENDPOINT"; -/// Environment variable containing the Run-scoped authentication token. -pub const AGENTCTL_TOKEN_ENV: &str = "PERSISTING_AGENTCTL_TOKEN"; -/// Environment variable containing [`AGENTCTL_VERSION`]. -pub const AGENTCTL_VERSION_ENV: &str = "PERSISTING_AGENTCTL_VERSION"; -/// Environment variable naming the transport, currently `unix`. -pub const AGENTCTL_TRANSPORT_ENV: &str = "PERSISTING_AGENTCTL_TRANSPORT"; - -/// A request sent by a runtime client to the server. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(tag = "type", rename_all = "snake_case")] -pub enum AgentRequest { - /// Authenticate a runtime client and create a Session. - Hello { - /// Protocol version spoken by the client. - version: u32, - /// Run-scoped bearer token injected by the server. - token: String, - /// Non-empty client identity, unique among live Sessions in the Run. - client_id: String, - }, - /// Report current cooperative state and obtain the server's current directive. - Sync { - /// Protocol version spoken by the client. - version: u32, - /// Random Session identifier returned by [`AgentResponse::Welcome`]. - session_id: String, - /// Client state at the time this request was formed. - state: AgentState, - }, -} - -/// Cooperative state reported by one runtime client. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(tag = "kind", rename_all = "snake_case")] -pub enum AgentState { - /// The client may be running or draining work and is not at a safe point. - Active, - /// The client has no current work but may accept work until quiescence. - Idle, - /// The client reached the safe boundary for the named checkpoint. - Quiesced { - /// Checkpoint whose `Quiesce` directive the client observed. - checkpoint_id: String, - }, -} - -/// Desired cooperative state published by the server. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(tag = "kind", rename_all = "snake_case")] -pub enum AgentDirective { - /// Admit and run ordinary work. - Continue, - /// Stop admitting work and reach a safe checkpoint boundary. - Quiesce { - /// Identifier clients must echo in [`AgentState::Quiesced`]. - checkpoint_id: String, - /// Optional absolute deadline for reaching the boundary. - #[serde(default, skip_serializing_if = "Option::is_none")] - deadline_unix_ms: Option, - }, - /// Terminate the runtime client. - Shutdown { - /// Optional human-readable shutdown reason. - #[serde(default, skip_serializing_if = "Option::is_none")] - reason: Option, - }, -} - -/// Stable machine-readable category for an AgentCtl protocol failure. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum AgentErrorCode { - /// The request is malformed or contains invalid values. - InvalidRequest, - /// The Run token or Session ID is invalid. - Unauthorized, - /// The request does not speak [`AGENTCTL_VERSION`]. - VersionMismatch, - /// The request conflicts with a live Session or active checkpoint. - Conflict, -} - -/// A response sent by the server to a runtime client. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(tag = "type", rename_all = "snake_case")] -pub enum AgentResponse { - /// Successful Session creation. - Welcome { - /// Random bearer identifier for subsequent Sync requests. - session_id: String, - /// Recommended period between Sync requests. - sync_interval_ms: u64, - /// Current directive, which the client must observe before work starts. - directive: AgentDirective, - }, - /// Successful state synchronization. - Synced { - /// Current directive after the reported state was accepted. - directive: AgentDirective, - }, - /// Rejected request. - Error { - /// Stable category used for client control flow. - code: AgentErrorCode, - /// Human-readable diagnostic context, not a machine contract. - message: String, - }, -} - -#[cfg(test)] -mod tests { - use super::*; - use proptest::prelude::*; - - fn text_strategy() -> impl Strategy { - proptest::string::string_regex("[a-zA-Z0-9_ ./:-]{0,32}").unwrap() - } - - fn state_strategy() -> impl Strategy { - prop_oneof![ - Just(AgentState::Active), - Just(AgentState::Idle), - text_strategy().prop_map(|checkpoint_id| AgentState::Quiesced { checkpoint_id }), - ] - } - - fn directive_strategy() -> impl Strategy { - prop_oneof![ - Just(AgentDirective::Continue), - (text_strategy(), prop::option::of(any::())).prop_map( - |(checkpoint_id, deadline_unix_ms)| AgentDirective::Quiesce { - checkpoint_id, - deadline_unix_ms, - }, - ), - prop::option::of(text_strategy()) - .prop_map(|reason| AgentDirective::Shutdown { reason }), - ] - } - - fn request_strategy() -> impl Strategy { - prop_oneof![ - (any::(), text_strategy(), text_strategy()).prop_map( - |(version, token, client_id)| AgentRequest::Hello { - version, - token, - client_id, - }, - ), - (any::(), text_strategy(), state_strategy()).prop_map( - |(version, session_id, state)| AgentRequest::Sync { - version, - session_id, - state, - }, - ), - ] - } - - fn response_strategy() -> impl Strategy { - prop_oneof![ - (text_strategy(), any::(), directive_strategy()).prop_map( - |(session_id, sync_interval_ms, directive)| AgentResponse::Welcome { - session_id, - sync_interval_ms, - directive, - }, - ), - directive_strategy().prop_map(|directive| AgentResponse::Synced { directive }), - ( - prop_oneof![ - Just(AgentErrorCode::InvalidRequest), - Just(AgentErrorCode::Unauthorized), - Just(AgentErrorCode::VersionMismatch), - Just(AgentErrorCode::Conflict), - ], - text_strategy() - ) - .prop_map(|(code, message)| AgentResponse::Error { code, message }), - ] - } - - #[test] - fn hello_has_the_v1_wire_shape() { - let request = AgentRequest::Hello { - version: 1, - token: "secret".into(), - client_id: "worker-1".into(), - }; - - assert_eq!( - serde_json::to_value(&request).unwrap(), - serde_json::json!({ - "type": "hello", - "version": 1, - "token": "secret", - "client_id": "worker-1" - }) - ); - assert_eq!(AGENTCTL_VERSION, 1); - } - - #[test] - fn quiesced_sync_and_quiesce_response_roundtrip() { - let request = AgentRequest::Sync { - version: 1, - session_id: "session-1".into(), - state: AgentState::Quiesced { - checkpoint_id: "checkpoint-7".into(), - }, - }; - let response = AgentResponse::Synced { - directive: AgentDirective::Quiesce { - checkpoint_id: "checkpoint-7".into(), - deadline_unix_ms: Some(1_786_890_000_000), - }, - }; - - assert_eq!( - serde_json::from_value::(serde_json::to_value(&request).unwrap()) - .unwrap(), - request - ); - assert_eq!( - serde_json::from_value::(serde_json::to_value(&response).unwrap()) - .unwrap(), - response - ); - } - - #[test] - fn optional_directive_fields_are_omitted() { - assert_eq!( - serde_json::to_value(AgentDirective::Continue).unwrap(), - serde_json::json!({ "kind": "continue" }) - ); - assert_eq!( - serde_json::to_value(AgentDirective::Shutdown { reason: None }).unwrap(), - serde_json::json!({ "kind": "shutdown" }) - ); - } - - #[test] - fn error_codes_use_stable_snake_case_names() { - let cases = [ - (AgentErrorCode::InvalidRequest, "invalid_request"), - (AgentErrorCode::Unauthorized, "unauthorized"), - (AgentErrorCode::VersionMismatch, "version_mismatch"), - (AgentErrorCode::Conflict, "conflict"), - ]; - - for (code, expected) in cases { - assert_eq!(serde_json::to_value(code).unwrap(), expected); - } - } - - #[test] - fn welcome_and_error_have_exact_v1_wire_shapes() { - let welcome = AgentResponse::Welcome { - session_id: "session-1".into(), - sync_interval_ms: 1_000, - directive: AgentDirective::Continue, - }; - assert_eq!( - serde_json::to_value(welcome).unwrap(), - serde_json::json!({ - "type": "welcome", - "session_id": "session-1", - "sync_interval_ms": 1_000, - "directive": { "kind": "continue" } - }) - ); - - let error = AgentResponse::Error { - code: AgentErrorCode::Conflict, - message: "checkpoint is active".into(), - }; - assert_eq!( - serde_json::to_value(error).unwrap(), - serde_json::json!({ - "type": "error", - "code": "conflict", - "message": "checkpoint is active" - }) - ); - } - - #[test] - fn idle_and_directive_variants_have_exact_v1_wire_shapes() { - assert_eq!( - serde_json::to_value(AgentState::Idle).unwrap(), - serde_json::json!({ "kind": "idle" }) - ); - assert_eq!( - serde_json::to_value(AgentDirective::Quiesce { - checkpoint_id: "cp".into(), - deadline_unix_ms: None, - }) - .unwrap(), - serde_json::json!({ "kind": "quiesce", "checkpoint_id": "cp" }) - ); - assert_eq!( - serde_json::to_value(AgentDirective::Shutdown { - reason: Some("maintenance".into()), - }) - .unwrap(), - serde_json::json!({ "kind": "shutdown", "reason": "maintenance" }) - ); - } - - proptest! { - #[test] - fn every_request_roundtrips_through_json(request in request_strategy()) { - let encoded = serde_json::to_value(&request).unwrap(); - let decoded = serde_json::from_value::(encoded).unwrap(); - prop_assert_eq!(decoded, request); - } - - #[test] - fn every_response_roundtrips_through_json(response in response_strategy()) { - let encoded = serde_json::to_value(&response).unwrap(); - let decoded = serde_json::from_value::(encoded).unwrap(); - prop_assert_eq!(decoded, response); - } - - #[test] - fn optional_directive_fields_follow_presence_semantics( - checkpoint_id in text_strategy(), - deadline_unix_ms in prop::option::of(any::()), - reason in prop::option::of(text_strategy()), - ) { - let quiesce = serde_json::to_value(AgentDirective::Quiesce { - checkpoint_id, - deadline_unix_ms, - }).unwrap(); - prop_assert_eq!( - quiesce.get("deadline_unix_ms").is_some(), - deadline_unix_ms.is_some(), - ); - - let shutdown = serde_json::to_value(AgentDirective::Shutdown { - reason: reason.clone(), - }).unwrap(); - prop_assert_eq!(shutdown.get("reason").is_some(), reason.is_some()); - } - } -} diff --git a/crates/persisting-agentctl/src/runtime.rs b/crates/persisting-agentctl/src/runtime.rs deleted file mode 100644 index 2cd76a570..000000000 --- a/crates/persisting-agentctl/src/runtime.rs +++ /dev/null @@ -1,956 +0,0 @@ -//! Stable value types shared by capture, storage, and runtime integrations. -//! -//! The runtime and narrative dimensions are deliberately orthogonal: -//! -//! - one [`RunId`] may have multiple execution [`AttemptId`]s; -//! - events may additionally belong to a [`StorylineId`], turn, and call. - -use serde::{Deserialize, Serialize}; -use serde_json::Value; -use std::collections::BTreeMap; -use std::fmt; - -pub const RUNTIME_SCHEMA_VERSION: u32 = 1; - -fn default_supervisor_connect_timeout_ms() -> u64 { - 500 -} - -fn default_attempt_ttl_ms() -> u64 { - 15_000 -} - -macro_rules! string_id { - ($name:ident) => { - #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] - #[serde(transparent)] - pub struct $name(pub String); - - impl $name { - pub fn new(value: impl Into) -> Self { - Self(value.into()) - } - - pub fn as_str(&self) -> &str { - &self.0 - } - - pub fn is_empty(&self) -> bool { - self.0.trim().is_empty() - } - } - - impl fmt::Display for $name { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(&self.0) - } - } - - impl From for $name { - fn from(value: String) -> Self { - Self(value) - } - } - - impl From<&str> for $name { - fn from(value: &str) -> Self { - Self(value.to_string()) - } - } - }; -} - -string_id!(RunId); -string_id!(AttemptId); -string_id!(StorylineId); - -/// Connection material supplied by a coordinator in a RunSpec. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct SupervisorBootstrap { - pub endpoint: String, - pub token: String, - pub controller_epoch: u64, - #[serde(default = "default_supervisor_connect_timeout_ms")] - pub connect_timeout_ms: u64, - /// pChronicle root used to publish durable Attempt liveness and - /// terminal results. This is optional for standalone Runs. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub attempt_registry_uri: Option, - #[serde(default = "default_attempt_ttl_ms")] - pub attempt_ttl_ms: u64, -} - -/// A versioned logical Agent reference. It describes identity, not placement. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct AgentRef { - pub name: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub version: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub adapter: Option, -} - -impl AgentRef { - pub fn new(name: impl Into) -> Self { - Self { - name: name.into(), - version: None, - adapter: None, - } - } -} - -/// One semantic Agent execution described by a runtime integration. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct RunSpec { - #[serde(default = "runtime_schema_version")] - pub schema_version: u32, - pub run_id: RunId, - /// Monotonic coordinator ownership generation. Zero is reserved for callers that - /// do not use durable orchestration/fencing. - #[serde(default)] - pub lease_epoch: u64, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub task_id: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub parent_run_id: Option, - pub agent: AgentRef, - pub invocation: RunInvocation, - #[serde(default)] - pub input: Value, - #[serde(default)] - pub runtime: RuntimeConfig, - #[serde(default)] - pub capabilities: CapabilitySet, - /// Optional coordinator control channel. Absence, connection failure, or later - /// disconnection never prevents standalone execution. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub supervisor: Option, - #[serde(default)] - pub metadata: BTreeMap, -} - -impl RunSpec { - pub fn process( - run_id: impl Into, - agent: impl Into, - program: impl Into, - ) -> Self { - Self { - schema_version: RUNTIME_SCHEMA_VERSION, - run_id: run_id.into(), - lease_epoch: 0, - task_id: None, - parent_run_id: None, - agent: AgentRef::new(agent), - invocation: RunInvocation::Process(ProcessInvocation::new(program)), - input: Value::Null, - runtime: RuntimeConfig::default(), - capabilities: CapabilitySet::default(), - supervisor: None, - metadata: BTreeMap::new(), - } - } -} - -fn runtime_schema_version() -> u32 { - RUNTIME_SCHEMA_VERSION -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(tag = "kind", rename_all = "snake_case")] -pub enum RunInvocation { - Process(ProcessInvocation), -} - -/// Local or container-hosted process invocation. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ProcessInvocation { - pub program: String, - #[serde(default)] - pub args: Vec, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub cwd: Option, - #[serde(default)] - pub env: BTreeMap, - #[serde(default = "default_true")] - pub inherit_env: bool, - #[serde(default)] - pub stdin: StdioMode, - #[serde(default)] - pub stdout: StdioMode, - #[serde(default)] - pub stderr: StdioMode, -} - -impl ProcessInvocation { - pub fn new(program: impl Into) -> Self { - Self { - program: program.into(), - args: Vec::new(), - cwd: None, - env: BTreeMap::new(), - inherit_env: true, - stdin: StdioMode::Inherit, - stdout: StdioMode::Inherit, - stderr: StdioMode::Inherit, - } - } -} - -fn default_true() -> bool { - true -} - -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum StdioMode { - #[default] - Inherit, - Capture, - Null, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct RuntimeConfig { - /// Wall-clock limit for one Attempt. `None` means no execution deadline. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub timeout_ms: Option, - /// Grace period between a cooperative process-tree termination request and - /// forced termination. - #[serde(default = "default_termination_grace_ms")] - pub termination_grace_ms: u64, - /// Maximum retained bytes for each captured stdout/stderr stream. - #[serde(default = "default_max_output_bytes")] - pub max_output_bytes: usize, - /// Optional process-tree resource budget. Executors must report which - /// fields they can enforce; an unset field is intentionally unlimited. - #[serde(default)] - pub resource_limits: ResourceLimits, - #[serde(default)] - pub policy_mode: PolicyMode, -} - -impl Default for RuntimeConfig { - fn default() -> Self { - Self { - timeout_ms: None, - termination_grace_ms: default_termination_grace_ms(), - max_output_bytes: default_max_output_bytes(), - resource_limits: ResourceLimits::default(), - policy_mode: PolicyMode::Audit, - } - } -} - -#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] -pub struct ResourceLimits { - /// Aggregate resident-memory budget when the executor has a process-tree - /// controller; otherwise an address-space limit may be used and reported. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub memory_bytes: Option, - /// Maximum number of processes/threads admitted for the Run. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub processes: Option, - /// CPU time budget in milliseconds. Native rlimit backends round this up - /// to whole seconds. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub cpu_time_ms: Option, - /// Maximum number of open file descriptors inherited by descendants. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub open_files: Option, - /// Maximum size of a file created by the process. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub file_size_bytes: Option, -} - -impl ResourceLimits { - pub fn is_empty(&self) -> bool { - self.memory_bytes.is_none() - && self.processes.is_none() - && self.cpu_time_ms.is_none() - && self.open_files.is_none() - && self.file_size_bytes.is_none() - } -} - -fn default_max_output_bytes() -> usize { - 1024 * 1024 -} - -fn default_termination_grace_ms() -> u64 { - 2_000 -} - -/// `Audit` records requested capabilities. `Enforce` requires an executor that -/// can actually prevent ambient access. -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum PolicyMode { - #[default] - Audit, - Enforce, -} - -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct CapabilitySet { - #[serde(default)] - pub models: Vec, - #[serde(default)] - pub tools: Vec, - #[serde(default)] - pub filesystem: Vec, - #[serde(default)] - pub network: NetworkCapability, - #[serde(default)] - pub secrets: Vec, - #[serde(default)] - pub allow_subprocess: bool, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct FilesystemCapability { - pub path: String, - pub access: FilesystemAccess, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum FilesystemAccess { - Read, - ReadWrite, -} - -#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(tag = "mode", rename_all = "snake_case")] -pub enum NetworkCapability { - /// Use the executor's ambient network. Only valid for audit/compatibility runs. - #[default] - Ambient, - Deny, - AllowList { - /// Legacy host-only rules. Empty port/transport constraints mean any. - #[serde(default)] - hosts: Vec, - /// Structured rules for protocol- and port-scoped access. - #[serde(default, skip_serializing_if = "Vec::is_empty")] - rules: Vec, - }, - /// Ordered policy surface: explicit denies win, then the default action - /// determines whether an allow rule is required. - Policy { - default_action: NetworkDefaultAction, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - allow: Vec, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - deny: Vec, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - limits: Vec, - }, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum NetworkDefaultAction { - Allow, - Deny, -} - -/// One declarative network grant. `host` accepts an exact hostname, -/// `*.suffix`, an IP literal, or a CIDR. Empty `ports` or `transports` mean -/// unrestricted for that dimension. Hostname rules reject private and -/// loopback resolved addresses unless `allow_private_ips` is enabled; other -/// special-purpose addresses still require an explicit IP or CIDR grant. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct NetworkAccessRule { - pub host: String, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub ports: Vec, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub transports: Vec, - #[serde(default)] - pub allow_private_ips: bool, -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] -pub struct NetworkBandwidthLimit { - /// Absent means all intercepted destinations. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub host: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub port: Option, - pub bytes_per_second: u64, -} - -/// Runtime-neutral network request presented to a controller for authorization. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct NetworkAccessRequest { - #[serde(default, skip_serializing_if = "Option::is_none")] - pub run_id: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub attempt_id: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub storyline_id: Option, - pub host: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub port: Option, - pub transport: NetworkTransport, - /// Address selected from the request hostname's current DNS result. This - /// is absent only during a pre-resolution policy check. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub resolved_ip: Option, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum NetworkTransport { - Http, - Https, - TcpTunnel, -} - -/// Model invocation metadata. Request/response bodies intentionally stay in -/// Capture; the controller receives only the information needed for policy and audit. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ModelCallRequest { - #[serde(default, skip_serializing_if = "Option::is_none")] - pub run_id: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub attempt_id: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub storyline_id: Option, - pub call_id: String, - pub client_model: String, - pub upstream_model: String, - pub provider: String, - pub protocol: String, - pub upstream_host: String, -} - -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct ModelAccessPolicy { - #[serde(default)] - pub allowed_models: Vec, - #[serde(default)] - pub allowed_providers: Vec, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum AccessEffect { - Allow, - Deny, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum AccessReason { - AmbientNetwork, - TrustedLocal, - NetworkAllowList, - NetworkDenied, - NetworkAllowListEmpty, - HostNotAllowed, - PortNotAllowed, - TransportNotAllowed, - ResolvedAddressNotAllowed, - ExplicitlyDenied, - ModelAllowed, - ModelNotAllowed, - ProviderNotAllowed, -} - -impl AccessReason { - pub fn code(self) -> &'static str { - match self { - Self::AmbientNetwork => "ambient-network", - Self::TrustedLocal => "trusted-local", - Self::NetworkAllowList => "network-allowlist", - Self::NetworkDenied => "network-denied", - Self::NetworkAllowListEmpty => "network-allowlist-empty", - Self::HostNotAllowed => "host-not-allowed", - Self::PortNotAllowed => "port-not-allowed", - Self::TransportNotAllowed => "transport-not-allowed", - Self::ResolvedAddressNotAllowed => "resolved-address-not-allowed", - Self::ExplicitlyDenied => "explicitly-denied", - Self::ModelAllowed => "model-allowed", - Self::ModelNotAllowed => "model-not-allowed", - Self::ProviderNotAllowed => "provider-not-allowed", - } - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum RunState { - Created, - Starting, - Running, - Checkpointing, - Suspended, - Cancelling, - Completed, - Failed, - Cancelled, -} - -impl RunState { - pub fn is_terminal(self) -> bool { - matches!(self, Self::Completed | Self::Failed | Self::Cancelled) - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum ExecutorKind { - Process, - Container, - VirtualMachine, - Wasm, - Remote, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum IsolationKind { - HostProcess, - /// Unprivileged process isolated by a user namespace and host-kernel policy. - RootlessProcess, - /// Host process constrained by a platform sandbox such as macOS Seatbelt. - SandboxedProcess, - Container, - VirtualMachine, - Wasm, -} - -/// Strength of the boundary claimed for one capability dimension. -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum EnforcementLevel { - #[default] - Unenforced, - Cooperative, - Enforced, -} - -/// Independently enforceable dimensions of a Run policy. -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum CapabilityDimension { - Models, - Tools, - FilesystemRead, - FilesystemWrite, - Network, - Secrets, - Subprocess, - Resources, -} - -impl fmt::Display for CapabilityDimension { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let name = match self { - Self::Models => "models", - Self::Tools => "tools", - Self::FilesystemRead => "filesystem_read", - Self::FilesystemWrite => "filesystem_write", - Self::Network => "network", - Self::Secrets => "secrets", - Self::Subprocess => "subprocess", - Self::Resources => "resources", - }; - f.write_str(name) - } -} - -/// Auditable claim for one dimension. `mechanisms` names the concrete boundary -/// rather than inferring enforcement from an executor or isolation label. -#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] -pub struct EnforcementEvidence { - #[serde(default)] - pub level: EnforcementLevel, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub mechanisms: Vec, -} - -impl EnforcementEvidence { - pub fn new(level: EnforcementLevel, mechanism: impl Into) -> Self { - Self { - level, - mechanisms: vec![mechanism.into()], - } - } - - pub fn is_enforced(&self) -> bool { - self.level == EnforcementLevel::Enforced - } -} - -/// Sparse, per-dimension enforcement evidence advertised by an executor and -/// augmented by runtime drivers for a specific Run. -#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] -pub struct CapabilityEnforcementEvidence { - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub dimensions: BTreeMap, -} - -impl CapabilityEnforcementEvidence { - pub fn record( - &mut self, - dimension: CapabilityDimension, - level: EnforcementLevel, - mechanism: impl Into, - ) { - let mechanism = mechanism.into(); - let evidence = self.dimensions.entry(dimension).or_default(); - if level > evidence.level { - evidence.level = level; - } - if !evidence.mechanisms.contains(&mechanism) { - evidence.mechanisms.push(mechanism); - } - } - - pub fn enforced( - mut self, - dimension: CapabilityDimension, - mechanism: impl Into, - ) -> Self { - self.record(dimension, EnforcementLevel::Enforced, mechanism); - self - } - - pub fn cooperative( - mut self, - dimension: CapabilityDimension, - mechanism: impl Into, - ) -> Self { - self.record(dimension, EnforcementLevel::Cooperative, mechanism); - self - } - - pub fn evidence(&self, dimension: CapabilityDimension) -> Option<&EnforcementEvidence> { - self.dimensions.get(&dimension) - } - - pub fn is_enforced(&self, dimension: CapabilityDimension) -> bool { - self.evidence(dimension) - .is_some_and(EnforcementEvidence::is_enforced) - } - - pub fn missing_dimensions( - &self, - capabilities: &CapabilitySet, - resources: &ResourceLimits, - ) -> Vec { - requested_enforcement_dimensions(capabilities, resources) - .into_iter() - .filter(|dimension| !self.is_enforced(*dimension)) - .collect() - } -} - -/// Dimensions that require non-bypassable evidence when `PolicyMode::Enforce` -/// is selected. Network is always included because `Ambient` is explicitly an -/// audit/compatibility mode, not an enforceable boundary. -pub fn requested_enforcement_dimensions( - capabilities: &CapabilitySet, - resources: &ResourceLimits, -) -> Vec { - use std::collections::BTreeSet; - - let mut requested = BTreeSet::new(); - if !capabilities.models.is_empty() { - requested.insert(CapabilityDimension::Models); - } - if !capabilities.tools.is_empty() { - requested.insert(CapabilityDimension::Tools); - } - for filesystem in &capabilities.filesystem { - requested.insert(CapabilityDimension::FilesystemRead); - if filesystem.access == FilesystemAccess::ReadWrite { - requested.insert(CapabilityDimension::FilesystemWrite); - } - } - // PolicyMode::Enforce must fail closed for `Ambient` too: selecting ambient - // access is not evidence that direct sockets are confined by a boundary. - requested.insert(CapabilityDimension::Network); - if !capabilities.secrets.is_empty() { - requested.insert(CapabilityDimension::Secrets); - } - // `false` is a deny policy, not the absence of a request. Enforce mode must - // have a non-bypassable mechanism that prevents fork/exec. When subprocesses - // are explicitly allowed there is no deny boundary to prove. - if !capabilities.allow_subprocess { - requested.insert(CapabilityDimension::Subprocess); - } - if !resources.is_empty() { - requested.insert(CapabilityDimension::Resources); - } - requested.into_iter().collect() -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ExecutorDescriptor { - pub name: String, - pub kind: ExecutorKind, - pub isolation: IsolationKind, - #[serde(default)] - pub capability_enforcement: CapabilityEnforcementEvidence, - pub supports_checkpoint: bool, - pub supports_migration: bool, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct AttemptInfo { - pub attempt_id: AttemptId, - #[serde(default)] - pub lease_epoch: u64, - pub number: u32, - pub executor: ExecutorDescriptor, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub started_at_unix_ms: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub finished_at_unix_ms: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct RunStatus { - pub run_id: RunId, - pub state: RunState, - pub attempt: AttemptInfo, - pub updated_at_unix_ms: u64, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub message: Option, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum RunFailureKind { - InvalidSpec, - Unsupported, - Spawn, - ProcessExit, - Workload, - DeadlineExceeded, - Infrastructure, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct RunFailure { - pub kind: RunFailureKind, - pub message: String, - #[serde(default)] - pub retryable: bool, -} - -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct ProcessOutput { - #[serde(default, skip_serializing_if = "Option::is_none")] - pub stdout: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub stderr: Option, - #[serde(default)] - pub stdout_truncated: bool, - #[serde(default)] - pub stderr_truncated: bool, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ArtifactRef { - pub name: String, - pub uri: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub media_type: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub digest: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct RunResult { - pub run_id: RunId, - pub attempt_id: AttemptId, - #[serde(default)] - pub lease_epoch: u64, - pub state: RunState, - pub started_at_unix_ms: u64, - pub finished_at_unix_ms: u64, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub exit_code: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub failure: Option, - #[serde(default)] - pub output: ProcessOutput, - /// Small structured workload result. Large outputs belong in `artifacts`. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub value: Option, - #[serde(default)] - pub metrics: BTreeMap, - #[serde(default)] - pub artifacts: Vec, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub event_stream_ref: Option, - #[serde(default)] - pub warnings: Vec, -} - -/// The current execution owner for one logical Run. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct RunLeaseRecord { - pub run_id: RunId, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub task_id: Option, - pub epoch: u64, - pub owner: String, - pub issued_at_unix_ms: u64, - pub expires_at_unix_ms: u64, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub attempt_id: Option, -} - -/// Immutable terminal commit request. `result_digest` binds the commit to the -/// durable completion record without embedding an arbitrarily large -/// result in the control object. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct RunCommitRequest { - pub run_id: RunId, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub task_id: Option, - pub attempt_id: AttemptId, - pub lease_epoch: u64, - pub state: RunState, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub event_high_watermark: Option, - pub result_digest: String, -} - -/// The sole terminal result visible for a Run. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct RunCommit { - #[serde(flatten)] - pub request: RunCommitRequest, - pub committed_at_unix_ms: u64, -} - -/// CAS-managed pChronicle control record. Lease acquisition and terminal -/// commit update this same object, closing the stale-lease/commit race. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct RunControlRecord { - pub revision: u64, - pub run_id: RunId, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub lease: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub commit: Option, -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn process_run_spec_json_roundtrips() { - let mut spec = RunSpec::process("run-1", "codex", "codex"); - let RunInvocation::Process(process) = &mut spec.invocation; - process.args = vec!["exec".into(), "review".into()]; - process.stdout = StdioMode::Capture; - spec.capabilities.network = NetworkCapability::AllowList { - hosts: vec!["api.openai.com".into()], - rules: Vec::new(), - }; - - let json = serde_json::to_string(&spec).unwrap(); - let decoded: RunSpec = serde_json::from_str(&json).unwrap(); - assert_eq!(decoded.run_id.as_str(), "run-1"); - assert_eq!(decoded.schema_version, RUNTIME_SCHEMA_VERSION); - assert!(matches!( - decoded.capabilities.network, - NetworkCapability::AllowList { .. } - )); - } - - #[test] - fn network_policy_roundtrips_deny_and_bandwidth_limits() { - let capability = NetworkCapability::Policy { - default_action: NetworkDefaultAction::Deny, - allow: vec![NetworkAccessRule { - host: "api.example.com".into(), - ports: vec![443], - transports: vec![NetworkTransport::TcpTunnel], - allow_private_ips: false, - }], - deny: vec![NetworkAccessRule { - host: "169.254.0.0/16".into(), - ports: Vec::new(), - transports: Vec::new(), - allow_private_ips: false, - }], - limits: vec![NetworkBandwidthLimit { - host: None, - port: None, - bytes_per_second: 1_250_000, - }], - }; - let json = serde_json::to_string(&capability).unwrap(); - assert_eq!( - serde_json::from_str::(&json).unwrap(), - capability - ); - } - - #[test] - fn enforcement_evidence_checks_only_requested_dimensions() { - let capabilities = CapabilitySet { - filesystem: vec![FilesystemCapability { - path: "/workspace".into(), - access: FilesystemAccess::ReadWrite, - }], - network: NetworkCapability::Deny, - ..CapabilitySet::default() - }; - let evidence = CapabilityEnforcementEvidence::default() - .enforced(CapabilityDimension::FilesystemRead, "mount-namespace") - .enforced(CapabilityDimension::FilesystemWrite, "mount-namespace"); - - assert_eq!( - evidence.missing_dimensions(&capabilities, &ResourceLimits::default()), - vec![ - CapabilityDimension::Network, - CapabilityDimension::Subprocess - ] - ); - } - - #[test] - fn subprocess_deny_requires_evidence_but_explicit_allow_does_not() { - let denied = CapabilitySet::default(); - assert!( - requested_enforcement_dimensions(&denied, &ResourceLimits::default()) - .contains(&CapabilityDimension::Subprocess) - ); - - let allowed = CapabilitySet { - allow_subprocess: true, - ..CapabilitySet::default() - }; - assert!( - !requested_enforcement_dimensions(&allowed, &ResourceLimits::default()) - .contains(&CapabilityDimension::Subprocess) - ); - } - - #[test] - fn descriptor_without_new_evidence_field_remains_readable() { - let descriptor: ExecutorDescriptor = serde_json::from_value(serde_json::json!({ - "name": "legacy", - "kind": "process", - "isolation": "host_process", - "enforces_capabilities": false, - "supports_checkpoint": false, - "supports_migration": false - })) - .unwrap(); - - assert!(descriptor.capability_enforcement.dimensions.is_empty()); - } -} diff --git a/crates/persisting-agentctl/src/supervisor.rs b/crates/persisting-agentctl/src/supervisor.rs deleted file mode 100644 index be081daf8..000000000 --- a/crates/persisting-agentctl/src/supervisor.rs +++ /dev/null @@ -1,94 +0,0 @@ -//! Versioned Supervisor protocol shared by the control and execution planes. - -use crate::{AttemptId, NetworkBandwidthLimit, RunId}; -use serde::{Deserialize, Serialize}; - -pub const SUPERVISOR_PROTOCOL_VERSION: u32 = 1; - -fn supervisor_protocol_version() -> u32 { - SUPERVISOR_PROTOCOL_VERSION -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct SupervisorRegistration { - #[serde(default = "supervisor_protocol_version")] - pub protocol_version: u32, - pub token: String, - pub run_id: RunId, - pub attempt_id: AttemptId, - pub lease_epoch: u64, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct SupervisorHeartbeat { - pub last_applied_directive_seq: u64, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct SupervisorDirectiveAck { - pub directive_seq: u64, - pub applied: bool, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(tag = "type", rename_all = "snake_case")] -pub enum SupervisorClientMessage { - Register(SupervisorRegistration), - Heartbeat(SupervisorHeartbeat), - Ack(SupervisorDirectiveAck), -} - -/// A time-bounded rate grant enforced locally by a runtime integration. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct SupervisorNetworkQuotaGrant { - pub grant_id: String, - pub quota_epoch: u64, - pub valid_until_unix_ms: u64, - pub limit: NetworkBandwidthLimit, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(tag = "kind", rename_all = "snake_case")] -pub enum SupervisorDirective { - GrantNetworkQuota(SupervisorNetworkQuotaGrant), - Cancel, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct SupervisorDirectiveEnvelope { - pub controller_epoch: u64, - pub lease_epoch: u64, - pub directive_seq: u64, - pub directive: SupervisorDirective, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(tag = "type", rename_all = "snake_case")] -pub enum SupervisorServerMessage { - Registered { - controller_epoch: u64, - directives: Vec, - }, - Directive(SupervisorDirectiveEnvelope), - Error { - message: String, - }, -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn wire_messages_roundtrip_as_json() { - let message = SupervisorServerMessage::Directive(SupervisorDirectiveEnvelope { - controller_epoch: 4, - lease_epoch: 9, - directive_seq: 2, - directive: SupervisorDirective::Cancel, - }); - let encoded = serde_json::to_vec(&message).unwrap(); - let decoded: SupervisorServerMessage = serde_json::from_slice(&encoded).unwrap(); - assert_eq!(decoded, message); - } -} diff --git a/crates/persisting-events/Cargo.toml b/crates/persisting-events/Cargo.toml deleted file mode 100644 index 548e69d70..000000000 --- a/crates/persisting-events/Cargo.toml +++ /dev/null @@ -1,29 +0,0 @@ -[package] -name = "persisting-events" -version.workspace = true -edition.workspace = true -authors.workspace = true -license.workspace = true -description = "Shared runtime event contracts for Persisting components" - -[features] -default = [] -control = [ - "dep:anyhow", - "dep:async-trait", - "dep:persisting-agentctl", - "dep:tokio", -] - -[dependencies] -anyhow = { workspace = true, optional = true } -async-trait = { workspace = true, optional = true } -persisting-agentctl = { workspace = true, optional = true } -serde = { workspace = true, features = ["derive"] } -serde_json.workspace = true -thiserror.workspace = true -tokio = { workspace = true, features = ["io-util", "net", "process", "sync"], optional = true } - -[dev-dependencies] -proptest.workspace = true -tokio = { workspace = true, features = ["macros", "rt"] } diff --git a/crates/persisting-events/src/control.rs b/crates/persisting-events/src/control.rs deleted file mode 100644 index 68e210810..000000000 --- a/crates/persisting-events/src/control.rs +++ /dev/null @@ -1,1129 +0,0 @@ -//! Lightweight, versioned control-plane client for the standalone pChronicle process. -//! -//! This optional module contains no storage engine. pChronicle implements the -//! durable operations; local integrations depend only on these -//! contracts and the long-lived process transport. - -use crate::{EventRecord, unix_now_ms}; -use anyhow::{Context, Result, bail}; -use async_trait::async_trait; -use persisting_agentctl::{ - AttemptId, RunCommit, RunCommitRequest, RunControlRecord, RunId, RunLeaseRecord, -}; -use serde::{Deserialize, Serialize}; -use serde_json::Value; -use std::collections::BTreeMap; -use std::net::SocketAddr; -use std::path::{Path, PathBuf}; -use std::process::Stdio; -use std::sync::Arc; -use std::sync::Mutex as StdMutex; -use std::sync::atomic::{AtomicU64, Ordering}; -use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; -use tokio::net::TcpStream; -use tokio::process::{Child, Command}; -use tokio::sync::Mutex; - -pub const CHRONICLE_CONTROL_VERSION: u32 = 2; -pub const CHRONICLE_CONTROL_MAX_FRAME_BYTES: usize = 8 * 1024 * 1024; - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub enum LeaseAcquireOutcome { - Acquired(RunLeaseRecord), - Held(RunLeaseRecord), - AlreadyCommitted(RunCommit), -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub enum CommitRunOutcome { - Committed(RunCommit), - AlreadyCommitted(RunCommit), - StaleLease { - supplied_epoch: u64, - current_epoch: Option, - }, - Conflict(RunCommit), -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum AttemptRecordState { - Active, - Terminal, -} - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct AttemptRecord { - pub revision: u64, - pub run_id: String, - pub attempt_id: String, - pub lease_epoch: u64, - pub state: AttemptRecordState, - pub heartbeat_at_unix_ms: u64, - pub expires_at_unix_ms: u64, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub terminal_result: Option, -} - -impl AttemptRecord { - pub fn is_live_at(&self, now_unix_ms: u64) -> bool { - self.state == AttemptRecordState::Active && self.expires_at_unix_ms > now_unix_ms - } -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct TrajectoryAppendRequest { - pub storage: String, - pub agent_id: String, - pub session_id: String, - /// Physical warehouse representation. Older callers default to Lance. - #[serde(default, skip_serializing_if = "TrajectoryFormat::is_default")] - pub format: TrajectoryFormat, - #[serde(default)] - pub root_session_id: Option, - pub records: Vec, -} - -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] -pub enum TrajectoryFormat { - Json, - #[default] - Lance, -} - -impl TrajectoryFormat { - fn is_default(value: &Self) -> bool { - *value == Self::Lance - } -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct TrajectoryAppendResponse { - pub storage: String, - pub agent_id: String, - pub session_id: String, - pub accepted_records: usize, - pub dataset: String, - pub status: String, - pub note: String, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ChronicleControlEnvelope { - pub version: u32, - pub request_id: u64, - pub auth_token: String, - pub request: ChronicleControlRequest, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(tag = "type", content = "payload", rename_all = "snake_case")] -pub enum ChronicleControlRequest { - Ping, - AcquireLease { - run_id: RunId, - task_id: Option, - owner: String, - ttl_ms: u64, - }, - TakeoverLease { - run_id: RunId, - task_id: Option, - owner: String, - ttl_ms: u64, - }, - BindAttempt { - run_id: RunId, - epoch: u64, - attempt_id: AttemptId, - }, - RenewLease { - run_id: RunId, - epoch: u64, - owner: String, - ttl_ms: u64, - }, - CommitRun(RunCommitRequest), - GetRun { - run_id: RunId, - }, - ListRuns, - GetAttempt { - run_id: String, - }, - PublishAttemptActive { - run_id: String, - attempt_id: String, - lease_epoch: u64, - ttl_ms: u64, - }, - HeartbeatAttempt { - run_id: String, - attempt_id: String, - lease_epoch: u64, - ttl_ms: u64, - }, - PublishAttemptTerminal { - run_id: String, - attempt_id: String, - lease_epoch: u64, - result: Value, - }, - AppendTrajectory(TrajectoryAppendRequest), -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ChronicleControlResponseEnvelope { - pub version: u32, - pub request_id: u64, - pub response: ChronicleControlResponse, -} - -pub const CHRONICLE_SERVE_READY_VERSION: u32 = 2; - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct ChronicleServeControlReady { - pub endpoint: String, - pub auth_token: String, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct ChronicleServeReady { - pub version: u32, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub warehouse_endpoint: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub control: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub gateway_endpoint: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub gateway_admin_endpoint: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub gateway_dataset: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub gateway_split: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(tag = "type", content = "payload", rename_all = "snake_case")] -pub enum ChronicleControlResponse { - Pong, - LeaseAcquire(LeaseAcquireOutcome), - CommitRun(CommitRunOutcome), - Boolean(bool), - Run(Option), - Runs(Vec), - Attempt(Option), - TrajectoryAppend(TrajectoryAppendResponse), - Error { message: String }, -} - -#[async_trait] -#[allow( - clippy::double_must_use, - reason = "async_trait adds must_use to futures" -)] -pub trait ChronicleControl: Send + Sync { - fn root_uri(&self) -> &str; - - async fn acquire_lease( - &self, - run_id: &RunId, - task_id: Option<&str>, - owner: &str, - ttl_ms: u64, - ) -> Result; - async fn takeover_lease( - &self, - run_id: &RunId, - task_id: Option<&str>, - owner: &str, - ttl_ms: u64, - ) -> Result; - async fn bind_attempt(&self, run_id: &RunId, epoch: u64, attempt_id: AttemptId) - -> Result; - async fn renew_lease( - &self, - run_id: &RunId, - epoch: u64, - owner: &str, - ttl_ms: u64, - ) -> Result; - async fn commit_run(&self, request: RunCommitRequest) -> Result; - async fn get_run(&self, run_id: &RunId) -> Result>; - async fn list_runs(&self) -> Result>; - async fn get_attempt(&self, run_id: &str) -> Result>; - async fn publish_attempt_active( - &self, - run_id: &str, - attempt_id: &str, - lease_epoch: u64, - ttl_ms: u64, - ) -> Result; - async fn heartbeat_attempt( - &self, - run_id: &str, - attempt_id: &str, - lease_epoch: u64, - ttl_ms: u64, - ) -> Result; - async fn publish_attempt_terminal( - &self, - run_id: &str, - attempt_id: &str, - lease_epoch: u64, - result: Value, - ) -> Result; - async fn append_trajectory( - &self, - request: TrajectoryAppendRequest, - ) -> Result; -} - -struct ProcessState { - child: Mutex, - next_request_id: AtomicU64, -} - -impl Drop for ProcessState { - fn drop(&mut self) { - let _ = self.child.get_mut().start_kill(); - } -} - -#[derive(Clone)] -pub struct ChronicleServeProcessClient { - root_uri: String, - binary: PathBuf, - endpoint: SocketAddr, - auth_token: String, - state: Arc, -} - -impl std::fmt::Debug for ChronicleServeProcessClient { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter - .debug_struct("ChronicleServeProcessClient") - .field("root_uri", &self.root_uri) - .field("binary", &self.binary) - .field("endpoint", &self.endpoint) - .finish_non_exhaustive() - } -} - -impl ChronicleServeProcessClient { - pub async fn spawn(binary: impl AsRef, root_uri: impl Into) -> Result { - let requested_binary = binary.as_ref().to_path_buf(); - let binary = if requested_binary.components().count() == 1 { - std::env::current_exe() - .ok() - .and_then(|current| { - let parent = current.parent()?; - let sibling = parent.join(&requested_binary); - if sibling.is_file() { - return Some(sibling); - } - parent - .parent() - .map(|profile| profile.join(&requested_binary)) - .filter(|candidate| candidate.is_file()) - }) - .unwrap_or(requested_binary) - } else { - requested_binary - }; - let root_uri = root_uri.into(); - let mut child = Command::new(&binary) - .arg("serve") - .arg("--storage") - .arg(&root_uri) - .arg("--control") - .arg("127.0.0.1:0") - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::inherit()) - .kill_on_drop(true) - .spawn() - .with_context(|| format!("spawn pChronicle serve process {}", binary.display()))?; - let stdout = child - .stdout - .take() - .context("pChronicle serve stdout unavailable")?; - let mut stdout = BufReader::new(stdout); - let mut ready = String::new(); - let bytes = stdout - .read_line(&mut ready) - .await - .context("read pChronicle serve readiness")?; - anyhow::ensure!( - bytes > 0, - "pChronicle serve process exited before readiness" - ); - let ready: ChronicleServeReady = - serde_json::from_str(&ready).context("decode pChronicle readiness")?; - anyhow::ensure!( - ready.version == CHRONICLE_SERVE_READY_VERSION, - "unsupported pChronicle serve readiness version {}", - ready.version - ); - let control = ready - .control - .context("pChronicle serve readiness omitted Control")?; - let endpoint = control - .endpoint - .parse::() - .context("parse pChronicle control endpoint")?; - anyhow::ensure!( - endpoint.ip().is_loopback(), - "pChronicle returned a non-loopback endpoint" - ); - let client = Self { - root_uri, - binary, - endpoint, - auth_token: control.auth_token, - state: Arc::new(ProcessState { - child: Mutex::new(child), - next_request_id: AtomicU64::new(1), - }), - }; - match client.call(ChronicleControlRequest::Ping).await? { - ChronicleControlResponse::Pong => Ok(client), - other => bail!("unexpected pChronicle handshake response: {other:?}"), - } - } - - async fn call(&self, request: ChronicleControlRequest) -> Result { - let request_id = self.state.next_request_id.fetch_add(1, Ordering::Relaxed); - let envelope = ChronicleControlEnvelope { - version: CHRONICLE_CONTROL_VERSION, - request_id, - auth_token: self.auth_token.clone(), - request, - }; - let mut frame = serde_json::to_vec(&envelope).context("encode pChronicle request")?; - anyhow::ensure!( - frame.len() <= CHRONICLE_CONTROL_MAX_FRAME_BYTES, - "pChronicle request frame exceeds {} bytes", - CHRONICLE_CONTROL_MAX_FRAME_BYTES - ); - frame.push(b'\n'); - let stream = TcpStream::connect(self.endpoint) - .await - .with_context(|| format!("connect pChronicle control at {}", self.endpoint))?; - stream - .set_nodelay(true) - .context("configure pChronicle control socket")?; - let (read, mut write) = stream.into_split(); - write - .write_all(&frame) - .await - .context("write pChronicle request")?; - write.flush().await.context("flush pChronicle request")?; - - let mut line = String::new(); - let bytes = BufReader::new(read) - .read_line(&mut line) - .await - .context("read pChronicle response")?; - if bytes == 0 { - let status = self - .state - .child - .lock() - .await - .try_wait() - .context("inspect pChronicle process")?; - bail!("pChronicle control process closed stdout (status: {status:?})"); - } - anyhow::ensure!( - bytes <= CHRONICLE_CONTROL_MAX_FRAME_BYTES, - "pChronicle response frame exceeds {} bytes", - CHRONICLE_CONTROL_MAX_FRAME_BYTES - ); - let response: ChronicleControlResponseEnvelope = - serde_json::from_str(&line).context("decode pChronicle response")?; - anyhow::ensure!( - response.version == CHRONICLE_CONTROL_VERSION, - "unsupported pChronicle control version {}", - response.version - ); - anyhow::ensure!( - response.request_id == request_id, - "pChronicle response id mismatch: expected {request_id}, got {}", - response.request_id - ); - match response.response { - ChronicleControlResponse::Error { message } => bail!("pChronicle: {message}"), - response => Ok(response), - } - } -} - -macro_rules! expect_response { - ($value:expr, $variant:path, $name:literal) => { - match $value { - $variant(value) => Ok(value), - other => bail!("unexpected pChronicle response to {}: {other:?}", $name), - } - }; -} - -#[async_trait] -impl ChronicleControl for ChronicleServeProcessClient { - fn root_uri(&self) -> &str { - &self.root_uri - } - - async fn acquire_lease( - &self, - run_id: &RunId, - task_id: Option<&str>, - owner: &str, - ttl_ms: u64, - ) -> Result { - expect_response!( - self.call(ChronicleControlRequest::AcquireLease { - run_id: run_id.clone(), - task_id: task_id.map(str::to_owned), - owner: owner.to_owned(), - ttl_ms - }) - .await?, - ChronicleControlResponse::LeaseAcquire, - "acquire_lease" - ) - } - async fn takeover_lease( - &self, - run_id: &RunId, - task_id: Option<&str>, - owner: &str, - ttl_ms: u64, - ) -> Result { - expect_response!( - self.call(ChronicleControlRequest::TakeoverLease { - run_id: run_id.clone(), - task_id: task_id.map(str::to_owned), - owner: owner.to_owned(), - ttl_ms - }) - .await?, - ChronicleControlResponse::LeaseAcquire, - "takeover_lease" - ) - } - async fn bind_attempt( - &self, - run_id: &RunId, - epoch: u64, - attempt_id: AttemptId, - ) -> Result { - expect_response!( - self.call(ChronicleControlRequest::BindAttempt { - run_id: run_id.clone(), - epoch, - attempt_id - }) - .await?, - ChronicleControlResponse::Boolean, - "bind_attempt" - ) - } - async fn renew_lease( - &self, - run_id: &RunId, - epoch: u64, - owner: &str, - ttl_ms: u64, - ) -> Result { - expect_response!( - self.call(ChronicleControlRequest::RenewLease { - run_id: run_id.clone(), - epoch, - owner: owner.to_owned(), - ttl_ms - }) - .await?, - ChronicleControlResponse::Boolean, - "renew_lease" - ) - } - async fn commit_run(&self, request: RunCommitRequest) -> Result { - expect_response!( - self.call(ChronicleControlRequest::CommitRun(request)) - .await?, - ChronicleControlResponse::CommitRun, - "commit_run" - ) - } - async fn get_run(&self, run_id: &RunId) -> Result> { - expect_response!( - self.call(ChronicleControlRequest::GetRun { - run_id: run_id.clone() - }) - .await?, - ChronicleControlResponse::Run, - "get_run" - ) - } - async fn list_runs(&self) -> Result> { - expect_response!( - self.call(ChronicleControlRequest::ListRuns).await?, - ChronicleControlResponse::Runs, - "list_runs" - ) - } - async fn get_attempt(&self, run_id: &str) -> Result> { - expect_response!( - self.call(ChronicleControlRequest::GetAttempt { - run_id: run_id.to_owned() - }) - .await?, - ChronicleControlResponse::Attempt, - "get_attempt" - ) - } - async fn publish_attempt_active( - &self, - run_id: &str, - attempt_id: &str, - lease_epoch: u64, - ttl_ms: u64, - ) -> Result { - expect_response!( - self.call(ChronicleControlRequest::PublishAttemptActive { - run_id: run_id.to_owned(), - attempt_id: attempt_id.to_owned(), - lease_epoch, - ttl_ms - }) - .await?, - ChronicleControlResponse::Boolean, - "publish_attempt_active" - ) - } - - async fn heartbeat_attempt( - &self, - run_id: &str, - attempt_id: &str, - lease_epoch: u64, - ttl_ms: u64, - ) -> Result { - expect_response!( - self.call(ChronicleControlRequest::HeartbeatAttempt { - run_id: run_id.to_owned(), - attempt_id: attempt_id.to_owned(), - lease_epoch, - ttl_ms - }) - .await?, - ChronicleControlResponse::Boolean, - "heartbeat_attempt" - ) - } - - async fn publish_attempt_terminal( - &self, - run_id: &str, - attempt_id: &str, - lease_epoch: u64, - result: Value, - ) -> Result { - expect_response!( - self.call(ChronicleControlRequest::PublishAttemptTerminal { - run_id: run_id.to_owned(), - attempt_id: attempt_id.to_owned(), - lease_epoch, - result - }) - .await?, - ChronicleControlResponse::Boolean, - "publish_attempt_terminal" - ) - } - async fn append_trajectory( - &self, - request: TrajectoryAppendRequest, - ) -> Result { - expect_response!( - self.call(ChronicleControlRequest::AppendTrajectory(request)) - .await?, - ChronicleControlResponse::TrajectoryAppend, - "append_trajectory" - ) - } -} - -/// Deterministic embedded control implementation for orchestration tests and -/// callers that explicitly do not need persistence across process restarts. -pub struct MemoryChronicleControl { - root_uri: String, - runs: StdMutex>, - attempts: StdMutex>, -} - -impl std::fmt::Debug for MemoryChronicleControl { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter - .debug_struct("MemoryChronicleControl") - .field("root_uri", &self.root_uri) - .finish_non_exhaustive() - } -} - -impl MemoryChronicleControl { - pub fn new(root_uri: impl Into) -> Self { - Self { - root_uri: root_uri.into(), - runs: StdMutex::new(BTreeMap::new()), - attempts: StdMutex::new(BTreeMap::new()), - } - } - - fn acquire_inner( - &self, - run_id: &RunId, - task_id: Option<&str>, - owner: &str, - ttl_ms: u64, - force: bool, - ) -> Result { - anyhow::ensure!( - !owner.trim().is_empty(), - "Run lease owner must not be empty" - ); - let now = unix_now_ms(); - let mut runs = self - .runs - .lock() - .map_err(|_| anyhow::anyhow!("Run control lock poisoned"))?; - let current = runs.get(run_id).cloned(); - if let Some(commit) = current.as_ref().and_then(|record| record.commit.clone()) { - return Ok(LeaseAcquireOutcome::AlreadyCommitted(commit)); - } - if let Some(existing) = current.as_ref().and_then(|record| record.lease.as_ref()) { - if existing.owner != owner && existing.expires_at_unix_ms > now && !force { - return Ok(LeaseAcquireOutcome::Held(existing.clone())); - } - if existing.owner == owner && !force { - let mut lease = existing.clone(); - lease.expires_at_unix_ms = now.saturating_add(ttl_ms.max(1)); - let mut record = current.expect("lease requires record"); - record.revision = record.revision.saturating_add(1); - record.lease = Some(lease.clone()); - runs.insert(run_id.clone(), record); - return Ok(LeaseAcquireOutcome::Acquired(lease)); - } - } - let epoch = current - .as_ref() - .and_then(|record| record.lease.as_ref().map(|lease| lease.epoch)) - .unwrap_or(0) - .checked_add(1) - .context("Run lease epoch overflow")?; - let lease = RunLeaseRecord { - run_id: run_id.clone(), - task_id: task_id.map(str::to_owned), - epoch, - owner: owner.to_owned(), - issued_at_unix_ms: now, - expires_at_unix_ms: now.saturating_add(ttl_ms.max(1)), - attempt_id: None, - }; - let revision = current - .as_ref() - .map_or(1, |record| record.revision.saturating_add(1)); - runs.insert( - run_id.clone(), - RunControlRecord { - revision, - run_id: run_id.clone(), - lease: Some(lease.clone()), - commit: None, - }, - ); - Ok(LeaseAcquireOutcome::Acquired(lease)) - } -} - -#[async_trait] -impl ChronicleControl for MemoryChronicleControl { - fn root_uri(&self) -> &str { - &self.root_uri - } - - async fn acquire_lease( - &self, - run_id: &RunId, - task_id: Option<&str>, - owner: &str, - ttl_ms: u64, - ) -> Result { - self.acquire_inner(run_id, task_id, owner, ttl_ms, false) - } - - async fn takeover_lease( - &self, - run_id: &RunId, - task_id: Option<&str>, - owner: &str, - ttl_ms: u64, - ) -> Result { - self.acquire_inner(run_id, task_id, owner, ttl_ms, true) - } - - async fn bind_attempt( - &self, - run_id: &RunId, - epoch: u64, - attempt_id: AttemptId, - ) -> Result { - let now = unix_now_ms(); - let mut runs = self - .runs - .lock() - .map_err(|_| anyhow::anyhow!("Run control lock poisoned"))?; - let Some(record) = runs.get_mut(run_id) else { - return Ok(false); - }; - let Some(lease) = record.lease.as_mut() else { - return Ok(false); - }; - if record.commit.is_some() || lease.epoch != epoch || lease.expires_at_unix_ms <= now { - return Ok(false); - } - if lease - .attempt_id - .as_ref() - .is_some_and(|current| current != &attempt_id) - { - return Ok(false); - } - if lease.attempt_id.is_none() { - lease.attempt_id = Some(attempt_id); - record.revision = record.revision.saturating_add(1); - } - Ok(true) - } - - async fn renew_lease( - &self, - run_id: &RunId, - epoch: u64, - owner: &str, - ttl_ms: u64, - ) -> Result { - let now = unix_now_ms(); - let mut runs = self - .runs - .lock() - .map_err(|_| anyhow::anyhow!("Run control lock poisoned"))?; - let Some(record) = runs.get_mut(run_id) else { - return Ok(false); - }; - let Some(lease) = record.lease.as_mut() else { - return Ok(false); - }; - if record.commit.is_some() - || lease.epoch != epoch - || lease.owner != owner - || lease.expires_at_unix_ms <= now - { - return Ok(false); - } - lease.expires_at_unix_ms = now.saturating_add(ttl_ms.max(1)); - record.revision = record.revision.saturating_add(1); - Ok(true) - } - - async fn commit_run(&self, request: RunCommitRequest) -> Result { - anyhow::ensure!( - request.state.is_terminal(), - "RunCommit requires a terminal Run state" - ); - anyhow::ensure!( - request.lease_epoch > 0, - "RunCommit requires a non-zero lease epoch" - ); - anyhow::ensure!( - !request.result_digest.trim().is_empty(), - "RunCommit result_digest must not be empty" - ); - let now = unix_now_ms(); - let mut runs = self - .runs - .lock() - .map_err(|_| anyhow::anyhow!("Run control lock poisoned"))?; - let Some(record) = runs.get_mut(&request.run_id) else { - return Ok(CommitRunOutcome::StaleLease { - supplied_epoch: request.lease_epoch, - current_epoch: None, - }); - }; - if let Some(existing) = record.commit.clone() { - return Ok(if existing.request == request { - CommitRunOutcome::AlreadyCommitted(existing) - } else { - CommitRunOutcome::Conflict(existing) - }); - } - let current_epoch = record.lease.as_ref().map(|lease| lease.epoch); - let matches = record.lease.as_ref().is_some_and(|lease| { - lease.epoch == request.lease_epoch - && lease.expires_at_unix_ms > now - && lease - .attempt_id - .as_ref() - .is_none_or(|attempt| attempt == &request.attempt_id) - }); - if !matches { - return Ok(CommitRunOutcome::StaleLease { - supplied_epoch: request.lease_epoch, - current_epoch, - }); - } - let commit = RunCommit { - request: request.clone(), - committed_at_unix_ms: now, - }; - if let Some(lease) = record.lease.as_mut() { - lease.attempt_id.get_or_insert(request.attempt_id.clone()); - } - record.commit = Some(commit.clone()); - record.revision = record.revision.saturating_add(1); - Ok(CommitRunOutcome::Committed(commit)) - } - - async fn get_run(&self, run_id: &RunId) -> Result> { - Ok(self - .runs - .lock() - .map_err(|_| anyhow::anyhow!("Run control lock poisoned"))? - .get(run_id) - .cloned()) - } - - async fn list_runs(&self) -> Result> { - Ok(self - .runs - .lock() - .map_err(|_| anyhow::anyhow!("Run control lock poisoned"))? - .values() - .cloned() - .collect()) - } - - async fn get_attempt(&self, run_id: &str) -> Result> { - Ok(self - .attempts - .lock() - .map_err(|_| anyhow::anyhow!("Attempt registry lock poisoned"))? - .get(run_id) - .cloned()) - } - - async fn publish_attempt_active( - &self, - run_id: &str, - attempt_id: &str, - lease_epoch: u64, - ttl_ms: u64, - ) -> Result { - anyhow::ensure!( - !run_id.trim().is_empty() && !attempt_id.trim().is_empty() && lease_epoch > 0, - "invalid Attempt identity" - ); - let now = unix_now_ms(); - let mut attempts = self - .attempts - .lock() - .map_err(|_| anyhow::anyhow!("Attempt registry lock poisoned"))?; - let current = attempts.get(run_id); - if current.is_some_and(|record| { - record.lease_epoch > lease_epoch - || (record.lease_epoch == lease_epoch - && (record.attempt_id != attempt_id - || record.state == AttemptRecordState::Terminal)) - }) { - return Ok(false); - } - let revision = current.map_or(1, |record| record.revision.saturating_add(1)); - attempts.insert( - run_id.to_owned(), - AttemptRecord { - revision, - run_id: run_id.to_owned(), - attempt_id: attempt_id.to_owned(), - lease_epoch, - state: AttemptRecordState::Active, - heartbeat_at_unix_ms: now, - expires_at_unix_ms: now.saturating_add(ttl_ms.max(1)), - terminal_result: None, - }, - ); - Ok(true) - } - - async fn publish_attempt_terminal( - &self, - run_id: &str, - attempt_id: &str, - lease_epoch: u64, - result: Value, - ) -> Result { - let now = unix_now_ms(); - let mut attempts = self - .attempts - .lock() - .map_err(|_| anyhow::anyhow!("Attempt registry lock poisoned"))?; - let Some(record) = attempts.get_mut(run_id) else { - return Ok(false); - }; - if record.attempt_id != attempt_id || record.lease_epoch != lease_epoch { - return Ok(false); - } - if record.state == AttemptRecordState::Terminal { - return Ok(record.terminal_result.as_ref() == Some(&result)); - } - record.revision = record.revision.saturating_add(1); - record.state = AttemptRecordState::Terminal; - record.heartbeat_at_unix_ms = now; - record.expires_at_unix_ms = now; - record.terminal_result = Some(result); - Ok(true) - } - - async fn heartbeat_attempt( - &self, - run_id: &str, - attempt_id: &str, - lease_epoch: u64, - ttl_ms: u64, - ) -> Result { - let now = unix_now_ms(); - let mut attempts = self - .attempts - .lock() - .map_err(|_| anyhow::anyhow!("Attempt registry lock poisoned"))?; - let Some(record) = attempts.get_mut(run_id) else { - return Ok(false); - }; - if record.attempt_id != attempt_id - || record.lease_epoch != lease_epoch - || record.state != AttemptRecordState::Active - { - return Ok(false); - } - record.revision = record.revision.saturating_add(1); - record.heartbeat_at_unix_ms = now; - record.expires_at_unix_ms = now.saturating_add(ttl_ms.max(1)); - Ok(true) - } - - async fn append_trajectory( - &self, - request: TrajectoryAppendRequest, - ) -> Result { - let accepted_records = request.records.len(); - Ok(TrajectoryAppendResponse { - dataset: format!( - "{}/{}/{}/events.lance", - request.storage.trim_end_matches('/'), - request.agent_id, - request.session_id - ), - storage: request.storage, - agent_id: request.agent_id, - session_id: request.session_id, - accepted_records, - status: "ok".into(), - note: "in-memory control transport; trajectory was not persisted".into(), - }) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn serve_ready_omits_disabled_services_and_rejects_unknown_members() { - let ready = ChronicleServeReady { - version: CHRONICLE_SERVE_READY_VERSION, - warehouse_endpoint: None, - control: Some(ChronicleServeControlReady { - endpoint: "127.0.0.1:4000".into(), - auth_token: "secret".into(), - }), - gateway_endpoint: None, - gateway_admin_endpoint: None, - gateway_dataset: None, - gateway_split: None, - }; - - let value = serde_json::to_value(ready).unwrap(); - assert_eq!(value["version"], 2); - assert_eq!(value["control"]["endpoint"], "127.0.0.1:4000"); - assert_eq!(value["control"]["auth_token"], "secret"); - assert!(value.get("warehouse_endpoint").is_none()); - assert!(value.get("gateway_endpoint").is_none()); - assert!(value.get("gateway_admin_endpoint").is_none()); - assert!(value.get("gateway_dataset").is_none()); - assert!(value.get("gateway_split").is_none()); - - let malformed = serde_json::json!({ - "version": 2, - "control": { - "endpoint": "127.0.0.1:4000", - "auth_token": "secret" - }, - "unexpected": true - }); - assert!(serde_json::from_value::(malformed).is_err()); - } - - #[tokio::test] - async fn attempt_heartbeat_renews_only_the_active_fenced_attempt() { - let control = MemoryChronicleControl::new("memory"); - assert!( - control - .publish_attempt_active("run-1", "attempt-1", 3, 1_000) - .await - .unwrap() - ); - let initial = control.get_attempt("run-1").await.unwrap().unwrap(); - assert!( - !control - .heartbeat_attempt("run-1", "attempt-old", 2, 2_000) - .await - .unwrap() - ); - assert!( - control - .heartbeat_attempt("run-1", "attempt-1", 3, 2_000) - .await - .unwrap() - ); - let renewed = control.get_attempt("run-1").await.unwrap().unwrap(); - assert_eq!(renewed.revision, initial.revision + 1); - assert!(renewed.expires_at_unix_ms >= initial.expires_at_unix_ms); - - assert!( - control - .publish_attempt_terminal("run-1", "attempt-1", 3, serde_json::json!({"ok": true})) - .await - .unwrap() - ); - assert!( - !control - .heartbeat_attempt("run-1", "attempt-1", 3, 2_000) - .await - .unwrap() - ); - } -} diff --git a/crates/persisting-events/src/lib.rs b/crates/persisting-events/src/lib.rs deleted file mode 100644 index 4fb40eb5c..000000000 --- a/crates/persisting-events/src/lib.rs +++ /dev/null @@ -1,143 +0,0 @@ -//! Storage-independent runtime event contracts shared by Persisting components. -//! -//! Producers such as Gateway emit these records. Consumers such as -//! pChronicle decide how to persist, query, and project them. -//! The optional `control` feature also carries the lightweight, versioned -//! sidecar protocol so callers do not need another protocol-only package. - -use serde::{Deserialize, Serialize}; -use serde_json::Value; - -mod time; -pub use time::unix_now_ms; - -#[cfg(feature = "control")] -mod control; -#[cfg(feature = "control")] -pub use control::*; - -/// Runtime identity shared by lifecycle and trajectory events. -/// -/// The fields are flattened into [`EventRecord`] on the wire. A storage -/// adapter may fill missing routing identities, but must reject conflicts. -#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] -pub struct EventIdentity { - #[serde(default, skip_serializing_if = "Option::is_none")] - pub event_id: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub run_id: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub attempt_id: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub storyline_id: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub turn_id: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub timestamp_unix_ms: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub producer: Option, -} - -/// Canonical storage-independent Agent runtime event. -/// -/// Ordering within one Attempt is defined by `seq`; wall-clock timestamps are -/// evidence for correlation and display, not the source of ordering truth. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct EventRecord { - #[serde(flatten)] - pub identity: EventIdentity, - pub seq: u64, - pub source: String, - pub kind: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub timestamp: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub session_id: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub agent_id: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub parent_uuid: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub trace_id: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub call_id: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub subagent_id: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub parent_agent_id: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub branch: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub parent_call_id: Option, - pub payload: Value, -} - -impl EventRecord { - /// Validate the stable event envelope independently of any storage engine. - pub fn validate(&self) -> Result<(), EventValidationError> { - if self.source.trim().is_empty() { - return Err(EventValidationError::MissingSource); - } - if self.kind.trim().is_empty() { - return Err(EventValidationError::MissingKind); - } - Ok(()) - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] -pub enum EventValidationError { - #[error("event source is required")] - MissingSource, - #[error("event kind is required")] - MissingKind, -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn identity_is_flattened_and_wire_compatible() { - let record = EventRecord { - identity: EventIdentity { - event_id: Some("event-1".into()), - run_id: Some("run-1".into()), - ..EventIdentity::default() - }, - seq: 0, - source: "runtime".into(), - kind: "run.created".into(), - timestamp: None, - session_id: None, - agent_id: None, - parent_uuid: None, - trace_id: None, - call_id: None, - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload: Value::Object(Default::default()), - }; - let encoded = serde_json::to_value(record).unwrap(); - assert_eq!(encoded["event_id"], "event-1"); - assert_eq!(encoded["run_id"], "run-1"); - assert!(encoded.get("identity").is_none()); - assert!(encoded.get("schema_version").is_none()); - } - - #[test] - fn validates_required_routing_fields() { - let mut record: EventRecord = serde_json::from_value(serde_json::json!({ - "seq": 0, - "source": "runtime", - "kind": "run.created", - "payload": {} - })) - .unwrap(); - record.validate().unwrap(); - record.kind.clear(); - assert_eq!(record.validate(), Err(EventValidationError::MissingKind)); - } -} diff --git a/crates/persisting-events/src/time.rs b/crates/persisting-events/src/time.rs deleted file mode 100644 index d57f1cb80..000000000 --- a/crates/persisting-events/src/time.rs +++ /dev/null @@ -1,15 +0,0 @@ -//! Shared wall-clock helpers for runtime event timestamps. - -use std::time::{SystemTime, UNIX_EPOCH}; - -/// Return the current Unix wall-clock time in milliseconds. -/// -/// Event timestamps are observational metadata rather than ordering truth. -/// A clock before the Unix epoch is represented as zero, and conversion -/// saturates at `u64::MAX` instead of truncating. -pub fn unix_now_ms() -> u64 { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .map(|duration| duration.as_millis().min(u64::MAX as u128) as u64) - .unwrap_or(0) -} diff --git a/crates/persisting-events/tests/event_properties.rs b/crates/persisting-events/tests/event_properties.rs deleted file mode 100644 index 4325e3d62..000000000 --- a/crates/persisting-events/tests/event_properties.rs +++ /dev/null @@ -1,160 +0,0 @@ -//! Property tests for the storage-independent event contract. -//! -//! Keep generators here focused on wire-level invariants. Product-specific -//! behavior belongs in the producer crate; this suite verifies that any -//! producer can serialize, validate, and round-trip the shared envelope. - -use persisting_events::{EventIdentity, EventRecord, EventValidationError}; -#[cfg(feature = "control")] -use persisting_events::{TrajectoryAppendRequest, TrajectoryFormat}; -use proptest::prelude::*; -use serde_json::Value; - -fn non_empty_string() -> impl Strategy { - "[a-zA-Z][a-zA-Z0-9_.-]{0,24}" -} - -fn optional_string() -> impl Strategy> { - prop::option::of("[a-zA-Z0-9_.-]{0,24}") -} - -fn payload() -> impl Strategy { - prop::collection::btree_map( - "[a-z][a-z0-9_]{0,8}", - prop_oneof![ - any::().prop_map(Value::Bool), - any::().prop_map(Value::from), - "[a-zA-Z0-9 _.-]{0,32}".prop_map(Value::String), - ], - 0..8, - ) - .prop_map(|fields| Value::Object(fields.into_iter().collect())) -} - -prop_compose! { - fn event_record_strategy()( - event_id in optional_string(), - run_id in optional_string(), - attempt_id in optional_string(), - storyline_id in optional_string(), - turn_id in optional_string(), - timestamp_unix_ms in prop::option::of(any::()), - producer in optional_string(), - seq in any::(), - source in non_empty_string(), - kind in non_empty_string(), - timestamp in optional_string(), - session_id in optional_string(), - agent_id in optional_string(), - parent_uuid in optional_string(), - trace_id in optional_string(), - call_id in optional_string(), - subagent_id in optional_string(), - parent_agent_id in optional_string(), - branch in optional_string(), - parent_call_id in optional_string(), - payload in payload(), - ) -> EventRecord { - EventRecord { - identity: EventIdentity { - event_id, - run_id, - attempt_id, - storyline_id, - turn_id, - timestamp_unix_ms, - producer, - }, - seq, - source, - kind, - timestamp, - session_id, - agent_id, - parent_uuid, - trace_id, - call_id, - subagent_id, - parent_agent_id, - branch, - parent_call_id, - payload, - } - } -} - -proptest! { - #![proptest_config(ProptestConfig { cases: 128, .. ProptestConfig::default() })] - - #[test] - fn event_record_json_roundtrip_preserves_the_flattened_contract( - record in event_record_strategy() - ) { - let encoded = serde_json::to_value(&record).expect("serialize EventRecord"); - prop_assert!(encoded.get("identity").is_none()); - - let decoded: EventRecord = - serde_json::from_value(encoded).expect("deserialize EventRecord"); - prop_assert_eq!(decoded, record); - } - - #[test] - fn generated_records_satisfy_required_routing_fields(record in event_record_strategy()) { - prop_assert_eq!(record.validate(), Ok(())); - } - - #[test] - fn validation_rejects_each_missing_routing_field(record in event_record_strategy()) { - let mut missing_source = record.clone(); - missing_source.source.clear(); - prop_assert_eq!( - missing_source.validate(), - Err(EventValidationError::MissingSource) - ); - - let mut missing_kind = record; - missing_kind.kind.clear(); - prop_assert_eq!(missing_kind.validate(), Err(EventValidationError::MissingKind)); - } -} - -#[cfg(feature = "control")] -prop_compose! { - fn append_request_strategy()( - storage in non_empty_string(), - agent_id in non_empty_string(), - session_id in non_empty_string(), - format in prop_oneof![Just(TrajectoryFormat::Json), Just(TrajectoryFormat::Lance)], - root_session_id in optional_string(), - records in prop::collection::vec(event_record_strategy(), 0..4), - ) -> TrajectoryAppendRequest { - TrajectoryAppendRequest { - storage, - agent_id, - session_id, - format, - root_session_id, - records, - } - } -} - -#[cfg(feature = "control")] -proptest! { - #![proptest_config(ProptestConfig { cases: 128, .. ProptestConfig::default() })] - - #[test] - fn trajectory_append_json_roundtrip_preserves_format_and_records( - request in append_request_strategy() - ) { - let encoded = serde_json::to_value(&request).expect("serialize append request"); - let decoded: TrajectoryAppendRequest = - serde_json::from_value(encoded).expect("deserialize append request"); - prop_assert_eq!(decoded.format, request.format); - prop_assert_eq!(decoded.records, request.records); - prop_assert_eq!(decoded.storage, request.storage); - prop_assert_eq!(decoded.agent_id, request.agent_id); - prop_assert_eq!(decoded.session_id, request.session_id); - prop_assert_eq!(decoded.root_session_id, request.root_session_id); - } -} diff --git a/crates/persisting-gateway/Cargo.toml b/crates/persisting-gateway/Cargo.toml deleted file mode 100644 index d6478e421..000000000 --- a/crates/persisting-gateway/Cargo.toml +++ /dev/null @@ -1,73 +0,0 @@ -[package] -name = "persisting-gateway" -version.workspace = true -edition.workspace = true -authors.workspace = true -license.workspace = true -description = "Agent protocol gateway for forwarding and trajectory capture" - -[features] -default = ["echo-server"] -echo-server = ["dep:base64"] - -[dependencies] -anyhow.workspace = true -async-trait.workspace = true -axum = { workspace = true, features = ["http1", "json", "tokio"] } -base64 = { workspace = true, optional = true } -bytes.workspace = true -chrono = { workspace = true, features = ["serde"] } -http-body-util.workspace = true -futures-util = { workspace = true, features = ["std"] } -reqwest = { workspace = true, features = ["json", "rustls-tls", "stream"] } -serde = { workspace = true, features = ["derive"] } -serde_json.workspace = true -toml.workspace = true -serde_yaml.workspace = true -dirs.workspace = true -tokio = { workspace = true, features = ["rt-multi-thread", "sync", "net"] } -tokio-stream.workspace = true -tracing.workspace = true -url.workspace = true -regex.workspace = true -pulsing-actor = { workspace = true } -dashmap.workspace = true -blake3.workspace = true -hostname.workspace = true -persisting-agentctl.workspace = true -persisting-events.workspace = true -persisting-overlaynet.workspace = true -persisting-pchronicle = { workspace = true, default-features = false } - -[dev-dependencies] -proptest.workspace = true -tempfile.workspace = true -tokio = { workspace = true, features = ["macros"] } - -[[test]] -name = "llm_fixtures" -path = "tests/llm_fixtures.rs" - -[[test]] -name = "ag_fixture_tests" -path = "tests/ag_fixture_tests.rs" - -[[test]] -name = "capture_apps_claude" -path = "tests/capture_apps_claude.rs" - -[[test]] -name = "agenticmd_golden" -path = "tests/agenticmd_golden.rs" - -[[test]] -name = "agenticmd_bridge" -path = "tests/agenticmd_bridge.rs" - -[[test]] -name = "network_policy_http" -path = "tests/network_policy_http.rs" - -[[test]] -name = "model_api_forwarding" -path = "tests/model_api_forwarding.rs" diff --git a/crates/persisting-gateway/README.md b/crates/persisting-gateway/README.md deleted file mode 100644 index 791846837..000000000 --- a/crates/persisting-gateway/README.md +++ /dev/null @@ -1,42 +0,0 @@ -# persisting-gateway - -**pChronicle Agent protocol gateway: LLM HTTP forwarding and canonical -trajectory capture.** - -Owns the application-level path from Agent/LLM HTTP exchanges to trajectory -events: protocol recognition and adaptation, upstream selection, run/session/ -story/call correlation, canonical pChronicle event emission, WAL coordination, -and live human-readable projections. - -Does not own the proxy data plane or the canonical trajectory storage format. -[`persisting-overlaynet`](../persisting-overlaynet/README.md) owns proxy -transport, access enforcement, and generic sink dispatch. -[`persisting-pchronicle`](../persisting-pchronicle/README.md) owns schemas, -persistence, reading, replay, conversion, and derived views. - -Capture runs through `pchronicle serve --gateway-config`. Gateway is an internal -library; external execution components can also integrate it. - -This crate implements `persisting-overlaynet::OverlaySink`. Protocol rendering -and capture share one in-memory `LlmRequestEventPayload` (`llm/v1`). Provider -wire formats are never chained through Chat Completions as an intermediate -protocol. Storyline is a derived trajectory view and is not part of the online -protocol-conversion path. - -## Develop - -```bash -just test persisting-gateway -# or: just test-crate capture -just test-capture-fixtures -just echo -``` - -`just echo` starts the loopback-only `pchronicle echo` upstream used by Gateway -benchmarks and regressions. It does not start Gateway itself. - -## Links - -- [Gateway forwarding, rewriting, and capture](../../docs/src/en/pchronicle/guides/serve-gateway.md) -- [`persisting-overlaynet`](../persisting-overlaynet/README.md) -- [`persisting-pchronicle`](../persisting-pchronicle/README.md) diff --git a/crates/persisting-gateway/fuzz/Cargo.lock b/crates/persisting-gateway/fuzz/Cargo.lock deleted file mode 100644 index c13033c4f..000000000 --- a/crates/persisting-gateway/fuzz/Cargo.lock +++ /dev/null @@ -1,2688 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 4 - -[[package]] -name = "aho-corasick" -version = "1.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" -dependencies = [ - "memchr", -] - -[[package]] -name = "allocator-api2" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" - -[[package]] -name = "android_system_properties" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" -dependencies = [ - "libc", -] - -[[package]] -name = "anyhow" -version = "1.0.104" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" - -[[package]] -name = "arbitrary" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" - -[[package]] -name = "arrayref" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" - -[[package]] -name = "arrayvec" -version = "0.7.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" - -[[package]] -name = "async-trait" -version = "0.1.92" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.3", -] - -[[package]] -name = "atomic-waker" -version = "1.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" - -[[package]] -name = "autocfg" -version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" - -[[package]] -name = "axum" -version = "0.8.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" -dependencies = [ - "axum-core", - "bytes", - "futures-util", - "http", - "http-body", - "http-body-util", - "hyper", - "hyper-util", - "itoa", - "matchit", - "memchr", - "mime", - "percent-encoding", - "pin-project-lite", - "serde_core", - "serde_json", - "serde_path_to_error", - "sync_wrapper", - "tokio", - "tower", - "tower-layer", - "tower-service", -] - -[[package]] -name = "axum-core" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" -dependencies = [ - "bytes", - "futures-core", - "http", - "http-body", - "http-body-util", - "mime", - "pin-project-lite", - "sync_wrapper", - "tower-layer", - "tower-service", -] - -[[package]] -name = "base64" -version = "0.21.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" - -[[package]] -name = "base64" -version = "0.22.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" - -[[package]] -name = "better_io" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef0a3155e943e341e557863e69a708999c94ede624e37865c8e2a91b94efa78f" - -[[package]] -name = "bincode" -version = "1.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" -dependencies = [ - "serde", -] - -[[package]] -name = "bitflags" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" - -[[package]] -name = "bitflags" -version = "2.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" -dependencies = [ - "serde_core", -] - -[[package]] -name = "blake3" -version = "1.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76ae7bad254120e9e4c63bafc385310756f90c484eac0e36b8317cf09cb92a77" -dependencies = [ - "arrayref", - "arrayvec", - "cc", - "cfg-if", - "constant_time_eq", - "cpufeatures", -] - -[[package]] -name = "bumpalo" -version = "3.20.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" - -[[package]] -name = "byteorder" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" - -[[package]] -name = "bytes" -version = "1.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" - -[[package]] -name = "cc" -version = "1.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "509591b7bcd67f4ef775afad7662703b4935daaa6ec0e5605cfb1090b32a2b6d" -dependencies = [ - "find-msvc-tools", - "jobserver", - "libc", - "shlex", -] - -[[package]] -name = "cfg-if" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" - -[[package]] -name = "cfg_aliases" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" - -[[package]] -name = "chacha20" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" -dependencies = [ - "cfg-if", - "cpufeatures", - "rand_core 0.10.1", -] - -[[package]] -name = "chrono" -version = "0.4.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" -dependencies = [ - "iana-time-zone", - "js-sys", - "num-traits", - "serde", - "wasm-bindgen", - "windows-link", -] - -[[package]] -name = "constant_time_eq" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" - -[[package]] -name = "core-foundation-sys" -version = "0.8.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" - -[[package]] -name = "cpufeatures" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" -dependencies = [ - "libc", -] - -[[package]] -name = "crossbeam-utils" -version = "0.8.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" - -[[package]] -name = "crunchy" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" - -[[package]] -name = "dashmap" -version = "6.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c" -dependencies = [ - "cfg-if", - "crossbeam-utils", - "hashbrown 0.14.5", - "lock_api", - "once_cell", - "parking_lot_core", -] - -[[package]] -name = "defmt" -version = "0.3.100" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0963443817029b2024136fc4dd07a5107eb8f977eaf18fcd1fdeb11306b64ad" -dependencies = [ - "defmt 1.1.1", -] - -[[package]] -name = "defmt" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1" -dependencies = [ - "bitflags 1.3.2", - "defmt-macros", -] - -[[package]] -name = "defmt-macros" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8" -dependencies = [ - "defmt-parser", - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "defmt-parser" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" -dependencies = [ - "thiserror 2.0.20", -] - -[[package]] -name = "dirs" -version = "6.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3e8aa94d75141228480295a7d0e7feb620b1a5ad9f12bc40be62411e38cce4e" -dependencies = [ - "dirs-sys", -] - -[[package]] -name = "dirs-sys" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" -dependencies = [ - "libc", - "option-ext", - "redox_users", - "windows-sys 0.61.2", -] - -[[package]] -name = "displaydoc" -version = "0.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.3", -] - -[[package]] -name = "dtype_dispatch" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3a5ccdfd6c5e7e2fea9c5cf256f2a08216047fab19c621c3da64e9ae4a1462d" - -[[package]] -name = "equivalent" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" - -[[package]] -name = "errno" -version = "0.3.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" -dependencies = [ - "libc", - "windows-sys 0.61.2", -] - -[[package]] -name = "find-msvc-tools" -version = "0.1.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" - -[[package]] -name = "fnv" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" - -[[package]] -name = "foldhash" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" - -[[package]] -name = "form_urlencoded" -version = "1.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" -dependencies = [ - "percent-encoding", -] - -[[package]] -name = "fs2" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" -dependencies = [ - "libc", - "winapi", -] - -[[package]] -name = "futures" -version = "0.3.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" -dependencies = [ - "futures-channel", - "futures-core", - "futures-executor", - "futures-io", - "futures-sink", - "futures-task", - "futures-util", -] - -[[package]] -name = "futures-channel" -version = "0.3.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" -dependencies = [ - "futures-core", - "futures-sink", -] - -[[package]] -name = "futures-core" -version = "0.3.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" - -[[package]] -name = "futures-executor" -version = "0.3.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" -dependencies = [ - "futures-core", - "futures-task", - "futures-util", -] - -[[package]] -name = "futures-io" -version = "0.3.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" - -[[package]] -name = "futures-macro" -version = "0.3.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.3", -] - -[[package]] -name = "futures-sink" -version = "0.3.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" - -[[package]] -name = "futures-task" -version = "0.3.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" - -[[package]] -name = "futures-util" -version = "0.3.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" -dependencies = [ - "futures-channel", - "futures-core", - "futures-io", - "futures-macro", - "futures-sink", - "futures-task", - "memchr", - "pin-project-lite", - "slab", -] - -[[package]] -name = "getrandom" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" -dependencies = [ - "cfg-if", - "js-sys", - "libc", - "wasi", - "wasm-bindgen", -] - -[[package]] -name = "getrandom" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" -dependencies = [ - "cfg-if", - "libc", - "r-efi 5.3.0", - "wasip2", -] - -[[package]] -name = "getrandom" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" -dependencies = [ - "cfg-if", - "js-sys", - "libc", - "r-efi 6.0.0", - "rand_core 0.10.1", - "wasm-bindgen", -] - -[[package]] -name = "glob" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" - -[[package]] -name = "h2" -version = "0.4.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" -dependencies = [ - "atomic-waker", - "bytes", - "fnv", - "futures-core", - "futures-sink", - "http", - "indexmap", - "slab", - "tokio", - "tokio-util", - "tracing", -] - -[[package]] -name = "half" -version = "2.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" -dependencies = [ - "cfg-if", - "crunchy", - "zerocopy", -] - -[[package]] -name = "hash32" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47d60b12902ba28e2730cd37e95b8c9223af2808df9e902d4df49588d1470606" -dependencies = [ - "byteorder", -] - -[[package]] -name = "hashbrown" -version = "0.14.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" - -[[package]] -name = "hashbrown" -version = "0.15.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" -dependencies = [ - "allocator-api2", - "equivalent", - "foldhash", -] - -[[package]] -name = "hashbrown" -version = "0.17.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" - -[[package]] -name = "heapless" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25ba4bd83f9415b58b4ed8dc5714c76e626a105be4646c02630ad730ad3b5aa4" -dependencies = [ - "hash32", - "stable_deref_trait", -] - -[[package]] -name = "hostname" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "617aaa3557aef3810a6369d0a99fac8a080891b68bd9f9812a1eeda0c0730cbd" -dependencies = [ - "cfg-if", - "libc", - "windows-link", -] - -[[package]] -name = "http" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" -dependencies = [ - "bytes", - "itoa", -] - -[[package]] -name = "http-body" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" -dependencies = [ - "bytes", - "http", -] - -[[package]] -name = "http-body-util" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" -dependencies = [ - "bytes", - "futures-core", - "http", - "http-body", - "pin-project-lite", -] - -[[package]] -name = "httparse" -version = "1.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" - -[[package]] -name = "httpdate" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" - -[[package]] -name = "hyper" -version = "1.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d22053281f852e11534f5198498373cbb59295120a20771d90f7ed1897490a72" -dependencies = [ - "atomic-waker", - "bytes", - "futures-channel", - "futures-core", - "h2", - "http", - "http-body", - "httparse", - "httpdate", - "itoa", - "pin-project-lite", - "smallvec", - "tokio", - "want", -] - -[[package]] -name = "hyper-rustls" -version = "0.27.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" -dependencies = [ - "http", - "hyper", - "hyper-util", - "rustls", - "tokio", - "tokio-rustls", - "tower-service", - "webpki-roots", -] - -[[package]] -name = "hyper-util" -version = "0.1.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" -dependencies = [ - "base64 0.22.1", - "bytes", - "futures-channel", - "futures-util", - "http", - "http-body", - "hyper", - "ipnet", - "libc", - "percent-encoding", - "pin-project-lite", - "socket2", - "tokio", - "tower-service", - "tracing", -] - -[[package]] -name = "iana-time-zone" -version = "0.1.65" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" -dependencies = [ - "android_system_properties", - "core-foundation-sys", - "iana-time-zone-haiku", - "js-sys", - "log", - "wasm-bindgen", - "windows-core", -] - -[[package]] -name = "iana-time-zone-haiku" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" -dependencies = [ - "cc", -] - -[[package]] -name = "icu_collections" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" -dependencies = [ - "displaydoc", - "potential_utf", - "utf8_iter", - "yoke", - "zerofrom", - "zerovec", -] - -[[package]] -name = "icu_locale_core" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" -dependencies = [ - "displaydoc", - "litemap", - "tinystr", - "writeable", - "zerovec", -] - -[[package]] -name = "icu_normalizer" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" -dependencies = [ - "icu_collections", - "icu_normalizer_data", - "icu_properties", - "icu_provider", - "smallvec", - "zerovec", -] - -[[package]] -name = "icu_normalizer_data" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" - -[[package]] -name = "icu_properties" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" -dependencies = [ - "displaydoc", - "icu_collections", - "icu_locale_core", - "icu_properties_data", - "icu_provider", - "zerotrie", - "zerovec", -] - -[[package]] -name = "icu_properties_data" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" - -[[package]] -name = "icu_provider" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92a7ed671a6aad807a8651a2e1782a6598fda9ce5185dd8158549e95a91c6428" -dependencies = [ - "displaydoc", - "icu_locale_core", - "writeable", - "yoke", - "zerofrom", - "zerotrie", - "zerovec", -] - -[[package]] -name = "idna" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" -dependencies = [ - "idna_adapter", - "smallvec", - "utf8_iter", -] - -[[package]] -name = "idna_adapter" -version = "1.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" -dependencies = [ - "icu_normalizer", - "icu_properties", -] - -[[package]] -name = "indexmap" -version = "2.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" -dependencies = [ - "equivalent", - "hashbrown 0.17.1", -] - -[[package]] -name = "ipnet" -version = "2.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a756c3fac73139e83f14c2d742155dd2b78d3ee56597b419a0579b7bdd6dd78" - -[[package]] -name = "itoa" -version = "1.0.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" - -[[package]] -name = "jobserver" -version = "0.1.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" -dependencies = [ - "getrandom 0.4.3", - "libc", -] - -[[package]] -name = "js-sys" -version = "0.3.104" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a" -dependencies = [ - "cfg-if", - "futures-util", - "wasm-bindgen", -] - -[[package]] -name = "lazy_static" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" - -[[package]] -name = "libc" -version = "0.2.189" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" - -[[package]] -name = "libfuzzer-sys" -version = "0.4.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9fd2f41a1cba099f79a0b6b6c35656cf7c03351a7bae8ff0f28f25270f929d2" -dependencies = [ - "arbitrary", - "cc", -] - -[[package]] -name = "libredox" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2026a5056764a10b2bf5d56488cba40da507f5493a6a429340e2004d9ed085fa" -dependencies = [ - "libc", -] - -[[package]] -name = "litemap" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" - -[[package]] -name = "lock_api" -version = "0.4.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" -dependencies = [ - "scopeguard", -] - -[[package]] -name = "log" -version = "0.4.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" - -[[package]] -name = "lru" -version = "0.12.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" -dependencies = [ - "hashbrown 0.15.5", -] - -[[package]] -name = "lru-slab" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" - -[[package]] -name = "managed" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ca88d725a0a943b096803bd34e73a4437208b6077654cc4ecb2947a5f91618d" - -[[package]] -name = "matchers" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" -dependencies = [ - "regex-automata", -] - -[[package]] -name = "matchit" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" - -[[package]] -name = "memchr" -version = "2.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" - -[[package]] -name = "memmap2" -version = "0.9.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0" -dependencies = [ - "libc", -] - -[[package]] -name = "mime" -version = "0.3.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" - -[[package]] -name = "mio" -version = "1.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" -dependencies = [ - "libc", - "wasi", - "windows-sys 0.61.2", -] - -[[package]] -name = "nu-ansi-term" -version = "0.50.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" -dependencies = [ - "windows-sys 0.61.2", -] - -[[package]] -name = "num-traits" -version = "0.2.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" -dependencies = [ - "autocfg", -] - -[[package]] -name = "once_cell" -version = "1.21.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" - -[[package]] -name = "opentelemetry" -version = "0.27.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab70038c28ed37b97d8ed414b6429d343a8bbf44c9f79ec854f3a643029ba6d7" -dependencies = [ - "futures-core", - "futures-sink", - "js-sys", - "pin-project-lite", - "thiserror 1.0.69", - "tracing", -] - -[[package]] -name = "opentelemetry_sdk" -version = "0.27.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "231e9d6ceef9b0b2546ddf52335785ce41252bc7474ee8ba05bfad277be13ab8" -dependencies = [ - "async-trait", - "futures-channel", - "futures-executor", - "futures-util", - "glob", - "opentelemetry", - "percent-encoding", - "rand 0.8.7", - "serde_json", - "thiserror 1.0.69", - "tokio", - "tokio-stream", - "tracing", -] - -[[package]] -name = "option-ext" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" - -[[package]] -name = "parking_lot_core" -version = "0.9.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" -dependencies = [ - "cfg-if", - "libc", - "redox_syscall", - "smallvec", - "windows-link", -] - -[[package]] -name = "pco" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42382de9fb564e2d10cb4d5ca97cc06d928f0f9667bbef456b57e60827b6548b" -dependencies = [ - "better_io", - "dtype_dispatch", - "half", - "rand_xoshiro", -] - -[[package]] -name = "percent-encoding" -version = "2.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" - -[[package]] -name = "persisting-agentctl" -version = "0.2.0" -dependencies = [ - "anyhow", - "ipnet", - "serde", - "serde_json", -] - -[[package]] -name = "persisting-gateway" -version = "0.2.0" -dependencies = [ - "anyhow", - "async-trait", - "axum", - "base64 0.22.1", - "blake3", - "bytes", - "chrono", - "dashmap", - "dirs", - "futures-util", - "hostname", - "http-body-util", - "persisting-agentctl", - "persisting-overlaynet", - "persisting-pchronicle", - "pulsing-actor", - "regex", - "reqwest", - "ron", - "serde", - "serde_json", - "serde_yaml", - "tokio", - "tokio-stream", - "toml", - "tracing", - "url", -] - -[[package]] -name = "persisting-gateway-fuzz" -version = "0.0.0" -dependencies = [ - "bytes", - "libfuzzer-sys", - "persisting-gateway", -] - -[[package]] -name = "persisting-overlaynet" -version = "0.2.0" -dependencies = [ - "anyhow", - "async-trait", - "axum", - "futures-util", - "hyper", - "hyper-util", - "persisting-agentctl", - "reqwest", - "serde", - "smoltcp", - "thiserror 2.0.20", - "tokio", - "tracing", -] - -[[package]] -name = "persisting-pchronicle" -version = "0.2.0" -dependencies = [ - "anyhow", - "async-trait", - "base64 0.22.1", - "blake3", - "chrono", - "fs2", - "persisting-agentctl", - "reqwest", - "ron", - "serde", - "serde_json", - "serde_yaml", - "thiserror 2.0.20", - "tokio", - "uuid", - "zstd", -] - -[[package]] -name = "pin-project-lite" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" - -[[package]] -name = "pkg-config" -version = "0.3.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" - -[[package]] -name = "potential_utf" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" -dependencies = [ - "zerovec", -] - -[[package]] -name = "ppv-lite86" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" -dependencies = [ - "zerocopy", -] - -[[package]] -name = "probing-memtable" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5551a1889a0365fa7167159a40f3e8a87557fea36dd77cfa204cd3f766b632a7" -dependencies = [ - "libc", - "memmap2", - "pco", - "xxhash-rust", -] - -[[package]] -name = "proc-macro2" -version = "1.0.107" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "pulsing-actor" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a751c49db41d362084cb011c77391c1b0e857d11f4e6a839f6452c722a090a6" -dependencies = [ - "anyhow", - "async-trait", - "bincode", - "bytes", - "dashmap", - "futures", - "http-body-util", - "hyper", - "hyper-util", - "lru", - "opentelemetry", - "opentelemetry_sdk", - "probing-memtable", - "rand 0.9.5", - "serde", - "serde_json", - "thiserror 2.0.20", - "tokio", - "tokio-stream", - "tokio-util", - "tracing", - "tracing-opentelemetry", - "tracing-subscriber", - "uuid", -] - -[[package]] -name = "quinn" -version = "0.11.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" -dependencies = [ - "bytes", - "cfg_aliases", - "pin-project-lite", - "quinn-proto", - "quinn-udp", - "rustc-hash", - "rustls", - "socket2", - "thiserror 2.0.20", - "tokio", - "tracing", - "web-time", -] - -[[package]] -name = "quinn-proto" -version = "0.11.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" -dependencies = [ - "bytes", - "getrandom 0.4.3", - "lru-slab", - "rand 0.10.2", - "rand_pcg", - "ring", - "rustc-hash", - "rustls", - "rustls-pki-types", - "slab", - "thiserror 2.0.20", - "tinyvec", - "tracing", - "web-time", -] - -[[package]] -name = "quinn-udp" -version = "0.5.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" -dependencies = [ - "cfg_aliases", - "libc", - "once_cell", - "socket2", - "tracing", - "windows-sys 0.61.2", -] - -[[package]] -name = "quote" -version = "1.0.47" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "r-efi" -version = "5.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" - -[[package]] -name = "r-efi" -version = "6.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" - -[[package]] -name = "rand" -version = "0.8.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" -dependencies = [ - "libc", - "rand_chacha 0.3.1", - "rand_core 0.6.4", -] - -[[package]] -name = "rand" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" -dependencies = [ - "rand_chacha 0.9.0", - "rand_core 0.9.5", -] - -[[package]] -name = "rand" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" -dependencies = [ - "chacha20", - "getrandom 0.4.3", - "rand_core 0.10.1", -] - -[[package]] -name = "rand_chacha" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" -dependencies = [ - "ppv-lite86", - "rand_core 0.6.4", -] - -[[package]] -name = "rand_chacha" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" -dependencies = [ - "ppv-lite86", - "rand_core 0.9.5", -] - -[[package]] -name = "rand_core" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" -dependencies = [ - "getrandom 0.2.17", -] - -[[package]] -name = "rand_core" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" -dependencies = [ - "getrandom 0.3.4", -] - -[[package]] -name = "rand_core" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" - -[[package]] -name = "rand_pcg" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" -dependencies = [ - "rand_core 0.10.1", -] - -[[package]] -name = "rand_xoshiro" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f97cdb2a36ed4183de61b2f824cc45c9f1037f28afe0a322e9fff4c108b5aaa" -dependencies = [ - "rand_core 0.6.4", -] - -[[package]] -name = "redox_syscall" -version = "0.5.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" -dependencies = [ - "bitflags 2.13.1", -] - -[[package]] -name = "redox_users" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" -dependencies = [ - "getrandom 0.2.17", - "libredox", - "thiserror 2.0.20", -] - -[[package]] -name = "regex" -version = "1.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" -dependencies = [ - "aho-corasick", - "memchr", - "regex-automata", - "regex-syntax", -] - -[[package]] -name = "regex-automata" -version = "0.4.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" -dependencies = [ - "aho-corasick", - "memchr", - "regex-syntax", -] - -[[package]] -name = "regex-syntax" -version = "0.8.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" - -[[package]] -name = "reqwest" -version = "0.12.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" -dependencies = [ - "base64 0.22.1", - "bytes", - "futures-core", - "futures-util", - "http", - "http-body", - "http-body-util", - "hyper", - "hyper-rustls", - "hyper-util", - "js-sys", - "log", - "percent-encoding", - "pin-project-lite", - "quinn", - "rustls", - "rustls-pki-types", - "serde", - "serde_json", - "serde_urlencoded", - "sync_wrapper", - "tokio", - "tokio-rustls", - "tokio-util", - "tower", - "tower-http", - "tower-service", - "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "wasm-streams", - "web-sys", - "webpki-roots", -] - -[[package]] -name = "ring" -version = "0.17.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" -dependencies = [ - "cc", - "cfg-if", - "getrandom 0.2.17", - "libc", - "untrusted", - "windows-sys 0.52.0", -] - -[[package]] -name = "ron" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b91f7eff05f748767f183df4320a63d6936e9c6107d97c9e6bdd9784f4289c94" -dependencies = [ - "base64 0.21.7", - "bitflags 2.13.1", - "serde", - "serde_derive", -] - -[[package]] -name = "rustc-hash" -version = "2.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" - -[[package]] -name = "rustls" -version = "0.23.43" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06" -dependencies = [ - "once_cell", - "ring", - "rustls-pki-types", - "rustls-webpki", - "subtle", - "zeroize", -] - -[[package]] -name = "rustls-pki-types" -version = "1.15.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" -dependencies = [ - "web-time", - "zeroize", -] - -[[package]] -name = "rustls-webpki" -version = "0.103.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0527518605e68109d875e248ea259b6758801cf165e4b2c2733ae3b51f12535a" -dependencies = [ - "ring", - "rustls-pki-types", - "untrusted", -] - -[[package]] -name = "rustversion" -version = "1.0.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" - -[[package]] -name = "ryu" -version = "1.0.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" - -[[package]] -name = "scopeguard" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" - -[[package]] -name = "serde" -version = "1.0.229" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" -dependencies = [ - "serde_core", - "serde_derive", -] - -[[package]] -name = "serde_core" -version = "1.0.229" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.229" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.3", -] - -[[package]] -name = "serde_json" -version = "1.0.151" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" -dependencies = [ - "itoa", - "memchr", - "serde", - "serde_core", - "zmij", -] - -[[package]] -name = "serde_path_to_error" -version = "0.1.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" -dependencies = [ - "itoa", - "serde", - "serde_core", -] - -[[package]] -name = "serde_spanned" -version = "0.6.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" -dependencies = [ - "serde", -] - -[[package]] -name = "serde_urlencoded" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" -dependencies = [ - "form_urlencoded", - "itoa", - "ryu", - "serde", -] - -[[package]] -name = "serde_yaml" -version = "0.9.34+deprecated" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" -dependencies = [ - "indexmap", - "itoa", - "ryu", - "serde", - "unsafe-libyaml", -] - -[[package]] -name = "sha1_smol" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d" - -[[package]] -name = "sharded-slab" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" -dependencies = [ - "lazy_static", -] - -[[package]] -name = "shlex" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" - -[[package]] -name = "signal-hook-registry" -version = "1.4.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" -dependencies = [ - "errno", - "libc", -] - -[[package]] -name = "slab" -version = "0.4.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" - -[[package]] -name = "smallvec" -version = "1.15.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" - -[[package]] -name = "smoltcp" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f73d40463bba65efc9adc6370b56df76d563cc46e2482bba58351b4afb7535e" -dependencies = [ - "bitflags 1.3.2", - "byteorder", - "cfg-if", - "defmt 0.3.100", - "heapless", - "log", - "managed", -] - -[[package]] -name = "socket2" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" -dependencies = [ - "libc", - "windows-sys 0.61.2", -] - -[[package]] -name = "stable_deref_trait" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" - -[[package]] -name = "subtle" -version = "2.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" - -[[package]] -name = "syn" -version = "2.0.119" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "syn" -version = "3.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "sync_wrapper" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" -dependencies = [ - "futures-core", -] - -[[package]] -name = "synstructure" -version = "0.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "thiserror" -version = "1.0.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" -dependencies = [ - "thiserror-impl 1.0.69", -] - -[[package]] -name = "thiserror" -version = "2.0.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" -dependencies = [ - "thiserror-impl 2.0.20", -] - -[[package]] -name = "thiserror-impl" -version = "1.0.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "thiserror-impl" -version = "2.0.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.3", -] - -[[package]] -name = "thread_local" -version = "1.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "tinystr" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" -dependencies = [ - "displaydoc", - "zerovec", -] - -[[package]] -name = "tinyvec" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" -dependencies = [ - "tinyvec_macros", -] - -[[package]] -name = "tinyvec_macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" - -[[package]] -name = "tokio" -version = "1.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" -dependencies = [ - "bytes", - "libc", - "mio", - "pin-project-lite", - "signal-hook-registry", - "socket2", - "tokio-macros", - "windows-sys 0.61.2", -] - -[[package]] -name = "tokio-macros" -version = "2.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.3", -] - -[[package]] -name = "tokio-rustls" -version = "0.26.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" -dependencies = [ - "rustls", - "tokio", -] - -[[package]] -name = "tokio-stream" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" -dependencies = [ - "futures-core", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "tokio-util" -version = "0.7.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" -dependencies = [ - "bytes", - "futures-core", - "futures-sink", - "futures-util", - "libc", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "toml" -version = "0.8.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" -dependencies = [ - "serde", - "serde_spanned", - "toml_datetime", - "toml_edit", -] - -[[package]] -name = "toml_datetime" -version = "0.6.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" -dependencies = [ - "serde", -] - -[[package]] -name = "toml_edit" -version = "0.22.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" -dependencies = [ - "indexmap", - "serde", - "serde_spanned", - "toml_datetime", - "toml_write", - "winnow", -] - -[[package]] -name = "toml_write" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" - -[[package]] -name = "tower" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" -dependencies = [ - "futures-core", - "futures-util", - "pin-project-lite", - "sync_wrapper", - "tokio", - "tower-layer", - "tower-service", -] - -[[package]] -name = "tower-http" -version = "0.6.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" -dependencies = [ - "bitflags 2.13.1", - "bytes", - "futures-util", - "http", - "http-body", - "pin-project-lite", - "tower", - "tower-layer", - "tower-service", - "url", -] - -[[package]] -name = "tower-layer" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" - -[[package]] -name = "tower-service" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" - -[[package]] -name = "tracing" -version = "0.1.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" -dependencies = [ - "pin-project-lite", - "tracing-attributes", - "tracing-core", -] - -[[package]] -name = "tracing-attributes" -version = "0.1.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "tracing-core" -version = "0.1.36" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" -dependencies = [ - "once_cell", - "valuable", -] - -[[package]] -name = "tracing-log" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" -dependencies = [ - "log", - "once_cell", - "tracing-core", -] - -[[package]] -name = "tracing-opentelemetry" -version = "0.28.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97a971f6058498b5c0f1affa23e7ea202057a7301dbff68e968b2d578bcbd053" -dependencies = [ - "js-sys", - "once_cell", - "opentelemetry", - "opentelemetry_sdk", - "smallvec", - "tracing", - "tracing-core", - "tracing-log", - "tracing-subscriber", - "web-time", -] - -[[package]] -name = "tracing-serde" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "704b1aeb7be0d0a84fc9828cae51dab5970fee5088f83d1dd7ee6f6246fc6ff1" -dependencies = [ - "serde", - "tracing-core", -] - -[[package]] -name = "tracing-subscriber" -version = "0.3.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" -dependencies = [ - "matchers", - "nu-ansi-term", - "once_cell", - "regex-automata", - "serde", - "serde_json", - "sharded-slab", - "smallvec", - "thread_local", - "tracing", - "tracing-core", - "tracing-log", - "tracing-serde", -] - -[[package]] -name = "try-lock" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" - -[[package]] -name = "unicode-ident" -version = "1.0.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" - -[[package]] -name = "unsafe-libyaml" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" - -[[package]] -name = "untrusted" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" - -[[package]] -name = "url" -version = "2.5.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" -dependencies = [ - "form_urlencoded", - "idna", - "percent-encoding", - "serde", -] - -[[package]] -name = "utf8_iter" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" - -[[package]] -name = "uuid" -version = "1.24.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" -dependencies = [ - "getrandom 0.4.3", - "js-sys", - "rand 0.10.2", - "sha1_smol", - "wasm-bindgen", -] - -[[package]] -name = "valuable" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" - -[[package]] -name = "want" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" -dependencies = [ - "try-lock", -] - -[[package]] -name = "wasi" -version = "0.11.1+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" - -[[package]] -name = "wasip2" -version = "1.0.4+wasi-0.2.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" -dependencies = [ - "wit-bindgen", -] - -[[package]] -name = "wasm-bindgen" -version = "0.2.127" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70" -dependencies = [ - "cfg-if", - "once_cell", - "rustversion", - "wasm-bindgen-macro", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-futures" -version = "0.4.77" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b7777d5cc23d0e91404e53ce2d5e8ec7acae3026b16233dba62cd3246457950" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - -[[package]] -name = "wasm-bindgen-macro" -version = "0.2.127" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1" -dependencies = [ - "quote", - "wasm-bindgen-macro-support", -] - -[[package]] -name = "wasm-bindgen-macro-support" -version = "0.2.127" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284" -dependencies = [ - "bumpalo", - "proc-macro2", - "quote", - "syn 2.0.119", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-shared" -version = "0.2.127" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "wasm-streams" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" -dependencies = [ - "futures-util", - "js-sys", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", -] - -[[package]] -name = "web-sys" -version = "0.3.104" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c435338968042f4f59a557f690a253676d47ce13ceb55d70100e7facf6620a30" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - -[[package]] -name = "web-time" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - -[[package]] -name = "webpki-roots" -version = "1.0.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" -dependencies = [ - "rustls-pki-types", -] - -[[package]] -name = "winapi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" -dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", -] - -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - -[[package]] -name = "windows-core" -version = "0.62.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" -dependencies = [ - "windows-implement", - "windows-interface", - "windows-link", - "windows-result", - "windows-strings", -] - -[[package]] -name = "windows-implement" -version = "0.60.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "windows-interface" -version = "0.59.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "windows-link" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" - -[[package]] -name = "windows-result" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" -dependencies = [ - "windows-link", -] - -[[package]] -name = "windows-strings" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" -dependencies = [ - "windows-link", -] - -[[package]] -name = "windows-sys" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" -dependencies = [ - "windows-targets", -] - -[[package]] -name = "windows-sys" -version = "0.61.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" -dependencies = [ - "windows-link", -] - -[[package]] -name = "windows-targets" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" -dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", -] - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" - -[[package]] -name = "windows_i686_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" - -[[package]] -name = "windows_i686_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" - -[[package]] -name = "windows_i686_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" - -[[package]] -name = "winnow" -version = "0.7.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" -dependencies = [ - "memchr", -] - -[[package]] -name = "wit-bindgen" -version = "0.57.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" - -[[package]] -name = "writeable" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" - -[[package]] -name = "xxhash-rust" -version = "0.8.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aee1b19627c7c60102ab80d3a9cbe18de90bfe03bfa6c3715447681f0e8c8af6" - -[[package]] -name = "yoke" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" -dependencies = [ - "stable_deref_trait", - "yoke-derive", - "zerofrom", -] - -[[package]] -name = "yoke-derive" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", - "synstructure", -] - -[[package]] -name = "zerocopy" -version = "0.8.56" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb" -dependencies = [ - "zerocopy-derive", -] - -[[package]] -name = "zerocopy-derive" -version = "0.8.56" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "zerofrom" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" -dependencies = [ - "zerofrom-derive", -] - -[[package]] -name = "zerofrom-derive" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", - "synstructure", -] - -[[package]] -name = "zeroize" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" - -[[package]] -name = "zerotrie" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" -dependencies = [ - "displaydoc", - "yoke", - "zerofrom", -] - -[[package]] -name = "zerovec" -version = "0.11.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94b5c6b5976d66c1d703c4fd17d3f5e43c8cedaacf604961b171adc7130896d8" -dependencies = [ - "yoke", - "zerofrom", - "zerovec-derive", -] - -[[package]] -name = "zerovec-derive" -version = "0.11.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47402523226a02bfe5230160dc3ccc089aa6f6f19e7fcbb4e6f824bbb1b4aa62" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.3", -] - -[[package]] -name = "zmij" -version = "1.0.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" - -[[package]] -name = "zstd" -version = "0.13.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" -dependencies = [ - "zstd-safe", -] - -[[package]] -name = "zstd-safe" -version = "7.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" -dependencies = [ - "zstd-sys", -] - -[[package]] -name = "zstd-sys" -version = "2.0.16+zstd.1.5.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" -dependencies = [ - "cc", - "pkg-config", -] diff --git a/crates/persisting-gateway/fuzz/Cargo.toml b/crates/persisting-gateway/fuzz/Cargo.toml deleted file mode 100644 index b2574173b..000000000 --- a/crates/persisting-gateway/fuzz/Cargo.toml +++ /dev/null @@ -1,30 +0,0 @@ -[package] -name = "persisting-gateway-fuzz" -version = "0.0.0" -publish = false -edition = "2021" - -[package.metadata] -cargo-fuzz = true - -[dependencies] -bytes = "1" -libfuzzer-sys = "0.4" -persisting-gateway = { path = ".." } - -[[bin]] -name = "llm_request_conversion" -path = "fuzz_targets/llm_request_conversion.rs" -test = false -doc = false -bench = false - -[[bin]] -name = "llm_stream_conversion" -path = "fuzz_targets/llm_stream_conversion.rs" -test = false -doc = false -bench = false - -[workspace] -members = ["."] diff --git a/crates/persisting-gateway/fuzz/fuzz_targets/llm_request_conversion.rs b/crates/persisting-gateway/fuzz/fuzz_targets/llm_request_conversion.rs deleted file mode 100644 index bcc915017..000000000 --- a/crates/persisting-gateway/fuzz/fuzz_targets/llm_request_conversion.rs +++ /dev/null @@ -1,38 +0,0 @@ -#![no_main] - -use bytes::Bytes; -use libfuzzer_sys::fuzz_target; -use persisting_gateway::conversion::{ - completions_request_to_gemini, completions_response_to_messages, - gemini_response_to_completions, messages_request_to_completions, - responses_request_to_completions, -}; -use persisting_gateway::protocol::ProtocolKind; -use persisting_gateway::understanding::understand_request; - -fuzz_target!(|data: &[u8]| { - let Some((&selector, body)) = data.split_first() else { - return; - }; - let body = Bytes::copy_from_slice(body); - match selector % 5 { - 0 => { - let _ = understand_request(ProtocolKind::Messages, &body); - let _ = messages_request_to_completions(&body, "gpt-5.6-terra"); - } - 1 => { - let _ = understand_request(ProtocolKind::Responses, &body); - let _ = responses_request_to_completions(&body, "deepseek-chat", None); - } - 2 => { - let _ = understand_request(ProtocolKind::ChatCompletions, &body); - let _ = completions_response_to_messages(&body, "claude-client"); - } - 3 => { - let _ = completions_request_to_gemini(&body, "gemini-2.5-pro"); - } - _ => { - let _ = gemini_response_to_completions(&body, "gemini-2.5-pro"); - } - } -}); diff --git a/crates/persisting-gateway/fuzz/fuzz_targets/llm_stream_conversion.rs b/crates/persisting-gateway/fuzz/fuzz_targets/llm_stream_conversion.rs deleted file mode 100644 index 6e346e867..000000000 --- a/crates/persisting-gateway/fuzz/fuzz_targets/llm_stream_conversion.rs +++ /dev/null @@ -1,48 +0,0 @@ -#![no_main] - -use libfuzzer_sys::fuzz_target; -use persisting_gateway::conversion::{ProtocolBridge, StreamTranslator}; -use persisting_gateway::protocol::ProtocolKind; - -fuzz_target!(|data: &[u8]| { - let bridges = [ - (ProtocolBridge::Passthrough, ProtocolKind::ChatCompletions), - ( - ProtocolBridge::MessagesToCompletions, - ProtocolKind::Messages, - ), - ( - ProtocolBridge::ResponsesToCompletions, - ProtocolKind::Responses, - ), - ( - ProtocolBridge::CompletionsToGemini, - ProtocolKind::ChatCompletions, - ), - (ProtocolBridge::MessagesToGemini, ProtocolKind::Messages), - (ProtocolBridge::ResponsesToGemini, ProtocolKind::Responses), - ]; - let chunk_size = data.first().map_or(1, |byte| usize::from(*byte).max(1)); - for (bridge, protocol) in bridges { - let Some(mut translator) = StreamTranslator::new(bridge, protocol, "fuzz-model") else { - continue; - }; - let mut output = Vec::new(); - let mut succeeded = true; - for chunk in data.chunks(chunk_size) { - match translator.push_chunk(chunk) { - Ok(rendered) => output.extend_from_slice(&rendered), - Err(_) => succeeded = false, - } - } - match translator.finish_stream() { - Ok(rendered) => output.extend_from_slice(&rendered), - Err(_) => succeeded = false, - } - let _ = translator.semantic_response(); - if bridge == ProtocolBridge::Passthrough { - assert!(succeeded); - assert_eq!(output, data); - } - } -}); diff --git a/crates/persisting-gateway/src/config.rs b/crates/persisting-gateway/src/config.rs deleted file mode 100644 index e4ab20377..000000000 --- a/crates/persisting-gateway/src/config.rs +++ /dev/null @@ -1,577 +0,0 @@ -//! Proxy configuration (TOML): `models[]` entries + optional `forward` to another model name. - -use std::collections::HashSet; -use std::path::Path; - -use serde::{Deserialize, Serialize}; -use url::Url; - -pub use crate::protocol::ProtocolKind; -pub use crate::provider::ProviderKind; -pub use persisting_overlaynet::{NetworkConfig, NetworkMode}; - -/// Top-level config for capture proxy / daemon. -#[derive(Debug, Clone, Deserialize, Serialize)] -#[serde(deny_unknown_fields)] -pub struct ProxyConfig { - pub listen: String, - /// Admin API for `capture status` (default `127.0.0.1:9876`). - #[serde(default = "default_admin_listen")] - pub admin_listen: String, - #[serde(default = "default_agent_id")] - pub agent_id: String, - #[serde(default = "default_session_header")] - pub session_header: String, - /// What to persist in trajectory: `summary` (metadata), `dialogue` (default, user/assistant text), `full` (raw bodies). - #[serde(default)] - pub capture_level: CaptureLevel, - /// Log every proxied / captured HTTP request to stderr and `{storage}/.capture/debug.log`. - #[serde(default)] - pub debug: bool, - /// Harbor-aligned egress policy for forward-proxy traffic (`CONNECT` + absolute-URI). - #[serde(default)] - pub network: NetworkConfig, - /// Optional overlay configuration for integrations; Gateway does not mount it. - #[serde(default)] - pub overlay: OverlayConfig, - pub models: Vec, -} - -/// Filesystem overlay settings retained in capture TOML for integrations. -/// -/// Model: **target** (read-only base / apply destination) + **staging** (upper -/// holds deltas). The Agent sees `merged`; changes do **not** touch `target` -/// until an explicit runtime overlay is applied. -#[derive(Debug, Clone, Default, Deserialize, Serialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct OverlayConfig { - /// Whether the integration requests an overlay for the Attempt. - #[serde(default)] - pub enabled: bool, - /// Target filesystem: primary lower layer and destination for `apply`. - /// Prefer this over listing the same path in `lower_dirs`. - #[serde(default)] - pub target: Option, - /// Prevent explicit or automatic apply from modifying the lower target. - #[serde(default)] - pub protect_target: bool, - /// Read-only compose layers stacked above `target`, highest priority first. - #[serde(default)] - pub lower_dirs: Vec, - /// Root for staging (`upper` / `work` / `merged`). Default: - /// `{capture_storage}/.overlay/{session_id}/`. - #[serde(default)] - pub stage_dir: Option, - /// Writable upper backend. `directory` is the default; `jujutsu` adds - /// named persistent forks in a shared repository. - #[serde(default)] - pub backend: OverlayBackend, - /// Shared Jujutsu store. All named workspaces use the same object store and - /// operation log (overrides `{storage}/.overlay/jujutsu`). - #[serde(default)] - pub jujutsu_store_path: Option, - /// Jujutsu workspace/fork name for the integration. - #[serde(default)] - pub jujutsu_workspace: Option, - /// Writable upper directory (overrides `{stage_dir}/upper` when set). - #[serde(default)] - pub upper_dir: Option, - /// Overlay work directory (overrides `{stage_dir}/work` when set). - #[serde(default)] - pub work_dir: Option, - /// Merged mount point (overrides `{stage_dir}/merged` when set). - #[serde(default)] - pub merged_dir: Option, - /// If true, apply staging onto `target` automatically when the Attempt ends. - /// Default false; effects require an explicit apply or discard decision. - #[serde(default)] - pub auto_apply: bool, - /// If true, discard staging automatically when the Attempt ends. - #[serde(default)] - pub auto_discard: bool, -} - -#[derive(Debug, Clone, Copy, Default, Deserialize, Serialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum OverlayBackend { - #[default] - Directory, - Jujutsu, -} - -fn default_admin_listen() -> String { - "127.0.0.1:9876".to_string() -} - -fn default_agent_id() -> String { - "default".to_string() -} - -fn default_session_header() -> String { - "x-persisting-session-id".to_string() -} - -/// Controls how much request/response content is written to trajectory records. -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize, serde::Serialize)] -#[serde(rename_all = "snake_case")] -pub enum CaptureLevel { - /// Model, path, byte counts — no message text. - Summary, - /// User / assistant dialogue text (default). - #[default] - Dialogue, - /// Full parsed JSON bodies in `payload.body`. - Full, -} - -impl CaptureLevel { - pub fn includes_user_text(self) -> bool { - !matches!(self, Self::Summary) - } - - pub fn includes_assistant_text(self) -> bool { - !matches!(self, Self::Summary) - } - - pub fn includes_full_body(self) -> bool { - matches!(self, Self::Full) - } -} - -#[derive(Debug, Clone, Deserialize, Serialize)] -#[serde(deny_unknown_fields)] -pub struct ModelRoute { - /// Match pattern (exact, `prefix*`, `*suffix`, `*`) or target model id. - pub name: String, - /// `openai` | `anthropic` | `gemini` | `vertex` | `bedrock` | `azure` | `copilot` | `custom` - #[serde(default)] - pub provider: Option, - /// OpenAI-compatible upstream base (include API prefix, e.g. `https://api.deepseek.com/v1`). - #[serde(default)] - pub upstream: Option, - /// Anthropic-compatible upstream (e.g. `https://api.deepseek.com/anthropic/v1`). Falls back to `upstream`. - #[serde(default)] - pub upstream_anthropic: Option, - #[serde(default)] - pub api_key_env: Option, - #[serde(default)] - pub api_key: Option, - /// Forward to another `models[].name` (exact id): use its upstream and rewrite request `model`. - #[serde(default)] - pub forward: Option, -} - -impl ProxyConfig { - pub fn from_toml_str(s: &str) -> anyhow::Result { - let cfg: Self = toml::from_str(s)?; - cfg.validate()?; - Ok(cfg) - } - - pub fn from_toml_file(path: &Path) -> anyhow::Result { - let s = std::fs::read_to_string(path)?; - Self::from_toml_str(&s) - } - - /// Load a TOML proxy configuration. - pub fn from_file(path: &Path) -> anyhow::Result { - match path - .extension() - .and_then(|e| e.to_str()) - .map(str::to_ascii_lowercase) - .as_deref() - { - Some("toml") | None => Self::from_toml_file(path), - Some(ext) => anyhow::bail!("unsupported proxy config extension `.{ext}` (use .toml)"), - } - } - - pub fn to_toml_string(&self) -> anyhow::Result { - Ok(toml::to_string_pretty(self)?) - } - - /// Validate model entries, `forward` references, network policy, and duplicate names. - pub fn validate(&self) -> anyhow::Result<()> { - persisting_overlaynet::policy::validate_network_config(&self.network)?; - if self.overlay.auto_apply && self.overlay.auto_discard { - anyhow::bail!("overlay auto_apply and auto_discard are mutually exclusive"); - } - match self.overlay.backend { - OverlayBackend::Directory => { - if self.overlay.jujutsu_store_path.is_some() - || self.overlay.jujutsu_workspace.is_some() - { - anyhow::bail!("overlay backend `directory` cannot use Jujutsu options"); - } - } - OverlayBackend::Jujutsu => { - if self.overlay.upper_dir.is_some() || self.overlay.work_dir.is_some() { - anyhow::bail!("overlay backend `jujutsu` cannot use directory upper options"); - } - } - } - let mut seen = HashSet::new(); - for route in &self.models { - if !seen.insert(route.name.clone()) { - anyhow::bail!("duplicate models[].name `{}`", route.name); - } - match (&route.forward, &route.upstream) { - (Some(fwd), None) => { - if fwd == &route.name { - anyhow::bail!( - "models[].forward must reference another entry, not `{}`", - route.name - ); - } - } - (None, None) => { - anyhow::bail!( - "models[] entry `{}` needs `upstream` or `forward`", - route.name - ); - } - (Some(_), Some(_)) => { - anyhow::bail!( - "models[] entry `{}` cannot set both `forward` and `upstream`", - route.name - ); - } - (None, Some(_)) => {} - } - } - for route in &self.models { - let Some(fwd) = &route.forward else { - continue; - }; - let target = self - .models - .iter() - .find(|r| r.name == *fwd) - .ok_or_else(|| anyhow::anyhow!("forward target `{fwd}` not found"))?; - if target.forward.is_some() { - anyhow::bail!("forward target `{fwd}` cannot forward again"); - } - if target.upstream.is_none() { - anyhow::bail!("forward target `{fwd}` missing upstream"); - } - } - Ok(()) - } -} - -impl ModelRoute { - pub fn provider_kind(&self) -> ProviderKind { - self.provider - .as_deref() - .map(ProviderKind::parse) - .unwrap_or(ProviderKind::OpenAi) - } - - /// Provider used for indexing / cost when protocol selects an Anthropic upstream. - pub fn effective_provider(&self, protocol: ProtocolKind) -> ProviderKind { - if protocol == ProtocolKind::Messages && self.upstream_anthropic.is_some() { - return ProviderKind::Anthropic; - } - self.provider_kind() - } - - fn effective_upstream_base(&self, protocol: ProtocolKind) -> anyhow::Result<&str> { - if protocol == ProtocolKind::Messages - && let Some(ref u) = self.upstream_anthropic - { - return Ok(u.as_str()); - } - self.upstream - .as_deref() - .ok_or_else(|| anyhow::anyhow!("model `{}` has no upstream", self.name)) - } - - pub fn resolve_upstream_url( - &self, - incoming_path: &str, - protocol: ProtocolKind, - ) -> anyhow::Result { - let base_str = self.effective_upstream_base(protocol)?; - let mut base = Url::parse(base_str) - .map_err(|e| anyhow::anyhow!("invalid upstream for model {}: {e}", self.name))?; - let api_prefix = detect_incoming_api_prefix(incoming_path).to_string(); - let suffix = strip_incoming_api_prefix(incoming_path, &api_prefix); - let base_path = base.path().trim_end_matches('/'); - - let final_path = if base_path.is_empty() || base_path == "/" { - join_api_path(&api_prefix, &suffix) - } else if base_includes_api_prefix(base_path, &api_prefix) { - join_api_path(base_path, &suffix) - } else { - join_api_path(&format!("{base_path}{api_prefix}"), &suffix) - }; - - base.set_path(&final_path); - Ok(base) - } - - pub fn api_key_value(&self) -> anyhow::Result> { - if let Some(ref k) = self.api_key { - return Ok(Some(k.clone())); - } - if let Some(ref env) = self.api_key_env { - return Ok(lookup_env_var(env)); - } - Ok(None) - } -} - -/// Read an API-key env var plus known Claude Code / provider aliases. -pub fn lookup_env_var(name: &str) -> Option { - if let Ok(v) = std::env::var(name) { - let v = v.trim(); - if !v.is_empty() { - return Some(v.to_string()); - } - } - for alias in api_key_env_aliases(name) { - if let Ok(v) = std::env::var(alias) { - let v = v.trim(); - if !v.is_empty() { - return Some(v.to_string()); - } - } - } - None -} - -pub fn api_key_env_aliases(primary: &str) -> &'static [&'static str] { - match primary { - "DEEPSEEK_API_KEY" => &["ANTHROPIC_AUTH_TOKEN", "ANTHROPIC_API_KEY"], - "ANTHROPIC_API_KEY" => &["ANTHROPIC_AUTH_TOKEN", "DEEPSEEK_API_KEY"], - "ANTHROPIC_AUTH_TOKEN" => &["ANTHROPIC_API_KEY", "DEEPSEEK_API_KEY"], - "OPENAI_API_KEY" => &["ANTHROPIC_AUTH_TOKEN"], - "GEMINI_API_KEY" => &["GOOGLE_API_KEY"], - "GOOGLE_API_KEY" => &["GEMINI_API_KEY"], - _ => &[], - } -} - -/// Detect API version prefix from the client request path (`/v1/messages` → `/v1`). -fn detect_incoming_api_prefix(incoming_path: &str) -> &'static str { - let incoming = incoming_path.trim_start_matches('/'); - if incoming.starts_with("v1beta/") || incoming == "v1beta" { - "/v1beta" - } else { - "/v1" - } -} - -fn base_includes_api_prefix(base_path: &str, api_prefix: &str) -> bool { - let base = base_path.trim_end_matches('/'); - let prefix = api_prefix.trim_start_matches('/').trim_end_matches('/'); - if base.is_empty() { - return false; - } - base == prefix || base.ends_with(&format!("/{prefix}")) -} - -/// Strip the incoming API version prefix (e.g. `/v1/messages` → `messages`). -fn strip_incoming_api_prefix(incoming_path: &str, api_prefix: &str) -> String { - let incoming = incoming_path.trim_start_matches('/'); - let prefix = api_prefix.trim_start_matches('/').trim_end_matches('/'); - if incoming == prefix { - return String::new(); - } - if let Some(rest) = incoming.strip_prefix(&format!("{prefix}/")) { - return rest.to_string(); - } - incoming.to_string() -} - -fn join_api_path(prefix: &str, suffix: &str) -> String { - let prefix = prefix.trim_end_matches('/'); - if suffix.is_empty() { - if prefix.is_empty() { - "/".to_string() - } else { - prefix.to_string() - } - } else if prefix.is_empty() { - format!("/{suffix}") - } else { - format!("{prefix}/{suffix}") - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::protocol::ProtocolKind; - - fn route(upstream: &str) -> ModelRoute { - route_with_anthropic(upstream, None) - } - - fn route_with_anthropic(upstream: &str, upstream_anthropic: Option<&str>) -> ModelRoute { - ModelRoute { - name: "test".into(), - provider: None, - upstream: Some(upstream.into()), - upstream_anthropic: upstream_anthropic.map(str::to_string), - api_key_env: None, - api_key: None, - forward: None, - } - } - - #[test] - fn upstream_url_strips_duplicate_v1() { - let r = route("http://127.0.0.1:19080/v1"); - let url = r - .resolve_upstream_url("/v1/messages", ProtocolKind::Messages) - .unwrap(); - assert_eq!(url.as_str(), "http://127.0.0.1:19080/v1/messages"); - - let url = r - .resolve_upstream_url("/v1/chat/completions", ProtocolKind::ChatCompletions) - .unwrap(); - assert_eq!(url.as_str(), "http://127.0.0.1:19080/v1/chat/completions"); - } - - #[test] - fn upstream_url_anthropic_host_with_v1_in_upstream() { - let r = route("https://api.anthropic.com/v1"); - let url = r - .resolve_upstream_url("/v1/messages", ProtocolKind::Messages) - .unwrap(); - assert_eq!(url.as_str(), "https://api.anthropic.com/v1/messages"); - } - - #[test] - fn upstream_url_when_base_already_has_prefix() { - let r = route("https://api.openai.com/v1"); - let url = r - .resolve_upstream_url("/v1/chat/completions", ProtocolKind::ChatCompletions) - .unwrap(); - assert_eq!(url.as_str(), "https://api.openai.com/v1/chat/completions"); - } - - #[test] - fn upstream_url_v1beta_prefix() { - let r = route("https://generativelanguage.googleapis.com/v1beta"); - let url = r - .resolve_upstream_url( - "/v1beta/models/gemini-pro:generateContent", - ProtocolKind::Unknown, - ) - .unwrap(); - assert_eq!( - url.as_str(), - "https://generativelanguage.googleapis.com/v1beta/models/gemini-pro:generateContent" - ); - } - - #[test] - fn upstream_url_deepseek_anthropic_base() { - let r = route_with_anthropic( - "https://api.deepseek.com/v1", - Some("https://api.deepseek.com/anthropic/v1"), - ); - let url = r - .resolve_upstream_url("/v1/messages", ProtocolKind::Messages) - .unwrap(); - assert_eq!( - url.as_str(), - "https://api.deepseek.com/anthropic/v1/messages" - ); - let url = r - .resolve_upstream_url("/v1/chat/completions", ProtocolKind::ChatCompletions) - .unwrap(); - assert_eq!(url.as_str(), "https://api.deepseek.com/v1/chat/completions"); - } - - #[test] - fn effective_provider_anthropic_when_dual_upstream() { - let r = route_with_anthropic( - "https://api.deepseek.com/v1", - Some("https://api.deepseek.com/anthropic/v1"), - ); - assert_eq!( - r.effective_provider(ProtocolKind::Messages), - ProviderKind::Anthropic - ); - assert_eq!( - r.effective_provider(ProtocolKind::ChatCompletions), - ProviderKind::OpenAi - ); - } - - #[test] - fn toml_roundtrip_and_deny_unknown_field() { - let src = r#" -listen = "127.0.0.1:1" - -[[models]] -name = "*" -upstream = "http://example.com/v1" -unknown_field = true -"#; - assert!(ProxyConfig::from_toml_str(src).is_err()); - - let cfg = ProxyConfig::from_toml_str( - r#" -listen = "127.0.0.1:1" - -[[models]] -name = "*" -upstream = "http://example.com/v1" -"#, - ) - .unwrap(); - let again = ProxyConfig::from_toml_str(&cfg.to_toml_string().unwrap()).unwrap(); - assert_eq!(cfg.listen, again.listen); - } - - #[test] - fn config_rejects_allow_entries_without_allowlist_mode() { - let error = ProxyConfig::from_toml_str( - r#" -listen = "127.0.0.1:1" - -[network] -allowed_hosts = ["api.example.com"] - -[[models]] -name = "*" -upstream = "http://example.com/v1" -"#, - ) - .unwrap_err(); - assert!( - error - .to_string() - .contains("network allow entries require mode") - ); - } - - #[test] - fn config_rejects_invalid_bandwidth_limits_at_load_time() { - let error = ProxyConfig::from_toml_str( - r#" -listen = "127.0.0.1:1" - -[network] -mode = "public" - -[[network.limits]] -bytes_per_second = 0 - -[[models]] -name = "*" -upstream = "http://example.com/v1" -"#, - ) - .unwrap_err(); - assert!( - error - .to_string() - .contains("network bandwidth limit must be greater than zero") - ); - } -} diff --git a/crates/persisting-gateway/src/conversion/gemini_native.rs b/crates/persisting-gateway/src/conversion/gemini_native.rs deleted file mode 100644 index 37ba95959..000000000 --- a/crates/persisting-gateway/src/conversion/gemini_native.rs +++ /dev/null @@ -1,1011 +0,0 @@ -//! OpenAI Chat Completions ↔ Gemini native `generateContent` wire format. -//! -//! Adapted from agentgateway's `conversion::vertex_gemini` contract. This module intentionally -//! contains no Google SDK or control-plane dependency; Gateway routing, auth, capture and WAL stay -//! outside the protocol adapter. - -use std::collections::HashMap; - -use anyhow::{Context, Result}; -use bytes::Bytes; -use serde_json::{Map, Value, json}; - -const THOUGHT_SIGNATURE_SEPARATOR: &str = "__thought__"; - -pub fn completions_request_to_gemini(body: &Bytes, model: &str) -> Result { - let request: Value = serde_json::from_slice(body).context("parse completions request")?; - let object = request - .as_object() - .context("completions request must be a JSON object")?; - let messages = object - .get("messages") - .and_then(Value::as_array) - .context("completions request missing messages")?; - - let (system, mut contents) = messages_to_contents(messages)?; - if contents.is_empty() { - contents.push(json!({"role":"user", "parts":[{"text":" "}]})); - } - - let mut output = json!({"contents": contents}); - if !system.is_empty() { - output["systemInstruction"] = json!({ - "parts": [{"text": system.join("\n")}] - }); - } - if let Some(tools) = build_tools(object) { - output["tools"] = tools; - } - if let Some(tool_config) = build_tool_config(object) { - output["toolConfig"] = tool_config; - } - if let Some(generation_config) = build_generation_config(object, model) { - output["generationConfig"] = generation_config; - } - for (source, target) in [ - ("cachedContent", "cachedContent"), - ("cached_content", "cachedContent"), - ("safetySettings", "safetySettings"), - ("safety_settings", "safetySettings"), - ("labels", "labels"), - ] { - if output.get(target).is_none() - && let Some(value) = object.get(source).filter(|value| !value.is_null()) - { - output[target] = value.clone(); - } - } - // Gemini cached content already owns system instructions and tools. - if output.get("cachedContent").is_some() { - output - .as_object_mut() - .expect("object") - .remove("systemInstruction"); - output.as_object_mut().expect("object").remove("tools"); - output.as_object_mut().expect("object").remove("toolConfig"); - } - - Ok(Bytes::from(serde_json::to_vec(&output)?)) -} - -fn messages_to_contents(messages: &[Value]) -> Result<(Vec, Vec)> { - let mut system = Vec::new(); - let mut contents = Vec::new(); - let mut calls: HashMap = HashMap::new(); - - for message in messages { - let role = message - .get("role") - .and_then(Value::as_str) - .unwrap_or("user"); - let content = message.get("content").unwrap_or(&Value::Null); - match role { - "system" | "developer" => { - if let Some(text) = content_text(content).filter(|text| !text.is_empty()) { - system.push(text); - } - } - "user" => push_content(&mut contents, "user", user_parts(content)?), - "assistant" => { - let mut parts = assistant_text_parts(content); - if let Some(tool_calls) = message.get("tool_calls").and_then(Value::as_array) { - for (index, call) in tool_calls.iter().enumerate() { - if let (Some(id), Some(name)) = ( - call.get("id").and_then(Value::as_str), - call.get("function") - .and_then(|function| function.get("name")) - .and_then(Value::as_str), - ) { - calls.insert( - split_tool_call_id(id).0.to_string(), - (name.to_string(), index), - ); - } - parts.push(function_call_part(call)); - } - } - push_content(&mut contents, "model", parts); - } - "tool" | "function" => { - let raw_id = message - .get("tool_call_id") - .and_then(Value::as_str) - .unwrap_or_default(); - let base_id = split_tool_call_id(raw_id).0; - let name = calls - .get(base_id) - .map(|(name, _)| name.as_str()) - .or_else(|| message.get("name").and_then(Value::as_str)) - .unwrap_or_default(); - let response = content_text(content) - .map(|text| json!({"content": text})) - .unwrap_or_else(|| json!({})); - let mut part = json!({ - "functionResponse": { - "name": name, - "response": response - } - }); - if !base_id.is_empty() { - // Transient correlation key. Gemini rejects this field, so the ordering pass - // below removes it before serialization. - part["functionResponse"]["id"] = json!(base_id); - } - push_content(&mut contents, "user", vec![part]); - } - _ => {} - } - } - - // Gemini correlates parallel function responses positionally. - for content in &mut contents { - let Some(parts) = content.get_mut("parts").and_then(Value::as_array_mut) else { - continue; - }; - if !parts - .iter() - .any(|part| part.get("functionResponse").is_some()) - { - continue; - } - parts.sort_by_key(|part| { - part.get("functionResponse") - .and_then(|response| response.get("id")) - .and_then(Value::as_str) - .and_then(|id| calls.get(id)) - .map(|(_, index)| *index) - .unwrap_or(usize::MAX) - }); - for part in parts { - if let Some(response) = part - .get_mut("functionResponse") - .and_then(Value::as_object_mut) - { - response.remove("id"); - } - } - } - Ok((system, contents)) -} - -fn push_content(contents: &mut Vec, role: &str, mut parts: Vec) { - if parts.is_empty() { - return; - } - let function_response = parts - .iter() - .any(|part| part.get("functionResponse").is_some()); - let can_merge = contents.last().is_some_and(|last| { - last.get("role").and_then(Value::as_str) == Some(role) - && last - .get("parts") - .and_then(Value::as_array) - .is_some_and(|last_parts| { - last_parts - .iter() - .any(|part| part.get("functionResponse").is_some()) - == function_response - }) - }); - if can_merge { - if role == "user" - && !function_response - && !contents - .last() - .and_then(|last| last.get("parts")) - .and_then(Value::as_array) - .is_some_and(|parts| parts.iter().any(|part| part.get("text").is_some())) - && !parts.iter().any(|part| part.get("text").is_some()) - { - parts.push(json!({"text":" "})); - } - contents - .last_mut() - .and_then(|last| last.get_mut("parts")) - .and_then(Value::as_array_mut) - .expect("checked above") - .extend(parts); - } else { - if role == "user" - && !function_response - && !parts.iter().any(|part| part.get("text").is_some()) - { - parts.push(json!({"text":" "})); - } - contents.push(json!({"role": role, "parts": parts})); - } -} - -fn content_text(content: &Value) -> Option { - match content { - Value::String(text) => Some(text.clone()), - Value::Array(parts) => Some( - parts - .iter() - .filter(|part| part.get("type").and_then(Value::as_str) == Some("text")) - .filter_map(|part| part.get("text").and_then(Value::as_str)) - .collect::(), - ), - _ => None, - } -} - -fn assistant_text_parts(content: &Value) -> Vec { - match content { - Value::String(text) if !text.is_empty() => vec![json!({"text":text})], - Value::Array(parts) => parts - .iter() - .filter(|part| part.get("type").and_then(Value::as_str) == Some("text")) - .filter_map(|part| part.get("text").and_then(Value::as_str)) - .map(|text| json!({"text":text})) - .collect(), - _ => Vec::new(), - } -} - -fn user_parts(content: &Value) -> Result> { - match content { - Value::String(text) => Ok(vec![json!({"text":text})]), - Value::Array(parts) => parts - .iter() - .filter_map(|part| match part.get("type").and_then(Value::as_str) { - Some("text") => part - .get("text") - .and_then(Value::as_str) - .map(|text| Ok(json!({"text":text}))), - Some("image_url") => Some(image_part(part.get("image_url"))), - _ => None, - }) - .collect(), - _ => Ok(Vec::new()), - } -} - -fn image_part(image_url: Option<&Value>) -> Result { - let image_url = image_url.context("image_url content part missing image_url")?; - let url = image_url - .get("url") - .and_then(Value::as_str) - .context("image_url missing url")?; - if let Some(raw) = url.strip_prefix("data:") { - let (meta, data) = raw.split_once(',').context("invalid image data URL")?; - let (mime, encoding) = meta.split_once(';').context("invalid image data URL")?; - anyhow::ensure!( - encoding.eq_ignore_ascii_case("base64"), - "image data URL must be base64" - ); - let mime = if mime == "image/jpg" { - "image/jpeg" - } else { - mime - }; - return Ok(json!({"inlineData":{"mimeType":mime,"data":data}})); - } - anyhow::ensure!( - url.starts_with("gs://"), - "native Gemini accepts data: or gs:// media URLs" - ); - let mime = image_url - .get("format") - .or_else(|| image_url.get("mime_type")) - .or_else(|| image_url.get("content_type")) - .and_then(Value::as_str) - .and_then(mime_hint) - .or_else(|| url.rsplit_once('.').and_then(|(_, ext)| mime_hint(ext))) - .context("gs:// media URL needs a recognized extension or MIME hint")?; - Ok(json!({"fileData":{"mimeType":mime,"fileUri":url}})) -} - -fn mime_hint(value: &str) -> Option<&'static str> { - Some(match value.to_ascii_lowercase().as_str() { - "image/png" | "png" => "image/png", - "image/jpeg" | "image/jpg" | "jpg" | "jpeg" => "image/jpeg", - "image/webp" | "webp" => "image/webp", - "image/gif" | "gif" => "image/gif", - "image/heic" | "heic" => "image/heic", - "image/heif" | "heif" => "image/heif", - "application/pdf" | "pdf" => "application/pdf", - "audio/mpeg" | "mp3" => "audio/mpeg", - "audio/wav" | "wav" => "audio/wav", - "video/mp4" | "mp4" => "video/mp4", - "video/quicktime" | "mov" => "video/quicktime", - "video/webm" | "webm" => "video/webm", - "text/plain" | "txt" => "text/plain", - _ => return None, - }) -} - -fn function_call_part(call: &Value) -> Value { - let function = call.get("function").unwrap_or(&Value::Null); - let args = function - .get("arguments") - .and_then(Value::as_str) - .and_then(|arguments| serde_json::from_str(arguments).ok()) - .unwrap_or_else(|| json!({})); - let mut part = json!({ - "functionCall": { - "name": function.get("name").and_then(Value::as_str).unwrap_or_default(), - "args": args - } - }); - if let Some(signature) = call - .get("id") - .and_then(Value::as_str) - .and_then(|id| split_tool_call_id(id).1) - { - part["thoughtSignature"] = json!(signature); - } - part -} - -fn split_tool_call_id(raw: &str) -> (&str, Option<&str>) { - match raw.split_once(THOUGHT_SIGNATURE_SEPARATOR) { - Some((base, signature)) if !signature.is_empty() => (base, Some(signature)), - _ => (raw, None), - } -} - -fn build_tools(object: &Map) -> Option { - let declarations = object - .get("tools")? - .as_array()? - .iter() - .filter_map(|tool| tool.get("function")) - .map(|function| { - let mut declaration = json!({ - "name": function.get("name").and_then(Value::as_str).unwrap_or_default() - }); - if let Some(description) = function.get("description").filter(|value| !value.is_null()) - { - declaration["description"] = description.clone(); - } - if let Some(parameters) = function.get("parameters") { - declaration["parameters"] = normalize_gemini_schema(parameters); - } - declaration - }) - .collect::>(); - (!declarations.is_empty()).then(|| json!([{"functionDeclarations":declarations}])) -} - -fn build_tool_config(object: &Map) -> Option { - let choice = object.get("tool_choice")?; - let (mode, allowed) = match choice { - Value::String(value) if value == "none" => ("NONE", None), - Value::String(value) if value == "required" => ("ANY", None), - Value::String(_) => ("AUTO", None), - Value::Object(_) => ( - "ANY", - choice - .get("function") - .and_then(|function| function.get("name")) - .and_then(Value::as_str), - ), - _ => return None, - }; - let mut config = json!({"mode":mode}); - if let Some(name) = allowed { - config["allowedFunctionNames"] = json!([name]); - } - Some(json!({"functionCallingConfig":config})) -} - -fn build_generation_config(object: &Map, model: &str) -> Option { - let mut config = Map::new(); - for (source, target) in [ - ("temperature", "temperature"), - ("top_p", "topP"), - ("top_k", "topK"), - ("frequency_penalty", "frequencyPenalty"), - ("presence_penalty", "presencePenalty"), - ("seed", "seed"), - ("n", "candidateCount"), - ] { - if let Some(value) = object.get(source).filter(|value| !value.is_null()) { - config.insert(target.to_string(), value.clone()); - } - } - if let Some(value) = object - .get("max_completion_tokens") - .or_else(|| object.get("max_tokens")) - .filter(|value| !value.is_null()) - { - config.insert("maxOutputTokens".into(), value.clone()); - } - if let Some(stop) = object.get("stop") { - let sequences = match stop { - Value::String(text) => json!([text]), - Value::Array(_) => stop.clone(), - _ => Value::Null, - }; - if !sequences.is_null() { - config.insert("stopSequences".into(), sequences); - } - } - if let Some(response_format) = object.get("response_format") { - match response_format.get("type").and_then(Value::as_str) { - Some("json_object") => { - config.insert("responseMimeType".into(), json!("application/json")); - } - Some("json_schema") => { - config.insert("responseMimeType".into(), json!("application/json")); - if let Some(schema) = response_format - .get("json_schema") - .and_then(|schema| schema.get("schema")) - { - config.insert("responseSchema".into(), normalize_gemini_schema(schema)); - } - } - _ => {} - } - } - if let Some(thinking) = object - .get("thinking_config") - .or_else(|| object.get("thinkingConfig")) - { - config.insert("thinkingConfig".into(), thinking.clone()); - } else if let Some(effort) = object.get("reasoning_effort").and_then(Value::as_str) { - let thinking = if model.contains("gemini-3") { - let level = match effort { - "none" => None, - "minimal" => Some("minimal"), - "low" => Some("low"), - "medium" => Some("medium"), - _ => Some("high"), - }; - level.map(|level| json!({"thinkingLevel":level,"includeThoughts":true})) - } else { - let budget = match effort { - "none" => None, - "minimal" | "low" => Some(1024), - "medium" => Some(2048), - "high" => Some(4096), - "xhigh" => Some(8192), - "max" => Some(16384), - _ => None, - }; - budget.map(|budget| json!({"thinkingBudget":budget,"includeThoughts":true})) - }; - if let Some(thinking) = thinking { - config.insert("thinkingConfig".into(), thinking); - } - } - (!config.is_empty()).then_some(Value::Object(config)) -} - -pub(crate) fn normalize_gemini_schema(schema: &Value) -> Value { - let mut output = schema.clone(); - let definitions = take_definitions(&mut output); - inline_refs(&mut output, &definitions, &mut Vec::new()); - clean_schema_node(&mut output); - output -} - -fn take_definitions(root: &mut Value) -> Map { - let mut definitions = Map::new(); - if let Value::Object(object) = root { - for key in ["$defs", "definitions"] { - if let Some(Value::Object(values)) = object.remove(key) { - definitions.extend(values); - } - } - } - definitions -} - -fn for_each_child_schema(object: &mut Map, mut visit: impl FnMut(&mut Value)) { - if let Some(items) = object.get_mut("items") { - visit(items); - } - if let Some(Value::Object(properties)) = object.get_mut("properties") { - properties.values_mut().for_each(&mut visit); - } - for key in ["anyOf", "allOf"] { - if let Some(Value::Array(values)) = object.get_mut(key) { - values.iter_mut().for_each(&mut visit); - } - } -} - -fn inline_refs(value: &mut Value, definitions: &Map, chain: &mut Vec) { - let reference_name = value.get("$ref").and_then(Value::as_str).map(|reference| { - reference - .rsplit('/') - .next() - .unwrap_or(reference) - .to_string() - }); - if let Some(name) = reference_name { - if chain.contains(&name) { - return; - } - if let Some(definition) = definitions.get(&name) { - let mut resolved = definition.clone(); - if let (Value::Object(resolved), Value::Object(original)) = (&mut resolved, &*value) { - for (key, child) in original { - if key != "$ref" { - resolved.insert(key.clone(), child.clone()); - } - } - } - chain.push(name); - inline_refs(&mut resolved, definitions, chain); - chain.pop(); - *value = resolved; - } - return; - } - match value { - Value::Object(object) => { - object.remove("$defs"); - object.remove("definitions"); - for_each_child_schema(object, |child| inline_refs(child, definitions, chain)); - } - Value::Array(array) => array - .iter_mut() - .for_each(|child| inline_refs(child, definitions, chain)), - _ => {} - } -} - -fn rewrite_schema_structure(object: &mut Map) -> bool { - let mut changed = false; - - if let Some(Value::Array(members)) = object.remove("allOf") { - changed = true; - for member in members { - let Value::Object(member) = member else { - continue; - }; - for (key, value) in member { - if key == "properties" - && let Value::Object(ref source) = value - && let Value::Object(target) = - object.entry("properties").or_insert_with(|| json!({})) - { - for (name, schema) in source { - target.entry(name).or_insert(schema.clone()); - } - } else if key == "required" - && let Value::Array(ref source) = value - && let Value::Array(target) = - object.entry("required").or_insert_with(|| json!([])) - { - for required in source { - if !target.contains(required) { - target.push(required.clone()); - } - } - } else { - object.entry(key).or_insert(value); - } - } - } - } - - if let Some(constant) = object.remove("const") { - changed = true; - let schema_type = schema_type_for_constant(&constant); - object.insert("enum".into(), Value::Array(vec![constant])); - object.entry("type").or_insert_with(|| json!(schema_type)); - } - - if let Some(Value::Array(types)) = object.get("type").cloned() { - changed = true; - let names = types - .iter() - .filter_map(Value::as_str) - .map(str::to_string) - .collect::>(); - let nullable = names.iter().any(|name| name == "null"); - let non_null = names - .into_iter() - .filter(|name| name != "null") - .collect::>(); - object.remove("type"); - if nullable { - object.insert("nullable".into(), Value::Bool(true)); - } - match non_null.as_slice() { - [] => {} - [only] => { - object.insert("type".into(), json!(only)); - } - many => { - object.insert( - "anyOf".into(), - Value::Array(many.iter().map(|kind| json!({"type":kind})).collect()), - ); - } - } - } - - let collapsible_any_of = object - .get("anyOf") - .and_then(Value::as_array) - .is_some_and(|members| { - members.len() == 1 - || members - .iter() - .any(|member| member.get("type").and_then(Value::as_str) == Some("null")) - }); - if collapsible_any_of && let Some(Value::Array(members)) = object.remove("anyOf") { - changed = true; - let nullable = members - .iter() - .any(|member| member.get("type").and_then(Value::as_str) == Some("null")); - let non_null = members - .into_iter() - .filter(|member| member.get("type").and_then(Value::as_str) != Some("null")) - .collect::>(); - if nullable { - object.insert("nullable".into(), Value::Bool(true)); - } - match non_null.len() { - 0 => {} - 1 => { - if let Some(Value::Object(member)) = non_null.into_iter().next() { - for (key, value) in member { - object.entry(key).or_insert(value); - } - } - } - _ => { - object.insert("anyOf".into(), Value::Array(non_null)); - } - } - } - changed -} - -fn schema_type_for_constant(value: &Value) -> &'static str { - match value { - Value::String(_) => "string", - Value::Bool(_) => "boolean", - Value::Number(number) if number.is_f64() => "number", - Value::Number(_) => "integer", - Value::Array(_) => "array", - _ => "object", - } -} - -fn clean_schema_node(value: &mut Value) { - const ALLOWED: &[&str] = &[ - "type", - "format", - "description", - "title", - "nullable", - "enum", - "items", - "properties", - "required", - "anyOf", - "default", - "minLength", - "maxLength", - "pattern", - "minimum", - "maximum", - "exclusiveMinimum", - "exclusiveMaximum", - "propertyOrdering", - ]; - let object = match value { - Value::Object(object) => object, - Value::Array(values) => { - values.iter_mut().for_each(clean_schema_node); - return; - } - _ => return, - }; - - let mut guard = 0; - while rewrite_schema_structure(object) && guard < 64 { - guard += 1; - } - - if object.get("type").and_then(Value::as_str) == Some("array") && !object.contains_key("items") - { - object.insert("items".into(), json!({"type":"object"})); - } - if object.contains_key("enum") { - match object.get("type").and_then(Value::as_str) { - Some("string") => {} - Some(_) => { - object.remove("enum"); - } - None => { - object.insert("type".into(), json!("string")); - } - } - } - object.remove("additionalProperties"); - if !object.contains_key("type") && !object.contains_key("anyOf") && !object.contains_key("enum") - { - object.insert("type".into(), json!("object")); - } - if object - .get("format") - .and_then(Value::as_str) - .is_some_and(|format| format != "enum" && format != "date-time") - { - object.remove("format"); - } - for_each_child_schema(object, clean_schema_node); - object.retain(|key, _| ALLOWED.contains(&key.as_str())); -} - -pub fn gemini_response_to_completions(body: &Bytes, fallback_model: &str) -> Result { - let native: Value = serde_json::from_slice(body).context("parse Gemini response")?; - Ok(Bytes::from(serde_json::to_vec( - &gemini_value_to_completions(&native, fallback_model)?, - )?)) -} - -pub fn gemini_value_to_completions(native: &Value, fallback_model: &str) -> Result { - let model = native - .get("modelVersion") - .and_then(Value::as_str) - .unwrap_or(fallback_model); - let id = native - .get("responseId") - .and_then(Value::as_str) - .map(str::to_string) - .unwrap_or_else(|| format!("gemini-{}", chrono::Utc::now().timestamp_millis())); - let candidates = native.get("candidates").and_then(Value::as_array); - let choices = if candidates.is_none_or(Vec::is_empty) { - let blocked = native - .get("promptFeedback") - .and_then(|feedback| feedback.get("blockReason")) - .is_some(); - vec![json!({ - "message":{"content":"","role":"assistant"}, - "index":0, - "finish_reason": if blocked {"content_filter"} else {"stop"} - })] - } else { - candidates - .expect("checked") - .iter() - .enumerate() - .map(|(index, candidate)| build_choice(candidate, index, &id)) - .collect() - }; - let mut response = json!({ - "model": model, - "choices": choices, - "id": id, - "created": chrono::Utc::now().timestamp(), - "object": "chat.completion" - }); - if let Some(usage) = native.get("usageMetadata") { - response["usage"] = gemini_usage(usage); - } - Ok(response) -} - -fn build_choice(candidate: &Value, index: usize, response_id: &str) -> Value { - let decoded = decode_parts( - candidate - .get("content") - .and_then(|content| content.get("parts")) - .and_then(Value::as_array), - ); - let mut message = json!({"role":"assistant"}); - if !decoded.text.is_empty() || (decoded.tool_calls.is_empty() && decoded.reasoning.is_empty()) { - message["content"] = json!(decoded.text); - } - if !decoded.reasoning.is_empty() { - message["reasoning_content"] = json!(decoded.reasoning); - } - if !decoded.tool_calls.is_empty() { - message["tool_calls"] = json!( - decoded - .tool_calls - .iter() - .enumerate() - .map(|(tool_index, call)| tool_call_json(call, response_id, tool_index)) - .collect::>() - ); - } - let finish = finish_reason( - candidate.get("finishReason").and_then(Value::as_str), - !decoded.tool_calls.is_empty(), - ); - json!({"message":message,"index":index,"finish_reason":finish}) -} - -#[derive(Default)] -struct DecodedParts { - text: String, - reasoning: String, - tool_calls: Vec, -} - -struct NativeToolCall { - id: Option, - name: String, - args: Value, - signature: Option, -} - -fn decode_parts(parts: Option<&Vec>) -> DecodedParts { - let mut decoded = DecodedParts::default(); - for part in parts.into_iter().flatten() { - if let Some(text) = part.get("text").and_then(Value::as_str) { - if part - .get("thought") - .and_then(Value::as_bool) - .unwrap_or(false) - { - decoded.reasoning.push_str(text); - } else { - decoded.text.push_str(text); - } - } - if let Some(call) = part.get("functionCall") { - decoded.tool_calls.push(NativeToolCall { - id: call.get("id").and_then(Value::as_str).map(str::to_string), - name: call - .get("name") - .and_then(Value::as_str) - .unwrap_or_default() - .to_string(), - args: call.get("args").cloned().unwrap_or_else(|| json!({})), - signature: part - .get("thoughtSignature") - .and_then(Value::as_str) - .map(str::to_string), - }); - } - } - decoded -} - -fn tool_call_json(call: &NativeToolCall, response_id: &str, index: usize) -> Value { - let base = call - .id - .clone() - .unwrap_or_else(|| format!("call_{response_id}_{index}")); - let id = call - .signature - .as_deref() - .map(|signature| format!("{base}{THOUGHT_SIGNATURE_SEPARATOR}{signature}")) - .unwrap_or(base); - json!({ - "type":"function", - "id":id, - "function":{ - "name":call.name, - "arguments":serde_json::to_string(&call.args).unwrap_or_else(|_| "{}".into()) - } - }) -} - -fn finish_reason(reason: Option<&str>, has_tools: bool) -> &'static str { - match reason { - Some("MAX_TOKENS") => "length", - Some( - "SAFETY" - | "RECITATION" - | "LANGUAGE" - | "BLOCKLIST" - | "PROHIBITED_CONTENT" - | "SPII" - | "UNEXPECTED_TOOL_CALL" - | "TOO_MANY_TOOL_CALLS" - | "IMAGE_SAFETY" - | "IMAGE_PROHIBITED_CONTENT" - | "IMAGE_RECITATION", - ) => "content_filter", - _ if has_tools => "tool_calls", - _ => "stop", - } -} - -fn gemini_usage(usage: &Value) -> Value { - let input = usage - .get("promptTokenCount") - .and_then(Value::as_u64) - .unwrap_or(0); - let output = usage - .get("candidatesTokenCount") - .and_then(Value::as_u64) - .unwrap_or(0); - let mut result = json!({ - "prompt_tokens":input, - "completion_tokens":output, - "total_tokens":usage.get("totalTokenCount").and_then(Value::as_u64).unwrap_or(input + output) - }); - if let Some(cached) = usage.get("cachedContentTokenCount").and_then(Value::as_u64) { - result["prompt_tokens_details"] = json!({"cached_tokens":cached}); - } - if let Some(reasoning) = usage.get("thoughtsTokenCount").and_then(Value::as_u64) { - result["completion_tokens_details"] = json!({"reasoning_tokens":reasoning}); - } - result -} - -pub fn translate_gemini_error(body: &Bytes) -> Result { - let value: Value = serde_json::from_slice(body).context("parse Gemini error")?; - let root = value - .as_array() - .and_then(|array| array.first()) - .unwrap_or(&value); - let error = root.get("error").unwrap_or(root); - let status = error.get("status").and_then(Value::as_str); - let error_type = match status { - Some("INVALID_ARGUMENT" | "FAILED_PRECONDITION") => "invalid_request_error", - Some("NOT_FOUND") => "not_found_error", - Some("PERMISSION_DENIED" | "UNAUTHENTICATED") => "authentication_error", - Some("RESOURCE_EXHAUSTED") => "rate_limit_error", - _ => "api_error", - }; - Ok(Bytes::from(serde_json::to_vec(&json!({ - "error":{ - "type":error_type, - "message":error.get("message").and_then(Value::as_str).unwrap_or("Gemini request failed"), - "param":null, - "code":status - } - }))?)) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn parallel_same_name_tool_results_restore_call_order() { - let body = Bytes::from_static( - br#"{ - "model":"gemini-2.5-pro", - "messages":[ - {"role":"user","content":"compare"}, - {"role":"assistant","tool_calls":[ - {"id":"call_a","type":"function","function":{"name":"lookup","arguments":"{\"id\":\"a\"}"}}, - {"id":"call_b","type":"function","function":{"name":"lookup","arguments":"{\"id\":\"b\"}"}} - ]}, - {"role":"tool","tool_call_id":"call_b","content":"result-b"}, - {"role":"tool","tool_call_id":"call_a","content":"result-a"} - ] - }"#, - ); - let translated = completions_request_to_gemini(&body, "gemini-2.5-pro").unwrap(); - let value: Value = serde_json::from_slice(&translated).unwrap(); - let responses = value["contents"][2]["parts"].as_array().unwrap(); - assert_eq!( - responses[0]["functionResponse"]["response"]["content"], - "result-a" - ); - assert_eq!( - responses[1]["functionResponse"]["response"]["content"], - "result-b" - ); - assert!( - responses - .iter() - .all(|response| response["functionResponse"].get("id").is_none()) - ); - } - - #[test] - fn schema_normalization_flattens_and_makes_nullable() { - let schema = json!({ - "allOf":[ - {"type":"object","properties":{"name":{"type":"string"}},"required":["name"]}, - {"properties":{"age":{"type":["integer","null"]}},"required":["age"]} - ], - "additionalProperties":false - }); - let normalized = normalize_gemini_schema(&schema); - assert_eq!(normalized["type"], "object"); - assert_eq!(normalized["properties"]["name"]["type"], "string"); - assert_eq!(normalized["properties"]["age"]["type"], "integer"); - assert_eq!(normalized["properties"]["age"]["nullable"], true); - assert_eq!(normalized["required"], json!(["name", "age"])); - assert!(normalized.get("allOf").is_none()); - assert!(normalized.get("additionalProperties").is_none()); - } -} diff --git a/crates/persisting-gateway/src/conversion/messages_completions.rs b/crates/persisting-gateway/src/conversion/messages_completions.rs deleted file mode 100644 index c19a6f6b9..000000000 --- a/crates/persisting-gateway/src/conversion/messages_completions.rs +++ /dev/null @@ -1,440 +0,0 @@ -//! Anthropic Messages ↔ OpenAI Chat Completions. -//! -//! The wire mapping follows agentgateway's `agent-llm` conversion contract. Keep -//! this module lossless for agent-critical semantics (tools, tool results and -//! multimodal content); unsupported server-side tools are the only intentional -//! omission because Chat Completions has no equivalent execution model. - -use anyhow::{Context, Result}; -use bytes::Bytes; -use serde_json::{Value, json}; - -/// Convert Anthropic `/v1/messages` JSON body to OpenAI `/v1/chat/completions`. -pub fn messages_request_to_completions(body: &Bytes, upstream_model: &str) -> Result { - let v: Value = serde_json::from_slice(body).context("parse messages request")?; - let obj = v - .as_object() - .context("messages request must be a JSON object")?; - let supports_prompt_cache = supports_prompt_cache_breakpoint(upstream_model); - let mut out_messages = Vec::new(); - if let Some(system) = obj.get("system") { - append_system_message(system, supports_prompt_cache, &mut out_messages); - } - - if let Some(msgs) = obj.get("messages").and_then(Value::as_array) { - for msg in msgs { - let role = msg.get("role").and_then(|r| r.as_str()).unwrap_or("user"); - append_message( - role, - msg.get("content").unwrap_or(&Value::Null), - supports_prompt_cache, - &mut out_messages, - ); - } - } - - let stream = obj.get("stream").and_then(Value::as_bool).unwrap_or(false); - let mut out = json!({ - "messages": out_messages, - "model": upstream_model, - "max_completion_tokens": obj.get("max_tokens").cloned().unwrap_or(json!(1024)), - "stream": stream, - }); - - copy_non_null(obj, &mut out, "temperature", "temperature"); - copy_non_null(obj, &mut out, "top_p", "top_p"); - if let Some(stop) = obj.get("stop_sequences").filter(|v| !is_empty_array(v)) { - out["stop"] = stop.clone(); - } - if stream { - out["stream_options"] = json!({"include_usage": true}); - } - let mut has_function_tools = false; - if let Some(tools) = obj.get("tools").and_then(Value::as_array) { - let converted = tools.iter().filter_map(convert_tool).collect::>(); - if !converted.is_empty() { - has_function_tools = true; - out["tools"] = json!(converted); - } - } - if let Some(choice) = obj.get("tool_choice") { - let (tool_choice, parallel) = convert_tool_choice(choice); - if let Some(tool_choice) = tool_choice { - out["tool_choice"] = tool_choice; - } - if let Some(parallel) = parallel { - out["parallel_tool_calls"] = json!(parallel); - } - } - if let Some(effort) = reasoning_effort(obj, upstream_model, has_function_tools) { - out["reasoning_effort"] = json!(effort); - } - if let Some(format) = response_format(obj) { - out["response_format"] = format; - } - if let Some(user_id) = obj - .get("metadata") - .and_then(|v| v.get("user_id")) - .filter(|v| !v.is_null()) - { - out["user"] = user_id.clone(); - } - - Ok(Bytes::from( - serde_json::to_vec(&out).context("serialize completions request")?, - )) -} - -/// Convert OpenAI chat completion JSON to Anthropic messages response shape. -pub fn completions_response_to_messages(body: &Bytes, client_model: &str) -> Result { - let v: Value = serde_json::from_slice(body).context("parse completions response")?; - Ok(Bytes::from( - serde_json::to_vec(&completions_value_to_messages(&v, client_model)?) - .context("serialize messages response")?, - )) -} - -pub fn completions_value_to_messages(body: &Value, client_model: &str) -> Result { - let choice = body - .get("choices") - .and_then(|c| c.as_array()) - .and_then(|a| a.first()) - .context("completions response missing choices[0]")?; - let message = choice.get("message").context("choice missing message")?; - let text = message.get("content").and_then(Value::as_str).unwrap_or(""); - let finish = choice - .get("finish_reason") - .and_then(|f| f.as_str()) - .unwrap_or("stop"); - let id = body - .get("id") - .and_then(|i| i.as_str()) - .unwrap_or("msg_proxy"); - let mut content = Vec::new(); - if !text.is_empty() { - content.push(json!({"type": "text", "text": text})); - } - if let Some(tool_calls) = message.get("tool_calls").and_then(Value::as_array) { - for tool_call in tool_calls { - let Some(function) = tool_call.get("function") else { - continue; - }; - let input = function - .get("arguments") - .and_then(Value::as_str) - .map(|arguments| { - serde_json::from_str::(arguments).unwrap_or_else(|_| json!(arguments)) - }) - .unwrap_or_else(|| json!({})); - content.push(json!({ - "type": "tool_use", - "id": tool_call.get("id").cloned().unwrap_or(json!("call_proxy")), - "name": function.get("name").cloned().unwrap_or(json!("")), - "input": input, - })); - } - } - let usage = openai_usage_to_anthropic(body); - - Ok(json!({ - "id": id, - "type": "message", - "role": "assistant", - "model": client_model, - "content": content, - "stop_reason": openai_finish_to_anthropic(finish), - "stop_sequence": null, - "usage": usage, - })) -} - -fn append_system_message(system: &Value, prompt_cache: bool, messages: &mut Vec) { - let content = match system { - Value::String(text) if !text.is_empty() => json!(text), - Value::Array(blocks) => json!( - blocks - .iter() - .filter_map(|block| text_part(block, prompt_cache)) - .collect::>() - ), - _ => return, - }; - if !is_empty_array(&content) { - messages.push(json!({"role": "system", "content": content})); - } -} - -fn append_message(role: &str, content: &Value, prompt_cache: bool, messages: &mut Vec) { - match content { - Value::String(text) => messages.push(json!({ - "role": role, - "content": [{"type": "text", "text": text}], - })), - Value::Array(blocks) => { - if role == "assistant" { - append_assistant_blocks(blocks, prompt_cache, messages); - return; - } - let mut parts = Vec::new(); - for block in blocks { - if block.get("type").and_then(Value::as_str) == Some("tool_result") { - flush_content_message(role, &mut parts, messages); - messages.push(json!({ - "role": "tool", - "tool_call_id": block.get("tool_use_id").cloned().unwrap_or(json!("")), - "content": tool_result_content(block, prompt_cache), - })); - continue; - } - match block.get("type").and_then(Value::as_str) { - Some("text") => { - if let Some(part) = text_part(block, prompt_cache) { - parts.push(part); - } - } - Some("image") => image_part(block).into_iter().for_each(|p| parts.push(p)), - _ => {} - } - } - flush_content_message(role, &mut parts, messages); - } - _ => {} - } -} - -fn append_assistant_blocks(blocks: &[Value], prompt_cache: bool, messages: &mut Vec) { - let mut content = Vec::new(); - let mut tool_calls = Vec::new(); - for block in blocks { - match block.get("type").and_then(Value::as_str) { - Some("text") => text_part(block, prompt_cache) - .into_iter() - .for_each(|p| content.push(p)), - Some("tool_use") => tool_calls.push(json!({ - "id": block.get("id").cloned().unwrap_or(json!("call_proxy")), - "type": "function", - "function": { - "name": block.get("name").cloned().unwrap_or(json!("")), - "arguments": serde_json::to_string(block.get("input").unwrap_or(&json!({}))) - .unwrap_or_else(|_| "{}".to_string()), - } - })), - _ => {} - } - } - if content.is_empty() && tool_calls.is_empty() { - return; - } - let mut message = json!({"role": "assistant"}); - if !content.is_empty() { - message["content"] = json!(content); - } - if !tool_calls.is_empty() { - message["tool_calls"] = json!(tool_calls); - } - messages.push(message); -} - -fn flush_content_message(role: &str, parts: &mut Vec, messages: &mut Vec) { - if !parts.is_empty() { - messages.push(json!({"role": role, "content": std::mem::take(parts)})); - } -} - -fn text_part(block: &Value, prompt_cache: bool) -> Option { - let text = block.get("text").and_then(Value::as_str)?; - let mut part = json!({"type": "text", "text": text}); - if prompt_cache && block.get("cache_control").is_some_and(Value::is_object) { - part["prompt_cache_breakpoint"] = json!({"mode": "explicit"}); - } - Some(part) -} - -fn image_part(block: &Value) -> Option { - let source = block.get("source")?; - let url = match source.get("type").and_then(Value::as_str)? { - "base64" => format!( - "data:{};base64,{}", - source.get("media_type")?.as_str()?, - source.get("data")?.as_str()? - ), - "url" => source.get("url")?.as_str()?.to_string(), - _ => return None, - }; - Some(json!({"type": "image_url", "image_url": {"url": url}})) -} - -fn tool_result_content(block: &Value, prompt_cache: bool) -> Value { - let outer_cache = prompt_cache && block.get("cache_control").is_some_and(Value::is_object); - match block.get("content") { - Some(Value::String(text)) if outer_cache => json!([{ - "type": "text", - "text": text, - "prompt_cache_breakpoint": {"mode": "explicit"}, - }]), - Some(Value::String(text)) => json!(text), - Some(Value::Array(parts)) => { - let mut converted = Vec::new(); - let mut trailing_cache = outer_cache; - for part in parts { - if part.get("type").and_then(Value::as_str) == Some("text") { - if let Some(text) = text_part(part, prompt_cache) { - converted.push(text); - } - } else if prompt_cache && part.get("cache_control").is_some_and(Value::is_object) { - trailing_cache = true; - } - } - if trailing_cache { - if let Some(last) = converted.last_mut() { - last["prompt_cache_breakpoint"] = json!({"mode": "explicit"}); - } else { - converted.push(json!({ - "type": "text", - "text": "", - "prompt_cache_breakpoint": {"mode": "explicit"}, - })); - } - } - json!(converted) - } - Some(other) => json!(other.to_string()), - None => json!(""), - } -} - -fn supports_prompt_cache_breakpoint(model: &str) -> bool { - model - .strip_prefix("gpt-") - .and_then(|model| model.split('-').next()) - .and_then(|version| version.split_once('.')) - .and_then(|(major, minor)| Some((major.parse::().ok()?, minor.parse::().ok()?))) - .is_some_and(|version| version >= (5, 6)) -} - -fn convert_tool(tool: &Value) -> Option { - let name = tool.get("name")?; - let schema = tool.get("input_schema")?; - Some(json!({ - "type": "function", - "function": { - "name": name, - "description": tool.get("description").cloned().unwrap_or(Value::Null), - "parameters": schema, - } - })) -} - -fn convert_tool_choice(choice: &Value) -> (Option, Option) { - let parallel = choice - .get("disable_parallel_tool_use") - .and_then(Value::as_bool) - .map(|disabled| !disabled); - let converted = match choice.get("type").and_then(Value::as_str) { - Some("auto") => Some(json!("auto")), - Some("any") => Some(json!("required")), - Some("none") => Some(json!("none")), - Some("tool") => Some(json!({ - "type": "function", - "function": {"name": choice.get("name").cloned().unwrap_or(json!(""))}, - })), - _ => None, - }; - (converted, parallel) -} - -fn reasoning_effort<'a>( - obj: &'a serde_json::Map, - model: &str, - has_function_tools: bool, -) -> Option<&'a str> { - let adaptive = obj - .get("thinking") - .and_then(|v| v.get("type")) - .and_then(Value::as_str) - == Some("adaptive"); - adaptive.then(|| { - if has_function_tools && !supports_reasoning_with_tools(model) { - return "none"; - } - obj.get("output_config") - .and_then(|v| v.get("effort")) - .and_then(Value::as_str) - .unwrap_or("high") - }) -} - -fn supports_reasoning_with_tools(model: &str) -> bool { - !model.starts_with("gpt-") - || model == "gpt-5" - || model.starts_with("gpt-5-") - || model.starts_with("gpt-5.1") - || model.starts_with("gpt-5.2") -} - -fn response_format(obj: &serde_json::Map) -> Option { - let format = obj.get("output_config")?.get("format")?; - let schema = format.get("schema")?; - Some(json!({ - "type": "json_schema", - "json_schema": {"name": "structured_output", "schema": schema}, - })) -} - -fn copy_non_null( - source: &serde_json::Map, - target: &mut Value, - source_key: &str, - target_key: &str, -) { - if let Some(value) = source.get(source_key).filter(|v| !v.is_null()) { - target[target_key] = value.clone(); - } -} - -fn is_empty_array(value: &Value) -> bool { - value.as_array().is_some_and(Vec::is_empty) -} - -fn openai_finish_to_anthropic(finish: &str) -> &'static str { - match finish { - "length" => "max_tokens", - "tool_calls" => "tool_use", - "content_filter" => "refusal", - _ => "end_turn", - } -} - -fn openai_usage_to_anthropic(body: &Value) -> Value { - let usage = body.get("usage"); - let prompt_tokens = usage - .and_then(|u| u.get("prompt_tokens")) - .and_then(Value::as_u64) - .unwrap_or(0); - let prompt_details = usage.and_then(|u| u.get("prompt_tokens_details")); - let cache_read_tokens = prompt_details - .and_then(|details| details.get("cached_tokens")) - .and_then(Value::as_u64); - let cache_creation_tokens = prompt_details - .and_then(|details| details.get("cache_write_tokens")) - .and_then(Value::as_u64); - let mut out = json!({ - "input_tokens": prompt_tokens - .saturating_sub(cache_read_tokens.unwrap_or(0)) - .saturating_sub(cache_creation_tokens.unwrap_or(0)), - "output_tokens": usage - .and_then(|u| u.get("completion_tokens")) - .and_then(Value::as_u64) - .unwrap_or(0), - }); - if let Some(service_tier) = body.get("service_tier").filter(|v| !v.is_null()) { - out["service_tier"] = service_tier.clone(); - } - if let Some(tokens) = cache_creation_tokens { - out["cache_creation_input_tokens"] = json!(tokens); - } - if let Some(tokens) = cache_read_tokens { - out["cache_read_input_tokens"] = json!(tokens); - } - out -} diff --git a/crates/persisting-gateway/src/conversion/mod.rs b/crates/persisting-gateway/src/conversion/mod.rs deleted file mode 100644 index f280919b7..000000000 --- a/crates/persisting-gateway/src/conversion/mod.rs +++ /dev/null @@ -1,560 +0,0 @@ -//! LLM protocol conversion around Chat Completions and Gemini native generateContent. - -mod gemini_native; -mod messages_completions; -mod responses_completions; -mod semantic; -mod tool_call; -mod typed_stream; - -/// Maximum accepted JSON request body. Accepted bodies are retained verbatim -/// for replay, so the capture and conversion paths share the same bound. -pub const MAX_REQUEST_BODY_BYTES: usize = 16 * 1024 * 1024; -/// Maximum buffered non-streaming provider response. -pub const MAX_RESPONSE_BODY_BYTES: usize = 32 * 1024 * 1024; -/// Maximum single incomplete SSE frame retained by a translator. -pub const MAX_SSE_FRAME_BYTES: usize = 2 * 1024 * 1024; -/// Maximum complete streaming response retained for durable raw capture. -pub const MAX_STREAM_CAPTURE_BYTES: usize = 64 * 1024 * 1024; - -pub use gemini_native::{completions_request_to_gemini, gemini_response_to_completions}; -pub use messages_completions::{completions_response_to_messages, messages_request_to_completions}; -pub use responses_completions::{ - completions_response_to_responses, responses_request_to_completions, -}; -pub use tool_call::{decode_stream_arguments_delta, unquote_chat_tool_arguments}; -pub use typed_stream::TypedStreamTranslator as StreamTranslator; - -use crate::config::ModelRoute; -use crate::protocol::ProtocolKind; -use crate::provider::ProviderKind; -use bytes::Bytes; - -/// Whether the proxy must translate request/response bodies between protocols. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum ProtocolBridge { - /// Same wire format end-to-end. - Passthrough, - /// Client `/v1/messages` → upstream `/v1/chat/completions`. - MessagesToCompletions, - /// Client `/v1/responses` → upstream `/v1/chat/completions` (Codex + DeepSeek). - ResponsesToCompletions, - /// Client `/v1/chat/completions` → Gemini native `generateContent`. - CompletionsToGemini, - /// Client `/v1/messages` → Chat Completions → Gemini native `generateContent`. - MessagesToGemini, - /// Client `/v1/responses` → Chat Completions → Gemini native `generateContent`. - ResponsesToGemini, -} - -impl ProtocolBridge { - pub fn needed(client: ProtocolKind, route: &ModelRoute) -> Self { - if route.provider_kind() == ProviderKind::Gemini { - return match client { - ProtocolKind::ChatCompletions => Self::CompletionsToGemini, - ProtocolKind::Messages => Self::MessagesToGemini, - ProtocolKind::Responses => Self::ResponsesToGemini, - _ => Self::Passthrough, - }; - } - match client { - ProtocolKind::Messages if route.upstream_anthropic.is_none() => { - Self::MessagesToCompletions - } - ProtocolKind::Responses if !route_supports_native_responses(route) => { - Self::ResponsesToCompletions - } - _ => Self::Passthrough, - } - } - - pub fn needs_request_translation(self) -> bool { - !matches!(self, Self::Passthrough) - } - - pub fn needs_response_translation(self) -> bool { - !matches!(self, Self::Passthrough) - } - - pub fn upstream_path( - self, - client_path: &str, - model: &str, - streaming: bool, - ) -> anyhow::Result { - match self { - Self::Passthrough => Ok(client_path.to_string()), - Self::MessagesToCompletions => { - if client_path.contains("/v1/") { - Ok(client_path.replacen("/messages", "/chat/completions", 1)) - } else { - Ok("/v1/chat/completions".to_string()) - } - } - Self::ResponsesToCompletions => { - if client_path.contains("/v1/") { - Ok(client_path.replacen("/responses", "/chat/completions", 1)) - } else { - Ok("/v1/chat/completions".to_string()) - } - } - Self::CompletionsToGemini | Self::MessagesToGemini | Self::ResponsesToGemini => { - let model = model.strip_prefix("models/").unwrap_or(model); - anyhow::ensure!( - !model.is_empty() - && model - .bytes() - .all(|byte| byte.is_ascii_alphanumeric() || b"-._".contains(&byte)), - "invalid Gemini model id `{model}`" - ); - let method = if streaming { - "streamGenerateContent" - } else { - "generateContent" - }; - Ok(format!("/v1beta/models/{model}:{method}")) - } - } - } - - pub fn upstream_protocol(self, client: ProtocolKind) -> ProtocolKind { - match self { - Self::Passthrough => client, - Self::MessagesToCompletions | Self::ResponsesToCompletions => { - ProtocolKind::ChatCompletions - } - Self::CompletionsToGemini | Self::MessagesToGemini | Self::ResponsesToGemini => { - ProtocolKind::Gemini - } - } - } -} - -/// Translate client request body to upstream wire format for [`ProtocolBridge`]. -pub fn translate_request_for_bridge( - bridge: ProtocolBridge, - semantic: &persisting_pchronicle::model::LlmRequestEventPayload, - upstream_model: &str, - reasoning_cache: Option<&crate::gateway::ReasoningCacheHandle>, -) -> anyhow::Result { - match bridge { - ProtocolBridge::Passthrough => anyhow::bail!("passthrough requests have no renderer"), - ProtocolBridge::MessagesToCompletions | ProtocolBridge::ResponsesToCompletions => { - semantic::request_to_chat_completions(semantic, upstream_model, reasoning_cache) - } - ProtocolBridge::CompletionsToGemini - | ProtocolBridge::MessagesToGemini - | ProtocolBridge::ResponsesToGemini => { - semantic::request_to_gemini(semantic, upstream_model) - } - } -} - -/// A provider response after its single parse into Chronicle semantics. -pub struct TranslatedResponse { - pub body: Bytes, - pub semantic: std::sync::Arc, -} - -/// Parse an upstream response once and render it to the client wire protocol. -pub fn translate_response_for_bridge( - bridge: ProtocolBridge, - body: &Bytes, - client_protocol: ProtocolKind, - client_model: &str, -) -> anyhow::Result { - let upstream_protocol = bridge.upstream_protocol(client_protocol); - let value: serde_json::Value = serde_json::from_slice(body)?; - let semantic = crate::understanding::understand_response_value(upstream_protocol, &value)?; - let rendered = if bridge == ProtocolBridge::Passthrough { - body.clone() - } else { - semantic::response_to_wire(&semantic, client_protocol.into(), client_model)? - }; - Ok(TranslatedResponse { - body: rendered, - semantic: std::sync::Arc::new(semantic), - }) -} - -/// Translate an upstream error without running it through a success-body parser. -pub fn translate_error_for_bridge( - bridge: ProtocolBridge, - body: &Bytes, - status: axum::http::StatusCode, -) -> anyhow::Result { - match bridge { - ProtocolBridge::MessagesToCompletions => openai_error_to_messages(body, status), - ProtocolBridge::MessagesToGemini => { - let openai = gemini_native::translate_gemini_error(body)?; - openai_error_to_messages(&openai, status) - } - ProtocolBridge::CompletionsToGemini | ProtocolBridge::ResponsesToGemini => { - gemini_native::translate_gemini_error(body) - } - // Responses and Chat Completions use the same OpenAI-compatible error envelope. - ProtocolBridge::ResponsesToCompletions | ProtocolBridge::Passthrough => Ok(body.clone()), - } -} - -fn openai_error_to_messages(body: &Bytes, status: axum::http::StatusCode) -> anyhow::Result { - let value: serde_json::Value = serde_json::from_slice(body).unwrap_or_else( - |_| serde_json::json!({"error": {"message": String::from_utf8_lossy(body)}}), - ); - let error = value.get("error").unwrap_or(&value); - let message = error - .get("message") - .and_then(serde_json::Value::as_str) - .unwrap_or("upstream request failed"); - let error_type = error - .get("type") - .and_then(serde_json::Value::as_str) - .unwrap_or_else(|| normalized_anthropic_error_type(status)); - Ok(Bytes::from(serde_json::to_vec(&serde_json::json!({ - "type": "error", - "error": {"type": error_type, "message": message}, - }))?)) -} - -fn normalized_anthropic_error_type(status: axum::http::StatusCode) -> &'static str { - match status { - axum::http::StatusCode::BAD_REQUEST => "invalid_request_error", - axum::http::StatusCode::UNAUTHORIZED | axum::http::StatusCode::FORBIDDEN => { - "authentication_error" - } - axum::http::StatusCode::NOT_FOUND => "not_found_error", - axum::http::StatusCode::TOO_MANY_REQUESTS => "rate_limit_error", - _ => "api_error", - } -} - -fn route_supports_native_responses(route: &ModelRoute) -> bool { - route.upstream.as_deref().is_some_and(|u| { - let lower = u.to_ascii_lowercase(); - lower.contains("api.openai.com") || lower.contains("openai.azure.com") - }) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::config::ProxyConfig; - use proptest::prelude::*; - - fn text_strategy() -> impl Strategy { - "[a-zA-Z0-9][a-zA-Z0-9 .,!?_-]{0,63}" - } - - #[test] - fn response_bridge_parses_once_then_renders_messages() { - let upstream = Bytes::from_static( - br#"{"id":"chat-1","model":"upstream","choices":[{"index":0,"message":{"role":"assistant","content":"hello"},"finish_reason":"stop"}],"usage":{"prompt_tokens":2,"completion_tokens":1,"total_tokens":3}}"#, - ); - let translated = translate_response_for_bridge( - ProtocolBridge::MessagesToCompletions, - &upstream, - ProtocolKind::Messages, - "client-model", - ) - .unwrap(); - let wire: serde_json::Value = serde_json::from_slice(&translated.body).unwrap(); - assert_eq!(wire["model"], "client-model"); - assert_eq!(wire["content"][0]["text"], "hello"); - assert_eq!( - translated.semantic.output_format, - persisting_pchronicle::model::LlmProtocol::ChatCompletions - ); - assert_eq!( - translated - .semantic - .response - .usage - .as_ref() - .unwrap() - .total_tokens, - 3 - ); - } - - #[test] - fn gemini_response_renders_responses_without_chat_intermediate() { - let upstream = Bytes::from_static( - br#"{"responseId":"g-1","modelVersion":"gemini-upstream","candidates":[{"index":0,"content":{"role":"model","parts":[{"text":"hello"}]},"finishReason":"STOP"}]}"#, - ); - let translated = translate_response_for_bridge( - ProtocolBridge::ResponsesToGemini, - &upstream, - ProtocolKind::Responses, - "client-model", - ) - .unwrap(); - let wire: serde_json::Value = serde_json::from_slice(&translated.body).unwrap(); - assert_eq!(wire["model"], "client-model"); - assert_eq!(wire["output"][0]["content"][0]["text"], "hello"); - assert_eq!( - translated.semantic.output_format, - persisting_pchronicle::model::LlmProtocol::Gemini - ); - } - - proptest! { - #[test] - fn responses_bridge_selects_chat_completions_for_deepseek( - model in "deepseek-[a-z][a-z0-9-]{0,20}", - streaming in any::(), - ) { - let config = format!( - "listen = \"127.0.0.1:1\"\n\n[[models]]\nname = \"{model}\"\nupstream = \"https://api.deepseek.com/v1\"\n" - ); - let route = ProxyConfig::from_toml_str(&config) - .unwrap() - .models - .into_iter() - .next() - .unwrap(); - let bridge = ProtocolBridge::needed(ProtocolKind::Responses, &route); - - prop_assert_eq!(bridge, ProtocolBridge::ResponsesToCompletions); - prop_assert_eq!( - bridge.upstream_protocol(ProtocolKind::Responses), - ProtocolKind::ChatCompletions - ); - prop_assert_eq!( - bridge - .upstream_path("/v1/responses", &model, streaming) - .unwrap(), - "/v1/chat/completions" - ); - } - - #[test] - fn responses_bridge_passthrough_covers_native_openai_hosts( - model in "[a-z][a-z0-9._-]{0,24}", - host in prop_oneof![Just("api.openai.com"), Just("openai.azure.com")], - ) { - let config = format!( - "listen = \"127.0.0.1:1\"\n\n[[models]]\nname = \"{model}\"\nupstream = \"https://{host}/v1\"\n" - ); - let route = ProxyConfig::from_toml_str(&config) - .unwrap() - .models - .into_iter() - .next() - .unwrap(); - - prop_assert_eq!( - ProtocolBridge::needed(ProtocolKind::Responses, &route), - ProtocolBridge::Passthrough - ); - } - - #[test] - fn messages_error_conversion_preserves_generated_error_details( - message in text_strategy(), - error_type in "[a-z][a-z0-9_-]{0,16}", - code in prop::option::of("[a-z][a-z0-9_-]{0,16}"), - status_code in prop_oneof![Just(400u16), Just(429u16), Just(500u16)], - ) { - let body = Bytes::from(serde_json::to_vec(&serde_json::json!({ - "error": { - "message": message.clone(), - "type": error_type.clone(), - "code": code, - } - })).unwrap()); - let translated = translate_error_for_bridge( - ProtocolBridge::MessagesToCompletions, - &body, - axum::http::StatusCode::from_u16(status_code).unwrap(), - ).unwrap(); - let value: serde_json::Value = serde_json::from_slice(&translated).unwrap(); - - prop_assert_eq!(value["type"].as_str(), Some("error")); - prop_assert_eq!(value["error"]["type"].as_str(), Some(error_type.as_str())); - prop_assert_eq!(value["error"]["message"].as_str(), Some(message.as_str())); - } - - #[test] - fn responses_error_conversion_preserves_arbitrary_bytes( - body in prop::collection::vec(any::(), 0..=256), - status_code in prop_oneof![Just(400u16), Just(429u16), Just(500u16)], - ) { - let body = Bytes::from(body); - let translated = translate_error_for_bridge( - ProtocolBridge::ResponsesToCompletions, - &body, - axum::http::StatusCode::from_u16(status_code).unwrap(), - ).unwrap(); - - prop_assert_eq!(translated, body); - } - - #[test] - fn gemini_bridge_preserves_model_and_stream_path( - model in "[a-z][a-z0-9._-]{0,24}", - prefixed in any::(), - streaming in any::(), - ) { - let route_model = if prefixed { - format!("models/{model}") - } else { - model.clone() - }; - let config = format!( - "listen = \"127.0.0.1:1\"\n\n[[models]]\nname = \"{route_model}\"\nprovider = \"gemini\"\nupstream = \"https://generativelanguage.googleapis.com/v1beta\"\n" - ); - let route = ProxyConfig::from_toml_str(&config) - .unwrap() - .models - .into_iter() - .next() - .unwrap(); - let bridge = ProtocolBridge::needed(ProtocolKind::ChatCompletions, &route); - let method = if streaming { - "streamGenerateContent" - } else { - "generateContent" - }; - - prop_assert_eq!(bridge, ProtocolBridge::CompletionsToGemini); - prop_assert_eq!( - bridge.upstream_protocol(ProtocolKind::ChatCompletions), - ProtocolKind::Gemini - ); - prop_assert_eq!( - bridge - .upstream_path("/v1/chat/completions", &route_model, streaming) - .unwrap(), - format!("/v1beta/models/{model}:{method}") - ); - } - - #[test] - fn messages_request_conversion_preserves_text_and_stream_flag( - text in text_strategy(), - system in prop::option::of(text_strategy()), - stream in any::(), - ) { - let request = serde_json::json!({ - "model": "claude-test", - "system": system.clone(), - "messages": [{"role": "user", "content": text.clone()}], - "stream": stream, - }); - let translated = messages_request_to_completions( - &Bytes::from(serde_json::to_vec(&request).unwrap()), - "upstream-model", - ) - .unwrap(); - let wire: serde_json::Value = serde_json::from_slice(&translated).unwrap(); - - prop_assert_eq!(wire["model"].as_str(), Some("upstream-model")); - prop_assert_eq!(wire["stream"].as_bool(), Some(stream)); - let user_index = if let Some(system) = system.as_ref() { - prop_assert_eq!(wire["messages"][0]["role"].as_str(), Some("system")); - prop_assert_eq!(wire["messages"][0]["content"].as_str(), Some(system.as_str())); - 1 - } else { - 0 - }; - prop_assert_eq!(wire["messages"][user_index]["role"].as_str(), Some("user")); - prop_assert_eq!(wire["messages"][user_index]["content"][0]["type"].as_str(), Some("text")); - prop_assert_eq!( - wire["messages"][user_index]["content"][0]["text"].as_str(), - Some(text.as_str()) - ); - prop_assert!(wire.get("system").is_none()); - } - - #[test] - fn responses_request_conversion_preserves_text_and_stream_flag( - text in text_strategy(), - instructions in prop::option::of(text_strategy()), - stream in any::(), - ) { - let request = serde_json::json!({ - "model": "responses-test", - "instructions": instructions.clone(), - "input": text.clone(), - "stream": stream, - }); - let translated = responses_request_to_completions( - &Bytes::from(serde_json::to_vec(&request).unwrap()), - "upstream-model", - None, - ) - .unwrap(); - let wire: serde_json::Value = serde_json::from_slice(&translated).unwrap(); - - prop_assert_eq!(wire["model"].as_str(), Some("upstream-model")); - prop_assert_eq!(wire["stream"].as_bool(), Some(stream)); - let user_index = if let Some(instructions) = instructions.as_ref() { - prop_assert_eq!(wire["messages"][0]["role"].as_str(), Some("system")); - prop_assert_eq!(wire["messages"][0]["content"].as_str(), Some(instructions.as_str())); - 1 - } else { - 0 - }; - prop_assert_eq!(wire["messages"][user_index]["role"].as_str(), Some("user")); - prop_assert_eq!(wire["messages"][user_index]["content"].as_str(), Some(text.as_str())); - } - - #[test] - fn completion_response_conversion_preserves_wire_shape_and_usage( - text in text_strategy(), - model in "[a-z][a-z0-9_-]{0,24}", - finish_reason in prop_oneof![Just("stop"), Just("length"), Just("content_filter")], - input_tokens in 0u64..=10_000, - output_tokens in 0u64..=10_000, - ) { - let response = serde_json::json!({ - "id": "chatcmpl-property", - "model": "upstream-model", - "choices": [{ - "index": 0, - "message": {"role": "assistant", "content": text}, - "finish_reason": finish_reason - }], - "usage": { - "prompt_tokens": input_tokens, - "completion_tokens": output_tokens, - "total_tokens": input_tokens + output_tokens - } - }); - let body = Bytes::from(serde_json::to_vec(&response).unwrap()); - - let messages = completions_response_to_messages(&body, &model).unwrap(); - let messages_wire: serde_json::Value = serde_json::from_slice(&messages).unwrap(); - let expected_stop = match finish_reason { - "length" => "max_tokens", - "content_filter" => "refusal", - _ => "end_turn", - }; - prop_assert_eq!(messages_wire["type"].as_str(), Some("message")); - prop_assert_eq!(messages_wire["role"].as_str(), Some("assistant")); - prop_assert_eq!(messages_wire["model"].as_str(), Some(model.as_str())); - prop_assert_eq!(messages_wire["content"][0]["type"].as_str(), Some("text")); - prop_assert_eq!(messages_wire["content"][0]["text"].as_str(), Some(text.as_str())); - prop_assert_eq!(messages_wire["stop_reason"].as_str(), Some(expected_stop)); - prop_assert_eq!(messages_wire["usage"]["input_tokens"].as_u64(), Some(input_tokens)); - prop_assert_eq!(messages_wire["usage"]["output_tokens"].as_u64(), Some(output_tokens)); - - let responses = completions_response_to_responses(&body, &model).unwrap(); - let responses_wire: serde_json::Value = serde_json::from_slice(&responses).unwrap(); - let expected_status = match finish_reason { - "length" => "incomplete", - "content_filter" => "failed", - _ => "completed", - }; - prop_assert_eq!(responses_wire["object"].as_str(), Some("response")); - prop_assert_eq!(responses_wire["status"].as_str(), Some(expected_status)); - prop_assert_eq!(responses_wire["model"].as_str(), Some(model.as_str())); - prop_assert_eq!(responses_wire["output"][0]["type"].as_str(), Some("message")); - prop_assert_eq!(responses_wire["output"][0]["content"][0]["type"].as_str(), Some("output_text")); - prop_assert_eq!(responses_wire["output"][0]["content"][0]["text"].as_str(), Some(text.as_str())); - prop_assert_eq!(responses_wire["usage"]["input_tokens"].as_u64(), Some(input_tokens)); - prop_assert_eq!(responses_wire["usage"]["output_tokens"].as_u64(), Some(output_tokens)); - prop_assert_eq!(responses_wire["usage"]["total_tokens"].as_u64(), Some(input_tokens + output_tokens)); - } - } -} diff --git a/crates/persisting-gateway/src/conversion/responses_completions.rs b/crates/persisting-gateway/src/conversion/responses_completions.rs deleted file mode 100644 index f518f1820..000000000 --- a/crates/persisting-gateway/src/conversion/responses_completions.rs +++ /dev/null @@ -1,547 +0,0 @@ -//! OpenAI Responses API ↔ Chat Completions (Codex / DeepSeek subset). - -use anyhow::{Context, Result}; -use bytes::Bytes; -use serde_json::{Value, json}; - -use super::tool_call::{ - chat_tool_call_from_input_item, is_tool_call_input_type, is_tool_call_output_type, - unquote_chat_tool_arguments, -}; - -/// Convert `/v1/responses` JSON body to `/v1/chat/completions`. -pub fn responses_request_to_completions( - body: &Bytes, - upstream_model: &str, - reasoning_cache: Option<&crate::gateway::ReasoningCacheHandle>, -) -> Result { - let mut v: Value = serde_json::from_slice(body).context("parse responses request")?; - let obj = v - .as_object_mut() - .context("responses request must be a JSON object")?; - - let mut messages = Vec::new(); - - if let Some(instructions) = obj.remove("instructions").and_then(|i| value_to_text(&i)) - && !instructions.is_empty() - { - messages.push(json!({"role": "system", "content": instructions})); - } - - if let Some(input) = obj.remove("input") { - append_input_as_messages(&input, &mut messages); - } - - let stream = obj.get("stream").and_then(|s| s.as_bool()).unwrap_or(false); - - let mut out = json!({ - "model": upstream_model, - "messages": messages, - "stream": stream, - }); - - if stream { - out["stream_options"] = json!({"include_usage": true}); - } - - if let Some(t) = obj.get("temperature").filter(|v| !v.is_null()) { - out["temperature"] = t.clone(); - } - if let Some(t) = obj.get("top_p").filter(|v| !v.is_null()) { - out["top_p"] = t.clone(); - } - if let Some(n) = obj.get("max_output_tokens").filter(|v| !v.is_null()) { - out["max_tokens"] = n.clone(); - } - for key in ["tools", "tool_choice", "parallel_tool_calls"] { - if let Some(v) = obj.get(key).filter(|v| !v.is_null()) { - if key == "tools" { - out[key] = normalize_responses_tools(v); - } else { - out[key] = v.clone(); - } - } - } - - if let Some(cache) = reasoning_cache - && let Some(msgs) = out.get_mut("messages").and_then(|m| m.as_array_mut()) - { - cache.apply_to_messages(msgs); - } - - Ok(Bytes::from( - serde_json::to_vec(&out).context("serialize completions request")?, - )) -} - -/// Convert chat completion JSON to OpenAI Responses response shape. -pub fn completions_response_to_responses(body: &Bytes, client_model: &str) -> Result { - let v: Value = serde_json::from_slice(body).context("parse completions response")?; - Ok(Bytes::from( - serde_json::to_vec(&completions_value_to_responses(&v, client_model)?) - .context("serialize responses body")?, - )) -} - -pub fn completions_value_to_responses(body: &Value, client_model: &str) -> Result { - let choice = body - .get("choices") - .and_then(|c| c.as_array()) - .and_then(|a| a.first()) - .context("completions response missing choices[0]")?; - let message = choice.get("message").context("choice missing message")?; - let text = message - .get("content") - .and_then(|c| c.as_str()) - .unwrap_or(""); - let finish = choice - .get("finish_reason") - .and_then(|f| f.as_str()) - .unwrap_or("stop"); - let response_id = format!("resp_{}", body_id_suffix(body)); - let message_id = format!("msg_{}", body_id_suffix(body)); - - let status = match finish { - "length" => "incomplete", - "content_filter" => "failed", - _ => "completed", - }; - - let usage = body.get("usage"); - let input_tokens = usage - .and_then(|u| u.get("prompt_tokens")) - .and_then(|v| v.as_u64()) - .unwrap_or(0); - let output_tokens = usage - .and_then(|u| u.get("completion_tokens")) - .and_then(|v| v.as_u64()) - .unwrap_or(0); - let total_tokens = usage - .and_then(|u| u.get("total_tokens")) - .and_then(|v| v.as_u64()) - .unwrap_or(input_tokens + output_tokens); - - let mut output = Vec::new(); - if !text.is_empty() { - output.push(json!({ - "type": "message", - "id": message_id, - "role": "assistant", - "status": "completed", - "content": [{ - "type": "output_text", - "text": text, - "annotations": [] - }] - })); - } - if let Some(tool_calls) = message.get("tool_calls").and_then(|t| t.as_array()) { - for tc in tool_calls { - let call_id = tc - .get("id") - .and_then(|v| v.as_str()) - .unwrap_or("call_proxy"); - let name = tc - .get("function") - .and_then(|f| f.get("name")) - .and_then(|v| v.as_str()) - .unwrap_or(""); - let arguments = tc - .get("function") - .and_then(|f| f.get("arguments")) - .map(unquote_chat_tool_arguments) - .unwrap_or_else(|| "{}".to_string()); - output.push(json!({ - "type": "function_call", - "id": call_id, - "call_id": call_id, - "name": name, - "arguments": arguments, - "status": "completed" - })); - } - } - - Ok(json!({ - "id": response_id, - "object": "response", - "created_at": body.get("created").cloned().unwrap_or(json!(0)), - "status": status, - "model": client_model, - "output": output, - "usage": { - "input_tokens": input_tokens, - "output_tokens": output_tokens, - "total_tokens": total_tokens - } - })) -} - -fn body_id_suffix(body: &Value) -> String { - body.get("id") - .and_then(|i| i.as_str()) - .map(|s| { - s.chars() - .filter(|c| c.is_ascii_alphanumeric()) - .take(16) - .collect() - }) - .filter(|s: &String| !s.is_empty()) - .unwrap_or_else(|| "proxy".to_string()) -} - -fn normalize_responses_tools(tools: &Value) -> Value { - let Some(items) = tools.as_array() else { - return tools.clone(); - }; - json!( - items - .iter() - .filter_map(|tool| { - let tool_type = tool.get("type").and_then(|t| t.as_str()).unwrap_or(""); - if tool_type != "function" { - return None; - } - if tool.get("function").is_some() { - return Some(tool.clone()); - } - Some(json!({ - "type": "function", - "function": { - "name": tool.get("name").cloned().unwrap_or(json!("")), - "description": tool.get("description").cloned().unwrap_or(Value::Null), - "parameters": tool.get("parameters").cloned().unwrap_or(json!({"type":"object","properties":{}})), - "strict": tool.get("strict").cloned().unwrap_or(Value::Null), - } - })) - }) - .collect::>() - ) -} - -fn append_input_as_messages(input: &Value, messages: &mut Vec) { - match input { - Value::String(s) if !s.trim().is_empty() => { - messages.push(json!({"role": "user", "content": s})); - } - Value::Array(items) => { - let mut converter = InputConverter::default(); - for item in items { - converter.push_item(item); - } - converter.flush(&mut *messages); - } - _ => {} - } -} - -/// Batches consecutive Responses `function_call` items into one assistant message -/// (moon-bridge `convertInput` / `pendingFCBlocks` pattern). -#[derive(Default)] -struct InputConverter { - pending_tool_calls: Vec, - legacy_messages: Option>, -} - -impl InputConverter { - fn push_item(&mut self, item: &Value) { - if let Some(item_type) = item.get("type").and_then(|t| t.as_str()) { - if is_tool_call_output_type(item_type) { - self.flush_pending_tool_calls(); - self.push_tool_result(item); - return; - } - if is_tool_call_input_type(item_type) { - self.pending_tool_calls - .push(chat_tool_call_from_input_item(item)); - return; - } - if item_type == "reasoning" { - // Codex reasoning items are not replayed to Chat Completions upstreams. - return; - } - if item_type == "message" { - self.flush_pending_tool_calls(); - append_message_input_item(item, self.pending_messages()); - return; - } - } - - self.flush_pending_tool_calls(); - append_legacy_input_item(item, self.pending_messages()); - } - - fn flush(&mut self, messages: &mut Vec) { - self.flush_pending_tool_calls(); - messages.extend(self.drain_pending_messages()); - } - - fn pending_messages(&mut self) -> &mut Vec { - // Lazily allocate only when legacy path needs it. - self.legacy_messages.get_or_insert_with(Vec::new) - } - - fn drain_pending_messages(&mut self) -> Vec { - self.legacy_messages.take().unwrap_or_default() - } - - fn flush_pending_tool_calls(&mut self) { - if self.pending_tool_calls.is_empty() { - return; - } - let tool_calls = std::mem::take(&mut self.pending_tool_calls); - self.pending_messages().push(json!({ - "role": "assistant", - "content": null, - "tool_calls": tool_calls - })); - } - - fn push_tool_result(&mut self, item: &Value) { - let call_id = item.get("call_id").and_then(|v| v.as_str()).unwrap_or(""); - let output = function_call_output_text(item.get("output")); - self.pending_messages().push(json!({ - "role": "tool", - "tool_call_id": call_id, - "content": output - })); - } -} - -fn append_legacy_input_item(item: &Value, messages: &mut Vec) { - let role = item - .get("role") - .or_else(|| item.get("type").filter(|t| is_role_name(t))) - .and_then(|r| r.as_str()) - .map(normalize_role) - .unwrap_or("user"); - - if let Some(Value::Array(parts)) = item.get("content") { - for part in parts { - let text = content_part_to_text(part); - if !text.is_empty() { - messages.push(json!({"role": role, "content": text})); - } - } - return; - } - - let text = input_item_text(item); - if text.is_empty() { - return; - } - - messages.push(json!({"role": role, "content": text})); -} - -fn append_message_input_item(item: &Value, messages: &mut Vec) { - let role = item - .get("role") - .and_then(|r| r.as_str()) - .map(normalize_role) - .unwrap_or("user"); - if let Some(Value::Array(parts)) = item.get("content") { - for part in parts { - let text = content_part_to_text(part); - if !text.is_empty() { - messages.push(json!({"role": role, "content": text})); - } - } - return; - } - let text = input_item_text(item); - if !text.is_empty() { - messages.push(json!({"role": role, "content": text})); - } -} - -fn function_call_output_text(output: Option<&Value>) -> String { - match output { - Some(Value::String(s)) => s.clone(), - Some(Value::Array(parts)) => parts - .iter() - .filter_map(|part| { - let t = part.get("type").and_then(|x| x.as_str()).unwrap_or(""); - match t { - "input_text" | "text" | "output_text" => part - .get("text") - .and_then(|x| x.as_str()) - .map(str::to_string), - _ => value_to_text(part), - } - }) - .collect::>() - .join("\n"), - Some(v) => value_to_text(v).unwrap_or_default(), - None => String::new(), - } -} - -fn content_part_to_text(part: &Value) -> String { - let t = part.get("type").and_then(|x| x.as_str()).unwrap_or(""); - match t { - "input_text" | "text" | "output_text" => part - .get("text") - .and_then(|x| x.as_str()) - .unwrap_or("") - .to_string(), - _ => value_to_text(part).unwrap_or_default(), - } -} - -fn is_role_name(v: &Value) -> bool { - matches!( - v.as_str(), - Some("user") | Some("assistant") | Some("system") | Some("developer") - ) -} - -fn normalize_role(role: &str) -> &str { - match role { - "developer" => "system", - other => other, - } -} - -fn input_item_text(item: &Value) -> String { - if let Some(s) = item.as_str() { - return s.to_string(); - } - if let Some(content) = item.get("content") { - return input_content_to_text(content); - } - value_to_text(item).unwrap_or_default() -} - -fn input_content_to_text(content: &Value) -> String { - match content { - Value::String(s) => s.clone(), - Value::Array(parts) => parts - .iter() - .filter_map(|part| { - let t = part.get("type").and_then(|x| x.as_str()).unwrap_or(""); - match t { - "input_text" | "text" | "output_text" => part - .get("text") - .and_then(|x| x.as_str()) - .map(str::to_string), - _ => value_to_text(part), - } - }) - .collect::>() - .join("\n\n"), - _ => value_to_text(content).unwrap_or_default(), - } -} - -fn value_to_text(v: &Value) -> Option { - match v { - Value::String(s) if !s.trim().is_empty() => Some(s.clone()), - Value::Array(parts) => { - let joined = parts - .iter() - .filter_map(value_to_text) - .collect::>() - .join("\n\n"); - if joined.is_empty() { - None - } else { - Some(joined) - } - } - _ => None, - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn codex_like_request_to_completions() { - let body = Bytes::from_static(include_bytes!( - "../../tests/fixtures/local/requests/responses/codex_basic.json" - )); - let out = responses_request_to_completions(&body, "deepseek-chat", None).unwrap(); - let v: Value = serde_json::from_slice(&out).unwrap(); - assert_eq!(v["model"], "deepseek-chat"); - assert_eq!(v["stream"], true); - let msgs = v["messages"].as_array().unwrap(); - assert!(msgs.len() >= 2); - assert_eq!(msgs[0]["role"], "user"); - assert!(msgs[0]["content"].as_str().unwrap().contains("permissions")); - assert!( - msgs.last().unwrap()["content"] - .as_str() - .unwrap() - .contains("hi") - ); - } - - #[test] - fn tool_roundtrip_request_to_completions() { - let body = Bytes::from_static(include_bytes!( - "../../tests/fixtures/local/requests/responses/codex_tool_roundtrip.json" - )); - let out = responses_request_to_completions(&body, "deepseek-chat", None).unwrap(); - let v: Value = serde_json::from_slice(&out).unwrap(); - assert!(v.get("tools").is_some()); - let tool = &v["tools"][0]; - assert!(tool.get("function").is_some()); - assert_eq!(tool["function"]["name"], "shell"); - let msgs = v["messages"].as_array().unwrap(); - assert_eq!(msgs[0]["role"], "user"); - assert_eq!(msgs[1]["role"], "assistant"); - assert_eq!(msgs[1]["tool_calls"][0]["function"]["name"], "shell"); - assert_eq!(msgs[2]["role"], "tool"); - assert_eq!(msgs[2]["tool_call_id"], "call_abc123"); - assert!(msgs[2]["content"].as_str().unwrap().contains("Cargo.toml")); - } - - #[test] - fn completions_tool_call_to_responses_output() { - let body = Bytes::from_static(include_bytes!( - "../../tests/fixtures/local/response/completions/tool_call.json" - )); - let out = completions_response_to_responses(&body, "gpt-5.5").unwrap(); - let v: Value = serde_json::from_slice(&out).unwrap(); - assert_eq!(v["output"][0]["type"], "function_call"); - assert_eq!(v["output"][0]["call_id"], "call_abc123"); - assert_eq!(v["output"][0]["name"], "shell"); - assert!( - v["output"][0]["arguments"] - .as_str() - .unwrap() - .contains("ls -la") - ); - } - - #[test] - fn batches_consecutive_function_calls_into_one_assistant_message() { - let body = json!({ - "model": "gpt-5.5", - "stream": true, - "input": [ - {"type":"message","role":"user","content":[{"type":"input_text","text":"go"}]}, - {"type":"function_call","call_id":"call_a","name":"shell","arguments":"{}"}, - {"type":"function_call","call_id":"call_b","name":"read","arguments":"{}"}, - {"type":"function_call_output","call_id":"call_a","output":"ok"}, - {"type":"function_call_output","call_id":"call_b","output":"file"} - ] - }); - let out = responses_request_to_completions( - &Bytes::from(serde_json::to_vec(&body).unwrap()), - "deepseek-chat", - None, - ) - .unwrap(); - let v: Value = serde_json::from_slice(&out).unwrap(); - let msgs = v["messages"].as_array().unwrap(); - assert_eq!(msgs[1]["role"], "assistant"); - assert_eq!(msgs[1]["tool_calls"].as_array().unwrap().len(), 2); - assert_eq!(msgs[2]["role"], "tool"); - assert_eq!(msgs[2]["tool_call_id"], "call_a"); - assert_eq!(msgs[3]["role"], "tool"); - assert_eq!(msgs[3]["tool_call_id"], "call_b"); - } -} diff --git a/crates/persisting-gateway/src/conversion/semantic.rs b/crates/persisting-gateway/src/conversion/semantic.rs deleted file mode 100644 index 13cef708a..000000000 --- a/crates/persisting-gateway/src/conversion/semantic.rs +++ /dev/null @@ -1,1123 +0,0 @@ -//! Chronicle typed LLM payload → provider/client wire renderers. -//! -//! This module is the only semantic conversion boundary used by the runtime. -//! Wire JSON is parsed into Chronicle types before entering here; renderers -//! never reinterpret a second protocol-shaped JSON document. - -use std::collections::HashMap; - -use anyhow::{Context, Result}; -use bytes::Bytes; -use persisting_pchronicle::model::{ - LlmCandidate, LlmContentPart, LlmImageSource, LlmMessage, LlmProtocol, LlmRequest, - LlmRequestEventPayload, LlmResponse, LlmResponseEventPayload, LlmRole, LlmToolChoiceMode, - LlmUsage, -}; -use serde_json::{Value, json}; - -pub fn request_to_chat_completions( - semantic: &LlmRequestEventPayload, - upstream_model: &str, - reasoning_cache: Option<&crate::gateway::ReasoningCacheHandle>, -) -> Result { - let mut rendered = - render_chat_completions(&semantic.request, &semantic.input_format, upstream_model); - if semantic.input_format == LlmProtocol::Responses - && let Some(cache) = reasoning_cache - && let Some(messages) = rendered.get_mut("messages").and_then(Value::as_array_mut) - { - cache.apply_to_messages(messages); - } - Ok(Bytes::from(serde_json::to_vec(&rendered)?)) -} - -pub fn request_to_gemini(semantic: &LlmRequestEventPayload, upstream_model: &str) -> Result { - let request = &semantic.request; - let mut calls = HashMap::::new(); - let mut contents = Vec::new(); - let mut inline_system = Vec::new(); - - for message in &request.messages { - if matches!(message.role, LlmRole::System | LlmRole::Developer) { - inline_system.extend(message.parts.iter().filter_map(|part| match part { - LlmContentPart::Text { text, .. } if !text.is_empty() => Some(text.as_str()), - _ => None, - })); - continue; - } - let role = if message.role == LlmRole::Assistant { - "model" - } else { - "user" - }; - let mut parts = Vec::new(); - let function_response = message - .parts - .iter() - .any(|part| matches!(part, LlmContentPart::ToolResult { .. })); - for (part_index, part) in message.parts.iter().enumerate() { - match part { - LlmContentPart::Text { text, .. } if !text.is_empty() => { - parts.push(json!({"text": text})); - } - LlmContentPart::Image { - source, media_type, .. - } => parts.push(render_gemini_image(source, media_type.as_deref())?), - LlmContentPart::Reasoning { text, signature } => { - if let Some(text) = text.as_deref().filter(|text| !text.is_empty()) { - let mut value = json!({"text": text, "thought": true}); - if let Some(signature) = signature { - value["thoughtSignature"] = json!(signature); - } - parts.push(value); - } - } - LlmContentPart::ToolCall { - id, - name, - arguments, - signature, - } => { - calls.insert(id.clone(), (name.clone(), part_index)); - let mut value = json!({ - "functionCall": {"name": name, "args": arguments} - }); - if let Some(signature) = signature { - value["thoughtSignature"] = json!(signature); - } - parts.push(value); - } - LlmContentPart::ToolResult { - call_id, - name, - content, - is_error, - .. - } => { - let resolved_name = name - .as_deref() - .or_else(|| calls.get(call_id).map(|(name, _)| name.as_str())) - .unwrap_or_default(); - let response = match content { - Value::Object(_) => content.clone(), - Value::String(text) => json!({"content": text}), - other => json!({"content": other}), - }; - let mut value = json!({ - "functionResponse": {"name": resolved_name, "response": response} - }); - if is_error == &Some(true) { - value["functionResponse"]["response"]["error"] = json!(true); - } - // Used only for deterministic ordering below; Gemini does not - // receive this correlation field. - value["functionResponse"]["id"] = json!(call_id); - parts.push(value); - } - LlmContentPart::Unknown { value, .. } => parts.push(value.clone()), - _ => {} - } - } - if role == "user" && !function_response && !parts.iter().any(|p| p.get("text").is_some()) { - parts.push(json!({"text":" "})); - } - push_gemini_content(&mut contents, role, function_response, parts); - } - - for content in &mut contents { - let Some(parts) = content.get_mut("parts").and_then(Value::as_array_mut) else { - continue; - }; - if parts - .iter() - .any(|part| part.get("functionResponse").is_some()) - { - parts.sort_by_key(|part| { - part.get("functionResponse") - .and_then(|response| response.get("id")) - .and_then(Value::as_str) - .and_then(|id| calls.get(id)) - .map(|(_, index)| *index) - .unwrap_or(usize::MAX) - }); - for part in parts { - if let Some(response) = part - .get_mut("functionResponse") - .and_then(Value::as_object_mut) - { - response.remove("id"); - } - } - } - } - if contents.is_empty() { - contents.push(json!({"role":"user", "parts":[{"text":" "}]})); - } - - let mut output = json!({"contents": contents}); - let mut system = request - .system - .iter() - .flat_map(|message| message.parts.iter()) - .filter_map(|part| match part { - LlmContentPart::Text { text, .. } if !text.is_empty() => Some(text.as_str()), - _ => None, - }) - .collect::>(); - system.extend(inline_system); - if !system.is_empty() { - output["systemInstruction"] = json!({"parts":[{"text":system.join("\n")}]}); - } - - if !request.tools.is_empty() { - let declarations = request - .tools - .iter() - .filter(|tool| tool.kind == "function") - .map(|tool| { - let mut declaration = json!({ - "name": tool.name, - "parameters": super::gemini_native::normalize_gemini_schema(&tool.input_schema), - }); - if let Some(description) = &tool.description { - declaration["description"] = json!(description); - } - declaration - }) - .collect::>(); - if !declarations.is_empty() { - output["tools"] = json!([{"functionDeclarations": declarations}]); - } - } - if let Some(choice) = &request.tool_choice { - let mode = match choice.mode { - LlmToolChoiceMode::None => "NONE", - LlmToolChoiceMode::Required | LlmToolChoiceMode::Tool => "ANY", - LlmToolChoiceMode::Auto => "AUTO", - }; - let mut config = json!({"mode": mode}); - if choice.mode == LlmToolChoiceMode::Tool - && let Some(name) = &choice.name - { - config["allowedFunctionNames"] = json!([name]); - } - output["toolConfig"] = json!({"functionCallingConfig": config}); - } - if let Some(config) = render_gemini_generation(request, upstream_model) { - output["generationConfig"] = config; - } - for (extension, field) in [ - ("google.cached_content", "cachedContent"), - ("google.safety_settings", "safetySettings"), - ("google.labels", "labels"), - ] { - if let Some(value) = request.extensions.get(extension) { - output[field] = value.clone(); - } - } - if output.get("cachedContent").is_some() { - let object = output.as_object_mut().expect("Gemini request is an object"); - object.remove("systemInstruction"); - object.remove("tools"); - object.remove("toolConfig"); - } - Ok(Bytes::from(serde_json::to_vec(&output)?)) -} - -fn push_gemini_content( - contents: &mut Vec, - role: &str, - function_response: bool, - parts: Vec, -) { - if parts.is_empty() { - return; - } - let can_merge = contents.last().is_some_and(|last| { - last.get("role").and_then(Value::as_str) == Some(role) - && last - .get("parts") - .and_then(Value::as_array) - .is_some_and(|parts| { - parts - .iter() - .any(|part| part.get("functionResponse").is_some()) - == function_response - }) - }); - if can_merge { - contents - .last_mut() - .and_then(|content| content.get_mut("parts")) - .and_then(Value::as_array_mut) - .expect("checked Gemini content parts") - .extend(parts); - } else { - contents.push(json!({"role":role, "parts":parts})); - } -} - -fn render_gemini_image(source: &LlmImageSource, media_type: Option<&str>) -> Result { - match source { - LlmImageSource::Data { data } => { - if let Some(raw) = data.strip_prefix("data:") { - let (meta, payload) = raw.split_once(',').context("invalid image data URL")?; - let (mime, encoding) = meta.split_once(';').context("invalid image data URL")?; - anyhow::ensure!( - encoding.eq_ignore_ascii_case("base64"), - "image data URL must be base64" - ); - Ok(json!({"inlineData":{"mimeType":normalize_mime(mime),"data":payload}})) - } else { - Ok( - json!({"inlineData":{"mimeType":media_type.unwrap_or("application/octet-stream"),"data":data}}), - ) - } - } - LlmImageSource::File { uri } if uri.starts_with("gs://") => Ok(json!({ - "fileData":{"mimeType":media_type.or_else(|| mime_from_uri(uri)).context("gs:// media needs a MIME type")?,"fileUri":uri} - })), - LlmImageSource::Url { url } | LlmImageSource::File { uri: url } => { - anyhow::bail!("native Gemini accepts data: or gs:// media URLs, got {url}") - } - } -} - -fn normalize_mime(mime: &str) -> &str { - if mime == "image/jpg" { - "image/jpeg" - } else { - mime - } -} - -fn mime_from_uri(uri: &str) -> Option<&'static str> { - match uri.rsplit_once('.')?.1.to_ascii_lowercase().as_str() { - "png" => Some("image/png"), - "jpg" | "jpeg" => Some("image/jpeg"), - "webp" => Some("image/webp"), - "gif" => Some("image/gif"), - "pdf" => Some("application/pdf"), - "mp3" => Some("audio/mpeg"), - "wav" => Some("audio/wav"), - "mp4" => Some("video/mp4"), - "mov" => Some("video/quicktime"), - "webm" => Some("video/webm"), - "txt" => Some("text/plain"), - _ => None, - } -} - -fn render_gemini_generation(request: &LlmRequest, model: &str) -> Option { - let generation = &request.generation; - let mut config = serde_json::Map::new(); - for (key, value) in [ - ("temperature", generation.temperature.map(Value::from)), - ("topP", generation.top_p.map(Value::from)), - ("topK", generation.top_k.map(Value::from)), - ( - "frequencyPenalty", - generation.frequency_penalty.map(Value::from), - ), - ( - "presencePenalty", - generation.presence_penalty.map(Value::from), - ), - ("seed", generation.seed.map(Value::from)), - ( - "candidateCount", - generation.candidate_count.map(Value::from), - ), - ( - "maxOutputTokens", - generation.max_output_tokens.map(Value::from), - ), - ] { - if let Some(value) = value { - config.insert(key.into(), value); - } - } - if !generation.stop_sequences.is_empty() { - config.insert("stopSequences".into(), json!(generation.stop_sequences)); - } - if let Some(format) = &request.response_format { - if format.kind == "json_object" || format.kind.contains("json") || format.schema.is_some() { - config.insert("responseMimeType".into(), json!("application/json")); - } - if let Some(schema) = &format.schema { - config.insert( - "responseSchema".into(), - super::gemini_native::normalize_gemini_schema(schema), - ); - } - } - let thinking = if let Some(budget) = generation.thinking_budget { - Some(json!({"thinkingBudget":budget,"includeThoughts":true})) - } else { - generation.reasoning_effort.as_ref().and_then(|value| { - let effort = value.as_str()?; - if model.contains("gemini-3") { - (effort != "none").then(|| json!({"thinkingLevel":effort,"includeThoughts":true})) - } else { - let budget = match effort { - "none" => return None, - "minimal" | "low" => 1024, - "medium" => 2048, - "high" => 4096, - "xhigh" => 8192, - "max" => 16384, - _ => return None, - }; - Some(json!({"thinkingBudget":budget,"includeThoughts":true})) - } - }) - }; - if let Some(thinking) = thinking { - config.insert("thinkingConfig".into(), thinking); - } - (!config.is_empty()).then_some(Value::Object(config)) -} - -pub fn response_to_wire( - semantic: &LlmResponseEventPayload, - target: LlmProtocol, - client_model: &str, -) -> Result { - let value = match target { - LlmProtocol::ChatCompletions => render_chat_response(&semantic.response, client_model), - LlmProtocol::Messages => render_messages_response(&semantic.response, client_model), - LlmProtocol::Responses => render_responses_response(&semantic.response, client_model), - LlmProtocol::Gemini => render_gemini_response(&semantic.response, client_model), - LlmProtocol::Unknown => anyhow::bail!("cannot render an unknown LLM response protocol"), - }; - Ok(Bytes::from(serde_json::to_vec(&value)?)) -} - -fn render_chat_response(response: &LlmResponse, fallback_model: &str) -> Value { - let id = response - .id - .clone() - .unwrap_or_else(|| format!("chatcmpl_{}", chrono::Utc::now().timestamp_millis())); - let choices = response - .candidates - .iter() - .map(|candidate| { - let mut message = json!({"role":"assistant"}); - let text = candidate_text(candidate); - if !text.is_empty() || !candidate_has_tools(candidate) { - message["content"] = json!(text); - } - let reasoning = candidate_reasoning(candidate); - if !reasoning.is_empty() { - message["reasoning_content"] = json!(reasoning); - } - let tool_calls = candidate_tool_calls(candidate, &id); - if !tool_calls.is_empty() { - message["tool_calls"] = Value::Array(tool_calls); - } - json!({ - "index": candidate.index, - "message": message, - "finish_reason": chat_finish_reason(candidate.finish_reason.as_deref(), candidate_has_tools(candidate)), - }) - }) - .collect::>(); - let choices = if choices.is_empty() { - vec![json!({"index":0,"message":{"role":"assistant","content":""},"finish_reason":"stop"})] - } else { - choices - }; - let mut value = json!({ - "id":id, - "object":"chat.completion", - "created":chrono::Utc::now().timestamp(), - "model":fallback_model, - "choices":choices, - }); - if let Some(usage) = &response.usage { - value["usage"] = render_chat_usage(usage); - } - value -} - -fn render_messages_response(response: &LlmResponse, fallback_model: &str) -> Value { - let candidate = response.candidates.first(); - let content = candidate - .map(|candidate| { - candidate - .message - .parts - .iter() - .filter_map(|part| match part { - LlmContentPart::Text { text, .. } if !text.is_empty() => { - Some(json!({"type":"text","text":text})) - } - LlmContentPart::Reasoning { text, signature } => { - let mut block = json!({"type":"thinking","thinking":text.as_deref().unwrap_or_default()}); - if let Some(signature) = signature { - block["signature"] = json!(signature); - } - Some(block) - } - LlmContentPart::ToolCall { id, name, arguments, .. } => Some(json!({ - "type":"tool_use","id":id,"name":name,"input":arguments - })), - LlmContentPart::Unknown { value, .. } => Some(value.clone()), - _ => None, - }) - .collect::>() - }) - .unwrap_or_default(); - let usage = response.usage.as_ref().cloned().unwrap_or_default(); - json!({ - "id":response.id.as_deref().unwrap_or("msg_proxy"), - "type":"message", - "role":"assistant", - "model":fallback_model, - "content":content, - "stop_reason":anthropic_finish_reason(candidate.and_then(|c| c.finish_reason.as_deref()), candidate.is_some_and(candidate_has_tools)), - "stop_sequence":null, - "usage":{ - "input_tokens":usage.input_tokens, - "output_tokens":usage.output_tokens, - "cache_read_input_tokens":usage.cache_read_tokens, - "cache_creation_input_tokens":usage.cache_write_tokens, - } - }) -} - -fn render_responses_response(response: &LlmResponse, fallback_model: &str) -> Value { - let suffix = response - .id - .as_deref() - .map(safe_id_suffix) - .filter(|value| !value.is_empty()) - .unwrap_or_else(|| "proxy".into()); - let response_id = format!("resp_{suffix}"); - let mut output = Vec::new(); - let mut status = "completed"; - for candidate in &response.candidates { - if candidate - .finish_reason - .as_deref() - .is_some_and(|reason| matches!(reason, "length" | "max_tokens" | "MAX_TOKENS")) - { - status = "incomplete"; - } - let text = candidate_text(candidate); - if !text.is_empty() { - output.push(json!({ - "type":"message", - "id":format!("msg_{suffix}_{}", candidate.index), - "role":"assistant", - "status":"completed", - "content":[{"type":"output_text","text":text,"annotations":[]}], - })); - } - for (part_index, part) in candidate.message.parts.iter().enumerate() { - match part { - LlmContentPart::Reasoning { text, signature } => { - let mut item = json!({ - "type":"reasoning", - "id":format!("rs_{suffix}_{part_index}"), - "summary":text.as_ref().map(|text| vec![json!({"type":"summary_text","text":text})]).unwrap_or_default(), - }); - if let Some(signature) = signature { - item["encrypted_content"] = json!(signature); - } - output.push(item); - } - LlmContentPart::ToolCall { - id, - name, - arguments, - .. - } => { - let call_id = if id.is_empty() { - format!("call_{suffix}_{part_index}") - } else { - id.clone() - }; - output.push(json!({ - "type":"function_call", - "id":call_id, - "call_id":call_id, - "name":name, - "arguments":serde_json::to_string(arguments).unwrap_or_else(|_| "{}".into()), - "status":"completed", - })); - } - LlmContentPart::Unknown { value, .. } => output.push(value.clone()), - _ => {} - } - } - } - let usage = response.usage.as_ref().cloned().unwrap_or_default(); - json!({ - "id":response_id, - "object":"response", - "created_at":chrono::Utc::now().timestamp(), - "status":status, - "model":fallback_model, - "output":output, - "usage":{ - "input_tokens":usage.input_tokens, - "output_tokens":usage.output_tokens, - "total_tokens":usage.total_tokens, - "input_tokens_details":{"cached_tokens":usage.cache_read_tokens}, - "output_tokens_details":{"reasoning_tokens":usage.reasoning_tokens}, - } - }) -} - -fn render_gemini_response(response: &LlmResponse, fallback_model: &str) -> Value { - let candidates = response - .candidates - .iter() - .map(|candidate| { - let parts = candidate - .message - .parts - .iter() - .filter_map(|part| match part { - LlmContentPart::Text { text, .. } => Some(json!({"text":text})), - LlmContentPart::Reasoning { text, signature } => { - let mut value = - json!({"text":text.as_deref().unwrap_or_default(),"thought":true}); - if let Some(signature) = signature { - value["thoughtSignature"] = json!(signature); - } - Some(value) - } - LlmContentPart::ToolCall { - name, - arguments, - signature, - .. - } => { - let mut value = json!({"functionCall":{"name":name,"args":arguments}}); - if let Some(signature) = signature { - value["thoughtSignature"] = json!(signature); - } - Some(value) - } - LlmContentPart::Unknown { value, .. } => Some(value.clone()), - _ => None, - }) - .collect::>(); - json!({ - "index":candidate.index, - "content":{"role":"model","parts":parts}, - "finishReason":gemini_finish_reason(candidate.finish_reason.as_deref()), - }) - }) - .collect::>(); - let mut value = json!({ - "responseId":response.id, - "modelVersion":fallback_model, - "candidates":candidates, - }); - if let Some(usage) = &response.usage { - value["usageMetadata"] = json!({ - "promptTokenCount":usage.input_tokens, - "candidatesTokenCount":usage.output_tokens, - "totalTokenCount":usage.total_tokens, - "cachedContentTokenCount":usage.cache_read_tokens, - "thoughtsTokenCount":usage.reasoning_tokens, - }); - } - remove_nulls(&mut value); - value -} - -fn candidate_text(candidate: &LlmCandidate) -> String { - candidate - .message - .parts - .iter() - .filter_map(|part| match part { - LlmContentPart::Text { text, .. } => Some(text.as_str()), - _ => None, - }) - .collect::() -} - -fn candidate_reasoning(candidate: &LlmCandidate) -> String { - candidate - .message - .parts - .iter() - .filter_map(|part| match part { - LlmContentPart::Reasoning { - text: Some(text), .. - } => Some(text.as_str()), - _ => None, - }) - .collect::() -} - -fn candidate_has_tools(candidate: &LlmCandidate) -> bool { - candidate - .message - .parts - .iter() - .any(|part| matches!(part, LlmContentPart::ToolCall { .. })) -} - -fn candidate_tool_calls(candidate: &LlmCandidate, response_id: &str) -> Vec { - candidate - .message - .parts - .iter() - .enumerate() - .filter_map(|(index, part)| match part { - LlmContentPart::ToolCall { id, name, arguments, signature } => { - let base = if id.is_empty() { format!("call_{response_id}_{index}") } else { id.clone() }; - let id = signature.as_ref().map(|signature| format!("{base}__thought__{signature}")).unwrap_or(base); - Some(json!({ - "id":id,"type":"function", - "function":{"name":name,"arguments":serde_json::to_string(arguments).unwrap_or_else(|_| "{}".into())} - })) - } - _ => None, - }) - .collect() -} - -pub(super) fn render_chat_usage(usage: &LlmUsage) -> Value { - json!({ - "prompt_tokens":usage.input_tokens, - "completion_tokens":usage.output_tokens, - "total_tokens":usage.total_tokens, - "prompt_tokens_details":{"cached_tokens":usage.cache_read_tokens}, - "completion_tokens_details":{"reasoning_tokens":usage.reasoning_tokens}, - }) -} - -pub(super) fn chat_finish_reason(reason: Option<&str>, has_tools: bool) -> &'static str { - match reason { - Some("length" | "max_tokens" | "MAX_TOKENS") => "length", - Some("content_filter" | "SAFETY" | "RECITATION" | "BLOCKLIST" | "PROHIBITED_CONTENT") => { - "content_filter" - } - Some("tool_calls" | "tool_use") => "tool_calls", - _ if has_tools => "tool_calls", - _ => "stop", - } -} - -pub(super) fn anthropic_finish_reason(reason: Option<&str>, has_tools: bool) -> &'static str { - match reason { - Some("length" | "max_tokens" | "MAX_TOKENS") => "max_tokens", - Some("tool_calls" | "tool_use") => "tool_use", - _ if has_tools => "tool_use", - _ => "end_turn", - } -} - -pub(super) fn gemini_finish_reason(reason: Option<&str>) -> &'static str { - match reason { - Some("length" | "max_tokens") => "MAX_TOKENS", - Some("content_filter") => "SAFETY", - _ => "STOP", - } -} - -pub(super) fn safe_id_suffix(raw: &str) -> String { - raw.chars() - .filter(char::is_ascii_alphanumeric) - .take(24) - .collect() -} - -fn render_chat_completions( - request: &LlmRequest, - source: &LlmProtocol, - upstream_model: &str, -) -> Value { - let prompt_cache = supports_prompt_cache_breakpoint(upstream_model); - let mut messages = Vec::new(); - for message in &request.system { - render_message(message, source, prompt_cache, &mut messages); - } - for message in &request.messages { - render_message(message, source, prompt_cache, &mut messages); - } - - let mut output = json!({ - "model": upstream_model, - "messages": messages, - "stream": request.stream, - }); - if request.stream { - output["stream_options"] = json!({"include_usage":true}); - } - let generation = &request.generation; - copy_option(&mut output, "temperature", generation.temperature); - copy_option(&mut output, "top_p", generation.top_p); - copy_option(&mut output, "seed", generation.seed); - copy_option( - &mut output, - "frequency_penalty", - generation.frequency_penalty, - ); - copy_option(&mut output, "presence_penalty", generation.presence_penalty); - if let Some(count) = generation.candidate_count { - output["n"] = json!(count); - } - if let Some(max) = generation.max_output_tokens { - let key = if *source == LlmProtocol::Messages { - "max_completion_tokens" - } else { - "max_tokens" - }; - output[key] = json!(max); - } else if *source == LlmProtocol::Messages { - output["max_completion_tokens"] = json!(1024); - } - if !generation.stop_sequences.is_empty() { - output["stop"] = json!(generation.stop_sequences); - } - - if !request.tools.is_empty() { - output["tools"] = Value::Array( - request - .tools - .iter() - .filter(|tool| tool.kind == "function") - .map(|tool| { - let mut function = json!({ - "name":tool.name, - "description":tool.description, - "parameters":tool.input_schema, - }); - if let Some(strict) = tool.strict { - function["strict"] = json!(strict); - } - json!({"type":"function","function":function}) - }) - .collect(), - ); - } - if let Some(choice) = &request.tool_choice { - output["tool_choice"] = match choice.mode { - LlmToolChoiceMode::Auto => json!("auto"), - LlmToolChoiceMode::None => json!("none"), - LlmToolChoiceMode::Required => json!("required"), - LlmToolChoiceMode::Tool => json!({ - "type":"function", - "function":{"name":choice.name.clone().unwrap_or_default()} - }), - }; - if let Some(parallel) = choice.parallel { - output["parallel_tool_calls"] = json!(parallel); - } - } else if let Some(parallel) = request - .extensions - .get("openai.parallel_tool_calls") - .and_then(Value::as_bool) - { - output["parallel_tool_calls"] = json!(parallel); - } - - if let Some(format) = &request.response_format { - if let Some(schema) = &format.schema { - output["response_format"] = json!({ - "type":"json_schema", - "json_schema":{ - "name":format.name.clone().unwrap_or_else(|| "structured_output".into()), - "schema":schema, - "strict":format.strict, - } - }); - remove_nulls(&mut output["response_format"]); - } else if format.kind == "json_object" || format.kind == "text" { - output["response_format"] = json!({"type":format.kind}); - } - } - - if *source == LlmProtocol::Messages { - if let Some(effort) = anthropic_reasoning_effort(request, upstream_model) { - output["reasoning_effort"] = Value::String(effort); - } - if let Some(user) = request - .metadata - .as_ref() - .and_then(|metadata| metadata.get("user_id")) - { - output["user"] = user.clone(); - } - } - output -} - -fn render_message( - message: &LlmMessage, - source: &LlmProtocol, - prompt_cache: bool, - output: &mut Vec, -) { - if message.role == LlmRole::Tool { - for part in &message.parts { - if let LlmContentPart::ToolResult { - call_id, - name, - content, - cache_control, - .. - } = part - { - let mut tool = json!({ - "role":"tool", - "tool_call_id":call_id, - "content":render_tool_result(content, cache_control.as_ref(), source, prompt_cache), - }); - if let Some(name) = name { - tool["name"] = Value::String(name.clone()); - } - output.push(tool); - } - } - return; - } - - let mut regular_parts = Vec::new(); - let mut tool_calls = Vec::new(); - for part in &message.parts { - match part { - LlmContentPart::ToolResult { - call_id, - name, - content, - cache_control, - .. - } => { - flush_regular_message(message, source, &mut regular_parts, &mut tool_calls, output); - let mut tool = json!({ - "role":"tool", - "tool_call_id":call_id, - "content":render_tool_result(content, cache_control.as_ref(), source, prompt_cache), - }); - if let Some(name) = name { - tool["name"] = Value::String(name.clone()); - } - output.push(tool); - } - LlmContentPart::ToolCall { - id, - name, - arguments, - signature, - } => { - let id = signature - .as_ref() - .map(|signature| format!("{id}__thought__{signature}")) - .unwrap_or_else(|| id.clone()); - tool_calls.push(json!({ - "id":id, - "type":"function", - "function":{ - "name":name, - "arguments":serde_json::to_string(arguments).unwrap_or_else(|_| "{}".into()) - } - })); - } - other => { - if let Some(rendered) = render_content_part(other, source, prompt_cache) { - regular_parts.push(rendered); - } - } - } - } - flush_regular_message(message, source, &mut regular_parts, &mut tool_calls, output); -} - -fn flush_regular_message( - message: &LlmMessage, - source: &LlmProtocol, - regular_parts: &mut Vec, - tool_calls: &mut Vec, - output: &mut Vec, -) { - if regular_parts.is_empty() && tool_calls.is_empty() { - return; - } - let role = match message.role { - LlmRole::System | LlmRole::Developer => "system", - LlmRole::Assistant => "assistant", - LlmRole::Tool => "tool", - LlmRole::User => "user", - }; - if *source == LlmProtocol::Responses && !regular_parts.is_empty() { - for content in std::mem::take(regular_parts) { - output.push(json!({"role":role,"content":content})); - } - if tool_calls.is_empty() { - return; - } - } - let mut rendered = json!({"role":role}); - if !regular_parts.is_empty() { - rendered["content"] = Value::Array(std::mem::take(regular_parts)); - } else if !tool_calls.is_empty() && *source == LlmProtocol::Responses { - rendered["content"] = Value::Null; - } - if !tool_calls.is_empty() { - rendered["tool_calls"] = Value::Array(std::mem::take(tool_calls)); - } - if let Some(name) = &message.name { - rendered["name"] = Value::String(name.clone()); - } - output.push(rendered); -} - -fn render_content_part( - part: &LlmContentPart, - source: &LlmProtocol, - prompt_cache: bool, -) -> Option { - match part { - LlmContentPart::Text { - text, - cache_control, - } => { - if *source == LlmProtocol::Responses { - return Some(Value::String(text.clone())); - } - let mut value = json!({"type":"text","text":text}); - if prompt_cache && cache_control.is_some() { - value["prompt_cache_breakpoint"] = json!({"mode":"explicit"}); - } - Some(value) - } - LlmContentPart::Image { - source: image, - media_type, - detail, - } => { - let url = match image { - LlmImageSource::Url { url } => url.clone(), - LlmImageSource::File { uri } => uri.clone(), - LlmImageSource::Data { data } if data.starts_with("data:") => data.clone(), - LlmImageSource::Data { data } => format!( - "data:{};base64,{data}", - media_type.as_deref().unwrap_or("application/octet-stream") - ), - }; - let mut image_url = json!({"url":url}); - if let Some(detail) = detail { - image_url["detail"] = Value::String(detail.clone()); - } - Some(json!({"type":"image_url","image_url":image_url})) - } - // Reasoning is retained in the Chronicle IR but Chat Completions has no - // lossless replay field for provider-signed thinking blocks. - LlmContentPart::Reasoning { .. } => None, - LlmContentPart::Unknown { value, .. } => Some(value.clone()), - _ => None, - } -} - -fn render_tool_result( - content: &Value, - outer_cache: Option<&Value>, - source: &LlmProtocol, - prompt_cache: bool, -) -> Value { - if *source == LlmProtocol::Responses { - return content.clone(); - } - let outer_cache = prompt_cache && outer_cache.is_some_and(Value::is_object); - match content { - Value::String(text) if outer_cache => json!([{ - "type":"text", - "text":text, - "prompt_cache_breakpoint":{"mode":"explicit"} - }]), - Value::Array(parts) => { - let trailing_cache = outer_cache - || (prompt_cache - && parts.iter().any(|part| { - part.get("type").and_then(Value::as_str) != Some("text") - && part.get("cache_control").is_some_and(Value::is_object) - })); - let mut rendered = parts - .iter() - .filter_map(|part| parse_tool_result_part(part, prompt_cache)) - .collect::>(); - if trailing_cache { - if let Some(last) = rendered.last_mut() { - last["prompt_cache_breakpoint"] = json!({"mode":"explicit"}); - } else { - rendered.push(json!({ - "type":"text", - "text":"", - "prompt_cache_breakpoint":{"mode":"explicit"} - })); - } - } - Value::Array(rendered) - } - value => value.clone(), - } -} - -fn parse_tool_result_part(part: &Value, prompt_cache: bool) -> Option { - let text = part.get("text").and_then(Value::as_str)?; - let mut rendered = json!({"type":"text","text":text}); - if prompt_cache && part.get("cache_control").is_some() { - rendered["prompt_cache_breakpoint"] = json!({"mode":"explicit"}); - } - Some(rendered) -} - -fn anthropic_reasoning_effort(request: &LlmRequest, model: &str) -> Option { - let thinking = request.extensions.get("anthropic.thinking")?; - if thinking.get("type").and_then(Value::as_str) != Some("adaptive") { - return None; - } - if !request.tools.is_empty() && !supports_reasoning_with_tools(model) { - return Some("none".into()); - } - Some( - request - .extensions - .get("anthropic.output_config") - .and_then(|value| value.get("effort")) - .and_then(Value::as_str) - .unwrap_or("high") - .into(), - ) -} - -fn supports_prompt_cache_breakpoint(model: &str) -> bool { - model - .strip_prefix("gpt-") - .and_then(|model| model.split('-').next()) - .and_then(|version| version.split_once('.')) - .and_then(|(major, minor)| Some((major.parse::().ok()?, minor.parse::().ok()?))) - .is_some_and(|version| version >= (5, 6)) -} - -fn supports_reasoning_with_tools(model: &str) -> bool { - !model.starts_with("gpt-") - || model == "gpt-5" - || model.starts_with("gpt-5-") - || model.starts_with("gpt-5.1") - || model.starts_with("gpt-5.2") -} - -fn copy_option(output: &mut Value, key: &str, value: Option) { - if let Some(value) = value { - output[key] = serde_json::to_value(value).unwrap_or(Value::Null); - } -} - -fn remove_nulls(value: &mut Value) { - if let Some(object) = value.as_object_mut() { - object.retain(|_, value| !value.is_null()); - for value in object.values_mut() { - remove_nulls(value); - } - } -} diff --git a/crates/persisting-gateway/src/conversion/tool_call.rs b/crates/persisting-gateway/src/conversion/tool_call.rs deleted file mode 100644 index cec8ca82f..000000000 --- a/crates/persisting-gateway/src/conversion/tool_call.rs +++ /dev/null @@ -1,120 +0,0 @@ -//! Shared tool-call helpers (moon-bridge Core `tool_use` / `tool_result` subset). - -use serde_json::{Value, json}; - -/// Responses input types that represent an outbound tool invocation. -pub fn is_tool_call_input_type(item_type: &str) -> bool { - matches!( - item_type, - "function_call" | "custom_tool_call" | "local_shell_call" - ) -} - -/// Responses input types that carry tool execution results back to the model. -pub fn is_tool_call_output_type(item_type: &str) -> bool { - matches!( - item_type, - "function_call_output" | "custom_tool_call_output" | "local_shell_call_output" - ) -} - -/// Build a Chat Completions `tool_calls[]` entry from a Responses input item. -pub fn chat_tool_call_from_input_item(item: &Value) -> Value { - let call_id = item - .get("call_id") - .or_else(|| item.get("id")) - .and_then(|v| v.as_str()) - .unwrap_or("call_proxy"); - let name = item.get("name").and_then(|v| v.as_str()).unwrap_or(""); - let arguments = tool_arguments_from_input_item(item); - json!({ - "id": call_id, - "type": "function", - "function": { - "name": name, - "arguments": arguments - } - }) -} - -fn tool_arguments_from_input_item(item: &Value) -> String { - if let Some(s) = item.get("arguments").and_then(|v| v.as_str()) { - return normalize_tool_arguments_string(s); - } - if let Some(input) = item.get("input").and_then(|v| v.as_str()) { - return serde_json::to_string(&json!({"input": input})) - .unwrap_or_else(|_| "{}".to_string()); - } - if let Some(action) = item.get("action") { - return serde_json::to_string(action).unwrap_or_else(|_| "{}".to_string()); - } - "{}".to_string() -} - -/// Normalize tool arguments for Chat Completions (must be a JSON object string). -pub fn normalize_tool_arguments_string(raw: &str) -> String { - let trimmed = raw.trim(); - if trimmed.is_empty() { - return "{}".to_string(); - } - if serde_json::from_str::(trimmed).is_ok() { - return trimmed.to_string(); - } - "{}".to_string() -} - -/// Decode a streaming `function.arguments` delta chunk. -/// -/// Some providers JSON-encode argument fragments; unwrap once when the chunk is -/// a quoted JSON string (moon-bridge `unquoteArguments` streaming variant). -pub fn decode_stream_arguments_delta(raw: &str) -> String { - if raw.is_empty() { - return String::new(); - } - if raw.starts_with('"') - && let Ok(decoded) = serde_json::from_str::(raw) - { - return decoded; - } - raw.to_string() -} - -/// Unwrap double-encoded tool arguments from a non-streaming Chat message. -pub fn unquote_chat_tool_arguments(raw: &Value) -> String { - match raw { - Value::String(s) => normalize_tool_arguments_string(s), - Value::Null => "{}".to_string(), - other => other.to_string(), - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn recognizes_tool_call_types() { - assert!(is_tool_call_input_type("function_call")); - assert!(is_tool_call_input_type("custom_tool_call")); - assert!(!is_tool_call_output_type("function_call")); - assert!(is_tool_call_output_type("function_call_output")); - } - - #[test] - fn chat_tool_call_from_responses_item() { - let item = json!({ - "type": "function_call", - "call_id": "call_1", - "name": "shell", - "arguments": "{\"command\":\"ls\"}" - }); - let tc = chat_tool_call_from_input_item(&item); - assert_eq!(tc["function"]["name"], "shell"); - assert_eq!(tc["id"], "call_1"); - } - - #[test] - fn decode_stream_arguments_delta_passthrough() { - assert_eq!(decode_stream_arguments_delta("{\"cmd\""), "{\"cmd\""); - } -} diff --git a/crates/persisting-gateway/src/conversion/typed_stream.rs b/crates/persisting-gateway/src/conversion/typed_stream.rs deleted file mode 100644 index d02f6de6d..000000000 --- a/crates/persisting-gateway/src/conversion/typed_stream.rs +++ /dev/null @@ -1,1405 +0,0 @@ -//! Provider SSE → Chronicle stream events → client SSE. -//! -//! Decoding and rendering are deliberately separate. No client/provider wire -//! shape is used as an intermediate protocol. - -use std::collections::{BTreeMap, HashMap}; -use std::time::Instant; - -use anyhow::{Context, Result}; -use bytes::Bytes; -use persisting_pchronicle::model::{ - LlmCandidate, LlmContentPart, LlmMessage, LlmProtocol, LlmResponse, LlmResponseEventPayload, - LlmRole, LlmStreamEvent, LlmUsage, -}; -use serde_json::{Value, json}; - -use super::{MAX_SSE_FRAME_BYTES, ProtocolBridge}; -use crate::protocol::ProtocolKind; -use crate::usage::{StreamMetrics, TokenUsage, extract_usage_from_response}; - -pub struct TypedStreamTranslator { - passthrough: bool, - decoder: StreamDecoder, - renderer: StreamRenderer, -} - -impl TypedStreamTranslator { - pub fn new(bridge: ProtocolBridge, client: ProtocolKind, client_model: &str) -> Option { - let upstream = bridge.upstream_protocol(client); - let upstream = llm_protocol(upstream)?; - let target = llm_protocol(client)?; - Some(Self { - passthrough: bridge == ProtocolBridge::Passthrough, - decoder: StreamDecoder::new(upstream, client_model), - renderer: StreamRenderer::new(target, client_model), - }) - } - - pub fn push_chunk(&mut self, chunk: &[u8]) -> Result { - if self.passthrough { - // Semantic capture is best-effort on an otherwise transparent route. - // Provider extensions or malformed frames must never alter or block - // the exact upstream bytes returned to the client. - if let Err(error) = self.decoder.push_chunk(chunk) { - tracing::warn!( - target: "persisting_gateway", - "passthrough stream semantic decode: {error:#}" - ); - } - return Ok(Bytes::copy_from_slice(chunk)); - } - let events = self.decoder.push_chunk(chunk)?; - let rendered = self.renderer.push_events(&events)?; - Ok(Bytes::from(rendered)) - } - - pub fn finish_stream(&mut self) -> Result { - if self.passthrough { - if let Err(error) = self.decoder.finish() { - tracing::warn!( - target: "persisting_gateway", - "passthrough stream semantic finish: {error:#}" - ); - } - return Ok(Bytes::new()); - } - let events = self.decoder.finish()?; - let rendered = self.renderer.finish(&events)?; - Ok(Bytes::from(rendered)) - } - - pub fn metrics(&self) -> &StreamMetrics { - self.decoder.metrics() - } - - pub fn upstream_snapshot(&self) -> &[u8] { - self.decoder.upstream_snapshot() - } - - pub fn streaming_capture_snapshot(&self) -> Option { - let text = self.decoder.accumulator.visible_text(); - (!text.trim().is_empty()).then_some(text) - } - - pub fn semantic_response(&self) -> LlmResponseEventPayload { - self.decoder.semantic_response() - } - - pub fn drain_reasoning_snapshot(&mut self) -> (Vec, String) { - self.decoder.accumulator.drain_reasoning_snapshot() - } -} - -fn llm_protocol(protocol: ProtocolKind) -> Option { - match protocol { - ProtocolKind::ChatCompletions => Some(LlmProtocol::ChatCompletions), - ProtocolKind::Messages => Some(LlmProtocol::Messages), - ProtocolKind::Responses => Some(LlmProtocol::Responses), - ProtocolKind::Gemini => Some(LlmProtocol::Gemini), - _ => None, - } -} - -struct StreamDecoder { - protocol: LlmProtocol, - fallback_model: String, - buffer: Vec, - upstream_raw: Vec, - started: Instant, - emitted_start: bool, - finished: bool, - metrics: StreamMetrics, - tool_ids: HashMap<(usize, usize), String>, - accumulator: ResponseAccumulator, -} - -impl StreamDecoder { - fn new(protocol: LlmProtocol, fallback_model: &str) -> Self { - Self { - protocol: protocol.clone(), - fallback_model: fallback_model.into(), - buffer: Vec::new(), - upstream_raw: Vec::new(), - started: Instant::now(), - emitted_start: false, - finished: false, - metrics: StreamMetrics::default(), - tool_ids: HashMap::new(), - accumulator: ResponseAccumulator::new(protocol), - } - } - - fn push_chunk(&mut self, chunk: &[u8]) -> Result> { - self.upstream_raw.extend_from_slice(chunk); - self.buffer.extend_from_slice(chunk); - let mut events = Vec::new(); - while let Some(frame) = next_sse_frame(&mut self.buffer) { - let frame = std::str::from_utf8(&frame).context("provider SSE frame is not UTF-8")?; - let data = sse_frame_data(frame); - if data.is_empty() { - continue; - } - if data == "[DONE]" { - continue; - } - let value: Value = serde_json::from_str(&data).context("parse provider SSE data")?; - match self.protocol { - LlmProtocol::Gemini => self.decode_gemini(&value, &mut events), - LlmProtocol::ChatCompletions => self.decode_chat(&value, &mut events), - LlmProtocol::Messages => self.decode_messages(&value, &mut events), - LlmProtocol::Responses => self.decode_responses(&value, &mut events), - LlmProtocol::Unknown => {} - } - } - anyhow::ensure!( - self.buffer.len() <= MAX_SSE_FRAME_BYTES, - "provider SSE frame exceeds {MAX_SSE_FRAME_BYTES} bytes" - ); - self.observe(&events); - Ok(events) - } - - fn finish(&mut self) -> Result> { - if self.finished { - return Ok(Vec::new()); - } - self.finished = true; - Ok(Vec::new()) - } - - fn decode_chat(&mut self, value: &Value, events: &mut Vec) { - self.start_from( - value.get("id").and_then(Value::as_str), - value.get("model").and_then(Value::as_str), - events, - ); - for (position, choice) in value - .get("choices") - .and_then(Value::as_array) - .into_iter() - .flatten() - .enumerate() - { - let candidate = choice - .get("index") - .and_then(Value::as_u64) - .and_then(|index| usize::try_from(index).ok()) - .unwrap_or(position); - let delta = choice.get("delta").unwrap_or(choice); - if let Some(text) = delta - .get("content") - .and_then(Value::as_str) - .filter(|s| !s.is_empty()) - { - events.push(LlmStreamEvent::TextDelta { - candidate, - text: text.into(), - }); - } - if let Some(text) = delta - .get("reasoning_content") - .or_else(|| delta.get("reasoning")) - .and_then(Value::as_str) - .filter(|s| !s.is_empty()) - { - events.push(LlmStreamEvent::ReasoningDelta { - candidate, - text: text.into(), - }); - } - for (tool_position, tool) in delta - .get("tool_calls") - .and_then(Value::as_array) - .into_iter() - .flatten() - .enumerate() - { - let index = tool - .get("index") - .and_then(Value::as_u64) - .and_then(|index| usize::try_from(index).ok()) - .unwrap_or(tool_position); - let function = tool.get("function").unwrap_or(tool); - let id = tool - .get("id") - .and_then(Value::as_str) - .filter(|id| !id.is_empty()) - .map(str::to_string) - .or_else(|| self.tool_ids.get(&(candidate, index)).cloned()) - .unwrap_or_else(|| format!("call_stream_{candidate}_{index}")); - let name = function - .get("name") - .and_then(Value::as_str) - .unwrap_or_default(); - if let std::collections::hash_map::Entry::Vacant(entry) = - self.tool_ids.entry((candidate, index)) - { - entry.insert(id.clone()); - events.push(LlmStreamEvent::ToolCallStart { - candidate, - id: id.clone(), - name: name.into(), - signature: None, - }); - } - if let Some(delta) = function - .get("arguments") - .and_then(Value::as_str) - .filter(|delta| !delta.is_empty()) - { - events.push(LlmStreamEvent::ToolArgumentsDelta { - candidate, - id, - delta: super::decode_stream_arguments_delta(delta), - }); - } - } - if let Some(reason) = choice.get("finish_reason").and_then(Value::as_str) { - events.push(LlmStreamEvent::Finish { - candidate, - reason: Some(reason.into()), - }); - } - } - self.usage_from(value, events); - } - - fn decode_gemini(&mut self, value: &Value, events: &mut Vec) { - self.start_from( - value.get("responseId").and_then(Value::as_str), - value.get("modelVersion").and_then(Value::as_str), - events, - ); - for (position, candidate_value) in value - .get("candidates") - .and_then(Value::as_array) - .into_iter() - .flatten() - .enumerate() - { - let candidate = candidate_value - .get("index") - .and_then(Value::as_u64) - .and_then(|index| usize::try_from(index).ok()) - .unwrap_or(position); - for (part_index, part) in candidate_value - .get("content") - .and_then(|content| content.get("parts")) - .and_then(Value::as_array) - .into_iter() - .flatten() - .enumerate() - { - if let Some(text) = part - .get("text") - .and_then(Value::as_str) - .filter(|s| !s.is_empty()) - { - if part - .get("thought") - .and_then(Value::as_bool) - .unwrap_or(false) - { - events.push(LlmStreamEvent::ReasoningDelta { - candidate, - text: text.into(), - }); - } else { - events.push(LlmStreamEvent::TextDelta { - candidate, - text: text.into(), - }); - } - } - if let Some(call) = part.get("functionCall") { - let id = call - .get("id") - .and_then(Value::as_str) - .filter(|id| !id.is_empty()) - .map(str::to_string) - .unwrap_or_else(|| format!("call_gemini_{candidate}_{part_index}")); - events.push(LlmStreamEvent::ToolCallStart { - candidate, - id: id.clone(), - name: call - .get("name") - .and_then(Value::as_str) - .unwrap_or_default() - .into(), - signature: part - .get("thoughtSignature") - .and_then(Value::as_str) - .map(str::to_string), - }); - events.push(LlmStreamEvent::ToolArgumentsDelta { - candidate, - id, - delta: serde_json::to_string(call.get("args").unwrap_or(&json!({}))) - .unwrap_or_else(|_| "{}".into()), - }); - } - } - if let Some(reason) = candidate_value.get("finishReason").and_then(Value::as_str) { - events.push(LlmStreamEvent::Finish { - candidate, - reason: Some(reason.into()), - }); - } - } - self.usage_from(value, events); - } - - fn decode_messages(&mut self, value: &Value, events: &mut Vec) { - match value.get("type").and_then(Value::as_str) { - Some("message_start") => { - let message = value.get("message").unwrap_or(value); - self.start_from( - message.get("id").and_then(Value::as_str), - message.get("model").and_then(Value::as_str), - events, - ); - self.usage_from(message, events); - } - Some("content_block_start") => { - let block = value.get("content_block").unwrap_or(value); - if block.get("type").and_then(Value::as_str) == Some("tool_use") { - let block_index = value - .get("index") - .and_then(Value::as_u64) - .and_then(|index| usize::try_from(index).ok()) - .unwrap_or(0); - let id = block - .get("id") - .and_then(Value::as_str) - .unwrap_or("call_stream") - .to_string(); - self.tool_ids.insert((0, block_index), id.clone()); - events.push(LlmStreamEvent::ToolCallStart { - candidate: 0, - id, - name: block - .get("name") - .and_then(Value::as_str) - .unwrap_or_default() - .into(), - signature: None, - }); - } - } - Some("content_block_delta") => { - let delta = value.get("delta").unwrap_or(value); - match delta.get("type").and_then(Value::as_str) { - Some("text_delta") => { - if let Some(text) = delta.get("text").and_then(Value::as_str) { - events.push(LlmStreamEvent::TextDelta { - candidate: 0, - text: text.into(), - }); - } - } - Some("thinking_delta") => { - if let Some(text) = delta.get("thinking").and_then(Value::as_str) { - events.push(LlmStreamEvent::ReasoningDelta { - candidate: 0, - text: text.into(), - }); - } - } - Some("input_json_delta") => { - if let Some(arguments) = delta.get("partial_json").and_then(Value::as_str) { - let block_index = value - .get("index") - .and_then(Value::as_u64) - .and_then(|index| usize::try_from(index).ok()) - .unwrap_or(0); - let id = self - .tool_ids - .get(&(0, block_index)) - .cloned() - .unwrap_or_else(|| "call_stream".into()); - events.push(LlmStreamEvent::ToolArgumentsDelta { - candidate: 0, - id, - delta: arguments.into(), - }); - } - } - _ => {} - } - } - Some("message_delta") => { - let delta = value.get("delta").unwrap_or(value); - if let Some(reason) = delta.get("stop_reason").and_then(Value::as_str) { - events.push(LlmStreamEvent::Finish { - candidate: 0, - reason: Some(reason.into()), - }); - } - self.usage_from(value, events); - } - _ => {} - } - } - - fn decode_responses(&mut self, value: &Value, events: &mut Vec) { - let event_type = value - .get("type") - .and_then(Value::as_str) - .unwrap_or_default(); - if !self.emitted_start { - let response = value.get("response").unwrap_or(value); - self.start_from( - response.get("id").and_then(Value::as_str), - response.get("model").and_then(Value::as_str), - events, - ); - } - match event_type { - "response.output_text.delta" => { - if let Some(text) = value.get("delta").and_then(Value::as_str) { - events.push(LlmStreamEvent::TextDelta { - candidate: 0, - text: text.into(), - }); - } - } - "response.reasoning_summary_text.delta" => { - if let Some(text) = value.get("delta").and_then(Value::as_str) { - events.push(LlmStreamEvent::ReasoningDelta { - candidate: 0, - text: text.into(), - }); - } - } - "response.output_item.added" => { - let item = value.get("item").unwrap_or(value); - if matches!( - item.get("type").and_then(Value::as_str), - Some("function_call" | "custom_tool_call") - ) { - let id = item - .get("call_id") - .or_else(|| item.get("id")) - .and_then(Value::as_str) - .unwrap_or("call_stream"); - events.push(LlmStreamEvent::ToolCallStart { - candidate: 0, - id: id.into(), - name: item - .get("name") - .and_then(Value::as_str) - .unwrap_or_default() - .into(), - signature: None, - }); - } - } - "response.function_call_arguments.delta" | "response.custom_tool_call_input.delta" => { - let id = value - .get("call_id") - .or_else(|| value.get("item_id")) - .and_then(Value::as_str) - .unwrap_or("call_stream"); - if let Some(delta) = value.get("delta").and_then(Value::as_str) { - events.push(LlmStreamEvent::ToolArgumentsDelta { - candidate: 0, - id: id.into(), - delta: delta.into(), - }); - } - } - "response.completed" | "response.incomplete" | "response.failed" => { - let response = value.get("response").unwrap_or(value); - self.usage_from(response, events); - events.push(LlmStreamEvent::Finish { - candidate: 0, - reason: response - .get("status") - .and_then(Value::as_str) - .map(str::to_string), - }); - } - _ => {} - } - } - - fn start_from( - &mut self, - id: Option<&str>, - model: Option<&str>, - events: &mut Vec, - ) { - if self.emitted_start { - if self.accumulator.model.is_none() { - self.accumulator.model = model.map(str::to_string); - } - return; - } - self.emitted_start = true; - let id = id.map(str::to_string); - let model = model - .map(str::to_string) - .or_else(|| Some(self.fallback_model.clone())); - events.push(LlmStreamEvent::Start { id, model }); - } - - fn usage_from(&self, value: &Value, events: &mut Vec) { - let usage = extract_usage_from_response(value); - if usage != TokenUsage::default() { - events.push(LlmStreamEvent::Usage { - usage: to_llm_usage(&usage), - }); - } - } - - fn observe(&mut self, events: &[LlmStreamEvent]) { - if self.metrics.ttft_ms.is_none() - && events.iter().any(|event| { - matches!( - event, - LlmStreamEvent::TextDelta { .. } - | LlmStreamEvent::ReasoningDelta { .. } - | LlmStreamEvent::ToolCallStart { .. } - ) - }) - { - self.metrics.ttft_ms = Some(self.started.elapsed().as_millis() as u64); - } - for event in events { - if let LlmStreamEvent::Usage { usage } = event { - self.metrics.usage = from_llm_usage(usage); - } - self.accumulator.apply(event); - } - } - - fn metrics(&self) -> &StreamMetrics { - &self.metrics - } - fn upstream_snapshot(&self) -> &[u8] { - &self.upstream_raw - } - - fn semantic_response(&self) -> LlmResponseEventPayload { - self.accumulator.response() - } -} - -#[derive(Default)] -struct CandidateState { - text: String, - reasoning: String, - tools: Vec, - finish_reason: Option, -} - -struct ToolState { - id: String, - name: String, - arguments: String, - signature: Option, -} - -struct ResponseAccumulator { - protocol: LlmProtocol, - id: Option, - model: Option, - usage: Option, - candidates: BTreeMap, -} - -impl ResponseAccumulator { - fn new(protocol: LlmProtocol) -> Self { - Self { - protocol, - id: None, - model: None, - usage: None, - candidates: BTreeMap::new(), - } - } - - fn apply(&mut self, event: &LlmStreamEvent) { - match event { - LlmStreamEvent::Start { id, model } => { - self.id = id.clone(); - self.model = model.clone(); - } - LlmStreamEvent::TextDelta { candidate, text } => self - .candidates - .entry(*candidate) - .or_default() - .text - .push_str(text), - LlmStreamEvent::ReasoningDelta { candidate, text } => self - .candidates - .entry(*candidate) - .or_default() - .reasoning - .push_str(text), - LlmStreamEvent::ToolCallStart { - candidate, - id, - name, - signature, - } => { - let state = self.candidates.entry(*candidate).or_default(); - if let Some(tool) = state.tools.iter_mut().find(|tool| tool.id == *id) { - tool.name = name.clone(); - tool.signature = signature.clone(); - } else { - state.tools.push(ToolState { - id: id.clone(), - name: name.clone(), - arguments: String::new(), - signature: signature.clone(), - }); - } - } - LlmStreamEvent::ToolArgumentsDelta { - candidate, - id, - delta, - } => { - let state = self.candidates.entry(*candidate).or_default(); - if let Some(tool) = state.tools.iter_mut().find(|tool| tool.id == *id) { - tool.arguments.push_str(delta); - } else { - state.tools.push(ToolState { - id: id.clone(), - name: String::new(), - arguments: delta.clone(), - signature: None, - }); - } - } - LlmStreamEvent::Usage { usage } => self.usage = Some(usage.clone()), - LlmStreamEvent::Finish { candidate, reason } => { - self.candidates.entry(*candidate).or_default().finish_reason = reason.clone() - } - LlmStreamEvent::Error { .. } => {} - } - } - - fn visible_text(&self) -> String { - let text = self - .candidates - .values() - .map(|candidate| candidate.text.as_str()) - .collect::(); - if text.trim().is_empty() { - self.candidates - .values() - .map(|candidate| candidate.reasoning.as_str()) - .collect() - } else { - text - } - } - - fn drain_reasoning_snapshot(&mut self) -> (Vec, String) { - let ids = self - .candidates - .values() - .flat_map(|candidate| candidate.tools.iter().map(|tool| tool.id.clone())) - .collect(); - let reasoning = self - .candidates - .values_mut() - .map(|candidate| std::mem::take(&mut candidate.reasoning)) - .collect(); - (ids, reasoning) - } - - fn response(&self) -> LlmResponseEventPayload { - let candidates = self - .candidates - .iter() - .map(|(index, state)| { - let mut parts = Vec::new(); - if !state.reasoning.is_empty() { - parts.push(LlmContentPart::Reasoning { - text: Some(state.reasoning.clone()), - signature: None, - }); - } - if !state.text.is_empty() { - parts.push(LlmContentPart::text(&state.text)); - } - for tool in &state.tools { - let arguments = serde_json::from_str(&tool.arguments) - .unwrap_or_else(|_| Value::String(tool.arguments.clone())); - parts.push(LlmContentPart::ToolCall { - id: tool.id.clone(), - name: tool.name.clone(), - arguments, - signature: tool.signature.clone(), - }); - } - LlmCandidate { - index: *index, - message: LlmMessage::new(LlmRole::Assistant, parts), - finish_reason: state.finish_reason.clone(), - extensions: BTreeMap::new(), - } - }) - .collect(); - LlmResponseEventPayload { - output_format: self.protocol.clone(), - response: LlmResponse { - id: self.id.clone(), - model: self.model.clone(), - candidates, - usage: self.usage.clone(), - extensions: BTreeMap::new(), - }, - } - } -} - -struct StreamRenderer { - protocol: LlmProtocol, - model: String, - id: String, - created: i64, - sequence: u64, - started: bool, - finished: bool, - usage: LlmUsage, - finish_reason: Option, - blocks: HashMap, - tool_names: HashMap, - tool_arguments: HashMap, - accumulated_text: String, - text_block: Option, - reasoning_block: Option, - next_block: usize, -} - -impl StreamRenderer { - fn new(protocol: LlmProtocol, model: &str) -> Self { - let timestamp = chrono::Utc::now().timestamp_millis(); - Self { - protocol, - model: model.into(), - id: format!("stream_{timestamp}"), - created: timestamp / 1000, - sequence: 0, - started: false, - finished: false, - usage: LlmUsage::default(), - finish_reason: None, - blocks: HashMap::new(), - tool_names: HashMap::new(), - tool_arguments: HashMap::new(), - accumulated_text: String::new(), - text_block: None, - reasoning_block: None, - next_block: 0, - } - } - - fn push_events(&mut self, events: &[LlmStreamEvent]) -> Result { - let mut output = String::new(); - for event in events { - match self.protocol { - LlmProtocol::ChatCompletions => self.render_chat(event, &mut output)?, - LlmProtocol::Messages => self.render_messages(event, &mut output)?, - LlmProtocol::Responses => self.render_responses(event, &mut output)?, - LlmProtocol::Gemini => self.render_gemini(event, &mut output)?, - LlmProtocol::Unknown => {} - } - } - Ok(output) - } - - fn finish(&mut self, events: &[LlmStreamEvent]) -> Result { - let mut output = self.push_events(events)?; - if self.finished { - return Ok(output); - } - self.finished = true; - match self.protocol { - LlmProtocol::ChatCompletions => output.push_str("data: [DONE]\n\n"), - LlmProtocol::Messages => self.finish_messages(&mut output)?, - LlmProtocol::Responses => self.finish_responses(&mut output)?, - LlmProtocol::Gemini | LlmProtocol::Unknown => {} - } - Ok(output) - } - - fn render_chat(&mut self, event: &LlmStreamEvent, output: &mut String) -> Result<()> { - let mut choice = None; - match event { - LlmStreamEvent::Start { id, .. } => { - if let Some(id) = id { - self.id = id.clone(); - } - choice = Some(json!({"index":0,"delta":{"role":"assistant"},"finish_reason":null})); - } - LlmStreamEvent::TextDelta { candidate, text } => { - choice = - Some(json!({"index":candidate,"delta":{"content":text},"finish_reason":null})) - } - LlmStreamEvent::ReasoningDelta { candidate, text } => { - choice = Some( - json!({"index":candidate,"delta":{"reasoning_content":text},"finish_reason":null}), - ) - } - LlmStreamEvent::ToolCallStart { - candidate, - id, - name, - signature, - } => { - let wire_id = signature - .as_ref() - .map(|signature| format!("{id}__thought__{signature}")) - .unwrap_or_else(|| id.clone()); - let index = self.ensure_chat_tool_index(id); - choice = Some( - json!({"index":candidate,"delta":{"tool_calls":[{"index":index,"id":wire_id,"type":"function","function":{"name":name,"arguments":""}}]},"finish_reason":null}), - ); - } - LlmStreamEvent::ToolArgumentsDelta { - candidate, - id, - delta, - } => { - let index = self.ensure_chat_tool_index(id); - choice = Some( - json!({"index":candidate,"delta":{"tool_calls":[{"index":index,"function":{"arguments":delta}}]},"finish_reason":null}), - ); - } - LlmStreamEvent::Usage { usage } => self.usage = usage.clone(), - LlmStreamEvent::Finish { candidate, reason } => { - self.finish_reason = reason.clone(); - choice = Some( - json!({"index":candidate,"delta":{},"finish_reason":super::semantic::chat_finish_reason(reason.as_deref(), !self.blocks.is_empty())}), - ); - } - LlmStreamEvent::Error { message, code } => { - output.push_str(&format_sse( - None, - &json!({"error":{"message":message,"code":code}}), - )?); - return Ok(()); - } - } - let mut value = json!({ - "id":self.id,"object":"chat.completion.chunk","created":self.created,"model":self.model, - "choices":choice.into_iter().collect::>(), - }); - if matches!(event, LlmStreamEvent::Usage { .. }) { - value["usage"] = super::semantic::render_chat_usage(&self.usage); - } - output.push_str(&format_sse(None, &value)?); - Ok(()) - } - - fn render_messages(&mut self, event: &LlmStreamEvent, output: &mut String) -> Result<()> { - self.ensure_messages_start(event, output)?; - match event { - LlmStreamEvent::TextDelta { text, .. } => { - self.accumulated_text.push_str(text); - let index = self.ensure_message_block("text", None, output)?; - output.push_str(&format_sse(Some("content_block_delta"), &json!({"type":"content_block_delta","index":index,"delta":{"type":"text_delta","text":text}}))?); - } - LlmStreamEvent::ReasoningDelta { text, .. } => { - let index = self.ensure_message_block("thinking", None, output)?; - output.push_str(&format_sse(Some("content_block_delta"), &json!({"type":"content_block_delta","index":index,"delta":{"type":"thinking_delta","thinking":text}}))?); - } - LlmStreamEvent::ToolCallStart { id, name, .. } => { - self.tool_names.insert(id.clone(), name.clone()); - self.ensure_message_block("tool_use", Some((id, name)), output)?; - } - LlmStreamEvent::ToolArgumentsDelta { id, delta, .. } => { - let empty_name = String::new(); - let index = - self.ensure_message_block("tool_use", Some((id, &empty_name)), output)?; - self.tool_arguments - .entry(id.clone()) - .or_default() - .push_str(delta); - output.push_str(&format_sse(Some("content_block_delta"), &json!({"type":"content_block_delta","index":index,"delta":{"type":"input_json_delta","partial_json":delta}}))?); - } - LlmStreamEvent::Usage { usage } => self.usage = usage.clone(), - LlmStreamEvent::Finish { reason, .. } => self.finish_reason = reason.clone(), - _ => {} - } - Ok(()) - } - - fn ensure_messages_start(&mut self, event: &LlmStreamEvent, output: &mut String) -> Result<()> { - if let LlmStreamEvent::Start { id: Some(id), .. } = event { - self.id = id.clone(); - } - if !self.started { - self.started = true; - output.push_str(&format_sse(Some("message_start"), &json!({"type":"message_start","message":{"id":self.id,"type":"message","role":"assistant","model":self.model,"content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":0,"output_tokens":0}}}))?); - } - Ok(()) - } - - fn ensure_message_block( - &mut self, - kind: &str, - tool: Option<(&String, &String)>, - output: &mut String, - ) -> Result { - let existing = match kind { - "text" => self.text_block, - "thinking" => self.reasoning_block, - _ => tool.and_then(|(id, _)| self.blocks.get(id).copied()), - }; - if let Some(index) = existing { - return Ok(index); - } - let index = self.next_block; - self.next_block += 1; - let block = match (kind, tool) { - ("tool_use", Some((id, name))) => { - self.blocks.insert(id.clone(), index); - self.tool_names.insert(id.clone(), name.clone()); - self.tool_arguments.entry(id.clone()).or_default(); - json!({"type":"tool_use","id":id,"name":name,"input":{}}) - } - ("thinking", _) => { - self.reasoning_block = Some(index); - json!({"type":"thinking","thinking":""}) - } - _ => { - self.text_block = Some(index); - json!({"type":"text","text":""}) - } - }; - output.push_str(&format_sse( - Some("content_block_start"), - &json!({"type":"content_block_start","index":index,"content_block":block}), - )?); - Ok(index) - } - - fn ensure_chat_tool_index(&mut self, id: &str) -> usize { - if let Some(index) = self.blocks.get(id) { - return *index; - } - let index = self.blocks.len(); - self.blocks.insert(id.to_string(), index); - index - } - - fn finish_messages(&mut self, output: &mut String) -> Result<()> { - for index in 0..self.next_block { - output.push_str(&format_sse( - Some("content_block_stop"), - &json!({"type":"content_block_stop","index":index}), - )?); - } - let mut usage = json!({ - "input_tokens": self.usage.input_tokens, - "output_tokens": self.usage.output_tokens, - "cache_read_input_tokens": self.usage.cache_read_tokens, - }); - if self.usage.cache_write_tokens > 0 { - usage["cache_creation_input_tokens"] = json!(self.usage.cache_write_tokens); - } - output.push_str(&format_sse(Some("message_delta"), &json!({"type":"message_delta","delta":{"stop_reason":super::semantic::anthropic_finish_reason(self.finish_reason.as_deref(), !self.blocks.is_empty()),"stop_sequence":null},"usage":usage}))?); - output.push_str(&format_sse( - Some("message_stop"), - &json!({"type":"message_stop"}), - )?); - Ok(()) - } - - fn render_responses(&mut self, event: &LlmStreamEvent, output: &mut String) -> Result<()> { - if let LlmStreamEvent::Start { id: Some(id), .. } = event { - self.id = format!("resp_{}", super::semantic::safe_id_suffix(id)); - } - if !self.started { - self.started = true; - self.emit_response_event("response.created", json!({"type":"response.created","response":{"id":self.id,"object":"response","status":"in_progress","model":self.model,"output":[]}}), output)?; - } - match event { - LlmStreamEvent::TextDelta { text, .. } => { - self.accumulated_text.push_str(text); - if self.text_block.is_none() { - let index = self.next_block; - self.next_block += 1; - self.text_block = Some(index); - self.emit_response_event("response.output_item.added", json!({"type":"response.output_item.added","output_index":index,"item":{"type":"message","id":format!("msg_{}",self.id),"role":"assistant","status":"in_progress","content":[]}}), output)?; - self.emit_response_event("response.content_part.added", json!({"type":"response.content_part.added","item_id":format!("msg_{}",self.id),"output_index":index,"content_index":0,"part":{"type":"output_text","text":"","annotations":[]}}), output)?; - } - let index = self.text_block.unwrap_or(0); - self.emit_response_event("response.output_text.delta", json!({"type":"response.output_text.delta","item_id":format!("msg_{}",self.id),"output_index":index,"content_index":0,"delta":text,"logprobs":[]}), output)?; - } - LlmStreamEvent::ToolCallStart { id, name, .. } => { - self.tool_names.insert(id.clone(), name.clone()); - self.tool_arguments.entry(id.clone()).or_default(); - if !self.blocks.contains_key(id) { - self.start_response_tool(id, name, output)?; - } - } - LlmStreamEvent::ToolArgumentsDelta { id, delta, .. } => { - let index = match self.blocks.get(id).copied() { - Some(index) => index, - None => self.start_response_tool(id, "", output)?, - }; - self.tool_arguments - .entry(id.clone()) - .or_default() - .push_str(delta); - self.emit_response_event("response.function_call_arguments.delta", json!({"type":"response.function_call_arguments.delta","item_id":format!("fc_{id}"),"output_index":index,"delta":delta}), output)?; - } - LlmStreamEvent::ReasoningDelta { .. } => {} - LlmStreamEvent::Usage { usage } => self.usage = usage.clone(), - LlmStreamEvent::Finish { reason, .. } => self.finish_reason = reason.clone(), - _ => {} - } - Ok(()) - } - - fn start_response_tool(&mut self, id: &str, name: &str, output: &mut String) -> Result { - let index = self.next_block; - self.next_block += 1; - self.blocks.insert(id.to_string(), index); - self.tool_names - .entry(id.to_string()) - .or_insert_with(|| name.to_string()); - self.tool_arguments.entry(id.to_string()).or_default(); - self.emit_response_event("response.output_item.added", json!({"type":"response.output_item.added","output_index":index,"item":{"type":"function_call","id":format!("fc_{id}"),"call_id":id,"name":name,"arguments":"","status":"in_progress"}}), output)?; - Ok(index) - } - - fn emit_response_event( - &mut self, - name: &str, - mut value: Value, - output: &mut String, - ) -> Result<()> { - self.sequence += 1; - value["sequence_number"] = json!(self.sequence); - output.push_str(&format_sse(Some(name), &value)?); - Ok(()) - } - - fn finish_responses(&mut self, output: &mut String) -> Result<()> { - let mut completed_output = Vec::new(); - if let Some(index) = self.text_block { - let item_id = format!("msg_{}", self.id); - let text = self.accumulated_text.clone(); - let item = json!({"type":"message","id":item_id,"role":"assistant","status":"completed","content":[{"type":"output_text","text":text,"annotations":[],"logprobs":[]}]}); - self.emit_response_event("response.output_text.done", json!({"type":"response.output_text.done","item_id":item_id,"output_index":index,"content_index":0,"text":text,"logprobs":[]}), output)?; - self.emit_response_event("response.content_part.done", json!({"type":"response.content_part.done","item_id":item_id,"output_index":index,"content_index":0,"part":{"type":"output_text","text":text,"annotations":[]}}), output)?; - self.emit_response_event( - "response.output_item.done", - json!({"type":"response.output_item.done","output_index":index,"item":item}), - output, - )?; - completed_output.push((index, item)); - } - let mut tools = self - .blocks - .iter() - .map(|(id, index)| { - ( - *index, - id.clone(), - self.tool_names.get(id).cloned().unwrap_or_default(), - self.tool_arguments.get(id).cloned().unwrap_or_default(), - ) - }) - .collect::>(); - tools.sort_by_key(|(index, ..)| *index); - for (index, id, name, arguments) in tools { - let item_id = format!("fc_{id}"); - let item = json!({"type":"function_call","id":item_id,"call_id":id,"name":name,"arguments":arguments,"status":"completed"}); - self.emit_response_event("response.function_call_arguments.done", json!({"type":"response.function_call_arguments.done","item_id":item_id,"output_index":index,"arguments":arguments}), output)?; - self.emit_response_event( - "response.output_item.done", - json!({"type":"response.output_item.done","output_index":index,"item":item}), - output, - )?; - completed_output.push((index, item)); - } - completed_output.sort_by_key(|(index, _)| *index); - let completed_output = completed_output - .into_iter() - .map(|(_, item)| item) - .collect::>(); - let status = match self.finish_reason.as_deref() { - Some("length" | "max_tokens" | "MAX_TOKENS" | "incomplete") => "incomplete", - Some("failed" | "content_filter") => "failed", - _ => "completed", - }; - let usage = self.usage.clone(); - self.emit_response_event("response.completed", json!({"type":"response.completed","response":{"id":self.id,"object":"response","status":status,"model":self.model,"output":completed_output,"usage":{"input_tokens":usage.input_tokens,"output_tokens":usage.output_tokens,"total_tokens":usage.total_tokens}}}), output) - } - - fn render_gemini(&mut self, event: &LlmStreamEvent, output: &mut String) -> Result<()> { - if let LlmStreamEvent::Start { id: Some(id), .. } = event { - self.id = id.clone(); - } - let mut parts = Vec::new(); - let mut finish = None; - match event { - LlmStreamEvent::TextDelta { text, .. } => parts.push(json!({"text":text})), - LlmStreamEvent::ReasoningDelta { text, .. } => { - parts.push(json!({"text":text,"thought":true})) - } - LlmStreamEvent::ToolCallStart { - id, - name, - signature, - .. - } => { - let mut p = json!({"functionCall":{"id":id,"name":name,"args":{}}}); - if let Some(signature) = signature { - p["thoughtSignature"] = json!(signature); - } - parts.push(p); - } - LlmStreamEvent::Usage { usage } => self.usage = usage.clone(), - LlmStreamEvent::Finish { reason, .. } => { - finish = Some(super::semantic::gemini_finish_reason(reason.as_deref())) - } - _ => {} - } - if !parts.is_empty() || finish.is_some() || matches!(event, LlmStreamEvent::Usage { .. }) { - let mut value = json!({"responseId":self.id,"modelVersion":self.model,"candidates":[]}); - if !parts.is_empty() || finish.is_some() { - value["candidates"] = json!([{"index":0,"content":{"role":"model","parts":parts},"finishReason":finish}]); - } - if matches!(event, LlmStreamEvent::Usage { .. }) { - value["usageMetadata"] = json!({"promptTokenCount":self.usage.input_tokens,"candidatesTokenCount":self.usage.output_tokens,"totalTokenCount":self.usage.total_tokens}); - } - output.push_str(&format_sse(None, &value)?); - } - Ok(()) - } -} - -fn next_sse_frame(buffer: &mut Vec) -> Option> { - let lf = buffer - .windows(2) - .position(|window| window == b"\n\n") - .map(|position| (position, 2)); - let crlf = buffer - .windows(4) - .position(|window| window == b"\r\n\r\n") - .map(|position| (position, 4)); - let (position, delimiter) = match (lf, crlf) { - (Some(left), Some(right)) => std::cmp::min(left, right), - (Some(found), None) | (None, Some(found)) => found, - (None, None) => return None, - }; - let remainder = buffer.split_off(position + delimiter); - buffer.truncate(position); - let frame = std::mem::replace(buffer, remainder); - Some(frame) -} - -fn sse_frame_data(frame: &str) -> String { - frame - .lines() - .filter_map(|line| line.trim().strip_prefix("data:").map(str::trim)) - .collect::>() - .join("\n") -} - -fn format_sse(event: Option<&str>, value: &Value) -> Result { - let data = serde_json::to_string(value)?; - Ok(match event { - Some(event) => format!("event: {event}\ndata: {data}\n\n"), - None => format!("data: {data}\n\n"), - }) -} - -fn to_llm_usage(usage: &TokenUsage) -> LlmUsage { - LlmUsage { - input_tokens: usage.input_tokens, - output_tokens: usage.output_tokens, - total_tokens: usage.total_tokens, - cache_read_tokens: usage.cache_read_tokens, - cache_write_tokens: usage.cache_write_tokens, - reasoning_tokens: usage.reasoning_tokens, - } -} - -fn from_llm_usage(usage: &LlmUsage) -> TokenUsage { - TokenUsage { - input_tokens: usage.input_tokens, - output_tokens: usage.output_tokens, - total_tokens: usage.total_tokens, - cache_read_tokens: usage.cache_read_tokens, - cache_write_tokens: usage.cache_write_tokens, - reasoning_tokens: usage.reasoning_tokens, - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn chat_stream_roundtrips_through_typed_events() { - let mut translator = TypedStreamTranslator::new( - ProtocolBridge::MessagesToCompletions, - ProtocolKind::Messages, - "client-model", - ) - .unwrap(); - let output = translator.push_chunk(b"data: {\"id\":\"x\",\"model\":\"m\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"hi\"},\"finish_reason\":null}]}\n\ndata: {\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":1,\"completion_tokens\":1,\"total_tokens\":2}}\n\n").unwrap(); - let tail = translator.finish_stream().unwrap(); - assert!( - std::str::from_utf8(&output) - .unwrap() - .contains("content_block_delta") - ); - assert!(std::str::from_utf8(&tail).unwrap().contains("message_stop")); - assert_eq!( - translator.semantic_response().response.candidates[0] - .message - .parts[0], - LlmContentPart::text("hi") - ); - assert_eq!(translator.metrics().usage.total_tokens, 2); - } - - #[test] - fn passthrough_preserves_unicode_split_at_every_byte_boundary() { - let input = "data: {\"choices\":[{\"delta\":{\"content\":\"你好🌍\"}}]}\n\n".as_bytes(); - for split in 0..=input.len() { - let mut translator = TypedStreamTranslator::new( - ProtocolBridge::Passthrough, - ProtocolKind::ChatCompletions, - "client-model", - ) - .unwrap(); - let mut output = Vec::new(); - output.extend_from_slice(&translator.push_chunk(&input[..split]).unwrap()); - output.extend_from_slice(&translator.push_chunk(&input[split..]).unwrap()); - output.extend_from_slice(&translator.finish_stream().unwrap()); - assert_eq!(output, input, "split at byte {split}"); - assert_eq!(translator.upstream_snapshot(), input); - } - } - - #[test] - fn translated_stream_decodes_unicode_only_after_complete_frame() { - let input = "data: {\"id\":\"x\",\"choices\":[{\"delta\":{\"content\":\"你好🌍\"}}]}\n\n"; - let split = input.find('好').unwrap() + 1; - let mut translator = TypedStreamTranslator::new( - ProtocolBridge::MessagesToCompletions, - ProtocolKind::Messages, - "client-model", - ) - .unwrap(); - assert!( - translator - .push_chunk(&input.as_bytes()[..split]) - .unwrap() - .is_empty() - ); - let output = translator.push_chunk(&input.as_bytes()[split..]).unwrap(); - assert!(std::str::from_utf8(&output).unwrap().contains("你好🌍")); - assert_eq!(translator.upstream_snapshot(), input.as_bytes()); - } - - #[test] - fn passthrough_preserves_malformed_and_non_utf8_provider_bytes() { - let input = b"data: \xff\xfe\n\ndata: not-json\n\n"; - let mut translator = TypedStreamTranslator::new( - ProtocolBridge::Passthrough, - ProtocolKind::ChatCompletions, - "client-model", - ) - .unwrap(); - let output = translator.push_chunk(input).unwrap(); - assert_eq!(output.as_ref(), input); - assert_eq!(translator.upstream_snapshot(), input); - } - - #[test] - fn responses_text_events_include_required_logprobs_fields() { - let input = b"data: {\"id\":\"x\",\"model\":\"m\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"hi\"},\"finish_reason\":null}]}\n\ndata: {\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\n"; - let mut translator = TypedStreamTranslator::new( - ProtocolBridge::ResponsesToCompletions, - ProtocolKind::Responses, - "client-model", - ) - .unwrap(); - let output = translator.push_chunk(input).unwrap(); - let tail = translator.finish_stream().unwrap(); - let output = std::str::from_utf8(&output).unwrap(); - assert!( - output.contains(r#""type":"response.output_text.delta""#), - "{output}" - ); - assert!(output.contains(r#""logprobs":[]"#)); - let tail = std::str::from_utf8(&tail).unwrap(); - assert!(tail.contains(r#""type":"response.output_text.done""#)); - assert!(tail.contains(r#""logprobs":[]"#)); - let completed = tail - .split("\n\n") - .find_map(|frame| { - let data = frame.lines().find_map(|line| line.strip_prefix("data: "))?; - let value: Value = serde_json::from_str(data).ok()?; - (value["type"] == "response.completed").then_some(value) - }) - .expect("response.completed event"); - assert_eq!(completed["response"]["output"][0]["type"], "message"); - assert_eq!( - completed["response"]["output"][0]["content"][0]["text"], - "hi" - ); - } - - #[test] - fn orphan_tool_delta_reserves_a_unique_responses_output_index() { - let mut renderer = StreamRenderer::new(LlmProtocol::Responses, "client-model"); - let orphan = renderer - .push_events(&[LlmStreamEvent::ToolArgumentsDelta { - candidate: 0, - id: "call-orphan".into(), - delta: "{\"city\":".into(), - }]) - .unwrap(); - assert!(orphan.contains("response.output_item.added")); - assert!(orphan.contains("\"output_index\":0")); - - let text = renderer - .push_events(&[LlmStreamEvent::TextDelta { - candidate: 0, - text: "hello".into(), - }]) - .unwrap(); - assert!(text.contains("\"output_index\":1")); - - let late_start = renderer - .push_events(&[LlmStreamEvent::ToolCallStart { - candidate: 0, - id: "call-orphan".into(), - name: "weather".into(), - signature: None, - }]) - .unwrap(); - assert!(!late_start.contains("response.output_item.added")); - - let finished = renderer.finish(&[]).unwrap(); - assert!(finished.contains("\"output_index\":0")); - assert!(finished.contains("\"name\":\"weather\"")); - let completed = finished - .split("\n\n") - .find_map(|frame| { - let data = frame.lines().find_map(|line| line.strip_prefix("data: "))?; - let value: Value = serde_json::from_str(data).ok()?; - (value["type"] == "response.completed").then_some(value) - }) - .expect("response.completed event"); - assert_eq!(completed["response"]["output"].as_array().unwrap().len(), 2); - assert_eq!(completed["response"]["output"][0]["type"], "function_call"); - assert_eq!(completed["response"]["output"][1]["type"], "message"); - } -} diff --git a/crates/persisting-gateway/src/dead_letter.rs b/crates/persisting-gateway/src/dead_letter.rs deleted file mode 100644 index a17007046..000000000 --- a/crates/persisting-gateway/src/dead_letter.rs +++ /dev/null @@ -1,674 +0,0 @@ -//! Failed capture events — append-only JSONL for recovery (`{storage}/.capture/dead_letter.jsonl`). - -use std::fs::File; -use std::io::{BufRead, BufReader, Write}; -use std::path::{Path, PathBuf}; - -use anyhow::{Context, Result}; -use bytes::Bytes; -use serde::{Deserialize, Serialize}; -use serde_json::Value; - -use crate::config::CaptureLevel; -use crate::engine::{ - Call, CallContext, CancelEvent, CompleteEvent, DraftEvent, Event, RequestEvent, -}; -use crate::protocol::ProtocolKind; -use crate::provider::ProviderKind; -use crate::runtime::open_private_append_file; -use crate::session::storage::CaptureRoute; -use crate::sink::{redact_sensitive_body, redact_sensitive_headers, redact_sensitive_url}; -use crate::usage::StreamMetrics; - -fn default_post() -> String { - "POST".into() -} - -const DEAD_LETTER_FILENAME: &str = "dead_letter.jsonl"; -const TRAJECTORY_DEAD_LETTER_FILENAME: &str = "trajectory_dead_letter.jsonl"; - -mod serde_compat { - use serde::{Deserialize, Deserializer, Serializer}; - - use crate::protocol::ProtocolKind; - use crate::provider::ProviderKind; - - pub mod provider { - use super::*; - - pub fn serialize(v: &ProviderKind, s: S) -> Result - where - S: Serializer, - { - s.serialize_str(v.as_str()) - } - - pub fn deserialize<'de, D>(d: D) -> Result - where - D: Deserializer<'de>, - { - let s = String::deserialize(d)?; - Ok(ProviderKind::parse(&s)) - } - } - - pub mod protocol { - use super::*; - - pub fn serialize(v: &ProtocolKind, s: S) -> Result - where - S: Serializer, - { - s.serialize_str(v.as_str()) - } - - pub fn deserialize<'de, D>(d: D) -> Result - where - D: Deserializer<'de>, - { - let s = String::deserialize(d)?; - Ok(ProtocolKind::parse(&s)) - } - } -} - -/// Serializable call context for dead-letter replay. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct DeadLetterContext { - pub route: CaptureRoute, - pub agent_id: String, - pub call: Call, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub request_headers: Vec<(String, String)>, - pub level: CaptureLevel, - pub client_model: String, - pub upstream_model: String, - #[serde(with = "serde_compat::provider")] - pub provider: ProviderKind, - #[serde(with = "serde_compat::protocol")] - pub protocol: ProtocolKind, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub client_peer: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub client_meta: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub http_version: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub upstream_url: Option, -} - -/// Serializable event (wire format for JSONL). -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(tag = "kind", rename_all = "snake_case")] -pub enum SerializableEvent { - Request { - path: String, - #[serde(default = "default_post")] - method: String, - #[serde(default)] - url: Option, - body_bytes: usize, - user_content: Option, - body_json: Option, - model_rewritten: bool, - #[serde(default)] - headers: Vec<(String, String)>, - }, - ResponseComplete { - status: u16, - resp_payload: RespPayload, - streaming: bool, - stream_metrics: Option, - assistant_content: Option, - #[serde(default)] - headers: Vec<(String, String)>, - }, - ResponseDraft { - status: u16, - assistant_content: String, - }, - Cancelled { - status: u16, - bytes_received: usize, - streaming: bool, - }, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(untagged)] -pub enum RespPayload { - Text(String), - Bytes(Vec), -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -pub struct DeadLetterEntry { - pub timestamp: String, - #[serde(alias = "invocation")] - pub context: DeadLetterContext, - pub event: SerializableEvent, - pub error: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub prepared_record_json: Option, -} - -pub fn dead_letter_path(storage: &Path) -> PathBuf { - storage.join(".capture").join(DEAD_LETTER_FILENAME) -} - -pub fn trajectory_dead_letter_path(storage: &Path) -> PathBuf { - storage - .join(".capture") - .join(TRAJECTORY_DEAD_LETTER_FILENAME) -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -pub struct TrajectoryDeadLetterEntry { - pub timestamp: String, - pub storage: String, - pub agent_id: String, - pub session_id: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub root_session: Option, - #[serde(default)] - pub records: Vec, - pub error: String, -} - -impl TrajectoryDeadLetterEntry { - pub fn decoded_records(&self) -> Result> { - Ok(self.records.clone()) - } -} - -pub fn append_trajectory_dead_letter( - storage: &Path, - agent_id: &str, - session_id: &str, - root_session: Option<&str>, - records: &[persisting_events::EventRecord], - error: &str, -) -> Result<()> { - let entry = TrajectoryDeadLetterEntry { - timestamp: chrono::Utc::now().to_rfc3339(), - storage: storage.display().to_string(), - agent_id: agent_id.to_string(), - session_id: session_id.to_string(), - root_session: root_session.map(str::to_string), - records: records.to_vec(), - error: error.to_string(), - }; - let path = trajectory_dead_letter_path(storage); - let mut file = open_private_append_file(&path) - .with_context(|| format!("open trajectory dead letter {}", path.display()))?; - let line = serde_json::to_string(&entry).context("serialize trajectory dead letter")?; - writeln!(file, "{line}").context("append trajectory dead letter")?; - Ok(()) -} - -pub fn read_trajectory_dead_letter_entries( - storage: &Path, -) -> Result> { - let path = trajectory_dead_letter_path(storage); - if !path.exists() { - return Ok(Vec::new()); - } - let file = File::open(&path).with_context(|| format!("open {}", path.display()))?; - let mut out = Vec::new(); - for (i, line) in BufReader::new(file).lines().enumerate() { - let line = line.with_context(|| format!("read trajectory dead letter line {i}"))?; - let trimmed = line.trim(); - if trimmed.is_empty() { - continue; - } - out.push( - serde_json::from_str(trimmed) - .with_context(|| format!("parse trajectory dead letter line {i}"))?, - ); - } - Ok(out) -} - -pub fn append_dead_letter( - storage: &Path, - ctx: &CallContext, - event: &Event, - error: &str, - prepared_record_json: Option, -) -> Result<()> { - let entry = DeadLetterEntry { - timestamp: chrono::Utc::now().to_rfc3339(), - context: DeadLetterContext::from_context(ctx), - event: SerializableEvent::from_event(event), - error: error.to_string(), - prepared_record_json, - }; - let path = dead_letter_path(storage); - let mut file = open_private_append_file(&path) - .with_context(|| format!("open dead letter {}", path.display()))?; - let line = serde_json::to_string(&entry).context("serialize dead letter entry")?; - writeln!(file, "{line}").context("append dead letter")?; - Ok(()) -} - -pub fn read_dead_letter_entries(storage: &Path) -> Result> { - let path = dead_letter_path(storage); - if !path.exists() { - return Ok(Vec::new()); - } - let file = File::open(&path).with_context(|| format!("open {}", path.display()))?; - let mut out = Vec::new(); - for (i, line) in BufReader::new(file).lines().enumerate() { - let line = line.with_context(|| format!("read dead letter line {i}"))?; - let trimmed = line.trim(); - if trimmed.is_empty() { - continue; - } - out.push( - serde_json::from_str(trimmed).with_context(|| format!("parse dead letter line {i}"))?, - ); - } - Ok(out) -} - -impl DeadLetterContext { - pub fn from_context(ctx: &CallContext) -> Self { - Self { - route: ctx.route().clone(), - agent_id: ctx.agent_id().to_string(), - call: ctx.call.clone(), - request_headers: redact_sensitive_headers(&ctx.request_headers), - level: ctx.level, - client_model: ctx.client_model.clone(), - upstream_model: ctx.upstream_model.clone(), - provider: ctx.provider, - protocol: ctx.protocol, - client_peer: ctx.client_peer.clone(), - client_meta: ctx.client_meta.clone(), - http_version: ctx.http_version.clone(), - upstream_url: ctx.upstream_url.as_deref().map(redact_sensitive_url), - } - } - - pub fn to_call_context(&self) -> CallContext { - let mut ctx = CallContext::new( - self.route.clone(), - self.agent_id.clone(), - self.call.clone(), - self.request_headers.clone(), - self.level, - self.client_model.clone(), - self.upstream_model.clone(), - self.provider, - self.protocol, - false, - ); - if let Some(peer) = &self.client_peer { - ctx.attach_client(peer.clone(), self.client_meta.clone()); - } else if self.client_meta.is_some() { - ctx.client_meta = self.client_meta.clone(); - } - if let Some(v) = &self.http_version { - ctx.attach_http_version(v.clone()); - } - if let Some(u) = &self.upstream_url { - ctx.attach_upstream_url(u.clone()); - } - ctx - } -} - -impl SerializableEvent { - pub fn from_event(event: &Event) -> Self { - match event { - Event::Request(e) => Self::Request { - path: redact_sensitive_url(&e.path), - method: e.method.clone(), - url: e.url.as_deref().map(redact_sensitive_url), - body_bytes: e.body_bytes, - user_content: e.user_content.clone(), - body_json: e.body_json.as_ref().map(redact_sensitive_body), - model_rewritten: e.model_rewritten, - headers: redact_sensitive_headers(&e.headers), - }, - Event::ResponseComplete(e) => Self::ResponseComplete { - status: e.status, - resp_payload: resp_to_payload(&e.resp_bytes), - streaming: e.streaming, - stream_metrics: e.stream_metrics.clone(), - assistant_content: e.assistant_content.clone(), - headers: redact_sensitive_headers(&e.headers), - }, - Event::ResponseDraft(e) => Self::ResponseDraft { - status: e.status, - assistant_content: e.assistant_content.clone(), - }, - Event::Cancelled(e) => Self::Cancelled { - status: e.status, - bytes_received: e.bytes_received, - streaming: e.streaming, - }, - } - } - - pub fn to_event(&self) -> Event { - match self { - Self::Request { - path, - method, - url, - body_bytes, - user_content, - body_json, - model_rewritten, - headers, - } => Event::Request(RequestEvent { - path: path.clone(), - method: method.clone(), - url: url.clone(), - body_bytes: *body_bytes, - user_content: user_content.clone(), - body_json: body_json.clone(), - // WAL/dead-letter rows retain one source of truth: the exact - // client JSON. Replay reconstructs the typed request in prepare. - semantic: None, - model_rewritten: *model_rewritten, - headers: headers.clone(), - }), - Self::ResponseComplete { - status, - resp_payload, - streaming, - stream_metrics, - assistant_content, - headers, - } => Event::ResponseComplete(CompleteEvent { - status: *status, - resp_bytes: payload_to_bytes(resp_payload), - streaming: *streaming, - stream_metrics: stream_metrics.clone(), - assistant_content: assistant_content.clone(), - semantic: None, - headers: headers.clone(), - }), - Self::ResponseDraft { - status, - assistant_content, - } => Event::ResponseDraft(DraftEvent { - status: *status, - assistant_content: assistant_content.clone(), - }), - Self::Cancelled { - status, - bytes_received, - streaming, - } => Event::Cancelled(CancelEvent { - status: *status, - bytes_received: *bytes_received, - streaming: *streaming, - }), - } - } -} - -fn resp_to_payload(bytes: &Bytes) -> RespPayload { - if let Ok(s) = std::str::from_utf8(bytes) { - RespPayload::Text(s.to_string()) - } else { - RespPayload::Bytes(bytes.to_vec()) - } -} - -fn payload_to_bytes(payload: &RespPayload) -> Bytes { - match payload { - RespPayload::Text(s) => Bytes::from(s.clone()), - RespPayload::Bytes(b) => Bytes::from(b.clone()), - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct DeadLetterReplaySummary { - pub attempted: usize, - pub succeeded: usize, - pub failed: usize, -} - -pub async fn replay_dead_letter( - storage: &Path, - engine: &crate::engine::CaptureEngine, -) -> Result { - let entries = read_dead_letter_entries(storage)?; - let mut summary = DeadLetterReplaySummary { - attempted: entries.len(), - succeeded: 0, - failed: 0, - }; - for entry in entries { - let ctx = entry.context.to_call_context(); - let event = entry.event.to_event(); - match engine.apply(&ctx, event).await { - Ok(()) => summary.succeeded += 1, - Err(e) => { - summary.failed += 1; - tracing::warn!("dead letter replay failed: {e:#}"); - } - } - } - Ok(summary) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::config::CaptureLevel; - use crate::protocol::ProtocolKind; - use crate::provider::ProviderKind; - - fn sample_ctx(_dir: &Path) -> CallContext { - CallContext::new( - CaptureRoute { - root_session: Some("run-1".into()), - session_id: "sess".into(), - storage_session_id: "run-1".into(), - subagent_id: None, - }, - "agent", - Call { - call_id: "c1".into(), - trace_id: "t1".into(), - started_at: "2026-01-01T00:00:00Z".into(), - }, - Vec::new(), - CaptureLevel::Dialogue, - "m", - "m", - ProviderKind::OpenAi, - ProtocolKind::ChatCompletions, - false, - ) - } - - #[test] - fn dead_letter_roundtrip_jsonl() { - let dir = tempfile::tempdir().unwrap(); - let ctx = sample_ctx(dir.path()); - let event = Event::Request(RequestEvent { - path: "/v1/chat/completions".into(), - method: "POST".into(), - url: None, - body_bytes: 10, - user_content: Some("hi".into()), - body_json: None, - semantic: None, - model_rewritten: false, - headers: vec![], - }); - append_dead_letter(dir.path(), &ctx, &event, "mailbox full", None).unwrap(); - let entries = read_dead_letter_entries(dir.path()).unwrap(); - assert_eq!(entries.len(), 1); - assert_eq!(entries[0].error, "mailbox full"); - assert!(matches!(entries[0].event.to_event(), Event::Request(_))); - assert_eq!(entries[0].context.provider, ProviderKind::OpenAi); - assert_eq!(entries[0].context.protocol, ProtocolKind::ChatCompletions); - } - - #[test] - fn dead_letter_redacts_credentials_before_writing() { - let dir = tempfile::tempdir().unwrap(); - let mut ctx = sample_ctx(dir.path()); - ctx.request_headers = vec![ - ("authorization".into(), "Bearer context-secret".into()), - ("x-request-id".into(), "req-safe".into()), - ]; - ctx.upstream_url = Some("https://upstream.example/v1?key=url-secret".into()); - let event = Event::Request(RequestEvent { - path: "/v1/chat/completions".into(), - method: "POST".into(), - url: Some("//gateway.example/v1?api_key=request-url-secret".into()), - body_bytes: 10, - user_content: Some("hi".into()), - body_json: Some(serde_json::json!({ - "api_key": "body-secret", - "nested": {"clientSecret": "nested-secret"}, - "safe": "kept" - })), - semantic: None, - model_rewritten: false, - headers: vec![("x-goog-api-key".into(), "event-secret".into())], - }); - - append_dead_letter(dir.path(), &ctx, &event, "mailbox full", None).unwrap(); - - let serialized = std::fs::read_to_string(dead_letter_path(dir.path())).unwrap(); - for secret in [ - "context-secret", - "event-secret", - "body-secret", - "nested-secret", - "url-secret", - "request-url-secret", - ] { - assert!( - !serialized.contains(secret), - "dead letter persisted credential {secret}: {serialized}" - ); - } - assert!(serialized.contains("req-safe")); - assert!(serialized.contains("kept")); - assert!(serialized.contains("")); - - let entries = read_dead_letter_entries(dir.path()).unwrap(); - assert_eq!(entries[0].context.request_headers[0].1, ""); - let SerializableEvent::Request { - body_json, headers, .. - } = &entries[0].event - else { - panic!("expected request event") - }; - assert_eq!(headers[0].1, ""); - assert_eq!(body_json.as_ref().unwrap()["api_key"], ""); - } - - #[cfg(unix)] - #[test] - fn dead_letter_uses_private_permissions() { - use std::os::unix::fs::PermissionsExt; - - let dir = tempfile::tempdir().unwrap(); - let ctx = sample_ctx(dir.path()); - let event = Event::Cancelled(CancelEvent { - status: 499, - bytes_received: 0, - streaming: true, - }); - append_dead_letter(dir.path(), &ctx, &event, "cancelled", None).unwrap(); - - let capture_dir = dir.path().join(".capture"); - assert_eq!( - std::fs::metadata(&capture_dir) - .unwrap() - .permissions() - .mode() - & 0o777, - 0o700 - ); - assert_eq!( - std::fs::metadata(dead_letter_path(dir.path())) - .unwrap() - .permissions() - .mode() - & 0o777, - 0o600 - ); - } - - #[test] - fn trajectory_dead_letter_roundtrip_jsonl() { - let dir = tempfile::tempdir().unwrap(); - append_trajectory_dead_letter( - dir.path(), - "agent", - "sess", - Some("run-1"), - &[persisting_events::EventRecord { - identity: Default::default(), - seq: 1, - source: "test".into(), - kind: "note".into(), - timestamp: None, - session_id: Some("sess".into()), - agent_id: Some("agent".into()), - parent_uuid: None, - trace_id: None, - call_id: None, - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload: serde_json::json!({"content":"retry"}), - }], - "engine invoke failed", - ) - .unwrap(); - let entries = read_trajectory_dead_letter_entries(dir.path()).unwrap(); - assert_eq!(entries.len(), 1); - assert_eq!(entries[0].error, "engine invoke failed"); - assert!( - serde_json::to_value(&entries[0]) - .unwrap() - .get("schema_version") - .is_none() - ); - assert_eq!(entries[0].decoded_records().unwrap().len(), 1); - } - - #[test] - fn dead_letter_context_roundtrips_legacy_string_provider_protocol() { - let json = r#"{ - "timestamp": "2026-01-01T00:00:00Z", - "context": { - "route": { - "root_session": "run-1", - "session_id": "sess", - "storage_session_id": "run-1", - "subagent_id": null - }, - "agent_id": "agent", - "call": {"call_id": "c1", "trace_id": "t1", "started_at": "2026-01-01T00:00:00Z"}, - "level": "dialogue", - "client_model": "m", - "upstream_model": "m", - "provider": "openai", - "protocol": "chat_completions" - }, - "event": {"kind": "cancelled", "status": 499, "bytes_received": 0, "streaming": true}, - "error": "test" - }"#; - let entry: DeadLetterEntry = serde_json::from_str(json).unwrap(); - assert_eq!(entry.context.provider, ProviderKind::OpenAi); - assert_eq!(entry.context.protocol, ProtocolKind::ChatCompletions); - let ctx = entry.context.to_call_context(); - assert_eq!(ctx.provider, ProviderKind::OpenAi); - assert_eq!(ctx.protocol, ProtocolKind::ChatCompletions); - } -} diff --git a/crates/persisting-gateway/src/dialogue_extract/assistant.rs b/crates/persisting-gateway/src/dialogue_extract/assistant.rs deleted file mode 100644 index 6d1966750..000000000 --- a/crates/persisting-gateway/src/dialogue_extract/assistant.rs +++ /dev/null @@ -1,73 +0,0 @@ -//! Cross-protocol assistant response extraction (SSE + JSON). - -use serde_json::Value; - -use super::completions::{completions_assistant_from_json, completions_sse_turn}; -use super::messages::{anthropic_assistant_from_json, anthropic_sse_turn}; -use super::responses::{responses_assistant_from_json, responses_sse_turn}; - -/// Visible assistant turn from SSE: Anthropic, OpenAI completions, or Responses events. -pub fn extract_assistant_turn_from_sse(raw: &str) -> String { - let anthropic = anthropic_sse_turn(raw); - if !anthropic.is_empty() { - return anthropic; - } - let completions = completions_sse_turn(raw); - if !completions.is_empty() { - return completions; - } - responses_sse_turn(raw) -} - -/// Assistant visible text from a JSON response body (OpenAI `choices` or Anthropic `content`). -pub fn extract_assistant_text_from_json(body: &Value) -> Option { - if let Some(s) = body.as_str() { - if s.contains("event:") && s.contains("data:") { - let text = extract_assistant_turn_from_sse(s); - if !text.is_empty() { - return Some(text); - } - } - return None; - } - completions_assistant_from_json(body) - .or_else(|| anthropic_assistant_from_json(body)) - .or_else(|| responses_assistant_from_json(body)) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn anthropic_sse_extracts_text_not_thinking() { - let sse = r#"event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":"secret"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"Hi"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"!"}} -"#; - assert_eq!(extract_assistant_turn_from_sse(sse), "Hi!"); - } - - #[test] - fn completions_sse_extracts_multiple_tool_calls() { - let sse = r#"data: {"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"name":"read_file","arguments":""}}]}}]} - -data: {"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"path\": \"a.rs\"}"}}]}}]} - -data: {"choices":[{"index":0,"delta":{"tool_calls":[{"index":1,"function":{"name":"read_file","arguments":""}}]}}]} - -data: {"choices":[{"index":0,"delta":{"tool_calls":[{"index":1,"function":{"arguments":"{\"path\": \"b.rs\"}"}}]}}]} - -data: {"choices":[{"index":0,"delta":{},"finish_reason":"tool_calls"}]} -"#; - let out = extract_assistant_turn_from_sse(sse); - assert!(out.contains("```tool:read_file")); - assert!(out.contains("a.rs")); - assert!(out.contains("b.rs")); - } -} diff --git a/crates/persisting-gateway/src/dialogue_extract/common.rs b/crates/persisting-gateway/src/dialogue_extract/common.rs deleted file mode 100644 index d493839d1..000000000 --- a/crates/persisting-gateway/src/dialogue_extract/common.rs +++ /dev/null @@ -1,310 +0,0 @@ -//! Shared text normalization and formatting helpers. - -use serde_json::Value; - -const IMAGE_PROMPT_PREVIEW_CHARS: usize = 200; -const HASH_HEX_CHARS: usize = 12; - -pub(crate) fn unwrap_session_tag(s: &str) -> &str { - let t = s.trim(); - if let Some(inner) = t - .strip_prefix("") - .and_then(|x| x.strip_suffix("")) - { - return inner.trim(); - } - t -} - -pub(crate) fn is_system_injection(s: &str) -> bool { - let t = s.trim_start(); - t.starts_with("") - || t.starts_with(" bool { - if is_system_injection(s) { - return true; - } - let t = s.trim_start(); - t.starts_with("") -} - -pub(crate) fn format_tool_result_block(part: &Value) -> Option { - let id = part - .get("tool_use_id") - .and_then(|v| v.as_str()) - .unwrap_or("tool"); - let content = tool_result_content(part.get("content")?); - Some(format!("```tool_result:{id}\n{content}\n```")) -} - -pub(crate) fn tool_result_content(v: &Value) -> String { - match v { - Value::String(s) => s.clone(), - Value::Array(parts) => parts - .iter() - .filter_map(|p| { - p.get("text") - .and_then(|t| t.as_str()) - .or_else(|| p.get("content").and_then(|c| c.as_str())) - }) - .collect::>() - .join("\n"), - other => serde_json::to_string_pretty(other).unwrap_or_else(|_| other.to_string()), - } -} - -pub(crate) fn normalize_json(raw: &str) -> String { - let trimmed = raw.trim(); - if trimmed.is_empty() { - return String::new(); - } - if let Ok(v) = serde_json::from_str::(trimmed) { - return serde_json::to_string_pretty(&v).unwrap_or_else(|_| trimmed.to_string()); - } - trimmed.to_string() -} - -/// Parse `data:{media_type};base64,{payload}` image URLs. -pub(crate) fn parse_data_url(url: &str) -> Option<(&str, &str)> { - let rest = url.trim().strip_prefix("data:")?; - rest.split_once(";base64,") -} - -pub(crate) fn format_image_url_placeholder(url: &str) -> String { - if let Some((media_type, data)) = parse_data_url(url) { - format_image_base64_placeholder(media_type, data) - } else { - format!("[image: url:{url}]") - } -} - -pub(crate) fn format_image_base64_placeholder(media_type: &str, base64_data: &str) -> String { - let approx_bytes = base64_data.len().saturating_mul(3) / 4; - let size = format_approx_bytes(approx_bytes); - let hash = short_content_hash(base64_data); - format!("[image: base64:{size} {media_type} hash={hash}]") -} - -pub(crate) fn format_anthropic_image_placeholder(part: &Value) -> Option { - let source = part.get("source")?; - match source.get("type")?.as_str()? { - "url" => { - let url = source.get("url")?.as_str()?; - Some(format_image_url_placeholder(url)) - } - "base64" => { - let media_type = source - .get("media_type") - .and_then(|m| m.as_str()) - .unwrap_or("image/*"); - let data = source.get("data")?.as_str()?; - Some(format_image_base64_placeholder(media_type, data)) - } - _ => None, - } -} - -pub(crate) fn format_openai_image_url_part(part: &Value) -> Option { - let url = part - .get("image_url") - .and_then(|iu| iu.get("url")) - .and_then(|u| u.as_str()) - .or_else(|| part.get("url").and_then(|u| u.as_str()))?; - Some(format_image_url_placeholder(url)) -} - -pub(crate) fn format_input_image_placeholder(part: &Value) -> Option { - part.get("image_url") - .and_then(|u| u.as_str()) - .map(format_image_url_placeholder) -} - -pub(crate) fn format_image_generation_placeholder(item: &Value) -> String { - let id = item - .get("id") - .or_else(|| item.get("item_id")) - .and_then(|v| v.as_str()) - .unwrap_or("ig"); - let format = item - .get("output_format") - .and_then(|f| f.as_str()) - .unwrap_or("png"); - let size = item - .get("size") - .and_then(|s| s.as_str()) - .filter(|s| !s.is_empty()); - let mut label = format!("[image_generated: {id}, {format}"); - if let Some(s) = size { - label.push_str(&format!(", {s}")); - } - if let Some(result) = item - .get("result") - .and_then(|r| r.as_str()) - .filter(|r| !r.is_empty()) - { - let approx_bytes = result.len().saturating_mul(3) / 4; - if approx_bytes > 0 { - label.push_str(&format!(", ~{}", format_approx_bytes(approx_bytes))); - } - } - label.push(']'); - if let Some(prompt) = item - .get("revised_prompt") - .and_then(|p| p.as_str()) - .filter(|p| !p.is_empty()) - { - let preview = truncate_preview(prompt, IMAGE_PROMPT_PREVIEW_CHARS); - return format!("{label}\nprompt: {preview}"); - } - label -} - -/// Append one user-visible content part (text, tool_result, or image) to `out`. -pub(crate) fn append_user_content_part(out: &mut Vec, part: &Value) { - let t = part.get("type").and_then(|x| x.as_str()).unwrap_or(""); - match t { - "text" | "input_text" => { - if let Some(text) = part.get("text").and_then(|x| x.as_str()) { - let text = unwrap_session_tag(text); - if !is_codex_context_injection(text) && !text.trim().is_empty() { - out.push(text.to_string()); - } - } - } - "tool_result" | "function_call_output" => { - if let Some(formatted) = format_tool_result_block(part) { - out.push(formatted); - } - } - "image" => { - if let Some(formatted) = format_anthropic_image_placeholder(part) { - out.push(formatted); - } - } - "image_url" => { - if let Some(formatted) = format_openai_image_url_part(part) { - out.push(formatted); - } - } - "input_image" => { - if let Some(formatted) = format_input_image_placeholder(part) { - out.push(formatted); - } - } - _ => {} - } -} - -pub(crate) fn user_content_part_is_visible(part: &Value) -> bool { - let t = part.get("type").and_then(|x| x.as_str()).unwrap_or(""); - match t { - "text" | "input_text" => part - .get("text") - .and_then(|x| x.as_str()) - .is_some_and(|text| { - let text = unwrap_session_tag(text); - !is_codex_context_injection(text) && !text.trim().is_empty() - }), - "tool_result" | "function_call_output" => format_tool_result_block(part).is_some(), - "image" => format_anthropic_image_placeholder(part).is_some(), - "image_url" => format_openai_image_url_part(part).is_some(), - "input_image" => format_input_image_placeholder(part).is_some(), - _ => false, - } -} - -pub(crate) fn format_visible_content_from_parts(parts: &[Value]) -> Option { - let mut out = Vec::new(); - for part in parts { - append_user_content_part(&mut out, part); - } - if out.is_empty() { - None - } else { - Some(out.join("\n\n")) - } -} - -fn format_approx_bytes(bytes: usize) -> String { - if bytes >= 1_048_576 { - format!("{}MB", bytes / 1_048_576) - } else if bytes >= 1024 { - format!("{}KB", bytes / 1024) - } else { - format!("{bytes}B") - } -} - -fn short_content_hash(data: &str) -> String { - blake3::hash(data.as_bytes()).to_hex()[..HASH_HEX_CHARS.min(blake3::OUT_LEN * 2)].to_string() -} - -fn truncate_preview(text: &str, max_chars: usize) -> String { - if text.chars().count() <= max_chars { - return text.to_string(); - } - let end = text - .char_indices() - .nth(max_chars) - .map(|(idx, _)| idx) - .unwrap_or(text.len()); - format!("{}…", &text[..end]) -} - -pub(crate) fn join_assistant_parts( - text: &str, - tools: impl IntoIterator, -) -> String { - let mut parts = Vec::new(); - if !text.trim().is_empty() { - parts.push(text.to_string()); - } - for (name, args) in tools { - if name.trim().is_empty() && args.trim().is_empty() { - continue; - } - let name = if name.trim().is_empty() { - "tool".to_string() - } else { - name - }; - let args = { - let normalized = normalize_json(&args); - if normalized.trim().is_empty() { - "{}".to_string() - } else { - normalized - } - }; - parts.push(format!("```tool:{name}\n{args}\n```")); - } - parts.join("\n\n") -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn data_url_image_placeholder_includes_size_and_hash() { - let url = "data:image/png;base64,iVBORw0KGgo="; - let out = format_image_url_placeholder(url); - assert!(out.starts_with("[image: base64:")); - assert!(out.contains("image/png")); - assert!(out.contains("hash=")); - } - - #[test] - fn https_image_url_placeholder() { - let out = format_image_url_placeholder("https://example.com/a.png"); - assert_eq!(out, "[image: url:https://example.com/a.png]"); - } -} diff --git a/crates/persisting-gateway/src/dialogue_extract/completions.rs b/crates/persisting-gateway/src/dialogue_extract/completions.rs deleted file mode 100644 index faa2a494d..000000000 --- a/crates/persisting-gateway/src/dialogue_extract/completions.rs +++ /dev/null @@ -1,262 +0,0 @@ -//! OpenAI Chat Completions API: choices, tool calls, SSE deltas. - -use std::collections::BTreeMap; - -use serde_json::Value; - -use crate::conversion::{decode_stream_arguments_delta, unquote_chat_tool_arguments}; - -use super::common::join_assistant_parts; - -/// Incremental OpenAI Completions SSE parser for tool-call partial capture (Codex upstream). -#[derive(Default)] -pub struct CompletionsSseToolParser { - line_carry: String, - text: String, - tool_calls: BTreeMap, - last_snapshot: Option, -} - -#[derive(Default)] -struct ToolAccum { - name: String, - arguments: String, -} - -impl CompletionsSseToolParser { - /// Incremental snapshot for Anthropic-style partial capture only. - pub fn push_chunk(&mut self, chunk: &str) -> Option { - self.ingest_chunk(chunk); - let snapshot = self.snapshot(); - if snapshot.is_empty() { - return None; - } - if self.last_snapshot.as_deref() == Some(snapshot.as_str()) { - return None; - } - self.last_snapshot = Some(snapshot.clone()); - Some(snapshot) - } - - pub fn finish(&mut self) { - if !self.line_carry.is_empty() { - let line = std::mem::take(&mut self.line_carry); - self.process_data_line(line.trim()); - } - } - - pub fn snapshot(&self) -> String { - join_assistant_parts( - &self.text, - self.tool_calls - .values() - .map(|t| (t.name.clone(), t.arguments.clone())), - ) - } - - fn ingest_chunk(&mut self, chunk: &str) { - self.line_carry.push_str(chunk); - while let Some(nl) = self.line_carry.find('\n') { - let line = self.line_carry[..nl].trim().to_string(); - self.line_carry.drain(..=nl); - self.process_data_line(&line); - } - } - - fn process_data_line(&mut self, line: &str) { - let Some(data) = line.strip_prefix("data:") else { - return; - }; - let data = data.trim(); - if data.is_empty() || data == "[DONE]" { - return; - } - let Ok(v) = serde_json::from_str::(data) else { - return; - }; - let Some(choice) = v - .get("choices") - .and_then(|c| c.as_array()) - .and_then(|a| a.first()) - else { - return; - }; - if let Some(delta) = choice.get("delta") { - if let Some(content) = delta.get("content").and_then(|c| c.as_str()) { - self.text.push_str(content); - } - if let Some(tcs) = delta.get("tool_calls").and_then(|t| t.as_array()) { - self.merge_tool_call_deltas(tcs); - } - } - if let Some(message) = choice.get("message") { - if let Some(content) = message.get("content").and_then(|c| c.as_str()) - && self.text.is_empty() - && !content.is_empty() - { - self.text.push_str(content); - } - if let Some(tcs) = message.get("tool_calls").and_then(|t| t.as_array()) { - self.merge_complete_tool_calls(tcs); - } - } - } - - fn merge_tool_call_deltas(&mut self, tcs: &[Value]) { - for tc in tcs { - let index = tc.get("index").and_then(|i| i.as_u64()).unwrap_or(0) as u32; - let entry = self.tool_calls.entry(index).or_default(); - if let Some(name) = tc - .get("function") - .and_then(|f| f.get("name")) - .and_then(|n| n.as_str()) - .filter(|n| !n.is_empty()) - { - entry.name = name.to_string(); - } - if let Some(args) = tc - .get("function") - .and_then(|f| f.get("arguments")) - .and_then(|a| a.as_str()) - .filter(|a| !a.is_empty()) - { - entry - .arguments - .push_str(&decode_stream_arguments_delta(args)); - } - } - } - - fn merge_complete_tool_calls(&mut self, tcs: &[Value]) { - for (index, tc) in tcs.iter().enumerate() { - let idx = tc - .get("index") - .and_then(|i| i.as_u64()) - .unwrap_or(index as u64) as u32; - let entry = self.tool_calls.entry(idx).or_default(); - if let Some(name) = tc - .get("function") - .and_then(|f| f.get("name")) - .and_then(|n| n.as_str()) - .filter(|n| !n.is_empty()) - { - entry.name = name.to_string(); - } - if let Some(args) = tc.get("function").and_then(|f| f.get("arguments")) { - entry.arguments = unquote_chat_tool_arguments(args); - } - } - } -} - -pub(crate) fn completions_sse_turn(raw: &str) -> String { - let mut parser = CompletionsSseToolParser::default(); - parser.ingest_chunk(raw); - parser.finish(); - parser.snapshot() -} - -pub(crate) fn completions_assistant_from_json(body: &Value) -> Option { - if let Some(content) = body - .get("choices") - .and_then(|c| c.as_array()) - .and_then(|a| a.first()) - .and_then(|c| c.get("message")) - .and_then(|m| m.get("content")) - && let Some(text) = super::messages::content_to_string(content) - { - return Some(text); - } - if let Some(message) = body - .get("choices") - .and_then(|c| c.as_array()) - .and_then(|a| a.first()) - .and_then(|c| c.get("message")) - { - return format_openai_tool_calls(message.get("tool_calls")); - } - None -} - -pub(crate) fn format_openai_tool_calls(tool_calls: Option<&Value>) -> Option { - let items = tool_calls?.as_array()?; - let mut parts = Vec::new(); - for tc in items { - let name = tc - .get("function") - .and_then(|f| f.get("name")) - .and_then(|n| n.as_str()) - .unwrap_or("tool"); - let args = tc - .get("function") - .and_then(|f| f.get("arguments")) - .and_then(|a| a.as_str()) - .unwrap_or("{}"); - parts.push(format!("```tool:{name}\n{args}\n```")); - } - if parts.is_empty() { - None - } else { - Some(parts.join("\n\n")) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn completions_sse_tool_parser_emits_tool_snapshot() { - let raw = - include_str!("../../tests/fixtures/local/response/completions/stream_tool_call.txt"); - let mut parser = CompletionsSseToolParser::default(); - let snap = parser.push_chunk(raw).expect("tool snapshot"); - assert!(snap.contains("```tool:shell")); - assert!(snap.contains("ls")); - } - - #[test] - fn completions_sse_extracts_tool_calls_without_trailing_newline() { - let raw = - include_str!("../../tests/fixtures/local/response/completions/stream_tool_call.txt"); - let trimmed = raw.trim_end(); - let out = completions_sse_turn(trimmed); - assert!(out.contains("```tool:shell")); - assert!(out.contains("\"command\": \"ls\"") || out.contains("\"command\":\"ls\"")); - } - - #[test] - fn completions_sse_extracts_multiple_tool_calls() { - let sse = r#"data: {"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"name":"read_file","arguments":""}}]}}]} - -data: {"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"path\": \"a.rs\"}"}}]}}]} - -data: {"choices":[{"index":0,"delta":{"tool_calls":[{"index":1,"function":{"name":"read_file","arguments":""}}]}}]} - -data: {"choices":[{"index":0,"delta":{"tool_calls":[{"index":1,"function":{"arguments":"{\"path\": \"b.rs\"}"}}]}}]} - -data: {"choices":[{"index":0,"delta":{},"finish_reason":"tool_calls"}]} -"#; - let out = completions_sse_turn(sse); - assert!(out.contains("```tool:read_file")); - assert!(out.contains("a.rs")); - assert!(out.contains("b.rs")); - } - - #[test] - fn openai_completions_sse_extracts_assistant_text() { - let raw = include_str!("../../tests/fixtures/local/response/completions/stream_head.txt"); - let out = completions_sse_turn(raw); - assert!(out.contains("Hi")); - assert!(out.contains("help")); - } - - #[test] - fn openai_completions_sse_extracts_tool_calls() { - let raw = - include_str!("../../tests/fixtures/local/response/completions/stream_tool_call.txt"); - let out = completions_sse_turn(raw); - assert!(out.contains("```tool:shell")); - assert!(out.contains("ls")); - } -} diff --git a/crates/persisting-gateway/src/dialogue_extract/heuristics.rs b/crates/persisting-gateway/src/dialogue_extract/heuristics.rs deleted file mode 100644 index 426daf103..000000000 --- a/crates/persisting-gateway/src/dialogue_extract/heuristics.rs +++ /dev/null @@ -1,97 +0,0 @@ -//! Claude Code request-shape heuristics (subagent probes, task notifications). - -use bytes::Bytes; -use serde_json::Value; - -/// Main agent turn delivered with a subagent's `x-claude-code-agent-id` header -/// (e.g. `` after background agent completion). -pub fn is_main_agent_continuation_payload(payload: &Value) -> bool { - if payload - .get("user_content") - .and_then(|s| s.as_str()) - .is_some_and(contains_task_notification) - { - return true; - } - payload - .get("messages") - .and_then(|m| m.as_array()) - .is_some_and(|msgs| msgs.iter().any(message_contains_task_notification)) -} - -pub fn is_main_agent_continuation(body: &Bytes) -> bool { - serde_json::from_slice::(body) - .ok() - .is_some_and(|v| is_main_agent_continuation_payload(&v)) -} - -fn message_contains_task_notification(msg: &Value) -> bool { - match msg.get("content") { - Some(Value::String(s)) => contains_task_notification(s), - Some(Value::Array(parts)) => parts.iter().any(|p| { - p.get("text") - .and_then(|t| t.as_str()) - .is_some_and(contains_task_notification) - }), - _ => false, - } -} - -fn contains_task_notification(s: &str) -> bool { - s.contains("") -} - -/// Claude Code subagent probe (`*-flash`/`*-haiku` + `` wrapper). -pub fn is_subagent_shape_payload(payload: &Value) -> bool { - let model = payload.get("model").and_then(|m| m.as_str()).unwrap_or(""); - if !model.contains("flash") && !model.contains("haiku") { - return false; - } - if payload - .get("user_content") - .and_then(|s| s.as_str()) - .is_some_and(|s| s.contains("")) - { - return true; - } - payload - .get("messages") - .and_then(|m| m.as_array()) - .is_some_and(|msgs| { - msgs.iter().any(|msg| { - msg.get("content") - .map(content_contains_session_tag) - .unwrap_or(false) - }) - }) -} - -fn content_contains_session_tag(c: &Value) -> bool { - match c { - Value::String(s) => s.contains(""), - Value::Array(parts) => parts.iter().any(|p| { - p.get("text") - .and_then(|t| t.as_str()) - .is_some_and(|t| t.contains("")) - }), - _ => false, - } -} - -#[cfg(test)] -mod tests { - use super::*; - use bytes::Bytes; - - #[test] - fn detects_task_notification_as_main_continuation() { - let body = br#"{"model":"deepseek-v4-pro","messages":[{"role":"user","content":[{"type":"text","text":"\nabc12345\n"}]}]}"#; - assert!(is_main_agent_continuation(&Bytes::from_static(body))); - } - - #[test] - fn subagent_flash_prompt_is_not_main_continuation() { - let body = br#"{"model":"deepseek-v4-flash","messages":[{"role":"user","content":"Review docs"}]}"#; - assert!(!is_main_agent_continuation(&Bytes::from_static(body))); - } -} diff --git a/crates/persisting-gateway/src/dialogue_extract/messages.rs b/crates/persisting-gateway/src/dialogue_extract/messages.rs deleted file mode 100644 index 93ce0a189..000000000 --- a/crates/persisting-gateway/src/dialogue_extract/messages.rs +++ /dev/null @@ -1,399 +0,0 @@ -//! Anthropic Messages API: user turns, content blocks, SSE parsing. - -use std::collections::BTreeMap; - -use serde_json::Value; - -use super::common::{ - format_visible_content_from_parts, is_system_injection, normalize_json, unwrap_session_tag, - user_content_part_is_visible, -}; - -pub(crate) fn extract_user_from_messages(v: &Value) -> Option { - let messages = v.get("messages")?.as_array()?; - for msg in messages.iter().rev() { - if msg.get("role").and_then(|r| r.as_str()) != Some("user") { - continue; - } - if let Some(s) = msg.get("content").and_then(|c| c.as_str()) { - let s = unwrap_session_tag(s); - if !is_system_injection(s) { - return Some(s.to_string()); - } - continue; - } - if let Some(parts) = msg.get("content").and_then(|c| c.as_array()) { - return format_visible_content_from_parts(parts); - } - } - None -} - -pub(crate) fn user_message_has_visible_text(msg: &Value) -> bool { - if msg.get("role").and_then(|r| r.as_str()) != Some("user") { - return false; - } - if let Some(s) = msg.get("content").and_then(|c| c.as_str()) { - let s = unwrap_session_tag(s); - return !is_system_injection(s) && !s.trim().is_empty(); - } - if let Some(parts) = msg.get("content").and_then(|c| c.as_array()) { - return parts.iter().any(user_content_part_is_visible); - } - false -} - -pub(crate) fn anthropic_sse_turn(raw: &str) -> String { - format_content_blocks(&parse_sse_content_blocks(raw)) -} - -pub(crate) fn anthropic_assistant_from_json(body: &Value) -> Option { - if let Some(parts) = body.get("content").and_then(|c| c.as_array()) { - let formatted = format_content_blocks_from_json_parts(parts); - if !formatted.is_empty() { - return Some(formatted); - } - } - None -} - -pub(crate) fn content_to_string(v: &Value) -> Option { - match v { - Value::String(s) => Some(unwrap_session_tag(s).to_string()), - Value::Array(parts) => { - let formatted = format_content_blocks_from_json_parts(parts); - if formatted.is_empty() { - None - } else { - Some(formatted) - } - } - _ => None, - } -} - -#[derive(Debug, Clone)] -pub enum ContentBlock { - Text(String), - ToolUse { name: String, input_json: String }, -} - -fn parse_sse_content_blocks(raw: &str) -> Vec { - let mut parser = SseStreamBlockParser::default(); - parser.push_chunk(raw); - parser.finish() -} - -/// Incremental Anthropic SSE parser; emits completed blocks on `content_block_stop`. -#[derive(Default)] -pub struct SseStreamBlockParser { - line_carry: String, - by_index: BTreeMap, - order: Vec, - completed: Vec, -} - -impl SseStreamBlockParser { - pub fn push_chunk(&mut self, chunk: &str) -> Vec { - self.line_carry.push_str(chunk); - let mut newly = Vec::new(); - while let Some(nl) = self.line_carry.find('\n') { - let line = self.line_carry[..nl].trim().to_string(); - self.line_carry.drain(..=nl); - if let Some(block) = self.process_line(&line) { - newly.push(block.clone()); - self.completed.push(block); - } - } - newly - } - - pub fn accumulated_assistant_text(&self) -> String { - format_content_blocks(&self.completed) - } - - pub fn finish(mut self) -> Vec { - for idx in self.order.clone() { - if self.by_index.contains_key(&idx) - && let Some(builder) = self.by_index.remove(&idx) - && let Some(block) = builder.finish() - { - self.completed.push(block); - } - } - self.completed - } - - fn process_line(&mut self, line: &str) -> Option { - let json_str = line.strip_prefix("data:")?; - let json_str = json_str.trim(); - if json_str.is_empty() || json_str == "[DONE]" { - return None; - } - let Ok(v) = serde_json::from_str::(json_str) else { - return None; - }; - match v.get("type").and_then(|t| t.as_str()) { - Some("content_block_start") => { - let index = v.get("index").and_then(|i| i.as_u64()).unwrap_or(0) as usize; - let block_type = v - .get("content_block") - .and_then(|b| b.get("type")) - .and_then(|t| t.as_str()) - .unwrap_or("text"); - let tool_name = v - .get("content_block") - .and_then(|b| b.get("name")) - .and_then(|n| n.as_str()) - .map(str::to_string); - if !self.by_index.contains_key(&index) { - self.order.push(index); - } - self.by_index.insert( - index, - BlockBuilder { - kind: match block_type { - "tool_use" => BlockKind::ToolUse { - name: tool_name.unwrap_or_else(|| "tool".into()), - }, - "thinking" => BlockKind::Thinking, - _ => BlockKind::Text, - }, - text: String::new(), - json: String::new(), - }, - ); - None - } - Some("content_block_delta") => { - let index = v.get("index").and_then(|i| i.as_u64()).unwrap_or(0) as usize; - let delta = v.get("delta")?; - let entry = self.by_index.entry(index).or_insert_with(|| { - self.order.push(index); - BlockBuilder { - kind: BlockKind::Text, - text: String::new(), - json: String::new(), - } - }); - match delta.get("type").and_then(|t| t.as_str()) { - Some("text_delta") => { - if let Some(t) = delta.get("text").and_then(|x| x.as_str()) { - entry.text.push_str(t); - } - } - Some("input_json_delta") => { - if let Some(t) = delta.get("partial_json").and_then(|x| x.as_str()) { - entry.json.push_str(t); - } - } - _ => {} - } - None - } - Some("content_block_stop") => { - let index = v.get("index").and_then(|i| i.as_u64()).unwrap_or(0) as usize; - self.by_index.remove(&index).and_then(|b| b.finish()) - } - _ => None, - } - } -} - -pub fn push_sse_tool_snapshot(parser: &mut SseStreamBlockParser, chunk: &str) -> Option { - let new = parser.push_chunk(chunk); - if new - .iter() - .any(|b| matches!(b, ContentBlock::ToolUse { .. })) - { - Some(parser.accumulated_assistant_text()) - } else { - None - } -} - -#[derive(Default)] -struct BlockBuilder { - kind: BlockKind, - text: String, - json: String, -} - -#[derive(Default)] -enum BlockKind { - #[default] - Text, - ToolUse { - name: String, - }, - Thinking, -} - -impl BlockBuilder { - fn finish(self) -> Option { - match self.kind { - BlockKind::Thinking => None, - BlockKind::ToolUse { name } => { - let input = normalize_json(&self.json); - if input.is_empty() && self.text.is_empty() { - return None; - } - Some(ContentBlock::ToolUse { - name, - input_json: if input.is_empty() { self.text } else { input }, - }) - } - BlockKind::Text => { - if self.text.is_empty() { - None - } else { - Some(ContentBlock::Text(self.text)) - } - } - } - } -} - -fn format_content_blocks(blocks: &[ContentBlock]) -> String { - blocks - .iter() - .map(format_content_block) - .collect::>() - .join("\n\n") -} - -fn format_content_blocks_from_json_parts(parts: &[Value]) -> String { - let mut blocks = Vec::new(); - for part in parts { - match part.get("type").and_then(|t| t.as_str()) { - Some("text") => { - if let Some(t) = part.get("text").and_then(|x| x.as_str()) { - blocks.push(ContentBlock::Text(t.to_string())); - } - } - Some("tool_use") => { - let name = part.get("name").and_then(|n| n.as_str()).unwrap_or("tool"); - let input = part - .get("input") - .map(|v| serde_json::to_string_pretty(v).unwrap_or_else(|_| v.to_string())) - .unwrap_or_else(|| "{}".into()); - blocks.push(ContentBlock::ToolUse { - name: name.to_string(), - input_json: input, - }); - } - Some("image") => { - if let Some(formatted) = super::common::format_anthropic_image_placeholder(part) { - blocks.push(ContentBlock::Text(formatted)); - } - } - _ => {} - } - } - format_content_blocks(&blocks) -} - -fn format_content_block(block: &ContentBlock) -> String { - match block { - ContentBlock::Text(t) => t.clone(), - ContentBlock::ToolUse { name, input_json } => { - format!("```tool:{name}\n{input_json}\n```") - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn anthropic_sse_extracts_text_not_thinking() { - let sse = r#"event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":"secret"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"Hi"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"!"}} -"#; - assert_eq!(anthropic_sse_turn(sse), "Hi!"); - } - - #[test] - fn sse_incremental_parser_emits_tool_on_stop() { - let sse = r#"event: content_block_start -data: {"type":"content_block_start","index":0,"content_block":{"type":"text"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Launching."}} - -event: content_block_stop -data: {"type":"content_block_stop","index":0} - -event: content_block_start -data: {"type":"content_block_start","index":1,"content_block":{"type":"tool_use","name":"Agent","id":"toolu_1"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"{\"prompt\":\"scan\"}"}} - -event: content_block_stop -data: {"type":"content_block_stop","index":1} -"#; - let mut parser = SseStreamBlockParser::default(); - let snap1 = push_sse_tool_snapshot(&mut parser, sse); - assert!(snap1.is_some()); - assert!(snap1.unwrap().contains("```tool:Agent")); - } - - #[test] - fn anthropic_sse_extracts_tool_use() { - let sse = r#"event: content_block_start -data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"我来 review。"}} - -event: content_block_start -data: {"type":"content_block_start","index":1,"content_block":{"type":"tool_use","id":"toolu_1","name":"Bash","input":{}}} - -event: content_block_delta -data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"{\"command\": \"git diff\"}"}} - -event: content_block_stop -data: {"type":"content_block_stop","index":1} -"#; - let out = anthropic_sse_turn(sse); - assert!(out.contains("我来 review。")); - assert!(out.contains("```tool:Bash")); - assert!(out.contains("git diff")); - } - - #[test] - fn thinking_only_sse_yields_empty_visible_text() { - let sse = r#"event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":"stay silent"}} - -event: message_stop -data: {"type":"message_stop"} -"#; - assert!(anthropic_sse_turn(sse).is_empty()); - } - - #[test] - fn anthropic_user_image_base64_placeholder() { - let body = serde_json::json!({ - "messages": [{ - "role": "user", - "content": [ - {"type":"text","text":"screenshot"}, - {"type":"image","source":{"type":"base64","media_type":"image/png","data":"iVBORw0KGgo="}} - ] - }] - }); - let out = extract_user_from_messages(&body).unwrap(); - assert!(out.contains("screenshot")); - assert!(out.contains("[image: base64:")); - assert!(out.contains("image/png")); - } -} diff --git a/crates/persisting-gateway/src/dialogue_extract/mod.rs b/crates/persisting-gateway/src/dialogue_extract/mod.rs deleted file mode 100644 index bb3637ef2..000000000 --- a/crates/persisting-gateway/src/dialogue_extract/mod.rs +++ /dev/null @@ -1,22 +0,0 @@ -//! Extract human-readable dialogue text from LLM request/response payloads. -//! -//! Submodules by wire protocol: -//! - `messages` — Anthropic Messages (`/v1/messages`) -//! - `responses` — OpenAI Responses (`/v1/responses`) -//! - `completions` — OpenAI Chat Completions (`/v1/chat/completions`) - -mod assistant; -mod common; -mod completions; -mod heuristics; -mod messages; -mod request; -mod responses; - -pub use assistant::{extract_assistant_text_from_json, extract_assistant_turn_from_sse}; -pub use completions::CompletionsSseToolParser; -pub use heuristics::{ - is_main_agent_continuation, is_main_agent_continuation_payload, is_subagent_shape_payload, -}; -pub use messages::{ContentBlock, SseStreamBlockParser, push_sse_tool_snapshot}; -pub use request::{count_visible_user_messages, extract_user_message_from_request_body}; diff --git a/crates/persisting-gateway/src/dialogue_extract/request.rs b/crates/persisting-gateway/src/dialogue_extract/request.rs deleted file mode 100644 index 0b0fd8ce4..000000000 --- a/crates/persisting-gateway/src/dialogue_extract/request.rs +++ /dev/null @@ -1,148 +0,0 @@ -//! Cross-protocol request body extraction. - -use bytes::Bytes; -use serde_json::Value; - -use super::messages::{extract_user_from_messages, user_message_has_visible_text}; -use super::responses::{count_responses_input_user_turns, extract_user_from_responses_input}; - -/// Last substantive user turn text (may include formatted `tool_result` blocks). -pub fn extract_user_message_from_request_body(body: &Bytes) -> Option { - let v: Value = serde_json::from_slice(body).ok()?; - extract_user_from_messages(&v).or_else(|| extract_user_from_responses_input(&v)) -} - -/// Count user-visible turns in a request body (Anthropic `messages` or Responses `input`). -pub fn count_visible_user_messages(v: &Value) -> usize { - if let Some(messages) = v.get("messages").and_then(|m| m.as_array()) { - return messages - .iter() - .filter(|msg| user_message_has_visible_text(msg)) - .count(); - } - if let Some(input) = v.get("input") { - return count_responses_input_user_turns(input); - } - 0 -} - -#[cfg(test)] -mod tests { - use super::*; - use bytes::Bytes; - use serde_json::json; - - #[test] - fn count_visible_user_messages_increments_with_new_user_turns() { - let one = json!({"messages":[{"role":"user","content":"hi"}]}); - assert_eq!(count_visible_user_messages(&one), 1); - let two = json!({"messages":[ - {"role":"user","content":"hi"}, - {"role":"assistant","content":"hello"}, - {"role":"user","content":"hi again"} - ]}); - assert_eq!(count_visible_user_messages(&two), 2); - let replay = json!({"messages":[ - {"role":"user","content":"hi"}, - {"role":"assistant","content":"hello"}, - {"role":"user","content":"hi again"} - ]}); - assert_eq!(count_visible_user_messages(&replay), 2); - } - - #[test] - fn anthropic_request_skips_system_reminder() { - let body = br#"{"messages":[{"role":"user","content":[ - {"type":"text","text":"\nskills\n"}, - {"type":"text","text":"hi"} - ]}]}"#; - assert_eq!( - extract_user_message_from_request_body(&Bytes::from_static(body)).as_deref(), - Some("hi") - ); - } - - #[test] - fn unwraps_session_tag() { - let body = r#"{"messages":[{"role":"user","content":[{"type":"text","text":"\nhi\n"}]}]}"#; - assert_eq!( - extract_user_message_from_request_body(&Bytes::copy_from_slice(body.as_bytes())) - .as_deref(), - Some("hi") - ); - } - - #[test] - fn skips_suggestion_mode_prompt() { - let body = br#"{"messages":[{"role":"user","content":[{"type":"text","text":"[SUGGESTION MODE: predict next input]\nReply ONLY the suggestion."}]}]}"#; - assert!(extract_user_message_from_request_body(&Bytes::from_static(body)).is_none()); - } - - #[test] - fn request_includes_tool_result() { - let body = br#"{"messages":[{"role":"user","content":[ - {"type":"tool_result","tool_use_id":"toolu_1","content":"file1.rs\nfile2.rs"} - ]}]}"#; - let out = extract_user_message_from_request_body(&Bytes::from_static(body)).unwrap(); - assert!(out.contains("```tool_result:toolu_1")); - assert!(out.contains("file1.rs")); - } - - #[test] - fn anthropic_user_message_with_image_url_placeholder() { - let body = br#"{"messages":[{"role":"user","content":[ - {"type":"text","text":"What's in this image?"}, - {"type":"image","source":{"type":"url","url":"https://example.com/a.png"}} - ]}]}"#; - let out = extract_user_message_from_request_body(&Bytes::from_static(body)).unwrap(); - assert!(out.contains("What's in this image?")); - assert!(out.contains("[image: url:https://example.com/a.png]")); - } - - #[test] - fn openai_completions_user_message_with_image_url() { - let body = br#"{"messages":[{"role":"user","content":[ - {"type":"text","text":"describe"}, - {"type":"image_url","image_url":{"url":"https://example.com/b.png"}} - ]}]}"#; - let v: Value = serde_json::from_slice(body).unwrap(); - assert_eq!(count_visible_user_messages(&v), 1); - let out = extract_user_message_from_request_body(&Bytes::from_static(body)).unwrap(); - assert!(out.contains("[image: url:https://example.com/b.png]")); - } - - #[test] - fn image_only_user_message_counts_as_visible_turn() { - let body = json!({"messages":[{"role":"user","content":[ - {"type":"image","source":{"type":"url","url":"https://example.com/only.png"}} - ]}]}); - assert_eq!(count_visible_user_messages(&body), 1); - let out = extract_user_message_from_request_body(&Bytes::copy_from_slice( - &serde_json::to_vec(&body).unwrap(), - )) - .unwrap(); - assert!(out.contains("[image: url:")); - } - - #[test] - fn codex_responses_input_image_counts_and_extracts() { - let body = json!({ - "model": "gpt-5", - "input": [{ - "type": "message", - "role": "user", - "content": [ - {"type":"input_text","text":"look"}, - {"type":"input_image","image_url":"https://example.com/c.png"} - ] - }] - }); - assert_eq!(count_visible_user_messages(&body), 1); - let out = extract_user_message_from_request_body(&Bytes::from( - serde_json::to_vec(&body).unwrap(), - )) - .unwrap(); - assert!(out.contains("look")); - assert!(out.contains("[image: url:https://example.com/c.png]")); - } -} diff --git a/crates/persisting-gateway/src/dialogue_extract/responses.rs b/crates/persisting-gateway/src/dialogue_extract/responses.rs deleted file mode 100644 index deb5ec38d..000000000 --- a/crates/persisting-gateway/src/dialogue_extract/responses.rs +++ /dev/null @@ -1,417 +0,0 @@ -//! OpenAI Responses API: `input` / `output` extraction and SSE events. - -use std::collections::BTreeMap; - -use serde_json::Value; - -use super::common::{ - format_image_generation_placeholder, format_visible_content_from_parts, - is_codex_context_injection, join_assistant_parts, unwrap_session_tag, - user_content_part_is_visible, -}; - -pub(crate) fn extract_user_from_responses_input(v: &Value) -> Option { - let input = v.get("input")?; - match input { - Value::String(s) if !s.trim().is_empty() => { - let s = unwrap_session_tag(s); - if is_codex_context_injection(s) { - None - } else { - Some(s.to_string()) - } - } - Value::Array(items) => extract_tail_from_responses_input(items), - _ => None, - } -} - -pub(crate) fn count_responses_input_user_turns(input: &Value) -> usize { - match input { - Value::String(s) => { - let s = unwrap_session_tag(s); - if is_codex_context_injection(s) || s.trim().is_empty() { - 0 - } else { - 1 - } - } - Value::Array(items) => items - .iter() - .filter(|item| responses_input_item_is_user_turn(item)) - .count(), - _ => 0, - } -} - -fn responses_input_item_is_user_turn(item: &Value) -> bool { - let item_type = item.get("type").and_then(|t| t.as_str()).unwrap_or(""); - match item_type { - "function_call" | "function_call_output" => false, - "message" => { - item.get("role").and_then(|r| r.as_str()) == Some("user") - && responses_user_content_is_visible(item.get("content")) - } - _ => responses_user_content_is_visible(item.get("content")), - } -} - -fn responses_user_content_is_visible(content: Option<&Value>) -> bool { - let Some(content) = content else { - return false; - }; - match content { - Value::String(s) => { - let s = unwrap_session_tag(s); - !is_codex_context_injection(s) && !s.trim().is_empty() - } - Value::Array(parts) => parts.iter().any(user_content_part_is_visible), - _ => false, - } -} - -fn extract_tail_from_responses_input(items: &[Value]) -> Option { - let mut outputs = Vec::new(); - for item in items.iter().rev() { - match item.get("type").and_then(|t| t.as_str()) { - Some("function_call_output") => { - if let Some(formatted) = format_responses_function_call_output(item) { - outputs.push(formatted); - } - } - _ => break, - } - } - if !outputs.is_empty() { - outputs.reverse(); - return Some(outputs.join("\n\n")); - } - for item in items.iter().rev() { - if let Some(text) = last_user_text_from_input_item(item) { - return Some(text); - } - } - None -} - -fn format_responses_function_call_output(item: &Value) -> Option { - let call_id = item - .get("call_id") - .or_else(|| item.get("id")) - .and_then(|v| v.as_str()) - .unwrap_or("call"); - let output = item.get("output")?; - let content = responses_output_to_text(output); - if content.trim().is_empty() { - return None; - } - Some(format!("```tool_result:{call_id}\n{content}\n```")) -} - -fn responses_output_to_text(output: &Value) -> String { - match output { - Value::String(s) => s.clone(), - Value::Array(parts) => parts - .iter() - .filter_map(|part| { - let t = part.get("type").and_then(|x| x.as_str()).unwrap_or(""); - match t { - "input_text" | "text" | "output_text" => part - .get("text") - .and_then(|x| x.as_str()) - .map(str::to_string), - _ => part - .get("text") - .and_then(|x| x.as_str()) - .map(str::to_string), - } - }) - .collect::>() - .join("\n"), - other => other.to_string(), - } -} - -fn last_user_text_from_input_item(item: &Value) -> Option { - let item_type = item.get("type").and_then(|t| t.as_str()).unwrap_or(""); - match item_type { - "function_call" | "function_call_output" => None, - "message" => { - let role = item.get("role").and_then(|r| r.as_str()).unwrap_or("user"); - if role != "user" { - return None; - } - format_responses_user_content(item.get("content")) - } - _ => format_responses_user_content(item.get("content")), - } -} - -fn format_responses_user_content(content: Option<&Value>) -> Option { - let content = content?; - match content { - Value::String(s) => { - let s = unwrap_session_tag(s); - if is_codex_context_injection(s) { - None - } else { - Some(s.to_string()) - } - } - Value::Array(parts) => format_visible_content_from_parts(parts), - _ => None, - } -} - -pub(crate) fn responses_sse_turn(raw: &str) -> String { - #[derive(Default)] - struct ToolAccum { - name: String, - arguments: String, - } - - let mut text = String::new(); - let mut tool_calls: BTreeMap = BTreeMap::new(); - let mut image_generations: BTreeMap = BTreeMap::new(); - let mut next_tool_index = 0u32; - - for line in raw.lines() { - let line = line.trim(); - let Some(data) = line.strip_prefix("data:") else { - continue; - }; - let data = data.trim(); - if data.is_empty() || data == "[DONE]" { - continue; - } - let Ok(v) = serde_json::from_str::(data) else { - continue; - }; - let event_type = v.get("type").and_then(|t| t.as_str()).unwrap_or(""); - match event_type { - "response.output_text.delta" => { - if let Some(delta) = v.get("delta").and_then(|d| d.as_str()) { - text.push_str(delta); - } - } - "response.output_text.done" => { - if let Some(done) = v.get("text").and_then(|t| t.as_str()) - && text.is_empty() - { - text.push_str(done); - } - } - "response.output_item.added" | "response.output_item.done" => { - if let Some(item) = v.get("item") { - if item.get("type").and_then(|t| t.as_str()) == Some("function_call") { - let name = item - .get("name") - .and_then(|n| n.as_str()) - .unwrap_or("tool") - .to_string(); - let output_index = v - .get("output_index") - .and_then(|i| i.as_u64()) - .unwrap_or(next_tool_index as u64) - as u32; - next_tool_index = next_tool_index.max(output_index + 1); - tool_calls.entry(output_index).or_default().name = name; - } else if item.get("type").and_then(|t| t.as_str()) - == Some("image_generation_call") - { - upsert_image_generation(&mut image_generations, item); - } - } - } - "response.function_call_arguments.delta" => { - if let Some(delta) = v.get("delta").and_then(|d| d.as_str()) { - let output_index = v - .get("output_index") - .and_then(|i| i.as_u64()) - .map(|i| i as u32); - if let Some(idx) = output_index { - tool_calls.entry(idx).or_default().arguments.push_str(delta); - } else if let Some((_, entry)) = tool_calls.iter_mut().next_back() { - entry.arguments.push_str(delta); - } - } - } - "response.function_call_arguments.done" => { - let args = v - .get("arguments") - .and_then(|a| a.as_str()) - .unwrap_or("{}") - .to_string(); - let output_index = v - .get("output_index") - .and_then(|i| i.as_u64()) - .map(|i| i as u32); - if let Some(idx) = output_index { - tool_calls.entry(idx).or_default().arguments = args; - } else if let Some((_, entry)) = tool_calls.iter_mut().next_back() { - entry.arguments = args; - } - } - "response.completed" => { - if let Some(output) = v - .get("response") - .and_then(|r| r.get("output")) - .and_then(|o| o.as_array()) - { - for item in output { - if item.get("type").and_then(|t| t.as_str()) - == Some("image_generation_call") - { - upsert_image_generation(&mut image_generations, item); - } - } - } - } - _ => {} - } - } - - let mut parts = Vec::new(); - let tool_text = join_assistant_parts( - &text, - tool_calls - .values() - .map(|t| (t.name.clone(), t.arguments.clone())), - ); - if !tool_text.is_empty() { - parts.push(tool_text); - } - parts.extend(image_generations.into_values()); - parts.join("\n\n") -} - -fn upsert_image_generation(out: &mut BTreeMap, item: &Value) { - let id = item - .get("id") - .or_else(|| item.get("item_id")) - .and_then(|v| v.as_str()) - .unwrap_or("ig") - .to_string(); - out.insert(id, format_image_generation_placeholder(item)); -} - -pub(crate) fn responses_assistant_from_json(body: &Value) -> Option { - let output = body.get("output")?.as_array()?; - let mut texts = Vec::new(); - for item in output { - match item.get("type").and_then(|t| t.as_str()) { - Some("message") => { - if let Some(parts) = item.get("content").and_then(|c| c.as_array()) { - for part in parts { - if part.get("type").and_then(|t| t.as_str()) == Some("output_text") - && let Some(t) = part.get("text").and_then(|x| x.as_str()) - { - texts.push(t.to_string()); - } - } - } - } - Some("function_call") => { - let name = item.get("name").and_then(|n| n.as_str()).unwrap_or("tool"); - let args = item - .get("arguments") - .and_then(|a| a.as_str()) - .unwrap_or("{}"); - texts.push(format!("```tool:{name}\n{args}\n```")); - } - Some("image_generation_call") => { - texts.push(format_image_generation_placeholder(item)); - } - _ => {} - } - } - if texts.is_empty() { - None - } else { - Some(texts.join("\n\n")) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use bytes::Bytes; - use serde_json::json; - - #[test] - fn codex_responses_input_extracts_tail_function_call_output() { - let body = serde_json::to_vec(&json!({ - "model": "gpt-5.5", - "input": [ - {"type":"message","role":"user","content":[{"type":"input_text","text":"review"}]}, - {"type":"function_call","call_id":"call_1","name":"exec_command","arguments":"{}"}, - {"type":"function_call_output","call_id":"call_1","output":"Cargo.toml\nsrc/"} - ] - })) - .unwrap(); - let v: Value = serde_json::from_slice(&body).unwrap(); - let out = extract_user_from_responses_input(&v).unwrap(); - assert!(out.contains("```tool_result:call_1")); - assert!(out.contains("Cargo.toml")); - } - - #[test] - fn codex_responses_input_extracts_last_user_text() { - let body = Bytes::from_static(include_bytes!( - "../../tests/fixtures/local/requests/responses/codex_basic.json" - )); - let v: Value = serde_json::from_slice(&body).unwrap(); - assert_eq!(extract_user_from_responses_input(&v).as_deref(), Some("hi")); - } - - #[test] - fn responses_sse_extracts_output_text_delta() { - let sse = r#"event: response.output_text.delta -data: {"type":"response.output_text.delta","delta":"Hello"} - -event: response.output_text.delta -data: {"type":"response.output_text.delta","delta":" world"} -"#; - assert_eq!(responses_sse_turn(sse), "Hello world"); - } - - #[test] - fn responses_sse_extracts_image_generation_call() { - let sse = r#"event: response.output_item.done -data: {"type":"response.output_item.done","item":{"id":"ig_1","type":"image_generation_call","output_format":"png","size":"1024x1024","result":"iVBORw0KGgo=","revised_prompt":"a cute cat"}} - -"#; - let out = responses_sse_turn(sse); - assert!(out.contains("[image_generated: ig_1, png, 1024x1024")); - assert!(out.contains("prompt: a cute cat")); - } - - #[test] - fn responses_json_extracts_image_generation_call() { - let body = json!({ - "output": [{ - "type": "image_generation_call", - "id": "ig_2", - "output_format": "png", - "size": "512x512", - "result": "abc", - "revised_prompt": "draw a dog" - }] - }); - let out = responses_assistant_from_json(&body).unwrap(); - assert!(out.contains("[image_generated: ig_2, png, 512x512")); - assert!(out.contains("draw a dog")); - } - - #[test] - fn codex_responses_image_only_input_counts_as_user_turn() { - let body = json!({ - "input": [{ - "type": "message", - "role": "user", - "content": [{"type":"input_image","image_url":"https://example.com/x.png"}] - }] - }); - assert_eq!(count_responses_input_user_turns(&body["input"]), 1); - } -} diff --git a/crates/persisting-gateway/src/echo.rs b/crates/persisting-gateway/src/echo.rs deleted file mode 100644 index 9e1968bd8..000000000 --- a/crates/persisting-gateway/src/echo.rs +++ /dev/null @@ -1,819 +0,0 @@ -//! Deterministic local LLM upstream for forwarding and conversion tests. - -use std::fmt; -use std::future::Future; - -use anyhow::{Context, Result, bail}; -use axum::body::{Body, Bytes}; -use axum::extract::{Path, State}; -use axum::http::{HeaderMap, HeaderName, HeaderValue, StatusCode}; -use axum::response::{IntoResponse, Response}; -use axum::routing::{get, post}; -use axum::{Json, Router}; -use base64::Engine; -use base64::engine::general_purpose::STANDARD; -use serde_json::{Value, json}; - -pub const ECHO_ENCODING_HEADER: &str = "x-persisting-echo-encoding"; -pub const ECHO_MODE_HEADER: &str = "x-persisting-echo-mode"; - -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] -enum EchoMode { - #[default] - Text, - Inspect, - Tool, - Reasoning, - Error, -} - -impl EchoMode { - fn from_headers(headers: &HeaderMap) -> Self { - match headers - .get(ECHO_MODE_HEADER) - .and_then(|value| value.to_str().ok()) - { - Some("inspect") => Self::Inspect, - Some("tool") => Self::Tool, - Some("reasoning") => Self::Reasoning, - Some("error") => Self::Error, - _ => Self::Text, - } - } -} - -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] -pub enum EchoEncoding { - #[default] - Plain, - Base64, -} - -impl EchoEncoding { - pub fn parse(value: &str) -> Result { - match value.trim().to_ascii_lowercase().as_str() { - "plain" | "text" | "direct" => Ok(Self::Plain), - "base64" => Ok(Self::Base64), - _ => bail!("unsupported echo encoding '{value}' (expected plain or base64)"), - } - } - - fn encode(self, input: &str) -> String { - match self { - Self::Plain => input.to_string(), - Self::Base64 => STANDARD.encode(input.as_bytes()), - } - } -} - -impl fmt::Display for EchoEncoding { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter.write_str(match self { - Self::Plain => "plain", - Self::Base64 => "base64", - }) - } -} - -#[derive(Debug, Clone, Copy, Default)] -pub struct EchoServerConfig { - pub default_encoding: EchoEncoding, -} - -pub fn router(config: EchoServerConfig) -> Router { - Router::new() - .route("/health", get(health)) - .route("/echo", post(raw_echo)) - .route("/v1/chat/completions", post(chat_completions)) - .route("/v1/messages", post(messages)) - .route("/v1/responses", post(responses)) - .route("/v1beta/models/{model_and_method}", post(gemini_generate)) - .with_state(config) -} - -pub async fn serve(listener: tokio::net::TcpListener, config: EchoServerConfig) -> Result<()> { - axum::serve(listener, router(config)) - .await - .context("serve Gateway echo upstream") -} - -pub async fn serve_with_shutdown( - listener: tokio::net::TcpListener, - config: EchoServerConfig, - shutdown: impl Future + Send + 'static, -) -> Result<()> { - axum::serve(listener, router(config)) - .with_graceful_shutdown(shutdown) - .await - .context("serve Gateway echo upstream") -} - -async fn health() -> Json { - Json(json!({"status": "ok", "service": "pchronicle-echo"})) -} - -async fn raw_echo( - State(config): State, - headers: HeaderMap, - body: Bytes, -) -> Response { - let encoding = match request_encoding(&headers, config) { - Ok(encoding) => encoding, - Err(error) => return error_response(&error), - }; - encoded_response( - StatusCode::OK, - "text/plain; charset=utf-8", - encoding, - encoding.encode(&String::from_utf8_lossy(&body)), - ) -} - -async fn chat_completions( - State(config): State, - headers: HeaderMap, - Json(request): Json, -) -> Response { - let mode = EchoMode::from_headers(&headers); - if mode == EchoMode::Error { - return controlled_error_response(); - } - let encoding = match request_encoding(&headers, config) { - Ok(encoding) => encoding, - Err(error) => return error_response(&error), - }; - let content = encoding.encode(&match mode { - EchoMode::Inspect => inspect_chat_input(&request), - _ => extract_chat_input(&request), - }); - let model = request_model(&request); - if mode == EchoMode::Tool { - if request_streams(&request) { - return sse_response(encoding, chat_tool_stream(&model)); - } - return json_response(encoding, chat_tool_body(&model)); - } - if mode == EchoMode::Reasoning { - if request_streams(&request) { - return sse_response(encoding, chat_reasoning_stream(&model, &content)); - } - return json_response(encoding, chat_reasoning_body(&model, &content)); - } - if request_streams(&request) { - return sse_response(encoding, chat_stream(&model, &content)); - } - json_response( - encoding, - json!({ - "id": "chatcmpl-echo", - "object": "chat.completion", - "created": 0, - "model": model, - "choices": [{ - "index": 0, - "message": {"role": "assistant", "content": content}, - "finish_reason": "stop" - }], - "usage": {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0} - }), - ) -} - -async fn messages( - State(config): State, - headers: HeaderMap, - Json(request): Json, -) -> Response { - let encoding = match request_encoding(&headers, config) { - Ok(encoding) => encoding, - Err(error) => return error_response(&error), - }; - let content = encoding.encode(&extract_chat_input(&request)); - let model = request_model(&request); - if request_streams(&request) { - return sse_response(encoding, messages_stream(&model, &content)); - } - json_response( - encoding, - json!({ - "id": "msg_echo", - "type": "message", - "role": "assistant", - "model": model, - "content": [{"type": "text", "text": content}], - "stop_reason": "end_turn", - "stop_sequence": null, - "usage": {"input_tokens": 0, "output_tokens": 0} - }), - ) -} - -async fn responses( - State(config): State, - headers: HeaderMap, - Json(request): Json, -) -> Response { - let encoding = match request_encoding(&headers, config) { - Ok(encoding) => encoding, - Err(error) => return error_response(&error), - }; - let content = encoding.encode(&extract_responses_input(&request)); - let model = request_model(&request); - if request_streams(&request) { - return sse_response(encoding, responses_stream(&model, &content)); - } - json_response(encoding, responses_body(&model, &content)) -} - -async fn gemini_generate( - State(config): State, - Path(model_and_method): Path, - headers: HeaderMap, - Json(request): Json, -) -> Response { - let encoding = match request_encoding(&headers, config) { - Ok(encoding) => encoding, - Err(error) => return error_response(&error), - }; - let (model, method) = model_and_method - .split_once(':') - .unwrap_or((&model_and_method, "generateContent")); - let content = encoding.encode(&extract_gemini_input(&request)); - let body = gemini_body(model, &content); - if method == "streamGenerateContent" { - let event = serde_json::to_string(&body).expect("serialize Gemini echo event"); - return sse_response(encoding, format!("data: {event}\n\n")); - } - json_response(encoding, body) -} - -fn request_encoding( - headers: &HeaderMap, - config: EchoServerConfig, -) -> std::result::Result { - let Some(value) = headers.get(ECHO_ENCODING_HEADER) else { - return Ok(config.default_encoding); - }; - let value = match value.to_str() { - Ok(value) => value, - Err(_) => return Err("echo encoding header is not valid UTF-8".to_string()), - }; - EchoEncoding::parse(value).map_err(|error| error.to_string()) -} - -fn request_model(request: &Value) -> String { - request - .get("model") - .and_then(Value::as_str) - .unwrap_or("echo-model") - .to_string() -} - -fn request_streams(request: &Value) -> bool { - request - .get("stream") - .and_then(Value::as_bool) - .unwrap_or(false) -} - -fn extract_chat_input(request: &Value) -> String { - request - .get("messages") - .and_then(Value::as_array) - .and_then(|messages| { - messages.iter().rev().find_map(|message| { - (message.get("role").and_then(Value::as_str) == Some("user")) - .then(|| text_content(message.get("content"))) - }) - }) - .filter(|text| !text.is_empty()) - .unwrap_or_else(|| extract_responses_input(request)) -} - -fn extract_responses_input(request: &Value) -> String { - let Some(input) = request.get("input") else { - return String::new(); - }; - match input { - Value::String(text) => text.clone(), - Value::Array(items) => items - .iter() - .rev() - .find_map(|item| { - let role_is_user = item.get("role").and_then(Value::as_str) == Some("user"); - role_is_user.then(|| text_content(item.get("content"))) - }) - .filter(|text| !text.is_empty()) - .unwrap_or_else(|| text_content(Some(input))), - _ => text_content(Some(input)), - } -} - -fn extract_gemini_input(request: &Value) -> String { - request - .get("contents") - .and_then(Value::as_array) - .and_then(|contents| { - contents.iter().rev().find_map(|content| { - let role = content.get("role").and_then(Value::as_str); - matches!(role, Some("user") | None).then(|| text_content(content.get("parts"))) - }) - }) - .unwrap_or_default() -} - -fn text_content(content: Option<&Value>) -> String { - match content { - Some(Value::String(text)) => text.clone(), - Some(Value::Array(parts)) => parts - .iter() - .filter_map(|part| match part { - Value::String(text) => Some(text.as_str()), - Value::Object(map) => map - .get("text") - .and_then(Value::as_str) - .or_else(|| map.get("input_text").and_then(Value::as_str)), - _ => None, - }) - .collect::>() - .join(""), - Some(Value::Object(map)) => map - .get("text") - .and_then(Value::as_str) - .unwrap_or_default() - .to_string(), - _ => String::new(), - } -} - -fn inspect_chat_input(request: &Value) -> String { - let content = request - .get("messages") - .and_then(Value::as_array) - .and_then(|messages| { - messages - .iter() - .rev() - .find(|message| message.get("role").and_then(Value::as_str) == Some("user")) - }) - .and_then(|message| message.get("content")); - let Some(Value::Array(parts)) = content else { - return "text=0 image=0".to_string(); - }; - let text = parts - .iter() - .filter(|part| { - matches!( - part.get("type").and_then(Value::as_str), - Some("text" | "input_text") - ) - }) - .count(); - let image = parts - .iter() - .filter(|part| { - matches!( - part.get("type").and_then(Value::as_str), - Some("image_url" | "input_image" | "image") - ) - }) - .count(); - format!("text={text} image={image}") -} - -fn chat_stream(model: &str, content: &str) -> String { - let start = json!({ - "id": "chatcmpl-echo", - "object": "chat.completion.chunk", - "created": 0, - "model": model, - "choices": [{"index": 0, "delta": {"role": "assistant", "content": content}, "finish_reason": null}] - }); - let stop = json!({ - "id": "chatcmpl-echo", - "object": "chat.completion.chunk", - "created": 0, - "model": model, - "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}] - }); - format!("data: {start}\n\ndata: {stop}\n\ndata: [DONE]\n\n") -} - -fn chat_tool_body(model: &str) -> Value { - json!({ - "id": "chatcmpl-echo-tool", - "object": "chat.completion", - "created": 0, - "model": model, - "choices": [{ - "index": 0, - "message": { - "role": "assistant", - "content": null, - "tool_calls": [{ - "id": "call_echo_weather", - "type": "function", - "function": {"name": "weather", "arguments": "{\"city\":\"Paris\"}"} - }] - }, - "finish_reason": "tool_calls" - }], - "usage": {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0} - }) -} - -fn chat_tool_stream(model: &str) -> String { - let start = json!({ - "id":"chatcmpl-echo-tool","object":"chat.completion.chunk","created":0,"model":model, - "choices":[{"index":0,"delta":{"role":"assistant","tool_calls":[{ - "index":0,"id":"call_echo_weather","type":"function", - "function":{"name":"weather","arguments":"{\"city\":\"Paris\"}"} - }]},"finish_reason":null}] - }); - let stop = json!({ - "id":"chatcmpl-echo-tool","object":"chat.completion.chunk","created":0,"model":model, - "choices":[{"index":0,"delta":{},"finish_reason":"tool_calls"}] - }); - format!("data: {start}\n\ndata: {stop}\n\ndata: [DONE]\n\n") -} - -fn chat_reasoning_body(model: &str, content: &str) -> Value { - json!({ - "id":"chatcmpl-echo-reasoning","object":"chat.completion","created":0,"model":model, - "choices":[{"index":0,"message":{ - "role":"assistant","reasoning_content":"echo-reasoning","content":content - },"finish_reason":"stop"}], - "usage":{"prompt_tokens":0,"completion_tokens":0,"total_tokens":0} - }) -} - -fn chat_reasoning_stream(model: &str, content: &str) -> String { - let reasoning = json!({ - "id":"chatcmpl-echo-reasoning","object":"chat.completion.chunk","created":0,"model":model, - "choices":[{"index":0,"delta":{"role":"assistant","reasoning_content":"echo-reasoning"},"finish_reason":null}] - }); - let text = json!({ - "id":"chatcmpl-echo-reasoning","object":"chat.completion.chunk","created":0,"model":model, - "choices":[{"index":0,"delta":{"content":content},"finish_reason":null}] - }); - let stop = json!({ - "id":"chatcmpl-echo-reasoning","object":"chat.completion.chunk","created":0,"model":model, - "choices":[{"index":0,"delta":{},"finish_reason":"stop"}] - }); - format!("data: {reasoning}\n\ndata: {text}\n\ndata: {stop}\n\ndata: [DONE]\n\n") -} - -fn messages_stream(model: &str, content: &str) -> String { - let events = [ - ( - "message_start", - json!({"type":"message_start","message":{"id":"msg_echo","type":"message","role":"assistant","model":model,"content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":0,"output_tokens":0}}}), - ), - ( - "content_block_start", - json!({"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}), - ), - ( - "content_block_delta", - json!({"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":content}}), - ), - ( - "content_block_stop", - json!({"type":"content_block_stop","index":0}), - ), - ( - "message_delta", - json!({"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"output_tokens":0}}), - ), - ("message_stop", json!({"type":"message_stop"})), - ]; - events - .into_iter() - .map(|(event, data)| format!("event: {event}\ndata: {data}\n\n")) - .collect() -} - -fn responses_body(model: &str, content: &str) -> Value { - json!({ - "id": "resp_echo", - "object": "response", - "created_at": 0, - "status": "completed", - "model": model, - "output": [{ - "id": "msg_echo", - "type": "message", - "status": "completed", - "role": "assistant", - "content": [{"type": "output_text", "text": content, "annotations": []}] - }], - "usage": {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0} - }) -} - -fn responses_stream(model: &str, content: &str) -> String { - let complete = responses_body(model, content); - let events = [ - json!({"type":"response.created","response":{"id":"resp_echo","object":"response","created_at":0,"status":"in_progress","model":model,"output":[]}}), - json!({"type":"response.output_item.added","output_index":0,"item":{"id":"msg_echo","type":"message","status":"in_progress","role":"assistant","content":[]}}), - json!({"type":"response.content_part.added","item_id":"msg_echo","output_index":0,"content_index":0,"part":{"type":"output_text","text":"","annotations":[]}}), - json!({"type":"response.output_text.delta","item_id":"msg_echo","output_index":0,"content_index":0,"delta":content,"logprobs":[]}), - json!({"type":"response.output_text.done","item_id":"msg_echo","output_index":0,"content_index":0,"text":content,"logprobs":[]}), - json!({"type":"response.completed","response":complete}), - ]; - events - .into_iter() - .enumerate() - .map(|(sequence_number, mut event)| { - event["sequence_number"] = json!(sequence_number); - format!( - "event: {}\ndata: {event}\n\n", - event["type"].as_str().unwrap_or("message") - ) - }) - .collect() -} - -fn gemini_body(model: &str, content: &str) -> Value { - json!({ - "candidates": [{ - "content": {"role": "model", "parts": [{"text": content}]}, - "finishReason": "STOP", - "index": 0 - }], - "usageMetadata": {"promptTokenCount": 0, "candidatesTokenCount": 0, "totalTokenCount": 0}, - "modelVersion": model - }) -} - -fn json_response(encoding: EchoEncoding, body: Value) -> Response { - encoded_response( - StatusCode::OK, - "application/json", - encoding, - serde_json::to_string(&body).expect("serialize echo response"), - ) -} - -fn sse_response(encoding: EchoEncoding, body: String) -> Response { - encoded_response(StatusCode::OK, "text/event-stream", encoding, body) -} - -fn encoded_response( - status: StatusCode, - content_type: &'static str, - encoding: EchoEncoding, - body: String, -) -> Response { - let mut response = Response::new(Body::from(body)); - *response.status_mut() = status; - response.headers_mut().insert( - axum::http::header::CONTENT_TYPE, - HeaderValue::from_static(content_type), - ); - response.headers_mut().insert( - HeaderName::from_static(ECHO_ENCODING_HEADER), - HeaderValue::from_str(&encoding.to_string()).expect("valid echo encoding header"), - ); - response -} - -fn error_response(message: &str) -> Response { - ( - StatusCode::BAD_REQUEST, - Json(json!({"error": {"message": message, "type": "echo_request_error"}})), - ) - .into_response() -} - -fn controlled_error_response() -> Response { - ( - StatusCode::TOO_MANY_REQUESTS, - Json(json!({ - "error": { - "message": "controlled echo failure", - "type": "echo_controlled_error", - "code": "echo_rate_limit" - } - })), - ) - .into_response() -} - -#[cfg(test)] -mod tests { - use super::*; - use std::sync::{Arc, Mutex}; - - fn test_client() -> reqwest::Client { - // Local echo binds 127.0.0.1; ignore ambient HTTP(S)_PROXY / ALL_PROXY - // (e.g. socks5h) which reqwest may not support without extra features. - reqwest::Client::builder() - .no_proxy() - .build() - .expect("reqwest client") - } - - async fn spawn_echo() -> (String, tokio::sync::oneshot::Sender<()>) { - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let address = listener.local_addr().unwrap(); - let (stop_tx, stop_rx) = tokio::sync::oneshot::channel(); - tokio::spawn(async move { - serve_with_shutdown(listener, EchoServerConfig::default(), async { - let _ = stop_rx.await; - }) - .await - .unwrap(); - }); - (format!("http://{address}"), stop_tx) - } - - #[tokio::test] - async fn raw_echo_supports_plain_and_base64() { - let (base, stop) = spawn_echo().await; - let client = test_client(); - let plain = client - .post(format!("{base}/echo")) - .body("hello") - .send() - .await - .unwrap(); - assert_eq!(plain.text().await.unwrap(), "hello"); - - let encoded = client - .post(format!("{base}/echo")) - .header(ECHO_ENCODING_HEADER, "base64") - .body("hello") - .send() - .await - .unwrap(); - assert_eq!(encoded.text().await.unwrap(), "aGVsbG8="); - let _ = stop.send(()); - } - - #[tokio::test] - async fn chat_echo_uses_last_user_message_and_streams() { - let (base, stop) = spawn_echo().await; - let client = test_client(); - let response: Value = client - .post(format!("{base}/v1/chat/completions")) - .header(ECHO_ENCODING_HEADER, "base64") - .json(&json!({ - "model": "echo-test", - "messages": [ - {"role":"user","content":"first"}, - {"role":"assistant","content":"ignored"}, - {"role":"user","content":"最后"} - ] - })) - .send() - .await - .unwrap() - .json() - .await - .unwrap(); - assert_eq!( - response["choices"][0]["message"]["content"], - STANDARD.encode("最后".as_bytes()) - ); - - let stream = client - .post(format!("{base}/v1/chat/completions")) - .json(&json!({"model":"echo-test","stream":true,"messages":[{"role":"user","content":"chunk"}]})) - .send() - .await - .unwrap(); - assert_eq!( - stream.headers()[axum::http::header::CONTENT_TYPE], - "text/event-stream" - ); - let stream = stream.text().await.unwrap(); - assert!(stream.contains("chunk")); - assert!(stream.contains("[DONE]")); - let _ = stop.send(()); - } - - #[tokio::test] - async fn native_protocol_endpoints_return_their_wire_shapes() { - let (base, stop) = spawn_echo().await; - let client = test_client(); - - let messages: Value = client - .post(format!("{base}/v1/messages")) - .json(&json!({"model":"m","messages":[{"role":"user","content":"message"}]})) - .send() - .await - .unwrap() - .json() - .await - .unwrap(); - assert_eq!(messages["content"][0]["text"], "message"); - - let responses: Value = client - .post(format!("{base}/v1/responses")) - .json(&json!({"model":"m","input":"response"})) - .send() - .await - .unwrap() - .json() - .await - .unwrap(); - assert_eq!(responses["output"][0]["content"][0]["text"], "response"); - - let gemini: Value = client - .post(format!("{base}/v1beta/models/gemini-test:generateContent")) - .json(&json!({"contents":[{"role":"user","parts":[{"text":"gemini"}]}]})) - .send() - .await - .unwrap() - .json() - .await - .unwrap(); - assert_eq!( - gemini["candidates"][0]["content"]["parts"][0]["text"], - "gemini" - ); - let _ = stop.send(()); - } - - #[tokio::test] - async fn pchronicle_echo_exercises_forward_rewrite_bridge_and_capture() { - let (echo_base, echo_stop) = spawn_echo().await; - let gateway_listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let gateway_address = gateway_listener.local_addr().unwrap(); - let admin_listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let admin_address = admin_listener.local_addr().unwrap(); - let config = crate::config::ProxyConfig::from_toml_str(&format!( - r#" -listen = "{gateway_address}" -admin_listen = "{admin_address}" -agent_id = "echo-test" -capture_level = "dialogue" - -[[models]] -name = "echo-upstream" -provider = "openai" -upstream = "{echo_base}/v1" - -[[models]] -name = "*" -forward = "echo-upstream" -"# - )) - .unwrap(); - let records = Arc::new(Mutex::new(Vec::new())); - let callback_records = Arc::clone(&records); - let sink: Arc = Arc::new( - crate::sink::CallbackSink::new("echo-test", move |_, _, record| { - callback_records.lock().unwrap().push(record); - Ok(()) - }), - ); - let state = tempfile::tempdir().unwrap(); - let (gateway_stop, gateway_stop_rx) = tokio::sync::oneshot::channel(); - let gateway = tokio::spawn(crate::serve_with_listeners_and_shutdown( - config, - state.path().to_path_buf(), - sink, - false, - gateway_listener, - admin_listener, - async { - let _ = gateway_stop_rx.await; - }, - )); - - let response = test_client() - .post(format!("http://{gateway_address}/v1/messages")) - .header(ECHO_ENCODING_HEADER, "base64") - .json(&json!({ - "model": "client-alias", - "max_tokens": 32, - "messages": [{"role":"user","content":"bridged"}] - })) - .send() - .await - .unwrap(); - assert_eq!(response.status(), reqwest::StatusCode::OK); - let response: Value = response.json().await.unwrap(); - assert_eq!(response["type"], "message"); - assert_eq!(response["model"], "client-alias"); - assert_eq!(response["content"][0]["text"], "YnJpZGdlZA=="); - - let _ = gateway_stop.send(()); - gateway.await.unwrap().unwrap(); - let _ = echo_stop.send(()); - - let records = records.lock().unwrap(); - assert!(records.iter().any(|record| { - record.kind == "llm.request" - && record.payload["model"] == "client-alias" - && record.payload["forward_to"] == "echo-upstream" - })); - assert!(records.iter().any(|record| { - record.kind == "llm.response" - && record.payload["assistant_content"] == "YnJpZGdlZA==" - && record.payload["forward_to"] == "echo-upstream" - })); - } -} diff --git a/crates/persisting-gateway/src/engine/actors/mod.rs b/crates/persisting-gateway/src/engine/actors/mod.rs deleted file mode 100644 index 44481e615..000000000 --- a/crates/persisting-gateway/src/engine/actors/mod.rs +++ /dev/null @@ -1,5 +0,0 @@ -mod run; -mod story; - -pub(crate) use run::RunActor; -pub(crate) use story::{StoryActor, StoryActorDeps}; diff --git a/crates/persisting-gateway/src/engine/actors/run.rs b/crates/persisting-gateway/src/engine/actors/run.rs deleted file mode 100644 index 3fdeee433..000000000 --- a/crates/persisting-gateway/src/engine/actors/run.rs +++ /dev/null @@ -1,122 +0,0 @@ -//! Run-scoped actor — subagent registry and cross-story links (one per capture runtime). - -use std::collections::HashMap; - -use pulsing_actor::prelude::*; - -use crate::engine::story::{RunId, StoryId}; -use crate::record::EventRecord; -use crate::subagent_link::{SubagentRegistry, enrich_record, main_route_for_backfill}; - -use super::super::wire::{RunCommand, RunReply, headers_to_header_map}; - -/// Run-level actor: subagent spawn links + active story index for one proxy instance. -pub(crate) struct RunActor { - registry: SubagentRegistry, - stories_by_run: HashMap>, -} - -impl RunActor { - pub fn new() -> Self { - Self { - registry: SubagentRegistry::default(), - stories_by_run: HashMap::new(), - } - } - - fn track_story(&mut self, run_id: Option<&RunId>, story_id: &StoryId) { - let Some(run_id) = run_id else { - return; - }; - let entry = self - .stories_by_run - .entry(run_id.as_str().to_string()) - .or_default(); - let sid = story_id.as_str().to_string(); - if !entry.contains(&sid) { - entry.push(sid); - } - } -} - -impl Default for RunActor { - fn default() -> Self { - Self::new() - } -} - -#[async_trait] -impl Actor for RunActor { - fn metadata(&self) -> HashMap { - HashMap::from([ - ("runs".into(), self.stories_by_run.len().to_string()), - ( - "tracked_runs".into(), - self.registry.tracked_run_count().to_string(), - ), - ]) - } - - async fn receive( - &mut self, - msg: Message, - _ctx: &mut ActorContext, - ) -> pulsing_actor::error::Result { - let cmd: RunCommand = msg.unpack()?; - let reply = match cmd { - RunCommand::Enrich { - record_bytes, - route, - headers, - body_bytes, - assistant_text, - story_id, - run_id, - } => { - if let Some(ref sid) = story_id { - self.track_story(run_id.as_ref(), sid); - } - let mut record: EventRecord = - serde_json::from_slice(&record_bytes).map_err(|e| { - pulsing_actor::error::PulsingError::from( - pulsing_actor::error::RuntimeError::Serialization(e.to_string()), - ) - })?; - let header_map = headers_to_header_map(&headers).map_err(|e| { - pulsing_actor::error::PulsingError::from( - pulsing_actor::error::RuntimeError::Other(e.to_string()), - ) - })?; - let body_val = body_bytes - .as_deref() - .map(serde_json::from_slice) - .transpose() - .map_err(|e| { - pulsing_actor::error::PulsingError::from( - pulsing_actor::error::RuntimeError::Serialization(e.to_string()), - ) - })?; - let outcome = enrich_record( - &mut record, - &route, - &header_map, - body_val.as_ref(), - assistant_text.as_deref(), - &mut self.registry, - ); - RunReply::Enrich { - record_bytes: serde_json::to_vec(&record).map_err(|e| { - pulsing_actor::error::PulsingError::from( - pulsing_actor::error::RuntimeError::Serialization(e.to_string()), - ) - })?, - backfills: outcome.spawn_link_backfills, - } - } - RunCommand::MainRoute { route } => { - RunReply::MainRoute(main_route_for_backfill(&route, &self.registry)) - } - }; - Message::pack(&reply) - } -} diff --git a/crates/persisting-gateway/src/engine/actors/story.rs b/crates/persisting-gateway/src/engine/actors/story.rs deleted file mode 100644 index ed94efe4a..000000000 --- a/crates/persisting-gateway/src/engine/actors/story.rs +++ /dev/null @@ -1,221 +0,0 @@ -//! Per-story I/O actor — one mailbox per `story_id`, owns turn state and [`CaptureEventSink`] writes. - -use std::collections::HashMap; -use std::path::PathBuf; -use std::sync::Arc; - -use anyhow::{Context, Result}; -use pulsing_actor::prelude::*; - -use super::super::story::{StoryId, TurnMachine}; -use super::super::wire::{CaptureAck, DraftPayload, StoryCommand, StoryReply, StoryScope}; -use crate::projection::dialogue::draft_stream_assistant_turn; -use crate::projection::frontmatter::refresh_document_frontmatter; -use crate::projection::markdown_pipeline::{LiveMarkdownWriter, MarkdownTarget}; -use crate::projection::markdown_policy::should_refresh_frontmatter; -use crate::sink::CaptureEventSink; - -/// Injected sink + markdown flag for each story actor instance. -#[derive(Clone)] -pub(crate) struct StoryActorDeps { - pub sink: Arc, - pub stream_markdown: bool, - pub storage: Arc, -} - -impl StoryActorDeps { - pub fn new( - sink: Arc, - storage: Arc, - stream_markdown: bool, - ) -> Self { - Self { - sink, - stream_markdown, - storage, - } - } -} - -/// Refresh frontmatter every Nth record so a single turn (request + response) -/// only triggers one rewrite instead of two. `Flush` always rewrites pending changes. -const FRONTMATTER_REFRESH_EVERY_N_RECORDS: u32 = 4; - -/// Per-story actor — serializes I/O and maintains [`TurnMachine`] for the narrative index. -pub(crate) struct StoryActor { - story_id: StoryId, - deps: StoryActorDeps, - turns: TurnMachine, - md: Option, - storage_session_id: Option, - /// Counter of records since last frontmatter rewrite — drives [`FRONTMATTER_REFRESH_EVERY_N_RECORDS`]. - frontmatter_pending: u32, - /// Last scope seen — `Flush` doesn't carry one but still needs to refresh frontmatter. - last_scope: Option, -} - -impl StoryActor { - pub fn new(story_id: StoryId, deps: StoryActorDeps) -> Self { - let turns = TurnMachine::new(story_id.clone()); - Self { - story_id, - deps, - turns, - md: None, - storage_session_id: None, - frontmatter_pending: 0, - last_scope: None, - } - } - - fn sync_scope(&mut self, scope: &StoryScope) { - self.storage_session_id = Some(scope.route().storage_session_id.clone()); - self.turns - .set_story_meta(scope.agent_id(), scope.context.run_id.clone()); - self.last_scope = Some(scope.clone()); - } - - fn md_writer(&mut self, scope: &StoryScope) -> &mut LiveMarkdownWriter { - if self.md.is_none() { - self.md = Some(LiveMarkdownWriter::new( - MarkdownTarget::new( - scope.route().clone(), - scope.agent_id().to_string(), - self.deps.storage.as_path().to_path_buf(), - ), - self.deps.stream_markdown, - )); - } - self.md.as_mut().expect("md writer initialized") - } - - /// Rewrite the markdown frontmatter for the current story, swallowing errors - /// (frontmatter is a derived projection — failure is non-fatal). - fn refresh_frontmatter(&mut self, scope: &StoryScope) { - let story = self.turns.snapshot(); - let user_turns = crate::engine::story_user_turn_count(&story); - let path = self.md_writer(scope).path(); - let _ = refresh_document_frontmatter( - self.deps.storage.as_path(), - scope.agent_id(), - scope.route(), - &path, - Some(user_turns), - ) - .map_err(|e| tracing::debug!("frontmatter refresh: {e:#}")); - self.frontmatter_pending = 0; - } - - fn flush_pending_frontmatter(&mut self, scope: &StoryScope) { - if self.frontmatter_pending > 0 { - self.refresh_frontmatter(scope); - } - } - - async fn handle(&mut self, cmd: StoryCommand) -> Result { - match cmd { - StoryCommand::Flush => { - // Drain any pending frontmatter writes so external observers see a - // consistent markdown file after `flush()` returns. - if self.frontmatter_pending > 0 - && let Some(scope) = self.last_scope.clone() - { - self.refresh_frontmatter(&scope); - } - return Ok(StoryReply::Ack(CaptureAck::ok())); - } - StoryCommand::LocalSnapshot => { - let storage_session_id = self - .storage_session_id - .clone() - .unwrap_or_else(|| self.story_id.as_str().to_string()); - return Ok(StoryReply::LocalSnapshot { - storage_session_id, - story: self.turns.snapshot(), - }); - } - StoryCommand::Snapshot { scope } => { - self.sync_scope(&scope); - self.flush_pending_frontmatter(&scope); - return Ok(StoryReply::Snapshot { - story: self.turns.snapshot(), - }); - } - _ => {} - } - let scope = cmd.scope().clone(); - self.sync_scope(&scope); - match cmd { - StoryCommand::PersistRecord { record_bytes, .. } => { - let mut rec: crate::record::EventRecord = serde_json::from_slice(&record_bytes)?; - let mut next_turns = self.turns.clone(); - next_turns.observe_record(&mut rec); - let sink = Arc::clone(&self.deps.sink); - let route = scope.route().clone(); - let agent_id = scope.agent_id().to_string(); - rec = tokio::task::spawn_blocking(move || { - sink.append(&route, &agent_id, &mut rec) - .context("capture append")?; - Ok::<_, anyhow::Error>(rec) - }) - .await - .context("join capture append")??; - self.turns = next_turns; - if let Err(error) = self.md_writer(&scope).write_record(&rec) { - tracing::warn!( - target: "persisting_gateway", - story_id = %self.story_id.as_str(), - "markdown projection lagged canonical event: {error:#}" - ); - } - if should_refresh_frontmatter(&rec) { - self.frontmatter_pending = self.frontmatter_pending.saturating_add(1); - if self.frontmatter_pending >= FRONTMATTER_REFRESH_EVERY_N_RECORDS { - self.refresh_frontmatter(&scope); - } - } - } - StoryCommand::UpsertDraft { draft_bytes, .. } => { - let draft: DraftPayload = serde_json::from_slice(&draft_bytes)?; - let mut rec: crate::record::EventRecord = - serde_json::from_slice(&draft.record_bytes)?; - rec.seq = self - .deps - .sink - .peek_next_seq(scope.route()) - .context("draft markdown requires sink peek_next_seq")?; - if let Some(turn) = draft_stream_assistant_turn(&rec, &draft.assistant_content)? { - self.md_writer(&scope) - .write_draft(rec.call_id.as_deref().unwrap_or(""), turn)?; - } - } - StoryCommand::Flush | StoryCommand::Snapshot { .. } | StoryCommand::LocalSnapshot => { - unreachable!() - } - } - Ok(StoryReply::Ack(CaptureAck::ok())) - } -} - -#[async_trait] -impl Actor for StoryActor { - fn metadata(&self) -> HashMap { - HashMap::from([ - ("story_id".into(), self.story_id.as_str().to_string()), - ("turns".into(), self.turns.turns().len().to_string()), - ]) - } - - async fn receive( - &mut self, - msg: Message, - _ctx: &mut ActorContext, - ) -> pulsing_actor::error::Result { - let cmd: StoryCommand = msg.unpack()?; - let reply = match self.handle(cmd).await { - Ok(r) => r, - Err(e) => StoryReply::Ack(CaptureAck::err(format!("{e:#}"))), - }; - Message::pack(&reply) - } -} diff --git a/crates/persisting-gateway/src/engine/apply_queue.rs b/crates/persisting-gateway/src/engine/apply_queue.rs deleted file mode 100644 index cd9f81ca1..000000000 --- a/crates/persisting-gateway/src/engine/apply_queue.rs +++ /dev/null @@ -1,386 +0,0 @@ -//! Per-`story_id` ordered capture apply queue — preserves event order within a story. - -use std::path::PathBuf; -use std::sync::{Arc, Mutex, mpsc as std_mpsc}; - -use dashmap::DashMap; -use tokio::sync::{mpsc, oneshot}; - -use super::coordinator::CaptureRuntimeInner; -use super::{CallContext, Event}; -use crate::dead_letter; - -const APPLY_QUEUE_CAPACITY: usize = 256; -const REJECTED_EVENT_QUEUE_CAPACITY: usize = 256; - -enum RejectedEventMessage { - Event { ctx: Arc, event: Event }, - Finish, -} - -struct RejectedEventWriter { - tx: std_mpsc::SyncSender, - worker: Mutex>>, -} - -impl RejectedEventWriter { - fn new(storage: Arc) -> Self { - let (tx, rx) = std_mpsc::sync_channel(REJECTED_EVENT_QUEUE_CAPACITY); - let worker = std::thread::Builder::new() - .name("persisting-dead-letter".to_string()) - .spawn(move || { - while let Ok(message) = rx.recv() { - let RejectedEventMessage::Event { ctx, event } = message else { - break; - }; - if let Err(error) = dead_letter::append_dead_letter( - storage.as_path(), - &ctx, - &event, - "apply queue full or closed", - None, - ) { - tracing::error!("dead letter write failed: {error:#}"); - } - } - }) - .expect("dead-letter writer thread must start with capture runtime"); - Self { - tx, - worker: Mutex::new(Some(worker)), - } - } - - fn try_record(&self, ctx: Arc, event: Event) { - if let Err(error) = self.tx.try_send(RejectedEventMessage::Event { ctx, event }) { - tracing::warn!( - target: "persisting_gateway", - "dead-letter queue rejected overloaded capture event: {error}" - ); - } - } -} - -impl Drop for RejectedEventWriter { - fn drop(&mut self) { - let _ = self.tx.send(RejectedEventMessage::Finish); - if let Some(worker) = self.worker.lock().expect("dead-letter worker mutex").take() { - let _ = worker.join(); - } - } -} - -enum ApplyJob { - Capture { - ctx: Arc, - event: Event, - /// WAL sequence to ack only after the canonical sink confirms success. - wal_seq: Option, - }, - Barrier { - ack: oneshot::Sender<()>, - }, -} - -/// Serializes `apply` calls per story while keeping the proxy non-blocking. -#[derive(Clone)] -pub(crate) struct ApplyDispatcher { - inner: Arc, - queues: Arc>>, - rejected: Arc, -} - -impl ApplyDispatcher { - pub(crate) fn new(inner: Arc) -> Self { - let rejected = Arc::new(RejectedEventWriter::new(Arc::clone( - &inner.story_deps.storage, - ))); - Self { - inner, - queues: Arc::new(DashMap::new()), - rejected, - } - } - - pub(crate) fn enqueue(&self, ctx: Arc, event: Event, wal_seq: Option) { - let story_id = ctx.story_id().as_str().to_string(); - let tx = self - .queues - .entry(story_id) - .or_insert_with(|| self.spawn_consumer()) - .clone(); - - let job = ApplyJob::Capture { - ctx, - event, - wal_seq, - }; - if let Err(e) = tx.try_send(job) { - self.record_rejected_job(e.into_inner()); - } - } - - /// Wait for all jobs already accepted by every per-story queue to finish. - pub(crate) async fn flush(&self) -> anyhow::Result<()> { - let queues: Vec<_> = self - .queues - .iter() - .map(|entry| entry.value().clone()) - .collect(); - for tx in queues { - let (ack, done) = oneshot::channel(); - tx.send(ApplyJob::Barrier { ack }) - .await - .map_err(|_| anyhow::anyhow!("apply queue closed while flushing"))?; - done.await - .map_err(|_| anyhow::anyhow!("apply queue barrier dropped"))?; - } - Ok(()) - } - - fn spawn_consumer(&self) -> mpsc::Sender { - let (tx, mut rx) = mpsc::channel::(APPLY_QUEUE_CAPACITY); - let inner = Arc::clone(&self.inner); - tokio::spawn(async move { - while let Some(job) = rx.recv().await { - match job { - ApplyJob::Capture { - ctx, - event, - wal_seq, - } => { - match inner.apply(&ctx, event).await { - Ok(()) => { - if let Some(seq) = wal_seq { - inner.wal.ack(seq); - } - } - Err(e) => { - // Keep the WAL entry pending. The dead letter is an - // operator diagnostic, not a durable-store substitute; - // restart replay may recover a transient sink failure. - tracing::warn!( - target: "persisting_gateway", - "capture apply: {e:#}" - ); - } - } - } - ApplyJob::Barrier { ack } => { - let _ = ack.send(()); - } - } - } - }); - tx - } - - fn record_rejected_job(&self, job: ApplyJob) { - let ApplyJob::Capture { - ctx, - event, - wal_seq: _, - } = job - else { - return; - }; - self.rejected.try_record(Arc::clone(&ctx), event); - // A rejected queue job never reached the durable sink, so its WAL row - // deliberately remains pending for restart replay. - tracing::warn!( - target: "persisting_gateway", - story_id = %ctx.story_id().as_str(), - "capture apply queue rejected job" - ); - } -} - -#[cfg(test)] -mod tests { - use std::collections::HashMap; - use std::sync::{Arc, Mutex}; - - use super::*; - use crate::Call; - use crate::config::CaptureLevel; - use crate::engine::CaptureEngine; - use crate::engine::{CompleteEvent, Event, RequestEvent}; - use crate::protocol::ProtocolKind; - use crate::provider::ProviderKind; - use crate::record::EventRecord; - use crate::session::index::SessionIndexStore; - use crate::session::storage::CaptureRoute; - use crate::sink::CaptureEventSink; - - struct OrderRecordingSink { - order: Mutex>, - next_seq: Mutex>, - } - - struct SlowSink; - - impl CaptureEventSink for SlowSink { - fn append( - &self, - _route: &CaptureRoute, - _agent_id: &str, - _record: &mut EventRecord, - ) -> anyhow::Result<()> { - std::thread::sleep(std::time::Duration::from_millis(400)); - Ok(()) - } - } - - impl OrderRecordingSink { - fn new() -> Arc { - Arc::new(Self { - order: Mutex::new(Vec::new()), - next_seq: Mutex::new(HashMap::new()), - }) - } - - fn drain_order(&self) -> Vec { - self.order.lock().unwrap().clone() - } - } - - impl CaptureEventSink for OrderRecordingSink { - fn append( - &self, - route: &CaptureRoute, - _agent_id: &str, - record: &mut EventRecord, - ) -> anyhow::Result<()> { - let mut guard = self.next_seq.lock().unwrap(); - let seq = guard.entry(route.seq_key()).or_insert(0); - record.seq = *seq; - *seq += 1; - drop(guard); - self.order.lock().unwrap().push(format!( - "{}:{}", - record.kind, - record.call_id.as_deref().unwrap_or("") - )); - Ok(()) - } - - fn peek_next_seq(&self, route: &CaptureRoute) -> Option { - Some( - self.next_seq - .lock() - .unwrap() - .get(&route.seq_key()) - .copied() - .unwrap_or(0), - ) - } - } - - fn sample_ctx(call_id: &str) -> CallContext { - CallContext::new( - CaptureRoute { - root_session: Some("run-1".into()), - session_id: "sess".into(), - storage_session_id: "run-1".into(), - subagent_id: None, - }, - "agent", - Call { - call_id: call_id.into(), - trace_id: "t1".into(), - started_at: "2026-01-01T00:00:00Z".into(), - }, - Vec::new(), - CaptureLevel::Dialogue, - "m", - "m", - ProviderKind::OpenAi, - ProtocolKind::ChatCompletions, - false, - ) - } - - #[tokio::test] - async fn dispatcher_preserves_request_before_response_order() { - let dir = tempfile::tempdir().unwrap(); - let storage = Arc::new(dir.path().to_path_buf()); - let sink = OrderRecordingSink::new(); - let index = SessionIndexStore::open(dir.path()).unwrap().clone_handle(); - let engine = CaptureEngine::new(sink.clone(), index, storage.clone(), false) - .await - .unwrap(); - - let ctx_req = sample_ctx("call-a"); - let ctx_resp = sample_ctx("call-a"); - engine.spawn_apply( - ctx_req, - Event::Request(RequestEvent { - path: "/v1/chat/completions".into(), - method: "POST".into(), - url: None, - body_bytes: 10, - user_content: Some("hi".into()), - body_json: None, - semantic: None, - model_rewritten: false, - headers: vec![], - }), - ); - engine.spawn_apply( - ctx_resp, - Event::ResponseComplete(CompleteEvent { - status: 200, - resp_bytes: bytes::Bytes::from_static( - br#"{"choices":[{"message":{"content":"ok"}}]}"#, - ), - streaming: false, - stream_metrics: None, - assistant_content: Some("ok".into()), - semantic: None, - headers: vec![], - }), - ); - - engine.flush().await.unwrap(); - - let order = sink.drain_order(); - assert!( - order.len() >= 2 - && order[0].starts_with("llm.request:") - && order[1].starts_with("llm.response"), - "expected request before response, got {order:?}" - ); - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 1)] - async fn durable_sink_wait_does_not_block_tokio_worker() { - let dir = tempfile::tempdir().unwrap(); - let storage = Arc::new(dir.path().to_path_buf()); - let index = SessionIndexStore::open(dir.path()).unwrap().clone_handle(); - let engine = CaptureEngine::new(Arc::new(SlowSink), index, storage, false) - .await - .unwrap(); - - let started = std::time::Instant::now(); - engine.spawn_apply( - sample_ctx("slow-call"), - Event::Request(RequestEvent { - path: "/v1/chat/completions".into(), - method: "POST".into(), - url: None, - body_bytes: 10, - user_content: Some("hi".into()), - body_json: None, - semantic: None, - model_rewritten: false, - headers: vec![], - }), - ); - tokio::time::sleep(std::time::Duration::from_millis(10)).await; - assert!( - started.elapsed() < std::time::Duration::from_millis(200), - "synchronous sink wait starved the only Tokio worker" - ); - engine.flush().await.unwrap(); - } -} diff --git a/crates/persisting-gateway/src/engine/coordinator.rs b/crates/persisting-gateway/src/engine/coordinator.rs deleted file mode 100644 index 92a12ee77..000000000 --- a/crates/persisting-gateway/src/engine/coordinator.rs +++ /dev/null @@ -1,369 +0,0 @@ -//! Capture actor coordinator: prepare → run actor → story actors. - -use std::path::{Path, PathBuf}; -use std::sync::Arc; - -use anyhow::{Context, Result}; -use dashmap::DashMap; -use dashmap::mapref::entry::Entry; -use pulsing_actor::prelude::*; - -use super::actors::{RunActor, StoryActor, StoryActorDeps}; -use super::apply_queue::ApplyDispatcher; -use super::egress::persist_story_snapshots; -use super::prepare::CapturePreparer; -use super::story::Story; -use super::story::{StoryContext, StoryId}; -use super::wal::{EventWal, replay_pending}; -use super::wire::{ - CaptureAck, RUN_ACTOR_NAME, StoryCommand, StoryReply, StoryScope, run_main_route, -}; -use super::{CallContext, Event}; -use crate::dead_letter; -use crate::session::index::SessionIndexHandle; -use crate::sink::CaptureEventSink; -use crate::subagent_link::{SpawnLinkBackfill, spawn_link_backfill_record}; - -const STORY_MAILBOX: usize = 256; - -pub(crate) struct CaptureRuntimeInner { - system: Arc, - preparer: Arc, - run: ActorRef, - pub(crate) story_deps: StoryActorDeps, - stories: Arc>, - pub(crate) wal: Arc, -} - -/// Actor topology (one ActorSystem per proxy): -/// -/// ```text -/// CaptureRuntime -/// ├── capture/run RunActor (subagent registry + run story index) -/// └── capture/story/{story_id} StoryActor (TurnMachine + sink/md I/O) -/// ``` -#[derive(Clone)] -pub struct CaptureRuntime { - inner: Arc, - apply_dispatcher: ApplyDispatcher, -} - -impl CaptureRuntime { - pub async fn new( - sink: Arc, - index: SessionIndexHandle, - storage: Arc, - stream_markdown: bool, - ) -> Result { - let system = ActorSystem::builder() - .mailbox_capacity(STORY_MAILBOX) - .build() - .await - .context("capture actor system")?; - - let run = system - .spawn_named(RUN_ACTOR_NAME, RunActor::new()) - .await - .map_err(pulsing_err)?; - - let sink = Arc::clone(&sink); - let wal = Arc::new(EventWal::open(storage.as_path())); - let inner = Arc::new(CaptureRuntimeInner { - system, - preparer: Arc::new(CapturePreparer { - index, - storage: Arc::clone(&storage), - stream_markdown, - }), - run, - story_deps: StoryActorDeps::new(sink, Arc::clone(&storage), stream_markdown), - stories: Arc::new(DashMap::new()), - wal, - }); - let apply_dispatcher = ApplyDispatcher::new(Arc::clone(&inner)); - let runtime = Self { - inner, - apply_dispatcher, - }; - - runtime.replay_wal(storage.as_path()).await; - - Ok(runtime) - } - - /// Replay events that were appended to the WAL but never acked - /// (process crash between [`Self::spawn_apply`] and apply completion). - async fn replay_wal(&self, storage: &Path) { - let pending = replay_pending(storage); - if pending.is_empty() { - return; - } - tracing::info!( - target: "persisting_gateway", - replayed = pending.len(), - "wal replay starting" - ); - for entry in pending { - let ctx = Arc::new(entry.context.to_call_context()); - let event = entry.event.to_event(); - // Replay through the same dispatcher the original call used so that - // ordering, dead-letter, and ack semantics all stay consistent. - self.apply_dispatcher.enqueue(ctx, event, Some(entry.seq)); - } - } - - /// Enqueue a capture event on the per-story ordered apply queue (non-blocking for callers). - /// - /// Accepts both `CallContext` (owned) and `Arc` so callers in the - /// streaming hot-path can share an `Arc` and avoid per-event clones, while - /// non-streaming callers can keep passing owned values. - /// - /// Best-effort submits to the bounded WAL writer before queuing. The submit - /// never waits for disk I/O; only records committed before a crash can be replayed. - pub fn spawn_apply(&self, ctx: impl Into>, event: Event) { - let ctx = ctx.into(); - let wal_seq = self.inner.wal.append_event(&ctx, &event); - self.apply_dispatcher.enqueue(ctx, event, wal_seq); - } - - pub async fn apply(&self, ctx: &CallContext, event: Event) -> Result<()> { - self.inner.apply(ctx, event).await - } - - pub async fn shutdown(self) -> Result<()> { - self.flush().await?; - let snapshots = self.inner.collect_local_snapshots().await?; - persist_story_snapshots(self.inner.story_deps.storage.as_path(), &snapshots)?; - // Never erase a failed or rejected capture. A fully acknowledged WAL - // can be truncated; otherwise its pending rows remain for restart replay. - if replay_pending(self.inner.story_deps.storage.as_path()).is_empty() - && let Err(e) = self.inner.wal.truncate() - { - tracing::warn!(target: "persisting_gateway", "wal truncate on shutdown: {e:#}"); - } - self.inner.system.shutdown().await.map_err(pulsing_err) - } - - /// Wait until accepted async apply jobs and story mailboxes have drained. - pub async fn flush(&self) -> Result<()> { - self.apply_dispatcher.flush().await?; - self.inner.flush_stories().await?; - self.inner.preparer.index.flush_if_dirty()?; - // ACKs are submitted asynchronously to the group-commit writer. Make - // their durability part of the runtime flush barrier before shutdown - // inspects pending WAL entries or decides that truncation is safe. - self.inner.wal.flush()?; - Ok(()) - } - - /// Read-model snapshot for one story (TurnMachine state inside StoryActor). - pub async fn story_snapshot(&self, context: &StoryContext) -> Result { - self.inner.story_snapshot(context).await - } -} - -impl CaptureRuntimeInner { - pub(crate) async fn apply(&self, ctx: &CallContext, event: Event) -> Result<()> { - self.apply_inner(ctx, &event).await - } - - async fn apply_inner(&self, ctx: &CallContext, event: &Event) -> Result<()> { - let mut prepared = match self.preparer.prepare(&self.run, ctx, event.clone()).await { - Ok(p) => p, - Err(e) => { - record_dead_letter(self.story_deps.storage.as_path(), ctx, event, &e, None); - return Err(e); - } - }; - - if !prepared.backfills.is_empty() { - let main = run_main_route(&self.run, prepared.ctx.route()).await?; - self.dispatch_backfills(&prepared.ctx, &main, &prepared.backfills) - .await?; - } - - if let Some(cmd) = prepared.take_story_command() { - let record_json = persist_record_json(&cmd); - let story_id = ctx.story_id().as_str().to_string(); - if let Err(e) = self.dispatch_story(&story_id, cmd).await { - record_dead_letter( - self.story_deps.storage.as_path(), - ctx, - event, - &e, - record_json, - ); - return Err(e); - } - } - - Ok(()) - } - - async fn flush_stories(&self) -> Result<()> { - for entry in self.stories.iter() { - let actor = entry.value().clone(); - let reply: StoryReply = actor.ask(StoryCommand::Flush).await.map_err(pulsing_err)?; - story_reply_ack(reply)?.into_result()?; - } - Ok(()) - } - - async fn collect_local_snapshots(&self) -> Result> { - let mut out = std::collections::HashMap::new(); - for entry in self.stories.iter() { - let actor = entry.value().clone(); - let reply: StoryReply = actor - .ask(StoryCommand::LocalSnapshot) - .await - .map_err(pulsing_err)?; - if let StoryReply::LocalSnapshot { - storage_session_id, - story, - } = reply - { - out.insert(storage_session_id, story); - } - } - Ok(out) - } - - async fn story_snapshot(&self, context: &StoryContext) -> Result { - let story_id = context.story_id.as_str(); - let actor = self.story_actor(story_id).await?; - let scope = StoryScope { - context: context.clone(), - }; - let reply: StoryReply = actor - .ask(StoryCommand::Snapshot { scope }) - .await - .map_err(pulsing_err)?; - match reply { - StoryReply::Snapshot { story } => Ok(story), - StoryReply::LocalSnapshot { story, .. } => Ok(story), - StoryReply::Ack(ack) => Err(anyhow::anyhow!( - "unexpected ack for snapshot: {}", - ack.error.unwrap_or_else(|| "unknown".into()) - )), - } - } - - async fn dispatch_story(&self, story_id: &str, cmd: StoryCommand) -> Result<()> { - let actor = self.story_actor(story_id).await?; - let reply: StoryReply = actor.ask(cmd).await.map_err(pulsing_err)?; - story_reply_ack(reply)?.into_result() - } - - async fn dispatch_backfills( - &self, - ctx: &CallContext, - main_route: &crate::session::storage::CaptureRoute, - backfills: &[SpawnLinkBackfill], - ) -> Result<()> { - let scope = StoryScope { - context: StoryContext::from_route(main_route.clone(), ctx.agent_id().to_string()), - }; - let actor = self - .story_actor( - StoryContext::from_route(main_route.clone(), ctx.agent_id()) - .story_id() - .as_str(), - ) - .await?; - for bf in backfills { - let rec = spawn_link_backfill_record(&bf.parent_call_id, &bf.links, &ctx.call); - let record_bytes = serde_json::to_vec(&rec)?; - let cmd = StoryCommand::persist_record(scope.clone(), record_bytes); - let reply: StoryReply = actor.ask(cmd).await.map_err(pulsing_err)?; - if let Err(e) = story_reply_ack(reply)?.into_result() { - tracing::warn!("spawn link backfill failed: {e:#}"); - } - } - Ok(()) - } - - async fn story_actor(&self, story_id: &str) -> Result { - if let Some(existing) = self.stories.get(story_id) { - return Ok(existing.clone()); - } - - let name = story_actor_name(story_id); - if let Ok(actor) = self.system.resolve(name.as_str()).await { - self.stories - .entry(story_id.to_string()) - .or_insert(actor.clone()); - return Ok(actor); - } - - let id = StoryId::new(story_id); - let actor = self - .system - .spawning() - .name(&name) - .supervision(SupervisionSpec::on_failure().with_max_restarts(3)) - .mailbox_capacity(STORY_MAILBOX) - .spawn(StoryActor::new(id, self.story_deps.clone())) - .await - .map_err(pulsing_err)?; - - match self.stories.entry(story_id.to_string()) { - Entry::Occupied(entry) => Ok(entry.get().clone()), - Entry::Vacant(entry) => { - entry.insert(actor.clone()); - Ok(actor) - } - } - } -} - -/// Public entry type (actor-backed capture runtime). -pub type CaptureEngine = CaptureRuntime; - -fn story_reply_ack(reply: StoryReply) -> Result { - match reply { - StoryReply::Ack(ack) => Ok(ack), - StoryReply::Snapshot { .. } | StoryReply::LocalSnapshot { .. } => { - Err(anyhow::anyhow!("unexpected snapshot reply")) - } - } -} - -/// Decode a `PersistRecord` command's JSON-encoded payload back to a `String` for -/// the dead-letter file (which carries `prepared_record_json` as a textual JSON). -/// Invalid UTF-8 should never happen — `serde_json::to_vec` always produces valid -/// UTF-8 — but if it ever does we drop the prepared JSON rather than panic. -fn persist_record_json(cmd: &StoryCommand) -> Option { - match cmd { - StoryCommand::PersistRecord { record_bytes, .. } => { - String::from_utf8(record_bytes.clone()).ok() - } - _ => None, - } -} - -fn record_dead_letter( - storage: &Path, - ctx: &CallContext, - event: &Event, - error: &anyhow::Error, - prepared_record_json: Option, -) { - if let Err(dl) = dead_letter::append_dead_letter( - storage, - ctx, - event, - &format!("{error:#}"), - prepared_record_json, - ) { - tracing::error!("dead letter write failed: {dl:#}"); - } -} - -fn story_actor_name(story_id: &str) -> String { - let sanitized = story_id.replace('|', "/"); - format!("capture/story/{sanitized}") -} - -fn pulsing_err(e: pulsing_actor::error::PulsingError) -> anyhow::Error { - anyhow::anyhow!("{e}") -} diff --git a/crates/persisting-gateway/src/engine/egress.rs b/crates/persisting-gateway/src/engine/egress.rs deleted file mode 100644 index 9f12f0708..000000000 --- a/crates/persisting-gateway/src/engine/egress.rs +++ /dev/null @@ -1,279 +0,0 @@ -//! Read-path: rebuild [`Story`] from persisted records and extract dialogue projections. - -use std::collections::{BTreeSet, HashMap}; -use std::path::{Path, PathBuf}; - -use anyhow::{Context, Result}; -use serde::{Deserialize, Serialize}; - -use super::story::{RunId, Story, StoryId, TurnKind, TurnMachine}; -use crate::record::EventRecord; -use crate::session::storage::CaptureRoute; - -/// On-disk cache of live story snapshots (keyed by markdown session stem). -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub(crate) struct StorySnapshotsFile { - pub updated_at: String, - pub stories: HashMap, - /// Denormalized for storage/frontmatter without importing [`Story`]. - #[serde(default, skip_serializing_if = "HashMap::is_empty")] - pub turn_counts: HashMap, -} - -/// Rebuild in-memory story state by replaying persisted records (offline egress). -fn rebuild_story_from_records( - story_id: StoryId, - agent_id: impl Into, - run_id: Option, - records: &[EventRecord], -) -> Story { - TurnMachine::replay_records(story_id, agent_id, run_id, records).snapshot() -} - -/// Rebuild story for one session markdown stem from Lance records. -pub fn rebuild_session_story( - session_id: &str, - root_session: &str, - records: &[EventRecord], -) -> Story { - let route = CaptureRoute::for_replay_stem(root_session, session_id); - let agent_id = records - .iter() - .find_map(|r| r.agent_id.clone()) - .unwrap_or_else(|| "unknown".into()); - let story_id = StoryId::from_seq_key(route.seq_key()); - let run_id = route.root_session.clone().map(RunId); - rebuild_story_from_records(story_id, agent_id, run_id, records) -} - -/// Visible call ids from the story read model. -pub fn story_call_ids(story: &Story) -> BTreeSet { - let mut ids = BTreeSet::new(); - for turn in &story.turns { - if let Some(user) = &turn.user - && let Some(call_id) = &user.call_id - { - ids.insert(call_id.as_str().to_string()); - } - if let Some(assistant) = &turn.assistant - && let Some(call_id) = &assistant.call_id - { - ids.insert(call_id.as_str().to_string()); - } - for call in &turn.calls { - ids.insert(call.call_id.as_str().to_string()); - } - } - ids -} - -/// Count user-visible dialogue turns (Dialogue kind with user text). -pub fn story_user_turn_count(story: &Story) -> u64 { - story - .turns - .iter() - .filter(|t| t.kind == TurnKind::Dialogue && t.user.is_some()) - .count() as u64 -} - -fn story_snapshots_path(storage: &Path) -> PathBuf { - storage.join(".capture").join("story_snapshots.json") -} - -pub fn persist_story_snapshots(storage: &Path, stories: &HashMap) -> Result<()> { - if stories.is_empty() { - return Ok(()); - } - let dir = storage.join(".capture"); - std::fs::create_dir_all(&dir).context("create .capture")?; - let path = story_snapshots_path(storage); - let turn_counts = stories - .iter() - .map(|(stem, story)| (stem.clone(), story_user_turn_count(story))) - .collect(); - let file = StorySnapshotsFile { - updated_at: chrono::Utc::now().to_rfc3339(), - stories: stories.clone(), - turn_counts, - }; - let json = serde_json::to_string_pretty(&file).context("serialize story snapshots")?; - std::fs::write(&path, json).with_context(|| format!("write {}", path.display()))?; - Ok(()) -} - -pub fn load_story_snapshots(storage: &Path) -> Result> { - let path = story_snapshots_path(storage); - if !path.is_file() { - return Ok(HashMap::new()); - } - let raw = std::fs::read_to_string(&path).with_context(|| format!("read {}", path.display()))?; - let file: StorySnapshotsFile = - serde_json::from_str(&raw).context("parse story_snapshots.json")?; - Ok(file.stories) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::Call; - use crate::config::CaptureLevel; - use crate::sink::{llm_request_summary_record, llm_response_record_with_content}; - - fn sample_call(id: &str) -> Call { - Call { - call_id: id.into(), - trace_id: "t".into(), - started_at: "2026-01-01T00:00:00Z".into(), - } - } - - #[test] - fn rebuild_story_counts_dialogue_turns() { - let call = sample_call("c1"); - let req = llm_request_summary_record( - Some("run".into()), - Some("agent".into()), - "m", - "/v1/chat/completions", - 10, - "chat", - "openai", - Some("hi".into()), - None, - &call, - CaptureLevel::Dialogue, - None, - ); - let resp = llm_response_record_with_content( - Some("run".into()), - Some("agent".into()), - 200, - &serde_json::json!({}), - false, - Some("ok".into()), - &call, - CaptureLevel::Dialogue, - ); - let story = rebuild_session_story("run", "run", &[req, resp]); - assert_eq!(story_user_turn_count(&story), 1); - assert!(story_call_ids(&story).contains("c1")); - } - - #[test] - fn rebuild_multi_turn_session_story() { - let mut records = Vec::new(); - for (i, text) in [("c1", "first"), ("c2", "second")] { - let call = sample_call(i); - records.push(llm_request_summary_record( - Some("run".into()), - Some("agent".into()), - "m", - "/v1/chat/completions", - 10, - "chat", - "openai", - Some(text.into()), - None, - &call, - CaptureLevel::Dialogue, - None, - )); - records.push(llm_response_record_with_content( - Some("run".into()), - Some("agent".into()), - 200, - &serde_json::json!({}), - false, - Some(format!("reply-{text}")), - &call, - CaptureLevel::Dialogue, - )); - } - let story = rebuild_session_story("run", "run", &records); - assert_eq!(story_user_turn_count(&story), 2); - assert_eq!(story.turns.len(), 2); - assert_eq!(story_call_ids(&story).len(), 2); - } - - #[test] - fn subagent_session_uses_root_scoped_story_id() { - let call = sample_call("c1"); - let req = llm_request_summary_record( - Some("agent-sub".into()), - Some("agent".into()), - "m", - "/v1/chat/completions", - 10, - "chat", - "openai", - Some("sub".into()), - None, - &call, - CaptureLevel::Dialogue, - None, - ); - let story = rebuild_session_story("agent-sub", "run-root", &[req]); - assert_eq!(story.story_id.as_str(), "run-root|agent-sub"); - assert_eq!(story.run_id.as_ref().map(|r| r.as_str()), Some("run-root")); - } - - #[test] - fn story_user_turn_count_ignores_autonomous_turn() { - let call = sample_call("tool-only"); - let req = llm_request_summary_record( - Some("run".into()), - Some("agent".into()), - "m", - "/v1/chat/completions", - 10, - "chat", - "openai", - None, - None, - &call, - CaptureLevel::Dialogue, - None, - ); - let story = rebuild_session_story("run", "run", &[req]); - assert_eq!(story.turns.len(), 1); - assert_eq!(story.turns[0].kind, TurnKind::Autonomous); - assert_eq!(story_user_turn_count(&story), 0); - } - - #[test] - fn persist_and_load_story_snapshots_roundtrip() { - let dir = tempfile::tempdir().unwrap(); - let call = sample_call("c1"); - let req = llm_request_summary_record( - Some("sess".into()), - Some("agent".into()), - "m", - "/v1/chat/completions", - 10, - "chat", - "openai", - Some("hi".into()), - None, - &call, - CaptureLevel::Dialogue, - None, - ); - let story = rebuild_session_story("sess", "sess", &[req]); - let mut stories = HashMap::new(); - stories.insert("sess".to_string(), story.clone()); - - persist_story_snapshots(dir.path(), &stories).unwrap(); - assert!(story_snapshots_path(dir.path()).is_file()); - - let loaded = load_story_snapshots(dir.path()).unwrap(); - assert_eq!(loaded.get("sess").unwrap().turns.len(), story.turns.len()); - assert_eq!(loaded.get("sess").unwrap().agent_id, story.agent_id); - } - - #[test] - fn persist_empty_snapshots_is_noop() { - let dir = tempfile::tempdir().unwrap(); - persist_story_snapshots(dir.path(), &HashMap::new()).unwrap(); - assert!(!story_snapshots_path(dir.path()).exists()); - } -} diff --git a/crates/persisting-gateway/src/engine/mod.rs b/crates/persisting-gateway/src/engine/mod.rs deleted file mode 100644 index 39c1422b1..000000000 --- a/crates/persisting-gateway/src/engine/mod.rs +++ /dev/null @@ -1,24 +0,0 @@ -mod actors; -mod apply_queue; -mod coordinator; -mod egress; -mod prepare; -mod story; -mod wal; -mod wire; - -pub(crate) use wire::headers_to_vec; - -pub use crate::projection::markdown_policy::{should_refresh_frontmatter, should_skip_record}; -pub use coordinator::CaptureEngine; -pub use egress::{ - load_story_snapshots, persist_story_snapshots, rebuild_session_story, story_call_ids, - story_user_turn_count, -}; -pub use story::{ - Call, CallContext, CancelEvent, CompleteEvent, DraftEvent, Event, RequestEvent, Story, - StoryContext, TurnKind, -}; - -#[cfg(test)] -mod tests; diff --git a/crates/persisting-gateway/src/engine/prepare.rs b/crates/persisting-gateway/src/engine/prepare.rs deleted file mode 100644 index 7c9c4c426..000000000 --- a/crates/persisting-gateway/src/engine/prepare.rs +++ /dev/null @@ -1,378 +0,0 @@ -//! Prepare phase — pure capture logic before story-local I/O. -//! -//! Runs on the runtime thread; may `ask` the run actor. Lance/markdown I/O only via [`StoryCommand`]. - -use anyhow::Result; -use pulsing_actor::ActorRef; -use serde_json::Value; - -use super::wire::{StoryCommand, StoryScope, run_enrich}; -use super::{CallContext, CancelEvent, CompleteEvent, DraftEvent, Event, RequestEvent}; -use crate::dialogue_extract::{extract_assistant_text_from_json, extract_assistant_turn_from_sse}; -use crate::runtime::debug; -use crate::sink::{ - attach_connection_and_client, attach_http_wire_request, attach_http_wire_response, - attach_recorded_headers, llm_request_summary_record, llm_response_record_with_content, -}; -use crate::subagent_link::SpawnLinkBackfill; -use crate::usage::{ - StreamMetrics, TokenUsage, estimate_cost_usd, extract_usage_from_response, - extract_usage_from_sse, -}; - -/// Index + config for the prepare phase (run actor accessed via wire client, not held here). -pub(crate) struct CapturePreparer { - pub index: crate::session::index::SessionIndexHandle, - pub storage: std::sync::Arc, - pub stream_markdown: bool, -} - -/// Outcome of prepare — at most one story command to dispatch. -pub(crate) struct PreparedCapture { - pub ctx: CallContext, - pub backfills: Vec, - story_cmd: Option, -} - -impl CapturePreparer { - pub async fn prepare( - &self, - run: &ActorRef, - ctx: &CallContext, - event: Event, - ) -> Result { - match event { - Event::Request(e) => self.prepare_request(run, ctx, e).await, - Event::ResponseDraft(e) => self.prepare_draft(ctx, e).await, - Event::ResponseComplete(e) => self.prepare_completed(run, ctx, e).await, - Event::Cancelled(e) => self.prepare_cancelled(ctx, e).await, - } - } - - async fn prepare_request( - &self, - run: &ActorRef, - ctx: &CallContext, - event: RequestEvent, - ) -> Result { - self.index.record_request( - ctx.agent_id(), - &ctx.route().session_id, - ctx.provider, - ctx.protocol.as_str(), - &ctx.client_model, - ); - // Session index is flushed in batch via CaptureEngine::flush / shutdown. - - // Runtime requests already carry the once-parsed semantic payload. The - // fallback keeps WAL rows written by older binaries replayable. - let semantic = event.semantic.clone().or_else(|| { - event.body_json.as_ref().and_then(|body| { - crate::understanding::understand_request_value(ctx.protocol, body) - .ok() - .map(std::sync::Arc::new) - }) - }); - let forward_to = event.model_rewritten.then_some(ctx.upstream_model.as_str()); - let mut rec = llm_request_summary_record( - Some(ctx.route().session_id.clone()), - Some(ctx.agent_id().to_string()), - &ctx.client_model, - &event.path, - event.body_bytes, - ctx.protocol.as_str(), - ctx.provider.as_str(), - event.user_content, - forward_to, - &ctx.call, - ctx.level, - event.body_json.as_ref(), - ); - attach_recorded_headers(&mut rec.payload, &event.headers); - // Prefer event headers for connection flags; fall back to ctx request headers. - let hdrs = if event.headers.is_empty() { - &ctx.request_headers - } else { - &event.headers - }; - attach_connection_and_client( - &mut rec.payload, - hdrs, - ctx.http_version.as_deref(), - ctx.client_peer.as_deref(), - ctx.client_meta.as_ref(), - ); - // Raw SoT (RFC-0002): always persist wire body into `http.request_body` when - // available. Flat `payload.body` remains gated by CaptureLevel::Full for - // dialogue-oriented consumers. - let body_for_wire = event.body_json.as_ref(); - attach_http_wire_request( - &mut rec.payload, - &event.method, - &event.path, - event.url.as_deref().or(ctx.upstream_url.as_deref()), - body_for_wire, - body_for_wire.is_some() && !event.headers.is_empty(), - ); - if let Some(semantic) = semantic { - rec.payload["llm_request"] = serde_json::to_value(semantic.as_ref())?; - } - let backfills = run_enrich(run, &mut rec, ctx, event.body_json.as_ref(), None).await?; - let scope = StoryScope::from_context(ctx); - let story_cmd = Some(StoryCommand::persist_record( - scope, - serde_json::to_vec(&rec)?, - )); - Ok(PreparedCapture { - ctx: ctx.clone(), - backfills, - story_cmd, - }) - } - - async fn prepare_draft(&self, ctx: &CallContext, event: DraftEvent) -> Result { - if !self.stream_markdown || !ctx.level.includes_assistant_text() { - return Ok(PreparedCapture { - ctx: ctx.clone(), - backfills: vec![], - story_cmd: None, - }); - } - if event.assistant_content.trim().is_empty() { - return Ok(PreparedCapture { - ctx: ctx.clone(), - backfills: vec![], - story_cmd: None, - }); - } - let rec = llm_response_record_with_content( - Some(ctx.route().session_id.clone()), - Some(ctx.agent_id().to_string()), - event.status, - &serde_json::json!({ "status": event.status, "draft": true }), - true, - Some(event.assistant_content.clone()), - &ctx.call, - ctx.level, - ); - let story_cmd = Some(StoryCommand::upsert_draft( - StoryScope::from_context(ctx), - serde_json::to_vec(&rec)?, - event.assistant_content, - )?); - Ok(PreparedCapture { - ctx: ctx.clone(), - backfills: vec![], - story_cmd, - }) - } - - async fn prepare_completed( - &self, - run: &ActorRef, - ctx: &CallContext, - event: CompleteEvent, - ) -> Result { - let resp_text = std::str::from_utf8(&event.resp_bytes).unwrap_or(""); - let usage = - resolve_response_usage(event.streaming, event.stream_metrics.as_ref(), resp_text); - let resp_json = if event.streaming { - Value::String(resp_text.to_string()) - } else { - serde_json::from_slice(&event.resp_bytes) - .unwrap_or_else(|_| Value::String(resp_text.to_string())) - }; - - let cost = estimate_cost_usd(&ctx.upstream_model, ctx.provider, &usage); - self.index.record_response( - ctx.agent_id(), - &ctx.route().session_id, - ctx.provider, - &ctx.client_model, - &usage, - cost, - ); - - if ctx.debug_on { - debug::log_llm_response( - self.storage.as_path(), - &ctx.route().session_id, - ctx.agent_id(), - &ctx.client_model, - event.status, - usage.total_tokens, - resp_text, - ); - } - - let mut resp_payload = serde_json::json!({ - "status": event.status, - "protocol": ctx.protocol.as_str(), - "provider": ctx.provider.as_str(), - "usage": usage, - "estimated_cost_usd": cost, - }); - if ctx.upstream_model != ctx.client_model { - resp_payload["forward_to"] = Value::String(ctx.upstream_model.clone()); - } - if ctx.level.includes_full_body() { - resp_payload["body"] = resp_json.clone(); - } - if let Some(m) = event.stream_metrics.as_ref() { - if let Some(ttft) = m.ttft_ms { - resp_payload["ttft_ms"] = Value::Number(ttft.into()); - } - if m.usage.reasoning_tokens > 0 { - resp_payload["reasoning_tokens"] = Value::Number(m.usage.reasoning_tokens.into()); - } - } - - let assistant_content = if ctx.level.includes_assistant_text() { - let from_stream = event.assistant_content.filter(|s| !s.trim().is_empty()); - from_stream.or_else(|| { - if event.streaming { - let extracted = extract_assistant_turn_from_sse(resp_text); - if extracted.trim().is_empty() { - None - } else { - Some(extracted) - } - } else { - extract_assistant_text_from_json(&resp_json) - } - }) - } else { - None - }; - - // Live responses carry the exact once-parsed semantic value used by the - // client renderer. The fallback is reserved for WAL/dead-letter replay. - let semantic_response = event.semantic.or_else(|| { - if event.streaming { - Some(std::sync::Arc::new( - crate::understanding::understand_stream_summary( - ctx.protocol, - &ctx.client_model, - assistant_content.as_deref(), - persisting_pchronicle::model::LlmUsage { - input_tokens: usage.input_tokens, - output_tokens: usage.output_tokens, - total_tokens: usage.total_tokens, - cache_read_tokens: usage.cache_read_tokens, - cache_write_tokens: usage.cache_write_tokens, - reasoning_tokens: usage.reasoning_tokens, - }, - ), - )) - } else { - crate::understanding::understand_response_value(ctx.protocol, &resp_json) - .ok() - .map(std::sync::Arc::new) - } - }); - - let mut rec = llm_response_record_with_content( - Some(ctx.route().session_id.clone()), - Some(ctx.agent_id().to_string()), - event.status, - &resp_payload, - event.streaming, - assistant_content.clone(), - &ctx.call, - ctx.level, - ); - if let Some(semantic) = semantic_response { - rec.payload["llm_response"] = serde_json::to_value(semantic.as_ref())?; - } - attach_recorded_headers(&mut rec.payload, &event.headers); - attach_connection_and_client( - &mut rec.payload, - &ctx.request_headers, - ctx.http_version.as_deref(), - ctx.client_peer.as_deref(), - ctx.client_meta.as_ref(), - ); - // Raw SoT: always mirror response bytes into `http.response_body`. - let body_for_wire = Some(&resp_json); - attach_http_wire_response( - &mut rec.payload, - event.status, - ctx.upstream_url.as_deref(), - body_for_wire, - true, - event.streaming, - !event.headers.is_empty(), - ); - let backfills = run_enrich(run, &mut rec, ctx, None, assistant_content.as_deref()).await?; - let scope = StoryScope::from_context(ctx); - let story_cmd = Some(StoryCommand::persist_record( - scope, - serde_json::to_vec(&rec)?, - )); - Ok(PreparedCapture { - ctx: ctx.clone(), - backfills, - story_cmd, - }) - } - - async fn prepare_cancelled( - &self, - ctx: &CallContext, - event: CancelEvent, - ) -> Result { - let rec = crate::record::EventRecord { - identity: persisting_events::EventIdentity::default(), - seq: 0, - source: "capture".into(), - kind: "llm.call.cancelled".into(), - timestamp: Some(crate::record::now_rfc3339()), - session_id: Some(ctx.route().session_id.clone()), - agent_id: Some(ctx.agent_id().to_string()), - parent_uuid: None, - trace_id: Some(ctx.call.trace_id.clone()), - call_id: Some(ctx.call.call_id.clone()), - subagent_id: ctx.route().subagent_id.clone(), - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload: serde_json::json!({ - "status": event.status, - "streaming": event.streaming, - "bytes_received": event.bytes_received, - }), - }; - let story_cmd = Some(StoryCommand::persist_record( - StoryScope::from_context(ctx), - serde_json::to_vec(&rec)?, - )); - Ok(PreparedCapture { - ctx: ctx.clone(), - backfills: vec![], - story_cmd, - }) - } -} - -impl PreparedCapture { - pub fn take_story_command(&mut self) -> Option { - self.story_cmd.take() - } -} - -fn resolve_response_usage( - streaming: bool, - stream_metrics: Option<&StreamMetrics>, - resp_text: &str, -) -> TokenUsage { - if streaming { - stream_metrics - .map(|m| m.usage.clone()) - .filter(|u| u.total_tokens > 0 || u.input_tokens > 0 || u.output_tokens > 0) - .unwrap_or_else(|| extract_usage_from_sse(resp_text)) - } else { - let resp_json: Value = serde_json::from_str(resp_text) - .unwrap_or_else(|_| Value::String(resp_text.to_string())); - extract_usage_from_response(&resp_json) - } -} diff --git a/crates/persisting-gateway/src/engine/story/call.rs b/crates/persisting-gateway/src/engine/story/call.rs deleted file mode 100644 index 30fbc7c35..000000000 --- a/crates/persisting-gateway/src/engine/story/call.rs +++ /dev/null @@ -1,28 +0,0 @@ -//! One LLM HTTP round-trip within a story. - -use axum::http::HeaderMap; -use serde::{Deserialize, Serialize}; - -use crate::record::now_rfc3339; -use crate::session::chain::{new_call_id, resolve_trace_id}; - -/// Identifiers for one proxied LLM HTTP call. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct Call { - pub call_id: String, - pub trace_id: String, - /// RFC3339 time when the HTTP request was accepted. - pub started_at: String, -} - -impl Call { - pub fn from_headers(headers: &HeaderMap) -> Self { - let call_id = new_call_id(); - let trace_id = resolve_trace_id(headers, &call_id); - Self { - call_id, - trace_id, - started_at: now_rfc3339(), - } - } -} diff --git a/crates/persisting-gateway/src/engine/story/context.rs b/crates/persisting-gateway/src/engine/story/context.rs deleted file mode 100644 index 53f9d9a69..000000000 --- a/crates/persisting-gateway/src/engine/story/context.rs +++ /dev/null @@ -1,124 +0,0 @@ -//! Run/story routing and per-call capture context. - -use serde::{Deserialize, Serialize}; - -use crate::config::CaptureLevel; -use crate::protocol::ProtocolKind; -use crate::provider::ProviderKind; -use crate::session::storage::CaptureRoute; - -use super::call::Call; -use super::ids::{RunId, StoryId}; - -/// Routing + identity for one story (one agent narrative line). -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct StoryContext { - pub run_id: Option, - pub story_id: StoryId, - pub route: CaptureRoute, - pub agent_id: String, -} - -impl StoryContext { - pub fn from_route(route: CaptureRoute, agent_id: impl Into) -> Self { - let story_id = StoryId::from_seq_key(route.seq_key()); - let run_id = route.root_session.clone().map(RunId); - Self { - run_id, - story_id, - route, - agent_id: agent_id.into(), - } - } - - pub fn story_id(&self) -> &StoryId { - &self.story_id - } -} - -/// Full capture context for one proxied LLM call (story + call + wire metadata). -#[derive(Clone)] -pub struct CallContext { - pub story: StoryContext, - pub call: Call, - pub request_headers: Vec<(String, String)>, - pub level: CaptureLevel, - pub client_model: String, - pub upstream_model: String, - pub provider: ProviderKind, - pub protocol: ProtocolKind, - pub debug_on: bool, - /// Client socket address (`ip:port`) from `ConnectInfo`, when known. - pub client_peer: Option, - /// Best-effort process / machine metadata for the peer. - pub client_meta: Option, - /// HTTP version string, e.g. `HTTP/1.1`. - pub http_version: Option, - /// Upstream request URL when known (proxy path). - pub upstream_url: Option, -} - -impl CallContext { - #[allow(clippy::too_many_arguments)] - pub fn new( - route: CaptureRoute, - agent_id: impl Into, - call: Call, - request_headers: Vec<(String, String)>, - level: CaptureLevel, - client_model: impl Into, - upstream_model: impl Into, - provider: ProviderKind, - protocol: ProtocolKind, - debug_on: bool, - ) -> Self { - Self { - story: StoryContext::from_route(route, agent_id), - call, - request_headers, - level, - client_model: client_model.into(), - upstream_model: upstream_model.into(), - provider, - protocol, - debug_on, - client_peer: None, - client_meta: None, - http_version: None, - upstream_url: None, - } - } - - pub fn attach_client( - &mut self, - peer: impl std::fmt::Display, - meta: Option, - ) { - self.client_peer = Some(peer.to_string()); - self.client_meta = meta; - } - - pub fn attach_http_version(&mut self, version: impl Into) { - self.http_version = Some(version.into()); - } - - pub fn attach_upstream_url(&mut self, url: impl Into) { - self.upstream_url = Some(url.into()); - } - - pub fn story_id(&self) -> &StoryId { - self.story.story_id() - } - - pub fn run_id(&self) -> Option<&RunId> { - self.story.run_id.as_ref() - } - - pub fn route(&self) -> &CaptureRoute { - &self.story.route - } - - pub fn agent_id(&self) -> &str { - &self.story.agent_id - } -} diff --git a/crates/persisting-gateway/src/engine/story/event.rs b/crates/persisting-gateway/src/engine/story/event.rs deleted file mode 100644 index 8f50273b9..000000000 --- a/crates/persisting-gateway/src/engine/story/event.rs +++ /dev/null @@ -1,58 +0,0 @@ -//! Capture events on a call: request → draft* → complete | cancel. - -use bytes::Bytes; -use serde_json::Value; - -use crate::usage::StreamMetrics; - -/// Something that happened on one LLM call during capture. -#[derive(Debug, Clone)] -pub enum Event { - Request(RequestEvent), - ResponseDraft(DraftEvent), - ResponseComplete(CompleteEvent), - Cancelled(CancelEvent), -} - -#[derive(Debug, Clone)] -pub struct CancelEvent { - pub status: u16, - pub bytes_received: usize, - pub streaming: bool, -} - -#[derive(Debug, Clone)] -pub struct DraftEvent { - pub status: u16, - pub assistant_content: String, -} - -#[derive(Debug, Clone)] -pub struct RequestEvent { - pub path: String, - /// HTTP method (`GET`, `POST`, …). - pub method: String, - /// Best-effort request URL (Host + path/query when available). - pub url: Option, - pub body_bytes: usize, - pub user_content: Option, - pub body_json: Option, - /// Provider-neutral Chronicle semantics parsed from the untouched client body. - pub semantic: Option>, - pub model_rewritten: bool, - /// HTTP request headers as `(name, value)` pairs (may include duplicates). - pub headers: Vec<(String, String)>, -} - -#[derive(Debug, Clone)] -pub struct CompleteEvent { - pub status: u16, - pub resp_bytes: Bytes, - pub streaming: bool, - pub stream_metrics: Option, - pub assistant_content: Option, - /// Provider-neutral response semantics parsed once from the upstream wire. - pub semantic: Option>, - /// HTTP response headers as `(name, value)` pairs (may include duplicates). - pub headers: Vec<(String, String)>, -} diff --git a/crates/persisting-gateway/src/engine/story/ids.rs b/crates/persisting-gateway/src/engine/story/ids.rs deleted file mode 100644 index ae837bad7..000000000 --- a/crates/persisting-gateway/src/engine/story/ids.rs +++ /dev/null @@ -1,75 +0,0 @@ -//! Strong ids for the run → story → turn → call narrative model. - -use serde::{Deserialize, Serialize}; - -/// One capture workspace (run directory / `root_session`). -#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] -#[serde(transparent)] -pub struct RunId(pub String); - -impl RunId { - pub fn as_str(&self) -> &str { - &self.0 - } -} - -/// One agent narrative line (≈ former `seq_key` / one trajectory file). -#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] -#[serde(transparent)] -pub struct StoryId(pub String); - -impl StoryId { - pub fn new(id: impl Into) -> Self { - Self(id.into()) - } - - pub fn from_seq_key(seq_key: impl Into) -> Self { - Self(seq_key.into()) - } - - pub fn as_str(&self) -> &str { - &self.0 - } -} - -/// One dialogue turn within a story (user → assistant round). -#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] -#[serde(transparent)] -pub struct TurnId(pub String); - -impl TurnId { - pub fn new(story_id: &StoryId, index: u32) -> Self { - Self(format!("{}#turn-{index}", story_id.as_str())) - } - - pub fn as_str(&self) -> &str { - &self.0 - } -} - -/// One LLM HTTP round-trip (`call_id`). -#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] -#[serde(transparent)] -pub struct CallId(pub String); - -impl CallId { - pub fn new(id: impl Into) -> Self { - Self(id.into()) - } - - pub fn as_str(&self) -> &str { - &self.0 - } -} - -impl From<&str> for CallId { - fn from(s: &str) -> Self { - Self(s.to_string()) - } -} - -impl From for CallId { - fn from(s: String) -> Self { - Self(s) - } -} diff --git a/crates/persisting-gateway/src/engine/story/mod.rs b/crates/persisting-gateway/src/engine/story/mod.rs deleted file mode 100644 index be7b73dfa..000000000 --- a/crates/persisting-gateway/src/engine/story/mod.rs +++ /dev/null @@ -1,22 +0,0 @@ -//! Capture narrative model: **RunId → Story → Turn**, with **Call + Event** at the wire layer. -//! -//! | Layer | Type | Role | -//! |-------|------|------| -//! | Container | [`RunId`] | One capture workspace (`root_session`) | -//! | Narrative | [`StoryId`] / [`Story`] | One agent's full line | -//! | Dialogue | [`TurnId`] / [`Turn`] | One user→assistant round | -//! | Wire | [`Call`] + [`Event`] | HTTP call and what happened on it | - -mod call; -mod context; -mod event; -mod ids; -mod model; -mod turn_machine; - -pub use call::Call; -pub use context::{CallContext, StoryContext}; -pub use event::{CancelEvent, CompleteEvent, DraftEvent, Event, RequestEvent}; -pub use ids::{RunId, StoryId}; -pub use model::{Story, TurnKind}; -pub(crate) use turn_machine::TurnMachine; diff --git a/crates/persisting-gateway/src/engine/story/model.rs b/crates/persisting-gateway/src/engine/story/model.rs deleted file mode 100644 index 02a52fbbb..000000000 --- a/crates/persisting-gateway/src/engine/story/model.rs +++ /dev/null @@ -1,108 +0,0 @@ -//! Read model: Story → Turn, with call/event detail in each turn. - -use serde::{Deserialize, Serialize}; - -use super::ids::{CallId, RunId, StoryId, TurnId}; -use crate::protocol::ProtocolKind; - -/// Link from a subagent story back to the spawn point in a parent story. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct StoryLink { - pub parent_story_id: StoryId, - pub spawn_call_id: CallId, - /// Turn index in the parent story when the subagent was spawned (0-based). - #[serde(default, skip_serializing_if = "Option::is_none")] - pub spawn_turn_index: Option, - #[serde(default)] - pub relation: StoryLinkRelation, -} - -/// How a child story relates to its parent link point. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] -#[serde(rename_all = "snake_case")] -pub enum StoryLinkRelation { - #[default] - Spawn, - MergeBack, -} - -/// One agent's full narrative line. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct Story { - pub story_id: StoryId, - pub run_id: Option, - pub agent_id: String, - pub parent: Option, - pub turns: Vec, -} - -/// One readable user→assistant round within a story. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct Turn { - pub turn_id: TurnId, - pub index: u32, - pub kind: TurnKind, - pub user: Option, - pub assistant: Option, - pub calls: Vec, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum TurnKind { - /// Visible user → assistant round. - Dialogue, - /// Agent-initiated stretch without user text on the opening request (tool loops, - /// system-only continuations, autonomous work — not limited to tool-call rounds). - #[serde(alias = "tool_loop")] - Autonomous, - /// Reserved for internal / non-dialogue indexing (currently unused). - Internal, -} - -/// Visible text (user or assistant). -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct TextBlock { - pub text: String, - pub call_id: Option, -} - -/// One call's event timeline within a turn. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct TurnCall { - pub call_id: CallId, - pub trace_id: String, - pub protocol: Option, - pub model: Option, - pub events: Vec, -} - -/// Phase of one call within a turn (read model; distinct from [`EventRecord::kind`]). -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum CallPhase { - Request, - Draft, - Complete, - Cancel, -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn call_phase_serializes_snake_case() { - let phase = CallPhase::Complete; - let json = serde_json::to_string(&phase).unwrap(); - assert_eq!(json, "\"complete\""); - let back: CallPhase = serde_json::from_str(&json).unwrap(); - assert_eq!(back, CallPhase::Complete); - } - - #[test] - fn turn_kind_deserializes_legacy_tool_loop_alias() { - let kind: TurnKind = serde_json::from_str("\"tool_loop\"").unwrap(); - assert_eq!(kind, TurnKind::Autonomous); - } -} diff --git a/crates/persisting-gateway/src/engine/story/turn_machine.rs b/crates/persisting-gateway/src/engine/story/turn_machine.rs deleted file mode 100644 index bcb87a3fa..000000000 --- a/crates/persisting-gateway/src/engine/story/turn_machine.rs +++ /dev/null @@ -1,562 +0,0 @@ -//! In-memory turn index updated as capture records flow through a story actor. - -use std::collections::HashMap; - -use serde_json::json; - -use super::ids::{CallId, StoryId, TurnId}; -use super::model::{CallPhase, Story, TextBlock, Turn, TurnCall, TurnKind}; -use crate::record::{EventRecord, EventRecordExt}; - -/// Outcome of observing one persisted capture record. -#[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) struct TurnObserveOutcome { - pub turn_id: Option, - pub turn_index: Option, -} - -/// Tracks turns for one story (single-writer, inside [`super::super::actors::StoryActor`]). -#[derive(Debug, Clone)] -pub struct TurnMachine { - story_id: StoryId, - agent_id: String, - run_id: Option, - turns: Vec, - active_turn: Option, - call_turns: HashMap, - next_index: u32, -} - -impl TurnMachine { - pub fn new(story_id: StoryId) -> Self { - Self { - story_id, - agent_id: String::new(), - run_id: None, - turns: Vec::new(), - active_turn: None, - call_turns: HashMap::new(), - next_index: 0, - } - } - - pub fn with_context( - story_id: StoryId, - agent_id: impl Into, - run_id: Option, - ) -> Self { - Self { - story_id, - agent_id: agent_id.into(), - run_id, - turns: Vec::new(), - active_turn: None, - call_turns: HashMap::new(), - next_index: 0, - } - } - - pub fn set_story_meta(&mut self, agent_id: &str, run_id: Option) { - if self.agent_id.is_empty() { - self.agent_id = agent_id.to_string(); - } - if self.run_id.is_none() { - self.run_id = run_id; - } - } - - pub fn turns(&self) -> &[Turn] { - &self.turns - } - - /// Build a read-model snapshot of the in-memory story state. - pub fn snapshot(&self) -> Story { - Story { - story_id: self.story_id.clone(), - run_id: self.run_id.clone(), - agent_id: self.agent_id.clone(), - parent: None, - turns: self.turns.clone(), - } - } - - /// Replay persisted records into a fresh turn machine (offline egress). - pub fn replay_records( - story_id: StoryId, - agent_id: impl Into, - run_id: Option, - records: &[EventRecord], - ) -> Self { - let mut tm = Self::with_context(story_id, agent_id, run_id); - for rec in records { - let mut r = rec.clone(); - tm.observe_record(&mut r); - } - tm - } - - /// Stamp `turn_id` / `turn_index` on the record and update in-memory story state. - pub fn observe_record(&mut self, rec: &mut EventRecord) -> TurnObserveOutcome { - let outcome = self.apply_record(rec); - if let Some(ref turn_id) = outcome.turn_id { - rec.payload["turn_id"] = json!(turn_id.as_str()); - } - if let Some(idx) = outcome.turn_index { - rec.payload["turn_index"] = json!(idx); - } - outcome - } - - fn apply_record(&mut self, rec: &EventRecord) -> TurnObserveOutcome { - let call_id = rec - .call_id - .as_deref() - .map(CallId::new) - .unwrap_or_else(|| CallId::new("unknown")); - - match rec.kind.as_str() { - "llm.request" => self.on_request(&call_id, rec), - "llm.response" | "llm.response.stream" => self.on_response(&call_id, rec), - "llm.call.cancelled" => self.on_cancel(&call_id), - _ => TurnObserveOutcome { - turn_id: self.active_turn.clone(), - turn_index: self.active_turn.as_ref().and_then(|t| self.turn_index(t)), - }, - } - } - - fn on_request(&mut self, call_id: &CallId, rec: &EventRecord) -> TurnObserveOutcome { - if rec.is_internal_llm_request() { - let turn_id = self.ensure_call_on_active_turn(call_id, CallPhase::Request); - return TurnObserveOutcome { - turn_index: turn_id.as_ref().and_then(|t| self.turn_index(t)), - turn_id, - }; - } - - if let Some(text) = rec.visible_user_text() { - let turn_id = TurnId::new(&self.story_id, self.next_index); - self.next_index += 1; - let turn = Turn { - turn_id: turn_id.clone(), - index: self.next_index - 1, - kind: TurnKind::Dialogue, - user: Some(TextBlock { - text, - call_id: Some(call_id.clone()), - }), - assistant: None, - calls: vec![self.call_snapshot(call_id)], - }; - self.turns.push(turn); - self.call_turns.insert(call_id.clone(), turn_id.clone()); - self.active_turn = Some(turn_id.clone()); - return TurnObserveOutcome { - turn_id: Some(turn_id), - turn_index: Some(self.next_index - 1), - }; - } - - if self.active_turn.is_some() { - let turn_id = self.ensure_call_on_active_turn(call_id, CallPhase::Request); - return TurnObserveOutcome { - turn_index: turn_id.as_ref().and_then(|t| self.turn_index(t)), - turn_id, - }; - } - - let turn_id = TurnId::new(&self.story_id, self.next_index); - self.next_index += 1; - let turn = Turn { - turn_id: turn_id.clone(), - index: self.next_index - 1, - kind: TurnKind::Autonomous, - user: None, - assistant: None, - calls: vec![self.call_snapshot(call_id)], - }; - self.turns.push(turn); - self.call_turns.insert(call_id.clone(), turn_id.clone()); - self.active_turn = Some(turn_id.clone()); - TurnObserveOutcome { - turn_id: Some(turn_id), - turn_index: Some(self.next_index - 1), - } - } - - fn on_response(&mut self, call_id: &CallId, rec: &EventRecord) -> TurnObserveOutcome { - let turn_id = self.ensure_call_for_call(call_id, CallPhase::Complete); - if let Some(text) = rec.visible_assistant_text() - && let Some(turn) = turn_id.as_ref().and_then(|id| self.turn_mut(id)) - { - turn.assistant = Some(TextBlock { - text, - call_id: Some(call_id.clone()), - }); - } - TurnObserveOutcome { - turn_index: turn_id.as_ref().and_then(|t| self.turn_index(t)), - turn_id, - } - } - - fn on_cancel(&mut self, call_id: &CallId) -> TurnObserveOutcome { - let turn_id = self.ensure_call_for_call(call_id, CallPhase::Cancel); - TurnObserveOutcome { - turn_index: turn_id.as_ref().and_then(|t| self.turn_index(t)), - turn_id, - } - } - - fn ensure_call_on_active_turn(&mut self, call_id: &CallId, kind: CallPhase) -> Option { - let turn_id = self.active_turn.clone()?; - self.ensure_call_in_turn(&turn_id, call_id, kind); - Some(turn_id) - } - - fn ensure_call_for_call(&mut self, call_id: &CallId, kind: CallPhase) -> Option { - let turn_id = self - .call_turns - .get(call_id) - .cloned() - .or_else(|| self.active_turn.clone())?; - self.ensure_call_in_turn(&turn_id, call_id, kind); - Some(turn_id) - } - - fn ensure_call_in_turn(&mut self, turn_id: &TurnId, call_id: &CallId, kind: CallPhase) { - if let Some(turn) = self.turn_mut(turn_id) { - if let Some(entry) = turn.calls.iter_mut().find(|c| c.call_id == *call_id) { - if !entry.events.contains(&kind) { - entry.events.push(kind); - } - self.call_turns.insert(call_id.clone(), turn_id.clone()); - return; - } - turn.calls.push(TurnCall { - call_id: call_id.clone(), - trace_id: String::new(), - protocol: None, - model: None, - events: vec![kind], - }); - self.call_turns.insert(call_id.clone(), turn_id.clone()); - } - } - - fn call_snapshot(&self, call_id: &CallId) -> TurnCall { - TurnCall { - call_id: call_id.clone(), - trace_id: String::new(), - protocol: None, - model: None, - events: vec![CallPhase::Request], - } - } - - fn turn_index(&self, turn_id: &TurnId) -> Option { - self.turns - .iter() - .find(|t| t.turn_id == *turn_id) - .map(|t| t.index) - } - - fn turn_mut(&mut self, turn_id: &TurnId) -> Option<&mut Turn> { - self.turns.iter_mut().find(|t| t.turn_id == *turn_id) - } -} - -#[cfg(test)] -mod tests { - use super::super::model::CallPhase; - use super::*; - use crate::config::CaptureLevel; - use crate::sink::{llm_request_summary_record, llm_response_record_with_content}; - - use crate::Call; - - fn sample_call(id: &str) -> Call { - Call { - call_id: id.into(), - trace_id: "t".into(), - started_at: "2026-01-01T00:00:00Z".into(), - } - } - - #[test] - fn opens_turn_on_visible_user_request() { - let story_id = StoryId::new("run|main"); - let mut tm = TurnMachine::new(story_id); - let call = sample_call("c1"); - let mut rec = llm_request_summary_record( - Some("s".into()), - Some("a".into()), - "m", - "/v1/chat/completions", - 10, - "chat_completions", - "openai", - Some("hello".into()), - None, - &call, - CaptureLevel::Dialogue, - None, - ); - let out = tm.observe_record(&mut rec); - assert!(out.turn_id.is_some()); - assert_eq!(tm.turns().len(), 1); - assert_eq!(tm.turns()[0].kind, TurnKind::Dialogue); - assert_eq!(tm.turns()[0].user.as_ref().unwrap().text, "hello"); - assert_eq!( - rec.payload.get("turn_id").and_then(|v| v.as_str()), - out.turn_id.as_ref().map(|t| t.as_str()) - ); - } - - #[test] - fn attaches_assistant_to_active_turn() { - let story_id = StoryId::new("run|main"); - let mut tm = TurnMachine::new(story_id); - let call = sample_call("c1"); - let mut req = llm_request_summary_record( - Some("s".into()), - Some("a".into()), - "m", - "/v1/chat/completions", - 10, - "chat_completions", - "openai", - Some("hi".into()), - None, - &call, - CaptureLevel::Dialogue, - None, - ); - tm.observe_record(&mut req); - - let mut resp = llm_response_record_with_content( - Some("s".into()), - Some("a".into()), - 200, - &json!({}), - false, - Some("ok".into()), - &call, - CaptureLevel::Dialogue, - ); - tm.observe_record(&mut resp); - assert_eq!(tm.turns()[0].assistant.as_ref().unwrap().text, "ok"); - } - - #[test] - fn internal_request_does_not_overwrite_dialogue_kind() { - let story_id = StoryId::new("run|main"); - let mut tm = TurnMachine::new(story_id); - let user_call = sample_call("c-user"); - let mut user_req = llm_request_summary_record( - Some("s".into()), - Some("a".into()), - "m", - "/v1/chat/completions", - 10, - "chat_completions", - "openai", - Some("hello".into()), - None, - &user_call, - CaptureLevel::Dialogue, - None, - ); - tm.observe_record(&mut user_req); - assert_eq!(tm.turns()[0].kind, TurnKind::Dialogue); - - let internal_call = sample_call("c-internal"); - let mut internal_req = llm_request_summary_record( - Some("s".into()), - Some("a".into()), - "m", - "/v1/messages/count_tokens", - 10, - "count_tokens", - "openai", - None, - None, - &internal_call, - CaptureLevel::Dialogue, - None, - ); - tm.observe_record(&mut internal_req); - - assert_eq!(tm.turns().len(), 1); - assert_eq!(tm.turns()[0].kind, TurnKind::Dialogue); - assert_eq!(tm.turns()[0].calls.len(), 2); - } - - #[test] - fn snapshot_reflects_turns() { - let story_id = StoryId::new("run|main"); - let mut tm = TurnMachine::with_context(story_id.clone(), "agent-1", None); - let call = sample_call("c1"); - let mut rec = llm_request_summary_record( - Some("s".into()), - Some("a".into()), - "m", - "/v1/chat/completions", - 10, - "chat_completions", - "openai", - Some("hi".into()), - None, - &call, - CaptureLevel::Dialogue, - None, - ); - tm.observe_record(&mut rec); - let snap = tm.snapshot(); - assert_eq!(snap.story_id, story_id); - assert_eq!(snap.agent_id, "agent-1"); - assert_eq!(snap.turns.len(), 1); - } - - #[test] - fn replay_records_matches_incremental_observe() { - let story_id = StoryId::new("run|main"); - let mut live = TurnMachine::with_context(story_id.clone(), "agent-1", None); - let mut records = Vec::new(); - - for (id, text) in [("c1", "one"), ("c2", "two")] { - let call = sample_call(id); - let mut req = llm_request_summary_record( - Some("s".into()), - Some("a".into()), - "m", - "/v1/chat/completions", - 10, - "chat_completions", - "openai", - Some(text.into()), - None, - &call, - CaptureLevel::Dialogue, - None, - ); - live.observe_record(&mut req); - records.push(req); - - let mut resp = llm_response_record_with_content( - Some("s".into()), - Some("a".into()), - 200, - &json!({}), - false, - Some(format!("ok-{text}")), - &call, - CaptureLevel::Dialogue, - ); - live.observe_record(&mut resp); - records.push(resp); - } - - let replayed = - TurnMachine::replay_records(story_id.clone(), "agent-1", None, &records).snapshot(); - assert_eq!(replayed, live.snapshot()); - } - - #[test] - fn cancel_records_call_phase() { - let story_id = StoryId::new("run|main"); - let mut tm = TurnMachine::new(story_id); - let call = sample_call("c1"); - let mut req = llm_request_summary_record( - Some("s".into()), - Some("a".into()), - "m", - "/v1/chat/completions", - 10, - "chat_completions", - "openai", - Some("hi".into()), - None, - &call, - CaptureLevel::Dialogue, - None, - ); - tm.observe_record(&mut req); - - let mut cancel = llm_response_record_with_content( - Some("s".into()), - Some("a".into()), - 499, - &json!({ "cancelled": true }), - true, - None, - &call, - CaptureLevel::Dialogue, - ); - cancel.kind = "llm.call.cancelled".into(); - tm.observe_record(&mut cancel); - - let phases = &tm.turns()[0].calls[0].events; - assert!(phases.contains(&CallPhase::Request)); - assert!(phases.contains(&CallPhase::Cancel)); - } - - #[test] - fn response_for_overlapping_call_attaches_to_original_turn() { - let story_id = StoryId::new("run|main"); - let mut tm = TurnMachine::new(story_id); - let call_a = sample_call("call-a"); - let call_b = sample_call("call-b"); - - let mut req_a = llm_request_summary_record( - Some("s".into()), - Some("a".into()), - "m", - "/v1/chat/completions", - 10, - "chat_completions", - "openai", - Some("first".into()), - None, - &call_a, - CaptureLevel::Dialogue, - None, - ); - tm.observe_record(&mut req_a); - - let mut req_b = llm_request_summary_record( - Some("s".into()), - Some("a".into()), - "m", - "/v1/chat/completions", - 10, - "chat_completions", - "openai", - Some("second".into()), - None, - &call_b, - CaptureLevel::Dialogue, - None, - ); - tm.observe_record(&mut req_b); - - let mut resp_a = llm_response_record_with_content( - Some("s".into()), - Some("a".into()), - 200, - &json!({}), - false, - Some("reply to first".into()), - &call_a, - CaptureLevel::Dialogue, - ); - let out = tm.observe_record(&mut resp_a); - - assert_eq!(out.turn_index, Some(0)); - assert_eq!( - tm.turns()[0].assistant.as_ref().map(|b| b.text.as_str()), - Some("reply to first") - ); - assert!(tm.turns()[1].assistant.is_none()); - } -} diff --git a/crates/persisting-gateway/src/engine/tests/events.rs b/crates/persisting-gateway/src/engine/tests/events.rs deleted file mode 100644 index 751485bd4..000000000 --- a/crates/persisting-gateway/src/engine/tests/events.rs +++ /dev/null @@ -1,144 +0,0 @@ -use super::fixtures::*; -use super::support::*; - -#[tokio::test] -async fn request_event_appends_single_llm_request() { - let sink = RecordingSink::new(); - let dir = tempfile::tempdir().unwrap(); - let engine = test_engine(sink.clone(), dir.path(), false).await; - let ctx = test_context(); - engine - .apply( - &ctx, - Event::Request(RequestEvent { - path: "/v1/chat/completions".into(), - method: "POST".into(), - url: None, - body_bytes: 12, - user_content: Some("hi".into()), - body_json: None, - semantic: None, - model_rewritten: false, - headers: vec![], - }), - ) - .await - .unwrap(); - flush_engine(&engine).await; - let records = sink.drain(); - assert_eq!(records.len(), 1); - assert_eq!(records[0].kind, "llm.request"); -} - -#[tokio::test] -async fn request_event_projects_original_body_and_typed_understanding() { - let sink = RecordingSink::new(); - let dir = tempfile::tempdir().unwrap(); - let engine = test_engine(sink.clone(), dir.path(), false).await; - let ctx = test_context(); - let original = serde_json::json!({ - "model": "deepseek-chat", - "stream": true, - "max_tokens": 64, - "messages": [{"role": "user", "content": "keep me"}], - "tools": [{"type": "function", "function": {"name": "shell"}}] - }); - engine - .apply( - &ctx, - Event::Request(RequestEvent { - path: "/v1/chat/completions".into(), - method: "POST".into(), - url: Some("//localhost/v1/chat/completions".into()), - body_bytes: serde_json::to_vec(&original).unwrap().len(), - user_content: Some("keep me".into()), - body_json: Some(original.clone()), - semantic: None, - model_rewritten: true, - headers: vec![("content-type".into(), "application/json".into())], - }), - ) - .await - .unwrap(); - flush_engine(&engine).await; - - let records = sink.drain(); - let payload = &records[0].payload; - assert_eq!(payload["http"]["request_body"], original); - assert!(payload["llm_request"].get("schema_version").is_none()); - assert_eq!(payload["llm_request"]["input_format"], "chat_completions"); - assert_eq!(payload["llm_request"]["request"]["model"], "deepseek-chat"); - assert_eq!(payload["llm_request"]["request"]["stream"], true); - assert_eq!( - payload["llm_request"]["request"]["generation"]["max_output_tokens"], - 64 - ); - assert_eq!( - payload["llm_request"]["request"]["tools"][0]["name"], - "shell" - ); -} - -#[tokio::test] -async fn response_event_appends_single_stream_record() { - let sink = RecordingSink::new(); - let dir = tempfile::tempdir().unwrap(); - let engine = test_engine(sink.clone(), dir.path(), false).await; - let ctx = test_context(); - engine - .apply( - &ctx, - Event::ResponseComplete(CompleteEvent { - status: 200, - resp_bytes: Bytes::from("data: [DONE]\n\n"), - streaming: true, - stream_metrics: None, - assistant_content: Some("hello".into()), - semantic: None, - headers: vec![], - }), - ) - .await - .unwrap(); - flush_engine(&engine).await; - let records = sink.drain(); - assert_eq!(records.len(), 1); - assert_eq!(records[0].kind, "llm.response.stream"); - assert_eq!( - records[0].payload["assistant_content"].as_str(), - Some("hello") - ); - assert!( - records[0].payload["llm_response"] - .get("schema_version") - .is_none() - ); - assert_eq!( - records[0].payload["llm_response"]["output_format"], - "chat_completions" - ); - assert_eq!( - records[0].payload["llm_response"]["response"]["candidates"][0]["message"]["parts"][0]["text"], - "hello" - ); -} - -#[tokio::test] -async fn draft_event_does_not_append_to_sink() { - let sink = RecordingSink::new(); - let dir = tempfile::tempdir().unwrap(); - let engine = test_engine(sink.clone(), dir.path(), true).await; - let ctx = test_context(); - engine - .apply( - &ctx, - Event::ResponseDraft(DraftEvent { - status: 200, - assistant_content: "partial".into(), - }), - ) - .await - .unwrap(); - flush_engine(&engine).await; - assert!(sink.drain().is_empty()); -} diff --git a/crates/persisting-gateway/src/engine/tests/fixtures.rs b/crates/persisting-gateway/src/engine/tests/fixtures.rs deleted file mode 100644 index be6f7f37b..000000000 --- a/crates/persisting-gateway/src/engine/tests/fixtures.rs +++ /dev/null @@ -1,102 +0,0 @@ -use std::collections::HashMap; -use std::sync::{Arc, Mutex}; - -use crate::Call; -use crate::config::CaptureLevel; -use crate::engine::{CallContext, CaptureEngine}; -use crate::protocol::ProtocolKind; -use crate::provider::ProviderKind; -use crate::record::EventRecord; -use crate::session::index::SessionIndexStore; -use crate::session::storage::CaptureRoute; -use crate::sink::CaptureEventSink; - -pub(crate) struct RecordingSink { - records: Mutex>, - next_seq: Mutex>, -} - -impl RecordingSink { - pub(crate) fn new() -> Arc { - Arc::new(Self { - records: Mutex::new(Vec::new()), - next_seq: Mutex::new(HashMap::new()), - }) - } - - pub(crate) fn drain(&self) -> Vec { - self.records.lock().unwrap().drain(..).collect() - } -} - -impl CaptureEventSink for RecordingSink { - fn append( - &self, - route: &CaptureRoute, - _agent_id: &str, - record: &mut EventRecord, - ) -> anyhow::Result<()> { - let mut guard = self.next_seq.lock().unwrap(); - let seq = guard.entry(route.seq_key()).or_insert(0); - record.seq = *seq; - *seq += 1; - drop(guard); - self.records.lock().unwrap().push(record.clone()); - Ok(()) - } - - fn peek_next_seq(&self, route: &CaptureRoute) -> Option { - Some( - self.next_seq - .lock() - .unwrap() - .get(&route.seq_key()) - .copied() - .unwrap_or(0), - ) - } -} - -pub(crate) fn test_context() -> CallContext { - CallContext::new( - CaptureRoute { - root_session: Some("run-test".into()), - session_id: "sess-1".into(), - storage_session_id: "sess-1".into(), - subagent_id: None, - }, - "agent-1", - Call { - call_id: "call-1".into(), - trace_id: "trace-1".into(), - started_at: "2026-01-01T00:00:00Z".into(), - }, - Vec::new(), - CaptureLevel::Dialogue, - "deepseek-chat", - "deepseek-chat", - ProviderKind::OpenAi, - ProtocolKind::ChatCompletions, - false, - ) -} - -pub(crate) async fn test_engine( - sink: Arc, - storage: &std::path::Path, - stream_markdown: bool, -) -> CaptureEngine { - let index = SessionIndexStore::open(storage).unwrap().clone_handle(); - CaptureEngine::new( - sink, - index, - Arc::new(storage.to_path_buf()), - stream_markdown, - ) - .await - .unwrap() -} - -pub(crate) async fn flush_engine(engine: &CaptureEngine) { - engine.flush().await.unwrap(); -} diff --git a/crates/persisting-gateway/src/engine/tests/markdown_stream.rs b/crates/persisting-gateway/src/engine/tests/markdown_stream.rs deleted file mode 100644 index 0e7e4d56b..000000000 --- a/crates/persisting-gateway/src/engine/tests/markdown_stream.rs +++ /dev/null @@ -1,239 +0,0 @@ -use super::fixtures::*; - -#[tokio::test] -async fn stream_markdown_keeps_user_block_when_assistant_upserts() { - use super::support::*; - use crate::session::storage::trajectory_run_dir; - use persisting_pchronicle::document::decode_agenticmd; - - let sink = RecordingSink::new(); - let dir = tempfile::tempdir().unwrap(); - let storage = dir.path().to_path_buf(); - let engine = test_engine(sink.clone(), &storage, true).await; - let ctx = test_context(); - - engine - .apply( - &ctx, - Event::Request(RequestEvent { - path: "/v1/chat/completions".into(), - method: "POST".into(), - url: None, - body_bytes: 12, - user_content: Some("hello user".into()), - body_json: None, - semantic: None, - model_rewritten: false, - headers: vec![], - }), - ) - .await - .unwrap(); - engine - .apply( - &ctx, - Event::ResponseDraft(DraftEvent { - status: 200, - assistant_content: "partial assistant".into(), - }), - ) - .await - .unwrap(); - engine - .apply( - &ctx, - Event::ResponseComplete(CompleteEvent { - status: 200, - resp_bytes: Bytes::from("data: [DONE]\n\n"), - streaming: true, - stream_metrics: None, - assistant_content: Some("final assistant".into()), - semantic: None, - headers: vec![], - }), - ) - .await - .unwrap(); - - flush_engine(&engine).await; - let run_dir = trajectory_run_dir(storage.as_path(), ctx.agent_id(), ctx.route()); - let md_path = session_markdown_write_path_for_key(&run_dir, &ctx.route().storage_session_id); - let story = decode_agenticmd(&std::fs::read_to_string(&md_path).unwrap()).unwrap(); - let records = sink.drain(); - assert_eq!(records.len(), 2); - assert_eq!(records[0].seq, 0); - assert_eq!(records[1].seq, 1); - assert_eq!(story.turns.len(), 2); - assert_eq!(story.turns[0].source, "user"); - assert_eq!(story.turns[0].message, serde_json::json!("hello user")); - assert_eq!(story.turns[0].id, 1); - assert_eq!(story.turns[1].source, "agent"); - assert_eq!(story.turns[1].message, serde_json::json!("final assistant")); - assert_eq!(story.turns[1].id, 2); -} - -#[tokio::test] -async fn draft_markdown_uses_peeked_seq_and_matches_final() { - use super::support::*; - use crate::session::storage::trajectory_run_dir; - use persisting_pchronicle::document::decode_agenticmd; - - let sink = RecordingSink::new(); - let dir = tempfile::tempdir().unwrap(); - let storage = dir.path().to_path_buf(); - let engine = test_engine(sink.clone(), &storage, true).await; - let ctx = test_context(); - - engine - .apply( - &ctx, - Event::Request(RequestEvent { - path: "/v1/chat/completions".into(), - method: "POST".into(), - url: None, - body_bytes: 12, - user_content: Some("hi".into()), - body_json: None, - semantic: None, - model_rewritten: false, - headers: vec![], - }), - ) - .await - .unwrap(); - engine - .apply( - &ctx, - Event::ResponseDraft(DraftEvent { - status: 200, - assistant_content: "wip".into(), - }), - ) - .await - .unwrap(); - - flush_engine(&engine).await; - let run_dir = trajectory_run_dir(storage.as_path(), ctx.agent_id(), ctx.route()); - let md_path = session_markdown_write_path_for_key(&run_dir, &ctx.route().storage_session_id); - let draft_story = decode_agenticmd(&std::fs::read_to_string(&md_path).unwrap()).unwrap(); - assert_eq!(draft_story.turns.len(), 2); - assert_eq!(draft_story.turns[1].id, 2); - assert_eq!(draft_story.turns[1].message, serde_json::json!("wip")); - - engine - .apply( - &ctx, - Event::ResponseComplete(CompleteEvent { - status: 200, - resp_bytes: Bytes::from("data: [DONE]\n\n"), - streaming: true, - stream_metrics: None, - assistant_content: Some("done".into()), - semantic: None, - headers: vec![], - }), - ) - .await - .unwrap(); - - flush_engine(&engine).await; - let final_story = decode_agenticmd(&std::fs::read_to_string(&md_path).unwrap()).unwrap(); - assert_eq!(final_story.turns[1].id, 2); - assert_eq!(final_story.turns[1].message, serde_json::json!("done")); - assert_eq!(sink.drain()[1].seq, 1); -} - -#[tokio::test] -async fn overlapping_calls_preserve_later_user_block_in_markdown() { - use super::support::*; - use crate::session::storage::trajectory_run_dir; - use persisting_pchronicle::document::decode_agenticmd; - - let sink = RecordingSink::new(); - let dir = tempfile::tempdir().unwrap(); - let storage = dir.path().to_path_buf(); - let engine = test_engine(sink.clone(), &storage, true).await; - let mut ctx_a = test_context(); - ctx_a.call = Call { - call_id: "call-a".into(), - trace_id: "trace-a".into(), - started_at: "2026-01-01T00:00:00Z".into(), - }; - - engine - .apply( - &ctx_a, - Event::Request(RequestEvent { - path: "/v1/chat/completions".into(), - method: "POST".into(), - url: None, - body_bytes: 12, - user_content: Some("req-a".into()), - body_json: None, - semantic: None, - model_rewritten: false, - headers: vec![], - }), - ) - .await - .unwrap(); - engine - .apply( - &ctx_a, - Event::ResponseDraft(DraftEvent { - status: 200, - assistant_content: "draft-a".into(), - }), - ) - .await - .unwrap(); - - let mut ctx_b = ctx_a.clone(); - ctx_b.call = Call { - call_id: "call-b".into(), - trace_id: "trace-b".into(), - started_at: "2026-01-01T00:00:01Z".into(), - }; - engine - .apply( - &ctx_b, - Event::Request(RequestEvent { - path: "/v1/chat/completions".into(), - method: "POST".into(), - url: None, - body_bytes: 12, - user_content: Some("req-b".into()), - body_json: None, - semantic: None, - model_rewritten: false, - headers: vec![], - }), - ) - .await - .unwrap(); - - engine - .apply( - &ctx_a, - Event::ResponseComplete(CompleteEvent { - status: 200, - resp_bytes: Bytes::from("data: [DONE]\n\n"), - streaming: true, - stream_metrics: None, - assistant_content: Some("final-a".into()), - semantic: None, - headers: vec![], - }), - ) - .await - .unwrap(); - - flush_engine(&engine).await; - let run_dir = trajectory_run_dir(storage.as_path(), ctx_a.agent_id(), ctx_a.route()); - let md_path = session_markdown_write_path_for_key(&run_dir, &ctx_a.route().storage_session_id); - let story = decode_agenticmd(&std::fs::read_to_string(&md_path).unwrap()).unwrap(); - assert_eq!(story.turns.len(), 3); - assert_eq!(story.turns[0].message, serde_json::json!("req-a")); - assert_eq!(story.turns[1].message, serde_json::json!("final-a")); - assert_eq!(story.turns[2].message, serde_json::json!("req-b")); -} diff --git a/crates/persisting-gateway/src/engine/tests/mod.rs b/crates/persisting-gateway/src/engine/tests/mod.rs deleted file mode 100644 index 39b60179e..000000000 --- a/crates/persisting-gateway/src/engine/tests/mod.rs +++ /dev/null @@ -1,10 +0,0 @@ -//! Integration tests for [`super::CaptureEngine`]. - -mod events; -mod fixtures; -mod markdown_stream; -mod reliability; -mod replay_dedup; -mod story; -mod support; -mod wal; diff --git a/crates/persisting-gateway/src/engine/tests/reliability.rs b/crates/persisting-gateway/src/engine/tests/reliability.rs deleted file mode 100644 index 179a85d20..000000000 --- a/crates/persisting-gateway/src/engine/tests/reliability.rs +++ /dev/null @@ -1,90 +0,0 @@ -use std::sync::Arc; - -use crate::dead_letter::read_dead_letter_entries; -use crate::engine::{CaptureEngine, Event, RequestEvent}; -use crate::record::EventRecord; -use crate::session::index::SessionIndexStore; -use crate::session::storage::CaptureRoute; -use crate::sink::CaptureEventSink; - -use super::fixtures::test_context; - -struct FailingSink; - -impl CaptureEventSink for FailingSink { - fn append( - &self, - _route: &CaptureRoute, - _agent_id: &str, - _record: &mut EventRecord, - ) -> anyhow::Result<()> { - Err(anyhow::anyhow!("sink unavailable")) - } -} - -#[tokio::test] -async fn session_sink_failure_writes_dead_letter_with_record() { - let dir = tempfile::tempdir().unwrap(); - let storage = Arc::new(dir.path().to_path_buf()); - let sink = Arc::new(FailingSink); - let index = SessionIndexStore::open(dir.path()).unwrap().clone_handle(); - let engine = CaptureEngine::new(sink, index, storage.clone(), false) - .await - .unwrap(); - let ctx = test_context(); - let event = Event::Request(RequestEvent { - path: "/v1/chat/completions".into(), - method: "POST".into(), - url: None, - body_bytes: 10, - user_content: Some("hi".into()), - body_json: None, - semantic: None, - model_rewritten: false, - headers: vec![], - }); - assert!(engine.apply(&ctx, event).await.is_err()); - engine.flush().await.unwrap(); - let entries = read_dead_letter_entries(dir.path()).unwrap(); - assert_eq!(entries.len(), 1); - assert!(entries[0].error.contains("sink unavailable")); - assert!(entries[0].prepared_record_json.is_some()); - let story = engine.story_snapshot(&ctx.story).await.unwrap(); - assert!( - story.turns.is_empty(), - "failed canonical append must not advance the story read model" - ); -} - -#[tokio::test] -async fn failed_durable_sink_keeps_wal_event_pending_after_shutdown() { - let dir = tempfile::tempdir().unwrap(); - let storage = Arc::new(dir.path().to_path_buf()); - let index = SessionIndexStore::open(dir.path()).unwrap().clone_handle(); - let engine = CaptureEngine::new(Arc::new(FailingSink), index, storage.clone(), false) - .await - .unwrap(); - engine.spawn_apply( - test_context(), - Event::Request(RequestEvent { - path: "/v1/chat/completions".into(), - method: "POST".into(), - url: None, - body_bytes: 10, - user_content: Some("retry me".into()), - body_json: None, - semantic: None, - model_rewritten: false, - headers: vec![], - }), - ); - - engine.flush().await.unwrap(); - assert_eq!(crate::engine::wal::replay_pending(dir.path()).len(), 1); - engine.shutdown().await.unwrap(); - assert_eq!( - crate::engine::wal::replay_pending(dir.path()).len(), - 1, - "clean shutdown must not erase an event rejected by the durable sink" - ); -} diff --git a/crates/persisting-gateway/src/engine/tests/replay_dedup.rs b/crates/persisting-gateway/src/engine/tests/replay_dedup.rs deleted file mode 100644 index c66fe6326..000000000 --- a/crates/persisting-gateway/src/engine/tests/replay_dedup.rs +++ /dev/null @@ -1,122 +0,0 @@ -use super::fixtures::*; - -fn claude_messages_body(user_lines: &[&str]) -> serde_json::Value { - let messages: Vec = user_lines - .iter() - .map(|line| { - serde_json::json!({ - "role": "user", - "content": [{"type": "text", "text": line}] - }) - }) - .collect(); - serde_json::json!({"messages": messages, "model": "deepseek-v4-pro"}) -} - -#[tokio::test] -async fn replay_dedup_omits_internal_claude_history_request_from_markdown() { - use crate::engine::tests::support::*; - use crate::session::storage::trajectory_run_dir; - use persisting_pchronicle::document::decode_agenticmd; - - let sink = RecordingSink::new(); - let dir = tempfile::tempdir().unwrap(); - let storage = dir.path().to_path_buf(); - let engine = test_engine(sink.clone(), &storage, true).await; - let ctx = test_context(); - - async fn user_turn( - engine: &CaptureEngine, - ctx: &CallContext, - call_id: &str, - user: &str, - prior_users: &[&str], - ) { - let mut lines: Vec<&str> = prior_users.to_vec(); - lines.push(user); - let body = claude_messages_body(&lines); - let mut call_ctx = ctx.clone(); - call_ctx.call = Call { - call_id: call_id.into(), - trace_id: call_id.into(), - started_at: "2026-01-01T00:00:00Z".into(), - }; - engine - .apply( - &call_ctx, - Event::Request(RequestEvent { - path: "/v1/messages".into(), - method: "POST".into(), - url: None, - body_bytes: 100, - user_content: Some(user.into()), - body_json: Some(body), - semantic: None, - model_rewritten: false, - headers: vec![], - }), - ) - .await - .unwrap(); - } - - async fn assistant_turn(engine: &CaptureEngine, ctx: &CallContext, call_id: &str, text: &str) { - let mut call_ctx = ctx.clone(); - call_ctx.call = Call { - call_id: call_id.into(), - trace_id: call_id.into(), - started_at: "2026-01-01T00:00:00Z".into(), - }; - engine - .apply( - &call_ctx, - Event::ResponseComplete(CompleteEvent { - status: 200, - resp_bytes: Bytes::from("data: [DONE]\n\n"), - streaming: true, - stream_metrics: None, - assistant_content: Some(text.into()), - semantic: None, - headers: vec![], - }), - ) - .await - .unwrap(); - } - - user_turn(&engine, &ctx, "call-1", "hi", &[]).await; - assistant_turn(&engine, &ctx, "call-1", "Hi! What can I help you with?").await; - user_turn(&engine, &ctx, "call-2", "hi", &["hi"]).await; - assistant_turn(&engine, &ctx, "call-2", "Hi! What can I help you with?").await; - user_turn(&engine, &ctx, "call-3", "hi", &["hi"]).await; - assistant_turn( - &engine, - &ctx, - "call-3", - "what's the status of the capture storage feature?", - ) - .await; - user_turn(&engine, &ctx, "call-4", "你好", &["hi", "hi"]).await; - assistant_turn(&engine, &ctx, "call-4", "你好！有什么我可以帮你的吗？").await; - - flush_engine(&engine).await; - let run_dir = trajectory_run_dir(storage.as_path(), ctx.agent_id(), ctx.route()); - let md_path = session_markdown_write_path_for_key(&run_dir, &ctx.route().storage_session_id); - let story = decode_agenticmd(&std::fs::read_to_string(&md_path).unwrap()).unwrap(); - let bodies: Vec<_> = story - .turns - .iter() - .map(|turn| turn.message.as_str().unwrap_or_default().to_string()) - .collect(); - assert_eq!( - bodies, - vec![ - "hi".to_string(), - "Hi! What can I help you with?".to_string(), - "hi".to_string(), - "Hi! What can I help you with?".to_string(), - "你好".to_string(), - "你好！有什么我可以帮你的吗？".to_string(), - ] - ); -} diff --git a/crates/persisting-gateway/src/engine/tests/story.rs b/crates/persisting-gateway/src/engine/tests/story.rs deleted file mode 100644 index 8895c3816..000000000 --- a/crates/persisting-gateway/src/engine/tests/story.rs +++ /dev/null @@ -1,165 +0,0 @@ -use crate::session::index::SessionIndexStore; - -use super::fixtures::*; -use super::support::*; - -#[tokio::test] -async fn story_snapshot_reflects_applied_turns() { - let sink = RecordingSink::new(); - let dir = tempfile::tempdir().unwrap(); - let engine = test_engine(sink.clone(), dir.path(), false).await; - let ctx = test_context(); - engine - .apply( - &ctx, - Event::Request(RequestEvent { - path: "/v1/chat/completions".into(), - method: "POST".into(), - url: None, - body_bytes: 12, - user_content: Some("hello".into()), - body_json: None, - semantic: None, - model_rewritten: false, - headers: vec![], - }), - ) - .await - .unwrap(); - flush_engine(&engine).await; - - let snap = engine.story_snapshot(&ctx.story).await.unwrap(); - assert_eq!(snap.story_id.as_str(), ctx.story_id().as_str()); - assert_eq!(snap.turns.len(), 1); - assert_eq!(snap.turns[0].user.as_ref().unwrap().text, "hello"); -} - -#[tokio::test] -async fn shutdown_persists_story_snapshots_for_active_stories() { - use crate::engine::load_story_snapshots; - - let sink = RecordingSink::new(); - let dir = tempfile::tempdir().unwrap(); - let engine = test_engine(sink.clone(), dir.path(), false).await; - let ctx = test_context(); - engine - .apply( - &ctx, - Event::Request(RequestEvent { - path: "/v1/chat/completions".into(), - method: "POST".into(), - url: None, - body_bytes: 12, - user_content: Some("hello".into()), - body_json: None, - semantic: None, - model_rewritten: false, - headers: vec![], - }), - ) - .await - .unwrap(); - flush_engine(&engine).await; - engine.shutdown().await.unwrap(); - - let snapshots = load_story_snapshots(dir.path()).unwrap(); - assert!(snapshots.contains_key("sess-1")); - assert_eq!(snapshots["sess-1"].turns.len(), 1); -} - -#[tokio::test] -async fn shutdown_drains_spawned_apply_before_snapshot() { - use crate::engine::load_story_snapshots; - - let sink = RecordingSink::new(); - let dir = tempfile::tempdir().unwrap(); - let engine = test_engine(sink.clone(), dir.path(), false).await; - let ctx = test_context(); - engine.spawn_apply( - ctx, - Event::Request(RequestEvent { - path: "/v1/chat/completions".into(), - method: "POST".into(), - url: None, - body_bytes: 12, - user_content: Some("queued hello".into()), - body_json: None, - semantic: None, - model_rewritten: false, - headers: vec![], - }), - ); - - engine.shutdown().await.unwrap(); - - let snapshots = load_story_snapshots(dir.path()).unwrap(); - assert_eq!( - snapshots["sess-1"].turns[0] - .user - .as_ref() - .map(|b| b.text.as_str()), - Some("queued hello") - ); -} - -#[tokio::test] -async fn flush_persists_dirty_session_index() { - let sink = RecordingSink::new(); - let dir = tempfile::tempdir().unwrap(); - let engine = test_engine(sink.clone(), dir.path(), false).await; - let ctx = test_context(); - engine - .apply( - &ctx, - Event::Request(RequestEvent { - path: "/v1/chat/completions".into(), - method: "POST".into(), - url: None, - body_bytes: 12, - user_content: Some("ping".into()), - body_json: None, - semantic: None, - model_rewritten: false, - headers: vec![], - }), - ) - .await - .unwrap(); - engine.flush().await.unwrap(); - - let index = SessionIndexStore::load(dir.path()).unwrap(); - assert_eq!(index.sessions.len(), 1); - assert_eq!(index.sessions[0].session_id, "sess-1"); - assert!(index.sessions[0].active); -} - -#[tokio::test] -async fn flush_drains_spawned_apply_without_sleep() { - let sink = RecordingSink::new(); - let dir = tempfile::tempdir().unwrap(); - let engine = test_engine(sink.clone(), dir.path(), false).await; - let ctx = test_context(); - engine.spawn_apply( - ctx, - Event::Request(RequestEvent { - path: "/v1/chat/completions".into(), - method: "POST".into(), - url: None, - body_bytes: 12, - user_content: Some("queued ping".into()), - body_json: None, - semantic: None, - model_rewritten: false, - headers: vec![], - }), - ); - - engine.flush().await.unwrap(); - - let records = sink.drain(); - assert_eq!(records.len(), 1); - assert_eq!( - records[0].payload["user_content"].as_str(), - Some("queued ping") - ); -} diff --git a/crates/persisting-gateway/src/engine/tests/support.rs b/crates/persisting-gateway/src/engine/tests/support.rs deleted file mode 100644 index 28f5b0880..000000000 --- a/crates/persisting-gateway/src/engine/tests/support.rs +++ /dev/null @@ -1,8 +0,0 @@ -//! Shared imports for engine integration test modules. - -pub(crate) use crate::Call; -pub(crate) use crate::engine::{ - CallContext, CaptureEngine, CompleteEvent, DraftEvent, Event, RequestEvent, -}; -pub(crate) use bytes::Bytes; -pub(crate) use persisting_pchronicle::storage::session_markdown_write_path_for_key; diff --git a/crates/persisting-gateway/src/engine/tests/wal.rs b/crates/persisting-gateway/src/engine/tests/wal.rs deleted file mode 100644 index d9e0bb4b3..000000000 --- a/crates/persisting-gateway/src/engine/tests/wal.rs +++ /dev/null @@ -1,56 +0,0 @@ -use super::fixtures::*; -use super::support::*; - -#[tokio::test] -async fn wal_replays_unacked_events_on_next_engine() { - let dir = tempfile::tempdir().unwrap(); - let storage = dir.path(); - - // Manually append an unacked event to the WAL — this models the case where - // the original process appended but crashed before the dispatcher applied it. - { - let wal = crate::engine::wal::EventWal::open(storage); - let appended = wal.append_event( - &test_context(), - &Event::Request(RequestEvent { - path: "/v1/chat/completions".into(), - method: "POST".into(), - url: None, - body_bytes: 5, - user_content: Some("survives crash".into()), - body_json: None, - semantic: None, - model_rewritten: false, - headers: vec![], - }), - ); - assert!(appended.is_some(), "wal append should have succeeded"); - } - - // Sanity check: the event is pending replay before any engine boots. - let pending = crate::engine::wal::replay_pending(storage); - assert_eq!(pending.len(), 1, "wal should hold one unacked entry"); - - // A fresh engine on the same storage replays the unacked event and routes - // it to its own sink, exactly as the original spawn_apply would have. - let sink = RecordingSink::new(); - let engine = test_engine(sink.clone(), storage, false).await; - engine.flush().await.unwrap(); - - let records = sink.drain(); - assert!( - records - .iter() - .any(|r| r.payload.get("user_content").and_then(|v| v.as_str()) - == Some("survives crash")), - "expected wal-replayed request to flow through new engine sink: {records:?}" - ); - - // Clean shutdown should truncate the WAL so a third engine sees no pending events. - engine.shutdown().await.unwrap(); - let pending_after = crate::engine::wal::replay_pending(storage); - assert!( - pending_after.is_empty(), - "wal not truncated on clean shutdown" - ); -} diff --git a/crates/persisting-gateway/src/engine/wal.rs b/crates/persisting-gateway/src/engine/wal.rs deleted file mode 100644 index b8c1f69b2..000000000 --- a/crates/persisting-gateway/src/engine/wal.rs +++ /dev/null @@ -1,685 +0,0 @@ -//! Event write-ahead log — append-before-apply durability for in-flight events. -//! -//! `spawn_apply` submits a JSONL `(CallContext, Event)` record to the bounded -//! writer queue before handing the event to [`super::apply_queue::ApplyDispatcher`]. -//! Submission never waits for filesystem I/O; accepted records are committed -//! in the background and may be lost if the process crashes inside that window. -//! After apply succeeds, an `ack` line for the same `seq` is appended. -//! On a clean shutdown the file is truncated only when every event is acked; -//! failed durable writes remain pending for restart replay. -//! -//! On startup [`replay_pending`] scans the file: events whose `seq` was -//! never acked are returned for replay. This complements -//! [`crate::dead_letter`] (which only catches application-layer failures) -//! by giving us a recovery path for OOM/panic/SIGKILL between -//! `spawn_apply` and `apply` completion. - -use std::fs::File; -use std::io::{BufRead, BufReader, Write}; -use std::path::{Path, PathBuf}; -use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::{Arc, Mutex, mpsc}; -use std::thread::JoinHandle; -use std::time::{Duration, Instant}; - -use anyhow::{Context, Result}; -use serde::{Deserialize, Serialize}; - -use super::{CallContext, Event}; -use crate::dead_letter::{DeadLetterContext, SerializableEvent}; - -const WAL_FILENAME: &str = "events.wal.jsonl"; -/// Maximum time the writer deliberately waits for peers to join a commit. -/// Disk scheduling can add latency beyond this coalescing window. -const GROUP_COMMIT_MAX_DELAY: Duration = Duration::from_millis(2); -const GROUP_COMMIT_MAX_LINES: usize = 256; -const WAL_QUEUE_CAPACITY: usize = 256; - -pub fn wal_path(storage: &Path) -> PathBuf { - storage.join(".capture").join(WAL_FILENAME) -} - -/// Payload of a WAL `Event` line — split out so the larger variant doesn't bloat -/// the `Ack`-only path through `WalLine`. -#[derive(Debug, Clone, Serialize, Deserialize)] -struct WalEventBody { - seq: u64, - timestamp: String, - context: DeadLetterContext, - event: SerializableEvent, -} - -/// One line in the WAL — either an enqueued event or an ack for a previously enqueued one. -/// `Event` carries its body via a `Box` so the enum's stack footprint is dominated by `Ack`, -/// which is by far the more common variant on the hot path. -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(tag = "kind", rename_all = "snake_case")] -enum WalLine { - Event { - #[serde(flatten)] - body: Box, - }, - Ack { - seq: u64, - }, -} - -type ControlResult = std::result::Result<(), String>; -type ControlCompletion = mpsc::SyncSender; - -enum WalCommand { - Line(WalLine), - Flush { completion: ControlCompletion }, - Truncate { completion: ControlCompletion }, - Shutdown, -} - -/// Append-only WAL with a process-local monotonic sequence and a dedicated -/// bounded-delay group-commit writer. -/// -/// Event and ACK submissions are both best-effort and asynchronous. Once a line -/// reaches the writer it is protected by group commit, while queue saturation or -/// a crash before the next commit may lose the WAL copy. Canonical capture keeps -/// its independent apply/Lance durability path. -pub(crate) struct EventWal { - next_seq: AtomicU64, - sender: Option>, - worker: Mutex>>, - #[cfg(test)] - sync_count: Arc, - enabled: bool, -} - -impl EventWal { - /// Open (or create) the WAL at `/.capture/events.wal.jsonl`. - /// On any I/O error the WAL falls back to disabled mode and the proxy keeps running - /// without durability — capture is best-effort, not critical-path. - pub fn open(storage: &Path) -> Self { - let path = wal_path(storage); - let enabled = match prepare_writer(&path) { - Ok(file) => { - let next_seq = next_sequence(&path); - let (sender, receiver) = mpsc::sync_channel(WAL_QUEUE_CAPACITY); - let writer_path = path.clone(); - let sync_count = Arc::new(AtomicU64::new(0)); - let worker_sync_count = Arc::clone(&sync_count); - match std::thread::Builder::new() - .name("persisting-wal-commit".to_string()) - .spawn(move || { - group_commit_loop(file, &writer_path, receiver, worker_sync_count) - }) { - Ok(worker) => { - return Self { - next_seq: AtomicU64::new(next_seq), - sender: Some(sender), - worker: Mutex::new(Some(worker)), - #[cfg(test)] - sync_count, - enabled: true, - }; - } - Err(e) => { - tracing::warn!(target: "persisting_gateway", "wal writer disabled: {e}"); - false - } - } - } - Err(e) => { - tracing::warn!(target: "persisting_gateway", "wal disabled: {e:#}"); - false - } - }; - Self { - next_seq: AtomicU64::new(0), - sender: None, - worker: Mutex::new(None), - #[cfg(test)] - sync_count: Arc::new(AtomicU64::new(0)), - enabled, - } - } - - /// Submit an event before dispatch without waiting for queue capacity or I/O. - /// Returns `Some(seq)` when accepted (must later be passed to [`Self::ack`]), - /// or `None` when WAL is disabled, full, or closed. - pub fn append_event(&self, ctx: &CallContext, event: &Event) -> Option { - if !self.enabled { - return None; - } - let seq = self.next_seq.fetch_add(1, Ordering::SeqCst); - let line = WalLine::Event { - body: Box::new(WalEventBody { - seq, - timestamp: chrono::Utc::now().to_rfc3339(), - context: DeadLetterContext::from_context(ctx), - event: SerializableEvent::from_event(event), - }), - }; - let Some(sender) = &self.sender else { - return None; - }; - match sender.try_send(WalCommand::Line(line)) { - Ok(()) => Some(seq), - Err(e) => { - tracing::warn!(target: "persisting_gateway", "wal append: {e}"); - None - } - } - } - - /// Mark a previously enqueued event as applied. - pub fn ack(&self, seq: u64) { - if !self.enabled { - return; - } - let Some(sender) = &self.sender else { - return; - }; - if let Err(e) = sender.try_send(WalCommand::Line(WalLine::Ack { seq })) { - tracing::warn!(target: "persisting_gateway", "wal ack: {e}"); - } - } - - /// Wait until every line submitted before this call has completed its group - /// commit. Required before inspecting or truncating the WAL during shutdown. - pub fn flush(&self) -> Result<()> { - self.control(|completion| WalCommand::Flush { completion }) - } - - /// Truncate the WAL after verifying that no unacknowledged event remains. - pub fn truncate(&self) -> Result<()> { - self.control(|completion| WalCommand::Truncate { completion }) - } - - fn control(&self, command: impl FnOnce(ControlCompletion) -> WalCommand) -> Result<()> { - if !self.enabled { - return Ok(()); - } - let (completion, completed) = mpsc::sync_channel(1); - let sender = self - .sender - .as_ref() - .ok_or_else(|| anyhow::anyhow!("wal not open"))?; - sender - .send(command(completion)) - .map_err(|_| anyhow::anyhow!("wal writer stopped"))?; - completed - .recv() - .map_err(|_| anyhow::anyhow!("wal writer dropped control result"))? - .map_err(anyhow::Error::msg) - } - - #[cfg(test)] - fn sync_count(&self) -> u64 { - self.sync_count.load(Ordering::Relaxed) - } -} - -impl Drop for EventWal { - fn drop(&mut self) { - if let Some(sender) = &self.sender { - let _ = sender.send(WalCommand::Shutdown); - } - if let Some(worker) = self.worker.lock().expect("wal worker mutex").take() - && worker.join().is_err() - { - tracing::warn!(target: "persisting_gateway", "wal writer thread panicked"); - } - } -} - -fn group_commit_loop( - mut file: File, - path: &Path, - receiver: mpsc::Receiver, - sync_count: Arc, -) { - let mut deferred = None; - loop { - let command = match deferred.take().or_else(|| receiver.recv().ok()) { - Some(command) => command, - None => return, - }; - match command { - WalCommand::Line(line) => { - let deadline = Instant::now() + GROUP_COMMIT_MAX_DELAY; - let mut batch = vec![line]; - let mut disconnected = false; - while batch.len() < GROUP_COMMIT_MAX_LINES { - let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { - break; - }; - match receiver.recv_timeout(remaining) { - Ok(WalCommand::Line(line)) => batch.push(line), - Ok(command) => { - deferred = Some(command); - break; - } - Err(mpsc::RecvTimeoutError::Timeout) => break, - Err(mpsc::RecvTimeoutError::Disconnected) => { - disconnected = true; - break; - } - } - } - commit_lines(&mut file, batch, &sync_count); - if disconnected { - return; - } - } - WalCommand::Flush { completion } => { - // FIFO command ordering means all preceding line batches have - // already completed `sync_data` when this barrier is observed. - let _ = completion.send(Ok(())); - } - WalCommand::Truncate { completion } => { - let result = (|| -> Result { - let truncated = crate::runtime::open_private_truncate_file(path) - .with_context(|| format!("truncate wal {}", path.display()))?; - truncated.sync_data().context("fsync truncated wal")?; - Ok(truncated) - })(); - match result { - Ok(truncated) => { - file = truncated; - sync_count.fetch_add(1, Ordering::Relaxed); - let _ = completion.send(Ok(())); - } - Err(error) => { - let _ = completion.send(Err(format!("{error:#}"))); - } - } - } - WalCommand::Shutdown => return, - } - } -} - -fn commit_lines(file: &mut File, batch: Vec, sync_count: &AtomicU64) { - let result = (|| -> Result<()> { - for line in &batch { - serde_json::to_writer(&mut *file, line).context("serialize wal line")?; - file.write_all(b"\n").context("append wal newline")?; - } - file.sync_data().context("fsync wal")?; - Ok(()) - })(); - let commit = sync_count.fetch_add(1, Ordering::Relaxed) + 1; - tracing::trace!( - target: "persisting_gateway", - commit, - lines = batch.len(), - "wal group commit" - ); - if let Err(error) = &result { - tracing::warn!( - target: "persisting_gateway", - lines = batch.len(), - "wal group commit failed: {error}" - ); - } -} - -fn prepare_writer(path: &Path) -> Result { - crate::runtime::open_private_append_file(path) - .with_context(|| format!("open wal {}", path.display())) -} - -/// One unacked entry recovered from the WAL. -#[derive(Debug, Clone)] -pub(crate) struct PendingEntry { - pub seq: u64, - pub context: DeadLetterContext, - pub event: SerializableEvent, -} - -/// Read the WAL and return events that were appended but never acked. -/// Missing or corrupt files are treated as "no pending events" with a warning. -pub(crate) fn replay_pending(storage: &Path) -> Vec { - let path = wal_path(storage); - if !path.exists() { - return Vec::new(); - } - let file = match File::open(&path) { - Ok(f) => f, - Err(e) => { - tracing::warn!(target: "persisting_gateway", "wal open: {e:#}"); - return Vec::new(); - } - }; - - let mut events: std::collections::BTreeMap = - std::collections::BTreeMap::new(); - let mut acked: std::collections::HashSet = std::collections::HashSet::new(); - for (i, line) in BufReader::new(file).lines().enumerate() { - let line = match line { - Ok(l) => l, - Err(e) => { - tracing::warn!(target: "persisting_gateway", "wal read line {i}: {e:#}"); - continue; - } - }; - let trimmed = line.trim(); - if trimmed.is_empty() { - continue; - } - match serde_json::from_str::(trimmed) { - Ok(WalLine::Event { body }) => { - let WalEventBody { - seq, - context, - event, - .. - } = *body; - events.insert( - seq, - PendingEntry { - seq, - context, - event, - }, - ); - } - Ok(WalLine::Ack { seq }) => { - acked.insert(seq); - } - Err(e) => { - tracing::warn!(target: "persisting_gateway", "wal parse line {i}: {e:#}"); - } - } - } - - events - .into_iter() - .filter_map(|(seq, entry)| (!acked.contains(&seq)).then_some(entry)) - .collect() -} - -fn next_sequence(path: &Path) -> u64 { - let Ok(file) = File::open(path) else { - return 0; - }; - BufReader::new(file) - .lines() - .map_while(std::result::Result::ok) - .filter_map(|line| serde_json::from_str::(line.trim()).ok()) - .map(|line| match line { - WalLine::Event { body } => body.seq, - WalLine::Ack { seq } => seq, - }) - .max() - .map_or(0, |seq| seq.saturating_add(1)) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::Call; - use crate::config::CaptureLevel; - use crate::engine::RequestEvent; - use crate::protocol::ProtocolKind; - use crate::provider::ProviderKind; - use crate::session::storage::CaptureRoute; - - fn sample_ctx() -> CallContext { - CallContext::new( - CaptureRoute { - root_session: Some("run-1".into()), - session_id: "sess".into(), - storage_session_id: "run-1".into(), - subagent_id: None, - }, - "agent", - Call { - call_id: "c1".into(), - trace_id: "t1".into(), - started_at: "2026-01-01T00:00:00Z".into(), - }, - Vec::new(), - CaptureLevel::Dialogue, - "m", - "m", - ProviderKind::OpenAi, - ProtocolKind::ChatCompletions, - false, - ) - } - - fn sample_event() -> Event { - Event::Request(RequestEvent { - path: "/v1/chat/completions".into(), - method: "POST".into(), - url: None, - body_bytes: 10, - user_content: Some("hi".into()), - body_json: None, - semantic: None, - model_rewritten: false, - headers: vec![], - }) - } - - #[test] - fn wal_replays_unacked_only() { - let dir = tempfile::tempdir().unwrap(); - let wal = EventWal::open(dir.path()); - - let ctx = sample_ctx(); - let event = sample_event(); - let s1 = wal.append_event(&ctx, &event).expect("seq"); - let s2 = wal.append_event(&ctx, &event).expect("seq"); - wal.ack(s1); - drop(wal); - - let pending = replay_pending(dir.path()); - assert_eq!(pending.len(), 1, "expected only s2 to be pending"); - assert_eq!(s2, 1); - } - - #[test] - fn writer_commits_queued_lines_with_one_fsync() { - let dir = tempfile::tempdir().unwrap(); - let path = wal_path(dir.path()); - let file = prepare_writer(&path).unwrap(); - let (sender, receiver) = mpsc::sync_channel(WAL_QUEUE_CAPACITY); - let sync_count = Arc::new(AtomicU64::new(0)); - for seq in 0..16 { - sender.send(WalCommand::Line(WalLine::Ack { seq })).unwrap(); - } - let (completion, completed) = mpsc::sync_channel(1); - sender.send(WalCommand::Flush { completion }).unwrap(); - sender.send(WalCommand::Shutdown).unwrap(); - - let writer_path = path.clone(); - let writer_sync_count = Arc::clone(&sync_count); - let worker = std::thread::spawn(move || { - group_commit_loop(file, &writer_path, receiver, writer_sync_count) - }); - completed.recv().unwrap().unwrap(); - worker.join().unwrap(); - - assert_eq!(sync_count.load(Ordering::Relaxed), 1); - assert_eq!(std::fs::read_to_string(path).unwrap().lines().count(), 16); - } - - #[test] - fn flush_makes_async_ack_visible_before_pending_check() { - let dir = tempfile::tempdir().unwrap(); - let wal = EventWal::open(dir.path()); - let seq = wal.append_event(&sample_ctx(), &sample_event()).unwrap(); - wal.ack(seq); - wal.flush().unwrap(); - - assert!(replay_pending(dir.path()).is_empty()); - assert!(wal.sync_count() >= 1); - } - - #[test] - fn truncate_clears_unacked() { - let dir = tempfile::tempdir().unwrap(); - let wal = EventWal::open(dir.path()); - wal.append_event(&sample_ctx(), &sample_event()).unwrap(); - wal.truncate().unwrap(); - drop(wal); - - let pending = replay_pending(dir.path()); - assert!(pending.is_empty()); - } - - #[test] - fn missing_file_replays_empty() { - let dir = tempfile::tempdir().unwrap(); - let pending = replay_pending(dir.path()); - assert!(pending.is_empty()); - } - - #[test] - fn reopen_continues_sequence_after_existing_entries() { - let dir = tempfile::tempdir().unwrap(); - let first = EventWal::open(dir.path()); - assert_eq!(first.append_event(&sample_ctx(), &sample_event()), Some(0)); - drop(first); - - let reopened = EventWal::open(dir.path()); - assert_eq!( - reopened.append_event(&sample_ctx(), &sample_event()), - Some(1) - ); - } - - #[test] - fn wal_retains_original_client_request_before_protocol_conversion() { - let dir = tempfile::tempdir().unwrap(); - let original_bytes = bytes::Bytes::from_static( - br#"{"model":"claude-client","max_tokens":16,"messages":[{"role":"user","content":"original"}]}"#, - ); - let original_json: serde_json::Value = serde_json::from_slice(&original_bytes).unwrap(); - let converted = - crate::conversion::messages_request_to_completions(&original_bytes, "upstream-model") - .unwrap(); - let converted_json: serde_json::Value = serde_json::from_slice(&converted).unwrap(); - assert_eq!(converted_json["model"], "upstream-model"); - - let wal = EventWal::open(dir.path()); - let semantic = - crate::understanding::understand_request(ProtocolKind::Messages, &original_bytes) - .unwrap() - .semantic; - wal.append_event( - &sample_ctx(), - &Event::Request(RequestEvent { - path: "/v1/messages".into(), - method: "POST".into(), - url: Some("//gateway/v1/messages".into()), - body_bytes: original_bytes.len(), - user_content: Some("original".into()), - body_json: Some(original_json.clone()), - semantic: Some(semantic), - model_rewritten: true, - headers: vec![("x-request-id".into(), "req-1".into())], - }), - ) - .unwrap(); - drop(wal); - - let serialized = std::fs::read_to_string(wal_path(dir.path())).unwrap(); - assert!(serialized.contains("body_json")); - assert!( - !serialized.contains("\"semantic\""), - "WAL must not duplicate the typed payload: {serialized}" - ); - - let pending = replay_pending(dir.path()); - assert_eq!(pending.len(), 1); - let Event::Request(replayed) = pending[0].event.to_event() else { - panic!("expected request event") - }; - assert_eq!(replayed.path, "/v1/messages"); - assert_eq!(replayed.url.as_deref(), Some("//gateway/v1/messages")); - assert_eq!(replayed.body_json, Some(original_json)); - assert_eq!(replayed.headers[0], ("x-request-id".into(), "req-1".into())); - } - - #[test] - fn wal_redacts_credentials_before_writing() { - let dir = tempfile::tempdir().unwrap(); - let mut ctx = sample_ctx(); - ctx.request_headers = vec![ - ("authorization".into(), "Bearer context-secret".into()), - ("x-request-id".into(), "req-safe".into()), - ]; - ctx.upstream_url = Some("https://upstream.example/v1?key=url-secret".into()); - let event = Event::Request(RequestEvent { - path: "/v1/chat/completions".into(), - method: "POST".into(), - url: Some("//gateway.example/v1?api_key=request-url-secret".into()), - body_bytes: 10, - user_content: Some("hi".into()), - body_json: Some(serde_json::json!({ - "apiKey": "body-secret", - "safe": "kept" - })), - semantic: None, - model_rewritten: false, - headers: vec![("x-api-key".into(), "event-secret".into())], - }); - - let wal = EventWal::open(dir.path()); - wal.append_event(&ctx, &event).unwrap(); - drop(wal); - - let serialized = std::fs::read_to_string(wal_path(dir.path())).unwrap(); - for secret in [ - "context-secret", - "event-secret", - "body-secret", - "url-secret", - "request-url-secret", - ] { - assert!( - !serialized.contains(secret), - "WAL persisted credential {secret}: {serialized}" - ); - } - assert!(serialized.contains("req-safe")); - assert!(serialized.contains("kept")); - assert!(serialized.contains("")); - - let pending = replay_pending(dir.path()); - assert_eq!(pending[0].context.request_headers[0].1, ""); - let Event::Request(replayed) = pending[0].event.to_event() else { - panic!("expected request event") - }; - assert_eq!(replayed.headers[0].1, ""); - assert_eq!(replayed.body_json.unwrap()["apiKey"], ""); - } - - #[cfg(unix)] - #[test] - fn wal_uses_private_permissions() { - use std::os::unix::fs::PermissionsExt; - - let dir = tempfile::tempdir().unwrap(); - let wal = EventWal::open(dir.path()); - wal.append_event(&sample_ctx(), &sample_event()).unwrap(); - drop(wal); - - let capture_dir = dir.path().join(".capture"); - assert_eq!( - std::fs::metadata(&capture_dir) - .unwrap() - .permissions() - .mode() - & 0o777, - 0o700 - ); - assert_eq!( - std::fs::metadata(wal_path(dir.path())) - .unwrap() - .permissions() - .mode() - & 0o777, - 0o600 - ); - } -} diff --git a/crates/persisting-gateway/src/engine/wire/headers.rs b/crates/persisting-gateway/src/engine/wire/headers.rs deleted file mode 100644 index 85729dea0..000000000 --- a/crates/persisting-gateway/src/engine/wire/headers.rs +++ /dev/null @@ -1,22 +0,0 @@ -use anyhow::{Context, Result}; -use axum::http::{HeaderMap, HeaderName, HeaderValue}; - -pub(crate) fn headers_to_vec(map: &HeaderMap) -> Vec<(String, String)> { - map.iter() - .filter_map(|(name, value)| { - Some((name.as_str().to_string(), value.to_str().ok()?.to_string())) - }) - .collect() -} - -pub(crate) fn headers_to_header_map(pairs: &[(String, String)]) -> Result { - let mut map = HeaderMap::new(); - for (name, value) in pairs { - let name = HeaderName::from_bytes(name.as_bytes()) - .with_context(|| format!("invalid header name: {name}"))?; - let value = HeaderValue::from_str(value) - .with_context(|| format!("invalid header value for {name}: {value}"))?; - map.append(name, value); - } - Ok(map) -} diff --git a/crates/persisting-gateway/src/engine/wire/mod.rs b/crates/persisting-gateway/src/engine/wire/mod.rs deleted file mode 100644 index af4c8dfe1..000000000 --- a/crates/persisting-gateway/src/engine/wire/mod.rs +++ /dev/null @@ -1,43 +0,0 @@ -//! Actor wire protocol — all cross-actor messages live here. - -mod headers; -mod run; -mod story; - -pub(crate) use headers::{headers_to_header_map, headers_to_vec}; -pub(crate) use run::{RUN_ACTOR_NAME, RunCommand, RunReply, run_enrich, run_main_route}; -pub(crate) use story::{DraftPayload, StoryCommand, StoryReply, StoryScope}; - -/// Unified ask/tell acknowledgement for story actors. -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] -pub(crate) struct CaptureAck { - pub ok: bool, - pub error: Option, -} - -impl CaptureAck { - pub fn ok() -> Self { - Self { - ok: true, - error: None, - } - } - - pub fn err(message: impl Into) -> Self { - Self { - ok: false, - error: Some(message.into()), - } - } - - pub fn into_result(self) -> anyhow::Result<()> { - if self.ok { - Ok(()) - } else { - Err(anyhow::anyhow!( - self.error - .unwrap_or_else(|| "capture command failed".into()) - )) - } - } -} diff --git a/crates/persisting-gateway/src/engine/wire/run.rs b/crates/persisting-gateway/src/engine/wire/run.rs deleted file mode 100644 index 74c3eec4c..000000000 --- a/crates/persisting-gateway/src/engine/wire/run.rs +++ /dev/null @@ -1,111 +0,0 @@ -//! Run actor command/reply pairs + runtime-side client helpers. - -use anyhow::Result; -use pulsing_actor::ActorRef; -use serde::{Deserialize, Serialize}; -use serde_json::Value; - -use crate::engine::story::{RunId, StoryId}; -use crate::record::EventRecord; -use crate::session::storage::CaptureRoute; -use crate::subagent_link::SpawnLinkBackfill; - -use super::super::CallContext; - -/// Global run actor path (one per capture runtime / ActorSystem). -pub(crate) const RUN_ACTOR_NAME: &str = "capture/run"; - -/// Commands handled by [`super::super::actors::run::RunActor`]. -/// -/// `record_bytes` / `body_bytes` are JSON-encoded payloads carried as -/// raw bytes — see [`super::story::StoryCommand`] for rationale. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub(crate) enum RunCommand { - Enrich { - record_bytes: Vec, - route: CaptureRoute, - headers: Vec<(String, String)>, - body_bytes: Option>, - assistant_text: Option, - story_id: Option, - run_id: Option, - }, - MainRoute { - route: CaptureRoute, - }, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub(crate) enum RunReply { - Enrich { - record_bytes: Vec, - backfills: Vec, - }, - MainRoute(CaptureRoute), -} - -pub(crate) async fn run_enrich( - run: &ActorRef, - rec: &mut EventRecord, - ctx: &CallContext, - body_json: Option<&Value>, - assistant_text: Option<&str>, -) -> Result> { - let story = ctx.story.clone(); - let reply: RunReply = run - .ask(RunCommand::Enrich { - record_bytes: serde_json::to_vec(rec)?, - route: ctx.route().clone(), - headers: ctx.request_headers.clone(), - body_bytes: body_json.map(serde_json::to_vec).transpose()?, - assistant_text: assistant_text.map(str::to_string), - story_id: Some(story.story_id.clone()), - run_id: story.run_id.clone(), - }) - .await - .map_err(|e| anyhow::anyhow!("{e}"))?; - match reply { - RunReply::Enrich { - record_bytes, - backfills, - } => { - *rec = serde_json::from_slice(&record_bytes)?; - Ok(backfills) - } - _ => Err(anyhow::anyhow!("unexpected run reply")), - } -} - -pub(crate) async fn run_main_route(run: &ActorRef, route: &CaptureRoute) -> Result { - let reply: RunReply = run - .ask(RunCommand::MainRoute { - route: route.clone(), - }) - .await - .map_err(|e| anyhow::anyhow!("{e}"))?; - match reply { - RunReply::MainRoute(r) => Ok(r), - _ => Err(anyhow::anyhow!("unexpected run reply")), - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn run_command_bincode_roundtrip() { - let route = CaptureRoute { - root_session: Some("run".into()), - session_id: "s".into(), - storage_session_id: "s".into(), - subagent_id: None, - }; - let cmd = RunCommand::MainRoute { - route: route.clone(), - }; - let packed = pulsing_actor::Message::pack(&cmd).expect("pack"); - let back: RunCommand = packed.unpack().expect("unpack"); - assert!(matches!(back, RunCommand::MainRoute { .. })); - } -} diff --git a/crates/persisting-gateway/src/engine/wire/story.rs b/crates/persisting-gateway/src/engine/wire/story.rs deleted file mode 100644 index 1ecd1e989..000000000 --- a/crates/persisting-gateway/src/engine/wire/story.rs +++ /dev/null @@ -1,159 +0,0 @@ -//! Per-story actor commands — one variant per I/O effect. - -use anyhow::Result; -use serde::{Deserialize, Serialize}; - -use super::super::story::{Story, StoryContext}; -use super::CaptureAck; - -/// Routing identity for every story command. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub(crate) struct StoryScope { - pub context: StoryContext, -} - -impl StoryScope { - pub fn from_context(ctx: &super::super::CallContext) -> Self { - Self { - context: ctx.story.clone(), - } - } - - pub fn route(&self) -> &crate::session::storage::CaptureRoute { - &self.context.route - } - - pub fn agent_id(&self) -> &str { - &self.context.agent_id - } -} - -/// Commands handled by [`super::super::actors::StoryActor`]. -/// -/// Inner payloads are sent as raw JSON bytes (`Vec`) rather than `String`: -/// - `bincode` (the actor wire format) length-prefixes both, but `String` -/// forces a UTF-8 validation pass on every serialize/deserialize, while -/// `Vec` is a memcpy. We're already JSON-encoding (and thus producing -/// valid UTF-8) at the producer, so the second validation is wasted work. -/// - The receiver does `serde_json::from_slice(&bytes)` directly without -/// first reconstructing a `String`. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub(crate) enum StoryCommand { - /// Append one record to Lance/sink and sync live markdown. - PersistRecord { - scope: StoryScope, - /// JSON-encoded [`crate::record::EventRecord`]. - record_bytes: Vec, - }, - /// Upsert streaming assistant draft in live markdown only (no sink append). - UpsertDraft { - scope: StoryScope, - draft_bytes: Vec, - }, - /// Drain mailbox before shutdown (no I/O). - Flush, - /// Read-model snapshot (no I/O). - Snapshot { scope: StoryScope }, - /// Read-model snapshot without scope (shutdown / persist). - LocalSnapshot, -} - -/// Reply from [`super::super::actors::StoryActor`]. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub(crate) enum StoryReply { - Ack(CaptureAck), - Snapshot { - story: Story, - }, - LocalSnapshot { - storage_session_id: String, - story: Story, - }, -} - -impl StoryCommand { - pub fn persist_record(scope: StoryScope, record_bytes: Vec) -> Self { - Self::PersistRecord { - scope, - record_bytes, - } - } - - pub fn upsert_draft( - scope: StoryScope, - record_bytes: Vec, - assistant_content: String, - ) -> Result { - let draft = DraftPayload { - record_bytes, - assistant_content, - }; - Ok(Self::UpsertDraft { - scope, - draft_bytes: serde_json::to_vec(&draft)?, - }) - } - - pub fn scope(&self) -> &StoryScope { - match self { - Self::PersistRecord { scope, .. } - | Self::UpsertDraft { scope, .. } - | Self::Snapshot { scope } => scope, - Self::Flush | Self::LocalSnapshot => panic!("command has no scope"), - } - } -} - -/// JSON payload for draft upsert: record template (seq assigned in StoryActor) + stream text. -#[derive(Debug, Serialize, Deserialize)] -pub(crate) struct DraftPayload { - pub record_bytes: Vec, - pub assistant_content: String, -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::Call; - use crate::config::CaptureLevel; - use crate::session::storage::CaptureRoute; - use crate::sink::llm_request_summary_record; - - #[test] - fn story_command_bincode_roundtrip() { - let scope = StoryScope { - context: StoryContext::from_route( - CaptureRoute { - root_session: Some("run".into()), - session_id: "s".into(), - storage_session_id: "s".into(), - subagent_id: None, - }, - "agent", - ), - }; - let call = Call { - call_id: "c".into(), - trace_id: "t".into(), - started_at: "2026-01-01T00:00:00Z".into(), - }; - let rec = llm_request_summary_record( - Some("s".into()), - Some("a".into()), - "model", - "/v1/chat/completions", - 12, - "chat_completions", - "openai", - Some("hi".into()), - None, - &call, - CaptureLevel::Dialogue, - None, - ); - let cmd = StoryCommand::persist_record(scope, serde_json::to_vec(&rec).unwrap()); - let packed = pulsing_actor::Message::pack(&cmd).expect("pack"); - let back: StoryCommand = packed.unpack().expect("unpack"); - assert!(matches!(back, StoryCommand::PersistRecord { .. })); - } -} diff --git a/crates/persisting-gateway/src/gateway/admin.rs b/crates/persisting-gateway/src/gateway/admin.rs deleted file mode 100644 index 04c8f65f0..000000000 --- a/crates/persisting-gateway/src/gateway/admin.rs +++ /dev/null @@ -1,90 +0,0 @@ -//! Gateway admin HTTP API: status + session list. - -use std::sync::Arc; - -use axum::extract::State; -use axum::routing::get; -use axum::{Json, Router}; -use serde::Serialize; - -use crate::session::index::{SessionIndex, SessionIndexHandle}; -use persisting_overlaynet::{InterceptionMetrics, InterceptionProfile, InterceptionSnapshot}; - -#[derive(Clone)] -pub struct AdminState { - pub index: SessionIndexHandle, - pub listen: String, - pub admin_listen: String, - pub started_at: String, - pub active_requests: Arc, - pub interception_metrics: InterceptionMetrics, -} - -#[derive(Serialize)] -pub struct StatusResponse { - pub listen: String, - pub admin: String, - pub started_at: String, - pub active_requests: usize, - pub interception: InterceptionProfile, - pub interception_metrics: InterceptionSnapshot, - pub sessions: Vec, -} - -pub fn admin_router(state: AdminState) -> Router { - Router::new() - .route("/admin/status", get(status_handler)) - .route("/admin/sessions", get(sessions_handler)) - .with_state(state) -} - -async fn status_handler(State(st): State) -> Json { - let index = st.index.snapshot(); - Json(StatusResponse { - listen: st.listen.clone(), - admin: st.admin_listen.clone(), - started_at: st.started_at.clone(), - active_requests: st - .active_requests - .load(std::sync::atomic::Ordering::Relaxed), - interception: InterceptionProfile::explicit_proxy(), - interception_metrics: st.interception_metrics.snapshot(), - sessions: index.sessions, - }) -} - -async fn sessions_handler(State(st): State) -> Json { - Json(st.index.snapshot()) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::session::index::SessionIndexStore; - - #[tokio::test] - async fn status_reports_both_listen_addresses() { - let storage = tempfile::tempdir().unwrap(); - let index = SessionIndexStore::open(storage.path()) - .unwrap() - .clone_handle(); - let response = status_handler(State(AdminState { - index, - listen: "127.0.0.1:9875".into(), - admin_listen: "127.0.0.1:9876".into(), - started_at: "2026-07-31T00:00:00Z".into(), - active_requests: Arc::new(std::sync::atomic::AtomicUsize::new(3)), - interception_metrics: InterceptionMetrics::default(), - })) - .await; - - assert_eq!(response.0.listen, "127.0.0.1:9875"); - assert_eq!(response.0.admin, "127.0.0.1:9876"); - assert_eq!(response.0.active_requests, 3); - assert!(!response.0.interception.is_enforcing()); - assert_eq!( - response.0.interception_metrics, - InterceptionSnapshot::default() - ); - } -} diff --git a/crates/persisting-gateway/src/gateway/auth.rs b/crates/persisting-gateway/src/gateway/auth.rs deleted file mode 100644 index daf5ceb5f..000000000 --- a/crates/persisting-gateway/src/gateway/auth.rs +++ /dev/null @@ -1,263 +0,0 @@ -//! Upstream API key resolution (OpenAI Bearer vs Anthropic x-api-key). - -use axum::http::HeaderMap; -use reqwest::RequestBuilder; - -use crate::config::ModelRoute; -use crate::protocol::ProtocolKind; -use crate::provider::ProviderKind; -use persisting_overlaynet::headers::skip_upstream_forward_header; - -pub fn resolve_upstream_api_key( - route: &ModelRoute, - client_headers: &HeaderMap, -) -> anyhow::Result<(Option, &'static str)> { - if let Some(k) = route.api_key_value()? { - let source = route - .api_key_env - .as_deref() - .map(|_| "daemon_env") - .unwrap_or("inline_api_key"); - return Ok((Some(k), source)); - } - let client_key = if route.provider_kind() == ProviderKind::Gemini { - google_api_key(client_headers).or_else(|| client_auth_token(client_headers)) - } else { - client_auth_token(client_headers) - }; - if let Some(k) = client_key { - return Ok((Some(k), "client_header")); - } - if route.api_key_env.is_some() || route.api_key.is_some() { - anyhow::bail!( - "api key not available for model route `{}` (set {} or pass client auth header)", - route.name, - route.api_key_env.as_deref().unwrap_or("api_key in config") - ); - } - Ok((None, "none")) -} - -/// Forward client headers and attach route API key with the correct scheme. -pub fn apply_upstream_headers( - mut req: RequestBuilder, - client_headers: &HeaderMap, - route: &ModelRoute, - protocol: ProtocolKind, -) -> anyhow::Result { - let provider = route.provider_kind(); - let anthropic_style = provider == ProviderKind::Anthropic || protocol == ProtocolKind::Messages; - - for (name, value) in client_headers.iter() { - if skip_upstream_forward_header(name.as_str()) { - continue; - } - req = req.header(name, value); - } - - let (key, _source) = resolve_upstream_api_key(route, client_headers)?; - - if let Some(key) = key { - if provider == ProviderKind::Gemini { - req = req.header("x-goog-api-key", key); - } else if anthropic_style { - req = req.header("x-api-key", key); - } else { - req = req.bearer_auth(key); - } - } else if route.api_key_env.is_some() || route.api_key.is_some() { - anyhow::bail!( - "api key not available for model route `{}` (set {} or pass client auth header)", - route.name, - route.api_key_env.as_deref().unwrap_or("api_key in config") - ); - } - - Ok(req) -} - -/// Client-side LLM credentials (Claude Code → `x-api-key` or `Authorization: Bearer`). -fn client_auth_token(headers: &HeaderMap) -> Option { - if let Some(v) = headers.get("x-api-key").and_then(|v| v.to_str().ok()) { - let v = v.trim(); - if !v.is_empty() { - return Some(v.to_string()); - } - } - headers - .get("authorization") - .and_then(|v| v.to_str().ok()) - .and_then(|v| v.strip_prefix("Bearer ")) - .map(str::trim) - .filter(|s| !s.is_empty()) - .map(str::to_string) -} - -fn google_api_key(headers: &HeaderMap) -> Option { - headers - .get("x-goog-api-key") - .and_then(|value| value.to_str().ok()) - .map(str::trim) - .filter(|value| !value.is_empty()) - .map(str::to_string) -} - -#[cfg(test)] -mod tests { - use super::*; - use axum::http::HeaderValue; - - fn route(provider: Option<&str>, key: Option<&str>) -> ModelRoute { - route_with_env(provider, key, None) - } - - fn route_with_env( - provider: Option<&str>, - api_key: Option<&str>, - api_key_env: Option<&str>, - ) -> ModelRoute { - ModelRoute { - name: "*".into(), - provider: provider.map(str::to_string), - upstream: Some("https://example.com/v1".into()), - upstream_anthropic: None, - api_key_env: api_key_env.map(str::to_string), - api_key: api_key.map(str::to_string), - forward: None, - } - } - - fn clear_api_key_env() { - for key in [ - "DEEPSEEK_API_KEY", - "OPENAI_API_KEY", - "ANTHROPIC_API_KEY", - "ANTHROPIC_AUTH_TOKEN", - ] { - unsafe { - std::env::remove_var(key); - } - } - } - - #[test] - fn anthropic_uses_x_api_key() { - let r = route(Some("anthropic"), Some("sk-test")); - let mut headers = HeaderMap::new(); - headers.insert("authorization", HeaderValue::from_static("Bearer client")); - headers.insert("x-api-key", HeaderValue::from_static("client-key")); - headers.insert("anthropic-version", HeaderValue::from_static("2023-06-01")); - - let rb = apply_upstream_headers( - reqwest::Client::new().post("https://example.com/v1/messages"), - &headers, - &r, - ProtocolKind::Messages, - ) - .unwrap(); - let built = rb.build().unwrap(); - assert_eq!(built.headers().get("x-api-key").unwrap(), "sk-test"); - assert!(built.headers().get("authorization").is_none()); - assert_eq!( - built.headers().get("anthropic-version").unwrap(), - "2023-06-01" - ); - } - - #[test] - fn openai_uses_bearer() { - let r = route(Some("openai"), Some("sk-test")); - let headers = HeaderMap::new(); - let rb = apply_upstream_headers( - reqwest::Client::new().post("https://example.com/v1/chat/completions"), - &headers, - &r, - ProtocolKind::ChatCompletions, - ) - .unwrap(); - let built = rb.build().unwrap(); - assert_eq!( - built.headers().get("authorization").unwrap(), - "Bearer sk-test" - ); - } - - #[test] - fn gemini_uses_google_api_key_header() { - let r = route(Some("gemini"), Some("gemini-test")); - let mut headers = HeaderMap::new(); - headers.insert("authorization", HeaderValue::from_static("Bearer client")); - let rb = apply_upstream_headers( - reqwest::Client::new() - .post("https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-pro:generateContent"), - &headers, - &r, - ProtocolKind::Gemini, - ) - .unwrap(); - let built = rb.build().unwrap(); - assert_eq!( - built.headers().get("x-goog-api-key").unwrap(), - "gemini-test" - ); - assert!(built.headers().get("authorization").is_none()); - } - - #[test] - fn gemini_can_use_client_google_api_key_without_forwarding_duplicates() { - let r = route(Some("gemini"), None); - let mut headers = HeaderMap::new(); - headers.insert("x-goog-api-key", HeaderValue::from_static("client-gemini")); - let rb = apply_upstream_headers( - reqwest::Client::new() - .post("https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-pro:generateContent"), - &headers, - &r, - ProtocolKind::Gemini, - ) - .unwrap(); - let built = rb.build().unwrap(); - assert_eq!(built.headers().get_all("x-goog-api-key").iter().count(), 1); - assert_eq!( - built.headers().get("x-goog-api-key").unwrap(), - "client-gemini" - ); - } - - #[test] - fn falls_back_to_client_x_api_key_when_daemon_env_missing() { - clear_api_key_env(); - let r = route_with_env(Some("anthropic"), None, Some("DEEPSEEK_API_KEY")); - let mut headers = HeaderMap::new(); - headers.insert("x-api-key", HeaderValue::from_static("sk-from-claude")); - let rb = apply_upstream_headers( - reqwest::Client::new().post("https://example.com/v1/messages"), - &headers, - &r, - ProtocolKind::Messages, - ) - .unwrap(); - let built = rb.build().unwrap(); - assert_eq!(built.headers().get("x-api-key").unwrap(), "sk-from-claude"); - } - - #[test] - fn falls_back_to_client_bearer_for_anthropic_upstream() { - clear_api_key_env(); - let r = route_with_env(Some("anthropic"), None, Some("DEEPSEEK_API_KEY")); - let mut headers = HeaderMap::new(); - headers.insert( - "authorization", - HeaderValue::from_static("Bearer sk-from-claude"), - ); - let rb = apply_upstream_headers( - reqwest::Client::new().post("https://example.com/v1/messages"), - &headers, - &r, - ProtocolKind::Messages, - ) - .unwrap(); - let built = rb.build().unwrap(); - assert_eq!(built.headers().get("x-api-key").unwrap(), "sk-from-claude"); - } -} diff --git a/crates/persisting-gateway/src/gateway/common.rs b/crates/persisting-gateway/src/gateway/common.rs deleted file mode 100644 index cb99b9fe6..000000000 --- a/crates/persisting-gateway/src/gateway/common.rs +++ /dev/null @@ -1,94 +0,0 @@ -//! Shared helpers for LLM gateway capture handlers. - -use std::sync::Arc; - -use axum::http::HeaderMap; -use bytes::Bytes; -use persisting_agentctl::ModelAccessPolicy; -use serde_json::Value; - -use super::state::GatewayState; -use crate::Call; -use crate::config::{CaptureLevel, ProxyConfig}; -use crate::engine::CallContext; -use crate::engine::headers_to_vec; -use crate::protocol::ProtocolKind; -use crate::provider::ProviderKind; -use crate::runtime::run_config::load_session_proxy_config; -use crate::session::storage::{CaptureRoute, route_config_key}; - -pub(crate) fn effective_config(state: &GatewayState, route: &CaptureRoute) -> Arc { - load_session_proxy_config(state.storage.as_path(), route_config_key(route)) - .map(Arc::new) - .unwrap_or_else(|| Arc::clone(&state.config)) -} - -pub(crate) fn model_access_policy(config: &ProxyConfig) -> ModelAccessPolicy { - let allowed_models = config - .models - .iter() - .map(|route| route.name.clone()) - .collect(); - let providers: Vec = config - .models - .iter() - .filter_map(|route| route.provider.clone()) - .collect(); - // An inferred/custom provider must remain representable during migration. - // An empty provider list means model identity is enforced but provider is open. - let allowed_providers = if providers.len() == config.models.len() { - providers - } else { - Vec::new() - }; - ModelAccessPolicy { - allowed_models, - allowed_providers, - } -} - -#[allow(clippy::too_many_arguments)] -pub(crate) fn call_context( - route: &CaptureRoute, - agent_id: &str, - call: &Call, - headers: &HeaderMap, - client_model: &str, - upstream_model: &str, - provider: ProviderKind, - protocol: ProtocolKind, - capture_level: CaptureLevel, - debug_on: bool, -) -> CallContext { - CallContext::new( - route.clone(), - agent_id, - call.clone(), - headers_to_vec(headers), - capture_level, - client_model, - upstream_model, - provider, - protocol, - debug_on, - ) -} - -pub(crate) fn extract_model(body: &Bytes) -> Option { - let v: Value = serde_json::from_slice(body).ok()?; - v.get("model")?.as_str().map(str::to_string) -} - -pub(crate) fn attach_capture_headers( - builder: axum::http::response::Builder, - call: &Call, -) -> axum::http::response::Builder { - builder - .header("x-persisting-call-id", call.call_id.as_str()) - .header("x-persisting-trace-id", call.trace_id.as_str()) -} - -pub(crate) fn is_models_list_path(path: &str) -> bool { - let p = path.trim_end_matches('/'); - p.ends_with("/models") || p == "models" || p.ends_with("/v1/models") -} diff --git a/crates/persisting-gateway/src/gateway/dispatch.rs b/crates/persisting-gateway/src/gateway/dispatch.rs deleted file mode 100644 index ac7d920ec..000000000 --- a/crates/persisting-gateway/src/gateway/dispatch.rs +++ /dev/null @@ -1,124 +0,0 @@ -//! Gateway sink adapter for the OverlayNet proxy server. - -use async_trait::async_trait; -use axum::Router; -use axum::extract::Request; -use axum::http::StatusCode; -use axum::response::{IntoResponse, Response}; -use bytes::Bytes; -use persisting_overlaynet::policy::{DenyReason, NetworkPolicy}; -use persisting_overlaynet::server::{OverlayRequestContext, OverlayServerState, OverlaySink}; - -use super::common::effective_config; -use super::llm_capture::llm_capture; -use super::state::GatewayState; -use crate::config::ProxyConfig; -use crate::runtime::debug::{self, is_debug_enabled}; -use crate::session::storage::resolve_capture_route; - -#[derive(Clone)] -pub(crate) struct GatewayRequestContext { - config: std::sync::Arc, - debug_on: bool, -} - -pub(crate) fn build_router(state: GatewayState) -> Router { - let server = OverlayServerState::new( - state.control_controller.clone(), - state.clone(), - state.active_requests.clone(), - ) - .with_interception_metrics(state.interception_metrics.clone()) - .with_bandwidth_registry(state.bandwidth_registry.clone()); - persisting_overlaynet::server::build_router(server) -} - -#[async_trait] -impl OverlaySink for GatewayState { - type RequestContext = GatewayRequestContext; - - fn request_context( - &self, - request: &Request, - ) -> anyhow::Result> { - let route = resolve_capture_route( - request.headers(), - &Bytes::new(), - &self.config.session_header, - self.storage.as_path(), - ); - let config = effective_config(self, &route); - let debug_on = is_debug_enabled(&config, self.storage.as_path()); - Ok(OverlayRequestContext { - policy: NetworkPolicy::from_config(config.as_ref())?, - run_id: route.root_session, - attempt_id: self.attempt_id.clone(), - storyline_id: Some(route.session_id.clone()), - session_id: route.session_id, - sink: GatewayRequestContext { config, debug_on }, - }) - } - - async fn handle( - &self, - request: Request, - peer: std::net::SocketAddr, - context: &OverlayRequestContext, - ) -> anyhow::Result { - if !self.gateway_enabled { - return Ok(( - StatusCode::BAD_REQUEST, - "relative request requires Gateway mode", - ) - .into_response()); - } - llm_capture( - self.clone(), - request, - peer, - std::sync::Arc::clone(&context.sink.config), - context.sink.debug_on, - ) - .await - } - - fn accepts(&self, request: &Request) -> bool { - self.gateway_enabled - && crate::protocol::ProtocolKind::from_path(request.uri().path()) - != crate::protocol::ProtocolKind::Unknown - } - - fn on_denied( - &self, - context: &OverlayRequestContext, - host: &str, - reason: &DenyReason, - ) { - if context.sink.debug_on { - debug::log_network_denied( - self.storage.as_path(), - host, - context.policy.mode_str(), - reason.as_str(), - &context.session_id, - ); - } - } - - fn on_dispatch( - &self, - context: &OverlayRequestContext, - request: &Request, - target: &str, - ) { - if context.sink.debug_on { - debug::log_dispatch( - self.storage.as_path(), - request.method().as_str(), - &request.uri().to_string(), - &context.session_id, - target, - ); - } - } -} diff --git a/crates/persisting-gateway/src/gateway/llm_capture.rs b/crates/persisting-gateway/src/gateway/llm_capture.rs deleted file mode 100644 index e966ca17d..000000000 --- a/crates/persisting-gateway/src/gateway/llm_capture.rs +++ /dev/null @@ -1,497 +0,0 @@ -//! LLM request/response capture handler (non-streaming path + upstream orchestration). - -use std::sync::Arc; - -use anyhow::Context; -use axum::body::{Body, to_bytes}; -use axum::extract::Request; -use axum::http::{Method, StatusCode}; -use axum::response::{IntoResponse, Response}; -use futures_util::StreamExt; -use persisting_agentctl::{ModelCallRequest, RunId, StorylineId}; -use serde_json::Value; - -use super::auth::{apply_upstream_headers, resolve_upstream_api_key}; -use super::common::{ - attach_capture_headers, call_context, extract_model, is_models_list_path, model_access_policy, -}; -use super::models_list::build_models_response; -use super::router::resolve_route; -use super::state::GatewayState; -use super::streaming::{should_stream_to_client, streaming_llm_response}; -use super::upstream::prepare_upstream_body; -use crate::Call; -use crate::config::ProxyConfig; -use crate::conversion::{ - MAX_REQUEST_BODY_BYTES, MAX_RESPONSE_BODY_BYTES, ProtocolBridge, translate_error_for_bridge, - translate_response_for_bridge, -}; -use crate::engine::headers_to_vec; -use crate::engine::{CompleteEvent, Event, RequestEvent}; -use crate::protocol::ProtocolKind; -use crate::runtime::debug::{self, truncate_body_bytes}; -use crate::session::storage::resolve_capture_route; -use crate::understanding::understand_request; -use persisting_overlaynet::headers::{is_websocket_upgrade, skip_response_header_after_reframing}; - -fn client_request_url(parts: &axum::http::request::Parts) -> String { - let path_and_query = parts - .uri - .path_and_query() - .map(|pq| pq.as_str()) - .unwrap_or(parts.uri.path()); - if let Some(host) = parts - .headers - .get(axum::http::header::HOST) - .and_then(|v| v.to_str().ok()) - { - // Scheme is not visible on the proxy request; record as authority+path for correlation. - format!("//{host}{path_and_query}") - } else { - path_and_query.to_string() - } -} - -fn http_version_label(version: axum::http::Version) -> String { - match version { - axum::http::Version::HTTP_09 => "HTTP/0.9".into(), - axum::http::Version::HTTP_10 => "HTTP/1.0".into(), - axum::http::Version::HTTP_11 => "HTTP/1.1".into(), - axum::http::Version::HTTP_2 => "HTTP/2".into(), - axum::http::Version::HTTP_3 => "HTTP/3".into(), - other => format!("{other:?}"), - } -} - -pub(super) async fn llm_capture( - state: GatewayState, - req: Request, - peer: std::net::SocketAddr, - cfg: Arc, - debug_on: bool, -) -> anyhow::Result { - let (parts, body) = req.into_parts(); - - if is_websocket_upgrade(&parts.headers) { - return Ok(Response::builder() - .status(StatusCode::NOT_IMPLEMENTED) - .header("content-type", "text/plain; charset=utf-8") - .body(Body::from( - "WebSocket transport is not supported by the capture gateway; use HTTPS", - )) - .map_err(|e| anyhow::anyhow!("websocket rejection response: {e}"))? - .into_response()); - } - - let body_bytes = match to_bytes(body, MAX_REQUEST_BODY_BYTES).await { - Ok(body) => body, - Err(error) => { - tracing::warn!( - target: "persisting_gateway", - %error, - limit = MAX_REQUEST_BODY_BYTES, - "rejecting oversized LLM request body" - ); - return Ok(( - StatusCode::PAYLOAD_TOO_LARGE, - format!("LLM request body exceeds {MAX_REQUEST_BODY_BYTES} bytes"), - ) - .into_response()); - } - }; - let path = parts.uri.path().to_string(); - let method = parts.method.clone(); - let protocol = ProtocolKind::from_path(&path); - - let capture_route = resolve_capture_route( - &parts.headers, - &body_bytes, - &state.config.session_header, - state.storage.as_path(), - ); - let call = Call::from_headers(&parts.headers); - let agent_id = cfg.agent_id.clone(); - let session_id = capture_route.session_id.clone(); - - if method == Method::GET && is_models_list_path(&path) { - let json = build_models_response(&cfg); - let bytes = serde_json::to_vec(&json).context("serialize /v1/models")?; - return Ok(Response::builder() - .status(StatusCode::OK) - .header("content-type", "application/json") - .body(Body::from(bytes)) - .context("build /v1/models response")? - .into_response()); - } - - let client_meta = - state - .session_clients - .ensure(state.storage.as_path(), &agent_id, &capture_route, peer); - - // Understand the untouched client request once. Routing, rendering, and live capture below - // share this typed value; WAL retains the pre-rewrite JSON for crash replay. - let mut parsed_request = understand_request(protocol, &body_bytes).ok(); - let stream_request = parsed_request - .as_ref() - .map(|parsed| parsed.semantic.request.stream) - .unwrap_or_else(|| super::streaming::request_wants_stream(&body_bytes)) - || path.ends_with(":streamGenerateContent"); - let client_model = parsed_request - .as_ref() - .and_then(|parsed| parsed.semantic.request.model.clone()) - .or_else(|| extract_model(&body_bytes)) - .or_else(|| gemini_model_from_path(&path)) - .unwrap_or_else(|| "_unknown".to_string()); - if let Some(parsed) = parsed_request.as_mut() { - let semantic = Arc::make_mut(&mut parsed.semantic); - semantic - .request - .model - .get_or_insert_with(|| client_model.clone()); - semantic.request.stream = stream_request; - } - let resolved = resolve_route(&cfg.models, &client_model)?; - let route = resolved.route; - let upstream_model = resolved.upstream_model.clone(); - let bridge = ProtocolBridge::needed(protocol, route); - let upstream_protocol = bridge.upstream_protocol(protocol); - let provider = route.effective_provider(upstream_protocol); - - // Wrap once in Arc so the request, response (or stream draft), and final events - // all share a single allocation; clones become refcount bumps. - let mut ctx = call_context( - &capture_route, - &agent_id, - &call, - &parts.headers, - &client_model, - &upstream_model, - provider, - protocol, - cfg.capture_level, - debug_on, - ); - ctx.attach_client(peer, client_meta); - ctx.attach_http_version(http_version_label(parts.version)); - let mut call_ctx: Arc<_> = Arc::new(ctx); - - { - let user_content = parsed_request - .as_ref() - .and_then(|parsed| parsed.latest_visible_user_content.clone()); - let body_json = parsed_request - .as_ref() - .map(|parsed| parsed.body_json.clone()) - .or_else(|| serde_json::from_slice::(&body_bytes).ok()); - let semantic = parsed_request - .as_ref() - .map(|parsed| Arc::clone(&parsed.semantic)); - state.capture_engine.spawn_apply( - Arc::clone(&call_ctx), - Event::Request(RequestEvent { - path: path.clone(), - method: method.as_str().to_string(), - url: Some(client_request_url(&parts)), - body_bytes: body_bytes.len(), - user_content, - body_json, - semantic, - model_rewritten: resolved.model_rewritten, - headers: headers_to_vec(&parts.headers), - }), - ); - } - - let upstream_body = prepare_upstream_body( - &body_bytes, - parsed_request - .as_ref() - .map(|parsed| parsed.semantic.as_ref()), - resolved.model_rewritten, - &upstream_model, - bridge, - Some(state.reasoning_cache.as_ref()), - )?; - - let upstream_path = bridge.upstream_path(&path, &upstream_model, stream_request)?; - let mut upstream_url = route.resolve_upstream_url(&upstream_path, upstream_protocol)?; - if bridge == ProtocolBridge::Passthrough { - if let Some(q) = parts.uri.query() { - upstream_url.set_query(Some(q)); - } - } else if matches!( - bridge, - ProtocolBridge::CompletionsToGemini - | ProtocolBridge::MessagesToGemini - | ProtocolBridge::ResponsesToGemini - ) && stream_request - { - upstream_url.query_pairs_mut().append_pair("alt", "sse"); - } else if let Some(q) = parts.uri.query() { - upstream_url.set_query(Some(q)); - } - - { - let mut ctx = (*call_ctx).clone(); - ctx.attach_upstream_url(upstream_url.as_str()); - call_ctx = Arc::new(ctx); - } - - let model_policy = model_access_policy(&cfg); - let model_request = ModelCallRequest { - run_id: capture_route.root_session.clone().map(RunId::new), - attempt_id: None, - storyline_id: Some(StorylineId::new(capture_route.session_id.clone())), - call_id: call.call_id.clone(), - client_model: client_model.clone(), - upstream_model: upstream_model.clone(), - provider: provider.as_str().to_string(), - protocol: protocol.as_str().to_string(), - upstream_host: upstream_url.host_str().unwrap_or_default().to_string(), - }; - let mut control = persisting_agentctl::ControlMachine::new(); - let control_transition = control - .authorize( - state.control_controller.as_ref(), - persisting_agentctl::ControlRequest::Model { - policy: &model_policy, - request: &model_request, - }, - ) - .expect("policy controller must return a valid authorization transition"); - if !control_transition.is_allowed() { - let reason = control_transition.reason; - let _applied_control = control - .applied() - .expect("a denied model control transition can be applied"); - tracing::warn!( - target: "persisting_gateway", - run_id = capture_route.root_session.as_deref().unwrap_or("-"), - storyline_id = %capture_route.session_id, - call_id = %call.call_id, - client_model = %client_model, - upstream_model = %upstream_model, - provider = provider.as_str(), - reason = reason.code(), - "pVisor denied model call" - ); - return Ok(( - StatusCode::FORBIDDEN, - format!( - "persisting-gateway gateway: pVisor denied model `{client_model}` ({})", - reason.code() - ), - ) - .into_response()); - } - let _applied_control = control - .applied() - .expect("an allowed model control transition can be applied"); - - if debug_on { - let body_preview = truncate_body_bytes(&upstream_body); - debug::log_llm_request( - state.storage.as_path(), - &session_id, - &agent_id, - &client_model, - protocol.as_str(), - &path, - upstream_url.as_str(), - &body_preview, - ); - } - - let (_, auth_source) = match resolve_upstream_api_key(route, &parts.headers) { - Ok(v) => v, - Err(e) => { - if debug_on { - debug::log_llm_upstream_error( - state.storage.as_path(), - &session_id, - &agent_id, - &client_model, - upstream_url.as_str(), - &format!("auth: {e:#}"), - ); - } - return Err(e); - } - }; - if debug_on { - debug::log_llm_auth_resolved(state.storage.as_path(), &session_id, auth_source); - } - - let mut upstream_req = state.client.request(method, upstream_url.clone()); - upstream_req = upstream_req.body(upstream_body.clone()); - upstream_req = - match apply_upstream_headers(upstream_req, &parts.headers, route, upstream_protocol) { - Ok(r) => r, - Err(e) => { - if debug_on { - debug::log_llm_upstream_error( - state.storage.as_path(), - &session_id, - &agent_id, - &client_model, - upstream_url.as_str(), - &format!("auth: {e:#}"), - ); - } - return Err(e); - } - }; - if debug_on { - debug::log_llm_upstream_sending( - state.storage.as_path(), - &session_id, - upstream_url.as_str(), - ); - } - - let upstream_resp = match upstream_req.send().await { - Ok(r) => r, - Err(e) => { - if debug_on { - debug::log_llm_upstream_error( - state.storage.as_path(), - &session_id, - &agent_id, - &client_model, - upstream_url.as_str(), - &e.to_string(), - ); - } - return Err(anyhow::anyhow!("upstream request: {e}")); - } - }; - let status = upstream_resp.status(); - let resp_headers = upstream_resp.headers().clone(); - if debug_on { - let content_type = resp_headers - .get("content-type") - .and_then(|v| v.to_str().ok()) - .unwrap_or("-"); - debug::log_llm_upstream_headers( - state.storage.as_path(), - &session_id, - &agent_id, - &client_model, - upstream_url.as_str(), - status.as_u16(), - content_type, - stream_request, - ); - } - - if !status.is_success() { - let raw_error = read_response_body_limited(upstream_resp, MAX_RESPONSE_BODY_BYTES).await?; - let client_error = translate_error_for_bridge(bridge, &raw_error, status)?; - let body_was_rewritten = client_error != raw_error; - state.capture_engine.spawn_apply( - Arc::clone(&call_ctx), - Event::ResponseComplete(CompleteEvent { - status: status.as_u16(), - resp_bytes: client_error.clone(), - streaming: false, - stream_metrics: None, - assistant_content: None, - semantic: None, - headers: headers_to_vec(&resp_headers), - }), - ); - let mut builder = Response::builder().status(status); - for (name, value) in &resp_headers { - if skip_response_header_after_reframing( - &resp_headers, - name.as_str(), - body_was_rewritten, - ) { - continue; - } - builder = builder.header(name, value); - } - if body_was_rewritten { - builder = builder.header("content-type", "application/json"); - } - builder = attach_capture_headers(builder, &call); - return Ok(builder - .body(Body::from(client_error)) - .map_err(|e| anyhow::anyhow!("error response body: {e}"))? - .into_response()); - } - - if should_stream_to_client(&resp_headers, &body_bytes) { - // streaming_llm_response takes an owned CallContext so unwrap the Arc when - // we know we're the only owner (we are — request emit was the only earlier clone). - let owned_ctx = Arc::try_unwrap(call_ctx).unwrap_or_else(|arc| (*arc).clone()); - return streaming_llm_response(upstream_resp, state, owned_ctx, bridge).await; - } - - let upstream_bytes = read_response_body_limited(upstream_resp, MAX_RESPONSE_BODY_BYTES).await?; - let body_was_rewritten = bridge.needs_response_translation(); - let translated = - translate_response_for_bridge(bridge, &upstream_bytes, protocol, &client_model)?; - let resp_bytes = translated.body; - state.capture_engine.spawn_apply( - Arc::clone(&call_ctx), - Event::ResponseComplete(CompleteEvent { - status: status.as_u16(), - resp_bytes: resp_bytes.clone(), - streaming: false, - stream_metrics: None, - assistant_content: None, - semantic: Some(translated.semantic), - headers: headers_to_vec(&resp_headers), - }), - ); - - let mut builder = Response::builder().status(status); - // The upstream body has been fully consumed and rebuilt even for protocol - // passthrough. Drop hop-by-hop framing and the stale content length on every - // buffered response. If translation changed the payload, also drop encoding - // and content type before installing the translated media type. - for (name, value) in resp_headers.iter() { - if skip_response_header_after_reframing(&resp_headers, name.as_str(), body_was_rewritten) { - continue; - } - builder = builder.header(name, value); - } - if body_was_rewritten { - builder = builder.header("content-type", "application/json"); - } - builder = attach_capture_headers(builder, &call); - Ok(builder - .body(Body::from(resp_bytes)) - .map_err(|e| anyhow::anyhow!("response body: {e}"))? - .into_response()) -} - -fn gemini_model_from_path(path: &str) -> Option { - let model = path.split("/models/").nth(1)?.split(':').next()?; - (!model.is_empty()).then(|| model.to_string()) -} - -async fn read_response_body_limited( - response: reqwest::Response, - limit: usize, -) -> anyhow::Result { - if response - .content_length() - .is_some_and(|content_length| content_length > limit as u64) - { - anyhow::bail!("upstream response body exceeds {limit} bytes"); - } - let mut body = bytes::BytesMut::new(); - let mut stream = response.bytes_stream(); - while let Some(chunk) = stream.next().await { - let chunk = chunk.context("read upstream response body")?; - anyhow::ensure!( - body.len().saturating_add(chunk.len()) <= limit, - "upstream response body exceeds {limit} bytes" - ); - body.extend_from_slice(&chunk); - } - Ok(body.freeze()) -} diff --git a/crates/persisting-gateway/src/gateway/mod.rs b/crates/persisting-gateway/src/gateway/mod.rs deleted file mode 100644 index afca885fd..000000000 --- a/crates/persisting-gateway/src/gateway/mod.rs +++ /dev/null @@ -1,22 +0,0 @@ -//! LLM protocol adaptation and trajectory capture on top of overlaynet. - -mod admin; -mod auth; -mod common; -mod dispatch; -mod llm_capture; -mod model; -pub mod models_list; -mod overlaynet_config; -mod reasoning; -mod router; -mod state; -mod streaming; -mod upstream; - -pub(crate) use reasoning::ReasoningCacheHandle; -pub(crate) use state::{GatewayRuntimeControl, serve_with_runtime_control_and_metrics}; -pub use state::{ - serve, serve_with_listeners_and_shutdown, serve_with_runtime_control, serve_with_shutdown, - serve_with_shutdown_and_ready, -}; diff --git a/crates/persisting-gateway/src/gateway/model.rs b/crates/persisting-gateway/src/gateway/model.rs deleted file mode 100644 index a3cda33b8..000000000 --- a/crates/persisting-gateway/src/gateway/model.rs +++ /dev/null @@ -1,43 +0,0 @@ -//! LLM request-body model rewrite for `forward` routes. - -use anyhow::{Context, Result, bail}; -use bytes::Bytes; -use serde_json::Value; - -/// Replace JSON `model` field before sending upstream. -pub fn rewrite_model_in_body(body: &Bytes, upstream_model: &str) -> Result { - if body.is_empty() { - return Ok(Bytes::new()); - } - let mut v: Value = serde_json::from_slice(body).context("parse request JSON")?; - let Some(obj) = v.as_object_mut() else { - bail!("request body must be a JSON object to rewrite model"); - }; - obj.insert( - "model".to_string(), - Value::String(upstream_model.to_string()), - ); - Ok(Bytes::from( - serde_json::to_vec(&v).context("serialize request JSON")?, - )) -} - -#[cfg(test)] -mod tests { - use super::*; - use serde_json::Value; - - #[test] - fn rewrite_model_in_request_body() { - let body = Bytes::from_static(br#"{"model":"claude-3","messages":[]}"#); - let out = rewrite_model_in_body(&body, "deepseek-chat").unwrap(); - let v: Value = serde_json::from_slice(&out).unwrap(); - assert_eq!(v["model"], "deepseek-chat"); - } - - #[test] - fn rewrite_model_empty_body_is_noop() { - let out = rewrite_model_in_body(&Bytes::new(), "deepseek-chat").unwrap(); - assert!(out.is_empty()); - } -} diff --git a/crates/persisting-gateway/src/gateway/models_list.rs b/crates/persisting-gateway/src/gateway/models_list.rs deleted file mode 100644 index 1c415166c..000000000 --- a/crates/persisting-gateway/src/gateway/models_list.rs +++ /dev/null @@ -1,104 +0,0 @@ -//! OpenAI-compatible `/v1/models` view of the capture gateway config. - -use serde_json::{Value, json}; - -use super::router::model_matches; -use crate::config::{ModelRoute, ProxyConfig}; - -/// Build model ids exposed to clients (OpenAI list format). -pub fn build_models_response(cfg: &ProxyConfig) -> Value { - let mut ids = Vec::new(); - for route in &cfg.models { - collect_model_ids(route, &cfg.models, &mut ids); - } - ids.sort(); - ids.dedup(); - let created = chrono::Utc::now().timestamp(); - let data = ids - .into_iter() - .map(|id| { - json!({ - "id": id, - "object": "model", - "created": created, - "owned_by": cfg.agent_id, - }) - }) - .collect::>(); - json!({ - "object": "list", - "data": data, - }) -} - -fn collect_model_ids(route: &ModelRoute, all: &[ModelRoute], out: &mut Vec) { - if route.upstream.is_some() { - push_id(out, &route.name); - } - if route.forward.is_some() { - return; - } - if route.upstream.is_none() { - return; - } - // Expose match patterns as discoverable ids when not wildcard-only routing entry. - if route.name.contains('*') { - for other in all { - if other.name == route.name { - continue; - } - if !other.name.contains('*') && other.upstream.is_some() { - push_id(out, &other.name); - } - } - } -} - -fn push_id(out: &mut Vec, id: &str) { - if id == "*" || id.contains('*') { - return; - } - out.push(id.to_string()); -} - -/// Match patterns that would answer for a given model id (for stub validation). -pub fn config_lists_model(cfg: &ProxyConfig, model_id: &str) -> bool { - cfg.models.iter().any(|r| { - if r.name == model_id && r.upstream.is_some() { - return true; - } - model_matches(&r.name, model_id) && r.upstream.is_some() - }) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::config::ProxyConfig; - - #[test] - fn models_list_from_config() { - let cfg = ProxyConfig::from_toml_str( - r#" -listen = "127.0.0.1:1" - -[[models]] -name = "deepseek-chat" -upstream = "http://x/v1" - -[[models]] -name = "*" -forward = "deepseek-chat" -"#, - ) - .unwrap(); - let v = build_models_response(&cfg); - let ids: Vec<_> = v["data"] - .as_array() - .unwrap() - .iter() - .map(|m| m["id"].as_str().unwrap()) - .collect(); - assert!(ids.contains(&"deepseek-chat")); - } -} diff --git a/crates/persisting-gateway/src/gateway/overlaynet_config.rs b/crates/persisting-gateway/src/gateway/overlaynet_config.rs deleted file mode 100644 index 4da561543..000000000 --- a/crates/persisting-gateway/src/gateway/overlaynet_config.rs +++ /dev/null @@ -1,11 +0,0 @@ -//! Adapt capture configuration to overlaynet's egress policy. - -pub use persisting_overlaynet::policy::*; - -use crate::config::ProxyConfig; - -impl PolicyConfig for ProxyConfig { - fn network(&self) -> &crate::config::NetworkConfig { - &self.network - } -} diff --git a/crates/persisting-gateway/src/gateway/reasoning.rs b/crates/persisting-gateway/src/gateway/reasoning.rs deleted file mode 100644 index 2a86d36c3..000000000 --- a/crates/persisting-gateway/src/gateway/reasoning.rs +++ /dev/null @@ -1,112 +0,0 @@ -//! DeepSeek Chat Completions multi-turn tool-call reasoning replay. - -use std::collections::HashMap; -use std::sync::Mutex; - -use serde_json::{Value, json}; - -/// Per-session cache: tool_call_id → reasoning_content to echo on follow-up requests. -#[derive(Debug, Default)] -pub struct ReasoningCache { - by_tool_call: HashMap, -} - -impl ReasoningCache { - pub fn remember(&mut self, tool_call_ids: &[String], reasoning: &str) { - if reasoning.is_empty() { - return; - } - for id in tool_call_ids { - if id.is_empty() { - continue; - } - self.by_tool_call.insert(id.clone(), reasoning.to_string()); - } - } - - pub fn get(&self, tool_call_id: &str) -> Option<&str> { - self.by_tool_call.get(tool_call_id).map(String::as_str) - } -} - -#[derive(Debug, Default)] -pub struct ReasoningCacheHandle { - inner: Mutex, -} - -impl ReasoningCacheHandle { - pub fn new() -> Self { - Self::default() - } - - pub fn remember(&self, tool_call_ids: &[String], reasoning: &str) { - if let Ok(mut g) = self.inner.lock() { - g.remember(tool_call_ids, reasoning); - } - } - - pub fn apply_to_messages(&self, messages: &mut [Value]) { - let Ok(cache) = self.inner.lock() else { - return; - }; - apply_deepseek_message_fixup(messages, &cache); - } -} - -/// Add cached or empty `reasoning_content` on assistant messages with `tool_calls`. -pub fn apply_deepseek_message_fixup(messages: &mut [Value], cache: &ReasoningCache) { - for msg in messages.iter_mut() { - let Some(obj) = msg.as_object_mut() else { - continue; - }; - if obj.get("role").and_then(|r| r.as_str()) != Some("assistant") { - continue; - } - let Some(tool_calls) = obj.get("tool_calls").and_then(|t| t.as_array()) else { - continue; - }; - if tool_calls.is_empty() { - continue; - } - if obj - .get("reasoning_content") - .and_then(|v| v.as_str()) - .is_some_and(|s| !s.is_empty()) - { - continue; - } - for tc in tool_calls { - let Some(id) = tc.get("id").and_then(|v| v.as_str()) else { - continue; - }; - if let Some(reasoning) = cache.get(id) { - obj.insert("reasoning_content".into(), json!(reasoning)); - break; - } - } - if !obj.contains_key("reasoning_content") { - obj.insert("reasoning_content".into(), json!("")); - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn injects_empty_reasoning_for_tool_call_assistant() { - let mut cache = ReasoningCache::default(); - let mut msgs = vec![json!({ - "role": "assistant", - "content": null, - "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "shell", "arguments": "{}"}}] - })]; - apply_deepseek_message_fixup(&mut msgs, &cache); - assert_eq!(msgs[0]["reasoning_content"], ""); - - cache.remember(&["call_1".into()], "thinking..."); - apply_deepseek_message_fixup(&mut msgs, &cache); - assert_eq!(msgs[0]["reasoning_content"], "thinking..."); - } -} diff --git a/crates/persisting-gateway/src/gateway/router.rs b/crates/persisting-gateway/src/gateway/router.rs deleted file mode 100644 index 77c4744d7..000000000 --- a/crates/persisting-gateway/src/gateway/router.rs +++ /dev/null @@ -1,178 +0,0 @@ -//! LLM model name → route selection and `forward` resolution. - -use anyhow::{Result, bail}; - -use crate::config::ModelRoute; - -/// Resolved upstream route after optional `forward`. -pub struct ResolvedRoute<'a> { - pub upstream_model: String, - pub route: &'a ModelRoute, - /// Request body `model` differs from client (via `forward`). - pub model_rewritten: bool, -} - -pub fn select_route<'a>(routes: &'a [ModelRoute], model: &str) -> Option<&'a ModelRoute> { - routes.iter().find(|r| model_matches(&r.name, model)) -} - -pub fn resolve_route<'a>( - routes: &'a [ModelRoute], - client_model: &str, -) -> Result> { - let matched = select_route(routes, client_model) - .ok_or_else(|| anyhow::anyhow!("no route for model {client_model}"))?; - - if let Some(ref target_name) = matched.forward { - let target = routes - .iter() - .find(|r| r.name == *target_name) - .ok_or_else(|| anyhow::anyhow!("forward target `{target_name}` not found"))?; - if target.forward.is_some() { - bail!("forward target `{target_name}` cannot forward again"); - } - if target.upstream.is_none() { - bail!("forward target `{target_name}` missing upstream"); - } - let upstream_model = target_name.clone(); - let model_rewritten = upstream_model != client_model; - return Ok(ResolvedRoute { - upstream_model, - route: target, - model_rewritten, - }); - } - - if matched.upstream.is_none() { - bail!( - "models[] entry `{}` has no upstream (set `upstream` or `forward`)", - matched.name - ); - } - - Ok(ResolvedRoute { - upstream_model: client_model.to_string(), - route: matched, - model_rewritten: false, - }) -} - -pub fn model_matches(pattern: &str, model: &str) -> bool { - if pattern == "*" { - return true; - } - if let Some(prefix) = pattern.strip_suffix('*') { - return !prefix.is_empty() && model.starts_with(prefix); - } - if let Some(suffix) = pattern.strip_prefix('*') { - return !suffix.is_empty() && model.ends_with(suffix); - } - pattern == model -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::config::ProxyConfig; - - fn target(name: &str, upstream: &str) -> ModelRoute { - ModelRoute { - name: name.into(), - provider: None, - upstream: Some(upstream.into()), - upstream_anthropic: None, - api_key_env: None, - api_key: None, - forward: None, - } - } - - fn forward_rule(match_pat: &str, to: &str) -> ModelRoute { - ModelRoute { - name: match_pat.into(), - provider: None, - upstream: None, - upstream_anthropic: None, - api_key_env: None, - api_key: None, - forward: Some(to.into()), - } - } - - #[test] - fn wildcard_star() { - assert!(model_matches("*", "gpt-4o")); - } - - #[test] - fn prefix_wildcard() { - assert!(model_matches("deepseek*", "deepseek-chat")); - assert!(!model_matches("deepseek*", "gpt-4")); - } - - #[test] - fn exact() { - assert!(model_matches("claude-3", "claude-3")); - assert!(!model_matches("claude-3", "claude-4")); - } - - #[test] - fn passthrough_uses_matched_upstream() { - let routes = vec![target("deepseek-chat", "http://ds/v1")]; - let r = resolve_route(&routes, "deepseek-chat").unwrap(); - assert!(!r.model_rewritten); - assert_eq!(r.upstream_model, "deepseek-chat"); - assert_eq!(r.route.upstream.as_deref(), Some("http://ds/v1")); - } - - #[test] - fn forward_rewrites_to_target() { - let routes = vec![ - target("deepseek-chat", "http://ds/v1"), - forward_rule("claude-*", "deepseek-chat"), - ]; - let r = resolve_route(&routes, "claude-sonnet-4").unwrap(); - assert!(r.model_rewritten); - assert_eq!(r.upstream_model, "deepseek-chat"); - assert_eq!(r.route.name, "deepseek-chat"); - } - - #[test] - fn first_match_wins() { - let routes = vec![ - target("deepseek-chat", "http://ds/v1"), - forward_rule("*", "deepseek-chat"), - ]; - let r = resolve_route(&routes, "deepseek-chat").unwrap(); - assert!(!r.model_rewritten); - } - - #[test] - fn config_validate_forward() { - let toml = r#" -listen = "127.0.0.1:1" - -[[models]] -name = "deepseek-chat" -upstream = "http://ds/v1" - -[[models]] -name = "claude-*" -forward = "deepseek-chat" -"#; - ProxyConfig::from_toml_str(toml).unwrap(); - } - - #[test] - fn config_rejects_forward_and_upstream() { - let toml = r#" -listen = "127.0.0.1:1" - -[[models]] -name = "bad" -upstream = "http://x/v1" -forward = "other" -"#; - assert!(ProxyConfig::from_toml_str(toml).is_err()); - } -} diff --git a/crates/persisting-gateway/src/gateway/state.rs b/crates/persisting-gateway/src/gateway/state.rs deleted file mode 100644 index b54740e3b..000000000 --- a/crates/persisting-gateway/src/gateway/state.rs +++ /dev/null @@ -1,318 +0,0 @@ -//! Gateway state and composition with OverlayNet. - -use std::path::Path; -use std::sync::Arc; -use std::sync::atomic::AtomicUsize; -use std::time::Duration; - -use anyhow::Context; -use persisting_agentctl::{ControlController, PolicyControlController}; -use persisting_overlaynet::{BandwidthRegistry, InterceptionMetrics}; -use tokio::task::JoinHandle; - -use super::admin::{AdminState, admin_router}; -use super::dispatch::build_router; -use super::reasoning::ReasoningCacheHandle; -use crate::config::ProxyConfig; -use crate::engine::CaptureEngine; -use crate::runtime::debug::{self, is_debug_enabled}; -use crate::session::client::SessionClientRegistry; -use crate::session::index::SessionIndexStore; -use crate::sink::CaptureEventSink; - -#[derive(Clone)] -pub(crate) struct GatewayState { - pub(crate) gateway_enabled: bool, - pub(crate) config: Arc, - pub(crate) storage: Arc, - pub(crate) client: reqwest::Client, - pub(crate) capture_engine: CaptureEngine, - pub(crate) session_clients: Arc, - pub(crate) reasoning_cache: Arc, - pub(crate) control_controller: Arc, - pub(crate) active_requests: Arc, - pub(crate) interception_metrics: InterceptionMetrics, - pub(crate) bandwidth_registry: BandwidthRegistry, - pub(crate) attempt_id: Option, -} - -pub(crate) struct GatewayRuntimeControl { - pub(crate) controller: Arc, - pub(crate) interception_metrics: InterceptionMetrics, - pub(crate) bandwidth_registry: BandwidthRegistry, - pub(crate) attempt_id: Option, - pub(crate) gateway_enabled: bool, -} - -pub async fn serve( - config: ProxyConfig, - storage: impl AsRef, - sink: Arc, - stream_markdown: bool, -) -> anyhow::Result<()> { - serve_with_shutdown( - config, - storage, - sink, - stream_markdown, - std::future::pending(), - ) - .await -} - -/// Run proxy until `shutdown` completes. Optionally signal bind readiness via `ready`. -pub async fn serve_with_shutdown( - config: ProxyConfig, - storage: impl AsRef, - sink: Arc, - stream_markdown: bool, - shutdown: impl std::future::Future + Send + 'static, -) -> anyhow::Result<()> { - serve_with_shutdown_and_ready(config, storage, sink, stream_markdown, None, shutdown).await -} - -pub async fn serve_with_shutdown_and_ready( - config: ProxyConfig, - storage: impl AsRef, - sink: Arc, - stream_markdown: bool, - ready: Option>, - shutdown: impl std::future::Future + Send + 'static, -) -> anyhow::Result<()> { - serve_with_runtime_control( - config, - storage, - sink, - stream_markdown, - Arc::new(PolicyControlController), - ready, - shutdown, - ) - .await -} - -/// Run the Gateway with already-bound proxy and admin listeners. -/// -/// Embedders can use this entry point to reserve every listener before -/// starting any service, report the resolved addresses for port `0`, and -/// coordinate one shutdown signal across multiple servers. -pub async fn serve_with_listeners_and_shutdown( - mut config: ProxyConfig, - storage: impl AsRef, - sink: Arc, - stream_markdown: bool, - listener: tokio::net::TcpListener, - admin_listener: tokio::net::TcpListener, - shutdown: impl std::future::Future + Send + 'static, -) -> anyhow::Result<()> { - config.listen = listener - .local_addr() - .context("read Gateway listen address")? - .to_string(); - config.admin_listen = admin_listener - .local_addr() - .context("read Gateway admin listen address")? - .to_string(); - config.validate()?; - serve_with_bound_listeners( - config, - storage, - sink, - stream_markdown, - GatewayRuntimeControl { - controller: Arc::new(PolicyControlController), - interception_metrics: InterceptionMetrics::default(), - bandwidth_registry: BandwidthRegistry::default(), - attempt_id: None, - gateway_enabled: true, - }, - listener, - admin_listener, - None, - shutdown, - ) - .await -} - -/// Gateway sink with an injected runtime control state controller. -/// -/// The caller supplies the controller; Gateway and OverlayNet apply model/network -/// transitions while retaining HTTP adaptation and trajectory extraction. -pub async fn serve_with_runtime_control( - config: ProxyConfig, - storage: impl AsRef, - sink: Arc, - stream_markdown: bool, - control_controller: Arc, - ready: Option>, - shutdown: impl std::future::Future + Send + 'static, -) -> anyhow::Result<()> { - serve_with_runtime_control_and_metrics( - config, - storage, - sink, - stream_markdown, - GatewayRuntimeControl { - controller: control_controller, - interception_metrics: InterceptionMetrics::default(), - bandwidth_registry: BandwidthRegistry::default(), - attempt_id: None, - gateway_enabled: true, - }, - ready, - shutdown, - ) - .await -} - -pub(crate) async fn serve_with_runtime_control_and_metrics( - config: ProxyConfig, - storage: impl AsRef, - sink: Arc, - stream_markdown: bool, - runtime_control: GatewayRuntimeControl, - ready: Option>, - shutdown: impl std::future::Future + Send + 'static, -) -> anyhow::Result<()> { - let admin_listen: std::net::SocketAddr = config.admin_listen.parse().with_context(|| { - format!( - "invalid capture admin listen address `{}`", - config.admin_listen - ) - })?; - let listen: std::net::SocketAddr = config - .listen - .parse() - .with_context(|| format!("invalid overlaynet listen address `{}`", config.listen))?; - let admin_listener = tokio::net::TcpListener::bind(admin_listen) - .await - .with_context(|| format!("bind capture admin API on {admin_listen}"))?; - let listener = tokio::net::TcpListener::bind(listen) - .await - .with_context(|| format!("bind overlaynet gateway on {listen}"))?; - serve_with_bound_listeners( - config, - storage, - sink, - stream_markdown, - runtime_control, - listener, - admin_listener, - ready, - shutdown, - ) - .await -} - -#[allow(clippy::too_many_arguments)] -async fn serve_with_bound_listeners( - config: ProxyConfig, - storage: impl AsRef, - sink: Arc, - stream_markdown: bool, - runtime_control: GatewayRuntimeControl, - listener: tokio::net::TcpListener, - admin_listener: tokio::net::TcpListener, - ready: Option>, - shutdown: impl std::future::Future + Send + 'static, -) -> anyhow::Result<()> { - let (stop_tx, stop_rx) = tokio::sync::watch::channel(()); - tokio::spawn(async move { - shutdown.await; - let _ = stop_tx.send(()); - }); - - let listen = listener - .local_addr() - .context("read Gateway listen address")?; - let admin_listen = admin_listener - .local_addr() - .context("read Gateway admin listen address")?; - let storage = Arc::new(storage.as_ref().to_path_buf()); - let index_store = SessionIndexStore::open(storage.as_path())?; - let index = index_store.clone_handle(); - let started_at = chrono::Utc::now().to_rfc3339(); - - if is_debug_enabled(&config, storage.as_path()) { - tracing::debug!( - target: "persisting_gateway", - "capture debug → {}", - debug::debug_log_path(storage.as_path()).display() - ); - debug::log_daemon_start(storage.as_path(), &config.listen, env!("CARGO_PKG_VERSION")); - } - - let active_requests = Arc::new(AtomicUsize::new(0)); - let interception_metrics = runtime_control.interception_metrics; - let capture_engine = CaptureEngine::new( - Arc::clone(&sink), - index.clone(), - Arc::clone(&storage), - stream_markdown, - ) - .await?; - let capture_for_shutdown = capture_engine.clone(); - let state = GatewayState { - config: Arc::new(config.clone()), - storage, - client: reqwest::Client::builder() - .no_proxy() - // Redirects must return to the proxy client so every destination - // gets a fresh OverlayNet authorization decision. Following a - // cross-origin Location here would bypass the policy gate. - .redirect(reqwest::redirect::Policy::none()) - .connect_timeout(Duration::from_secs(10)) - .timeout(Duration::from_secs(600)) - .build()?, - capture_engine, - session_clients: Arc::new(SessionClientRegistry::default()), - reasoning_cache: Arc::new(ReasoningCacheHandle::new()), - control_controller: runtime_control.controller, - active_requests: Arc::clone(&active_requests), - interception_metrics: interception_metrics.clone(), - bandwidth_registry: runtime_control.bandwidth_registry, - attempt_id: runtime_control.attempt_id, - gateway_enabled: runtime_control.gateway_enabled, - }; - - let admin_state = AdminState { - index, - listen: config.listen.clone(), - admin_listen: config.admin_listen.clone(), - started_at, - active_requests, - interception_metrics, - }; - let admin_app = admin_router(admin_state); - let admin_shutdown = wait_shutdown(stop_rx.clone()); - let app = build_router(state); - - let admin_handle: JoinHandle<()> = tokio::spawn(async move { - tracing::debug!(target: "persisting_gateway", "capture admin API on http://{admin_listen}"); - if let Err(error) = axum::serve(admin_listener, admin_app) - .with_graceful_shutdown(admin_shutdown) - .await - { - tracing::warn!(target: "persisting_gateway", %error, "capture admin API stopped"); - } - }); - - if let Some(tx) = ready { - let _ = tx.send(()); - } - tracing::debug!(target: "persisting_gateway", "capture LLM proxy on http://{listen}"); - let serve_result = axum::serve( - listener, - app.into_make_service_with_connect_info::(), - ) - .with_graceful_shutdown(wait_shutdown(stop_rx)) - .await; - admin_handle.abort(); - let shutdown_result = capture_for_shutdown.shutdown().await; - serve_result.context("serve overlaynet gateway")?; - shutdown_result -} - -async fn wait_shutdown(mut rx: tokio::sync::watch::Receiver<()>) { - let _ = rx.changed().await; -} diff --git a/crates/persisting-gateway/src/gateway/streaming.rs b/crates/persisting-gateway/src/gateway/streaming.rs deleted file mode 100644 index 7f0175aa7..000000000 --- a/crates/persisting-gateway/src/gateway/streaming.rs +++ /dev/null @@ -1,336 +0,0 @@ -//! SSE streaming upstream response: translate, forward, and capture drafts/final. - -use std::sync::Arc; -use std::time::Duration; - -use axum::body::Body; -use axum::http::HeaderMap; -use axum::response::{IntoResponse, Response}; -use bytes::{Bytes, BytesMut}; -use futures_util::StreamExt; -use serde_json::Value; -use tokio_stream::wrappers::ReceiverStream; - -use super::common::attach_capture_headers; -use super::state::GatewayState; -use crate::conversion::{MAX_STREAM_CAPTURE_BYTES, ProtocolBridge, StreamTranslator}; -use crate::engine::{ - CallContext, CancelEvent, CaptureEngine, CompleteEvent, DraftEvent, Event, headers_to_vec, -}; -use crate::runtime::debug; -use persisting_overlaynet::headers::skip_response_header_after_reframing; - -const STREAM_DRAFT_MD_INTERVAL: Duration = Duration::from_millis(150); -/// Bounded queue between upstream reader and client SSE writer (backpressure). -const STREAM_CLIENT_QUEUE: usize = 256; - -pub(super) fn request_wants_stream(body: &Bytes) -> bool { - serde_json::from_slice::(body) - .ok() - .and_then(|v| v.get("stream").and_then(|s| s.as_bool())) - .unwrap_or(false) -} - -pub(super) fn should_stream_to_client(headers: &HeaderMap, request_body: &Bytes) -> bool { - if request_wants_stream(request_body) { - return true; - } - headers - .get("content-type") - .and_then(|v| v.to_str().ok()) - .map(|ct| ct.contains("text/event-stream")) - .unwrap_or(false) -} - -pub(super) async fn streaming_llm_response( - upstream_resp: reqwest::Response, - state: GatewayState, - ctx: CallContext, - bridge: ProtocolBridge, -) -> anyhow::Result { - let status = upstream_resp.status(); - let resp_headers = upstream_resp.headers().clone(); - let recorded_resp_headers = headers_to_vec(&resp_headers); - let translate = bridge.needs_response_translation(); - let session_id = ctx.route().session_id.clone(); - let agent_id = ctx.agent_id().to_string(); - let client_model = ctx.client_model.clone(); - let debug_on = ctx.debug_on; - - if debug_on { - debug::log_llm_stream_start( - state.storage.as_path(), - &session_id, - &agent_id, - &client_model, - status.as_u16(), - ); - } - - let byte_stream = upstream_resp.bytes_stream(); - let (tx, rx) = tokio::sync::mpsc::channel::>(STREAM_CLIENT_QUEUE); - - let capture_engine = state.capture_engine.clone(); - // Share a single Arc across the streaming task and every emitted - // draft / final / cancel event, so the per-chunk `spawn_apply` only does an - // `Arc::clone` (refcount bump) instead of a deep `CallContext::clone` (Vec<(String,String)> - // headers + several String fields). - let ctx_bg: Arc = Arc::new(ctx.clone()); - let storage = Arc::clone(&state.storage); - let reasoning_cache = Arc::clone(&state.reasoning_cache); - - tokio::spawn(async move { - let mut buf = BytesMut::new(); - let mut translator = StreamTranslator::new(bridge, ctx_bg.protocol, &ctx_bg.client_model); - let mut last_draft_at = std::time::Instant::now(); - let mut last_draft_content = String::new(); - let mut client_disconnected = false; - let mut upstream_failed = false; - let mut stream = byte_stream; - while let Some(item) = stream.next().await { - match item { - Ok(chunk) => { - let retained = translator - .as_ref() - .map(|translator| translator.upstream_snapshot().len()) - .unwrap_or(buf.len()); - if retained.saturating_add(chunk.len()) > MAX_STREAM_CAPTURE_BYTES { - upstream_failed = true; - let message = format!( - "stream response exceeds durable capture limit of {MAX_STREAM_CAPTURE_BYTES} bytes" - ); - let _ = tx.send(Err(message.clone())).await; - if debug_on { - debug::log_llm_upstream_error( - storage.as_path(), - &ctx_bg.route().session_id, - ctx_bg.agent_id(), - &ctx_bg.client_model, - "stream", - &message, - ); - } - break; - } - if translator.is_none() { - buf.extend_from_slice(&chunk); - } - let out = if let Some(t) = translator.as_mut() { - match t.push_chunk(&chunk) { - Ok(bytes) if !bytes.is_empty() => bytes, - Ok(_) => { - maybe_emit_stream_draft( - &capture_engine, - &ctx_bg, - status.as_u16(), - t, - &mut last_draft_at, - &mut last_draft_content, - ); - continue; - } - Err(e) => { - tracing::warn!("stream translate: {e:#}"); - upstream_failed = true; - let message = format!("stream translation failed: {e:#}"); - let _ = tx.send(Err(message)).await; - break; - } - } - } else { - chunk - }; - if let Some(t) = translator.as_ref() { - maybe_emit_stream_draft( - &capture_engine, - &ctx_bg, - status.as_u16(), - t, - &mut last_draft_at, - &mut last_draft_content, - ); - } - if tx.send(Ok(out)).await.is_err() { - client_disconnected = true; - break; - } - } - Err(e) => { - upstream_failed = true; - let msg = e.to_string(); - let _ = tx.send(Err(msg.clone())).await; - if debug_on { - debug::log_llm_upstream_error( - storage.as_path(), - &ctx_bg.route().session_id, - ctx_bg.agent_id(), - &ctx_bg.client_model, - "stream", - &msg, - ); - } - break; - } - } - } - - if client_disconnected { - let bytes_received = translator - .as_ref() - .map(|translator| translator.upstream_snapshot().len()) - .unwrap_or(buf.len()); - capture_engine.spawn_apply( - Arc::clone(&ctx_bg), - Event::Cancelled(CancelEvent { - status: status.as_u16(), - bytes_received, - streaming: true, - }), - ); - return; - } - - if upstream_failed { - let bytes_received = translator - .as_ref() - .map(|translator| translator.upstream_snapshot().len()) - .unwrap_or(buf.len()); - capture_engine.spawn_apply( - Arc::clone(&ctx_bg), - Event::Cancelled(CancelEvent { - status: status.as_u16(), - bytes_received, - streaming: true, - }), - ); - return; - } - - let stream_metrics = translator.as_ref().map(|t| t.metrics().clone()); - let mut stream_semantic = None; - if let Some(t) = translator.as_mut() { - match t.finish_stream() { - Ok(tail) => { - if !tail.is_empty() && tx.send(Ok(tail)).await.is_err() { - capture_engine.spawn_apply( - Arc::clone(&ctx_bg), - Event::Cancelled(CancelEvent { - status: status.as_u16(), - bytes_received: t.upstream_snapshot().len(), - streaming: true, - }), - ); - return; - } - } - Err(error) => { - let _ = tx - .send(Err(format!("finish stream translation: {error:#}"))) - .await; - capture_engine.spawn_apply( - Arc::clone(&ctx_bg), - Event::Cancelled(CancelEvent { - status: status.as_u16(), - bytes_received: t.upstream_snapshot().len(), - streaming: true, - }), - ); - return; - } - } - stream_semantic = Some(Arc::new(t.semantic_response())); - let (tool_ids, reasoning) = t.drain_reasoning_snapshot(); - if !reasoning.is_empty() { - reasoning_cache.remember(&tool_ids, &reasoning); - } - } - let resp_bytes = translator - .as_ref() - .map(|t| Bytes::copy_from_slice(t.upstream_snapshot())) - .unwrap_or_else(|| buf.freeze()); - let stream_assistant_text = translator - .as_ref() - .and_then(|t| t.streaming_capture_snapshot()); - capture_engine.spawn_apply( - ctx_bg, - Event::ResponseComplete(CompleteEvent { - status: status.as_u16(), - resp_bytes, - streaming: true, - stream_metrics, - assistant_content: stream_assistant_text, - semantic: stream_semantic, - headers: recorded_resp_headers, - }), - ); - }); - - let body_stream = ReceiverStream::new(rx).map(|item| item.map_err(std::io::Error::other)); - - let mut builder = Response::builder().status(status); - // The client-facing SSE body is a new stream. Upstream hop-by-hop framing and - // content length never cross that boundary; translated streams additionally - // replace stale encoding and media type metadata. - for (name, value) in resp_headers.iter() { - if skip_response_header_after_reframing(&resp_headers, name.as_str(), translate) { - continue; - } - builder = builder.header(name, value); - } - if translate { - builder = builder.header("content-type", "text/event-stream"); - } - builder = attach_capture_headers(builder, &ctx.call); - Ok(builder - .body(Body::from_stream(body_stream)) - .map_err(|e| anyhow::anyhow!("response body: {e}"))? - .into_response()) -} - -fn maybe_emit_stream_draft( - engine: &CaptureEngine, - ctx: &Arc, - status: u16, - translator: &StreamTranslator, - last_draft_at: &mut std::time::Instant, - last_draft_content: &mut String, -) { - let Some(snapshot) = translator.streaming_capture_snapshot() else { - return; - }; - if snapshot == *last_draft_content { - return; - } - if !last_draft_content.is_empty() && last_draft_at.elapsed() < STREAM_DRAFT_MD_INTERVAL { - return; - } - // `Arc::clone` only bumps the refcount — no deep clone of headers/strings. - engine.spawn_apply( - Arc::clone(ctx), - Event::ResponseDraft(DraftEvent { - status, - assistant_content: snapshot.clone(), - }), - ); - *last_draft_content = snapshot; - *last_draft_at = std::time::Instant::now(); -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn stream_request_enables_passthrough() { - let body = Bytes::from_static(br#"{"model":"m","stream":true,"messages":[]}"#); - assert!(should_stream_to_client(&HeaderMap::new(), &body)); - } - - #[test] - fn sse_response_enables_passthrough() { - let mut h = HeaderMap::new(); - h.insert("content-type", "text/event-stream".parse().unwrap()); - let body = Bytes::from_static(b"{}"); - assert!(should_stream_to_client(&h, &body)); - } -} diff --git a/crates/persisting-gateway/src/gateway/upstream.rs b/crates/persisting-gateway/src/gateway/upstream.rs deleted file mode 100644 index 1a8e91fb8..000000000 --- a/crates/persisting-gateway/src/gateway/upstream.rs +++ /dev/null @@ -1,82 +0,0 @@ -//! Prepare upstream LLM request body after routing (model rewrite + protocol bridge). - -use anyhow::Result; -use bytes::Bytes; - -use crate::conversion::{ProtocolBridge, translate_request_for_bridge}; - -use super::model::rewrite_model_in_body; -use super::reasoning::ReasoningCacheHandle; - -/// Build the request body sent to the upstream LLM after routing. -pub fn prepare_upstream_body( - client_body: &Bytes, - semantic: Option<&persisting_pchronicle::model::LlmRequestEventPayload>, - model_rewritten: bool, - upstream_model: &str, - bridge: ProtocolBridge, - reasoning_cache: Option<&ReasoningCacheHandle>, -) -> Result { - if bridge.needs_request_translation() && !client_body.is_empty() { - let semantic = semantic.ok_or_else(|| { - anyhow::anyhow!("protocol translation requires a Chronicle typed LLM request") - })?; - return translate_request_for_bridge(bridge, semantic, upstream_model, reasoning_cache); - } - if model_rewritten { - return rewrite_model_in_body(client_body, upstream_model); - } - Ok(client_body.clone()) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::config::ModelRoute; - - fn route_with_upstream(upstream: &str) -> ModelRoute { - ModelRoute { - name: "*".into(), - provider: None, - upstream: Some(upstream.into()), - upstream_anthropic: None, - api_key_env: None, - api_key: None, - forward: None, - } - } - - #[test] - fn passthrough_without_rewrite() { - let body = Bytes::from_static(br#"{"model":"m","messages":[]}"#); - let out = prepare_upstream_body( - &body, - None, - false, - "m", - ProtocolBridge::needed( - crate::protocol::ProtocolKind::ChatCompletions, - &route_with_upstream("http://x/v1"), - ), - None, - ) - .unwrap(); - assert_eq!(out, body); - } - - #[test] - fn rewrites_model_when_forwarded() { - let body = Bytes::from_static(br#"{"model":"claude-3","messages":[]}"#); - let out = prepare_upstream_body( - &body, - None, - true, - "deepseek-chat", - ProtocolBridge::Passthrough, - None, - ) - .unwrap(); - let v: serde_json::Value = serde_json::from_slice(&out).unwrap(); - assert_eq!(v["model"], "deepseek-chat"); - } -} diff --git a/crates/persisting-gateway/src/injection/env.rs b/crates/persisting-gateway/src/injection/env.rs deleted file mode 100644 index f7eff4f07..000000000 --- a/crates/persisting-gateway/src/injection/env.rs +++ /dev/null @@ -1,156 +0,0 @@ -//! Child-process proxy / SDK base URL injection for `capture run`. - -use std::collections::HashMap; -use std::path::Path; - -pub use crate::runtime::run_env::ENV_SESSION_ID; - -/// Env vars that route HTTP clients through the capture proxy (must not be set on the daemon). -pub const CAPTURE_PROXY_ENV_KEYS: &[&str] = &[ - "HTTP_PROXY", - "HTTPS_PROXY", - "http_proxy", - "https_proxy", - "ALL_PROXY", - "all_proxy", -]; - -/// OpenAI-compatible gateway base (`http://127.0.0.1:PORT/v1`) for child LLM clients. -pub fn capture_openai_v1_base(listen: &str) -> String { - let base = if listen.starts_with("http://") || listen.starts_with("https://") { - listen.to_string() - } else { - format!("http://{listen}") - }; - format!("{}/v1", base.trim_end_matches('/')) -} - -/// Extra CLI flags for clients that ignore `OPENAI_BASE_URL` and need explicit config overrides. -/// -/// Codex reads `openai_base_url` from `config.toml` (via `-c`), not from `OPENAI_BASE_URL`. -pub fn client_gateway_config_args(program: &str, listen: &str) -> Vec { - let openai_v1 = capture_openai_v1_base(listen); - let name = Path::new(program) - .file_name() - .and_then(|s| s.to_str()) - .unwrap_or(program); - match name { - "codex" => vec!["-c".to_string(), format!("openai_base_url=\"{openai_v1}\"")], - _ => Vec::new(), - } -} - -/// Build env map for subprocess: HTTP(S) forward proxy + LLM SDK base URLs. -/// -/// Child processes that honor `HTTP_PROXY` / `HTTPS_PROXY` send **all** HTTP(S) traffic -/// to `listen`. The capture server: -/// - `CONNECT` → TCP tunnel (HTTPS and other TLS) -/// - absolute-URI HTTP → transparent forward, except LLM API paths (captured + yaml upstream) -/// - relative paths on `listen` (via `OPENAI_BASE_URL`) → LLM gateway + capture -pub fn proxy_environment(listen: &str, session_id: &str) -> HashMap { - proxy_environment_with_local_auth(listen, session_id, false) -} - -/// Build the proxy environment and, when the trusted Gateway owns upstream -/// credentials, provide non-secret Run-scoped placeholders required by SDKs. -pub fn proxy_environment_with_local_auth( - listen: &str, - session_id: &str, - local_auth: bool, -) -> HashMap { - let base = if listen.starts_with("http://") || listen.starts_with("https://") { - listen.to_string() - } else { - format!("http://{listen}") - }; - let base = base.trim_end_matches('/').to_string(); - let openai_v1 = capture_openai_v1_base(listen); - - let mut env = HashMap::new(); - for key in CAPTURE_PROXY_ENV_KEYS { - env.insert(key.to_string(), base.clone()); - } - // Loopback gateway requests must not be CONNECT-tunneled via HTTPS_PROXY. - env.insert("NO_PROXY".to_string(), "127.0.0.1,localhost".to_string()); - env.insert("no_proxy".to_string(), "127.0.0.1,localhost".to_string()); - for key in [ - "OPENAI_BASE_URL", - "OPENAI_API_BASE", - "AZURE_OPENAI_ENDPOINT", - ] { - env.insert(key.to_string(), openai_v1.clone()); - } - env.insert("ANTHROPIC_BASE_URL".to_string(), base.clone()); - env.insert("GEMINI_API_BASE".to_string(), format!("{base}/v1beta")); - env.insert(ENV_SESSION_ID.to_string(), session_id.to_string()); - - if local_auth { - let placeholder = format!("persisting-local-{session_id}"); - for key in [ - "OPENAI_API_KEY", - "ANTHROPIC_AUTH_TOKEN", - "ANTHROPIC_API_KEY", - "GEMINI_API_KEY", - ] { - env.insert(key.into(), placeholder.clone()); - } - } - - env -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn proxy_env_keys() { - let env = proxy_environment("127.0.0.1:8080", "sess-1"); - assert_eq!( - env.get("HTTP_PROXY").map(String::as_str), - Some("http://127.0.0.1:8080") - ); - assert_eq!( - env.get("OPENAI_BASE_URL").map(String::as_str), - Some("http://127.0.0.1:8080/v1") - ); - assert_eq!( - env.get("NO_PROXY").map(String::as_str), - Some("127.0.0.1,localhost") - ); - assert_eq!(env.get(ENV_SESSION_ID).map(String::as_str), Some("sess-1")); - } - - #[test] - fn codex_gateway_config_args() { - let args = client_gateway_config_args("codex", "127.0.0.1:19081"); - assert_eq!(args.len(), 2); - assert_eq!(args[0], "-c"); - assert_eq!(args[1], "openai_base_url=\"http://127.0.0.1:19081/v1\""); - } - - #[test] - fn proxy_environment_never_projects_upstream_credentials() { - unsafe { std::env::set_var("DEEPSEEK_API_KEY", "host-secret") }; - let env = proxy_environment("127.0.0.1:8080", "sess-1"); - assert!(!env.contains_key("DEEPSEEK_API_KEY")); - assert!(!env.contains_key("ANTHROPIC_AUTH_TOKEN")); - assert!(!env.contains_key("ANTHROPIC_API_KEY")); - unsafe { std::env::remove_var("DEEPSEEK_API_KEY") }; - } - - #[test] - fn local_auth_uses_run_scoped_placeholders_not_host_secrets() { - unsafe { std::env::set_var("OPENAI_API_KEY", "host-secret") }; - let env = proxy_environment_with_local_auth("127.0.0.1:8080", "sess-1", true); - assert_eq!( - env.get("OPENAI_API_KEY").map(String::as_str), - Some("persisting-local-sess-1") - ); - assert_eq!( - env.get("ANTHROPIC_AUTH_TOKEN").map(String::as_str), - Some("persisting-local-sess-1") - ); - unsafe { std::env::remove_var("OPENAI_API_KEY") }; - } -} diff --git a/crates/persisting-gateway/src/injection/host_identity.rs b/crates/persisting-gateway/src/injection/host_identity.rs deleted file mode 100644 index 258469a92..000000000 --- a/crates/persisting-gateway/src/injection/host_identity.rs +++ /dev/null @@ -1,321 +0,0 @@ -//! Stable one-way machine fingerprint for session client metadata. -//! -//! Stored as `machine_fp` on [`SessionClientMeta`](crate::session::client::SessionClientMeta) -//! (32-char hex = first 128 bits of BLAKE3). - -use std::net::{IpAddr, Ipv4Addr, SocketAddr}; -#[cfg(unix)] -use std::process::Command; - -/// Inputs for [`fingerprint_host`]; exposed for deterministic unit tests. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct HostIdentityInput { - pub hostname: String, - pub username: String, - /// IP used for identity (client peer when remote, else local primary IPv4). - pub identity_ip: String, - /// Linux `/etc/machine-id` when available — more stable than hostname alone. - pub machine_id: Option, -} - -/// One-way machine fingerprint (`persisting-machine-fp/v1`). -pub fn fingerprint_host(input: &HostIdentityInput) -> String { - let mut hasher = blake3::Hasher::new(); - hasher.update(b"persisting-machine-fp/v1\n"); - if let Some(id) = input - .machine_id - .as_deref() - .map(str::trim) - .filter(|s| !s.is_empty()) - { - hasher.update(b"machine_id="); - hasher.update(id.as_bytes()); - hasher.update(b"\n"); - } - hasher.update(b"hostname="); - hasher.update(normalize_hostname(&input.hostname).as_bytes()); - hasher.update(b"\n"); - hasher.update(b"username="); - hasher.update(normalize_username(&input.username).as_bytes()); - hasher.update(b"\n"); - hasher.update(b"ip="); - hasher.update(input.identity_ip.trim().as_bytes()); - hasher.update(b"\n"); - hex16(&hasher.finalize()) -} - -/// Best-effort fingerprint for the machine / OS user behind a proxied TCP peer. -/// -/// When `client_pid` is known, uses that process owner (`ps user=`); otherwise the -/// effective user of the capture proxy process. -pub fn machine_fingerprint_for_client(peer: SocketAddr, client_pid: Option) -> String { - let username = client_pid - .filter(|pid| *pid > 0) - .and_then(username_for_pid) - .unwrap_or_else(local_username); - fingerprint_host(&HostIdentityInput { - hostname: local_hostname(), - username, - identity_ip: identity_ip_for_peer(peer).to_string(), - machine_id: read_linux_machine_id(), - }) -} - -fn identity_ip_for_peer(peer: SocketAddr) -> IpAddr { - if peer.ip().is_loopback() { - local_primary_ipv4() - .map(IpAddr::V4) - .unwrap_or(IpAddr::V4(Ipv4Addr::LOCALHOST)) - } else { - peer.ip() - } -} - -fn normalize_hostname(host: &str) -> String { - let h = host.trim().to_lowercase(); - h.strip_suffix(".local").unwrap_or(&h).to_string() -} - -fn normalize_username(user: &str) -> String { - user.trim().to_lowercase() -} - -fn hex16(hash: &blake3::Hash) -> String { - hash.as_bytes()[..16] - .iter() - .map(|b| format!("{b:02x}")) - .collect() -} - -fn local_hostname() -> String { - hostname::get() - .ok() - .and_then(|h| h.into_string().ok()) - .filter(|s| !s.is_empty()) - .unwrap_or_else(|| "unknown".into()) -} - -fn local_username() -> String { - normalize_username( - &std::env::var("USER") - .or_else(|_| std::env::var("USERNAME")) - .unwrap_or_else(|_| "unknown".into()), - ) -} - -#[cfg(unix)] -fn username_for_pid(pid: u32) -> Option { - let output = Command::new("ps") - .args(["-p", &pid.to_string(), "-o", "user="]) - .output() - .ok()?; - if !output.status.success() { - return None; - } - let user = String::from_utf8_lossy(&output.stdout).trim().to_string(); - if user.is_empty() { - None - } else { - Some(normalize_username(&user)) - } -} - -#[cfg(not(unix))] -fn username_for_pid(_pid: u32) -> Option { - None -} - -/// Primary outbound IPv4 (UDP connect trick); no packets sent on most OSes. -fn local_primary_ipv4() -> Option { - let socket = std::net::UdpSocket::bind("0.0.0.0:0").ok()?; - socket.connect("8.8.8.8:80").ok()?; - match socket.local_addr().ok()?.ip() { - IpAddr::V4(v4) if !v4.is_loopback() => Some(v4), - _ => None, - } -} - -fn read_linux_machine_id() -> Option { - #[cfg(target_os = "linux")] - { - for path in ["/etc/machine-id", "/var/lib/dbus/machine-id"] { - if let Ok(text) = std::fs::read_to_string(path) { - let id = text.trim().to_string(); - if !id.is_empty() { - return Some(id); - } - } - } - } - #[cfg(not(target_os = "linux"))] - { - let _ = (); - } - None -} - -#[cfg(test)] -mod tests { - use super::*; - use proptest::prelude::*; - - fn sample_input() -> HostIdentityInput { - HostIdentityInput { - hostname: "MacBook.local".into(), - username: "alice".into(), - identity_ip: "192.168.1.42".into(), - machine_id: Some("abc-def".into()), - } - } - - #[test] - fn fingerprint_is_stable_and_hex32() { - let input = sample_input(); - let a = fingerprint_host(&input); - let b = fingerprint_host(&input); - assert_eq!(a, b); - assert_eq!(a.len(), 32); - assert!(a.chars().all(|c| c.is_ascii_hexdigit())); - } - - #[test] - fn hostname_normalization_strips_local_suffix() { - let with = HostIdentityInput { - hostname: "box.local".into(), - username: "bob".into(), - identity_ip: "10.0.0.1".into(), - machine_id: None, - }; - let without = HostIdentityInput { - hostname: "box".into(), - ..with.clone() - }; - assert_eq!(fingerprint_host(&with), fingerprint_host(&without)); - } - - #[test] - fn username_changes_fingerprint() { - let base = HostIdentityInput { - username: "alice".into(), - ..sample_input() - }; - let other = HostIdentityInput { - username: "bob".into(), - ..base.clone() - }; - assert_ne!(fingerprint_host(&base), fingerprint_host(&other)); - } - - #[test] - fn machine_id_changes_fingerprint() { - let base = HostIdentityInput { - machine_id: None, - ..sample_input() - }; - let with_id = HostIdentityInput { - machine_id: Some("id-1".into()), - ..base.clone() - }; - assert_ne!(fingerprint_host(&base), fingerprint_host(&with_id)); - } - - #[test] - fn loopback_peer_uses_localhost_ip_when_no_route() { - let peer: SocketAddr = "127.0.0.1:12345".parse().unwrap(); - let fp = machine_fingerprint_for_client(peer, None); - assert_eq!(fp.len(), 32); - } - - fn identity_text_strategy() -> impl Strategy { - proptest::string::string_regex("[a-zA-Z0-9._:-]{1,64}").unwrap() - } - - fn identity_strategy() -> impl Strategy { - ( - identity_text_strategy(), - identity_text_strategy(), - identity_text_strategy(), - prop::option::of(identity_text_strategy()), - ) - .prop_map( - |(hostname, username, identity_ip, machine_id)| HostIdentityInput { - hostname, - username, - identity_ip, - machine_id, - }, - ) - } - - proptest! { - #[test] - fn every_fingerprint_is_stable_lowercase_hex(input in identity_strategy()) { - let first = fingerprint_host(&input); - let second = fingerprint_host(&input); - prop_assert_eq!(&first, &second); - prop_assert_eq!(first.len(), 32); - prop_assert!(first.bytes().all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))); - } - - #[test] - fn hostname_case_whitespace_and_local_suffix_are_normalized( - hostname in proptest::string::string_regex("[a-zA-Z0-9-]{1,48}").unwrap(), - username in identity_text_strategy(), - identity_ip in identity_text_strategy(), - ) { - let canonical = HostIdentityInput { - hostname: hostname.to_ascii_lowercase(), - username: username.clone(), - identity_ip: identity_ip.clone(), - machine_id: None, - }; - let decorated = HostIdentityInput { - hostname: format!(" {}.LOCAL ", hostname.to_ascii_uppercase()), - ..canonical.clone() - }; - prop_assert_eq!(fingerprint_host(&canonical), fingerprint_host(&decorated)); - } - - #[test] - fn username_and_ip_outer_whitespace_do_not_change_identity( - hostname in identity_text_strategy(), - username in identity_text_strategy(), - identity_ip in identity_text_strategy(), - ) { - let canonical = HostIdentityInput { - hostname, - username: username.to_ascii_lowercase(), - identity_ip: identity_ip.clone(), - machine_id: None, - }; - let decorated = HostIdentityInput { - username: format!(" {} ", username.to_ascii_uppercase()), - identity_ip: format!(" {identity_ip} "), - ..canonical.clone() - }; - prop_assert_eq!(fingerprint_host(&canonical), fingerprint_host(&decorated)); - } - - #[test] - fn empty_machine_ids_are_equivalent_to_absent_ids( - mut input in identity_strategy(), - spaces in 0usize..32, - ) { - input.machine_id = None; - let absent = fingerprint_host(&input); - input.machine_id = Some(" ".repeat(spaces)); - prop_assert_eq!(fingerprint_host(&input), absent); - } - - #[test] - fn machine_id_outer_whitespace_is_ignored( - mut input in identity_strategy(), - machine_id in identity_text_strategy(), - ) { - input.machine_id = Some(machine_id.clone()); - let canonical = fingerprint_host(&input); - input.machine_id = Some(format!(" {machine_id} ")); - prop_assert_eq!(fingerprint_host(&input), canonical); - } - } -} diff --git a/crates/persisting-gateway/src/injection/mod.rs b/crates/persisting-gateway/src/injection/mod.rs deleted file mode 100644 index 530588e74..000000000 --- a/crates/persisting-gateway/src/injection/mod.rs +++ /dev/null @@ -1,16 +0,0 @@ -//! Agent-side injection: env vars for child processes and peer process detection. -//! -//! | Module | Role | -//! |--------|------| -//! | [`mod@env`] | `HTTP_PROXY`, SDK base URLs, session id for `capture run` children | -//! | [`peer`] | lsof/ps lookup of the TCP peer behind proxied connections | -//! | [`host_identity`] | one-way `machine_fp` from hostname + identity IP | - -pub mod env; -pub mod host_identity; -pub mod peer; - -pub use env::{ - CAPTURE_PROXY_ENV_KEYS, capture_openai_v1_base, client_gateway_config_args, proxy_environment, - proxy_environment_with_local_auth, -}; diff --git a/crates/persisting-gateway/src/injection/peer.rs b/crates/persisting-gateway/src/injection/peer.rs deleted file mode 100644 index c6fc153c5..000000000 --- a/crates/persisting-gateway/src/injection/peer.rs +++ /dev/null @@ -1,204 +0,0 @@ -//! Resolve the OS process behind an inbound proxy TCP peer (local ephemeral port). - -use std::net::SocketAddr; -use std::process::Command; - -use anyhow::{Context, Result}; - -use crate::session::client::SessionClientMeta; - -/// Look up the client process for a proxied connection's peer address. -pub(crate) fn resolve_peer_client(peer: SocketAddr) -> Result> { - #[cfg(unix)] - { - if let Some(pid) = lsof_client_pid_for_peer(peer)? - && let Some(command) = command_line_for_pid(pid)? - { - return Ok(Some(SessionClientMeta { - peer: peer.to_string(), - peer_port: peer.port(), - pid, - command, - machine_fp: None, - })); - } - Ok(None) - } - #[cfg(not(unix))] - { - let _ = peer; - Ok(None) - } -} - -#[cfg(unix)] -fn lsof_client_pid_for_peer(peer: SocketAddr) -> Result> { - let spec = lsof_tcp_spec(peer); - let output = Command::new("lsof") - .args(["-nP", "-F", "pnc", "-i", &spec, "-sTCP:ESTABLISHED"]) - .output() - .with_context(|| format!("lsof peer lookup for {peer}"))?; - if !output.status.success() && output.stdout.is_empty() { - return Ok(None); - } - let text = String::from_utf8_lossy(&output.stdout); - let entries = parse_lsof_pnc(&text); - let self_pid = std::process::id(); - - for entry in entries { - if entry.pid == 0 || entry.pid == self_pid { - continue; - } - if !entry - .names - .iter() - .any(|name| is_client_outbound_socket(name, peer)) - { - continue; - } - if command_line_for_pid(entry.pid)?.is_some() { - return Ok(Some(entry.pid)); - } - } - Ok(None) -} - -#[cfg(unix)] -struct LsofEntry { - pid: u32, - names: Vec, -} - -#[cfg(unix)] -fn parse_lsof_pnc(output: &str) -> Vec { - let mut entries = Vec::new(); - let mut pid = 0u32; - let mut names = Vec::new(); - - for line in output.lines() { - if line.is_empty() { - continue; - } - let Some(tag) = line.as_bytes().first() else { - continue; - }; - let rest = &line[1..]; - match *tag { - b'p' => { - if pid != 0 { - entries.push(LsofEntry { - pid, - names: std::mem::take(&mut names), - }); - } - pid = rest.trim().parse().unwrap_or(0); - } - b'n' => names.push(rest.to_string()), - _ => {} - } - } - if pid != 0 { - entries.push(LsofEntry { pid, names }); - } - entries -} - -#[cfg(unix)] -fn is_client_outbound_socket(name: &str, peer: SocketAddr) -> bool { - let local = format_socket_addr(peer); - name.split_once("->") - .is_some_and(|(lhs, _rhs)| lhs == local) -} - -#[cfg(unix)] -fn format_socket_addr(addr: SocketAddr) -> String { - match addr.ip() { - std::net::IpAddr::V4(v4) => format!("{v4}:{}", addr.port()), - std::net::IpAddr::V6(v6) => format!("[{v6}]:{}", addr.port()), - } -} - -#[cfg(unix)] -fn lsof_tcp_spec(peer: SocketAddr) -> String { - match peer.ip() { - std::net::IpAddr::V4(v4) => format!("TCP@{}:{}", v4, peer.port()), - std::net::IpAddr::V6(v6) => format!("TCP@[{}]:{}", v6, peer.port()), - } -} - -#[cfg(unix)] -fn command_line_for_pid(pid: u32) -> Result> { - #[cfg(target_os = "linux")] - { - let raw = std::fs::read(format!("/proc/{pid}/cmdline")).ok(); - if let Some(raw) = raw { - let cmd = raw - .split(|b| *b == 0) - .filter(|part| !part.is_empty()) - .map(|part| String::from_utf8_lossy(part).into_owned()) - .collect::>() - .join(" "); - if !cmd.is_empty() { - return Ok(Some(cmd)); - } - } - } - let output = Command::new("ps") - .args(["-p", &pid.to_string(), "-o", "args="]) - .output() - .with_context(|| format!("ps args for pid {pid}"))?; - if !output.status.success() { - return Ok(None); - } - let cmd = String::from_utf8_lossy(&output.stdout).trim().to_string(); - if cmd.is_empty() { - Ok(None) - } else { - Ok(Some(cmd)) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn lsof_tcp_spec_ipv4() { - let peer: SocketAddr = "127.0.0.1:54321".parse().unwrap(); - assert_eq!(lsof_tcp_spec(peer), "TCP@127.0.0.1:54321"); - } - - #[test] - fn lsof_tcp_spec_ipv6() { - let peer: SocketAddr = "[::1]:8080".parse().unwrap(); - assert_eq!(lsof_tcp_spec(peer), "TCP@[::1]:8080"); - } - - #[test] - fn client_outbound_socket_matches_local_peer_port() { - let peer: SocketAddr = "127.0.0.1:55522".parse().unwrap(); - assert!(is_client_outbound_socket( - "127.0.0.1:55522->127.0.0.1:19081", - peer - )); - assert!(!is_client_outbound_socket( - "127.0.0.1:19081->127.0.0.1:55522", - peer - )); - } - - #[test] - fn parse_lsof_pnc_groups_names_by_pid() { - let sample = "\ -p100 -n127.0.0.1:8080->127.0.0.1:55522 -p200 -n127.0.0.1:55522->127.0.0.1:8080 -"; - let entries = parse_lsof_pnc(sample); - assert_eq!(entries.len(), 2); - assert_eq!(entries[0].pid, 100); - assert_eq!(entries[1].pid, 200); - assert_eq!(entries[1].names[0], "127.0.0.1:55522->127.0.0.1:8080"); - } -} diff --git a/crates/persisting-gateway/src/lib.rs b/crates/persisting-gateway/src/lib.rs deleted file mode 100644 index 8a5c56e41..000000000 --- a/crates/persisting-gateway/src/lib.rs +++ /dev/null @@ -1,42 +0,0 @@ -//! Agent protocol gateway: forwarding, trajectory capture, and live projection. -//! -//! ## How to import -//! -//! Use the crate root for serving and modules for domain types: -//! -//! ```ignore -//! use persisting_gateway::config::ProxyConfig; -//! use persisting_gateway::record::EventRecord; -//! use persisting_gateway::engine::CaptureEngine; -//! use persisting_gateway::serve; -//! ``` - -pub mod config; -pub mod conversion; -pub mod dead_letter; -pub mod dialogue_extract; -#[cfg(feature = "echo-server")] -pub mod echo; -pub mod engine; -mod gateway; -pub mod injection; -pub mod lifecycle; -pub mod projection; -pub mod protocol; -pub mod provider; -pub mod record; -pub mod runtime; -pub mod session; -pub mod sink; -pub mod subagent_link; -pub mod understanding; -pub mod usage; - -pub use gateway::models_list; -pub use gateway::{ - serve, serve_with_listeners_and_shutdown, serve_with_runtime_control, serve_with_shutdown, - serve_with_shutdown_and_ready, -}; - -/// Used by [`sink`], [`record`], and gateway helpers (`crate::Call` internally). -pub use engine::Call; diff --git a/crates/persisting-gateway/src/lifecycle.rs b/crates/persisting-gateway/src/lifecycle.rs deleted file mode 100644 index 3ee63baf6..000000000 --- a/crates/persisting-gateway/src/lifecycle.rs +++ /dev/null @@ -1,187 +0,0 @@ -//! Session lifecycle events for the raw Lance event log. - -use serde::{Deserialize, Serialize}; -use serde_json::{Value, json}; - -use super::record::{EventRecord, now_rfc3339}; -use crate::session::storage::CaptureRoute; - -pub const SESSION_STARTED: &str = "session.started"; -pub const SESSION_ENDED: &str = "session.ended"; -pub const SESSION_STATE: &str = "session.state"; - -/// How capture was started (`payload.mode`). -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum CaptureMode { - Run, - Serve, - Daemon, -} - -impl CaptureMode { - pub fn as_str(self) -> &'static str { - match self { - Self::Run => "capture.run", - Self::Serve => "capture.serve", - Self::Daemon => "capture.daemon", - } - } -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct SessionLifecyclePayload { - pub action: String, - pub mode: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub listen: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub command: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub exit_code: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub reason: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub duration_ms: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub from: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub to: Option, - #[serde(flatten)] - pub extra: Value, -} - -pub fn session_started_record( - session_id: Option, - agent_id: Option, - mode: CaptureMode, - listen: Option<&str>, - command: Option<&str>, -) -> EventRecord { - lifecycle_record( - SESSION_STARTED, - session_id, - agent_id, - SessionLifecyclePayload { - action: "started".into(), - mode: mode.as_str().into(), - listen: listen.map(str::to_string), - command: command.map(str::to_string), - exit_code: None, - reason: None, - duration_ms: None, - from: None, - to: None, - extra: json!({}), - }, - ) -} - -pub fn session_ended_record( - session_id: Option, - agent_id: Option, - mode: CaptureMode, - reason: &str, - exit_code: Option, - duration_ms: Option, -) -> EventRecord { - lifecycle_record( - SESSION_ENDED, - session_id, - agent_id, - SessionLifecyclePayload { - action: "ended".into(), - mode: mode.as_str().into(), - listen: None, - command: None, - exit_code, - reason: Some(reason.into()), - duration_ms, - from: None, - to: None, - extra: json!({}), - }, - ) -} - -pub fn session_state_record( - session_id: Option, - agent_id: Option, - mode: CaptureMode, - from: &str, - to: &str, - reason: Option<&str>, -) -> EventRecord { - lifecycle_record( - SESSION_STATE, - session_id, - agent_id, - SessionLifecyclePayload { - action: "state".into(), - mode: mode.as_str().into(), - listen: None, - command: None, - exit_code: None, - reason: reason.map(str::to_string), - duration_ms: None, - from: Some(from.into()), - to: Some(to.into()), - extra: json!({}), - }, - ) -} - -fn lifecycle_record( - kind: &str, - session_id: Option, - agent_id: Option, - payload: SessionLifecyclePayload, -) -> EventRecord { - EventRecord { - identity: persisting_events::EventIdentity::default(), - seq: 0, - source: "persisting-gateway".into(), - kind: kind.into(), - timestamp: Some(now_rfc3339()), - session_id, - agent_id, - parent_uuid: None, - trace_id: None, - call_id: None, - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload: serde_json::to_value(payload).unwrap_or(json!({})), - } -} - -/// Route for daemon-level proxy lifecycle (`{agent_id}/_proxy/`). -pub fn proxy_lifecycle_route() -> CaptureRoute { - CaptureRoute { - root_session: None, - session_id: "_proxy".into(), - storage_session_id: "_proxy".into(), - subagent_id: None, - } -} - -/// Route for a capture run root session directory. -pub fn root_session_route(root_session: &str) -> CaptureRoute { - CaptureRoute { - root_session: Some(root_session.into()), - session_id: root_session.into(), - storage_session_id: root_session.into(), - subagent_id: None, - } -} - -pub fn append_lifecycle( - sink: &dyn super::sink::CaptureEventSink, - route: &CaptureRoute, - agent_id: &str, - mut record: EventRecord, -) -> anyhow::Result<()> { - sink.append(route, agent_id, &mut record)?; - Ok(()) -} diff --git a/crates/persisting-gateway/src/projection/dialogue/block.rs b/crates/persisting-gateway/src/projection/dialogue/block.rs deleted file mode 100644 index 89047f2e7..000000000 --- a/crates/persisting-gateway/src/projection/dialogue/block.rs +++ /dev/null @@ -1,88 +0,0 @@ -use anyhow::{Context, Result}; -use persisting_pchronicle::model::{EventRecord, StorylineTimestamp, StorylineTurn}; -use serde_json::{Map, Value}; - -use super::fields::role_and_body; - -/// Project one capture event into the authoritative Storyline turn model. -pub fn capture_record_to_storyline_turn(rec: &EventRecord) -> Result { - let (role, body) = role_and_body(rec)?; - let source = match role.as_str() { - "user" => "user", - "assistant" | "agent" => "agent", - _ => "system", - }; - let id = i64::try_from(rec.seq).context("event sequence exceeds Storyline turn id range")?; - let metrics = rec - .payload - .get("body") - .and_then(|body| body.get("usage")) - .or_else(|| rec.payload.get("usage")) - .cloned(); - let model_name = rec - .payload - .get("model") - .and_then(Value::as_str) - .map(str::to_owned); - let ttft_ms = rec.payload.get("ttft_ms").and_then(Value::as_i64); - let latency_ms = rec.payload.get("latency_ms").and_then(Value::as_i64); - let timestamp = rec - .timestamp - .as_deref() - .map(StorylineTimestamp::from_rfc3339) - .transpose() - .context("capture event timestamp is not valid RFC 3339")?; - - let mut extra = Map::new(); - insert_string(&mut extra, "producer", Some(rec.source.as_str())); - insert_string(&mut extra, "trace_id", rec.trace_id.as_deref()); - insert_string(&mut extra, "parent_uuid", rec.parent_uuid.as_deref()); - insert_string(&mut extra, "subagent_id", rec.subagent_id.as_deref()); - insert_string( - &mut extra, - "parent_agent_id", - rec.parent_agent_id.as_deref(), - ); - for key in [ - "path", - "status", - "draft", - "refs_subagent_ids", - "subagent_trajectories", - "subagent_trajectory", - "spawn_hints", - "spawn_links", - ] { - if let Some(value) = rec.payload.get(key) { - extra.insert(key.into(), value.clone()); - } - } - - Ok(StorylineTurn { - id, - kind: Some(rec.kind.clone()), - timestamp, - source: source.into(), - message: Value::String(body), - reasoning_content: None, - reasoning_effort: None, - tool_calls: None, - observation: None, - metrics, - model_name, - llm_call_count: (source == "agent").then_some(1), - is_copied_context: None, - latency_ms, - ttft_ms, - extra: (!extra.is_empty()).then_some(Value::Object(extra)), - env: None, - prompt: None, - finished_at: None, - }) -} - -fn insert_string(extra: &mut Map, key: &str, value: Option<&str>) { - if let Some(value) = value.filter(|value| !value.is_empty()) { - extra.insert(key.into(), Value::String(value.into())); - } -} diff --git a/crates/persisting-gateway/src/projection/dialogue/draft.rs b/crates/persisting-gateway/src/projection/dialogue/draft.rs deleted file mode 100644 index 1d9279b87..000000000 --- a/crates/persisting-gateway/src/projection/dialogue/draft.rs +++ /dev/null @@ -1,28 +0,0 @@ -use anyhow::Result; -use persisting_pchronicle::model::StorylineTurn; -use serde_json::json; - -use super::block::capture_record_to_storyline_turn; -use super::skip_markdown_block; -use crate::record::EventRecord; - -/// Build a streaming draft assistant block (markdown view only; not written to Lance). -pub fn draft_stream_assistant_turn( - rec: &EventRecord, - assistant_content: &str, -) -> Result> { - if assistant_content.trim().is_empty() { - return Ok(None); - } - let mut draft = rec.clone(); - draft.kind = "llm.response.stream".into(); - draft.payload = json!({ - "status": rec.payload.get("status").and_then(|v| v.as_u64()).unwrap_or(200), - "assistant_content": assistant_content, - "draft": true, - }); - if skip_markdown_block(&draft) { - return Ok(None); - } - Ok(Some(capture_record_to_storyline_turn(&draft)?)) -} diff --git a/crates/persisting-gateway/src/projection/dialogue/fields.rs b/crates/persisting-gateway/src/projection/dialogue/fields.rs deleted file mode 100644 index f76cc2240..000000000 --- a/crates/persisting-gateway/src/projection/dialogue/fields.rs +++ /dev/null @@ -1,26 +0,0 @@ -use anyhow::Result; -use serde_json::Value; - -use crate::record::{EventRecord, EventRecordExt, content_to_string}; - -pub(crate) fn role_and_body(rec: &EventRecord) -> Result<(String, String)> { - Ok(match rec.kind.as_str() { - "llm.request" => ("user".into(), rec.visible_user_text().unwrap_or_default()), - "llm.response" | "llm.response.stream" => ( - "assistant".into(), - rec.visible_assistant_text().unwrap_or_default(), - ), - "user" | "assistant" | "system" | "tool" | "note" => ( - rec.kind.clone(), - rec.payload - .get("content") - .and_then(content_to_string) - .unwrap_or_else(|| compact_json(&rec.payload)), - ), - _ => ("note".into(), compact_json(&rec.payload)), - }) -} - -pub(crate) fn compact_json(payload: &Value) -> String { - serde_json::to_string(payload).unwrap_or_else(|_| "{}".to_string()) -} diff --git a/crates/persisting-gateway/src/projection/dialogue/mod.rs b/crates/persisting-gateway/src/projection/dialogue/mod.rs deleted file mode 100644 index 349ef4f4c..000000000 --- a/crates/persisting-gateway/src/projection/dialogue/mod.rs +++ /dev/null @@ -1,16 +0,0 @@ -//! `EventRecord` ↔ markdown trajectory blocks. -//! -//! Markdown role/body text comes from Capture's [`crate::record::EventRecordExt`] -//! behavior (shared with turn indexing). - -mod block; -mod draft; -mod fields; - -#[cfg(test)] -#[path = "tests.rs"] -mod tests; - -pub use super::markdown_pipeline::skip_markdown_block; -pub use block::capture_record_to_storyline_turn; -pub use draft::draft_stream_assistant_turn; diff --git a/crates/persisting-gateway/src/projection/dialogue/tests.rs b/crates/persisting-gateway/src/projection/dialogue/tests.rs deleted file mode 100644 index 1b9411fc6..000000000 --- a/crates/persisting-gateway/src/projection/dialogue/tests.rs +++ /dev/null @@ -1,244 +0,0 @@ -use serde_json::{Value, json}; - -use super::*; -use crate::Call; -use crate::config::CaptureLevel; -use crate::sink::{ - llm_request_record, llm_request_summary_record, llm_response_record, - llm_response_record_with_content, -}; -fn test_call() -> Call { - Call { - call_id: "call-test".into(), - trace_id: "trace-test".into(), - started_at: "2026-01-01T00:00:00Z".into(), - } -} - -const LEVEL: CaptureLevel = CaptureLevel::Dialogue; - -#[test] -fn proxy_nested_body_writes_plain_content() { - let req = llm_request_record( - Some("sess".into()), - Some("agent".into()), - "mock-model", - "/v1/chat/completions", - &json!({ - "protocol": "chat_completions", - "provider": "openai", - "body": {"messages":[{"role":"user","content":"你好"}],"model":"mock-model"}, - }), - ); - let resp = llm_response_record( - Some("sess".into()), - Some("agent".into()), - 200, - &json!({ - "protocol": "chat_completions", - "provider": "openai", - "usage": {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3}, - "body": { - "choices":[{"message":{"role":"assistant","content":"你好！"}}], - }, - }), - false, - &test_call(), - ); - let t1 = capture_record_to_storyline_turn(&req).unwrap(); - let t2 = capture_record_to_storyline_turn(&resp).unwrap(); - assert_eq!(t1.message, json!("你好")); - assert_eq!(t2.message, json!("你好！")); -} - -#[test] -fn llm_pair_projects_to_storyline_semantics() { - let req = llm_request_record( - Some("sess".into()), - None, - "deepseek-chat", - "/v1/chat/completions", - &json!({"messages":[{"role":"user","content":"你好"}]}), - ); - let resp = llm_response_record( - Some("sess".into()), - None, - 200, - &json!({ - "choices":[{"message":{"role":"assistant","content":"你好！"}}], - "usage":{"prompt_tokens":12,"completion_tokens":18,"total_tokens":30} - }), - false, - &test_call(), - ); - let user = capture_record_to_storyline_turn(&req).unwrap(); - let assistant = capture_record_to_storyline_turn(&resp).unwrap(); - assert_eq!(user.source, "user"); - assert_eq!(user.message, json!("你好")); - assert_eq!(assistant.source, "agent"); - assert_eq!(assistant.message, json!("你好！")); - assert_eq!( - assistant.metrics, - Some(json!({ - "prompt_tokens": 12, - "completion_tokens": 18, - "total_tokens": 30 - })) - ); -} - -#[test] -fn summary_request_writes_user_content_not_json() { - let req = llm_request_summary_record( - Some("sess".into()), - Some("agent".into()), - "deepseek-v4-pro", - "/v1/messages", - 111_027, - "messages", - "anthropic", - Some("hi".into()), - None, - &test_call(), - LEVEL, - None, - ); - let turn = capture_record_to_storyline_turn(&req).unwrap(); - assert_eq!(turn.message, json!("hi")); -} - -#[test] -fn stream_response_with_assistant_content_writes_plain_text() { - let resp = llm_response_record_with_content( - Some("sess".into()), - Some("agent".into()), - 200, - &json!({"body": "event: x\ndata: {}\n"}), - true, - Some("Hi! How can I help you?".into()), - &test_call(), - LEVEL, - ); - let turn = capture_record_to_storyline_turn(&resp).unwrap(); - assert_eq!(turn.message, json!("Hi! How can I help you?")); -} - -#[test] -fn skip_internal_suggestion_request_and_silent_response() { - let req = llm_request_summary_record( - Some("s".into()), - None, - "m", - "/v1/messages", - 100, - "messages", - "anthropic", - None, - None, - &test_call(), - LEVEL, - None, - ); - assert!(skip_markdown_block(&req)); - let resp = llm_response_record_with_content( - Some("s".into()), - None, - 200, - &json!({"body": "event: x\ndata: {}\n"}), - true, - Some(String::new()), - &test_call(), - LEVEL, - ); - assert!(skip_markdown_block(&resp)); -} - -#[test] -fn skip_count_tokens_request() { - let req = llm_request_summary_record( - Some("s".into()), - None, - "m", - "/v1/messages/count_tokens", - 1000, - "count_tokens", - "anthropic", - Some("huge context".into()), - None, - &test_call(), - LEVEL, - None, - ); - assert!(skip_markdown_block(&req)); -} - -#[test] -fn subagent_link_fields_in_markdown_block() { - use crate::session::storage::CaptureRoute; - use crate::subagent_link::enrich_record; - - let mut registry = crate::subagent_link::SubagentRegistry::default(); - let route = CaptureRoute { - root_session: Some("run-1".into()), - session_id: "sess".into(), - storage_session_id: "agent-abc".into(), - subagent_id: Some("abc".into()), - }; - let mut rec = llm_response_record_with_content( - Some("sess".into()), - Some("proxy".into()), - 200, - &json!({}), - false, - Some("done".into()), - &test_call(), - LEVEL, - ); - let _outcome = enrich_record( - &mut rec, - &route, - &axum::http::HeaderMap::new(), - None, - Some("done"), - &mut registry, - ); - let turn = capture_record_to_storyline_turn(&rec).unwrap(); - assert_eq!( - turn.extra - .as_ref() - .and_then(Value::as_object) - .and_then(|extra| extra.get("subagent_id")) - .and_then(|v| v.as_str()), - Some("abc") - ); - assert_eq!( - turn.extra - .as_ref() - .and_then(Value::as_object) - .and_then(|extra| extra.get("subagent_trajectory")) - .and_then(|v| v.as_str()), - Some("agent-abc.md") - ); - assert_eq!(turn.message, json!("done")); -} - -#[test] -fn skip_main_flash_companion_user_duplicate() { - let mut rec = llm_request_summary_record( - Some("sess".into()), - Some("proxy".into()), - "deepseek-v4-flash", - "/v1/messages", - 100, - "messages", - "anthropic", - Some("再次开三个subagent".into()), - None, - &test_call(), - LEVEL, - None, - ); - assert!(skip_markdown_block(&rec)); - rec.subagent_id = Some("abc".into()); - assert!(!skip_markdown_block(&rec)); -} diff --git a/crates/persisting-gateway/src/projection/frontmatter.rs b/crates/persisting-gateway/src/projection/frontmatter.rs deleted file mode 100644 index 3587d8a2e..000000000 --- a/crates/persisting-gateway/src/projection/frontmatter.rs +++ /dev/null @@ -1,421 +0,0 @@ -//! Session-level YAML frontmatter rollup for live markdown trajectories. - -use std::path::{Path, PathBuf}; - -use anyhow::{Context, Result}; -use chrono::{DateTime, Utc}; -use serde::{Deserialize, Serialize}; -use serde_json::{Map, Value, json}; - -use super::markdown_trajectory::format_duration_human; -use crate::session::client::resolve_client_meta_for_run_dir; -use crate::session::index::{SessionIndexStore, SessionSummary}; -use crate::session::snapshots::load_snapshot_turn_counts; -use crate::session::storage::{CaptureRoute, trajectory_run_dir}; -use persisting_pchronicle::document::{ - decode_agenticmd, encode_agenticmd, rewrite_agenticmd_storyline_metadata, -}; -use persisting_pchronicle::model::StorylineDocument; -use persisting_pchronicle::storage::is_subagent_session_storage_key; - -#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)] -pub struct SessionFrontmatterSummary { - pub session: String, - pub agent: String, - pub model: Option, - pub provider: Option, - pub started: Option, - pub duration: Option, - pub turns: u64, - pub total_tokens: u64, - pub estimated_cost_usd: Option, - pub subagents: Vec, - pub client: Option, -} - -/// Serialize YAML frontmatter block (including opening/closing `---`). -pub fn format_session_frontmatter(summary: &SessionFrontmatterSummary) -> Result { - encode_agenticmd(&storyline_metadata(summary)) - .map_err(|e| anyhow::anyhow!("pchronicle AgenticMD Storyline metadata: {e}")) -} - -/// Build rollup from markdown blocks + `sessions.json` + run directory siblings. -/// -/// When `story_user_turn_count` is set, it overrides markdown block counting. -pub fn build_session_frontmatter_summary( - storage: &Path, - agent_id: &str, - route: &CaptureRoute, - md_path: &Path, - story_user_turn_count: Option, -) -> Result { - let run_dir = trajectory_run_dir(storage, agent_id, route); - let client = resolve_client_meta_for_run_dir(storage, &run_dir); - - let turns = match story_user_turn_count { - Some(n) => n, - None => count_user_turns(md_path)?, - }; - let index_row = lookup_session_index(storage, agent_id, &route.session_id); - - let (model, provider, started, duration, total_tokens, cost) = - index_stats(index_row.as_ref(), &route.session_id); - - Ok(SessionFrontmatterSummary { - session: route.session_id.clone(), - agent: agent_id.to_string(), - model, - provider, - started, - duration, - turns, - total_tokens, - estimated_cost_usd: cost, - subagents: list_subagent_stems(&run_dir, &route.storage_session_id), - client, - }) -} - -fn count_user_turns(md_path: &Path) -> Result { - if !md_path.is_file() { - return Ok(0); - } - let story = decode_agenticmd(&std::fs::read_to_string(md_path)?)?; - Ok(story - .turns - .iter() - .filter(|turn| turn.source == "user") - .count() as u64) -} - -fn lookup_session_index( - storage: &Path, - agent_id: &str, - session_id: &str, -) -> Option { - SessionIndexStore::load(storage).ok().and_then(|index| { - index - .sessions - .into_iter() - .find(|s| s.agent_id == agent_id && s.session_id == session_id) - }) -} - -#[allow(clippy::type_complexity)] -fn index_stats( - row: Option<&SessionSummary>, - fallback_session: &str, -) -> ( - Option, - Option, - Option, - Option, - u64, - Option, -) { - let Some(row) = row else { - return (None, None, None, None, 0, None); - }; - let model = (!row.model.is_empty() && row.model != "_unknown" && row.model != "unknown") - .then(|| row.model.clone()); - let provider = (row.provider != "unknown").then(|| row.provider.clone()); - let started = Some(row.first_seen.to_rfc3339()); - let duration = Some(format_duration_between(row.first_seen, row.last_seen)); - let total_tokens = row.usage.total_tokens; - let cost = (row.estimated_cost_usd > 0.0).then_some(row.estimated_cost_usd); - let _ = fallback_session; - (model, provider, started, duration, total_tokens, cost) -} - -fn format_duration_between(start: DateTime, end: DateTime) -> String { - let secs = (end - start).num_seconds().max(0) as u64; - format_duration_human(secs) -} - -/// List subagent markdown stems in a run directory (excluding the main session file). -pub fn list_subagent_stems(run_dir: &Path, main_storage_key: &str) -> Vec { - let Ok(entries) = std::fs::read_dir(run_dir) else { - return Vec::new(); - }; - let mut stems: Vec = entries - .filter_map(|e| e.ok()) - .map(|e| e.path()) - .filter(|p| p.extension().and_then(|s| s.to_str()) == Some("md")) - .filter_map(|p| p.file_stem().and_then(|s| s.to_str()).map(String::from)) - .filter(|stem| is_subagent_session_storage_key(stem)) - .filter(|stem| stem != main_storage_key) - .collect(); - stems.sort(); - stems.dedup(); - stems -} - -/// Replace the YAML preamble of an on-disk markdown trajectory (blocks unchanged). -pub fn refresh_document_frontmatter( - storage: &Path, - agent_id: &str, - route: &CaptureRoute, - md_path: &Path, - story_user_turn_count: Option, -) -> Result { - if !md_path.is_file() { - anyhow::bail!("markdown file missing: {}", md_path.display()); - } - let summary = build_session_frontmatter_summary( - storage, - agent_id, - route, - md_path, - story_user_turn_count, - )?; - let raw = - std::fs::read_to_string(md_path).with_context(|| format!("read {}", md_path.display()))?; - let mut document = - decode_agenticmd(&raw).with_context(|| format!("parse AgenticMD {}", md_path.display()))?; - apply_summary(&mut document, &summary); - rewrite_agenticmd_storyline_metadata(md_path, &document) - .with_context(|| format!("refresh frontmatter {}", md_path.display()))?; - Ok(summary) -} - -fn storyline_metadata(summary: &SessionFrontmatterSummary) -> StorylineDocument { - let mut document = StorylineDocument::new(&summary.session, &summary.agent); - apply_summary(&mut document, summary); - document -} - -fn apply_summary(document: &mut StorylineDocument, summary: &SessionFrontmatterSummary) { - document.session_id = summary.session.clone(); - document.agent.id = summary.agent.clone(); - document.agent.name = Some(summary.agent.clone()); - document.agent.model_name = summary.model.clone(); - document.child_session_ids = (!summary.subagents.is_empty()).then(|| summary.subagents.clone()); - - let mut agent_extra = Map::new(); - if let Some(provider) = &summary.provider { - agent_extra.insert("provider".into(), json!(provider)); - } - if let Some(client) = &summary.client - && let Ok(value) = serde_json::to_value(client) - { - agent_extra.insert("client".into(), value); - } - document.agent.extra = (!agent_extra.is_empty()).then_some(Value::Object(agent_extra)); - - let mut run_extra = Map::new(); - if let Some(started) = &summary.started { - run_extra.insert("started_at".into(), json!(started)); - } - if let Some(duration) = &summary.duration { - run_extra.insert("duration".into(), json!(duration)); - } - document.extra = (!run_extra.is_empty()).then_some(Value::Object(run_extra)); - document.final_metrics = Some(json!({ - "turn_count": summary.turns, - "total_tokens": summary.total_tokens, - "estimated_cost_usd": summary.estimated_cost_usd, - })); -} - -/// Refresh frontmatter for every `{run_dir}/*.md` file. -/// -/// Uses `.capture/story_snapshots.json` turn counts when `turn_counts` is not provided. -pub fn refresh_run_markdown_frontmatter( - storage: &Path, - agent_id: &str, - root_session: &str, - turn_counts: Option<&std::collections::HashMap>, -) -> Result> { - let run_dir = storage.join(agent_id).join(root_session); - if !run_dir.is_dir() { - return Ok(Vec::new()); - } - let loaded = load_snapshot_turn_counts(storage).unwrap_or_default(); - let turn_counts = turn_counts.unwrap_or(&loaded); - let mut out = Vec::new(); - for entry in std::fs::read_dir(&run_dir)? { - let entry = entry?; - let path = entry.path(); - if path.extension().and_then(|s| s.to_str()) != Some("md") { - continue; - } - let Some(stem) = path.file_stem().and_then(|s| s.to_str()) else { - continue; - }; - let route = CaptureRoute::for_run_markdown_stem(root_session, stem); - let turns = turn_counts.get(stem).copied(); - let summary = refresh_document_frontmatter(storage, agent_id, &route, &path, turns)?; - out.push((path, summary)); - } - out.sort_by(|a, b| a.0.file_name().cmp(&b.0.file_name())); - Ok(out) -} - -/// One-line human summary for stderr (run end). -pub fn format_run_summary_line(md_path: &Path, summary: &SessionFrontmatterSummary) -> String { - let cost = summary - .estimated_cost_usd - .map(|c| format!(" ${c:.4}")) - .unwrap_or_default(); - let duration = summary.duration.as_deref().unwrap_or("?"); - let subagents = if summary.subagents.is_empty() { - String::new() - } else { - format!(" | subagents: {}", summary.subagents.join(", ")) - }; - format!( - "Session: {} | turns: {} | tokens: {}{} | duration: {}{}", - md_path.display(), - summary.turns, - summary.total_tokens, - cost, - duration, - subagents - ) -} - -#[cfg(test)] -mod tests { - use super::*; - use persisting_pchronicle::model::StorylineTurn; - - fn user_document(body: &str) -> String { - let mut story = StorylineDocument::new("run-test", "agent"); - story.turns.push(StorylineTurn { - id: 1, - kind: Some("llm.request".into()), - timestamp: None, - source: "user".into(), - message: serde_json::json!(body), - reasoning_content: None, - reasoning_effort: None, - tool_calls: None, - observation: None, - metrics: None, - model_name: None, - llm_call_count: None, - is_copied_context: None, - latency_ms: None, - ttft_ms: None, - extra: None, - env: None, - prompt: None, - finished_at: None, - }); - encode_agenticmd(&story).unwrap() - } - - #[test] - fn refresh_frontmatter_preserves_blocks() { - let dir = tempfile::tempdir().unwrap(); - let md = dir.path().join("run-test.md"); - std::fs::write(&md, user_document("hello")).unwrap(); - let route = CaptureRoute { - root_session: Some("run-test".into()), - session_id: "run-test".into(), - storage_session_id: "run-test".into(), - subagent_id: None, - }; - let summary = refresh_document_frontmatter(dir.path(), "agent", &route, &md, None).unwrap(); - assert_eq!(summary.turns, 1); - assert_eq!(summary.session, "run-test"); - let text = std::fs::read_to_string(&md).unwrap(); - assert!(text.contains("turn_count: 1")); - assert!(text.contains("hello")); - } - - #[test] - fn frontmatter_prefers_story_turn_count_over_markdown_blocks() { - let dir = tempfile::tempdir().unwrap(); - let md = dir.path().join("run-test.md"); - std::fs::write(&md, user_document("only one block in md")).unwrap(); - let route = CaptureRoute { - root_session: Some("run-test".into()), - session_id: "run-test".into(), - storage_session_id: "run-test".into(), - subagent_id: None, - }; - - let call = crate::Call { - call_id: "c1".into(), - trace_id: "t".into(), - started_at: "2026-01-01T00:00:00Z".into(), - }; - let mut records = Vec::new(); - for text in ["first", "second"] { - records.push(crate::sink::llm_request_summary_record( - Some("run-test".into()), - Some("agent".into()), - "m", - "/v1/chat/completions", - 10, - "chat", - "openai", - Some(text.into()), - None, - &call, - crate::config::CaptureLevel::Dialogue, - None, - )); - } - let _story = crate::engine::rebuild_session_story("run-test", "run-test", &records); - - let from_md = - build_session_frontmatter_summary(dir.path(), "agent", &route, &md, None).unwrap(); - assert_eq!(from_md.turns, 1); - - let from_story = - build_session_frontmatter_summary(dir.path(), "agent", &route, &md, Some(2)).unwrap(); - assert_eq!(from_story.turns, 2); - } - - #[test] - fn refresh_run_uses_persisted_story_snapshots() { - use std::collections::HashMap; - - let dir = tempfile::tempdir().unwrap(); - let agent = "agent-1"; - let root = "run-test"; - let run_dir = dir.path().join(agent).join(root); - std::fs::create_dir_all(&run_dir).unwrap(); - let md = run_dir.join(format!("{root}.md")); - std::fs::write(&md, user_document("hello")).unwrap(); - - let call = crate::Call { - call_id: "c1".into(), - trace_id: "t".into(), - started_at: "2026-01-01T00:00:00Z".into(), - }; - let req = crate::sink::llm_request_summary_record( - Some(root.into()), - Some(agent.into()), - "m", - "/v1/chat/completions", - 10, - "chat", - "openai", - Some("hello".into()), - None, - &call, - crate::config::CaptureLevel::Dialogue, - None, - ); - let mut req2 = req.clone(); - req2.call_id = Some("c2".into()); - req2.payload["user_content"] = serde_json::json!("second turn"); - let story = crate::engine::rebuild_session_story(root, root, &[req, req2]); - assert_eq!(crate::engine::story_user_turn_count(&story), 2); - - let mut snapshots = HashMap::new(); - snapshots.insert(root.to_string(), story); - crate::engine::persist_story_snapshots(dir.path(), &snapshots).unwrap(); - - let entries = refresh_run_markdown_frontmatter(dir.path(), agent, root, None).unwrap(); - let summary = entries - .iter() - .find(|(p, _)| p == &md) - .map(|(_, s)| s) - .expect("main session summary"); - assert_eq!(summary.turns, 2); - } -} diff --git a/crates/persisting-gateway/src/projection/markdown.rs b/crates/persisting-gateway/src/projection/markdown.rs deleted file mode 100644 index 6bae5a015..000000000 --- a/crates/persisting-gateway/src/projection/markdown.rs +++ /dev/null @@ -1,70 +0,0 @@ -//! Session AgenticMD writes through the authoritative Storyline model. - -use std::path::Path; - -use anyhow::{Context, Result}; -use persisting_pchronicle::document::upsert_agenticmd_turn; -use persisting_pchronicle::model::{StorylineDocument, StorylineTurn}; -use serde_json::{Map, Value}; - -use crate::session::client::resolve_client_meta_for_run_dir; - -/// Insert or replace one Storyline turn in the session's readable AgenticMD view. -pub fn upsert_storyline_turn( - path: &Path, - document: &StorylineDocument, - edit_key: &str, - turn: &StorylineTurn, -) -> Result { - let document = if path.exists() { - document.clone() - } else { - document_with_client_metadata(path, document)? - }; - upsert_agenticmd_turn(path, &document, turn, edit_key) - .with_context(|| format!("upsert {}", path.display())) -} - -fn document_with_client_metadata( - path: &Path, - document: &StorylineDocument, -) -> Result { - let mut document = document.clone(); - let client = path.parent().and_then(|run_dir| { - run_dir - .parent() - .and_then(|agent_dir| agent_dir.parent()) - .and_then(|storage| resolve_client_meta_for_run_dir(storage, run_dir)) - }); - if let Some(client) = client { - let extra = document - .agent - .extra - .get_or_insert_with(|| Value::Object(Map::new())); - let object = extra - .as_object_mut() - .ok_or_else(|| anyhow::anyhow!("Storyline agent.extra must be an object"))?; - object.insert( - "client".into(), - serde_json::to_value(client).context("serialize session client metadata")?, - ); - } - Ok(document) -} - -/// Human-readable duration for session summaries (`42s`, `3m12s`, `1h5m`). -pub(crate) fn format_duration_human(secs: u64) -> String { - if secs < 60 { - return format!("{secs}s"); - } - if secs < 3600 { - return format!("{}m{}s", secs / 60, secs % 60); - } - let hours = secs / 3600; - let mins = (secs % 3600) / 60; - if mins == 0 { - format!("{hours}h") - } else { - format!("{hours}h{mins}m") - } -} diff --git a/crates/persisting-gateway/src/projection/mod.rs b/crates/persisting-gateway/src/projection/mod.rs deleted file mode 100644 index 7e5aa5250..000000000 --- a/crates/persisting-gateway/src/projection/mod.rs +++ /dev/null @@ -1,19 +0,0 @@ -//! Capture-time human-readable trajectory projection. -//! -//! This layer interprets provider payloads, applies live-only eligibility and -//! history deduplication, and orchestrates AgenticMD upserts. Generic formats, -//! schemas, offline projection, and physical storage remain pChronicle-owned. - -pub mod dialogue; -pub mod frontmatter; - -#[path = "markdown.rs"] -pub mod markdown_trajectory; - -#[path = "pipeline.rs"] -pub mod markdown_pipeline; - -#[path = "policy.rs"] -pub mod markdown_policy; - -pub mod reconcile; diff --git a/crates/persisting-gateway/src/projection/pipeline.rs b/crates/persisting-gateway/src/projection/pipeline.rs deleted file mode 100644 index 7bbb5702f..000000000 --- a/crates/persisting-gateway/src/projection/pipeline.rs +++ /dev/null @@ -1,593 +0,0 @@ -//! EventRecord → session markdown: eligibility, block build, live upsert. -//! -//! All paths (live `-f md`, materialize, reconcile) go through [`MarkdownPipeline`]. -//! Live session actors hold [`LiveMarkdownWriter`] (pipeline + target path + upsert). - -use std::collections::{BTreeSet, HashMap, HashSet}; -use std::path::PathBuf; - -use anyhow::{Context, Result}; -use persisting_pchronicle::model::{StorylineDocument, StorylineTurn}; -use serde_json::Value; - -use super::dialogue::capture_record_to_storyline_turn; -use super::markdown_policy::should_skip_record; -use super::markdown_trajectory::upsert_storyline_turn; -use crate::dialogue_extract::count_visible_user_messages; -use crate::record::{EventRecord, EventRecordExt}; -use crate::session::storage::{CaptureRoute, trajectory_run_dir}; -use persisting_pchronicle::storage::session_markdown_write_path_for_key; - -/// Per-session sequential state: static filters + Claude Code history-replay dedup. -#[derive(Debug, Default)] -pub struct MarkdownPipeline { - last_user_message_count: usize, - skipped_call_ids: HashSet, -} - -impl MarkdownPipeline { - pub fn static_skip(rec: &EventRecord) -> bool { - should_skip_record(rec) - } - - pub fn should_skip(&mut self, rec: &EventRecord) -> bool { - if self.skip_history_replay(rec) { - return true; - } - Self::static_skip(rec) - } - - pub fn skips_draft(&self, call_id: &str) -> bool { - self.skipped_call_ids.contains(call_id) - } - - pub fn try_storyline_turn(&mut self, rec: &EventRecord) -> Result> { - if self.should_skip(rec) { - return Ok(None); - } - Ok(Some(capture_record_to_storyline_turn(rec)?)) - } - - pub fn storyline_turns_from_records(records: &[EventRecord]) -> Result> { - let mut pipeline = Self::default(); - let mut turns = Vec::new(); - for rec in records { - if let Some(mut turn) = pipeline.try_storyline_turn(rec)? { - turn.id = turns.len() as i64 + 1; - turns.push(turn); - } - } - Ok(turns) - } - - pub fn call_ids_from_records(records: &[EventRecord]) -> BTreeSet { - let mut pipeline = Self::default(); - let mut ids = BTreeSet::new(); - for rec in records { - if pipeline.should_skip(rec) { - continue; - } - if let Some(id) = rec.call_id.as_deref().filter(|s| !s.is_empty()) { - ids.insert(id.to_string()); - } - } - ids - } - - fn skip_history_replay(&mut self, rec: &EventRecord) -> bool { - match rec.kind.as_str() { - "llm.request" => { - // Codex / Responses tool rounds resend the full `input` array; visible user - // turn count stays flat while `user_content` is a new tool_result block. - if is_tool_result_continuation(rec) { - return false; - } - let Some(count) = rec - .payload - .get("user_message_count") - .and_then(|v| v.as_u64()) - .map(|n| n as usize) - else { - return false; - }; - if count <= self.last_user_message_count { - if let Some(cid) = &rec.call_id { - self.skipped_call_ids.insert(cid.clone()); - } - return true; - } - self.last_user_message_count = count; - false - } - "llm.response" | "llm.response.stream" => rec - .call_id - .as_ref() - .is_some_and(|cid| self.skipped_call_ids.contains(cid)), - _ => false, - } - } -} - -/// Responses / Codex tool-output follow-up (not Claude history replay). -fn is_tool_result_continuation(rec: &EventRecord) -> bool { - rec.visible_user_text() - .is_some_and(|t| t.contains("```tool_result:")) -} - -/// Where live markdown for one session is written. -#[derive(Debug, Clone)] -pub struct MarkdownTarget { - pub route: CaptureRoute, - pub agent_id: String, - pub storage: PathBuf, -} - -impl MarkdownTarget { - pub fn new(route: CaptureRoute, agent_id: impl Into, storage: PathBuf) -> Self { - Self { - route, - agent_id: agent_id.into(), - storage, - } - } - - pub fn path(&self) -> PathBuf { - let run_dir = trajectory_run_dir(self.storage.as_path(), &self.agent_id, &self.route); - session_markdown_write_path_for_key(&run_dir, &self.route.storage_session_id) - } -} - -/// Live markdown upsert for one capture session (`-f md`). -#[derive(Debug)] -pub struct LiveMarkdownWriter { - target: MarkdownTarget, - enabled: bool, - pipeline: MarkdownPipeline, - document: StorylineDocument, - turn_ids: HashMap<(String, String), i64>, - next_turn_id: i64, -} - -impl LiveMarkdownWriter { - pub fn new(target: MarkdownTarget, enabled: bool) -> Self { - let mut document = StorylineDocument::new( - target.route.storage_session_id.clone(), - target.agent_id.clone(), - ); - document.run_id = target.route.root_session.clone(); - Self { - target, - enabled, - pipeline: MarkdownPipeline::default(), - document, - turn_ids: HashMap::new(), - next_turn_id: 1, - } - } - - pub fn path(&self) -> PathBuf { - self.target.path() - } - - pub fn write_record(&mut self, rec: &EventRecord) -> Result<()> { - if !self.enabled { - return Ok(()); - } - let Some(mut turn) = self.pipeline.try_storyline_turn(rec)? else { - return Ok(()); - }; - let call_id = rec.call_id.as_deref().unwrap_or(""); - turn.id = self.turn_id(call_id, &turn.source); - let path = self.target.path(); - upsert_storyline_turn(&path, &self.document, call_id, &turn) - .with_context(|| format!("markdown upsert {}", path.display()))?; - Ok(()) - } - - pub fn write_draft(&mut self, call_id: &str, mut turn: StorylineTurn) -> Result<()> { - if !self.enabled || self.pipeline.skips_draft(call_id) { - return Ok(()); - } - turn.id = self.turn_id(call_id, &turn.source); - let path = self.target.path(); - upsert_storyline_turn(&path, &self.document, call_id, &turn) - .with_context(|| format!("markdown draft upsert {}", path.display()))?; - Ok(()) - } - - fn turn_id(&mut self, edit_key: &str, source: &str) -> i64 { - let key = (edit_key.to_string(), source.to_string()); - if let Some(id) = self.turn_ids.get(&key) { - return *id; - } - let id = self.next_turn_id; - self.next_turn_id = self.next_turn_id.saturating_add(1); - self.turn_ids.insert(key, id); - id - } -} - -/// Stamp request payload fields consumed by [`MarkdownPipeline`]. -pub fn stamp_request_payload(payload: &mut Value, body_json: Option<&Value>) { - if let Some(body) = body_json { - payload["user_message_count"] = serde_json::json!(count_visible_user_messages(body)); - } -} - -/// Alias kept for tests and CLI view helpers. -pub fn skip_markdown_block(rec: &EventRecord) -> bool { - should_skip_record(rec) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::Call; - use crate::config::CaptureLevel; - use crate::session::storage::CaptureRoute; - use crate::sink::{llm_request_summary_record, llm_response_record_with_content}; - use persisting_pchronicle::document::decode_agenticmd; - use serde_json::json; - - const LEVEL: CaptureLevel = CaptureLevel::Dialogue; - - fn test_call(id: &str) -> Call { - Call { - call_id: id.into(), - trace_id: id.into(), - started_at: "2026-01-01T00:00:00Z".into(), - } - } - - fn route() -> CaptureRoute { - CaptureRoute { - root_session: Some("run-test".into()), - session_id: "sess-1".into(), - storage_session_id: "sess-1".into(), - subagent_id: None, - } - } - - fn request(id: &str, user: &str, user_message_count: u64, body: Option) -> EventRecord { - let mut rec = llm_request_summary_record( - Some("sess-1".into()), - Some("agent-1".into()), - "deepseek-v4-pro", - "/v1/messages", - 100, - "messages", - "anthropic", - Some(user.into()), - None, - &test_call(id), - LEVEL, - body.as_ref(), - ); - rec.payload["user_message_count"] = json!(user_message_count); - rec - } - - fn response(id: &str, text: &str) -> EventRecord { - llm_response_record_with_content( - Some("sess-1".into()), - Some("agent-1".into()), - 200, - &json!({"status": 200}), - true, - Some(text.into()), - &test_call(id), - LEVEL, - ) - } - - fn messages_body(user_lines: &[&str]) -> Value { - let messages: Vec = user_lines - .iter() - .map(|line| { - json!({ - "role": "user", - "content": [{"type": "text", "text": line}] - }) - }) - .collect(); - json!({"messages": messages, "model": "deepseek-v4-pro"}) - } - - #[test] - fn stamp_request_payload_sets_user_message_count() { - let body = messages_body(&["hi", "你好"]); - let mut payload = json!({}); - stamp_request_payload(&mut payload, Some(&body)); - assert_eq!(payload["user_message_count"], json!(2)); - } - - #[test] - fn stamp_request_payload_noop_without_body() { - let mut payload = json!({"model": "m"}); - stamp_request_payload(&mut payload, None); - assert!(payload.get("user_message_count").is_none()); - } - - #[test] - fn sink_summary_record_stamps_count_from_body_json() { - let body = messages_body(&["a", "b", "c"]); - let rec = llm_request_summary_record( - Some("s".into()), - None, - "m", - "/v1/messages", - 1, - "messages", - "anthropic", - Some("c".into()), - None, - &test_call("c1"), - LEVEL, - Some(&body), - ); - assert_eq!(rec.payload["user_message_count"], json!(3)); - } - - #[test] - fn static_skip_internal_count_tokens_and_empty_turns() { - let count_tokens = llm_request_summary_record( - Some("s".into()), - None, - "m", - "/v1/messages/count_tokens", - 1, - "count_tokens", - "anthropic", - Some("ctx".into()), - None, - &test_call("c1"), - LEVEL, - None, - ); - assert!(skip_markdown_block(&count_tokens)); - - let empty_user = request("c2", " ", 1, None); - assert!(skip_markdown_block(&empty_user)); - - let empty_resp = llm_response_record_with_content( - Some("s".into()), - None, - 200, - &json!({"status": 200}), - true, - Some("".into()), - &test_call("c3"), - LEVEL, - ); - assert!(skip_markdown_block(&empty_resp)); - - let mut partial = response("c4", "wip"); - partial.payload["stream_partial"] = json!(true); - assert!(skip_markdown_block(&partial)); - } - - #[test] - fn static_skip_spawn_link_is_visible() { - let mut rec = response("c1", "linked"); - rec.kind = "llm.spawn_link".into(); - assert!(!skip_markdown_block(&rec)); - } - - #[test] - fn intentional_duplicate_user_text_with_increasing_count_both_kept() { - let mut p = MarkdownPipeline::default(); - assert!( - p.try_storyline_turn(&request("c1", "hi", 1, None)) - .unwrap() - .is_some() - ); - assert!( - p.try_storyline_turn(&response("c1", "Hello")) - .unwrap() - .is_some() - ); - assert!( - p.try_storyline_turn(&request("c2", "hi", 2, None)) - .unwrap() - .is_some() - ); - assert!( - p.try_storyline_turn(&response("c2", "Hi again")) - .unwrap() - .is_some() - ); - } - - #[test] - fn skips_history_replay_without_new_user_turn() { - let mut p = MarkdownPipeline::default(); - assert!( - p.try_storyline_turn(&request("c1", "hi", 1, None)) - .unwrap() - .is_some() - ); - assert!( - p.try_storyline_turn(&response("c1", "Hello")) - .unwrap() - .is_some() - ); - - let replay = request("c3", "hi", 1, None); - assert!(p.try_storyline_turn(&replay).unwrap().is_none()); - assert!(p.skips_draft("c3")); - assert!( - p.try_storyline_turn(&response("c3", "internal")) - .unwrap() - .is_none() - ); - } - - #[test] - fn codex_tool_result_round_keeps_assistant_when_user_turn_count_flat() { - let mut tool_req = request("c-tool", "x", 2, None); - tool_req.payload["user_content"] = json!("```tool_result:call_x\nchunk output\n```"); - - let mut p = MarkdownPipeline::default(); - assert!( - p.try_storyline_turn(&request("c1", "hi", 1, None)) - .unwrap() - .is_some() - ); - assert!( - p.try_storyline_turn(&response("c1", "Hello")) - .unwrap() - .is_some() - ); - assert!( - p.try_storyline_turn(&request("c2", "review", 2, None)) - .unwrap() - .is_some() - ); - assert!( - p.try_storyline_turn(&response("c2", "running tools")) - .unwrap() - .is_some() - ); - assert!(p.try_storyline_turn(&tool_req).unwrap().is_some()); - assert!( - p.try_storyline_turn(&response("c-tool", "Let me dig in.")) - .unwrap() - .is_some() - ); - assert!( - p.try_storyline_turn(&response("c-final", "Full design review.")) - .unwrap() - .is_some() - ); - } - - #[test] - fn claude_code_three_turn_session_materializes_visible_dialogue_only() { - let records = vec![ - request("call-1", "hi", 1, None), - response("call-1", "Hi! What can I help you with?"), - request("call-2", "hi", 2, None), - response("call-2", "Hi! What can I help you with?"), - request("call-3", "hi", 2, None), - response( - "call-3", - "what's the status of the capture storage feature?", - ), - request("call-4", "你好", 3, None), - response("call-4", "你好！有什么我可以帮你的吗？"), - ]; - let turns = MarkdownPipeline::storyline_turns_from_records(&records).unwrap(); - let bodies: Vec<_> = turns - .iter() - .map(|turn| turn.message.as_str().unwrap_or_default().to_string()) - .collect(); - assert_eq!( - bodies, - vec![ - "hi".to_string(), - "Hi! What can I help you with?".to_string(), - "hi".to_string(), - "Hi! What can I help you with?".to_string(), - "你好".to_string(), - "你好！有什么我可以帮你的吗？".to_string(), - ] - ); - } - - #[test] - fn call_ids_from_records_matches_agenticmd_blocks_from_records() { - let records = vec![ - request("call-1", "hi", 1, None), - response("call-1", "a"), - request("call-2", "replay", 1, None), - response("call-2", "orphan"), - request("call-3", "next", 2, None), - response("call-3", "b"), - ]; - let ids = MarkdownPipeline::call_ids_from_records(&records); - assert_eq!(ids.len(), 2); - assert!(ids.contains("call-1")); - assert!(!ids.contains("call-2")); - assert!(ids.contains("call-3")); - } - - #[test] - fn skip_main_flash_companion_on_main_session() { - let rec = llm_request_summary_record( - Some("sess".into()), - Some("proxy".into()), - "deepseek-v4-flash", - "/v1/messages", - 100, - "messages", - "anthropic", - Some("再次开三个subagent".into()), - None, - &test_call("c1"), - LEVEL, - None, - ); - assert!(skip_markdown_block(&rec)); - } - - #[test] - fn live_writer_disabled_is_noop() { - let dir = tempfile::tempdir().unwrap(); - let target = MarkdownTarget::new(route(), "agent-1", dir.path().to_path_buf()); - let mut writer = LiveMarkdownWriter::new(target, false); - writer.write_record(&request("c1", "hi", 1, None)).unwrap(); - let path = MarkdownTarget::new(route(), "agent-1", dir.path().to_path_buf()).path(); - assert!(!path.exists()); - } - - #[test] - fn live_writer_persists_dialogue_and_skips_replay() { - let dir = tempfile::tempdir().unwrap(); - let target = MarkdownTarget::new(route(), "agent-1", dir.path().to_path_buf()); - let path = target.path(); - let mut writer = LiveMarkdownWriter::new(target, true); - - writer.write_record(&request("c1", "hi", 1, None)).unwrap(); - writer.write_record(&response("c1", "Hello")).unwrap(); - writer.write_record(&request("c2", "hi", 2, None)).unwrap(); - writer.write_record(&response("c2", "Hi again")).unwrap(); - writer.write_record(&request("c3", "hi", 2, None)).unwrap(); - writer - .write_record(&response( - "c3", - "what's the status of the capture storage feature?", - )) - .unwrap(); - writer - .write_record(&request("c4", "你好", 3, None)) - .unwrap(); - writer.write_record(&response("c4", "你好！")).unwrap(); - - let story = decode_agenticmd(&std::fs::read_to_string(&path).unwrap()).unwrap(); - assert_eq!(story.turns.len(), 6); - assert_eq!(story.turns[0].message, json!("hi")); - assert_eq!(story.turns[1].message, json!("Hello")); - assert_eq!(story.turns[2].message, json!("hi")); - assert_eq!(story.turns[3].message, json!("Hi again")); - assert_eq!(story.turns[4].message, json!("你好")); - assert_eq!(story.turns[5].message, json!("你好！")); - } - - #[test] - fn live_writer_skips_draft_for_replayed_call_id() { - let dir = tempfile::tempdir().unwrap(); - let target = MarkdownTarget::new(route(), "agent-1", dir.path().to_path_buf()); - let path = target.path(); - let mut writer = LiveMarkdownWriter::new(target, true); - - writer.write_record(&request("c1", "hi", 1, None)).unwrap(); - writer.write_record(&request("c2", "hi", 1, None)).unwrap(); - - let draft_turn = capture_record_to_storyline_turn(&response("c2", "draft text")).unwrap(); - writer.write_draft("c2", draft_turn).unwrap(); - - let story = decode_agenticmd(&std::fs::read_to_string(&path).unwrap()).unwrap(); - assert_eq!(story.turns.len(), 1); - assert_eq!(story.turns[0].message, json!("hi")); - } -} diff --git a/crates/persisting-gateway/src/projection/policy.rs b/crates/persisting-gateway/src/projection/policy.rs deleted file mode 100644 index 4e2716cba..000000000 --- a/crates/persisting-gateway/src/projection/policy.rs +++ /dev/null @@ -1,53 +0,0 @@ -//! Markdown / Lance eligibility rules for [`EventRecord`] (storage layer only). - -use crate::dialogue_extract::is_subagent_shape_payload; -use crate::record::{EventRecord, EventRecordExt}; - -pub fn should_skip_record(rec: &EventRecord) -> bool { - match rec.kind.as_str() { - "llm.request" => { - if rec.is_internal_llm_request() { - return true; - } - if should_skip_main_flash_companion_request(rec) { - return true; - } - rec.visible_user_text().is_none() - } - "llm.response" | "llm.response.stream" => { - if rec.payload.get("stream_partial").and_then(|v| v.as_bool()) == Some(true) { - return true; - } - rec.visible_assistant_text().is_none() - } - "llm.spawn_link" => false, - "llm.call.cancelled" => true, - k if k.starts_with("session.") => true, - _ => false, - } -} - -pub fn should_refresh_frontmatter(rec: &EventRecord) -> bool { - matches!( - rec.kind.as_str(), - "llm.request" | "llm.response" | "llm.response.stream" - ) -} - -fn should_skip_main_flash_companion_request(rec: &EventRecord) -> bool { - if rec.subagent_id.is_some() { - return false; - } - let model = rec - .payload - .get("model") - .and_then(|m| m.as_str()) - .unwrap_or(""); - if !model.contains("flash") && !model.contains("haiku") { - return false; - } - if rec.visible_user_text().is_none() { - return false; - } - !is_subagent_shape_payload(&rec.payload) -} diff --git a/crates/persisting-gateway/src/projection/reconcile.rs b/crates/persisting-gateway/src/projection/reconcile.rs deleted file mode 100644 index 1c9953e61..000000000 --- a/crates/persisting-gateway/src/projection/reconcile.rs +++ /dev/null @@ -1,473 +0,0 @@ -//! Lightweight post-run reconcile: compare live markdown call_ids vs Lance dialogue records. - -use std::collections::{BTreeMap, BTreeSet}; -use std::path::{Path, PathBuf}; - -use anyhow::{Context, Result}; -use serde::{Deserialize, Serialize}; - -use crate::engine::{rebuild_session_story, story_call_ids, story_user_turn_count}; -use crate::record::EventRecord; -use persisting_pchronicle::document::index_agenticmd_path; - -use super::markdown_pipeline::MarkdownPipeline; - -pub use persisting_pchronicle::document::list_agenticmd_paths as list_run_markdown_paths; - -/// Per-session reconcile outcome. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct SessionReconcile { - pub session_id: String, - pub md_path: String, - pub md_block_count: usize, - pub md_call_ids: Vec, - pub event_call_ids: Vec, - pub story_call_ids: Vec, - pub story_turn_count: u64, - pub missing_in_md: Vec, - pub extra_in_md: Vec, - pub story_missing_in_md: Vec, - pub story_extra_in_md: Vec, - pub structural_issues: Vec, -} - -impl SessionReconcile { - pub fn ok(&self) -> bool { - self.missing_in_md.is_empty() - && self.extra_in_md.is_empty() - && self.story_missing_in_md.is_empty() - && self.story_extra_in_md.is_empty() - && self.structural_issues.is_empty() - } -} - -/// Run-level reconcile report written to `{storage}/.capture/reconcile.json`. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct RunReconcileReport { - pub root_session: String, - pub agent_id: String, - pub ok: bool, - pub finished_at: String, - pub sessions: Vec, -} - -/// Extract visible dialogue `call_id`s that event records would materialize into markdown. -pub fn expected_markdown_call_ids(records: &[EventRecord]) -> BTreeSet { - MarkdownPipeline::call_ids_from_records(records) -} - -/// Index one live markdown file: block count, call_ids, structural issues. -pub fn index_markdown_path(path: &Path) -> Result<(usize, BTreeSet, Vec)> { - let index = index_agenticmd_path(path)?; - Ok((index.block_count, index.call_ids, index.structural_issues)) -} - -fn set_diff(expected: &BTreeSet, actual: &BTreeSet) -> (Vec, Vec) { - let missing: Vec<_> = expected.difference(actual).cloned().collect(); - let extra: Vec<_> = actual.difference(expected).cloned().collect(); - (missing, extra) -} - -fn sorted_vec(set: &BTreeSet) -> Vec { - set.iter().cloned().collect() -} - -/// Compare one markdown file against event records for the same session. -pub fn reconcile_session( - session_id: &str, - root_session: &str, - md_path: &Path, - event_records: &[EventRecord], -) -> Result { - let (md_block_count, md_ids, structural_issues) = index_markdown_path(md_path)?; - let event_ids = expected_markdown_call_ids(event_records); - let (missing_in_md, extra_in_md) = set_diff(&event_ids, &md_ids); - - let story = if event_records.is_empty() { - None - } else { - Some(rebuild_session_story( - session_id, - root_session, - event_records, - )) - }; - let (story_call_ids_vec, story_turn_count, story_missing_in_md, story_extra_in_md) = - if let Some(ref story) = story { - let ids = story_call_ids(story); - let (sm, se) = set_diff(&ids, &md_ids); - (sorted_vec(&ids), story_user_turn_count(story), sm, se) - } else { - (Vec::new(), 0, Vec::new(), Vec::new()) - }; - - Ok(SessionReconcile { - session_id: session_id.to_string(), - md_path: md_path.display().to_string(), - md_block_count, - md_call_ids: sorted_vec(&md_ids), - event_call_ids: sorted_vec(&event_ids), - story_call_ids: story_call_ids_vec, - story_turn_count, - missing_in_md, - extra_in_md, - story_missing_in_md, - story_extra_in_md, - structural_issues, - }) -} - -/// Build a run-level report from per-session event record batches keyed by storage session id. -pub fn build_run_report( - root_session: &str, - agent_id: &str, - run_dir: &Path, - events_by_session: &BTreeMap>, -) -> Result { - let mut sessions = Vec::new(); - for md_path in list_run_markdown_paths(run_dir)? { - let session_id = md_path - .file_stem() - .and_then(|s| s.to_str()) - .unwrap_or("unknown") - .to_string(); - let records = events_by_session - .get(&session_id) - .map(Vec::as_slice) - .unwrap_or(&[]); - sessions.push(reconcile_session( - &session_id, - root_session, - &md_path, - records, - )?); - } - let ok = sessions.iter().all(SessionReconcile::ok); - Ok(RunReconcileReport { - root_session: root_session.to_string(), - agent_id: agent_id.to_string(), - ok, - finished_at: chrono::Utc::now().to_rfc3339(), - sessions, - }) -} - -/// Write report to `{storage}/.capture/reconcile.json`. -pub fn write_run_reconcile_report(storage: &Path, report: &RunReconcileReport) -> Result { - let dir = storage.join(".capture"); - std::fs::create_dir_all(&dir).context("create .capture")?; - let path = dir.join("reconcile.json"); - let json = serde_json::to_string_pretty(report).context("serialize reconcile report")?; - std::fs::write(&path, json).with_context(|| format!("write {}", path.display()))?; - Ok(path) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::Call; - use crate::config::CaptureLevel; - use crate::projection::dialogue::capture_record_to_storyline_turn; - use crate::sink::{llm_request_summary_record, llm_response_record_with_content}; - use persisting_pchronicle::document::upsert_agenticmd_turn; - use persisting_pchronicle::model::{StorylineDocument, StorylineTurn}; - use std::collections::BTreeMap as Map; - - fn write_turns(path: &Path, turns: &[(&str, &str, &str)]) { - let story = StorylineDocument::new("run-test", "agent"); - for (index, (call_id, role, body)) in turns.iter().enumerate() { - let source = if *role == "assistant" { "agent" } else { *role }; - let turn = StorylineTurn { - id: index as i64 + 1, - kind: Some(if source == "user" { - "llm.request".into() - } else { - "llm.response".into() - }), - timestamp: None, - source: source.into(), - message: serde_json::json!(body), - reasoning_content: None, - reasoning_effort: None, - tool_calls: None, - observation: None, - metrics: None, - model_name: None, - llm_call_count: (source == "agent").then_some(1), - is_copied_context: None, - latency_ms: None, - ttft_ms: None, - extra: None, - env: None, - prompt: None, - finished_at: None, - }; - upsert_agenticmd_turn(path, &story, &turn, call_id).unwrap(); - } - } - - #[test] - fn reconcile_detects_missing_and_extra_call_ids() { - let dir = tempfile::tempdir().unwrap(); - let md = dir.path().join("run-test.md"); - - let call = Call { - call_id: "call-a".into(), - trace_id: "t".into(), - started_at: "2026-01-01T00:00:00Z".into(), - }; - let mut req = llm_request_summary_record( - Some("run-test".into()), - Some("agent".into()), - "m", - "/v1/chat/completions", - 1, - "chat", - "openai", - Some("hello".into()), - None, - &call, - CaptureLevel::Dialogue, - None, - ); - req.seq = 1; - let mut resp = llm_response_record_with_content( - Some("run-test".into()), - Some("agent".into()), - 200, - &serde_json::json!({ "status": 200 }), - false, - Some("hi".into()), - &call, - CaptureLevel::Dialogue, - ); - resp.seq = 2; - let story = StorylineDocument::new("run-test", "agent"); - upsert_agenticmd_turn( - &md, - &story, - &capture_record_to_storyline_turn(&req).unwrap(), - "call-a", - ) - .unwrap(); - upsert_agenticmd_turn( - &md, - &story, - &capture_record_to_storyline_turn(&resp).unwrap(), - "call-a", - ) - .unwrap(); - let mut extra_call = call; - extra_call.call_id = "call-b".into(); - let orphan = llm_request_summary_record( - Some("run-test".into()), - Some("agent".into()), - "m", - "/v1/chat/completions", - 1, - "chat", - "openai", - Some("orphan".into()), - None, - &extra_call, - CaptureLevel::Dialogue, - None, - ); - - let report = reconcile_session("run-test", "run-test", &md, &[req, resp, orphan]).unwrap(); - assert!(report.extra_in_md.is_empty()); - assert_eq!(report.missing_in_md, vec!["call-b"]); - assert!(!report.ok()); - } - - #[test] - fn structural_issue_flags_excessive_blank_lines() { - assert!( - persisting_pchronicle::document::agenticmd_structural_issues("a\n\n\n\nb") - .contains(&"excessive_blank_lines".into()) - ); - assert!(persisting_pchronicle::document::agenticmd_structural_issues("a\n\nb").is_empty()); - } - - #[test] - fn expected_call_ids_skips_replayed_requests() { - let call = Call { - call_id: "call-a".into(), - trace_id: "t".into(), - started_at: "2026-01-01T00:00:00Z".into(), - }; - let mut req1 = llm_request_summary_record( - Some("run".into()), - Some("agent".into()), - "m", - "/v1/messages", - 1, - "messages", - "anthropic", - Some("hi".into()), - None, - &call, - CaptureLevel::Dialogue, - None, - ); - req1.call_id = Some("call-a".into()); - req1.payload["user_message_count"] = serde_json::json!(1); - - let mut req2 = req1.clone(); - req2.call_id = Some("call-replay".into()); - - let mut req3 = req1.clone(); - req3.call_id = Some("call-b".into()); - req3.payload["user_message_count"] = serde_json::json!(2); - req3.payload["user_content"] = serde_json::json!("你好"); - - let ids = expected_markdown_call_ids(&[req1, req2, req3]); - assert_eq!(ids.len(), 2); - assert!(ids.contains("call-a")); - assert!(ids.contains("call-b")); - assert!(!ids.contains("call-replay")); - } - - #[test] - fn reconcile_story_projection_aligns_when_md_matches() { - let dir = tempfile::tempdir().unwrap(); - let md = dir.path().join("run-test.md"); - write_turns( - &md, - &[("call-a", "user", "hello"), ("call-a", "assistant", "hi")], - ); - - let call = Call { - call_id: "call-a".into(), - trace_id: "t".into(), - started_at: "2026-01-01T00:00:00Z".into(), - }; - let req = llm_request_summary_record( - Some("run-test".into()), - Some("agent".into()), - "m", - "/v1/chat/completions", - 1, - "chat", - "openai", - Some("hello".into()), - None, - &call, - CaptureLevel::Dialogue, - None, - ); - let resp = llm_response_record_with_content( - Some("run-test".into()), - Some("agent".into()), - 200, - &serde_json::json!({ "status": 200 }), - false, - Some("hi".into()), - &call, - CaptureLevel::Dialogue, - ); - - let report = reconcile_session("run-test", "run-test", &md, &[req, resp]).unwrap(); - assert!(report.ok()); - assert_eq!(report.story_turn_count, 1); - assert_eq!(report.story_call_ids, vec!["call-a"]); - assert!(report.story_missing_in_md.is_empty()); - assert!(report.story_extra_in_md.is_empty()); - } - - #[test] - fn reconcile_story_detects_extra_md_call_id() { - let dir = tempfile::tempdir().unwrap(); - let md = dir.path().join("run-test.md"); - write_turns( - &md, - &[ - ("call-a", "user", "hello"), - ("call-a", "assistant", "hi"), - ("call-ghost", "user", "phantom"), - ], - ); - - let call = Call { - call_id: "call-a".into(), - trace_id: "t".into(), - started_at: "2026-01-01T00:00:00Z".into(), - }; - let req = llm_request_summary_record( - Some("run-test".into()), - Some("agent".into()), - "m", - "/v1/chat/completions", - 1, - "chat", - "openai", - Some("hello".into()), - None, - &call, - CaptureLevel::Dialogue, - None, - ); - let resp = llm_response_record_with_content( - Some("run-test".into()), - Some("agent".into()), - 200, - &serde_json::json!({}), - false, - Some("hi".into()), - &call, - CaptureLevel::Dialogue, - ); - - let report = reconcile_session("run-test", "run-test", &md, &[req, resp]).unwrap(); - assert!(report.story_extra_in_md.contains(&"call-ghost".to_string())); - assert!(!report.ok()); - } - - #[test] - fn build_run_report_ok_when_all_sessions_match() { - let dir = tempfile::tempdir().unwrap(); - let run_dir = dir.path().join("run-test"); - std::fs::create_dir_all(&run_dir).unwrap(); - let md = run_dir.join("run-test.md"); - write_turns( - &md, - &[("call-a", "user", "hello"), ("call-a", "assistant", "hi")], - ); - - let call = Call { - call_id: "call-a".into(), - trace_id: "t".into(), - started_at: "2026-01-01T00:00:00Z".into(), - }; - let req = llm_request_summary_record( - Some("run-test".into()), - Some("agent".into()), - "m", - "/v1/chat/completions", - 1, - "chat", - "openai", - Some("hello".into()), - None, - &call, - CaptureLevel::Dialogue, - None, - ); - let resp = llm_response_record_with_content( - Some("run-test".into()), - Some("agent".into()), - 200, - &serde_json::json!({}), - false, - Some("hi".into()), - &call, - CaptureLevel::Dialogue, - ); - - let mut events_by_session = Map::new(); - events_by_session.insert("run-test".to_string(), vec![req, resp]); - let report = build_run_report("run-test", "agent", &run_dir, &events_by_session).unwrap(); - assert!(report.ok); - assert_eq!(report.sessions.len(), 1); - } -} diff --git a/crates/persisting-gateway/src/protocol.rs b/crates/persisting-gateway/src/protocol.rs deleted file mode 100644 index d5edb4139..000000000 --- a/crates/persisting-gateway/src/protocol.rs +++ /dev/null @@ -1,109 +0,0 @@ -//! LLM API path → protocol kind. - -use serde::{Deserialize, Serialize}; - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum ProtocolKind { - ChatCompletions, - Messages, - Responses, - /// Gemini native `models/*:generateContent` API. - Gemini, - Embeddings, - CountTokens, - Realtime, - Detect, - Unknown, -} - -impl ProtocolKind { - /// Infer from incoming HTTP path (OpenAI-style layout). - pub fn from_path(path: &str) -> Self { - let p = path.trim_end_matches('/'); - if p.ends_with("/chat/completions") || p.ends_with("chat/completions") { - return Self::ChatCompletions; - } - if p.ends_with("/messages") || p.contains("/messages") && !p.contains("count_tokens") { - return Self::Messages; - } - if p.ends_with("/responses") { - return Self::Responses; - } - if p.contains("/models/") - && (p.ends_with(":generateContent") || p.ends_with(":streamGenerateContent")) - { - return Self::Gemini; - } - if p.ends_with("/embeddings") { - return Self::Embeddings; - } - if p.contains("count_tokens") || p.ends_with("/count-tokens") { - return Self::CountTokens; - } - if p.contains("/realtime") { - return Self::Realtime; - } - if p.ends_with("/detect") { - return Self::Detect; - } - Self::Unknown - } - - pub fn as_str(self) -> &'static str { - match self { - Self::ChatCompletions => "chat_completions", - Self::Messages => "messages", - Self::Responses => "responses", - Self::Gemini => "gemini", - Self::Embeddings => "embeddings", - Self::CountTokens => "count_tokens", - Self::Realtime => "realtime", - Self::Detect => "detect", - Self::Unknown => "unknown", - } - } - - pub fn parse(s: &str) -> Self { - match s { - "chat_completions" => Self::ChatCompletions, - "messages" => Self::Messages, - "responses" => Self::Responses, - "gemini" => Self::Gemini, - "embeddings" => Self::Embeddings, - "count_tokens" => Self::CountTokens, - "realtime" => Self::Realtime, - "detect" => Self::Detect, - _ => Self::Unknown, - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn paths() { - assert_eq!( - ProtocolKind::from_path("/v1/chat/completions"), - ProtocolKind::ChatCompletions - ); - assert_eq!( - ProtocolKind::from_path("/v1/messages"), - ProtocolKind::Messages - ); - assert_eq!( - ProtocolKind::from_path("/v1/embeddings"), - ProtocolKind::Embeddings - ); - assert_eq!( - ProtocolKind::from_path("/v1beta/models/gemini-2.5-pro:generateContent"), - ProtocolKind::Gemini - ); - assert_eq!( - ProtocolKind::from_path("/v1beta/models/gemini-2.5-pro:streamGenerateContent"), - ProtocolKind::Gemini - ); - } -} diff --git a/crates/persisting-gateway/src/provider.rs b/crates/persisting-gateway/src/provider.rs deleted file mode 100644 index 98c2ddb5b..000000000 --- a/crates/persisting-gateway/src/provider.rs +++ /dev/null @@ -1,129 +0,0 @@ -//! LLM gateway provider kinds. - -use serde::{Deserialize, Serialize}; - -#[cfg(test)] -use proptest::prelude::*; - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, Default)] -#[serde(rename_all = "camelCase")] -pub enum ProviderKind { - #[default] - OpenAi, - Anthropic, - Gemini, - Vertex, - Bedrock, - Azure, - Copilot, - /// Custom upstream only (passthrough). - Custom, -} - -impl ProviderKind { - pub fn parse(s: &str) -> Self { - match s.to_ascii_lowercase().as_str() { - "openai" | "open_ai" => Self::OpenAi, - "anthropic" => Self::Anthropic, - "gemini" => Self::Gemini, - "vertex" => Self::Vertex, - "bedrock" => Self::Bedrock, - "azure" => Self::Azure, - "copilot" => Self::Copilot, - "custom" => Self::Custom, - _ => Self::Custom, - } - } - - pub fn as_str(self) -> &'static str { - match self { - Self::OpenAi => "openai", - Self::Anthropic => "anthropic", - Self::Gemini => "gemini", - Self::Vertex => "vertex", - Self::Bedrock => "bedrock", - Self::Azure => "azure", - Self::Copilot => "copilot", - Self::Custom => "custom", - } - } - - /// Default API host when `upstream` is not set (agentgateway defaults). - pub fn default_host(self) -> Option<&'static str> { - match self { - Self::OpenAi => Some("api.openai.com"), - Self::Anthropic => Some("api.anthropic.com"), - Self::Gemini => Some("generativelanguage.googleapis.com"), - Self::Copilot => Some("api.githubcopilot.com"), - Self::Vertex | Self::Bedrock | Self::Azure | Self::Custom => None, - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn provider_strategy() -> impl Strategy { - prop_oneof![ - Just(ProviderKind::OpenAi), - Just(ProviderKind::Anthropic), - Just(ProviderKind::Gemini), - Just(ProviderKind::Vertex), - Just(ProviderKind::Bedrock), - Just(ProviderKind::Azure), - Just(ProviderKind::Copilot), - Just(ProviderKind::Custom), - ] - } - - fn mixed_case(value: &str, uppercase: bool) -> String { - if uppercase { - value.to_ascii_uppercase() - } else { - value.to_ascii_lowercase() - } - } - - proptest! { - #[test] - fn canonical_names_roundtrip_through_parse(kind in provider_strategy()) { - prop_assert_eq!(ProviderKind::parse(kind.as_str()), kind); - } - - #[test] - fn canonical_names_are_case_insensitive( - kind in provider_strategy(), - uppercase in any::(), - ) { - prop_assert_eq!(ProviderKind::parse(&mixed_case(kind.as_str(), uppercase)), kind); - } - - #[test] - fn openai_accepts_both_public_spellings(uppercase in any::()) { - prop_assert_eq!( - ProviderKind::parse(&mixed_case("openai", uppercase)), - ProviderKind::OpenAi, - ); - prop_assert_eq!( - ProviderKind::parse(&mixed_case("open_ai", uppercase)), - ProviderKind::OpenAi, - ); - } - - #[test] - fn default_hosts_match_only_providers_with_builtin_endpoints(kind in provider_strategy()) { - let expected = match kind { - ProviderKind::OpenAi => Some("api.openai.com"), - ProviderKind::Anthropic => Some("api.anthropic.com"), - ProviderKind::Gemini => Some("generativelanguage.googleapis.com"), - ProviderKind::Copilot => Some("api.githubcopilot.com"), - ProviderKind::Vertex - | ProviderKind::Bedrock - | ProviderKind::Azure - | ProviderKind::Custom => None, - }; - prop_assert_eq!(kind.default_host(), expected); - } - } -} diff --git a/crates/persisting-gateway/src/record.rs b/crates/persisting-gateway/src/record.rs deleted file mode 100644 index 5eca6f8f8..000000000 --- a/crates/persisting-gateway/src/record.rs +++ /dev/null @@ -1,345 +0,0 @@ -//! Capture-side behavior over the shared -//! [`EventRecord`](persisting_events::EventRecord) schema. - -use serde_json::Value; - -use super::dialogue_extract::{extract_assistant_text_from_json, extract_assistant_turn_from_sse}; -use crate::protocol::ProtocolKind; - -pub use persisting_events::EventRecord; - -pub use persisting_events::unix_now_ms; - -/// Parse an RFC3339 event timestamp into Unix milliseconds. -pub fn unix_ms_from_rfc3339(timestamp: &str) -> Option { - chrono::DateTime::parse_from_rfc3339(timestamp) - .ok() - .map(|value| value.timestamp_millis().max(0) as u64) -} - -/// Make the shared wall-clock fields complete before an event is persisted. -/// -/// Gateway producers historically populated the RFC3339 `timestamp` field but -/// left the canonical `timestamp_unix_ms` identity field empty. The sink is -/// the last common boundary for all Gateway records, so it is the right place -/// to backfill both fields without changing event ordering (`seq`). -pub(crate) fn ensure_timestamp(record: &mut EventRecord) { - if record.timestamp.is_none() { - record.timestamp = Some(now_rfc3339()); - } - if record.identity.timestamp_unix_ms.is_none() { - record.identity.timestamp_unix_ms = record - .timestamp - .as_deref() - .and_then(unix_ms_from_rfc3339) - .or_else(|| Some(unix_now_ms())); - } -} - -/// Capture-only interpretation of raw proxy payloads. -/// -/// The record schema belongs to the shared events contract; SSE and provider -/// payload extraction remain producer concerns and are extension behavior. -pub trait EventRecordExt { - /// Internal traffic (e.g. `count_tokens`) — not a dialogue turn. - fn is_internal_llm_request(&self) -> bool; - - /// Visible user text used by Capture's live dialogue projection. - fn visible_user_text(&self) -> Option; - - /// Visible assistant text used by Capture's live dialogue projection. - fn visible_assistant_text(&self) -> Option; -} - -impl EventRecordExt for EventRecord { - fn is_internal_llm_request(&self) -> bool { - if self.kind != "llm.request" { - return false; - } - if self - .payload - .get("protocol") - .and_then(|p| p.as_str()) - .is_some_and(|p| p == ProtocolKind::CountTokens.as_str()) - { - return true; - } - self.payload - .get("path") - .and_then(|p| p.as_str()) - .is_some_and(|path| ProtocolKind::from_path(path) == ProtocolKind::CountTokens) - } - - fn visible_user_text(&self) -> Option { - visible_user_from_payload(&self.payload) - } - - fn visible_assistant_text(&self) -> Option { - visible_assistant_from_payload(&self.payload) - } -} - -/// Parse structured message `content` (string or Anthropic-style blocks). -pub(crate) fn content_to_string(v: &Value) -> Option { - match v { - Value::String(s) => Some(s.clone()), - Value::Array(parts) => { - let out: Vec<_> = parts - .iter() - .filter_map(|p| p.get("text").and_then(|t| t.as_str())) - .collect(); - if out.is_empty() { - None - } else { - Some(out.join("\n")) - } - } - _ => None, - } -} - -fn visible_user_from_payload(payload: &Value) -> Option { - if let Some(s) = payload.get("user_content").and_then(|v| v.as_str()) { - return non_empty(s); - } - let messages = llm_inner_body(payload) - .and_then(|b| b.get("messages")) - .or_else(|| payload.get("messages"))? - .as_array()?; - for msg in messages.iter().rev() { - if msg.get("role").and_then(|r| r.as_str()) == Some("user") - && let Some(text) = msg.get("content").and_then(content_to_string) - { - return non_empty(&text); - } - } - None -} - -fn visible_assistant_from_payload(payload: &Value) -> Option { - if let Some(s) = payload.get("assistant_content").and_then(|v| v.as_str()) { - return non_empty(s); - } - if let Some(s) = payload.get("body").and_then(|b| b.as_str()) { - let text = extract_assistant_turn_from_sse(s); - if let Some(t) = non_empty(&text) { - return Some(t); - } - } - if let Some(inner) = llm_inner_body(payload) { - if let Some(s) = inner.as_str() { - let text = extract_assistant_turn_from_sse(s); - if let Some(t) = non_empty(&text) { - return Some(t); - } - } - if let Some(text) = extract_assistant_text_from_json(inner) { - return non_empty(&text); - } - } - llm_inner_body(payload) - .and_then(|b| b.get("choices")) - .or_else(|| payload.get("body").and_then(|b| b.get("choices"))) - .or_else(|| payload.get("choices")) - .and_then(|c| c.as_array()) - .and_then(|a| a.first()) - .and_then(|c| c.get("message")) - .and_then(|m| m.get("content")) - .and_then(content_to_string) - .or_else(|| payload.get("content").and_then(content_to_string)) - .and_then(|s| non_empty(&s)) -} - -fn non_empty(s: &str) -> Option { - if s.trim().is_empty() { - None - } else { - Some(s.to_string()) - } -} - -/// Inner LLM JSON: `payload.body` or proxy wrapper `payload.body.body`. -fn llm_inner_body(payload: &Value) -> Option<&Value> { - let wrap = payload.get("body")?; - if wrap.get("messages").is_some() || wrap.get("choices").is_some() { - Some(wrap) - } else { - wrap.get("body") - } -} - -pub fn now_rfc3339() -> String { - chrono::Utc::now().to_rfc3339() -} - -#[cfg(test)] -mod timestamp_tests { - use proptest::prelude::*; - use serde_json::Value; - - use super::{EventRecord, ensure_timestamp, unix_ms_from_rfc3339}; - - #[test] - fn parses_rfc3339_to_unix_milliseconds() { - assert_eq!( - unix_ms_from_rfc3339("2026-01-01T00:00:00Z"), - Some(1_767_225_600_000) - ); - } - - #[test] - fn ensure_timestamp_backfills_both_wire_fields() { - let mut record = EventRecord { - identity: Default::default(), - seq: 0, - source: "persisting-proxy".into(), - kind: "llm.request".into(), - timestamp: None, - session_id: None, - agent_id: None, - parent_uuid: None, - trace_id: None, - call_id: None, - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload: Value::Null, - }; - ensure_timestamp(&mut record); - assert!(record.timestamp.is_some()); - assert!(record.identity.timestamp_unix_ms.is_some()); - } - - proptest! { - #[test] - fn ensure_timestamp_is_idempotent_for_existing_rfc3339( - milliseconds in 0u64..=4_000_000_000_000u64 - ) { - let timestamp = chrono::DateTime::from_timestamp_millis(milliseconds as i64) - .expect("generated timestamp is representable") - .to_rfc3339(); - let mut record = EventRecord { - identity: Default::default(), - seq: 0, - source: "persisting-proxy".into(), - kind: "llm.request".into(), - timestamp: Some(timestamp.clone()), - session_id: None, - agent_id: None, - parent_uuid: None, - trace_id: None, - call_id: None, - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload: Value::Null, - }; - - ensure_timestamp(&mut record); - let first = record.clone(); - ensure_timestamp(&mut record); - - prop_assert_eq!(record.timestamp.as_deref(), Some(timestamp.as_str())); - prop_assert_eq!(record.identity.timestamp_unix_ms, Some(milliseconds)); - prop_assert_eq!(record, first); - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::Call; - use crate::config::CaptureLevel; - use crate::sink::{llm_request_record, llm_request_summary_record, llm_response_record}; - - #[test] - fn internal_request_detects_count_tokens_path() { - let call = Call { - call_id: "c".into(), - trace_id: "t".into(), - started_at: "2026-01-01T00:00:00Z".into(), - }; - let rec = llm_request_summary_record( - Some("s".into()), - Some("a".into()), - "m", - "/v1/messages/count_tokens", - 10, - "count_tokens", - "openai", - None, - None, - &call, - CaptureLevel::Dialogue, - None, - ); - assert!(rec.is_internal_llm_request()); - } - - #[test] - fn visible_user_prefers_user_content_field() { - let rec = EventRecord { - identity: Default::default(), - seq: 0, - source: "test".into(), - kind: "llm.request".into(), - timestamp: None, - session_id: None, - agent_id: None, - parent_uuid: None, - trace_id: None, - call_id: None, - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload: serde_json::json!({ - "user_content": "hello", - "body": {"messages": [{"role": "user", "content": "ignored"}]} - }), - }; - assert_eq!(rec.visible_user_text().as_deref(), Some("hello")); - } - - #[test] - fn visible_user_reads_proxy_nested_body() { - let req = llm_request_record( - Some("sess".into()), - Some("agent".into()), - "mock-model", - "/v1/chat/completions", - &serde_json::json!({ - "protocol": "chat_completions", - "provider": "openai", - "body": {"messages":[{"role":"user","content":"你好"}],"model":"mock-model"}, - }), - ); - assert_eq!(req.visible_user_text().as_deref(), Some("你好")); - } - - #[test] - fn visible_assistant_reads_proxy_nested_body() { - let resp = llm_response_record( - Some("sess".into()), - Some("agent".into()), - 200, - &serde_json::json!({ - "protocol": "chat_completions", - "provider": "openai", - "body": { - "choices":[{"message":{"role":"assistant","content":"你好！"}}], - }, - }), - false, - &Call { - call_id: "c".into(), - trace_id: "t".into(), - started_at: "2026-01-01T00:00:00Z".into(), - }, - ); - assert_eq!(resp.visible_assistant_text().as_deref(), Some("你好！")); - } -} diff --git a/crates/persisting-gateway/src/runtime/debug.rs b/crates/persisting-gateway/src/runtime/debug.rs deleted file mode 100644 index 7fc3147a0..000000000 --- a/crates/persisting-gateway/src/runtime/debug.rs +++ /dev/null @@ -1,262 +0,0 @@ -//! Debug logging for capture proxy: all HTTP dispatch + captured LLM request/response. - -use std::fs::{self, OpenOptions}; -use std::io::Write; -use std::path::{Path, PathBuf}; -use std::sync::atomic::{AtomicBool, Ordering}; - -use crate::config::ProxyConfig; - -pub const ENV_CAPTURE_DEBUG: &str = "PERSISTING_CAPTURE_DEBUG"; -/// Mirror `[capture-debug]` lines to stderr (default: file only). -pub const ENV_CAPTURE_DEBUG_STDERR: &str = "PERSISTING_CAPTURE_DEBUG_STDERR"; - -const MAX_BODY_CHARS: usize = 8192; -static DEBUG_STDERR_ENABLED: AtomicBool = AtomicBool::new(false); - -pub fn debug_flag_path(storage: &Path) -> PathBuf { - storage.join(".capture").join("debug.enabled") -} - -pub fn debug_log_path(storage: &Path) -> PathBuf { - storage.join(".capture").join("debug.log") -} - -/// Enable debug until flag file is removed (`capture run --debug` / CLI helper). -pub fn enable_debug(storage: &Path) -> anyhow::Result<()> { - let path = debug_flag_path(storage); - if let Some(p) = path.parent() { - fs::create_dir_all(p)?; - } - fs::write(&path, "1\n")?; - Ok(()) -} - -pub fn is_debug_enabled(cfg: &ProxyConfig, storage: &Path) -> bool { - cfg.debug || env_truthy(ENV_CAPTURE_DEBUG) || debug_flag_path(storage).is_file() -} - -fn env_truthy(name: &str) -> bool { - std::env::var(name) - .ok() - .map(|v| { - let v = v.trim(); - v == "1" || v.eq_ignore_ascii_case("true") || v.eq_ignore_ascii_case("yes") - }) - .unwrap_or(false) -} - -/// Mirror capture diagnostics to stderr for the lifetime of this process. -/// -/// This is intended for foreground CLI commands. The atomic switch avoids -/// mutating process environment variables after an async runtime has started. -pub fn enable_debug_stderr() { - DEBUG_STDERR_ENABLED.store(true, Ordering::Relaxed); -} - -fn mirror_debug_to_stderr() -> bool { - DEBUG_STDERR_ENABLED.load(Ordering::Relaxed) || env_truthy(ENV_CAPTURE_DEBUG_STDERR) -} - -fn emit(storage: &Path, line: &str) { - let stamped = format!("{} {line}", chrono::Utc::now().to_rfc3339()); - if mirror_debug_to_stderr() { - eprintln!("[capture-debug] {line}"); - } - let log_path = debug_log_path(storage); - if let Some(dir) = log_path.parent() { - let _ = fs::create_dir_all(dir); - } - if let Ok(mut f) = OpenOptions::new().create(true).append(true).open(log_path) { - let _ = writeln!(f, "{stamped}"); - } -} - -pub fn truncate_body_bytes(raw: &[u8]) -> String { - if raw.len() <= MAX_BODY_CHARS { - return String::from_utf8_lossy(raw).into_owned(); - } - format!( - "{}…[truncated]", - String::from_utf8_lossy(&raw[..MAX_BODY_CHARS]) - ) -} - -pub fn truncate_body(raw: &str) -> String { - if raw.chars().count() <= MAX_BODY_CHARS { - return raw.to_string(); - } - let mut end = 0; - for (n, (i, c)) in raw.char_indices().enumerate() { - if n >= MAX_BODY_CHARS { - break; - } - end = i + c.len_utf8(); - } - format!("{}…[truncated]", &raw[..end]) -} - -pub fn log_daemon_start(storage: &Path, listen: &str, version: &str) { - emit( - storage, - &format!("capture daemon.start listen={listen} version={version}"), - ); -} - -pub fn log_connect(storage: &Path, target: &str, session_id: &str) { - emit( - storage, - &format!("CONNECT target={target} session={session_id}"), - ); -} - -pub fn log_network_denied(storage: &Path, host: &str, mode: &str, reason: &str, session_id: &str) { - emit( - storage, - &format!("network.denied host={host} mode={mode} reason={reason} session={session_id}"), - ); -} - -pub fn log_forward( - storage: &Path, - method: &str, - url: &str, - session_id: &str, - status: u16, - body: &str, -) { - emit( - storage, - &format!( - "forward {method} {url} session={session_id} status={status} body={}", - truncate_body(body) - ), - ); -} - -pub fn log_dispatch(storage: &Path, method: &str, uri: &str, session_id: &str, mode: &str) { - emit( - storage, - &format!("dispatch {method} {uri} session={session_id} mode={mode}"), - ); -} - -#[allow(clippy::too_many_arguments)] -pub fn log_llm_request( - storage: &Path, - session_id: &str, - agent_id: &str, - model: &str, - protocol: &str, - path: &str, - upstream: &str, - body: &str, -) { - emit( - storage, - &format!( - "capture llm.request session={session_id} agent={agent_id} model={model} \ - protocol={protocol} path={path} upstream={upstream} body={}", - truncate_body(body) - ), - ); -} - -pub fn log_llm_auth_resolved(storage: &Path, session_id: &str, source: &str) { - emit( - storage, - &format!("capture llm.auth session={session_id} source={source}"), - ); -} - -pub fn log_llm_upstream_sending(storage: &Path, session_id: &str, upstream: &str) { - emit( - storage, - &format!("capture llm.upstream.sending session={session_id} upstream={upstream}"), - ); -} - -#[allow(clippy::too_many_arguments)] -pub fn log_llm_upstream_headers( - storage: &Path, - session_id: &str, - agent_id: &str, - model: &str, - upstream: &str, - status: u16, - content_type: &str, - stream_request: bool, -) { - emit( - storage, - &format!( - "capture llm.upstream.headers session={session_id} agent={agent_id} model={model} \ - upstream={upstream} status={status} content_type={content_type} stream_request={stream_request}" - ), - ); -} - -pub fn log_llm_stream_start( - storage: &Path, - session_id: &str, - agent_id: &str, - model: &str, - status: u16, -) { - emit( - storage, - &format!( - "capture llm.response.start session={session_id} agent={agent_id} model={model} \ - status={status} streaming=true" - ), - ); -} - -pub fn log_llm_response( - storage: &Path, - session_id: &str, - agent_id: &str, - model: &str, - status: u16, - total_tokens: u64, - body: &str, -) { - emit( - storage, - &format!( - "capture llm.response session={session_id} agent={agent_id} model={model} \ - status={status} total_tokens={total_tokens} body={}", - truncate_body(body) - ), - ); -} - -pub fn log_llm_upstream_error( - storage: &Path, - session_id: &str, - agent_id: &str, - model: &str, - upstream: &str, - error: &str, -) { - emit( - storage, - &format!( - "capture llm.error session={session_id} agent={agent_id} model={model} \ - upstream={upstream} error={error}" - ), - ); -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn truncate_long_body() { - let s = "x".repeat(9000); - let t = truncate_body(&s); - assert!(t.contains("truncated")); - assert!(t.len() < 9000); - } -} diff --git a/crates/persisting-gateway/src/runtime/in_process.rs b/crates/persisting-gateway/src/runtime/in_process.rs deleted file mode 100644 index 6b419516a..000000000 --- a/crates/persisting-gateway/src/runtime/in_process.rs +++ /dev/null @@ -1,175 +0,0 @@ -//! In-process Gateway for one Attempt (no forked daemon). - -use std::net::TcpStream; -use std::path::PathBuf; -use std::sync::Arc; -use std::thread::JoinHandle; -use std::time::Duration; - -use anyhow::{Context, Result}; - -use crate::config::ProxyConfig; -use crate::sink::CaptureEventSink; -use persisting_agentctl::{ControlController, PolicyControlController}; -use persisting_overlaynet::{BandwidthRegistry, InterceptionMetrics, InterceptionSnapshot}; -use tokio::sync::oneshot; - -pub struct InProcessCapture { - shutdown_tx: Option>, - join: Option>>, - pub listen: String, - interception_metrics: InterceptionMetrics, -} - -/// Attempt-scoped network services shared by Gateway and other interception -/// drivers supplied by the caller. -#[derive(Clone)] -pub struct InProcessRuntime { - pub controller: Arc, - pub interception_metrics: InterceptionMetrics, - pub bandwidth_registry: BandwidthRegistry, - pub attempt_id: Option, - /// Disable LLM dispatch for Runs that only need the network proxy. - pub gateway_enabled: bool, -} - -impl Default for InProcessRuntime { - fn default() -> Self { - Self { - controller: Arc::new(PolicyControlController), - interception_metrics: InterceptionMetrics::default(), - bandwidth_registry: BandwidthRegistry::default(), - attempt_id: None, - gateway_enabled: true, - } - } -} - -impl InProcessCapture { - pub fn start( - config: ProxyConfig, - storage: PathBuf, - sink: Arc, - stream_markdown: bool, - ) -> Result { - Self::start_with_runtime( - config, - storage, - sink, - stream_markdown, - InProcessRuntime::default(), - ) - } - - pub fn start_with_runtime( - config: ProxyConfig, - storage: PathBuf, - sink: Arc, - stream_markdown: bool, - runtime: InProcessRuntime, - ) -> Result { - let listen = config.listen.clone(); - let (shutdown_tx, shutdown_rx) = oneshot::channel(); - let interception_metrics = runtime.interception_metrics.clone(); - let thread_metrics = interception_metrics.clone(); - - let join = std::thread::Builder::new() - .name("persisting-gateway".into()) - .spawn(move || { - let rt = tokio::runtime::Runtime::new().context("tokio runtime")?; - rt.block_on(crate::gateway::serve_with_runtime_control_and_metrics( - config, - storage, - sink, - stream_markdown, - crate::gateway::GatewayRuntimeControl { - controller: runtime.controller, - interception_metrics: thread_metrics, - bandwidth_registry: runtime.bandwidth_registry, - attempt_id: runtime.attempt_id, - gateway_enabled: runtime.gateway_enabled, - }, - None, - async { - let _ = shutdown_rx.await; - }, - )) - }) - .context("spawn in-process capture")?; - - wait_proxy_ready(&listen)?; - - Ok(Self { - shutdown_tx: Some(shutdown_tx), - join: Some(join), - listen, - interception_metrics, - }) - } - - /// Compatibility entry point for callers that only inject a controller. - pub fn start_with_control( - config: ProxyConfig, - storage: PathBuf, - sink: Arc, - stream_markdown: bool, - controller: Arc, - ) -> Result { - Self::start_with_runtime( - config, - storage, - sink, - stream_markdown, - InProcessRuntime { - controller, - ..InProcessRuntime::default() - }, - ) - } - - pub fn interception_snapshot(&self) -> InterceptionSnapshot { - self.interception_metrics.snapshot() - } - - pub fn shutdown(mut self) -> Result<()> { - self.shutdown_inner() - } - - fn shutdown_inner(&mut self) -> Result<()> { - if let Some(tx) = self.shutdown_tx.take() { - let _ = tx.send(()); - } - let Some(join) = self.join.take() else { - return Ok(()); - }; - join.join() - .map_err(|_| anyhow::anyhow!("in-process capture thread panicked"))??; - Ok(()) - } -} - -impl Drop for InProcessCapture { - fn drop(&mut self) { - if let Err(error) = self.shutdown_inner() { - tracing::warn!(%error, "failed to stop in-process Gateway during drop"); - } - } -} - -fn wait_proxy_ready(listen: &str) -> Result<()> { - for _ in 0..100 { - if tcp_bound(listen) { - return Ok(()); - } - std::thread::sleep(Duration::from_millis(50)); - } - anyhow::bail!("capture proxy did not become ready on http://{listen}"); -} - -fn tcp_bound(listen: &str) -> bool { - let addr = listen - .strip_prefix("http://") - .or_else(|| listen.strip_prefix("https://")) - .unwrap_or(listen); - TcpStream::connect(addr).is_ok() -} diff --git a/crates/persisting-gateway/src/runtime/mod.rs b/crates/persisting-gateway/src/runtime/mod.rs deleted file mode 100644 index 760a08bb1..000000000 --- a/crates/persisting-gateway/src/runtime/mod.rs +++ /dev/null @@ -1,8 +0,0 @@ -//! Run-scoped Gateway directories and debug sidecars. - -pub mod debug; -pub mod in_process; -mod private_fs; -pub(crate) use private_fs::{open_private_append_file, open_private_truncate_file}; -pub mod run_config; -pub mod run_env; diff --git a/crates/persisting-gateway/src/runtime/private_fs.rs b/crates/persisting-gateway/src/runtime/private_fs.rs deleted file mode 100644 index 7735619fb..000000000 --- a/crates/persisting-gateway/src/runtime/private_fs.rs +++ /dev/null @@ -1,220 +0,0 @@ -//! Private, crash-safe filesystem helpers for capture runtime state. - -use std::fs::{self, OpenOptions}; -use std::io::Write; -use std::path::Path; -use std::sync::atomic::{AtomicU64, Ordering}; - -use anyhow::{Context, Result}; - -static TEMP_FILE_SEQUENCE: AtomicU64 = AtomicU64::new(0); - -/// Create a directory and make it private to the current user. -pub(crate) fn ensure_private_dir(path: &Path) -> Result<()> { - fs::create_dir_all(path) - .with_context(|| format!("create private directory {}", path.display()))?; - set_private_dir_permissions(path) -} - -/// Open an append-only runtime-state file and make both it and its parent -/// private to the current user. Existing files are hardened as well. -pub(crate) fn open_private_append_file(path: &Path) -> Result { - open_private_file(path, true) -} - -/// Open and truncate a runtime-state file while preserving private ownership. -pub(crate) fn open_private_truncate_file(path: &Path) -> Result { - open_private_file(path, false) -} - -fn open_private_file(path: &Path, append: bool) -> Result { - let parent = path - .parent() - .with_context(|| format!("private file has no parent: {}", path.display()))?; - ensure_private_dir(parent)?; - - let mut options = OpenOptions::new(); - options.write(true).create(true); - if append { - options.append(true); - } else { - options.truncate(true); - } - #[cfg(unix)] - { - use std::os::unix::fs::OpenOptionsExt; - options.mode(0o600); - } - - let file = options - .open(path) - .with_context(|| format!("open private file {}", path.display()))?; - set_private_file_permissions(path)?; - Ok(file) -} - -/// Atomically replace a file with content readable only by the current user. -pub(crate) fn write_private_file(path: &Path, contents: &[u8]) -> Result<()> { - let parent = path - .parent() - .with_context(|| format!("private file has no parent: {}", path.display()))?; - ensure_private_dir(parent)?; - - let file_name = path - .file_name() - .and_then(|name| name.to_str()) - .unwrap_or("capture-state"); - let sequence = TEMP_FILE_SEQUENCE.fetch_add(1, Ordering::Relaxed); - let temp = parent.join(format!( - ".{file_name}.{}.{}.tmp", - std::process::id(), - sequence - )); - - let result = (|| -> Result<()> { - let mut options = OpenOptions::new(); - options.write(true).create_new(true); - #[cfg(unix)] - { - use std::os::unix::fs::OpenOptionsExt; - options.mode(0o600); - } - - let mut file = options - .open(&temp) - .with_context(|| format!("create private temporary file {}", temp.display()))?; - file.write_all(contents) - .with_context(|| format!("write private temporary file {}", temp.display()))?; - file.sync_all() - .with_context(|| format!("sync private temporary file {}", temp.display()))?; - drop(file); - - fs::rename(&temp, path).with_context(|| { - format!( - "atomically replace private file {} with {}", - path.display(), - temp.display() - ) - })?; - set_private_file_permissions(path)?; - - #[cfg(unix)] - fs::File::open(parent) - .and_then(|dir| dir.sync_all()) - .with_context(|| format!("sync private directory {}", parent.display()))?; - - Ok(()) - })(); - - if result.is_err() { - let _ = fs::remove_file(&temp); - } - result -} - -#[cfg(unix)] -fn set_private_dir_permissions(path: &Path) -> Result<()> { - use std::os::unix::fs::PermissionsExt; - - fs::set_permissions(path, fs::Permissions::from_mode(0o700)) - .with_context(|| format!("set private directory permissions on {}", path.display())) -} - -#[cfg(not(unix))] -fn set_private_dir_permissions(_path: &Path) -> Result<()> { - Ok(()) -} - -#[cfg(unix)] -fn set_private_file_permissions(path: &Path) -> Result<()> { - use std::os::unix::fs::PermissionsExt; - - fs::set_permissions(path, fs::Permissions::from_mode(0o600)) - .with_context(|| format!("set private file permissions on {}", path.display())) -} - -#[cfg(not(unix))] -fn set_private_file_permissions(_path: &Path) -> Result<()> { - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[cfg(unix)] - #[test] - fn private_write_hardens_existing_file_and_parent() { - use std::os::unix::fs::PermissionsExt; - - let temp = tempfile::tempdir().unwrap(); - let parent = temp.path().join("capture"); - fs::create_dir(&parent).unwrap(); - fs::set_permissions(&parent, fs::Permissions::from_mode(0o755)).unwrap(); - let path = parent.join("secret.json"); - fs::write(&path, b"old").unwrap(); - fs::set_permissions(&path, fs::Permissions::from_mode(0o644)).unwrap(); - - write_private_file(&path, b"new-secret").unwrap(); - - assert_eq!(fs::read(&path).unwrap(), b"new-secret"); - assert_eq!( - fs::metadata(&parent).unwrap().permissions().mode() & 0o777, - 0o700 - ); - assert_eq!( - fs::metadata(&path).unwrap().permissions().mode() & 0o777, - 0o600 - ); - } - - #[cfg(unix)] - #[test] - fn private_append_hardens_existing_file_and_parent() { - use std::os::unix::fs::PermissionsExt; - - let temp = tempfile::tempdir().unwrap(); - let parent = temp.path().join("capture"); - fs::create_dir(&parent).unwrap(); - fs::set_permissions(&parent, fs::Permissions::from_mode(0o755)).unwrap(); - let path = parent.join("events.jsonl"); - fs::write(&path, b"old\n").unwrap(); - fs::set_permissions(&path, fs::Permissions::from_mode(0o644)).unwrap(); - - let mut file = open_private_append_file(&path).unwrap(); - file.write_all(b"new\n").unwrap(); - drop(file); - - assert_eq!(fs::read(&path).unwrap(), b"old\nnew\n"); - assert_eq!( - fs::metadata(&parent).unwrap().permissions().mode() & 0o777, - 0o700 - ); - assert_eq!( - fs::metadata(&path).unwrap().permissions().mode() & 0o777, - 0o600 - ); - } - - #[test] - fn concurrent_replacements_never_leave_partial_content() { - let temp = tempfile::tempdir().unwrap(); - let path = std::sync::Arc::new(temp.path().join("capture").join("state.json")); - let payloads = [ - vec![b'a'; 8 * 1024], - vec![b'b'; 16 * 1024], - vec![b'c'; 32 * 1024], - vec![b'd'; 64 * 1024], - ]; - - std::thread::scope(|scope| { - for payload in &payloads { - let path = std::sync::Arc::clone(&path); - scope.spawn(move || write_private_file(path.as_ref(), payload).unwrap()); - } - }); - - let actual = fs::read(path.as_ref()).unwrap(); - assert!(payloads.iter().any(|payload| payload == &actual)); - } -} diff --git a/crates/persisting-gateway/src/runtime/run_config.rs b/crates/persisting-gateway/src/runtime/run_config.rs deleted file mode 100644 index 5f30e7d1d..000000000 --- a/crates/persisting-gateway/src/runtime/run_config.rs +++ /dev/null @@ -1,148 +0,0 @@ -//! Per-`capture run` proxy config snapshot (`{storage}/.capture/sessions/{session_id}/proxy.toml`). - -use std::path::{Path, PathBuf}; - -use anyhow::{Context, Result}; - -use crate::config::ProxyConfig; -use crate::runtime::private_fs::{ensure_private_dir, write_private_file}; -use crate::runtime::run_env::ensure_private_capture_dir; - -pub const SESSION_PROXY_FILENAME: &str = "proxy.toml"; -pub const LEGACY_SESSION_PROXY_FILENAME: &str = "proxy.yaml"; - -pub fn session_dir(storage: &Path, session_id: &str) -> PathBuf { - storage - .join(".capture") - .join("sessions") - .join(session_id.trim()) -} - -pub fn session_proxy_config_path(storage: &Path, session_id: &str) -> PathBuf { - session_dir(storage, session_id).join(SESSION_PROXY_FILENAME) -} - -/// Parse source config and write canonical TOML into the session snapshot (used by `capture run`). -pub fn snapshot_run_proxy_config( - storage: &Path, - session_id: &str, - source: &Path, -) -> Result { - let cfg = ProxyConfig::from_file(source)?; - snapshot_proxy_config(storage, session_id, &cfg) -} - -/// Write an already-resolved proxy configuration into the session snapshot. -/// Callers use this after resolving their configuration, so Gateway -/// runtime setup never needs to know where configuration originated. -pub fn snapshot_proxy_config( - storage: &Path, - session_id: &str, - cfg: &ProxyConfig, -) -> Result { - let dest = session_proxy_config_path(storage, session_id); - let capture_dir = ensure_private_capture_dir(storage)?; - let sessions_dir = capture_dir.join("sessions"); - ensure_private_dir(&sessions_dir)?; - ensure_private_dir(&session_dir(storage, session_id))?; - let toml = cfg.to_toml_string()?; - write_private_file(&dest, toml.as_bytes()) - .with_context(|| format!("write proxy config snapshot {}", dest.display()))?; - Ok(dest) -} - -/// Load snapshotted config for a session; `None` if no snapshot (fallback to daemon default). -pub fn load_session_proxy_config(storage: &Path, session_id: &str) -> Option { - let dir = session_dir(storage, session_id); - for name in [SESSION_PROXY_FILENAME, LEGACY_SESSION_PROXY_FILENAME] { - let path = dir.join(name); - if path.is_file() - && let Ok(cfg) = ProxyConfig::from_file(&path) - { - return Some(cfg); - } - } - None -} - -#[cfg(test)] -mod tests { - use super::*; - use std::io::Write; - - #[test] - fn snapshot_and_load_roundtrip() { - let tmp = tempfile::tempdir().unwrap(); - let storage = tmp.path(); - let src = storage.join("src.toml"); - std::fs::File::create(&src) - .unwrap() - .write_all( - br#"listen = "127.0.0.1:19999" -admin_listen = "127.0.0.1:19998" -agent_id = "snap-agent" - -[[models]] -name = "*" -upstream = "http://127.0.0.1:1" -"#, - ) - .unwrap(); - let dest = snapshot_run_proxy_config(storage, "sess-a", &src).unwrap(); - assert!(dest.ends_with("proxy.toml")); - let cfg = load_session_proxy_config(storage, "sess-a").unwrap(); - assert_eq!(cfg.agent_id, "snap-agent"); - assert_eq!(cfg.listen, "127.0.0.1:19999"); - } - - #[cfg(unix)] - #[test] - fn snapshot_is_private_even_when_replacing_a_public_file() { - use std::os::unix::fs::PermissionsExt; - - let tmp = tempfile::tempdir().unwrap(); - let storage = tmp.path(); - let cfg = ProxyConfig::from_toml_str( - r#" -listen = "127.0.0.1:19999" -admin_listen = "127.0.0.1:19998" -agent_id = "private-snapshot" - -[[models]] -name = "*" -upstream = "http://127.0.0.1:1" -api_key = "sk-inline-secret" -"#, - ) - .unwrap(); - let dest = session_proxy_config_path(storage, "sess-private"); - std::fs::create_dir_all(dest.parent().unwrap()).unwrap(); - std::fs::write(&dest, "legacy public content").unwrap(); - std::fs::set_permissions(&dest, std::fs::Permissions::from_mode(0o644)).unwrap(); - - snapshot_proxy_config(storage, "sess-private", &cfg).unwrap(); - - for dir in [ - storage.join(".capture"), - storage.join(".capture/sessions"), - session_dir(storage, "sess-private"), - ] { - assert_eq!( - std::fs::metadata(dir).unwrap().permissions().mode() & 0o777, - 0o700 - ); - } - assert_eq!( - std::fs::metadata(&dest).unwrap().permissions().mode() & 0o777, - 0o600 - ); - assert_eq!( - load_session_proxy_config(storage, "sess-private") - .unwrap() - .models[0] - .api_key - .as_deref(), - Some("sk-inline-secret") - ); - } -} diff --git a/crates/persisting-gateway/src/runtime/run_env.rs b/crates/persisting-gateway/src/runtime/run_env.rs deleted file mode 100644 index 139462014..000000000 --- a/crates/persisting-gateway/src/runtime/run_env.rs +++ /dev/null @@ -1,74 +0,0 @@ -//! Run-scoped capture environment files. - -use std::fs; -use std::path::{Path, PathBuf}; - -use anyhow::{Context, Result}; -use serde::{Deserialize, Serialize}; - -use crate::runtime::private_fs::{ensure_private_dir, write_private_file}; - -pub const ENV_SESSION_ID: &str = "PERSISTING_CAPTURE_SESSION_ID"; -pub use crate::injection::env::CAPTURE_PROXY_ENV_KEYS; - -pub fn run_session_file(storage: &Path) -> PathBuf { - storage.join(".capture").join("run_session") -} - -pub fn write_run_session(storage: &Path, session_id: &str) -> Result<()> { - let path = run_session_file(storage); - ensure_private_capture_dir(storage)?; - write_private_file(&path, session_id.trim().as_bytes()).context("write run_session")?; - Ok(()) -} - -pub fn read_run_session(storage: &Path) -> Option { - let path = run_session_file(storage); - let s = fs::read_to_string(&path).ok()?; - let s = s.trim(); - if s.is_empty() { - None - } else { - Some(s.to_string()) - } -} - -pub const RUN_CHILD_FILENAME: &str = "run_child.yaml"; - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct RunChildInfo { - pub pid: u32, - pub command: String, -} - -pub fn run_child_file(storage: &Path) -> PathBuf { - storage.join(".capture").join(RUN_CHILD_FILENAME) -} - -/// Written at `capture run` child spawn — authoritative client command (not the proxy). -pub fn write_run_child_info(storage: &Path, pid: u32, command: &[String]) -> Result<()> { - let path = run_child_file(storage); - ensure_private_capture_dir(storage)?; - let info = RunChildInfo { - pid, - command: command.join(" "), - }; - let yaml = serde_yaml::to_string(&info).context("serialize run_child.yaml")?; - write_private_file(&path, yaml.as_bytes()).context("write run_child.yaml") -} - -pub(crate) fn read_run_child_info(storage: &Path) -> Option { - let path = run_child_file(storage); - let text = fs::read_to_string(&path).ok()?; - serde_yaml::from_str(&text).ok() -} - -pub(crate) fn ensure_private_capture_dir(storage: &Path) -> Result { - let capture_dir = storage.join(".capture"); - ensure_private_dir(&capture_dir)?; - Ok(capture_dir) -} - -pub use crate::injection::env::{ - capture_openai_v1_base, client_gateway_config_args, proxy_environment, -}; diff --git a/crates/persisting-gateway/src/session/chain.rs b/crates/persisting-gateway/src/session/chain.rs deleted file mode 100644 index 24e8c558c..000000000 --- a/crates/persisting-gateway/src/session/chain.rs +++ /dev/null @@ -1,267 +0,0 @@ -//! Session / trace id extraction from HTTP headers (LiteLLM-compatible priority chain). - -use axum::http::HeaderMap; - -const EXPLICIT_SESSION_HEADERS: &[&str] = &[ - "x-persisting-trace-id", - "x-litellm-trace-id", - "x-litellm-session-id", -]; - -const GENERIC_SESSION_SUFFIX: &str = "-session-id"; -const CLAUDE_AGENT_ID_HEADER: &str = "x-claude-code-agent-id"; -const CLAUDE_PARENT_AGENT_ID_HEADER: &str = "x-claude-code-parent-agent-id"; - -/// Minimum length for vendor `x-*-session-id` values (UUIDs, hex ids, etc.). -const MIN_SESSION_ID_LEN: usize = 8; - -/// Claude Code subagent instance id (`agent-{id}` in `~/.claude/projects/.../subagents/`). -pub fn extract_claude_agent_id(headers: &HeaderMap) -> Option { - header_value(headers, CLAUDE_AGENT_ID_HEADER) -} - -/// Parent agent id when the request comes from a nested subagent spawn. -pub fn extract_claude_parent_agent_id(headers: &HeaderMap) -> Option { - header_value(headers, CLAUDE_PARENT_AGENT_ID_HEADER) -} - -/// Session id from request headers only (no job fallback). -/// -/// Priority: -/// 1. Configured `session_header` (e.g. `x-persisting-session-id`) -/// 2. `x-session-id` -/// 3. Explicit trace headers (`x-persisting-trace-id`, LiteLLM) -/// 4. Any `x--session-id` (e.g. Claude Code) -pub fn extract_session_from_headers(headers: &HeaderMap, session_header: &str) -> Option { - if let Some(v) = header_value(headers, session_header) { - return Some(v); - } - if let Some(v) = header_value(headers, "x-session-id") { - return Some(v); - } - for name in EXPLICIT_SESSION_HEADERS { - if name.eq_ignore_ascii_case(session_header) { - continue; - } - if let Some(v) = header_value(headers, name) { - return Some(v); - } - } - extract_generic_session_id(headers) -} - -pub fn ephemeral_session_id() -> String { - format!("ephemeral-{}", new_call_id().trim_start_matches("call-")) -} - -/// Trace id for correlating request/response pairs and retries within a logical unit. -/// -/// Falls back to `call_id` when no trace header is present. -pub fn resolve_trace_id(headers: &HeaderMap, call_id: &str) -> String { - for name in [ - "x-persisting-trace-id", - "x-litellm-trace-id", - "x-request-id", - ] { - if let Some(v) = header_value(headers, name) { - return v; - } - } - call_id.to_string() -} - -pub fn new_call_id() -> String { - use std::time::{SystemTime, UNIX_EPOCH}; - let ns = SystemTime::now() - .duration_since(UNIX_EPOCH) - .map(|d| d.as_nanos()) - .unwrap_or(0); - format!("call-{ns:x}") -} - -fn header_value(headers: &HeaderMap, name: &str) -> Option { - headers - .get(name) - .and_then(|v| v.to_str().ok()) - .map(str::trim) - .filter(|s| !s.is_empty()) - .map(str::to_string) -} - -fn extract_generic_session_id(headers: &HeaderMap) -> Option { - for (name, value) in headers.iter() { - let key = name.as_str(); - if !key.starts_with("x-") || !key.ends_with(GENERIC_SESSION_SUFFIX) { - continue; - } - if EXPLICIT_SESSION_HEADERS - .iter() - .any(|h| key.eq_ignore_ascii_case(h)) - { - continue; - } - if let Ok(v) = value.to_str() { - let v = v.trim(); - if plausible_session_value(v) { - return Some(v.to_string()); - } - } - } - None -} - -fn plausible_session_value(v: &str) -> bool { - if v.len() < MIN_SESSION_ID_LEN { - return false; - } - v.chars() - .all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.')) -} - -#[cfg(test)] -mod tests { - use super::*; - use axum::http::HeaderValue; - use proptest::prelude::*; - - fn headers(pairs: &[(&str, &str)]) -> HeaderMap { - use axum::http::header::HeaderName; - let mut h = HeaderMap::new(); - for (k, v) in pairs { - h.insert( - HeaderName::from_bytes(k.as_bytes()).expect("header name"), - HeaderValue::try_from(*v).expect("valid header value"), - ); - } - h - } - - #[test] - fn configured_session_header_wins() { - let h = headers(&[ - ("x-persisting-session-id", "sess-a"), - ("x-claude-code-session-id", "claude-b"), - ]); - assert_eq!( - extract_session_from_headers(&h, "x-persisting-session-id").as_deref(), - Some("sess-a") - ); - } - - #[test] - fn litellm_trace_id_used_when_no_configured_header() { - let h = headers(&[("x-litellm-trace-id", "trace-12345678")]); - assert_eq!( - extract_session_from_headers(&h, "x-persisting-session-id").as_deref(), - Some("trace-12345678") - ); - } - - #[test] - fn claude_code_session_id_from_header() { - let h = headers(&[( - "x-claude-code-session-id", - "e96634a3-fa28-4083-b354-55542e2dca01", - )]); - assert_eq!( - extract_session_from_headers(&h, "x-persisting-session-id").as_deref(), - Some("e96634a3-fa28-4083-b354-55542e2dca01") - ); - } - - #[test] - fn short_generic_session_id_ignored() { - let h = headers(&[("x-foo-session-id", "abc")]); - assert!(extract_session_from_headers(&h, "x-persisting-session-id").is_none()); - } - - #[test] - fn trace_id_from_header_or_call_id() { - let h = headers(&[("x-persisting-trace-id", "trace-abc")]); - assert_eq!(resolve_trace_id(&h, "call-deadbeef"), "trace-abc"); - assert_eq!( - resolve_trace_id(&HeaderMap::new(), "call-deadbeef"), - "call-deadbeef" - ); - } - - #[test] - fn claude_code_agent_id_from_header() { - let h = headers(&[("x-claude-code-agent-id", "a5aa661")]); - assert_eq!(extract_claude_agent_id(&h).as_deref(), Some("a5aa661")); - } - - fn header_token_strategy() -> impl Strategy { - proptest::string::string_regex("[a-zA-Z0-9._-]{1,64}").unwrap() - } - - fn plausible_generic_session_strategy() -> impl Strategy { - proptest::string::string_regex("[a-zA-Z0-9._-]{8,64}").unwrap() - } - - proptest! { - #[test] - fn configured_header_wins_the_complete_priority_chain( - configured in header_token_strategy(), - standard in header_token_strategy(), - trace in header_token_strategy(), - generic in plausible_generic_session_strategy(), - ) { - let h = headers(&[ - ("x-custom-session", &configured), - ("x-session-id", &standard), - ("x-persisting-trace-id", &trace), - ("x-vendor-session-id", &generic), - ]); - prop_assert_eq!( - extract_session_from_headers(&h, "x-custom-session"), - Some(configured), - ); - } - - #[test] - fn generic_session_acceptance_matches_length_and_alphabet( - value in proptest::string::string_regex("[a-zA-Z0-9._-]{0,96}").unwrap(), - ) { - prop_assert_eq!(plausible_session_value(&value), value.len() >= MIN_SESSION_ID_LEN); - } - - #[test] - fn generic_vendor_session_headers_roundtrip( - vendor in proptest::string::string_regex("[a-z][a-z0-9-]{0,20}").unwrap(), - value in plausible_generic_session_strategy(), - ) { - let name = format!("x-{vendor}-session-id"); - let h = headers(&[(&name, &value)]); - prop_assert_eq!( - extract_session_from_headers(&h, "x-custom-session"), - Some(value), - ); - } - - #[test] - fn invalid_generic_characters_are_rejected( - prefix in plausible_generic_session_strategy(), - invalid in prop_oneof![Just("/"), Just(" "), Just(":"), Just("💥")], - ) { - let value = format!("{prefix}{invalid}"); - prop_assert!(!plausible_session_value(&value)); - } - - #[test] - fn trace_resolution_uses_first_nonempty_header( - persisting in header_token_strategy(), - litellm in header_token_strategy(), - request in header_token_strategy(), - call_id in header_token_strategy(), - ) { - let h = headers(&[ - ("x-persisting-trace-id", &persisting), - ("x-litellm-trace-id", &litellm), - ("x-request-id", &request), - ]); - prop_assert_eq!(resolve_trace_id(&h, &call_id), persisting); - prop_assert_eq!(resolve_trace_id(&HeaderMap::new(), &call_id), call_id); - } - } -} diff --git a/crates/persisting-gateway/src/session/client.rs b/crates/persisting-gateway/src/session/client.rs deleted file mode 100644 index 01ef0e5e7..000000000 --- a/crates/persisting-gateway/src/session/client.rs +++ /dev/null @@ -1,325 +0,0 @@ -//! Session client detection: peer port → process command line, persisted for trajectory headers. - -use std::collections::HashSet; -use std::fs; -use std::net::SocketAddr; -use std::path::{Path, PathBuf}; -use std::sync::Mutex; - -use anyhow::{Context, Result}; -use serde::{Deserialize, Serialize}; - -use crate::injection::host_identity::machine_fingerprint_for_client; -use crate::injection::peer::resolve_peer_client; -use crate::runtime::run_env::{read_run_child_info, read_run_session}; - -use super::storage::{CaptureRoute, trajectory_run_dir}; - -/// Sidecar metadata for `capture serve` (no `run_session`). `capture run` uses `run_child.yaml` + markdown frontmatter instead. -pub const SESSION_CLIENT_META_FILENAME: &str = "session-meta.yaml"; - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct SessionClientMeta { - pub peer: String, - pub peer_port: u16, - pub pid: u32, - pub command: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub machine_fp: Option, -} - -fn finalize_client_meta(mut meta: SessionClientMeta, peer: SocketAddr) -> SessionClientMeta { - if meta.machine_fp.is_none() { - let pid = (meta.pid > 0).then_some(meta.pid); - meta.machine_fp = Some(machine_fingerprint_for_client(peer, pid)); - } - meta -} - -fn peer_for_finalize(meta: &SessionClientMeta) -> SocketAddr { - meta.peer - .parse() - .unwrap_or_else(|_| "127.0.0.1:0".parse().expect("loopback")) -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -struct SessionMetaFile { - client: SessionClientMeta, -} - -pub fn session_client_meta_path(storage: &Path, agent_id: &str, session_id: &str) -> PathBuf { - storage - .join(agent_id) - .join(session_id) - .join(SESSION_CLIENT_META_FILENAME) -} - -pub fn session_client_meta_path_in_dir(session_dir: &Path) -> PathBuf { - session_dir.join(SESSION_CLIENT_META_FILENAME) -} - -pub fn load_session_client_meta(path: &Path) -> Result> { - if !path.is_file() { - return Ok(None); - } - let text = fs::read_to_string(path).with_context(|| format!("read {}", path.display()))?; - let file: SessionMetaFile = - serde_yaml::from_str(&text).with_context(|| format!("parse {}", path.display()))?; - Ok(Some(file.client)) -} - -pub fn write_session_client_meta(path: &Path, meta: &SessionClientMeta) -> Result<()> { - if let Some(parent) = path.parent() { - fs::create_dir_all(parent)?; - } - let file = SessionMetaFile { - client: meta.clone(), - }; - let yaml = serde_yaml::to_string(&file).context("serialize session-meta.yaml")?; - fs::write(path, yaml).with_context(|| format!("write {}", path.display())) -} - -/// Client metadata for markdown frontmatter on a capture run directory. -pub fn resolve_client_meta_for_run_dir( - storage: &Path, - run_dir: &Path, -) -> Option { - if let Ok(Some(m)) = load_session_client_meta(&session_client_meta_path_in_dir(run_dir)) { - let peer = peer_for_finalize(&m); - return Some(finalize_client_meta(m, peer)); - } - let run_id = run_dir.file_name()?.to_str()?; - let route = CaptureRoute { - root_session: Some(run_id.to_string()), - session_id: run_id.to_string(), - storage_session_id: run_id.to_string(), - subagent_id: None, - }; - client_meta_from_run_child(storage, &route).map(|m| { - let peer: SocketAddr = "127.0.0.1:0".parse().expect("loopback"); - finalize_client_meta(m, peer) - }) -} - -/// Client metadata for markdown frontmatter: sidecar file, else `capture run` child snapshot. -pub fn resolve_client_meta_for_session_dir( - storage: &Path, - session_dir: &Path, -) -> Option { - resolve_client_meta_for_run_dir(storage, session_dir) -} - -fn is_capture_run_active(storage: &Path, route: &CaptureRoute) -> bool { - read_run_session(storage) - .as_deref() - .zip(route.root_session.as_deref()) - .is_some_and(|(a, b)| a == b) -} - -fn client_meta_from_run_child(storage: &Path, route: &CaptureRoute) -> Option { - if !is_capture_run_active(storage, route) { - return None; - } - let run_child = read_run_child_info(storage)?; - Some(SessionClientMeta { - peer: String::new(), - peer_port: 0, - pid: run_child.pid, - command: run_child.command, - machine_fp: None, - }) -} - -fn should_persist_session_meta_file(storage: &Path, route: &CaptureRoute) -> bool { - !is_capture_run_active(storage, route) -} - -/// Records client metadata once per session (best-effort; ignores lookup failures). -#[derive(Default)] -pub struct SessionClientRegistry { - recorded: Mutex>, -} - -impl SessionClientRegistry { - pub fn ensure( - &self, - storage: &Path, - agent_id: &str, - route: &CaptureRoute, - peer: SocketAddr, - ) -> Option { - let session_dir = trajectory_run_dir(storage, agent_id, route); - let path = session_client_meta_path_in_dir(&session_dir); - if should_persist_session_meta_file(storage, route) { - if let Ok(Some(existing)) = load_session_client_meta(&path) { - return Some(finalize_client_meta(existing, peer)); - } - } else if let Some(m) = client_meta_from_run_child(storage, route) { - return Some(finalize_client_meta( - SessionClientMeta { - peer: peer.to_string(), - peer_port: peer.port(), - ..m - }, - peer, - )); - } - - let key = format!("{}|{agent_id}|{}", storage.display(), route.seq_key()); - { - let guard = self.recorded.lock().unwrap(); - if guard.contains(&key) { - if should_persist_session_meta_file(storage, route) { - return load_session_client_meta(&path) - .ok() - .flatten() - .map(|m| finalize_client_meta(m, peer)); - } - return client_meta_from_run_child(storage, route).map(|m| { - finalize_client_meta( - SessionClientMeta { - peer: peer.to_string(), - peer_port: peer.port(), - ..m - }, - peer, - ) - }); - } - } - - let mut meta = if is_capture_run_active(storage, route) { - read_run_child_info(storage).map(|run_child| SessionClientMeta { - peer: peer.to_string(), - peer_port: peer.port(), - pid: run_child.pid, - command: run_child.command, - machine_fp: None, - }) - } else { - None - }; - - if meta.is_none() { - meta = match resolve_peer_client(peer) { - Ok(Some(m)) => Some(m), - Ok(None) => { - tracing::debug!( - target: "persisting_gateway", - %peer, - session_id = %route.session_id, - "session client: no process found for peer port" - ); - None - } - Err(e) => { - tracing::warn!( - target: "persisting_gateway", - %peer, - session_id = %route.session_id, - "session client lookup failed: {e:#}" - ); - None - } - }; - } - - let Some(meta) = meta else { - self.recorded.lock().unwrap().insert(key); - return None; - }; - - let meta = finalize_client_meta(meta, peer); - - if should_persist_session_meta_file(storage, route) - && write_session_client_meta(&path, &meta).is_err() - { - return None; - } - - self.recorded.lock().unwrap().insert(key); - tracing::debug!( - target: "persisting_gateway", - peer = %meta.peer, - pid = meta.pid, - session_id = %route.session_id, - command = %meta.command, - "session client recorded" - ); - Some(meta) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn session_meta_roundtrip() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("session-meta.yaml"); - let meta = SessionClientMeta { - peer: "127.0.0.1:54321".into(), - peer_port: 54321, - pid: 4242, - command: "python3 agent.py --verbose".into(), - machine_fp: Some("abc123".into()), - }; - write_session_client_meta(&path, &meta).unwrap(); - let loaded = load_session_client_meta(&path).unwrap().unwrap(); - assert_eq!(loaded, meta); - } - - #[test] - fn ensure_prefers_run_child_over_lsof() { - let dir = tempfile::tempdir().unwrap(); - let storage = dir.path(); - std::fs::create_dir_all(storage.join(".capture")).unwrap(); - std::fs::write(storage.join(".capture/run_session"), "sess-a").unwrap(); - crate::runtime::run_env::write_run_child_info( - storage, - 4242, - &["claude".into(), "--model".into()], - ) - .unwrap(); - - let registry = SessionClientRegistry::default(); - let peer: std::net::SocketAddr = "127.0.0.1:55522".parse().unwrap(); - let route = CaptureRoute { - root_session: Some("sess-a".into()), - session_id: "sess-a".into(), - storage_session_id: "sess-a".into(), - subagent_id: None, - }; - let meta = registry - .ensure(storage, "agent", &route, peer) - .expect("run_child metadata"); - assert_eq!(meta.command, "claude --model"); - assert_eq!(meta.pid, 4242); - assert_eq!(meta.peer_port, 55522); - assert!(meta.machine_fp.as_ref().is_some_and(|fp| fp.len() == 32)); - assert!( - !session_client_meta_path_in_dir(&storage.join("agent/sess-a")).exists(), - "capture run should not write session-meta.yaml" - ); - } - - #[test] - fn resolve_client_meta_from_run_child() { - let dir = tempfile::tempdir().unwrap(); - let storage = dir.path(); - std::fs::create_dir_all(storage.join(".capture")).unwrap(); - std::fs::write(storage.join(".capture/run_session"), "sess-b").unwrap(); - crate::runtime::run_env::write_run_child_info( - storage, - 99, - &["python3".into(), "agent.py".into()], - ) - .unwrap(); - let session_dir = storage.join("agent").join("sess-b"); - std::fs::create_dir_all(&session_dir).unwrap(); - let meta = resolve_client_meta_for_session_dir(storage, &session_dir).unwrap(); - assert_eq!(meta.command, "python3 agent.py"); - assert_eq!(meta.pid, 99); - } -} diff --git a/crates/persisting-gateway/src/session/index.rs b/crates/persisting-gateway/src/session/index.rs deleted file mode 100644 index 8a612fabb..000000000 --- a/crates/persisting-gateway/src/session/index.rs +++ /dev/null @@ -1,348 +0,0 @@ -//! Session index on disk for `capture list` without full Lance replay. - -use std::collections::HashMap; -use std::fs; -use std::path::{Path, PathBuf}; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Arc, RwLock}; - -use anyhow::{Context, Result}; -use chrono::{DateTime, Utc}; -use serde::{Deserialize, Serialize}; - -use crate::provider::ProviderKind; -use crate::usage::TokenUsage; - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct SessionIndex { - pub storage: String, - pub updated_at: DateTime, - pub sessions: Vec, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct SessionSummary { - pub agent_id: String, - pub session_id: String, - pub provider: String, - pub protocol: String, - pub model: String, - pub first_seen: DateTime, - pub last_seen: DateTime, - pub request_count: u64, - pub usage: TokenUsage, - pub estimated_cost_usd: f64, - pub active: bool, -} - -pub struct SessionIndexStore { - path: PathBuf, - inner: Arc>, - dirty: Arc, -} - -impl SessionIndexStore { - pub fn open(storage: &Path) -> Result { - fs::create_dir_all(storage)?; - let path = storage.join(".capture").join("sessions.json"); - if let Some(parent) = path.parent() { - fs::create_dir_all(parent)?; - } - let inner = if path.exists() { - let s = fs::read_to_string(&path)?; - serde_json::from_str(&s).unwrap_or_else(|_| SessionIndex { - storage: storage.display().to_string(), - updated_at: Utc::now(), - sessions: Vec::new(), - }) - } else { - SessionIndex { - storage: storage.display().to_string(), - updated_at: Utc::now(), - sessions: Vec::new(), - } - }; - Ok(Self { - path, - inner: Arc::new(RwLock::new(inner)), - dirty: Arc::new(AtomicBool::new(false)), - }) - } - - pub fn clone_handle(&self) -> SessionIndexHandle { - SessionIndexHandle { - inner: Arc::clone(&self.inner), - path: self.path.clone(), - dirty: Arc::clone(&self.dirty), - } - } - - pub fn load(storage: &Path) -> Result { - let path = storage.join(".capture").join("sessions.json"); - if !path.exists() { - return Ok(SessionIndex { - storage: storage.display().to_string(), - updated_at: Utc::now(), - sessions: Vec::new(), - }); - } - let s = fs::read_to_string(&path)?; - Ok(serde_json::from_str(&s)?) - } - - pub fn save(&self) -> Result<()> { - let mut guard = self.inner.write().unwrap(); - guard.updated_at = Utc::now(); - let json = serde_json::to_string_pretty(&*guard)?; - if let Some(parent) = self.path.parent() { - fs::create_dir_all(parent)?; - } - fs::write(&self.path, json)?; - Ok(()) - } -} - -#[derive(Clone)] -pub struct SessionIndexHandle { - inner: Arc>, - path: PathBuf, - dirty: Arc, -} - -impl SessionIndexHandle { - fn mark_dirty(&self) { - self.dirty.store(true, Ordering::Release); - } - - pub fn record_request( - &self, - agent_id: &str, - session_id: &str, - provider: ProviderKind, - protocol: &str, - model: &str, - ) { - let now = Utc::now(); - let mut guard = self.inner.write().unwrap(); - if let Some(s) = guard - .sessions - .iter_mut() - .find(|s| s.agent_id == agent_id && s.session_id == session_id) - { - s.last_seen = now; - s.request_count += 1; - s.active = true; - if !model.is_empty() && model != "_unknown" { - s.model = model.to_string(); - } - s.provider = provider.as_str().to_string(); - s.protocol = protocol.to_string(); - } else { - guard.sessions.push(SessionSummary { - agent_id: agent_id.to_string(), - session_id: session_id.to_string(), - provider: provider.as_str().to_string(), - protocol: protocol.to_string(), - model: model.to_string(), - first_seen: now, - last_seen: now, - request_count: 1, - usage: TokenUsage::default(), - estimated_cost_usd: 0.0, - active: true, - }); - } - self.mark_dirty(); - } - - pub fn record_response( - &self, - agent_id: &str, - session_id: &str, - provider: ProviderKind, - model: &str, - usage: &TokenUsage, - cost_usd: f64, - ) { - let now = Utc::now(); - let mut guard = self.inner.write().unwrap(); - if let Some(s) = guard - .sessions - .iter_mut() - .find(|s| s.agent_id == agent_id && s.session_id == session_id) - { - s.last_seen = now; - s.active = false; - s.usage.merge(usage); - s.estimated_cost_usd += cost_usd; - if !model.is_empty() { - s.model = model.to_string(); - } - s.provider = provider.as_str().to_string(); - } - self.mark_dirty(); - } - - pub fn set_active(&self, agent_id: &str, session_id: &str, active: bool) { - let mut guard = self.inner.write().unwrap(); - if let Some(s) = guard - .sessions - .iter_mut() - .find(|s| s.agent_id == agent_id && s.session_id == session_id) - { - s.active = active; - } - self.mark_dirty(); - } - - pub fn flush_if_dirty(&self) -> Result<()> { - if !self.dirty.load(Ordering::Acquire) { - return Ok(()); - } - self.flush() - } - - pub fn flush(&self) -> Result<()> { - let guard = self.inner.read().unwrap(); - let json = serde_json::to_string_pretty(&*guard)?; - if let Some(parent) = self.path.parent() { - fs::create_dir_all(parent)?; - } - fs::write(&self.path, json).context("write sessions.json")?; - self.dirty.store(false, Ordering::Release); - Ok(()) - } - - pub fn snapshot(&self) -> SessionIndex { - self.inner.read().unwrap().clone() - } -} - -/// Scan session layout dirs and merge with index file. -pub fn discover_sessions(storage: &Path) -> Result> { - let mut by_key: HashMap<(String, String), SessionSummary> = HashMap::new(); - if let Ok(index) = SessionIndexStore::load(storage) { - for s in index.sessions { - by_key.insert((s.agent_id.clone(), s.session_id.clone()), s); - } - } - if storage.is_dir() { - for agent_entry in fs::read_dir(storage)? { - let agent_entry = agent_entry?; - if !agent_entry.file_type()?.is_dir() { - continue; - } - let name = agent_entry.file_name().to_string_lossy().to_string(); - if name.starts_with('.') { - continue; - } - for sess_entry in fs::read_dir(agent_entry.path())? { - let sess_entry = sess_entry?; - if !sess_entry.file_type()?.is_dir() { - continue; - } - let sid = sess_entry.file_name().to_string_lossy().to_string(); - by_key - .entry((name.clone(), sid.clone())) - .or_insert_with(|| SessionSummary { - agent_id: name.clone(), - session_id: sid, - provider: "unknown".into(), - protocol: "unknown".into(), - model: "unknown".into(), - first_seen: Utc::now(), - last_seen: Utc::now(), - request_count: 0, - usage: TokenUsage::default(), - estimated_cost_usd: 0.0, - active: false, - }); - } - } - } - let mut out: Vec<_> = by_key.into_values().collect(); - out.sort_by_key(|b| std::cmp::Reverse(b.last_seen)); - Ok(out) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::provider::ProviderKind; - use crate::usage::TokenUsage; - - #[test] - fn flush_if_dirty_skips_when_not_dirty() { - let dir = tempfile::tempdir().unwrap(); - let store = SessionIndexStore::open(dir.path()).unwrap(); - let handle = store.clone_handle(); - handle.flush_if_dirty().unwrap(); - assert!(!dir.path().join(".capture").join("sessions.json").exists()); - } - - #[test] - fn flush_if_dirty_persists_after_record_request() { - let dir = tempfile::tempdir().unwrap(); - let store = SessionIndexStore::open(dir.path()).unwrap(); - let handle = store.clone_handle(); - handle.record_request("agent", "sess", ProviderKind::OpenAi, "chat", "gpt-4"); - handle.flush_if_dirty().unwrap(); - - let loaded = SessionIndexStore::load(dir.path()).unwrap(); - assert_eq!(loaded.sessions.len(), 1); - assert_eq!(loaded.sessions[0].request_count, 1); - assert!(loaded.sessions[0].active); - } - - #[test] - fn cloned_handles_share_dirty_flag() { - let dir = tempfile::tempdir().unwrap(); - let store = SessionIndexStore::open(dir.path()).unwrap(); - let writer = store.clone_handle(); - let reader = writer.clone(); - - writer.record_request("agent", "sess", ProviderKind::OpenAi, "chat", "m"); - reader.flush_if_dirty().unwrap(); - - writer.record_response( - "agent", - "sess", - ProviderKind::OpenAi, - "m", - &TokenUsage { - input_tokens: 10, - output_tokens: 5, - total_tokens: 15, - cache_read_tokens: 0, - cache_write_tokens: 0, - reasoning_tokens: 0, - }, - 0.01, - ); - writer.flush_if_dirty().unwrap(); - - let loaded = SessionIndexStore::load(dir.path()).unwrap(); - assert_eq!(loaded.sessions[0].request_count, 1); - assert_eq!(loaded.sessions[0].usage.total_tokens, 15); - assert!(!loaded.sessions[0].active); - } - - #[test] - fn second_flush_if_dirty_without_changes_is_noop() { - let dir = tempfile::tempdir().unwrap(); - let store = SessionIndexStore::open(dir.path()).unwrap(); - let handle = store.clone_handle(); - handle.record_request( - "agent", - "sess", - ProviderKind::Anthropic, - "messages", - "claude", - ); - handle.flush_if_dirty().unwrap(); - handle.flush_if_dirty().unwrap(); - - let loaded = SessionIndexStore::load(dir.path()).unwrap(); - assert_eq!(loaded.sessions.len(), 1); - } -} diff --git a/crates/persisting-gateway/src/session/mod.rs b/crates/persisting-gateway/src/session/mod.rs deleted file mode 100644 index 41a4bde80..000000000 --- a/crates/persisting-gateway/src/session/mod.rs +++ /dev/null @@ -1,7 +0,0 @@ -//! Request-scoped session identity and indexing. - -pub mod chain; -pub mod client; -pub mod index; -pub mod snapshots; -pub mod storage; diff --git a/crates/persisting-gateway/src/session/snapshots.rs b/crates/persisting-gateway/src/session/snapshots.rs deleted file mode 100644 index 187686934..000000000 --- a/crates/persisting-gateway/src/session/snapshots.rs +++ /dev/null @@ -1,101 +0,0 @@ -//! Read persisted story summaries without depending on capture engine types. - -use std::collections::HashMap; -use std::path::{Path, PathBuf}; - -use anyhow::{Context, Result}; -use serde_json::Value; - -fn snapshots_path(storage: &Path) -> PathBuf { - storage.join(".capture").join("story_snapshots.json") -} - -/// Per markdown session stem: user-visible dialogue turn count. -pub fn load_snapshot_turn_counts(storage: &Path) -> Result> { - let path = snapshots_path(storage); - if !path.is_file() { - return Ok(HashMap::new()); - } - let raw = std::fs::read_to_string(&path).with_context(|| format!("read {}", path.display()))?; - let root: Value = serde_json::from_str(&raw).context("parse story_snapshots.json")?; - if let Some(counts) = root.get("turn_counts").and_then(Value::as_object) { - let mut out = HashMap::new(); - for (stem, v) in counts { - if let Some(n) = v.as_u64() { - out.insert(stem.clone(), n); - } - } - if !out.is_empty() { - return Ok(out); - } - } - let mut out = HashMap::new(); - if let Some(stories) = root.get("stories").and_then(Value::as_object) { - for (stem, story) in stories { - out.insert(stem.clone(), user_turn_count_from_story_json(story)); - } - } - Ok(out) -} - -fn user_turn_count_from_story_json(story: &Value) -> u64 { - let Some(turns) = story.get("turns").and_then(Value::as_array) else { - return 0; - }; - turns - .iter() - .filter(|t| { - t.get("kind").and_then(|k| k.as_str()) == Some("dialogue") && t.get("user").is_some() - }) - .count() as u64 -} - -#[cfg(test)] -mod tests { - use super::*; - use serde_json::json; - - #[test] - fn load_prefers_turn_counts_field() { - let dir = tempfile::tempdir().unwrap(); - let capture = dir.path().join(".capture"); - std::fs::create_dir_all(&capture).unwrap(); - std::fs::write( - capture.join("story_snapshots.json"), - json!({ - "updated_at": "t", - "turn_counts": { "run-a": 3 }, - "stories": { "run-a": { "turns": [] } } - }) - .to_string(), - ) - .unwrap(); - let counts = load_snapshot_turn_counts(dir.path()).unwrap(); - assert_eq!(counts.get("run-a"), Some(&3)); - } - - #[test] - fn load_falls_back_to_stories_array() { - let dir = tempfile::tempdir().unwrap(); - let capture = dir.path().join(".capture"); - std::fs::create_dir_all(&capture).unwrap(); - std::fs::write( - capture.join("story_snapshots.json"), - json!({ - "updated_at": "t", - "stories": { - "run-a": { - "turns": [ - { "kind": "dialogue", "user": { "text": "hi" } }, - { "kind": "autonomous" } - ] - } - } - }) - .to_string(), - ) - .unwrap(); - let counts = load_snapshot_turn_counts(dir.path()).unwrap(); - assert_eq!(counts.get("run-a"), Some(&1)); - } -} diff --git a/crates/persisting-gateway/src/session/storage.rs b/crates/persisting-gateway/src/session/storage.rs deleted file mode 100644 index f4b978892..000000000 --- a/crates/persisting-gateway/src/session/storage.rs +++ /dev/null @@ -1,311 +0,0 @@ -//! Session routing and run-directory storage (`{run}/{session_id}.md`). - -use std::path::{Path, PathBuf}; - -use axum::http::HeaderMap; -use bytes::Bytes; - -use super::chain::{ephemeral_session_id, extract_claude_agent_id, extract_session_from_headers}; -use crate::dialogue_extract::{is_main_agent_continuation, is_subagent_shape_payload}; -use crate::runtime::run_env::read_run_session; -use persisting_pchronicle::storage::{ - is_subagent_session_storage_key, session_markdown_write_path_for_key, -}; - -/// Where to append captures for one proxied LLM request. -#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] -pub struct CaptureRoute { - /// Run directory name under `{storage}/{agent_id}/` (capture run or daily serve bucket). - pub root_session: Option, - /// Logical session from request headers (stored on - /// [`EventRecord`](crate::record::EventRecord)). - pub session_id: String, - /// Markdown filename stem and event-log key (`{storage_session_id}.md`). - pub storage_session_id: String, - /// Claude Code subagent instance id (`x-claude-code-agent-id`), when present. - pub subagent_id: Option, -} - -impl CaptureRoute { - pub fn seq_key(&self) -> String { - match &self.root_session { - Some(root) => format!("{root}|{}", self.storage_session_id), - None => self.storage_session_id.clone(), - } - } - - /// Run id for nested append (always set when [`Self::root_session`] is present). - pub fn append_root_session(&self) -> Option { - self.root_session.clone() - } - - /// Route for offline replay / reconcile (main session omits `root_session` wrapper). - pub fn for_replay_stem(root_session: &str, session_id: &str) -> Self { - let subagent_id = is_subagent_session_storage_key(session_id).then(|| { - session_id - .strip_prefix("agent-") - .unwrap_or(session_id) - .to_string() - }); - Self { - root_session: if session_id == root_session { - None - } else { - Some(root_session.to_string()) - }, - session_id: session_id.to_string(), - storage_session_id: session_id.to_string(), - subagent_id, - } - } - - /// Route for live run markdown frontmatter refresh (`root_session` always set). - pub fn for_run_markdown_stem(root_session: &str, stem: &str) -> Self { - let subagent_id = is_subagent_session_storage_key(stem) - .then(|| stem.strip_prefix("agent-").unwrap_or(stem).to_string()); - Self { - root_session: Some(root_session.to_string()), - session_id: stem.to_string(), - storage_session_id: stem.to_string(), - subagent_id, - } - } -} - -fn subagent_storage_leaf(claude_agent_id: &str) -> String { - format!("agent-{claude_agent_id}") -} - -fn main_storage_session_id(header_session: &Option, run_id: &str) -> String { - header_session.clone().unwrap_or_else(|| run_id.to_string()) -} - -/// Resolve routing: header session wins for filename; subagents use `agent-{id}.md` siblings. -pub fn resolve_capture_route( - headers: &HeaderMap, - body: &Bytes, - session_header: &str, - storage: &Path, -) -> CaptureRoute { - let root_session = read_run_session(storage); - let header_session = extract_session_from_headers(headers, session_header); - let claude_agent_id = extract_claude_agent_id(headers); - let _body_subagent = is_subagent_request_body(body); - - let session_id = header_session - .clone() - .unwrap_or_else(|| root_session.clone().unwrap_or_else(ephemeral_session_id)); - - match root_session { - None => { - let storage_session_id = if let Some(agent_id) = claude_agent_id.as_deref() { - subagent_storage_leaf(agent_id) - } else { - session_id.clone() - }; - CaptureRoute { - root_session: None, - session_id, - storage_session_id, - subagent_id: claude_agent_id, - } - } - Some(root) => { - if claude_agent_id.is_some() && is_main_agent_continuation(body) { - CaptureRoute { - root_session: Some(root.clone()), - session_id, - storage_session_id: main_storage_session_id(&header_session, &root), - subagent_id: None, - } - } else if let Some(agent_id) = claude_agent_id { - CaptureRoute { - root_session: Some(root), - session_id, - storage_session_id: subagent_storage_leaf(&agent_id), - subagent_id: Some(agent_id), - } - } else { - CaptureRoute { - root_session: Some(root.clone()), - session_id, - storage_session_id: main_storage_session_id(&header_session, &root), - subagent_id: None, - } - } - } - } -} - -fn is_subagent_request_body(body: &Bytes) -> bool { - serde_json::from_slice::(body) - .ok() - .is_some_and(|v| is_subagent_shape_payload(&v)) -} - -/// Run directory under `{storage}/{agent_id}/…` (all sessions in one capture run share this dir). -pub fn trajectory_run_dir(storage: &Path, agent_id: &str, route: &CaptureRoute) -> PathBuf { - let base = storage.join(agent_id); - match &route.root_session { - Some(root) => base.join(root), - None => base.join(&route.storage_session_id), - } -} - -/// Full path to the session markdown file (`{run_dir}/{storage_session_id}.md`). -pub fn trajectory_markdown_path(storage: &Path, agent_id: &str, route: &CaptureRoute) -> PathBuf { - let run_dir = trajectory_run_dir(storage, agent_id, route); - session_markdown_write_path_for_key(&run_dir, &route.storage_session_id) -} - -/// Config / metadata key (run root when present). -pub fn route_config_key(route: &CaptureRoute) -> &str { - route - .root_session - .as_deref() - .unwrap_or(route.storage_session_id.as_str()) -} - -#[cfg(test)] -mod tests { - use super::*; - use axum::http::HeaderValue; - use axum::http::header::HeaderName; - - fn headers(pairs: &[(&str, &str)]) -> HeaderMap { - let mut h = HeaderMap::new(); - for (k, v) in pairs { - h.insert( - HeaderName::from_bytes(k.as_bytes()).unwrap(), - HeaderValue::try_from(*v).unwrap(), - ); - } - h - } - - #[test] - fn main_agent_writes_header_session_md_in_run_dir() { - let dir = tempfile::tempdir().unwrap(); - std::fs::create_dir_all(dir.path().join(".capture")).unwrap(); - std::fs::write(dir.path().join(".capture/run_session"), "my-run").unwrap(); - let body = Bytes::from_static( - br#"{"model":"deepseek-chat","messages":[{"role":"user","content":"hi"}]}"#, - ); - let header_session = "e96634a3-fa28-4083-b354-55542e2dca01"; - let h = headers(&[("x-claude-code-session-id", header_session)]); - let route = resolve_capture_route(&h, &body, "x-persisting-session-id", dir.path()); - assert_eq!(route.root_session.as_deref(), Some("my-run")); - assert_eq!(route.storage_session_id, header_session); - assert_eq!(route.session_id, header_session); - assert!(route.subagent_id.is_none()); - let run = trajectory_run_dir(dir.path(), "agent", &route); - assert_eq!(run, dir.path().join("agent/my-run")); - let md = trajectory_markdown_path(dir.path(), "agent", &route); - assert_eq!(md, run.join(format!("{header_session}.md"))); - } - - #[test] - fn flash_probe_without_agent_id_writes_header_session_md() { - let dir = tempfile::tempdir().unwrap(); - std::fs::create_dir_all(dir.path().join(".capture")).unwrap(); - std::fs::write(dir.path().join(".capture/run_session"), "my-run").unwrap(); - let body = Bytes::from_static( - br#"{"model":"deepseek-v4-flash","messages":[{"role":"user","content":[{"type":"text","text":"\nexplore repo with three subagents\n"}]}]}"#, - ); - let header_session = "884e189a-9433-4996-a809-3a8232a8967d"; - let h = headers(&[("x-claude-code-session-id", header_session)]); - let route = resolve_capture_route(&h, &body, "x-persisting-session-id", dir.path()); - assert_eq!(route.storage_session_id, header_session); - let md = trajectory_markdown_path(dir.path(), "agent", &route); - assert!(md.ends_with(format!("{header_session}.md"))); - } - - #[test] - fn haiku_session_wrapper_without_agent_id_writes_header_session_md() { - let dir = tempfile::tempdir().unwrap(); - std::fs::create_dir_all(dir.path().join(".capture")).unwrap(); - std::fs::write(dir.path().join(".capture/run_session"), "my-run").unwrap(); - let body = Bytes::from_static( - br#"{"model":"claude-haiku-4-5","messages":[{"role":"user","content":[{"type":"text","text":"\ntask\n"}]}]}"#, - ); - let header_session = "sub-session-uuid-12345678"; - let h = headers(&[("x-claude-code-session-id", header_session)]); - let route = resolve_capture_route(&h, &body, "x-persisting-session-id", dir.path()); - assert_eq!(route.storage_session_id, header_session); - let md = trajectory_markdown_path(dir.path(), "agent", &route); - assert!(md.ends_with(format!("{header_session}.md"))); - } - - #[test] - fn parallel_subagents_use_sibling_agent_md_files() { - let dir = tempfile::tempdir().unwrap(); - std::fs::create_dir_all(dir.path().join(".capture")).unwrap(); - std::fs::write(dir.path().join(".capture/run_session"), "my-run").unwrap(); - let body = Bytes::from_static( - br#"{"model":"deepseek-v4-flash","messages":[{"role":"user","content":"explore auth"}]}"#, - ); - let shared_session = "e96634a3-fa28-4083-b354-55542e2dca01"; - for agent in ["a111111", "b222222", "c333333"] { - let h = headers(&[ - ("x-claude-code-session-id", shared_session), - ("x-claude-code-agent-id", agent), - ]); - let route = resolve_capture_route(&h, &body, "x-persisting-session-id", dir.path()); - assert_eq!(route.subagent_id.as_deref(), Some(agent)); - assert_eq!(route.storage_session_id, format!("agent-{agent}")); - let run = trajectory_run_dir(dir.path(), "agent", &route); - assert_eq!(run, dir.path().join("agent/my-run")); - let md = trajectory_markdown_path(dir.path(), "agent", &route); - assert_eq!(md, run.join(format!("agent-{agent}.md"))); - } - } - - #[test] - fn serve_flat_layout() { - let body = Bytes::from_static(b"{}"); - let h = headers(&[("x-persisting-session-id", "sess-flat")]); - let route = resolve_capture_route(&h, &body, "x-persisting-session-id", Path::new("/tmp")); - assert!(route.root_session.is_none()); - assert_eq!(route.storage_session_id, "sess-flat"); - let md = trajectory_markdown_path(Path::new("/tmp"), "agent", &route); - assert!(md.ends_with("sess-flat/sess-flat.md")); - } - - #[test] - fn no_header_falls_back_to_run_id_md() { - let dir = tempfile::tempdir().unwrap(); - std::fs::create_dir_all(dir.path().join(".capture")).unwrap(); - std::fs::write(dir.path().join(".capture/run_session"), "run-only").unwrap(); - let body = Bytes::from_static(b"{}"); - let route = resolve_capture_route( - &HeaderMap::new(), - &body, - "x-persisting-session-id", - dir.path(), - ); - assert_eq!(route.session_id, "run-only"); - assert_eq!(route.storage_session_id, "run-only"); - let md = trajectory_markdown_path(dir.path(), "agent", &route); - assert!(md.ends_with("run-only/run-only.md")); - } - - #[test] - fn task_notification_with_agent_id_writes_main_session_md() { - let dir = tempfile::tempdir().unwrap(); - std::fs::create_dir_all(dir.path().join(".capture")).unwrap(); - std::fs::write(dir.path().join(".capture/run_session"), "my-run").unwrap(); - let body = Bytes::from_static( - br#"{"model":"deepseek-v4-pro","messages":[{"role":"user","content":[{"type":"text","text":"\naeb40a1230926b4a8\ncompleted\n"}]}]}"#, - ); - let header_session = "c9df8c43-e129-47d3-93c3-7cedfab86931"; - let h = headers(&[ - ("x-claude-code-session-id", header_session), - ("x-claude-code-agent-id", "aeb40a1230926b4a8"), - ]); - let route = resolve_capture_route(&h, &body, "x-persisting-session-id", dir.path()); - assert_eq!(route.storage_session_id, header_session); - assert!(route.subagent_id.is_none()); - let md = trajectory_markdown_path(dir.path(), "agent", &route); - assert!(md.ends_with(format!("{header_session}.md"))); - } -} diff --git a/crates/persisting-gateway/src/sink.rs b/crates/persisting-gateway/src/sink.rs deleted file mode 100644 index 3fde04377..000000000 --- a/crates/persisting-gateway/src/sink.rs +++ /dev/null @@ -1,871 +0,0 @@ -//! Gateway event sink: append captured trajectory records per session. -//! -//! **Proxy capture path:** the internal `StoryActor` is the sole writer — it calls -//! [`CaptureEventSink::append`] on `PersistRecord` (and uses [`CaptureEventSink::peek_next_seq`] for streaming drafts). -//! The internal `CapturePreparer` only builds records and story commands. -//! Session lifecycle records may still append via [`super::lifecycle`]. - -use std::collections::HashMap; -use std::sync::{Arc, Mutex}; - -use anyhow::Result; -use serde_json::Value; - -use super::record::{EventRecord, ensure_timestamp, now_rfc3339}; -use crate::Call; -use crate::config::CaptureLevel; -use crate::projection::markdown_pipeline::stamp_request_payload; -use crate::session::storage::CaptureRoute; - -pub trait CaptureEventSink: Send + Sync { - /// Assign session-local `seq` on `record`, then persist. Mutates `record.seq` in place. - fn append(&self, route: &CaptureRoute, agent_id: &str, record: &mut EventRecord) -> Result<()>; - - /// Next `seq` that [`Self::append`] would assign (does not increment). - /// Returns `None` when the sink cannot predict seq (draft markdown preview unsupported). - fn peek_next_seq(&self, route: &CaptureRoute) -> Option { - let _ = route; - None - } -} - -/// Assigns monotonic `seq` per storage target without persisting (`-f md` capture path). -pub struct SeqOnlySink { - next_seq: Mutex>, -} - -impl Default for SeqOnlySink { - fn default() -> Self { - Self::new() - } -} - -impl SeqOnlySink { - pub fn new() -> Self { - Self { - next_seq: Mutex::new(HashMap::new()), - } - } - - fn assign_seq(&self, route: &CaptureRoute, record: &mut EventRecord) { - let mut guard = self.next_seq.lock().unwrap(); - let seq = guard.entry(route.seq_key()).or_insert(0); - record.seq = *seq; - *seq += 1; - record.session_id = Some(route.session_id.clone()); - record.subagent_id = route.subagent_id.clone(); - } -} - -impl CaptureEventSink for SeqOnlySink { - fn append( - &self, - route: &CaptureRoute, - _agent_id: &str, - record: &mut EventRecord, - ) -> Result<()> { - ensure_timestamp(record); - self.assign_seq(route, record); - Ok(()) - } - - fn peek_next_seq(&self, route: &CaptureRoute) -> Option { - Some( - self.next_seq - .lock() - .unwrap() - .get(&route.seq_key()) - .copied() - .unwrap_or(0), - ) - } -} - -/// Assigns monotonic `seq` per storage target and forwards typed records. -pub struct CallbackSink { - agent_id: String, - next_seq: Mutex>>>, - #[allow(clippy::type_complexity)] - callback: Box Result<()> + Send + Sync>, -} - -impl CallbackSink { - pub fn new(agent_id: impl Into, callback: F) -> Self - where - F: Fn(&CaptureRoute, &str, EventRecord) -> Result<()> + Send + Sync + 'static, - { - Self { - agent_id: agent_id.into(), - next_seq: Mutex::new(HashMap::new()), - callback: Box::new(callback), - } - } -} - -impl CaptureEventSink for CallbackSink { - fn append(&self, route: &CaptureRoute, agent_id: &str, record: &mut EventRecord) -> Result<()> { - ensure_timestamp(record); - let sequence = { - let mut guard = self.next_seq.lock().unwrap(); - Arc::clone( - guard - .entry(route.seq_key()) - .or_insert_with(|| Arc::new(Mutex::new(0))), - ) - }; - // Serialize one storage target through persistence. The sequence is - // advanced only after the callback accepts the record, so a rejected - // append can be retried without leaving a permanent gap. - let mut next_seq = sequence.lock().unwrap(); - record.seq = *next_seq; - let following_seq = next_seq - .checked_add(1) - .ok_or_else(|| anyhow::anyhow!("capture sequence exhausted for {}", route.seq_key()))?; - record.session_id = Some(route.session_id.clone()); - record.subagent_id = route.subagent_id.clone(); - let aid = if agent_id.is_empty() { - self.agent_id.as_str() - } else { - agent_id - }; - (self.callback)(route, aid, record.clone())?; - *next_seq = following_seq; - Ok(()) - } - - fn peek_next_seq(&self, route: &CaptureRoute) -> Option { - let sequence = self.next_seq.lock().unwrap().get(&route.seq_key()).cloned(); - Some(sequence.map_or(0, |sequence| *sequence.lock().unwrap())) - } -} - -#[cfg(test)] -mod sequence_tests { - use std::sync::atomic::{AtomicUsize, Ordering}; - - use super::*; - - fn record() -> EventRecord { - EventRecord { - identity: Default::default(), - seq: 99, - source: "test".into(), - kind: "test".into(), - timestamp: None, - session_id: None, - agent_id: None, - parent_uuid: None, - trace_id: None, - call_id: None, - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload: Value::Null, - } - } - - #[test] - fn callback_rejection_does_not_advance_sequence() { - let attempts = Arc::new(AtomicUsize::new(0)); - let observed = Arc::new(Mutex::new(Vec::new())); - let sink = CallbackSink::new("agent", { - let attempts = Arc::clone(&attempts); - let observed = Arc::clone(&observed); - move |_, _, record| { - observed.lock().unwrap().push(record.seq); - if attempts.fetch_add(1, Ordering::SeqCst) == 0 { - anyhow::bail!("reject first append"); - } - Ok(()) - } - }); - let route = CaptureRoute { - root_session: Some("run".into()), - session_id: "session".into(), - storage_session_id: "session".into(), - subagent_id: None, - }; - - let mut first = record(); - assert!(sink.append(&route, "agent", &mut first).is_err()); - assert_eq!(sink.peek_next_seq(&route), Some(0)); - - let mut retry = record(); - sink.append(&route, "agent", &mut retry).unwrap(); - assert_eq!(retry.seq, 0); - assert_eq!(sink.peek_next_seq(&route), Some(1)); - assert_eq!(*observed.lock().unwrap(), vec![0, 0]); - } -} - -/// Sensitive header names (lowercase) — values replaced with `` when recorded. -const REDACT_HEADER_NAMES: &[&str] = &[ - "authorization", - "proxy-authorization", - "cookie", - "set-cookie", - "x-api-key", - "api-key", - "x-goog-api-key", -]; - -const REDACTED_VALUE: &str = ""; - -fn is_sensitive_header_name(name: &str) -> bool { - let name = name.to_ascii_lowercase(); - REDACT_HEADER_NAMES.contains(&name.as_str()) - || name.ends_with("-api-key") - || name.ends_with("-token") - || name.contains("secret-key") -} - -fn is_sensitive_field_name(name: &str) -> bool { - const SECRET_FIELDS: &[&str] = &[ - "apikey", - "accesstoken", - "refreshtoken", - "authorization", - "password", - "secret", - "clientsecret", - "cookie", - "setcookie", - "token", - "idtoken", - "sessiontoken", - "bearertoken", - "secretaccesskey", - "privatekey", - ]; - let normalized = name - .chars() - .filter(|character| character.is_ascii_alphanumeric()) - .map(|character| character.to_ascii_lowercase()) - .collect::(); - SECRET_FIELDS.contains(&normalized.as_str()) -} - -fn is_sensitive_query_field_name(name: &str) -> bool { - name.eq_ignore_ascii_case("key") || is_sensitive_field_name(name) -} - -/// Return a persistence-safe copy of HTTP headers without mutating the wire -/// request. Header names and duplicates are retained for replay diagnostics. -pub(crate) fn redact_sensitive_headers(headers: &[(String, String)]) -> Vec<(String, String)> { - headers - .iter() - .map(|(name, value)| { - let value = if is_sensitive_header_name(name) { - REDACTED_VALUE.to_string() - } else { - value.clone() - }; - (name.clone(), value) - }) - .collect() -} - -/// Redact credentials carried in URL user-info or well-known query fields. -/// Invalid/relative URL syntax is retained, with its query sanitized when possible. -pub(crate) fn redact_sensitive_url(value: &str) -> String { - let (mut parsed, scheme_relative) = if value.starts_with("//") { - match url::Url::parse(&format!("http:{value}")) { - Ok(url) => (Some(url), true), - Err(_) => (None, false), - } - } else { - (url::Url::parse(value).ok(), false) - }; - - let value = if let Some(url) = parsed.as_mut() { - if !url.username().is_empty() { - let _ = url.set_username(REDACTED_VALUE); - } - if url.password().is_some() { - let _ = url.set_password(Some(REDACTED_VALUE)); - } - let rendered = url.to_string(); - if scheme_relative { - rendered - .strip_prefix("http:") - .unwrap_or(&rendered) - .to_string() - } else { - rendered - } - } else { - value.to_string() - }; - - let Some((prefix, query_and_fragment)) = value.split_once('?') else { - return value; - }; - let (query, fragment) = query_and_fragment - .split_once('#') - .map_or((query_and_fragment, None), |(query, fragment)| { - (query, Some(fragment)) - }); - let pairs = url::form_urlencoded::parse(query.as_bytes()).collect::>(); - if !pairs - .iter() - .any(|(name, _)| is_sensitive_query_field_name(name)) - { - return value; - } - let mut serializer = url::form_urlencoded::Serializer::new(String::new()); - for (name, field_value) in pairs { - serializer.append_pair( - &name, - if is_sensitive_query_field_name(&name) { - REDACTED_VALUE - } else { - &field_value - }, - ); - } - let mut redacted_url = format!("{prefix}?{}", serializer.finish()); - if let Some(fragment) = fragment { - redacted_url.push('#'); - redacted_url.push_str(fragment); - } - redacted_url -} - -/// Infer keep-alive / persistent connection flags from request headers + HTTP version. -pub fn infer_connection_persistent( - headers: &[(String, String)], - http_version: Option<&str>, -) -> (bool, Option, Option, Option) { - let mut connection_header = None; - let mut keep_alive = None; - let mut upgrade = None; - for (name, value) in headers { - match name.to_ascii_lowercase().as_str() { - "connection" => connection_header = Some(value.clone()), - "keep-alive" => keep_alive = Some(value.clone()), - "upgrade" => upgrade = Some(value.clone()), - _ => {} - } - } - let conn_l = connection_header - .as_deref() - .map(|s| s.to_ascii_lowercase()) - .unwrap_or_default(); - let persistent = if conn_l.split(',').any(|p| p.trim() == "close") { - false - } else if conn_l.split(',').any(|p| p.trim() == "keep-alive") { - true - } else { - !matches!(http_version.unwrap_or("HTTP/1.1"), v if v.starts_with("HTTP/1.0")) - }; - (persistent, connection_header, keep_alive, upgrade) -} - -/// Attach `connection.*` and `client.*` onto the event payload. -pub fn attach_connection_and_client( - payload: &mut Value, - headers: &[(String, String)], - http_version: Option<&str>, - client_peer: Option<&str>, - client_meta: Option<&crate::session::client::SessionClientMeta>, -) { - let (persistent, connection_header, keep_alive, upgrade) = - infer_connection_persistent(headers, http_version); - let mut connection = serde_json::Map::new(); - if let Some(v) = http_version { - connection.insert("http_version".into(), Value::String(v.to_string())); - } - connection.insert("persistent".into(), Value::Bool(persistent)); - if let Some(v) = connection_header { - connection.insert("connection_header".into(), Value::String(v)); - } - if let Some(v) = keep_alive { - connection.insert("keep_alive".into(), Value::String(v)); - } - if let Some(v) = upgrade { - connection.insert("upgrade".into(), Value::String(v)); - } - if let Some(peer) = client_peer { - // The accepted socket's peer address is stable for the lifetime of a - // keep-alive connection and does not expose credentials. - connection.insert("id".into(), Value::String(format!("client:{peer}"))); - } - let Some(obj) = payload.as_object_mut() else { - return; - }; - obj.insert("connection".into(), Value::Object(connection)); - - let mut client = serde_json::Map::new(); - if let Some(peer) = client_peer { - client.insert("peer".into(), Value::String(peer.to_string())); - if let Some((ip, port)) = peer.rsplit_once(':') { - client.insert("peer_ip".into(), Value::String(ip.to_string())); - if let Ok(p) = port.parse::() { - client.insert("peer_port".into(), Value::Number(p.into())); - } - } - } - if let Some(meta) = client_meta { - if client.get("peer").is_none() && !meta.peer.is_empty() { - client.insert("peer".into(), Value::String(meta.peer.clone())); - } - if client.get("peer_port").is_none() { - client.insert("peer_port".into(), Value::Number(meta.peer_port.into())); - } - if meta.pid > 0 { - client.insert("pid".into(), Value::Number(meta.pid.into())); - } - if !meta.command.is_empty() { - client.insert("command".into(), Value::String(meta.command.clone())); - } - if let Some(fp) = &meta.machine_fp { - client.insert("machine_fp".into(), Value::String(fp.clone())); - } - } - for (name, value) in headers { - if name.eq_ignore_ascii_case("user-agent") { - client.insert("user_agent".into(), Value::String(value.clone())); - break; - } - } - if !client.is_empty() { - obj.insert("client".into(), Value::Object(client)); - } -} - -/// Dual-write RFC-0002 `payload.http.*` request wire fields (keeps flat compat keys). -pub fn attach_http_wire_request( - payload: &mut Value, - method: &str, - path: &str, - url: Option<&str>, - body: Option<&Value>, - body_present: bool, -) { - let Some(obj) = payload.as_object_mut() else { - return; - }; - obj.insert("method".into(), Value::String(method.to_string())); - if let Some(u) = url { - obj.insert("url".into(), Value::String(u.to_string())); - } - let http = obj - .entry("http".to_string()) - .or_insert_with(|| Value::Object(serde_json::Map::new())); - if let Value::Object(http_obj) = http { - http_obj.insert("method".into(), Value::String(method.to_string())); - http_obj.insert("path".into(), Value::String(path.to_string())); - if let Some(u) = url { - http_obj.insert("url".into(), Value::String(u.to_string())); - } - if let Some(b) = body { - http_obj.insert("request_body".into(), redact_sensitive_body(b)); - http_obj.insert("body_encoding".into(), Value::String("json".into())); - } - } - if !body_present { - obj.insert("degraded".into(), Value::Bool(true)); - } -} - -/// Dual-write RFC-0002 `payload.http.*` response wire fields. -pub fn attach_http_wire_response( - payload: &mut Value, - status: u16, - url: Option<&str>, - body: Option<&Value>, - body_present: bool, - streaming: bool, - headers_present: bool, -) { - let Some(obj) = payload.as_object_mut() else { - return; - }; - if let Some(u) = url { - obj.insert("url".into(), Value::String(u.to_string())); - } - let http = obj - .entry("http".to_string()) - .or_insert_with(|| Value::Object(serde_json::Map::new())); - if let Value::Object(http_obj) = http { - http_obj.insert("status".into(), Value::Number(status.into())); - if let Some(u) = url { - http_obj.insert("url".into(), Value::String(u.to_string())); - } - http_obj.insert("streaming".into(), Value::Bool(streaming)); - if let Some(b) = body { - http_obj.insert("response_body".into(), redact_sensitive_body(b)); - let enc = if streaming { "sse-wire" } else { "json" }; - http_obj.insert("body_encoding".into(), Value::String(enc.into())); - } - } - if !body_present || !headers_present { - obj.insert("degraded".into(), Value::Bool(true)); - } -} - -/// Redact common credential fields recursively before a JSON body reaches the -/// canonical store. Provider-specific policies can pre-redact additional -/// fields; this is the non-disableable safety floor. -pub fn redact_sensitive_body(value: &Value) -> Value { - match value { - Value::Object(object) => Value::Object( - object - .iter() - .map(|(key, value)| { - let value = if is_sensitive_field_name(key) { - Value::String(REDACTED_VALUE.into()) - } else { - redact_sensitive_body(value) - }; - (key.clone(), value) - }) - .collect(), - ), - Value::Array(values) => Value::Array(values.iter().map(redact_sensitive_body).collect()), - _ => value.clone(), - } -} - -/// Persist HTTP headers onto an event payload (flat `headers` + nested `http.headers`). -/// -/// Sensitive values are replaced with `` and `headers_redacted` is set. -/// Empty `headers` still writes an empty object so callers can tell "recorded empty" -/// from "not recorded" only if they omit this call entirely. -pub fn attach_recorded_headers(payload: &mut Value, headers: &[(String, String)]) { - let mut map = serde_json::Map::new(); - let mut redacted = false; - for ((name, _), (_, value)) in headers.iter().zip(redact_sensitive_headers(headers)) { - let key = name.to_ascii_lowercase(); - if is_sensitive_header_name(name) { - redacted = true; - } - map.insert(key, Value::String(value)); - } - let headers_val = Value::Object(map); - let Some(obj) = payload.as_object_mut() else { - return; - }; - obj.insert("headers".into(), headers_val.clone()); - let http = obj - .entry("http".to_string()) - .or_insert_with(|| Value::Object(serde_json::Map::new())); - if let Value::Object(http_obj) = http { - http_obj.insert("headers".into(), headers_val); - if redacted { - http_obj.insert("headers_redacted".into(), Value::Bool(true)); - } - } - if redacted { - obj.insert("headers_redacted".into(), Value::Bool(true)); - } -} - -fn attach_call_context(rec: &mut EventRecord, call: &Call) { - rec.trace_id = Some(call.trace_id.clone()); - rec.call_id = Some(call.call_id.clone()); -} - -#[allow(clippy::too_many_arguments)] -pub fn llm_request_summary_record( - session_id: Option, - agent_id: Option, - model: &str, - path: &str, - body_bytes: usize, - protocol: &str, - provider: &str, - user_content: Option, - forward_to: Option<&str>, - call: &Call, - level: CaptureLevel, - body_json: Option<&Value>, -) -> EventRecord { - let mut payload = serde_json::json!({ - "model": model, - "path": path, - "body_bytes": body_bytes, - "protocol": protocol, - "provider": provider, - }); - if level.includes_user_text() - && let Some(content) = user_content.filter(|s| !s.is_empty()) - { - payload["user_content"] = serde_json::Value::String(content); - } - if let Some(fwd) = forward_to.filter(|s| !s.is_empty() && *s != model) { - payload["forward_to"] = serde_json::Value::String(fwd.to_string()); - } - if let Some(body) = body_json { - stamp_request_payload(&mut payload, Some(body)); - if level.includes_full_body() { - payload["body"] = body.clone(); - } - } - let mut rec = EventRecord { - identity: persisting_events::EventIdentity::default(), - seq: 0, - source: "persisting-proxy".to_string(), - kind: "llm.request".to_string(), - timestamp: Some(call.started_at.clone()), - session_id, - agent_id, - parent_uuid: None, - trace_id: None, - call_id: None, - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload, - }; - attach_call_context(&mut rec, call); - rec -} - -/// Full request body in payload — tests and fixtures only; production uses [`llm_request_summary_record`]. -#[doc(hidden)] -pub fn llm_request_record( - session_id: Option, - agent_id: Option, - model: &str, - path: &str, - body: &serde_json::Value, -) -> EventRecord { - EventRecord { - identity: persisting_events::EventIdentity::default(), - seq: 0, - source: "persisting-proxy".to_string(), - kind: "llm.request".to_string(), - timestamp: Some(now_rfc3339()), - session_id, - agent_id, - parent_uuid: None, - trace_id: None, - call_id: None, - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload: serde_json::json!({ - "model": model, - "path": path, - "body": body, - }), - } -} - -pub fn llm_response_record( - session_id: Option, - agent_id: Option, - status: u16, - body: &serde_json::Value, - streaming: bool, - call: &Call, -) -> EventRecord { - let mut rec = EventRecord { - identity: persisting_events::EventIdentity::default(), - seq: 0, - source: "persisting-proxy".to_string(), - kind: if streaming { - "llm.response.stream".to_string() - } else { - "llm.response".to_string() - }, - timestamp: Some(now_rfc3339()), - session_id, - agent_id, - parent_uuid: None, - trace_id: None, - call_id: None, - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload: serde_json::json!({ - "status": status, - "body": body, - }), - }; - attach_call_context(&mut rec, call); - rec -} - -#[allow(clippy::too_many_arguments)] -pub fn llm_response_record_with_content( - session_id: Option, - agent_id: Option, - status: u16, - payload: &serde_json::Value, - streaming: bool, - assistant_content: Option, - call: &Call, - level: CaptureLevel, -) -> EventRecord { - let mut payload = payload.clone(); - payload["status"] = serde_json::json!(status); - if level.includes_assistant_text() - && let Some(content) = assistant_content.filter(|s| !s.is_empty()) - { - payload["assistant_content"] = serde_json::Value::String(content); - } - let kind = if streaming { - "llm.response.stream" - } else { - "llm.response" - }; - let mut rec = EventRecord { - identity: persisting_events::EventIdentity::default(), - seq: 0, - source: "persisting-proxy".to_string(), - kind: kind.to_string(), - timestamp: Some(now_rfc3339()), - session_id, - agent_id, - parent_uuid: None, - trace_id: None, - call_id: None, - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload, - }; - attach_call_context(&mut rec, call); - rec -} - -#[cfg(test)] -mod header_tests { - use super::{attach_recorded_headers, redact_sensitive_headers, redact_sensitive_url}; - use serde_json::json; - - #[test] - fn infer_connection_persistent_http11_default() { - let (p, _, _, _) = super::infer_connection_persistent(&[], Some("HTTP/1.1")); - assert!(p); - let (p, _, _, _) = super::infer_connection_persistent( - &[("Connection".into(), "close".into())], - Some("HTTP/1.1"), - ); - assert!(!p); - } - - #[test] - fn attach_connection_and_client_writes_peer() { - let mut payload = json!({}); - super::attach_connection_and_client( - &mut payload, - &[("Connection".into(), "keep-alive".into())], - Some("HTTP/1.1"), - Some("127.0.0.1:9"), - None, - ); - assert_eq!(payload["connection"]["persistent"], true); - assert_eq!(payload["client"]["peer"], "127.0.0.1:9"); - assert_eq!(payload["client"]["peer_port"], 9); - } - - #[test] - fn attach_http_wire_request_sets_nested_fields() { - let mut payload = json!({"path": "/v1/chat/completions"}); - let body = json!({"messages":[{"role":"user","content":"hi"}]}); - super::attach_http_wire_request( - &mut payload, - "POST", - "/v1/chat/completions", - Some("//localhost/v1/chat/completions"), - Some(&body), - true, - ); - assert_eq!(payload["method"], "POST"); - assert_eq!(payload["http"]["method"], "POST"); - assert_eq!(payload["http"]["path"], "/v1/chat/completions"); - assert_eq!(payload["http"]["url"], "//localhost/v1/chat/completions"); - assert_eq!( - payload["http"]["request_body"]["messages"][0]["content"], - "hi" - ); - assert!(payload.get("degraded").is_none()); - } - - #[test] - fn attach_http_wire_marks_degraded_without_body() { - let mut payload = json!({}); - super::attach_http_wire_request(&mut payload, "GET", "/v1/models", None, None, false); - assert_eq!(payload["degraded"], true); - } - - #[test] - fn nested_body_credentials_are_redacted() { - let body = serde_json::json!({ - "request": { - "api_key": "sk-live", - "clientSecret": "client-live", - "session-token": "session-live", - "safe": "x" - } - }); - let redacted = super::redact_sensitive_body(&body); - assert_eq!(redacted["request"]["api_key"], ""); - assert_eq!(redacted["request"]["clientSecret"], ""); - assert_eq!(redacted["request"]["session-token"], ""); - assert_eq!(redacted["request"]["safe"], "x"); - } - - #[test] - fn persistence_header_copy_redacts_common_and_vendor_credentials() { - let headers = vec![ - ("Authorization".into(), "Bearer live".into()), - ("x-goog-api-key".into(), "google-live".into()), - ("x-vendor-access-token".into(), "vendor-live".into()), - ("x-request-id".into(), "req-1".into()), - ]; - let redacted = redact_sensitive_headers(&headers); - assert_eq!(redacted[0].1, ""); - assert_eq!(redacted[1].1, ""); - assert_eq!(redacted[2].1, ""); - assert_eq!(redacted[3].1, "req-1"); - assert_eq!( - headers[0].1, "Bearer live", - "wire headers must be untouched" - ); - } - - #[test] - fn persisted_urls_redact_userinfo_and_sensitive_query_fields() { - let redacted = redact_sensitive_url( - "https://live-user:live-password@example.com/v1/models?key=live-key&alt=sse", - ); - assert!(!redacted.contains("live-user")); - assert!(!redacted.contains("live-password")); - assert!(!redacted.contains("live-key")); - assert!(redacted.contains("alt=sse")); - - assert_eq!( - redact_sensitive_url("//example.com/v1/models?api_key=live-key&safe=kept"), - "//example.com/v1/models?api_key=%3Credacted%3E&safe=kept" - ); - assert_eq!( - redact_sensitive_url("/v1/models?safe=kept"), - "/v1/models?safe=kept" - ); - } - - #[test] - fn attach_recorded_headers_redacts_authorization() { - let mut payload = json!({"path": "/v1/chat/completions"}); - attach_recorded_headers( - &mut payload, - &[ - ("Content-Type".into(), "application/json".into()), - ("Authorization".into(), "Bearer secret".into()), - ], - ); - assert_eq!(payload["headers"]["content-type"], "application/json"); - assert_eq!(payload["headers"]["authorization"], ""); - assert_eq!(payload["headers_redacted"], true); - assert_eq!(payload["http"]["headers"]["authorization"], ""); - } -} diff --git a/crates/persisting-gateway/src/subagent_link/enrich.rs b/crates/persisting-gateway/src/subagent_link/enrich.rs deleted file mode 100644 index b9792ff76..000000000 --- a/crates/persisting-gateway/src/subagent_link/enrich.rs +++ /dev/null @@ -1,143 +0,0 @@ -//! Enrich capture records with subagent cross-refs. - -use axum::http::HeaderMap; -use serde_json::{Value, json}; - -use super::extract::{ - extract_spawn_hints_from_assistant, extract_subagent_ids_from_request, - extract_subagent_ids_from_text, -}; -use super::match_::{ - apply_spawn_links, merge_string_array_payload, spawn_hint_to_json, spawn_link_to_json, -}; -use super::types::{EnrichOutcome, SpawnLink, SubagentRegistry}; -use crate::record::EventRecord; -use crate::session::chain::extract_claude_parent_agent_id; -use crate::session::storage::CaptureRoute; - -pub fn spawn_link_backfill_record( - parent_call_id: &str, - links: &[SpawnLink], - call: &crate::Call, -) -> EventRecord { - let mut payload = serde_json::json!({ - "parent_call_id": parent_call_id, - "spawn_links": links.iter().map(spawn_link_to_json).collect::>(), - }); - let ids: Vec<_> = links.iter().map(|l| l.subagent_id.clone()).collect(); - let paths: Vec<_> = links - .iter() - .map(|l| l.subagent_trajectory.clone()) - .collect(); - payload["refs_subagent_ids"] = json!(ids); - payload["subagent_trajectories"] = json!(paths); - - EventRecord { - identity: persisting_events::EventIdentity::default(), - seq: 0, - source: "persisting-proxy".to_string(), - kind: "llm.spawn_link".to_string(), - timestamp: None, - session_id: None, - agent_id: None, - parent_uuid: None, - trace_id: Some(call.trace_id.clone()), - call_id: Some(parent_call_id.to_string()), - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: Some(parent_call_id.to_string()), - payload, - } -} -pub fn main_route_for_backfill(route: &CaptureRoute, registry: &SubagentRegistry) -> CaptureRoute { - let run_key = run_registry_key(route); - let storage_session_id = registry - .runs - .get(&run_key) - .and_then(|r| r.main_storage_session_id.clone()) - .or_else(|| route.root_session.clone()) - .unwrap_or_else(|| route.storage_session_id.clone()); - CaptureRoute { - root_session: route.root_session.clone(), - session_id: route.session_id.clone(), - storage_session_id, - subagent_id: None, - } -} - -pub fn run_registry_key(route: &CaptureRoute) -> String { - route - .root_session - .clone() - .unwrap_or_else(|| route.storage_session_id.clone()) -} -/// Attach cross-reference fields to a capture record (metadata + payload). -pub fn enrich_record( - rec: &mut EventRecord, - route: &CaptureRoute, - headers: &HeaderMap, - request_body: Option<&Value>, - assistant_text: Option<&str>, - registry: &mut SubagentRegistry, -) -> EnrichOutcome { - let mut outcome = EnrichOutcome::default(); - let run_key = run_registry_key(route); - registry.register_route( - &run_key, - route, - rec.call_id.as_deref().unwrap_or(""), - request_body, - ); - - if let Some(id) = route.subagent_id.clone() { - rec.subagent_id = Some(id.clone()); - let trajectory = super::types::subagent_trajectory_path(&route.storage_session_id); - rec.payload["subagent_trajectory"] = json!(trajectory); - if let Some(parent) = extract_claude_parent_agent_id(headers) { - rec.parent_agent_id = Some(parent.clone()); - rec.payload["parent_agent_id"] = json!(parent); - } - outcome.spawn_link_backfills = registry.on_subagent_registered(&run_key, &id); - return outcome; - } - - if let Some(parent) = extract_claude_parent_agent_id(headers) { - rec.parent_agent_id = Some(parent.clone()); - rec.payload["parent_agent_id"] = json!(parent); - } - - if let Some(body) = request_body { - let refs = extract_subagent_ids_from_request(body); - if !refs.is_empty() { - let trajectories = registry.trajectory_paths(&run_key, &refs); - rec.payload["refs_subagent_ids"] = json!(refs); - if !trajectories.is_empty() { - rec.payload["subagent_trajectories"] = json!(trajectories); - } - } - } - - if let Some(text) = assistant_text.filter(|s| !s.is_empty()) { - let hints = extract_spawn_hints_from_assistant(text); - if !hints.is_empty() { - rec.payload["spawn_hints"] = - json!(hints.iter().map(spawn_hint_to_json).collect::>()); - let links = registry.remember_main_spawn( - &run_key, - rec.call_id.as_deref().unwrap_or(""), - &hints, - ); - apply_spawn_links(rec, &links); - } - let spawned = extract_subagent_ids_from_text(text); - if !spawned.is_empty() { - let trajectories = registry.trajectory_paths(&run_key, &spawned); - merge_string_array_payload(rec, "refs_subagent_ids", &spawned); - if !trajectories.is_empty() { - merge_string_array_payload(rec, "subagent_trajectories", &trajectories); - } - } - } - outcome -} diff --git a/crates/persisting-gateway/src/subagent_link/extract.rs b/crates/persisting-gateway/src/subagent_link/extract.rs deleted file mode 100644 index 4f3a5728c..000000000 --- a/crates/persisting-gateway/src/subagent_link/extract.rs +++ /dev/null @@ -1,163 +0,0 @@ -//! Extract subagent ids and spawn hints from Claude traffic. - -use std::collections::HashSet; -use std::sync::LazyLock; - -use regex::Regex; -use serde_json::Value; - -use super::types::SpawnHint; - -static AGENT_ID_JSON: LazyLock = LazyLock::new(|| { - Regex::new(r#"(?i)"agentId"\s*:\s*"([a-zA-Z0-9_-]{4,})""#).expect("agentId json") -}); -/// Claude Code tool_result line: `agentId: a583b6c00c3d06436` (not CLI `--agent-id`). -static AGENT_ID_TOOL_RESULT: LazyLock = LazyLock::new(|| { - Regex::new(r"(?m)(?:^|\n)\s*agentId:\s*([a-zA-Z0-9_-]{4,})").expect("agentId tool result") -}); -static TOOL_AGENT_BLOCK: LazyLock = LazyLock::new(|| { - Regex::new(r"```tool:(?:Agent|Task)\s*\n([\s\S]*?)```").expect("tool agent block") -}); -/// Design-doc paths in prompts (`*.md` or legacy `*.zh.md` under docs/src/design/). -static DOC_TARGET: LazyLock = - LazyLock::new(|| Regex::new(r"docs/src/design/[^\s\)\]]+\.md").expect("doc target path")); -static TASK_ID_XML: LazyLock = LazyLock::new(|| { - Regex::new(r"(?i)\s*([a-zA-Z0-9_-]{4,})\s*").expect("task-id xml") -}); - -pub(crate) fn extract_doc_target(text: &str) -> Option { - DOC_TARGET.find(text).map(|m| m.as_str().to_string()) -} - -pub fn extract_subagent_ids_from_request(body: &Value) -> Vec { - let mut ids = HashSet::new(); - let Some(messages) = body.get("messages").and_then(|m| m.as_array()) else { - return Vec::new(); - }; - for msg in messages { - let Some(content) = msg.get("content") else { - continue; - }; - match content { - Value::Array(parts) => { - for part in parts { - if part.get("type").and_then(|t| t.as_str()) == Some("tool_result") { - if let Some(inner) = part.get("content") { - collect_subagent_ids_from_value(inner, &mut ids); - } - } else if let Some(text) = part.get("text").and_then(|t| t.as_str()) { - for id in extract_subagent_ids_from_text(text) { - ids.insert(id); - } - } else { - collect_subagent_ids_from_value(part, &mut ids); - } - } - } - other => collect_subagent_ids_from_value(other, &mut ids), - } - } - let mut out: Vec<_> = ids.into_iter().collect(); - out.sort(); - out -} - -fn collect_subagent_ids_from_value(v: &Value, out: &mut HashSet) { - match v { - Value::String(s) => { - for id in extract_subagent_ids_from_text(s) { - out.insert(id); - } - } - Value::Array(parts) => { - for p in parts { - if let Some(t) = p.get("text").and_then(|x| x.as_str()) { - for id in extract_subagent_ids_from_text(t) { - out.insert(id); - } - } - } - } - Value::Object(map) => { - if let Some(id) = map - .get("agentId") - .or_else(|| map.get("agent_id")) - .and_then(|x| x.as_str()) - .filter(|s| plausible_agent_id(s)) - { - out.insert(id.to_string()); - } - for v in map.values() { - collect_subagent_ids_from_value(v, out); - } - } - _ => {} - } -} - -pub fn extract_subagent_ids_from_text(text: &str) -> Vec { - let mut ids = HashSet::new(); - for cap in AGENT_ID_JSON.captures_iter(text) { - if let Some(id) = cap.get(1).map(|m| m.as_str()) - && plausible_agent_id(id) - { - ids.insert(id.to_string()); - } - } - for cap in AGENT_ID_TOOL_RESULT.captures_iter(text) { - if let Some(id) = cap.get(1).map(|m| m.as_str()) - && plausible_agent_id(id) - { - ids.insert(id.to_string()); - } - } - for cap in TASK_ID_XML.captures_iter(text) { - if let Some(id) = cap.get(1).map(|m| m.as_str()) - && plausible_agent_id(id) - { - ids.insert(id.to_string()); - } - } - let mut out: Vec<_> = ids.into_iter().collect(); - out.sort(); - out -} - -pub fn extract_spawn_hints_from_assistant(text: &str) -> Vec { - let mut hints = Vec::new(); - for cap in TOOL_AGENT_BLOCK.captures_iter(text) { - let Some(json_str) = cap.get(1).map(|m| m.as_str()) else { - continue; - }; - let Ok(v) = serde_json::from_str::(json_str.trim()) else { - continue; - }; - let subagent_type = v - .get("subagent_type") - .or_else(|| v.get("agent_type")) - .or_else(|| v.get("description")) - .and_then(|x| x.as_str()) - .unwrap_or("Agent"); - let description = v - .get("description") - .and_then(|x| x.as_str()) - .map(str::to_string); - let prompt = v.get("prompt").and_then(|x| x.as_str()).map(str::to_string); - let doc_target = prompt.as_deref().and_then(extract_doc_target); - hints.push(SpawnHint { - subagent_type: subagent_type.to_string(), - description, - prompt, - doc_target, - }); - } - hints -} - -fn plausible_agent_id(id: &str) -> bool { - id.len() >= 4 - && id.len() <= 64 - && id - .chars() - .all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_') -} diff --git a/crates/persisting-gateway/src/subagent_link/match_.rs b/crates/persisting-gateway/src/subagent_link/match_.rs deleted file mode 100644 index a0008eb6a..000000000 --- a/crates/persisting-gateway/src/subagent_link/match_.rs +++ /dev/null @@ -1,139 +0,0 @@ -//! Hint/link matching and payload merge helpers. - -use std::collections::{HashMap, HashSet}; - -use bytes::Bytes; -use serde_json::{Value, json}; - -use crate::dialogue_extract::extract_user_message_from_request_body; -use crate::record::EventRecord; - -use super::types::{AgentEntry, SpawnHint, SpawnLink}; - -pub(crate) fn first_user_message_from_json(body: &Value) -> Option { - let bytes = Bytes::from(serde_json::to_vec(body).ok()?); - extract_user_message_from_request_body(&bytes) -} - -pub(crate) fn find_agent_for_spawn<'a>( - agents: &'a HashMap, - hint: &SpawnHint, - used: &HashSet, -) -> Option<&'a AgentEntry> { - if let Some(doc) = hint.doc_target.as_deref() - && let Some(entry) = agents - .values() - .find(|e| !used.contains(&e.agent_id) && e.doc_target.as_deref() == Some(doc)) - { - return Some(entry); - } - if let Some(prompt) = hint.prompt.as_deref() - && let Some(entry) = agents.values().find(|e| { - !used.contains(&e.agent_id) - && e.first_prompt - .as_deref() - .is_some_and(|p| prompts_match(p, prompt)) - }) - { - return Some(entry); - } - None -} - -fn prompts_match(a: &str, b: &str) -> bool { - let na = normalize_prompt(a); - let nb = normalize_prompt(b); - if na.len() < 40 || nb.len() < 40 { - return false; - } - let prefix_len = 80.min(na.len()).min(nb.len()); - na[..prefix_len] == nb[..prefix_len] -} - -fn normalize_prompt(s: &str) -> String { - s.chars().filter(|c| !c.is_whitespace()).collect() -} - -pub(crate) fn spawn_hint_to_json(h: &SpawnHint) -> Value { - let mut obj = json!({ "subagent_type": h.subagent_type }); - if let Some(d) = &h.description { - obj["description"] = json!(d); - } - if let Some(p) = &h.prompt { - obj["prompt"] = json!(p); - } - if let Some(d) = &h.doc_target { - obj["doc_target"] = json!(d); - } - obj -} - -pub(crate) fn spawn_link_to_json(l: &SpawnLink) -> Value { - let mut obj = json!({ - "subagent_type": l.subagent_type, - "subagent_id": l.subagent_id, - "subagent_trajectory": l.subagent_trajectory, - }); - if let Some(d) = &l.description { - obj["description"] = json!(d); - } - if let Some(d) = &l.doc_target { - obj["doc_target"] = json!(d); - } - obj -} - -pub(crate) fn spawn_hint_matches_link(hint: &SpawnHint, link: &SpawnLink) -> bool { - if hint.doc_target.is_some() && hint.doc_target == link.doc_target { - return true; - } - hint.description.is_some() && hint.description == link.description -} - -fn same_spawn_hint(a: &SpawnHint, b: &SpawnHint) -> bool { - if a.doc_target.is_some() && a.doc_target == b.doc_target { - return true; - } - a.description.is_some() && a.description == b.description -} - -pub(crate) fn merge_spawn_hints(into: &mut Vec, new_hints: &[SpawnHint]) { - for hint in new_hints { - if !into.iter().any(|h| same_spawn_hint(h, hint)) { - into.push(hint.clone()); - } - } -} - -pub(crate) fn apply_spawn_links(rec: &mut EventRecord, links: &[SpawnLink]) { - if links.is_empty() { - return; - } - rec.payload["spawn_links"] = json!(links.iter().map(spawn_link_to_json).collect::>()); - let ids: Vec<_> = links.iter().map(|l| l.subagent_id.clone()).collect(); - let paths: Vec<_> = links - .iter() - .map(|l| l.subagent_trajectory.clone()) - .collect(); - merge_string_array_payload(rec, "refs_subagent_ids", &ids); - merge_string_array_payload(rec, "subagent_trajectories", &paths); -} - -pub(crate) fn merge_string_array_payload(rec: &mut EventRecord, key: &str, values: &[String]) { - let mut set = HashSet::new(); - if let Some(existing) = rec.payload.get(key).and_then(|v| v.as_array()) { - for v in existing { - if let Some(s) = v.as_str() { - set.insert(s.to_string()); - } - } - } - for v in values { - set.insert(v.clone()); - } - if !set.is_empty() { - let mut merged: Vec<_> = set.into_iter().collect(); - merged.sort(); - rec.payload[key] = json!(merged); - } -} diff --git a/crates/persisting-gateway/src/subagent_link/mod.rs b/crates/persisting-gateway/src/subagent_link/mod.rs deleted file mode 100644 index 74fb6fae3..000000000 --- a/crates/persisting-gateway/src/subagent_link/mod.rs +++ /dev/null @@ -1,23 +0,0 @@ -//! Main agent ↔ subagent cross-reference for trajectory capture. - -mod enrich; -mod extract; -mod match_; -mod registry; -mod types; - -#[cfg(test)] -#[path = "tests.rs"] -mod tests; - -pub use enrich::{ - enrich_record, main_route_for_backfill, run_registry_key, spawn_link_backfill_record, -}; -pub use extract::{ - extract_spawn_hints_from_assistant, extract_subagent_ids_from_request, - extract_subagent_ids_from_text, -}; -pub use types::subagent_trajectory_path; -pub use types::{ - AgentEntry, EnrichOutcome, SpawnHint, SpawnLink, SpawnLinkBackfill, SubagentRegistry, -}; diff --git a/crates/persisting-gateway/src/subagent_link/registry.rs b/crates/persisting-gateway/src/subagent_link/registry.rs deleted file mode 100644 index 53b6d674e..000000000 --- a/crates/persisting-gateway/src/subagent_link/registry.rs +++ /dev/null @@ -1,191 +0,0 @@ -//! Per-run subagent registry and pending spawn matching. - -use std::collections::{HashMap, HashSet}; - -use serde_json::Value; - -use super::extract::extract_doc_target; -use super::match_::{ - find_agent_for_spawn, first_user_message_from_json, merge_spawn_hints, spawn_hint_matches_link, -}; -use super::types::{ - AgentEntry, PendingMainSpawn, RunSubagents, SpawnHint, SpawnLink, SpawnLinkBackfill, - SubagentRegistry, subagent_trajectory_path, -}; -use crate::session::storage::CaptureRoute; - -impl SubagentRegistry { - pub fn tracked_run_count(&self) -> usize { - self.runs.len() - } - - fn run_mut(&mut self, run_key: &str) -> &mut RunSubagents { - self.runs.entry(run_key.to_string()).or_default() - } - - pub fn register_route( - &mut self, - run_key: &str, - route: &CaptureRoute, - call_id: &str, - request_body: Option<&Value>, - ) { - let run = self.run_mut(run_key); - if route.subagent_id.is_none() { - run.main_storage_session_id = Some(route.storage_session_id.clone()); - } - let Some(id) = route.subagent_id.as_deref() else { - return; - }; - let prompt = request_body.and_then(first_user_message_from_json); - let doc_target = prompt.as_deref().and_then(extract_doc_target); - run.agents - .entry(id.to_string()) - .and_modify(|e| { - if e.first_prompt.is_none() { - e.first_prompt = prompt.clone(); - } - if e.doc_target.is_none() { - e.doc_target = doc_target.clone(); - } - }) - .or_insert_with(|| AgentEntry { - agent_id: id.to_string(), - storage_leaf: route.storage_session_id.clone(), - first_call_id: Some(call_id.to_string()), - first_prompt: prompt, - doc_target, - }); - } - - pub fn trajectory_paths(&self, run_key: &str, ids: &[String]) -> Vec { - let Some(run) = self.runs.get(run_key) else { - return Vec::new(); - }; - ids.iter() - .filter_map(|id| { - run.agents - .get(id) - .map(|e| subagent_trajectory_path(&e.storage_leaf)) - }) - .collect() - } - - /// Match ```tool:Agent``` spawn hints to subagents already registered in this run. - pub fn match_spawns_to_agents(&self, run_key: &str, hints: &[SpawnHint]) -> Vec { - let Some(run) = self.runs.get(run_key) else { - return Vec::new(); - }; - let mut used = HashSet::new(); - let mut links = Vec::new(); - for hint in hints { - let Some(agent) = find_agent_for_spawn(&run.agents, hint, &used) else { - continue; - }; - used.insert(agent.agent_id.clone()); - links.push(SpawnLink { - subagent_type: hint.subagent_type.clone(), - description: hint.description.clone(), - doc_target: hint.doc_target.clone(), - subagent_id: agent.agent_id.clone(), - subagent_trajectory: subagent_trajectory_path(&agent.storage_leaf), - }); - } - links - } - - pub(crate) fn remember_main_spawn( - &mut self, - run_key: &str, - parent_call_id: &str, - hints: &[SpawnHint], - ) -> Vec { - if hints.is_empty() { - return Vec::new(); - } - let merged = { - let run = self.run_mut(run_key); - if let Some(existing) = run - .pending_mains - .iter_mut() - .find(|p| p.parent_call_id == parent_call_id) - { - merge_spawn_hints(&mut existing.hints, hints); - existing.hints.clone() - } else { - run.pending_mains.push(PendingMainSpawn { - parent_call_id: parent_call_id.to_string(), - hints: hints.to_vec(), - links: Vec::new(), - matched_agents: HashSet::new(), - }); - hints.to_vec() - } - }; - let links = self.match_spawns_to_agents(run_key, &merged); - let run = self.run_mut(run_key); - if let Some(existing) = run - .pending_mains - .iter_mut() - .find(|p| p.parent_call_id == parent_call_id) - { - existing.hints = merged; - existing.links = links.clone(); - existing.matched_agents = links.iter().map(|l| l.subagent_id.clone()).collect(); - } - links - } - - pub(crate) fn on_subagent_registered( - &mut self, - run_key: &str, - agent_id: &str, - ) -> Vec { - let Some(agent) = self - .runs - .get(run_key) - .and_then(|r| r.agents.get(agent_id).cloned()) - else { - return Vec::new(); - }; - let run = self.run_mut(run_key); - let mut backfills = Vec::new(); - for pending in &mut run.pending_mains { - if pending.matched_agents.contains(agent_id) { - continue; - } - let mut used = pending.matched_agents.clone(); - let mut new_links = Vec::new(); - for hint in &pending.hints { - if pending - .links - .iter() - .any(|l| spawn_hint_matches_link(hint, l)) - { - continue; - } - let single = HashMap::from([(agent.agent_id.clone(), agent.clone())]); - if find_agent_for_spawn(&single, hint, &used).is_some() { - let link = SpawnLink { - subagent_type: hint.subagent_type.clone(), - description: hint.description.clone(), - doc_target: hint.doc_target.clone(), - subagent_id: agent.agent_id.clone(), - subagent_trajectory: subagent_trajectory_path(&agent.storage_leaf), - }; - used.insert(agent.agent_id.clone()); - pending.links.push(link.clone()); - new_links.push(link); - } - } - if !new_links.is_empty() { - pending.matched_agents = used; - backfills.push(SpawnLinkBackfill { - parent_call_id: pending.parent_call_id.clone(), - links: new_links, - }); - } - } - backfills - } -} diff --git a/crates/persisting-gateway/src/subagent_link/tests.rs b/crates/persisting-gateway/src/subagent_link/tests.rs deleted file mode 100644 index 1608b3332..000000000 --- a/crates/persisting-gateway/src/subagent_link/tests.rs +++ /dev/null @@ -1,363 +0,0 @@ -use axum::http::{HeaderMap, HeaderValue}; -use serde_json::json; - -use crate::record::EventRecord; -use crate::session::storage::CaptureRoute; - -use super::*; - -fn headers(pairs: &[(&str, &str)]) -> HeaderMap { - use axum::http::header::HeaderName; - let mut h = HeaderMap::new(); - for (k, v) in pairs { - h.insert( - HeaderName::from_bytes(k.as_bytes()).unwrap(), - HeaderValue::try_from(*v).unwrap(), - ); - } - h -} - -#[test] -fn ignores_cli_agent_id_flag_in_document_text() { - let body = json!({ - "messages": [{ - "role": "user", - "content": [{ - "type": "tool_result", - "content": "--agent-id my-repo --session-id abc" - }] - }] - }); - assert!(extract_subagent_ids_from_request(&body).is_empty()); -} - -#[test] -fn extracts_agent_id_from_task_notification() { - let text = "\naeb40a1230926b4a8\n"; - assert_eq!( - extract_subagent_ids_from_text(text), - vec!["aeb40a1230926b4a8".to_string()] - ); - let body = json!({ - "messages": [{ - "role": "user", - "content": [{"type": "text", "text": text}] - }] - }); - assert_eq!( - extract_subagent_ids_from_request(&body), - vec!["aeb40a1230926b4a8".to_string()] - ); -} - -#[test] -fn extracts_agent_id_from_tool_result_line() { - let text = "Async agent launched.\nagentId: a483eb5f3a45995b0 done\n"; - assert_eq!( - extract_subagent_ids_from_text(text), - vec!["a483eb5f3a45995b0".to_string()] - ); -} - -#[test] -fn extracts_agent_id_from_tool_result_json() { - let body = json!({ - "messages": [{ - "role": "user", - "content": [{ - "type": "tool_result", - "tool_use_id": "toolu_1", - "content": [{"type":"text","text":"{\"agentId\":\"a5aa661\",\"status\":\"complete\"}"}] - }] - }] - }); - assert_eq!( - extract_subagent_ids_from_request(&body), - vec!["a5aa661".to_string()] - ); -} - -#[test] -fn extracts_spawn_hints_from_agent_tool_block() { - let text = r#"Delegated. - -```tool:Agent -{ - "subagent_type": "Explore", - "prompt": "scan src/" -} -```"#; - let hints = extract_spawn_hints_from_assistant(text); - assert_eq!(hints.len(), 1); - assert_eq!(hints[0].subagent_type, "Explore"); - assert_eq!(hints[0].prompt.as_deref(), Some("scan src/")); -} - -#[test] -fn match_spawns_by_doc_target_when_subagents_registered_first() { - let mut registry = SubagentRegistry::default(); - let run_key = "run-20260524-021032"; - let doc = "docs/src/design/cli-pvisor.md"; - let sub_body = json!({ - "messages": [{ - "role": "user", - "content": format!("Review the Chinese design document at /Users/reiase/workspace/Persisting/{doc}") - }] - }); - let sub_route = CaptureRoute { - root_session: Some(run_key.into()), - session_id: "sess".into(), - storage_session_id: "agent-a583b6c00c3d06436".into(), - subagent_id: Some("a583b6c00c3d06436".into()), - }; - registry.register_route(run_key, &sub_route, "call-sub", Some(&sub_body)); - - let assistant = format!( - r#"Launching review. - -```tool:Agent -{{ - "description": "Review cli-capture design doc", - "prompt": "Review the Chinese design document at /Users/reiase/workspace/Persisting/{doc}", - "subagent_type": "general-purpose" -}} -```"# - ); - let hints = extract_spawn_hints_from_assistant(&assistant); - assert_eq!(hints.len(), 1); - assert_eq!(hints[0].doc_target.as_deref(), Some(doc)); - - let links = registry.match_spawns_to_agents(run_key, &hints); - assert_eq!(links.len(), 1); - assert_eq!(links[0].subagent_id, "a583b6c00c3d06436"); - assert_eq!(links[0].subagent_trajectory, "agent-a583b6c00c3d06436.md"); - - let main_route = CaptureRoute { - root_session: Some(run_key.into()), - session_id: "sess".into(), - storage_session_id: run_key.into(), - subagent_id: None, - }; - let mut rec = EventRecord { - identity: Default::default(), - seq: 3, - source: "persisting-proxy".into(), - kind: "llm.response.stream".into(), - timestamp: None, - session_id: None, - agent_id: None, - parent_uuid: None, - trace_id: None, - call_id: Some("call-main".into()), - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload: json!({}), - }; - let outcome = enrich_record( - &mut rec, - &main_route, - &HeaderMap::new(), - None, - Some(&assistant), - &mut registry, - ); - assert!(outcome.spawn_link_backfills.is_empty()); - assert_eq!( - rec.payload["spawn_links"][0]["subagent_id"], - json!("a583b6c00c3d06436") - ); - assert_eq!( - rec.payload["refs_subagent_ids"], - json!(["a583b6c00c3d06436"]) - ); -} - -#[test] -fn backfill_spawn_link_when_subagent_registers_after_assistant() { - let mut registry = SubagentRegistry::default(); - let run_key = "run-backfill"; - let doc = "docs/src/design/capture.md"; - let assistant = format!( - r#"Starting. - -```tool:Agent -{{ - "description": "Review capture design", - "prompt": "Review {doc}", - "subagent_type": "general-purpose" -}} -```"# - ); - let main_route = CaptureRoute { - root_session: Some(run_key.into()), - session_id: "sess".into(), - storage_session_id: run_key.into(), - subagent_id: None, - }; - let mut main_rec = EventRecord { - identity: Default::default(), - seq: 2, - source: "t".into(), - kind: "llm.response.stream".into(), - timestamp: None, - session_id: None, - agent_id: None, - parent_uuid: None, - trace_id: None, - call_id: Some("call-main".into()), - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload: json!({}), - }; - enrich_record( - &mut main_rec, - &main_route, - &HeaderMap::new(), - None, - Some(&assistant), - &mut registry, - ); - assert!(main_rec.payload.get("spawn_links").is_none()); - - let sub_body = json!({"messages":[{"role":"user","content": format!("Review {doc}")}]}); - let sub_route = CaptureRoute { - root_session: Some(run_key.into()), - session_id: "sess".into(), - storage_session_id: "agent-deadbeef".into(), - subagent_id: Some("deadbeef".into()), - }; - let mut sub_rec = EventRecord { - identity: Default::default(), - seq: 0, - source: "t".into(), - kind: "llm.request".into(), - timestamp: None, - session_id: None, - agent_id: None, - parent_uuid: None, - trace_id: None, - call_id: Some("call-sub".into()), - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload: json!({}), - }; - let outcome = enrich_record( - &mut sub_rec, - &sub_route, - &HeaderMap::new(), - Some(&sub_body), - None, - &mut registry, - ); - assert_eq!(outcome.spawn_link_backfills.len(), 1); - assert_eq!(outcome.spawn_link_backfills[0].parent_call_id, "call-main"); - assert_eq!( - outcome.spawn_link_backfills[0].links[0].subagent_id, - "deadbeef" - ); -} - -#[test] -fn enrich_links_main_request_to_subagent_trajectory() { - let mut registry = SubagentRegistry::default(); - let run_key = "run-1"; - let sub_route = CaptureRoute { - root_session: Some(run_key.into()), - session_id: "sess".into(), - storage_session_id: "agent-abc1234".into(), - subagent_id: Some("abc1234".into()), - }; - registry.register_route(run_key, &sub_route, "call-sub", None); - - let body = json!({ - "messages": [{ - "role": "user", - "content": [{ - "type": "tool_result", - "content": "agentId: abc1234 done" - }] - }] - }); - let main_route = CaptureRoute { - root_session: Some(run_key.into()), - session_id: "sess".into(), - storage_session_id: run_key.into(), - subagent_id: None, - }; - let mut rec = EventRecord { - identity: Default::default(), - seq: 0, - source: "t".into(), - kind: "llm.request".into(), - timestamp: None, - session_id: None, - agent_id: None, - parent_uuid: None, - trace_id: None, - call_id: Some("call-main".into()), - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload: json!({}), - }; - enrich_record( - &mut rec, - &main_route, - &HeaderMap::new(), - Some(&body), - None, - &mut registry, - ); - assert_eq!(rec.payload["refs_subagent_ids"], json!(["abc1234"])); - assert_eq!( - rec.payload["subagent_trajectories"], - json!(["agent-abc1234.md"]) - ); -} - -#[test] -fn enrich_subagent_record_has_self_trajectory() { - let mut registry = SubagentRegistry::default(); - let route = CaptureRoute { - root_session: Some("run-1".into()), - session_id: "sess".into(), - storage_session_id: "agent-xyz".into(), - subagent_id: Some("xyz".into()), - }; - let mut rec = EventRecord { - identity: Default::default(), - seq: 0, - source: "t".into(), - kind: "llm.request".into(), - timestamp: None, - session_id: None, - agent_id: None, - parent_uuid: None, - trace_id: None, - call_id: Some("call-1".into()), - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload: json!({}), - }; - enrich_record( - &mut rec, - &route, - &headers(&[("x-claude-code-parent-agent-id", "main01")]), - None, - None, - &mut registry, - ); - assert_eq!(rec.subagent_id.as_deref(), Some("xyz")); - assert_eq!(rec.parent_agent_id.as_deref(), Some("main01")); - assert_eq!(rec.payload["subagent_trajectory"], json!("agent-xyz.md")); -} diff --git a/crates/persisting-gateway/src/subagent_link/types.rs b/crates/persisting-gateway/src/subagent_link/types.rs deleted file mode 100644 index ffc59336c..000000000 --- a/crates/persisting-gateway/src/subagent_link/types.rs +++ /dev/null @@ -1,66 +0,0 @@ -//! Spawn-link value types. - -use std::collections::{HashMap, HashSet}; - -#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] -pub struct SpawnHint { - pub subagent_type: String, - pub description: Option, - pub prompt: Option, - pub doc_target: Option, -} - -#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] -pub struct SpawnLink { - pub subagent_type: String, - pub description: Option, - pub doc_target: Option, - pub subagent_id: String, - pub subagent_trajectory: String, -} - -#[derive(Debug, Clone, Default)] -pub struct AgentEntry { - pub agent_id: String, - pub storage_leaf: String, - pub first_call_id: Option, - pub first_prompt: Option, - pub doc_target: Option, -} - -#[derive(Debug, Clone, Default)] -pub(crate) struct PendingMainSpawn { - pub(crate) parent_call_id: String, - pub(crate) hints: Vec, - pub(crate) links: Vec, - pub(crate) matched_agents: HashSet, -} - -#[derive(Debug, Clone, Default)] -pub(crate) struct RunSubagents { - pub(crate) agents: HashMap, - pub(crate) pending_mains: Vec, - pub(crate) main_storage_session_id: Option, -} - -/// Tracks subagent instances seen during one `capture run` root session. -#[derive(Debug, Default)] -pub struct SubagentRegistry { - pub(crate) runs: HashMap, -} - -#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] -pub struct SpawnLinkBackfill { - pub parent_call_id: String, - pub links: Vec, -} - -/// Side effects from [`enrich_record`](super::enrich::enrich_record). -#[derive(Debug, Clone, Default)] -pub struct EnrichOutcome { - pub spawn_link_backfills: Vec, -} - -pub fn subagent_trajectory_path(storage_leaf: &str) -> String { - persisting_pchronicle::storage::session_markdown_filename(storage_leaf) -} diff --git a/crates/persisting-gateway/src/understanding.rs b/crates/persisting-gateway/src/understanding.rs deleted file mode 100644 index 79ad1746f..000000000 --- a/crates/persisting-gateway/src/understanding.rs +++ /dev/null @@ -1,1211 +0,0 @@ -//! Parse provider wire requests into Chronicle's typed LLM event payload. -//! -//! The original JSON remains available for exact wire capture. This module -//! extracts the provider-neutral semantics once, before model rewrite or -//! protocol conversion, so capture and future renderers share one contract. - -use std::collections::BTreeMap; -use std::sync::Arc; - -use anyhow::{Context, Result}; -use bytes::Bytes; -use persisting_pchronicle::model::{ - LlmCandidate, LlmContentPart, LlmGenerationParams, LlmImageSource, LlmMessage, LlmProtocol, - LlmRequest, LlmRequestEventPayload, LlmResponse, LlmResponseEventPayload, LlmResponseFormat, - LlmRole, LlmToolChoice, LlmToolChoiceMode, LlmToolDefinition, LlmUsage, -}; -use serde_json::{Map, Value}; - -use crate::dialogue_extract::extract_user_message_from_request_body; -use crate::protocol::ProtocolKind; - -#[derive(Debug, Clone)] -pub struct ParsedRequest { - pub body_json: Value, - pub semantic: Arc, - /// Capture-oriented visible text applies product filters (for example - /// Claude Code system reminders) that do not belong in the semantic IR. - pub latest_visible_user_content: Option, -} - -pub fn understand_request(protocol: ProtocolKind, body: &Bytes) -> Result { - let body_json: Value = serde_json::from_slice(body).context("parse LLM request")?; - let semantic = understand_request_value(protocol, &body_json)?; - Ok(ParsedRequest { - body_json, - semantic: Arc::new(semantic), - latest_visible_user_content: extract_user_message_from_request_body(body), - }) -} - -pub fn understand_request_value( - protocol: ProtocolKind, - body: &Value, -) -> Result { - let object = body - .as_object() - .context("LLM request must be a JSON object")?; - let request = match protocol { - ProtocolKind::Responses => parse_responses_request(object), - ProtocolKind::Gemini => parse_gemini_request(object), - ProtocolKind::Messages => parse_messages_request(object), - _ => parse_chat_request(object), - }; - Ok(LlmRequestEventPayload { - input_format: protocol.into(), - request, - }) -} - -pub fn understand_response_value( - protocol: ProtocolKind, - body: &Value, -) -> Result { - let object = body - .as_object() - .context("LLM response must be a JSON object")?; - let response = match protocol { - ProtocolKind::Messages => parse_messages_response(object), - ProtocolKind::Responses => parse_responses_response(object), - ProtocolKind::Gemini => parse_gemini_response(object), - _ => parse_chat_response(object), - }; - Ok(LlmResponseEventPayload { - output_format: protocol.into(), - response, - }) -} - -pub fn understand_stream_summary( - protocol: ProtocolKind, - model: &str, - assistant_content: Option<&str>, - usage: LlmUsage, -) -> LlmResponseEventPayload { - let candidates = assistant_content - .filter(|content| !content.is_empty()) - .map(|content| { - vec![LlmCandidate { - index: 0, - message: LlmMessage::new(LlmRole::Assistant, vec![LlmContentPart::text(content)]), - finish_reason: None, - extensions: BTreeMap::from([( - "capture.completeness".into(), - Value::String("visible_text".into()), - )]), - }] - }) - .unwrap_or_default(); - LlmResponseEventPayload { - output_format: protocol.into(), - response: LlmResponse { - id: None, - model: Some(model.into()), - candidates, - usage: Some(usage), - extensions: BTreeMap::from([("capture.streaming".into(), Value::Bool(true))]), - }, - } -} - -impl From for LlmProtocol { - fn from(value: ProtocolKind) -> Self { - match value { - ProtocolKind::ChatCompletions => Self::ChatCompletions, - ProtocolKind::Messages => Self::Messages, - ProtocolKind::Responses => Self::Responses, - ProtocolKind::Gemini => Self::Gemini, - _ => Self::Unknown, - } - } -} - -fn base_request(object: &Map) -> LlmRequest { - LlmRequest { - model: object - .get("model") - .and_then(Value::as_str) - .map(str::to_string), - system: Vec::new(), - messages: Vec::new(), - tools: parse_tools(object.get("tools")), - tool_choice: parse_tool_choice(object.get("tool_choice")), - generation: parse_generation(object), - response_format: parse_response_format(object), - stream: object - .get("stream") - .and_then(Value::as_bool) - .unwrap_or(false), - metadata: object.get("metadata").filter(|v| !v.is_null()).cloned(), - extensions: selected_extensions(object), - } -} - -fn parse_chat_request(object: &Map) -> LlmRequest { - let mut request = base_request(object); - for message in object - .get("messages") - .and_then(Value::as_array) - .into_iter() - .flatten() - { - push_wire_message(message, &mut request); - } - request -} - -fn parse_chat_response(object: &Map) -> LlmResponse { - let candidates = object - .get("choices") - .and_then(Value::as_array) - .into_iter() - .flatten() - .enumerate() - .map(|(position, choice)| { - let message = choice.get("message").unwrap_or(choice); - let mut request = LlmRequest { - model: None, - system: Vec::new(), - messages: Vec::new(), - tools: Vec::new(), - tool_choice: None, - generation: LlmGenerationParams::default(), - response_format: None, - stream: false, - metadata: None, - extensions: BTreeMap::new(), - }; - push_wire_message(message, &mut request); - let message = request - .messages - .pop() - .unwrap_or_else(|| LlmMessage::new(LlmRole::Assistant, Vec::new())); - LlmCandidate { - index: choice - .get("index") - .and_then(Value::as_u64) - .and_then(|index| usize::try_from(index).ok()) - .unwrap_or(position), - message, - finish_reason: choice - .get("finish_reason") - .and_then(Value::as_str) - .map(str::to_string), - extensions: BTreeMap::new(), - } - }) - .collect(); - LlmResponse { - id: object.get("id").and_then(Value::as_str).map(str::to_string), - model: object - .get("model") - .and_then(Value::as_str) - .map(str::to_string), - candidates, - usage: object.get("usage").map(parse_usage), - extensions: BTreeMap::new(), - } -} - -fn parse_messages_response(object: &Map) -> LlmResponse { - let parts = parse_content_parts(object.get("content").unwrap_or(&Value::Null)); - let candidates = if parts.is_empty() { - Vec::new() - } else { - vec![LlmCandidate { - index: 0, - message: LlmMessage::new(LlmRole::Assistant, parts), - finish_reason: object - .get("stop_reason") - .and_then(Value::as_str) - .map(str::to_string), - extensions: BTreeMap::new(), - }] - }; - LlmResponse { - id: object.get("id").and_then(Value::as_str).map(str::to_string), - model: object - .get("model") - .and_then(Value::as_str) - .map(str::to_string), - candidates, - usage: object.get("usage").map(parse_usage), - extensions: BTreeMap::new(), - } -} - -fn parse_responses_response(object: &Map) -> LlmResponse { - let mut parts = Vec::new(); - for item in object - .get("output") - .and_then(Value::as_array) - .into_iter() - .flatten() - { - match item.get("type").and_then(Value::as_str).unwrap_or("") { - "message" => parts.extend(parse_content_parts( - item.get("content").unwrap_or(&Value::Null), - )), - "function_call" | "custom_tool_call" => parts.push(tool_call_part(item)), - "reasoning" => parts.push(LlmContentPart::Reasoning { - text: item.get("summary").and_then(value_text), - signature: item - .get("encrypted_content") - .and_then(Value::as_str) - .map(str::to_string), - }), - kind if !kind.is_empty() => parts.push(LlmContentPart::Unknown { - kind: kind.into(), - value: item.clone(), - }), - _ => {} - } - } - let candidates = if parts.is_empty() { - Vec::new() - } else { - vec![LlmCandidate { - index: 0, - message: LlmMessage::new(LlmRole::Assistant, parts), - finish_reason: object - .get("status") - .and_then(Value::as_str) - .map(str::to_string), - extensions: BTreeMap::new(), - }] - }; - LlmResponse { - id: object.get("id").and_then(Value::as_str).map(str::to_string), - model: object - .get("model") - .and_then(Value::as_str) - .map(str::to_string), - candidates, - usage: object.get("usage").map(parse_usage), - extensions: BTreeMap::new(), - } -} - -fn parse_gemini_response(object: &Map) -> LlmResponse { - let response_id = object - .get("responseId") - .and_then(Value::as_str) - .unwrap_or("gemini"); - let candidates = object - .get("candidates") - .and_then(Value::as_array) - .into_iter() - .flatten() - .enumerate() - .map(|(position, candidate)| { - let index = candidate - .get("index") - .and_then(Value::as_u64) - .and_then(|index| usize::try_from(index).ok()) - .unwrap_or(position); - let mut parts = parse_gemini_parts( - candidate - .get("content") - .and_then(|content| content.get("parts")) - .unwrap_or(&Value::Null), - ); - for (part_index, part) in parts.iter_mut().enumerate() { - if let LlmContentPart::ToolCall { id, .. } = part - && id.is_empty() - { - *id = format!("call_{response_id}_{index}_{part_index}"); - } - } - LlmCandidate { - index, - message: LlmMessage::new(LlmRole::Assistant, parts), - finish_reason: candidate - .get("finishReason") - .and_then(Value::as_str) - .map(str::to_string), - extensions: BTreeMap::new(), - } - }) - .collect(); - LlmResponse { - id: object - .get("responseId") - .and_then(Value::as_str) - .map(str::to_string), - model: object - .get("modelVersion") - .and_then(Value::as_str) - .map(str::to_string), - candidates, - usage: object.get("usageMetadata").map(parse_usage), - extensions: BTreeMap::new(), - } -} - -fn parse_messages_request(object: &Map) -> LlmRequest { - let mut request = base_request(object); - request.system.extend( - parse_content(object.get("system").unwrap_or(&Value::Null)) - .into_iter() - .next() - .map(|parts| LlmMessage::new(LlmRole::System, parts)), - ); - for message in object - .get("messages") - .and_then(Value::as_array) - .into_iter() - .flatten() - { - push_wire_message(message, &mut request); - } - request -} - -fn parse_responses_request(object: &Map) -> LlmRequest { - let mut request = base_request(object); - if let Some(instructions) = object.get("instructions") { - let parts = parse_content_parts(instructions); - if !parts.is_empty() { - request - .system - .push(LlmMessage::new(LlmRole::Developer, parts)); - } - } - match object.get("input") { - Some(Value::String(text)) => request.messages.push(LlmMessage::new( - LlmRole::User, - vec![LlmContentPart::text(text)], - )), - Some(Value::Array(items)) => { - let mut pending_calls = Vec::new(); - for item in items { - let item_type = item.get("type").and_then(Value::as_str).unwrap_or(""); - match item_type { - "function_call" | "custom_tool_call" => { - pending_calls.push(tool_call_part(item)); - } - "function_call_output" | "custom_tool_call_output" => { - flush_pending_calls(&mut pending_calls, &mut request.messages); - request.messages.push(LlmMessage::new( - LlmRole::Tool, - vec![LlmContentPart::ToolResult { - call_id: string_field(item, "call_id"), - name: item.get("name").and_then(Value::as_str).map(str::to_string), - content: item.get("output").cloned().unwrap_or(Value::Null), - is_error: item.get("is_error").and_then(Value::as_bool), - cache_control: item.get("cache_control").cloned(), - }], - )); - } - "reasoning" => { - flush_pending_calls(&mut pending_calls, &mut request.messages); - request.messages.push(LlmMessage::new( - LlmRole::Assistant, - vec![LlmContentPart::Reasoning { - text: item - .get("summary") - .or_else(|| item.get("content")) - .and_then(value_text), - signature: item - .get("encrypted_content") - .and_then(Value::as_str) - .map(str::to_string), - }], - )); - } - "message" => { - flush_pending_calls(&mut pending_calls, &mut request.messages); - push_wire_message(item, &mut request); - } - _ => { - flush_pending_calls(&mut pending_calls, &mut request.messages); - let parts = parse_content_parts(item); - if !parts.is_empty() { - request.messages.push(LlmMessage::new(LlmRole::User, parts)); - } - } - } - } - flush_pending_calls(&mut pending_calls, &mut request.messages); - } - _ => {} - } - request -} - -fn parse_gemini_request(object: &Map) -> LlmRequest { - let mut request = base_request(object); - request.stream = false; - if let Some(instruction) = object.get("systemInstruction") { - let parts = parse_gemini_parts(instruction.get("parts").unwrap_or(instruction)); - if !parts.is_empty() { - request.system.push(LlmMessage::new(LlmRole::System, parts)); - } - } - for (message_index, content) in object - .get("contents") - .and_then(Value::as_array) - .into_iter() - .flatten() - .enumerate() - { - let role = match content.get("role").and_then(Value::as_str) { - Some("model") => LlmRole::Assistant, - _ => LlmRole::User, - }; - let mut parts = parse_gemini_parts(content.get("parts").unwrap_or(&Value::Null)); - for (part_index, part) in parts.iter_mut().enumerate() { - if let LlmContentPart::ToolCall { id, .. } = part - && id.is_empty() - { - *id = format!("call-{message_index}-{part_index}"); - } - } - if !parts.is_empty() { - request.messages.push(LlmMessage::new(role, parts)); - } - } - if let Some(config) = object.get("generationConfig").and_then(Value::as_object) { - request.generation = parse_generation(config); - request.response_format = parse_gemini_response_format(config); - } - request.tools = parse_gemini_tools(object.get("tools")); - request.tool_choice = parse_gemini_tool_choice(object.get("toolConfig")); - request -} - -fn push_wire_message(message: &Value, request: &mut LlmRequest) { - let role = parse_role( - message - .get("role") - .or_else(|| message.get("type")) - .and_then(Value::as_str) - .unwrap_or("user"), - ); - let mut parts = if role == LlmRole::Tool { - vec![LlmContentPart::ToolResult { - call_id: string_field(message, "tool_call_id"), - name: message - .get("name") - .and_then(Value::as_str) - .map(str::to_string), - content: message.get("content").cloned().unwrap_or(Value::Null), - is_error: None, - cache_control: message.get("cache_control").cloned(), - }] - } else { - parse_content_parts(message.get("content").unwrap_or(&Value::Null)) - }; - if let Some(calls) = message.get("tool_calls").and_then(Value::as_array) { - parts.extend(calls.iter().map(tool_call_part)); - } - if let Some(reasoning) = message - .get("reasoning_content") - .or_else(|| message.get("reasoning")) - .and_then(Value::as_str) - .filter(|reasoning| !reasoning.is_empty()) - { - parts.insert( - 0, - LlmContentPart::Reasoning { - text: Some(reasoning.into()), - signature: message - .get("reasoning_signature") - .and_then(Value::as_str) - .map(str::to_string), - }, - ); - } - if let Some(call) = message.get("function_call") { - parts.push(tool_call_part(call)); - } - if parts.is_empty() { - return; - } - let mut parsed = LlmMessage::new(role.clone(), parts); - parsed.name = message - .get("name") - .and_then(Value::as_str) - .map(str::to_string); - request.messages.push(parsed); -} - -fn parse_content(value: &Value) -> Option> { - let parts = parse_content_parts(value); - (!parts.is_empty()).then_some(parts) -} - -fn parse_content_parts(value: &Value) -> Vec { - match value { - Value::String(text) => vec![LlmContentPart::text(text)], - Value::Array(parts) => parts.iter().filter_map(parse_content_part).collect(), - Value::Object(_) => parse_content_part(value).into_iter().collect(), - _ => Vec::new(), - } -} - -fn parse_content_part(part: &Value) -> Option { - let kind = part.get("type").and_then(Value::as_str).unwrap_or(""); - match kind { - "text" | "input_text" | "output_text" => Some(LlmContentPart::Text { - text: string_field(part, "text"), - cache_control: part - .get("cache_control") - .cloned() - .or_else(|| part.get("prompt_cache_breakpoint").cloned()), - }), - "image_url" | "input_image" => { - let image = part.get("image_url").unwrap_or(part); - let url = image - .as_str() - .or_else(|| image.get("url").and_then(Value::as_str))?; - Some(LlmContentPart::Image { - source: image_source(url), - media_type: image - .get("mime_type") - .or_else(|| image.get("format")) - .and_then(Value::as_str) - .map(str::to_string), - detail: image - .get("detail") - .and_then(Value::as_str) - .map(str::to_string), - }) - } - "image" => parse_anthropic_image(part), - "tool_use" | "function_call" | "custom_tool_call" => Some(tool_call_part(part)), - "tool_result" | "function_call_output" | "custom_tool_call_output" => { - Some(LlmContentPart::ToolResult { - call_id: part - .get("tool_use_id") - .or_else(|| part.get("call_id")) - .and_then(Value::as_str) - .unwrap_or_default() - .to_string(), - name: part.get("name").and_then(Value::as_str).map(str::to_string), - content: part - .get("content") - .or_else(|| part.get("output")) - .cloned() - .unwrap_or(Value::Null), - is_error: part.get("is_error").and_then(Value::as_bool), - cache_control: part.get("cache_control").cloned(), - }) - } - "thinking" | "reasoning" => Some(LlmContentPart::Reasoning { - text: part - .get("thinking") - .or_else(|| part.get("text")) - .or_else(|| part.get("summary")) - .and_then(value_text), - signature: part - .get("signature") - .or_else(|| part.get("encrypted_content")) - .and_then(Value::as_str) - .map(str::to_string), - }), - _ => value_text(part).map(LlmContentPart::text).or_else(|| { - (!kind.is_empty()).then(|| LlmContentPart::Unknown { - kind: kind.to_string(), - value: part.clone(), - }) - }), - } -} - -fn parse_anthropic_image(part: &Value) -> Option { - let source = part.get("source")?; - let image_source = match source.get("type").and_then(Value::as_str) { - Some("base64") => LlmImageSource::Data { - data: string_field(source, "data"), - }, - Some("url") => LlmImageSource::Url { - url: string_field(source, "url"), - }, - _ => return None, - }; - Some(LlmContentPart::Image { - source: image_source, - media_type: source - .get("media_type") - .and_then(Value::as_str) - .map(str::to_string), - detail: None, - }) -} - -fn parse_gemini_parts(value: &Value) -> Vec { - value - .as_array() - .into_iter() - .flatten() - .filter_map(|part| { - if let Some(text) = part.get("text").and_then(Value::as_str) { - if part.get("thought").and_then(Value::as_bool) == Some(true) { - return Some(LlmContentPart::Reasoning { - text: Some(text.into()), - signature: part - .get("thoughtSignature") - .and_then(Value::as_str) - .map(str::to_string), - }); - } - return Some(LlmContentPart::Text { - text: text.into(), - cache_control: None, - }); - } - if let Some(data) = part.get("inlineData") { - return Some(LlmContentPart::Image { - source: LlmImageSource::Data { - data: string_field(data, "data"), - }, - media_type: data - .get("mimeType") - .and_then(Value::as_str) - .map(str::to_string), - detail: None, - }); - } - if let Some(file) = part.get("fileData") { - return Some(LlmContentPart::Image { - source: LlmImageSource::File { - uri: string_field(file, "fileUri"), - }, - media_type: file - .get("mimeType") - .and_then(Value::as_str) - .map(str::to_string), - detail: None, - }); - } - if let Some(call) = part.get("functionCall") { - return Some(LlmContentPart::ToolCall { - id: call - .get("id") - .and_then(Value::as_str) - .unwrap_or_default() - .into(), - name: string_field(call, "name"), - arguments: call - .get("args") - .cloned() - .unwrap_or_else(|| Value::Object(Map::new())), - signature: part - .get("thoughtSignature") - .and_then(Value::as_str) - .map(str::to_string), - }); - } - part.get("functionResponse") - .map(|response| LlmContentPart::ToolResult { - call_id: response - .get("id") - .and_then(Value::as_str) - .unwrap_or_default() - .into(), - name: response - .get("name") - .and_then(Value::as_str) - .map(str::to_string), - content: response.get("response").cloned().unwrap_or(Value::Null), - is_error: None, - cache_control: None, - }) - }) - .collect() -} - -fn parse_tools(value: Option<&Value>) -> Vec { - value - .and_then(Value::as_array) - .into_iter() - .flatten() - .filter_map(|tool| { - let function = tool.get("function").unwrap_or(tool); - let name = function.get("name")?.as_str()?.to_string(); - let kind = tool - .get("type") - .and_then(Value::as_str) - .unwrap_or("function") - .to_string(); - let mut extensions = BTreeMap::new(); - for (key, value) in tool.as_object().into_iter().flatten() { - if !matches!( - key.as_str(), - "type" | "name" | "description" | "input_schema" | "function" - ) { - extensions.insert(format!("wire.{key}"), value.clone()); - } - } - Some(LlmToolDefinition { - kind, - name, - description: function - .get("description") - .and_then(Value::as_str) - .map(str::to_string), - input_schema: function - .get("parameters") - .or_else(|| function.get("input_schema")) - .cloned() - .unwrap_or_else(|| serde_json::json!({"type":"object","properties":{}})), - strict: function.get("strict").and_then(Value::as_bool), - extensions, - }) - }) - .collect() -} - -fn parse_gemini_tools(value: Option<&Value>) -> Vec { - value - .and_then(Value::as_array) - .into_iter() - .flatten() - .flat_map(|group| { - group - .get("functionDeclarations") - .and_then(Value::as_array) - .into_iter() - .flatten() - }) - .filter_map(|function| { - Some(LlmToolDefinition { - kind: "function".into(), - name: function.get("name")?.as_str()?.into(), - description: function - .get("description") - .and_then(Value::as_str) - .map(str::to_string), - input_schema: function - .get("parameters") - .cloned() - .unwrap_or_else(|| serde_json::json!({"type":"object","properties":{}})), - strict: None, - extensions: BTreeMap::new(), - }) - }) - .collect() -} - -fn parse_tool_choice(value: Option<&Value>) -> Option { - let value = value?; - let (mode, name) = match value { - Value::String(mode) => ( - match mode.as_str() { - "none" => LlmToolChoiceMode::None, - "required" | "any" => LlmToolChoiceMode::Required, - _ => LlmToolChoiceMode::Auto, - }, - None, - ), - Value::Object(object) => { - let kind = object.get("type").and_then(Value::as_str).unwrap_or("auto"); - let name = object - .get("name") - .or_else(|| object.get("function").and_then(|f| f.get("name"))) - .and_then(Value::as_str) - .map(str::to_string); - let mode = if kind == "tool" || kind == "function" { - LlmToolChoiceMode::Tool - } else if kind == "any" || kind == "required" { - LlmToolChoiceMode::Required - } else if kind == "none" { - LlmToolChoiceMode::None - } else { - LlmToolChoiceMode::Auto - }; - (mode, name) - } - _ => return None, - }; - Some(LlmToolChoice { - mode, - name, - parallel: value - .get("disable_parallel_tool_use") - .and_then(Value::as_bool) - .map(|disabled| !disabled), - extensions: BTreeMap::new(), - }) -} - -fn parse_gemini_tool_choice(value: Option<&Value>) -> Option { - let config = value?.get("functionCallingConfig")?; - let mode = match config.get("mode").and_then(Value::as_str) { - Some("NONE") => LlmToolChoiceMode::None, - Some("ANY") => { - if config - .get("allowedFunctionNames") - .and_then(Value::as_array) - .is_some_and(|names| names.len() == 1) - { - LlmToolChoiceMode::Tool - } else { - LlmToolChoiceMode::Required - } - } - _ => LlmToolChoiceMode::Auto, - }; - let name = config - .get("allowedFunctionNames") - .and_then(Value::as_array) - .and_then(|names| names.first()) - .and_then(Value::as_str) - .map(str::to_string); - Some(LlmToolChoice { - mode, - name, - parallel: None, - extensions: BTreeMap::new(), - }) -} - -fn parse_generation(object: &Map) -> LlmGenerationParams { - let stop_sequences = object - .get("stop") - .or_else(|| object.get("stop_sequences")) - .map(|stop| match stop { - Value::String(value) => vec![value.clone()], - Value::Array(values) => values - .iter() - .filter_map(Value::as_str) - .map(str::to_string) - .collect(), - _ => Vec::new(), - }) - .unwrap_or_default(); - LlmGenerationParams { - temperature: number_f64(object, &["temperature"]), - top_p: number_f64(object, &["top_p", "topP"]), - top_k: number_u64(object, &["top_k", "topK"]), - max_output_tokens: number_u64( - object, - &[ - "max_tokens", - "max_output_tokens", - "max_completion_tokens", - "maxOutputTokens", - ], - ), - stop_sequences, - seed: object.get("seed").and_then(Value::as_i64), - frequency_penalty: number_f64(object, &["frequency_penalty", "frequencyPenalty"]), - presence_penalty: number_f64(object, &["presence_penalty", "presencePenalty"]), - candidate_count: number_u64(object, &["n", "candidateCount"]), - reasoning_effort: object - .get("reasoning_effort") - .or_else(|| object.get("reasoning")) - .or_else(|| object.get("thinking")) - .filter(|value| !value.is_null()) - .cloned(), - thinking_budget: object - .get("thinkingConfig") - .and_then(|value| value.get("thinkingBudget")) - .and_then(Value::as_i64), - extensions: BTreeMap::new(), - } -} - -fn parse_response_format(object: &Map) -> Option { - let format = object - .get("response_format") - .or_else(|| object.get("text").and_then(|text| text.get("format"))) - .or_else(|| { - object - .get("output_config") - .and_then(|value| value.get("format")) - })?; - let kind = format - .get("type") - .and_then(Value::as_str) - .unwrap_or("json_schema") - .to_string(); - let json_schema = format.get("json_schema").unwrap_or(format); - Some(LlmResponseFormat { - kind, - name: json_schema - .get("name") - .and_then(Value::as_str) - .map(str::to_string), - schema: json_schema.get("schema").cloned(), - strict: json_schema.get("strict").and_then(Value::as_bool), - extensions: BTreeMap::new(), - }) -} - -fn parse_gemini_response_format(object: &Map) -> Option { - let mime = object.get("responseMimeType")?.as_str()?; - Some(LlmResponseFormat { - kind: mime.into(), - name: None, - schema: object.get("responseJsonSchema").cloned(), - strict: None, - extensions: BTreeMap::new(), - }) -} - -fn selected_extensions(object: &Map) -> BTreeMap { - let mut extensions = BTreeMap::new(); - for (source, name) in [ - ("cache_control", "anthropic.cache_control"), - ("thinking", "anthropic.thinking"), - ("output_config", "anthropic.output_config"), - ("cachedContent", "google.cached_content"), - ("safetySettings", "google.safety_settings"), - ("labels", "google.labels"), - ("parallel_tool_calls", "openai.parallel_tool_calls"), - ("user", "openai.user"), - ] { - if let Some(value) = object.get(source).filter(|value| !value.is_null()) { - extensions.insert(name.into(), value.clone()); - } - } - extensions -} - -fn parse_usage(value: &Value) -> LlmUsage { - let input_tokens = first_u64( - value, - &["input_tokens", "prompt_tokens", "promptTokenCount"], - ); - let output_tokens = first_u64( - value, - &["output_tokens", "completion_tokens", "candidatesTokenCount"], - ); - let cache_read_tokens = first_u64( - value, - &["cache_read_input_tokens", "cachedContentTokenCount"], - ) - .max( - value - .get("prompt_tokens_details") - .and_then(|details| details.get("cached_tokens")) - .and_then(Value::as_u64) - .unwrap_or(0), - ); - let cache_write_tokens = first_u64(value, &["cache_creation_input_tokens"]); - let reasoning_tokens = first_u64(value, &["thoughtsTokenCount"]).max( - value - .get("completion_tokens_details") - .and_then(|details| details.get("reasoning_tokens")) - .and_then(Value::as_u64) - .unwrap_or(0), - ); - LlmUsage { - input_tokens, - output_tokens, - total_tokens: first_u64(value, &["total_tokens", "totalTokenCount"]) - .max(input_tokens.saturating_add(output_tokens)), - cache_read_tokens, - cache_write_tokens, - reasoning_tokens, - } -} - -fn first_u64(value: &Value, keys: &[&str]) -> u64 { - keys.iter() - .find_map(|key| value.get(*key).and_then(Value::as_u64)) - .unwrap_or(0) -} - -fn tool_call_part(value: &Value) -> LlmContentPart { - let function = value.get("function").unwrap_or(value); - let arguments = function - .get("arguments") - .or_else(|| value.get("arguments")) - .or_else(|| value.get("input")) - .cloned() - .unwrap_or_else(|| Value::Object(Map::new())); - let arguments = match arguments { - Value::String(raw) if raw.trim().is_empty() => Value::Object(Map::new()), - Value::String(raw) => serde_json::from_str(&raw).unwrap_or(Value::String(raw)), - value => value, - }; - LlmContentPart::ToolCall { - id: value - .get("id") - .or_else(|| value.get("call_id")) - .and_then(Value::as_str) - .unwrap_or_default() - .into(), - name: function - .get("name") - .or_else(|| value.get("name")) - .and_then(Value::as_str) - .unwrap_or_default() - .into(), - arguments, - signature: value - .get("signature") - .or_else(|| value.get("thoughtSignature")) - .and_then(Value::as_str) - .map(str::to_string), - } -} - -fn flush_pending_calls(calls: &mut Vec, messages: &mut Vec) { - if !calls.is_empty() { - messages.push(LlmMessage::new(LlmRole::Assistant, std::mem::take(calls))); - } -} - -fn parse_role(role: &str) -> LlmRole { - match role { - "system" => LlmRole::System, - "developer" => LlmRole::Developer, - "assistant" | "model" => LlmRole::Assistant, - "tool" | "function" => LlmRole::Tool, - _ => LlmRole::User, - } -} - -fn image_source(url: &str) -> LlmImageSource { - if url.starts_with("data:") { - LlmImageSource::Data { data: url.into() } - } else if url.starts_with("gs://") { - LlmImageSource::File { uri: url.into() } - } else { - LlmImageSource::Url { url: url.into() } - } -} - -fn value_text(value: &Value) -> Option { - match value { - Value::String(text) if !text.is_empty() => Some(text.clone()), - Value::Array(parts) => { - let text = parts - .iter() - .filter_map(value_text) - .collect::>() - .join("\n"); - (!text.is_empty()).then_some(text) - } - Value::Object(object) => object - .get("text") - .or_else(|| object.get("content")) - .and_then(value_text), - _ => None, - } -} - -fn string_field(value: &Value, field: &str) -> String { - value - .get(field) - .and_then(Value::as_str) - .unwrap_or_default() - .to_string() -} - -fn number_f64(object: &Map, keys: &[&str]) -> Option { - keys.iter() - .find_map(|key| object.get(*key).and_then(Value::as_f64)) -} - -fn number_u64(object: &Map, keys: &[&str]) -> Option { - keys.iter() - .find_map(|key| object.get(*key).and_then(Value::as_u64)) -} - -#[cfg(test)] -mod tests { - use super::*; - use serde_json::json; - - #[test] - fn parses_messages_semantics_without_chat_completions_hop() { - let body = json!({ - "model":"claude-test", - "system":[{"type":"text","text":"system","cache_control":{"type":"ephemeral"}}], - "messages":[ - {"role":"user","content":[{"type":"text","text":"hello"}]}, - {"role":"assistant","content":[{"type":"tool_use","id":"call-1","name":"shell","input":{"cmd":"pwd"}}]}, - {"role":"user","content":[{"type":"tool_result","tool_use_id":"call-1","content":"ok"}]} - ], - "tools":[{"name":"shell","input_schema":{"type":"object"}}], - "max_tokens":64, - "stream":true - }); - let parsed = understand_request_value(ProtocolKind::Messages, &body).unwrap(); - assert_eq!(parsed.input_format, LlmProtocol::Messages); - assert_eq!(parsed.request.model.as_deref(), Some("claude-test")); - assert_eq!(parsed.request.system.len(), 1); - assert_eq!(parsed.request.messages.len(), 3); - assert_eq!(parsed.request.tool_names(), ["shell"]); - assert_eq!(parsed.request.visible_user_turns(), 1); - assert_eq!(parsed.request.generation.max_output_tokens, Some(64)); - assert!(matches!( - &parsed.request.messages[1].parts[0], - LlmContentPart::ToolCall { arguments, .. } - if arguments.get("cmd").and_then(Value::as_str) == Some("pwd") - )); - } - - #[test] - fn parses_gemini_native_contents_for_capture() { - let body = json!({ - "systemInstruction":{"parts":[{"text":"system"}]}, - "contents":[ - {"role":"user","parts":[{"text":"hello"}]}, - {"role":"model","parts":[{"functionCall":{"name":"shell","args":{"cmd":"pwd"}},"thoughtSignature":"sig"}]} - ], - "generationConfig":{"maxOutputTokens":32,"temperature":0.2} - }); - let parsed = understand_request_value(ProtocolKind::Gemini, &body).unwrap(); - assert_eq!(parsed.request.latest_user_text().as_deref(), Some("hello")); - assert_eq!(parsed.request.generation.max_output_tokens, Some(32)); - assert!(matches!( - parsed.request.messages[1].parts[0], - LlmContentPart::ToolCall { ref signature, .. } if signature.as_deref() == Some("sig") - )); - } - - #[test] - fn parses_responses_tool_sequence() { - let body = json!({ - "model":"gpt-test", - "instructions":"system", - "input":[ - {"type":"message","role":"user","content":[{"type":"input_text","text":"run"}]}, - {"type":"function_call","call_id":"call-1","name":"shell","arguments":"{\"cmd\":\"pwd\"}"}, - {"type":"function_call_output","call_id":"call-1","output":"ok"} - ] - }); - let parsed = understand_request_value(ProtocolKind::Responses, &body).unwrap(); - assert_eq!(parsed.request.system.len(), 1); - assert_eq!(parsed.request.messages.len(), 3); - assert!(matches!( - parsed.request.messages[1].parts[0], - LlmContentPart::ToolCall { ref id, .. } if id == "call-1" - )); - } - - #[test] - fn parses_gemini_response_usage_and_tool_call() { - let body = json!({ - "responseId":"response-1", - "modelVersion":"gemini-test", - "candidates":[{ - "index":0, - "content":{"role":"model","parts":[{ - "functionCall":{"name":"shell","args":{"cmd":"pwd"}}, - "thoughtSignature":"sig" - }]}, - "finishReason":"STOP" - }], - "usageMetadata":{ - "promptTokenCount":3, - "candidatesTokenCount":2, - "thoughtsTokenCount":1, - "totalTokenCount":6 - } - }); - let parsed = understand_response_value(ProtocolKind::Gemini, &body).unwrap(); - assert_eq!(parsed.response.id.as_deref(), Some("response-1")); - assert_eq!(parsed.response.usage.unwrap().reasoning_tokens, 1); - assert!(matches!( - parsed.response.candidates[0].message.parts[0], - LlmContentPart::ToolCall { ref signature, .. } if signature.as_deref() == Some("sig") - )); - } -} diff --git a/crates/persisting-gateway/src/usage.rs b/crates/persisting-gateway/src/usage.rs deleted file mode 100644 index 8944b2042..000000000 --- a/crates/persisting-gateway/src/usage.rs +++ /dev/null @@ -1,229 +0,0 @@ -//! Token usage extraction and rough USD cost estimation. - -use serde::{Deserialize, Serialize}; -use serde_json::Value; - -use crate::provider::ProviderKind; - -#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] -pub struct TokenUsage { - pub input_tokens: u64, - pub output_tokens: u64, - pub total_tokens: u64, - pub cache_read_tokens: u64, - pub cache_write_tokens: u64, - #[serde(default)] - pub reasoning_tokens: u64, -} - -/// Streaming capture metrics (TTFT + incremental usage). -#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] -pub struct StreamMetrics { - pub usage: TokenUsage, - pub ttft_ms: Option, -} - -impl TokenUsage { - pub fn merge(&mut self, other: &TokenUsage) { - self.input_tokens += other.input_tokens; - self.output_tokens += other.output_tokens; - self.total_tokens += other.total_tokens; - self.cache_read_tokens += other.cache_read_tokens; - self.cache_write_tokens += other.cache_write_tokens; - self.reasoning_tokens += other.reasoning_tokens; - } -} - -/// Extract usage from OpenAI-style `usage` or Anthropic `usage` blocks. -/// Merge usage from SSE `data:` lines (OpenAI / Anthropic streaming). -pub fn extract_usage_from_sse(raw: &str) -> TokenUsage { - let mut merged = TokenUsage::default(); - for line in raw.lines() { - let line = line.trim(); - let Some(json_str) = line.strip_prefix("data:") else { - continue; - }; - let json_str = json_str.trim(); - if json_str.is_empty() || json_str == "[DONE]" { - continue; - } - if let Ok(v) = serde_json::from_str::(json_str) { - merged.merge(&extract_usage_from_response(&v)); - if let Some(msg) = v.get("message") { - merged.merge(&extract_usage_from_response(msg)); - } - } - } - merged -} - -pub fn extract_usage_from_response(body: &Value) -> TokenUsage { - let mut u = TokenUsage::default(); - let usage = body - .get("usage") - .or_else(|| body.get("Usage")) - .or_else(|| body.get("usageMetadata")); - let Some(usage) = usage else { - return u; - }; - u.input_tokens = pick_u64( - usage, - &["prompt_tokens", "input_tokens", "promptTokenCount"], - ); - u.output_tokens = pick_u64( - usage, - &["completion_tokens", "output_tokens", "candidatesTokenCount"], - ); - u.total_tokens = pick_u64(usage, &["total_tokens", "totalTokenCount"]); - if u.total_tokens == 0 { - u.total_tokens = u.input_tokens + u.output_tokens; - } - if let Some(details) = usage.get("prompt_tokens_details") { - u.cache_read_tokens = pick_u64(details, &["cached_tokens"]); - u.cache_write_tokens = pick_u64(details, &["cache_write_tokens"]); - } - if usage.get("cache_creation_input_tokens").is_some() { - u.cache_write_tokens = pick_u64(usage, &["cache_creation_input_tokens"]); - } - if let Some(details) = usage.get("completion_tokens_details") { - u.reasoning_tokens = pick_u64(details, &["reasoning_tokens"]); - } - if usage.get("cache_read_input_tokens").is_some() { - u.cache_read_tokens = pick_u64(usage, &["cache_read_input_tokens"]); - } - if usage.get("cachedContentTokenCount").is_some() { - u.cache_read_tokens = pick_u64(usage, &["cachedContentTokenCount"]); - } - if usage.get("thoughtsTokenCount").is_some() { - u.reasoning_tokens = pick_u64(usage, &["thoughtsTokenCount"]); - } - u -} - -fn pick_u64(v: &Value, keys: &[&str]) -> u64 { - for k in keys { - if let Some(n) = v.get(*k).and_then(|x| x.as_u64()) { - return n; - } - } - 0 -} - -/// Per-million-token USD list prices (rough public list; override via config later). -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ModelPricing { - pub input_per_million: f64, - pub output_per_million: f64, -} - -impl Default for ModelPricing { - fn default() -> Self { - Self { - input_per_million: 1.0, - output_per_million: 2.0, - } - } -} - -pub fn default_pricing_table() -> Vec<(&'static str, ModelPricing)> { - vec![ - ( - "gpt-4o", - ModelPricing { - input_per_million: 2.5, - output_per_million: 10.0, - }, - ), - ( - "gpt-4o-mini", - ModelPricing { - input_per_million: 0.15, - output_per_million: 0.6, - }, - ), - ( - "claude-3-5-sonnet", - ModelPricing { - input_per_million: 3.0, - output_per_million: 15.0, - }, - ), - ( - "deepseek-chat", - ModelPricing { - input_per_million: 0.27, - output_per_million: 1.1, - }, - ), - ( - "gemini-2.0-flash", - ModelPricing { - input_per_million: 0.1, - output_per_million: 0.4, - }, - ), - ] -} - -pub fn estimate_cost_usd(model: &str, provider: ProviderKind, usage: &TokenUsage) -> f64 { - let table = default_pricing_table(); - let model_key = model.to_ascii_lowercase(); - let fallback = ModelPricing { - input_per_million: match provider { - ProviderKind::Anthropic => 3.0, - ProviderKind::Gemini | ProviderKind::Vertex => 0.15, - _ => 1.0, - }, - output_per_million: match provider { - ProviderKind::Anthropic => 15.0, - ProviderKind::Gemini | ProviderKind::Vertex => 0.6, - _ => 2.0, - }, - }; - let price = table - .iter() - .find(|(m, _)| model_key.contains(m)) - .map(|(_, p)| p) - .unwrap_or(&fallback); - (usage.input_tokens as f64 / 1_000_000.0) * price.input_per_million - + (usage.output_tokens as f64 / 1_000_000.0) * price.output_per_million -} - -#[cfg(test)] -mod tests { - use super::*; - use serde_json::json; - - #[test] - fn openai_usage() { - let body = json!({ - "usage": { - "prompt_tokens": 10, - "completion_tokens": 20, - "total_tokens": 30 - } - }); - let u = extract_usage_from_response(&body); - assert_eq!(u.input_tokens, 10); - assert_eq!(u.output_tokens, 20); - } - - #[test] - fn anthropic_sse_usage() { - let sse = "event: message_delta\n\ - data: {\"type\":\"message_delta\",\"usage\":{\"output_tokens\":12}}\n\n"; - let u = extract_usage_from_sse(sse); - assert_eq!(u.output_tokens, 12); - } - - #[test] - fn cost_positive() { - let u = TokenUsage { - input_tokens: 1_000_000, - output_tokens: 0, - ..Default::default() - }; - let c = estimate_cost_usd("gpt-4o", ProviderKind::OpenAi, &u); - assert!(c > 2.0); - } -} diff --git a/crates/persisting-gateway/tests/README.md b/crates/persisting-gateway/tests/README.md deleted file mode 100644 index f22d0f272..000000000 --- a/crates/persisting-gateway/tests/README.md +++ /dev/null @@ -1,32 +0,0 @@ -# persisting-gateway 集成测试 - -**Gateway conversion / capture fixture 回归，以及 Claude 专有轨迹 fixture。** - -拥有本 crate 的集成测试入口与 fixture 读取辅助。不拥有 Gateway 实现 -（[`../README.md`](../README.md)）或 pChronicle 存储格式。`fixtures/**/README.md` -是 fixture 数据说明，不是组件文档。 - -## Fixture 数据 - -| 目录 / 文件 | 说明 | -|-------------|------| -| [`fixtures/`](fixtures/README.md) | 主要来自 agentgateway LLM 测试集（含致谢与许可说明） | -| [`fixtures/local/`](fixtures/local/README.md) | Persisting 自有补充 fixture | -| [`support/ag_fixtures.rs`](support/ag_fixtures.rs) | 读取 fixture、解析 AG `.snap`、归一化比对 | -| [`ag_fixture_tests.rs`](ag_fixture_tests.rs) | 基于 AG 数据的 conversion + capture 回归 | -| [`llm_fixtures.rs`](llm_fixtures.rs) | 基础 smoke 测试 | -| [`model_api_forwarding.rs`](model_api_forwarding.rs) | Chat Completions、Responses、Messages 的文本与 tool call 非流式/SSE 同协议转发测试 | -| [`capture/apps/claude/`](capture/apps/claude/) | Claude 专有轨迹 fixture（与 AG 无关） | - -## Run - -```bash -just test-capture-fixtures -just test-capture-claude -cargo nextest run -p persisting-gateway --test model_api_forwarding --locked -``` - -## Links - -- [`persisting-gateway`](../README.md) -- [Capture trajectories](../../../docs/src/en/pchronicle/guides/serve-gateway.md) diff --git a/crates/persisting-gateway/tests/ag_fixture_tests.rs b/crates/persisting-gateway/tests/ag_fixture_tests.rs deleted file mode 100644 index 02ab8f102..000000000 --- a/crates/persisting-gateway/tests/ag_fixture_tests.rs +++ /dev/null @@ -1,669 +0,0 @@ -//! Conversion + capture regression tests driven by agentgateway LLM fixtures. -//! -//! Source and license: `tests/fixtures/README.md`. - -mod support; - -use std::collections::BTreeMap; - -use persisting_gateway::conversion::{ - ProtocolBridge, StreamTranslator, completions_request_to_gemini, - completions_response_to_messages, gemini_response_to_completions, - messages_request_to_completions, responses_request_to_completions, - translate_request_for_bridge, -}; -use persisting_gateway::dialogue_extract::{ - count_visible_user_messages, extract_assistant_text_from_json, extract_assistant_turn_from_sse, - extract_user_message_from_request_body, -}; -use persisting_gateway::protocol::ProtocolKind; -use persisting_gateway::understanding::understand_request; -use persisting_gateway::usage::{extract_usage_from_response, extract_usage_from_sse}; -use serde_json::Value; -use support::ag_capture_cases::{ - ASSISTANT_JSON_CASES, ASSISTANT_SSE_CASES, USAGE_JSON_FIXTURES, USER_CAPTURE_CASES, -}; -use support::ag_fixtures::{ - COMPLETIONS_TO_GEMINI, COMPLETIONS_TO_MESSAGES, CaseReport, GEMINI_TO_COMPLETIONS, - MESSAGES_TO_COMPLETIONS, RESPONSES_TO_COMPLETIONS, ag_snap_request, ag_snap_response, - assert_json_eq, assert_messages_response_eq, client_model_from_completions_fixture, - completions_messages_snap, fixture_exists, for_each_existing, for_each_existing_case, - load_json_fixture, messages_completions_snap, parse_ag_sse_snap, parse_sse_events, - read_fixture, read_fixture_bytes, sse_event_names, translate_openai_sse_fixture, - upstream_model_from_messages_fixture, -}; - -// --- conversion: messages ↔ completions --- - -#[test] -fn ag_messages_request_matches_completions_snap() { - let report = for_each_existing_case( - MESSAGES_TO_COMPLETIONS, - "requests/messages/", - ".completions.snap", - |case| { - let snap_path = messages_completions_snap(case); - let body = read_fixture_bytes(&format!("requests/messages/{case}.json")); - let upstream = upstream_model_from_messages_fixture(case); - let out = messages_request_to_completions(&body, &upstream).unwrap(); - let actual: Value = serde_json::from_slice(&out).unwrap(); - let expected = ag_snap_request(&snap_path); - assert_json_eq(&actual, &expected, case); - }, - ); - report.assert_min_ran(3, "messages→completions snap"); -} - -#[test] -fn chronicle_semantic_ir_messages_request_matches_completions_snap() { - let report = for_each_existing_case( - MESSAGES_TO_COMPLETIONS, - "requests/messages/", - ".completions.snap", - |case| { - let snap_path = messages_completions_snap(case); - let body = read_fixture_bytes(&format!("requests/messages/{case}.json")); - let upstream = upstream_model_from_messages_fixture(case); - let parsed = - understand_request(persisting_gateway::protocol::ProtocolKind::Messages, &body) - .unwrap(); - let out = translate_request_for_bridge( - ProtocolBridge::MessagesToCompletions, - &parsed.semantic, - &upstream, - None, - ) - .unwrap(); - let actual: Value = serde_json::from_slice(&out).unwrap(); - let expected = ag_snap_request(&snap_path); - assert_json_eq(&actual, &expected, case); - }, - ); - report.assert_min_ran(3, "Chronicle semantic IR messages→completions snap"); -} - -#[test] -fn ag_messages_tool_history_survives_without_tool_declarations() { - let body = read_fixture_bytes("requests/messages/tool_history_without_tools.json"); - let out = messages_request_to_completions(&body, "upstream-model").unwrap(); - let value: Value = serde_json::from_slice(&out).unwrap(); - assert!(value.get("tools").is_none()); - assert_eq!(value["messages"][0]["role"], "assistant"); - assert_eq!(value["messages"][0]["tool_calls"][0]["id"], "call_1"); - assert_eq!( - value["messages"][0]["tool_calls"][0]["function"]["name"], - "get_weather" - ); - assert_eq!(value["messages"][1]["role"], "tool"); - assert_eq!(value["messages"][1]["tool_call_id"], "call_1"); -} - -#[test] -fn ag_completions_response_matches_messages_snap() { - let report = for_each_existing_case( - COMPLETIONS_TO_MESSAGES, - "response/completions/", - ".completions-messages.snap", - |case| { - let snap_path = completions_messages_snap(case); - let body = read_fixture_bytes(&format!("response/completions/{case}.json")); - let client_model = client_model_from_completions_fixture(case); - let out = completions_response_to_messages(&body, &client_model).unwrap(); - let actual: Value = serde_json::from_slice(&out).unwrap(); - let expected = ag_snap_response(&snap_path); - assert_messages_response_eq(&actual, &expected, case); - }, - ); - report.assert_min_ran(5, "completions→messages snap"); -} - -// --- conversion: responses → completions --- - -#[test] -fn ag_responses_request_to_completions() { - let report = for_each_existing_case( - RESPONSES_TO_COMPLETIONS, - "requests/responses/", - ".json", - |case| { - let body = read_fixture_bytes(&format!("requests/responses/{case}.json")); - let out = responses_request_to_completions(&body, "upstream-model", None).unwrap(); - let v: Value = serde_json::from_slice(&out).unwrap(); - assert_eq!(v["model"], "upstream-model", "{case}"); - assert!( - v.get("messages") - .and_then(|m| m.as_array()) - .is_some_and(|a| !a.is_empty()), - "{case}: expected non-empty messages" - ); - }, - ); - report.assert_min_ran(5, "responses→completions"); -} - -#[test] -fn chronicle_semantic_ir_responses_bridge_is_usable() { - let report = for_each_existing_case( - RESPONSES_TO_COMPLETIONS, - "requests/responses/", - ".json", - |case| { - let body = read_fixture_bytes(&format!("requests/responses/{case}.json")); - let parsed = - understand_request(persisting_gateway::protocol::ProtocolKind::Responses, &body) - .unwrap(); - let semantic = translate_request_for_bridge( - ProtocolBridge::ResponsesToCompletions, - &parsed.semantic, - "upstream-model", - None, - ) - .unwrap(); - let semantic: Value = serde_json::from_slice(&semantic).unwrap(); - assert_eq!(semantic["model"], "upstream-model", "{case}"); - assert!( - semantic["messages"] - .as_array() - .is_some_and(|messages| !messages.is_empty()), - "{case}: semantic renderer returned no messages" - ); - }, - ); - report.assert_min_ran(5, "Chronicle semantic IR responses→completions"); -} - -// --- conversion: chat completions ↔ Gemini native --- - -#[test] -fn ag_completions_request_matches_gemini_native_goldens() { - let report = for_each_existing_case( - COMPLETIONS_TO_GEMINI, - "requests/completions/", - ".vertex-gemini.snap", - |case| { - let body = read_fixture_bytes(&format!("requests/completions/{case}.json")); - let request: Value = serde_json::from_slice(&body).unwrap(); - let model = request - .get("model") - .and_then(Value::as_str) - .unwrap_or("gemini-2.5-pro"); - let out = completions_request_to_gemini(&body, model).unwrap(); - let actual: Value = serde_json::from_slice(&out).unwrap(); - let expected = - ag_snap_request(&format!("requests/completions/{case}.vertex-gemini.snap")); - assert_json_eq(&actual, &expected, case); - - let parsed = understand_request( - persisting_gateway::protocol::ProtocolKind::ChatCompletions, - &body, - ) - .unwrap(); - let typed = translate_request_for_bridge( - ProtocolBridge::CompletionsToGemini, - &parsed.semantic, - model, - None, - ) - .unwrap(); - let typed: Value = serde_json::from_slice(&typed).unwrap(); - assert_json_eq(&typed, &expected, &format!("typed:{case}")); - }, - ); - report.assert_min_ran(10, "completions→Gemini native snap"); -} - -#[test] -fn ag_gemini_native_response_matches_completions_goldens() { - let report = for_each_existing_case( - GEMINI_TO_COMPLETIONS, - "response/vertex-gemini/", - ".vertex-gemini-completions.snap", - |case| { - let body = read_fixture_bytes(&format!("response/vertex-gemini/{case}.json")); - let out = gemini_response_to_completions(&body, "gemini-2.5-pro").unwrap(); - let mut actual: Value = serde_json::from_slice(&out).unwrap(); - actual["id"] = serde_json::json!("[id]"); - actual["created"] = serde_json::json!("[date]"); - let expected = ag_snap_response(&format!( - "response/vertex-gemini/{case}.vertex-gemini-completions.snap" - )); - assert_json_eq(&actual, &expected, case); - }, - ); - report.assert_min_ran(4, "Gemini native→completions snap"); -} - -#[test] -fn ag_gemini_native_stream_matches_completions_golden() { - let input = read_fixture("response/vertex-gemini/stream_tool.json"); - let mut translator = StreamTranslator::new( - ProtocolBridge::CompletionsToGemini, - ProtocolKind::ChatCompletions, - "gemini-2.5-pro", - ) - .unwrap(); - let mut actual = Vec::new(); - for chunk in input.as_bytes().chunks(37) { - actual.extend_from_slice(&translator.push_chunk(chunk).unwrap()); - } - actual.extend_from_slice(&translator.finish_stream().unwrap()); - let actual = String::from_utf8(actual).unwrap(); - - let expected = parse_ag_sse_snap( - "response/vertex-gemini/stream_tool.vertex-gemini-completions-streaming.snap", - ); - assert_eq!(fold_chat_sse(&actual), fold_chat_sse(&expected)); - assert_eq!(translator.metrics().usage.input_tokens, 15); - assert_eq!(translator.metrics().usage.output_tokens, 12); -} - -#[test] -fn gemini_native_stream_composes_with_messages_and_responses_bridges() { - let input = read_fixture("response/vertex-gemini/stream_tool.json"); - - let mut messages = StreamTranslator::new( - ProtocolBridge::MessagesToGemini, - ProtocolKind::Messages, - "claude-gemini-client", - ) - .unwrap(); - let mut messages_sse = messages.push_chunk(input.as_bytes()).unwrap().to_vec(); - messages_sse.extend_from_slice(&messages.finish_stream().unwrap()); - let messages_sse = String::from_utf8(messages_sse).unwrap(); - let message_events = sse_event_names(&messages_sse); - assert!(message_events.contains(&"message_start")); - assert!(message_events.contains(&"content_block_start")); - assert!(message_events.contains(&"message_stop")); - - let mut responses = StreamTranslator::new( - ProtocolBridge::ResponsesToGemini, - ProtocolKind::Responses, - "responses-gemini-client", - ) - .unwrap(); - let mut responses_sse = responses.push_chunk(input.as_bytes()).unwrap().to_vec(); - responses_sse.extend_from_slice(&responses.finish_stream().unwrap()); - let responses_sse = String::from_utf8(responses_sse).unwrap(); - let response_events = sse_event_names(&responses_sse); - assert!(response_events.contains(&"response.created")); - assert!(response_events.contains(&"response.output_item.added")); - assert!(response_events.contains(&"response.completed")); -} - -// --- streaming translation --- - -#[test] -fn ag_completions_stream_translates_to_messages_sse() { - let path = "response/completions/stream.json"; - if !fixture_exists(path) { - panic!("missing required fixture {path}"); - } - let client_model = "claude-test"; - let mut translator = StreamTranslator::new( - ProtocolBridge::MessagesToCompletions, - ProtocolKind::Messages, - client_model, - ) - .unwrap(); - let out = translate_openai_sse_fixture(path, |chunk| translator.push_chunk(chunk)); - let events = sse_event_names(&out); - assert!(events.contains(&"message_start"), "events: {events:?}"); - assert!(events.contains(&"content_block_delta")); - let text = translator.streaming_capture_snapshot().unwrap_or_default(); - assert!(text.contains("Hi"), "accumulated: {text}"); - assert!(text.contains("help")); -} - -#[test] -fn ag_stream_snap_contains_expected_events() { - let snap_path = "response/completions/stream.completions-messages-streaming.snap"; - if !fixture_exists(snap_path) { - panic!("missing required fixture {snap_path}"); - } - let snap = parse_ag_sse_snap(snap_path); - let events = sse_event_names(&snap); - assert!(events.contains(&"message_start")); - assert!(events.contains(&"content_block_delta")); -} - -#[test] -fn ag_completions_stream_matches_messages_golden() { - for (case, model) in [ - ("stream", "gpt-5-nano-2025-08-07"), - ("stream_tool_empty_content", "claude-opus-4-8"), - ] { - let input = format!("response/completions/{case}.json"); - let snapshot = format!("response/completions/{case}.completions-messages-streaming.snap"); - let mut translator = StreamTranslator::new( - ProtocolBridge::MessagesToCompletions, - ProtocolKind::Messages, - model, - ) - .unwrap(); - let mut actual = translate_openai_sse_fixture(&input, |chunk| translator.push_chunk(chunk)); - actual.push_str(std::str::from_utf8(&translator.finish_stream().unwrap()).unwrap()); - let expected = parse_ag_sse_snap(&snapshot); - assert_eq!( - normalized_message_events(&actual), - normalized_message_events(&expected), - "{case}" - ); - } -} - -#[test] -fn ag_local_stream_head_fixture() { - let raw = read_fixture("local/response/completions/stream_head.txt"); - let out = completions_sse_to_messages(&raw, "claude-test"); - assert!(out.contains("message_start")); - assert!(out.contains("content_block_delta")); -} - -// --- capture: user dialogue matrix --- - -#[test] -fn ag_capture_user_dialogue_matrix() { - let mut report = CaseReport::default(); - for case in USER_CAPTURE_CASES { - if !fixture_exists(case.path) { - report.record_skipped(); - continue; - } - let body = read_fixture_bytes(case.path); - let v = load_json_fixture(case.path); - let turns = count_visible_user_messages(&v); - assert!( - turns >= case.min_turns, - "{}: expected >={} user turns, got {turns}", - case.path, - case.min_turns - ); - let user = extract_user_message_from_request_body(&body) - .unwrap_or_else(|| panic!("{}: expected user content", case.path)); - assert!(!user.trim().is_empty(), "{}: empty user content", case.path); - for needle in case.must_contain { - assert!( - user.contains(needle), - "{}: user content missing {needle:?}\n---\n{user}", - case.path - ); - } - report.record_ran(); - } - report.assert_min_ran(8, "user dialogue matrix"); -} - -#[test] -fn ag_capture_user_from_messages_requests() { - let report = for_each_existing_case( - MESSAGES_TO_COMPLETIONS, - "requests/messages/", - ".json", - |case| { - let body = read_fixture_bytes(&format!("requests/messages/{case}.json")); - let user = extract_user_message_from_request_body(&body); - assert!(user.is_some(), "{case}: expected user text"); - assert!(!user.unwrap().trim().is_empty(), "{case}: empty user text"); - }, - ); - report.assert_min_ran(3, "messages request user extract"); -} - -#[test] -fn ag_capture_user_from_responses_requests() { - let report = for_each_existing_case( - RESPONSES_TO_COMPLETIONS, - "requests/responses/", - ".json", - |case| { - let body = read_fixture_bytes(&format!("requests/responses/{case}.json")); - let user = extract_user_message_from_request_body(&body); - assert!(user.is_some(), "{case}: expected user text"); - }, - ); - report.assert_min_ran(5, "responses request user extract"); -} - -// --- capture: assistant dialogue matrix --- - -#[test] -fn ag_capture_assistant_json_matrix() { - let mut report = CaseReport::default(); - for case in ASSISTANT_JSON_CASES { - if !fixture_exists(case.path) { - report.record_skipped(); - continue; - } - let body = load_json_fixture(case.path); - let text = extract_assistant_text_from_json(&body) - .unwrap_or_else(|| panic!("{}: expected assistant text", case.path)); - for needle in case.must_contain { - assert!( - text.contains(needle), - "{}: assistant missing {needle:?}\n---\n{text}", - case.path - ); - } - report.record_ran(); - } - report.assert_min_ran(5, "assistant JSON matrix"); -} - -#[test] -fn ag_capture_assistant_sse_matrix() { - let mut report = CaseReport::default(); - for case in ASSISTANT_SSE_CASES { - if !fixture_exists(case.path) { - report.record_skipped(); - continue; - } - let raw = read_fixture(case.path); - let sse = if case.translate_completions_to_messages { - completions_sse_to_messages(&raw, "claude-test") - } else { - raw - }; - let text = extract_assistant_turn_from_sse(&sse); - for needle in case.must_contain { - assert!( - text.contains(needle), - "{}: SSE assistant missing {needle:?}\n---\n{text}", - case.path - ); - } - report.record_ran(); - } - report.assert_min_ran(4, "assistant SSE matrix"); -} - -#[test] -fn ag_capture_assistant_from_completions_response() { - let body: Value = load_json_fixture("response/completions/basic.json"); - let text = extract_assistant_text_from_json(&body).unwrap(); - assert!(text.contains("Sorry") || text.contains("provider")); -} - -#[test] -fn ag_capture_assistant_from_stream_fixture() { - let raw = read_fixture("local/response/completions/stream_head.txt"); - let text = extract_assistant_turn_from_sse(&completions_sse_to_messages(&raw, "claude-test")); - assert!(text.contains("Hi")); -} - -fn completions_sse_to_messages(raw: &str, model: &str) -> String { - let mut translator = StreamTranslator::new( - ProtocolBridge::MessagesToCompletions, - ProtocolKind::Messages, - model, - ) - .unwrap(); - let mut output = Vec::new(); - for chunk in raw.as_bytes().chunks(512) { - output.extend_from_slice(&translator.push_chunk(chunk).unwrap()); - } - output.extend_from_slice(&translator.finish_stream().unwrap()); - String::from_utf8(output).unwrap() -} - -fn fold_chat_sse(sse: &str) -> Value { - let mut id = None; - let mut model = None; - let mut role = None; - let mut content = String::new(); - let mut finish_reason = None; - let mut tools: BTreeMap = BTreeMap::new(); - let mut usage = None; - for line in sse.lines().filter_map(|line| line.strip_prefix("data: ")) { - if line == "[DONE]" { - continue; - } - let value: Value = serde_json::from_str(line).unwrap(); - id = value - .get("id") - .and_then(Value::as_str) - .map(str::to_string) - .or(id); - model = value - .get("model") - .and_then(Value::as_str) - .map(str::to_string) - .or(model); - if let Some(choice) = value - .get("choices") - .and_then(Value::as_array) - .and_then(|choices| choices.first()) - { - let delta = choice.get("delta").unwrap_or(&Value::Null); - role = delta - .get("role") - .and_then(Value::as_str) - .map(str::to_string) - .or(role); - if let Some(text) = delta.get("content").and_then(Value::as_str) { - content.push_str(text); - } - for call in delta - .get("tool_calls") - .and_then(Value::as_array) - .into_iter() - .flatten() - { - let index = call.get("index").and_then(Value::as_u64).unwrap_or(0); - let tool = tools.entry(index).or_default(); - if let Some(call_id) = call.get("id").and_then(Value::as_str) { - tool.0 = call_id.to_string(); - } - if let Some(name) = call.pointer("/function/name").and_then(Value::as_str) { - tool.1 = name.to_string(); - } - if let Some(arguments) = call.pointer("/function/arguments").and_then(Value::as_str) - { - tool.2.push_str(arguments); - } - } - finish_reason = choice - .get("finish_reason") - .and_then(Value::as_str) - .map(str::to_string) - .or(finish_reason); - } - if let Some(value_usage) = value.get("usage") { - usage = Some(serde_json::json!({ - "prompt_tokens": value_usage.get("prompt_tokens").and_then(Value::as_u64).unwrap_or(0), - "completion_tokens": value_usage.get("completion_tokens").and_then(Value::as_u64).unwrap_or(0), - "total_tokens": value_usage.get("total_tokens").and_then(Value::as_u64).unwrap_or(0), - })); - } - } - serde_json::json!({ - "id": id, - "model": model, - "role": role, - "content": content, - "finish_reason": finish_reason, - "tool_calls": tools.into_values().map(|(id, name, arguments)| serde_json::json!({"id":id,"name":name,"arguments":arguments})).collect::>(), - "usage": usage, - }) -} - -fn normalized_message_events(sse: &str) -> Vec<(String, Value)> { - let mut events = parse_sse_events(sse); - for (_, value) in &mut events { - let Some(usage) = value.get_mut("usage").and_then(Value::as_object_mut) else { - continue; - }; - for key in ["cache_read_input_tokens", "cache_creation_input_tokens"] { - if usage.get(key).and_then(Value::as_u64) == Some(0) { - usage.remove(key); - } - } - } - events -} - -// --- capture: usage --- - -#[test] -fn ag_capture_usage_json_matrix() { - let report = for_each_existing(USAGE_JSON_FIXTURES, |path| { - let body = load_json_fixture(path); - let usage = extract_usage_from_response(&body); - assert!( - usage.total_tokens > 0 || usage.input_tokens + usage.output_tokens > 0, - "{path}: expected non-zero usage, got {usage:?}" - ); - }); - report.assert_min_ran(5, "usage JSON matrix"); -} - -#[test] -fn ag_capture_usage_from_completions_stream() { - let raw = read_fixture("local/response/completions/stream_tool_call.txt"); - let usage = extract_usage_from_sse(&raw); - assert!(usage.total_tokens > 0, "usage: {usage:?}"); -} - -#[test] -fn ag_capture_usage_from_completions_response() { - let body = load_json_fixture("response/completions/basic.json"); - let usage = extract_usage_from_response(&body); - assert!(usage.total_tokens > 0 || usage.input_tokens + usage.output_tokens > 0); -} - -// --- capture: multimodal (Phase 0) --- - -#[test] -fn ag_completions_full_fixture_includes_image_placeholder() { - let path = "requests/completions/full.json"; - if !fixture_exists(path) { - panic!("missing required fixture {path}"); - } - let body = read_fixture_bytes(path); - let user = extract_user_message_from_request_body(&body).unwrap(); - assert!(user.contains("What's in this image?")); - assert!(user.contains("[image: base64:")); - assert!(user.contains("image/png")); -} - -#[test] -fn ag_messages_full_converted_user_still_has_image_placeholder() { - let path = "requests/completions/full.json"; - if !fixture_exists(path) { - return; - } - let body = read_fixture_bytes(path); - let user = extract_user_message_from_request_body(&body).unwrap(); - assert!(user.contains("[image: base64:")); -} - -#[test] -fn ag_responses_stream_image_fixture_has_generation_events() { - let path = "response/responses/stream-image.json"; - if !fixture_exists(path) { - panic!("missing required fixture {path}"); - } - let raw = read_fixture(path); - assert!(raw.contains("image_generation_call")); - let text = extract_assistant_turn_from_sse(&raw); - assert!(text.contains("[image_generated:")); -} diff --git a/crates/persisting-gateway/tests/agenticmd_bridge.rs b/crates/persisting-gateway/tests/agenticmd_bridge.rs deleted file mode 100644 index 79aa99a1b..000000000 --- a/crates/persisting-gateway/tests/agenticmd_bridge.rs +++ /dev/null @@ -1,87 +0,0 @@ -//! Capture → Storyline → AgenticMD bridge tests. - -use persisting_gateway::Call; -use persisting_gateway::projection::dialogue::capture_record_to_storyline_turn; -use persisting_gateway::projection::markdown_pipeline::MarkdownPipeline; -use persisting_gateway::record::EventRecord; -use persisting_gateway::sink::{llm_request_record, llm_response_record}; -use persisting_pchronicle::document::{decode_agenticmd, write_agenticmd_storyline}; -use persisting_pchronicle::model::StorylineDocument; -use serde_json::json; - -fn fixture() -> &'static str { - include_str!("fixtures/agenticmd/demo-run-001.md") -} - -fn materialize_records(path: &std::path::Path, records: &[EventRecord]) -> anyhow::Result<()> { - let turns = MarkdownPipeline::storyline_turns_from_records(records)?; - let mut story = StorylineDocument::new("s1", "gateway"); - story.turns = turns; - write_agenticmd_storyline(path, &story)?; - Ok(()) -} - -fn pair() -> [EventRecord; 2] { - let call = Call { - call_id: "c1".into(), - trace_id: "t1".into(), - started_at: "2026-01-01T00:00:00Z".into(), - }; - let mut req = llm_request_record( - Some("s1".into()), - Some("gateway".into()), - "m", - "/v1/chat/completions", - &json!({"messages":[{"role":"user","content":"hi"}]}), - ); - req.seq = 1; - req.call_id = Some("c1".into()); - req.timestamp = Some("2026-01-01T00:00:00Z".into()); - let mut resp = llm_response_record( - Some("s1".into()), - Some("gateway".into()), - 200, - &json!({"choices":[{"message":{"role":"assistant","content":"yo"}}]}), - false, - &call, - ); - resp.seq = 2; - resp.call_id = Some("c1".into()); - resp.timestamp = Some("2026-01-01T00:00:01Z".into()); - [req, resp] -} - -#[test] -fn capture_turns_roundtrip_through_public_storyline_api() { - let [req, resp] = pair(); - let mut story = StorylineDocument::new("s1", "gateway"); - story.turns = vec![ - capture_record_to_storyline_turn(&req).unwrap(), - capture_record_to_storyline_turn(&resp).unwrap(), - ]; - - let encoded = persisting_pchronicle::document::encode_agenticmd(&story).unwrap(); - assert_eq!(decode_agenticmd(&encoded).unwrap(), story); -} - -#[test] -fn legacy_fixture_without_authoritative_storyline_is_rejected() { - let error = decode_agenticmd(fixture()).unwrap_err(); - assert_eq!( - error.to_string(), - "missing authoritative Storyline metadata" - ); -} - -#[test] -fn materialize_writes_storyline_parseable_markdown() { - let records = pair(); - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("sess.md"); - materialize_records(&path, &records).unwrap(); - let story = decode_agenticmd(&std::fs::read_to_string(&path).unwrap()).unwrap(); - assert_eq!(story.session_id, "s1"); - assert_eq!(story.turns.len(), 2); - assert_eq!(story.turns[0].message, json!("hi")); - assert_eq!(story.turns[1].message, json!("yo")); -} diff --git a/crates/persisting-gateway/tests/agenticmd_golden.rs b/crates/persisting-gateway/tests/agenticmd_golden.rs deleted file mode 100644 index b8af2124f..000000000 --- a/crates/persisting-gateway/tests/agenticmd_golden.rs +++ /dev/null @@ -1,64 +0,0 @@ -//! Golden AgenticMD semantics through the public Storyline API. - -use persisting_gateway::Call; -use persisting_gateway::projection::dialogue::capture_record_to_storyline_turn; -use persisting_gateway::sink::{llm_request_record, llm_response_record}; -use persisting_pchronicle::document::{decode_agenticmd, encode_agenticmd}; -use persisting_pchronicle::model::StorylineDocument; -use serde_json::json; - -fn demo_storyline() -> StorylineDocument { - let call = Call { - call_id: "call-demo-1".into(), - trace_id: "trace-demo-1".into(), - started_at: "2026-01-01T00:00:00Z".into(), - }; - let mut req = llm_request_record( - Some("demo-run-001".into()), - Some("demo-agent".into()), - "deepseek-chat", - "/v1/chat/completions", - &json!({"messages":[{"role":"user","content":"你好"}]}), - ); - req.seq = 1; - req.call_id = Some("call-demo-1".into()); - req.timestamp = Some("2026-01-01T00:00:00Z".into()); - let mut resp = llm_response_record( - Some("demo-run-001".into()), - Some("demo-agent".into()), - 200, - &json!({ - "choices":[{"message":{"role":"assistant","content":"你好！有什么可以帮你的？"}}], - "usage":{"prompt_tokens":12,"completion_tokens":18,"total_tokens":30} - }), - false, - &call, - ); - resp.seq = 2; - resp.timestamp = Some("2026-01-01T00:00:01Z".into()); - - let mut story = StorylineDocument::new("demo-run-001", "demo-agent"); - story.turns = vec![ - capture_record_to_storyline_turn(&req).unwrap(), - capture_record_to_storyline_turn(&resp).unwrap(), - ]; - story -} - -#[test] -fn generated_agenticmd_preserves_golden_storyline_semantics() { - let story = demo_storyline(); - let encoded = encode_agenticmd(&story).unwrap(); - assert_eq!(decode_agenticmd(&encoded).unwrap(), story); -} - -#[test] -fn checked_in_legacy_golden_without_authoritative_storyline_is_rejected() { - let fixture = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) - .join("tests/fixtures/agenticmd/demo-run-001.md"); - let error = decode_agenticmd(&std::fs::read_to_string(&fixture).unwrap()).unwrap_err(); - assert_eq!( - error.to_string(), - "missing authoritative Storyline metadata" - ); -} diff --git a/crates/persisting-gateway/tests/capture/apps/claude/dialogue_extract.rs b/crates/persisting-gateway/tests/capture/apps/claude/dialogue_extract.rs deleted file mode 100644 index ba1bd3f51..000000000 --- a/crates/persisting-gateway/tests/capture/apps/claude/dialogue_extract.rs +++ /dev/null @@ -1,64 +0,0 @@ -//! Visible dialogue extraction from Claude Code Anthropic payloads. - -use persisting_gateway::dialogue_extract::{ - SseStreamBlockParser, extract_assistant_turn_from_sse, extract_user_message_from_request_body, - is_main_agent_continuation, push_sse_tool_snapshot, -}; - -use super::support::fixture_body; - -#[test] -fn skips_suggestion_mode_system_injection_in_request_extract() { - let body = super::support::body_bytes( - r#"{"messages":[{"role":"user","content":[{"type":"text","text":"[SUGGESTION MODE: predict next input]\nReply ONLY the suggestion."}]}]}"#, - ); - assert!(extract_user_message_from_request_body(&body).is_none()); -} - -#[test] -fn extracts_user_turn_with_tool_result_blocks() { - let body = fixture_body("agent_launch_tool_result.json"); - let text = extract_user_message_from_request_body(&body).expect("user text"); - assert!(text.contains("```tool_result:")); - assert!(text.contains("agentId: a483eb5f3a45995b0")); -} - -#[test] -fn unwraps_session_tag_wrapper_in_user_text() { - let body = fixture_body("flash_session_probe.json"); - let text = extract_user_message_from_request_body(&body).expect("user text"); - assert_eq!(text, "Review design docs with subagents"); - assert!(!text.contains("")); -} - -#[test] -fn detects_task_notification_as_main_continuation() { - let body = fixture_body("task_notification.json"); - assert!(is_main_agent_continuation(&body)); -} - -#[test] -fn subagent_explore_prompt_is_not_main_continuation() { - let body = fixture_body("subagent_explore_flash.json"); - assert!(!is_main_agent_continuation(&body)); -} - -#[test] -fn sse_incremental_parser_emits_tool_agent_on_content_block_stop() { - let sse = include_str!("fixtures/agent_tool_stream.sse"); - let mut parser = SseStreamBlockParser::default(); - let snap = push_sse_tool_snapshot(&mut parser, sse).expect("partial snapshot"); - assert!(snap.contains("```tool:Agent")); - assert!(snap.contains("cli-pvisor.md")); -} - -#[test] -fn sse_visible_text_excludes_thinking_blocks() { - let sse = r#"event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":"secret"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"Hi"}} -"#; - assert_eq!(extract_assistant_turn_from_sse(sse), "Hi"); -} diff --git a/crates/persisting-gateway/tests/capture/apps/claude/fixtures/agent_launch_tool_result.json b/crates/persisting-gateway/tests/capture/apps/claude/fixtures/agent_launch_tool_result.json deleted file mode 100644 index d237be815..000000000 --- a/crates/persisting-gateway/tests/capture/apps/claude/fixtures/agent_launch_tool_result.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "model": "deepseek-v4-pro", - "messages": [ - { - "role": "user", - "content": [ - { - "type": "tool_result", - "tool_use_id": "call_00_33N2dbzetZy8qNYgNwTx8513", - "content": "Async agent launched successfully.\nagentId: a483eb5f3a45995b0 (internal ID - do not mention to user.)\nThe agent is working in the background." - } - ] - } - ] -} diff --git a/crates/persisting-gateway/tests/capture/apps/claude/fixtures/agent_tool_stream.sse b/crates/persisting-gateway/tests/capture/apps/claude/fixtures/agent_tool_stream.sse deleted file mode 100644 index c9fc2ded4..000000000 --- a/crates/persisting-gateway/tests/capture/apps/claude/fixtures/agent_tool_stream.sse +++ /dev/null @@ -1,17 +0,0 @@ -event: content_block_start -data: {"type":"content_block_start","index":0,"content_block":{"type":"text"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Launching."}} - -event: content_block_stop -data: {"type":"content_block_stop","index":0} - -event: content_block_start -data: {"type":"content_block_start","index":1,"content_block":{"type":"tool_use","name":"Agent","id":"toolu_agent_1"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"{\"description\":\"Review capture command design\",\"subagent_type\":\"Explore\",\"prompt\":\"Review docs/src/design/cli-pvisor.md\"}"}} - -event: content_block_stop -data: {"type":"content_block_stop","index":1} diff --git a/crates/persisting-gateway/tests/capture/apps/claude/fixtures/flash_session_probe.json b/crates/persisting-gateway/tests/capture/apps/claude/fixtures/flash_session_probe.json deleted file mode 100644 index af79d7a6b..000000000 --- a/crates/persisting-gateway/tests/capture/apps/claude/fixtures/flash_session_probe.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "model": "deepseek-v4-flash", - "messages": [ - { - "role": "user", - "content": [ - { - "type": "text", - "text": "\nReview design docs with subagents\n" - } - ] - } - ] -} diff --git a/crates/persisting-gateway/tests/capture/apps/claude/fixtures/main_user_pro.json b/crates/persisting-gateway/tests/capture/apps/claude/fixtures/main_user_pro.json deleted file mode 100644 index 49e9fac33..000000000 --- a/crates/persisting-gateway/tests/capture/apps/claude/fixtures/main_user_pro.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "model": "deepseek-v4-pro", - "messages": [ - { - "role": "user", - "content": "再次开三个subagent，帮我看看设计文档的问题" - } - ] -} diff --git a/crates/persisting-gateway/tests/capture/apps/claude/fixtures/subagent_explore_flash.json b/crates/persisting-gateway/tests/capture/apps/claude/fixtures/subagent_explore_flash.json deleted file mode 100644 index 6c57ff4e5..000000000 --- a/crates/persisting-gateway/tests/capture/apps/claude/fixtures/subagent_explore_flash.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "model": "deepseek-v4-flash", - "messages": [ - { - "role": "user", - "content": "Review the design document at docs/src/design/cli-pvisor.md\n\nPlease review for clarity, consistency, and actionable gaps.\nProvide bullet points in Chinese." - } - ] -} diff --git a/crates/persisting-gateway/tests/capture/apps/claude/fixtures/task_notification.json b/crates/persisting-gateway/tests/capture/apps/claude/fixtures/task_notification.json deleted file mode 100644 index 645f71fbc..000000000 --- a/crates/persisting-gateway/tests/capture/apps/claude/fixtures/task_notification.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "model": "deepseek-v4-pro", - "messages": [ - { - "role": "user", - "content": [ - { - "type": "text", - "text": "\naeb40a1230926b4a8\ncompleted\nAgent completed\n" - } - ] - } - ] -} diff --git a/crates/persisting-gateway/tests/capture/apps/claude/markdown_filter.rs b/crates/persisting-gateway/tests/capture/apps/claude/markdown_filter.rs deleted file mode 100644 index bd87277c4..000000000 --- a/crates/persisting-gateway/tests/capture/apps/claude/markdown_filter.rs +++ /dev/null @@ -1,105 +0,0 @@ -//! Markdown trajectory filters for Claude Code traffic patterns. - -use persisting_gateway::config::CaptureLevel; -use persisting_gateway::engine::Call; -use persisting_gateway::projection::dialogue::skip_markdown_block; -use persisting_gateway::record::EventRecord; -use persisting_gateway::sink::llm_request_summary_record; -use serde_json::json; - -use super::support::fixture_body; - -fn test_call() -> Call { - Call { - call_id: "call-test".into(), - trace_id: "trace-test".into(), - started_at: "2026-05-24T00:00:00Z".into(), - } -} - -fn request_record( - model: &str, - user: &str, - subagent_id: Option<&str>, - body: Option<&serde_json::Value>, -) -> EventRecord { - let mut rec = llm_request_summary_record( - Some(super::support::CLAUDE_SESSION.into()), - Some(super::support::PROXY_AGENT.into()), - model, - "/v1/messages", - 100, - "messages", - "anthropic", - Some(user.into()), - None, - &test_call(), - CaptureLevel::Dialogue, - body, - ); - rec.subagent_id = subagent_id.map(str::to_string); - if let Some(body) = body { - if let Some(messages) = body.get("messages") { - rec.payload["messages"] = messages.clone(); - } - if let Some(model) = body.get("model").and_then(|m| m.as_str()) { - rec.payload["model"] = json!(model); - } - } - rec -} - -#[test] -fn skips_main_session_flash_companion_before_pro_turn() { - let rec = request_record( - "deepseek-v4-flash", - "再次开三个subagent，帮我看看设计文档的问题", - None, - None, - ); - assert!(skip_markdown_block(&rec)); -} - -#[test] -fn keeps_main_session_pro_user_turn() { - let rec = request_record( - "deepseek-v4-pro", - "再次开三个subagent，帮我看看设计文档的问题", - None, - None, - ); - assert!(!skip_markdown_block(&rec)); -} - -#[test] -fn keeps_subagent_flash_explore_prompt() { - let body: serde_json::Value = - serde_json::from_slice(&fixture_body("subagent_explore_flash.json")).unwrap(); - let rec = request_record( - "deepseek-v4-flash", - "Review docs/src/design/cli-pvisor.md", - Some("ad20bef53147678d4"), - Some(&body), - ); - assert!(!skip_markdown_block(&rec)); -} - -#[test] -fn keeps_flash_session_title_probe_on_main() { - let body: serde_json::Value = - serde_json::from_slice(&fixture_body("flash_session_probe.json")).unwrap(); - let rec = request_record( - "deepseek-v4-flash", - "\nReview design docs with subagents\n", - None, - Some(&body), - ); - assert!(!skip_markdown_block(&rec)); -} - -#[test] -fn skips_count_tokens_internal_requests() { - let mut rec = request_record("deepseek-v4-pro", "huge context", None, None); - rec.payload["path"] = json!("/v1/messages/count_tokens"); - assert!(skip_markdown_block(&rec)); -} diff --git a/crates/persisting-gateway/tests/capture/apps/claude/mod.rs b/crates/persisting-gateway/tests/capture/apps/claude/mod.rs deleted file mode 100644 index 5e55fdcd9..000000000 --- a/crates/persisting-gateway/tests/capture/apps/claude/mod.rs +++ /dev/null @@ -1,9 +0,0 @@ -//! Claude Code–specific capture regressions (routing, spawn links, markdown filters). -//! -//! See `docs/src/design/capture.md` §8 and `trajectory.md` §7 for the contract these tests guard. - -mod dialogue_extract; -mod markdown_filter; -mod routing; -mod spawn_link; -mod support; diff --git a/crates/persisting-gateway/tests/capture/apps/claude/routing.rs b/crates/persisting-gateway/tests/capture/apps/claude/routing.rs deleted file mode 100644 index 26ca67d67..000000000 --- a/crates/persisting-gateway/tests/capture/apps/claude/routing.rs +++ /dev/null @@ -1,126 +0,0 @@ -//! Claude Code HTTP routing: main run dir + `{session_id}.md` siblings. - -use super::support::{ - CLAUDE_SESSION, PROXY_AGENT, RUN_ROOT, body_bytes, claude_run_storage, fixture_body, headers, - main_trajectory_dir, resolve_claude_route, subagent_markdown_path, -}; -use persisting_gateway::session::storage::resolve_capture_route; - -#[test] -fn main_agent_pro_turn_writes_header_session_md() { - let dir = claude_run_storage(); - let body = fixture_body("main_user_pro.json"); - let route = resolve_claude_route( - dir.path(), - &[("x-claude-code-session-id", CLAUDE_SESSION)], - &body, - ); - assert_eq!(route.root_session.as_deref(), Some(RUN_ROOT)); - assert_eq!(route.storage_session_id, CLAUDE_SESSION); - assert_eq!(route.session_id, CLAUDE_SESSION); - assert!(route.subagent_id.is_none()); - assert_eq!( - main_trajectory_dir(dir.path(), &route), - dir.path().join(format!("{PROXY_AGENT}/{RUN_ROOT}")) - ); - assert!(subagent_markdown_path(dir.path(), &route).ends_with(format!("{CLAUDE_SESSION}.md"))); -} - -#[test] -fn flash_session_probe_without_agent_id_writes_header_session_md() { - let dir = claude_run_storage(); - let body = fixture_body("flash_session_probe.json"); - let header_session = "884e189a-9433-4996-a809-3a8232a8967d"; - let route = resolve_claude_route( - dir.path(), - &[("x-claude-code-session-id", header_session)], - &body, - ); - assert_eq!(route.storage_session_id, header_session); - assert!(route.subagent_id.is_none()); - assert_eq!(route.session_id, header_session); - assert!(subagent_markdown_path(dir.path(), &route).ends_with(format!("{header_session}.md"))); -} - -#[test] -fn explore_subagent_with_agent_id_writes_sibling_md() { - let dir = claude_run_storage(); - let body = fixture_body("subagent_explore_flash.json"); - let agent = "ad20bef53147678d4"; - let route = resolve_claude_route( - dir.path(), - &[ - ("x-claude-code-session-id", CLAUDE_SESSION), - ("x-claude-code-agent-id", agent), - ], - &body, - ); - assert_eq!(route.subagent_id.as_deref(), Some(agent)); - assert_eq!(route.storage_session_id, format!("agent-{agent}")); - assert_eq!( - main_trajectory_dir(dir.path(), &route), - dir.path().join(format!("{PROXY_AGENT}/{RUN_ROOT}")) - ); - assert_eq!( - subagent_markdown_path(dir.path(), &route), - dir.path() - .join(format!("{PROXY_AGENT}/{RUN_ROOT}/agent-{agent}.md")) - ); -} - -#[test] -fn parallel_subagents_get_isolated_md_files() { - let dir = claude_run_storage(); - let body = fixture_body("subagent_explore_flash.json"); - for agent in [ - "a1111111111111111", - "b2222222222222222", - "c3333333333333333", - ] { - let route = resolve_claude_route( - dir.path(), - &[ - ("x-claude-code-session-id", CLAUDE_SESSION), - ("x-claude-code-agent-id", agent), - ], - &body, - ); - assert_eq!(route.storage_session_id, format!("agent-{agent}")); - assert_eq!( - subagent_markdown_path(dir.path(), &route), - dir.path() - .join(format!("{PROXY_AGENT}/{RUN_ROOT}/agent-{agent}.md")) - ); - } -} - -#[test] -fn task_notification_with_agent_id_header_routes_to_main_session_md() { - let dir = claude_run_storage(); - let body = fixture_body("task_notification.json"); - let agent = "aeb40a1230926b4a8"; - let route = resolve_claude_route( - dir.path(), - &[ - ("x-claude-code-session-id", CLAUDE_SESSION), - ("x-claude-code-agent-id", agent), - ], - &body, - ); - assert_eq!(route.storage_session_id, CLAUDE_SESSION); - assert!(route.subagent_id.is_none()); - assert!(subagent_markdown_path(dir.path(), &route).ends_with(format!("{CLAUDE_SESSION}.md"))); -} - -#[test] -fn capture_serve_flat_layout_without_run_root() { - let body = body_bytes(r#"{"model":"deepseek-v4-pro","messages":[]}"#); - let route = resolve_capture_route( - &headers(&[("x-persisting-session-id", "sess-flat")]), - &body, - "x-persisting-session-id", - std::path::Path::new("/tmp/unused"), - ); - assert!(route.root_session.is_none()); - assert_eq!(route.storage_session_id, "sess-flat"); -} diff --git a/crates/persisting-gateway/tests/capture/apps/claude/spawn_link.rs b/crates/persisting-gateway/tests/capture/apps/claude/spawn_link.rs deleted file mode 100644 index ea553c9e0..000000000 --- a/crates/persisting-gateway/tests/capture/apps/claude/spawn_link.rs +++ /dev/null @@ -1,147 +0,0 @@ -//! Main agent ↔ subagent cross-reference (`spawn_links`, `refs_subagent_ids`). - -use axum::http::HeaderMap; -use persisting_gateway::projection::dialogue::skip_markdown_block; -use persisting_gateway::record::EventRecord; -use persisting_gateway::session::storage::CaptureRoute; -use persisting_gateway::subagent_link::{ - SubagentRegistry, enrich_record, extract_spawn_hints_from_assistant, - extract_subagent_ids_from_request, extract_subagent_ids_from_text, -}; -use serde_json::json; - -use super::support::fixture_body; - -#[test] -fn parses_agent_tool_blocks_into_spawn_hints() { - let text = r#"Launching reviewers. - -```tool:Agent -{ - "description": "Review capture command design", - "subagent_type": "Explore", - "prompt": "Review docs/src/design/cli-pvisor.md" -} -```"#; - let hints = extract_spawn_hints_from_assistant(text); - assert_eq!(hints.len(), 1); - assert_eq!(hints[0].subagent_type, "Explore"); - assert_eq!( - hints[0].doc_target.as_deref(), - Some("docs/src/design/cli-pvisor.md") - ); -} - -#[test] -fn matches_spawn_hint_to_registered_subagent_by_doc_target() { - let mut registry = SubagentRegistry::default(); - let run_key = super::support::RUN_ROOT.to_string(); - let route = CaptureRoute { - root_session: Some(super::support::RUN_ROOT.into()), - session_id: super::support::CLAUDE_SESSION.into(), - storage_session_id: super::support::RUN_ROOT.into(), - subagent_id: None, - }; - let sub_route = CaptureRoute { - root_session: Some(super::support::RUN_ROOT.into()), - session_id: super::support::CLAUDE_SESSION.into(), - storage_session_id: "agent-ad20bef53147678d4".into(), - subagent_id: Some("ad20bef53147678d4".into()), - }; - let body: serde_json::Value = - serde_json::from_slice(&fixture_body("subagent_explore_flash.json")).unwrap(); - registry.register_route(&run_key, &sub_route, "call-sub-first", Some(&body)); - let assistant = r#"```tool:Agent -{"description":"Review capture command design","subagent_type":"Explore","prompt":"Review docs/src/design/cli-pvisor.md"} -```"#; - let mut rec = EventRecord { - identity: Default::default(), - seq: 0, - source: "persisting-proxy".into(), - kind: "llm.response.stream".into(), - timestamp: None, - session_id: Some(super::support::CLAUDE_SESSION.into()), - agent_id: Some(super::support::PROXY_AGENT.into()), - parent_uuid: None, - trace_id: Some("trace-main".into()), - call_id: Some("call-main".into()), - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload: json!({}), - }; - let outcome = enrich_record( - &mut rec, - &route, - &HeaderMap::new(), - None, - Some(assistant), - &mut registry, - ); - assert!(outcome.spawn_link_backfills.is_empty()); - let links = rec.payload["spawn_links"].as_array().expect("spawn_links"); - assert_eq!(links.len(), 1); - assert_eq!(links[0]["subagent_id"], "ad20bef53147678d4"); - assert_eq!( - links[0]["subagent_trajectory"], - "agent-ad20bef53147678d4.md" - ); -} - -#[test] -fn tool_result_agent_id_line_populates_refs() { - let body: serde_json::Value = - serde_json::from_slice(&fixture_body("agent_launch_tool_result.json")).unwrap(); - assert_eq!( - extract_subagent_ids_from_request(&body), - vec!["a483eb5f3a45995b0".to_string()] - ); -} - -#[test] -fn task_notification_task_id_populates_refs() { - let text = "\na206e346f287da063\n"; - assert_eq!( - extract_subagent_ids_from_text(text), - vec!["a206e346f287da063".to_string()] - ); -} - -#[test] -fn ignores_documentation_cli_agent_id_flag() { - let body = json!({ - "messages": [{ - "role": "user", - "content": [{ - "type": "tool_result", - "content": "--agent-id my-repo --session-id abc" - }] - }] - }); - assert!(extract_subagent_ids_from_request(&body).is_empty()); -} - -#[test] -fn spawn_link_records_are_never_skipped_in_markdown() { - let rec = EventRecord { - identity: Default::default(), - seq: 7, - source: "persisting-proxy".into(), - kind: "llm.spawn_link".into(), - timestamp: None, - session_id: None, - agent_id: None, - parent_uuid: None, - trace_id: None, - call_id: Some("call-parent".into()), - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: Some("call-parent".into()), - payload: json!({ - "spawn_links": [{"subagent_id": "abc1234", "subagent_trajectory": "agent-abc1234.md"}] - }), - }; - assert!(!skip_markdown_block(&rec)); -} diff --git a/crates/persisting-gateway/tests/capture/apps/claude/support.rs b/crates/persisting-gateway/tests/capture/apps/claude/support.rs deleted file mode 100644 index 1a361544a..000000000 --- a/crates/persisting-gateway/tests/capture/apps/claude/support.rs +++ /dev/null @@ -1,62 +0,0 @@ -use std::path::{Path, PathBuf}; - -use axum::http::HeaderMap; -use axum::http::HeaderValue; -use axum::http::header::HeaderName; -use bytes::Bytes; -use persisting_gateway::session::storage::{ - CaptureRoute, resolve_capture_route, trajectory_markdown_path, trajectory_run_dir, -}; - -pub const RUN_ROOT: &str = "run-20260524-test"; -pub const PROXY_AGENT: &str = "deepseek-proxy"; -pub const CLAUDE_SESSION: &str = "5e27e4a7-f42a-42a9-8448-79608bd95c53"; -pub const SESSION_HEADER: &str = "x-persisting-session-id"; - -/// Temp storage root with `.capture/run_session` (mimics `capture run`). -pub fn claude_run_storage() -> tempfile::TempDir { - let dir = tempfile::tempdir().expect("tempdir"); - std::fs::create_dir_all(dir.path().join(".capture")).expect("mkdir .capture"); - std::fs::write(dir.path().join(".capture/run_session"), RUN_ROOT).expect("write run_session"); - dir -} - -pub fn headers(pairs: &[(&str, &str)]) -> HeaderMap { - let mut h = HeaderMap::new(); - for (k, v) in pairs { - h.insert( - HeaderName::from_bytes(k.as_bytes()).expect("header name"), - HeaderValue::try_from(*v).expect("header value"), - ); - } - h -} - -pub fn body_bytes(json: &str) -> Bytes { - Bytes::copy_from_slice(json.as_bytes()) -} - -pub fn fixture_body(name: &str) -> Bytes { - let path = format!( - "{}/tests/capture/apps/claude/fixtures/{name}", - env!("CARGO_MANIFEST_DIR") - ); - let data = std::fs::read(&path).unwrap_or_else(|e| panic!("read fixture {path}: {e}")); - Bytes::from(data) -} - -pub fn resolve_claude_route( - storage: &Path, - header_pairs: &[(&str, &str)], - body: &Bytes, -) -> CaptureRoute { - resolve_capture_route(&headers(header_pairs), body, SESSION_HEADER, storage) -} - -pub fn main_trajectory_dir(storage: &Path, route: &CaptureRoute) -> PathBuf { - trajectory_run_dir(storage, PROXY_AGENT, route) -} - -pub fn subagent_markdown_path(storage: &Path, route: &CaptureRoute) -> PathBuf { - trajectory_markdown_path(storage, PROXY_AGENT, route) -} diff --git a/crates/persisting-gateway/tests/capture/apps/mod.rs b/crates/persisting-gateway/tests/capture/apps/mod.rs deleted file mode 100644 index e098b0fa0..000000000 --- a/crates/persisting-gateway/tests/capture/apps/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod claude; diff --git a/crates/persisting-gateway/tests/capture/mod.rs b/crates/persisting-gateway/tests/capture/mod.rs deleted file mode 100644 index c2b34bdcb..000000000 --- a/crates/persisting-gateway/tests/capture/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod apps; diff --git a/crates/persisting-gateway/tests/capture_apps_claude.rs b/crates/persisting-gateway/tests/capture_apps_claude.rs deleted file mode 100644 index 13a71a6ab..000000000 --- a/crates/persisting-gateway/tests/capture_apps_claude.rs +++ /dev/null @@ -1,5 +0,0 @@ -//! Claude Code trajectory capture regression suite. -//! -//! Run: `cargo test -p persisting-gateway --test capture_apps_claude` - -mod capture; diff --git a/crates/persisting-gateway/tests/fixtures/README.md b/crates/persisting-gateway/tests/fixtures/README.md deleted file mode 100644 index 5525aea85..000000000 --- a/crates/persisting-gateway/tests/fixtures/README.md +++ /dev/null @@ -1,66 +0,0 @@ -# LLM 测试数据（fixtures） - -本目录大部分文件来自 [agentgateway](https://github.com/agentgateway/agentgateway) 的 LLM 协议转换回归测试集，供 `persisting-gateway` 的 conversion 与 capture 测试使用。 - -## 来源 - -| 项目 | 路径 | -|------|------| -| **上游仓库** | https://github.com/agentgateway/agentgateway | -| **上游目录** | `crates/llm/src/tests/` | -| **同步基线** | `82cbbf6f5bd988899302c8bce2a8d006355355ef` | - -手动更新时，将上游该目录内容复制到本目录（**保留** [`local/`](local/README.md) 子目录，勿覆盖）。 - -## 目录结构 - -与 agentgateway 保持一致： - -``` -fixtures/ -├── requests/ # 各协议请求样例（.json）及期望转换 snap（.*.snap） -│ ├── completions/ -│ ├── messages/ -│ ├── responses/ -│ ├── embeddings/ -│ ├── count-tokens/ -│ └── policies/ -├── response/ # 各协议响应样例与跨协议 snap -│ ├── completions/ -│ ├── anthropic/ -│ ├── bedrock/ -│ ├── responses/ -│ └── detect/ -└── local/ # Persisting 自有 fixture（非 AG），见 local/README.md -``` - -- **`.json`** / 无后缀文本：原始 wire 输入（请求或上游响应）。 -- **`.snap`**：agentgateway 用 [insta](https://github.com/mitsuhiko/insta) 生成的期望输出；YAML frontmatter（`---` … `---`）后为 JSON 或 SSE 正文。 -- 原始 SSE fixture 必须保留事件后的空行分隔符，包括文件末尾的 `\n\n`；该空行属于 wire 协议，不能作为多余尾部空白删除。 - -## 在本项目中的用途 - -| 测试 | 说明 | -|------|------| -| `tests/ag_fixture_tests.rs` | 对照 AG snap 做 messages/responses ↔ completions 与 Gemini native 转换；capture 用户/助手/usage 矩阵（见 `tests/support/ag_capture_cases.rs`） | -| `tests/llm_fixtures.rs` | 基础 smoke 回归 | -| `src/conversion/*`、`src/dialogue_extract/*` | 部分单元测试通过 `include_str!` 引用本目录 | - -协议解析与流式转换还通过 `crates/persisting-gateway/fuzz/` 下的两个 cargo-fuzz target -覆盖任意输入、分块边界和不完整 SSE frame。 - -Persisting 当前实现 **Messages ↔ Completions**、**Responses ↔ Completions**，以及 -**Completions/Messages/Responses ↔ Gemini native generateContent** bridge。支持的请求、响应和 -SSE 用例会与上游 snap 做严格 JSON/事件比较；Bedrock、Vertex 和 Detect 等其他 adapter 的 -snap 仅作为后续扩展参考，不代表已经实现。 - -## 致谢与许可 - -测试数据源自 **agentgateway** 项目及其贡献者。agentgateway 以 [Apache License 2.0](https://github.com/agentgateway/agentgateway/blob/main/LICENSE) 发布。 - -在此复制并用于回归测试，遵循 Apache-2.0 对再分发与修改的要求。若上游 fixture 有更新，请同步复制并保留本 README 中的来源说明。 - -``` -Copyright agentgateway contributors -SPDX-License-Identifier: Apache-2.0 -``` diff --git a/crates/persisting-gateway/tests/fixtures/agenticmd/demo-run-001.md b/crates/persisting-gateway/tests/fixtures/agenticmd/demo-run-001.md deleted file mode 100644 index d69a53835..000000000 --- a/crates/persisting-gateway/tests/fixtures/agenticmd/demo-run-001.md +++ /dev/null @@ -1,19 +0,0 @@ ---- -format: persisting -block: |+ - - - message body - -session_id: demo-run-001 -agent_id: demo-agent -turn_count: 1 ---- - - - -你好 - - - -你好！有什么可以帮你的？ diff --git a/crates/persisting-gateway/tests/fixtures/local/README.md b/crates/persisting-gateway/tests/fixtures/local/README.md deleted file mode 100644 index bfbb50114..000000000 --- a/crates/persisting-gateway/tests/fixtures/local/README.md +++ /dev/null @@ -1,27 +0,0 @@ -# Persisting 自有 fixture（`local/`） - -本目录 **不属于** [agentgateway](https://github.com/agentgateway/agentgateway) 上游测试集，由 Persisting 维护，用于 AG 未覆盖的场景（如 Codex 多轮、tool roundtrip、截断 SSE 片段）。 - -## 与 `fixtures/` 其余内容的关系 - -从 agentgateway 手动复制 `crates/agentgateway/src/llm/tests/` 时： - -- 只覆盖 `fixtures/` 下除 **`local/`** 以外的文件； -- **不要** 删除或覆盖 `local/` 中的内容。 - -## 当前文件 - -| 路径 | 用途 | -|------|------| -| `requests/responses/codex_basic.json` | Codex 风格 Responses 请求（permissions + 多轮 input） | -| `requests/responses/codex_tool_roundtrip.json` | Responses tool call / tool output 往返 | -| `response/completions/stream_head.txt` | OpenAI completions SSE 前缀（流式解析单测） | -| `response/completions/stream_tool_call.txt` | 含 tool call delta 的 SSE | -| `response/completions/tool_call.json` | 非流式 tool call completions 响应 | - -## 测试引用 - -- `src/conversion/responses_completions.rs` -- `src/conversion/stream.rs`、`responses_stream.rs` -- `src/storage/dialogue_extract/completions.rs` -- `tests/ag_fixture_tests.rs`（部分 capture case） diff --git a/crates/persisting-gateway/tests/fixtures/local/requests/responses/codex_basic.json b/crates/persisting-gateway/tests/fixtures/local/requests/responses/codex_basic.json deleted file mode 100644 index 458e74656..000000000 --- a/crates/persisting-gateway/tests/fixtures/local/requests/responses/codex_basic.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "model": "gpt-5.5", - "stream": true, - "include": ["reasoning.encrypted_content"], - "input": [ - { - "content": [ - { - "type": "input_text", - "text": "\nFilesystem sandboxing example\n" - }, - { - "type": "input_text", - "text": "hi" - } - ] - } - ], - "client_metadata": { - "x-codex-installation-id": "test-installation" - } -} diff --git a/crates/persisting-gateway/tests/fixtures/local/requests/responses/codex_tool_roundtrip.json b/crates/persisting-gateway/tests/fixtures/local/requests/responses/codex_tool_roundtrip.json deleted file mode 100644 index e08d69fcc..000000000 --- a/crates/persisting-gateway/tests/fixtures/local/requests/responses/codex_tool_roundtrip.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "model": "gpt-5.5", - "stream": true, - "tools": [ - { - "type": "function", - "name": "shell", - "description": "Run a shell command", - "parameters": { - "type": "object", - "properties": { - "command": { "type": "string" } - }, - "required": ["command"] - } - } - ], - "input": [ - { - "type": "message", - "role": "user", - "content": [{ "type": "input_text", "text": "list files" }] - }, - { - "type": "function_call", - "call_id": "call_abc123", - "name": "shell", - "arguments": "{\"command\":\"ls\"}" - }, - { - "type": "function_call_output", - "call_id": "call_abc123", - "output": "Cargo.toml\nsrc" - } - ] -} diff --git a/crates/persisting-gateway/tests/fixtures/local/response/completions/stream_head.txt b/crates/persisting-gateway/tests/fixtures/local/response/completions/stream_head.txt deleted file mode 100644 index bb7daad72..000000000 --- a/crates/persisting-gateway/tests/fixtures/local/response/completions/stream_head.txt +++ /dev/null @@ -1,26 +0,0 @@ -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"finish_reason":null}],"usage":null,"obfuscation":"Hxo8P"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"Hi"},"finish_reason":null}],"usage":null,"obfuscation":"9AL59"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" there"},"finish_reason":null}],"usage":null,"obfuscation":"J"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"!"},"finish_reason":null}],"usage":null,"obfuscation":"sXPZoY"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" How"},"finish_reason":null}],"usage":null,"obfuscation":"32m"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" can"},"finish_reason":null}],"usage":null,"obfuscation":"z6C"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" I"},"finish_reason":null}],"usage":null,"obfuscation":"5MelW"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" help"},"finish_reason":null}],"usage":null,"obfuscation":"Lu"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" today"},"finish_reason":null}],"usage":null,"obfuscation":"m"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"?"},"finish_reason":null}],"usage":null,"obfuscation":"ZS1MkT"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" If"},"finish_reason":null}],"usage":null,"obfuscation":"cHSc"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" you"},"finish_reason":null}],"usage":null,"obfuscation":"1eg"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"’re"},"finish_reason":null}],"usage":null,"obfuscation":"3DOB"} -data: [DONE] diff --git a/crates/persisting-gateway/tests/fixtures/local/response/completions/stream_tool_call.txt b/crates/persisting-gateway/tests/fixtures/local/response/completions/stream_tool_call.txt deleted file mode 100644 index a670c292b..000000000 --- a/crates/persisting-gateway/tests/fixtures/local/response/completions/stream_tool_call.txt +++ /dev/null @@ -1,11 +0,0 @@ -data: {"id":"chatcmpl-tool-stream","object":"chat.completion.chunk","created":1772656915,"model":"deepseek-chat","choices":[{"index":0,"delta":{"role":"assistant","content":null},"finish_reason":null}]} - -data: {"id":"chatcmpl-tool-stream","object":"chat.completion.chunk","created":1772656915,"model":"deepseek-chat","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"id":"call_abc123","type":"function","function":{"name":"shell","arguments":""}}]},"finish_reason":null}]} - -data: {"id":"chatcmpl-tool-stream","object":"chat.completion.chunk","created":1772656915,"model":"deepseek-chat","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"command\""}}]},"finish_reason":null}]} - -data: {"id":"chatcmpl-tool-stream","object":"chat.completion.chunk","created":1772656915,"model":"deepseek-chat","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":":\"ls\"}"}}]},"finish_reason":null}]} - -data: {"id":"chatcmpl-tool-stream","object":"chat.completion.chunk","created":1772656915,"model":"deepseek-chat","choices":[{"index":0,"delta":{},"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":10,"completion_tokens":5,"total_tokens":15}} - -data: [DONE] diff --git a/crates/persisting-gateway/tests/fixtures/local/response/completions/tool_call.json b/crates/persisting-gateway/tests/fixtures/local/response/completions/tool_call.json deleted file mode 100644 index 51f95fcc1..000000000 --- a/crates/persisting-gateway/tests/fixtures/local/response/completions/tool_call.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "id": "chatcmpl-tool-1", - "object": "chat.completion", - "created": 1772656915, - "model": "deepseek-chat", - "choices": [ - { - "index": 0, - "message": { - "role": "assistant", - "content": null, - "tool_calls": [ - { - "id": "call_abc123", - "type": "function", - "function": { - "name": "shell", - "arguments": "{\"command\":\"ls -la\"}" - } - } - ] - }, - "finish_reason": "tool_calls" - } - ], - "usage": { - "prompt_tokens": 42, - "completion_tokens": 18, - "total_tokens": 60 - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/completions/basic.anthropic.snap b/crates/persisting-gateway/tests/fixtures/requests/completions/basic.anthropic.snap deleted file mode 100644 index a91ffcde2..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/completions/basic.anthropic.snap +++ /dev/null @@ -1,47 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/completions/basic.json -info: - model: replaceme - messages: - - role: system - content: You are a helpful assistant. - - role: user - content: What is the name of the LLM provider? ---- -{ - "request": { - "messages": [ - { - "role": "user", - "content": [ - { - "type": "text", - "text": "What is the name of the LLM provider?" - } - ] - } - ], - "system": "You are a helpful assistant.", - "model": "replaceme", - "max_tokens": 4096 - }, - "parsed": { - "input_format": "Completions", - "cache_convention": "InputIncludesCache", - "request_model": "replaceme", - "provider": "anthropic", - "streaming": false, - "params": {}, - "prompt": [ - { - "role": "system", - "content": "You are a helpful assistant." - }, - { - "role": "user", - "content": "What is the name of the LLM provider?" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/completions/basic.bedrock.snap b/crates/persisting-gateway/tests/fixtures/requests/completions/basic.bedrock.snap deleted file mode 100644 index f39e5ffcb..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/completions/basic.bedrock.snap +++ /dev/null @@ -1,51 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/completions/basic.json -info: - model: replaceme - messages: - - role: system - content: You are a helpful assistant. - - role: user - content: What is the name of the LLM provider? ---- -{ - "request": { - "messages": [ - { - "role": "user", - "content": [ - { - "text": "What is the name of the LLM provider?" - } - ] - } - ], - "system": [ - { - "text": "You are a helpful assistant." - } - ], - "inferenceConfig": { - "maxTokens": 4096 - } - }, - "parsed": { - "input_format": "Completions", - "cache_convention": "InputIncludesCache", - "request_model": "replaceme", - "provider": "bedrock", - "streaming": false, - "params": {}, - "prompt": [ - { - "role": "system", - "content": "You are a helpful assistant." - }, - { - "role": "user", - "content": "What is the name of the LLM provider?" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/completions/basic.json b/crates/persisting-gateway/tests/fixtures/requests/completions/basic.json deleted file mode 100644 index fa0dd3c12..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/completions/basic.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "model": "replaceme", - "messages": [ - { - "role": "system", - "content": "You are a helpful assistant." - }, - { - "role": "user", - "content": "What is the name of the LLM provider?" - } - ] -} - diff --git a/crates/persisting-gateway/tests/fixtures/requests/completions/basic.vertex-gemini.snap b/crates/persisting-gateway/tests/fixtures/requests/completions/basic.vertex-gemini.snap deleted file mode 100644 index 511c3da16..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/completions/basic.vertex-gemini.snap +++ /dev/null @@ -1,50 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/completions/basic.json -info: - model: replaceme - messages: - - role: system - content: You are a helpful assistant. - - role: user - content: What is the name of the LLM provider? ---- -{ - "request": { - "contents": [ - { - "role": "user", - "parts": [ - { - "text": "What is the name of the LLM provider?" - } - ] - } - ], - "systemInstruction": { - "parts": [ - { - "text": "You are a helpful assistant." - } - ] - } - }, - "parsed": { - "input_format": "Completions", - "cache_convention": "InputIncludesCache", - "request_model": "replaceme", - "provider": "vertex", - "streaming": false, - "params": {}, - "prompt": [ - { - "role": "system", - "content": "You are a helpful assistant." - }, - { - "role": "user", - "content": "What is the name of the LLM provider?" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/completions/full.anthropic.snap b/crates/persisting-gateway/tests/fixtures/requests/completions/full.anthropic.snap deleted file mode 100644 index 8a3ce90b3..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/completions/full.anthropic.snap +++ /dev/null @@ -1,199 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/completions/full.json -info: - model: gpt-4-turbo-preview - messages: - - role: system - content: You are a helpful assistant that specializes in technical documentation and code analysis. - name: system_assistant - - role: user - content: "Please analyze this code snippet and explain what it does: function fibonacci(n) { return n <= 1 ? n : fibonacci(n-1) + fibonacci(n-2); }" - name: user_1 - - role: assistant - content: This is a recursive implementation of the Fibonacci sequence. It calculates the nth Fibonacci number by adding the previous two numbers. - name: assistant_1 - - role: user - content: - - type: text - text: "What's in this image?" - - type: image_url - image_url: - url: "data:image/png;base64,iVBORw0KGgo=" - max_tokens: 1000 - temperature: 0.7 - top_p: 0.9 - n: 3 - stream: false - stop: - - "\n\n" - - END - presence_penalty: 0.1 - frequency_penalty: 0.2 - logit_bias: - "50256": -100 - user: user_12345 - response_format: - type: json_object - seed: 42 - tool_choice: - type: function - function: - name: my_function - tools: - - type: function - function: - name: get_weather - description: Get the current weather in a given location - parameters: - type: object - properties: - location: - type: string - description: "The city and state, e.g. San Francisco, CA" - unit: - type: string - enum: - - celsius - - fahrenheit - required: - - location - functions: - - name: calculate_fibonacci - description: Calculate the nth Fibonacci number using an optimized algorithm - parameters: - type: object - properties: - n: - type: integer - description: The position in the Fibonacci sequence (0-indexed) - minimum: 0 - maximum: 100 - required: - - n - function_call: auto ---- -{ - "request": { - "messages": [ - { - "role": "user", - "content": [ - { - "type": "text", - "text": "Please analyze this code snippet and explain what it does: function fibonacci(n) { return n <= 1 ? n : fibonacci(n-1) + fibonacci(n-2); }" - } - ] - }, - { - "role": "assistant", - "content": [ - { - "type": "text", - "text": "This is a recursive implementation of the Fibonacci sequence. It calculates the nth Fibonacci number by adding the previous two numbers." - } - ] - }, - { - "role": "user", - "content": [ - { - "type": "text", - "text": "What's in this image?" - }, - { - "type": "image", - "source": { - "type": "base64", - "media_type": "image/png", - "data": "iVBORw0KGgo=" - } - } - ] - } - ], - "system": "You are a helpful assistant that specializes in technical documentation and code analysis.", - "model": "gpt-4-turbo-preview", - "max_tokens": 1000, - "stop_sequences": [ - "\n\n", - "END" - ], - "temperature": 0.7, - "top_p": 0.9, - "tools": [ - { - "name": "get_weather", - "description": "Get the current weather in a given location", - "input_schema": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA" - }, - "unit": { - "type": "string", - "enum": [ - "celsius", - "fahrenheit" - ] - } - }, - "required": [ - "location" - ] - } - } - ], - "tool_choice": { - "type": "tool", - "name": "my_function" - }, - "metadata": { - "user_id": "user_12345" - }, - "output_config": { - "format": { - "type": "json_schema", - "schema": { - "type": "object", - "additionalProperties": true - } - } - } - }, - "parsed": { - "input_format": "Completions", - "cache_convention": "InputIncludesCache", - "request_model": "gpt-4-turbo-preview", - "provider": "anthropic", - "streaming": false, - "params": { - "temperature": 0.699999988079071, - "top_p": 0.8999999761581421, - "frequency_penalty": 0.20000000298023224, - "presence_penalty": 0.10000000149011612, - "seed": 42, - "max_tokens": 1000 - }, - "prompt": [ - { - "role": "system", - "content": "You are a helpful assistant that specializes in technical documentation and code analysis." - }, - { - "role": "user", - "content": "Please analyze this code snippet and explain what it does: function fibonacci(n) { return n <= 1 ? n : fibonacci(n-1) + fibonacci(n-2); }" - }, - { - "role": "assistant", - "content": "This is a recursive implementation of the Fibonacci sequence. It calculates the nth Fibonacci number by adding the previous two numbers." - }, - { - "role": "user", - "content": "What's in this image?" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/completions/full.bedrock.snap b/crates/persisting-gateway/tests/fixtures/requests/completions/full.bedrock.snap deleted file mode 100644 index f1b962839..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/completions/full.bedrock.snap +++ /dev/null @@ -1,206 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/completions/full.json -info: - model: gpt-4-turbo-preview - messages: - - role: system - content: You are a helpful assistant that specializes in technical documentation and code analysis. - name: system_assistant - - role: user - content: "Please analyze this code snippet and explain what it does: function fibonacci(n) { return n <= 1 ? n : fibonacci(n-1) + fibonacci(n-2); }" - name: user_1 - - role: assistant - content: This is a recursive implementation of the Fibonacci sequence. It calculates the nth Fibonacci number by adding the previous two numbers. - name: assistant_1 - - role: user - content: - - type: text - text: "What's in this image?" - - type: image_url - image_url: - url: "data:image/png;base64,iVBORw0KGgo=" - max_tokens: 1000 - temperature: 0.7 - top_p: 0.9 - n: 3 - stream: false - stop: - - "\n\n" - - END - presence_penalty: 0.1 - frequency_penalty: 0.2 - logit_bias: - "50256": -100 - user: user_12345 - response_format: - type: json_object - seed: 42 - tool_choice: - type: function - function: - name: my_function - tools: - - type: function - function: - name: get_weather - description: Get the current weather in a given location - parameters: - type: object - properties: - location: - type: string - description: "The city and state, e.g. San Francisco, CA" - unit: - type: string - enum: - - celsius - - fahrenheit - required: - - location - functions: - - name: calculate_fibonacci - description: Calculate the nth Fibonacci number using an optimized algorithm - parameters: - type: object - properties: - n: - type: integer - description: The position in the Fibonacci sequence (0-indexed) - minimum: 0 - maximum: 100 - required: - - n - function_call: auto ---- -{ - "request": { - "messages": [ - { - "role": "user", - "content": [ - { - "text": "Please analyze this code snippet and explain what it does: function fibonacci(n) { return n <= 1 ? n : fibonacci(n-1) + fibonacci(n-2); }" - } - ] - }, - { - "role": "assistant", - "content": [ - { - "text": "This is a recursive implementation of the Fibonacci sequence. It calculates the nth Fibonacci number by adding the previous two numbers." - } - ] - }, - { - "role": "user", - "content": [ - { - "text": "What's in this image?" - }, - { - "image": { - "format": "png", - "source": { - "bytes": "iVBORw0KGgo=" - } - } - } - ] - } - ], - "system": [ - { - "text": "You are a helpful assistant that specializes in technical documentation and code analysis." - } - ], - "inferenceConfig": { - "maxTokens": 1000, - "temperature": 0.7, - "topP": 0.9, - "stopSequences": [ - "\n\n", - "END" - ] - }, - "outputConfig": { - "textFormat": { - "type": "json_schema", - "structure": { - "jsonSchema": { - "schema": "{\"type\":\"object\",\"additionalProperties\":true}" - } - } - } - }, - "toolConfig": { - "tools": [ - { - "toolSpec": { - "name": "get_weather", - "description": "Get the current weather in a given location", - "inputSchema": { - "json": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA" - }, - "unit": { - "type": "string", - "enum": [ - "celsius", - "fahrenheit" - ] - } - }, - "required": [ - "location" - ] - } - } - } - } - ], - "tool_choice": { - "tool": { - "name": "my_function" - } - } - } - }, - "parsed": { - "input_format": "Completions", - "cache_convention": "InputIncludesCache", - "request_model": "gpt-4-turbo-preview", - "provider": "bedrock", - "streaming": false, - "params": { - "temperature": 0.699999988079071, - "top_p": 0.8999999761581421, - "frequency_penalty": 0.20000000298023224, - "presence_penalty": 0.10000000149011612, - "seed": 42, - "max_tokens": 1000 - }, - "prompt": [ - { - "role": "system", - "content": "You are a helpful assistant that specializes in technical documentation and code analysis." - }, - { - "role": "user", - "content": "Please analyze this code snippet and explain what it does: function fibonacci(n) { return n <= 1 ? n : fibonacci(n-1) + fibonacci(n-2); }" - }, - { - "role": "assistant", - "content": "This is a recursive implementation of the Fibonacci sequence. It calculates the nth Fibonacci number by adding the previous two numbers." - }, - { - "role": "user", - "content": "What's in this image?" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/completions/full.get-messages-completions.snap b/crates/persisting-gateway/tests/fixtures/requests/completions/full.get-messages-completions.snap deleted file mode 100644 index 7a63eddec..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/completions/full.get-messages-completions.snap +++ /dev/null @@ -1,93 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/completions/full.json -info: - model: gpt-4-turbo-preview - messages: - - role: system - content: You are a helpful assistant that specializes in technical documentation and code analysis. - name: system_assistant - - role: user - content: "Please analyze this code snippet and explain what it does: function fibonacci(n) { return n <= 1 ? n : fibonacci(n-1) + fibonacci(n-2); }" - name: user_1 - - role: assistant - content: This is a recursive implementation of the Fibonacci sequence. It calculates the nth Fibonacci number by adding the previous two numbers. - name: assistant_1 - - role: user - content: - - type: text - text: "What's in this image?" - - type: image_url - image_url: - url: "https://upload.wikimedia.org/wikipedia/commons/4/47/PNG_transparency_demonstration_1.png" - max_tokens: 1000 - temperature: 0.7 - top_p: 0.9 - n: 3 - stream: false - stop: - - "\n\n" - - END - presence_penalty: 0.1 - frequency_penalty: 0.2 - logit_bias: - "50256": -100 - user: user_12345 - response_format: - type: json_object - seed: 42 - tool_choice: - type: function - function: - name: my_function - tools: - - type: function - function: - name: get_weather - description: Get the current weather in a given location - parameters: - type: object - properties: - location: - type: string - description: "The city and state, e.g. San Francisco, CA" - unit: - type: string - enum: - - celsius - - fahrenheit - required: - - location - functions: - - name: calculate_fibonacci - description: Calculate the nth Fibonacci number using an optimized algorithm - parameters: - type: object - properties: - n: - type: integer - description: The position in the Fibonacci sequence (0-indexed) - minimum: 0 - maximum: 100 - required: - - n - function_call: auto ---- -[ - { - "role": "system", - "content": "You are a helpful assistant that specializes in technical documentation and code analysis." - }, - { - "role": "user", - "content": "Please analyze this code snippet and explain what it does: function fibonacci(n) { return n <= 1 ? n : fibonacci(n-1) + fibonacci(n-2); }" - }, - { - "role": "assistant", - "content": "This is a recursive implementation of the Fibonacci sequence. It calculates the nth Fibonacci number by adding the previous two numbers." - }, - { - "role": "user", - "content": "What's in this image?" - } -] diff --git a/crates/persisting-gateway/tests/fixtures/requests/completions/full.get-messages-messages.snap b/crates/persisting-gateway/tests/fixtures/requests/completions/full.get-messages-messages.snap deleted file mode 100644 index 7a63eddec..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/completions/full.get-messages-messages.snap +++ /dev/null @@ -1,93 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/completions/full.json -info: - model: gpt-4-turbo-preview - messages: - - role: system - content: You are a helpful assistant that specializes in technical documentation and code analysis. - name: system_assistant - - role: user - content: "Please analyze this code snippet and explain what it does: function fibonacci(n) { return n <= 1 ? n : fibonacci(n-1) + fibonacci(n-2); }" - name: user_1 - - role: assistant - content: This is a recursive implementation of the Fibonacci sequence. It calculates the nth Fibonacci number by adding the previous two numbers. - name: assistant_1 - - role: user - content: - - type: text - text: "What's in this image?" - - type: image_url - image_url: - url: "https://upload.wikimedia.org/wikipedia/commons/4/47/PNG_transparency_demonstration_1.png" - max_tokens: 1000 - temperature: 0.7 - top_p: 0.9 - n: 3 - stream: false - stop: - - "\n\n" - - END - presence_penalty: 0.1 - frequency_penalty: 0.2 - logit_bias: - "50256": -100 - user: user_12345 - response_format: - type: json_object - seed: 42 - tool_choice: - type: function - function: - name: my_function - tools: - - type: function - function: - name: get_weather - description: Get the current weather in a given location - parameters: - type: object - properties: - location: - type: string - description: "The city and state, e.g. San Francisco, CA" - unit: - type: string - enum: - - celsius - - fahrenheit - required: - - location - functions: - - name: calculate_fibonacci - description: Calculate the nth Fibonacci number using an optimized algorithm - parameters: - type: object - properties: - n: - type: integer - description: The position in the Fibonacci sequence (0-indexed) - minimum: 0 - maximum: 100 - required: - - n - function_call: auto ---- -[ - { - "role": "system", - "content": "You are a helpful assistant that specializes in technical documentation and code analysis." - }, - { - "role": "user", - "content": "Please analyze this code snippet and explain what it does: function fibonacci(n) { return n <= 1 ? n : fibonacci(n-1) + fibonacci(n-2); }" - }, - { - "role": "assistant", - "content": "This is a recursive implementation of the Fibonacci sequence. It calculates the nth Fibonacci number by adding the previous two numbers." - }, - { - "role": "user", - "content": "What's in this image?" - } -] diff --git a/crates/persisting-gateway/tests/fixtures/requests/completions/full.json b/crates/persisting-gateway/tests/fixtures/requests/completions/full.json deleted file mode 100644 index 0b3b06848..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/completions/full.json +++ /dev/null @@ -1,91 +0,0 @@ -{ - "model": "gpt-4-turbo-preview", - "messages": [ - { - "role": "system", - "content": "You are a helpful assistant that specializes in technical documentation and code analysis.", - "name": "system_assistant" - }, - { - "role": "user", - "content": "Please analyze this code snippet and explain what it does: function fibonacci(n) { return n <= 1 ? n : fibonacci(n-1) + fibonacci(n-2); }", - "name": "user_1" - }, - { - "role": "assistant", - "content": "This is a recursive implementation of the Fibonacci sequence. It calculates the nth Fibonacci number by adding the previous two numbers.", - "name": "assistant_1" - }, - { - "role": "user", - "content": [ - { "type": "text", "text": "What's in this image?" }, - { - "type": "image_url", - "image_url": { - "url": "data:image/png;base64,iVBORw0KGgo=" - } - } - ] - } - ], - "max_tokens": 1000, - "temperature": 0.7, - "top_p": 0.9, - "n": 3, - "stream": false, - "stop": ["\n\n", "END"], - "presence_penalty": 0.1, - "frequency_penalty": 0.2, - "logit_bias": { - "50256": -100 - }, - "user": "user_12345", - "response_format": { - "type": "json_object" - }, - "seed": 42, - "tool_choice": {"type": "function", "function": {"name": "my_function"}}, - "tools": [ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get the current weather in a given location", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA" - }, - "unit": { - "type": "string", - "enum": ["celsius", "fahrenheit"] - } - }, - "required": ["location"] - } - } - } - ], - "functions": [ - { - "name": "calculate_fibonacci", - "description": "Calculate the nth Fibonacci number using an optimized algorithm", - "parameters": { - "type": "object", - "properties": { - "n": { - "type": "integer", - "description": "The position in the Fibonacci sequence (0-indexed)", - "minimum": 0, - "maximum": 100 - } - }, - "required": ["n"] - } - } - ], - "function_call": "auto" -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/completions/generation-config.json b/crates/persisting-gateway/tests/fixtures/requests/completions/generation-config.json deleted file mode 100644 index 4992a7cfb..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/completions/generation-config.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "model": "gemini-2.5-pro", - "messages": [ - { "role": "system", "content": "You are a helpful assistant." }, - { "role": "user", "content": "What is the weather?" } - ], - "max_tokens": 1000, - "temperature": 0.7, - "top_p": 0.9, - "top_k": 40, - "n": 3, - "stop": ["\n\n", "END"], - "presence_penalty": 0.1, - "frequency_penalty": 0.2, - "seed": 42, - "response_format": { "type": "json_object" }, - "tool_choice": { "type": "function", "function": { "name": "get_weather" } }, - "tools": [ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get the current weather in a location", - "parameters": { - "type": "object", - "properties": { "location": { "type": "string" } }, - "required": ["location"] - } - } - } - ] -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/completions/generation-config.vertex-gemini.snap b/crates/persisting-gateway/tests/fixtures/requests/completions/generation-config.vertex-gemini.snap deleted file mode 100644 index de81204ee..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/completions/generation-config.vertex-gemini.snap +++ /dev/null @@ -1,130 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/completions/generation-config.json -info: - model: gemini-2.5-pro - messages: - - role: system - content: You are a helpful assistant. - - role: user - content: What is the weather? - max_tokens: 1000 - temperature: 0.7 - top_p: 0.9 - top_k: 40 - n: 3 - stop: - - "\n\n" - - END - presence_penalty: 0.1 - frequency_penalty: 0.2 - seed: 42 - response_format: - type: json_object - tool_choice: - type: function - function: - name: get_weather - tools: - - type: function - function: - name: get_weather - description: Get the current weather in a location - parameters: - type: object - properties: - location: - type: string - required: - - location ---- -{ - "request": { - "contents": [ - { - "role": "user", - "parts": [ - { - "text": "What is the weather?" - } - ] - } - ], - "systemInstruction": { - "parts": [ - { - "text": "You are a helpful assistant." - } - ] - }, - "tools": [ - { - "functionDeclarations": [ - { - "name": "get_weather", - "description": "Get the current weather in a location", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string" - } - }, - "required": [ - "location" - ] - } - } - ] - } - ], - "toolConfig": { - "functionCallingConfig": { - "mode": "ANY", - "allowedFunctionNames": [ - "get_weather" - ] - } - }, - "generationConfig": { - "temperature": 0.7, - "topP": 0.9, - "topK": 40, - "frequencyPenalty": 0.2, - "presencePenalty": 0.1, - "maxOutputTokens": 1000, - "stopSequences": [ - "\n\n", - "END" - ], - "candidateCount": 3, - "seed": 42, - "responseMimeType": "application/json" - } - }, - "parsed": { - "input_format": "Completions", - "cache_convention": "InputIncludesCache", - "request_model": "gemini-2.5-pro", - "provider": "vertex", - "streaming": false, - "params": { - "temperature": 0.699999988079071, - "top_p": 0.8999999761581421, - "frequency_penalty": 0.20000000298023224, - "presence_penalty": 0.10000000149011612, - "seed": 42, - "max_tokens": 1000 - }, - "prompt": [ - { - "role": "system", - "content": "You are a helpful assistant." - }, - { - "role": "user", - "content": "What is the weather?" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/completions/image-file.json b/crates/persisting-gateway/tests/fixtures/requests/completions/image-file.json deleted file mode 100644 index 9e4f4b187..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/completions/image-file.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "model": "gemini-2.5-pro", - "messages": [ - { - "role": "user", - "content": [ - { "type": "text", "text": "Describe this image." }, - { - "type": "image_url", - "image_url": { - "url": "gs://my-bucket/photo.png" - } - } - ] - } - ] -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/completions/image-file.vertex-gemini.snap b/crates/persisting-gateway/tests/fixtures/requests/completions/image-file.vertex-gemini.snap deleted file mode 100644 index ea2996322..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/completions/image-file.vertex-gemini.snap +++ /dev/null @@ -1,48 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/completions/image-file.json -info: - model: gemini-2.5-pro - messages: - - role: user - content: - - type: text - text: Describe this image. - - type: image_url - image_url: - url: "gs://my-bucket/photo.png" ---- -{ - "request": { - "contents": [ - { - "role": "user", - "parts": [ - { - "text": "Describe this image." - }, - { - "fileData": { - "mimeType": "image/png", - "fileUri": "gs://my-bucket/photo.png" - } - } - ] - } - ] - }, - "parsed": { - "input_format": "Completions", - "cache_convention": "InputIncludesCache", - "request_model": "gemini-2.5-pro", - "provider": "vertex", - "streaming": false, - "params": {}, - "prompt": [ - { - "role": "user", - "content": "Describe this image." - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/completions/image-inline.json b/crates/persisting-gateway/tests/fixtures/requests/completions/image-inline.json deleted file mode 100644 index b64d2c40d..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/completions/image-inline.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "model": "gemini-2.5-pro", - "messages": [ - { - "role": "user", - "content": [ - { "type": "text", "text": "What's in this image?" }, - { - "type": "image_url", - "image_url": { - "url": "data:image/jpg;base64,/9j/4AAQSkZJRgABAQ==" - } - } - ] - } - ] -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/completions/image-inline.vertex-gemini.snap b/crates/persisting-gateway/tests/fixtures/requests/completions/image-inline.vertex-gemini.snap deleted file mode 100644 index 3cabd5fa2..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/completions/image-inline.vertex-gemini.snap +++ /dev/null @@ -1,48 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/completions/image-inline.json -info: - model: gemini-2.5-pro - messages: - - role: user - content: - - type: text - text: "What's in this image?" - - type: image_url - image_url: - url: "data:image/jpg;base64,/9j/4AAQSkZJRgABAQ==" ---- -{ - "request": { - "contents": [ - { - "role": "user", - "parts": [ - { - "text": "What's in this image?" - }, - { - "inlineData": { - "mimeType": "image/jpeg", - "data": "/9j/4AAQSkZJRgABAQ==" - } - } - ] - } - ] - }, - "parsed": { - "input_format": "Completions", - "cache_convention": "InputIncludesCache", - "request_model": "gemini-2.5-pro", - "provider": "vertex", - "streaming": false, - "params": {}, - "prompt": [ - { - "role": "user", - "content": "What's in this image?" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/completions/image-url.anthropic.snap b/crates/persisting-gateway/tests/fixtures/requests/completions/image-url.anthropic.snap deleted file mode 100644 index 3238a6c85..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/completions/image-url.anthropic.snap +++ /dev/null @@ -1,52 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/completions/image-url.json -info: - model: claude-sonnet-4-20250514 - messages: - - role: user - content: - - type: text - text: "What's in this image?" - - type: image_url - image_url: - url: "https://example.com/image.png" ---- -{ - "request": { - "messages": [ - { - "role": "user", - "content": [ - { - "type": "text", - "text": "What's in this image?" - }, - { - "type": "image", - "source": { - "type": "url", - "url": "https://example.com/image.png" - } - } - ] - } - ], - "model": "claude-sonnet-4-20250514", - "max_tokens": 4096 - }, - "parsed": { - "input_format": "Completions", - "cache_convention": "InputIncludesCache", - "request_model": "claude-sonnet-4-20250514", - "provider": "anthropic", - "streaming": false, - "params": {}, - "prompt": [ - { - "role": "user", - "content": "What's in this image?" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/completions/image-url.json b/crates/persisting-gateway/tests/fixtures/requests/completions/image-url.json deleted file mode 100644 index e9839bb65..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/completions/image-url.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "model": "claude-sonnet-4-20250514", - "messages": [ - { - "role": "user", - "content": [ - { "type": "text", "text": "What's in this image?" }, - { - "type": "image_url", - "image_url": { - "url": "https://example.com/image.png" - } - } - ] - } - ] -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/completions/multi-turn-tools.json b/crates/persisting-gateway/tests/fixtures/requests/completions/multi-turn-tools.json deleted file mode 100644 index 83123edbc..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/completions/multi-turn-tools.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "model": "gemini-2.5-pro", - "messages": [ - { "role": "user", "content": "Weather in Columbus and Berlin?" }, - { - "role": "assistant", - "content": null, - "tool_calls": [ - { - "id": "call_1", - "type": "function", - "function": { "name": "get_weather", "arguments": "{\"location\":\"Columbus\"}" } - } - ] - }, - { "role": "tool", "tool_call_id": "call_1", "name": "get_weather", "content": "{\"temp\":15}" }, - { - "role": "assistant", - "content": null, - "tool_calls": [ - { - "id": "call_2", - "type": "function", - "function": { "name": "get_weather", "arguments": "{\"location\":\"Berlin\"}" } - } - ] - }, - { "role": "tool", "tool_call_id": "call_2", "name": "get_weather", "content": "{\"temp\":9}" } - ], - "tools": [ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get the current weather in a location", - "parameters": { - "type": "object", - "properties": { "location": { "type": "string" } }, - "required": ["location"] - } - } - } - ] -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/completions/multi-turn-tools.vertex-gemini.snap b/crates/persisting-gateway/tests/fixtures/requests/completions/multi-turn-tools.vertex-gemini.snap deleted file mode 100644 index b9c6f457f..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/completions/multi-turn-tools.vertex-gemini.snap +++ /dev/null @@ -1,162 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/completions/multi-turn-tools.json -info: - model: gemini-2.5-pro - messages: - - role: user - content: Weather in Columbus and Berlin? - - role: assistant - content: ~ - tool_calls: - - id: call_1 - type: function - function: - name: get_weather - arguments: "{\"location\":\"Columbus\"}" - - role: tool - tool_call_id: call_1 - name: get_weather - content: "{\"temp\":15}" - - role: assistant - content: ~ - tool_calls: - - id: call_2 - type: function - function: - name: get_weather - arguments: "{\"location\":\"Berlin\"}" - - role: tool - tool_call_id: call_2 - name: get_weather - content: "{\"temp\":9}" - tools: - - type: function - function: - name: get_weather - description: Get the current weather in a location - parameters: - type: object - properties: - location: - type: string - required: - - location ---- -{ - "request": { - "contents": [ - { - "role": "user", - "parts": [ - { - "text": "Weather in Columbus and Berlin?" - } - ] - }, - { - "role": "model", - "parts": [ - { - "functionCall": { - "name": "get_weather", - "args": { - "location": "Columbus" - } - } - } - ] - }, - { - "role": "user", - "parts": [ - { - "functionResponse": { - "name": "get_weather", - "response": { - "content": "{\"temp\":15}" - } - } - } - ] - }, - { - "role": "model", - "parts": [ - { - "functionCall": { - "name": "get_weather", - "args": { - "location": "Berlin" - } - } - } - ] - }, - { - "role": "user", - "parts": [ - { - "functionResponse": { - "name": "get_weather", - "response": { - "content": "{\"temp\":9}" - } - } - } - ] - } - ], - "tools": [ - { - "functionDeclarations": [ - { - "name": "get_weather", - "description": "Get the current weather in a location", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string" - } - }, - "required": [ - "location" - ] - } - } - ] - } - ] - }, - "parsed": { - "input_format": "Completions", - "cache_convention": "InputIncludesCache", - "request_model": "gemini-2.5-pro", - "provider": "vertex", - "streaming": false, - "params": {}, - "prompt": [ - { - "role": "user", - "content": "Weather in Columbus and Berlin?" - }, - { - "role": "assistant", - "content": "" - }, - { - "role": "tool", - "content": "{\"temp\":15}" - }, - { - "role": "assistant", - "content": "" - }, - { - "role": "tool", - "content": "{\"temp\":9}" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/completions/parallel-tool-call.bedrock.snap b/crates/persisting-gateway/tests/fixtures/requests/completions/parallel-tool-call.bedrock.snap deleted file mode 100644 index 950633093..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/completions/parallel-tool-call.bedrock.snap +++ /dev/null @@ -1,136 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/completions/parallel-tool-call.json -info: - model: gpt-3.5-turbo - messages: - - role: user - content: How is the weather in Columbus and New York? - - role: assistant - content: ~ - tool_calls: - - id: call_1 - type: function - function: - name: get_weather - arguments: "{\"location\":\"Columbus, OH\"}" - - id: call_2 - type: function - function: - name: get_weather - arguments: "{\"location\":\"New York, NY\"}" - - role: tool - tool_call_id: call_1 - content: "{ \"temperature\": 15, \"condition\": \"Cloudy\" }" - - role: tool - tool_call_id: call_2 - content: "{ \"temperature\": 22, \"condition\": \"Sunny\" }" ---- -{ - "request": { - "messages": [ - { - "role": "user", - "content": [ - { - "text": "How is the weather in Columbus and New York?" - } - ] - }, - { - "role": "assistant", - "content": [ - { - "toolUse": { - "toolUseId": "call_1", - "name": "get_weather", - "input": { - "location": "Columbus, OH" - } - } - }, - { - "toolUse": { - "toolUseId": "call_2", - "name": "get_weather", - "input": { - "location": "New York, NY" - } - } - } - ] - }, - { - "role": "user", - "content": [ - { - "toolResult": { - "toolUseId": "call_1", - "content": [ - { - "text": "{ \"temperature\": 15, \"condition\": \"Cloudy\" }" - } - ], - "status": null - } - }, - { - "toolResult": { - "toolUseId": "call_2", - "content": [ - { - "text": "{ \"temperature\": 22, \"condition\": \"Sunny\" }" - } - ], - "status": null - } - } - ] - } - ], - "inferenceConfig": { - "maxTokens": 4096 - }, - "toolConfig": { - "tools": [ - { - "toolSpec": { - "name": "agentgateway_dummy_do_not_call", - "description": null, - "inputSchema": { - "json": { - "type": "object" - } - } - } - } - ] - } - }, - "parsed": { - "input_format": "Completions", - "cache_convention": "InputIncludesCache", - "request_model": "gpt-3.5-turbo", - "provider": "bedrock", - "streaming": false, - "params": {}, - "prompt": [ - { - "role": "user", - "content": "How is the weather in Columbus and New York?" - }, - { - "role": "assistant", - "content": "" - }, - { - "role": "tool", - "content": "{ \"temperature\": 15, \"condition\": \"Cloudy\" }" - }, - { - "role": "tool", - "content": "{ \"temperature\": 22, \"condition\": \"Sunny\" }" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/completions/parallel-tool-call.json b/crates/persisting-gateway/tests/fixtures/requests/completions/parallel-tool-call.json deleted file mode 100644 index 4a22de28d..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/completions/parallel-tool-call.json +++ /dev/null @@ -1,41 +0,0 @@ -{ - "model": "gpt-3.5-turbo", - "messages": [ - { - "role": "user", - "content": "How is the weather in Columbus and New York?" - }, - { - "role": "assistant", - "content": null, - "tool_calls": [ - { - "id": "call_1", - "type": "function", - "function": { - "name": "get_weather", - "arguments": "{\"location\":\"Columbus, OH\"}" - } - }, - { - "id": "call_2", - "type": "function", - "function": { - "name": "get_weather", - "arguments": "{\"location\":\"New York, NY\"}" - } - } - ] - }, - { - "role": "tool", - "tool_call_id": "call_1", - "content": "{ \"temperature\": 15, \"condition\": \"Cloudy\" }" - }, - { - "role": "tool", - "tool_call_id": "call_2", - "content": "{ \"temperature\": 22, \"condition\": \"Sunny\" }" - } - ] -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/completions/parallel-tool-call.vertex-gemini.snap b/crates/persisting-gateway/tests/fixtures/requests/completions/parallel-tool-call.vertex-gemini.snap deleted file mode 100644 index 7e397f83f..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/completions/parallel-tool-call.vertex-gemini.snap +++ /dev/null @@ -1,110 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/completions/parallel-tool-call.json -info: - model: gpt-3.5-turbo - messages: - - role: user - content: How is the weather in Columbus and New York? - - role: assistant - content: ~ - tool_calls: - - id: call_1 - type: function - function: - name: get_weather - arguments: "{\"location\":\"Columbus, OH\"}" - - id: call_2 - type: function - function: - name: get_weather - arguments: "{\"location\":\"New York, NY\"}" - - role: tool - tool_call_id: call_1 - content: "{ \"temperature\": 15, \"condition\": \"Cloudy\" }" - - role: tool - tool_call_id: call_2 - content: "{ \"temperature\": 22, \"condition\": \"Sunny\" }" ---- -{ - "request": { - "contents": [ - { - "role": "user", - "parts": [ - { - "text": "How is the weather in Columbus and New York?" - } - ] - }, - { - "role": "model", - "parts": [ - { - "functionCall": { - "name": "get_weather", - "args": { - "location": "Columbus, OH" - } - } - }, - { - "functionCall": { - "name": "get_weather", - "args": { - "location": "New York, NY" - } - } - } - ] - }, - { - "role": "user", - "parts": [ - { - "functionResponse": { - "name": "get_weather", - "response": { - "content": "{ \"temperature\": 15, \"condition\": \"Cloudy\" }" - } - } - }, - { - "functionResponse": { - "name": "get_weather", - "response": { - "content": "{ \"temperature\": 22, \"condition\": \"Sunny\" }" - } - } - } - ] - } - ] - }, - "parsed": { - "input_format": "Completions", - "cache_convention": "InputIncludesCache", - "request_model": "gpt-3.5-turbo", - "provider": "vertex", - "streaming": false, - "params": {}, - "prompt": [ - { - "role": "user", - "content": "How is the weather in Columbus and New York?" - }, - { - "role": "assistant", - "content": "" - }, - { - "role": "tool", - "content": "{ \"temperature\": 15, \"condition\": \"Cloudy\" }" - }, - { - "role": "tool", - "content": "{ \"temperature\": 22, \"condition\": \"Sunny\" }" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/completions/reasoning.anthropic.snap b/crates/persisting-gateway/tests/fixtures/requests/completions/reasoning.anthropic.snap deleted file mode 100644 index 77c938a8e..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/completions/reasoning.anthropic.snap +++ /dev/null @@ -1,45 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/completions/reasoning.json -info: - model: gpt-3.5-turbo - messages: - - role: user - content: How is the current weather in Columbus? - reasoning_effort: medium ---- -{ - "request": { - "messages": [ - { - "role": "user", - "content": [ - { - "type": "text", - "text": "How is the current weather in Columbus?" - } - ] - } - ], - "model": "gpt-3.5-turbo", - "max_tokens": 4096, - "thinking": { - "type": "enabled", - "budget_tokens": 2048 - } - }, - "parsed": { - "input_format": "Completions", - "cache_convention": "InputIncludesCache", - "request_model": "gpt-3.5-turbo", - "provider": "anthropic", - "streaming": false, - "params": {}, - "prompt": [ - { - "role": "user", - "content": "How is the current weather in Columbus?" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/completions/reasoning.bedrock.snap b/crates/persisting-gateway/tests/fixtures/requests/completions/reasoning.bedrock.snap deleted file mode 100644 index 3ae10c06e..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/completions/reasoning.bedrock.snap +++ /dev/null @@ -1,47 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/completions/reasoning.json -info: - model: gpt-3.5-turbo - messages: - - role: user - content: How is the current weather in Columbus? - reasoning_effort: medium ---- -{ - "request": { - "messages": [ - { - "role": "user", - "content": [ - { - "text": "How is the current weather in Columbus?" - } - ] - } - ], - "inferenceConfig": { - "maxTokens": 4096 - }, - "additionalModelRequestFields": { - "thinking": { - "type": "enabled", - "budget_tokens": 2048 - } - } - }, - "parsed": { - "input_format": "Completions", - "cache_convention": "InputIncludesCache", - "request_model": "gpt-3.5-turbo", - "provider": "bedrock", - "streaming": false, - "params": {}, - "prompt": [ - { - "role": "user", - "content": "How is the current weather in Columbus?" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/completions/reasoning.json b/crates/persisting-gateway/tests/fixtures/requests/completions/reasoning.json deleted file mode 100644 index bdbb1c15f..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/completions/reasoning.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "model": "gpt-3.5-turbo", - "messages": [ - { - "role": "user", - "content": "How is the current weather in Columbus?" - } - ], - "reasoning_effort": "medium" -} \ No newline at end of file diff --git a/crates/persisting-gateway/tests/fixtures/requests/completions/reasoning.vertex-gemini.snap b/crates/persisting-gateway/tests/fixtures/requests/completions/reasoning.vertex-gemini.snap deleted file mode 100644 index 6afc346c0..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/completions/reasoning.vertex-gemini.snap +++ /dev/null @@ -1,44 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/completions/reasoning.json -info: - model: gpt-3.5-turbo - messages: - - role: user - content: How is the current weather in Columbus? - reasoning_effort: medium ---- -{ - "request": { - "contents": [ - { - "role": "user", - "parts": [ - { - "text": "How is the current weather in Columbus?" - } - ] - } - ], - "generationConfig": { - "thinkingConfig": { - "thinkingBudget": 2048, - "includeThoughts": true - } - } - }, - "parsed": { - "input_format": "Completions", - "cache_convention": "InputIncludesCache", - "request_model": "gpt-3.5-turbo", - "provider": "vertex", - "streaming": false, - "params": {}, - "prompt": [ - { - "role": "user", - "content": "How is the current weather in Columbus?" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/completions/reasoning_max.anthropic.snap b/crates/persisting-gateway/tests/fixtures/requests/completions/reasoning_max.anthropic.snap deleted file mode 100644 index 999237fae..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/completions/reasoning_max.anthropic.snap +++ /dev/null @@ -1,45 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/completions/reasoning_max.json -info: - model: claude-opus-4-6 - messages: - - role: user - content: Deeply analyze quantum gravity - reasoning_effort: xhigh ---- -{ - "request": { - "messages": [ - { - "role": "user", - "content": [ - { - "type": "text", - "text": "Deeply analyze quantum gravity" - } - ] - } - ], - "model": "claude-opus-4-6", - "max_tokens": 4096, - "thinking": { - "type": "enabled", - "budget_tokens": 4095 - } - }, - "parsed": { - "input_format": "Completions", - "cache_convention": "InputIncludesCache", - "request_model": "claude-opus-4-6", - "provider": "anthropic", - "streaming": false, - "params": {}, - "prompt": [ - { - "role": "user", - "content": "Deeply analyze quantum gravity" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/completions/reasoning_max.json b/crates/persisting-gateway/tests/fixtures/requests/completions/reasoning_max.json deleted file mode 100644 index bbae635e6..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/completions/reasoning_max.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "model": "claude-opus-4-6", - "messages": [ - { - "role": "user", - "content": "Deeply analyze quantum gravity" - } - ], - "reasoning_effort": "xhigh" -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/completions/reasoning_max.vertex-gemini.snap b/crates/persisting-gateway/tests/fixtures/requests/completions/reasoning_max.vertex-gemini.snap deleted file mode 100644 index 2df84b1a7..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/completions/reasoning_max.vertex-gemini.snap +++ /dev/null @@ -1,44 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/completions/reasoning_max.json -info: - model: claude-opus-4-6 - messages: - - role: user - content: Deeply analyze quantum gravity - reasoning_effort: xhigh ---- -{ - "request": { - "contents": [ - { - "role": "user", - "parts": [ - { - "text": "Deeply analyze quantum gravity" - } - ] - } - ], - "generationConfig": { - "thinkingConfig": { - "thinkingBudget": 8192, - "includeThoughts": true - } - } - }, - "parsed": { - "input_format": "Completions", - "cache_convention": "InputIncludesCache", - "request_model": "claude-opus-4-6", - "provider": "vertex", - "streaming": false, - "params": {}, - "prompt": [ - { - "role": "user", - "content": "Deeply analyze quantum gravity" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/completions/reasoning_replay.bedrock.snap b/crates/persisting-gateway/tests/fixtures/requests/completions/reasoning_replay.bedrock.snap deleted file mode 100644 index d6edbd254..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/completions/reasoning_replay.bedrock.snap +++ /dev/null @@ -1,78 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/completions/reasoning_replay.json -info: - model: "anthropic.claude-3-7-sonnet-20250219-v1:0" - messages: - - role: user - content: What is 2+2? - - role: assistant - content: "4" - reasoning_content: Two plus two. - reasoning_signature: SIG-replay== - - role: user - content: And 3+3? ---- -{ - "request": { - "messages": [ - { - "role": "user", - "content": [ - { - "text": "What is 2+2?" - } - ] - }, - { - "role": "assistant", - "content": [ - { - "reasoningContent": { - "reasoningText": { - "text": "Two plus two.", - "signature": "SIG-replay==" - } - } - }, - { - "text": "4" - } - ] - }, - { - "role": "user", - "content": [ - { - "text": "And 3+3?" - } - ] - } - ], - "inferenceConfig": { - "maxTokens": 4096 - } - }, - "parsed": { - "input_format": "Completions", - "cache_convention": "InputIncludesCache", - "request_model": "anthropic.claude-3-7-sonnet-20250219-v1:0", - "provider": "bedrock", - "streaming": false, - "params": {}, - "prompt": [ - { - "role": "user", - "content": "What is 2+2?" - }, - { - "role": "assistant", - "content": "4" - }, - { - "role": "user", - "content": "And 3+3?" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/completions/reasoning_replay.json b/crates/persisting-gateway/tests/fixtures/requests/completions/reasoning_replay.json deleted file mode 100644 index 246c0d233..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/completions/reasoning_replay.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "model": "anthropic.claude-3-7-sonnet-20250219-v1:0", - "messages": [ - { "role": "user", "content": "What is 2+2?" }, - { - "role": "assistant", - "content": "4", - "reasoning_content": "Two plus two.", - "reasoning_signature": "SIG-replay==" - }, - { "role": "user", "content": "And 3+3?" } - ] -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/completions/reasoning_replay_unsigned.bedrock.snap b/crates/persisting-gateway/tests/fixtures/requests/completions/reasoning_replay_unsigned.bedrock.snap deleted file mode 100644 index 2b107ab97..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/completions/reasoning_replay_unsigned.bedrock.snap +++ /dev/null @@ -1,69 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/completions/reasoning_replay_unsigned.json -info: - model: "anthropic.claude-3-7-sonnet-20250219-v1:0" - messages: - - role: user - content: Hi - - role: assistant - content: Hello - reasoning_content: no signature here - - role: user - content: Continue ---- -{ - "request": { - "messages": [ - { - "role": "user", - "content": [ - { - "text": "Hi" - } - ] - }, - { - "role": "assistant", - "content": [ - { - "text": "Hello" - } - ] - }, - { - "role": "user", - "content": [ - { - "text": "Continue" - } - ] - } - ], - "inferenceConfig": { - "maxTokens": 4096 - } - }, - "parsed": { - "input_format": "Completions", - "cache_convention": "InputIncludesCache", - "request_model": "anthropic.claude-3-7-sonnet-20250219-v1:0", - "provider": "bedrock", - "streaming": false, - "params": {}, - "prompt": [ - { - "role": "user", - "content": "Hi" - }, - { - "role": "assistant", - "content": "Hello" - }, - { - "role": "user", - "content": "Continue" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/completions/reasoning_replay_unsigned.json b/crates/persisting-gateway/tests/fixtures/requests/completions/reasoning_replay_unsigned.json deleted file mode 100644 index 5ed6d7781..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/completions/reasoning_replay_unsigned.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "model": "anthropic.claude-3-7-sonnet-20250219-v1:0", - "messages": [ - { "role": "user", "content": "Hi" }, - { - "role": "assistant", - "content": "Hello", - "reasoning_content": "no signature here" - }, - { "role": "user", "content": "Continue" } - ] -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/completions/structured-output.json b/crates/persisting-gateway/tests/fixtures/requests/completions/structured-output.json deleted file mode 100644 index 153499ad9..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/completions/structured-output.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "model": "gemini-2.5-pro", - "messages": [ - { "role": "user", "content": "Extract the person from: Ada Lovelace, age 36." } - ], - "response_format": { - "type": "json_schema", - "json_schema": { - "name": "person", - "schema": { - "type": "object", - "properties": { - "person": { "$ref": "#/$defs/Person" } - }, - "required": ["person"], - "$defs": { - "Person": { - "type": "object", - "properties": { - "name": { "type": "string" }, - "age": { "type": "integer" } - }, - "required": ["name"] - } - } - } - } - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/completions/structured-output.vertex-gemini.snap b/crates/persisting-gateway/tests/fixtures/requests/completions/structured-output.vertex-gemini.snap deleted file mode 100644 index 966ac9efd..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/completions/structured-output.vertex-gemini.snap +++ /dev/null @@ -1,83 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/completions/structured-output.json -info: - model: gemini-2.5-pro - messages: - - role: user - content: "Extract the person from: Ada Lovelace, age 36." - response_format: - type: json_schema - json_schema: - name: person - schema: - type: object - properties: - person: - $ref: "#/$defs/Person" - required: - - person - $defs: - Person: - type: object - properties: - name: - type: string - age: - type: integer - required: - - name ---- -{ - "request": { - "contents": [ - { - "role": "user", - "parts": [ - { - "text": "Extract the person from: Ada Lovelace, age 36." - } - ] - } - ], - "generationConfig": { - "responseMimeType": "application/json", - "responseSchema": { - "type": "object", - "properties": { - "person": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "age": { - "type": "integer" - } - }, - "required": [ - "name" - ] - } - }, - "required": [ - "person" - ] - } - } - }, - "parsed": { - "input_format": "Completions", - "cache_convention": "InputIncludesCache", - "request_model": "gemini-2.5-pro", - "provider": "vertex", - "streaming": false, - "params": {}, - "prompt": [ - { - "role": "user", - "content": "Extract the person from: Ada Lovelace, age 36." - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/completions/tool-call.anthropic.snap b/crates/persisting-gateway/tests/fixtures/requests/completions/tool-call.anthropic.snap deleted file mode 100644 index 760a151e0..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/completions/tool-call.anthropic.snap +++ /dev/null @@ -1,113 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/completions/tool-call.json -info: - model: gpt-3.5-turbo - messages: - - role: user - content: How is the current weather in Columbus? - - role: assistant - content: ~ - tool_calls: - - id: call_iMGPsr4Xx1u0G5sOzFsTCbQU - type: function - function: - name: get_weather - arguments: "{\"format\":\"celsius\",\"location\":\"Columbus, OH\"}" - - id: call_5rQmXtLpZ9wKvNbJhGyDsFcE - type: function - function: - name: get_time - arguments: "" - - role: tool - tool_call_id: call_iMGPsr4Xx1u0G5sOzFsTCbQU - name: get_weather - content: "{ \"temperature\": 15, \"condition\": \"Cloudy\" }" - - role: tool - tool_call_id: call_5rQmXtLpZ9wKvNbJhGyDsFcE - name: get_time - content: "{ \"time\": \"2026-08-12T12:00:00Z\" }" ---- -{ - "request": { - "messages": [ - { - "role": "user", - "content": [ - { - "type": "text", - "text": "How is the current weather in Columbus?" - } - ] - }, - { - "role": "assistant", - "content": [ - { - "type": "tool_use", - "id": "call_iMGPsr4Xx1u0G5sOzFsTCbQU", - "name": "get_weather", - "input": { - "format": "celsius", - "location": "Columbus, OH" - } - }, - { - "type": "tool_use", - "id": "call_5rQmXtLpZ9wKvNbJhGyDsFcE", - "name": "get_time", - "input": {} - } - ] - }, - { - "role": "user", - "content": [ - { - "type": "tool_result", - "tool_use_id": "call_iMGPsr4Xx1u0G5sOzFsTCbQU", - "content": "{ \"temperature\": 15, \"condition\": \"Cloudy\" }" - } - ] - }, - { - "role": "user", - "content": [ - { - "type": "tool_result", - "tool_use_id": "call_5rQmXtLpZ9wKvNbJhGyDsFcE", - "content": "{ \"time\": \"2026-08-12T12:00:00Z\" }" - } - ] - } - ], - "model": "gpt-3.5-turbo", - "max_tokens": 4096 - }, - "parsed": { - "input_format": "Completions", - "cache_convention": "InputIncludesCache", - "request_model": "gpt-3.5-turbo", - "provider": "anthropic", - "streaming": false, - "params": {}, - "prompt": [ - { - "role": "user", - "content": "How is the current weather in Columbus?" - }, - { - "role": "assistant", - "content": "" - }, - { - "role": "tool", - "content": "{ \"temperature\": 15, \"condition\": \"Cloudy\" }" - }, - { - "role": "tool", - "content": "{ \"time\": \"2026-08-12T12:00:00Z\" }" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/completions/tool-call.bedrock.snap b/crates/persisting-gateway/tests/fixtures/requests/completions/tool-call.bedrock.snap deleted file mode 100644 index d00421c4a..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/completions/tool-call.bedrock.snap +++ /dev/null @@ -1,137 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/completions/tool-call.json -info: - model: gpt-3.5-turbo - messages: - - role: user - content: How is the current weather in Columbus? - - role: assistant - content: ~ - tool_calls: - - id: call_iMGPsr4Xx1u0G5sOzFsTCbQU - type: function - function: - name: get_weather - arguments: "{\"format\":\"celsius\",\"location\":\"Columbus, OH\"}" - - id: call_5rQmXtLpZ9wKvNbJhGyDsFcE - type: function - function: - name: get_time - arguments: "" - - role: tool - tool_call_id: call_iMGPsr4Xx1u0G5sOzFsTCbQU - name: get_weather - content: "{ \"temperature\": 15, \"condition\": \"Cloudy\" }" - - role: tool - tool_call_id: call_5rQmXtLpZ9wKvNbJhGyDsFcE - name: get_time - content: "{ \"time\": \"2026-08-12T12:00:00Z\" }" ---- -{ - "request": { - "messages": [ - { - "role": "user", - "content": [ - { - "text": "How is the current weather in Columbus?" - } - ] - }, - { - "role": "assistant", - "content": [ - { - "toolUse": { - "toolUseId": "call_iMGPsr4Xx1u0G5sOzFsTCbQU", - "name": "get_weather", - "input": { - "format": "celsius", - "location": "Columbus, OH" - } - } - }, - { - "toolUse": { - "toolUseId": "call_5rQmXtLpZ9wKvNbJhGyDsFcE", - "name": "get_time", - "input": {} - } - } - ] - }, - { - "role": "user", - "content": [ - { - "toolResult": { - "toolUseId": "call_iMGPsr4Xx1u0G5sOzFsTCbQU", - "content": [ - { - "text": "{ \"temperature\": 15, \"condition\": \"Cloudy\" }" - } - ], - "status": null - } - }, - { - "toolResult": { - "toolUseId": "call_5rQmXtLpZ9wKvNbJhGyDsFcE", - "content": [ - { - "text": "{ \"time\": \"2026-08-12T12:00:00Z\" }" - } - ], - "status": null - } - } - ] - } - ], - "inferenceConfig": { - "maxTokens": 4096 - }, - "toolConfig": { - "tools": [ - { - "toolSpec": { - "name": "agentgateway_dummy_do_not_call", - "description": null, - "inputSchema": { - "json": { - "type": "object" - } - } - } - } - ] - } - }, - "parsed": { - "input_format": "Completions", - "cache_convention": "InputIncludesCache", - "request_model": "gpt-3.5-turbo", - "provider": "bedrock", - "streaming": false, - "params": {}, - "prompt": [ - { - "role": "user", - "content": "How is the current weather in Columbus?" - }, - { - "role": "assistant", - "content": "" - }, - { - "role": "tool", - "content": "{ \"temperature\": 15, \"condition\": \"Cloudy\" }" - }, - { - "role": "tool", - "content": "{ \"time\": \"2026-08-12T12:00:00Z\" }" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/completions/tool-call.json b/crates/persisting-gateway/tests/fixtures/requests/completions/tool-call.json deleted file mode 100644 index 50f69a022..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/completions/tool-call.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "model": "gpt-3.5-turbo", - "messages": [ - { - "role": "user", - "content": "How is the current weather in Columbus?" - }, - { - "role": "assistant", - "content": null, - "tool_calls": [ - { - "id": "call_iMGPsr4Xx1u0G5sOzFsTCbQU", - "type": "function", - "function": { - "name": "get_weather", - "arguments": "{\"format\":\"celsius\",\"location\":\"Columbus, OH\"}" - } - }, - { - "id": "call_5rQmXtLpZ9wKvNbJhGyDsFcE", - "type": "function", - "function": { - "name": "get_time", - "arguments": "" - } - } - ] - }, - { - "role": "tool", - "tool_call_id": "call_iMGPsr4Xx1u0G5sOzFsTCbQU", - "name": "get_weather", - "content": "{ \"temperature\": 15, \"condition\": \"Cloudy\" }" - }, - { - "role": "tool", - "tool_call_id": "call_5rQmXtLpZ9wKvNbJhGyDsFcE", - "name": "get_time", - "content": "{ \"time\": \"2026-08-12T12:00:00Z\" }" - } - ] -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/completions/tool-call.vertex-gemini.snap b/crates/persisting-gateway/tests/fixtures/requests/completions/tool-call.vertex-gemini.snap deleted file mode 100644 index ba9de9822..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/completions/tool-call.vertex-gemini.snap +++ /dev/null @@ -1,111 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/completions/tool-call.json -info: - model: gpt-3.5-turbo - messages: - - role: user - content: How is the current weather in Columbus? - - role: assistant - content: ~ - tool_calls: - - id: call_iMGPsr4Xx1u0G5sOzFsTCbQU - type: function - function: - name: get_weather - arguments: "{\"format\":\"celsius\",\"location\":\"Columbus, OH\"}" - - id: call_5rQmXtLpZ9wKvNbJhGyDsFcE - type: function - function: - name: get_time - arguments: "" - - role: tool - tool_call_id: call_iMGPsr4Xx1u0G5sOzFsTCbQU - name: get_weather - content: "{ \"temperature\": 15, \"condition\": \"Cloudy\" }" - - role: tool - tool_call_id: call_5rQmXtLpZ9wKvNbJhGyDsFcE - name: get_time - content: "{ \"time\": \"2026-08-12T12:00:00Z\" }" ---- -{ - "request": { - "contents": [ - { - "role": "user", - "parts": [ - { - "text": "How is the current weather in Columbus?" - } - ] - }, - { - "role": "model", - "parts": [ - { - "functionCall": { - "name": "get_weather", - "args": { - "format": "celsius", - "location": "Columbus, OH" - } - } - }, - { - "functionCall": { - "name": "get_time", - "args": {} - } - } - ] - }, - { - "role": "user", - "parts": [ - { - "functionResponse": { - "name": "get_weather", - "response": { - "content": "{ \"temperature\": 15, \"condition\": \"Cloudy\" }" - } - } - }, - { - "functionResponse": { - "name": "get_time", - "response": { - "content": "{ \"time\": \"2026-08-12T12:00:00Z\" }" - } - } - } - ] - } - ] - }, - "parsed": { - "input_format": "Completions", - "cache_convention": "InputIncludesCache", - "request_model": "gpt-3.5-turbo", - "provider": "vertex", - "streaming": false, - "params": {}, - "prompt": [ - { - "role": "user", - "content": "How is the current weather in Columbus?" - }, - { - "role": "assistant", - "content": "" - }, - { - "role": "tool", - "content": "{ \"temperature\": 15, \"condition\": \"Cloudy\" }" - }, - { - "role": "tool", - "content": "{ \"time\": \"2026-08-12T12:00:00Z\" }" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/count-tokens/basic.anthropic.snap b/crates/persisting-gateway/tests/fixtures/requests/count-tokens/basic.anthropic.snap deleted file mode 100644 index d5e9ce5eb..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/count-tokens/basic.anthropic.snap +++ /dev/null @@ -1,28 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/count-tokens/basic.json -info: - model: claude-3-5-sonnet-20241022 - messages: - - role: user - content: "Hello, how are you?" ---- -{ - "request": { - "messages": [ - { - "role": "user", - "content": "Hello, how are you?" - } - ], - "model": "claude-3-5-sonnet-20241022" - }, - "parsed": { - "input_format": "CountTokens", - "cache_convention": "InputIncludesCache", - "request_model": "claude-3-5-sonnet-20241022", - "provider": "anthropic", - "streaming": false, - "params": {} - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/count-tokens/basic.bedrock.snap b/crates/persisting-gateway/tests/fixtures/requests/count-tokens/basic.bedrock.snap deleted file mode 100644 index 92b09bc3f..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/count-tokens/basic.bedrock.snap +++ /dev/null @@ -1,26 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/count-tokens/basic.json -info: - model: claude-3-5-sonnet-20241022 - messages: - - role: user - content: "Hello, how are you?" ---- -{ - "request": { - "input": { - "invokeModel": { - "body": "eyJtZXNzYWdlcyI6W3sicm9sZSI6InVzZXIiLCJjb250ZW50IjoiSGVsbG8sIGhvdyBhcmUgeW91PyJ9XSwibWF4X3Rva2VucyI6MSwiYW50aHJvcGljX3ZlcnNpb24iOiIyMDIzLTA2LTAxIn0=" - } - } - }, - "parsed": { - "input_format": "CountTokens", - "cache_convention": "InputIncludesCache", - "request_model": "claude-3-5-sonnet-20241022", - "provider": "bedrock", - "streaming": false, - "params": {} - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/count-tokens/basic.json b/crates/persisting-gateway/tests/fixtures/requests/count-tokens/basic.json deleted file mode 100644 index 94871bc1b..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/count-tokens/basic.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "model": "claude-3-5-sonnet-20241022", - "messages": [ - {"role": "user", "content": "Hello, how are you?"} - ] -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/count-tokens/basic.vertex.snap b/crates/persisting-gateway/tests/fixtures/requests/count-tokens/basic.vertex.snap deleted file mode 100644 index ed1b64849..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/count-tokens/basic.vertex.snap +++ /dev/null @@ -1,29 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/count-tokens/basic.json -info: - model: claude-3-5-sonnet-20241022 - messages: - - role: user - content: "Hello, how are you?" ---- -{ - "request": { - "messages": [ - { - "role": "user", - "content": "Hello, how are you?" - } - ], - "model": "anthropic/claude-sonnet-4-5", - "anthropic_version": "vertex-2023-10-16" - }, - "parsed": { - "input_format": "CountTokens", - "cache_convention": "InputIncludesCache", - "request_model": "claude-3-5-sonnet-20241022", - "provider": "vertex", - "streaming": false, - "params": {} - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/count-tokens/with_system.anthropic.snap b/crates/persisting-gateway/tests/fixtures/requests/count-tokens/with_system.anthropic.snap deleted file mode 100644 index aa3ad4383..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/count-tokens/with_system.anthropic.snap +++ /dev/null @@ -1,30 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/count-tokens/with_system.json -info: - model: claude-3-5-sonnet-20241022 - system: You are a helpful assistant. - messages: - - role: user - content: What is the weather today? ---- -{ - "request": { - "messages": [ - { - "role": "user", - "content": "What is the weather today?" - } - ], - "model": "claude-3-5-sonnet-20241022", - "system": "You are a helpful assistant." - }, - "parsed": { - "input_format": "CountTokens", - "cache_convention": "InputIncludesCache", - "request_model": "claude-3-5-sonnet-20241022", - "provider": "anthropic", - "streaming": false, - "params": {} - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/count-tokens/with_system.bedrock.snap b/crates/persisting-gateway/tests/fixtures/requests/count-tokens/with_system.bedrock.snap deleted file mode 100644 index 713ac365c..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/count-tokens/with_system.bedrock.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/count-tokens/with_system.json -info: - model: claude-3-5-sonnet-20241022 - system: You are a helpful assistant. - messages: - - role: user - content: What is the weather today? ---- -{ - "request": { - "input": { - "invokeModel": { - "body": "eyJtZXNzYWdlcyI6W3sicm9sZSI6InVzZXIiLCJjb250ZW50IjoiV2hhdCBpcyB0aGUgd2VhdGhlciB0b2RheT8ifV0sInN5c3RlbSI6IllvdSBhcmUgYSBoZWxwZnVsIGFzc2lzdGFudC4iLCJtYXhfdG9rZW5zIjoxLCJhbnRocm9waWNfdmVyc2lvbiI6IjIwMjMtMDYtMDEifQ==" - } - } - }, - "parsed": { - "input_format": "CountTokens", - "cache_convention": "InputIncludesCache", - "request_model": "claude-3-5-sonnet-20241022", - "provider": "bedrock", - "streaming": false, - "params": {} - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/count-tokens/with_system.json b/crates/persisting-gateway/tests/fixtures/requests/count-tokens/with_system.json deleted file mode 100644 index d2511ac36..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/count-tokens/with_system.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "model": "claude-3-5-sonnet-20241022", - "system": "You are a helpful assistant.", - "messages": [ - {"role": "user", "content": "What is the weather today?"} - ] -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/count-tokens/with_system.vertex.snap b/crates/persisting-gateway/tests/fixtures/requests/count-tokens/with_system.vertex.snap deleted file mode 100644 index d007e2d30..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/count-tokens/with_system.vertex.snap +++ /dev/null @@ -1,31 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/count-tokens/with_system.json -info: - model: claude-3-5-sonnet-20241022 - system: You are a helpful assistant. - messages: - - role: user - content: What is the weather today? ---- -{ - "request": { - "messages": [ - { - "role": "user", - "content": "What is the weather today?" - } - ], - "model": "anthropic/claude-sonnet-4-5", - "system": "You are a helpful assistant.", - "anthropic_version": "vertex-2023-10-16" - }, - "parsed": { - "input_format": "CountTokens", - "cache_convention": "InputIncludesCache", - "request_model": "claude-3-5-sonnet-20241022", - "provider": "vertex", - "streaming": false, - "params": {} - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/embeddings/array.bedrock-cohere.snap b/crates/persisting-gateway/tests/fixtures/requests/embeddings/array.bedrock-cohere.snap deleted file mode 100644 index 547e84ffb..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/embeddings/array.bedrock-cohere.snap +++ /dev/null @@ -1,26 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/embeddings/array.json -info: - model: text-embedding-3-small - input: - - hello - - world ---- -{ - "request": { - "texts": [ - "hello", - "world" - ], - "input_type": "search_query" - }, - "parsed": { - "input_format": "Embeddings", - "cache_convention": "InputIncludesCache", - "request_model": "text-embedding-3-small", - "provider": "bedrock", - "streaming": false, - "params": {} - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/embeddings/array.json b/crates/persisting-gateway/tests/fixtures/requests/embeddings/array.json deleted file mode 100644 index c96013a6a..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/embeddings/array.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "model": "text-embedding-3-small", - "input": ["hello", "world"] -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/embeddings/array.openai.snap b/crates/persisting-gateway/tests/fixtures/requests/embeddings/array.openai.snap deleted file mode 100644 index 63d9f47fc..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/embeddings/array.openai.snap +++ /dev/null @@ -1,26 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/embeddings/array.json -info: - model: text-embedding-3-small - input: - - hello - - world ---- -{ - "request": { - "model": "text-embedding-3-small", - "input": [ - "hello", - "world" - ] - }, - "parsed": { - "input_format": "Embeddings", - "cache_convention": "InputIncludesCache", - "request_model": "text-embedding-3-small", - "provider": "openai", - "streaming": false, - "params": {} - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/embeddings/array.vertex.snap b/crates/persisting-gateway/tests/fixtures/requests/embeddings/array.vertex.snap deleted file mode 100644 index ee2af4f54..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/embeddings/array.vertex.snap +++ /dev/null @@ -1,31 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/embeddings/array.json -info: - model: text-embedding-3-small - input: - - hello - - world ---- -{ - "request": { - "instances": [ - { - "content": "hello", - "task_type": "RETRIEVAL_QUERY" - }, - { - "content": "world", - "task_type": "RETRIEVAL_QUERY" - } - ] - }, - "parsed": { - "input_format": "Embeddings", - "cache_convention": "InputIncludesCache", - "request_model": "text-embedding-3-small", - "provider": "vertex", - "streaming": false, - "params": {} - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/embeddings/basic.bedrock-cohere.snap b/crates/persisting-gateway/tests/fixtures/requests/embeddings/basic.bedrock-cohere.snap deleted file mode 100644 index 4e48b49ff..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/embeddings/basic.bedrock-cohere.snap +++ /dev/null @@ -1,26 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/embeddings/basic.json -info: - model: text-embedding-3-small - input: hello world - dimensions: 1024 ---- -{ - "request": { - "texts": [ - "hello world" - ], - "input_type": "search_query" - }, - "parsed": { - "input_format": "Embeddings", - "cache_convention": "InputIncludesCache", - "request_model": "text-embedding-3-small", - "provider": "bedrock", - "streaming": false, - "params": { - "dimensions": 1024 - } - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/embeddings/basic.bedrock-nova.snap b/crates/persisting-gateway/tests/fixtures/requests/embeddings/basic.bedrock-nova.snap deleted file mode 100644 index ee6098eb0..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/embeddings/basic.bedrock-nova.snap +++ /dev/null @@ -1,31 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/embeddings/basic.json -info: - model: text-embedding-3-small - input: hello world - dimensions: 1024 ---- -{ - "request": { - "taskType": "SINGLE_EMBEDDING", - "singleEmbeddingParams": { - "embeddingPurpose": "GENERIC_INDEX", - "embeddingDimension": 1024, - "text": { - "truncationMode": "END", - "value": "hello world" - } - } - }, - "parsed": { - "input_format": "Embeddings", - "cache_convention": "InputIncludesCache", - "request_model": "text-embedding-3-small", - "provider": "bedrock", - "streaming": false, - "params": { - "dimensions": 1024 - } - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/embeddings/basic.bedrock-titan.snap b/crates/persisting-gateway/tests/fixtures/requests/embeddings/basic.bedrock-titan.snap deleted file mode 100644 index 11c07e3a4..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/embeddings/basic.bedrock-titan.snap +++ /dev/null @@ -1,24 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/embeddings/basic.json -info: - model: text-embedding-3-small - input: hello world - dimensions: 1024 ---- -{ - "request": { - "inputText": "hello world", - "dimensions": 1024 - }, - "parsed": { - "input_format": "Embeddings", - "cache_convention": "InputIncludesCache", - "request_model": "text-embedding-3-small", - "provider": "bedrock", - "streaming": false, - "params": { - "dimensions": 1024 - } - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/embeddings/basic.json b/crates/persisting-gateway/tests/fixtures/requests/embeddings/basic.json deleted file mode 100644 index c6718fec5..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/embeddings/basic.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "model": "text-embedding-3-small", - "input": "hello world", - "dimensions": 1024 -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/embeddings/basic.openai.snap b/crates/persisting-gateway/tests/fixtures/requests/embeddings/basic.openai.snap deleted file mode 100644 index e9f4f59f2..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/embeddings/basic.openai.snap +++ /dev/null @@ -1,25 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/embeddings/basic.json -info: - model: text-embedding-3-small - input: hello world - dimensions: 1024 ---- -{ - "request": { - "model": "text-embedding-3-small", - "input": "hello world", - "dimensions": 1024 - }, - "parsed": { - "input_format": "Embeddings", - "cache_convention": "InputIncludesCache", - "request_model": "text-embedding-3-small", - "provider": "openai", - "streaming": false, - "params": { - "dimensions": 1024 - } - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/embeddings/basic.vertex.snap b/crates/persisting-gateway/tests/fixtures/requests/embeddings/basic.vertex.snap deleted file mode 100644 index 9cec737aa..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/embeddings/basic.vertex.snap +++ /dev/null @@ -1,31 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/embeddings/basic.json -info: - model: text-embedding-3-small - input: hello world - dimensions: 1024 ---- -{ - "request": { - "instances": [ - { - "content": "hello world", - "task_type": "RETRIEVAL_QUERY" - } - ], - "parameters": { - "outputDimensionality": 1024 - } - }, - "parsed": { - "input_format": "Embeddings", - "cache_convention": "InputIncludesCache", - "request_model": "text-embedding-3-small", - "provider": "vertex", - "streaming": false, - "params": { - "dimensions": 1024 - } - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/embeddings/cohere-v4.bedrock-cohere.snap b/crates/persisting-gateway/tests/fixtures/requests/embeddings/cohere-v4.bedrock-cohere.snap deleted file mode 100644 index 0ca05d485..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/embeddings/cohere-v4.bedrock-cohere.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/embeddings/cohere-v4.json -info: - model: "cohere.embed-v4:0" - input: hello world - dimensions: 512 ---- -{ - "request": { - "texts": [ - "hello world" - ], - "input_type": "search_query", - "output_dimension": 512 - }, - "parsed": { - "input_format": "Embeddings", - "cache_convention": "InputIncludesCache", - "request_model": "cohere.embed-v4:0", - "provider": "bedrock", - "streaming": false, - "params": { - "dimensions": 512 - } - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/embeddings/cohere-v4.json b/crates/persisting-gateway/tests/fixtures/requests/embeddings/cohere-v4.json deleted file mode 100644 index 97c2390f9..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/embeddings/cohere-v4.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "model": "cohere.embed-v4:0", - "input": "hello world", - "dimensions": 512 -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/embeddings/full.json b/crates/persisting-gateway/tests/fixtures/requests/embeddings/full.json deleted file mode 100644 index dd1233cd7..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/embeddings/full.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "model": "text-embedding-004", - "input": ["hello", "world"], - "dimensions": 512, - "task_type": "RETRIEVAL_DOCUMENT", - "title": "My Document", - "auto_truncate": true -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/embeddings/full.vertex.snap b/crates/persisting-gateway/tests/fixtures/requests/embeddings/full.vertex.snap deleted file mode 100644 index c661d7faf..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/embeddings/full.vertex.snap +++ /dev/null @@ -1,43 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/embeddings/full.json -info: - model: text-embedding-004 - input: - - hello - - world - dimensions: 512 - task_type: RETRIEVAL_DOCUMENT - title: My Document - auto_truncate: true ---- -{ - "request": { - "instances": [ - { - "content": "hello", - "task_type": "RETRIEVAL_DOCUMENT", - "title": "My Document" - }, - { - "content": "world", - "task_type": "RETRIEVAL_DOCUMENT", - "title": "My Document" - } - ], - "parameters": { - "autoTruncate": true, - "outputDimensionality": 512 - } - }, - "parsed": { - "input_format": "Embeddings", - "cache_convention": "InputIncludesCache", - "request_model": "text-embedding-004", - "provider": "vertex", - "streaming": false, - "params": { - "dimensions": 512 - } - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/messages/basic.anthropic.snap b/crates/persisting-gateway/tests/fixtures/requests/messages/basic.anthropic.snap deleted file mode 100644 index 6ff09b673..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/messages/basic.anthropic.snap +++ /dev/null @@ -1,38 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/messages/basic.json -info: - model: claude-sonnet-4-20250514 - max_tokens: 1024 - messages: - - role: user - content: "Hello, world" ---- -{ - "request": { - "messages": [ - { - "role": "user", - "content": "Hello, world" - } - ], - "model": "claude-sonnet-4-20250514", - "max_tokens": 1024 - }, - "parsed": { - "input_format": "Messages", - "cache_convention": "InputIncludesCache", - "request_model": "claude-sonnet-4-20250514", - "provider": "anthropic", - "streaming": false, - "params": { - "max_tokens": 1024 - }, - "prompt": [ - { - "role": "user", - "content": "Hello, world" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/messages/basic.bedrock.snap b/crates/persisting-gateway/tests/fixtures/requests/messages/basic.bedrock.snap deleted file mode 100644 index a49ce7c4e..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/messages/basic.bedrock.snap +++ /dev/null @@ -1,43 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/messages/basic.json -info: - model: claude-sonnet-4-20250514 - max_tokens: 1024 - messages: - - role: user - content: "Hello, world" ---- -{ - "request": { - "messages": [ - { - "role": "user", - "content": [ - { - "text": "Hello, world" - } - ] - } - ], - "inferenceConfig": { - "maxTokens": 1024 - } - }, - "parsed": { - "input_format": "Messages", - "cache_convention": "InputIncludesCache", - "request_model": "claude-sonnet-4-20250514", - "provider": "bedrock", - "streaming": false, - "params": { - "max_tokens": 1024 - }, - "prompt": [ - { - "role": "user", - "content": "Hello, world" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/messages/basic.completions.snap b/crates/persisting-gateway/tests/fixtures/requests/messages/basic.completions.snap deleted file mode 100644 index 8b2c4d9b8..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/messages/basic.completions.snap +++ /dev/null @@ -1,44 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/messages/basic.json -info: - model: claude-sonnet-4-20250514 - max_tokens: 1024 - messages: - - role: user - content: "Hello, world" ---- -{ - "request": { - "messages": [ - { - "role": "user", - "content": [ - { - "type": "text", - "text": "Hello, world" - } - ] - } - ], - "model": "claude-sonnet-4-20250514", - "max_completion_tokens": 1024, - "stream": false - }, - "parsed": { - "input_format": "Messages", - "cache_convention": "InputIncludesCache", - "request_model": "claude-sonnet-4-20250514", - "provider": "openai", - "streaming": false, - "params": { - "max_tokens": 1024 - }, - "prompt": [ - { - "role": "user", - "content": "Hello, world" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/messages/basic.json b/crates/persisting-gateway/tests/fixtures/requests/messages/basic.json deleted file mode 100644 index 2bab7ae06..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/messages/basic.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "model": "claude-sonnet-4-20250514", - "max_tokens": 1024, - "messages": [ - {"role": "user", "content": "Hello, world"} - ] -} \ No newline at end of file diff --git a/crates/persisting-gateway/tests/fixtures/requests/messages/basic.vertex.snap b/crates/persisting-gateway/tests/fixtures/requests/messages/basic.vertex.snap deleted file mode 100644 index 5e14c62f8..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/messages/basic.vertex.snap +++ /dev/null @@ -1,38 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/messages/basic.json -info: - model: claude-sonnet-4-20250514 - max_tokens: 1024 - messages: - - role: user - content: "Hello, world" ---- -{ - "request": { - "messages": [ - { - "role": "user", - "content": "Hello, world" - } - ], - "anthropic_version": "vertex-2023-10-16", - "max_tokens": 1024 - }, - "parsed": { - "input_format": "Messages", - "cache_convention": "InputIncludesCache", - "request_model": "claude-sonnet-4-20250514", - "provider": "vertex", - "streaming": false, - "params": { - "max_tokens": 1024 - }, - "prompt": [ - { - "role": "user", - "content": "Hello, world" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/messages/cache_control.completions.snap b/crates/persisting-gateway/tests/fixtures/requests/messages/cache_control.completions.snap deleted file mode 100644 index 557bbd67b..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/messages/cache_control.completions.snap +++ /dev/null @@ -1,287 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/messages/cache_control.json -info: - model: gpt-5.6-terra - max_tokens: 1024 - system: - - type: text - text: First system block - - type: text - text: Second system block - cache_control: - type: ephemeral - messages: - - role: assistant - content: - - type: text - text: First assistant block - - type: text - text: Second assistant block - cache_control: - type: ephemeral - - role: user - content: - - type: image - source: - type: base64 - media_type: image/png - data: aGVsbG8= - cache_control: - type: ephemeral - - role: assistant - content: - - type: tool_use - id: call_text - name: lookup - input: {} - - type: tool_use - id: call_array - name: lookup - input: {} - - type: tool_use - id: call_image - name: lookup - input: {} - - type: tool_use - id: call_mixed - name: lookup - input: {} - - role: user - content: - - type: tool_result - tool_use_id: call_text - content: text result - cache_control: - type: ephemeral - - type: tool_result - tool_use_id: call_array - content: - - type: text - text: array result - cache_control: - type: ephemeral - - type: tool_result - tool_use_id: call_image - content: - - type: image - source: - type: base64 - media_type: image/png - data: aGVsbG8= - cache_control: - type: ephemeral - - type: tool_result - tool_use_id: call_mixed - content: - - type: text - text: before image - - type: image - source: - type: base64 - media_type: image/png - data: aGVsbG8= - cache_control: - type: ephemeral - - type: text - text: after image - - role: system - content: - - type: text - text: System-role block - cache_control: - type: ephemeral ---- -{ - "request": { - "messages": [ - { - "role": "system", - "content": [ - { - "type": "text", - "text": "First system block" - }, - { - "type": "text", - "text": "Second system block", - "prompt_cache_breakpoint": { - "mode": "explicit" - } - } - ] - }, - { - "role": "assistant", - "content": [ - { - "type": "text", - "text": "First assistant block" - }, - { - "type": "text", - "text": "Second assistant block", - "prompt_cache_breakpoint": { - "mode": "explicit" - } - } - ] - }, - { - "role": "user", - "content": [ - { - "type": "image_url", - "image_url": { - "url": "data:image/png;base64,aGVsbG8=" - } - } - ] - }, - { - "role": "assistant", - "tool_calls": [ - { - "type": "function", - "id": "call_text", - "function": { - "name": "lookup", - "arguments": "{}" - } - }, - { - "type": "function", - "id": "call_array", - "function": { - "name": "lookup", - "arguments": "{}" - } - }, - { - "type": "function", - "id": "call_image", - "function": { - "name": "lookup", - "arguments": "{}" - } - }, - { - "type": "function", - "id": "call_mixed", - "function": { - "name": "lookup", - "arguments": "{}" - } - } - ] - }, - { - "role": "tool", - "content": [ - { - "type": "text", - "text": "text result", - "prompt_cache_breakpoint": { - "mode": "explicit" - } - } - ], - "tool_call_id": "call_text" - }, - { - "role": "tool", - "content": [ - { - "type": "text", - "text": "array result", - "prompt_cache_breakpoint": { - "mode": "explicit" - } - } - ], - "tool_call_id": "call_array" - }, - { - "role": "tool", - "content": [ - { - "type": "text", - "text": "", - "prompt_cache_breakpoint": { - "mode": "explicit" - } - } - ], - "tool_call_id": "call_image" - }, - { - "role": "tool", - "content": [ - { - "type": "text", - "text": "before image" - }, - { - "type": "text", - "text": "after image", - "prompt_cache_breakpoint": { - "mode": "explicit" - } - } - ], - "tool_call_id": "call_mixed" - }, - { - "role": "system", - "content": [ - { - "type": "text", - "text": "System-role block", - "prompt_cache_breakpoint": { - "mode": "explicit" - } - } - ] - } - ], - "model": "gpt-5.6-terra", - "max_completion_tokens": 1024, - "stream": false - }, - "parsed": { - "input_format": "Messages", - "cache_convention": "InputIncludesCache", - "request_model": "gpt-5.6-terra", - "provider": "openai", - "streaming": false, - "params": { - "max_tokens": 1024 - }, - "prompt": [ - { - "role": "system", - "content": "First system block\nSecond system block" - }, - { - "role": "assistant", - "content": "First assistant block Second assistant block" - }, - { - "role": "user", - "content": "" - }, - { - "role": "assistant", - "content": "" - }, - { - "role": "user", - "content": "" - }, - { - "role": "system", - "content": "System-role block" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/messages/cache_control.json b/crates/persisting-gateway/tests/fixtures/requests/messages/cache_control.json deleted file mode 100644 index b2ba0629a..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/messages/cache_control.json +++ /dev/null @@ -1,160 +0,0 @@ -{ - "model": "gpt-5.6-terra", - "max_tokens": 1024, - "system": [ - { - "type": "text", - "text": "First system block" - }, - { - "type": "text", - "text": "Second system block", - "cache_control": { - "type": "ephemeral" - } - } - ], - "messages": [ - { - "role": "assistant", - "content": [ - { - "type": "text", - "text": "First assistant block" - }, - { - "type": "text", - "text": "Second assistant block", - "cache_control": { - "type": "ephemeral" - } - } - ] - }, - { - "role": "user", - "content": [ - { - "type": "image", - "source": { - "type": "base64", - "media_type": "image/png", - "data": "aGVsbG8=" - }, - "cache_control": { - "type": "ephemeral" - } - } - ] - }, - { - "role": "assistant", - "content": [ - { - "type": "tool_use", - "id": "call_text", - "name": "lookup", - "input": {} - }, - { - "type": "tool_use", - "id": "call_array", - "name": "lookup", - "input": {} - }, - { - "type": "tool_use", - "id": "call_image", - "name": "lookup", - "input": {} - }, - { - "type": "tool_use", - "id": "call_mixed", - "name": "lookup", - "input": {} - } - ] - }, - { - "role": "user", - "content": [ - { - "type": "tool_result", - "tool_use_id": "call_text", - "content": "text result", - "cache_control": { - "type": "ephemeral" - } - }, - { - "type": "tool_result", - "tool_use_id": "call_array", - "content": [ - { - "type": "text", - "text": "array result", - "cache_control": { - "type": "ephemeral" - } - } - ] - }, - { - "type": "tool_result", - "tool_use_id": "call_image", - "content": [ - { - "type": "image", - "source": { - "type": "base64", - "media_type": "image/png", - "data": "aGVsbG8=" - }, - "cache_control": { - "type": "ephemeral" - } - } - ] - }, - { - "type": "tool_result", - "tool_use_id": "call_mixed", - "content": [ - { - "type": "text", - "text": "before image" - }, - { - "type": "image", - "source": { - "type": "base64", - "media_type": "image/png", - "data": "aGVsbG8=" - }, - "cache_control": { - "type": "ephemeral" - } - }, - { - "type": "text", - "text": "after image" - } - ] - } - ] - }, - { - "role": "system", - "content": [ - { - "type": "text", - "text": "System-role block", - "cache_control": { - "type": "ephemeral" - } - } - ] - } - ] -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/messages/gpt_adaptive_thinking_with_tools.completions.snap b/crates/persisting-gateway/tests/fixtures/requests/messages/gpt_adaptive_thinking_with_tools.completions.snap deleted file mode 100644 index cdac246b9..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/messages/gpt_adaptive_thinking_with_tools.completions.snap +++ /dev/null @@ -1,77 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/messages/gpt_adaptive_thinking_with_tools.json -info: - model: gpt-5.6-terra - max_tokens: 64 - thinking: - type: adaptive - tools: - - name: get_weather - description: Get the weather for a location - input_schema: - type: object - properties: - location: - type: string - required: - - location - messages: - - role: user - content: Say hi ---- -{ - "request": { - "messages": [ - { - "role": "user", - "content": [ - { - "type": "text", - "text": "Say hi" - } - ] - } - ], - "model": "gpt-5.6-terra", - "reasoning_effort": "none", - "max_completion_tokens": 64, - "stream": false, - "tools": [ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get the weather for a location", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string" - } - }, - "required": [ - "location" - ] - } - } - } - ] - }, - "parsed": { - "input_format": "Messages", - "cache_convention": "InputIncludesCache", - "request_model": "gpt-5.6-terra", - "provider": "openai", - "streaming": false, - "params": { - "max_tokens": 64 - }, - "prompt": [ - { - "role": "user", - "content": "Say hi" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/messages/gpt_adaptive_thinking_with_tools.json b/crates/persisting-gateway/tests/fixtures/requests/messages/gpt_adaptive_thinking_with_tools.json deleted file mode 100644 index 377da9c7e..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/messages/gpt_adaptive_thinking_with_tools.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "model": "gpt-5.6-terra", - "max_tokens": 64, - "thinking": { - "type": "adaptive" - }, - "tools": [ - { - "name": "get_weather", - "description": "Get the weather for a location", - "input_schema": { - "type": "object", - "properties": { - "location": { - "type": "string" - } - }, - "required": [ - "location" - ] - } - } - ], - "messages": [ - { - "role": "user", - "content": "Say hi" - } - ] -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/messages/metadata.anthropic.snap b/crates/persisting-gateway/tests/fixtures/requests/messages/metadata.anthropic.snap deleted file mode 100644 index eb9bda631..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/messages/metadata.anthropic.snap +++ /dev/null @@ -1,45 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/messages/metadata.json -info: - model: anthropic.claude-3-sonnet - max_tokens: 100 - metadata: - user_id: "{\"device_id\":\"704cb53c2074e9\",\"account_uuid\":\"\",\"session_id\":\"180423cd-fe24-4f48-bbde-b4ab5bfd36e7\"}" - department: engineering - messages: - - role: user - content: Hello ---- -{ - "request": { - "messages": [ - { - "role": "user", - "content": "Hello" - } - ], - "model": "anthropic.claude-3-sonnet", - "max_tokens": 100, - "metadata": { - "department": "engineering", - "user_id": "{\"device_id\":\"704cb53c2074e9\",\"account_uuid\":\"\",\"session_id\":\"180423cd-fe24-4f48-bbde-b4ab5bfd36e7\"}" - } - }, - "parsed": { - "input_format": "Messages", - "cache_convention": "InputIncludesCache", - "request_model": "anthropic.claude-3-sonnet", - "provider": "anthropic", - "streaming": false, - "params": { - "max_tokens": 100 - }, - "prompt": [ - { - "role": "user", - "content": "Hello" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/messages/metadata.bedrock.snap b/crates/persisting-gateway/tests/fixtures/requests/messages/metadata.bedrock.snap deleted file mode 100644 index 7bce8236f..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/messages/metadata.bedrock.snap +++ /dev/null @@ -1,52 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/messages/metadata.json -info: - model: anthropic.claude-3-sonnet - max_tokens: 100 - metadata: - user_id: "{\"device_id\":\"704cb53c2074e9\",\"account_uuid\":\"\",\"session_id\":\"180423cd-fe24-4f48-bbde-b4ab5bfd36e7\"}" - department: engineering - messages: - - role: user - content: Hello ---- -{ - "request": { - "messages": [ - { - "role": "user", - "content": [ - { - "text": "Hello" - } - ] - } - ], - "inferenceConfig": { - "maxTokens": 100 - }, - "additionalModelRequestFields": { - "metadata": { - "department": "engineering", - "user_id": "{\"device_id\":\"704cb53c2074e9\",\"account_uuid\":\"\",\"session_id\":\"180423cd-fe24-4f48-bbde-b4ab5bfd36e7\"}" - } - } - }, - "parsed": { - "input_format": "Messages", - "cache_convention": "InputIncludesCache", - "request_model": "anthropic.claude-3-sonnet", - "provider": "bedrock", - "streaming": false, - "params": { - "max_tokens": 100 - }, - "prompt": [ - { - "role": "user", - "content": "Hello" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/messages/metadata.completions.snap b/crates/persisting-gateway/tests/fixtures/requests/messages/metadata.completions.snap deleted file mode 100644 index 7f04908d8..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/messages/metadata.completions.snap +++ /dev/null @@ -1,48 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/messages/metadata.json -info: - model: anthropic.claude-3-sonnet - max_tokens: 100 - metadata: - user_id: "{\"device_id\":\"704cb53c2074e9\",\"account_uuid\":\"\",\"session_id\":\"180423cd-fe24-4f48-bbde-b4ab5bfd36e7\"}" - department: engineering - messages: - - role: user - content: Hello ---- -{ - "request": { - "messages": [ - { - "role": "user", - "content": [ - { - "type": "text", - "text": "Hello" - } - ] - } - ], - "model": "anthropic.claude-3-sonnet", - "max_completion_tokens": 100, - "stream": false, - "user": "{\"device_id\":\"704cb53c2074e9\",\"account_uuid\":\"\",\"session_id\":\"180423cd-fe24-4f48-bbde-b4ab5bfd36e7\"}" - }, - "parsed": { - "input_format": "Messages", - "cache_convention": "InputIncludesCache", - "request_model": "anthropic.claude-3-sonnet", - "provider": "openai", - "streaming": false, - "params": { - "max_tokens": 100 - }, - "prompt": [ - { - "role": "user", - "content": "Hello" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/messages/metadata.json b/crates/persisting-gateway/tests/fixtures/requests/messages/metadata.json deleted file mode 100644 index 958a093bd..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/messages/metadata.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "model": "anthropic.claude-3-sonnet", - "max_tokens": 100, - "metadata": { - "user_id": "{\"device_id\":\"704cb53c2074e9\",\"account_uuid\":\"\",\"session_id\":\"180423cd-fe24-4f48-bbde-b4ab5bfd36e7\"}", - "department": "engineering" - }, - "messages": [ - { - "role": "user", - "content": "Hello" - } - ] -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/messages/metadata.vertex.snap b/crates/persisting-gateway/tests/fixtures/requests/messages/metadata.vertex.snap deleted file mode 100644 index 2f952c93f..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/messages/metadata.vertex.snap +++ /dev/null @@ -1,45 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/messages/metadata.json -info: - model: anthropic.claude-3-sonnet - max_tokens: 100 - metadata: - user_id: "{\"device_id\":\"704cb53c2074e9\",\"account_uuid\":\"\",\"session_id\":\"180423cd-fe24-4f48-bbde-b4ab5bfd36e7\"}" - department: engineering - messages: - - role: user - content: Hello ---- -{ - "request": { - "messages": [ - { - "role": "user", - "content": "Hello" - } - ], - "anthropic_version": "vertex-2023-10-16", - "max_tokens": 100, - "metadata": { - "department": "engineering", - "user_id": "{\"device_id\":\"704cb53c2074e9\",\"account_uuid\":\"\",\"session_id\":\"180423cd-fe24-4f48-bbde-b4ab5bfd36e7\"}" - } - }, - "parsed": { - "input_format": "Messages", - "cache_convention": "InputIncludesCache", - "request_model": "anthropic.claude-3-sonnet", - "provider": "vertex", - "streaming": false, - "params": { - "max_tokens": 100 - }, - "prompt": [ - { - "role": "user", - "content": "Hello" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/messages/reasoning.anthropic.snap b/crates/persisting-gateway/tests/fixtures/requests/messages/reasoning.anthropic.snap deleted file mode 100644 index d3389ab1a..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/messages/reasoning.anthropic.snap +++ /dev/null @@ -1,64 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/messages/reasoning.json -info: - model: claude-opus-4-6 - max_tokens: 1024 - thinking: - type: adaptive - output_config: - effort: high - output_format: markdown - messages: - - role: user - content: - - type: text - text: Give one concise insight about distributed consensus. - cache_control: - type: ephemeral - scope: turn ---- -{ - "request": { - "messages": [ - { - "role": "user", - "content": [ - { - "type": "text", - "text": "Give one concise insight about distributed consensus.", - "cache_control": { - "type": "ephemeral", - "scope": "turn" - } - } - ] - } - ], - "model": "claude-opus-4-6", - "max_tokens": 1024, - "thinking": { - "type": "adaptive" - }, - "output_config": { - "effort": "high" - }, - "output_format": "markdown" - }, - "parsed": { - "input_format": "Messages", - "cache_convention": "InputIncludesCache", - "request_model": "claude-opus-4-6", - "provider": "anthropic", - "streaming": false, - "params": { - "max_tokens": 1024 - }, - "prompt": [ - { - "role": "user", - "content": "Give one concise insight about distributed consensus." - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/messages/reasoning.bedrock.snap b/crates/persisting-gateway/tests/fixtures/requests/messages/reasoning.bedrock.snap deleted file mode 100644 index 519b71100..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/messages/reasoning.bedrock.snap +++ /dev/null @@ -1,66 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/messages/reasoning.json -info: - model: claude-opus-4-6 - max_tokens: 1024 - thinking: - type: adaptive - output_config: - effort: high - output_format: markdown - messages: - - role: user - content: - - type: text - text: Give one concise insight about distributed consensus. - cache_control: - type: ephemeral - scope: turn ---- -{ - "request": { - "messages": [ - { - "role": "user", - "content": [ - { - "text": "Give one concise insight about distributed consensus." - }, - { - "cachePoint": { - "type": "default" - } - } - ] - } - ], - "inferenceConfig": { - "maxTokens": 1024 - }, - "additionalModelRequestFields": { - "thinking": { - "type": "adaptive" - }, - "output_config": { - "effort": "high" - } - } - }, - "parsed": { - "input_format": "Messages", - "cache_convention": "InputIncludesCache", - "request_model": "claude-opus-4-6", - "provider": "bedrock", - "streaming": false, - "params": { - "max_tokens": 1024 - }, - "prompt": [ - { - "role": "user", - "content": "Give one concise insight about distributed consensus." - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/messages/reasoning.completions.snap b/crates/persisting-gateway/tests/fixtures/requests/messages/reasoning.completions.snap deleted file mode 100644 index 4bd264e2a..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/messages/reasoning.completions.snap +++ /dev/null @@ -1,55 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/messages/reasoning.json -info: - model: claude-opus-4-6 - max_tokens: 1024 - thinking: - type: adaptive - output_config: - effort: high - output_format: markdown - messages: - - role: user - content: - - type: text - text: Give one concise insight about distributed consensus. - cache_control: - type: ephemeral - scope: turn ---- -{ - "request": { - "messages": [ - { - "role": "user", - "content": [ - { - "type": "text", - "text": "Give one concise insight about distributed consensus." - } - ] - } - ], - "model": "claude-opus-4-6", - "reasoning_effort": "high", - "max_completion_tokens": 1024, - "stream": false - }, - "parsed": { - "input_format": "Messages", - "cache_convention": "InputIncludesCache", - "request_model": "claude-opus-4-6", - "provider": "openai", - "streaming": false, - "params": { - "max_tokens": 1024 - }, - "prompt": [ - { - "role": "user", - "content": "Give one concise insight about distributed consensus." - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/messages/reasoning.json b/crates/persisting-gateway/tests/fixtures/requests/messages/reasoning.json deleted file mode 100644 index e1ee045be..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/messages/reasoning.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "model": "claude-opus-4-6", - "max_tokens": 1024, - "thinking": { - "type": "adaptive" - }, - "output_config": { - "effort": "high" - }, - "output_format": "markdown", - "messages": [ - { - "role": "user", - "content": [ - { - "type": "text", - "text": "Give one concise insight about distributed consensus.", - "cache_control": { - "type": "ephemeral", - "scope": "turn" - } - } - ] - } - ] -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/messages/reasoning.vertex.snap b/crates/persisting-gateway/tests/fixtures/requests/messages/reasoning.vertex.snap deleted file mode 100644 index 1497d26a9..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/messages/reasoning.vertex.snap +++ /dev/null @@ -1,62 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/messages/reasoning.json -info: - model: claude-opus-4-6 - max_tokens: 1024 - thinking: - type: adaptive - output_config: - effort: high - output_format: markdown - messages: - - role: user - content: - - type: text - text: Give one concise insight about distributed consensus. - cache_control: - type: ephemeral - scope: turn ---- -{ - "request": { - "messages": [ - { - "role": "user", - "content": [ - { - "type": "text", - "text": "Give one concise insight about distributed consensus.", - "cache_control": { - "type": "ephemeral" - } - } - ] - } - ], - "anthropic_version": "vertex-2023-10-16", - "max_tokens": 1024, - "thinking": { - "type": "adaptive" - }, - "output_config": { - "effort": "high" - } - }, - "parsed": { - "input_format": "Messages", - "cache_convention": "InputIncludesCache", - "request_model": "claude-opus-4-6", - "provider": "vertex", - "streaming": false, - "params": { - "max_tokens": 1024 - }, - "prompt": [ - { - "role": "user", - "content": "Give one concise insight about distributed consensus." - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/messages/reasoning_replay.bedrock.snap b/crates/persisting-gateway/tests/fixtures/requests/messages/reasoning_replay.bedrock.snap deleted file mode 100644 index 7f0c56bce..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/messages/reasoning_replay.bedrock.snap +++ /dev/null @@ -1,88 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/messages/reasoning_replay.json -info: - model: "claude-3-7-sonnet-20250219-v1:0" - max_tokens: 1024 - messages: - - role: user - content: - - type: text - text: What is 2+2? - - role: assistant - content: - - type: thinking - thinking: Two plus two. - signature: SIG-replay== - - type: text - text: "4" - - role: user - content: - - type: text - text: And 3+3? ---- -{ - "request": { - "messages": [ - { - "role": "user", - "content": [ - { - "text": "What is 2+2?" - } - ] - }, - { - "role": "assistant", - "content": [ - { - "reasoningContent": { - "reasoningText": { - "text": "Two plus two.", - "signature": "SIG-replay==" - } - } - }, - { - "text": "4" - } - ] - }, - { - "role": "user", - "content": [ - { - "text": "And 3+3?" - } - ] - } - ], - "inferenceConfig": { - "maxTokens": 1024 - } - }, - "parsed": { - "input_format": "Messages", - "cache_convention": "InputIncludesCache", - "request_model": "claude-3-7-sonnet-20250219-v1:0", - "provider": "bedrock", - "streaming": false, - "params": { - "max_tokens": 1024 - }, - "prompt": [ - { - "role": "user", - "content": "What is 2+2?" - }, - { - "role": "assistant", - "content": "4" - }, - { - "role": "user", - "content": "And 3+3?" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/messages/reasoning_replay.json b/crates/persisting-gateway/tests/fixtures/requests/messages/reasoning_replay.json deleted file mode 100644 index 2ff6d0f94..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/messages/reasoning_replay.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "model": "claude-3-7-sonnet-20250219-v1:0", - "max_tokens": 1024, - "messages": [ - { - "role": "user", - "content": [{ "type": "text", "text": "What is 2+2?" }] - }, - { - "role": "assistant", - "content": [ - { - "type": "thinking", - "thinking": "Two plus two.", - "signature": "SIG-replay==" - }, - { "type": "text", "text": "4" } - ] - }, - { - "role": "user", - "content": [{ "type": "text", "text": "And 3+3?" }] - } - ] -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/messages/server_tools.anthropic.snap b/crates/persisting-gateway/tests/fixtures/requests/messages/server_tools.anthropic.snap deleted file mode 100644 index 37198e193..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/messages/server_tools.anthropic.snap +++ /dev/null @@ -1,60 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/messages/server_tools.json -info: - model: claude-opus-4-6 - max_tokens: 64 - messages: - - role: user - content: "Search the web for today's date" - tools: - - type: web_search_20250305 - name: web_search - max_uses: 5 - - name: get_weather - description: Get the weather - input_schema: - type: object ---- -{ - "request": { - "messages": [ - { - "role": "user", - "content": "Search the web for today's date" - } - ], - "model": "claude-opus-4-6", - "max_tokens": 64, - "tools": [ - { - "type": "web_search_20250305", - "name": "web_search", - "max_uses": 5 - }, - { - "name": "get_weather", - "description": "Get the weather", - "input_schema": { - "type": "object" - } - } - ] - }, - "parsed": { - "input_format": "Messages", - "cache_convention": "InputIncludesCache", - "request_model": "claude-opus-4-6", - "provider": "anthropic", - "streaming": false, - "params": { - "max_tokens": 64 - }, - "prompt": [ - { - "role": "user", - "content": "Search the web for today's date" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/messages/server_tools.bedrock.snap b/crates/persisting-gateway/tests/fixtures/requests/messages/server_tools.bedrock.snap deleted file mode 100644 index 18d5e8d50..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/messages/server_tools.bedrock.snap +++ /dev/null @@ -1,66 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/messages/server_tools.json -info: - model: claude-opus-4-6 - max_tokens: 64 - messages: - - role: user - content: "Search the web for today's date" - tools: - - type: web_search_20250305 - name: web_search - max_uses: 5 - - name: get_weather - description: Get the weather - input_schema: - type: object ---- -{ - "request": { - "messages": [ - { - "role": "user", - "content": [ - { - "text": "Search the web for today's date" - } - ] - } - ], - "inferenceConfig": { - "maxTokens": 64 - }, - "toolConfig": { - "tools": [ - { - "toolSpec": { - "name": "get_weather", - "description": "Get the weather", - "inputSchema": { - "json": { - "type": "object" - } - } - } - } - ] - } - }, - "parsed": { - "input_format": "Messages", - "cache_convention": "InputIncludesCache", - "request_model": "claude-opus-4-6", - "provider": "bedrock", - "streaming": false, - "params": { - "max_tokens": 64 - }, - "prompt": [ - { - "role": "user", - "content": "Search the web for today's date" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/messages/server_tools.completions.snap b/crates/persisting-gateway/tests/fixtures/requests/messages/server_tools.completions.snap deleted file mode 100644 index bedf3cbf0..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/messages/server_tools.completions.snap +++ /dev/null @@ -1,64 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/messages/server_tools.json -info: - model: claude-opus-4-6 - max_tokens: 64 - messages: - - role: user - content: "Search the web for today's date" - tools: - - type: web_search_20250305 - name: web_search - max_uses: 5 - - name: get_weather - description: Get the weather - input_schema: - type: object ---- -{ - "request": { - "messages": [ - { - "role": "user", - "content": [ - { - "type": "text", - "text": "Search the web for today's date" - } - ] - } - ], - "model": "claude-opus-4-6", - "max_completion_tokens": 64, - "stream": false, - "tools": [ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get the weather", - "parameters": { - "type": "object" - } - } - } - ] - }, - "parsed": { - "input_format": "Messages", - "cache_convention": "InputIncludesCache", - "request_model": "claude-opus-4-6", - "provider": "openai", - "streaming": false, - "params": { - "max_tokens": 64 - }, - "prompt": [ - { - "role": "user", - "content": "Search the web for today's date" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/messages/server_tools.json b/crates/persisting-gateway/tests/fixtures/requests/messages/server_tools.json deleted file mode 100644 index e15324ee7..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/messages/server_tools.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "model": "claude-opus-4-6", - "max_tokens": 64, - "messages": [ - { - "role": "user", - "content": "Search the web for today's date" - } - ], - "tools": [ - { - "type": "web_search_20250305", - "name": "web_search", - "max_uses": 5 - }, - { - "name": "get_weather", - "description": "Get the weather", - "input_schema": { - "type": "object" - } - } - ] -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/messages/server_tools.vertex.snap b/crates/persisting-gateway/tests/fixtures/requests/messages/server_tools.vertex.snap deleted file mode 100644 index f43399008..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/messages/server_tools.vertex.snap +++ /dev/null @@ -1,60 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/messages/server_tools.json -info: - model: claude-opus-4-6 - max_tokens: 64 - messages: - - role: user - content: "Search the web for today's date" - tools: - - type: web_search_20250305 - name: web_search - max_uses: 5 - - name: get_weather - description: Get the weather - input_schema: - type: object ---- -{ - "request": { - "messages": [ - { - "role": "user", - "content": "Search the web for today's date" - } - ], - "anthropic_version": "vertex-2023-10-16", - "max_tokens": 64, - "tools": [ - { - "type": "web_search_20250305", - "name": "web_search", - "max_uses": 5 - }, - { - "name": "get_weather", - "description": "Get the weather", - "input_schema": { - "type": "object" - } - } - ] - }, - "parsed": { - "input_format": "Messages", - "cache_convention": "InputIncludesCache", - "request_model": "claude-opus-4-6", - "provider": "vertex", - "streaming": false, - "params": { - "max_tokens": 64 - }, - "prompt": [ - { - "role": "user", - "content": "Search the web for today's date" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/messages/structured-output.anthropic.snap b/crates/persisting-gateway/tests/fixtures/requests/messages/structured-output.anthropic.snap deleted file mode 100644 index 540bb25c2..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/messages/structured-output.anthropic.snap +++ /dev/null @@ -1,76 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/messages/structured-output.json -info: - model: anthropic.claude-3-sonnet - max_tokens: 100 - temperature: 0.7 - top_p: 0.8 - top_k: 50 - output_config: - effort: high - format: - type: json_schema - schema: - type: object - properties: - answer: - type: number - required: - - answer - additionalProperties: false - messages: - - role: user - content: What is 2+2? ---- -{ - "request": { - "messages": [ - { - "role": "user", - "content": "What is 2+2?" - } - ], - "model": "anthropic.claude-3-sonnet", - "top_p": 0.8, - "temperature": 0.7, - "max_tokens": 100, - "top_k": 50, - "output_config": { - "effort": "high", - "format": { - "type": "json_schema", - "schema": { - "type": "object", - "properties": { - "answer": { - "type": "number" - } - }, - "required": [ - "answer" - ], - "additionalProperties": false - } - } - } - }, - "parsed": { - "input_format": "Messages", - "cache_convention": "InputIncludesCache", - "request_model": "anthropic.claude-3-sonnet", - "provider": "anthropic", - "streaming": false, - "params": { - "temperature": 0.699999988079071, - "top_p": 0.800000011920929, - "max_tokens": 100 - }, - "prompt": [ - { - "role": "user", - "content": "What is 2+2?" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/messages/structured-output.bedrock.snap b/crates/persisting-gateway/tests/fixtures/requests/messages/structured-output.bedrock.snap deleted file mode 100644 index e97d9f0fb..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/messages/structured-output.bedrock.snap +++ /dev/null @@ -1,78 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/messages/structured-output.json -info: - model: anthropic.claude-3-sonnet - max_tokens: 100 - temperature: 0.7 - top_p: 0.8 - top_k: 50 - output_config: - effort: high - format: - type: json_schema - schema: - type: object - properties: - answer: - type: number - required: - - answer - additionalProperties: false - messages: - - role: user - content: What is 2+2? ---- -{ - "request": { - "messages": [ - { - "role": "user", - "content": [ - { - "text": "What is 2+2?" - } - ] - } - ], - "inferenceConfig": { - "maxTokens": 100, - "temperature": 0.7, - "topP": 0.8 - }, - "outputConfig": { - "textFormat": { - "type": "json_schema", - "structure": { - "jsonSchema": { - "schema": "{\"type\":\"object\",\"properties\":{\"answer\":{\"type\":\"number\"}},\"required\":[\"answer\"],\"additionalProperties\":false}" - } - } - } - }, - "additionalModelRequestFields": { - "top_k": 50, - "output_config": { - "effort": "high" - } - } - }, - "parsed": { - "input_format": "Messages", - "cache_convention": "InputIncludesCache", - "request_model": "anthropic.claude-3-sonnet", - "provider": "bedrock", - "streaming": false, - "params": { - "temperature": 0.699999988079071, - "top_p": 0.800000011920929, - "max_tokens": 100 - }, - "prompt": [ - { - "role": "user", - "content": "What is 2+2?" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/messages/structured-output.completions.snap b/crates/persisting-gateway/tests/fixtures/requests/messages/structured-output.completions.snap deleted file mode 100644 index 5ac213fb1..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/messages/structured-output.completions.snap +++ /dev/null @@ -1,81 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/messages/structured-output.json -info: - model: anthropic.claude-3-sonnet - max_tokens: 100 - temperature: 0.7 - top_p: 0.8 - top_k: 50 - output_config: - effort: high - format: - type: json_schema - schema: - type: object - properties: - answer: - type: number - required: - - answer - additionalProperties: false - messages: - - role: user - content: What is 2+2? ---- -{ - "request": { - "messages": [ - { - "role": "user", - "content": [ - { - "type": "text", - "text": "What is 2+2?" - } - ] - } - ], - "model": "anthropic.claude-3-sonnet", - "max_completion_tokens": 100, - "response_format": { - "type": "json_schema", - "json_schema": { - "name": "structured_output", - "schema": { - "type": "object", - "properties": { - "answer": { - "type": "number" - } - }, - "required": [ - "answer" - ], - "additionalProperties": false - } - } - }, - "stream": false, - "temperature": 0.7, - "top_p": 0.8 - }, - "parsed": { - "input_format": "Messages", - "cache_convention": "InputIncludesCache", - "request_model": "anthropic.claude-3-sonnet", - "provider": "openai", - "streaming": false, - "params": { - "temperature": 0.699999988079071, - "top_p": 0.800000011920929, - "max_tokens": 100 - }, - "prompt": [ - { - "role": "user", - "content": "What is 2+2?" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/messages/structured-output.json b/crates/persisting-gateway/tests/fixtures/requests/messages/structured-output.json deleted file mode 100644 index c91d2142a..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/messages/structured-output.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "model": "anthropic.claude-3-sonnet", - "max_tokens": 100, - "temperature": 0.7, - "top_p": 0.8, - "top_k": 50, - "output_config": { - "effort": "high", - "format": { - "type": "json_schema", - "schema": { - "type": "object", - "properties": { - "answer": { "type": "number" } - }, - "required": ["answer"], - "additionalProperties": false - } - } - }, - "messages": [ - { - "role": "user", - "content": "What is 2+2?" - } - ] -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/messages/structured-output.vertex.snap b/crates/persisting-gateway/tests/fixtures/requests/messages/structured-output.vertex.snap deleted file mode 100644 index 55725ac7e..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/messages/structured-output.vertex.snap +++ /dev/null @@ -1,76 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/messages/structured-output.json -info: - model: anthropic.claude-3-sonnet - max_tokens: 100 - temperature: 0.7 - top_p: 0.8 - top_k: 50 - output_config: - effort: high - format: - type: json_schema - schema: - type: object - properties: - answer: - type: number - required: - - answer - additionalProperties: false - messages: - - role: user - content: What is 2+2? ---- -{ - "request": { - "messages": [ - { - "role": "user", - "content": "What is 2+2?" - } - ], - "anthropic_version": "vertex-2023-10-16", - "top_p": 0.8, - "temperature": 0.7, - "max_tokens": 100, - "top_k": 50, - "output_config": { - "effort": "high", - "format": { - "type": "json_schema", - "schema": { - "type": "object", - "properties": { - "answer": { - "type": "number" - } - }, - "required": [ - "answer" - ], - "additionalProperties": false - } - } - } - }, - "parsed": { - "input_format": "Messages", - "cache_convention": "InputIncludesCache", - "request_model": "anthropic.claude-3-sonnet", - "provider": "vertex", - "streaming": false, - "params": { - "temperature": 0.699999988079071, - "top_p": 0.800000011920929, - "max_tokens": 100 - }, - "prompt": [ - { - "role": "user", - "content": "What is 2+2?" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/messages/system_message.anthropic.snap b/crates/persisting-gateway/tests/fixtures/requests/messages/system_message.anthropic.snap deleted file mode 100644 index 3499eac9d..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/messages/system_message.anthropic.snap +++ /dev/null @@ -1,66 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/messages/system_message.json -info: - model: claude-opus-4-8 - max_tokens: 1024 - messages: - - role: user - content: - - type: text - text: "\nAs you answer, be concise." - - type: text - text: hola - cache_control: - type: ephemeral - - role: system - content: The following skills are available for use. ---- -{ - "request": { - "messages": [ - { - "role": "user", - "content": [ - { - "type": "text", - "text": "\nAs you answer, be concise." - }, - { - "type": "text", - "text": "hola", - "cache_control": { - "type": "ephemeral" - } - } - ] - }, - { - "role": "system", - "content": "The following skills are available for use." - } - ], - "model": "claude-opus-4-8", - "max_tokens": 1024 - }, - "parsed": { - "input_format": "Messages", - "cache_convention": "InputIncludesCache", - "request_model": "claude-opus-4-8", - "provider": "anthropic", - "streaming": false, - "params": { - "max_tokens": 1024 - }, - "prompt": [ - { - "role": "user", - "content": "\nAs you answer, be concise. hola" - }, - { - "role": "system", - "content": "The following skills are available for use." - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/messages/system_message.bedrock.snap b/crates/persisting-gateway/tests/fixtures/requests/messages/system_message.bedrock.snap deleted file mode 100644 index 9740054ca..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/messages/system_message.bedrock.snap +++ /dev/null @@ -1,68 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/messages/system_message.json -info: - model: claude-opus-4-8 - max_tokens: 1024 - messages: - - role: user - content: - - type: text - text: "\nAs you answer, be concise." - - type: text - text: hola - cache_control: - type: ephemeral - - role: system - content: The following skills are available for use. ---- -{ - "request": { - "messages": [ - { - "role": "user", - "content": [ - { - "text": "\nAs you answer, be concise." - }, - { - "text": "hola" - }, - { - "cachePoint": { - "type": "default" - } - } - ] - } - ], - "system": [ - { - "text": "The following skills are available for use." - } - ], - "inferenceConfig": { - "maxTokens": 1024 - } - }, - "parsed": { - "input_format": "Messages", - "cache_convention": "InputIncludesCache", - "request_model": "claude-opus-4-8", - "provider": "bedrock", - "streaming": false, - "params": { - "max_tokens": 1024 - }, - "prompt": [ - { - "role": "user", - "content": "\nAs you answer, be concise. hola" - }, - { - "role": "system", - "content": "The following skills are available for use." - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/messages/system_message.completions.snap b/crates/persisting-gateway/tests/fixtures/requests/messages/system_message.completions.snap deleted file mode 100644 index 39c94adbd..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/messages/system_message.completions.snap +++ /dev/null @@ -1,69 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/messages/system_message.json -info: - model: claude-opus-4-8 - max_tokens: 1024 - messages: - - role: user - content: - - type: text - text: "\nAs you answer, be concise." - - type: text - text: hola - cache_control: - type: ephemeral - - role: system - content: The following skills are available for use. ---- -{ - "request": { - "messages": [ - { - "role": "user", - "content": [ - { - "type": "text", - "text": "\nAs you answer, be concise." - }, - { - "type": "text", - "text": "hola" - } - ] - }, - { - "role": "system", - "content": [ - { - "type": "text", - "text": "The following skills are available for use." - } - ] - } - ], - "model": "claude-opus-4-8", - "max_completion_tokens": 1024, - "stream": false - }, - "parsed": { - "input_format": "Messages", - "cache_convention": "InputIncludesCache", - "request_model": "claude-opus-4-8", - "provider": "openai", - "streaming": false, - "params": { - "max_tokens": 1024 - }, - "prompt": [ - { - "role": "user", - "content": "\nAs you answer, be concise. hola" - }, - { - "role": "system", - "content": "The following skills are available for use." - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/messages/system_message.json b/crates/persisting-gateway/tests/fixtures/requests/messages/system_message.json deleted file mode 100644 index a52e3ee56..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/messages/system_message.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "model": "claude-opus-4-8", - "max_tokens": 1024, - "messages": [ - { - "role": "user", - "content": [ - { - "type": "text", - "text": "\nAs you answer, be concise." - }, - { - "type": "text", - "text": "hola", - "cache_control": { - "type": "ephemeral" - } - } - ] - }, - { - "role": "system", - "content": "The following skills are available for use." - } - ] -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/messages/system_message.vertex.snap b/crates/persisting-gateway/tests/fixtures/requests/messages/system_message.vertex.snap deleted file mode 100644 index 727e7eae1..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/messages/system_message.vertex.snap +++ /dev/null @@ -1,66 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/messages/system_message.json -info: - model: claude-opus-4-8 - max_tokens: 1024 - messages: - - role: user - content: - - type: text - text: "\nAs you answer, be concise." - - type: text - text: hola - cache_control: - type: ephemeral - - role: system - content: The following skills are available for use. ---- -{ - "request": { - "messages": [ - { - "role": "user", - "content": [ - { - "type": "text", - "text": "\nAs you answer, be concise." - }, - { - "type": "text", - "text": "hola", - "cache_control": { - "type": "ephemeral" - } - } - ] - }, - { - "role": "system", - "content": "The following skills are available for use." - } - ], - "anthropic_version": "vertex-2023-10-16", - "max_tokens": 1024 - }, - "parsed": { - "input_format": "Messages", - "cache_convention": "InputIncludesCache", - "request_model": "claude-opus-4-8", - "provider": "vertex", - "streaming": false, - "params": { - "max_tokens": 1024 - }, - "prompt": [ - { - "role": "user", - "content": "\nAs you answer, be concise. hola" - }, - { - "role": "system", - "content": "The following skills are available for use." - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/messages/tool_history_without_tools.bedrock.snap b/crates/persisting-gateway/tests/fixtures/requests/messages/tool_history_without_tools.bedrock.snap deleted file mode 100644 index 37638fe25..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/messages/tool_history_without_tools.bedrock.snap +++ /dev/null @@ -1,94 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/messages/tool_history_without_tools.json -info: - model: "anthropic.claude-sonnet-4-20250514-v1:0" - max_tokens: 1024 - messages: - - role: assistant - content: - - type: tool_use - id: call_1 - name: get_weather - input: - location: Seattle - - role: user - content: - - type: tool_result - tool_use_id: call_1 - content: rain ---- -{ - "request": { - "messages": [ - { - "role": "assistant", - "content": [ - { - "toolUse": { - "toolUseId": "call_1", - "name": "get_weather", - "input": { - "location": "Seattle" - } - } - } - ] - }, - { - "role": "user", - "content": [ - { - "toolResult": { - "toolUseId": "call_1", - "content": [ - { - "text": "rain" - } - ], - "status": null - } - } - ] - } - ], - "inferenceConfig": { - "maxTokens": 1024 - }, - "toolConfig": { - "tools": [ - { - "toolSpec": { - "name": "agentgateway_dummy_do_not_call", - "description": null, - "inputSchema": { - "json": { - "type": "object" - } - } - } - } - ] - } - }, - "parsed": { - "input_format": "Messages", - "cache_convention": "InputIncludesCache", - "request_model": "anthropic.claude-sonnet-4-20250514-v1:0", - "provider": "bedrock", - "streaming": false, - "params": { - "max_tokens": 1024 - }, - "prompt": [ - { - "role": "assistant", - "content": "" - }, - { - "role": "user", - "content": "" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/messages/tool_history_without_tools.json b/crates/persisting-gateway/tests/fixtures/requests/messages/tool_history_without_tools.json deleted file mode 100644 index eb850cae6..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/messages/tool_history_without_tools.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "model": "anthropic.claude-sonnet-4-20250514-v1:0", - "max_tokens": 1024, - "messages": [ - { - "role": "assistant", - "content": [ - { - "type": "tool_use", - "id": "call_1", - "name": "get_weather", - "input": { - "location": "Seattle" - } - } - ] - }, - { - "role": "user", - "content": [ - { - "type": "tool_result", - "tool_use_id": "call_1", - "content": "rain" - } - ] - } - ] -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/messages/tools.anthropic.snap b/crates/persisting-gateway/tests/fixtures/requests/messages/tools.anthropic.snap deleted file mode 100644 index de4c06a0d..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/messages/tools.anthropic.snap +++ /dev/null @@ -1,67 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/messages/tools.json -info: - model: claude-opus-4-6 - max_tokens: 1024 - tools: - - name: get_weather - description: Get the current weather in a given location - input_schema: - type: object - properties: - location: - type: string - description: "The city and state, e.g. San Francisco, CA" - required: - - location - messages: - - role: user - content: What is the weather like in San Francisco? ---- -{ - "request": { - "messages": [ - { - "role": "user", - "content": "What is the weather like in San Francisco?" - } - ], - "model": "claude-opus-4-6", - "max_tokens": 1024, - "tools": [ - { - "name": "get_weather", - "description": "Get the current weather in a given location", - "input_schema": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA" - } - }, - "required": [ - "location" - ] - } - } - ] - }, - "parsed": { - "input_format": "Messages", - "cache_convention": "InputIncludesCache", - "request_model": "claude-opus-4-6", - "provider": "anthropic", - "streaming": false, - "params": { - "max_tokens": 1024 - }, - "prompt": [ - { - "role": "user", - "content": "What is the weather like in San Francisco?" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/messages/tools.bedrock.snap b/crates/persisting-gateway/tests/fixtures/requests/messages/tools.bedrock.snap deleted file mode 100644 index 57f9df7de..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/messages/tools.bedrock.snap +++ /dev/null @@ -1,78 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/messages/tools.json -info: - model: claude-opus-4-6 - max_tokens: 1024 - tools: - - name: get_weather - description: Get the current weather in a given location - input_schema: - type: object - properties: - location: - type: string - description: "The city and state, e.g. San Francisco, CA" - required: - - location - messages: - - role: user - content: What is the weather like in San Francisco? ---- -{ - "request": { - "messages": [ - { - "role": "user", - "content": [ - { - "text": "What is the weather like in San Francisco?" - } - ] - } - ], - "inferenceConfig": { - "maxTokens": 1024 - }, - "toolConfig": { - "tools": [ - { - "toolSpec": { - "name": "get_weather", - "description": "Get the current weather in a given location", - "inputSchema": { - "json": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA" - } - }, - "required": [ - "location" - ] - } - } - } - } - ] - } - }, - "parsed": { - "input_format": "Messages", - "cache_convention": "InputIncludesCache", - "request_model": "claude-opus-4-6", - "provider": "bedrock", - "streaming": false, - "params": { - "max_tokens": 1024 - }, - "prompt": [ - { - "role": "user", - "content": "What is the weather like in San Francisco?" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/messages/tools.completions.snap b/crates/persisting-gateway/tests/fixtures/requests/messages/tools.completions.snap deleted file mode 100644 index e1709389d..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/messages/tools.completions.snap +++ /dev/null @@ -1,76 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/messages/tools.json -info: - model: claude-opus-4-6 - max_tokens: 1024 - tools: - - name: get_weather - description: Get the current weather in a given location - input_schema: - type: object - properties: - location: - type: string - description: "The city and state, e.g. San Francisco, CA" - required: - - location - messages: - - role: user - content: What is the weather like in San Francisco? ---- -{ - "request": { - "messages": [ - { - "role": "user", - "content": [ - { - "type": "text", - "text": "What is the weather like in San Francisco?" - } - ] - } - ], - "model": "claude-opus-4-6", - "max_completion_tokens": 1024, - "stream": false, - "tools": [ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get the current weather in a given location", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA" - } - }, - "required": [ - "location" - ] - } - } - } - ] - }, - "parsed": { - "input_format": "Messages", - "cache_convention": "InputIncludesCache", - "request_model": "claude-opus-4-6", - "provider": "openai", - "streaming": false, - "params": { - "max_tokens": 1024 - }, - "prompt": [ - { - "role": "user", - "content": "What is the weather like in San Francisco?" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/messages/tools.json b/crates/persisting-gateway/tests/fixtures/requests/messages/tools.json deleted file mode 100644 index 48ea9fc44..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/messages/tools.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "model": "claude-opus-4-6", - "max_tokens": 1024, - "tools": [ - { - "name": "get_weather", - "description": "Get the current weather in a given location", - "input_schema": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA" - } - }, - "required": [ - "location" - ] - } - } - ], - "messages": [ - { - "role": "user", - "content": "What is the weather like in San Francisco?" - } - ] -} \ No newline at end of file diff --git a/crates/persisting-gateway/tests/fixtures/requests/messages/tools.vertex.snap b/crates/persisting-gateway/tests/fixtures/requests/messages/tools.vertex.snap deleted file mode 100644 index 2e34ad275..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/messages/tools.vertex.snap +++ /dev/null @@ -1,67 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/messages/tools.json -info: - model: claude-opus-4-6 - max_tokens: 1024 - tools: - - name: get_weather - description: Get the current weather in a given location - input_schema: - type: object - properties: - location: - type: string - description: "The city and state, e.g. San Francisco, CA" - required: - - location - messages: - - role: user - content: What is the weather like in San Francisco? ---- -{ - "request": { - "messages": [ - { - "role": "user", - "content": "What is the weather like in San Francisco?" - } - ], - "anthropic_version": "vertex-2023-10-16", - "max_tokens": 1024, - "tools": [ - { - "name": "get_weather", - "description": "Get the current weather in a given location", - "input_schema": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA" - } - }, - "required": [ - "location" - ] - } - } - ] - }, - "parsed": { - "input_format": "Messages", - "cache_convention": "InputIncludesCache", - "request_model": "claude-opus-4-6", - "provider": "vertex", - "streaming": false, - "params": { - "max_tokens": 1024 - }, - "prompt": [ - { - "role": "user", - "content": "What is the weather like in San Francisco?" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/policies/anthropic_with_system.anthropic.snap b/crates/persisting-gateway/tests/fixtures/requests/policies/anthropic_with_system.anthropic.snap deleted file mode 100644 index fe1cb1e21..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/policies/anthropic_with_system.anthropic.snap +++ /dev/null @@ -1,89 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/policies/anthropic_with_system.json -info: - model: claude-sonnet-4-20250514 - max_tokens: 1024 - system: system prompt - messages: - - role: user - content: "Hello, world" ---- -{ - "request": { - "messages": [ - { - "role": "user", - "content": "prepend user message" - }, - { - "role": "assistant", - "content": "prepend assistant message" - }, - { - "role": "user", - "content": "Hello, world" - }, - { - "role": "user", - "content": "append user message" - }, - { - "role": "assistant", - "content": "append assistant prompt" - } - ], - "model": "claude-sonnet-4-20250514", - "system": [ - { - "type": "text", - "text": "prepend system prompt" - }, - { - "type": "text", - "text": "system prompt" - }, - { - "type": "text", - "text": "append system prompt" - } - ], - "max_tokens": 1024 - }, - "parsed": { - "input_format": "Messages", - "cache_convention": "InputIncludesCache", - "request_model": "claude-sonnet-4-20250514", - "provider": "anthropic", - "streaming": false, - "params": { - "max_tokens": 1024 - }, - "prompt": [ - { - "role": "system", - "content": "prepend system prompt\nsystem prompt\nappend system prompt" - }, - { - "role": "user", - "content": "prepend user message" - }, - { - "role": "assistant", - "content": "prepend assistant message" - }, - { - "role": "user", - "content": "Hello, world" - }, - { - "role": "user", - "content": "append user message" - }, - { - "role": "assistant", - "content": "append assistant prompt" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/policies/anthropic_with_system.json b/crates/persisting-gateway/tests/fixtures/requests/policies/anthropic_with_system.json deleted file mode 100644 index cf75b7141..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/policies/anthropic_with_system.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "model": "claude-sonnet-4-20250514", - "max_tokens": 1024, - "system": "system prompt", - "messages": [{ "role": "user", "content": "Hello, world" }] -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/policies/openai_with_inputs.json b/crates/persisting-gateway/tests/fixtures/requests/policies/openai_with_inputs.json deleted file mode 100644 index 27cbc7cea..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/policies/openai_with_inputs.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "model": "gpt-4o", - "max_output_tokens": 1024, - "input": [ - { - "type": "message", - "role": "user", - "content": [{ "type": "input_text", "text": "Hello, world" }] - } - ] -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/policies/openai_with_inputs.openai.snap b/crates/persisting-gateway/tests/fixtures/requests/policies/openai_with_inputs.openai.snap deleted file mode 100644 index bca8d4f66..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/policies/openai_with_inputs.openai.snap +++ /dev/null @@ -1,121 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/policies/openai_with_inputs.json -info: - model: gpt-4o - max_output_tokens: 1024 - input: - - type: message - role: user - content: - - type: input_text - text: "Hello, world" ---- -{ - "request": { - "input": [ - { - "type": "message", - "content": [ - { - "type": "input_text", - "text": "prepend system prompt" - } - ], - "role": "system" - }, - { - "type": "message", - "content": [ - { - "type": "input_text", - "text": "prepend user message" - } - ], - "role": "user" - }, - { - "type": "message", - "role": "assistant", - "content": "prepend assistant message" - }, - { - "type": "message", - "role": "user", - "content": [ - { - "type": "input_text", - "text": "Hello, world" - } - ] - }, - { - "type": "message", - "content": [ - { - "type": "input_text", - "text": "append user message" - } - ], - "role": "user" - }, - { - "type": "message", - "content": [ - { - "type": "input_text", - "text": "append system prompt" - } - ], - "role": "system" - }, - { - "type": "message", - "role": "assistant", - "content": "append assistant prompt" - } - ], - "model": "gpt-4o", - "max_output_tokens": 1024 - }, - "parsed": { - "input_format": "Responses", - "cache_convention": "InputIncludesCache", - "request_model": "gpt-4o", - "provider": "openai", - "streaming": false, - "params": { - "max_tokens": 1024 - }, - "prompt": [ - { - "role": "system", - "content": "prepend system prompt" - }, - { - "role": "user", - "content": "prepend user message" - }, - { - "role": "assistant", - "content": "prepend assistant message" - }, - { - "role": "user", - "content": "Hello, world" - }, - { - "role": "user", - "content": "append user message" - }, - { - "role": "system", - "content": "append system prompt" - }, - { - "role": "assistant", - "content": "append assistant prompt" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/policies/openai_with_messages.json b/crates/persisting-gateway/tests/fixtures/requests/policies/openai_with_messages.json deleted file mode 100644 index 449bb4e49..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/policies/openai_with_messages.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "model": "gpt-4o", - "max_tokens": 1024, - "messages": [ - { - "role": "user", - "content": "Hello, world" - } - ] -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/policies/openai_with_messages.openai.snap b/crates/persisting-gateway/tests/fixtures/requests/policies/openai_with_messages.openai.snap deleted file mode 100644 index ab6da6603..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/policies/openai_with_messages.openai.snap +++ /dev/null @@ -1,86 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/policies/openai_with_messages.json -info: - model: gpt-4o - max_tokens: 1024 - messages: - - role: user - content: "Hello, world" ---- -{ - "request": { - "messages": [ - { - "role": "system", - "content": "prepend system prompt" - }, - { - "role": "user", - "content": "prepend user message" - }, - { - "role": "assistant", - "content": "prepend assistant message" - }, - { - "role": "user", - "content": "Hello, world" - }, - { - "role": "user", - "content": "append user message" - }, - { - "role": "system", - "content": "append system prompt" - }, - { - "role": "assistant", - "content": "append assistant prompt" - } - ], - "model": "gpt-4o", - "max_tokens": 1024 - }, - "parsed": { - "input_format": "Completions", - "cache_convention": "InputIncludesCache", - "request_model": "gpt-4o", - "provider": "openai", - "streaming": false, - "params": { - "max_tokens": 1024 - }, - "prompt": [ - { - "role": "system", - "content": "prepend system prompt" - }, - { - "role": "user", - "content": "prepend user message" - }, - { - "role": "assistant", - "content": "prepend assistant message" - }, - { - "role": "user", - "content": "Hello, world" - }, - { - "role": "user", - "content": "append user message" - }, - { - "role": "system", - "content": "append system prompt" - }, - { - "role": "assistant", - "content": "append assistant prompt" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/policies/openai_with_text_input.json b/crates/persisting-gateway/tests/fixtures/requests/policies/openai_with_text_input.json deleted file mode 100644 index cb6dc99d6..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/policies/openai_with_text_input.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "model": "gpt-4o", - "max_output_tokens": 1024, - "input": "Hello, world" -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/policies/openai_with_text_input.openai.snap b/crates/persisting-gateway/tests/fixtures/requests/policies/openai_with_text_input.openai.snap deleted file mode 100644 index 07833e643..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/policies/openai_with_text_input.openai.snap +++ /dev/null @@ -1,116 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/policies/openai_with_text_input.json -info: - model: gpt-4o - max_output_tokens: 1024 - input: "Hello, world" ---- -{ - "request": { - "input": [ - { - "type": "message", - "content": [ - { - "type": "input_text", - "text": "prepend system prompt" - } - ], - "role": "system" - }, - { - "type": "message", - "content": [ - { - "type": "input_text", - "text": "prepend user message" - } - ], - "role": "user" - }, - { - "type": "message", - "role": "assistant", - "content": "prepend assistant message" - }, - { - "type": "message", - "content": [ - { - "type": "input_text", - "text": "Hello, world" - } - ], - "role": "user" - }, - { - "type": "message", - "content": [ - { - "type": "input_text", - "text": "append user message" - } - ], - "role": "user" - }, - { - "type": "message", - "content": [ - { - "type": "input_text", - "text": "append system prompt" - } - ], - "role": "system" - }, - { - "type": "message", - "role": "assistant", - "content": "append assistant prompt" - } - ], - "model": "gpt-4o", - "max_output_tokens": 1024 - }, - "parsed": { - "input_format": "Responses", - "cache_convention": "InputIncludesCache", - "request_model": "gpt-4o", - "provider": "openai", - "streaming": false, - "params": { - "max_tokens": 1024 - }, - "prompt": [ - { - "role": "system", - "content": "prepend system prompt" - }, - { - "role": "user", - "content": "prepend user message" - }, - { - "role": "assistant", - "content": "prepend assistant message" - }, - { - "role": "user", - "content": "Hello, world" - }, - { - "role": "user", - "content": "append user message" - }, - { - "role": "system", - "content": "append system prompt" - }, - { - "role": "assistant", - "content": "append assistant prompt" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/rerank/basic.bedrock.snap b/crates/persisting-gateway/tests/fixtures/requests/rerank/basic.bedrock.snap deleted file mode 100644 index b34e85457..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/rerank/basic.bedrock.snap +++ /dev/null @@ -1,70 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/rerank/basic.json -info: - model: rerank-v3.5 - query: What is the capital of the United States? - documents: - - Carson City is the capital city of the American state of Nevada. - - text: "Washington, D.C. is the capital of the United States." - - Capital punishment has existed in the United States since colonial times. - top_n: 2 ---- -{ - "request": { - "queries": [ - { - "type": "TEXT", - "textQuery": { - "text": "What is the capital of the United States?" - } - } - ], - "sources": [ - { - "type": "INLINE", - "inlineDocumentSource": { - "type": "TEXT", - "textDocument": { - "text": "Carson City is the capital city of the American state of Nevada." - } - } - }, - { - "type": "INLINE", - "inlineDocumentSource": { - "type": "TEXT", - "textDocument": { - "text": "Washington, D.C. is the capital of the United States." - } - } - }, - { - "type": "INLINE", - "inlineDocumentSource": { - "type": "TEXT", - "textDocument": { - "text": "Capital punishment has existed in the United States since colonial times." - } - } - } - ], - "rerankingConfiguration": { - "type": "BEDROCK_RERANKING_MODEL", - "bedrockRerankingConfiguration": { - "modelConfiguration": { - "modelArn": "arn:aws:bedrock:us-west-2::foundation-model/cohere.rerank-v3-5:0" - }, - "numberOfResults": 2 - } - } - }, - "parsed": { - "input_format": "Rerank", - "cache_convention": "InputIncludesCache", - "request_model": "rerank-v3.5", - "provider": "bedrock", - "streaming": false, - "params": {} - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/rerank/basic.cohere.snap b/crates/persisting-gateway/tests/fixtures/requests/rerank/basic.cohere.snap deleted file mode 100644 index edae69c90..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/rerank/basic.cohere.snap +++ /dev/null @@ -1,34 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/rerank/basic.json -info: - model: rerank-v3.5 - query: What is the capital of the United States? - documents: - - Carson City is the capital city of the American state of Nevada. - - text: "Washington, D.C. is the capital of the United States." - - Capital punishment has existed in the United States since colonial times. - top_n: 2 ---- -{ - "request": { - "model": "rerank-v3.5", - "query": "What is the capital of the United States?", - "documents": [ - "Carson City is the capital city of the American state of Nevada.", - { - "text": "Washington, D.C. is the capital of the United States." - }, - "Capital punishment has existed in the United States since colonial times." - ], - "top_n": 2 - }, - "parsed": { - "input_format": "Rerank", - "cache_convention": "InputIncludesCache", - "request_model": "rerank-v3.5", - "provider": "cohere", - "streaming": false, - "params": {} - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/rerank/basic.json b/crates/persisting-gateway/tests/fixtures/requests/rerank/basic.json deleted file mode 100644 index b568cb489..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/rerank/basic.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "model": "rerank-v3.5", - "query": "What is the capital of the United States?", - "documents": [ - "Carson City is the capital city of the American state of Nevada.", - { "text": "Washington, D.C. is the capital of the United States." }, - "Capital punishment has existed in the United States since colonial times." - ], - "top_n": 2 -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/rerank/basic.vertex.snap b/crates/persisting-gateway/tests/fixtures/requests/rerank/basic.vertex.snap deleted file mode 100644 index 1c9185e05..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/rerank/basic.vertex.snap +++ /dev/null @@ -1,42 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/rerank/basic.json -info: - model: rerank-v3.5 - query: What is the capital of the United States? - documents: - - Carson City is the capital city of the American state of Nevada. - - text: "Washington, D.C. is the capital of the United States." - - Capital punishment has existed in the United States since colonial times. - top_n: 2 ---- -{ - "request": { - "model": "rerank-v3.5", - "query": "What is the capital of the United States?", - "records": [ - { - "id": "0", - "content": "Carson City is the capital city of the American state of Nevada." - }, - { - "id": "1", - "content": "Washington, D.C. is the capital of the United States." - }, - { - "id": "2", - "content": "Capital punishment has existed in the United States since colonial times." - } - ], - "topN": 2, - "ignoreRecordDetailsInResponse": true - }, - "parsed": { - "input_format": "Rerank", - "cache_convention": "InputIncludesCache", - "request_model": "rerank-v3.5", - "provider": "vertex", - "streaming": false, - "params": {} - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/rerank/passthrough-fields.bedrock.snap b/crates/persisting-gateway/tests/fixtures/requests/rerank/passthrough-fields.bedrock.snap deleted file mode 100644 index de36e0ea8..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/rerank/passthrough-fields.bedrock.snap +++ /dev/null @@ -1,56 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/rerank/passthrough-fields.json -info: - model: rerank-v3.5 - query: q - documents: - - a - max_tokens_per_doc: 512 - return_documents: false - truncation: END ---- -{ - "request": { - "queries": [ - { - "type": "TEXT", - "textQuery": { - "text": "q" - } - } - ], - "sources": [ - { - "type": "INLINE", - "inlineDocumentSource": { - "type": "TEXT", - "textDocument": { - "text": "a" - } - } - } - ], - "rerankingConfiguration": { - "type": "BEDROCK_RERANKING_MODEL", - "bedrockRerankingConfiguration": { - "modelConfiguration": { - "modelArn": "arn:aws:bedrock:us-west-2::foundation-model/cohere.rerank-v3-5:0", - "additionalModelRequestFields": { - "truncation": "END", - "max_tokens_per_doc": 512 - } - }, - "numberOfResults": 1 - } - } - }, - "parsed": { - "input_format": "Rerank", - "cache_convention": "InputIncludesCache", - "request_model": "rerank-v3.5", - "provider": "bedrock", - "streaming": false, - "params": {} - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/rerank/passthrough-fields.cohere.snap b/crates/persisting-gateway/tests/fixtures/requests/rerank/passthrough-fields.cohere.snap deleted file mode 100644 index 31ac48b88..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/rerank/passthrough-fields.cohere.snap +++ /dev/null @@ -1,32 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/rerank/passthrough-fields.json -info: - model: rerank-v3.5 - query: q - documents: - - a - max_tokens_per_doc: 512 - return_documents: false - truncation: END ---- -{ - "request": { - "model": "rerank-v3.5", - "query": "q", - "documents": [ - "a" - ], - "return_documents": false, - "max_tokens_per_doc": 512, - "truncation": "END" - }, - "parsed": { - "input_format": "Rerank", - "cache_convention": "InputIncludesCache", - "request_model": "rerank-v3.5", - "provider": "cohere", - "streaming": false, - "params": {} - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/rerank/passthrough-fields.json b/crates/persisting-gateway/tests/fixtures/requests/rerank/passthrough-fields.json deleted file mode 100644 index 0cfcb3f17..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/rerank/passthrough-fields.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "model": "rerank-v3.5", - "query": "q", - "documents": ["a"], - "max_tokens_per_doc": 512, - "return_documents": false, - "truncation": "END" -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/rerank/passthrough-fields.vertex.snap b/crates/persisting-gateway/tests/fixtures/requests/rerank/passthrough-fields.vertex.snap deleted file mode 100644 index 2d9daa83c..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/rerank/passthrough-fields.vertex.snap +++ /dev/null @@ -1,33 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/rerank/passthrough-fields.json -info: - model: rerank-v3.5 - query: q - documents: - - a - max_tokens_per_doc: 512 - return_documents: false - truncation: END ---- -{ - "request": { - "model": "rerank-v3.5", - "query": "q", - "records": [ - { - "id": "0", - "content": "a" - } - ], - "ignoreRecordDetailsInResponse": true - }, - "parsed": { - "input_format": "Rerank", - "cache_convention": "InputIncludesCache", - "request_model": "rerank-v3.5", - "provider": "vertex", - "streaming": false, - "params": {} - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/responses/assistant-history.get-messages-responses.snap b/crates/persisting-gateway/tests/fixtures/requests/responses/assistant-history.get-messages-responses.snap deleted file mode 100644 index 6e3a250aa..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/responses/assistant-history.get-messages-responses.snap +++ /dev/null @@ -1,30 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/responses/assistant-history.json -info: - model: gpt-4o - input: - - type: message - role: assistant - phase: final_answer - content: - - type: output_text - text: Here is the answer. - - type: refusal - text: I cannot help with that part. - - type: message - role: user - content: - - type: input_text - text: Follow up question. ---- -[ - { - "role": "assistant", - "content": "Here is the answer." - }, - { - "role": "user", - "content": "Follow up question." - } -] diff --git a/crates/persisting-gateway/tests/fixtures/requests/responses/assistant-history.json b/crates/persisting-gateway/tests/fixtures/requests/responses/assistant-history.json deleted file mode 100644 index 0df557f25..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/responses/assistant-history.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "gpt-4o", - "input": [ - { - "type": "message", - "role": "assistant", - "phase": "final_answer", - "content": [ - {"type": "output_text", "text": "Here is the answer."}, - {"type": "refusal", "text": "I cannot help with that part."} - ] - }, - { - "type": "message", - "role": "user", - "content": [{"type": "input_text", "text": "Follow up question."}] - } - ] -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/responses/assistant-history.openai.snap b/crates/persisting-gateway/tests/fixtures/requests/responses/assistant-history.openai.snap deleted file mode 100644 index c0b7cd74c..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/responses/assistant-history.openai.snap +++ /dev/null @@ -1,144 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/responses/assistant-history.json -info: - model: gpt-4o - input: - - type: message - role: assistant - phase: final_answer - content: - - type: output_text - text: Here is the answer. - - type: refusal - text: I cannot help with that part. - - type: message - role: user - content: - - type: input_text - text: Follow up question. ---- -{ - "request": { - "input": [ - { - "type": "message", - "content": [ - { - "type": "input_text", - "text": "prepend system prompt" - } - ], - "role": "system" - }, - { - "type": "message", - "content": [ - { - "type": "input_text", - "text": "prepend user message" - } - ], - "role": "user" - }, - { - "type": "message", - "role": "assistant", - "content": "prepend assistant message" - }, - { - "type": "message", - "role": "assistant", - "phase": "final_answer", - "content": [ - { - "type": "output_text", - "text": "Here is the answer." - }, - { - "type": "refusal", - "text": "I cannot help with that part." - } - ] - }, - { - "type": "message", - "role": "user", - "content": [ - { - "type": "input_text", - "text": "Follow up question." - } - ] - }, - { - "type": "message", - "content": [ - { - "type": "input_text", - "text": "append user message" - } - ], - "role": "user" - }, - { - "type": "message", - "content": [ - { - "type": "input_text", - "text": "append system prompt" - } - ], - "role": "system" - }, - { - "type": "message", - "role": "assistant", - "content": "append assistant prompt" - } - ], - "model": "gpt-4o" - }, - "parsed": { - "input_format": "Responses", - "cache_convention": "InputIncludesCache", - "request_model": "gpt-4o", - "provider": "openai", - "streaming": false, - "params": {}, - "prompt": [ - { - "role": "system", - "content": "prepend system prompt" - }, - { - "role": "user", - "content": "prepend user message" - }, - { - "role": "assistant", - "content": "prepend assistant message" - }, - { - "role": "assistant", - "content": "Here is the answer." - }, - { - "role": "user", - "content": "Follow up question." - }, - { - "role": "user", - "content": "append user message" - }, - { - "role": "system", - "content": "append system prompt" - }, - { - "role": "assistant", - "content": "append assistant prompt" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/responses/basic.bedrock.snap b/crates/persisting-gateway/tests/fixtures/requests/responses/basic.bedrock.snap deleted file mode 100644 index beeb3f458..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/responses/basic.bedrock.snap +++ /dev/null @@ -1,41 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/responses/basic.json -info: - model: gpt-4o - max_output_tokens: 1024 - input: "Hello, world" ---- -{ - "request": { - "messages": [ - { - "role": "user", - "content": [ - { - "text": "Hello, world" - } - ] - } - ], - "inferenceConfig": { - "maxTokens": 1024 - } - }, - "parsed": { - "input_format": "Responses", - "cache_convention": "InputIncludesCache", - "request_model": "gpt-4o", - "provider": "bedrock", - "streaming": false, - "params": { - "max_tokens": 1024 - }, - "prompt": [ - { - "role": "user", - "content": "Hello, world" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/responses/basic.gemini.snap b/crates/persisting-gateway/tests/fixtures/requests/responses/basic.gemini.snap deleted file mode 100644 index f2ea6c82b..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/responses/basic.gemini.snap +++ /dev/null @@ -1,42 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/responses/basic.json -info: - model: gpt-4o - max_output_tokens: 1024 - input: "Hello, world" ---- -{ - "request": { - "messages": [ - { - "role": "user", - "content": [ - { - "type": "text", - "text": "Hello, world" - } - ] - } - ], - "model": "gpt-4o", - "max_completion_tokens": 1024, - "stream": false - }, - "parsed": { - "input_format": "Responses", - "cache_convention": "InputIncludesCache", - "request_model": "gpt-4o", - "provider": "gemini", - "streaming": false, - "params": { - "max_tokens": 1024 - }, - "prompt": [ - { - "role": "user", - "content": "Hello, world" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/responses/basic.json b/crates/persisting-gateway/tests/fixtures/requests/responses/basic.json deleted file mode 100644 index cb6dc99d6..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/responses/basic.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "model": "gpt-4o", - "max_output_tokens": 1024, - "input": "Hello, world" -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/responses/cache_control.gemini.snap b/crates/persisting-gateway/tests/fixtures/requests/responses/cache_control.gemini.snap deleted file mode 100644 index 834833c4c..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/responses/cache_control.gemini.snap +++ /dev/null @@ -1,69 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/responses/cache_control.json -info: - model: gpt-5 - input: - - role: user - content: - - type: input_text - text: Cached user input - prompt_cache_breakpoint: - mode: explicit - - role: system - content: - - type: input_text - text: Cached system input - prompt_cache_breakpoint: - mode: explicit ---- -{ - "request": { - "messages": [ - { - "role": "user", - "content": [ - { - "type": "text", - "text": "Cached user input", - "prompt_cache_breakpoint": { - "mode": "explicit" - } - } - ] - }, - { - "role": "developer", - "content": [ - { - "type": "text", - "text": "Cached system input", - "prompt_cache_breakpoint": { - "mode": "explicit" - } - } - ] - } - ], - "model": "gpt-5", - "stream": false - }, - "parsed": { - "input_format": "Responses", - "cache_convention": "InputIncludesCache", - "request_model": "gpt-5", - "provider": "gemini", - "streaming": false, - "params": {}, - "prompt": [ - { - "role": "user", - "content": "Cached user input" - }, - { - "role": "system", - "content": "Cached system input" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/responses/cache_control.json b/crates/persisting-gateway/tests/fixtures/requests/responses/cache_control.json deleted file mode 100644 index ec97de25f..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/responses/cache_control.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "model": "gpt-5", - "input": [ - { - "role": "user", - "content": [ - { - "type": "input_text", - "text": "Cached user input", - "prompt_cache_breakpoint": { - "mode": "explicit" - } - } - ] - }, - { - "role": "system", - "content": [ - { - "type": "input_text", - "text": "Cached system input", - "prompt_cache_breakpoint": { - "mode": "explicit" - } - } - ] - } - ] -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/responses/input-list.bedrock.snap b/crates/persisting-gateway/tests/fixtures/requests/responses/input-list.bedrock.snap deleted file mode 100644 index 9ca934105..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/responses/input-list.bedrock.snap +++ /dev/null @@ -1,41 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/responses/input-list.json -info: - input: - - role: user - content: Hello - model: gpt-4o - stream: true ---- -{ - "request": { - "messages": [ - { - "role": "user", - "content": [ - { - "text": "Hello" - } - ] - } - ], - "inferenceConfig": { - "maxTokens": 4096 - } - }, - "parsed": { - "input_format": "Responses", - "cache_convention": "InputIncludesCache", - "request_model": "gpt-4o", - "provider": "bedrock", - "streaming": true, - "params": {}, - "prompt": [ - { - "role": "user", - "content": "Hello" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/responses/input-list.gemini.snap b/crates/persisting-gateway/tests/fixtures/requests/responses/input-list.gemini.snap deleted file mode 100644 index 24e05e097..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/responses/input-list.gemini.snap +++ /dev/null @@ -1,39 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/responses/input-list.json -info: - input: - - role: user - content: Hello - model: gpt-4o - stream: true ---- -{ - "request": { - "messages": [ - { - "role": "user", - "content": "Hello" - } - ], - "model": "gpt-4o", - "stream": true, - "stream_options": { - "include_usage": true - } - }, - "parsed": { - "input_format": "Responses", - "cache_convention": "InputIncludesCache", - "request_model": "gpt-4o", - "provider": "gemini", - "streaming": true, - "params": {}, - "prompt": [ - { - "role": "user", - "content": "Hello" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/responses/input-list.json b/crates/persisting-gateway/tests/fixtures/requests/responses/input-list.json deleted file mode 100644 index 230442f12..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/responses/input-list.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "input": [{"role": "user", "content": "Hello"}], - "model": "gpt-4o", - "stream": true -} \ No newline at end of file diff --git a/crates/persisting-gateway/tests/fixtures/requests/responses/input-media.bedrock.snap b/crates/persisting-gateway/tests/fixtures/requests/responses/input-media.bedrock.snap deleted file mode 100644 index db2aa769a..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/responses/input-media.bedrock.snap +++ /dev/null @@ -1,115 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/responses/input-media.json -info: - model: gpt-4o - max_output_tokens: 64 - input: - - role: user - content: - - type: input_text - text: Compare these documents and image. - - type: input_image - image_url: "data:image/png;base64,iVBORw0KGgo=" - detail: auto - - type: input_file - file_data: "data:application/pdf;base64,JVBERi0=" - filename: report.pdf - - type: input_file - file_url: "data:text/csv;base64,YSxiLGM=" - filename: data.csv - - type: input_file - file_data: "data:application/octet-stream;base64,dGVzdA==" - filename: notes.md - - type: input_file - file_data: "data:application/pdf;base64,JVBERi1=" - filename: report.pdf - - type: input_file - file_data: "data:text/plain;base64,dGVzdA==" ---- -{ - "request": { - "messages": [ - { - "role": "user", - "content": [ - { - "text": "Compare these documents and image." - }, - { - "image": { - "format": "png", - "source": { - "bytes": "iVBORw0KGgo=" - } - } - }, - { - "document": { - "format": "pdf", - "name": "report", - "source": { - "bytes": "JVBERi0=" - } - } - }, - { - "document": { - "format": "csv", - "name": "data", - "source": { - "bytes": "YSxiLGM=" - } - } - }, - { - "document": { - "format": "md", - "name": "notes", - "source": { - "bytes": "dGVzdA==" - } - } - }, - { - "document": { - "format": "pdf", - "name": "report [2]", - "source": { - "bytes": "JVBERi1=" - } - } - }, - { - "document": { - "format": "txt", - "name": "document", - "source": { - "bytes": "dGVzdA==" - } - } - } - ] - } - ], - "inferenceConfig": { - "maxTokens": 64 - } - }, - "parsed": { - "input_format": "Responses", - "cache_convention": "InputIncludesCache", - "request_model": "gpt-4o", - "provider": "bedrock", - "streaming": false, - "params": { - "max_tokens": 64 - }, - "prompt": [ - { - "role": "user", - "content": "Compare these documents and image." - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/responses/input-media.json b/crates/persisting-gateway/tests/fixtures/requests/responses/input-media.json deleted file mode 100644 index 6039130be..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/responses/input-media.json +++ /dev/null @@ -1,41 +0,0 @@ -{ - "model": "gpt-4o", - "max_output_tokens": 64, - "input": [ - { - "role": "user", - "content": [ - { "type": "input_text", "text": "Compare these documents and image." }, - { - "type": "input_image", - "image_url": "data:image/png;base64,iVBORw0KGgo=", - "detail": "auto" - }, - { - "type": "input_file", - "file_data": "data:application/pdf;base64,JVBERi0=", - "filename": "report.pdf" - }, - { - "type": "input_file", - "file_url": "data:text/csv;base64,YSxiLGM=", - "filename": "data.csv" - }, - { - "type": "input_file", - "file_data": "data:application/octet-stream;base64,dGVzdA==", - "filename": "notes.md" - }, - { - "type": "input_file", - "file_data": "data:application/pdf;base64,JVBERi1=", - "filename": "report.pdf" - }, - { - "type": "input_file", - "file_data": "data:text/plain;base64,dGVzdA==" - } - ] - } - ] -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/responses/instructions.bedrock.snap b/crates/persisting-gateway/tests/fixtures/requests/responses/instructions.bedrock.snap deleted file mode 100644 index ff57b7165..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/responses/instructions.bedrock.snap +++ /dev/null @@ -1,51 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/responses/instructions.json -info: - model: gpt-4o - max_output_tokens: 1024 - instructions: Always use the get_weather tool for weather questions. - input: What is the weather like in San Francisco? ---- -{ - "request": { - "messages": [ - { - "role": "user", - "content": [ - { - "text": "What is the weather like in San Francisco?" - } - ] - } - ], - "system": [ - { - "text": "Always use the get_weather tool for weather questions." - } - ], - "inferenceConfig": { - "maxTokens": 1024 - } - }, - "parsed": { - "input_format": "Responses", - "cache_convention": "InputIncludesCache", - "request_model": "gpt-4o", - "provider": "bedrock", - "streaming": false, - "params": { - "max_tokens": 1024 - }, - "prompt": [ - { - "role": "system", - "content": "Always use the get_weather tool for weather questions." - }, - { - "role": "user", - "content": "What is the weather like in San Francisco?" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/responses/instructions.gemini.snap b/crates/persisting-gateway/tests/fixtures/requests/responses/instructions.gemini.snap deleted file mode 100644 index f818ba043..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/responses/instructions.gemini.snap +++ /dev/null @@ -1,51 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/responses/instructions.json -info: - model: gpt-4o - max_output_tokens: 1024 - instructions: Always use the get_weather tool for weather questions. - input: What is the weather like in San Francisco? ---- -{ - "request": { - "messages": [ - { - "role": "developer", - "content": "Always use the get_weather tool for weather questions." - }, - { - "role": "user", - "content": [ - { - "type": "text", - "text": "What is the weather like in San Francisco?" - } - ] - } - ], - "model": "gpt-4o", - "max_completion_tokens": 1024, - "stream": false - }, - "parsed": { - "input_format": "Responses", - "cache_convention": "InputIncludesCache", - "request_model": "gpt-4o", - "provider": "gemini", - "streaming": false, - "params": { - "max_tokens": 1024 - }, - "prompt": [ - { - "role": "system", - "content": "Always use the get_weather tool for weather questions." - }, - { - "role": "user", - "content": "What is the weather like in San Francisco?" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/responses/instructions.json b/crates/persisting-gateway/tests/fixtures/requests/responses/instructions.json deleted file mode 100644 index 57725ea79..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/responses/instructions.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "model": "gpt-4o", - "max_output_tokens": 1024, - "instructions": "Always use the get_weather tool for weather questions.", - "input": "What is the weather like in San Francisco?" -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/responses/parallel-tool-call.bedrock.snap b/crates/persisting-gateway/tests/fixtures/requests/responses/parallel-tool-call.bedrock.snap deleted file mode 100644 index 6cdeeed27..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/responses/parallel-tool-call.bedrock.snap +++ /dev/null @@ -1,125 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/responses/parallel-tool-call.json -info: - model: gpt-4o - max_output_tokens: 1024 - input: - - type: message - role: user - content: - - type: input_text - text: How is the weather in Columbus and New York? - - type: function_call - call_id: call_1 - name: get_weather - arguments: "{\"location\":\"Columbus, OH\"}" - - type: function_call - call_id: call_2 - name: get_weather - arguments: "{\"location\":\"New York, NY\"}" - - type: function_call_output - call_id: call_1 - output: "{ \"temperature\": 15, \"condition\": \"Cloudy\" }" - - type: function_call_output - call_id: call_2 - output: "{ \"temperature\": 22, \"condition\": \"Sunny\" }" ---- -{ - "request": { - "messages": [ - { - "role": "user", - "content": [ - { - "text": "How is the weather in Columbus and New York?" - } - ] - }, - { - "role": "assistant", - "content": [ - { - "toolUse": { - "toolUseId": "call_1", - "name": "get_weather", - "input": { - "location": "Columbus, OH" - } - } - }, - { - "toolUse": { - "toolUseId": "call_2", - "name": "get_weather", - "input": { - "location": "New York, NY" - } - } - } - ] - }, - { - "role": "user", - "content": [ - { - "toolResult": { - "toolUseId": "call_1", - "content": [ - { - "text": "{ \"temperature\": 15, \"condition\": \"Cloudy\" }" - } - ], - "status": null - } - }, - { - "toolResult": { - "toolUseId": "call_2", - "content": [ - { - "text": "{ \"temperature\": 22, \"condition\": \"Sunny\" }" - } - ], - "status": null - } - } - ] - } - ], - "inferenceConfig": { - "maxTokens": 1024 - }, - "toolConfig": { - "tools": [ - { - "toolSpec": { - "name": "agentgateway_dummy_do_not_call", - "description": null, - "inputSchema": { - "json": { - "type": "object" - } - } - } - } - ] - } - }, - "parsed": { - "input_format": "Responses", - "cache_convention": "InputIncludesCache", - "request_model": "gpt-4o", - "provider": "bedrock", - "streaming": false, - "params": { - "max_tokens": 1024 - }, - "prompt": [ - { - "role": "user", - "content": "How is the weather in Columbus and New York?" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/responses/parallel-tool-call.gemini.snap b/crates/persisting-gateway/tests/fixtures/requests/responses/parallel-tool-call.gemini.snap deleted file mode 100644 index fe7fce570..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/responses/parallel-tool-call.gemini.snap +++ /dev/null @@ -1,92 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/responses/parallel-tool-call.json -info: - model: gpt-4o - max_output_tokens: 1024 - input: - - type: message - role: user - content: - - type: input_text - text: How is the weather in Columbus and New York? - - type: function_call - call_id: call_1 - name: get_weather - arguments: "{\"location\":\"Columbus, OH\"}" - - type: function_call - call_id: call_2 - name: get_weather - arguments: "{\"location\":\"New York, NY\"}" - - type: function_call_output - call_id: call_1 - output: "{ \"temperature\": 15, \"condition\": \"Cloudy\" }" - - type: function_call_output - call_id: call_2 - output: "{ \"temperature\": 22, \"condition\": \"Sunny\" }" ---- -{ - "request": { - "messages": [ - { - "role": "user", - "content": [ - { - "type": "text", - "text": "How is the weather in Columbus and New York?" - } - ] - }, - { - "role": "assistant", - "tool_calls": [ - { - "type": "function", - "id": "call_1", - "function": { - "name": "get_weather", - "arguments": "{\"location\":\"Columbus, OH\"}" - } - }, - { - "type": "function", - "id": "call_2", - "function": { - "name": "get_weather", - "arguments": "{\"location\":\"New York, NY\"}" - } - } - ] - }, - { - "role": "tool", - "content": "{ \"temperature\": 15, \"condition\": \"Cloudy\" }", - "tool_call_id": "call_1" - }, - { - "role": "tool", - "content": "{ \"temperature\": 22, \"condition\": \"Sunny\" }", - "tool_call_id": "call_2" - } - ], - "model": "gpt-4o", - "max_completion_tokens": 1024, - "stream": false - }, - "parsed": { - "input_format": "Responses", - "cache_convention": "InputIncludesCache", - "request_model": "gpt-4o", - "provider": "gemini", - "streaming": false, - "params": { - "max_tokens": 1024 - }, - "prompt": [ - { - "role": "user", - "content": "How is the weather in Columbus and New York?" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/responses/parallel-tool-call.json b/crates/persisting-gateway/tests/fixtures/requests/responses/parallel-tool-call.json deleted file mode 100644 index 9e7cdcf7e..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/responses/parallel-tool-call.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "model": "gpt-4o", - "max_output_tokens": 1024, - "input": [ - { - "type": "message", - "role": "user", - "content": [{"type": "input_text", "text": "How is the weather in Columbus and New York?"}] - }, - { - "type": "function_call", - "call_id": "call_1", - "name": "get_weather", - "arguments": "{\"location\":\"Columbus, OH\"}" - }, - { - "type": "function_call", - "call_id": "call_2", - "name": "get_weather", - "arguments": "{\"location\":\"New York, NY\"}" - }, - { - "type": "function_call_output", - "call_id": "call_1", - "output": "{ \"temperature\": 15, \"condition\": \"Cloudy\" }" - }, - { - "type": "function_call_output", - "call_id": "call_2", - "output": "{ \"temperature\": 22, \"condition\": \"Sunny\" }" - } - ] -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/responses/structured-output.bedrock.snap b/crates/persisting-gateway/tests/fixtures/requests/responses/structured-output.bedrock.snap deleted file mode 100644 index 9910275d1..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/responses/structured-output.bedrock.snap +++ /dev/null @@ -1,66 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/requests/responses/structured-output.json -info: - model: gpt-4o - max_output_tokens: 64 - input: Extract the city name. - text: - format: - type: json_schema - name: city_schema - description: Structured city extraction - schema: - type: object - properties: - city: - type: string - required: - - city - additionalProperties: false ---- -{ - "request": { - "messages": [ - { - "role": "user", - "content": [ - { - "text": "Extract the city name." - } - ] - } - ], - "inferenceConfig": { - "maxTokens": 64 - }, - "outputConfig": { - "textFormat": { - "type": "json_schema", - "structure": { - "jsonSchema": { - "schema": "{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false}", - "name": "city_schema", - "description": "Structured city extraction" - } - } - } - } - }, - "parsed": { - "input_format": "Responses", - "cache_convention": "InputIncludesCache", - "request_model": "gpt-4o", - "provider": "bedrock", - "streaming": false, - "params": { - "max_tokens": 64 - }, - "prompt": [ - { - "role": "user", - "content": "Extract the city name." - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/requests/responses/structured-output.json b/crates/persisting-gateway/tests/fixtures/requests/responses/structured-output.json deleted file mode 100644 index e6ddd95a6..000000000 --- a/crates/persisting-gateway/tests/fixtures/requests/responses/structured-output.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "model": "gpt-4o", - "max_output_tokens": 64, - "input": "Extract the city name.", - "text": { - "format": { - "type": "json_schema", - "name": "city_schema", - "description": "Structured city extraction", - "schema": { - "type": "object", - "properties": { - "city": { "type": "string" } - }, - "required": ["city"], - "additionalProperties": false - } - } - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/anthropic/basic.json b/crates/persisting-gateway/tests/fixtures/response/anthropic/basic.json deleted file mode 100644 index 47ed0eeaa..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/anthropic/basic.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "id": "msg_012JYtNYb8sv6Hb67TcGT7xW", - "type": "message", - "role": "assistant", - "model": "claude-haiku-4-5-20251001", - "content": [ - { - "type": "text", - "text": "Hi there! How are you doing today? Is there anything I can help you with?" - } - ], - "stop_reason": "end_turn", - "stop_sequence": null, - "usage": { - "input_tokens": 15, - "cache_creation_input_tokens": 13, - "cache_read_input_tokens": 12, - "output_tokens": 21, - "service_tier": "standard" - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/anthropic/basic.messages-completions.snap b/crates/persisting-gateway/tests/fixtures/response/anthropic/basic.messages-completions.snap deleted file mode 100644 index 3cb3e9e59..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/anthropic/basic.messages-completions.snap +++ /dev/null @@ -1,59 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/anthropic/basic.json -info: - id: msg_012JYtNYb8sv6Hb67TcGT7xW - type: message - role: assistant - model: claude-haiku-4-5-20251001 - content: - - type: text - text: Hi there! How are you doing today? Is there anything I can help you with? - stop_reason: end_turn - stop_sequence: ~ - usage: - input_tokens: 15 - cache_creation_input_tokens: 13 - cache_read_input_tokens: 12 - output_tokens: 21 - service_tier: standard ---- -{ - "response": { - "model": "claude-haiku-4-5-20251001", - "service_tier": "standard", - "usage": { - "prompt_tokens": 15, - "completion_tokens": 21, - "total_tokens": 36, - "prompt_tokens_details": { - "cached_tokens": 12, - "cache_write_tokens": 13 - }, - "cache_read_input_tokens": 12, - "cache_creation_input_tokens": 13 - }, - "choices": [ - { - "message": { - "content": "Hi there! How are you doing today? Is there anything I can help you with?", - "role": "assistant" - }, - "index": 0, - "finish_reason": "stop" - } - ], - "id": "[id]", - "created": "[date]", - "object": "chat.completion" - }, - "parsed": { - "input_tokens": 15, - "output_tokens": 21, - "total_tokens": 36, - "cache_creation_input_tokens": 13, - "cached_input_tokens": 12, - "service_tier": "standard", - "provider_model": "claude-haiku-4-5-20251001" - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/anthropic/basic.messages-detect.snap b/crates/persisting-gateway/tests/fixtures/response/anthropic/basic.messages-detect.snap deleted file mode 100644 index f392446ca..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/anthropic/basic.messages-detect.snap +++ /dev/null @@ -1,52 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/anthropic/basic.json -info: - id: msg_012JYtNYb8sv6Hb67TcGT7xW - type: message - role: assistant - model: claude-haiku-4-5-20251001 - content: - - type: text - text: Hi there! How are you doing today? Is there anything I can help you with? - stop_reason: end_turn - stop_sequence: ~ - usage: - input_tokens: 15 - cache_creation_input_tokens: 13 - cache_read_input_tokens: 12 - output_tokens: 21 - service_tier: standard ---- -{ - "response": { - "id": "[id]", - "type": "message", - "role": "assistant", - "model": "claude-haiku-4-5-20251001", - "content": [ - { - "type": "text", - "text": "Hi there! How are you doing today? Is there anything I can help you with?" - } - ], - "stop_reason": "end_turn", - "stop_sequence": null, - "usage": { - "input_tokens": 15, - "cache_creation_input_tokens": 13, - "cache_read_input_tokens": 12, - "output_tokens": 21, - "service_tier": "standard" - } - }, - "parsed": { - "input_tokens": 15, - "output_tokens": 21, - "total_tokens": 36, - "cache_creation_input_tokens": 13, - "cached_input_tokens": 12, - "service_tier": "standard", - "provider_model": "claude-haiku-4-5-20251001" - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/anthropic/basic.messages-messages.snap b/crates/persisting-gateway/tests/fixtures/response/anthropic/basic.messages-messages.snap deleted file mode 100644 index 38ba51b24..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/anthropic/basic.messages-messages.snap +++ /dev/null @@ -1,52 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/anthropic/basic.json -info: - id: msg_012JYtNYb8sv6Hb67TcGT7xW - type: message - role: assistant - model: claude-haiku-4-5-20251001 - content: - - type: text - text: Hi there! How are you doing today? Is there anything I can help you with? - stop_reason: end_turn - stop_sequence: ~ - usage: - input_tokens: 15 - cache_creation_input_tokens: 13 - cache_read_input_tokens: 12 - output_tokens: 21 - service_tier: standard ---- -{ - "response": { - "id": "[id]", - "type": "message", - "role": "assistant", - "model": "claude-haiku-4-5-20251001", - "stop_reason": "end_turn", - "stop_sequence": null, - "usage": { - "input_tokens": 15, - "output_tokens": 21, - "cache_creation_input_tokens": 13, - "cache_read_input_tokens": 12, - "service_tier": "standard" - }, - "content": [ - { - "text": "Hi there! How are you doing today? Is there anything I can help you with?", - "type": "text" - } - ] - }, - "parsed": { - "input_tokens": 15, - "output_tokens": 21, - "total_tokens": 36, - "cache_creation_input_tokens": 13, - "cached_input_tokens": 12, - "service_tier": "standard", - "provider_model": "claude-haiku-4-5-20251001" - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/anthropic/count_tokens.anthropic.snap b/crates/persisting-gateway/tests/fixtures/response/anthropic/count_tokens.anthropic.snap deleted file mode 100644 index a1550ca4c..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/anthropic/count_tokens.anthropic.snap +++ /dev/null @@ -1,10 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/anthropic/count_tokens.json -info: - input_tokens: 42 ---- -{ - "input_tokens": 42, - "token_count": 42 -} diff --git a/crates/persisting-gateway/tests/fixtures/response/anthropic/count_tokens.json b/crates/persisting-gateway/tests/fixtures/response/anthropic/count_tokens.json deleted file mode 100644 index 0a9e80978..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/anthropic/count_tokens.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "input_tokens": 42 -} diff --git a/crates/persisting-gateway/tests/fixtures/response/anthropic/multiple_text_blocks.json b/crates/persisting-gateway/tests/fixtures/response/anthropic/multiple_text_blocks.json deleted file mode 100644 index 7738de115..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/anthropic/multiple_text_blocks.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "id": "msg_012JYtNYb8sv6Hb67TcGT7xW", - "type": "message", - "role": "assistant", - "model": "claude-haiku-4-5-20251001", - "content": [ - { - "type": "text", - "text": "The sky is " - }, - { - "type": "text", - "text": "blue during the day", - "citations": [ - { - "type": "char_location", - "cited_text": "the sky appears blue", - "document_index": 0, - "document_title": "Sky Facts", - "start_char_index": 0, - "end_char_index": 20 - } - ] - }, - { - "type": "text", - "text": "." - } - ], - "stop_reason": "end_turn", - "stop_sequence": null, - "usage": { - "input_tokens": 15, - "output_tokens": 21, - "service_tier": "standard" - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/anthropic/multiple_text_blocks.messages-completions.snap b/crates/persisting-gateway/tests/fixtures/response/anthropic/multiple_text_blocks.messages-completions.snap deleted file mode 100644 index 8a33f3512..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/anthropic/multiple_text_blocks.messages-completions.snap +++ /dev/null @@ -1,60 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/anthropic/multiple_text_blocks.json -info: - id: msg_012JYtNYb8sv6Hb67TcGT7xW - type: message - role: assistant - model: claude-haiku-4-5-20251001 - content: - - type: text - text: "The sky is " - - type: text - text: blue during the day - citations: - - type: char_location - cited_text: the sky appears blue - document_index: 0 - document_title: Sky Facts - start_char_index: 0 - end_char_index: 20 - - type: text - text: "." - stop_reason: end_turn - stop_sequence: ~ - usage: - input_tokens: 15 - output_tokens: 21 - service_tier: standard ---- -{ - "response": { - "model": "claude-haiku-4-5-20251001", - "service_tier": "standard", - "usage": { - "prompt_tokens": 15, - "completion_tokens": 21, - "total_tokens": 36 - }, - "choices": [ - { - "message": { - "content": "The sky is blue during the day.", - "role": "assistant" - }, - "index": 0, - "finish_reason": "stop" - } - ], - "id": "[id]", - "created": "[date]", - "object": "chat.completion" - }, - "parsed": { - "input_tokens": 15, - "output_tokens": 21, - "total_tokens": 36, - "service_tier": "standard", - "provider_model": "claude-haiku-4-5-20251001" - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/anthropic/multiple_text_blocks.messages-detect.snap b/crates/persisting-gateway/tests/fixtures/response/anthropic/multiple_text_blocks.messages-detect.snap deleted file mode 100644 index c7f4c596f..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/anthropic/multiple_text_blocks.messages-detect.snap +++ /dev/null @@ -1,75 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/anthropic/multiple_text_blocks.json -info: - id: msg_012JYtNYb8sv6Hb67TcGT7xW - type: message - role: assistant - model: claude-haiku-4-5-20251001 - content: - - type: text - text: "The sky is " - - type: text - text: blue during the day - citations: - - type: char_location - cited_text: the sky appears blue - document_index: 0 - document_title: Sky Facts - start_char_index: 0 - end_char_index: 20 - - type: text - text: "." - stop_reason: end_turn - stop_sequence: ~ - usage: - input_tokens: 15 - output_tokens: 21 - service_tier: standard ---- -{ - "response": { - "id": "[id]", - "type": "message", - "role": "assistant", - "model": "claude-haiku-4-5-20251001", - "content": [ - { - "type": "text", - "text": "The sky is " - }, - { - "type": "text", - "text": "blue during the day", - "citations": [ - { - "type": "char_location", - "cited_text": "the sky appears blue", - "document_index": 0, - "document_title": "Sky Facts", - "start_char_index": 0, - "end_char_index": 20 - } - ] - }, - { - "type": "text", - "text": "." - } - ], - "stop_reason": "end_turn", - "stop_sequence": null, - "usage": { - "input_tokens": 15, - "output_tokens": 21, - "service_tier": "standard" - } - }, - "parsed": { - "input_tokens": 15, - "output_tokens": 21, - "total_tokens": 36, - "service_tier": "standard", - "provider_model": "claude-haiku-4-5-20251001" - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/anthropic/multiple_text_blocks.messages-messages.snap b/crates/persisting-gateway/tests/fixtures/response/anthropic/multiple_text_blocks.messages-messages.snap deleted file mode 100644 index 1db5857e4..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/anthropic/multiple_text_blocks.messages-messages.snap +++ /dev/null @@ -1,75 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/anthropic/multiple_text_blocks.json -info: - id: msg_012JYtNYb8sv6Hb67TcGT7xW - type: message - role: assistant - model: claude-haiku-4-5-20251001 - content: - - type: text - text: "The sky is " - - type: text - text: blue during the day - citations: - - type: char_location - cited_text: the sky appears blue - document_index: 0 - document_title: Sky Facts - start_char_index: 0 - end_char_index: 20 - - type: text - text: "." - stop_reason: end_turn - stop_sequence: ~ - usage: - input_tokens: 15 - output_tokens: 21 - service_tier: standard ---- -{ - "response": { - "id": "[id]", - "type": "message", - "role": "assistant", - "model": "claude-haiku-4-5-20251001", - "stop_reason": "end_turn", - "stop_sequence": null, - "usage": { - "input_tokens": 15, - "output_tokens": 21, - "service_tier": "standard" - }, - "content": [ - { - "text": "The sky is ", - "type": "text" - }, - { - "text": "blue during the day", - "type": "text", - "citations": [ - { - "type": "char_location", - "cited_text": "the sky appears blue", - "document_index": 0, - "document_title": "Sky Facts", - "start_char_index": 0, - "end_char_index": 20 - } - ] - }, - { - "text": ".", - "type": "text" - } - ] - }, - "parsed": { - "input_tokens": 15, - "output_tokens": 21, - "total_tokens": 36, - "service_tier": "standard", - "provider_model": "claude-haiku-4-5-20251001" - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/anthropic/stream_basic.json b/crates/persisting-gateway/tests/fixtures/response/anthropic/stream_basic.json deleted file mode 100644 index 169a4de66..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/anthropic/stream_basic.json +++ /dev/null @@ -1,33 +0,0 @@ -event: message_start -data: {"type":"message_start","message":{"id":"msg_016wbjv5k86nxxd8CEZwSQnA","type":"message","role":"assistant","model":"claude-haiku-4-5-20251001","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":15,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":5,"service_tier":"standard"}} } - -event: content_block_start -data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""} } - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Hi there! How are"} } - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" you doing today?"} } - -event: ping -data: {"type": "ping"} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" Is"} } - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" there anything I can help"} } - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" you with?"} } - -event: content_block_stop -data: {"type":"content_block_stop","index":0 } - -event: message_delta -data: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"input_tokens":15,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":21} } - -event: message_stop -data: {"type":"message_stop" } - diff --git a/crates/persisting-gateway/tests/fixtures/response/anthropic/stream_basic.messages-completions-streaming.snap b/crates/persisting-gateway/tests/fixtures/response/anthropic/stream_basic.messages-completions-streaming.snap deleted file mode 100644 index 6ec73dbe7..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/anthropic/stream_basic.messages-completions-streaming.snap +++ /dev/null @@ -1,32 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/anthropic/stream_basic.json ---- -data: {"id":"msg_016wbjv5k86nxxd8CEZwSQnA","choices":[{"index":0,"delta":{"content":"Hi there! How are","role":"assistant"}}],"created":123,"model":"claude-haiku-4-5-20251001","service_tier":"standard","system_fingerprint":null,"object":"chat.completion.chunk","usage":null} - -data: {"id":"msg_016wbjv5k86nxxd8CEZwSQnA","choices":[{"index":0,"delta":{"content":" you doing today?"}}],"created":123,"model":"claude-haiku-4-5-20251001","service_tier":"standard","system_fingerprint":null,"object":"chat.completion.chunk","usage":null} - -data: {"id":"msg_016wbjv5k86nxxd8CEZwSQnA","choices":[{"index":0,"delta":{"content":" Is"}}],"created":123,"model":"claude-haiku-4-5-20251001","service_tier":"standard","system_fingerprint":null,"object":"chat.completion.chunk","usage":null} - -data: {"id":"msg_016wbjv5k86nxxd8CEZwSQnA","choices":[{"index":0,"delta":{"content":" there anything I can help"}}],"created":123,"model":"claude-haiku-4-5-20251001","service_tier":"standard","system_fingerprint":null,"object":"chat.completion.chunk","usage":null} - -data: {"id":"msg_016wbjv5k86nxxd8CEZwSQnA","choices":[{"index":0,"delta":{"content":" you with?"}}],"created":123,"model":"claude-haiku-4-5-20251001","service_tier":"standard","system_fingerprint":null,"object":"chat.completion.chunk","usage":null} - -data: {"id":"msg_016wbjv5k86nxxd8CEZwSQnA","choices":[{"index":0,"delta":{"content":null},"finish_reason":"stop"}],"created":123,"model":"claude-haiku-4-5-20251001","service_tier":"standard","system_fingerprint":null,"object":"chat.completion.chunk","usage":{"prompt_tokens":15,"completion_tokens":21,"total_tokens":36,"prompt_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"cache_read_input_tokens":0,"cache_creation_input_tokens":0}} - -data: [DONE] - - - -{ - "input_tokens": 15, - "output_tokens": 21, - "total_tokens": 36, - "cache_creation_input_tokens": 0, - "cached_input_tokens": 0, - "service_tier": "standard", - "provider_model": "claude-haiku-4-5-20251001", - "completion": [ - "Hi there! How are you doing today? Is there anything I can help you with?" - ] -} diff --git a/crates/persisting-gateway/tests/fixtures/response/anthropic/stream_basic.messages-detect-streaming.snap b/crates/persisting-gateway/tests/fixtures/response/anthropic/stream_basic.messages-detect-streaming.snap deleted file mode 100644 index 25da97240..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/anthropic/stream_basic.messages-detect-streaming.snap +++ /dev/null @@ -1,47 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/anthropic/stream_basic.json ---- -event: message_start -data: {"type":"message_start","message":{"id":"msg_016wbjv5k86nxxd8CEZwSQnA","type":"message","role":"assistant","model":"claude-haiku-4-5-20251001","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":15,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":5,"service_tier":"standard"}} } - -event: content_block_start -data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""} } - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Hi there! How are"} } - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" you doing today?"} } - -event: ping -data: {"type": "ping"} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" Is"} } - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" there anything I can help"} } - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" you with?"} } - -event: content_block_stop -data: {"type":"content_block_stop","index":0 } - -event: message_delta -data: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"input_tokens":15,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":21} } - -event: message_stop -data: {"type":"message_stop" } - - - -{ - "input_tokens": 15, - "output_tokens": 21, - "total_tokens": 36, - "cache_creation_input_tokens": 0, - "cached_input_tokens": 0, - "provider_model": "claude-haiku-4-5-20251001" -} diff --git a/crates/persisting-gateway/tests/fixtures/response/anthropic/stream_basic.messages-messages-streaming.snap b/crates/persisting-gateway/tests/fixtures/response/anthropic/stream_basic.messages-messages-streaming.snap deleted file mode 100644 index 7c6a2c118..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/anthropic/stream_basic.messages-messages-streaming.snap +++ /dev/null @@ -1,51 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/anthropic/stream_basic.json ---- -event: message_start -data: {"type":"message_start","message":{"id":"msg_016wbjv5k86nxxd8CEZwSQnA","type":"message","role":"assistant","model":"claude-haiku-4-5-20251001","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":15,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":5,"service_tier":"standard"}} } - -event: content_block_start -data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""} } - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Hi there! How are"} } - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" you doing today?"} } - -event: ping -data: {"type": "ping"} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" Is"} } - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" there anything I can help"} } - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" you with?"} } - -event: content_block_stop -data: {"type":"content_block_stop","index":0 } - -event: message_delta -data: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"input_tokens":15,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":21} } - -event: message_stop -data: {"type":"message_stop" } - - - -{ - "input_tokens": 15, - "output_tokens": 21, - "total_tokens": 36, - "cache_creation_input_tokens": 0, - "cached_input_tokens": 0, - "service_tier": "standard", - "provider_model": "claude-haiku-4-5-20251001", - "completion": [ - "Hi there! How are you doing today? Is there anything I can help you with?" - ] -} diff --git a/crates/persisting-gateway/tests/fixtures/response/anthropic/stream_message_delta_usage.json b/crates/persisting-gateway/tests/fixtures/response/anthropic/stream_message_delta_usage.json deleted file mode 100644 index f7dc93962..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/anthropic/stream_message_delta_usage.json +++ /dev/null @@ -1,9 +0,0 @@ -event:message_start -data:{"type":"message_start","message":{"id":"msg","type":"message","role":"assistant","content":[],"model":"glm-5.2","stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":34824,"output_tokens":0}}} - -event:message_delta -data:{"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"input_tokens":3883,"output_tokens":49,"cache_creation_input_tokens":0,"cache_read_input_tokens":30464,"prompt_tokens_details":{"cached_tokens":30464}}} - -event:message_stop -data:{"type":"message_stop"} - diff --git a/crates/persisting-gateway/tests/fixtures/response/anthropic/stream_message_delta_usage.messages-completions-streaming.snap b/crates/persisting-gateway/tests/fixtures/response/anthropic/stream_message_delta_usage.messages-completions-streaming.snap deleted file mode 100644 index 752252eeb..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/anthropic/stream_message_delta_usage.messages-completions-streaming.snap +++ /dev/null @@ -1,21 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/anthropic/stream_message_delta_usage.json ---- -data: {"id":"msg","choices":[{"index":0,"delta":{"content":null,"role":"assistant"},"finish_reason":"stop"}],"created":123,"model":"glm-5.2","service_tier":null,"system_fingerprint":null,"object":"chat.completion.chunk","usage":{"prompt_tokens":3883,"completion_tokens":49,"total_tokens":3932,"prompt_tokens_details":{"cached_tokens":30464,"cache_write_tokens":0},"cache_read_input_tokens":30464,"cache_creation_input_tokens":0}} - -data: [DONE] - - - -{ - "input_tokens": 3883, - "output_tokens": 49, - "total_tokens": 3932, - "cache_creation_input_tokens": 0, - "cached_input_tokens": 30464, - "provider_model": "glm-5.2", - "completion": [ - "" - ] -} diff --git a/crates/persisting-gateway/tests/fixtures/response/anthropic/stream_message_delta_usage.messages-messages-streaming.snap b/crates/persisting-gateway/tests/fixtures/response/anthropic/stream_message_delta_usage.messages-messages-streaming.snap deleted file mode 100644 index 18b9f076d..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/anthropic/stream_message_delta_usage.messages-messages-streaming.snap +++ /dev/null @@ -1,26 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/anthropic/stream_message_delta_usage.json ---- -event:message_start -data:{"type":"message_start","message":{"id":"msg","type":"message","role":"assistant","content":[],"model":"glm-5.2","stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":34824,"output_tokens":0}}} - -event:message_delta -data:{"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"input_tokens":3883,"output_tokens":49,"cache_creation_input_tokens":0,"cache_read_input_tokens":30464,"prompt_tokens_details":{"cached_tokens":30464}}} - -event:message_stop -data:{"type":"message_stop"} - - - -{ - "input_tokens": 3883, - "output_tokens": 49, - "total_tokens": 3932, - "cache_creation_input_tokens": 0, - "cached_input_tokens": 30464, - "provider_model": "glm-5.2", - "completion": [ - "" - ] -} diff --git a/crates/persisting-gateway/tests/fixtures/response/anthropic/stream_thinking.json b/crates/persisting-gateway/tests/fixtures/response/anthropic/stream_thinking.json deleted file mode 100644 index 28ffba845..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/anthropic/stream_thinking.json +++ /dev/null @@ -1,51 +0,0 @@ -event: message_start -data: {"type":"message_start","message":{"id":"msg_018XasA6jgabsibaUtstx4bg","type":"message","role":"assistant","model":"claude-opus-4-6","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":47,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":4,"service_tier":"standard"}} } - -event: content_block_start -data: {"type":"content_block_start","index":0,"content_block":{"type":"thinking","thinking":"","signature":""} } - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":"The user is"} } - -event: ping -data: {"type": "ping"} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":" asking me about the weather in"} } - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":" San Francisco. However, I don"} } - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":"d suggest how they can get current weather information."}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"signature_delta","signature":"EsgDCkYIBxgCKkDIY0kwFva57N6iFPjhIgYy221Ub+EFYZWV+OTmoLv3y/bbXg5rjd6MgPsrwJKf75dB+6WUHs2F/epN1HbNrR68EgznKMtCqednHsUIeZcaDDK1y0Z+c/3JokR/uCIw16faEoFv4stNb8SgXMiRRJECbRtkS86bv2cn9Fv75SOv8/DFTFt1+QRtPLsTRwTIKq8CFUnvffH/J0Ei3Pc2oaWuLuPA9eSAm4I89gEge/Et3d9f9wdXDy5Y/OztcD9S04lC7H/3iOgUJOMGQGqEV+k87Sv2fa+cEaNjjCouz4odDIB/e/+pR5vYPltOj/bdlf1RNHT7KMnCEc/rGDJ9aY2JZXrbMmA2i+SJN4WoAQ+86D2KDO3NWwFT3V7d2YOkqN/nX6ehZnts6pl6/ABi2EH0qH6B8tWBccP2FcaUZXFoYpQbhDpN++eYQp8m9+ehqEf/G66s30Y5bjwW2qwW56GzGx7nt04n8cxNLL/5SpeTJoUu4jgYSx4ae7AA9xpbnRDNO6xjKou4TnUnLcOBvszoi+RmjJMQ/fWquBOTFjdsrg6CGTz8ewgZSwcnnrU73G2XCXOgnOl7OF4t50bL87hOGAE="} } - -event: content_block_stop -data: {"type":"content_block_stop","index":0 } - -event: content_block_start -data: {"type":"content_block_start","index":1,"content_block":{"type":"text","text":""} } - -event: content_block_delta -data: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"I don't have access to real"} } - -event: content_block_delta -data: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"\" in your browser\n\nIs there something"} } - -event: content_block_delta -data: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":" specific about San Francisco's climate patterns"} } - -event: content_block_delta -data: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":" you'd like to know more about?"} } - -event: content_block_stop -data: {"type":"content_block_stop","index":1 } - -event: message_delta -data: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"input_tokens":47,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":264} } - -event: message_stop -data: {"type":"message_stop" } - diff --git a/crates/persisting-gateway/tests/fixtures/response/anthropic/stream_thinking.messages-completions-streaming.snap b/crates/persisting-gateway/tests/fixtures/response/anthropic/stream_thinking.messages-completions-streaming.snap deleted file mode 100644 index 9b123b065..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/anthropic/stream_thinking.messages-completions-streaming.snap +++ /dev/null @@ -1,38 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/anthropic/stream_thinking.json ---- -data: {"id":"msg_018XasA6jgabsibaUtstx4bg","choices":[{"index":0,"delta":{"content":null,"role":"assistant","reasoning_content":"The user is"}}],"created":123,"model":"claude-opus-4-6","service_tier":"standard","system_fingerprint":null,"object":"chat.completion.chunk","usage":null} - -data: {"id":"msg_018XasA6jgabsibaUtstx4bg","choices":[{"index":0,"delta":{"content":null,"reasoning_content":" asking me about the weather in"}}],"created":123,"model":"claude-opus-4-6","service_tier":"standard","system_fingerprint":null,"object":"chat.completion.chunk","usage":null} - -data: {"id":"msg_018XasA6jgabsibaUtstx4bg","choices":[{"index":0,"delta":{"content":null,"reasoning_content":" San Francisco. However, I don"}}],"created":123,"model":"claude-opus-4-6","service_tier":"standard","system_fingerprint":null,"object":"chat.completion.chunk","usage":null} - -data: {"id":"msg_018XasA6jgabsibaUtstx4bg","choices":[{"index":0,"delta":{"content":null,"reasoning_content":"d suggest how they can get current weather information."}}],"created":123,"model":"claude-opus-4-6","service_tier":"standard","system_fingerprint":null,"object":"chat.completion.chunk","usage":null} - -data: {"id":"msg_018XasA6jgabsibaUtstx4bg","choices":[{"index":0,"delta":{"content":"I don't have access to real"}}],"created":123,"model":"claude-opus-4-6","service_tier":"standard","system_fingerprint":null,"object":"chat.completion.chunk","usage":null} - -data: {"id":"msg_018XasA6jgabsibaUtstx4bg","choices":[{"index":0,"delta":{"content":"\" in your browser\n\nIs there something"}}],"created":123,"model":"claude-opus-4-6","service_tier":"standard","system_fingerprint":null,"object":"chat.completion.chunk","usage":null} - -data: {"id":"msg_018XasA6jgabsibaUtstx4bg","choices":[{"index":0,"delta":{"content":" specific about San Francisco's climate patterns"}}],"created":123,"model":"claude-opus-4-6","service_tier":"standard","system_fingerprint":null,"object":"chat.completion.chunk","usage":null} - -data: {"id":"msg_018XasA6jgabsibaUtstx4bg","choices":[{"index":0,"delta":{"content":" you'd like to know more about?"}}],"created":123,"model":"claude-opus-4-6","service_tier":"standard","system_fingerprint":null,"object":"chat.completion.chunk","usage":null} - -data: {"id":"msg_018XasA6jgabsibaUtstx4bg","choices":[{"index":0,"delta":{"content":null},"finish_reason":"stop"}],"created":123,"model":"claude-opus-4-6","service_tier":"standard","system_fingerprint":null,"object":"chat.completion.chunk","usage":{"prompt_tokens":47,"completion_tokens":264,"total_tokens":311,"prompt_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"cache_read_input_tokens":0,"cache_creation_input_tokens":0}} - -data: [DONE] - - - -{ - "input_tokens": 47, - "output_tokens": 264, - "total_tokens": 311, - "cache_creation_input_tokens": 0, - "cached_input_tokens": 0, - "service_tier": "standard", - "provider_model": "claude-opus-4-6", - "completion": [ - "I don't have access to real\" in your browser\n\nIs there something specific about San Francisco's climate patterns you'd like to know more about?" - ] -} diff --git a/crates/persisting-gateway/tests/fixtures/response/anthropic/stream_thinking.messages-detect-streaming.snap b/crates/persisting-gateway/tests/fixtures/response/anthropic/stream_thinking.messages-detect-streaming.snap deleted file mode 100644 index 328884b3d..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/anthropic/stream_thinking.messages-detect-streaming.snap +++ /dev/null @@ -1,65 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/anthropic/stream_thinking.json ---- -event: message_start -data: {"type":"message_start","message":{"id":"msg_018XasA6jgabsibaUtstx4bg","type":"message","role":"assistant","model":"claude-opus-4-6","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":47,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":4,"service_tier":"standard"}} } - -event: content_block_start -data: {"type":"content_block_start","index":0,"content_block":{"type":"thinking","thinking":"","signature":""} } - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":"The user is"} } - -event: ping -data: {"type": "ping"} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":" asking me about the weather in"} } - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":" San Francisco. However, I don"} } - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":"d suggest how they can get current weather information."}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"signature_delta","signature":"EsgDCkYIBxgCKkDIY0kwFva57N6iFPjhIgYy221Ub+EFYZWV+OTmoLv3y/bbXg5rjd6MgPsrwJKf75dB+6WUHs2F/epN1HbNrR68EgznKMtCqednHsUIeZcaDDK1y0Z+c/3JokR/uCIw16faEoFv4stNb8SgXMiRRJECbRtkS86bv2cn9Fv75SOv8/DFTFt1+QRtPLsTRwTIKq8CFUnvffH/J0Ei3Pc2oaWuLuPA9eSAm4I89gEge/Et3d9f9wdXDy5Y/OztcD9S04lC7H/3iOgUJOMGQGqEV+k87Sv2fa+cEaNjjCouz4odDIB/e/+pR5vYPltOj/bdlf1RNHT7KMnCEc/rGDJ9aY2JZXrbMmA2i+SJN4WoAQ+86D2KDO3NWwFT3V7d2YOkqN/nX6ehZnts6pl6/ABi2EH0qH6B8tWBccP2FcaUZXFoYpQbhDpN++eYQp8m9+ehqEf/G66s30Y5bjwW2qwW56GzGx7nt04n8cxNLL/5SpeTJoUu4jgYSx4ae7AA9xpbnRDNO6xjKou4TnUnLcOBvszoi+RmjJMQ/fWquBOTFjdsrg6CGTz8ewgZSwcnnrU73G2XCXOgnOl7OF4t50bL87hOGAE="} } - -event: content_block_stop -data: {"type":"content_block_stop","index":0 } - -event: content_block_start -data: {"type":"content_block_start","index":1,"content_block":{"type":"text","text":""} } - -event: content_block_delta -data: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"I don't have access to real"} } - -event: content_block_delta -data: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"\" in your browser\n\nIs there something"} } - -event: content_block_delta -data: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":" specific about San Francisco's climate patterns"} } - -event: content_block_delta -data: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":" you'd like to know more about?"} } - -event: content_block_stop -data: {"type":"content_block_stop","index":1 } - -event: message_delta -data: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"input_tokens":47,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":264} } - -event: message_stop -data: {"type":"message_stop" } - - - -{ - "input_tokens": 47, - "output_tokens": 264, - "total_tokens": 311, - "cache_creation_input_tokens": 0, - "cached_input_tokens": 0, - "provider_model": "claude-opus-4-6" -} diff --git a/crates/persisting-gateway/tests/fixtures/response/anthropic/stream_thinking.messages-messages-streaming.snap b/crates/persisting-gateway/tests/fixtures/response/anthropic/stream_thinking.messages-messages-streaming.snap deleted file mode 100644 index 71863cbeb..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/anthropic/stream_thinking.messages-messages-streaming.snap +++ /dev/null @@ -1,69 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/anthropic/stream_thinking.json ---- -event: message_start -data: {"type":"message_start","message":{"id":"msg_018XasA6jgabsibaUtstx4bg","type":"message","role":"assistant","model":"claude-opus-4-6","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":47,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":4,"service_tier":"standard"}} } - -event: content_block_start -data: {"type":"content_block_start","index":0,"content_block":{"type":"thinking","thinking":"","signature":""} } - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":"The user is"} } - -event: ping -data: {"type": "ping"} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":" asking me about the weather in"} } - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":" San Francisco. However, I don"} } - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":"d suggest how they can get current weather information."}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"signature_delta","signature":"EsgDCkYIBxgCKkDIY0kwFva57N6iFPjhIgYy221Ub+EFYZWV+OTmoLv3y/bbXg5rjd6MgPsrwJKf75dB+6WUHs2F/epN1HbNrR68EgznKMtCqednHsUIeZcaDDK1y0Z+c/3JokR/uCIw16faEoFv4stNb8SgXMiRRJECbRtkS86bv2cn9Fv75SOv8/DFTFt1+QRtPLsTRwTIKq8CFUnvffH/J0Ei3Pc2oaWuLuPA9eSAm4I89gEge/Et3d9f9wdXDy5Y/OztcD9S04lC7H/3iOgUJOMGQGqEV+k87Sv2fa+cEaNjjCouz4odDIB/e/+pR5vYPltOj/bdlf1RNHT7KMnCEc/rGDJ9aY2JZXrbMmA2i+SJN4WoAQ+86D2KDO3NWwFT3V7d2YOkqN/nX6ehZnts6pl6/ABi2EH0qH6B8tWBccP2FcaUZXFoYpQbhDpN++eYQp8m9+ehqEf/G66s30Y5bjwW2qwW56GzGx7nt04n8cxNLL/5SpeTJoUu4jgYSx4ae7AA9xpbnRDNO6xjKou4TnUnLcOBvszoi+RmjJMQ/fWquBOTFjdsrg6CGTz8ewgZSwcnnrU73G2XCXOgnOl7OF4t50bL87hOGAE="} } - -event: content_block_stop -data: {"type":"content_block_stop","index":0 } - -event: content_block_start -data: {"type":"content_block_start","index":1,"content_block":{"type":"text","text":""} } - -event: content_block_delta -data: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"I don't have access to real"} } - -event: content_block_delta -data: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"\" in your browser\n\nIs there something"} } - -event: content_block_delta -data: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":" specific about San Francisco's climate patterns"} } - -event: content_block_delta -data: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":" you'd like to know more about?"} } - -event: content_block_stop -data: {"type":"content_block_stop","index":1 } - -event: message_delta -data: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"input_tokens":47,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":264} } - -event: message_stop -data: {"type":"message_stop" } - - - -{ - "input_tokens": 47, - "output_tokens": 264, - "total_tokens": 311, - "cache_creation_input_tokens": 0, - "cached_input_tokens": 0, - "service_tier": "standard", - "provider_model": "claude-opus-4-6", - "completion": [ - "I don't have access to real\" in your browser\n\nIs there something specific about San Francisco's climate patterns you'd like to know more about?" - ] -} diff --git a/crates/persisting-gateway/tests/fixtures/response/anthropic/stream_tool.json b/crates/persisting-gateway/tests/fixtures/response/anthropic/stream_tool.json deleted file mode 100644 index 89bb93d15..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/anthropic/stream_tool.json +++ /dev/null @@ -1,22 +0,0 @@ -event: message_start -data: {"type":"message_start","message":{"id":"msg_tool","type":"message","role":"assistant","model":"claude-haiku-4-5-20251001","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":15,"output_tokens":1}}} - -event: content_block_start -data: {"type":"content_block_start","index":0,"content_block":{"type":"tool_use","id":"toolu_01A","name":"get_weather","input":{}}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"{\"location\":"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"\"San Francisco\"}"}} - -event: content_block_stop -data: {"type":"content_block_stop","index":0} - -event: message_delta -data: {"type":"message_delta","delta":{"stop_reason":"tool_use","stop_sequence":null},"usage":{"input_tokens":15,"output_tokens":12}} - -event: message_stop -data: {"type":"message_stop"} - -: stream complete diff --git a/crates/persisting-gateway/tests/fixtures/response/anthropic/stream_tool.messages-completions-streaming.snap b/crates/persisting-gateway/tests/fixtures/response/anthropic/stream_tool.messages-completions-streaming.snap deleted file mode 100644 index bdfdbd421..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/anthropic/stream_tool.messages-completions-streaming.snap +++ /dev/null @@ -1,41 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/anthropic/stream_tool.json ---- -data: {"id":"msg_tool","choices":[{"index":0,"delta":{"content":null,"tool_calls":[{"index":0,"id":"toolu_01A","type":"function","function":{"name":"get_weather"}}],"role":"assistant"}}],"created":123,"model":"claude-haiku-4-5-20251001","service_tier":null,"system_fingerprint":null,"object":"chat.completion.chunk","usage":null} - -data: {"id":"msg_tool","choices":[{"index":0,"delta":{"content":null,"tool_calls":[{"index":0,"id":null,"type":null,"function":{"arguments":"{\"location\":"}}]}}],"created":123,"model":"claude-haiku-4-5-20251001","service_tier":null,"system_fingerprint":null,"object":"chat.completion.chunk","usage":null} - -data: {"id":"msg_tool","choices":[{"index":0,"delta":{"content":null,"tool_calls":[{"index":0,"id":null,"type":null,"function":{"arguments":"\"San Francisco\"}"}}]}}],"created":123,"model":"claude-haiku-4-5-20251001","service_tier":null,"system_fingerprint":null,"object":"chat.completion.chunk","usage":null} - -data: {"id":"msg_tool","choices":[{"index":0,"delta":{"content":null},"finish_reason":"tool_calls"}],"created":123,"model":"claude-haiku-4-5-20251001","service_tier":null,"system_fingerprint":null,"object":"chat.completion.chunk","usage":{"prompt_tokens":15,"completion_tokens":12,"total_tokens":27}} - -data: [DONE] - - - -{ - "input_tokens": 15, - "output_tokens": 12, - "total_tokens": 27, - "provider_model": "claude-haiku-4-5-20251001", - "completion": [ - "" - ], - "output_messages": [ - { - "role": "assistant", - "content": [ - { - "type": "tool_call", - "id": "toolu_01A", - "name": "get_weather", - "arguments": { - "location": "San Francisco" - } - } - ], - "finish_reason": "tool_use" - } - ] -} diff --git a/crates/persisting-gateway/tests/fixtures/response/anthropic/stream_tool.messages-messages-streaming.snap b/crates/persisting-gateway/tests/fixtures/response/anthropic/stream_tool.messages-messages-streaming.snap deleted file mode 100644 index dc51b79b7..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/anthropic/stream_tool.messages-messages-streaming.snap +++ /dev/null @@ -1,53 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/anthropic/stream_tool.json ---- -event: message_start -data: {"type":"message_start","message":{"id":"msg_tool","type":"message","role":"assistant","model":"claude-haiku-4-5-20251001","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":15,"output_tokens":1}}} - -event: content_block_start -data: {"type":"content_block_start","index":0,"content_block":{"type":"tool_use","id":"toolu_01A","name":"get_weather","input":{}}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"{\"location\":"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"\"San Francisco\"}"}} - -event: content_block_stop -data: {"type":"content_block_stop","index":0} - -event: message_delta -data: {"type":"message_delta","delta":{"stop_reason":"tool_use","stop_sequence":null},"usage":{"input_tokens":15,"output_tokens":12}} - -event: message_stop -data: {"type":"message_stop"} - -: stream complete - - -{ - "input_tokens": 15, - "output_tokens": 12, - "total_tokens": 27, - "provider_model": "claude-haiku-4-5-20251001", - "completion": [ - "" - ], - "output_messages": [ - { - "role": "assistant", - "content": [ - { - "type": "tool_call", - "id": "toolu_01A", - "name": "get_weather", - "arguments": { - "location": "San Francisco" - } - } - ], - "finish_reason": "tool_use" - } - ] -} diff --git a/crates/persisting-gateway/tests/fixtures/response/anthropic/stream_tool_empty_args.json b/crates/persisting-gateway/tests/fixtures/response/anthropic/stream_tool_empty_args.json deleted file mode 100644 index 764f97edd..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/anthropic/stream_tool_empty_args.json +++ /dev/null @@ -1,19 +0,0 @@ -event: message_start -data: {"type":"message_start","message":{"id":"msg_tool_empty","type":"message","role":"assistant","model":"claude-haiku-4-5-20251001","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":15,"output_tokens":1}}} - -event: content_block_start -data: {"type":"content_block_start","index":0,"content_block":{"type":"tool_use","id":"toolu_01A","name":"get_time","input":{}}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":""}} - -event: content_block_stop -data: {"type":"content_block_stop","index":0} - -event: message_delta -data: {"type":"message_delta","delta":{"stop_reason":"tool_use","stop_sequence":null},"usage":{"input_tokens":15,"output_tokens":8}} - -event: message_stop -data: {"type":"message_stop"} - -: stream complete diff --git a/crates/persisting-gateway/tests/fixtures/response/anthropic/stream_tool_empty_args.messages-completions-streaming.snap b/crates/persisting-gateway/tests/fixtures/response/anthropic/stream_tool_empty_args.messages-completions-streaming.snap deleted file mode 100644 index fb7c4ed35..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/anthropic/stream_tool_empty_args.messages-completions-streaming.snap +++ /dev/null @@ -1,37 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/anthropic/stream_tool_empty_args.json ---- -data: {"id":"msg_tool_empty","choices":[{"index":0,"delta":{"content":null,"tool_calls":[{"index":0,"id":"toolu_01A","type":"function","function":{"name":"get_time"}}],"role":"assistant"}}],"created":123,"model":"claude-haiku-4-5-20251001","service_tier":null,"system_fingerprint":null,"object":"chat.completion.chunk","usage":null} - -data: {"id":"msg_tool_empty","choices":[{"index":0,"delta":{"content":null,"tool_calls":[{"index":0,"id":null,"type":null,"function":{"arguments":"{}"}}]}}],"created":123,"model":"claude-haiku-4-5-20251001","service_tier":null,"system_fingerprint":null,"object":"chat.completion.chunk","usage":null} - -data: {"id":"msg_tool_empty","choices":[{"index":0,"delta":{"content":null},"finish_reason":"tool_calls"}],"created":123,"model":"claude-haiku-4-5-20251001","service_tier":null,"system_fingerprint":null,"object":"chat.completion.chunk","usage":{"prompt_tokens":15,"completion_tokens":8,"total_tokens":23}} - -data: [DONE] - - - -{ - "input_tokens": 15, - "output_tokens": 8, - "total_tokens": 23, - "provider_model": "claude-haiku-4-5-20251001", - "completion": [ - "" - ], - "output_messages": [ - { - "role": "assistant", - "content": [ - { - "type": "tool_call", - "id": "toolu_01A", - "name": "get_time", - "arguments": {} - } - ], - "finish_reason": "tool_use" - } - ] -} diff --git a/crates/persisting-gateway/tests/fixtures/response/anthropic/stream_tool_empty_args.messages-messages-streaming.snap b/crates/persisting-gateway/tests/fixtures/response/anthropic/stream_tool_empty_args.messages-messages-streaming.snap deleted file mode 100644 index e6eb15f1b..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/anthropic/stream_tool_empty_args.messages-messages-streaming.snap +++ /dev/null @@ -1,48 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/anthropic/stream_tool_empty_args.json ---- -event: message_start -data: {"type":"message_start","message":{"id":"msg_tool_empty","type":"message","role":"assistant","model":"claude-haiku-4-5-20251001","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":15,"output_tokens":1}}} - -event: content_block_start -data: {"type":"content_block_start","index":0,"content_block":{"type":"tool_use","id":"toolu_01A","name":"get_time","input":{}}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":""}} - -event: content_block_stop -data: {"type":"content_block_stop","index":0} - -event: message_delta -data: {"type":"message_delta","delta":{"stop_reason":"tool_use","stop_sequence":null},"usage":{"input_tokens":15,"output_tokens":8}} - -event: message_stop -data: {"type":"message_stop"} - -: stream complete - - -{ - "input_tokens": 15, - "output_tokens": 8, - "total_tokens": 23, - "provider_model": "claude-haiku-4-5-20251001", - "completion": [ - "" - ], - "output_messages": [ - { - "role": "assistant", - "content": [ - { - "type": "tool_call", - "id": "toolu_01A", - "name": "get_time", - "arguments": {} - } - ], - "finish_reason": "tool_use" - } - ] -} diff --git a/crates/persisting-gateway/tests/fixtures/response/anthropic/thinking.json b/crates/persisting-gateway/tests/fixtures/response/anthropic/thinking.json deleted file mode 100644 index 2ecd46fc6..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/anthropic/thinking.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "id": "msg_01BE9GBzjs95TqdG8FeUanUn", - "type": "message", - "role": "assistant", - "model": "claude-opus-4-6", - "content": [ - { - "type": "thinking", - "thinking": "The user is asking me about the current weather in San Francisco", - "signature": "Es8DCkYI" - }, - { - "type": "text", - "text": "I don't have access to real-time weather data" - } - ], - "stop_reason": "end_turn", - "stop_sequence": null, - "usage": { - "input_tokens": 47, - "cache_creation_input_tokens": 1, - "cache_read_input_tokens": 2, - "cache_creation": { - "ephemeral_5m_input_tokens": 0, - "ephemeral_1h_input_tokens": 0 - }, - "output_tokens": 251, - "service_tier": "standard" - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/anthropic/thinking.messages-completions.snap b/crates/persisting-gateway/tests/fixtures/response/anthropic/thinking.messages-completions.snap deleted file mode 100644 index 5a8888fdf..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/anthropic/thinking.messages-completions.snap +++ /dev/null @@ -1,66 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/anthropic/thinking.json -info: - id: msg_01BE9GBzjs95TqdG8FeUanUn - type: message - role: assistant - model: claude-opus-4-6 - content: - - type: thinking - thinking: The user is asking me about the current weather in San Francisco - signature: Es8DCkYI - - type: text - text: "I don't have access to real-time weather data" - stop_reason: end_turn - stop_sequence: ~ - usage: - input_tokens: 47 - cache_creation_input_tokens: 1 - cache_read_input_tokens: 2 - cache_creation: - ephemeral_5m_input_tokens: 0 - ephemeral_1h_input_tokens: 0 - output_tokens: 251 - service_tier: standard ---- -{ - "response": { - "model": "claude-opus-4-6", - "service_tier": "standard", - "usage": { - "prompt_tokens": 47, - "completion_tokens": 251, - "total_tokens": 298, - "prompt_tokens_details": { - "cached_tokens": 2, - "cache_write_tokens": 1 - }, - "cache_read_input_tokens": 2, - "cache_creation_input_tokens": 1 - }, - "choices": [ - { - "message": { - "content": "I don't have access to real-time weather data", - "role": "assistant", - "reasoning_content": "The user is asking me about the current weather in San Francisco" - }, - "index": 0, - "finish_reason": "stop" - } - ], - "id": "[id]", - "created": "[date]", - "object": "chat.completion" - }, - "parsed": { - "input_tokens": 47, - "output_tokens": 251, - "total_tokens": 298, - "cache_creation_input_tokens": 1, - "cached_input_tokens": 2, - "service_tier": "standard", - "provider_model": "claude-opus-4-6" - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/anthropic/thinking.messages-detect.snap b/crates/persisting-gateway/tests/fixtures/response/anthropic/thinking.messages-detect.snap deleted file mode 100644 index 43ab015fa..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/anthropic/thinking.messages-detect.snap +++ /dev/null @@ -1,67 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/anthropic/thinking.json -info: - id: msg_01BE9GBzjs95TqdG8FeUanUn - type: message - role: assistant - model: claude-opus-4-6 - content: - - type: thinking - thinking: The user is asking me about the current weather in San Francisco - signature: Es8DCkYI - - type: text - text: "I don't have access to real-time weather data" - stop_reason: end_turn - stop_sequence: ~ - usage: - input_tokens: 47 - cache_creation_input_tokens: 1 - cache_read_input_tokens: 2 - cache_creation: - ephemeral_5m_input_tokens: 0 - ephemeral_1h_input_tokens: 0 - output_tokens: 251 - service_tier: standard ---- -{ - "response": { - "id": "[id]", - "type": "message", - "role": "assistant", - "model": "claude-opus-4-6", - "content": [ - { - "type": "thinking", - "thinking": "The user is asking me about the current weather in San Francisco", - "signature": "Es8DCkYI" - }, - { - "type": "text", - "text": "I don't have access to real-time weather data" - } - ], - "stop_reason": "end_turn", - "stop_sequence": null, - "usage": { - "input_tokens": 47, - "cache_creation_input_tokens": 1, - "cache_read_input_tokens": 2, - "cache_creation": { - "ephemeral_5m_input_tokens": 0, - "ephemeral_1h_input_tokens": 0 - }, - "output_tokens": 251, - "service_tier": "standard" - } - }, - "parsed": { - "input_tokens": 47, - "output_tokens": 251, - "total_tokens": 298, - "cache_creation_input_tokens": 1, - "cached_input_tokens": 2, - "service_tier": "standard", - "provider_model": "claude-opus-4-6" - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/anthropic/thinking.messages-messages.snap b/crates/persisting-gateway/tests/fixtures/response/anthropic/thinking.messages-messages.snap deleted file mode 100644 index b7f493b0f..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/anthropic/thinking.messages-messages.snap +++ /dev/null @@ -1,67 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/anthropic/thinking.json -info: - id: msg_01BE9GBzjs95TqdG8FeUanUn - type: message - role: assistant - model: claude-opus-4-6 - content: - - type: thinking - thinking: The user is asking me about the current weather in San Francisco - signature: Es8DCkYI - - type: text - text: "I don't have access to real-time weather data" - stop_reason: end_turn - stop_sequence: ~ - usage: - input_tokens: 47 - cache_creation_input_tokens: 1 - cache_read_input_tokens: 2 - cache_creation: - ephemeral_5m_input_tokens: 0 - ephemeral_1h_input_tokens: 0 - output_tokens: 251 - service_tier: standard ---- -{ - "response": { - "id": "[id]", - "type": "message", - "role": "assistant", - "model": "claude-opus-4-6", - "stop_reason": "end_turn", - "stop_sequence": null, - "usage": { - "input_tokens": 47, - "output_tokens": 251, - "cache_creation_input_tokens": 1, - "cache_read_input_tokens": 2, - "service_tier": "standard", - "cache_creation": { - "ephemeral_5m_input_tokens": 0, - "ephemeral_1h_input_tokens": 0 - } - }, - "content": [ - { - "type": "thinking", - "thinking": "The user is asking me about the current weather in San Francisco", - "signature": "Es8DCkYI" - }, - { - "text": "I don't have access to real-time weather data", - "type": "text" - } - ] - }, - "parsed": { - "input_tokens": 47, - "output_tokens": 251, - "total_tokens": 298, - "cache_creation_input_tokens": 1, - "cached_input_tokens": 2, - "service_tier": "standard", - "provider_model": "claude-opus-4-6" - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/anthropic/tool.json b/crates/persisting-gateway/tests/fixtures/response/anthropic/tool.json deleted file mode 100644 index 5a6338fd2..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/anthropic/tool.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "id": "msg_01Aq9w938a90dw8q", - "type": "message", - "model": "claude-opus-4-6", - "stop_reason": "tool_use", - "role": "assistant", - "content": [ - { - "type": "text", - "text": "I need to call the get_weather function, and the user wants SF, which is likely San Francisco, CA." - }, - { - "type": "tool_use", - "id": "toolu_01A09q90qw90lq917835lq9", - "name": "get_weather", - "input": {"location": "San Francisco, CA", "unit": "celsius"} - } - ], - "usage": { - "input_tokens": 558, - "cache_creation_input_tokens": 1, - "cache_read_input_tokens": 2, - "output_tokens": 48, - "service_tier": "standard" - } -} \ No newline at end of file diff --git a/crates/persisting-gateway/tests/fixtures/response/anthropic/tool.messages-completions.snap b/crates/persisting-gateway/tests/fixtures/response/anthropic/tool.messages-completions.snap deleted file mode 100644 index 4e3ced478..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/anthropic/tool.messages-completions.snap +++ /dev/null @@ -1,91 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/anthropic/tool.json -info: - id: msg_01Aq9w938a90dw8q - type: message - model: claude-opus-4-6 - stop_reason: tool_use - role: assistant - content: - - type: text - text: "I need to call the get_weather function, and the user wants SF, which is likely San Francisco, CA." - - type: tool_use - id: toolu_01A09q90qw90lq917835lq9 - name: get_weather - input: - location: "San Francisco, CA" - unit: celsius - usage: - input_tokens: 558 - cache_creation_input_tokens: 1 - cache_read_input_tokens: 2 - output_tokens: 48 - service_tier: standard ---- -{ - "response": { - "model": "claude-opus-4-6", - "service_tier": "standard", - "usage": { - "prompt_tokens": 558, - "completion_tokens": 48, - "total_tokens": 606, - "prompt_tokens_details": { - "cached_tokens": 2, - "cache_write_tokens": 1 - }, - "cache_read_input_tokens": 2, - "cache_creation_input_tokens": 1 - }, - "choices": [ - { - "message": { - "content": "I need to call the get_weather function, and the user wants SF, which is likely San Francisco, CA.", - "role": "assistant", - "tool_calls": [ - { - "type": "function", - "id": "toolu_01A09q90qw90lq917835lq9", - "function": { - "name": "get_weather", - "arguments": "{\"location\":\"San Francisco, CA\",\"unit\":\"celsius\"}" - } - } - ] - }, - "index": 0, - "finish_reason": "tool_calls" - } - ], - "id": "[id]", - "created": "[date]", - "object": "chat.completion" - }, - "parsed": { - "input_tokens": 558, - "output_tokens": 48, - "total_tokens": 606, - "cache_creation_input_tokens": 1, - "cached_input_tokens": 2, - "service_tier": "standard", - "provider_model": "claude-opus-4-6", - "output_messages": [ - { - "role": "assistant", - "content": [ - { - "type": "tool_call", - "id": "toolu_01A09q90qw90lq917835lq9", - "name": "get_weather", - "arguments": { - "location": "San Francisco, CA", - "unit": "celsius" - } - } - ], - "finish_reason": "tool_calls" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/anthropic/tool.messages-detect.snap b/crates/persisting-gateway/tests/fixtures/response/anthropic/tool.messages-detect.snap deleted file mode 100644 index db329e136..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/anthropic/tool.messages-detect.snap +++ /dev/null @@ -1,65 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/anthropic/tool.json -info: - id: msg_01Aq9w938a90dw8q - type: message - model: claude-opus-4-6 - stop_reason: tool_use - role: assistant - content: - - type: text - text: "I need to call the get_weather function, and the user wants SF, which is likely San Francisco, CA." - - type: tool_use - id: toolu_01A09q90qw90lq917835lq9 - name: get_weather - input: - location: "San Francisco, CA" - unit: celsius - usage: - input_tokens: 558 - cache_creation_input_tokens: 1 - cache_read_input_tokens: 2 - output_tokens: 48 - service_tier: standard ---- -{ - "response": { - "id": "[id]", - "type": "message", - "model": "claude-opus-4-6", - "stop_reason": "tool_use", - "role": "assistant", - "content": [ - { - "type": "text", - "text": "I need to call the get_weather function, and the user wants SF, which is likely San Francisco, CA." - }, - { - "type": "tool_use", - "id": "toolu_01A09q90qw90lq917835lq9", - "name": "get_weather", - "input": { - "location": "San Francisco, CA", - "unit": "celsius" - } - } - ], - "usage": { - "input_tokens": 558, - "cache_creation_input_tokens": 1, - "cache_read_input_tokens": 2, - "output_tokens": 48, - "service_tier": "standard" - } - }, - "parsed": { - "input_tokens": 558, - "output_tokens": 48, - "total_tokens": 606, - "cache_creation_input_tokens": 1, - "cached_input_tokens": 2, - "service_tier": "standard", - "provider_model": "claude-opus-4-6" - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/anthropic/tool.messages-messages.snap b/crates/persisting-gateway/tests/fixtures/response/anthropic/tool.messages-messages.snap deleted file mode 100644 index 56bccb276..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/anthropic/tool.messages-messages.snap +++ /dev/null @@ -1,83 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/anthropic/tool.json -info: - id: msg_01Aq9w938a90dw8q - type: message - model: claude-opus-4-6 - stop_reason: tool_use - role: assistant - content: - - type: text - text: "I need to call the get_weather function, and the user wants SF, which is likely San Francisco, CA." - - type: tool_use - id: toolu_01A09q90qw90lq917835lq9 - name: get_weather - input: - location: "San Francisco, CA" - unit: celsius - usage: - input_tokens: 558 - cache_creation_input_tokens: 1 - cache_read_input_tokens: 2 - output_tokens: 48 - service_tier: standard ---- -{ - "response": { - "id": "[id]", - "type": "message", - "role": "assistant", - "model": "claude-opus-4-6", - "stop_reason": "tool_use", - "stop_sequence": null, - "usage": { - "input_tokens": 558, - "output_tokens": 48, - "cache_creation_input_tokens": 1, - "cache_read_input_tokens": 2, - "service_tier": "standard" - }, - "content": [ - { - "text": "I need to call the get_weather function, and the user wants SF, which is likely San Francisco, CA.", - "type": "text" - }, - { - "type": "tool_use", - "id": "toolu_01A09q90qw90lq917835lq9", - "name": "get_weather", - "input": { - "location": "San Francisco, CA", - "unit": "celsius" - } - } - ] - }, - "parsed": { - "input_tokens": 558, - "output_tokens": 48, - "total_tokens": 606, - "cache_creation_input_tokens": 1, - "cached_input_tokens": 2, - "service_tier": "standard", - "provider_model": "claude-opus-4-6", - "output_messages": [ - { - "role": "assistant", - "content": [ - { - "type": "tool_call", - "id": "toolu_01A09q90qw90lq917835lq9", - "name": "get_weather", - "arguments": { - "location": "San Francisco, CA", - "unit": "celsius" - } - } - ], - "finish_reason": "tool_use" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/bedrock-cohere/embeddings.bedrock-cohere.snap b/crates/persisting-gateway/tests/fixtures/response/bedrock-cohere/embeddings.bedrock-cohere.snap deleted file mode 100644 index a5335b96f..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/bedrock-cohere/embeddings.bedrock-cohere.snap +++ /dev/null @@ -1,51 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/bedrock-cohere/embeddings.json -info: - embeddings: - - - 0.1 - - 0.2 - - 0.3 - - - 0.4 - - 0.5 - - 0.6 - id: "123" - texts: - - hello - - world ---- -{ - "response": { - "object": "list", - "model": "cohere.embed-english-v3", - "usage": { - "prompt_tokens": 0, - "total_tokens": 0 - }, - "data": [ - { - "index": 0, - "object": "embedding", - "embedding": [ - 0.10000000149011612, - 0.20000000298023224, - 0.30000001192092896 - ] - }, - { - "index": 1, - "object": "embedding", - "embedding": [ - 0.4000000059604645, - 0.5, - 0.6000000238418579 - ] - } - ] - }, - "parsed": { - "input_tokens": 0, - "total_tokens": 0, - "provider_model": "cohere.embed-english-v3" - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/bedrock-cohere/embeddings.json b/crates/persisting-gateway/tests/fixtures/response/bedrock-cohere/embeddings.json deleted file mode 100644 index 77b9fb50f..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/bedrock-cohere/embeddings.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "embeddings": [[0.1, 0.2, 0.3], [0.4, 0.5, 0.6]], - "id": "123", - "texts": ["hello", "world"] -} diff --git a/crates/persisting-gateway/tests/fixtures/response/bedrock-nova/embeddings.bedrock-nova.snap b/crates/persisting-gateway/tests/fixtures/response/bedrock-nova/embeddings.bedrock-nova.snap deleted file mode 100644 index e0d4ef8be..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/bedrock-nova/embeddings.bedrock-nova.snap +++ /dev/null @@ -1,37 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/bedrock-nova/embeddings.json -info: - embeddings: - - embedding: - - -0.006603240966796875 - - -0.054168701171875 - - 0.0121917724609375 - embeddingType: TEXT ---- -{ - "response": { - "object": "list", - "model": "amazon.nova-2-multimodal-embeddings-v1:0", - "usage": { - "prompt_tokens": 0, - "total_tokens": 0 - }, - "data": [ - { - "index": 0, - "object": "embedding", - "embedding": [ - -0.006603240966796875, - -0.054168701171875, - 0.0121917724609375 - ] - } - ] - }, - "parsed": { - "input_tokens": 0, - "total_tokens": 0, - "provider_model": "amazon.nova-2-multimodal-embeddings-v1:0" - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/bedrock-nova/embeddings.json b/crates/persisting-gateway/tests/fixtures/response/bedrock-nova/embeddings.json deleted file mode 100644 index d1a95b697..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/bedrock-nova/embeddings.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "embeddings": [ - { - "embedding": [-0.006603240966796875,-0.054168701171875,0.0121917724609375], - "embeddingType": "TEXT" - } - ] -} diff --git a/crates/persisting-gateway/tests/fixtures/response/bedrock-titan/embeddings.bedrock-titan.snap b/crates/persisting-gateway/tests/fixtures/response/bedrock-titan/embeddings.bedrock-titan.snap deleted file mode 100644 index c24b9e8fa..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/bedrock-titan/embeddings.bedrock-titan.snap +++ /dev/null @@ -1,36 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/bedrock-titan/embeddings.json -info: - embedding: - - -0.006603240966796875 - - -0.054168701171875 - - 0.0121917724609375 - inputTextTokenCount: 3 ---- -{ - "response": { - "object": "list", - "model": "amazon.titan-embed-text-v2:0", - "usage": { - "prompt_tokens": 3, - "total_tokens": 3 - }, - "data": [ - { - "index": 0, - "object": "embedding", - "embedding": [ - -0.006603240966796875, - -0.054168701171875, - 0.0121917724609375 - ] - } - ] - }, - "parsed": { - "input_tokens": 3, - "total_tokens": 3, - "provider_model": "amazon.titan-embed-text-v2:0" - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/bedrock-titan/embeddings.json b/crates/persisting-gateway/tests/fixtures/response/bedrock-titan/embeddings.json deleted file mode 100644 index b318fcdb4..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/bedrock-titan/embeddings.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "embedding": [-0.006603240966796875,-0.054168701171875,0.0121917724609375], - "inputTextTokenCount": 3 -} diff --git a/crates/persisting-gateway/tests/fixtures/response/bedrock/basic.bedrock-completions-streaming.snap b/crates/persisting-gateway/tests/fixtures/response/bedrock/basic.bedrock-completions-streaming.snap deleted file mode 100644 index 924174482..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/bedrock/basic.bedrock-completions-streaming.snap +++ /dev/null @@ -1,28 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/bedrock/basic.bin ---- -data: {"id":"request_id","choices":[{"index":0,"delta":{"content":null,"role":"assistant"}}],"created":123,"model":"input-model","service_tier":null,"system_fingerprint":null,"object":"chat.completion.chunk","usage":null} - -data: {"id":"request_id","choices":[{"index":0,"delta":{"content":"\nOnce upon a time, there was a little boy named Jack who loved to play with his friends in the park. His favorite activity was to climb the tallest tree and swing on the strongest branch. One day, Jac"}}],"created":123,"model":"input-model","service_tier":null,"system_fingerprint":null,"object":"chat.completion.chunk","usage":null} - -data: {"id":"request_id","choices":[{"index":0,"delta":{"content":"k and his friends were playing when they heard a loud noise coming from the tree. They ran over to investigate and found a large bird trapped inside.\n\nJack knew that he had to help the bird, so he climbed up the tree and gently removed the bird from the branches. The bird was grateful and thanked Jack for his help"}}],"created":123,"model":"input-model","service_tier":null,"system_fingerprint":null,"object":"chat.completion.chunk","usage":null} - -data: {"id":"request_id","choices":[{"index":0,"delta":{"content":". From that day on, Jack became known as the bird whisperer in the park and everyone loved to hear his stories about the birds he had helped."}}],"created":123,"model":"input-model","service_tier":null,"system_fingerprint":null,"object":"chat.completion.chunk","usage":null} - -data: {"id":"request_id","choices":[{"index":0,"delta":{"content":null},"finish_reason":"stop"}],"created":123,"model":"input-model","service_tier":null,"system_fingerprint":null,"object":"chat.completion.chunk","usage":null} - -data: {"id":"request_id","choices":[],"created":123,"model":"input-model","service_tier":null,"system_fingerprint":null,"object":"chat.completion.chunk","usage":{"prompt_tokens":15,"completion_tokens":142,"total_tokens":157}} - -data: [DONE] - - - -{ - "input_tokens": 15, - "output_tokens": 142, - "total_tokens": 157, - "completion": [ - "\nOnce upon a time, there was a little boy named Jack who loved to play with his friends in the park. His favorite activity was to climb the tallest tree and swing on the strongest branch. One day, Jack and his friends were playing when they heard a loud noise coming from the tree. They ran over to investigate and found a large bird trapped inside.\n\nJack knew that he had to help the bird, so he climbed up the tree and gently removed the bird from the branches. The bird was grateful and thanked Jack for his help. From that day on, Jack became known as the bird whisperer in the park and everyone loved to hear his stories about the birds he had helped." - ] -} diff --git a/crates/persisting-gateway/tests/fixtures/response/bedrock/basic.bedrock-completions.snap b/crates/persisting-gateway/tests/fixtures/response/bedrock/basic.bedrock-completions.snap deleted file mode 100644 index 5ebe26f83..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/bedrock/basic.bedrock-completions.snap +++ /dev/null @@ -1,46 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/bedrock/basic.json -info: - output: - message: - role: assistant - content: - - text: Hello world - stopReason: end_turn - usage: - inputTokens: 18 - outputTokens: 46 - totalTokens: 64 - metrics: - latencyMs: 2083 ---- -{ - "response": { - "model": "input-model", - "usage": { - "prompt_tokens": 18, - "completion_tokens": 46, - "total_tokens": 64 - }, - "choices": [ - { - "message": { - "content": "Hello world", - "role": "assistant" - }, - "index": 0, - "finish_reason": "stop" - } - ], - "id": "[id]", - "created": "[date]", - "object": "chat.completion" - }, - "parsed": { - "input_tokens": 18, - "output_tokens": 46, - "total_tokens": 64, - "provider_model": "input-model" - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/bedrock/basic.bedrock-messages-streaming.snap b/crates/persisting-gateway/tests/fixtures/response/bedrock/basic.bedrock-messages-streaming.snap deleted file mode 100644 index 267c4962c..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/bedrock/basic.bedrock-messages-streaming.snap +++ /dev/null @@ -1,38 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/bedrock/basic.bin ---- -event: message_start -data: {"type":"message_start","message":{"id":"","type":"message","role":"assistant","content":[],"model":"input-model","stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":0,"output_tokens":0}}} - -event: content_block_start -data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"\nOnce upon a time, there was a little boy named Jack who loved to play with his friends in the park. His favorite activity was to climb the tallest tree and swing on the strongest branch. One day, Jac"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"k and his friends were playing when they heard a loud noise coming from the tree. They ran over to investigate and found a large bird trapped inside.\n\nJack knew that he had to help the bird, so he climbed up the tree and gently removed the bird from the branches. The bird was grateful and thanked Jack for his help"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":". From that day on, Jack became known as the bird whisperer in the park and everyone loved to hear his stories about the birds he had helped."}} - -event: content_block_stop -data: {"type":"content_block_stop","index":0} - -event: message_delta -data: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"input_tokens":15,"output_tokens":142}} - -event: message_stop -data: {"type":"message_stop"} - - - -{ - "input_tokens": 15, - "output_tokens": 142, - "total_tokens": 157, - "completion": [ - "\nOnce upon a time, there was a little boy named Jack who loved to play with his friends in the park. His favorite activity was to climb the tallest tree and swing on the strongest branch. One day, Jack and his friends were playing when they heard a loud noise coming from the tree. They ran over to investigate and found a large bird trapped inside.\n\nJack knew that he had to help the bird, so he climbed up the tree and gently removed the bird from the branches. The bird was grateful and thanked Jack for his help. From that day on, Jack became known as the bird whisperer in the park and everyone loved to hear his stories about the birds he had helped." - ] -} diff --git a/crates/persisting-gateway/tests/fixtures/response/bedrock/basic.bedrock-messages.snap b/crates/persisting-gateway/tests/fixtures/response/bedrock/basic.bedrock-messages.snap deleted file mode 100644 index 501296197..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/bedrock/basic.bedrock-messages.snap +++ /dev/null @@ -1,43 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/bedrock/basic.json -info: - output: - message: - role: assistant - content: - - text: Hello world - stopReason: end_turn - usage: - inputTokens: 18 - outputTokens: 46 - totalTokens: 64 - metrics: - latencyMs: 2083 ---- -{ - "response": { - "id": "[id]", - "type": "message", - "role": "assistant", - "model": "input-model", - "stop_reason": "end_turn", - "stop_sequence": null, - "usage": { - "input_tokens": 18, - "output_tokens": 46 - }, - "content": [ - { - "text": "Hello world", - "type": "text" - } - ] - }, - "parsed": { - "input_tokens": 18, - "output_tokens": 46, - "total_tokens": 64, - "provider_model": "input-model" - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/bedrock/basic.bedrock-responses-streaming.snap b/crates/persisting-gateway/tests/fixtures/response/bedrock/basic.bedrock-responses-streaming.snap deleted file mode 100644 index 0b222a8af..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/bedrock/basic.bedrock-responses-streaming.snap +++ /dev/null @@ -1,35 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/bedrock/basic.bin ---- -event: event -data: {"type":"response.created","sequence_number":1,"response":{"created_at":123,"id":"","model":"input-model","object":"response","output":[],"status":"in_progress"}} - -event: event -data: {"type":"response.output_item.added","sequence_number":2,"output_index":0,"item":{"type":"message","content":[],"id":"","role":"assistant","status":"in_progress"}} - -event: event -data: {"type":"response.output_text.delta","sequence_number":3,"item_id":"","output_index":0,"content_index":0,"delta":"\nOnce upon a time, there was a little boy named Jack who loved to play with his friends in the park. His favorite activity was to climb the tallest tree and swing on the strongest branch. One day, Jac"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":4,"item_id":"","output_index":0,"content_index":0,"delta":"k and his friends were playing when they heard a loud noise coming from the tree. They ran over to investigate and found a large bird trapped inside.\n\nJack knew that he had to help the bird, so he climbed up the tree and gently removed the bird from the branches. The bird was grateful and thanked Jack for his help"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":5,"item_id":"","output_index":0,"content_index":0,"delta":". From that day on, Jack became known as the bird whisperer in the park and everyone loved to hear his stories about the birds he had helped."} - -event: event -data: {"type":"response.output_item.done","sequence_number":6,"output_index":0,"item":{"type":"message","content":[],"id":"","role":"assistant","status":"completed"}} - -event: event -data: {"type":"response.completed","sequence_number":7,"response":{"created_at":123,"id":"","model":"input-model","object":"response","output":[],"status":"completed","usage":{"input_tokens":15,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":null},"output_tokens":142,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":157}}} - - - -{ - "input_tokens": 15, - "output_tokens": 142, - "total_tokens": 157, - "completion": [ - "\nOnce upon a time, there was a little boy named Jack who loved to play with his friends in the park. His favorite activity was to climb the tallest tree and swing on the strongest branch. One day, Jack and his friends were playing when they heard a loud noise coming from the tree. They ran over to investigate and found a large bird trapped inside.\n\nJack knew that he had to help the bird, so he climbed up the tree and gently removed the bird from the branches. The bird was grateful and thanked Jack for his help. From that day on, Jack became known as the bird whisperer in the park and everyone loved to hear his stories about the birds he had helped." - ] -} diff --git a/crates/persisting-gateway/tests/fixtures/response/bedrock/basic.bedrock-responses.snap b/crates/persisting-gateway/tests/fixtures/response/bedrock/basic.bedrock-responses.snap deleted file mode 100644 index 66b852a6c..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/bedrock/basic.bedrock-responses.snap +++ /dev/null @@ -1,61 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/bedrock/basic.json -info: - output: - message: - role: assistant - content: - - text: Hello world - stopReason: end_turn - usage: - inputTokens: 18 - outputTokens: 46 - totalTokens: 64 - metrics: - latencyMs: 2083 ---- -{ - "response": { - "id": "[id]", - "status": "completed", - "output": [ - { - "type": "message", - "content": [ - { - "type": "output_text", - "annotations": [], - "logprobs": null, - "text": "Hello world" - } - ], - "id": "[id]", - "role": "assistant", - "status": "completed" - } - ], - "model": "input-model", - "usage": { - "input_tokens": 18, - "output_tokens": 46, - "input_tokens_details": { - "cached_tokens": 0 - }, - "output_tokens_details": { - "reasoning_tokens": 0 - }, - "total_tokens": 64 - }, - "created_at": "[date]", - "object": "response" - }, - "parsed": { - "input_tokens": 18, - "output_tokens": 46, - "total_tokens": 64, - "reasoning_tokens": 0, - "cached_input_tokens": 0, - "provider_model": "input-model" - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/bedrock/basic.bin b/crates/persisting-gateway/tests/fixtures/response/bedrock/basic.bin deleted file mode 100644 index 88991f9db..000000000 Binary files a/crates/persisting-gateway/tests/fixtures/response/bedrock/basic.bin and /dev/null differ diff --git a/crates/persisting-gateway/tests/fixtures/response/bedrock/basic.json b/crates/persisting-gateway/tests/fixtures/response/bedrock/basic.json deleted file mode 100644 index 4af253ee5..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/bedrock/basic.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "output": { - "message": { - "role": "assistant", - "content": [ - { - "text": "Hello world" - } - ] - } - }, - "stopReason": "end_turn", - "usage": { - "inputTokens": 18, - "outputTokens": 46, - "totalTokens": 64 - }, - "metrics": { - "latencyMs": 2083 - } -} \ No newline at end of file diff --git a/crates/persisting-gateway/tests/fixtures/response/bedrock/cache_write.bedrock-completions.snap b/crates/persisting-gateway/tests/fixtures/response/bedrock/cache_write.bedrock-completions.snap deleted file mode 100644 index 5b387aba8..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/bedrock/cache_write.bedrock-completions.snap +++ /dev/null @@ -1,56 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/bedrock/cache_write.json -info: - output: - message: - role: assistant - content: - - text: ok - stopReason: end_turn - usage: - inputTokens: 16 - outputTokens: 9 - totalTokens: 2412 - cacheReadInputTokens: 0 - cacheWriteInputTokens: 2387 - metrics: - latencyMs: 2083 ---- -{ - "response": { - "model": "input-model", - "usage": { - "prompt_tokens": 16, - "completion_tokens": 9, - "total_tokens": 2412, - "prompt_tokens_details": { - "cached_tokens": 0, - "cache_write_tokens": 2387 - }, - "cache_read_input_tokens": 0, - "cache_creation_input_tokens": 2387 - }, - "choices": [ - { - "message": { - "content": "ok", - "role": "assistant" - }, - "index": 0, - "finish_reason": "stop" - } - ], - "id": "[id]", - "created": "[date]", - "object": "chat.completion" - }, - "parsed": { - "input_tokens": 16, - "output_tokens": 9, - "total_tokens": 2412, - "cache_creation_input_tokens": 2387, - "cached_input_tokens": 0, - "provider_model": "input-model" - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/bedrock/cache_write.bedrock-responses.snap b/crates/persisting-gateway/tests/fixtures/response/bedrock/cache_write.bedrock-responses.snap deleted file mode 100644 index be026a099..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/bedrock/cache_write.bedrock-responses.snap +++ /dev/null @@ -1,65 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/bedrock/cache_write.json -info: - output: - message: - role: assistant - content: - - text: ok - stopReason: end_turn - usage: - inputTokens: 16 - outputTokens: 9 - totalTokens: 2412 - cacheReadInputTokens: 0 - cacheWriteInputTokens: 2387 - metrics: - latencyMs: 2083 ---- -{ - "response": { - "id": "[id]", - "status": "completed", - "output": [ - { - "type": "message", - "content": [ - { - "type": "output_text", - "annotations": [], - "logprobs": null, - "text": "ok" - } - ], - "id": "[id]", - "role": "assistant", - "status": "completed" - } - ], - "model": "input-model", - "usage": { - "input_tokens": 16, - "output_tokens": 9, - "input_tokens_details": { - "cached_tokens": 0, - "cache_write_tokens": 2387 - }, - "output_tokens_details": { - "reasoning_tokens": 0 - }, - "total_tokens": 2412 - }, - "created_at": "[date]", - "object": "response" - }, - "parsed": { - "input_tokens": 16, - "output_tokens": 9, - "total_tokens": 2412, - "reasoning_tokens": 0, - "cache_creation_input_tokens": 2387, - "cached_input_tokens": 0, - "provider_model": "input-model" - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/bedrock/cache_write.json b/crates/persisting-gateway/tests/fixtures/response/bedrock/cache_write.json deleted file mode 100644 index ecf96f506..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/bedrock/cache_write.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "output": { - "message": { - "role": "assistant", - "content": [ - { - "text": "ok" - } - ] - } - }, - "stopReason": "end_turn", - "usage": { - "inputTokens": 16, - "outputTokens": 9, - "totalTokens": 2412, - "cacheReadInputTokens": 0, - "cacheWriteInputTokens": 2387 - }, - "metrics": { - "latencyMs": 2083 - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/bedrock/max_tokens.bedrock-responses.snap b/crates/persisting-gateway/tests/fixtures/response/bedrock/max_tokens.bedrock-responses.snap deleted file mode 100644 index 7ef28d2c3..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/bedrock/max_tokens.bedrock-responses.snap +++ /dev/null @@ -1,62 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/bedrock/max_tokens.json -info: - output: - message: - role: assistant - content: - - text: Partial response - stopReason: max_tokens - usage: - inputTokens: 18 - outputTokens: 46 - totalTokens: 64 ---- -{ - "response": { - "id": "[id]", - "status": "incomplete", - "output": [ - { - "type": "message", - "content": [ - { - "type": "output_text", - "annotations": [], - "logprobs": null, - "text": "Partial response" - } - ], - "id": "[id]", - "role": "assistant", - "status": "incomplete" - } - ], - "model": "input-model", - "usage": { - "input_tokens": 18, - "output_tokens": 46, - "input_tokens_details": { - "cached_tokens": 0 - }, - "output_tokens_details": { - "reasoning_tokens": 0 - }, - "total_tokens": 64 - }, - "created_at": "[date]", - "incomplete_details": { - "reason": "max_tokens" - }, - "object": "response" - }, - "parsed": { - "input_tokens": 18, - "output_tokens": 46, - "total_tokens": 64, - "reasoning_tokens": 0, - "cached_input_tokens": 0, - "provider_model": "input-model" - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/bedrock/max_tokens.json b/crates/persisting-gateway/tests/fixtures/response/bedrock/max_tokens.json deleted file mode 100644 index 9e860f466..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/bedrock/max_tokens.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "output": { - "message": { - "role": "assistant", - "content": [ - { - "text": "Partial response" - } - ] - } - }, - "stopReason": "max_tokens", - "usage": { - "inputTokens": 18, - "outputTokens": 46, - "totalTokens": 64 - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/bedrock/reasoning.bedrock-completions-streaming.snap b/crates/persisting-gateway/tests/fixtures/response/bedrock/reasoning.bedrock-completions-streaming.snap deleted file mode 100644 index 4f9032f6e..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/bedrock/reasoning.bedrock-completions-streaming.snap +++ /dev/null @@ -1,28 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/bedrock/reasoning.bin ---- -data: {"id":"request_id","choices":[{"index":0,"delta":{"content":null,"role":"assistant"}}],"created":123,"model":"input-model","service_tier":null,"system_fingerprint":null,"object":"chat.completion.chunk","usage":null} - -data: {"id":"request_id","choices":[{"index":0,"delta":{"content":null,"reasoning_content":"Two plus two is four."}}],"created":123,"model":"input-model","service_tier":null,"system_fingerprint":null,"object":"chat.completion.chunk","usage":null} - -data: {"id":"request_id","choices":[{"index":0,"delta":{"content":null,"reasoning_signature":"SIG-xyz=="}}],"created":123,"model":"input-model","service_tier":null,"system_fingerprint":null,"object":"chat.completion.chunk","usage":null} - -data: {"id":"request_id","choices":[{"index":0,"delta":{"content":"The answer is 4."}}],"created":123,"model":"input-model","service_tier":null,"system_fingerprint":null,"object":"chat.completion.chunk","usage":null} - -data: {"id":"request_id","choices":[{"index":0,"delta":{"content":null},"finish_reason":"stop"}],"created":123,"model":"input-model","service_tier":null,"system_fingerprint":null,"object":"chat.completion.chunk","usage":null} - -data: {"id":"request_id","choices":[],"created":123,"model":"input-model","service_tier":null,"system_fingerprint":null,"object":"chat.completion.chunk","usage":{"prompt_tokens":18,"completion_tokens":46,"total_tokens":64}} - -data: [DONE] - - - -{ - "input_tokens": 18, - "output_tokens": 46, - "total_tokens": 64, - "completion": [ - "The answer is 4." - ] -} diff --git a/crates/persisting-gateway/tests/fixtures/response/bedrock/reasoning.bedrock-completions.snap b/crates/persisting-gateway/tests/fixtures/response/bedrock/reasoning.bedrock-completions.snap deleted file mode 100644 index d281b02c7..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/bedrock/reasoning.bedrock-completions.snap +++ /dev/null @@ -1,52 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/bedrock/reasoning.json -info: - output: - message: - role: assistant - content: - - reasoningContent: - reasoningText: - text: Two plus two is four. - signature: SIG-xyz== - - text: The answer is 4. - stopReason: end_turn - usage: - inputTokens: 18 - outputTokens: 46 - totalTokens: 64 - metrics: - latencyMs: 2083 ---- -{ - "response": { - "model": "input-model", - "usage": { - "prompt_tokens": 18, - "completion_tokens": 46, - "total_tokens": 64 - }, - "choices": [ - { - "message": { - "content": "The answer is 4.", - "role": "assistant", - "reasoning_content": "Two plus two is four.", - "reasoning_signature": "SIG-xyz==" - }, - "index": 0, - "finish_reason": "stop" - } - ], - "id": "[id]", - "created": "[date]", - "object": "chat.completion" - }, - "parsed": { - "input_tokens": 18, - "output_tokens": 46, - "total_tokens": 64, - "provider_model": "input-model" - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/bedrock/reasoning.bedrock-messages-streaming.snap b/crates/persisting-gateway/tests/fixtures/response/bedrock/reasoning.bedrock-messages-streaming.snap deleted file mode 100644 index 60d6562ff..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/bedrock/reasoning.bedrock-messages-streaming.snap +++ /dev/null @@ -1,38 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/bedrock/reasoning.bin ---- -event: message_start -data: {"type":"message_start","message":{"id":"","type":"message","role":"assistant","content":[],"model":"input-model","stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":0,"output_tokens":0}}} - -event: content_block_start -data: {"type":"content_block_start","index":0,"content_block":{"type":"thinking","thinking":"","signature":""}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":"Two plus two is four."}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"signature_delta","signature":"SIG-xyz=="}} - -event: content_block_start -data: {"type":"content_block_start","index":1,"content_block":{"type":"text","text":""}} - -event: content_block_delta -data: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"The answer is 4."}} - -event: message_delta -data: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"input_tokens":18,"output_tokens":46}} - -event: message_stop -data: {"type":"message_stop"} - - - -{ - "input_tokens": 18, - "output_tokens": 46, - "total_tokens": 64, - "completion": [ - "The answer is 4." - ] -} diff --git a/crates/persisting-gateway/tests/fixtures/response/bedrock/reasoning.bedrock-messages.snap b/crates/persisting-gateway/tests/fixtures/response/bedrock/reasoning.bedrock-messages.snap deleted file mode 100644 index 8047050ca..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/bedrock/reasoning.bedrock-messages.snap +++ /dev/null @@ -1,52 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/bedrock/reasoning.json -info: - output: - message: - role: assistant - content: - - reasoningContent: - reasoningText: - text: Two plus two is four. - signature: SIG-xyz== - - text: The answer is 4. - stopReason: end_turn - usage: - inputTokens: 18 - outputTokens: 46 - totalTokens: 64 - metrics: - latencyMs: 2083 ---- -{ - "response": { - "id": "[id]", - "type": "message", - "role": "assistant", - "model": "input-model", - "stop_reason": "end_turn", - "stop_sequence": null, - "usage": { - "input_tokens": 18, - "output_tokens": 46 - }, - "content": [ - { - "type": "thinking", - "thinking": "Two plus two is four.", - "signature": "SIG-xyz==" - }, - { - "text": "The answer is 4.", - "type": "text" - } - ] - }, - "parsed": { - "input_tokens": 18, - "output_tokens": 46, - "total_tokens": 64, - "provider_model": "input-model" - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/bedrock/reasoning.bedrock-responses-streaming.snap b/crates/persisting-gateway/tests/fixtures/response/bedrock/reasoning.bedrock-responses-streaming.snap deleted file mode 100644 index ac3487fab..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/bedrock/reasoning.bedrock-responses-streaming.snap +++ /dev/null @@ -1,32 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/bedrock/reasoning.bin ---- -event: event -data: {"type":"response.created","sequence_number":1,"response":{"created_at":123,"id":"","model":"input-model","object":"response","output":[],"status":"in_progress"}} - -event: event -data: {"type":"response.output_item.added","sequence_number":2,"output_index":0,"item":{"type":"message","content":[],"id":"","role":"assistant","status":"in_progress"}} - -event: event -data: {"type":"response.output_text.delta","sequence_number":3,"item_id":"","output_index":0,"content_index":0,"delta":"Two plus two is four."} - -event: event -data: {"type":"response.output_text.delta","sequence_number":4,"item_id":"","output_index":0,"content_index":0,"delta":"The answer is 4."} - -event: event -data: {"type":"response.output_item.done","sequence_number":5,"output_index":0,"item":{"type":"message","content":[],"id":"","role":"assistant","status":"completed"}} - -event: event -data: {"type":"response.completed","sequence_number":6,"response":{"created_at":123,"id":"","model":"input-model","object":"response","output":[],"status":"completed","usage":{"input_tokens":18,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":null},"output_tokens":46,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":64}}} - - - -{ - "input_tokens": 18, - "output_tokens": 46, - "total_tokens": 64, - "completion": [ - "The answer is 4." - ] -} diff --git a/crates/persisting-gateway/tests/fixtures/response/bedrock/reasoning.bedrock-responses.snap b/crates/persisting-gateway/tests/fixtures/response/bedrock/reasoning.bedrock-responses.snap deleted file mode 100644 index 5baa91c52..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/bedrock/reasoning.bedrock-responses.snap +++ /dev/null @@ -1,71 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/bedrock/reasoning.json -info: - output: - message: - role: assistant - content: - - reasoningContent: - reasoningText: - text: Two plus two is four. - signature: SIG-xyz== - - text: The answer is 4. - stopReason: end_turn - usage: - inputTokens: 18 - outputTokens: 46 - totalTokens: 64 - metrics: - latencyMs: 2083 ---- -{ - "response": { - "id": "[id]", - "status": "completed", - "output": [ - { - "type": "message", - "content": [ - { - "type": "output_text", - "annotations": [], - "logprobs": null, - "text": "Two plus two is four." - }, - { - "type": "output_text", - "annotations": [], - "logprobs": null, - "text": "The answer is 4." - } - ], - "id": "[id]", - "role": "assistant", - "status": "completed" - } - ], - "model": "input-model", - "usage": { - "input_tokens": 18, - "output_tokens": 46, - "input_tokens_details": { - "cached_tokens": 0 - }, - "output_tokens_details": { - "reasoning_tokens": 0 - }, - "total_tokens": 64 - }, - "created_at": "[date]", - "object": "response" - }, - "parsed": { - "input_tokens": 18, - "output_tokens": 46, - "total_tokens": 64, - "reasoning_tokens": 0, - "cached_input_tokens": 0, - "provider_model": "input-model" - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/bedrock/reasoning.bin b/crates/persisting-gateway/tests/fixtures/response/bedrock/reasoning.bin deleted file mode 100644 index 0ca138b65..000000000 Binary files a/crates/persisting-gateway/tests/fixtures/response/bedrock/reasoning.bin and /dev/null differ diff --git a/crates/persisting-gateway/tests/fixtures/response/bedrock/reasoning.json b/crates/persisting-gateway/tests/fixtures/response/bedrock/reasoning.json deleted file mode 100644 index a0e62fd7d..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/bedrock/reasoning.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "output": { - "message": { - "role": "assistant", - "content": [ - { - "reasoningContent": { - "reasoningText": { - "text": "Two plus two is four.", - "signature": "SIG-xyz==" - } - } - }, - { - "text": "The answer is 4." - } - ] - } - }, - "stopReason": "end_turn", - "usage": { - "inputTokens": 18, - "outputTokens": 46, - "totalTokens": 64 - }, - "metrics": { - "latencyMs": 2083 - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/bedrock/reasoning_unsigned.bedrock-completions.snap b/crates/persisting-gateway/tests/fixtures/response/bedrock/reasoning_unsigned.bedrock-completions.snap deleted file mode 100644 index 55dbaef27..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/bedrock/reasoning_unsigned.bedrock-completions.snap +++ /dev/null @@ -1,50 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/bedrock/reasoning_unsigned.json -info: - output: - message: - role: assistant - content: - - reasoningContent: - reasoningText: - text: Two plus two is four. - - text: The answer is 4. - stopReason: end_turn - usage: - inputTokens: 18 - outputTokens: 46 - totalTokens: 64 - metrics: - latencyMs: 2083 ---- -{ - "response": { - "model": "input-model", - "usage": { - "prompt_tokens": 18, - "completion_tokens": 46, - "total_tokens": 64 - }, - "choices": [ - { - "message": { - "content": "The answer is 4.", - "role": "assistant", - "reasoning_content": "Two plus two is four." - }, - "index": 0, - "finish_reason": "stop" - } - ], - "id": "[id]", - "created": "[date]", - "object": "chat.completion" - }, - "parsed": { - "input_tokens": 18, - "output_tokens": 46, - "total_tokens": 64, - "provider_model": "input-model" - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/bedrock/reasoning_unsigned.bedrock-messages.snap b/crates/persisting-gateway/tests/fixtures/response/bedrock/reasoning_unsigned.bedrock-messages.snap deleted file mode 100644 index 06f55b3f4..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/bedrock/reasoning_unsigned.bedrock-messages.snap +++ /dev/null @@ -1,51 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/bedrock/reasoning_unsigned.json -info: - output: - message: - role: assistant - content: - - reasoningContent: - reasoningText: - text: Two plus two is four. - - text: The answer is 4. - stopReason: end_turn - usage: - inputTokens: 18 - outputTokens: 46 - totalTokens: 64 - metrics: - latencyMs: 2083 ---- -{ - "response": { - "id": "[id]", - "type": "message", - "role": "assistant", - "model": "input-model", - "stop_reason": "end_turn", - "stop_sequence": null, - "usage": { - "input_tokens": 18, - "output_tokens": 46 - }, - "content": [ - { - "type": "thinking", - "thinking": "Two plus two is four.", - "signature": "" - }, - { - "text": "The answer is 4.", - "type": "text" - } - ] - }, - "parsed": { - "input_tokens": 18, - "output_tokens": 46, - "total_tokens": 64, - "provider_model": "input-model" - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/bedrock/reasoning_unsigned.bedrock-responses.snap b/crates/persisting-gateway/tests/fixtures/response/bedrock/reasoning_unsigned.bedrock-responses.snap deleted file mode 100644 index 31ef4cbb4..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/bedrock/reasoning_unsigned.bedrock-responses.snap +++ /dev/null @@ -1,70 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/bedrock/reasoning_unsigned.json -info: - output: - message: - role: assistant - content: - - reasoningContent: - reasoningText: - text: Two plus two is four. - - text: The answer is 4. - stopReason: end_turn - usage: - inputTokens: 18 - outputTokens: 46 - totalTokens: 64 - metrics: - latencyMs: 2083 ---- -{ - "response": { - "id": "[id]", - "status": "completed", - "output": [ - { - "type": "message", - "content": [ - { - "type": "output_text", - "annotations": [], - "logprobs": null, - "text": "Two plus two is four." - }, - { - "type": "output_text", - "annotations": [], - "logprobs": null, - "text": "The answer is 4." - } - ], - "id": "[id]", - "role": "assistant", - "status": "completed" - } - ], - "model": "input-model", - "usage": { - "input_tokens": 18, - "output_tokens": 46, - "input_tokens_details": { - "cached_tokens": 0 - }, - "output_tokens_details": { - "reasoning_tokens": 0 - }, - "total_tokens": 64 - }, - "created_at": "[date]", - "object": "response" - }, - "parsed": { - "input_tokens": 18, - "output_tokens": 46, - "total_tokens": 64, - "reasoning_tokens": 0, - "cached_input_tokens": 0, - "provider_model": "input-model" - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/bedrock/reasoning_unsigned.json b/crates/persisting-gateway/tests/fixtures/response/bedrock/reasoning_unsigned.json deleted file mode 100644 index b9fc3c716..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/bedrock/reasoning_unsigned.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "output": { - "message": { - "role": "assistant", - "content": [ - { - "reasoningContent": { - "reasoningText": { - "text": "Two plus two is four." - } - } - }, - { - "text": "The answer is 4." - } - ] - } - }, - "stopReason": "end_turn", - "usage": { - "inputTokens": 18, - "outputTokens": 46, - "totalTokens": 64 - }, - "metrics": { - "latencyMs": 2083 - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/bedrock/rerank.bedrock.snap b/crates/persisting-gateway/tests/fixtures/response/bedrock/rerank.bedrock.snap deleted file mode 100644 index 3d5260a45..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/bedrock/rerank.bedrock.snap +++ /dev/null @@ -1,25 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/bedrock/rerank.json -info: - results: - - index: 2 - relevanceScore: 0.9 - - index: 0 - relevanceScore: 0.1 ---- -{ - "response": { - "results": [ - { - "index": 2, - "relevance_score": 0.9 - }, - { - "index": 0, - "relevance_score": 0.1 - } - ] - }, - "parsed": {} -} diff --git a/crates/persisting-gateway/tests/fixtures/response/bedrock/rerank.json b/crates/persisting-gateway/tests/fixtures/response/bedrock/rerank.json deleted file mode 100644 index 4947eaa7d..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/bedrock/rerank.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "results": [ - { "index": 2, "relevanceScore": 0.9 }, - { "index": 0, "relevanceScore": 0.1 } - ] -} diff --git a/crates/persisting-gateway/tests/fixtures/response/bedrock/tool.bedrock-completions-streaming.snap b/crates/persisting-gateway/tests/fixtures/response/bedrock/tool.bedrock-completions-streaming.snap deleted file mode 100644 index b1bc58df0..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/bedrock/tool.bedrock-completions-streaming.snap +++ /dev/null @@ -1,58 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/bedrock/tool.bin ---- -data: {"id":"request_id","choices":[{"index":0,"delta":{"content":null,"role":"assistant"}}],"created":123,"model":"input-model","service_tier":null,"system_fingerprint":null,"object":"chat.completion.chunk","usage":null} - -data: {"id":"request_id","choices":[{"index":0,"delta":{"content":"I'll call both tools."}}],"created":123,"model":"input-model","service_tier":null,"system_fingerprint":null,"object":"chat.completion.chunk","usage":null} - -data: {"id":"request_id","choices":[{"index":0,"delta":{"content":null,"tool_calls":[{"index":0,"id":"tooluse_kZJMlvQmRJ6eAyJE5GIl7Q","type":"function","function":{"name":"top_song"}}]}}],"created":123,"model":"input-model","service_tier":null,"system_fingerprint":null,"object":"chat.completion.chunk","usage":null} - -data: {"id":"request_id","choices":[{"index":0,"delta":{"content":null,"tool_calls":[{"index":0,"id":null,"type":null,"function":{"arguments":"{\"sign\":"}}]}}],"created":123,"model":"input-model","service_tier":null,"system_fingerprint":null,"object":"chat.completion.chunk","usage":null} - -data: {"id":"request_id","choices":[{"index":0,"delta":{"content":null,"tool_calls":[{"index":0,"id":null,"type":null,"function":{"arguments":"\"WZPZ\"}"}}]}}],"created":123,"model":"input-model","service_tier":null,"system_fingerprint":null,"object":"chat.completion.chunk","usage":null} - -data: {"id":"request_id","choices":[{"index":0,"delta":{"content":null,"tool_calls":[{"index":1,"id":"tooluse_kZJMlvQmRJ6eAyxxx","type":"function","function":{"name":"hello"}}]}}],"created":123,"model":"input-model","service_tier":null,"system_fingerprint":null,"object":"chat.completion.chunk","usage":null} - -data: {"id":"request_id","choices":[{"index":0,"delta":{"content":null,"tool_calls":[{"index":1,"id":null,"type":null,"function":{"arguments":"{\"sign\":\"world\"}"}}]}}],"created":123,"model":"input-model","service_tier":null,"system_fingerprint":null,"object":"chat.completion.chunk","usage":null} - -data: {"id":"request_id","choices":[{"index":0,"delta":{"content":null},"finish_reason":"tool_calls"}],"created":123,"model":"input-model","service_tier":null,"system_fingerprint":null,"object":"chat.completion.chunk","usage":null} - -data: {"id":"request_id","choices":[],"created":123,"model":"input-model","service_tier":null,"system_fingerprint":null,"object":"chat.completion.chunk","usage":{"prompt_tokens":150,"completion_tokens":45,"total_tokens":195}} - -data: [DONE] - - - -{ - "input_tokens": 150, - "output_tokens": 45, - "total_tokens": 195, - "completion": [ - "I'll call both tools." - ], - "output_messages": [ - { - "role": "assistant", - "content": [ - { - "type": "tool_call", - "id": "tooluse_kZJMlvQmRJ6eAyJE5GIl7Q", - "name": "top_song", - "arguments": { - "sign": "WZPZ" - } - }, - { - "type": "tool_call", - "id": "tooluse_kZJMlvQmRJ6eAyxxx", - "name": "hello", - "arguments": { - "sign": "world" - } - } - ], - "finish_reason": "tool_calls" - } - ] -} diff --git a/crates/persisting-gateway/tests/fixtures/response/bedrock/tool.bedrock-completions.snap b/crates/persisting-gateway/tests/fixtures/response/bedrock/tool.bedrock-completions.snap deleted file mode 100644 index 9d18f4b66..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/bedrock/tool.bedrock-completions.snap +++ /dev/null @@ -1,90 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/bedrock/tool.json -info: - output: - message: - role: assistant - content: - - toolUse: - toolUseId: tooluse_kZJMlvQmRJ6eAyJE5GIl7Q - name: top_song - input: - sign: WZPZ - - toolUse: - toolUseId: tooluse_kZJMlvQmRJ6eAyxxx - name: hello - input: - sign: world - stopReason: tool_use ---- -{ - "response": { - "model": "input-model", - "usage": { - "prompt_tokens": 0, - "completion_tokens": 0, - "total_tokens": 0 - }, - "choices": [ - { - "message": { - "role": "assistant", - "tool_calls": [ - { - "type": "function", - "id": "tooluse_kZJMlvQmRJ6eAyJE5GIl7Q", - "function": { - "name": "top_song", - "arguments": "{\"sign\":\"WZPZ\"}" - } - }, - { - "type": "function", - "id": "tooluse_kZJMlvQmRJ6eAyxxx", - "function": { - "name": "hello", - "arguments": "{\"sign\":\"world\"}" - } - } - ] - }, - "index": 0, - "finish_reason": "tool_calls" - } - ], - "id": "[id]", - "created": "[date]", - "object": "chat.completion" - }, - "parsed": { - "input_tokens": 0, - "output_tokens": 0, - "total_tokens": 0, - "provider_model": "input-model", - "output_messages": [ - { - "role": "assistant", - "content": [ - { - "type": "tool_call", - "id": "tooluse_kZJMlvQmRJ6eAyJE5GIl7Q", - "name": "top_song", - "arguments": { - "sign": "WZPZ" - } - }, - { - "type": "tool_call", - "id": "tooluse_kZJMlvQmRJ6eAyxxx", - "name": "hello", - "arguments": { - "sign": "world" - } - } - ], - "finish_reason": "tool_calls" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/bedrock/tool.bedrock-messages-streaming.snap b/crates/persisting-gateway/tests/fixtures/response/bedrock/tool.bedrock-messages-streaming.snap deleted file mode 100644 index 4fa342a5d..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/bedrock/tool.bedrock-messages-streaming.snap +++ /dev/null @@ -1,77 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/bedrock/tool.bin ---- -event: message_start -data: {"type":"message_start","message":{"id":"","type":"message","role":"assistant","content":[],"model":"input-model","stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":0,"output_tokens":0}}} - -event: content_block_start -data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"I'll call both tools."}} - -event: content_block_stop -data: {"type":"content_block_stop","index":0} - -event: content_block_start -data: {"type":"content_block_start","index":1,"content_block":{"type":"tool_use","id":"tooluse_kZJMlvQmRJ6eAyJE5GIl7Q","name":"top_song","input":{}}} - -event: content_block_delta -data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"{\"sign\":"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"\"WZPZ\"}"}} - -event: content_block_stop -data: {"type":"content_block_stop","index":1} - -event: content_block_start -data: {"type":"content_block_start","index":2,"content_block":{"type":"tool_use","id":"tooluse_kZJMlvQmRJ6eAyxxx","name":"hello","input":{}}} - -event: content_block_delta -data: {"type":"content_block_delta","index":2,"delta":{"type":"input_json_delta","partial_json":"{\"sign\":\"world\"}"}} - -event: content_block_stop -data: {"type":"content_block_stop","index":2} - -event: message_delta -data: {"type":"message_delta","delta":{"stop_reason":"tool_use","stop_sequence":null},"usage":{"input_tokens":150,"output_tokens":45}} - -event: message_stop -data: {"type":"message_stop"} - - - -{ - "input_tokens": 150, - "output_tokens": 45, - "total_tokens": 195, - "completion": [ - "I'll call both tools." - ], - "output_messages": [ - { - "role": "assistant", - "content": [ - { - "type": "tool_call", - "id": "tooluse_kZJMlvQmRJ6eAyJE5GIl7Q", - "name": "top_song", - "arguments": { - "sign": "WZPZ" - } - }, - { - "type": "tool_call", - "id": "tooluse_kZJMlvQmRJ6eAyxxx", - "name": "hello", - "arguments": { - "sign": "world" - } - } - ], - "finish_reason": "tool_use" - } - ] -} diff --git a/crates/persisting-gateway/tests/fixtures/response/bedrock/tool.bedrock-messages.snap b/crates/persisting-gateway/tests/fixtures/response/bedrock/tool.bedrock-messages.snap deleted file mode 100644 index fcf8490da..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/bedrock/tool.bedrock-messages.snap +++ /dev/null @@ -1,82 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/bedrock/tool.json -info: - output: - message: - role: assistant - content: - - toolUse: - toolUseId: tooluse_kZJMlvQmRJ6eAyJE5GIl7Q - name: top_song - input: - sign: WZPZ - - toolUse: - toolUseId: tooluse_kZJMlvQmRJ6eAyxxx - name: hello - input: - sign: world - stopReason: tool_use ---- -{ - "response": { - "id": "[id]", - "type": "message", - "role": "assistant", - "model": "input-model", - "stop_reason": "tool_use", - "stop_sequence": null, - "usage": { - "input_tokens": 0, - "output_tokens": 0 - }, - "content": [ - { - "type": "tool_use", - "id": "tooluse_kZJMlvQmRJ6eAyJE5GIl7Q", - "name": "top_song", - "input": { - "sign": "WZPZ" - } - }, - { - "type": "tool_use", - "id": "tooluse_kZJMlvQmRJ6eAyxxx", - "name": "hello", - "input": { - "sign": "world" - } - } - ] - }, - "parsed": { - "input_tokens": 0, - "output_tokens": 0, - "total_tokens": 0, - "provider_model": "input-model", - "output_messages": [ - { - "role": "assistant", - "content": [ - { - "type": "tool_call", - "id": "tooluse_kZJMlvQmRJ6eAyJE5GIl7Q", - "name": "top_song", - "arguments": { - "sign": "WZPZ" - } - }, - { - "type": "tool_call", - "id": "tooluse_kZJMlvQmRJ6eAyxxx", - "name": "hello", - "arguments": { - "sign": "world" - } - } - ], - "finish_reason": "tool_use" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/bedrock/tool.bedrock-responses-streaming.snap b/crates/persisting-gateway/tests/fixtures/response/bedrock/tool.bedrock-responses-streaming.snap deleted file mode 100644 index a7ce8bc12..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/bedrock/tool.bedrock-responses-streaming.snap +++ /dev/null @@ -1,80 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/bedrock/tool.bin ---- -event: event -data: {"type":"response.created","sequence_number":1,"response":{"created_at":123,"id":"","model":"input-model","object":"response","output":[],"status":"in_progress"}} - -event: event -data: {"type":"response.output_item.added","sequence_number":2,"output_index":0,"item":{"type":"message","content":[],"id":"","role":"assistant","status":"in_progress"}} - -event: event -data: {"type":"response.output_text.delta","sequence_number":3,"item_id":"","output_index":0,"content_index":0,"delta":"I'll call both tools."} - -event: event -data: {"type":"response.output_item.added","sequence_number":4,"output_index":1,"item":{"type":"function_call","arguments":"","call_id":"call_xxx","name":"top_song","id":"","status":"in_progress"}} - -event: event -data: {"type":"response.function_call_arguments.delta","sequence_number":5,"item_id":"","output_index":1,"delta":"{\"sign\":"} - -event: event -data: {"type":"response.function_call_arguments.delta","sequence_number":6,"item_id":"","output_index":1,"delta":"\"WZPZ\"}"} - -event: event -data: {"type":"response.function_call_arguments.done","name":"top_song","sequence_number":7,"item_id":"","output_index":1,"arguments":"{\"sign\":\"WZPZ\"}"} - -event: event -data: {"type":"response.output_item.done","sequence_number":8,"output_index":1,"item":{"type":"function_call","arguments":"{\"sign\":\"WZPZ\"}","call_id":"call_xxx","name":"top_song","id":"","status":"completed"}} - -event: event -data: {"type":"response.output_item.added","sequence_number":9,"output_index":2,"item":{"type":"function_call","arguments":"","call_id":"call_xxx","name":"hello","id":"","status":"in_progress"}} - -event: event -data: {"type":"response.function_call_arguments.delta","sequence_number":10,"item_id":"","output_index":2,"delta":"{\"sign\":\"world\"}"} - -event: event -data: {"type":"response.function_call_arguments.done","name":"hello","sequence_number":11,"item_id":"","output_index":2,"arguments":"{\"sign\":\"world\"}"} - -event: event -data: {"type":"response.output_item.done","sequence_number":12,"output_index":2,"item":{"type":"function_call","arguments":"{\"sign\":\"world\"}","call_id":"call_xxx","name":"hello","id":"","status":"completed"}} - -event: event -data: {"type":"response.output_item.done","sequence_number":13,"output_index":0,"item":{"type":"message","content":[],"id":"","role":"assistant","status":"completed"}} - -event: event -data: {"type":"response.completed","sequence_number":14,"response":{"created_at":123,"id":"","model":"input-model","object":"response","output":[],"status":"completed","usage":{"input_tokens":150,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":null},"output_tokens":45,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":195}}} - - - -{ - "input_tokens": 150, - "output_tokens": 45, - "total_tokens": 195, - "completion": [ - "I'll call both tools." - ], - "output_messages": [ - { - "role": "assistant", - "content": [ - { - "type": "tool_call", - "id": "tooluse_kZJMlvQmRJ6eAyJE5GIl7Q", - "name": "top_song", - "arguments": { - "sign": "WZPZ" - } - }, - { - "type": "tool_call", - "id": "tooluse_kZJMlvQmRJ6eAyxxx", - "name": "hello", - "arguments": { - "sign": "world" - } - } - ], - "finish_reason": "completed" - } - ] -} diff --git a/crates/persisting-gateway/tests/fixtures/response/bedrock/tool.bedrock-responses.snap b/crates/persisting-gateway/tests/fixtures/response/bedrock/tool.bedrock-responses.snap deleted file mode 100644 index a18636a60..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/bedrock/tool.bedrock-responses.snap +++ /dev/null @@ -1,74 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/bedrock/tool.json -info: - output: - message: - role: assistant - content: - - toolUse: - toolUseId: tooluse_kZJMlvQmRJ6eAyJE5GIl7Q - name: top_song - input: - sign: WZPZ - - toolUse: - toolUseId: tooluse_kZJMlvQmRJ6eAyxxx - name: hello - input: - sign: world - stopReason: tool_use ---- -{ - "response": { - "id": "[id]", - "status": "completed", - "output": [ - { - "type": "function_call", - "arguments": "{\"sign\":\"WZPZ\"}", - "call_id": "tooluse_kZJMlvQmRJ6eAyJE5GIl7Q", - "name": "top_song", - "id": "[id]", - "status": "completed" - }, - { - "type": "function_call", - "arguments": "{\"sign\":\"world\"}", - "call_id": "tooluse_kZJMlvQmRJ6eAyxxx", - "name": "hello", - "id": "[id]", - "status": "completed" - } - ], - "model": "input-model", - "created_at": "[date]", - "object": "response" - }, - "parsed": { - "provider_model": "input-model", - "output_messages": [ - { - "role": "assistant", - "content": [ - { - "type": "tool_call", - "id": "tooluse_kZJMlvQmRJ6eAyJE5GIl7Q", - "name": "top_song", - "arguments": { - "sign": "WZPZ" - } - }, - { - "type": "tool_call", - "id": "tooluse_kZJMlvQmRJ6eAyxxx", - "name": "hello", - "arguments": { - "sign": "world" - } - } - ], - "finish_reason": "completed" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/bedrock/tool.bin b/crates/persisting-gateway/tests/fixtures/response/bedrock/tool.bin deleted file mode 100644 index 325a3d7ac..000000000 Binary files a/crates/persisting-gateway/tests/fixtures/response/bedrock/tool.bin and /dev/null differ diff --git a/crates/persisting-gateway/tests/fixtures/response/bedrock/tool.json b/crates/persisting-gateway/tests/fixtures/response/bedrock/tool.json deleted file mode 100644 index 1fe2314e4..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/bedrock/tool.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "output": { - "message": { - "role": "assistant", - "content": [ - { - "toolUse": { - "toolUseId": "tooluse_kZJMlvQmRJ6eAyJE5GIl7Q", - "name": "top_song", - "input": { - "sign": "WZPZ" - } - } - }, - { - "toolUse": { - "toolUseId": "tooluse_kZJMlvQmRJ6eAyxxx", - "name": "hello", - "input": { - "sign": "world" - } - } - } - ] - } - }, - "stopReason": "tool_use" -} \ No newline at end of file diff --git a/crates/persisting-gateway/tests/fixtures/response/cohere/rerank.cohere.snap b/crates/persisting-gateway/tests/fixtures/response/cohere/rerank.cohere.snap deleted file mode 100644 index 60288259e..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/cohere/rerank.cohere.snap +++ /dev/null @@ -1,45 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/cohere/rerank.json -info: - id: abc123 - results: - - index: 2 - relevance_score: 0.91 - - index: 0 - relevance_score: 0.42 - meta: - billed_units: - search_units: 1 - tokens: - input_tokens: 214 - output_tokens: 2 ---- -{ - "response": { - "id": "[id]", - "results": [ - { - "index": 2, - "relevance_score": 0.91 - }, - { - "index": 0, - "relevance_score": 0.42 - } - ], - "meta": { - "billed_units": { - "search_units": 1 - }, - "tokens": { - "input_tokens": 214.0, - "output_tokens": 2.0 - } - } - }, - "parsed": { - "input_tokens": 214, - "total_tokens": 214 - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/cohere/rerank.json b/crates/persisting-gateway/tests/fixtures/response/cohere/rerank.json deleted file mode 100644 index bda1d3ffa..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/cohere/rerank.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "id": "abc123", - "results": [ - { "index": 2, "relevance_score": 0.91 }, - { "index": 0, "relevance_score": 0.42 } - ], - "meta": { - "billed_units": { "search_units": 1 }, - "tokens": { "input_tokens": 214.0, "output_tokens": 2.0 } - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/completions/audio.completions-completions.snap b/crates/persisting-gateway/tests/fixtures/response/completions/audio.completions-completions.snap deleted file mode 100644 index 56e49b315..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/completions/audio.completions-completions.snap +++ /dev/null @@ -1,92 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/completions/audio.json -info: - id: chatcmpl-audioTokens123 - object: chat.completion - created: 1755008546 - model: gpt-4o-audio-preview - choices: - - index: 0 - message: - role: assistant - content: Here is the transcription and spoken answer. - audio: - id: audio_junk_123 - data: ZmFrZS1hdWRpby1ieXRlcw== - expires_at: 1755009546 - transcript: Here is the transcription and spoken answer. - refusal: ~ - annotations: [] - logprobs: ~ - finish_reason: stop - usage: - prompt_tokens: 320 - completion_tokens: 180 - total_tokens: 500 - prompt_tokens_details: - cached_tokens: 12 - audio_tokens: 240 - completion_tokens_details: - reasoning_tokens: 15 - audio_tokens: 90 - accepted_prediction_tokens: 0 - rejected_prediction_tokens: 0 - service_tier: default - system_fingerprint: ~ ---- -{ - "response": { - "model": "gpt-4o-audio-preview", - "service_tier": "default", - "usage": { - "prompt_tokens": 320, - "completion_tokens": 180, - "total_tokens": 500, - "completion_tokens_details": { - "reasoning_tokens": 15, - "audio_tokens": 90, - "accepted_prediction_tokens": 0, - "rejected_prediction_tokens": 0 - }, - "prompt_tokens_details": { - "cached_tokens": 12, - "audio_tokens": 240 - } - }, - "choices": [ - { - "message": { - "content": "Here is the transcription and spoken answer.", - "role": "assistant", - "audio": { - "id": "audio_junk_123", - "data": "ZmFrZS1hdWRpby1ieXRlcw==", - "expires_at": 1755009546, - "transcript": "Here is the transcription and spoken answer." - }, - "refusal": null, - "annotations": [] - }, - "index": 0, - "logprobs": null, - "finish_reason": "stop" - } - ], - "id": "[id]", - "object": "chat.completion", - "created": "[date]", - "system_fingerprint": null - }, - "parsed": { - "input_tokens": 320, - "input_audio_tokens": 240, - "output_tokens": 180, - "output_audio_tokens": 90, - "total_tokens": 500, - "reasoning_tokens": 15, - "cached_input_tokens": 12, - "service_tier": "default", - "provider_model": "gpt-4o-audio-preview" - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/completions/audio.completions-detect.snap b/crates/persisting-gateway/tests/fixtures/response/completions/audio.completions-detect.snap deleted file mode 100644 index f2cd669bf..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/completions/audio.completions-detect.snap +++ /dev/null @@ -1,92 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/completions/audio.json -info: - id: chatcmpl-audioTokens123 - object: chat.completion - created: 1755008546 - model: gpt-4o-audio-preview - choices: - - index: 0 - message: - role: assistant - content: Here is the transcription and spoken answer. - audio: - id: audio_junk_123 - data: ZmFrZS1hdWRpby1ieXRlcw== - expires_at: 1755009546 - transcript: Here is the transcription and spoken answer. - refusal: ~ - annotations: [] - logprobs: ~ - finish_reason: stop - usage: - prompt_tokens: 320 - completion_tokens: 180 - total_tokens: 500 - prompt_tokens_details: - cached_tokens: 12 - audio_tokens: 240 - completion_tokens_details: - reasoning_tokens: 15 - audio_tokens: 90 - accepted_prediction_tokens: 0 - rejected_prediction_tokens: 0 - service_tier: default - system_fingerprint: ~ ---- -{ - "response": { - "id": "[id]", - "object": "chat.completion", - "created": "[date]", - "model": "gpt-4o-audio-preview", - "choices": [ - { - "index": 0, - "message": { - "role": "assistant", - "content": "Here is the transcription and spoken answer.", - "audio": { - "id": "audio_junk_123", - "data": "ZmFrZS1hdWRpby1ieXRlcw==", - "expires_at": 1755009546, - "transcript": "Here is the transcription and spoken answer." - }, - "refusal": null, - "annotations": [] - }, - "logprobs": null, - "finish_reason": "stop" - } - ], - "usage": { - "prompt_tokens": 320, - "completion_tokens": 180, - "total_tokens": 500, - "prompt_tokens_details": { - "cached_tokens": 12, - "audio_tokens": 240 - }, - "completion_tokens_details": { - "reasoning_tokens": 15, - "audio_tokens": 90, - "accepted_prediction_tokens": 0, - "rejected_prediction_tokens": 0 - } - }, - "service_tier": "default", - "system_fingerprint": null - }, - "parsed": { - "input_tokens": 320, - "input_audio_tokens": 240, - "output_tokens": 180, - "output_audio_tokens": 90, - "total_tokens": 500, - "reasoning_tokens": 15, - "cached_input_tokens": 12, - "service_tier": "default", - "provider_model": "gpt-4o-audio-preview" - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/completions/audio.completions-messages.snap b/crates/persisting-gateway/tests/fixtures/response/completions/audio.completions-messages.snap deleted file mode 100644 index fb645302c..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/completions/audio.completions-messages.snap +++ /dev/null @@ -1,69 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/completions/audio.json -info: - id: chatcmpl-audioTokens123 - object: chat.completion - created: 1755008546 - model: gpt-4o-audio-preview - choices: - - index: 0 - message: - role: assistant - content: Here is the transcription and spoken answer. - audio: - id: audio_junk_123 - data: ZmFrZS1hdWRpby1ieXRlcw== - expires_at: 1755009546 - transcript: Here is the transcription and spoken answer. - refusal: ~ - annotations: [] - logprobs: ~ - finish_reason: stop - usage: - prompt_tokens: 320 - completion_tokens: 180 - total_tokens: 500 - prompt_tokens_details: - cached_tokens: 12 - audio_tokens: 240 - completion_tokens_details: - reasoning_tokens: 15 - audio_tokens: 90 - accepted_prediction_tokens: 0 - rejected_prediction_tokens: 0 - service_tier: default - system_fingerprint: ~ ---- -{ - "response": { - "id": "[id]", - "type": "message", - "role": "assistant", - "content": [ - { - "type": "text", - "text": "Here is the transcription and spoken answer." - } - ], - "model": "gpt-4o-audio-preview", - "stop_reason": "end_turn", - "stop_sequence": null, - "usage": { - "input_tokens": 308, - "output_tokens": 180, - "cache_read_input_tokens": 12, - "service_tier": "default" - } - }, - "parsed": { - "input_tokens": 308, - "input_audio_tokens": 240, - "output_tokens": 180, - "output_audio_tokens": 90, - "total_tokens": 488, - "cached_input_tokens": 12, - "service_tier": "default", - "provider_model": "gpt-4o-audio-preview" - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/completions/audio.completions-responses.snap b/crates/persisting-gateway/tests/fixtures/response/completions/audio.completions-responses.snap deleted file mode 100644 index a7f28c612..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/completions/audio.completions-responses.snap +++ /dev/null @@ -1,81 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/completions/audio.json -info: - id: chatcmpl-audioTokens123 - object: chat.completion - created: 1755008546 - model: gpt-4o-audio-preview - choices: - - index: 0 - message: - role: assistant - content: Here is the transcription and spoken answer. - audio: - id: audio_junk_123 - data: ZmFrZS1hdWRpby1ieXRlcw== - expires_at: 1755009546 - transcript: Here is the transcription and spoken answer. - refusal: ~ - annotations: [] - logprobs: ~ - finish_reason: stop - usage: - prompt_tokens: 320 - completion_tokens: 180 - total_tokens: 500 - prompt_tokens_details: - cached_tokens: 12 - audio_tokens: 240 - completion_tokens_details: - reasoning_tokens: 15 - audio_tokens: 90 - accepted_prediction_tokens: 0 - rejected_prediction_tokens: 0 - service_tier: default - system_fingerprint: ~ ---- -{ - "response": { - "id": "[id]", - "status": "completed", - "output": [ - { - "type": "message", - "content": [ - { - "type": "output_text", - "annotations": [], - "logprobs": null, - "text": "Here is the transcription and spoken answer." - } - ], - "id": "[id]", - "role": "assistant", - "status": "completed" - } - ], - "model": "input-model", - "usage": { - "input_tokens": 320, - "output_tokens": 180, - "input_tokens_details": { - "cached_tokens": 12 - }, - "output_tokens_details": { - "reasoning_tokens": 15 - }, - "total_tokens": 500 - }, - "created_at": "[date]", - "object": "response" - }, - "parsed": { - "input_tokens": 320, - "output_tokens": 180, - "total_tokens": 500, - "reasoning_tokens": 15, - "cached_input_tokens": 12, - "provider_model": "input-model" - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/completions/audio.json b/crates/persisting-gateway/tests/fixtures/response/completions/audio.json deleted file mode 100644 index fea073b7b..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/completions/audio.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "id": "chatcmpl-audioTokens123", - "object": "chat.completion", - "created": 1755008546, - "model": "gpt-4o-audio-preview", - "choices": [ - { - "index": 0, - "message": { - "role": "assistant", - "content": "Here is the transcription and spoken answer.", - "audio": { - "id": "audio_junk_123", - "data": "ZmFrZS1hdWRpby1ieXRlcw==", - "expires_at": 1755009546, - "transcript": "Here is the transcription and spoken answer." - }, - "refusal": null, - "annotations": [] - }, - "logprobs": null, - "finish_reason": "stop" - } - ], - "usage": { - "prompt_tokens": 320, - "completion_tokens": 180, - "total_tokens": 500, - "prompt_tokens_details": { - "cached_tokens": 12, - "audio_tokens": 240 - }, - "completion_tokens_details": { - "reasoning_tokens": 15, - "audio_tokens": 90, - "accepted_prediction_tokens": 0, - "rejected_prediction_tokens": 0 - } - }, - "service_tier": "default", - "system_fingerprint": null -} diff --git a/crates/persisting-gateway/tests/fixtures/response/completions/basic.completions-completions.snap b/crates/persisting-gateway/tests/fixtures/response/completions/basic.completions-completions.snap deleted file mode 100644 index 21de0b7d7..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/completions/basic.completions-completions.snap +++ /dev/null @@ -1,81 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/completions/basic.json -info: - id: chatcmpl-C3kBOLGARFj2WJrm5MRyuTiK4u6eF - object: chat.completion - created: 1755008546 - model: gpt-3.5-turbo-0125 - choices: - - index: 0 - message: - role: assistant - content: "Sorry, I couldn't find the name of the LLM provider. Could you please provide more information or context?" - refusal: ~ - annotations: [] - logprobs: ~ - finish_reason: stop - usage: - prompt_tokens: 17 - completion_tokens: 23 - total_tokens: 40 - prompt_tokens_details: - cached_tokens: 0 - audio_tokens: 0 - completion_tokens_details: - reasoning_tokens: 0 - audio_tokens: 0 - accepted_prediction_tokens: 0 - rejected_prediction_tokens: 0 - service_tier: default - system_fingerprint: ~ ---- -{ - "response": { - "model": "gpt-3.5-turbo-0125", - "service_tier": "default", - "usage": { - "prompt_tokens": 17, - "completion_tokens": 23, - "total_tokens": 40, - "completion_tokens_details": { - "reasoning_tokens": 0, - "audio_tokens": 0, - "accepted_prediction_tokens": 0, - "rejected_prediction_tokens": 0 - }, - "prompt_tokens_details": { - "cached_tokens": 0, - "audio_tokens": 0 - } - }, - "choices": [ - { - "message": { - "content": "Sorry, I couldn't find the name of the LLM provider. Could you please provide more information or context?", - "role": "assistant", - "refusal": null, - "annotations": [] - }, - "index": 0, - "logprobs": null, - "finish_reason": "stop" - } - ], - "id": "[id]", - "object": "chat.completion", - "created": "[date]", - "system_fingerprint": null - }, - "parsed": { - "input_tokens": 17, - "input_audio_tokens": 0, - "output_tokens": 23, - "output_audio_tokens": 0, - "total_tokens": 40, - "reasoning_tokens": 0, - "cached_input_tokens": 0, - "service_tier": "default", - "provider_model": "gpt-3.5-turbo-0125" - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/completions/basic.completions-detect.snap b/crates/persisting-gateway/tests/fixtures/response/completions/basic.completions-detect.snap deleted file mode 100644 index a772efbab..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/completions/basic.completions-detect.snap +++ /dev/null @@ -1,81 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/completions/basic.json -info: - id: chatcmpl-C3kBOLGARFj2WJrm5MRyuTiK4u6eF - object: chat.completion - created: 1755008546 - model: gpt-3.5-turbo-0125 - choices: - - index: 0 - message: - role: assistant - content: "Sorry, I couldn't find the name of the LLM provider. Could you please provide more information or context?" - refusal: ~ - annotations: [] - logprobs: ~ - finish_reason: stop - usage: - prompt_tokens: 17 - completion_tokens: 23 - total_tokens: 40 - prompt_tokens_details: - cached_tokens: 0 - audio_tokens: 0 - completion_tokens_details: - reasoning_tokens: 0 - audio_tokens: 0 - accepted_prediction_tokens: 0 - rejected_prediction_tokens: 0 - service_tier: default - system_fingerprint: ~ ---- -{ - "response": { - "id": "[id]", - "object": "chat.completion", - "created": "[date]", - "model": "gpt-3.5-turbo-0125", - "choices": [ - { - "index": 0, - "message": { - "role": "assistant", - "content": "Sorry, I couldn't find the name of the LLM provider. Could you please provide more information or context?", - "refusal": null, - "annotations": [] - }, - "logprobs": null, - "finish_reason": "stop" - } - ], - "usage": { - "prompt_tokens": 17, - "completion_tokens": 23, - "total_tokens": 40, - "prompt_tokens_details": { - "cached_tokens": 0, - "audio_tokens": 0 - }, - "completion_tokens_details": { - "reasoning_tokens": 0, - "audio_tokens": 0, - "accepted_prediction_tokens": 0, - "rejected_prediction_tokens": 0 - } - }, - "service_tier": "default", - "system_fingerprint": null - }, - "parsed": { - "input_tokens": 17, - "input_audio_tokens": 0, - "output_tokens": 23, - "output_audio_tokens": 0, - "total_tokens": 40, - "reasoning_tokens": 0, - "cached_input_tokens": 0, - "service_tier": "default", - "provider_model": "gpt-3.5-turbo-0125" - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/completions/basic.completions-messages.snap b/crates/persisting-gateway/tests/fixtures/response/completions/basic.completions-messages.snap deleted file mode 100644 index f53c36ff3..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/completions/basic.completions-messages.snap +++ /dev/null @@ -1,64 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/completions/basic.json -info: - id: chatcmpl-C3kBOLGARFj2WJrm5MRyuTiK4u6eF - object: chat.completion - created: 1755008546 - model: gpt-3.5-turbo-0125 - choices: - - index: 0 - message: - role: assistant - content: "Sorry, I couldn't find the name of the LLM provider. Could you please provide more information or context?" - refusal: ~ - annotations: [] - logprobs: ~ - finish_reason: stop - usage: - prompt_tokens: 17 - completion_tokens: 23 - total_tokens: 40 - prompt_tokens_details: - cached_tokens: 0 - audio_tokens: 0 - completion_tokens_details: - reasoning_tokens: 0 - audio_tokens: 0 - accepted_prediction_tokens: 0 - rejected_prediction_tokens: 0 - service_tier: default - system_fingerprint: ~ ---- -{ - "response": { - "id": "[id]", - "type": "message", - "role": "assistant", - "content": [ - { - "type": "text", - "text": "Sorry, I couldn't find the name of the LLM provider. Could you please provide more information or context?" - } - ], - "model": "gpt-3.5-turbo-0125", - "stop_reason": "end_turn", - "stop_sequence": null, - "usage": { - "input_tokens": 17, - "output_tokens": 23, - "cache_read_input_tokens": 0, - "service_tier": "default" - } - }, - "parsed": { - "input_tokens": 17, - "input_audio_tokens": 0, - "output_tokens": 23, - "output_audio_tokens": 0, - "total_tokens": 40, - "cached_input_tokens": 0, - "service_tier": "default", - "provider_model": "gpt-3.5-turbo-0125" - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/completions/basic.completions-responses.snap b/crates/persisting-gateway/tests/fixtures/response/completions/basic.completions-responses.snap deleted file mode 100644 index 34eb5198a..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/completions/basic.completions-responses.snap +++ /dev/null @@ -1,76 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/completions/basic.json -info: - id: chatcmpl-C3kBOLGARFj2WJrm5MRyuTiK4u6eF - object: chat.completion - created: 1755008546 - model: gpt-3.5-turbo-0125 - choices: - - index: 0 - message: - role: assistant - content: "Sorry, I couldn't find the name of the LLM provider. Could you please provide more information or context?" - refusal: ~ - annotations: [] - logprobs: ~ - finish_reason: stop - usage: - prompt_tokens: 17 - completion_tokens: 23 - total_tokens: 40 - prompt_tokens_details: - cached_tokens: 0 - audio_tokens: 0 - completion_tokens_details: - reasoning_tokens: 0 - audio_tokens: 0 - accepted_prediction_tokens: 0 - rejected_prediction_tokens: 0 - service_tier: default - system_fingerprint: ~ ---- -{ - "response": { - "id": "[id]", - "status": "completed", - "output": [ - { - "type": "message", - "content": [ - { - "type": "output_text", - "annotations": [], - "logprobs": null, - "text": "Sorry, I couldn't find the name of the LLM provider. Could you please provide more information or context?" - } - ], - "id": "[id]", - "role": "assistant", - "status": "completed" - } - ], - "model": "input-model", - "usage": { - "input_tokens": 17, - "output_tokens": 23, - "input_tokens_details": { - "cached_tokens": 0 - }, - "output_tokens_details": { - "reasoning_tokens": 0 - }, - "total_tokens": 40 - }, - "created_at": "[date]", - "object": "response" - }, - "parsed": { - "input_tokens": 17, - "output_tokens": 23, - "total_tokens": 40, - "reasoning_tokens": 0, - "cached_input_tokens": 0, - "provider_model": "input-model" - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/completions/basic.json b/crates/persisting-gateway/tests/fixtures/response/completions/basic.json deleted file mode 100644 index 13300d1a8..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/completions/basic.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "id": "chatcmpl-C3kBOLGARFj2WJrm5MRyuTiK4u6eF", - "object": "chat.completion", - "created": 1755008546, - "model": "gpt-3.5-turbo-0125", - "choices": [ - { - "index": 0, - "message": { - "role": "assistant", - "content": "Sorry, I couldn't find the name of the LLM provider. Could you please provide more information or context?", - "refusal": null, - "annotations": [] - }, - "logprobs": null, - "finish_reason": "stop" - } - ], - "usage": { - "prompt_tokens": 17, - "completion_tokens": 23, - "total_tokens": 40, - "prompt_tokens_details": { - "cached_tokens": 0, - "audio_tokens": 0 - }, - "completion_tokens_details": { - "reasoning_tokens": 0, - "audio_tokens": 0, - "accepted_prediction_tokens": 0, - "rejected_prediction_tokens": 0 - } - }, - "service_tier": "default", - "system_fingerprint": null -} - diff --git a/crates/persisting-gateway/tests/fixtures/response/completions/cache_write.completions-completions.snap b/crates/persisting-gateway/tests/fixtures/response/completions/cache_write.completions-completions.snap deleted file mode 100644 index 5afd804a1..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/completions/cache_write.completions-completions.snap +++ /dev/null @@ -1,66 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/completions/cache_write.json -info: - id: chatcmpl-cache-write - object: chat.completion - created: 1784217600 - model: gpt-5.6 - choices: - - index: 0 - message: - role: assistant - content: cached response - finish_reason: stop - usage: - prompt_tokens: 1000 - completion_tokens: 10 - total_tokens: 1010 - prompt_tokens_details: - cached_tokens: 300 - cache_write_tokens: 200 - completion_tokens_details: - reasoning_tokens: 0 - service_tier: default ---- -{ - "response": { - "model": "gpt-5.6", - "service_tier": "default", - "usage": { - "prompt_tokens": 1000, - "completion_tokens": 10, - "total_tokens": 1010, - "completion_tokens_details": { - "reasoning_tokens": 0 - }, - "prompt_tokens_details": { - "cached_tokens": 300, - "cache_write_tokens": 200 - } - }, - "choices": [ - { - "message": { - "content": "cached response", - "role": "assistant" - }, - "index": 0, - "finish_reason": "stop" - } - ], - "id": "[id]", - "object": "chat.completion", - "created": "[date]" - }, - "parsed": { - "input_tokens": 1000, - "output_tokens": 10, - "total_tokens": 1010, - "reasoning_tokens": 0, - "cache_creation_input_tokens": 200, - "cached_input_tokens": 300, - "service_tier": "default", - "provider_model": "gpt-5.6" - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/completions/cache_write.completions-messages.snap b/crates/persisting-gateway/tests/fixtures/response/completions/cache_write.completions-messages.snap deleted file mode 100644 index 28c38293e..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/completions/cache_write.completions-messages.snap +++ /dev/null @@ -1,57 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/completions/cache_write.json -info: - id: chatcmpl-cache-write - object: chat.completion - created: 1784217600 - model: gpt-5.6 - choices: - - index: 0 - message: - role: assistant - content: cached response - finish_reason: stop - usage: - prompt_tokens: 1000 - completion_tokens: 10 - total_tokens: 1010 - prompt_tokens_details: - cached_tokens: 300 - cache_write_tokens: 200 - completion_tokens_details: - reasoning_tokens: 0 - service_tier: default ---- -{ - "response": { - "id": "[id]", - "type": "message", - "role": "assistant", - "content": [ - { - "type": "text", - "text": "cached response" - } - ], - "model": "gpt-5.6", - "stop_reason": "end_turn", - "stop_sequence": null, - "usage": { - "input_tokens": 500, - "output_tokens": 10, - "cache_creation_input_tokens": 200, - "cache_read_input_tokens": 300, - "service_tier": "default" - } - }, - "parsed": { - "input_tokens": 500, - "output_tokens": 10, - "total_tokens": 510, - "cache_creation_input_tokens": 200, - "cached_input_tokens": 300, - "service_tier": "default", - "provider_model": "gpt-5.6" - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/completions/cache_write.json b/crates/persisting-gateway/tests/fixtures/response/completions/cache_write.json deleted file mode 100644 index 7a8bc53da..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/completions/cache_write.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "id": "chatcmpl-cache-write", - "object": "chat.completion", - "created": 1784217600, - "model": "gpt-5.6", - "choices": [ - { - "index": 0, - "message": { - "role": "assistant", - "content": "cached response" - }, - "finish_reason": "stop" - } - ], - "usage": { - "prompt_tokens": 1000, - "completion_tokens": 10, - "total_tokens": 1010, - "prompt_tokens_details": { - "cached_tokens": 300, - "cache_write_tokens": 200 - }, - "completion_tokens_details": { - "reasoning_tokens": 0 - } - }, - "service_tier": "default" -} diff --git a/crates/persisting-gateway/tests/fixtures/response/completions/gemini_with_completion_tokens.completions-completions.snap b/crates/persisting-gateway/tests/fixtures/response/completions/gemini_with_completion_tokens.completions-completions.snap deleted file mode 100644 index 82230276d..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/completions/gemini_with_completion_tokens.completions-completions.snap +++ /dev/null @@ -1,48 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/completions/gemini_with_completion_tokens.json -info: - model: gemini-2.5-flash - usage: - prompt_tokens: 7 - completion_tokens: 12 - total_tokens: 402 - choices: - - message: - content: "Hello! I'm functioning perfectly and ready to assist.\n\n" - role: assistant - finish_reason: length - index: 0 - created: 1775595257 - id: 9W7VaabzK7KPjMcPnYGiuAU - object: chat.completion ---- -{ - "response": { - "model": "gemini-2.5-flash", - "usage": { - "prompt_tokens": 7, - "completion_tokens": 12, - "total_tokens": 402 - }, - "choices": [ - { - "message": { - "content": "Hello! I'm functioning perfectly and ready to assist.\n\n", - "role": "assistant" - }, - "finish_reason": "length", - "index": 0 - } - ], - "created": "[date]", - "id": "[id]", - "object": "chat.completion" - }, - "parsed": { - "input_tokens": 7, - "output_tokens": 12, - "total_tokens": 402, - "provider_model": "gemini-2.5-flash" - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/completions/gemini_with_completion_tokens.completions-detect.snap b/crates/persisting-gateway/tests/fixtures/response/completions/gemini_with_completion_tokens.completions-detect.snap deleted file mode 100644 index 82230276d..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/completions/gemini_with_completion_tokens.completions-detect.snap +++ /dev/null @@ -1,48 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/completions/gemini_with_completion_tokens.json -info: - model: gemini-2.5-flash - usage: - prompt_tokens: 7 - completion_tokens: 12 - total_tokens: 402 - choices: - - message: - content: "Hello! I'm functioning perfectly and ready to assist.\n\n" - role: assistant - finish_reason: length - index: 0 - created: 1775595257 - id: 9W7VaabzK7KPjMcPnYGiuAU - object: chat.completion ---- -{ - "response": { - "model": "gemini-2.5-flash", - "usage": { - "prompt_tokens": 7, - "completion_tokens": 12, - "total_tokens": 402 - }, - "choices": [ - { - "message": { - "content": "Hello! I'm functioning perfectly and ready to assist.\n\n", - "role": "assistant" - }, - "finish_reason": "length", - "index": 0 - } - ], - "created": "[date]", - "id": "[id]", - "object": "chat.completion" - }, - "parsed": { - "input_tokens": 7, - "output_tokens": 12, - "total_tokens": 402, - "provider_model": "gemini-2.5-flash" - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/completions/gemini_with_completion_tokens.completions-messages.snap b/crates/persisting-gateway/tests/fixtures/response/completions/gemini_with_completion_tokens.completions-messages.snap deleted file mode 100644 index 997d02fa5..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/completions/gemini_with_completion_tokens.completions-messages.snap +++ /dev/null @@ -1,45 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/completions/gemini_with_completion_tokens.json -info: - model: gemini-2.5-flash - usage: - prompt_tokens: 7 - completion_tokens: 12 - total_tokens: 402 - choices: - - message: - content: "Hello! I'm functioning perfectly and ready to assist.\n\n" - role: assistant - finish_reason: length - index: 0 - created: 1775595257 - id: 9W7VaabzK7KPjMcPnYGiuAU - object: chat.completion ---- -{ - "response": { - "id": "[id]", - "type": "message", - "role": "assistant", - "content": [ - { - "type": "text", - "text": "Hello! I'm functioning perfectly and ready to assist.\n\n" - } - ], - "model": "gemini-2.5-flash", - "stop_reason": "max_tokens", - "stop_sequence": null, - "usage": { - "input_tokens": 7, - "output_tokens": 12 - } - }, - "parsed": { - "input_tokens": 7, - "output_tokens": 12, - "total_tokens": 19, - "provider_model": "gemini-2.5-flash" - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/completions/gemini_with_completion_tokens.completions-responses.snap b/crates/persisting-gateway/tests/fixtures/response/completions/gemini_with_completion_tokens.completions-responses.snap deleted file mode 100644 index 36d776930..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/completions/gemini_with_completion_tokens.completions-responses.snap +++ /dev/null @@ -1,66 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/completions/gemini_with_completion_tokens.json -info: - model: gemini-2.5-flash - usage: - prompt_tokens: 7 - completion_tokens: 12 - total_tokens: 402 - choices: - - message: - content: "Hello! I'm functioning perfectly and ready to assist.\n\n" - role: assistant - finish_reason: length - index: 0 - created: 1775595257 - id: 9W7VaabzK7KPjMcPnYGiuAU - object: chat.completion ---- -{ - "response": { - "id": "[id]", - "status": "incomplete", - "output": [ - { - "type": "message", - "content": [ - { - "type": "output_text", - "annotations": [], - "logprobs": null, - "text": "Hello! I'm functioning perfectly and ready to assist.\n\n" - } - ], - "id": "[id]", - "role": "assistant", - "status": "completed" - } - ], - "model": "input-model", - "usage": { - "input_tokens": 7, - "output_tokens": 12, - "input_tokens_details": { - "cached_tokens": 0 - }, - "output_tokens_details": { - "reasoning_tokens": 0 - }, - "total_tokens": 402 - }, - "created_at": "[date]", - "incomplete_details": { - "reason": "max_tokens" - }, - "object": "response" - }, - "parsed": { - "input_tokens": 7, - "output_tokens": 12, - "total_tokens": 402, - "reasoning_tokens": 0, - "cached_input_tokens": 0, - "provider_model": "input-model" - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/completions/gemini_with_completion_tokens.json b/crates/persisting-gateway/tests/fixtures/response/completions/gemini_with_completion_tokens.json deleted file mode 100644 index 56343f5f4..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/completions/gemini_with_completion_tokens.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "model": "gemini-2.5-flash", - "usage": { - "prompt_tokens": 7, - "completion_tokens": 12, - "total_tokens": 402 - }, - "choices": [ - { - "message": { - "content": "Hello! I'm functioning perfectly and ready to assist.\n\n", - "role": "assistant" - }, - "finish_reason": "length", - "index": 0 - } - ], - "created": 1775595257, - "id": "9W7VaabzK7KPjMcPnYGiuAU", - "object": "chat.completion" -} diff --git a/crates/persisting-gateway/tests/fixtures/response/completions/gemini_zero_completion_tokens.completions-completions.snap b/crates/persisting-gateway/tests/fixtures/response/completions/gemini_zero_completion_tokens.completions-completions.snap deleted file mode 100644 index 07084afd5..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/completions/gemini_zero_completion_tokens.completions-completions.snap +++ /dev/null @@ -1,46 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/completions/gemini_zero_completion_tokens.json -info: - model: gemini-2.5-flash - usage: - prompt_tokens: 7 - completion_tokens: 0 - total_tokens: 8 - choices: - - message: - role: assistant - finish_reason: length - index: 0 - created: 1775595226 - id: 2m7Vaca_HdfC1MkP1trPwAM - object: chat.completion ---- -{ - "response": { - "model": "gemini-2.5-flash", - "usage": { - "prompt_tokens": 7, - "completion_tokens": 0, - "total_tokens": 8 - }, - "choices": [ - { - "message": { - "role": "assistant" - }, - "finish_reason": "length", - "index": 0 - } - ], - "created": "[date]", - "id": "[id]", - "object": "chat.completion" - }, - "parsed": { - "input_tokens": 7, - "output_tokens": 0, - "total_tokens": 8, - "provider_model": "gemini-2.5-flash" - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/completions/gemini_zero_completion_tokens.completions-detect.snap b/crates/persisting-gateway/tests/fixtures/response/completions/gemini_zero_completion_tokens.completions-detect.snap deleted file mode 100644 index 07084afd5..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/completions/gemini_zero_completion_tokens.completions-detect.snap +++ /dev/null @@ -1,46 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/completions/gemini_zero_completion_tokens.json -info: - model: gemini-2.5-flash - usage: - prompt_tokens: 7 - completion_tokens: 0 - total_tokens: 8 - choices: - - message: - role: assistant - finish_reason: length - index: 0 - created: 1775595226 - id: 2m7Vaca_HdfC1MkP1trPwAM - object: chat.completion ---- -{ - "response": { - "model": "gemini-2.5-flash", - "usage": { - "prompt_tokens": 7, - "completion_tokens": 0, - "total_tokens": 8 - }, - "choices": [ - { - "message": { - "role": "assistant" - }, - "finish_reason": "length", - "index": 0 - } - ], - "created": "[date]", - "id": "[id]", - "object": "chat.completion" - }, - "parsed": { - "input_tokens": 7, - "output_tokens": 0, - "total_tokens": 8, - "provider_model": "gemini-2.5-flash" - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/completions/gemini_zero_completion_tokens.completions-messages.snap b/crates/persisting-gateway/tests/fixtures/response/completions/gemini_zero_completion_tokens.completions-messages.snap deleted file mode 100644 index e318afe79..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/completions/gemini_zero_completion_tokens.completions-messages.snap +++ /dev/null @@ -1,39 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/completions/gemini_zero_completion_tokens.json -info: - model: gemini-2.5-flash - usage: - prompt_tokens: 7 - completion_tokens: 0 - total_tokens: 8 - choices: - - message: - role: assistant - finish_reason: length - index: 0 - created: 1775595226 - id: 2m7Vaca_HdfC1MkP1trPwAM - object: chat.completion ---- -{ - "response": { - "id": "[id]", - "type": "message", - "role": "assistant", - "content": [], - "model": "gemini-2.5-flash", - "stop_reason": "max_tokens", - "stop_sequence": null, - "usage": { - "input_tokens": 7, - "output_tokens": 0 - } - }, - "parsed": { - "input_tokens": 7, - "output_tokens": 0, - "total_tokens": 7, - "provider_model": "gemini-2.5-flash" - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/completions/gemini_zero_completion_tokens.completions-responses.snap b/crates/persisting-gateway/tests/fixtures/response/completions/gemini_zero_completion_tokens.completions-responses.snap deleted file mode 100644 index 9d63b9b5b..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/completions/gemini_zero_completion_tokens.completions-responses.snap +++ /dev/null @@ -1,50 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/completions/gemini_zero_completion_tokens.json -info: - model: gemini-2.5-flash - usage: - prompt_tokens: 7 - completion_tokens: 0 - total_tokens: 8 - choices: - - message: - role: assistant - finish_reason: length - index: 0 - created: 1775595226 - id: 2m7Vaca_HdfC1MkP1trPwAM - object: chat.completion ---- -{ - "response": { - "id": "[id]", - "status": "incomplete", - "output": [], - "model": "input-model", - "usage": { - "input_tokens": 7, - "output_tokens": 1, - "input_tokens_details": { - "cached_tokens": 0 - }, - "output_tokens_details": { - "reasoning_tokens": 0 - }, - "total_tokens": 8 - }, - "created_at": "[date]", - "incomplete_details": { - "reason": "max_tokens" - }, - "object": "response" - }, - "parsed": { - "input_tokens": 7, - "output_tokens": 1, - "total_tokens": 8, - "reasoning_tokens": 0, - "cached_input_tokens": 0, - "provider_model": "input-model" - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/completions/gemini_zero_completion_tokens.json b/crates/persisting-gateway/tests/fixtures/response/completions/gemini_zero_completion_tokens.json deleted file mode 100644 index c28082326..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/completions/gemini_zero_completion_tokens.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "model": "gemini-2.5-flash", - "usage": { - "prompt_tokens": 7, - "completion_tokens": 0, - "total_tokens": 8 - }, - "choices": [ - { - "message": { - "role": "assistant" - }, - "finish_reason": "length", - "index": 0 - } - ], - "created": 1775595226, - "id": "2m7Vaca_HdfC1MkP1trPwAM", - "object": "chat.completion" -} diff --git a/crates/persisting-gateway/tests/fixtures/response/completions/openrouter_reasoning.completions-completions.snap b/crates/persisting-gateway/tests/fixtures/response/completions/openrouter_reasoning.completions-completions.snap deleted file mode 100644 index 870a27a78..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/completions/openrouter_reasoning.completions-completions.snap +++ /dev/null @@ -1,72 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/completions/openrouter_reasoning.json -info: - id: gen-1758648153-xeB95xIQsxABYYNTJLAj - provider: test - model: test - object: chat.completion - created: 1758648153 - choices: - - logprobs: ~ - finish_reason: stop - native_finish_reason: stop - index: 0 - message: - role: assistant - content: hello world! - refusal: ~ - reasoning: "Okay, the user wants me to write a story" - reasoning_details: - - type: reasoning.text - text: "Okay, the user wants me to write a story" - format: unknown - index: 0 - usage: - prompt_tokens: 23 - completion_tokens: 796 - total_tokens: 819 - prompt_tokens_details: ~ ---- -{ - "response": { - "model": "test", - "usage": { - "prompt_tokens": 23, - "completion_tokens": 796, - "total_tokens": 819 - }, - "choices": [ - { - "message": { - "content": "hello world!", - "role": "assistant", - "refusal": null, - "reasoning": "Okay, the user wants me to write a story", - "reasoning_details": [ - { - "type": "reasoning.text", - "text": "Okay, the user wants me to write a story", - "format": "unknown", - "index": 0 - } - ] - }, - "logprobs": null, - "finish_reason": "stop", - "native_finish_reason": "stop", - "index": 0 - } - ], - "id": "[id]", - "provider": "test", - "object": "chat.completion", - "created": "[date]" - }, - "parsed": { - "input_tokens": 23, - "output_tokens": 796, - "total_tokens": 819, - "provider_model": "test" - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/completions/openrouter_reasoning.completions-detect.snap b/crates/persisting-gateway/tests/fixtures/response/completions/openrouter_reasoning.completions-detect.snap deleted file mode 100644 index fe50d6659..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/completions/openrouter_reasoning.completions-detect.snap +++ /dev/null @@ -1,73 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/completions/openrouter_reasoning.json -info: - id: gen-1758648153-xeB95xIQsxABYYNTJLAj - provider: test - model: test - object: chat.completion - created: 1758648153 - choices: - - logprobs: ~ - finish_reason: stop - native_finish_reason: stop - index: 0 - message: - role: assistant - content: hello world! - refusal: ~ - reasoning: "Okay, the user wants me to write a story" - reasoning_details: - - type: reasoning.text - text: "Okay, the user wants me to write a story" - format: unknown - index: 0 - usage: - prompt_tokens: 23 - completion_tokens: 796 - total_tokens: 819 - prompt_tokens_details: ~ ---- -{ - "response": { - "id": "[id]", - "provider": "test", - "model": "test", - "object": "chat.completion", - "created": "[date]", - "choices": [ - { - "logprobs": null, - "finish_reason": "stop", - "native_finish_reason": "stop", - "index": 0, - "message": { - "role": "assistant", - "content": "hello world!", - "refusal": null, - "reasoning": "Okay, the user wants me to write a story", - "reasoning_details": [ - { - "type": "reasoning.text", - "text": "Okay, the user wants me to write a story", - "format": "unknown", - "index": 0 - } - ] - } - } - ], - "usage": { - "prompt_tokens": 23, - "completion_tokens": 796, - "total_tokens": 819, - "prompt_tokens_details": null - } - }, - "parsed": { - "input_tokens": 23, - "output_tokens": 796, - "total_tokens": 819, - "provider_model": "test" - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/completions/openrouter_reasoning.completions-messages.snap b/crates/persisting-gateway/tests/fixtures/response/completions/openrouter_reasoning.completions-messages.snap deleted file mode 100644 index d84006b5f..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/completions/openrouter_reasoning.completions-messages.snap +++ /dev/null @@ -1,56 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/completions/openrouter_reasoning.json -info: - id: gen-1758648153-xeB95xIQsxABYYNTJLAj - provider: test - model: test - object: chat.completion - created: 1758648153 - choices: - - logprobs: ~ - finish_reason: stop - native_finish_reason: stop - index: 0 - message: - role: assistant - content: hello world! - refusal: ~ - reasoning: "Okay, the user wants me to write a story" - reasoning_details: - - type: reasoning.text - text: "Okay, the user wants me to write a story" - format: unknown - index: 0 - usage: - prompt_tokens: 23 - completion_tokens: 796 - total_tokens: 819 - prompt_tokens_details: ~ ---- -{ - "response": { - "id": "[id]", - "type": "message", - "role": "assistant", - "content": [ - { - "type": "text", - "text": "hello world!" - } - ], - "model": "test", - "stop_reason": "end_turn", - "stop_sequence": null, - "usage": { - "input_tokens": 23, - "output_tokens": 796 - } - }, - "parsed": { - "input_tokens": 23, - "output_tokens": 796, - "total_tokens": 819, - "provider_model": "test" - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/completions/openrouter_reasoning.completions-responses.snap b/crates/persisting-gateway/tests/fixtures/response/completions/openrouter_reasoning.completions-responses.snap deleted file mode 100644 index 3f54f9b03..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/completions/openrouter_reasoning.completions-responses.snap +++ /dev/null @@ -1,74 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/completions/openrouter_reasoning.json -info: - id: gen-1758648153-xeB95xIQsxABYYNTJLAj - provider: test - model: test - object: chat.completion - created: 1758648153 - choices: - - logprobs: ~ - finish_reason: stop - native_finish_reason: stop - index: 0 - message: - role: assistant - content: hello world! - refusal: ~ - reasoning: "Okay, the user wants me to write a story" - reasoning_details: - - type: reasoning.text - text: "Okay, the user wants me to write a story" - format: unknown - index: 0 - usage: - prompt_tokens: 23 - completion_tokens: 796 - total_tokens: 819 - prompt_tokens_details: ~ ---- -{ - "response": { - "id": "[id]", - "status": "completed", - "output": [ - { - "type": "message", - "content": [ - { - "type": "output_text", - "annotations": [], - "logprobs": null, - "text": "hello world!" - } - ], - "id": "[id]", - "role": "assistant", - "status": "completed" - } - ], - "model": "input-model", - "usage": { - "input_tokens": 23, - "output_tokens": 796, - "input_tokens_details": { - "cached_tokens": 0 - }, - "output_tokens_details": { - "reasoning_tokens": 0 - }, - "total_tokens": 819 - }, - "created_at": "[date]", - "object": "response" - }, - "parsed": { - "input_tokens": 23, - "output_tokens": 796, - "total_tokens": 819, - "reasoning_tokens": 0, - "cached_input_tokens": 0, - "provider_model": "input-model" - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/completions/openrouter_reasoning.json b/crates/persisting-gateway/tests/fixtures/response/completions/openrouter_reasoning.json deleted file mode 100644 index 61d4aabda..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/completions/openrouter_reasoning.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "id": "gen-1758648153-xeB95xIQsxABYYNTJLAj", - "provider": "test", - "model": "test", - "object": "chat.completion", - "created": 1758648153, - "choices": [ - { - "logprobs": null, - "finish_reason": "stop", - "native_finish_reason": "stop", - "index": 0, - "message": { - "role": "assistant", - "content": "hello world!", - "refusal": null, - "reasoning": "Okay, the user wants me to write a story", - "reasoning_details": [ - { - "type": "reasoning.text", - "text": "Okay, the user wants me to write a story", - "format": "unknown", - "index": 0 - } - ] - } - } - ], - "usage": { - "prompt_tokens": 23, - "completion_tokens": 796, - "total_tokens": 819, - "prompt_tokens_details": null - } -} \ No newline at end of file diff --git a/crates/persisting-gateway/tests/fixtures/response/completions/stream.completions-completions-streaming.snap b/crates/persisting-gateway/tests/fixtures/response/completions/stream.completions-completions-streaming.snap deleted file mode 100644 index 22df00e62..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/completions/stream.completions-completions-streaming.snap +++ /dev/null @@ -1,198 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/completions/stream.json ---- -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"finish_reason":null}],"usage":null,"obfuscation":"Hxo8P"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"Hi"},"finish_reason":null}],"usage":null,"obfuscation":"9AL59"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" there"},"finish_reason":null}],"usage":null,"obfuscation":"J"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"!"},"finish_reason":null}],"usage":null,"obfuscation":"sXPZoY"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" How"},"finish_reason":null}],"usage":null,"obfuscation":"32m"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" can"},"finish_reason":null}],"usage":null,"obfuscation":"z6C"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" I"},"finish_reason":null}],"usage":null,"obfuscation":"5MelW"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" help"},"finish_reason":null}],"usage":null,"obfuscation":"Lu"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" today"},"finish_reason":null}],"usage":null,"obfuscation":"m"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"?"},"finish_reason":null}],"usage":null,"obfuscation":"ZS1MkT"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" If"},"finish_reason":null}],"usage":null,"obfuscation":"cHSc"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" you"},"finish_reason":null}],"usage":null,"obfuscation":"1eg"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"’re"},"finish_reason":null}],"usage":null,"obfuscation":"3DOB"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" not"},"finish_reason":null}],"usage":null,"obfuscation":"xn3"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" sure"},"finish_reason":null}],"usage":null,"obfuscation":"iD"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" where"},"finish_reason":null}],"usage":null,"obfuscation":"U"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" to"},"finish_reason":null}],"usage":null,"obfuscation":"i6Ye"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" start"},"finish_reason":null}],"usage":null,"obfuscation":"h"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":","},"finish_reason":null}],"usage":null,"obfuscation":"JVen3S"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" I"},"finish_reason":null}],"usage":null,"obfuscation":"MR5FG"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" can"},"finish_reason":null}],"usage":null,"obfuscation":"COt"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":":\n\n"},"finish_reason":null}],"usage":null,"obfuscation":"V2"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"-"},"finish_reason":null}],"usage":null,"obfuscation":"bqgHn4"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" Answer"},"finish_reason":null}],"usage":null,"obfuscation":""} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" questions"},"finish_reason":null}],"usage":null,"obfuscation":"D028i7q5qrSJS"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" or"},"finish_reason":null}],"usage":null,"obfuscation":"BK28"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" explain"},"finish_reason":null}],"usage":null,"obfuscation":"n84w87BHVWEslzL"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" concepts"},"finish_reason":null}],"usage":null,"obfuscation":"P1xxk8T6rSL0Kl"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"\n"},"finish_reason":null}],"usage":null,"obfuscation":"sjhfW"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"-"},"finish_reason":null}],"usage":null,"obfuscation":"DzYwmk"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" Help"},"finish_reason":null}],"usage":null,"obfuscation":"K0"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" with"},"finish_reason":null}],"usage":null,"obfuscation":"pU"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" writing"},"finish_reason":null}],"usage":null,"obfuscation":"qnGfPwOY2xLGZLG"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":","},"finish_reason":null}],"usage":null,"obfuscation":"qxmaOy"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" editing"},"finish_reason":null}],"usage":null,"obfuscation":"67kPNCpqBGqivPn"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":","},"finish_reason":null}],"usage":null,"obfuscation":"RMjzHK"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" or"},"finish_reason":null}],"usage":null,"obfuscation":"PluF"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" brainstorming"},"finish_reason":null}],"usage":null,"obfuscation":"SnGLuyRcF"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"\n"},"finish_reason":null}],"usage":null,"obfuscation":"etEgZ"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"-"},"finish_reason":null}],"usage":null,"obfuscation":"LoHdtI"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" Assist"},"finish_reason":null}],"usage":null,"obfuscation":""} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" with"},"finish_reason":null}],"usage":null,"obfuscation":"U8"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" coding"},"finish_reason":null}],"usage":null,"obfuscation":""} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":","},"finish_reason":null}],"usage":null,"obfuscation":"wVSaDj"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" math"},"finish_reason":null}],"usage":null,"obfuscation":"lO"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":","},"finish_reason":null}],"usage":null,"obfuscation":"1u7ath"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" or"},"finish_reason":null}],"usage":null,"obfuscation":"pPDH"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" data"},"finish_reason":null}],"usage":null,"obfuscation":"cz"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" tasks"},"finish_reason":null}],"usage":null,"obfuscation":"b"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"\n"},"finish_reason":null}],"usage":null,"obfuscation":"mjT2x"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"-"},"finish_reason":null}],"usage":null,"obfuscation":"6hQM4Z"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" Plan"},"finish_reason":null}],"usage":null,"obfuscation":"N8"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" something"},"finish_reason":null}],"usage":null,"obfuscation":"GfNRAPMrpn7g4"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" ("},"finish_reason":null}],"usage":null,"obfuscation":"wrazg"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"a"},"finish_reason":null}],"usage":null,"obfuscation":"ey8bHF"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" trip"},"finish_reason":null}],"usage":null,"obfuscation":"Zd"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":","},"finish_reason":null}],"usage":null,"obfuscation":"n9Ll9x"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" a"},"finish_reason":null}],"usage":null,"obfuscation":"CDcC3"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" project"},"finish_reason":null}],"usage":null,"obfuscation":"wePlKVyiChPIbuB"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":","},"finish_reason":null}],"usage":null,"obfuscation":"ahbhFR"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" a"},"finish_reason":null}],"usage":null,"obfuscation":"FbtRz"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" study"},"finish_reason":null}],"usage":null,"obfuscation":"T"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" schedule"},"finish_reason":null}],"usage":null,"obfuscation":"hQyDaU8iWbhy0a"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":")\n"},"finish_reason":null}],"usage":null,"obfuscation":"DK9z"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"-"},"finish_reason":null}],"usage":null,"obfuscation":"5rPABI"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" Translate"},"finish_reason":null}],"usage":null,"obfuscation":"iWxBkG5xhLK9R"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" or"},"finish_reason":null}],"usage":null,"obfuscation":"g0nH"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" practice"},"finish_reason":null}],"usage":null,"obfuscation":"mKNK8BwUuLhWKY"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" a"},"finish_reason":null}],"usage":null,"obfuscation":"Ris3I"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" language"},"finish_reason":null}],"usage":null,"obfuscation":"pRKfKHu39ZhgAF"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"\n\n"},"finish_reason":null}],"usage":null,"obfuscation":"27l"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"Tell"},"finish_reason":null}],"usage":null,"obfuscation":"MxT"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" me"},"finish_reason":null}],"usage":null,"obfuscation":"Td2S"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" what"},"finish_reason":null}],"usage":null,"obfuscation":"H6"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" you"},"finish_reason":null}],"usage":null,"obfuscation":"AWF"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"’re"},"finish_reason":null}],"usage":null,"obfuscation":"yZ4S"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" aiming"},"finish_reason":null}],"usage":null,"obfuscation":""} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" for"},"finish_reason":null}],"usage":null,"obfuscation":"Y6z"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":","},"finish_reason":null}],"usage":null,"obfuscation":"qjc2ki"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" and"},"finish_reason":null}],"usage":null,"obfuscation":"94I"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" I"},"finish_reason":null}],"usage":null,"obfuscation":"JQ6UH"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"’ll"},"finish_reason":null}],"usage":null,"obfuscation":"HHic"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" tailor"},"finish_reason":null}],"usage":null,"obfuscation":""} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" my"},"finish_reason":null}],"usage":null,"obfuscation":"HxFu"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" help"},"finish_reason":null}],"usage":null,"obfuscation":"Bs"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"."},"finish_reason":null}],"usage":null,"obfuscation":"mUqC4Z"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{},"finish_reason":"stop"}],"usage":null,"obfuscation":"n"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[],"usage":{"prompt_tokens":18,"completion_tokens":286,"total_tokens":304,"prompt_tokens_details":{"cached_tokens":0,"audio_tokens":0},"completion_tokens_details":{"reasoning_tokens":192,"audio_tokens":0,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0}},"obfuscation":"Od"} - -data: [DONE] - - - -{ - "input_tokens": 18, - "input_audio_tokens": 0, - "output_tokens": 286, - "output_audio_tokens": 0, - "total_tokens": 304, - "reasoning_tokens": 192, - "cached_input_tokens": 0, - "service_tier": "default", - "provider_model": "gpt-5-nano-2025-08-07", - "completion": [ - "Hi there! How can I help today? If you’re not sure where to start, I can:\n\n- Answer questions or explain concepts\n- Help with writing, editing, or brainstorming\n- Assist with coding, math, or data tasks\n- Plan something (a trip, a project, a study schedule)\n- Translate or practice a language\n\nTell me what you’re aiming for, and I’ll tailor my help." - ] -} diff --git a/crates/persisting-gateway/tests/fixtures/response/completions/stream.completions-detect-streaming.snap b/crates/persisting-gateway/tests/fixtures/response/completions/stream.completions-detect-streaming.snap deleted file mode 100644 index b69ad63d0..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/completions/stream.completions-detect-streaming.snap +++ /dev/null @@ -1,195 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/completions/stream.json ---- -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"finish_reason":null}],"usage":null,"obfuscation":"Hxo8P"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"Hi"},"finish_reason":null}],"usage":null,"obfuscation":"9AL59"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" there"},"finish_reason":null}],"usage":null,"obfuscation":"J"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"!"},"finish_reason":null}],"usage":null,"obfuscation":"sXPZoY"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" How"},"finish_reason":null}],"usage":null,"obfuscation":"32m"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" can"},"finish_reason":null}],"usage":null,"obfuscation":"z6C"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" I"},"finish_reason":null}],"usage":null,"obfuscation":"5MelW"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" help"},"finish_reason":null}],"usage":null,"obfuscation":"Lu"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" today"},"finish_reason":null}],"usage":null,"obfuscation":"m"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"?"},"finish_reason":null}],"usage":null,"obfuscation":"ZS1MkT"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" If"},"finish_reason":null}],"usage":null,"obfuscation":"cHSc"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" you"},"finish_reason":null}],"usage":null,"obfuscation":"1eg"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"’re"},"finish_reason":null}],"usage":null,"obfuscation":"3DOB"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" not"},"finish_reason":null}],"usage":null,"obfuscation":"xn3"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" sure"},"finish_reason":null}],"usage":null,"obfuscation":"iD"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" where"},"finish_reason":null}],"usage":null,"obfuscation":"U"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" to"},"finish_reason":null}],"usage":null,"obfuscation":"i6Ye"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" start"},"finish_reason":null}],"usage":null,"obfuscation":"h"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":","},"finish_reason":null}],"usage":null,"obfuscation":"JVen3S"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" I"},"finish_reason":null}],"usage":null,"obfuscation":"MR5FG"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" can"},"finish_reason":null}],"usage":null,"obfuscation":"COt"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":":\n\n"},"finish_reason":null}],"usage":null,"obfuscation":"V2"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"-"},"finish_reason":null}],"usage":null,"obfuscation":"bqgHn4"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" Answer"},"finish_reason":null}],"usage":null,"obfuscation":""} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" questions"},"finish_reason":null}],"usage":null,"obfuscation":"D028i7q5qrSJS"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" or"},"finish_reason":null}],"usage":null,"obfuscation":"BK28"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" explain"},"finish_reason":null}],"usage":null,"obfuscation":"n84w87BHVWEslzL"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" concepts"},"finish_reason":null}],"usage":null,"obfuscation":"P1xxk8T6rSL0Kl"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"\n"},"finish_reason":null}],"usage":null,"obfuscation":"sjhfW"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"-"},"finish_reason":null}],"usage":null,"obfuscation":"DzYwmk"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" Help"},"finish_reason":null}],"usage":null,"obfuscation":"K0"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" with"},"finish_reason":null}],"usage":null,"obfuscation":"pU"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" writing"},"finish_reason":null}],"usage":null,"obfuscation":"qnGfPwOY2xLGZLG"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":","},"finish_reason":null}],"usage":null,"obfuscation":"qxmaOy"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" editing"},"finish_reason":null}],"usage":null,"obfuscation":"67kPNCpqBGqivPn"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":","},"finish_reason":null}],"usage":null,"obfuscation":"RMjzHK"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" or"},"finish_reason":null}],"usage":null,"obfuscation":"PluF"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" brainstorming"},"finish_reason":null}],"usage":null,"obfuscation":"SnGLuyRcF"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"\n"},"finish_reason":null}],"usage":null,"obfuscation":"etEgZ"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"-"},"finish_reason":null}],"usage":null,"obfuscation":"LoHdtI"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" Assist"},"finish_reason":null}],"usage":null,"obfuscation":""} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" with"},"finish_reason":null}],"usage":null,"obfuscation":"U8"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" coding"},"finish_reason":null}],"usage":null,"obfuscation":""} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":","},"finish_reason":null}],"usage":null,"obfuscation":"wVSaDj"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" math"},"finish_reason":null}],"usage":null,"obfuscation":"lO"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":","},"finish_reason":null}],"usage":null,"obfuscation":"1u7ath"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" or"},"finish_reason":null}],"usage":null,"obfuscation":"pPDH"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" data"},"finish_reason":null}],"usage":null,"obfuscation":"cz"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" tasks"},"finish_reason":null}],"usage":null,"obfuscation":"b"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"\n"},"finish_reason":null}],"usage":null,"obfuscation":"mjT2x"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"-"},"finish_reason":null}],"usage":null,"obfuscation":"6hQM4Z"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" Plan"},"finish_reason":null}],"usage":null,"obfuscation":"N8"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" something"},"finish_reason":null}],"usage":null,"obfuscation":"GfNRAPMrpn7g4"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" ("},"finish_reason":null}],"usage":null,"obfuscation":"wrazg"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"a"},"finish_reason":null}],"usage":null,"obfuscation":"ey8bHF"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" trip"},"finish_reason":null}],"usage":null,"obfuscation":"Zd"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":","},"finish_reason":null}],"usage":null,"obfuscation":"n9Ll9x"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" a"},"finish_reason":null}],"usage":null,"obfuscation":"CDcC3"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" project"},"finish_reason":null}],"usage":null,"obfuscation":"wePlKVyiChPIbuB"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":","},"finish_reason":null}],"usage":null,"obfuscation":"ahbhFR"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" a"},"finish_reason":null}],"usage":null,"obfuscation":"FbtRz"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" study"},"finish_reason":null}],"usage":null,"obfuscation":"T"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" schedule"},"finish_reason":null}],"usage":null,"obfuscation":"hQyDaU8iWbhy0a"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":")\n"},"finish_reason":null}],"usage":null,"obfuscation":"DK9z"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"-"},"finish_reason":null}],"usage":null,"obfuscation":"5rPABI"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" Translate"},"finish_reason":null}],"usage":null,"obfuscation":"iWxBkG5xhLK9R"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" or"},"finish_reason":null}],"usage":null,"obfuscation":"g0nH"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" practice"},"finish_reason":null}],"usage":null,"obfuscation":"mKNK8BwUuLhWKY"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" a"},"finish_reason":null}],"usage":null,"obfuscation":"Ris3I"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" language"},"finish_reason":null}],"usage":null,"obfuscation":"pRKfKHu39ZhgAF"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"\n\n"},"finish_reason":null}],"usage":null,"obfuscation":"27l"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"Tell"},"finish_reason":null}],"usage":null,"obfuscation":"MxT"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" me"},"finish_reason":null}],"usage":null,"obfuscation":"Td2S"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" what"},"finish_reason":null}],"usage":null,"obfuscation":"H6"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" you"},"finish_reason":null}],"usage":null,"obfuscation":"AWF"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"’re"},"finish_reason":null}],"usage":null,"obfuscation":"yZ4S"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" aiming"},"finish_reason":null}],"usage":null,"obfuscation":""} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" for"},"finish_reason":null}],"usage":null,"obfuscation":"Y6z"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":","},"finish_reason":null}],"usage":null,"obfuscation":"qjc2ki"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" and"},"finish_reason":null}],"usage":null,"obfuscation":"94I"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" I"},"finish_reason":null}],"usage":null,"obfuscation":"JQ6UH"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"’ll"},"finish_reason":null}],"usage":null,"obfuscation":"HHic"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" tailor"},"finish_reason":null}],"usage":null,"obfuscation":""} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" my"},"finish_reason":null}],"usage":null,"obfuscation":"HxFu"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" help"},"finish_reason":null}],"usage":null,"obfuscation":"Bs"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"."},"finish_reason":null}],"usage":null,"obfuscation":"mUqC4Z"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{},"finish_reason":"stop"}],"usage":null,"obfuscation":"n"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":123,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[],"usage":{"prompt_tokens":18,"completion_tokens":286,"total_tokens":304,"prompt_tokens_details":{"cached_tokens":0,"audio_tokens":0},"completion_tokens_details":{"reasoning_tokens":192,"audio_tokens":0,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0}},"obfuscation":"Od"} - -data: [DONE] - - - -{ - "input_tokens": 18, - "input_audio_tokens": 0, - "output_tokens": 286, - "output_audio_tokens": 0, - "total_tokens": 304, - "reasoning_tokens": 192, - "cached_input_tokens": 0, - "service_tier": "default", - "provider_model": "gpt-5-nano-2025-08-07" -} diff --git a/crates/persisting-gateway/tests/fixtures/response/completions/stream.completions-messages-streaming.snap b/crates/persisting-gateway/tests/fixtures/response/completions/stream.completions-messages-streaming.snap deleted file mode 100644 index fc0d040b1..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/completions/stream.completions-messages-streaming.snap +++ /dev/null @@ -1,283 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/completions/stream.json ---- -event: message_start -data: {"type":"message_start","message":{"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","type":"message","role":"assistant","content":[],"model":"gpt-5-nano-2025-08-07","stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":0,"output_tokens":0}}} - -event: content_block_start -data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Hi"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" there"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"!"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" How"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" can"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" I"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" help"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" today"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"?"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" If"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" you"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"’re"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" not"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" sure"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" where"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" to"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" start"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":","}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" I"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" can"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":":\n\n"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"-"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" Answer"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" questions"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" or"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" explain"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" concepts"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"\n"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"-"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" Help"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" with"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" writing"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":","}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" editing"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":","}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" or"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" brainstorming"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"\n"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"-"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" Assist"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" with"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" coding"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":","}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" math"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":","}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" or"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" data"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" tasks"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"\n"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"-"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" Plan"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" something"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" ("}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"a"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" trip"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":","}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" a"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" project"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":","}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" a"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" study"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" schedule"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":")\n"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"-"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" Translate"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" or"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" practice"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" a"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" language"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"\n\n"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Tell"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" me"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" what"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" you"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"’re"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" aiming"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" for"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":","}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" and"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" I"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"’ll"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" tailor"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" my"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" help"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"."}} - -event: content_block_stop -data: {"type":"content_block_stop","index":0} - -event: message_delta -data: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"input_tokens":18,"output_tokens":286,"cache_read_input_tokens":0}} - -event: message_stop -data: {"type":"message_stop"} - - - -{ - "input_tokens": 18, - "output_tokens": 286, - "total_tokens": 304, - "cached_input_tokens": 0, - "provider_model": "gpt-5-nano-2025-08-07" -} diff --git a/crates/persisting-gateway/tests/fixtures/response/completions/stream.completions-responses-streaming.snap b/crates/persisting-gateway/tests/fixtures/response/completions/stream.completions-responses-streaming.snap deleted file mode 100644 index 892c29651..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/completions/stream.completions-responses-streaming.snap +++ /dev/null @@ -1,294 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/completions/stream.json ---- -event: event -data: {"type":"response.created","sequence_number":1,"response":{"created_at":123,"id":"","model":"gpt-5-nano-2025-08-07","object":"response","output":[],"status":"in_progress"}} - -event: event -data: {"type":"response.output_item.added","sequence_number":2,"output_index":0,"item":{"type":"message","content":[],"id":"","role":"assistant","status":"in_progress"}} - -event: event -data: {"type":"response.content_part.added","sequence_number":3,"item_id":"","output_index":0,"content_index":0,"part":{"type":"output_text","annotations":[],"logprobs":null,"text":""}} - -event: event -data: {"type":"response.output_text.delta","sequence_number":4,"item_id":"","output_index":0,"content_index":0,"delta":""} - -event: event -data: {"type":"response.output_text.delta","sequence_number":5,"item_id":"","output_index":0,"content_index":0,"delta":"Hi"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":6,"item_id":"","output_index":0,"content_index":0,"delta":" there"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":7,"item_id":"","output_index":0,"content_index":0,"delta":"!"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":8,"item_id":"","output_index":0,"content_index":0,"delta":" How"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":9,"item_id":"","output_index":0,"content_index":0,"delta":" can"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":10,"item_id":"","output_index":0,"content_index":0,"delta":" I"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":11,"item_id":"","output_index":0,"content_index":0,"delta":" help"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":12,"item_id":"","output_index":0,"content_index":0,"delta":" today"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":13,"item_id":"","output_index":0,"content_index":0,"delta":"?"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":14,"item_id":"","output_index":0,"content_index":0,"delta":" If"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":15,"item_id":"","output_index":0,"content_index":0,"delta":" you"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":16,"item_id":"","output_index":0,"content_index":0,"delta":"’re"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":17,"item_id":"","output_index":0,"content_index":0,"delta":" not"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":18,"item_id":"","output_index":0,"content_index":0,"delta":" sure"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":19,"item_id":"","output_index":0,"content_index":0,"delta":" where"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":20,"item_id":"","output_index":0,"content_index":0,"delta":" to"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":21,"item_id":"","output_index":0,"content_index":0,"delta":" start"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":22,"item_id":"","output_index":0,"content_index":0,"delta":","} - -event: event -data: {"type":"response.output_text.delta","sequence_number":23,"item_id":"","output_index":0,"content_index":0,"delta":" I"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":24,"item_id":"","output_index":0,"content_index":0,"delta":" can"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":25,"item_id":"","output_index":0,"content_index":0,"delta":":\n\n"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":26,"item_id":"","output_index":0,"content_index":0,"delta":"-"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":27,"item_id":"","output_index":0,"content_index":0,"delta":" Answer"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":28,"item_id":"","output_index":0,"content_index":0,"delta":" questions"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":29,"item_id":"","output_index":0,"content_index":0,"delta":" or"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":30,"item_id":"","output_index":0,"content_index":0,"delta":" explain"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":31,"item_id":"","output_index":0,"content_index":0,"delta":" concepts"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":32,"item_id":"","output_index":0,"content_index":0,"delta":"\n"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":33,"item_id":"","output_index":0,"content_index":0,"delta":"-"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":34,"item_id":"","output_index":0,"content_index":0,"delta":" Help"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":35,"item_id":"","output_index":0,"content_index":0,"delta":" with"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":36,"item_id":"","output_index":0,"content_index":0,"delta":" writing"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":37,"item_id":"","output_index":0,"content_index":0,"delta":","} - -event: event -data: {"type":"response.output_text.delta","sequence_number":38,"item_id":"","output_index":0,"content_index":0,"delta":" editing"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":39,"item_id":"","output_index":0,"content_index":0,"delta":","} - -event: event -data: {"type":"response.output_text.delta","sequence_number":40,"item_id":"","output_index":0,"content_index":0,"delta":" or"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":41,"item_id":"","output_index":0,"content_index":0,"delta":" brainstorming"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":42,"item_id":"","output_index":0,"content_index":0,"delta":"\n"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":43,"item_id":"","output_index":0,"content_index":0,"delta":"-"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":44,"item_id":"","output_index":0,"content_index":0,"delta":" Assist"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":45,"item_id":"","output_index":0,"content_index":0,"delta":" with"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":46,"item_id":"","output_index":0,"content_index":0,"delta":" coding"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":47,"item_id":"","output_index":0,"content_index":0,"delta":","} - -event: event -data: {"type":"response.output_text.delta","sequence_number":48,"item_id":"","output_index":0,"content_index":0,"delta":" math"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":49,"item_id":"","output_index":0,"content_index":0,"delta":","} - -event: event -data: {"type":"response.output_text.delta","sequence_number":50,"item_id":"","output_index":0,"content_index":0,"delta":" or"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":51,"item_id":"","output_index":0,"content_index":0,"delta":" data"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":52,"item_id":"","output_index":0,"content_index":0,"delta":" tasks"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":53,"item_id":"","output_index":0,"content_index":0,"delta":"\n"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":54,"item_id":"","output_index":0,"content_index":0,"delta":"-"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":55,"item_id":"","output_index":0,"content_index":0,"delta":" Plan"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":56,"item_id":"","output_index":0,"content_index":0,"delta":" something"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":57,"item_id":"","output_index":0,"content_index":0,"delta":" ("} - -event: event -data: {"type":"response.output_text.delta","sequence_number":58,"item_id":"","output_index":0,"content_index":0,"delta":"a"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":59,"item_id":"","output_index":0,"content_index":0,"delta":" trip"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":60,"item_id":"","output_index":0,"content_index":0,"delta":","} - -event: event -data: {"type":"response.output_text.delta","sequence_number":61,"item_id":"","output_index":0,"content_index":0,"delta":" a"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":62,"item_id":"","output_index":0,"content_index":0,"delta":" project"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":63,"item_id":"","output_index":0,"content_index":0,"delta":","} - -event: event -data: {"type":"response.output_text.delta","sequence_number":64,"item_id":"","output_index":0,"content_index":0,"delta":" a"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":65,"item_id":"","output_index":0,"content_index":0,"delta":" study"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":66,"item_id":"","output_index":0,"content_index":0,"delta":" schedule"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":67,"item_id":"","output_index":0,"content_index":0,"delta":")\n"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":68,"item_id":"","output_index":0,"content_index":0,"delta":"-"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":69,"item_id":"","output_index":0,"content_index":0,"delta":" Translate"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":70,"item_id":"","output_index":0,"content_index":0,"delta":" or"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":71,"item_id":"","output_index":0,"content_index":0,"delta":" practice"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":72,"item_id":"","output_index":0,"content_index":0,"delta":" a"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":73,"item_id":"","output_index":0,"content_index":0,"delta":" language"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":74,"item_id":"","output_index":0,"content_index":0,"delta":"\n\n"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":75,"item_id":"","output_index":0,"content_index":0,"delta":"Tell"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":76,"item_id":"","output_index":0,"content_index":0,"delta":" me"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":77,"item_id":"","output_index":0,"content_index":0,"delta":" what"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":78,"item_id":"","output_index":0,"content_index":0,"delta":" you"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":79,"item_id":"","output_index":0,"content_index":0,"delta":"’re"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":80,"item_id":"","output_index":0,"content_index":0,"delta":" aiming"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":81,"item_id":"","output_index":0,"content_index":0,"delta":" for"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":82,"item_id":"","output_index":0,"content_index":0,"delta":","} - -event: event -data: {"type":"response.output_text.delta","sequence_number":83,"item_id":"","output_index":0,"content_index":0,"delta":" and"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":84,"item_id":"","output_index":0,"content_index":0,"delta":" I"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":85,"item_id":"","output_index":0,"content_index":0,"delta":"’ll"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":86,"item_id":"","output_index":0,"content_index":0,"delta":" tailor"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":87,"item_id":"","output_index":0,"content_index":0,"delta":" my"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":88,"item_id":"","output_index":0,"content_index":0,"delta":" help"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":89,"item_id":"","output_index":0,"content_index":0,"delta":"."} - -event: event -data: {"type":"response.content_part.done","sequence_number":90,"item_id":"","output_index":0,"content_index":0,"part":{"type":"output_text","annotations":[],"logprobs":null,"text":""}} - -event: event -data: {"type":"response.output_item.done","sequence_number":91,"output_index":0,"item":{"type":"message","content":[],"id":"","role":"assistant","status":"completed"}} - -event: event -data: {"type":"response.completed","sequence_number":92,"response":{"created_at":123,"id":"","model":"gpt-5-nano-2025-08-07","object":"response","output":[],"status":"completed","usage":{"input_tokens":18,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":null},"output_tokens":286,"output_tokens_details":{"reasoning_tokens":192},"total_tokens":304}}} - - - -{ - "input_tokens": 18, - "output_tokens": 286, - "total_tokens": 304, - "reasoning_tokens": 192, - "cached_input_tokens": 0, - "service_tier": "default", - "provider_model": "gpt-5-nano-2025-08-07", - "completion": [ - "Hi there! How can I help today? If you’re not sure where to start, I can:\n\n- Answer questions or explain concepts\n- Help with writing, editing, or brainstorming\n- Assist with coding, math, or data tasks\n- Plan something (a trip, a project, a study schedule)\n- Translate or practice a language\n\nTell me what you’re aiming for, and I’ll tailor my help." - ] -} diff --git a/crates/persisting-gateway/tests/fixtures/response/completions/stream.json b/crates/persisting-gateway/tests/fixtures/response/completions/stream.json deleted file mode 100644 index f291333d4..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/completions/stream.json +++ /dev/null @@ -1,178 +0,0 @@ -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"finish_reason":null}],"usage":null,"obfuscation":"Hxo8P"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"Hi"},"finish_reason":null}],"usage":null,"obfuscation":"9AL59"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" there"},"finish_reason":null}],"usage":null,"obfuscation":"J"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"!"},"finish_reason":null}],"usage":null,"obfuscation":"sXPZoY"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" How"},"finish_reason":null}],"usage":null,"obfuscation":"32m"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" can"},"finish_reason":null}],"usage":null,"obfuscation":"z6C"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" I"},"finish_reason":null}],"usage":null,"obfuscation":"5MelW"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" help"},"finish_reason":null}],"usage":null,"obfuscation":"Lu"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" today"},"finish_reason":null}],"usage":null,"obfuscation":"m"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"?"},"finish_reason":null}],"usage":null,"obfuscation":"ZS1MkT"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" If"},"finish_reason":null}],"usage":null,"obfuscation":"cHSc"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" you"},"finish_reason":null}],"usage":null,"obfuscation":"1eg"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"’re"},"finish_reason":null}],"usage":null,"obfuscation":"3DOB"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" not"},"finish_reason":null}],"usage":null,"obfuscation":"xn3"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" sure"},"finish_reason":null}],"usage":null,"obfuscation":"iD"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" where"},"finish_reason":null}],"usage":null,"obfuscation":"U"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" to"},"finish_reason":null}],"usage":null,"obfuscation":"i6Ye"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" start"},"finish_reason":null}],"usage":null,"obfuscation":"h"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":","},"finish_reason":null}],"usage":null,"obfuscation":"JVen3S"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" I"},"finish_reason":null}],"usage":null,"obfuscation":"MR5FG"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" can"},"finish_reason":null}],"usage":null,"obfuscation":"COt"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":":\n\n"},"finish_reason":null}],"usage":null,"obfuscation":"V2"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"-"},"finish_reason":null}],"usage":null,"obfuscation":"bqgHn4"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" Answer"},"finish_reason":null}],"usage":null,"obfuscation":""} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" questions"},"finish_reason":null}],"usage":null,"obfuscation":"D028i7q5qrSJS"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" or"},"finish_reason":null}],"usage":null,"obfuscation":"BK28"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" explain"},"finish_reason":null}],"usage":null,"obfuscation":"n84w87BHVWEslzL"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" concepts"},"finish_reason":null}],"usage":null,"obfuscation":"P1xxk8T6rSL0Kl"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"\n"},"finish_reason":null}],"usage":null,"obfuscation":"sjhfW"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"-"},"finish_reason":null}],"usage":null,"obfuscation":"DzYwmk"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" Help"},"finish_reason":null}],"usage":null,"obfuscation":"K0"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" with"},"finish_reason":null}],"usage":null,"obfuscation":"pU"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" writing"},"finish_reason":null}],"usage":null,"obfuscation":"qnGfPwOY2xLGZLG"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":","},"finish_reason":null}],"usage":null,"obfuscation":"qxmaOy"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" editing"},"finish_reason":null}],"usage":null,"obfuscation":"67kPNCpqBGqivPn"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":","},"finish_reason":null}],"usage":null,"obfuscation":"RMjzHK"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" or"},"finish_reason":null}],"usage":null,"obfuscation":"PluF"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" brainstorming"},"finish_reason":null}],"usage":null,"obfuscation":"SnGLuyRcF"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"\n"},"finish_reason":null}],"usage":null,"obfuscation":"etEgZ"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"-"},"finish_reason":null}],"usage":null,"obfuscation":"LoHdtI"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" Assist"},"finish_reason":null}],"usage":null,"obfuscation":""} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" with"},"finish_reason":null}],"usage":null,"obfuscation":"U8"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" coding"},"finish_reason":null}],"usage":null,"obfuscation":""} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":","},"finish_reason":null}],"usage":null,"obfuscation":"wVSaDj"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" math"},"finish_reason":null}],"usage":null,"obfuscation":"lO"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":","},"finish_reason":null}],"usage":null,"obfuscation":"1u7ath"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" or"},"finish_reason":null}],"usage":null,"obfuscation":"pPDH"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" data"},"finish_reason":null}],"usage":null,"obfuscation":"cz"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" tasks"},"finish_reason":null}],"usage":null,"obfuscation":"b"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"\n"},"finish_reason":null}],"usage":null,"obfuscation":"mjT2x"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"-"},"finish_reason":null}],"usage":null,"obfuscation":"6hQM4Z"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" Plan"},"finish_reason":null}],"usage":null,"obfuscation":"N8"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" something"},"finish_reason":null}],"usage":null,"obfuscation":"GfNRAPMrpn7g4"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" ("},"finish_reason":null}],"usage":null,"obfuscation":"wrazg"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"a"},"finish_reason":null}],"usage":null,"obfuscation":"ey8bHF"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" trip"},"finish_reason":null}],"usage":null,"obfuscation":"Zd"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":","},"finish_reason":null}],"usage":null,"obfuscation":"n9Ll9x"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" a"},"finish_reason":null}],"usage":null,"obfuscation":"CDcC3"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" project"},"finish_reason":null}],"usage":null,"obfuscation":"wePlKVyiChPIbuB"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":","},"finish_reason":null}],"usage":null,"obfuscation":"ahbhFR"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" a"},"finish_reason":null}],"usage":null,"obfuscation":"FbtRz"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" study"},"finish_reason":null}],"usage":null,"obfuscation":"T"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" schedule"},"finish_reason":null}],"usage":null,"obfuscation":"hQyDaU8iWbhy0a"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":")\n"},"finish_reason":null}],"usage":null,"obfuscation":"DK9z"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"-"},"finish_reason":null}],"usage":null,"obfuscation":"5rPABI"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" Translate"},"finish_reason":null}],"usage":null,"obfuscation":"iWxBkG5xhLK9R"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" or"},"finish_reason":null}],"usage":null,"obfuscation":"g0nH"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" practice"},"finish_reason":null}],"usage":null,"obfuscation":"mKNK8BwUuLhWKY"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" a"},"finish_reason":null}],"usage":null,"obfuscation":"Ris3I"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" language"},"finish_reason":null}],"usage":null,"obfuscation":"pRKfKHu39ZhgAF"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"\n\n"},"finish_reason":null}],"usage":null,"obfuscation":"27l"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"Tell"},"finish_reason":null}],"usage":null,"obfuscation":"MxT"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" me"},"finish_reason":null}],"usage":null,"obfuscation":"Td2S"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" what"},"finish_reason":null}],"usage":null,"obfuscation":"H6"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" you"},"finish_reason":null}],"usage":null,"obfuscation":"AWF"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"’re"},"finish_reason":null}],"usage":null,"obfuscation":"yZ4S"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" aiming"},"finish_reason":null}],"usage":null,"obfuscation":""} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" for"},"finish_reason":null}],"usage":null,"obfuscation":"Y6z"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":","},"finish_reason":null}],"usage":null,"obfuscation":"qjc2ki"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" and"},"finish_reason":null}],"usage":null,"obfuscation":"94I"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" I"},"finish_reason":null}],"usage":null,"obfuscation":"JQ6UH"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"’ll"},"finish_reason":null}],"usage":null,"obfuscation":"HHic"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" tailor"},"finish_reason":null}],"usage":null,"obfuscation":""} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" my"},"finish_reason":null}],"usage":null,"obfuscation":"HxFu"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" help"},"finish_reason":null}],"usage":null,"obfuscation":"Bs"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"."},"finish_reason":null}],"usage":null,"obfuscation":"mUqC4Z"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{},"finish_reason":"stop"}],"usage":null,"obfuscation":"n"} - -data: {"id":"chatcmpl-DFnKV81lMKGBRRh35sxZO8I88ireo","object":"chat.completion.chunk","created":1772656915,"model":"gpt-5-nano-2025-08-07","service_tier":"default","system_fingerprint":null,"choices":[],"usage":{"prompt_tokens":18,"completion_tokens":286,"total_tokens":304,"prompt_tokens_details":{"cached_tokens":0,"audio_tokens":0},"completion_tokens_details":{"reasoning_tokens":192,"audio_tokens":0,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0}},"obfuscation":"Od"} - -data: [DONE] - diff --git a/crates/persisting-gateway/tests/fixtures/response/completions/stream_tool_empty_content.completions-messages-streaming.snap b/crates/persisting-gateway/tests/fixtures/response/completions/stream_tool_empty_content.completions-messages-streaming.snap deleted file mode 100644 index 829cf16d0..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/completions/stream_tool_empty_content.completions-messages-streaming.snap +++ /dev/null @@ -1,49 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/completions/stream_tool_empty_content.json ---- -event: message_start -data: {"type":"message_start","message":{"id":"chatcmpl-tool-empty-content","type":"message","role":"assistant","content":[],"model":"claude-opus-4-8","stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":0,"output_tokens":0}}} - -event: content_block_start -data: {"type":"content_block_start","index":0,"content_block":{"type":"tool_use","id":"toolu_empty_content","name":"Bash","input":{}}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"{\"com"}} - -event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"mand\":\"pwd\"}"}} - -event: content_block_stop -data: {"type":"content_block_stop","index":0} - -event: message_delta -data: {"type":"message_delta","delta":{"stop_reason":"tool_use","stop_sequence":null},"usage":{"input_tokens":12,"output_tokens":8}} - -event: message_stop -data: {"type":"message_stop"} - - - -{ - "input_tokens": 12, - "output_tokens": 8, - "total_tokens": 20, - "provider_model": "claude-opus-4-8", - "output_messages": [ - { - "role": "assistant", - "content": [ - { - "type": "tool_call", - "id": "toolu_empty_content", - "name": "Bash", - "arguments": { - "command": "pwd" - } - } - ], - "finish_reason": "tool_use" - } - ] -} diff --git a/crates/persisting-gateway/tests/fixtures/response/completions/stream_tool_empty_content.completions-responses-streaming.snap b/crates/persisting-gateway/tests/fixtures/response/completions/stream_tool_empty_content.completions-responses-streaming.snap deleted file mode 100644 index 4930fe19f..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/completions/stream_tool_empty_content.completions-responses-streaming.snap +++ /dev/null @@ -1,77 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/completions/stream_tool_empty_content.json ---- -event: event -data: {"type":"response.created","sequence_number":1,"response":{"created_at":123,"id":"","model":"claude-opus-4-8","object":"response","output":[],"status":"in_progress"}} - -event: event -data: {"type":"response.output_item.added","sequence_number":2,"output_index":0,"item":{"type":"message","content":[],"id":"","role":"assistant","status":"in_progress"}} - -event: event -data: {"type":"response.content_part.added","sequence_number":3,"item_id":"","output_index":0,"content_index":0,"part":{"type":"output_text","annotations":[],"logprobs":null,"text":""}} - -event: event -data: {"type":"response.output_text.delta","sequence_number":4,"item_id":"","output_index":0,"content_index":0,"delta":""} - -event: event -data: {"type":"response.output_item.added","sequence_number":5,"output_index":1,"item":{"type":"function_call","arguments":"","call_id":"call_xxx","name":"Bash","id":"","status":"in_progress"}} - -event: event -data: {"type":"response.output_text.delta","sequence_number":6,"item_id":"","output_index":0,"content_index":0,"delta":""} - -event: event -data: {"type":"response.function_call_arguments.delta","sequence_number":7,"item_id":"","output_index":1,"delta":"{\"com"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":8,"item_id":"","output_index":0,"content_index":0,"delta":""} - -event: event -data: {"type":"response.function_call_arguments.delta","sequence_number":9,"item_id":"","output_index":1,"delta":"mand\":\"pwd\"}"} - -event: event -data: {"type":"response.output_text.delta","sequence_number":10,"item_id":"","output_index":0,"content_index":0,"delta":""} - -event: event -data: {"type":"response.function_call_arguments.done","name":"Bash","sequence_number":11,"item_id":"","output_index":1,"arguments":"{\"command\":\"pwd\"}"} - -event: event -data: {"type":"response.output_item.done","sequence_number":12,"output_index":1,"item":{"type":"function_call","arguments":"{\"command\":\"pwd\"}","call_id":"call_xxx","name":"Bash","id":"","status":"completed"}} - -event: event -data: {"type":"response.content_part.done","sequence_number":13,"item_id":"","output_index":0,"content_index":0,"part":{"type":"output_text","annotations":[],"logprobs":null,"text":""}} - -event: event -data: {"type":"response.output_item.done","sequence_number":14,"output_index":0,"item":{"type":"message","content":[],"id":"","role":"assistant","status":"completed"}} - -event: event -data: {"type":"response.completed","sequence_number":15,"response":{"created_at":123,"id":"","model":"claude-opus-4-8","object":"response","output":[],"status":"completed","usage":{"input_tokens":12,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":null},"output_tokens":8,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":20}}} - - - -{ - "input_tokens": 12, - "output_tokens": 8, - "total_tokens": 20, - "service_tier": "default", - "provider_model": "claude-opus-4-8", - "completion": [ - "" - ], - "output_messages": [ - { - "role": "assistant", - "content": [ - { - "type": "tool_call", - "id": "toolu_empty_content", - "name": "Bash", - "arguments": { - "command": "pwd" - } - } - ], - "finish_reason": "completed" - } - ] -} diff --git a/crates/persisting-gateway/tests/fixtures/response/completions/stream_tool_empty_content.json b/crates/persisting-gateway/tests/fixtures/response/completions/stream_tool_empty_content.json deleted file mode 100644 index c7bb9c752..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/completions/stream_tool_empty_content.json +++ /dev/null @@ -1,20 +0,0 @@ -data: {"id":"chatcmpl-tool-empty-content","object":"chat.completion.chunk","created":1772656915,"model":"claude-opus-4-8","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null}],"usage":null} - -data: {"id":"chatcmpl-tool-empty-content","object":"chat.completion.chunk","created":1772656915,"model":"claude-opus-4-8","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"id":"toolu_empty_content","type":"function","function":{"name":"Bash"}}]},"finish_reason":null}],"usage":null} - -data: {"id":"chatcmpl-tool-empty-content","object":"chat.completion.chunk","created":1772656915,"model":"claude-opus-4-8","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":""},"finish_reason":null}],"usage":null} - -data: {"id":"chatcmpl-tool-empty-content","object":"chat.completion.chunk","created":1772656915,"model":"claude-opus-4-8","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"com"}}]},"finish_reason":null}],"usage":null} - -data: {"id":"chatcmpl-tool-empty-content","object":"chat.completion.chunk","created":1772656915,"model":"claude-opus-4-8","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":""},"finish_reason":null}],"usage":null} - -data: {"id":"chatcmpl-tool-empty-content","object":"chat.completion.chunk","created":1772656915,"model":"claude-opus-4-8","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"mand\":\"pwd\"}"}}]},"finish_reason":null}],"usage":null} - -data: {"id":"chatcmpl-tool-empty-content","object":"chat.completion.chunk","created":1772656915,"model":"claude-opus-4-8","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":""},"finish_reason":null}],"usage":null} - -data: {"id":"chatcmpl-tool-empty-content","object":"chat.completion.chunk","created":1772656915,"model":"claude-opus-4-8","service_tier":"default","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":null},"finish_reason":"tool_calls"}],"usage":null} - -data: {"id":"chatcmpl-tool-empty-content","object":"chat.completion.chunk","created":1772656915,"model":"claude-opus-4-8","service_tier":"default","system_fingerprint":null,"choices":[],"usage":{"prompt_tokens":12,"completion_tokens":8,"total_tokens":20}} - -data: [DONE] - diff --git a/crates/persisting-gateway/tests/fixtures/response/completions/tool_call.completions-completions.snap b/crates/persisting-gateway/tests/fixtures/response/completions/tool_call.completions-completions.snap deleted file mode 100644 index f8b22e922..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/completions/tool_call.completions-completions.snap +++ /dev/null @@ -1,124 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/completions/tool_call.json -info: - id: chatcmpl-tool-call-test - object: chat.completion - created: 1772656915 - model: gpt-4-turbo - choices: - - index: 0 - message: - role: assistant - content: ~ - tool_calls: - - id: call_abc123 - type: function - function: - name: get_weather - arguments: "{\"location\": \"San Francisco\", \"unit\": \"celsius\"}" - - id: call_xyz789 - type: function - function: - name: search_web - arguments: "{\"query\": \"Claude AI\"}" - finish_reason: tool_calls - usage: - prompt_tokens: 25 - completion_tokens: 18 - total_tokens: 43 - prompt_tokens_details: - cached_tokens: 0 - audio_tokens: 0 - completion_tokens_details: - reasoning_tokens: 0 - audio_tokens: 0 - service_tier: default - system_fingerprint: ~ ---- -{ - "response": { - "model": "gpt-4-turbo", - "service_tier": "default", - "usage": { - "prompt_tokens": 25, - "completion_tokens": 18, - "total_tokens": 43, - "completion_tokens_details": { - "reasoning_tokens": 0, - "audio_tokens": 0 - }, - "prompt_tokens_details": { - "cached_tokens": 0, - "audio_tokens": 0 - } - }, - "choices": [ - { - "message": { - "role": "assistant", - "tool_calls": [ - { - "id": "call_abc123", - "type": "function", - "function": { - "name": "get_weather", - "arguments": "{\"location\": \"San Francisco\", \"unit\": \"celsius\"}" - } - }, - { - "id": "call_xyz789", - "type": "function", - "function": { - "name": "search_web", - "arguments": "{\"query\": \"Claude AI\"}" - } - } - ] - }, - "index": 0, - "finish_reason": "tool_calls" - } - ], - "id": "[id]", - "object": "chat.completion", - "created": "[date]", - "system_fingerprint": null - }, - "parsed": { - "input_tokens": 25, - "input_audio_tokens": 0, - "output_tokens": 18, - "output_audio_tokens": 0, - "total_tokens": 43, - "reasoning_tokens": 0, - "cached_input_tokens": 0, - "service_tier": "default", - "provider_model": "gpt-4-turbo", - "output_messages": [ - { - "role": "assistant", - "content": [ - { - "type": "tool_call", - "id": "call_abc123", - "name": "get_weather", - "arguments": { - "location": "San Francisco", - "unit": "celsius" - } - }, - { - "type": "tool_call", - "id": "call_xyz789", - "name": "search_web", - "arguments": { - "query": "Claude AI" - } - } - ], - "finish_reason": "tool_calls" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/completions/tool_call.completions-detect.snap b/crates/persisting-gateway/tests/fixtures/response/completions/tool_call.completions-detect.snap deleted file mode 100644 index 466159a97..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/completions/tool_call.completions-detect.snap +++ /dev/null @@ -1,100 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/completions/tool_call.json -info: - id: chatcmpl-tool-call-test - object: chat.completion - created: 1772656915 - model: gpt-4-turbo - choices: - - index: 0 - message: - role: assistant - content: ~ - tool_calls: - - id: call_abc123 - type: function - function: - name: get_weather - arguments: "{\"location\": \"San Francisco\", \"unit\": \"celsius\"}" - - id: call_xyz789 - type: function - function: - name: search_web - arguments: "{\"query\": \"Claude AI\"}" - finish_reason: tool_calls - usage: - prompt_tokens: 25 - completion_tokens: 18 - total_tokens: 43 - prompt_tokens_details: - cached_tokens: 0 - audio_tokens: 0 - completion_tokens_details: - reasoning_tokens: 0 - audio_tokens: 0 - service_tier: default - system_fingerprint: ~ ---- -{ - "response": { - "id": "[id]", - "object": "chat.completion", - "created": "[date]", - "model": "gpt-4-turbo", - "choices": [ - { - "index": 0, - "message": { - "role": "assistant", - "content": null, - "tool_calls": [ - { - "id": "call_abc123", - "type": "function", - "function": { - "name": "get_weather", - "arguments": "{\"location\": \"San Francisco\", \"unit\": \"celsius\"}" - } - }, - { - "id": "call_xyz789", - "type": "function", - "function": { - "name": "search_web", - "arguments": "{\"query\": \"Claude AI\"}" - } - } - ] - }, - "finish_reason": "tool_calls" - } - ], - "usage": { - "prompt_tokens": 25, - "completion_tokens": 18, - "total_tokens": 43, - "prompt_tokens_details": { - "cached_tokens": 0, - "audio_tokens": 0 - }, - "completion_tokens_details": { - "reasoning_tokens": 0, - "audio_tokens": 0 - } - }, - "service_tier": "default", - "system_fingerprint": null - }, - "parsed": { - "input_tokens": 25, - "input_audio_tokens": 0, - "output_tokens": 18, - "output_audio_tokens": 0, - "total_tokens": 43, - "reasoning_tokens": 0, - "cached_input_tokens": 0, - "service_tier": "default", - "provider_model": "gpt-4-turbo" - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/completions/tool_call.completions-messages.snap b/crates/persisting-gateway/tests/fixtures/response/completions/tool_call.completions-messages.snap deleted file mode 100644 index f8ce29b70..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/completions/tool_call.completions-messages.snap +++ /dev/null @@ -1,108 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/completions/tool_call.json -info: - id: chatcmpl-tool-call-test - object: chat.completion - created: 1772656915 - model: gpt-4-turbo - choices: - - index: 0 - message: - role: assistant - content: ~ - tool_calls: - - id: call_abc123 - type: function - function: - name: get_weather - arguments: "{\"location\": \"San Francisco\", \"unit\": \"celsius\"}" - - id: call_xyz789 - type: function - function: - name: search_web - arguments: "{\"query\": \"Claude AI\"}" - finish_reason: tool_calls - usage: - prompt_tokens: 25 - completion_tokens: 18 - total_tokens: 43 - prompt_tokens_details: - cached_tokens: 0 - audio_tokens: 0 - completion_tokens_details: - reasoning_tokens: 0 - audio_tokens: 0 - service_tier: default - system_fingerprint: ~ ---- -{ - "response": { - "id": "[id]", - "type": "message", - "role": "assistant", - "content": [ - { - "type": "tool_use", - "id": "call_abc123", - "name": "get_weather", - "input": { - "location": "San Francisco", - "unit": "celsius" - } - }, - { - "type": "tool_use", - "id": "call_xyz789", - "name": "search_web", - "input": { - "query": "Claude AI" - } - } - ], - "model": "gpt-4-turbo", - "stop_reason": "tool_use", - "stop_sequence": null, - "usage": { - "input_tokens": 25, - "output_tokens": 18, - "cache_read_input_tokens": 0, - "service_tier": "default" - } - }, - "parsed": { - "input_tokens": 25, - "input_audio_tokens": 0, - "output_tokens": 18, - "output_audio_tokens": 0, - "total_tokens": 43, - "cached_input_tokens": 0, - "service_tier": "default", - "provider_model": "gpt-4-turbo", - "output_messages": [ - { - "role": "assistant", - "content": [ - { - "type": "tool_call", - "id": "call_abc123", - "name": "get_weather", - "arguments": { - "location": "San Francisco", - "unit": "celsius" - } - }, - { - "type": "tool_call", - "id": "call_xyz789", - "name": "search_web", - "arguments": { - "query": "Claude AI" - } - } - ], - "finish_reason": "tool_use" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/completions/tool_call.completions-responses.snap b/crates/persisting-gateway/tests/fixtures/response/completions/tool_call.completions-responses.snap deleted file mode 100644 index b3498fde6..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/completions/tool_call.completions-responses.snap +++ /dev/null @@ -1,109 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/completions/tool_call.json -info: - id: chatcmpl-tool-call-test - object: chat.completion - created: 1772656915 - model: gpt-4-turbo - choices: - - index: 0 - message: - role: assistant - content: ~ - tool_calls: - - id: call_abc123 - type: function - function: - name: get_weather - arguments: "{\"location\": \"San Francisco\", \"unit\": \"celsius\"}" - - id: call_xyz789 - type: function - function: - name: search_web - arguments: "{\"query\": \"Claude AI\"}" - finish_reason: tool_calls - usage: - prompt_tokens: 25 - completion_tokens: 18 - total_tokens: 43 - prompt_tokens_details: - cached_tokens: 0 - audio_tokens: 0 - completion_tokens_details: - reasoning_tokens: 0 - audio_tokens: 0 - service_tier: default - system_fingerprint: ~ ---- -{ - "response": { - "id": "[id]", - "status": "completed", - "output": [ - { - "type": "function_call", - "arguments": "{\"location\": \"San Francisco\", \"unit\": \"celsius\"}", - "call_id": "call_abc123", - "name": "get_weather", - "id": "[id]", - "status": "completed" - }, - { - "type": "function_call", - "arguments": "{\"query\": \"Claude AI\"}", - "call_id": "call_xyz789", - "name": "search_web", - "id": "[id]", - "status": "completed" - } - ], - "model": "input-model", - "usage": { - "input_tokens": 25, - "output_tokens": 18, - "input_tokens_details": { - "cached_tokens": 0 - }, - "output_tokens_details": { - "reasoning_tokens": 0 - }, - "total_tokens": 43 - }, - "created_at": "[date]", - "object": "response" - }, - "parsed": { - "input_tokens": 25, - "output_tokens": 18, - "total_tokens": 43, - "reasoning_tokens": 0, - "cached_input_tokens": 0, - "provider_model": "input-model", - "output_messages": [ - { - "role": "assistant", - "content": [ - { - "type": "tool_call", - "id": "call_abc123", - "name": "get_weather", - "arguments": { - "location": "San Francisco", - "unit": "celsius" - } - }, - { - "type": "tool_call", - "id": "call_xyz789", - "name": "search_web", - "arguments": { - "query": "Claude AI" - } - } - ], - "finish_reason": "completed" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/completions/tool_call.json b/crates/persisting-gateway/tests/fixtures/response/completions/tool_call.json deleted file mode 100644 index 87d556a22..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/completions/tool_call.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "id": "chatcmpl-tool-call-test", - "object": "chat.completion", - "created": 1772656915, - "model": "gpt-4-turbo", - "choices": [ - { - "index": 0, - "message": { - "role": "assistant", - "content": null, - "tool_calls": [ - { - "id": "call_abc123", - "type": "function", - "function": { - "name": "get_weather", - "arguments": "{\"location\": \"San Francisco\", \"unit\": \"celsius\"}" - } - }, - { - "id": "call_xyz789", - "type": "function", - "function": { - "name": "search_web", - "arguments": "{\"query\": \"Claude AI\"}" - } - } - ] - }, - "finish_reason": "tool_calls" - } - ], - "usage": { - "prompt_tokens": 25, - "completion_tokens": 18, - "total_tokens": 43, - "prompt_tokens_details": { - "cached_tokens": 0, - "audio_tokens": 0 - }, - "completion_tokens_details": { - "reasoning_tokens": 0, - "audio_tokens": 0 - } - }, - "service_tier": "default", - "system_fingerprint": null -} diff --git a/crates/persisting-gateway/tests/fixtures/response/completions/truncated_tool_call.completions-completions.snap b/crates/persisting-gateway/tests/fixtures/response/completions/truncated_tool_call.completions-completions.snap deleted file mode 100644 index b9d374e49..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/completions/truncated_tool_call.completions-completions.snap +++ /dev/null @@ -1,77 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/completions/truncated_tool_call.json -info: - id: chatcmpl-truncated-tool-call - object: chat.completion - created: 1700000000 - model: gpt-x - choices: - - index: 0 - message: - role: assistant - content: ~ - tool_calls: - - id: call_1 - type: function - function: - name: get_weather - arguments: "{\"location\": \"Par" - finish_reason: length - usage: - prompt_tokens: 20 - completion_tokens: 15 - total_tokens: 35 ---- -{ - "response": { - "model": "gpt-x", - "usage": { - "prompt_tokens": 20, - "completion_tokens": 15, - "total_tokens": 35 - }, - "choices": [ - { - "message": { - "role": "assistant", - "tool_calls": [ - { - "id": "call_1", - "type": "function", - "function": { - "name": "get_weather", - "arguments": "{\"location\": \"Par" - } - } - ] - }, - "index": 0, - "finish_reason": "length" - } - ], - "id": "[id]", - "object": "chat.completion", - "created": "[date]" - }, - "parsed": { - "input_tokens": 20, - "output_tokens": 15, - "total_tokens": 35, - "provider_model": "gpt-x", - "output_messages": [ - { - "role": "assistant", - "content": [ - { - "type": "tool_call", - "id": "call_1", - "name": "get_weather", - "arguments": "{\"location\": \"Par" - } - ], - "finish_reason": "length" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/completions/truncated_tool_call.completions-messages.snap b/crates/persisting-gateway/tests/fixtures/response/completions/truncated_tool_call.completions-messages.snap deleted file mode 100644 index 675e53f74..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/completions/truncated_tool_call.completions-messages.snap +++ /dev/null @@ -1,67 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/completions/truncated_tool_call.json -info: - id: chatcmpl-truncated-tool-call - object: chat.completion - created: 1700000000 - model: gpt-x - choices: - - index: 0 - message: - role: assistant - content: ~ - tool_calls: - - id: call_1 - type: function - function: - name: get_weather - arguments: "{\"location\": \"Par" - finish_reason: length - usage: - prompt_tokens: 20 - completion_tokens: 15 - total_tokens: 35 ---- -{ - "response": { - "id": "[id]", - "type": "message", - "role": "assistant", - "content": [ - { - "type": "tool_use", - "id": "call_1", - "name": "get_weather", - "input": "{\"location\": \"Par" - } - ], - "model": "gpt-x", - "stop_reason": "max_tokens", - "stop_sequence": null, - "usage": { - "input_tokens": 20, - "output_tokens": 15 - } - }, - "parsed": { - "input_tokens": 20, - "output_tokens": 15, - "total_tokens": 35, - "provider_model": "gpt-x", - "output_messages": [ - { - "role": "assistant", - "content": [ - { - "type": "tool_call", - "id": "call_1", - "name": "get_weather", - "arguments": "{\"location\": \"Par" - } - ], - "finish_reason": "max_tokens" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/completions/truncated_tool_call.completions-responses.snap b/crates/persisting-gateway/tests/fixtures/response/completions/truncated_tool_call.completions-responses.snap deleted file mode 100644 index 5c1680d3b..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/completions/truncated_tool_call.completions-responses.snap +++ /dev/null @@ -1,80 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/completions/truncated_tool_call.json -info: - id: chatcmpl-truncated-tool-call - object: chat.completion - created: 1700000000 - model: gpt-x - choices: - - index: 0 - message: - role: assistant - content: ~ - tool_calls: - - id: call_1 - type: function - function: - name: get_weather - arguments: "{\"location\": \"Par" - finish_reason: length - usage: - prompt_tokens: 20 - completion_tokens: 15 - total_tokens: 35 ---- -{ - "response": { - "id": "[id]", - "status": "incomplete", - "output": [ - { - "type": "function_call", - "arguments": "{\"location\": \"Par", - "call_id": "call_1", - "name": "get_weather", - "id": "[id]", - "status": "completed" - } - ], - "model": "input-model", - "usage": { - "input_tokens": 20, - "output_tokens": 15, - "input_tokens_details": { - "cached_tokens": 0 - }, - "output_tokens_details": { - "reasoning_tokens": 0 - }, - "total_tokens": 35 - }, - "created_at": "[date]", - "incomplete_details": { - "reason": "max_tokens" - }, - "object": "response" - }, - "parsed": { - "input_tokens": 20, - "output_tokens": 15, - "total_tokens": 35, - "reasoning_tokens": 0, - "cached_input_tokens": 0, - "provider_model": "input-model", - "output_messages": [ - { - "role": "assistant", - "content": [ - { - "type": "tool_call", - "id": "call_1", - "name": "get_weather", - "arguments": "{\"location\": \"Par" - } - ], - "finish_reason": "incomplete" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/completions/truncated_tool_call.json b/crates/persisting-gateway/tests/fixtures/response/completions/truncated_tool_call.json deleted file mode 100644 index 251f4a360..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/completions/truncated_tool_call.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "id": "chatcmpl-truncated-tool-call", - "object": "chat.completion", - "created": 1700000000, - "model": "gpt-x", - "choices": [ - { - "index": 0, - "message": { - "role": "assistant", - "content": null, - "tool_calls": [ - { - "id": "call_1", - "type": "function", - "function": { - "name": "get_weather", - "arguments": "{\"location\": \"Par" - } - } - ] - }, - "finish_reason": "length" - } - ], - "usage": { - "prompt_tokens": 20, - "completion_tokens": 15, - "total_tokens": 35 - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/detect/bedrock-basic.bedrock-detect-streaming.snap b/crates/persisting-gateway/tests/fixtures/response/detect/bedrock-basic.bedrock-detect-streaming.snap deleted file mode 100644 index 465761b0f..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/detect/bedrock-basic.bedrock-detect-streaming.snap +++ /dev/null @@ -1,11 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/detect/bedrock-basic.bin ---- -base64: AAAAhwAAAFLjgYMzCzpldmVudC10eXBlBwAMbWVzc2FnZVN0YXJ0DTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsicCI6ImFiY2RlZmdoaWoiLCJyb2xlIjoiYXNzaXN0YW50In10WSZKAAABkwAAAFfN15VbCzpldmVudC10eXBlBwARY29udGVudEJsb2NrRGVsdGENOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJjb250ZW50QmxvY2tJbmRleCI6MCwiZGVsdGEiOnsidGV4dCI6IlxuT25jZSB1cG9uIGEgdGltZSwgdGhlcmUgd2FzIGEgbGl0dGxlIGJveSBuYW1lZCBKYWNrIHdobyBsb3ZlZCB0byBwbGF5IHdpdGggaGlzIGZyaWVuZHMgaW4gdGhlIHBhcmsuIEhpcyBmYXZvcml0ZSBhY3Rpdml0eSB3YXMgdG8gY2xpbWIgdGhlIHRhbGxlc3QgdHJlZSBhbmQgc3dpbmcgb24gdGhlIHN0cm9uZ2VzdCBicmFuY2guIE9uZSBkYXksIEphYyJ9LCJwIjoiYWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNERUZHSElKS0xNTk9QUVJTVFVWVyJ9G4FU0gAAAfwAAABX1rV1xws6ZXZlbnQtdHlwZQcAEWNvbnRlbnRCbG9ja0RlbHRhDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiY29udGVudEJsb2NrSW5kZXgiOjAsImRlbHRhIjp7InRleHQiOiJrIGFuZCBoaXMgZnJpZW5kcyB3ZXJlIHBsYXlpbmcgd2hlbiB0aGV5IGhlYXJkIGEgbG91ZCBub2lzZSBjb21pbmcgZnJvbSB0aGUgdHJlZS4gVGhleSByYW4gb3ZlciB0byBpbnZlc3RpZ2F0ZSBhbmQgZm91bmQgYSBsYXJnZSBiaXJkIHRyYXBwZWQgaW5zaWRlLlxuXG5KYWNrIGtuZXcgdGhhdCBoZSBoYWQgdG8gaGVscCB0aGUgYmlyZCwgc28gaGUgY2xpbWJlZCB1cCB0aGUgdHJlZSBhbmQgZ2VudGx5IHJlbW92ZWQgdGhlIGJpcmQgZnJvbSB0aGUgYnJhbmNoZXMuIFRoZSBiaXJkIHdhcyBncmF0ZWZ1bCBhbmQgdGhhbmtlZCBKYWNrIGZvciBoaXMgaGVscCJ9LCJwIjoiYWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNERUZHSElKS0wifbI8DeYAAAFEAAAAV/YCNhILOmV2ZW50LXR5cGUHABFjb250ZW50QmxvY2tEZWx0YQ06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImNvbnRlbnRCbG9ja0luZGV4IjowLCJkZWx0YSI6eyJ0ZXh0IjoiLiBGcm9tIHRoYXQgZGF5IG9uLCBKYWNrIGJlY2FtZSBrbm93biBhcyB0aGUgYmlyZCB3aGlzcGVyZXIgaW4gdGhlIHBhcmsgYW5kIGV2ZXJ5b25lIGxvdmVkIHRvIGhlYXIgaGlzIHN0b3JpZXMgYWJvdXQgdGhlIGJpcmRzIGhlIGhhZCBoZWxwZWQuIn0sInAiOiJhYmNkZWZnaGlqa2xtbm9wcXJzdHV2d3h5ekFCQ0QifQu6PB4AAAC2AAAAVnit1hwLOmV2ZW50LXR5cGUHABBjb250ZW50QmxvY2tTdG9wDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiY29udGVudEJsb2NrSW5kZXgiOjAsInAiOiJhYmNkZWZnaGlqa2xtbm9wcXJzdHV2d3h5ekFCQ0RFRkdISUpLTE1OT1BRUlNUVVZXWCJ9FNaQ0wAAALUAAABRoWk5bws6ZXZlbnQtdHlwZQcAC21lc3NhZ2VTdG9wDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6QUJDREVGR0hJSktMTU5PUFFSU1RVVldYWVoiLCJzdG9wUmVhc29uIjoiZW5kX3R1cm4ifdXu6+EAAADrAAAATqtCRTALOmV2ZW50LXR5cGUHAAhtZXRhZGF0YQ06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7Im1ldHJpY3MiOnsibGF0ZW5jeU1zIjo1ODQxfSwicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6QUJDREVGR0hJSktMTU4iLCJ1c2FnZSI6eyJpbnB1dFRva2VucyI6MTUsIm91dHB1dFRva2VucyI6MTQyLCJ0b3RhbFRva2VucyI6MTU3fX2dC3OX - -{ - "input_tokens": 15, - "output_tokens": 142, - "total_tokens": 157 -} diff --git a/crates/persisting-gateway/tests/fixtures/response/detect/bedrock-basic.bedrock-detect.snap b/crates/persisting-gateway/tests/fixtures/response/detect/bedrock-basic.bedrock-detect.snap deleted file mode 100644 index 58dee65fd..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/detect/bedrock-basic.bedrock-detect.snap +++ /dev/null @@ -1,9 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/detect/bedrock-basic.bin -info: "\u0000\u0000\u0000�\u0000\u0000\u0000Rぃ3\u000b:event-type\u0007\u0000\fmessageStart\r:content-type\u0007\u0000\u0010application/json\r:message-type\u0007\u0000\u0005event{\"p\":\"abcdefghij\",\"role\":\"assistant\"}tY&J\u0000\u0000\u0001�\u0000\u0000\u0000W�ו[\u000b:event-type\u0007\u0000\u0011contentBlockDelta\r:content-type\u0007\u0000\u0010application/json\r:message-type\u0007\u0000\u0005event{\"contentBlockIndex\":0,\"delta\":{\"text\":\"\\nOnce upon a time, there was a little boy named Jack who loved to play with his friends in the park. His favorite activity was to climb the tallest tree and swing on the strongest branch. One day, Jac\"},\"p\":\"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVW\"}\u001b�T�\u0000\u0000\u0001�\u0000\u0000\u0000Wֵu�\u000b:event-type\u0007\u0000\u0011contentBlockDelta\r:content-type\u0007\u0000\u0010application/json\r:message-type\u0007\u0000\u0005event{\"contentBlockIndex\":0,\"delta\":{\"text\":\"k and his friends were playing when they heard a loud noise coming from the tree. They ran over to investigate and found a large bird trapped inside.\\n\\nJack knew that he had to help the bird, so he climbed up the tree and gently removed the bird from the branches. The bird was grateful and thanked Jack for his help\"},\"p\":\"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKL\"}�<\r�\u0000\u0000\u0001D\u0000\u0000\u0000W�\u00026\u0012\u000b:event-type\u0007\u0000\u0011contentBlockDelta\r:content-type\u0007\u0000\u0010application/json\r:message-type\u0007\u0000\u0005event{\"contentBlockIndex\":0,\"delta\":{\"text\":\". From that day on, Jack became known as the bird whisperer in the park and everyone loved to hear his stories about the birds he had helped.\"},\"p\":\"abcdefghijklmnopqrstuvwxyzABCD\"}\u000b�<\u001e\u0000\u0000\u0000�\u0000\u0000\u0000Vx��\u001c\u000b:event-type\u0007\u0000\u0010contentBlockStop\r:content-type\u0007\u0000\u0010application/json\r:message-type\u0007\u0000\u0005event{\"contentBlockIndex\":0,\"p\":\"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWX\"}\u0014֐�\u0000\u0000\u0000�\u0000\u0000\u0000Q�i9o\u000b:event-type\u0007\u0000\u000bmessageStop\r:content-type\u0007\u0000\u0010application/json\r:message-type\u0007\u0000\u0005event{\"p\":\"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ\",\"stopReason\":\"end_turn\"}����\u0000\u0000\u0000�\u0000\u0000\u0000N�BE0\u000b:event-type\u0007\u0000\bmetadata\r:content-type\u0007\u0000\u0010application/json\r:message-type\u0007\u0000\u0005event{\"metrics\":{\"latencyMs\":5841},\"p\":\"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMN\",\"usage\":{\"inputTokens\":15,\"outputTokens\":142,\"totalTokens\":157}}�\u000bs�" ---- -{ - "response": "\u0000\u0000\u0000�\u0000\u0000\u0000Rぃ3\u000b:event-type\u0007\u0000\fmessageStart\r:content-type\u0007\u0000\u0010application/json\r:message-type\u0007\u0000\u0005event{\"p\":\"abcdefghij\",\"role\":\"assistant\"}tY&J\u0000\u0000\u0001�\u0000\u0000\u0000W�ו[\u000b:event-type\u0007\u0000\u0011contentBlockDelta\r:content-type\u0007\u0000\u0010application/json\r:message-type\u0007\u0000\u0005event{\"contentBlockIndex\":0,\"delta\":{\"text\":\"\\nOnce upon a time, there was a little boy named Jack who loved to play with his friends in the park. His favorite activity was to climb the tallest tree and swing on the strongest branch. One day, Jac\"},\"p\":\"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVW\"}\u001b�T�\u0000\u0000\u0001�\u0000\u0000\u0000Wֵu�\u000b:event-type\u0007\u0000\u0011contentBlockDelta\r:content-type\u0007\u0000\u0010application/json\r:message-type\u0007\u0000\u0005event{\"contentBlockIndex\":0,\"delta\":{\"text\":\"k and his friends were playing when they heard a loud noise coming from the tree. They ran over to investigate and found a large bird trapped inside.\\n\\nJack knew that he had to help the bird, so he climbed up the tree and gently removed the bird from the branches. The bird was grateful and thanked Jack for his help\"},\"p\":\"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKL\"}�<\r�\u0000\u0000\u0001D\u0000\u0000\u0000W�\u00026\u0012\u000b:event-type\u0007\u0000\u0011contentBlockDelta\r:content-type\u0007\u0000\u0010application/json\r:message-type\u0007\u0000\u0005event{\"contentBlockIndex\":0,\"delta\":{\"text\":\". From that day on, Jack became known as the bird whisperer in the park and everyone loved to hear his stories about the birds he had helped.\"},\"p\":\"abcdefghijklmnopqrstuvwxyzABCD\"}\u000b�<\u001e\u0000\u0000\u0000�\u0000\u0000\u0000Vx��\u001c\u000b:event-type\u0007\u0000\u0010contentBlockStop\r:content-type\u0007\u0000\u0010application/json\r:message-type\u0007\u0000\u0005event{\"contentBlockIndex\":0,\"p\":\"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWX\"}\u0014֐�\u0000\u0000\u0000�\u0000\u0000\u0000Q�i9o\u000b:event-type\u0007\u0000\u000bmessageStop\r:content-type\u0007\u0000\u0010application/json\r:message-type\u0007\u0000\u0005event{\"p\":\"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ\",\"stopReason\":\"end_turn\"}����\u0000\u0000\u0000�\u0000\u0000\u0000N�BE0\u000b:event-type\u0007\u0000\bmetadata\r:content-type\u0007\u0000\u0010application/json\r:message-type\u0007\u0000\u0005event{\"metrics\":{\"latencyMs\":5841},\"p\":\"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMN\",\"usage\":{\"inputTokens\":15,\"outputTokens\":142,\"totalTokens\":157}}�\u000bs�", - "parsed": {} -} diff --git a/crates/persisting-gateway/tests/fixtures/response/detect/bedrock-basic.bin b/crates/persisting-gateway/tests/fixtures/response/detect/bedrock-basic.bin deleted file mode 100644 index 88991f9db..000000000 Binary files a/crates/persisting-gateway/tests/fixtures/response/detect/bedrock-basic.bin and /dev/null differ diff --git a/crates/persisting-gateway/tests/fixtures/response/detect/bedrock-broken.bedrock-detect-streaming.snap b/crates/persisting-gateway/tests/fixtures/response/detect/bedrock-broken.bedrock-detect-streaming.snap deleted file mode 100644 index c6f3a2e36..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/detect/bedrock-broken.bedrock-detect-streaming.snap +++ /dev/null @@ -1,7 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/detect/bedrock-broken.bin ---- -base64: AAAA77+9AAAAUuOBgzMLOmV2ZW50LXR5cGUHAAxtZXNzYWdlU3RhcnQNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlsbW5vcHFyc3R1dnd4eXpBQkNERUZHSElKS0xNTiIsInVzYWdlIjp7ImlucHV0VG9rZW5zIjoxNSwib3V0cHV0VG9rZW5zIjoxNDIsInRvdGFsVG9rZW5zIjoxNTd9fe+/vQtz77+9 - -{} diff --git a/crates/persisting-gateway/tests/fixtures/response/detect/bedrock-broken.bedrock-detect.snap b/crates/persisting-gateway/tests/fixtures/response/detect/bedrock-broken.bedrock-detect.snap deleted file mode 100644 index 6739377c1..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/detect/bedrock-broken.bedrock-detect.snap +++ /dev/null @@ -1,9 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/detect/bedrock-broken.bin -info: "\u0000\u0000\u0000�\u0000\u0000\u0000Rぃ3\u000b:event-type\u0007\u0000\fmessageStart\r:content-type\u0007\u0000\u0010application/json\r:message-tylmnopqrstuvwxyzABCDEFGHIJKLMN\",\"usage\":{\"inputTokens\":15,\"outputTokens\":142,\"totalTokens\":157}}�\u000bs�" ---- -{ - "response": "\u0000\u0000\u0000�\u0000\u0000\u0000Rぃ3\u000b:event-type\u0007\u0000\fmessageStart\r:content-type\u0007\u0000\u0010application/json\r:message-tylmnopqrstuvwxyzABCDEFGHIJKLMN\",\"usage\":{\"inputTokens\":15,\"outputTokens\":142,\"totalTokens\":157}}�\u000bs�", - "parsed": {} -} diff --git a/crates/persisting-gateway/tests/fixtures/response/detect/bedrock-broken.bin b/crates/persisting-gateway/tests/fixtures/response/detect/bedrock-broken.bin deleted file mode 100644 index 5a37e02db..000000000 Binary files a/crates/persisting-gateway/tests/fixtures/response/detect/bedrock-broken.bin and /dev/null differ diff --git a/crates/persisting-gateway/tests/fixtures/response/detect/bedrock-invoke.bedrock-detect-streaming.snap b/crates/persisting-gateway/tests/fixtures/response/detect/bedrock-invoke.bedrock-detect-streaming.snap deleted file mode 100644 index c047a762f..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/detect/bedrock-invoke.bedrock-detect-streaming.snap +++ /dev/null @@ -1,12 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/detect/bedrock-invoke.bin ---- -base64: AAAAqAAAAEvEe5MmCzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SnRaWE56WVdkbFUzUmhjblFpT25zaWNtOXNaU0k2SW1GemMybHpkR0Z1ZENKOWZRPT0iLCJwIjoiYWJjZGVmIn3MQ7GMAAAA9QAAAEsE+JhcCzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SmpiMjUwWlc1MFFteHZZMnRFWld4MFlTSTZleUprWld4MFlTSTZleUowWlhoMElqb2lTR1ZzYkc4aElFaHZkeUJqWVc0Z1NTQm9aV3h3SW4wc0ltTnZiblJsYm5SQ2JHOWphMGx1WkdWNElqb3dmWDA9IiwicCI6ImFiY2RlZmdoaWprbG1ub3BxcnMiffFHRW8AAADHAAAAS98Zc7oLOmV2ZW50LXR5cGUHAAVjaHVuaw06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImJ5dGVzIjoiZXlKamIyNTBaVzUwUW14dlkydFRkRzl3SWpwN0ltTnZiblJsYm5SQ2JHOWphMGx1WkdWNElqb3dmWDA9IiwicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6QUJDIn0wmTQPAAAA8AAAAEvMGBcsCzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SmpiMjUwWlc1MFFteHZZMnRFWld4MFlTSTZleUprWld4MFlTSTZleUowWlhoMElqb2lJSGx2ZFNCMGIyUmhlVDhpZlN3aVkyOXVkR1Z1ZEVKc2IyTnJTVzVrWlhnaU9qRjlmUT09IiwicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6In1Un/2pAAAAvAAAAEtRG6JkCzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SmpiMjUwWlc1MFFteHZZMnRUZEc5d0lqcDdJbU52Ym5SbGJuUkNiRzlqYTBsdVpHVjRJam94ZlgwPSIsInAiOiJhYmNkZWZnaGlqa2xtbm9wcXIifbbzlK4AAADRAAAASzC5EZgLOmV2ZW50LXR5cGUHAAVjaHVuaw06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImJ5dGVzIjoiZXlKamIyNTBaVzUwUW14dlkydEVaV3gwWVNJNmV5SmtaV3gwWVNJNmV5SjBaWGgwSWpvaUluMHNJbU52Ym5SbGJuUkNiRzlqYTBsdVpHVjRJam95ZlgwPSIsInAiOiJhYmNkZWZnaGlqayJ9A1k+owAAAMkAAABLYCnN2ws6ZXZlbnQtdHlwZQcABWNodW5rDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiYnl0ZXMiOiJleUpqYjI1MFpXNTBRbXh2WTJ0VGRHOXdJanA3SW1OdmJuUmxiblJDYkc5amEwbHVaR1Y0SWpveWZYMD0iLCJwIjoiYWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNERSJ9LkWMKgAAAOAAAABLrPiArgs6ZXZlbnQtdHlwZQcABWNodW5rDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiYnl0ZXMiOiJleUp0WlhOellXZGxVM1J2Y0NJNmV5SnpkRzl3VW1WaGMyOXVJam9pWlc1a1gzUjFjbTRpZlgwPSIsInAiOiJhYmNkZWZnaGlqa2xtbm9wcXJzdHV2d3h5ekFCQ0RFRkdISUpLTE1OT1BRUlNUVVZXWFlaMDEyMzQ1In34tO/RAAACPwAAAEvqlcktCzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SnRaWFJoWkdGMFlTSTZleUoxYzJGblpTSTZleUpwYm5CMWRGUnZhMlZ1Y3lJNk5EY3NJbTkxZEhCMWRGUnZhMlZ1Y3lJNk1USXNJbU5oWTJobFVtVmhaRWx1Y0hWMFZHOXJaVzVEYjNWdWRDSTZNQ3dpWTJGamFHVlhjbWwwWlVsdWNIVjBWRzlyWlc1RGIzVnVkQ0k2TUgwc0ltMWxkSEpwWTNNaU9udDlMQ0owY21GalpTSTZlMzE5TENKaGJXRjZiMjR0WW1Wa2NtOWpheTFwYm5adlkyRjBhVzl1VFdWMGNtbGpjeUk2ZXlKcGJuQjFkRlJ2YTJWdVEyOTFiblFpT2pRM0xDSnZkWFJ3ZFhSVWIydGxia052ZFc1MElqb3hNaXdpYVc1MmIyTmhkR2x2Ymt4aGRHVnVZM2tpT2pReE5pd2labWx5YzNSQ2VYUmxUR0YwWlc1amVTSTZNekF3TENKallXTm9aVkpsWVdSSmJuQjFkRlJ2YTJWdVEyOTFiblFpT2pBc0ltTmhZMmhsVjNKcGRHVkpibkIxZEZSdmEyVnVRMjkxYm5RaU9qQjlmUT09IiwicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHkifVASiX4= - -{ - "input_tokens": 47, - "output_tokens": 12, - "total_tokens": 59, - "cached_input_tokens": 0 -} diff --git a/crates/persisting-gateway/tests/fixtures/response/detect/bedrock-invoke.bedrock-detect.snap b/crates/persisting-gateway/tests/fixtures/response/detect/bedrock-invoke.bedrock-detect.snap deleted file mode 100644 index 496fb802d..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/detect/bedrock-invoke.bedrock-detect.snap +++ /dev/null @@ -1,9 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/detect/bedrock-invoke.bin -info: "\u0000\u0000\u0000�\u0000\u0000\u0000K�{�&\u000b:event-type\u0007\u0000\u0005chunk\r:content-type\u0007\u0000\u0010application/json\r:message-type\u0007\u0000\u0005event{\"bytes\":\"eyJtZXNzYWdlU3RhcnQiOnsicm9sZSI6ImFzc2lzdGFudCJ9fQ==\",\"p\":\"abcdef\"}�C��\u0000\u0000\u0000�\u0000\u0000\u0000K\u0004��\\\u000b:event-type\u0007\u0000\u0005chunk\r:content-type\u0007\u0000\u0010application/json\r:message-type\u0007\u0000\u0005event{\"bytes\":\"eyJjb250ZW50QmxvY2tEZWx0YSI6eyJkZWx0YSI6eyJ0ZXh0IjoiSGVsbG8hIEhvdyBjYW4gSSBoZWxwIn0sImNvbnRlbnRCbG9ja0luZGV4IjowfX0=\",\"p\":\"abcdefghijklmnopqrs\"}�GEo\u0000\u0000\u0000�\u0000\u0000\u0000K�\u0019s�\u000b:event-type\u0007\u0000\u0005chunk\r:content-type\u0007\u0000\u0010application/json\r:message-type\u0007\u0000\u0005event{\"bytes\":\"eyJjb250ZW50QmxvY2tTdG9wIjp7ImNvbnRlbnRCbG9ja0luZGV4IjowfX0=\",\"p\":\"abcdefghijklmnopqrstuvwxyzABC\"}0�4\u000f\u0000\u0000\u0000�\u0000\u0000\u0000K�\u0018\u0017,\u000b:event-type\u0007\u0000\u0005chunk\r:content-type\u0007\u0000\u0010application/json\r:message-type\u0007\u0000\u0005event{\"bytes\":\"eyJjb250ZW50QmxvY2tEZWx0YSI6eyJkZWx0YSI6eyJ0ZXh0IjoiIHlvdSB0b2RheT8ifSwiY29udGVudEJsb2NrSW5kZXgiOjF9fQ==\",\"p\":\"abcdefghijklmnopqrstuvwxyz\"}T���\u0000\u0000\u0000�\u0000\u0000\u0000KQ\u001b�d\u000b:event-type\u0007\u0000\u0005chunk\r:content-type\u0007\u0000\u0010application/json\r:message-type\u0007\u0000\u0005event{\"bytes\":\"eyJjb250ZW50QmxvY2tTdG9wIjp7ImNvbnRlbnRCbG9ja0luZGV4IjoxfX0=\",\"p\":\"abcdefghijklmnopqr\"}��\u0000\u0000\u0000�\u0000\u0000\u0000K0�\u0011�\u000b:event-type\u0007\u0000\u0005chunk\r:content-type\u0007\u0000\u0010application/json\r:message-type\u0007\u0000\u0005event{\"bytes\":\"eyJjb250ZW50QmxvY2tEZWx0YSI6eyJkZWx0YSI6eyJ0ZXh0IjoiIn0sImNvbnRlbnRCbG9ja0luZGV4IjoyfX0=\",\"p\":\"abcdefghijk\"}\u0003Y>�\u0000\u0000\u0000�\u0000\u0000\u0000K`)��\u000b:event-type\u0007\u0000\u0005chunk\r:content-type\u0007\u0000\u0010application/json\r:message-type\u0007\u0000\u0005event{\"bytes\":\"eyJjb250ZW50QmxvY2tTdG9wIjp7ImNvbnRlbnRCbG9ja0luZGV4IjoyfX0=\",\"p\":\"abcdefghijklmnopqrstuvwxyzABCDE\"}.E�*\u0000\u0000\u0000�\u0000\u0000\u0000K����\u000b:event-type\u0007\u0000\u0005chunk\r:content-type\u0007\u0000\u0010application/json\r:message-type\u0007\u0000\u0005event{\"bytes\":\"eyJtZXNzYWdlU3RvcCI6eyJzdG9wUmVhc29uIjoiZW5kX3R1cm4ifX0=\",\"p\":\"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ012345\"}����\u0000\u0000\u0002?\u0000\u0000\u0000K��-\u000b:event-type\u0007\u0000\u0005chunk\r:content-type\u0007\u0000\u0010application/json\r:message-type\u0007\u0000\u0005event{\"bytes\":\"eyJtZXRhZGF0YSI6eyJ1c2FnZSI6eyJpbnB1dFRva2VucyI6NDcsIm91dHB1dFRva2VucyI6MTIsImNhY2hlUmVhZElucHV0VG9rZW5Db3VudCI6MCwiY2FjaGVXcml0ZUlucHV0VG9rZW5Db3VudCI6MH0sIm1ldHJpY3MiOnt9LCJ0cmFjZSI6e319LCJhbWF6b24tYmVkcm9jay1pbnZvY2F0aW9uTWV0cmljcyI6eyJpbnB1dFRva2VuQ291bnQiOjQ3LCJvdXRwdXRUb2tlbkNvdW50IjoxMiwiaW52b2NhdGlvbkxhdGVuY3kiOjQxNiwiZmlyc3RCeXRlTGF0ZW5jeSI6MzAwLCJjYWNoZVJlYWRJbnB1dFRva2VuQ291bnQiOjAsImNhY2hlV3JpdGVJbnB1dFRva2VuQ291bnQiOjB9fQ==\",\"p\":\"abcdefghijklmnopqrstuvwxy\"}P\u0012�~" ---- -{ - "response": "\u0000\u0000\u0000�\u0000\u0000\u0000K�{�&\u000b:event-type\u0007\u0000\u0005chunk\r:content-type\u0007\u0000\u0010application/json\r:message-type\u0007\u0000\u0005event{\"bytes\":\"eyJtZXNzYWdlU3RhcnQiOnsicm9sZSI6ImFzc2lzdGFudCJ9fQ==\",\"p\":\"abcdef\"}�C��\u0000\u0000\u0000�\u0000\u0000\u0000K\u0004��\\\u000b:event-type\u0007\u0000\u0005chunk\r:content-type\u0007\u0000\u0010application/json\r:message-type\u0007\u0000\u0005event{\"bytes\":\"eyJjb250ZW50QmxvY2tEZWx0YSI6eyJkZWx0YSI6eyJ0ZXh0IjoiSGVsbG8hIEhvdyBjYW4gSSBoZWxwIn0sImNvbnRlbnRCbG9ja0luZGV4IjowfX0=\",\"p\":\"abcdefghijklmnopqrs\"}�GEo\u0000\u0000\u0000�\u0000\u0000\u0000K�\u0019s�\u000b:event-type\u0007\u0000\u0005chunk\r:content-type\u0007\u0000\u0010application/json\r:message-type\u0007\u0000\u0005event{\"bytes\":\"eyJjb250ZW50QmxvY2tTdG9wIjp7ImNvbnRlbnRCbG9ja0luZGV4IjowfX0=\",\"p\":\"abcdefghijklmnopqrstuvwxyzABC\"}0�4\u000f\u0000\u0000\u0000�\u0000\u0000\u0000K�\u0018\u0017,\u000b:event-type\u0007\u0000\u0005chunk\r:content-type\u0007\u0000\u0010application/json\r:message-type\u0007\u0000\u0005event{\"bytes\":\"eyJjb250ZW50QmxvY2tEZWx0YSI6eyJkZWx0YSI6eyJ0ZXh0IjoiIHlvdSB0b2RheT8ifSwiY29udGVudEJsb2NrSW5kZXgiOjF9fQ==\",\"p\":\"abcdefghijklmnopqrstuvwxyz\"}T���\u0000\u0000\u0000�\u0000\u0000\u0000KQ\u001b�d\u000b:event-type\u0007\u0000\u0005chunk\r:content-type\u0007\u0000\u0010application/json\r:message-type\u0007\u0000\u0005event{\"bytes\":\"eyJjb250ZW50QmxvY2tTdG9wIjp7ImNvbnRlbnRCbG9ja0luZGV4IjoxfX0=\",\"p\":\"abcdefghijklmnopqr\"}��\u0000\u0000\u0000�\u0000\u0000\u0000K0�\u0011�\u000b:event-type\u0007\u0000\u0005chunk\r:content-type\u0007\u0000\u0010application/json\r:message-type\u0007\u0000\u0005event{\"bytes\":\"eyJjb250ZW50QmxvY2tEZWx0YSI6eyJkZWx0YSI6eyJ0ZXh0IjoiIn0sImNvbnRlbnRCbG9ja0luZGV4IjoyfX0=\",\"p\":\"abcdefghijk\"}\u0003Y>�\u0000\u0000\u0000�\u0000\u0000\u0000K`)��\u000b:event-type\u0007\u0000\u0005chunk\r:content-type\u0007\u0000\u0010application/json\r:message-type\u0007\u0000\u0005event{\"bytes\":\"eyJjb250ZW50QmxvY2tTdG9wIjp7ImNvbnRlbnRCbG9ja0luZGV4IjoyfX0=\",\"p\":\"abcdefghijklmnopqrstuvwxyzABCDE\"}.E�*\u0000\u0000\u0000�\u0000\u0000\u0000K����\u000b:event-type\u0007\u0000\u0005chunk\r:content-type\u0007\u0000\u0010application/json\r:message-type\u0007\u0000\u0005event{\"bytes\":\"eyJtZXNzYWdlU3RvcCI6eyJzdG9wUmVhc29uIjoiZW5kX3R1cm4ifX0=\",\"p\":\"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ012345\"}����\u0000\u0000\u0002?\u0000\u0000\u0000K��-\u000b:event-type\u0007\u0000\u0005chunk\r:content-type\u0007\u0000\u0010application/json\r:message-type\u0007\u0000\u0005event{\"bytes\":\"eyJtZXRhZGF0YSI6eyJ1c2FnZSI6eyJpbnB1dFRva2VucyI6NDcsIm91dHB1dFRva2VucyI6MTIsImNhY2hlUmVhZElucHV0VG9rZW5Db3VudCI6MCwiY2FjaGVXcml0ZUlucHV0VG9rZW5Db3VudCI6MH0sIm1ldHJpY3MiOnt9LCJ0cmFjZSI6e319LCJhbWF6b24tYmVkcm9jay1pbnZvY2F0aW9uTWV0cmljcyI6eyJpbnB1dFRva2VuQ291bnQiOjQ3LCJvdXRwdXRUb2tlbkNvdW50IjoxMiwiaW52b2NhdGlvbkxhdGVuY3kiOjQxNiwiZmlyc3RCeXRlTGF0ZW5jeSI6MzAwLCJjYWNoZVJlYWRJbnB1dFRva2VuQ291bnQiOjAsImNhY2hlV3JpdGVJbnB1dFRva2VuQ291bnQiOjB9fQ==\",\"p\":\"abcdefghijklmnopqrstuvwxy\"}P\u0012�~", - "parsed": {} -} diff --git a/crates/persisting-gateway/tests/fixtures/response/detect/bedrock-invoke.bin b/crates/persisting-gateway/tests/fixtures/response/detect/bedrock-invoke.bin deleted file mode 100644 index 1e98c12ce..000000000 Binary files a/crates/persisting-gateway/tests/fixtures/response/detect/bedrock-invoke.bin and /dev/null differ diff --git a/crates/persisting-gateway/tests/fixtures/response/detect/broken-sse b/crates/persisting-gateway/tests/fixtures/response/detect/broken-sse deleted file mode 100644 index 221a5cbde..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/detect/broken-sse +++ /dev/null @@ -1,5 +0,0 @@ -data: {} - -data: ha this isn't json! - -this isn't sse at all! \ No newline at end of file diff --git a/crates/persisting-gateway/tests/fixtures/response/detect/broken-sse.completions-detect-streaming.snap b/crates/persisting-gateway/tests/fixtures/response/detect/broken-sse.completions-detect-streaming.snap deleted file mode 100644 index cdf3051bc..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/detect/broken-sse.completions-detect-streaming.snap +++ /dev/null @@ -1,11 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/detect/broken-sse ---- -data: {} - -data: ha this isn't json! - -this isn't sse at all! - -{} diff --git a/crates/persisting-gateway/tests/fixtures/response/detect/broken-sse.completions-detect.snap b/crates/persisting-gateway/tests/fixtures/response/detect/broken-sse.completions-detect.snap deleted file mode 100644 index bbacec1ea..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/detect/broken-sse.completions-detect.snap +++ /dev/null @@ -1,9 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/detect/broken-sse -info: "data: {}\n\ndata: ha this isn't json!\n\nthis isn't sse at all!" ---- -{ - "response": "data: {}\n\ndata: ha this isn't json!\n\nthis isn't sse at all!", - "parsed": {} -} diff --git a/crates/persisting-gateway/tests/fixtures/response/detect/non-json b/crates/persisting-gateway/tests/fixtures/response/detect/non-json deleted file mode 100644 index 15e410283..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/detect/non-json +++ /dev/null @@ -1,3 +0,0 @@ -{ - this isn't json! -} \ No newline at end of file diff --git a/crates/persisting-gateway/tests/fixtures/response/detect/non-json.completions-detect-streaming.snap b/crates/persisting-gateway/tests/fixtures/response/detect/non-json.completions-detect-streaming.snap deleted file mode 100644 index 10cfe1c3d..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/detect/non-json.completions-detect-streaming.snap +++ /dev/null @@ -1,9 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/detect/non-json ---- -{ - this isn't json! -} - -{} diff --git a/crates/persisting-gateway/tests/fixtures/response/detect/non-json.completions-detect.snap b/crates/persisting-gateway/tests/fixtures/response/detect/non-json.completions-detect.snap deleted file mode 100644 index c5b3c6152..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/detect/non-json.completions-detect.snap +++ /dev/null @@ -1,9 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/detect/non-json -info: "{\n this isn't json!\n}" ---- -{ - "response": "{\n this isn't json!\n}", - "parsed": {} -} diff --git a/crates/persisting-gateway/tests/fixtures/response/detect/stream-image-generation b/crates/persisting-gateway/tests/fixtures/response/detect/stream-image-generation deleted file mode 100644 index a839b4fdd..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/detect/stream-image-generation +++ /dev/null @@ -1,3 +0,0 @@ -event: image_generation.completed -data: {"created_at":1774968276,"type":"image_generation.completed","b64_json":"i","background":"opaque","output_format":"png","quality":"high","sequence_number":0,"size":"1024x1024","usage":{"input_tokens":12,"input_tokens_details":{"image_tokens":0,"text_tokens":12},"output_tokens":4493,"output_tokens_details":{"image_tokens":4160,"text_tokens":333},"total_tokens":4505}} - diff --git a/crates/persisting-gateway/tests/fixtures/response/detect/stream-image-generation.completions-detect-streaming.snap b/crates/persisting-gateway/tests/fixtures/response/detect/stream-image-generation.completions-detect-streaming.snap deleted file mode 100644 index 957176f67..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/detect/stream-image-generation.completions-detect-streaming.snap +++ /dev/null @@ -1,18 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/detect/stream-image-generation ---- -event: image_generation.completed -data: {"created_at":123,"type":"image_generation.completed","b64_json":"i","background":"opaque","output_format":"png","quality":"high","sequence_number":0,"size":"1024x1024","usage":{"input_tokens":12,"input_tokens_details":{"image_tokens":0,"text_tokens":12},"output_tokens":4493,"output_tokens_details":{"image_tokens":4160,"text_tokens":333},"total_tokens":4505}} - - - -{ - "input_tokens": 12, - "input_image_tokens": 0, - "input_text_tokens": 12, - "output_tokens": 4493, - "output_image_tokens": 4160, - "output_text_tokens": 333, - "total_tokens": 4505 -} diff --git a/crates/persisting-gateway/tests/fixtures/response/detect/stream-image-generation.completions-detect.snap b/crates/persisting-gateway/tests/fixtures/response/detect/stream-image-generation.completions-detect.snap deleted file mode 100644 index 383140b84..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/detect/stream-image-generation.completions-detect.snap +++ /dev/null @@ -1,9 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/detect/stream-image-generation -info: "event: image_generation.completed\ndata: {\"created_at\":1774968276,\"type\":\"image_generation.completed\",\"b64_json\":\"i\",\"background\":\"opaque\",\"output_format\":\"png\",\"quality\":\"high\",\"sequence_number\":0,\"size\":\"1024x1024\",\"usage\":{\"input_tokens\":12,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":12},\"output_tokens\":4493,\"output_tokens_details\":{\"image_tokens\":4160,\"text_tokens\":333},\"total_tokens\":4505}}\n\n" ---- -{ - "response": "event: image_generation.completed\ndata: {\"created_at\":1774968276,\"type\":\"image_generation.completed\",\"b64_json\":\"i\",\"background\":\"opaque\",\"output_format\":\"png\",\"quality\":\"high\",\"sequence_number\":0,\"size\":\"1024x1024\",\"usage\":{\"input_tokens\":12,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":12},\"output_tokens\":4493,\"output_tokens_details\":{\"image_tokens\":4160,\"text_tokens\":333},\"total_tokens\":4505}}\n\n", - "parsed": {} -} diff --git a/crates/persisting-gateway/tests/fixtures/response/openai/embeddings.json b/crates/persisting-gateway/tests/fixtures/response/openai/embeddings.json deleted file mode 100644 index 41217c686..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/openai/embeddings.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "object": "list", - "data": [ - { - "object": "embedding", - "index": 0, - "embedding": [ - -0.029852388, - 0.00078823726, - -0.08126111 - ] - } - ], - "model": "text-embedding-3-small", - "usage": { - "prompt_tokens": 8, - "total_tokens": 8 - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/openai/embeddings.openai.snap b/crates/persisting-gateway/tests/fixtures/response/openai/embeddings.openai.snap deleted file mode 100644 index 1f0e73ff6..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/openai/embeddings.openai.snap +++ /dev/null @@ -1,43 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/openai/embeddings.json -info: - object: list - data: - - object: embedding - index: 0 - embedding: - - -0.029852388 - - 0.00078823726 - - -0.08126111 - model: text-embedding-3-small - usage: - prompt_tokens: 8 - total_tokens: 8 ---- -{ - "response": { - "object": "list", - "model": "text-embedding-3-small", - "usage": { - "prompt_tokens": 8, - "total_tokens": 8 - }, - "data": [ - { - "object": "embedding", - "index": 0, - "embedding": [ - -0.029852388, - 0.00078823726, - -0.08126111 - ] - } - ] - }, - "parsed": { - "input_tokens": 8, - "total_tokens": 8, - "provider_model": "text-embedding-3-small" - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/openai/gemini-embeddings.json b/crates/persisting-gateway/tests/fixtures/response/openai/gemini-embeddings.json deleted file mode 100644 index 424f1ac05..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/openai/gemini-embeddings.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "object": "list", - "data": [ - { - "object": "embedding", - "embedding": [ - -0.024183841422200203, - -0.0059304218739271164 - ] - } - ], - "model": "gemini-embedding-001" -} diff --git a/crates/persisting-gateway/tests/fixtures/response/openai/gemini-embeddings.openai.snap b/crates/persisting-gateway/tests/fixtures/response/openai/gemini-embeddings.openai.snap deleted file mode 100644 index e244cc71b..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/openai/gemini-embeddings.openai.snap +++ /dev/null @@ -1,31 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/openai/gemini-embeddings.json -info: - object: list - data: - - object: embedding - embedding: - - -0.024183841422200203 - - -0.005930421873927116 - model: gemini-embedding-001 ---- -{ - "response": { - "object": "list", - "model": "gemini-embedding-001", - "usage": null, - "data": [ - { - "object": "embedding", - "embedding": [ - -0.024183841422200203, - -0.005930421873927116 - ] - } - ] - }, - "parsed": { - "provider_model": "gemini-embedding-001" - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/responses/basic.json b/crates/persisting-gateway/tests/fixtures/response/responses/basic.json deleted file mode 100644 index a6982e630..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/responses/basic.json +++ /dev/null @@ -1,71 +0,0 @@ -{ - "id": "resp_0fd43e1b46691ead0069a89a2144808199a06ce29faf5798da", - "object": "response", - "created_at": 1772657185, - "status": "completed", - "background": false, - "billing": { - "payer": "developer" - }, - "completed_at": 1772657185, - "error": null, - "frequency_penalty": 0.0, - "incomplete_details": null, - "instructions": null, - "max_output_tokens": 1024, - "max_tool_calls": null, - "model": "gpt-4o-2024-08-06", - "output": [ - { - "id": "msg_0fd43e1b46691ead0069a89a21ea1c8199ae1962065e1d3995", - "type": "message", - "status": "completed", - "content": [ - { - "type": "output_text", - "annotations": [], - "logprobs": [], - "text": "Hello! How can I assist you today?" - } - ], - "role": "assistant" - } - ], - "parallel_tool_calls": true, - "presence_penalty": 0.0, - "previous_response_id": null, - "prompt_cache_key": null, - "prompt_cache_retention": null, - "reasoning": { - "effort": null, - "summary": null - }, - "safety_identifier": null, - "service_tier": "default", - "store": true, - "temperature": 1.0, - "text": { - "format": { - "type": "text" - }, - "verbosity": "medium" - }, - "tool_choice": "auto", - "tools": [], - "top_logprobs": 0, - "top_p": 1.0, - "truncation": "disabled", - "usage": { - "input_tokens": 10, - "input_tokens_details": { - "cached_tokens": 0 - }, - "output_tokens": 10, - "output_tokens_details": { - "reasoning_tokens": 0 - }, - "total_tokens": 20 - }, - "user": null, - "metadata": {} -} \ No newline at end of file diff --git a/crates/persisting-gateway/tests/fixtures/response/responses/basic.responses-detect.snap b/crates/persisting-gateway/tests/fixtures/response/responses/basic.responses-detect.snap deleted file mode 100644 index fbdab902a..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/responses/basic.responses-detect.snap +++ /dev/null @@ -1,143 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/responses/basic.json -info: - id: resp_0fd43e1b46691ead0069a89a2144808199a06ce29faf5798da - object: response - created_at: 1772657185 - status: completed - background: false - billing: - payer: developer - completed_at: 1772657185 - error: ~ - frequency_penalty: 0 - incomplete_details: ~ - instructions: ~ - max_output_tokens: 1024 - max_tool_calls: ~ - model: gpt-4o-2024-08-06 - output: - - id: msg_0fd43e1b46691ead0069a89a21ea1c8199ae1962065e1d3995 - type: message - status: completed - content: - - type: output_text - annotations: [] - logprobs: [] - text: Hello! How can I assist you today? - role: assistant - parallel_tool_calls: true - presence_penalty: 0 - previous_response_id: ~ - prompt_cache_key: ~ - prompt_cache_retention: ~ - reasoning: - effort: ~ - summary: ~ - safety_identifier: ~ - service_tier: default - store: true - temperature: 1 - text: - format: - type: text - verbosity: medium - tool_choice: auto - tools: [] - top_logprobs: 0 - top_p: 1 - truncation: disabled - usage: - input_tokens: 10 - input_tokens_details: - cached_tokens: 0 - output_tokens: 10 - output_tokens_details: - reasoning_tokens: 0 - total_tokens: 20 - user: ~ - metadata: {} ---- -{ - "response": { - "id": "[id]", - "object": "response", - "created_at": "[date]", - "status": "completed", - "background": false, - "billing": { - "payer": "developer" - }, - "completed_at": 1772657185, - "error": null, - "frequency_penalty": 0.0, - "incomplete_details": null, - "instructions": null, - "max_output_tokens": 1024, - "max_tool_calls": null, - "model": "gpt-4o-2024-08-06", - "output": [ - { - "id": "[id]", - "type": "message", - "status": "completed", - "content": [ - { - "type": "output_text", - "annotations": [], - "logprobs": [], - "text": "Hello! How can I assist you today?" - } - ], - "role": "assistant" - } - ], - "parallel_tool_calls": true, - "presence_penalty": 0.0, - "previous_response_id": null, - "prompt_cache_key": null, - "prompt_cache_retention": null, - "reasoning": { - "effort": null, - "summary": null - }, - "safety_identifier": null, - "service_tier": "default", - "store": true, - "temperature": 1.0, - "text": { - "format": { - "type": "text" - }, - "verbosity": "medium" - }, - "tool_choice": "auto", - "tools": [], - "top_logprobs": 0, - "top_p": 1.0, - "truncation": "disabled", - "usage": { - "input_tokens": 10, - "input_tokens_details": { - "cached_tokens": 0 - }, - "output_tokens": 10, - "output_tokens_details": { - "reasoning_tokens": 0 - }, - "total_tokens": 20 - }, - "user": null, - "metadata": {} - }, - "parsed": { - "input_tokens": 10, - "output_tokens": 10, - "total_tokens": 20, - "reasoning_tokens": 0, - "cached_input_tokens": 0, - "service_tier": "default", - "provider_model": "gpt-4o-2024-08-06" - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/responses/basic.responses-responses.snap b/crates/persisting-gateway/tests/fixtures/response/responses/basic.responses-responses.snap deleted file mode 100644 index 3a0374a70..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/responses/basic.responses-responses.snap +++ /dev/null @@ -1,143 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/responses/basic.json -info: - id: resp_0fd43e1b46691ead0069a89a2144808199a06ce29faf5798da - object: response - created_at: 1772657185 - status: completed - background: false - billing: - payer: developer - completed_at: 1772657185 - error: ~ - frequency_penalty: 0 - incomplete_details: ~ - instructions: ~ - max_output_tokens: 1024 - max_tool_calls: ~ - model: gpt-4o-2024-08-06 - output: - - id: msg_0fd43e1b46691ead0069a89a21ea1c8199ae1962065e1d3995 - type: message - status: completed - content: - - type: output_text - annotations: [] - logprobs: [] - text: Hello! How can I assist you today? - role: assistant - parallel_tool_calls: true - presence_penalty: 0 - previous_response_id: ~ - prompt_cache_key: ~ - prompt_cache_retention: ~ - reasoning: - effort: ~ - summary: ~ - safety_identifier: ~ - service_tier: default - store: true - temperature: 1 - text: - format: - type: text - verbosity: medium - tool_choice: auto - tools: [] - top_logprobs: 0 - top_p: 1 - truncation: disabled - usage: - input_tokens: 10 - input_tokens_details: - cached_tokens: 0 - output_tokens: 10 - output_tokens_details: - reasoning_tokens: 0 - total_tokens: 20 - user: ~ - metadata: {} ---- -{ - "response": { - "id": "[id]", - "status": "completed", - "output": [ - { - "type": "message", - "content": [ - { - "type": "output_text", - "annotations": [], - "logprobs": [], - "text": "Hello! How can I assist you today?" - } - ], - "id": "[id]", - "role": "assistant", - "status": "completed" - } - ], - "model": "gpt-4o-2024-08-06", - "service_tier": "default", - "usage": { - "input_tokens": 10, - "output_tokens": 10, - "input_tokens_details": { - "cached_tokens": 0 - }, - "output_tokens_details": { - "reasoning_tokens": 0 - }, - "total_tokens": 20 - }, - "object": "response", - "created_at": "[date]", - "background": false, - "billing": { - "payer": "developer" - }, - "completed_at": 1772657185, - "error": null, - "frequency_penalty": 0.0, - "incomplete_details": null, - "instructions": null, - "max_output_tokens": 1024, - "max_tool_calls": null, - "parallel_tool_calls": true, - "presence_penalty": 0.0, - "previous_response_id": null, - "prompt_cache_key": null, - "prompt_cache_retention": null, - "reasoning": { - "effort": null, - "summary": null - }, - "safety_identifier": null, - "store": true, - "temperature": 1.0, - "text": { - "format": { - "type": "text" - }, - "verbosity": "medium" - }, - "tool_choice": "auto", - "tools": [], - "top_logprobs": 0, - "top_p": 1.0, - "truncation": "disabled", - "user": null, - "metadata": {} - }, - "parsed": { - "input_tokens": 10, - "output_tokens": 10, - "total_tokens": 20, - "reasoning_tokens": 0, - "cached_input_tokens": 0, - "service_tier": "default", - "provider_model": "gpt-4o-2024-08-06" - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/responses/custom-tool.json b/crates/persisting-gateway/tests/fixtures/response/responses/custom-tool.json deleted file mode 100644 index 3f891f52c..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/responses/custom-tool.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "id": "resp_123", - "status": "completed", - "model": "gpt-5.6-luna", - "output": [ - { - "type": "custom_tool_call", - "id": "ctc_123", - "call_id": "call_123", - "name": "exec", - "input": "{\"cmd\":\"echo\"}" - } - ], - "usage": { - "input_tokens": 10, - "input_tokens_details": { - "cached_tokens": 4 - }, - "output_tokens": 5, - "output_tokens_details": { - "reasoning_tokens": 0 - }, - "total_tokens": 15 - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/responses/custom-tool.responses-responses.snap b/crates/persisting-gateway/tests/fixtures/response/responses/custom-tool.responses-responses.snap deleted file mode 100644 index a3c04e524..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/responses/custom-tool.responses-responses.snap +++ /dev/null @@ -1,73 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/responses/custom-tool.json -info: - id: resp_123 - status: completed - model: gpt-5.6-luna - output: - - type: custom_tool_call - id: ctc_123 - call_id: call_123 - name: exec - input: "{\"cmd\":\"echo\"}" - usage: - input_tokens: 10 - input_tokens_details: - cached_tokens: 4 - output_tokens: 5 - output_tokens_details: - reasoning_tokens: 0 - total_tokens: 15 ---- -{ - "response": { - "id": "[id]", - "status": "completed", - "output": [ - { - "type": "custom_tool_call", - "call_id": "call_123", - "input": "{\"cmd\":\"echo\"}", - "name": "exec", - "id": "[id]" - } - ], - "model": "gpt-5.6-luna", - "usage": { - "input_tokens": 10, - "output_tokens": 5, - "input_tokens_details": { - "cached_tokens": 4 - }, - "output_tokens_details": { - "reasoning_tokens": 0 - }, - "total_tokens": 15 - } - }, - "parsed": { - "input_tokens": 10, - "output_tokens": 5, - "total_tokens": 15, - "reasoning_tokens": 0, - "cached_input_tokens": 4, - "provider_model": "gpt-5.6-luna", - "output_messages": [ - { - "role": "assistant", - "content": [ - { - "type": "tool_call", - "id": "call_123", - "name": "exec", - "arguments": { - "cmd": "echo" - } - } - ], - "finish_reason": "completed" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/responses/stream-custom-tool.json b/crates/persisting-gateway/tests/fixtures/response/responses/stream-custom-tool.json deleted file mode 100644 index 8b84ea9df..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/responses/stream-custom-tool.json +++ /dev/null @@ -1,23 +0,0 @@ -event: response.created -data: {"type":"response.created","sequence_number":0,"response":{"created_at":123,"id":"resp_123","model":"gpt-5.6-luna","object":"response","output":[],"status":"in_progress"}} - -event: response.output_item.added -data: {"type":"response.output_item.added","sequence_number":1,"output_index":0,"item":{"type":"custom_tool_call","id":"ctc_123","call_id":"call_123","name":"exec","input":""}} - -event: response.custom_tool_call_input.delta -data: {"type":"response.custom_tool_call_input.delta","sequence_number":2,"output_index":0,"item_id":"ctc_123","delta":"const r = await tools.exec_command({cmd:\"echo\"}); "} - -event: response.custom_tool_call_input.delta -data: {"type":"response.custom_tool_call_input.delta","sequence_number":3,"output_index":0,"item_id":"ctc_123","delta":"text(r.output);"} - -event: response.custom_tool_call_input.done -data: {"type":"response.custom_tool_call_input.done","sequence_number":4,"output_index":0,"item_id":"ctc_123","input":"const r = await tools.exec_command({cmd:\"echo\"}); text(r.output);"} - -event: response.output_item.done -data: {"type":"response.output_item.done","sequence_number":5,"output_index":0,"item":{"type":"custom_tool_call","id":"ctc_123","call_id":"call_123","name":"exec","input":"const r = await tools.exec_command({cmd:\"echo\"}); text(r.output);"}} - -event: response.completed -data: {"type":"response.completed","sequence_number":6,"response":{"created_at":123,"id":"resp_123","model":"gpt-5.6-luna","object":"response","output":[{"type":"custom_tool_call","id":"ctc_123","call_id":"call_123","name":"exec","input":"const r = await tools.exec_command({cmd:\"echo\"}); text(r.output);"}],"status":"completed","usage":{"input_tokens":10,"input_tokens_details":{"cached_tokens":4},"output_tokens":5,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":15}}} - -data: [DONE] - diff --git a/crates/persisting-gateway/tests/fixtures/response/responses/stream-custom-tool.responses-responses-streaming.snap b/crates/persisting-gateway/tests/fixtures/response/responses/stream-custom-tool.responses-responses-streaming.snap deleted file mode 100644 index f7c02a6b3..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/responses/stream-custom-tool.responses-responses-streaming.snap +++ /dev/null @@ -1,54 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/responses/stream-custom-tool.json ---- -event: response.created -data: {"type":"response.created","sequence_number":0,"response":{"created_at":123,"id":"","model":"gpt-5.6-luna","object":"response","output":[],"status":"in_progress"}} - -event: response.output_item.added -data: {"type":"response.output_item.added","sequence_number":1,"output_index":0,"item":{"type":"custom_tool_call","id":"ctc_123","call_id":"call_xxx","name":"exec","input":""}} - -event: response.custom_tool_call_input.delta -data: {"type":"response.custom_tool_call_input.delta","sequence_number":2,"output_index":0,"item_id":"ctc_123","delta":"const r = await tools.exec_command({cmd:\"echo\"}); "} - -event: response.custom_tool_call_input.delta -data: {"type":"response.custom_tool_call_input.delta","sequence_number":3,"output_index":0,"item_id":"ctc_123","delta":"text(r.output);"} - -event: response.custom_tool_call_input.done -data: {"type":"response.custom_tool_call_input.done","sequence_number":4,"output_index":0,"item_id":"ctc_123","input":"const r = await tools.exec_command({cmd:\"echo\"}); text(r.output);"} - -event: response.output_item.done -data: {"type":"response.output_item.done","sequence_number":5,"output_index":0,"item":{"type":"custom_tool_call","id":"ctc_123","call_id":"call_xxx","name":"exec","input":"const r = await tools.exec_command({cmd:\"echo\"}); text(r.output);"}} - -event: response.completed -data: {"type":"response.completed","sequence_number":6,"response":{"created_at":123,"id":"","model":"gpt-5.6-luna","object":"response","output":[{"type":"custom_tool_call","id":"ctc_123","call_id":"call_xxx","name":"exec","input":"const r = await tools.exec_command({cmd:\"echo\"}); text(r.output);"}],"status":"completed","usage":{"input_tokens":10,"input_tokens_details":{"cached_tokens":4},"output_tokens":5,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":15}}} - -data: [DONE] - - - -{ - "input_tokens": 10, - "output_tokens": 5, - "total_tokens": 15, - "reasoning_tokens": 0, - "cached_input_tokens": 4, - "provider_model": "gpt-5.6-luna", - "completion": [ - "" - ], - "output_messages": [ - { - "role": "assistant", - "content": [ - { - "type": "tool_call", - "id": "call_123", - "name": "exec", - "arguments": "const r = await tools.exec_command({cmd:\"echo\"}); text(r.output);" - } - ], - "finish_reason": "completed" - } - ] -} diff --git a/crates/persisting-gateway/tests/fixtures/response/responses/stream-image.json b/crates/persisting-gateway/tests/fixtures/response/responses/stream-image.json deleted file mode 100644 index 5c6140be9..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/responses/stream-image.json +++ /dev/null @@ -1,48 +0,0 @@ -event: response.created -data: {"type":"response.created","response":{"id":"resp_062bc30e2ee47b4c0069cbe41e31188190aff11955555346a5","object":"response","created_at":1774969886,"status":"in_progress","background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5-2025-08-07","output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":null,"reasoning":{"effort":"medium","summary":null},"safety_identifier":null,"service_tier":"auto","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[{"type":"image_generation","background":"auto","model":"gpt-image-1","moderation":"auto","n":1,"output_compression":100,"output_format":"png","quality":"auto","size":"auto"}],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":0} - -event: response.in_progress -data: {"type":"response.in_progress","response":{"id":"resp_062bc30e2ee47b4c0069cbe41e31188190aff11955555346a5","object":"response","created_at":1774969886,"status":"in_progress","background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5-2025-08-07","output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":null,"reasoning":{"effort":"medium","summary":null},"safety_identifier":null,"service_tier":"auto","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[{"type":"image_generation","background":"auto","model":"gpt-image-1","moderation":"auto","n":1,"output_compression":100,"output_format":"png","quality":"auto","size":"auto"}],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":1} - -event: response.output_item.added -data: {"type":"response.output_item.added","item":{"id":"rs_062bc30e2ee47b4c0069cbe41ee4348190a0a693a17fd5c3bb","type":"reasoning","summary":[]},"output_index":0,"sequence_number":2} - -event: response.output_item.done -data: {"type":"response.output_item.done","item":{"id":"rs_062bc30e2ee47b4c0069cbe41ee4348190a0a693a17fd5c3bb","type":"reasoning","summary":[]},"output_index":0,"sequence_number":3} - -event: response.output_item.added -data: {"type":"response.output_item.added","item":{"id":"ig_062bc30e2ee47b4c0069cbe42d60848190b958fbcd59659f71","type":"image_generation_call","status":"in_progress"},"output_index":1,"sequence_number":4} - -event: response.image_generation_call.in_progress -data: {"type":"response.image_generation_call.in_progress","item_id":"ig_062bc30e2ee47b4c0069cbe42d60848190b958fbcd59659f71","output_index":1,"sequence_number":5} - -event: response.image_generation_call.generating -data: {"type":"response.image_generation_call.generating","item_id":"ig_062bc30e2ee47b4c0069cbe42d60848190b958fbcd59659f71","output_index":1,"sequence_number":6} - -event: keepalive -data: {"type":"keepalive","sequence_number":7} - -event: response.image_generation_call.partial_image -data: {"type":"response.image_generation_call.partial_image","background":"opaque","item_id":"ig_062bc30e2ee47b4c0069cbe42d60848190b958fbcd59659f71","output_format":"png","output_index":1,"partial_image_b64":"i","partial_image_index":0,"quality":"high","sequence_number":8,"size":"1024x1536"} - -event: response.output_item.done -data: {"type":"response.output_item.done","item":{"id":"ig_062bc30e2ee47b4c0069cbe42d60848190b958fbcd59659f71","type":"image_generation_call","status":"generating","action":"generate","background":"opaque","output_format":"png","quality":"high","result":"i","revised_prompt":"Create a high-resolution, cute digital illustration of a gray tabby cat hugging an otter. The cat has classic tabby markings (M on the forehead, soft gray stripes) and a sweet, content expression. The otter is wearing a cozy orange scarf wrapped around its neck. They’re in a warm, affectionate pose with the cat’s paws around the otter. Use soft, warm lighting and a simple pastel background for a wholesome, heartwarming vibe. Clean lines, soft shading, and vibrant yet gentle colors.","size":"1024x1536"},"output_index":1,"sequence_number":9} - -event: response.output_item.added -data: {"type":"response.output_item.added","item":{"id":"msg_062bc30e2ee47b4c0069cbe4642f448190b58bbdfedbe4aa41","type":"message","status":"in_progress","content":[],"role":"assistant"},"output_index":2,"sequence_number":10} - -event: response.content_part.added -data: {"type":"response.content_part.added","content_index":0,"item_id":"msg_062bc30e2ee47b4c0069cbe4642f448190b58bbdfedbe4aa41","output_index":2,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":""},"sequence_number":11} - -event: response.output_text.done -data: {"type":"response.output_text.done","content_index":0,"item_id":"msg_062bc30e2ee47b4c0069cbe4642f448190b58bbdfedbe4aa41","logprobs":[],"output_index":2,"sequence_number":12,"text":""} - -event: response.content_part.done -data: {"type":"response.content_part.done","content_index":0,"item_id":"msg_062bc30e2ee47b4c0069cbe4642f448190b58bbdfedbe4aa41","output_index":2,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":""},"sequence_number":13} - -event: response.output_item.done -data: {"type":"response.output_item.done","item":{"id":"msg_062bc30e2ee47b4c0069cbe4642f448190b58bbdfedbe4aa41","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":""}],"role":"assistant"},"output_index":2,"sequence_number":14} - -event: response.completed -data: {"type":"response.completed","response":{"id":"resp_062bc30e2ee47b4c0069cbe41e31188190aff11955555346a5","object":"response","created_at":1774969886,"status":"completed","background":false,"completed_at":1774969956,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5-2025-08-07","output":[{"id":"rs_062bc30e2ee47b4c0069cbe41ee4348190a0a693a17fd5c3bb","type":"reasoning","summary":[]},{"id":"ig_062bc30e2ee47b4c0069cbe42d60848190b958fbcd59659f71","type":"image_generation_call","status":"generating","action":"generate","background":"opaque","output_format":"png","quality":"high","result":"iVBORw0KGgoAAAANSUhEUgAABAAAAAYACAIAAABn4K39AADryWNhQlgAAOvJanVtYgAAAB5qdW1kYzJwYQARABCAAACqADibcQNjMnBhAAAANzRqdW1iAAAAR2p1bWRjMm1hABEAEIAAAKoAOJtxA3VybjpjMnBhOjNiM2MxYjZhLThkZTctNDc4NC1hYjczLTc0MGUxYjRlZDlkMAAAAAHwanVtYgAAAClqdW1kYzJhcwARABCAAACqADibcQNjMnBhLmFzc2VydGlvbnMAAAAA/Gp1bWIAAABBanVtZGNib3IAEQAQgAAAqgA4m3ETYzJwYS5hY3Rpb25zLnYyAAAAABhjMnNoLjFj4NmU52zUvdtvhyjNngAAALNjYm9yoWdhY3Rpb25zgqNmYWN0aW9ubGMycGEuY3JlYXRlZG1zb2Z0d2FyZUFnZW50oWRuYW1lZkdQVC00b3FkaWdpdGFsU291cmNlVHlwZXhGaHR0cDovL2N2LmlwdGMub3JnL25ld3Njb2Rlcy9kaWdpdGFsc291cmNldHlwZS90cmFpbmVkQWxnb3JpdGhtaWNNZWRpYaFmYWN0aW9ubmMycGEuY29udmVydGVkAAAAw2p1bWIAAABAanVtZGNib3IAEQAQgAAAqgA4m3ETYzJwYS5oYXNoLmRhdGEAAAAAGGMyc2jUI7MAl9yOwt9D++Q+g7K7AAAAe2Nib3KlamV4Y2x1c2lvbnOBomVzdGFydBghZmxlbmd0aBk3ZmRuYW1lbmp1bWJmIG1hbmlmZXN0Y2FsZ2ZzaGEyNTZkaGFzaFggidJ65CbguXAk+1Lqu7ZkWetoCpg7tg7TShwJWbsND1pjcGFkSAAAAAAAAAAAAAAB/mp1bWIAAAAnanVtZGMyY2wAEQAQgAAAqgA4m3EDYzJwYS5jbGFpbS52MgAAAAHPY2JvcqdqaW5zdGFuY2VJRHgseG1wOmlpZDo4NGQxM2M0Yy02MmY2LTQ1ZDgtYjZjYy1m","revised_prompt":"Create a high-resolution, cute digital illustration of a gray tabby cat hugging an otter. The cat has classic tabby markings (M on the forehead, soft gray stripes) and a sweet, content expression. The otter is wearing a cozy orange scarf wrapped around its neck. They’re in a warm, affectionate pose with the cat’s paws around the otter. Use soft, warm lighting and a simple pastel background for a wholesome, heartwarming vibe. Clean lines, soft shading, and vibrant yet gentle colors.","size":"1024x1536"},{"id":"msg_062bc30e2ee47b4c0069cbe4642f448190b58bbdfedbe4aa41","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":""}],"role":"assistant"}],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":null,"reasoning":{"effort":"medium","summary":null},"safety_identifier":null,"service_tier":"default","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[{"type":"image_generation","background":"auto","model":"gpt-image-1","moderation":"auto","n":1,"output_compression":100,"output_format":"png","quality":"auto","size":"auto"}],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":{"input_tokens":1755,"input_tokens_details":{"cached_tokens":0},"output_tokens":1314,"output_tokens_details":{"reasoning_tokens":1152},"total_tokens":3069},"user":null,"metadata":{}},"sequence_number":15} - diff --git a/crates/persisting-gateway/tests/fixtures/response/responses/stream-image.responses-detect-streaming.snap b/crates/persisting-gateway/tests/fixtures/response/responses/stream-image.responses-detect-streaming.snap deleted file mode 100644 index 1d26e532a..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/responses/stream-image.responses-detect-streaming.snap +++ /dev/null @@ -1,62 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/responses/stream-image.json ---- -event: response.created -data: {"type":"response.created","response":{"id":"","object":"response","created_at":123,"status":"in_progress","background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5-2025-08-07","output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":null,"reasoning":{"effort":"medium","summary":null},"safety_identifier":null,"service_tier":"auto","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[{"type":"image_generation","background":"auto","model":"gpt-image-1","moderation":"auto","n":1,"output_compression":100,"output_format":"png","quality":"auto","size":"auto"}],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":0} - -event: response.in_progress -data: {"type":"response.in_progress","response":{"id":"","object":"response","created_at":123,"status":"in_progress","background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5-2025-08-07","output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":null,"reasoning":{"effort":"medium","summary":null},"safety_identifier":null,"service_tier":"auto","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[{"type":"image_generation","background":"auto","model":"gpt-image-1","moderation":"auto","n":1,"output_compression":100,"output_format":"png","quality":"auto","size":"auto"}],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":1} - -event: response.output_item.added -data: {"type":"response.output_item.added","item":{"id":"rs_062bc30e2ee47b4c0069cbe41ee4348190a0a693a17fd5c3bb","type":"reasoning","summary":[]},"output_index":0,"sequence_number":2} - -event: response.output_item.done -data: {"type":"response.output_item.done","item":{"id":"rs_062bc30e2ee47b4c0069cbe41ee4348190a0a693a17fd5c3bb","type":"reasoning","summary":[]},"output_index":0,"sequence_number":3} - -event: response.output_item.added -data: {"type":"response.output_item.added","item":{"id":"ig_062bc30e2ee47b4c0069cbe42d60848190b958fbcd59659f71","type":"image_generation_call","status":"in_progress"},"output_index":1,"sequence_number":4} - -event: response.image_generation_call.in_progress -data: {"type":"response.image_generation_call.in_progress","item_id":"ig_062bc30e2ee47b4c0069cbe42d60848190b958fbcd59659f71","output_index":1,"sequence_number":5} - -event: response.image_generation_call.generating -data: {"type":"response.image_generation_call.generating","item_id":"ig_062bc30e2ee47b4c0069cbe42d60848190b958fbcd59659f71","output_index":1,"sequence_number":6} - -event: keepalive -data: {"type":"keepalive","sequence_number":7} - -event: response.image_generation_call.partial_image -data: {"type":"response.image_generation_call.partial_image","background":"opaque","item_id":"ig_062bc30e2ee47b4c0069cbe42d60848190b958fbcd59659f71","output_format":"png","output_index":1,"partial_image_b64":"i","partial_image_index":0,"quality":"high","sequence_number":8,"size":"1024x1536"} - -event: response.output_item.done -data: {"type":"response.output_item.done","item":{"id":"ig_062bc30e2ee47b4c0069cbe42d60848190b958fbcd59659f71","type":"image_generation_call","status":"generating","action":"generate","background":"opaque","output_format":"png","quality":"high","result":"i","revised_prompt":"Create a high-resolution, cute digital illustration of a gray tabby cat hugging an otter. The cat has classic tabby markings (M on the forehead, soft gray stripes) and a sweet, content expression. The otter is wearing a cozy orange scarf wrapped around its neck. They’re in a warm, affectionate pose with the cat’s paws around the otter. Use soft, warm lighting and a simple pastel background for a wholesome, heartwarming vibe. Clean lines, soft shading, and vibrant yet gentle colors.","size":"1024x1536"},"output_index":1,"sequence_number":9} - -event: response.output_item.added -data: {"type":"response.output_item.added","item":{"id":"","type":"message","status":"in_progress","content":[],"role":"assistant"},"output_index":2,"sequence_number":10} - -event: response.content_part.added -data: {"type":"response.content_part.added","content_index":0,"item_id":"","output_index":2,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":""},"sequence_number":11} - -event: response.output_text.done -data: {"type":"response.output_text.done","content_index":0,"item_id":"","logprobs":[],"output_index":2,"sequence_number":12,"text":""} - -event: response.content_part.done -data: {"type":"response.content_part.done","content_index":0,"item_id":"","output_index":2,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":""},"sequence_number":13} - -event: response.output_item.done -data: {"type":"response.output_item.done","item":{"id":"","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":""}],"role":"assistant"},"output_index":2,"sequence_number":14} - -event: response.completed -data: {"type":"response.completed","response":{"id":"","object":"response","created_at":123,"status":"completed","background":false,"completed_at":1774969956,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5-2025-08-07","output":[{"id":"rs_062bc30e2ee47b4c0069cbe41ee4348190a0a693a17fd5c3bb","type":"reasoning","summary":[]},{"id":"ig_062bc30e2ee47b4c0069cbe42d60848190b958fbcd59659f71","type":"image_generation_call","status":"generating","action":"generate","background":"opaque","output_format":"png","quality":"high","result":"iVBORw0KGgoAAAANSUhEUgAABAAAAAYACAIAAABn4K39AADryWNhQlgAAOvJanVtYgAAAB5qdW1kYzJwYQARABCAAACqADibcQNjMnBhAAAANzRqdW1iAAAAR2p1bWRjMm1hABEAEIAAAKoAOJtxA3VybjpjMnBhOjNiM2MxYjZhLThkZTctNDc4NC1hYjczLTc0MGUxYjRlZDlkMAAAAAHwanVtYgAAAClqdW1kYzJhcwARABCAAACqADibcQNjMnBhLmFzc2VydGlvbnMAAAAA/Gp1bWIAAABBanVtZGNib3IAEQAQgAAAqgA4m3ETYzJwYS5hY3Rpb25zLnYyAAAAABhjMnNoLjFj4NmU52zUvdtvhyjNngAAALNjYm9yoWdhY3Rpb25zgqNmYWN0aW9ubGMycGEuY3JlYXRlZG1zb2Z0d2FyZUFnZW50oWRuYW1lZkdQVC00b3FkaWdpdGFsU291cmNlVHlwZXhGaHR0cDovL2N2LmlwdGMub3JnL25ld3Njb2Rlcy9kaWdpdGFsc291cmNldHlwZS90cmFpbmVkQWxnb3JpdGhtaWNNZWRpYaFmYWN0aW9ubmMycGEuY29udmVydGVkAAAAw2p1bWIAAABAanVtZGNib3IAEQAQgAAAqgA4m3ETYzJwYS5oYXNoLmRhdGEAAAAAGGMyc2jUI7MAl9yOwt9D++Q+g7K7AAAAe2Nib3KlamV4Y2x1c2lvbnOBomVzdGFydBghZmxlbmd0aBk3ZmRuYW1lbmp1bWJmIG1hbmlmZXN0Y2FsZ2ZzaGEyNTZkaGFzaFggidJ65CbguXAk+1Lqu7ZkWetoCpg7tg7TShwJWbsND1pjcGFkSAAAAAAAAAAAAAAB/mp1bWIAAAAnanVtZGMyY2wAEQAQgAAAqgA4m3EDYzJwYS5jbGFpbS52MgAAAAHPY2JvcqdqaW5zdGFuY2VJRHgseG1wOmlpZDo4NGQxM2M0Yy02MmY2LTQ1ZDgtYjZjYy1m","revised_prompt":"Create a high-resolution, cute digital illustration of a gray tabby cat hugging an otter. The cat has classic tabby markings (M on the forehead, soft gray stripes) and a sweet, content expression. The otter is wearing a cozy orange scarf wrapped around its neck. They’re in a warm, affectionate pose with the cat’s paws around the otter. Use soft, warm lighting and a simple pastel background for a wholesome, heartwarming vibe. Clean lines, soft shading, and vibrant yet gentle colors.","size":"1024x1536"},{"id":"","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":""}],"role":"assistant"}],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":null,"reasoning":{"effort":"medium","summary":null},"safety_identifier":null,"service_tier":"default","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[{"type":"image_generation","background":"auto","model":"gpt-image-1","moderation":"auto","n":1,"output_compression":100,"output_format":"png","quality":"auto","size":"auto"}],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":{"input_tokens":1755,"input_tokens_details":{"cached_tokens":0},"output_tokens":1314,"output_tokens_details":{"reasoning_tokens":1152},"total_tokens":3069},"user":null,"metadata":{}},"sequence_number":15} - - - -{ - "input_tokens": 1755, - "output_tokens": 1314, - "total_tokens": 3069, - "reasoning_tokens": 1152, - "cached_input_tokens": 0, - "service_tier": "default" -} diff --git a/crates/persisting-gateway/tests/fixtures/response/responses/stream-image.responses-responses-streaming.snap b/crates/persisting-gateway/tests/fixtures/response/responses/stream-image.responses-responses-streaming.snap deleted file mode 100644 index b12d743a5..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/responses/stream-image.responses-responses-streaming.snap +++ /dev/null @@ -1,66 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/responses/stream-image.json ---- -event: response.created -data: {"type":"response.created","response":{"id":"","object":"response","created_at":123,"status":"in_progress","background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5-2025-08-07","output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":null,"reasoning":{"effort":"medium","summary":null},"safety_identifier":null,"service_tier":"auto","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[{"type":"image_generation","background":"auto","model":"gpt-image-1","moderation":"auto","n":1,"output_compression":100,"output_format":"png","quality":"auto","size":"auto"}],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":0} - -event: response.in_progress -data: {"type":"response.in_progress","response":{"id":"","object":"response","created_at":123,"status":"in_progress","background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5-2025-08-07","output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":null,"reasoning":{"effort":"medium","summary":null},"safety_identifier":null,"service_tier":"auto","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[{"type":"image_generation","background":"auto","model":"gpt-image-1","moderation":"auto","n":1,"output_compression":100,"output_format":"png","quality":"auto","size":"auto"}],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":1} - -event: response.output_item.added -data: {"type":"response.output_item.added","item":{"id":"rs_062bc30e2ee47b4c0069cbe41ee4348190a0a693a17fd5c3bb","type":"reasoning","summary":[]},"output_index":0,"sequence_number":2} - -event: response.output_item.done -data: {"type":"response.output_item.done","item":{"id":"rs_062bc30e2ee47b4c0069cbe41ee4348190a0a693a17fd5c3bb","type":"reasoning","summary":[]},"output_index":0,"sequence_number":3} - -event: response.output_item.added -data: {"type":"response.output_item.added","item":{"id":"ig_062bc30e2ee47b4c0069cbe42d60848190b958fbcd59659f71","type":"image_generation_call","status":"in_progress"},"output_index":1,"sequence_number":4} - -event: response.image_generation_call.in_progress -data: {"type":"response.image_generation_call.in_progress","item_id":"ig_062bc30e2ee47b4c0069cbe42d60848190b958fbcd59659f71","output_index":1,"sequence_number":5} - -event: response.image_generation_call.generating -data: {"type":"response.image_generation_call.generating","item_id":"ig_062bc30e2ee47b4c0069cbe42d60848190b958fbcd59659f71","output_index":1,"sequence_number":6} - -event: keepalive -data: {"type":"keepalive","sequence_number":7} - -event: response.image_generation_call.partial_image -data: {"type":"response.image_generation_call.partial_image","background":"opaque","item_id":"ig_062bc30e2ee47b4c0069cbe42d60848190b958fbcd59659f71","output_format":"png","output_index":1,"partial_image_b64":"i","partial_image_index":0,"quality":"high","sequence_number":8,"size":"1024x1536"} - -event: response.output_item.done -data: {"type":"response.output_item.done","item":{"id":"ig_062bc30e2ee47b4c0069cbe42d60848190b958fbcd59659f71","type":"image_generation_call","status":"generating","action":"generate","background":"opaque","output_format":"png","quality":"high","result":"i","revised_prompt":"Create a high-resolution, cute digital illustration of a gray tabby cat hugging an otter. The cat has classic tabby markings (M on the forehead, soft gray stripes) and a sweet, content expression. The otter is wearing a cozy orange scarf wrapped around its neck. They’re in a warm, affectionate pose with the cat’s paws around the otter. Use soft, warm lighting and a simple pastel background for a wholesome, heartwarming vibe. Clean lines, soft shading, and vibrant yet gentle colors.","size":"1024x1536"},"output_index":1,"sequence_number":9} - -event: response.output_item.added -data: {"type":"response.output_item.added","item":{"id":"","type":"message","status":"in_progress","content":[],"role":"assistant"},"output_index":2,"sequence_number":10} - -event: response.content_part.added -data: {"type":"response.content_part.added","content_index":0,"item_id":"","output_index":2,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":""},"sequence_number":11} - -event: response.output_text.done -data: {"type":"response.output_text.done","content_index":0,"item_id":"","logprobs":[],"output_index":2,"sequence_number":12,"text":""} - -event: response.content_part.done -data: {"type":"response.content_part.done","content_index":0,"item_id":"","output_index":2,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":""},"sequence_number":13} - -event: response.output_item.done -data: {"type":"response.output_item.done","item":{"id":"","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":""}],"role":"assistant"},"output_index":2,"sequence_number":14} - -event: response.completed -data: {"type":"response.completed","response":{"id":"","object":"response","created_at":123,"status":"completed","background":false,"completed_at":1774969956,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5-2025-08-07","output":[{"id":"rs_062bc30e2ee47b4c0069cbe41ee4348190a0a693a17fd5c3bb","type":"reasoning","summary":[]},{"id":"ig_062bc30e2ee47b4c0069cbe42d60848190b958fbcd59659f71","type":"image_generation_call","status":"generating","action":"generate","background":"opaque","output_format":"png","quality":"high","result":"iVBORw0KGgoAAAANSUhEUgAABAAAAAYACAIAAABn4K39AADryWNhQlgAAOvJanVtYgAAAB5qdW1kYzJwYQARABCAAACqADibcQNjMnBhAAAANzRqdW1iAAAAR2p1bWRjMm1hABEAEIAAAKoAOJtxA3VybjpjMnBhOjNiM2MxYjZhLThkZTctNDc4NC1hYjczLTc0MGUxYjRlZDlkMAAAAAHwanVtYgAAAClqdW1kYzJhcwARABCAAACqADibcQNjMnBhLmFzc2VydGlvbnMAAAAA/Gp1bWIAAABBanVtZGNib3IAEQAQgAAAqgA4m3ETYzJwYS5hY3Rpb25zLnYyAAAAABhjMnNoLjFj4NmU52zUvdtvhyjNngAAALNjYm9yoWdhY3Rpb25zgqNmYWN0aW9ubGMycGEuY3JlYXRlZG1zb2Z0d2FyZUFnZW50oWRuYW1lZkdQVC00b3FkaWdpdGFsU291cmNlVHlwZXhGaHR0cDovL2N2LmlwdGMub3JnL25ld3Njb2Rlcy9kaWdpdGFsc291cmNldHlwZS90cmFpbmVkQWxnb3JpdGhtaWNNZWRpYaFmYWN0aW9ubmMycGEuY29udmVydGVkAAAAw2p1bWIAAABAanVtZGNib3IAEQAQgAAAqgA4m3ETYzJwYS5oYXNoLmRhdGEAAAAAGGMyc2jUI7MAl9yOwt9D++Q+g7K7AAAAe2Nib3KlamV4Y2x1c2lvbnOBomVzdGFydBghZmxlbmd0aBk3ZmRuYW1lbmp1bWJmIG1hbmlmZXN0Y2FsZ2ZzaGEyNTZkaGFzaFggidJ65CbguXAk+1Lqu7ZkWetoCpg7tg7TShwJWbsND1pjcGFkSAAAAAAAAAAAAAAB/mp1bWIAAAAnanVtZGMyY2wAEQAQgAAAqgA4m3EDYzJwYS5jbGFpbS52MgAAAAHPY2JvcqdqaW5zdGFuY2VJRHgseG1wOmlpZDo4NGQxM2M0Yy02MmY2LTQ1ZDgtYjZjYy1m","revised_prompt":"Create a high-resolution, cute digital illustration of a gray tabby cat hugging an otter. The cat has classic tabby markings (M on the forehead, soft gray stripes) and a sweet, content expression. The otter is wearing a cozy orange scarf wrapped around its neck. They’re in a warm, affectionate pose with the cat’s paws around the otter. Use soft, warm lighting and a simple pastel background for a wholesome, heartwarming vibe. Clean lines, soft shading, and vibrant yet gentle colors.","size":"1024x1536"},{"id":"","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":""}],"role":"assistant"}],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":null,"reasoning":{"effort":"medium","summary":null},"safety_identifier":null,"service_tier":"default","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[{"type":"image_generation","background":"auto","model":"gpt-image-1","moderation":"auto","n":1,"output_compression":100,"output_format":"png","quality":"auto","size":"auto"}],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":{"input_tokens":1755,"input_tokens_details":{"cached_tokens":0},"output_tokens":1314,"output_tokens_details":{"reasoning_tokens":1152},"total_tokens":3069},"user":null,"metadata":{}},"sequence_number":15} - - - -{ - "input_tokens": 1755, - "output_tokens": 1314, - "total_tokens": 3069, - "reasoning_tokens": 1152, - "cached_input_tokens": 0, - "service_tier": "default", - "provider_model": "gpt-5-2025-08-07", - "completion": [ - "" - ] -} diff --git a/crates/persisting-gateway/tests/fixtures/response/responses/stream.json b/crates/persisting-gateway/tests/fixtures/response/responses/stream.json deleted file mode 100644 index f3dea4cdd..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/responses/stream.json +++ /dev/null @@ -1,35 +0,0 @@ -event: response.created -data: {"type":"response.created","sequence_number":1,"response":{"created_at":123,"id":"","model":"gpt-4.1-mini","object":"response","output":[],"status":"in_progress"}} - -event: response.output_item.added -data: {"type":"response.output_item.added","sequence_number":2,"output_index":0,"item":{"type":"message","content":[],"id":"","role":"assistant","status":"in_progress"}} - -event: response.output_text.delta -data: {"type":"response.output_text.delta","sequence_number":3,"item_id":"","output_index":0,"content_index":0,"delta":"Hello"} - -event: response.output_text.done -data: {"type":"response.output_text.done","sequence_number":4,"item_id":"","output_index":0,"content_index":0,"text":"Hello"} - -event: response.output_item.done -data: {"type":"response.output_item.done","sequence_number":5,"output_index":0,"item":{"type":"message","content":[{"type":"output_text","text":"Hello"}],"id":"","role":"assistant","status":"completed"}} - -event: response.output_item.added -data: {"type":"response.output_item.added","sequence_number":6,"output_index":1,"item":{"type":"function_call","arguments":"","call_id":"call_xxx","name":"get_weather","id":"","status":"in_progress"}} - -event: response.function_call_arguments.delta -data: {"type":"response.function_call_arguments.delta","sequence_number":7,"item_id":"","output_index":1,"delta":"{\"loc"} - -event: response.function_call_arguments.delta -data: {"type":"response.function_call_arguments.delta","sequence_number":8,"item_id":"","output_index":1,"delta":"ation\":\"San Francisco\"}"} - -event: response.function_call_arguments.done -data: {"type":"response.function_call_arguments.done","name":"get_weather","sequence_number":9,"item_id":"","output_index":1,"arguments":"{\"location\":\"San Francisco\"}"} - -event: response.output_item.done -data: {"type":"response.output_item.done","sequence_number":10,"output_index":1,"item":{"type":"function_call","arguments":"{\"location\":\"San Francisco\"}","call_id":"call_xxx","name":"get_weather","id":"","status":"completed"}} - -event: response.completed -data: {"type":"response.completed","sequence_number":11,"response":{"created_at":123,"id":"","model":"gpt-4.1-mini","object":"response","output":[],"status":"completed","usage":{"input_tokens":12,"input_tokens_details":{"cached_tokens":0},"output_tokens":8,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":20}}} - -data: [DONE] - diff --git a/crates/persisting-gateway/tests/fixtures/response/responses/stream.responses-detect-streaming.snap b/crates/persisting-gateway/tests/fixtures/response/responses/stream.responses-detect-streaming.snap deleted file mode 100644 index 4cb4adb71..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/responses/stream.responses-detect-streaming.snap +++ /dev/null @@ -1,48 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/responses/stream.json ---- -event: response.created -data: {"type":"response.created","sequence_number":1,"response":{"created_at":123,"id":"","model":"gpt-4.1-mini","object":"response","output":[],"status":"in_progress"}} - -event: response.output_item.added -data: {"type":"response.output_item.added","sequence_number":2,"output_index":0,"item":{"type":"message","content":[],"id":"","role":"assistant","status":"in_progress"}} - -event: response.output_text.delta -data: {"type":"response.output_text.delta","sequence_number":3,"item_id":"","output_index":0,"content_index":0,"delta":"Hello"} - -event: response.output_text.done -data: {"type":"response.output_text.done","sequence_number":4,"item_id":"","output_index":0,"content_index":0,"text":"Hello"} - -event: response.output_item.done -data: {"type":"response.output_item.done","sequence_number":5,"output_index":0,"item":{"type":"message","content":[{"type":"output_text","text":"Hello"}],"id":"","role":"assistant","status":"completed"}} - -event: response.output_item.added -data: {"type":"response.output_item.added","sequence_number":6,"output_index":1,"item":{"type":"function_call","arguments":"","call_id":"call_xxx","name":"get_weather","id":"","status":"in_progress"}} - -event: response.function_call_arguments.delta -data: {"type":"response.function_call_arguments.delta","sequence_number":7,"item_id":"","output_index":1,"delta":"{\"loc"} - -event: response.function_call_arguments.delta -data: {"type":"response.function_call_arguments.delta","sequence_number":8,"item_id":"","output_index":1,"delta":"ation\":\"San Francisco\"}"} - -event: response.function_call_arguments.done -data: {"type":"response.function_call_arguments.done","name":"get_weather","sequence_number":9,"item_id":"","output_index":1,"arguments":"{\"location\":\"San Francisco\"}"} - -event: response.output_item.done -data: {"type":"response.output_item.done","sequence_number":10,"output_index":1,"item":{"type":"function_call","arguments":"{\"location\":\"San Francisco\"}","call_id":"call_xxx","name":"get_weather","id":"","status":"completed"}} - -event: response.completed -data: {"type":"response.completed","sequence_number":11,"response":{"created_at":123,"id":"","model":"gpt-4.1-mini","object":"response","output":[],"status":"completed","usage":{"input_tokens":12,"input_tokens_details":{"cached_tokens":0},"output_tokens":8,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":20}}} - -data: [DONE] - - - -{ - "input_tokens": 12, - "output_tokens": 8, - "total_tokens": 20, - "reasoning_tokens": 0, - "cached_input_tokens": 0 -} diff --git a/crates/persisting-gateway/tests/fixtures/response/responses/stream.responses-responses-streaming.snap b/crates/persisting-gateway/tests/fixtures/response/responses/stream.responses-responses-streaming.snap deleted file mode 100644 index a394ed85b..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/responses/stream.responses-responses-streaming.snap +++ /dev/null @@ -1,68 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/responses/stream.json ---- -event: response.created -data: {"type":"response.created","sequence_number":1,"response":{"created_at":123,"id":"","model":"gpt-4.1-mini","object":"response","output":[],"status":"in_progress"}} - -event: response.output_item.added -data: {"type":"response.output_item.added","sequence_number":2,"output_index":0,"item":{"type":"message","content":[],"id":"","role":"assistant","status":"in_progress"}} - -event: response.output_text.delta -data: {"type":"response.output_text.delta","sequence_number":3,"item_id":"","output_index":0,"content_index":0,"delta":"Hello"} - -event: response.output_text.done -data: {"type":"response.output_text.done","sequence_number":4,"item_id":"","output_index":0,"content_index":0,"text":"Hello"} - -event: response.output_item.done -data: {"type":"response.output_item.done","sequence_number":5,"output_index":0,"item":{"type":"message","content":[{"type":"output_text","text":"Hello"}],"id":"","role":"assistant","status":"completed"}} - -event: response.output_item.added -data: {"type":"response.output_item.added","sequence_number":6,"output_index":1,"item":{"type":"function_call","arguments":"","call_id":"call_xxx","name":"get_weather","id":"","status":"in_progress"}} - -event: response.function_call_arguments.delta -data: {"type":"response.function_call_arguments.delta","sequence_number":7,"item_id":"","output_index":1,"delta":"{\"loc"} - -event: response.function_call_arguments.delta -data: {"type":"response.function_call_arguments.delta","sequence_number":8,"item_id":"","output_index":1,"delta":"ation\":\"San Francisco\"}"} - -event: response.function_call_arguments.done -data: {"type":"response.function_call_arguments.done","name":"get_weather","sequence_number":9,"item_id":"","output_index":1,"arguments":"{\"location\":\"San Francisco\"}"} - -event: response.output_item.done -data: {"type":"response.output_item.done","sequence_number":10,"output_index":1,"item":{"type":"function_call","arguments":"{\"location\":\"San Francisco\"}","call_id":"call_xxx","name":"get_weather","id":"","status":"completed"}} - -event: response.completed -data: {"type":"response.completed","sequence_number":11,"response":{"created_at":123,"id":"","model":"gpt-4.1-mini","object":"response","output":[],"status":"completed","usage":{"input_tokens":12,"input_tokens_details":{"cached_tokens":0},"output_tokens":8,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":20}}} - -data: [DONE] - - - -{ - "input_tokens": 12, - "output_tokens": 8, - "total_tokens": 20, - "reasoning_tokens": 0, - "cached_input_tokens": 0, - "provider_model": "gpt-4.1-mini", - "completion": [ - "Hello" - ], - "output_messages": [ - { - "role": "assistant", - "content": [ - { - "type": "tool_call", - "id": "call_xxx", - "name": "get_weather", - "arguments": { - "location": "San Francisco" - } - } - ], - "finish_reason": "completed" - } - ] -} diff --git a/crates/persisting-gateway/tests/fixtures/response/responses/truncated_tool_call.json b/crates/persisting-gateway/tests/fixtures/response/responses/truncated_tool_call.json deleted file mode 100644 index aee3c60bf..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/responses/truncated_tool_call.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "id": "resp_truncated_tool_call", - "status": "incomplete", - "model": "gpt-x", - "output": [ - { - "type": "function_call", - "id": "fc_1", - "call_id": "call_1", - "name": "get_weather", - "arguments": "{\"location\": \"Par", - "status": "completed" - } - ] -} diff --git a/crates/persisting-gateway/tests/fixtures/response/responses/truncated_tool_call.responses-responses.snap b/crates/persisting-gateway/tests/fixtures/response/responses/truncated_tool_call.responses-responses.snap deleted file mode 100644 index 1194b1d71..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/responses/truncated_tool_call.responses-responses.snap +++ /dev/null @@ -1,49 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/responses/truncated_tool_call.json -info: - id: resp_truncated_tool_call - status: incomplete - model: gpt-x - output: - - type: function_call - id: fc_1 - call_id: call_1 - name: get_weather - arguments: "{\"location\": \"Par" - status: completed ---- -{ - "response": { - "id": "[id]", - "status": "incomplete", - "output": [ - { - "type": "function_call", - "arguments": "{\"location\": \"Par", - "call_id": "call_1", - "name": "get_weather", - "id": "[id]", - "status": "completed" - } - ], - "model": "gpt-x" - }, - "parsed": { - "provider_model": "gpt-x", - "output_messages": [ - { - "role": "assistant", - "content": [ - { - "type": "tool_call", - "id": "call_1", - "name": "get_weather", - "arguments": "{\"location\": \"Par" - } - ], - "finish_reason": "incomplete" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/vertex-gemini/basic.json b/crates/persisting-gateway/tests/fixtures/response/vertex-gemini/basic.json deleted file mode 100644 index 37ba80d40..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/vertex-gemini/basic.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "candidates": [ - { - "content": { - "role": "model", - "parts": [{ "text": "The provider is Gemini." }] - }, - "finishReason": "STOP", - "index": 0 - } - ], - "usageMetadata": { - "promptTokenCount": 8, - "candidatesTokenCount": 6, - "totalTokenCount": 14 - }, - "modelVersion": "gemini-2.5-pro", - "responseId": "resp-basic-1" -} diff --git a/crates/persisting-gateway/tests/fixtures/response/vertex-gemini/basic.vertex-gemini-completions.snap b/crates/persisting-gateway/tests/fixtures/response/vertex-gemini/basic.vertex-gemini-completions.snap deleted file mode 100644 index d2ec5279a..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/vertex-gemini/basic.vertex-gemini-completions.snap +++ /dev/null @@ -1,47 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/vertex-gemini/basic.json -info: - candidates: - - content: - role: model - parts: - - text: The provider is Gemini. - finishReason: STOP - index: 0 - usageMetadata: - promptTokenCount: 8 - candidatesTokenCount: 6 - totalTokenCount: 14 - modelVersion: gemini-2.5-pro - responseId: resp-basic-1 ---- -{ - "response": { - "model": "gemini-2.5-pro", - "usage": { - "prompt_tokens": 8, - "completion_tokens": 6, - "total_tokens": 14 - }, - "choices": [ - { - "message": { - "content": "The provider is Gemini.", - "role": "assistant" - }, - "index": 0, - "finish_reason": "stop" - } - ], - "id": "[id]", - "created": "[date]", - "object": "chat.completion" - }, - "parsed": { - "input_tokens": 8, - "output_tokens": 6, - "total_tokens": 14, - "provider_model": "gemini-2.5-pro" - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/vertex-gemini/blocked.json b/crates/persisting-gateway/tests/fixtures/response/vertex-gemini/blocked.json deleted file mode 100644 index 698630ecb..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/vertex-gemini/blocked.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "candidates": [], - "promptFeedback": { "blockReason": "SAFETY" }, - "usageMetadata": { - "promptTokenCount": 12, - "totalTokenCount": 12 - }, - "modelVersion": "gemini-2.5-pro" -} diff --git a/crates/persisting-gateway/tests/fixtures/response/vertex-gemini/blocked.vertex-gemini-completions.snap b/crates/persisting-gateway/tests/fixtures/response/vertex-gemini/blocked.vertex-gemini-completions.snap deleted file mode 100644 index 6a7fcc1ef..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/vertex-gemini/blocked.vertex-gemini-completions.snap +++ /dev/null @@ -1,41 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/vertex-gemini/blocked.json -info: - candidates: [] - promptFeedback: - blockReason: SAFETY - usageMetadata: - promptTokenCount: 12 - totalTokenCount: 12 - modelVersion: gemini-2.5-pro ---- -{ - "response": { - "model": "gemini-2.5-pro", - "usage": { - "prompt_tokens": 12, - "completion_tokens": 0, - "total_tokens": 12 - }, - "choices": [ - { - "message": { - "content": "", - "role": "assistant" - }, - "index": 0, - "finish_reason": "content_filter" - } - ], - "id": "[id]", - "created": "[date]", - "object": "chat.completion" - }, - "parsed": { - "input_tokens": 12, - "output_tokens": 0, - "total_tokens": 12, - "provider_model": "gemini-2.5-pro" - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/vertex-gemini/reasoning.json b/crates/persisting-gateway/tests/fixtures/response/vertex-gemini/reasoning.json deleted file mode 100644 index 6b946a1e0..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/vertex-gemini/reasoning.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "candidates": [ - { - "content": { - "role": "model", - "parts": [ - { "text": "Let me think about the weather.", "thought": true }, - { "text": "It is sunny." } - ] - }, - "finishReason": "STOP", - "index": 0 - } - ], - "usageMetadata": { - "promptTokenCount": 10, - "candidatesTokenCount": 12, - "thoughtsTokenCount": 5, - "totalTokenCount": 27 - }, - "modelVersion": "gemini-2.5-pro" -} diff --git a/crates/persisting-gateway/tests/fixtures/response/vertex-gemini/reasoning.vertex-gemini-completions.snap b/crates/persisting-gateway/tests/fixtures/response/vertex-gemini/reasoning.vertex-gemini-completions.snap deleted file mode 100644 index 239d8dbee..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/vertex-gemini/reasoning.vertex-gemini-completions.snap +++ /dev/null @@ -1,54 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/vertex-gemini/reasoning.json -info: - candidates: - - content: - role: model - parts: - - text: Let me think about the weather. - thought: true - - text: It is sunny. - finishReason: STOP - index: 0 - usageMetadata: - promptTokenCount: 10 - candidatesTokenCount: 12 - thoughtsTokenCount: 5 - totalTokenCount: 27 - modelVersion: gemini-2.5-pro ---- -{ - "response": { - "model": "gemini-2.5-pro", - "usage": { - "prompt_tokens": 10, - "completion_tokens": 12, - "total_tokens": 27, - "completion_tokens_details": { - "reasoning_tokens": 5 - } - }, - "choices": [ - { - "message": { - "content": "It is sunny.", - "role": "assistant", - "reasoning_content": "Let me think about the weather." - }, - "index": 0, - "finish_reason": "stop" - } - ], - "id": "[id]", - "created": "[date]", - "object": "chat.completion" - }, - "parsed": { - "input_tokens": 10, - "output_tokens": 12, - "total_tokens": 27, - "reasoning_tokens": 5, - "provider_model": "gemini-2.5-pro" - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/vertex-gemini/stream_tool.json b/crates/persisting-gateway/tests/fixtures/response/vertex-gemini/stream_tool.json deleted file mode 100644 index d2e209473..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/vertex-gemini/stream_tool.json +++ /dev/null @@ -1,2 +0,0 @@ -data: {"responseId":"gemini-tool","modelVersion":"gemini-2.5-pro","candidates":[{"content":{"role":"model","parts":[{"functionCall":{"id":"fc_native","name":"get_weather","args":{"location":"Tokyo"}}}]},"finishReason":"STOP"}],"usageMetadata":{"promptTokenCount":15,"candidatesTokenCount":12,"totalTokenCount":27}} - diff --git a/crates/persisting-gateway/tests/fixtures/response/vertex-gemini/stream_tool.vertex-gemini-completions-streaming.snap b/crates/persisting-gateway/tests/fixtures/response/vertex-gemini/stream_tool.vertex-gemini-completions-streaming.snap deleted file mode 100644 index a135338e2..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/vertex-gemini/stream_tool.vertex-gemini-completions-streaming.snap +++ /dev/null @@ -1,35 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/vertex-gemini/stream_tool.json ---- -data: {"id":"gemini-tool","choices":[{"index":0,"delta":{"content":null,"tool_calls":[{"index":0,"id":"fc_native","type":"function","function":{"name":"get_weather","arguments":"{\"location\":\"Tokyo\"}"}}],"role":"assistant"},"finish_reason":"tool_calls"}],"created":123,"model":"gemini-2.5-pro","service_tier":null,"system_fingerprint":null,"object":"chat.completion.chunk","usage":{"prompt_tokens":15,"completion_tokens":12,"total_tokens":27}} - -data: [DONE] - - - -{ - "input_tokens": 15, - "output_tokens": 12, - "total_tokens": 27, - "provider_model": "gemini-2.5-pro", - "completion": [ - "" - ], - "output_messages": [ - { - "role": "assistant", - "content": [ - { - "type": "tool_call", - "id": "fc_native", - "name": "get_weather", - "arguments": { - "location": "Tokyo" - } - } - ], - "finish_reason": "tool_calls" - } - ] -} diff --git a/crates/persisting-gateway/tests/fixtures/response/vertex-gemini/tool.json b/crates/persisting-gateway/tests/fixtures/response/vertex-gemini/tool.json deleted file mode 100644 index fac7655d6..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/vertex-gemini/tool.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "candidates": [ - { - "content": { - "role": "model", - "parts": [ - { - "functionCall": { - "name": "get_weather", - "id": "fc_get_weather_1", - "args": { "location": "Columbus" } - } - } - ] - }, - "finishReason": "STOP", - "index": 0 - } - ], - "usageMetadata": { - "promptTokenCount": 20, - "candidatesTokenCount": 8, - "totalTokenCount": 28 - }, - "modelVersion": "gemini-2.5-pro" -} diff --git a/crates/persisting-gateway/tests/fixtures/response/vertex-gemini/tool.vertex-gemini-completions.snap b/crates/persisting-gateway/tests/fixtures/response/vertex-gemini/tool.vertex-gemini-completions.snap deleted file mode 100644 index eafca42a9..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/vertex-gemini/tool.vertex-gemini-completions.snap +++ /dev/null @@ -1,75 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/vertex-gemini/tool.json -info: - candidates: - - content: - role: model - parts: - - functionCall: - name: get_weather - id: fc_get_weather_1 - args: - location: Columbus - finishReason: STOP - index: 0 - usageMetadata: - promptTokenCount: 20 - candidatesTokenCount: 8 - totalTokenCount: 28 - modelVersion: gemini-2.5-pro ---- -{ - "response": { - "model": "gemini-2.5-pro", - "usage": { - "prompt_tokens": 20, - "completion_tokens": 8, - "total_tokens": 28 - }, - "choices": [ - { - "message": { - "role": "assistant", - "tool_calls": [ - { - "type": "function", - "id": "fc_get_weather_1", - "function": { - "name": "get_weather", - "arguments": "{\"location\":\"Columbus\"}" - } - } - ] - }, - "index": 0, - "finish_reason": "tool_calls" - } - ], - "id": "[id]", - "created": "[date]", - "object": "chat.completion" - }, - "parsed": { - "input_tokens": 20, - "output_tokens": 8, - "total_tokens": 28, - "provider_model": "gemini-2.5-pro", - "output_messages": [ - { - "role": "assistant", - "content": [ - { - "type": "tool_call", - "id": "fc_get_weather_1", - "name": "get_weather", - "arguments": { - "location": "Columbus" - } - } - ], - "finish_reason": "tool_calls" - } - ] - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/vertex/embeddings.json b/crates/persisting-gateway/tests/fixtures/response/vertex/embeddings.json deleted file mode 100644 index c6ade9432..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/vertex/embeddings.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "predictions": [ - { - "embeddings": { - "values": [-0.006603240966796875,-0.054168701171875,0.0121917724609375], - "statistics": { - "token_count": 4, - "truncated": false - } - } - }, - { - "embeddings": { - "values": [0.024810791015625,0.070068359375,0.04156494140625], - "statistics": { - "token_count": 5, - "truncated": false - } - } - } - ] -} diff --git a/crates/persisting-gateway/tests/fixtures/response/vertex/embeddings.vertex.snap b/crates/persisting-gateway/tests/fixtures/response/vertex/embeddings.vertex.snap deleted file mode 100644 index fa05fd794..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/vertex/embeddings.vertex.snap +++ /dev/null @@ -1,57 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/vertex/embeddings.json -info: - predictions: - - embeddings: - values: - - -0.006603240966796875 - - -0.054168701171875 - - 0.0121917724609375 - statistics: - token_count: 4 - truncated: false - - embeddings: - values: - - 0.024810791015625 - - 0.070068359375 - - 0.04156494140625 - statistics: - token_count: 5 - truncated: false ---- -{ - "response": { - "object": "list", - "model": "text-embedding-004", - "usage": { - "prompt_tokens": 9, - "total_tokens": 9 - }, - "data": [ - { - "index": 0, - "object": "embedding", - "embedding": [ - -0.006603240966796875, - -0.054168701171875, - 0.0121917724609375 - ] - }, - { - "index": 1, - "object": "embedding", - "embedding": [ - 0.024810791015625, - 0.070068359375, - 0.04156494140625 - ] - } - ] - }, - "parsed": { - "input_tokens": 9, - "total_tokens": 9, - "provider_model": "text-embedding-004" - } -} diff --git a/crates/persisting-gateway/tests/fixtures/response/vertex/rerank-no-details.json b/crates/persisting-gateway/tests/fixtures/response/vertex/rerank-no-details.json deleted file mode 100644 index 9f3dc71ad..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/vertex/rerank-no-details.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "records": [ - { "id": "1" }, - { "id": "0" } - ] -} diff --git a/crates/persisting-gateway/tests/fixtures/response/vertex/rerank-no-details.vertex.snap b/crates/persisting-gateway/tests/fixtures/response/vertex/rerank-no-details.vertex.snap deleted file mode 100644 index 52bbb247e..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/vertex/rerank-no-details.vertex.snap +++ /dev/null @@ -1,23 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/vertex/rerank-no-details.json -info: - records: - - id: "1" - - id: "0" ---- -{ - "response": { - "results": [ - { - "index": 1, - "relevance_score": 1.0 - }, - { - "index": 0, - "relevance_score": 1.0 - } - ] - }, - "parsed": {} -} diff --git a/crates/persisting-gateway/tests/fixtures/response/vertex/rerank.json b/crates/persisting-gateway/tests/fixtures/response/vertex/rerank.json deleted file mode 100644 index 87ae40faf..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/vertex/rerank.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "records": [ - { "id": "2", "score": 0.97 }, - { "id": "0", "score": 0.30 } - ] -} diff --git a/crates/persisting-gateway/tests/fixtures/response/vertex/rerank.vertex.snap b/crates/persisting-gateway/tests/fixtures/response/vertex/rerank.vertex.snap deleted file mode 100644 index 4d11c507e..000000000 --- a/crates/persisting-gateway/tests/fixtures/response/vertex/rerank.vertex.snap +++ /dev/null @@ -1,25 +0,0 @@ ---- -source: crates/llm/src/golden_tests.rs -description: src/tests/response/vertex/rerank.json -info: - records: - - id: "2" - score: 0.97 - - id: "0" - score: 0.3 ---- -{ - "response": { - "results": [ - { - "index": 2, - "relevance_score": 0.97 - }, - { - "index": 0, - "relevance_score": 0.3 - } - ] - }, - "parsed": {} -} diff --git a/crates/persisting-gateway/tests/llm_fixtures.rs b/crates/persisting-gateway/tests/llm_fixtures.rs deleted file mode 100644 index 8943dc38a..000000000 --- a/crates/persisting-gateway/tests/llm_fixtures.rs +++ /dev/null @@ -1,121 +0,0 @@ -//! Regression tests using agentgateway LLM JSON fixtures (copied under `tests/fixtures/`). -//! -//! Source and license: `tests/fixtures/README.md`. - -use bytes::Bytes; -use persisting_gateway::config::ProxyConfig; -use persisting_gateway::conversion::{ - ProtocolBridge, StreamTranslator, completions_response_to_messages, - messages_request_to_completions, -}; -use persisting_gateway::models_list::build_models_response; -use persisting_gateway::protocol::ProtocolKind; -use persisting_gateway::usage::extract_usage_from_response; -use serde_json::Value; - -const COMPLETIONS_BASIC_REQ: &str = include_str!("fixtures/requests/completions/basic.json"); - -#[test] -fn fixture_request_parses_model() { - let v: Value = serde_json::from_str(COMPLETIONS_BASIC_REQ).unwrap(); - let model = v.get("model").and_then(|m| m.as_str()).unwrap(); - assert!(!model.is_empty()); -} - -#[test] -fn fixture_response_usage() { - let body: Value = - serde_json::from_str(include_str!("fixtures/response/completions/basic.json")).unwrap(); - let u = extract_usage_from_response(&body); - assert!(u.total_tokens > 0 || u.input_tokens > 0 || u.output_tokens > 0); -} - -#[test] -fn protocol_from_openai_paths() { - assert_eq!( - ProtocolKind::from_path("/v1/chat/completions"), - ProtocolKind::ChatCompletions - ); - assert_eq!( - ProtocolKind::from_path("/v1/messages"), - ProtocolKind::Messages - ); -} - -#[test] -fn messages_to_completions_fixture() { - let body = Bytes::from_static(include_bytes!("fixtures/requests/messages/basic.json")); - let out = messages_request_to_completions(&body, "upstream-model").unwrap(); - let v: Value = serde_json::from_slice(&out).unwrap(); - assert_eq!(v["model"], "upstream-model"); - assert_eq!(v["messages"][0]["content"][0]["text"], "Hello, world"); -} - -#[test] -fn completions_to_messages_fixture() { - let body = Bytes::from_static(include_bytes!("fixtures/response/completions/basic.json")); - let out = completions_response_to_messages(&body, "claude-test").unwrap(); - let v: Value = serde_json::from_slice(&out).unwrap(); - assert_eq!(v["type"], "message"); - assert_eq!(v["content"][0]["type"], "text"); -} - -#[test] -fn stream_translate_fixture() { - let raw = include_str!("fixtures/local/response/completions/stream_head.txt"); - let mut translator = StreamTranslator::new( - ProtocolBridge::MessagesToCompletions, - ProtocolKind::Messages, - "claude-test", - ) - .unwrap(); - let mut out = translator.push_chunk(raw.as_bytes()).unwrap().to_vec(); - out.extend_from_slice(&translator.finish_stream().unwrap()); - let out = String::from_utf8(out).unwrap(); - assert!(out.contains("message_start")); - assert!(out.contains("content_block_delta")); -} - -#[test] -fn protocol_bridge_when_no_anthropic_upstream() { - let cfg = ProxyConfig::from_toml_str( - r#" -listen = "127.0.0.1:1" - -[[models]] -name = "*" -upstream = "http://x/v1" -"#, - ) - .unwrap(); - assert_eq!( - ProtocolBridge::needed(ProtocolKind::Messages, &cfg.models[0]), - ProtocolBridge::MessagesToCompletions - ); -} - -#[test] -fn models_list_forward_target() { - let cfg = ProxyConfig::from_toml_str( - r#" -listen = "127.0.0.1:1" - -[[models]] -name = "deepseek-chat" -upstream = "http://x/v1" - -[[models]] -name = "*" -forward = "deepseek-chat" -"#, - ) - .unwrap(); - let list = build_models_response(&cfg); - let ids: Vec<_> = list["data"] - .as_array() - .unwrap() - .iter() - .map(|m| m["id"].as_str().unwrap()) - .collect(); - assert!(ids.contains(&"deepseek-chat")); -} diff --git a/crates/persisting-gateway/tests/markdown_trajectory.rs b/crates/persisting-gateway/tests/markdown_trajectory.rs deleted file mode 100644 index 6883a0fea..000000000 --- a/crates/persisting-gateway/tests/markdown_trajectory.rs +++ /dev/null @@ -1,103 +0,0 @@ -//! Capture-owned metadata and semantic live upsert tests. - -use persisting_gateway::projection::markdown_trajectory::upsert_storyline_turn; -use persisting_gateway::session::client::{ - SESSION_CLIENT_META_FILENAME, SessionClientMeta, write_session_client_meta, -}; -use persisting_pchronicle::document::decode_agenticmd; -use persisting_pchronicle::model::{StorylineDocument, StorylineTurn}; -use serde_json::json; - -fn turn(id: i64, source: &str, body: &str, draft: bool) -> StorylineTurn { - StorylineTurn { - id, - kind: Some(if draft { - "llm.response.stream".into() - } else if source == "user" { - "llm.request".into() - } else { - "llm.response".into() - }), - timestamp: None, - source: source.into(), - message: json!(body), - reasoning_content: None, - reasoning_effort: None, - tool_calls: None, - observation: None, - metrics: None, - model_name: None, - llm_call_count: (source == "agent").then_some(1), - is_copied_context: None, - latency_ms: None, - ttft_ms: None, - extra: None, - env: None, - prompt: None, - finished_at: None, - } -} - -#[test] -fn new_document_includes_session_client_metadata() { - let dir = tempfile::tempdir().unwrap(); - let session_dir = dir.path().join("demo-agent").join("sess-1"); - std::fs::create_dir_all(&session_dir).unwrap(); - write_session_client_meta( - &session_dir.join(SESSION_CLIENT_META_FILENAME), - &SessionClientMeta { - peer: "127.0.0.1:54321".into(), - peer_port: 54321, - pid: 999, - command: "claude --model deepseek".into(), - machine_fp: None, - }, - ) - .unwrap(); - - let path = session_dir.join("sess-1.md"); - let story = StorylineDocument::new("sess-1", "demo-agent"); - upsert_storyline_turn(&path, &story, "call-1", &turn(1, "agent", "hi", false)).unwrap(); - - let parsed = decode_agenticmd(&std::fs::read_to_string(&path).unwrap()).unwrap(); - assert_eq!(parsed.turns[0].message, json!("hi")); - assert_eq!( - parsed.agent.extra.as_ref().unwrap()["client"]["peer_port"], - 54321 - ); - assert_eq!( - parsed.agent.extra.as_ref().unwrap()["client"]["command"], - "claude --model deepseek" - ); -} - -#[test] -fn live_upsert_replaces_draft_by_edit_key() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("sess.md"); - let story = StorylineDocument::new("sess-1", "demo-agent"); - assert!( - !upsert_storyline_turn(&path, &story, "call-1", &turn(1, "agent", "draft", true),).unwrap() - ); - assert!( - upsert_storyline_turn( - &path, - &story, - "call-1", - &turn(1, "agent", "complete", false), - ) - .unwrap() - ); - - let parsed = decode_agenticmd(&std::fs::read_to_string(&path).unwrap()).unwrap(); - assert_eq!(parsed.turns.len(), 1); - assert_eq!(parsed.turns[0].message, json!("complete")); - assert_eq!(parsed.turns[0].kind.as_deref(), Some("llm.response")); - assert!( - parsed.turns[0] - .extra - .as_ref() - .and_then(|extra| extra.get("call_id")) - .is_none() - ); -} diff --git a/crates/persisting-gateway/tests/model_api_forwarding.rs b/crates/persisting-gateway/tests/model_api_forwarding.rs deleted file mode 100644 index e5c23c842..000000000 --- a/crates/persisting-gateway/tests/model_api_forwarding.rs +++ /dev/null @@ -1,731 +0,0 @@ -//! End-to-end request and response forwarding checks for supported model APIs. - -use std::collections::HashMap; -use std::sync::{Arc, Mutex}; - -use axum::Router; -use axum::body::{Body, to_bytes}; -use axum::extract::{Request, State}; -use axum::http::StatusCode; -use axum::response::Response; -use axum::routing::post; -use persisting_gateway::config::ProxyConfig; -use persisting_gateway::serve_with_listeners_and_shutdown; -use persisting_gateway::sink::SeqOnlySink; -use serde_json::{Value, json}; -use tokio::sync::oneshot; - -// The request/response shapes and tool-call fields follow recordings from the -// supplied Ornith-1.5-35B-A3B service; the test itself replays them locally. -const MODEL: &str = "Ornith-1.5-35B-A3B"; -const UPSTREAM_KEY: &str = "upstream-test-key"; -const CLIENT_TRACE: &str = "client-trace-42"; -const MODEL_REQUEST_ID: &str = "model-request-42"; - -#[derive(Clone, Copy, Debug)] -enum Api { - ChatCompletions, - Responses, - Messages, -} - -impl Api { - fn name(self) -> &'static str { - match self { - Self::ChatCompletions => "chat_completions", - Self::Responses => "responses", - Self::Messages => "messages", - } - } - - fn path(self) -> &'static str { - match self { - Self::ChatCompletions => "/v1/chat/completions", - Self::Responses => "/v1/responses", - Self::Messages => "/v1/messages", - } - } - - fn request(self, stream: bool, tool_call: bool) -> Value { - let mut request = match self { - Self::ChatCompletions => json!({ - "model": MODEL, - "stream": stream, - "temperature": 0.25, - "messages": [{"role": "user", "content": "chat request"}] - }), - Self::Responses => json!({ - "model": MODEL, - "stream": stream, - "instructions": "Answer briefly.", - "input": "responses request" - }), - Self::Messages => json!({ - "model": MODEL, - "stream": stream, - "max_tokens": 64, - "system": "Answer briefly.", - "messages": [{"role": "user", "content": "messages request"}] - }), - }; - if tool_call { - request["tools"] = match self { - Self::ChatCompletions => json!([{ - "type": "function", - "function": { - "name": "get_weather", - "description": "Get the weather.", - "parameters": { - "type": "object", - "properties": {"city": {"type": "string"}}, - "required": ["city"] - } - } - }]), - Self::Responses => json!([{ - "type": "function", - "name": "get_weather", - "description": "Get the weather.", - "parameters": { - "type": "object", - "properties": {"city": {"type": "string"}}, - "required": ["city"] - } - }]), - Self::Messages => json!([{ - "name": "get_weather", - "description": "Get the weather.", - "input_schema": { - "type": "object", - "properties": {"city": {"type": "string"}}, - "required": ["city"] - } - }]), - }; - request["tool_choice"] = match self { - Self::Messages => json!({"type": "tool", "name": "get_weather"}), - Self::ChatCompletions | Self::Responses => json!("required"), - }; - } - request - } - - fn non_streaming_response(self) -> Value { - match self { - Self::ChatCompletions => json!({ - "id": "chatcmpl-mock", - "object": "chat.completion", - "created": 0, - "model": MODEL, - "choices": [{ - "index": 0, - "message": {"role": "assistant", "content": "chat response"}, - "finish_reason": "stop" - }], - "usage": {"prompt_tokens": 2, "completion_tokens": 2, "total_tokens": 4} - }), - Self::Responses => json!({ - "id": "resp-mock", - "object": "response", - "created_at": 0, - "status": "completed", - "model": MODEL, - "output": [{ - "id": "msg-mock", - "type": "message", - "status": "completed", - "role": "assistant", - "content": [{ - "type": "output_text", - "text": "responses response", - "annotations": [] - }] - }], - "usage": {"input_tokens": 2, "output_tokens": 2, "total_tokens": 4} - }), - Self::Messages => json!({ - "id": "msg-mock", - "type": "message", - "role": "assistant", - "model": MODEL, - "content": [{"type": "text", "text": "messages response"}], - "stop_reason": "end_turn", - "stop_sequence": null, - "usage": {"input_tokens": 2, "output_tokens": 2} - }), - } - } - - fn tool_non_streaming_response(self) -> Value { - match self { - Self::ChatCompletions => json!({ - "id": "chatcmpl-tool-mock", - "object": "chat.completion", - "created": 0, - "model": MODEL, - "choices": [{ - "index": 0, - "message": { - "role": "assistant", - "content": null, - "tool_calls": [{ - "id": "call-weather-42", - "type": "function", - "function": { - "name": "get_weather", - "arguments": "{\"city\":\"Paris\"}" - } - }] - }, - "finish_reason": "tool_calls" - }], - "usage": {"prompt_tokens": 3, "completion_tokens": 4, "total_tokens": 7} - }), - Self::Responses => json!({ - "id": "resp-tool-mock", - "object": "response", - "created_at": 0, - "status": "completed", - "model": MODEL, - "output": [{ - "id": "fc-item-42", - "type": "function_call", - "status": "completed", - "call_id": "call-weather-42", - "name": "get_weather", - "arguments": "{\"city\":\"Paris\"}" - }], - "usage": {"input_tokens": 3, "output_tokens": 4, "total_tokens": 7} - }), - Self::Messages => json!({ - "id": "msg-tool-mock", - "type": "message", - "role": "assistant", - "model": MODEL, - "content": [{ - "type": "tool_use", - "id": "call-weather-42", - "name": "get_weather", - "input": {"city": "Paris"} - }], - "stop_reason": "tool_use", - "stop_sequence": null, - "usage": {"input_tokens": 3, "output_tokens": 4} - }), - } - } - - fn streaming_response(self) -> String { - match self { - Self::ChatCompletions => concat!( - "data: {\"id\":\"chatcmpl-mock\",\"object\":\"chat.completion.chunk\",\"created\":0,\"model\":\"Ornith-1.5-35B-A3B\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"chat stream\"},\"finish_reason\":null}]}\n\n", - "data: {\"id\":\"chatcmpl-mock\",\"object\":\"chat.completion.chunk\",\"created\":0,\"model\":\"Ornith-1.5-35B-A3B\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\n", - "data: [DONE]\n\n" - ) - .to_string(), - Self::Responses => concat!( - "event: response.created\n", - "data: {\"type\":\"response.created\",\"sequence_number\":0,\"response\":{\"id\":\"resp-mock\",\"object\":\"response\",\"created_at\":0,\"status\":\"in_progress\",\"model\":\"Ornith-1.5-35B-A3B\",\"output\":[]}}\n\n", - "event: response.output_text.delta\n", - "data: {\"type\":\"response.output_text.delta\",\"sequence_number\":1,\"item_id\":\"msg-mock\",\"output_index\":0,\"content_index\":0,\"delta\":\"responses stream\",\"logprobs\":[]}\n\n", - "event: response.completed\n", - "data: {\"type\":\"response.completed\",\"sequence_number\":2,\"response\":{\"id\":\"resp-mock\",\"object\":\"response\",\"created_at\":0,\"status\":\"completed\",\"model\":\"Ornith-1.5-35B-A3B\",\"output\":[{\"id\":\"msg-mock\",\"type\":\"message\",\"status\":\"completed\",\"role\":\"assistant\",\"content\":[{\"type\":\"output_text\",\"text\":\"responses stream\",\"annotations\":[]}]}]}}\n\n" - ) - .to_string(), - Self::Messages => concat!( - "event: message_start\n", - "data: {\"type\":\"message_start\",\"message\":{\"id\":\"msg-mock\",\"type\":\"message\",\"role\":\"assistant\",\"model\":\"Ornith-1.5-35B-A3B\",\"content\":[],\"stop_reason\":null,\"stop_sequence\":null,\"usage\":{\"input_tokens\":2,\"output_tokens\":0}}}\n\n", - "event: content_block_start\n", - "data: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"text\",\"text\":\"\"}}\n\n", - "event: content_block_delta\n", - "data: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"messages stream\"}}\n\n", - "event: content_block_stop\n", - "data: {\"type\":\"content_block_stop\",\"index\":0}\n\n", - "event: message_delta\n", - "data: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\",\"stop_sequence\":null},\"usage\":{\"output_tokens\":2}}\n\n", - "event: message_stop\n", - "data: {\"type\":\"message_stop\"}\n\n" - ) - .to_string(), - } - } - - fn tool_streaming_response(self) -> String { - match self { - Self::ChatCompletions => [ - sse_data(json!({ - "id": "chatcmpl-tool-mock", "object": "chat.completion.chunk", "created": 0, "model": MODEL, - "choices": [{"index": 0, "delta": {"role": "assistant", "tool_calls": [{ - "index": 0, "id": "call-weather-42", "type": "function", - "function": {"name": "get_weather", "arguments": "{\"city\":"} - }]}, "finish_reason": null}] - })), - sse_data(json!({ - "id": "chatcmpl-tool-mock", "object": "chat.completion.chunk", "created": 0, "model": MODEL, - "choices": [{"index": 0, "delta": {"tool_calls": [{"index": 0, - "function": {"arguments": "\"Paris\"}"} - }]}, "finish_reason": null}] - })), - sse_data(json!({ - "id": "chatcmpl-tool-mock", "object": "chat.completion.chunk", "created": 0, "model": MODEL, - "choices": [{"index": 0, "delta": {}, "finish_reason": "tool_calls"}] - })), - "data: [DONE]\n\n".to_string(), - ] - .concat(), - Self::Responses => [ - sse_event("response.created", json!({ - "type": "response.created", "sequence_number": 0, - "response": {"id": "resp-tool-mock", "object": "response", "created_at": 0, - "status": "in_progress", "model": MODEL, "output": []} - })), - sse_event("response.output_item.added", json!({ - "type": "response.output_item.added", "sequence_number": 1, "output_index": 0, - "item": {"id": "fc-item-42", "type": "function_call", "status": "in_progress", - "call_id": "call-weather-42", "name": "get_weather", "arguments": ""} - })), - sse_event("response.function_call_arguments.delta", json!({ - "type": "response.function_call_arguments.delta", "sequence_number": 2, - "item_id": "fc-item-42", "output_index": 0, "delta": "{\"city\":" - })), - sse_event("response.function_call_arguments.delta", json!({ - "type": "response.function_call_arguments.delta", "sequence_number": 3, - "item_id": "fc-item-42", "output_index": 0, "delta": "\"Paris\"}" - })), - sse_event("response.function_call_arguments.done", json!({ - "type": "response.function_call_arguments.done", "sequence_number": 4, - "item_id": "fc-item-42", "output_index": 0, "name": "get_weather", - "arguments": "{\"city\":\"Paris\"}" - })), - sse_event("response.output_item.done", json!({ - "type": "response.output_item.done", "sequence_number": 5, "output_index": 0, - "item": {"id": "fc-item-42", "type": "function_call", "status": "completed", - "call_id": "call-weather-42", "name": "get_weather", "arguments": "{\"city\":\"Paris\"}"} - })), - sse_event("response.completed", json!({ - "type": "response.completed", "sequence_number": 6, - "response": {"id": "resp-tool-mock", "object": "response", "created_at": 0, - "status": "completed", "model": MODEL, "output": [{"id": "fc-item-42", - "type": "function_call", "status": "completed", "call_id": "call-weather-42", - "name": "get_weather", "arguments": "{\"city\":\"Paris\"}"}]} - })), - ] - .concat(), - Self::Messages => [ - sse_event("message_start", json!({ - "type": "message_start", "message": {"id": "msg-tool-mock", "type": "message", - "role": "assistant", "model": MODEL, "content": [], "stop_reason": null, - "stop_sequence": null, "usage": {"input_tokens": 3, "output_tokens": 0}} - })), - sse_event("content_block_start", json!({ - "type": "content_block_start", "index": 0, - "content_block": {"type": "tool_use", "id": "call-weather-42", "name": "get_weather", "input": {}} - })), - sse_event("content_block_delta", json!({ - "type": "content_block_delta", "index": 0, - "delta": {"type": "input_json_delta", "partial_json": "{\"city\":"} - })), - sse_event("content_block_delta", json!({ - "type": "content_block_delta", "index": 0, - "delta": {"type": "input_json_delta", "partial_json": "\"Paris\"}"} - })), - sse_event("content_block_stop", json!({"type": "content_block_stop", "index": 0})), - sse_event("message_delta", json!({ - "type": "message_delta", "delta": {"stop_reason": "tool_use", "stop_sequence": null}, - "usage": {"output_tokens": 4} - })), - sse_event("message_stop", json!({"type": "message_stop"})), - ] - .concat(), - } - } -} - -fn sse_data(value: Value) -> String { - format!( - "data: {}\n\n", - serde_json::to_string(&value).expect("serialize SSE data") - ) -} - -fn sse_event(event: &str, value: Value) -> String { - format!("event: {event}\n{}", sse_data(value)) -} - -#[derive(Clone)] -struct MockState { - response_content_type: &'static str, - response_body: String, - captured: Arc>>, -} - -#[derive(Clone, Debug)] -struct CapturedRequest { - method: String, - path_and_query: String, - headers: HashMap, - body: Value, -} - -async fn mock_model(State(state): State, request: Request) -> Response { - let (parts, body) = request.into_parts(); - let body = to_bytes(body, 1024 * 1024) - .await - .expect("read mock model request body"); - let headers = parts - .headers - .iter() - .filter_map(|(name, value)| { - value - .to_str() - .ok() - .map(|value| (name.as_str().to_string(), value.to_string())) - }) - .collect(); - let captured = CapturedRequest { - method: parts.method.to_string(), - path_and_query: parts - .uri - .path_and_query() - .map(|value| value.as_str()) - .unwrap_or(parts.uri.path()) - .to_string(), - headers, - body: serde_json::from_slice(&body).expect("mock model request is JSON"), - }; - *state.captured.lock().expect("lock captured request") = Some(captured); - - Response::builder() - .status(StatusCode::OK) - .header("content-type", state.response_content_type) - .header("x-model-request-id", MODEL_REQUEST_ID) - .body(Body::from(state.response_body)) - .expect("build mock model response") -} - -async fn spawn_mock_model( - response_content_type: &'static str, - response_body: String, -) -> ( - String, - Arc>>, - oneshot::Sender<()>, - tokio::task::JoinHandle<()>, -) { - let listener = tokio::net::TcpListener::bind("127.0.0.1:0") - .await - .expect("bind mock model service"); - let address = listener.local_addr().expect("mock model address"); - let captured = Arc::new(Mutex::new(None)); - let app = Router::new() - .fallback(post(mock_model)) - .with_state(MockState { - response_content_type, - response_body, - captured: Arc::clone(&captured), - }); - let (stop_tx, stop_rx) = oneshot::channel(); - let task = tokio::spawn(async move { - axum::serve(listener, app) - .with_graceful_shutdown(async { - let _ = stop_rx.await; - }) - .await - .expect("serve mock model service"); - }); - (format!("http://{address}"), captured, stop_tx, task) -} - -async fn spawn_gateway( - api: Api, - mock_base: &str, -) -> ( - String, - tempfile::TempDir, - oneshot::Sender<()>, - tokio::task::JoinHandle>, -) { - let gateway_listener = tokio::net::TcpListener::bind("127.0.0.1:0") - .await - .expect("bind Gateway"); - let gateway_address = gateway_listener.local_addr().expect("Gateway address"); - let admin_listener = tokio::net::TcpListener::bind("127.0.0.1:0") - .await - .expect("bind Gateway admin API"); - let admin_address = admin_listener.local_addr().expect("Gateway admin address"); - - let (provider, upstream, upstream_anthropic) = match api { - Api::ChatCompletions => ("openai", format!("{mock_base}/v1"), None), - Api::Responses => { - // Native Responses routing recognizes official OpenAI endpoints. A URL fragment - // supplies that marker for this loopback mock and, by HTTP definition, is never - // included in the request sent to the model service. - ("openai", format!("{mock_base}/v1#api.openai.com"), None) - } - Api::Messages => ( - "anthropic", - format!("{mock_base}/v1"), - Some(format!("{mock_base}/v1")), - ), - }; - let upstream_anthropic = upstream_anthropic - .map(|value| format!("upstream_anthropic = {value:?}\n")) - .unwrap_or_default(); - let config = ProxyConfig::from_toml_str(&format!( - r#" -listen = "{gateway_address}" -admin_listen = "{admin_address}" -agent_id = "model-api-forwarding-test" -capture_level = "summary" - -[[models]] -name = "*" -provider = "{provider}" -upstream = {upstream:?} -{upstream_anthropic}api_key = "{UPSTREAM_KEY}" -"# - )) - .expect("Gateway config"); - let storage = tempfile::tempdir().expect("Gateway state directory"); - let storage_path = storage.path().to_path_buf(); - let sink: Arc = Arc::new(SeqOnlySink::new()); - let (stop_tx, stop_rx) = oneshot::channel(); - let task = tokio::spawn(async move { - serve_with_listeners_and_shutdown( - config, - storage_path, - sink, - false, - gateway_listener, - admin_listener, - async { - let _ = stop_rx.await; - }, - ) - .await - }); - tokio::task::yield_now().await; - - (format!("http://{gateway_address}"), storage, stop_tx, task) -} - -async fn assert_round_trip(api: Api, stream: bool, tool_call: bool) { - let expected_request = api.request(stream, tool_call); - let (response_content_type, expected_response) = if stream { - ( - "text/event-stream", - if tool_call { - api.tool_streaming_response() - } else { - api.streaming_response() - }, - ) - } else { - ( - "application/json", - serde_json::to_string(&if tool_call { - api.tool_non_streaming_response() - } else { - api.non_streaming_response() - }) - .expect("serialize response"), - ) - }; - let (mock_base, captured, mock_stop, mock_task) = - spawn_mock_model(response_content_type, expected_response.clone()).await; - let (gateway_base, _storage, gateway_stop, gateway_task) = spawn_gateway(api, &mock_base).await; - - let mut request = reqwest::Client::builder() - .no_proxy() - .build() - .expect("HTTP client") - .post(format!("{gateway_base}{}", api.path())) - .header("content-type", "application/json") - .header("x-client-trace", CLIENT_TRACE) - .body(serde_json::to_vec(&expected_request).expect("serialize request")); - request = match api { - Api::Messages => request - .header("x-api-key", "client-key-must-not-reach-upstream") - .header("anthropic-version", "2023-06-01"), - Api::ChatCompletions | Api::Responses => { - request.header("authorization", "Bearer client-key-must-not-reach-upstream") - } - }; - let response = request.send().await.expect("request Gateway"); - let status = response.status(); - let headers = response.headers().clone(); - let actual_response = response.text().await.expect("read Gateway response"); - - let captured = captured - .lock() - .expect("lock captured request") - .clone() - .unwrap_or_else(|| panic!("{} mock received no request", api.name())); - - let _ = gateway_stop.send(()); - gateway_task - .await - .expect("join Gateway task") - .expect("Gateway shutdown"); - let _ = mock_stop.send(()); - mock_task.await.expect("join mock model task"); - - assert_eq!(status, StatusCode::OK, "{} status", api.name()); - assert_eq!( - headers - .get("content-type") - .and_then(|value| value.to_str().ok()), - Some(response_content_type), - "{} response content type", - api.name() - ); - assert_eq!( - headers - .get("x-model-request-id") - .and_then(|value| value.to_str().ok()), - Some(MODEL_REQUEST_ID), - "{} response header", - api.name() - ); - assert_eq!( - actual_response, - expected_response, - "{} response body", - api.name() - ); - - assert_eq!(captured.method, "POST", "{} method", api.name()); - assert_eq!( - captured.path_and_query, - api.path(), - "{} upstream path", - api.name() - ); - assert_eq!( - captured.body, - expected_request, - "{} upstream JSON body", - api.name() - ); - if tool_call { - assert_eq!( - captured.body["tools"][0]["type"], - if matches!(api, Api::Messages) { - Value::Null - } else { - Value::String("function".into()) - }, - "{} tool type", - api.name() - ); - assert_eq!( - captured.body["tools"][0]["name"], - if matches!(api, Api::ChatCompletions) { - Value::Null - } else { - Value::String("get_weather".into()) - }, - "{} tool name", - api.name() - ); - let function_name = captured.body["tools"][0] - .get("function") - .and_then(|function| function.get("name")) - .and_then(Value::as_str); - assert_eq!( - function_name, - matches!(api, Api::ChatCompletions).then_some("get_weather"), - "{} function tool definition", - api.name() - ); - } - assert_eq!( - captured.headers.get("x-client-trace").map(String::as_str), - Some(CLIENT_TRACE), - "{} forwarded client header", - api.name() - ); - match api { - Api::Messages => { - assert_eq!( - captured.headers.get("x-api-key").map(String::as_str), - Some(UPSTREAM_KEY) - ); - assert_eq!( - captured - .headers - .get("anthropic-version") - .map(String::as_str), - Some("2023-06-01") - ); - assert!(!captured.headers.contains_key("authorization")); - } - Api::ChatCompletions | Api::Responses => { - assert_eq!( - captured.headers.get("authorization").map(String::as_str), - Some("Bearer upstream-test-key") - ); - assert!(!captured.headers.contains_key("x-api-key")); - } - } -} - -#[tokio::test] -async fn chat_completions_non_streaming_round_trip() { - assert_round_trip(Api::ChatCompletions, false, false).await; -} - -#[tokio::test] -async fn chat_completions_sse_round_trip() { - assert_round_trip(Api::ChatCompletions, true, false).await; -} - -#[tokio::test] -async fn responses_non_streaming_round_trip() { - assert_round_trip(Api::Responses, false, false).await; -} - -#[tokio::test] -async fn responses_sse_round_trip() { - assert_round_trip(Api::Responses, true, false).await; -} - -#[tokio::test] -async fn messages_non_streaming_round_trip() { - assert_round_trip(Api::Messages, false, false).await; -} - -#[tokio::test] -async fn messages_sse_round_trip() { - assert_round_trip(Api::Messages, true, false).await; -} - -#[tokio::test] -async fn chat_completions_tool_call_non_streaming_round_trip() { - assert_round_trip(Api::ChatCompletions, false, true).await; -} - -#[tokio::test] -async fn chat_completions_tool_call_sse_round_trip() { - assert_round_trip(Api::ChatCompletions, true, true).await; -} - -#[tokio::test] -async fn responses_tool_call_non_streaming_round_trip() { - assert_round_trip(Api::Responses, false, true).await; -} - -#[tokio::test] -async fn responses_tool_call_sse_round_trip() { - assert_round_trip(Api::Responses, true, true).await; -} - -#[tokio::test] -async fn messages_tool_call_non_streaming_round_trip() { - assert_round_trip(Api::Messages, false, true).await; -} - -#[tokio::test] -async fn messages_tool_call_sse_round_trip() { - assert_round_trip(Api::Messages, true, true).await; -} diff --git a/crates/persisting-gateway/tests/network_policy_http.rs b/crates/persisting-gateway/tests/network_policy_http.rs deleted file mode 100644 index 9df1541d2..000000000 --- a/crates/persisting-gateway/tests/network_policy_http.rs +++ /dev/null @@ -1,1447 +0,0 @@ -//! End-to-end HTTP checks for Harbor-style `[network]` egress policy. - -use std::net::SocketAddr; -use std::sync::Arc; -use std::time::Duration; - -use axum::Router; -use axum::extract::Path; -use axum::http::{HeaderMap, StatusCode, Uri}; -use axum::response::{IntoResponse, Response}; -use axum::routing::{get, post}; -use persisting_agentctl::{ - ControlController, ControlReason, ControlRequest, ControlTransition, PolicyControlController, -}; -use persisting_gateway::config::ProxyConfig; -use persisting_gateway::runtime::in_process::{InProcessCapture, InProcessRuntime}; -use persisting_gateway::sink::SeqOnlySink; -use persisting_gateway::{serve_with_runtime_control, serve_with_shutdown_and_ready}; -use tokio::sync::oneshot; - -struct DenyModelController; - -impl ControlController for DenyModelController { - fn authorize(&self, request: ControlRequest<'_>) -> ControlTransition { - match request { - ControlRequest::Model { .. } => { - ControlTransition::denied(ControlReason::ModelNotAllowed) - } - network => PolicyControlController.authorize(network), - } - } -} - -fn free_port() -> u16 { - std::net::TcpListener::bind("127.0.0.1:0") - .unwrap() - .local_addr() - .unwrap() - .port() -} - -async fn spawn_mock_http() -> (u16, oneshot::Sender<()>) { - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let port = listener.local_addr().unwrap().port(); - let redirect_target = format!("http://127.0.0.1:{port}/"); - let (stop_tx, stop_rx) = oneshot::channel::<()>(); - tokio::spawn(async move { - let app = Router::new() - .route("/", get(|| async { "ok" })) - .route( - "/bytes/{size}", - get(|Path(size): Path| async move { vec![b'x'; size.min(1_048_576)] }), - ) - .route( - "/redirect", - get(move || { - let target = redirect_target.clone(); - async move { (StatusCode::FOUND, [(axum::http::header::LOCATION, target)]) } - }), - ) - .route( - "/v1/chat/completions", - post(|| async { - axum::Json(serde_json::json!({ - "id": "chatcmpl-test", - "object": "chat.completion", - "choices": [{ - "index": 0, - "message": {"role": "assistant", "content": "hi"}, - "finish_reason": "stop" - }], - "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2} - })) - }), - ) - .route( - "/echo-size", - post(|body: axum::body::Bytes| async move { body.len().to_string() }), - ); - axum::serve(listener, app) - .with_graceful_shutdown(async { - let _ = stop_rx.await; - }) - .await - .ok(); - }); - tokio::task::yield_now().await; - (port, stop_tx) -} - -async fn spawn_chunked_llm_http() -> (u16, oneshot::Sender<()>) { - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let port = listener.local_addr().unwrap().port(); - let (stop_tx, stop_rx) = oneshot::channel::<()>(); - tokio::spawn(async move { - let app = Router::new().route( - "/v1/chat/completions", - post(|| async { - let chunks = futures_util::stream::iter([ - Ok::<_, std::convert::Infallible>(axum::body::Bytes::from_static( - br#"{"id":"chatcmpl-chunked","object":"chat.completion","choices":[{"index":0,"message":{"role":"assistant","content":"#, - )), - Ok(axum::body::Bytes::from_static( - br#""chunked"},"finish_reason":"stop"}],"usage":{"prompt_tokens":1,"completion_tokens":1,"total_tokens":2}}"#, - )), - ]); - Response::builder() - .status(StatusCode::OK) - .header("content-type", "application/json") - .header("connection", "x-upstream-hop") - .header("x-upstream-hop", "must-not-cross-proxy") - .body(axum::body::Body::from_stream(chunks)) - .unwrap() - }), - ); - axum::serve(listener, app) - .with_graceful_shutdown(async { - let _ = stop_rx.await; - }) - .await - .ok(); - }); - tokio::task::yield_now().await; - (port, stop_tx) -} - -async fn spawn_capturing_llm_http() -> ( - u16, - Arc>>, - oneshot::Sender<()>, -) { - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let port = listener.local_addr().unwrap().port(); - let captured = Arc::new(std::sync::Mutex::new(None)); - let captured_for_app = Arc::clone(&captured); - let (stop_tx, stop_rx) = oneshot::channel::<()>(); - tokio::spawn(async move { - let app = Router::new().route( - "/v1/chat/completions", - post(move |body: axum::body::Bytes| { - let captured = Arc::clone(&captured_for_app); - async move { - *captured.lock().unwrap() = serde_json::from_slice(&body).ok(); - axum::Json(serde_json::json!({ - "id": "chatcmpl-bridge", - "object": "chat.completion", - "model": "upstream-model", - "choices": [{ - "index": 0, - "message": {"role": "assistant", "content": "bridged"}, - "finish_reason": "stop" - }], - "usage": {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4} - })) - } - }), - ); - axum::serve(listener, app) - .with_graceful_shutdown(async { - let _ = stop_rx.await; - }) - .await - .ok(); - }); - tokio::task::yield_now().await; - (port, captured, stop_tx) -} - -async fn spawn_capturing_gemini_http() -> ( - u16, - Arc>>, - oneshot::Sender<()>, -) { - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let port = listener.local_addr().unwrap().port(); - let captured = Arc::new(std::sync::Mutex::new(None)); - let captured_for_app = Arc::clone(&captured); - let (stop_tx, stop_rx) = oneshot::channel::<()>(); - tokio::spawn(async move { - let app = Router::new().fallback(post( - move |uri: Uri, headers: HeaderMap, body: axum::body::Bytes| { - let captured = Arc::clone(&captured_for_app); - async move { - let body: serde_json::Value = - serde_json::from_slice(&body).expect("Gemini request JSON"); - *captured.lock().unwrap() = Some(serde_json::json!({ - "path": uri.path(), - "query": uri.query(), - "api_key": headers - .get("x-goog-api-key") - .and_then(|value| value.to_str().ok()), - "authorization": headers - .get("authorization") - .and_then(|value| value.to_str().ok()), - "body": body, - })); - if uri.path().ends_with(":streamGenerateContent") { - Response::builder() - .status(StatusCode::OK) - .header("content-type", "text/event-stream") - .body(axum::body::Body::from( - "data: {\"responseId\":\"gemini-stream-e2e\",\"modelVersion\":\"gemini-2.5-pro\",\"candidates\":[{\"content\":{\"role\":\"model\",\"parts\":[{\"text\":\"native streamed\"}]},\"finishReason\":\"STOP\"}],\"usageMetadata\":{\"promptTokenCount\":3,\"candidatesTokenCount\":2,\"totalTokenCount\":5}}\n\n", - )) - .unwrap() - .into_response() - } else { - axum::Json(serde_json::json!({ - "responseId": "gemini-e2e", - "modelVersion": "gemini-2.5-pro", - "candidates": [{ - "index": 0, - "content": { - "role": "model", - "parts": [{"text": "native bridged"}] - }, - "finishReason": "STOP" - }], - "usageMetadata": { - "promptTokenCount": 3, - "candidatesTokenCount": 2, - "totalTokenCount": 5 - } - })) - .into_response() - } - } - }, - )); - axum::serve(listener, app) - .with_graceful_shutdown(async { - let _ = stop_rx.await; - }) - .await - .ok(); - }); - tokio::task::yield_now().await; - (port, captured, stop_tx) -} - -async fn spawn_proxy(toml: &str) -> (String, tempfile::TempDir, oneshot::Sender<()>) { - let listen_port = free_port(); - let admin_port = free_port(); - let toml = toml - .replace("{{LISTEN}}", &format!("127.0.0.1:{listen_port}")) - .replace("{{ADMIN}}", &format!("127.0.0.1:{admin_port}")); - let cfg = ProxyConfig::from_toml_str(&toml).expect("proxy toml"); - let tmp = tempfile::tempdir().unwrap(); - let (ready_tx, ready_rx) = oneshot::channel(); - let (stop_tx, stop_rx) = oneshot::channel::<()>(); - let storage = tmp.path().to_path_buf(); - let sink: Arc = Arc::new(SeqOnlySink::new()); - tokio::spawn(async move { - let _ = - serve_with_shutdown_and_ready(cfg, storage, sink, false, Some(ready_tx), async move { - let _ = stop_rx.await; - }) - .await; - }); - ready_rx.await.expect("proxy ready"); - (format!("http://127.0.0.1:{listen_port}"), tmp, stop_tx) -} - -async fn spawn_proxy_with_controller( - toml: &str, - controller: Arc, -) -> (String, tempfile::TempDir, oneshot::Sender<()>) { - let listen_port = free_port(); - let admin_port = free_port(); - let toml = toml - .replace("{{LISTEN}}", &format!("127.0.0.1:{listen_port}")) - .replace("{{ADMIN}}", &format!("127.0.0.1:{admin_port}")); - let cfg = ProxyConfig::from_toml_str(&toml).expect("proxy toml"); - let tmp = tempfile::tempdir().unwrap(); - let (ready_tx, ready_rx) = oneshot::channel(); - let (stop_tx, stop_rx) = oneshot::channel::<()>(); - let storage = tmp.path().to_path_buf(); - let sink: Arc = Arc::new(SeqOnlySink::new()); - tokio::spawn(async move { - let _ = serve_with_runtime_control( - cfg, - storage, - sink, - false, - controller, - Some(ready_tx), - async move { - let _ = stop_rx.await; - }, - ) - .await; - }); - ready_rx.await.expect("proxy ready"); - (format!("http://127.0.0.1:{listen_port}"), tmp, stop_tx) -} - -async fn raw_connect(proxy: &str, authority: &str) -> (StatusCode, String) { - use tokio::io::{AsyncReadExt, AsyncWriteExt}; - - let addr: SocketAddr = proxy - .trim_start_matches("http://") - .parse() - .expect("proxy addr"); - let mut stream = tokio::net::TcpStream::connect(addr).await.unwrap(); - let req = format!("CONNECT {authority} HTTP/1.1\r\nHost: {authority}\r\n\r\n"); - stream.write_all(req.as_bytes()).await.unwrap(); - - let mut buf = vec![0u8; 2048]; - let n = stream.read(&mut buf).await.unwrap(); - let text = String::from_utf8_lossy(&buf[..n]); - let status_line = text.lines().next().unwrap_or(""); - let code = status_line - .split_whitespace() - .nth(1) - .and_then(|c| c.parse::().ok()) - .unwrap_or(0); - let status = StatusCode::from_u16(code).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR); - let body = text.split("\r\n\r\n").nth(1).unwrap_or("").to_string(); - (status, body) -} - -async fn tunneled_get(proxy: &str, authority: &str, path: &str) -> (Duration, Vec) { - use tokio::io::{AsyncReadExt, AsyncWriteExt}; - - let addr: SocketAddr = proxy.trim_start_matches("http://").parse().unwrap(); - let mut stream = tokio::net::TcpStream::connect(addr).await.unwrap(); - let connect = format!("CONNECT {authority} HTTP/1.1\r\nHost: {authority}\r\n\r\n"); - stream.write_all(connect.as_bytes()).await.unwrap(); - let mut headers = Vec::new(); - let mut byte = [0_u8; 1]; - while !headers.ends_with(b"\r\n\r\n") { - stream.read_exact(&mut byte).await.unwrap(); - headers.push(byte[0]); - assert!(headers.len() < 8_192, "CONNECT response headers too large"); - } - assert!( - String::from_utf8_lossy(&headers).starts_with("HTTP/1.1 200"), - "{}", - String::from_utf8_lossy(&headers) - ); - - let started = tokio::time::Instant::now(); - let request = format!("GET {path} HTTP/1.1\r\nHost: {authority}\r\nConnection: close\r\n\r\n"); - stream.write_all(request.as_bytes()).await.unwrap(); - let mut response = Vec::new(); - tokio::time::timeout(Duration::from_secs(8), stream.read_to_end(&mut response)) - .await - .expect("tunneled response timeout") - .unwrap(); - (started.elapsed(), response) -} - -#[tokio::test] -async fn e2e_allowlist_denies_connect_to_unlisted_host() { - let (proxy, _tmp, stop) = spawn_proxy( - r#" -listen = "{{LISTEN}}" -admin_listen = "{{ADMIN}}" -agent_id = "t" - -[network] -mode = "allowlist" -allowed_hosts = [ - "pypi.org", -] - -[[models]] -name = "*" -upstream = "http://127.0.0.1:9/v1" -"#, - ) - .await; - - let (status, body) = raw_connect(&proxy, "github.com:443").await; - assert_eq!(status, StatusCode::FORBIDDEN); - assert!(body.contains("github.com"), "{body}"); - let _ = stop.send(()); -} - -#[tokio::test] -async fn e2e_allowlist_permits_connect_to_listed_host() { - let (mock_port, mock_stop) = spawn_mock_http().await; - let (proxy, _tmp, stop) = spawn_proxy( - r#" -listen = "{{LISTEN}}" -admin_listen = "{{ADMIN}}" -agent_id = "t" - -[network] -mode = "allowlist" -allowed_hosts = [ - "127.0.0.1", -] - -[[models]] -name = "*" -upstream = "http://127.0.0.1:9/v1" -"#, - ) - .await; - - let (status, _) = raw_connect(&proxy, &format!("127.0.0.1:{mock_port}")).await; - assert_eq!(status, StatusCode::OK); - let _ = stop.send(()); - let _ = mock_stop.send(()); -} - -#[tokio::test] -async fn e2e_model_upstream_is_not_an_agent_egress_grant() { - let (mock_port, mock_stop) = spawn_mock_http().await; - let toml = format!( - r#" -listen = "{{{{LISTEN}}}}" -admin_listen = "{{{{ADMIN}}}}" -agent_id = "t" - -[network] -mode = "allowlist" -allowed_hosts = ["pypi.org"] - -[[models]] -name = "*" -upstream = "http://127.0.0.1:{mock_port}/v1" -"# - ); - let (proxy, _tmp, stop) = spawn_proxy(&toml).await; - - let (status, body) = raw_connect(&proxy, &format!("127.0.0.1:{mock_port}")).await; - assert_eq!(status, StatusCode::FORBIDDEN); - assert!(body.contains("not-in-allowlist"), "{body}"); - let _ = stop.send(()); - let _ = mock_stop.send(()); -} - -#[tokio::test] -async fn e2e_structured_rule_scopes_port_and_transport() { - let (mock_port, mock_stop) = spawn_mock_http().await; - let toml = format!( - r#" -listen = "{{{{LISTEN}}}}" -admin_listen = "{{{{ADMIN}}}}" -agent_id = "t" - -[network] -mode = "allowlist" - -[[network.rules]] -host = "127.0.0.1" -ports = [{mock_port}] -transports = ["tcp_tunnel"] - -[[models]] -name = "*" -upstream = "http://127.0.0.1:9/v1" -"# - ); - let (proxy, _tmp, stop) = spawn_proxy(&toml).await; - - let (allowed, _) = raw_connect(&proxy, &format!("127.0.0.1:{mock_port}")).await; - assert_eq!(allowed, StatusCode::OK); - - let (wrong_port, body) = raw_connect(&proxy, "127.0.0.1:1").await; - assert_eq!(wrong_port, StatusCode::FORBIDDEN); - assert!(body.contains("port-not-allowed"), "{body}"); - - let client = reqwest::Client::builder() - .proxy(reqwest::Proxy::all(&proxy).unwrap()) - .timeout(Duration::from_secs(5)) - .build() - .unwrap(); - let response = client - .get(format!("http://127.0.0.1:{mock_port}/")) - .send() - .await - .unwrap(); - assert_eq!(response.status(), StatusCode::FORBIDDEN); - assert!( - response - .text() - .await - .unwrap() - .contains("transport-not-allowed") - ); - let _ = stop.send(()); - let _ = mock_stop.send(()); -} - -#[tokio::test] -async fn e2e_connect_does_not_report_success_before_upstream_connects() { - let closed_port = free_port(); - let (proxy, _tmp, stop) = spawn_proxy( - r#" -listen = "{{LISTEN}}" -admin_listen = "{{ADMIN}}" -agent_id = "t" - -[network] -mode = "public" - -[[models]] -name = "*" -upstream = "http://127.0.0.1:9/v1" -"#, - ) - .await; - - let (status, body) = raw_connect(&proxy, &format!("127.0.0.1:{closed_port}")).await; - assert_eq!(status, StatusCode::BAD_GATEWAY); - assert!(body.contains("CONNECT"), "{body}"); - let _ = stop.send(()); -} - -#[tokio::test] -async fn e2e_malformed_connect_authorities_fail_closed() { - let (proxy, _tmp, stop) = spawn_proxy( - r#" -listen = "{{LISTEN}}" -admin_listen = "{{ADMIN}}" -agent_id = "t" - -[network] -mode = "public" - -[[models]] -name = "*" -upstream = "http://127.0.0.1:9/v1" -"#, - ) - .await; - for authority in ["example.com:0", "example.com:65536", "user@example.com:443"] { - let (status, _) = raw_connect(&proxy, authority).await; - assert_eq!(status, StatusCode::BAD_REQUEST, "{authority}"); - } - let _ = stop.send(()); -} - -#[tokio::test] -async fn e2e_no_network_denies_connect_including_loopback() { - let (proxy, _tmp, stop) = spawn_proxy( - r#" -listen = "{{LISTEN}}" -admin_listen = "{{ADMIN}}" -agent_id = "t" - -[network] -mode = "no-network" - -[[models]] -name = "*" -upstream = "http://127.0.0.1:9/v1" -"#, - ) - .await; - - let (denied, _) = raw_connect(&proxy, "example.com:443").await; - assert_eq!(denied, StatusCode::FORBIDDEN); - let (loopback, _) = raw_connect(&proxy, "127.0.0.1:9").await; - assert_eq!(loopback, StatusCode::FORBIDDEN); - let _ = stop.send(()); -} - -#[tokio::test] -async fn e2e_explicit_deny_overrides_public_default() { - let (mock_port, mock_stop) = spawn_mock_http().await; - let toml = format!( - r#" -listen = "{{{{LISTEN}}}}" -admin_listen = "{{{{ADMIN}}}}" -agent_id = "t" - -[network] -mode = "public" - -[[network.deny_rules]] -host = "127.0.0.1" -ports = [{mock_port}] - -[[models]] -name = "*" -upstream = "http://127.0.0.1:9/v1" -"# - ); - let (proxy, _tmp, stop) = spawn_proxy(&toml).await; - let (status, body) = raw_connect(&proxy, &format!("127.0.0.1:{mock_port}")).await; - assert_eq!(status, StatusCode::FORBIDDEN); - assert!(body.contains("explicit-deny"), "{body}"); - let _ = stop.send(()); - let _ = mock_stop.send(()); -} - -#[tokio::test] -async fn e2e_cidr_deny_filters_hostname_dns_results() { - let (mock_port, mock_stop) = spawn_mock_http().await; - let toml = r#" -listen = "{{LISTEN}}" -admin_listen = "{{ADMIN}}" -agent_id = "t" - -[network] -mode = "public" - -[[network.deny_rules]] -host = "127.0.0.0/8" - -[[network.deny_rules]] -host = "::1/128" - -[[models]] -name = "*" -upstream = "http://127.0.0.1:9/v1" -"# - .to_string(); - let (proxy, _tmp, stop) = spawn_proxy(&toml).await; - let (status, body) = raw_connect(&proxy, &format!("localhost:{mock_port}")).await; - assert_eq!(status, StatusCode::FORBIDDEN); - assert!(body.contains("explicit-deny"), "{body}"); - let _ = stop.send(()); - let _ = mock_stop.send(()); -} - -#[tokio::test] -async fn e2e_global_bandwidth_limit_throttles_forwarded_body() { - let (mock_port, mock_stop) = spawn_mock_http().await; - let (proxy, _tmp, stop) = spawn_proxy( - r#" -listen = "{{LISTEN}}" -admin_listen = "{{ADMIN}}" -agent_id = "t" - -[network] -mode = "public" - -[[network.limits]] -bytes_per_second = 32768 - -[[models]] -name = "*" -upstream = "http://127.0.0.1:9/v1" -"#, - ) - .await; - let client = reqwest::Client::builder() - .proxy(reqwest::Proxy::all(&proxy).unwrap()) - .timeout(Duration::from_secs(8)) - .build() - .unwrap(); - let started = tokio::time::Instant::now(); - let response = client - .get(format!("http://127.0.0.1:{mock_port}/bytes/32768")) - .send() - .await - .unwrap(); - assert_eq!(response.status(), StatusCode::OK); - assert_eq!(response.bytes().await.unwrap().len(), 32768); - assert!( - started.elapsed() >= Duration::from_millis(800), - "bandwidth limit completed too quickly: {:?}", - started.elapsed() - ); - let _ = stop.send(()); - let _ = mock_stop.send(()); -} - -#[tokio::test] -async fn e2e_bandwidth_limit_throttles_http_upload() { - let (mock_port, mock_stop) = spawn_mock_http().await; - let (proxy, _tmp, stop) = spawn_proxy( - r#" -listen = "{{LISTEN}}" -admin_listen = "{{ADMIN}}" -agent_id = "t" - -[network] -mode = "public" - -[[network.limits]] -bytes_per_second = 32768 - -[[models]] -name = "*" -upstream = "http://127.0.0.1:9/v1" -"#, - ) - .await; - let client = reqwest::Client::builder() - .proxy(reqwest::Proxy::all(&proxy).unwrap()) - .timeout(Duration::from_secs(8)) - .build() - .unwrap(); - let started = tokio::time::Instant::now(); - let response = client - .post(format!("http://127.0.0.1:{mock_port}/echo-size")) - .body(vec![b'x'; 32768]) - .send() - .await - .unwrap(); - assert_eq!(response.text().await.unwrap(), "32768"); - assert!(started.elapsed() >= Duration::from_millis(800)); - let _ = stop.send(()); - let _ = mock_stop.send(()); -} - -#[tokio::test] -async fn e2e_bandwidth_limit_is_shared_by_concurrent_requests() { - let (mock_port, mock_stop) = spawn_mock_http().await; - let (proxy, _tmp, stop) = spawn_proxy( - r#" -listen = "{{LISTEN}}" -admin_listen = "{{ADMIN}}" -agent_id = "t" - -[network] -mode = "public" - -[[network.limits]] -bytes_per_second = 32768 - -[[models]] -name = "*" -upstream = "http://127.0.0.1:9/v1" -"#, - ) - .await; - let client = reqwest::Client::builder() - .proxy(reqwest::Proxy::all(&proxy).unwrap()) - .timeout(Duration::from_secs(8)) - .build() - .unwrap(); - let url = format!("http://127.0.0.1:{mock_port}/bytes/16384"); - let started = tokio::time::Instant::now(); - let (first, second) = tokio::join!(client.get(&url).send(), client.get(&url).send()); - let (first, second) = (first.unwrap(), second.unwrap()); - assert_eq!(first.bytes().await.unwrap().len(), 16384); - assert_eq!(second.bytes().await.unwrap().len(), 16384); - assert!(started.elapsed() >= Duration::from_millis(800)); - let _ = stop.send(()); - let _ = mock_stop.send(()); -} - -#[tokio::test] -async fn e2e_bandwidth_limit_throttles_connect_tunnel_bytes() { - let (mock_port, mock_stop) = spawn_mock_http().await; - let (proxy, _tmp, stop) = spawn_proxy( - r#" -listen = "{{LISTEN}}" -admin_listen = "{{ADMIN}}" -agent_id = "t" - -[network] -mode = "public" - -[[network.limits]] -bytes_per_second = 32768 - -[[models]] -name = "*" -upstream = "http://127.0.0.1:9/v1" -"#, - ) - .await; - let (elapsed, response) = - tunneled_get(&proxy, &format!("127.0.0.1:{mock_port}"), "/bytes/32768").await; - assert!(response.starts_with(b"HTTP/1.1 200")); - assert!(response.ends_with(&vec![b'x'; 32768])); - assert!( - elapsed >= Duration::from_millis(800), - "elapsed: {elapsed:?}" - ); - let _ = stop.send(()); - let _ = mock_stop.send(()); -} - -#[tokio::test] -async fn e2e_allowlist_denies_absolute_uri_forward() { - let (proxy, _tmp, stop) = spawn_proxy( - r#" -listen = "{{LISTEN}}" -admin_listen = "{{ADMIN}}" -agent_id = "t" - -[network] -mode = "allowlist" -allowed_hosts = [ - "pypi.org", -] - -[[models]] -name = "*" -upstream = "http://127.0.0.1:9/v1" -"#, - ) - .await; - - let client = reqwest::Client::builder() - .proxy(reqwest::Proxy::all(&proxy).unwrap()) - .timeout(Duration::from_secs(5)) - .build() - .unwrap(); - let resp = client.get("http://github.com/").send().await.unwrap(); - assert_eq!(resp.status(), StatusCode::FORBIDDEN); - let body = resp.text().await.unwrap(); - assert!(body.contains("github.com"), "{body}"); - let _ = stop.send(()); -} - -#[tokio::test] -async fn e2e_allowlist_allows_absolute_uri_to_local_mock() { - let (mock_port, mock_stop) = spawn_mock_http().await; - let (proxy, _tmp, stop) = spawn_proxy( - r#" -listen = "{{LISTEN}}" -admin_listen = "{{ADMIN}}" -agent_id = "t" - -[network] -mode = "allowlist" -allowed_hosts = [ - "127.0.0.1", -] - -[[models]] -name = "*" -upstream = "http://127.0.0.1:9/v1" -"#, - ) - .await; - - let client = reqwest::Client::builder() - .proxy(reqwest::Proxy::all(&proxy).unwrap()) - .timeout(Duration::from_secs(5)) - .build() - .unwrap(); - let resp = client - .get(format!("http://127.0.0.1:{mock_port}/")) - .send() - .await - .unwrap(); - assert_eq!(resp.status(), StatusCode::OK); - assert_eq!(resp.text().await.unwrap(), "ok"); - let _ = stop.send(()); - let _ = mock_stop.send(()); -} - -#[tokio::test] -async fn e2e_hostname_rule_rejects_private_dns_result_by_default() { - let (mock_port, mock_stop) = spawn_mock_http().await; - let toml = format!( - r#" -listen = "{{{{LISTEN}}}}" -admin_listen = "{{{{ADMIN}}}}" -agent_id = "t" - -[network] -mode = "allowlist" - -[[network.rules]] -host = "localhost" -ports = [{mock_port}] -transports = ["http"] - -[[models]] -name = "*" -upstream = "http://127.0.0.1:9/v1" -"# - ); - let (proxy, _tmp, stop) = spawn_proxy(&toml).await; - let client = reqwest::Client::builder() - .proxy(reqwest::Proxy::all(&proxy).unwrap()) - .timeout(Duration::from_secs(5)) - .build() - .unwrap(); - - let response = client - .get(format!("http://localhost:{mock_port}/")) - .send() - .await - .unwrap(); - assert_eq!(response.status(), StatusCode::FORBIDDEN); - assert!( - response - .text() - .await - .unwrap() - .contains("resolved-address-not-allowed") - ); - let _ = stop.send(()); - let _ = mock_stop.send(()); -} - -#[tokio::test] -async fn e2e_hostname_rule_can_explicitly_allow_private_dns_results() { - let (mock_port, mock_stop) = spawn_mock_http().await; - let toml = format!( - r#" -listen = "{{{{LISTEN}}}}" -admin_listen = "{{{{ADMIN}}}}" -agent_id = "t" - -[network] -mode = "allowlist" - -[[network.rules]] -host = "localhost" -ports = [{mock_port}] -transports = ["http"] -allow_private_ips = true - -[[models]] -name = "*" -upstream = "http://127.0.0.1:9/v1" -"# - ); - let (proxy, _tmp, stop) = spawn_proxy(&toml).await; - let client = reqwest::Client::builder() - .proxy(reqwest::Proxy::all(&proxy).unwrap()) - .timeout(Duration::from_secs(5)) - .build() - .unwrap(); - - let response = client - .get(format!("http://localhost:{mock_port}/")) - .send() - .await - .unwrap(); - assert_eq!(response.status(), StatusCode::OK); - assert_eq!(response.text().await.unwrap(), "ok"); - let _ = stop.send(()); - let _ = mock_stop.send(()); -} - -#[tokio::test] -async fn e2e_cross_host_redirect_returns_through_policy_gate() { - let (mock_port, mock_stop) = spawn_mock_http().await; - let toml = format!( - r#" -listen = "{{{{LISTEN}}}}" -admin_listen = "{{{{ADMIN}}}}" -agent_id = "t" - -[network] -mode = "allowlist" - -[[network.rules]] -host = "localhost" -ports = [{mock_port}] -transports = ["http"] -allow_private_ips = true - -[[models]] -name = "*" -upstream = "http://127.0.0.1:9/v1" -"#, - ); - let (proxy, _tmp, stop) = spawn_proxy(&toml).await; - - let client = reqwest::Client::builder() - .proxy(reqwest::Proxy::all(&proxy).unwrap()) - .timeout(Duration::from_secs(5)) - .build() - .unwrap(); - let response = client - .get(format!("http://localhost:{mock_port}/redirect")) - .send() - .await - .unwrap(); - assert_eq!(response.status(), StatusCode::FORBIDDEN); - assert!(response.text().await.unwrap().contains("127.0.0.1")); - let _ = stop.send(()); - let _ = mock_stop.send(()); -} - -#[tokio::test] -async fn e2e_absolute_uri_llm_keeps_gateway_model_routing() { - let (mock_port, captured, mock_stop) = spawn_capturing_llm_http().await; - let config = format!( - r#" -listen = "{{{{LISTEN}}}}" -admin_listen = "{{{{ADMIN}}}}" - -[network] -mode = "allowlist" -allowed_hosts = ["127.0.0.1"] - -[[models]] -name = "client-model" -forward = "upstream-model" - -[[models]] -name = "upstream-model" -upstream = "http://127.0.0.1:{mock_port}/v1" -"# - ); - let (proxy, _storage, stop) = spawn_proxy(&config).await; - let response = reqwest::Client::builder() - .proxy(reqwest::Proxy::all(&proxy).unwrap()) - .timeout(Duration::from_secs(5)) - .build() - .unwrap() - // The URI destination differs from the configured model upstream. - .post("http://127.0.0.1:9/v1/chat/completions") - .header("content-type", "application/json") - .body(r#"{"model":"client-model","messages":[{"role":"user","content":"hi"}]}"#) - .send() - .await - .unwrap(); - assert_eq!(response.status(), StatusCode::OK); - let _ = response.text().await.unwrap(); - assert_eq!( - captured.lock().unwrap().as_ref().unwrap()["model"], - "upstream-model" - ); - let _ = stop.send(()); - let _ = mock_stop.send(()); -} - -#[tokio::test] -async fn e2e_network_only_allowlist_forwards_absolute_llm_uri() { - let (mock_port, mock_stop) = spawn_mock_http().await; - let listen = format!("127.0.0.1:{}", free_port()); - let admin = format!("127.0.0.1:{}", free_port()); - let config = ProxyConfig::from_toml_str(&format!( - r#"listen = "{listen}" -admin_listen = "{admin}" -agent_id = "t" -models = [] -[network] -mode = "allowlist" -allowed_hosts = ["127.0.0.1"] -"# - )) - .unwrap(); - let storage = tempfile::tempdir().unwrap(); - let proxy = InProcessCapture::start_with_runtime( - config, - storage.path().to_path_buf(), - Arc::new(SeqOnlySink::new()), - false, - InProcessRuntime { - gateway_enabled: false, - ..InProcessRuntime::default() - }, - ) - .unwrap(); - let response = reqwest::Client::builder() - .proxy(reqwest::Proxy::all(format!("http://{}", proxy.listen)).unwrap()) - .timeout(Duration::from_secs(5)) - .build() - .unwrap() - .post(format!("http://127.0.0.1:{mock_port}/v1/chat/completions")) - .body(r#"not JSON; OverlayNet must forward it unchanged"#) - .send() - .await - .unwrap(); - assert_eq!(response.status(), StatusCode::OK); - assert!(response.text().await.unwrap().contains("chatcmpl-test")); - proxy.shutdown().unwrap(); - let _ = mock_stop.send(()); -} - -#[tokio::test] -async fn e2e_relative_llm_gateway_bypasses_host_allowlist() { - let (mock_port, mock_stop) = spawn_mock_http().await; - let toml = format!( - r#" -listen = "{{{{LISTEN}}}}" -admin_listen = "{{{{ADMIN}}}}" -agent_id = "t" - -[network] -mode = "allowlist" -allowed_hosts = [ - "pypi.org", -] - -[[models]] -name = "*" -upstream = "http://127.0.0.1:{mock_port}/v1" -"# - ); - let (proxy, _tmp, stop) = spawn_proxy(&toml).await; - - // No HTTP_PROXY: hit listen with a relative path (LLM gateway). - let client = reqwest::Client::builder() - .no_proxy() - .timeout(Duration::from_secs(5)) - .build() - .unwrap(); - let resp = client - .post(format!("{proxy}/v1/chat/completions")) - .header("content-type", "application/json") - .body(r#"{"model":"test","messages":[{"role":"user","content":"hi"}]}"#) - .send() - .await - .unwrap(); - // Must not be blocked by network policy (403). Upstream mock returns 200. - assert_ne!(resp.status(), StatusCode::FORBIDDEN); - assert_eq!(resp.status(), StatusCode::OK); - let _ = stop.send(()); - let _ = mock_stop.send(()); -} - -#[tokio::test] -async fn e2e_non_streaming_chunked_upstream_returns_complete_json() { - let (mock_port, mock_stop) = spawn_chunked_llm_http().await; - let toml = format!( - r#" -listen = "{{{{LISTEN}}}}" -admin_listen = "{{{{ADMIN}}}}" -agent_id = "t" - -[network] -mode = "public" - -[[models]] -name = "*" -upstream = "http://127.0.0.1:{mock_port}/v1" -"#, - ); - let (proxy, _tmp, stop) = spawn_proxy(&toml).await; - - let response = reqwest::Client::builder() - .no_proxy() - .timeout(Duration::from_secs(5)) - .build() - .unwrap() - .post(format!("{proxy}/v1/chat/completions")) - .json(&serde_json::json!({ - "model": "test", - "stream": false, - "messages": [{"role": "user", "content": "hi"}] - })) - .send() - .await - .unwrap(); - - assert_eq!(response.status(), StatusCode::OK); - assert!(response.headers().get("x-upstream-hop").is_none()); - let body: serde_json::Value = response.json().await.unwrap(); - assert_eq!(body["choices"][0]["message"]["content"], "chunked"); - - let _ = stop.send(()); - let _ = mock_stop.send(()); -} - -#[tokio::test] -async fn e2e_messages_bridge_keeps_forwarding_and_translates_both_directions() { - let (mock_port, captured, mock_stop) = spawn_capturing_llm_http().await; - let toml = format!( - r#" -listen = "{{{{LISTEN}}}}" -admin_listen = "{{{{ADMIN}}}}" -agent_id = "t" - -[network] -mode = "public" - -[[models]] -name = "claude-client" -forward = "upstream-model" - -[[models]] -name = "upstream-model" -upstream = "http://127.0.0.1:{mock_port}/v1" -"# - ); - let (proxy, _tmp, stop) = spawn_proxy(&toml).await; - let original = serde_json::json!({ - "model": "claude-client", - "max_tokens": 32, - "messages": [{"role": "user", "content": "hello"}], - "tools": [{ - "name": "shell", - "description": "run a command", - "input_schema": {"type": "object", "properties": {}} - }] - }); - - let response = reqwest::Client::builder() - .no_proxy() - .timeout(Duration::from_secs(5)) - .build() - .unwrap() - .post(format!("{proxy}/v1/messages")) - .header("content-type", "application/json") - .json(&original) - .send() - .await - .unwrap(); - assert_eq!(response.status(), StatusCode::OK); - let client_body: serde_json::Value = response.json().await.unwrap(); - assert_eq!(client_body["type"], "message"); - assert_eq!(client_body["model"], "claude-client"); - assert_eq!(client_body["content"][0]["text"], "bridged"); - - let upstream = captured.lock().unwrap().clone().expect("upstream request"); - assert_eq!(upstream["model"], "upstream-model"); - assert_eq!(upstream["max_completion_tokens"], 32); - assert_eq!(upstream["messages"][0]["content"][0]["text"], "hello"); - assert_eq!(upstream["tools"][0]["function"]["name"], "shell"); - - let _ = stop.send(()); - let _ = mock_stop.send(()); -} - -#[tokio::test] -async fn e2e_messages_bridge_uses_gemini_native_forwarding_and_google_auth() { - let (mock_port, captured, mock_stop) = spawn_capturing_gemini_http().await; - let toml = format!( - r#" -listen = "{{{{LISTEN}}}}" -admin_listen = "{{{{ADMIN}}}}" -agent_id = "t" - -[network] -mode = "public" - -[[models]] -name = "claude-client" -forward = "gemini-2.5-pro" - -[[models]] -name = "gemini-2.5-pro" -provider = "gemini" -upstream = "http://127.0.0.1:{mock_port}/v1beta" -api_key = "gemini-secret" -"#, - ); - let (proxy, _tmp, stop) = spawn_proxy(&toml).await; - let response = reqwest::Client::builder() - .no_proxy() - .timeout(Duration::from_secs(5)) - .build() - .unwrap() - .post(format!("{proxy}/v1/messages")) - .header("content-type", "application/json") - .header("authorization", "Bearer client-secret") - .json(&serde_json::json!({ - "model": "claude-client", - "max_tokens": 32, - "system": "Be concise.", - "messages": [{"role": "user", "content": "hello"}] - })) - .send() - .await - .unwrap(); - - assert_eq!(response.status(), StatusCode::OK); - let client_body: serde_json::Value = response.json().await.unwrap(); - assert_eq!(client_body["type"], "message"); - assert_eq!(client_body["model"], "claude-client"); - assert_eq!(client_body["content"][0]["text"], "native bridged"); - - let upstream = captured.lock().unwrap().clone().expect("upstream request"); - assert_eq!( - upstream["path"], - "/v1beta/models/gemini-2.5-pro:generateContent" - ); - assert_eq!(upstream["query"], serde_json::Value::Null); - assert_eq!(upstream["api_key"], "gemini-secret"); - assert_eq!(upstream["authorization"], serde_json::Value::Null); - assert_eq!(upstream["body"]["contents"][0]["role"], "user"); - assert_eq!(upstream["body"]["contents"][0]["parts"][0]["text"], "hello"); - assert_eq!( - upstream["body"]["systemInstruction"]["parts"][0]["text"], - "Be concise." - ); - assert_eq!(upstream["body"]["generationConfig"]["maxOutputTokens"], 32); - - let _ = stop.send(()); - let _ = mock_stop.send(()); -} - -#[tokio::test] -async fn e2e_completions_bridge_uses_gemini_native_stream_endpoint() { - let (mock_port, captured, mock_stop) = spawn_capturing_gemini_http().await; - let toml = format!( - r#" -listen = "{{{{LISTEN}}}}" -admin_listen = "{{{{ADMIN}}}}" -agent_id = "t" - -[network] -mode = "public" - -[[models]] -name = "gemini-2.5-pro" -provider = "gemini" -upstream = "http://127.0.0.1:{mock_port}/v1beta" -api_key = "gemini-secret" -"#, - ); - let (proxy, _tmp, stop) = spawn_proxy(&toml).await; - let response = reqwest::Client::builder() - .no_proxy() - .timeout(Duration::from_secs(5)) - .build() - .unwrap() - .post(format!("{proxy}/v1/chat/completions")) - .header("content-type", "application/json") - .json(&serde_json::json!({ - "model": "gemini-2.5-pro", - "stream": true, - "messages": [{"role": "user", "content": "hello"}] - })) - .send() - .await - .unwrap(); - - assert_eq!(response.status(), StatusCode::OK); - assert_eq!( - response - .headers() - .get("content-type") - .and_then(|value| value.to_str().ok()), - Some("text/event-stream") - ); - let client_sse = response.text().await.unwrap(); - assert!(client_sse.contains("chat.completion.chunk"), "{client_sse}"); - assert!(client_sse.contains("native streamed"), "{client_sse}"); - assert!(client_sse.contains("data: [DONE]"), "{client_sse}"); - - let upstream = captured.lock().unwrap().clone().expect("upstream request"); - assert_eq!( - upstream["path"], - "/v1beta/models/gemini-2.5-pro:streamGenerateContent" - ); - assert_eq!(upstream["query"], "alt=sse"); - assert_eq!(upstream["api_key"], "gemini-secret"); - assert!(upstream["body"].get("stream").is_none()); - - let _ = stop.send(()); - let _ = mock_stop.send(()); -} - -#[tokio::test] -async fn e2e_injected_pvisor_controller_denies_model_before_upstream() { - let (mock_port, mock_stop) = spawn_mock_http().await; - let toml = format!( - r#" -listen = "{{{{LISTEN}}}}" -admin_listen = "{{{{ADMIN}}}}" -agent_id = "t" - -[network] -mode = "public" - -[[models]] -name = "*" -upstream = "http://127.0.0.1:{mock_port}/v1" -"# - ); - let (proxy, _tmp, stop) = - spawn_proxy_with_controller(&toml, Arc::new(DenyModelController)).await; - - let client = reqwest::Client::builder() - .no_proxy() - .timeout(Duration::from_secs(5)) - .build() - .unwrap(); - let resp = client - .post(format!("{proxy}/v1/chat/completions")) - .header("content-type", "application/json") - .body(r#"{"model":"test","messages":[{"role":"user","content":"hi"}]}"#) - .send() - .await - .unwrap(); - assert_eq!(resp.status(), StatusCode::FORBIDDEN); - assert!( - resp.text().await.unwrap().contains("model-not-allowed"), - "denial should expose the stable pVisor reason" - ); - let _ = stop.send(()); - let _ = mock_stop.send(()); -} - -#[tokio::test] -async fn e2e_public_mode_allows_absolute_uri_forward() { - let (mock_port, mock_stop) = spawn_mock_http().await; - let (proxy, _tmp, stop) = spawn_proxy( - r#" -listen = "{{LISTEN}}" -admin_listen = "{{ADMIN}}" -agent_id = "t" - -[network] -mode = "public" - -[[models]] -name = "*" -upstream = "http://127.0.0.1:9/v1" -"#, - ) - .await; - - let client = reqwest::Client::builder() - .proxy(reqwest::Proxy::all(&proxy).unwrap()) - .timeout(Duration::from_secs(5)) - .build() - .unwrap(); - let resp = client - .get(format!("http://127.0.0.1:{mock_port}/")) - .send() - .await - .unwrap(); - assert_eq!(resp.status(), StatusCode::OK); - let _ = stop.send(()); - let _ = mock_stop.send(()); -} - -#[tokio::test] -async fn e2e_absolute_uri_llm_path_still_checks_host() { - let (proxy, _tmp, stop) = spawn_proxy( - r#" -listen = "{{LISTEN}}" -admin_listen = "{{ADMIN}}" -agent_id = "t" - -[network] -mode = "allowlist" -allowed_hosts = [ - "pypi.org", -] - -[[models]] -name = "*" -upstream = "http://127.0.0.1:9/v1" -"#, - ) - .await; - - let client = reqwest::Client::builder() - .proxy(reqwest::Proxy::all(&proxy).unwrap()) - .timeout(Duration::from_secs(5)) - .build() - .unwrap(); - // Absolute-URI LLM path to an unlisted host must be denied before capture/upstream. - let resp = client - .post("http://api.openai.com/v1/chat/completions") - .header("content-type", "application/json") - .body(r#"{"model":"gpt","messages":[{"role":"user","content":"x"}]}"#) - .send() - .await - .unwrap(); - assert_eq!(resp.status(), StatusCode::FORBIDDEN); - let body = resp.text().await.unwrap(); - assert!(body.contains("api.openai.com"), "{body}"); - let _ = stop.send(()); -} diff --git a/crates/persisting-gateway/tests/support/ag_capture_cases.rs b/crates/persisting-gateway/tests/support/ag_capture_cases.rs deleted file mode 100644 index 09b41b6c5..000000000 --- a/crates/persisting-gateway/tests/support/ag_capture_cases.rs +++ /dev/null @@ -1,130 +0,0 @@ -//! Expected capture outcomes for agentgateway fixture files. - -/// User-turn extraction expectations for a request fixture. -pub struct UserCaptureCase { - pub path: &'static str, - pub min_turns: usize, - pub must_contain: &'static [&'static str], -} - -pub const USER_CAPTURE_CASES: &[UserCaptureCase] = &[ - UserCaptureCase { - path: "requests/completions/basic.json", - min_turns: 1, - must_contain: &[], - }, - UserCaptureCase { - path: "requests/completions/full.json", - min_turns: 1, - must_contain: &["What's in this image?", "[image: base64:"], - }, - UserCaptureCase { - path: "requests/completions/tool-call.json", - min_turns: 1, - must_contain: &["Columbus"], - }, - UserCaptureCase { - path: "requests/completions/parallel-tool-call.json", - min_turns: 1, - must_contain: &["Columbus"], - }, - UserCaptureCase { - path: "requests/messages/basic.json", - min_turns: 1, - must_contain: &["Hello"], - }, - UserCaptureCase { - path: "requests/messages/tools.json", - min_turns: 1, - must_contain: &[], - }, - UserCaptureCase { - path: "requests/responses/basic.json", - min_turns: 1, - must_contain: &[], - }, - UserCaptureCase { - path: "requests/responses/assistant-history.json", - min_turns: 1, - must_contain: &["Follow up question"], - }, - UserCaptureCase { - path: "local/requests/responses/codex_basic.json", - min_turns: 1, - must_contain: &[], - }, - UserCaptureCase { - path: "local/requests/responses/codex_tool_roundtrip.json", - min_turns: 1, - must_contain: &["```tool_result:", "Cargo.toml"], - }, -]; - -/// Assistant text extraction from JSON response bodies. -pub struct AssistantJsonCase { - pub path: &'static str, - pub must_contain: &'static [&'static str], -} - -pub const ASSISTANT_JSON_CASES: &[AssistantJsonCase] = &[ - AssistantJsonCase { - path: "response/completions/basic.json", - must_contain: &["Sorry"], - }, - AssistantJsonCase { - path: "local/response/completions/tool_call.json", - must_contain: &["```tool:shell", "ls"], - }, - AssistantJsonCase { - path: "response/anthropic/basic.json", - must_contain: &["Hi there"], - }, - AssistantJsonCase { - path: "response/anthropic/tool.json", - must_contain: &["```tool:get_weather", "San Francisco"], - }, - AssistantJsonCase { - path: "response/responses/basic.json", - must_contain: &["Hello"], - }, -]; - -/// Assistant extraction from SSE wire fixtures (OpenAI completions or Responses). -pub struct AssistantSseCase { - pub path: &'static str, - pub translate_completions_to_messages: bool, - pub must_contain: &'static [&'static str], -} - -pub const ASSISTANT_SSE_CASES: &[AssistantSseCase] = &[ - AssistantSseCase { - path: "local/response/completions/stream_head.txt", - translate_completions_to_messages: true, - must_contain: &["Hi"], - }, - AssistantSseCase { - path: "local/response/completions/stream_tool_call.txt", - translate_completions_to_messages: false, - must_contain: &["```tool:shell", "ls"], - }, - AssistantSseCase { - path: "response/completions/stream.json", - translate_completions_to_messages: true, - must_contain: &["Hi", "help"], - }, - AssistantSseCase { - path: "response/responses/stream-image.json", - translate_completions_to_messages: false, - must_contain: &["[image_generated:", "gray tabby cat"], - }, -]; - -/// Fixtures that should yield non-zero token usage when parsed. -pub const USAGE_JSON_FIXTURES: &[&str] = &[ - "response/completions/basic.json", - "response/completions/gemini_with_completion_tokens.json", - "response/completions/gemini_zero_completion_tokens.json", - "response/anthropic/basic.json", - "response/anthropic/tool.json", - "response/responses/basic.json", -]; diff --git a/crates/persisting-gateway/tests/support/ag_fixtures.rs b/crates/persisting-gateway/tests/support/ag_fixtures.rs deleted file mode 100644 index cfd6b1dac..000000000 --- a/crates/persisting-gateway/tests/support/ag_fixtures.rs +++ /dev/null @@ -1,299 +0,0 @@ -//! Helpers for agentgateway LLM fixtures under `tests/fixtures/`. -//! -//! Fixture provenance and Apache-2.0 attribution: see `tests/fixtures/README.md`. - -use std::fs; -use std::path::PathBuf; - -use bytes::Bytes; -use serde_json::{Value, json}; - -/// Root of agentgateway LLM fixtures (`crates/persisting-gateway/tests/fixtures/`). -pub fn fixtures_root() -> PathBuf { - PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures") -} - -pub fn fixture_path(relative: &str) -> PathBuf { - fixtures_root().join(relative) -} - -pub fn read_fixture(relative: &str) -> String { - fs::read_to_string(fixture_path(relative)) - .unwrap_or_else(|e| panic!("read fixture {relative}: {e}")) -} - -pub fn read_fixture_bytes(relative: &str) -> Bytes { - Bytes::from( - fs::read(fixture_path(relative)) - .unwrap_or_else(|e| panic!("read fixture bytes {relative}: {e}")), - ) -} - -/// Strip insta YAML frontmatter (`---` … `---`) from AG snapshot files. -pub fn strip_snap_frontmatter(raw: &str) -> &str { - if let Some(stripped) = raw.strip_prefix("---\n") - && let Some(end) = stripped.find("\n---\n") - { - return stripped[end + 5..].trim_start(); - } - raw.trim() -} - -/// Parse JSON body from an AG `.snap` file (after frontmatter). -pub fn parse_ag_json_snap(relative: &str) -> Value { - let raw = read_fixture(relative); - let body = strip_snap_frontmatter(&raw); - serde_json::from_str(body).unwrap_or_else(|e| panic!("parse snap JSON {relative}: {e}")) -} - -/// AG response snaps wrap wire output in `{ "response": ..., "parsed": ... }`. -pub fn ag_snap_response(relative: &str) -> Value { - let v = parse_ag_json_snap(relative); - v.get("response") - .cloned() - .unwrap_or_else(|| panic!("snap {relative} missing .response")) -} - -/// AG request snaps wrap wire output in `{ "request": ..., "parsed": ... }`. -pub fn ag_snap_request(relative: &str) -> Value { - let value = parse_ag_json_snap(relative); - value.get("request").cloned().unwrap_or(value) -} - -/// AG streaming snaps are SSE text after frontmatter. -pub fn parse_ag_sse_snap(relative: &str) -> String { - strip_snap_frontmatter(&read_fixture(relative)).to_string() -} - -/// Normalize Anthropic message response for comparison with AG snaps. -pub fn normalize_messages_response(v: &mut Value) { - if let Some(id) = v.get_mut("id") { - *id = json!("[id]"); - } - if let Some(content) = v.get_mut("content").and_then(|c| c.as_array_mut()) { - // Persisting bridge emits an empty text block when upstream choice has no content. - if content.len() == 1 - && content[0].get("type").and_then(|t| t.as_str()) == Some("text") - && content[0] - .get("text") - .and_then(|t| t.as_str()) - .is_some_and(str::is_empty) - { - content.clear(); - } - } - if let Some(model) = v.get("model").and_then(|m| m.as_str()) { - // AG snaps keep upstream model from completions response; client_model may differ. - let _ = model; - } -} - -pub fn assert_json_eq(actual: &Value, expected: &Value, context: &str) { - assert_eq!(actual, expected, "{context}"); -} - -pub fn assert_messages_response_eq(actual: &Value, expected: &Value, context: &str) { - let mut a = actual.clone(); - let mut e = expected.clone(); - normalize_messages_response(&mut a); - normalize_messages_response(&mut e); - assert_eq!(a, e, "{context}"); -} - -/// Feed full upstream OpenAI SSE fixture through a translator callback. -pub fn translate_openai_sse_fixture(relative: &str, mut translate: F) -> String -where - F: FnMut(&[u8]) -> anyhow::Result, -{ - let raw = read_fixture(relative); - let mut out = Vec::new(); - for chunk in raw.as_bytes().chunks(512) { - out.extend_from_slice(&translate(chunk).expect("translate SSE chunk")); - } - String::from_utf8(out).expect("translated SSE must be UTF-8") -} - -pub fn sse_event_names(sse: &str) -> Vec<&str> { - sse.lines() - .filter(|l| l.starts_with("event: ")) - .map(|l| l.strip_prefix("event: ").unwrap_or("")) - .collect() -} - -/// Parse SSE into protocol events, ignoring agentgateway's trailing parsed-info JSON. -pub fn parse_sse_events(sse: &str) -> Vec<(String, Value)> { - let mut events = Vec::new(); - let mut event_name = None; - for line in sse.lines() { - if let Some(name) = line.strip_prefix("event: ") { - event_name = Some(name.to_string()); - } else if let Some(data) = line.strip_prefix("data: ") { - if data == "[DONE]" { - continue; - } - if let (Some(name), Ok(value)) = (event_name.take(), serde_json::from_str(data)) { - events.push((name, value)); - } - } - } - events -} - -pub fn fixture_exists(relative: &str) -> bool { - fixture_path(relative).is_file() -} - -/// Case tables aligned with agentgateway `llm/tests.rs` (Persisting-supported bridges only). -pub const MESSAGES_TO_COMPLETIONS: &[&str] = &[ - "basic", - "cache_control", - "gpt_adaptive_thinking_with_tools", - "metadata", - "reasoning", - "server_tools", - "structured-output", - "system_message", - "tools", -]; - -pub const COMPLETIONS_TO_MESSAGES: &[&str] = &[ - "basic", - "cache_write", - "gemini_with_completion_tokens", - "gemini_zero_completion_tokens", - "openrouter_reasoning", - "audio", - "tool_call", - "truncated_tool_call", -]; - -pub const RESPONSES_TO_COMPLETIONS: &[&str] = &[ - "basic", - "instructions", - "input-list", - "assistant-history", - "parallel-tool-call", -]; - -/// Gemini native `generateContent` request goldens. The upstream corpus names the shared -/// Google wire contract `vertex-gemini`; Gemini API uses the same request/response schema. -pub const COMPLETIONS_TO_GEMINI: &[&str] = &[ - "basic", - "generation-config", - "image-file", - "image-inline", - "multi-turn-tools", - "parallel-tool-call", - "reasoning", - "reasoning_max", - "structured-output", - "tool-call", -]; - -pub const GEMINI_TO_COMPLETIONS: &[&str] = &["basic", "tool", "reasoning", "blocked"]; - -/// Tracks how many fixture cases actually ran (guards against silent `continue` on missing files). -#[derive(Debug, Default, Clone, Copy)] -pub struct CaseReport { - pub ran: usize, - pub skipped: usize, -} - -impl CaseReport { - pub fn record_ran(&mut self) { - self.ran += 1; - } - - pub fn record_skipped(&mut self) { - self.skipped += 1; - } - - pub fn assert_min_ran(&self, min: usize, label: &str) { - assert!( - self.ran >= min, - "{label}: expected >= {min} cases, ran {} skipped {}", - self.ran, - self.skipped - ); - } -} - -pub fn load_json_fixture(relative: &str) -> Value { - serde_json::from_slice(&read_fixture_bytes(relative)) - .unwrap_or_else(|e| panic!("parse JSON fixture {relative}: {e}")) -} - -pub fn for_each_existing(relative_paths: &[&str], mut f: impl FnMut(&str)) -> CaseReport { - let mut report = CaseReport::default(); - for path in relative_paths { - if fixture_exists(path) { - f(path); - report.record_ran(); - } else { - report.record_skipped(); - } - } - report -} - -pub fn for_each_existing_case( - cases: &[&str], - prefix: &str, - suffix: &str, - mut f: impl FnMut(&str), -) -> CaseReport { - let mut report = CaseReport::default(); - for case in cases { - let path = format!("{prefix}{case}{suffix}"); - if fixture_exists(&path) { - f(case); - report.record_ran(); - } else { - report.record_skipped(); - } - } - report -} - -pub fn messages_completions_snap(case: &str) -> String { - format!("requests/messages/{case}.completions.snap") -} - -pub fn completions_messages_snap(case: &str) -> String { - format!("response/completions/{case}.completions-messages.snap") -} - -pub fn upstream_model_from_messages_fixture(case: &str) -> String { - let v: Value = serde_json::from_str(&read_fixture(&format!("requests/messages/{case}.json"))) - .expect("parse messages fixture"); - v.get("model") - .and_then(|m| m.as_str()) - .unwrap_or("upstream-model") - .to_string() -} - -pub fn client_model_from_completions_fixture(case: &str) -> String { - let v: Value = - serde_json::from_str(&read_fixture(&format!("response/completions/{case}.json"))) - .expect("parse completions fixture"); - v.get("model") - .and_then(|m| m.as_str()) - .unwrap_or("client-model") - .to_string() -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn fixtures_root_exists() { - assert!(fixtures_root().is_dir()); - } - - #[test] - fn parse_basic_completions_snap() { - let v = ag_snap_request("requests/messages/basic.completions.snap"); - assert!(v.get("messages").is_some()); - } -} diff --git a/crates/persisting-gateway/tests/support/mod.rs b/crates/persisting-gateway/tests/support/mod.rs deleted file mode 100644 index a249a1137..000000000 --- a/crates/persisting-gateway/tests/support/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub mod ag_capture_cases; -pub mod ag_fixtures; diff --git a/crates/persisting-overlaynet/Cargo.toml b/crates/persisting-overlaynet/Cargo.toml deleted file mode 100644 index e72e2c357..000000000 --- a/crates/persisting-overlaynet/Cargo.toml +++ /dev/null @@ -1,27 +0,0 @@ -[package] -name = "persisting-overlaynet" -version.workspace = true -edition.workspace = true -authors.workspace = true -license.workspace = true -description = "Network interception and egress policy for trajectory capture" - -[dependencies] -anyhow.workspace = true -async-trait.workspace = true -axum = { workspace = true, features = ["http1", "tokio"] } -futures-util = { workspace = true, features = ["async-await", "std"] } -hyper.workspace = true -hyper-util = { workspace = true, features = ["tokio"] } -persisting-agentctl.workspace = true -reqwest = { workspace = true, features = ["rustls-tls", "stream"] } -serde = { workspace = true, features = ["derive"] } -smoltcp = { workspace = true, features = ["std", "log", "medium-ethernet", "proto-ipv4", "proto-dhcpv4", "proto-dns", "socket-tcp", "socket-udp", "socket-tcp-pause-synack"] } -tokio = { workspace = true, features = ["net", "io-util", "rt", "sync", "time"] } -tracing.workspace = true -thiserror.workspace = true - -[dev-dependencies] -http-body-util.workspace = true -proptest.workspace = true -tokio = { workspace = true, features = ["macros", "rt", "test-util"] } diff --git a/crates/persisting-overlaynet/README.md b/crates/persisting-overlaynet/README.md deleted file mode 100644 index 992e77a38..000000000 --- a/crates/persisting-overlaynet/README.md +++ /dev/null @@ -1,29 +0,0 @@ -# persisting-overlaynet - -**Network interception and egress-policy data planes for trajectory capture.** - -Owns the proxy data plane: request classification, HTTP `CONNECT`, -absolute-URI forwarding, access enforcement through -[`persisting-agentctl`](../persisting-agentctl/README.md), request accounting, -shared proxy header safety, and dispatch to one caller-supplied `OverlaySink`. - -Does not own LLM protocol adaptation, upstream selection, session correlation, -capture events, WAL, or pChronicle writes. -[`persisting-gateway`](../persisting-gateway/README.md) implements `OverlaySink` -for those. - -The explicit-proxy profile is deliberately marked `cooperative`: policy -decisions over intercepted requests are not non-bypassable enforcement. -`no-network` and `allowlist` mean "for traffic that reached this proxy"; -direct sockets and clients that remove proxy variables remain ambient. - -## Develop - -```bash -just test persisting-overlaynet -``` - -## Links - -- [`persisting-gateway`](../persisting-gateway/README.md) -- [`persisting-agentctl`](../persisting-agentctl/README.md) diff --git a/crates/persisting-overlaynet/src/bandwidth.rs b/crates/persisting-overlaynet/src/bandwidth.rs deleted file mode 100644 index 0fd0fd90b..000000000 --- a/crates/persisting-overlaynet/src/bandwidth.rs +++ /dev/null @@ -1,269 +0,0 @@ -//! Shared aggregate bandwidth scheduling for intercepted streams. - -use std::collections::HashMap; -use std::sync::Arc; -use std::time::Duration; - -use axum::body::Body; -use futures_util::StreamExt; -use tokio::sync::Mutex; -use tokio::time::Instant; - -use crate::policy::NetworkBandwidthLimit; - -#[derive(Debug, Clone, Default)] -pub struct BandwidthRegistry { - buckets: Arc>>>, -} - -impl BandwidthRegistry { - pub(crate) async fn session(&self, limits: Vec) -> BandwidthSession { - let mut registry = self.buckets.lock().await; - let mut buckets = Vec::with_capacity(limits.len()); - for limit in limits { - let rate = limit.bytes_per_second; - let bucket = registry - .entry(limit) - .or_insert_with(|| Arc::new(BandwidthBucket::new(rate))) - .clone(); - if !buckets.iter().any(|current| Arc::ptr_eq(current, &bucket)) { - buckets.push(bucket); - } - } - BandwidthSession { buckets } - } -} - -#[derive(Debug, Clone, Default)] -pub(crate) struct BandwidthSession { - buckets: Vec>, -} - -impl BandwidthSession { - pub(crate) async fn throttle(&self, bytes: usize) { - if bytes == 0 || self.buckets.is_empty() { - return; - } - let mut ready_at = Instant::now(); - for bucket in &self.buckets { - ready_at = ready_at.max(bucket.reserve(bytes).await); - } - tokio::time::sleep_until(ready_at).await; - } - - pub(crate) fn is_limited(&self) -> bool { - !self.buckets.is_empty() - } -} - -#[derive(Debug)] -struct BandwidthBucket { - bytes_per_second: u64, - next_available: Mutex, -} - -impl BandwidthBucket { - fn new(bytes_per_second: u64) -> Self { - Self { - bytes_per_second, - next_available: Mutex::new(Instant::now()), - } - } - - async fn reserve(&self, bytes: usize) -> Instant { - let now = Instant::now(); - let mut next_available = self.next_available.lock().await; - let starts_at = (*next_available).max(now); - let nanos = (bytes as u128) - .saturating_mul(1_000_000_000) - .div_ceil(self.bytes_per_second as u128) - .min(u64::MAX as u128) as u64; - let ready_at = starts_at + Duration::from_nanos(nanos); - *next_available = ready_at; - ready_at - } -} - -pub(crate) fn throttle_body(body: Body, bandwidth: BandwidthSession) -> Body { - if !bandwidth.is_limited() { - return body; - } - Body::from_stream(body.into_data_stream().then(move |result| { - let bandwidth = bandwidth.clone(); - async move { - if let Ok(bytes) = &result { - bandwidth.throttle(bytes.len()).await; - } - result - } - })) -} - -#[cfg(test)] -mod tests { - use super::*; - use http_body_util::BodyExt; - use proptest::prelude::*; - - proptest! { - #[test] - fn sequential_reservations_have_the_declared_duration( - bytes_per_second in 1u64..=1_000_000_000, - bytes in 0usize..=100_000, - ) { - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_time() - .build() - .unwrap(); - runtime.block_on(async { - let bucket = BandwidthBucket::new(bytes_per_second); - let first = bucket.reserve(bytes).await; - let second = bucket.reserve(bytes).await; - let expected_nanos = (bytes as u128) - .saturating_mul(1_000_000_000) - .div_ceil(bytes_per_second as u128) - .min(u64::MAX as u128) as u64; - prop_assert!( - second.duration_since(first) >= Duration::from_nanos(expected_nanos), - "reservation interval was shorter than the declared rate" - ); - Ok(()) - })?; - } - - #[test] - fn registry_session_contains_one_bucket_per_distinct_limit( - limits in prop::collection::vec( - ( - prop::option::of(proptest::string::string_regex("[a-z]{1,8}\\.example\\.com").unwrap()), - prop::option::of(1u16..=u16::MAX), - 1u64..=1_000_000_000, - ).prop_map(|(host, port, bytes_per_second)| NetworkBandwidthLimit { - host, - port, - bytes_per_second, - }), - 0..16, - ), - ) { - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_time() - .build() - .unwrap(); - runtime.block_on(async { - let expected = limits.iter().collect::>().len(); - let session = BandwidthRegistry::default().session(limits).await; - prop_assert_eq!(session.buckets.len(), expected); - Ok(()) - })?; - } - } - - #[tokio::test] - async fn registry_shares_identical_limits_between_sessions() { - let registry = BandwidthRegistry::default(); - let limit = NetworkBandwidthLimit { - host: None, - port: None, - bytes_per_second: 1_000, - }; - let first = registry.session(vec![limit.clone()]).await; - let second = registry.session(vec![limit]).await; - assert!(Arc::ptr_eq(&first.buckets[0], &second.buckets[0])); - } - - #[tokio::test] - async fn reservation_serializes_aggregate_bytes() { - let bucket = BandwidthBucket::new(1_000); - let first = bucket.reserve(1_000).await; - let second = bucket.reserve(1_000).await; - assert!(second.duration_since(first) >= Duration::from_secs(1)); - } - - #[tokio::test(start_paused = true)] - async fn throttle_waits_for_reserved_capacity() { - let registry = BandwidthRegistry::default(); - let session = registry - .session(vec![NetworkBandwidthLimit { - host: None, - port: None, - bytes_per_second: 1_000, - }]) - .await; - let task = tokio::spawn(async move { session.throttle(1_000).await }); - tokio::task::yield_now().await; - assert!(!task.is_finished()); - tokio::time::advance(Duration::from_millis(999)).await; - assert!(!task.is_finished()); - tokio::time::advance(Duration::from_millis(1)).await; - task.await.unwrap(); - } - - #[tokio::test(start_paused = true)] - async fn identical_limits_aggregate_concurrent_sessions() { - let registry = BandwidthRegistry::default(); - let limit = NetworkBandwidthLimit { - host: None, - port: None, - bytes_per_second: 1_000, - }; - let first = registry.session(vec![limit.clone()]).await; - let second = registry.session(vec![limit]).await; - let first = tokio::spawn(async move { first.throttle(1_000).await }); - let second = tokio::spawn(async move { second.throttle(1_000).await }); - tokio::task::yield_now().await; - tokio::task::yield_now().await; - - tokio::time::advance(Duration::from_secs(1)).await; - tokio::task::yield_now().await; - assert_ne!(first.is_finished(), second.is_finished()); - tokio::time::advance(Duration::from_secs(1)).await; - first.await.unwrap(); - second.await.unwrap(); - } - - #[tokio::test(start_paused = true)] - async fn stacked_limits_use_the_strictest_schedule() { - let registry = BandwidthRegistry::default(); - let session = registry - .session(vec![ - NetworkBandwidthLimit { - host: None, - port: None, - bytes_per_second: 1_000, - }, - NetworkBandwidthLimit { - host: Some("api.example.com".into()), - port: None, - bytes_per_second: 500, - }, - ]) - .await; - let task = tokio::spawn(async move { session.throttle(1_000).await }); - tokio::task::yield_now().await; - tokio::time::advance(Duration::from_millis(1_999)).await; - assert!(!task.is_finished()); - tokio::time::advance(Duration::from_millis(1)).await; - task.await.unwrap(); - } - - #[tokio::test(start_paused = true)] - async fn throttled_body_preserves_bytes_and_obeys_schedule() { - let registry = BandwidthRegistry::default(); - let session = registry - .session(vec![NetworkBandwidthLimit { - host: None, - port: None, - bytes_per_second: 1_000, - }]) - .await; - let body = throttle_body(Body::from(vec![b'x'; 1_000]), session); - let task = tokio::spawn(async move { body.collect().await.unwrap().to_bytes() }); - tokio::task::yield_now().await; - assert!(!task.is_finished()); - tokio::time::advance(Duration::from_secs(1)).await; - let bytes = task.await.unwrap(); - assert_eq!(bytes.len(), 1_000); - assert!(bytes.iter().all(|byte| *byte == b'x')); - } -} diff --git a/crates/persisting-overlaynet/src/egress.rs b/crates/persisting-overlaynet/src/egress.rs deleted file mode 100644 index ab116eec1..000000000 --- a/crates/persisting-overlaynet/src/egress.rs +++ /dev/null @@ -1,144 +0,0 @@ -//! VM egress authorization, resolution, and rate limiting. - -use std::net::SocketAddr; -use std::sync::Arc; -use std::time::Duration; - -use crate::bandwidth::{BandwidthRegistry, BandwidthSession}; -use crate::policy::{DenyReason, NetworkPolicy}; -use crate::resolver::{ - ResolvedAddressPolicy, TargetAuthorizationError, authorize_target_with_policy, -}; -use persisting_agentctl::{ - AttemptId, ControlController, NetworkAccessRequest, NetworkTransport, RunId, StorylineId, -}; -use tokio::net::TcpStream; -use tokio::time::timeout; - -pub(crate) const CONNECT_TIMEOUT: Duration = Duration::from_secs(10); - -#[derive(Debug, Clone, Default)] -pub struct EgressContext { - pub run_id: Option, - pub attempt_id: Option, - pub storyline_id: Option, -} - -#[derive(Debug, thiserror::Error)] -pub(crate) enum EgressError { - #[error("egress denied: {0:?}")] - Denied(DenyReason), - #[error("egress resolution failed: {0:#}")] - Resolve(anyhow::Error), - #[error("egress connection to {host}:{port} timed out")] - ConnectTimeout { host: String, port: u16 }, - #[error("egress connection to {host}:{port} failed: {source}")] - Connect { - host: String, - port: u16, - #[source] - source: std::io::Error, - }, -} - -/// Attempt-scoped authorization services for transparent VM flows. -/// -/// The compiled policy remains an invariant and an injected controller may -/// only narrow it. The injected registry shares identical bandwidth buckets -/// with the explicit proxy running in the same Attempt. -#[derive(Clone)] -pub struct EgressRuntime { - policy: NetworkPolicy, - controller: Arc, - bandwidth: BandwidthRegistry, -} - -impl std::fmt::Debug for EgressRuntime { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("EgressRuntime") - .field("policy", &self.policy) - .finish_non_exhaustive() - } -} - -impl EgressRuntime { - pub fn with_bandwidth_registry( - policy: NetworkPolicy, - controller: Arc, - bandwidth: BandwidthRegistry, - ) -> Self { - Self { - policy, - controller, - bandwidth, - } - } - - pub(crate) async fn authorize_tcp( - &self, - context: &EgressContext, - host: &str, - port: u16, - ) -> Result<(Vec, BandwidthSession), EgressError> { - let request = NetworkAccessRequest { - run_id: context.run_id.clone().map(RunId), - attempt_id: context.attempt_id.clone().map(AttemptId), - storyline_id: context.storyline_id.clone().map(StorylineId), - host: host.to_owned(), - port: Some(port), - transport: NetworkTransport::TcpTunnel, - resolved_ip: None, - }; - let authorized = authorize_target_with_policy( - self.controller.as_ref(), - &self.policy, - request, - ResolvedAddressPolicy::HostConnectorAliases, - ) - .await - .map_err(|error| match error { - TargetAuthorizationError::Denied(reason) => EgressError::Denied(reason), - TargetAuthorizationError::Resolve(error) => EgressError::Resolve(error), - })?; - let bandwidth = self - .bandwidth - .session( - self.policy - .matching_limits(host, Some(port), &authorized.addresses), - ) - .await; - Ok((authorized.addresses, bandwidth)) - } -} - -pub(crate) async fn connect_tcp_addresses( - addresses: &[SocketAddr], - host: &str, - port: u16, -) -> Result { - let mut last_error = None; - timeout(CONNECT_TIMEOUT, async { - for address in addresses { - match TcpStream::connect(address).await { - Ok(stream) => return Ok(stream), - Err(error) => last_error = Some(error), - } - } - Err(last_error.unwrap_or_else(|| { - std::io::Error::new( - std::io::ErrorKind::AddrNotAvailable, - "no authorized address", - ) - })) - }) - .await - .map_err(|_| EgressError::ConnectTimeout { - host: host.to_owned(), - port, - })? - .map_err(|source| EgressError::Connect { - host: host.to_owned(), - port, - source, - }) -} diff --git a/crates/persisting-overlaynet/src/forward.rs b/crates/persisting-overlaynet/src/forward.rs deleted file mode 100644 index 746fcc242..000000000 --- a/crates/persisting-overlaynet/src/forward.rs +++ /dev/null @@ -1,277 +0,0 @@ -//! Explicit HTTP proxy forwarding: CONNECT tunnel and absolute-URI requests. - -use axum::body::Body; -use axum::extract::Request; -use axum::http::uri::Authority; -use axum::http::{Method, StatusCode, Uri}; -use axum::response::{IntoResponse, Response}; -use futures_util::StreamExt; -use hyper::upgrade::OnUpgrade; -use hyper_util::rt::TokioIo; -use std::time::Duration; -use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; - -use crate::bandwidth::BandwidthSession; -use crate::egress::{CONNECT_TIMEOUT, connect_tcp_addresses}; -use crate::headers::skip_transparent_forward_header_for; -use crate::resolver::AuthorizedTarget; - -#[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) struct ConnectTarget { - pub host: String, - pub port: u16, - pub authority: String, -} - -pub(crate) async fn handle_connect_authorized( - req: Request, - target: ConnectTarget, - authorized: &AuthorizedTarget, - bandwidth: BandwidthSession, -) -> anyhow::Result { - let on_upgrade: OnUpgrade = hyper::upgrade::on(req); - // Establish the upstream before reporting success. Returning 200 first - // makes a refused or unroutable destination look like an accepted tunnel. - let mut destination = connect_tcp_addresses(&authorized.addresses, &target.host, target.port) - .await - .map_err(|error| anyhow::anyhow!("CONNECT to {} failed: {error}", target.authority))?; - tokio::spawn(async move { - let Ok(upgraded) = on_upgrade.await else { - return; - }; - let client = TokioIo::new(upgraded); - let (client_read, client_write) = tokio::io::split(client); - let (destination_read, destination_write) = destination.split(); - let upload = copy_limited(client_read, destination_write, bandwidth.clone()); - let download = copy_limited(destination_read, client_write, bandwidth); - let _ = tokio::join!(upload, download); - }); - Ok(Response::builder() - .status(StatusCode::OK) - .body(Body::empty()) - .expect("CONNECT response") - .into_response()) -} - -pub(crate) fn parse_connect_target(authority: &str) -> anyhow::Result { - if authority.contains('@') { - anyhow::bail!("CONNECT authority must not include user information"); - } - let parsed = authority - .parse::() - .map_err(|error| anyhow::anyhow!("invalid CONNECT authority `{authority}`: {error}"))?; - let parsed_host = parsed.host(); - let host = parsed_host - .strip_prefix('[') - .and_then(|host| host.strip_suffix(']')) - .unwrap_or(parsed_host); - if host.is_empty() { - anyhow::bail!("CONNECT authority must include a host"); - } - let explicit_port = if authority.starts_with('[') { - authority - .find(']') - .and_then(|end| authority[end + 1..].strip_prefix(':')) - } else { - authority.rsplit_once(':').map(|(_, port)| port) - }; - let port = explicit_port - .map(|port| { - port.parse::() - .map_err(|_| anyhow::anyhow!("CONNECT authority port is out of range")) - }) - .transpose()? - .unwrap_or(443); - if port == 0 { - anyhow::bail!("CONNECT authority port must not be zero"); - } - let normalized_authority = if host.contains(':') { - format!("[{host}]:{port}") - } else { - format!("{host}:{port}") - }; - Ok(ConnectTarget { - host: host.to_string(), - port, - authority: normalized_authority, - }) -} - -pub fn is_forward_proxy_request(method: &Method, uri: &Uri) -> bool { - method == Method::CONNECT || uri.scheme().is_some() -} - -pub(crate) async fn transparent_forward_authorized( - req: Request, - target: &AuthorizedTarget, - bandwidth: BandwidthSession, -) -> anyhow::Result> { - let (parts, body) = req.into_parts(); - let url = parts.uri.to_string(); - - let mut client = reqwest::Client::builder() - .no_proxy() - .redirect(reqwest::redirect::Policy::none()) - .connect_timeout(CONNECT_TIMEOUT) - .timeout(Duration::from_secs(600)); - if target.host.parse::().is_err() { - client = client.resolve_to_addrs(&target.host, &target.addresses); - } - let client = client - .build() - .map_err(|error| anyhow::anyhow!("build pinned forward client: {error}"))?; - let mut request = client.request(parts.method, url); - for (name, value) in &parts.headers { - if !skip_transparent_forward_header_for(&parts.headers, name.as_str()) { - request = request.header(name, value); - } - } - let upload_bandwidth = bandwidth.clone(); - let upload = body.into_data_stream().then(move |result| { - let bandwidth = upload_bandwidth.clone(); - async move { - if let Ok(bytes) = &result { - bandwidth.throttle(bytes.len()).await; - } - result - } - }); - let response = request - .body(reqwest::Body::wrap_stream(upload)) - .send() - .await - .map_err(|error| anyhow::anyhow!("forward request: {error}"))?; - let status = response.status(); - let headers = response.headers().clone(); - - let mut builder = Response::builder().status(status); - for (name, value) in &headers { - if !skip_transparent_forward_header_for(&headers, name.as_str()) { - builder = builder.header(name, value); - } - } - let download = response.bytes_stream().then(move |result| { - let bandwidth = bandwidth.clone(); - async move { - if let Ok(bytes) = &result { - bandwidth.throttle(bytes.len()).await; - } - result - } - }); - builder - .body(Body::from_stream(download)) - .map_err(|error| anyhow::anyhow!("build response: {error}")) -} - -async fn copy_limited( - mut reader: R, - mut writer: W, - bandwidth: BandwidthSession, -) -> std::io::Result -where - R: AsyncRead + Unpin, - W: AsyncWrite + Unpin, -{ - let mut copied = 0_u64; - let mut buffer = vec![0_u8; 16 * 1024]; - loop { - let read = reader.read(&mut buffer).await?; - if read == 0 { - writer.shutdown().await?; - return Ok(copied); - } - bandwidth.throttle(read).await; - writer.write_all(&buffer[..read]).await?; - copied = copied.saturating_add(read as u64); - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::bandwidth::BandwidthRegistry; - use crate::policy::NetworkBandwidthLimit; - - #[test] - fn detects_explicit_proxy_requests() { - let uri: Uri = "http://example.com/path".parse().unwrap(); - assert!(is_forward_proxy_request(&Method::GET, &uri)); - assert!(!is_forward_proxy_request( - &Method::GET, - &"/v1/messages".parse().unwrap() - )); - } - - #[test] - fn connect_target_defaults_to_https_port() { - assert_eq!( - parse_connect_target("example.com").unwrap(), - ConnectTarget { - host: "example.com".into(), - port: 443, - authority: "example.com:443".into(), - } - ); - assert_eq!( - parse_connect_target("[::1]").unwrap(), - ConnectTarget { - host: "::1".into(), - port: 443, - authority: "[::1]:443".into(), - } - ); - assert_eq!(parse_connect_target("example.com:8443").unwrap().port, 8443); - assert!(parse_connect_target(":443").is_err()); - assert!(parse_connect_target("example.com:0").is_err()); - assert!(parse_connect_target("user@example.com:443").is_err()); - } - - #[test] - fn rejects_malformed_connect_authorities() { - for authority in [ - "", - ":443", - "example.com:0", - "example.com:65536", - "user@example.com:443", - "example.com/path", - "[::1", - "::1:443", - "example.com:443 extra", - ] { - assert!( - parse_connect_target(authority).is_err(), - "accepted malformed authority {authority:?}" - ); - } - } - - #[tokio::test(start_paused = true)] - async fn limited_copy_delays_and_preserves_tunnel_bytes() { - let registry = BandwidthRegistry::default(); - let bandwidth = registry - .session(vec![NetworkBandwidthLimit { - host: None, - port: None, - bytes_per_second: 1_000, - }]) - .await; - let (mut source_writer, source_reader) = tokio::io::duplex(2_048); - let (destination_writer, mut destination_reader) = tokio::io::duplex(2_048); - source_writer.write_all(&vec![b'x'; 1_000]).await.unwrap(); - source_writer.shutdown().await.unwrap(); - let copy = tokio::spawn(copy_limited(source_reader, destination_writer, bandwidth)); - let read = tokio::spawn(async move { - let mut bytes = Vec::new(); - destination_reader.read_to_end(&mut bytes).await.unwrap(); - bytes - }); - tokio::task::yield_now().await; - assert!(!read.is_finished()); - tokio::time::advance(Duration::from_secs(1)).await; - assert_eq!(copy.await.unwrap().unwrap(), 1_000); - let bytes = read.await.unwrap(); - assert_eq!(bytes, vec![b'x'; 1_000]); - } -} diff --git a/crates/persisting-overlaynet/src/headers.rs b/crates/persisting-overlaynet/src/headers.rs deleted file mode 100644 index 9a25f1242..000000000 --- a/crates/persisting-overlaynet/src/headers.rs +++ /dev/null @@ -1,258 +0,0 @@ -//! Shared HTTP header filtering for proxy forwarding. - -use axum::http::HeaderMap; - -const HOP_BY_HOP: &[&str] = &[ - "connection", - "proxy-connection", - "keep-alive", - "proxy-authenticate", - "proxy-authorization", - "te", - "trailers", - "transfer-encoding", - "upgrade", -]; - -pub fn is_hop_by_hop(name: &str) -> bool { - HOP_BY_HOP.iter().any(|h| name.eq_ignore_ascii_case(h)) -} - -pub fn skip_transparent_forward_header(name: &str) -> bool { - is_hop_by_hop(name) || name.eq_ignore_ascii_case("host") -} - -/// RFC 9110 allows `Connection` to nominate additional hop-by-hop headers. -/// Those fields must be removed alongside the fixed proxy header set. -pub fn skip_transparent_forward_header_for(headers: &HeaderMap, name: &str) -> bool { - skip_transparent_forward_header(name) || connection_nominates_header(headers, name) -} - -pub fn skip_upstream_forward_header(name: &str) -> bool { - matches!( - name.to_ascii_lowercase().as_str(), - "host" | "content-length" | "x-api-key" | "x-goog-api-key" | "authorization" | "expect" - ) || is_hop_by_hop(name) -} - -pub fn skip_response_header_when_body_changed(name: &str) -> bool { - is_hop_by_hop(name) - || matches!( - name.to_ascii_lowercase().as_str(), - "content-length" | "content-encoding" | "content-type" - ) -} - -/// Filter upstream response headers after a proxy has consumed and rebuilt the -/// response body. HTTP framing is always local to the current hop, so stale -/// transfer metadata must not survive even when the payload bytes are unchanged. -pub fn skip_response_header_after_reframing( - headers: &HeaderMap, - name: &str, - body_changed: bool, -) -> bool { - is_hop_by_hop(name) - || connection_nominates_header(headers, name) - || name.eq_ignore_ascii_case("content-length") - || (body_changed - && matches!( - name.to_ascii_lowercase().as_str(), - "content-encoding" | "content-type" - )) -} - -fn connection_nominates_header(headers: &HeaderMap, name: &str) -> bool { - headers - .get_all("connection") - .iter() - .filter_map(|value| value.to_str().ok()) - .flat_map(|value| value.split(',')) - .map(str::trim) - .any(|token| token.eq_ignore_ascii_case(name)) -} - -pub fn is_websocket_upgrade(headers: &HeaderMap) -> bool { - headers - .get_all("upgrade") - .iter() - .filter_map(|value| value.to_str().ok()) - .flat_map(|value| value.split(',')) - .map(str::trim) - .any(|protocol| protocol.eq_ignore_ascii_case("websocket")) -} - -#[cfg(test)] -mod tests { - use super::*; - use proptest::prelude::*; - - fn token_strategy() -> impl Strategy { - proptest::string::string_regex("[a-z][a-z0-9-]{0,12}").unwrap() - } - - fn hop_header_strategy() -> impl Strategy { - prop_oneof![ - Just("connection"), - Just("proxy-connection"), - Just("keep-alive"), - Just("proxy-authenticate"), - Just("proxy-authorization"), - Just("te"), - Just("trailers"), - Just("transfer-encoding"), - Just("upgrade"), - ] - } - - #[test] - fn filters_hop_by_hop_and_rewritten_body_headers() { - assert!(skip_transparent_forward_header("host")); - assert!(skip_upstream_forward_header("authorization")); - assert!(skip_response_header_when_body_changed("content-length")); - assert!(!skip_response_header_when_body_changed("x-request-id")); - } - - #[test] - fn reframed_responses_drop_upstream_framing_and_connection_extensions() { - let mut headers = HeaderMap::new(); - headers.insert("connection", "keep-alive, x-upstream-hop".parse().unwrap()); - - assert!(skip_response_header_after_reframing( - &headers, - "transfer-encoding", - false - )); - assert!(skip_response_header_after_reframing( - &headers, - "content-length", - false - )); - assert!(skip_response_header_after_reframing( - &headers, - "x-upstream-hop", - false - )); - assert!(!skip_response_header_after_reframing( - &headers, - "content-type", - false - )); - assert!(skip_response_header_after_reframing( - &headers, - "content-type", - true - )); - } - - #[test] - fn connection_header_can_nominate_extra_hop_by_hop_fields() { - let mut headers = HeaderMap::new(); - headers.insert("connection", "keep-alive, x-private-hop".parse().unwrap()); - headers.insert("x-private-hop", "secret".parse().unwrap()); - - assert!(skip_transparent_forward_header_for( - &headers, - "x-private-hop" - )); - assert!(!skip_transparent_forward_header_for( - &headers, - "x-end-to-end" - )); - } - - #[test] - fn every_fixed_hop_header_is_filtered_case_insensitively() { - for name in HOP_BY_HOP { - assert!(is_hop_by_hop(name)); - assert!(is_hop_by_hop(&name.to_ascii_uppercase())); - assert!(skip_transparent_forward_header(name)); - } - assert!(!is_hop_by_hop("x-request-id")); - } - - #[test] - fn multiple_connection_values_nominate_trimmed_tokens() { - let mut headers = HeaderMap::new(); - headers.append("connection", "keep-alive".parse().unwrap()); - headers.append("connection", " x-first , X-Second ".parse().unwrap()); - assert!(skip_transparent_forward_header_for(&headers, "x-first")); - assert!(skip_transparent_forward_header_for(&headers, "x-second")); - assert!(!skip_transparent_forward_header_for(&headers, "x-third")); - } - - #[test] - fn websocket_upgrade_is_a_case_insensitive_protocol_token() { - let mut headers = HeaderMap::new(); - headers.insert("upgrade", "h2c, WebSocket".parse().unwrap()); - assert!(is_websocket_upgrade(&headers)); - headers.insert("upgrade", "h2c".parse().unwrap()); - assert!(!is_websocket_upgrade(&headers)); - } - - proptest! { - #[test] - fn fixed_hop_headers_are_case_insensitive(name in hop_header_strategy()) { - prop_assert!(is_hop_by_hop(name)); - prop_assert!(is_hop_by_hop(&name.to_ascii_uppercase())); - prop_assert!(skip_transparent_forward_header(&name.to_ascii_uppercase())); - } - - #[test] - fn connection_tokens_nominate_headers_with_arbitrary_spacing_and_case( - token in token_strategy(), - uppercase in any::(), - ) { - let nominated = if uppercase { - token.to_ascii_uppercase() - } else { - token.clone() - }; - let mut headers = HeaderMap::new(); - headers.insert( - "connection", - format!(" keep-alive , {nominated} ").parse().unwrap(), - ); - let non_nominated_header = format!("{}-other", token); - prop_assert!(skip_transparent_forward_header_for(&headers, &token)); - prop_assert!(!skip_transparent_forward_header_for( - &headers, - &non_nominated_header, - )); - } - - #[test] - fn reframing_always_removes_content_length_and_conditionally_removes_content_metadata( - body_changed in any::(), - ) { - let headers = HeaderMap::new(); - prop_assert!(skip_response_header_after_reframing( - &headers, - "content-length", - body_changed, - )); - prop_assert_eq!( - skip_response_header_after_reframing(&headers, "content-type", body_changed), - body_changed, - ); - prop_assert_eq!( - skip_response_header_after_reframing(&headers, "content-encoding", body_changed), - body_changed, - ); - } - - #[test] - fn websocket_upgrade_detection_is_token_based_not_substring( - prefix in token_strategy(), - suffix in token_strategy(), - ) { - let mut headers = HeaderMap::new(); - headers.insert( - "upgrade", - format!("{prefix}, WebSocket, {suffix}").parse().unwrap(), - ); - prop_assert!(is_websocket_upgrade(&headers)); - headers.insert("upgrade", format!("{prefix}websocket{suffix}").parse().unwrap()); - prop_assert!(!is_websocket_upgrade(&headers)); - } - } -} diff --git a/crates/persisting-overlaynet/src/interception.rs b/crates/persisting-overlaynet/src/interception.rs deleted file mode 100644 index 60166d6e1..000000000 --- a/crates/persisting-overlaynet/src/interception.rs +++ /dev/null @@ -1,337 +0,0 @@ -//! Observable interception coverage for OverlayNet drivers. -//! -//! Coverage is deliberately separate from egress policy. A deny decision is -//! only an enforcement guarantee when the active driver is non-bypassable. - -use serde::{Deserialize, Serialize}; -use std::sync::Arc; -use std::sync::atomic::{AtomicU64, Ordering}; - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] -pub enum InterceptionDriver { - ExplicitProxy, - VmSmoltcp, - LinuxNetns, - LinuxSeccompNotify, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] -pub enum InterceptionStrength { - /// Only traffic that cooperates with the configured proxy is observed. - Cooperative, - /// The process tree has no network path around the driver. - NonBypassable, -} - -/// Stable description of what an active interception driver actually covers. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct InterceptionProfile { - pub driver: InterceptionDriver, - pub strength: InterceptionStrength, - pub http: bool, - pub tcp_connect: bool, - pub dns: bool, - pub udp: bool, - pub inherited_by_children: bool, -} - -impl InterceptionProfile { - pub const fn explicit_proxy() -> Self { - Self { - driver: InterceptionDriver::ExplicitProxy, - strength: InterceptionStrength::Cooperative, - http: true, - tcp_connect: true, - dns: false, - udp: false, - inherited_by_children: false, - } - } - - pub const fn is_enforcing(&self) -> bool { - matches!(self.strength, InterceptionStrength::NonBypassable) - } - - pub const fn vm_smoltcp() -> Self { - Self { - driver: InterceptionDriver::VmSmoltcp, - strength: InterceptionStrength::NonBypassable, - http: false, - tcp_connect: true, - dns: true, - udp: false, - inherited_by_children: true, - } - } -} - -#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(default)] -pub struct InterceptionSnapshot { - pub requests_seen: u64, - pub policy_allowed: u64, - pub policy_denied: u64, - pub connect_requests: u64, - pub absolute_http_requests: u64, - pub sink_requests: u64, - pub failures: u64, - pub dns_queries: u64, - pub dns_answers: u64, - pub tcp_flows_opened: u64, - pub tcp_flows_denied: u64, - pub tcp_connect_failures: u64, - pub bytes_guest_to_host: u64, - pub bytes_host_to_guest: u64, - pub unsupported_packets: u64, - pub active_tcp_flows: u64, - pub peak_tcp_flows: u64, -} - -#[derive(Default)] -struct Counters { - requests_seen: AtomicU64, - policy_allowed: AtomicU64, - policy_denied: AtomicU64, - connect_requests: AtomicU64, - absolute_http_requests: AtomicU64, - sink_requests: AtomicU64, - failures: AtomicU64, - dns_queries: AtomicU64, - dns_answers: AtomicU64, - tcp_flows_opened: AtomicU64, - tcp_flows_denied: AtomicU64, - tcp_connect_failures: AtomicU64, - bytes_guest_to_host: AtomicU64, - bytes_host_to_guest: AtomicU64, - unsupported_packets: AtomicU64, - active_tcp_flows: AtomicU64, - peak_tcp_flows: AtomicU64, -} - -/// Cloneable, lock-free counters for the traffic that reached OverlayNet. -/// -/// These counters measure intercepted traffic, not traffic that bypassed a -/// cooperative driver. The accompanying [`InterceptionProfile`] carries that -/// distinction. -#[derive(Clone, Default)] -pub struct InterceptionMetrics { - counters: Arc, -} - -impl InterceptionMetrics { - pub fn snapshot(&self) -> InterceptionSnapshot { - InterceptionSnapshot { - requests_seen: self.counters.requests_seen.load(Ordering::Relaxed), - policy_allowed: self.counters.policy_allowed.load(Ordering::Relaxed), - policy_denied: self.counters.policy_denied.load(Ordering::Relaxed), - connect_requests: self.counters.connect_requests.load(Ordering::Relaxed), - absolute_http_requests: self.counters.absolute_http_requests.load(Ordering::Relaxed), - sink_requests: self.counters.sink_requests.load(Ordering::Relaxed), - failures: self.counters.failures.load(Ordering::Relaxed), - dns_queries: self.counters.dns_queries.load(Ordering::Relaxed), - dns_answers: self.counters.dns_answers.load(Ordering::Relaxed), - tcp_flows_opened: self.counters.tcp_flows_opened.load(Ordering::Relaxed), - tcp_flows_denied: self.counters.tcp_flows_denied.load(Ordering::Relaxed), - tcp_connect_failures: self.counters.tcp_connect_failures.load(Ordering::Relaxed), - bytes_guest_to_host: self.counters.bytes_guest_to_host.load(Ordering::Relaxed), - bytes_host_to_guest: self.counters.bytes_host_to_guest.load(Ordering::Relaxed), - unsupported_packets: self.counters.unsupported_packets.load(Ordering::Relaxed), - active_tcp_flows: self.counters.active_tcp_flows.load(Ordering::Relaxed), - peak_tcp_flows: self.counters.peak_tcp_flows.load(Ordering::Relaxed), - } - } - - pub(crate) fn request_seen(&self) { - self.counters.requests_seen.fetch_add(1, Ordering::Relaxed); - } - - pub(crate) fn policy_allowed(&self) { - self.counters.policy_allowed.fetch_add(1, Ordering::Relaxed); - } - - pub(crate) fn policy_denied(&self) { - self.counters.policy_denied.fetch_add(1, Ordering::Relaxed); - } - - pub(crate) fn connect_request(&self) { - self.counters - .connect_requests - .fetch_add(1, Ordering::Relaxed); - } - - pub(crate) fn absolute_http_request(&self) { - self.counters - .absolute_http_requests - .fetch_add(1, Ordering::Relaxed); - } - - pub(crate) fn sink_request(&self) { - self.counters.sink_requests.fetch_add(1, Ordering::Relaxed); - } - - pub(crate) fn failure(&self) { - self.counters.failures.fetch_add(1, Ordering::Relaxed); - } - - pub(crate) fn dns_query(&self) { - self.counters.dns_queries.fetch_add(1, Ordering::Relaxed); - } - - pub(crate) fn dns_answer(&self) { - self.counters.dns_answers.fetch_add(1, Ordering::Relaxed); - } - - pub(crate) fn tcp_flow_opened(&self) { - self.counters - .tcp_flows_opened - .fetch_add(1, Ordering::Relaxed); - let active = self - .counters - .active_tcp_flows - .fetch_add(1, Ordering::Relaxed) - + 1; - self.counters - .peak_tcp_flows - .fetch_max(active, Ordering::Relaxed); - } - - pub(crate) fn tcp_flow_closed(&self) { - let active_tcp_flows = &self.counters.active_tcp_flows; - let mut current = active_tcp_flows.load(Ordering::Relaxed); - while current > 0 { - match active_tcp_flows.compare_exchange_weak( - current, - current - 1, - Ordering::Relaxed, - Ordering::Relaxed, - ) { - Ok(_) => break, - Err(next) => current = next, - } - } - } - - pub(crate) fn tcp_flow_denied(&self) { - self.counters - .tcp_flows_denied - .fetch_add(1, Ordering::Relaxed); - self.policy_denied(); - } - - pub(crate) fn tcp_connect_failure(&self) { - self.counters - .tcp_connect_failures - .fetch_add(1, Ordering::Relaxed); - self.failure(); - } - - pub(crate) fn guest_to_host(&self, bytes: usize) { - self.counters - .bytes_guest_to_host - .fetch_add(bytes as u64, Ordering::Relaxed); - } - - pub(crate) fn host_to_guest(&self, bytes: usize) { - self.counters - .bytes_host_to_guest - .fetch_add(bytes as u64, Ordering::Relaxed); - } - - pub(crate) fn unsupported_packet(&self) { - self.counters - .unsupported_packets - .fetch_add(1, Ordering::Relaxed); - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn explicit_proxy_never_claims_non_bypassable_enforcement() { - let profile = InterceptionProfile::explicit_proxy(); - assert_eq!(profile.strength, InterceptionStrength::Cooperative); - assert!(!profile.is_enforcing()); - assert!(!profile.dns); - assert!(!profile.udp); - } - - #[test] - fn vm_smoltcp_claims_only_implemented_protocols() { - let profile = InterceptionProfile::vm_smoltcp(); - assert!(profile.is_enforcing()); - assert!(profile.tcp_connect); - assert!(profile.dns); - assert!(profile.inherited_by_children); - assert!(!profile.http); - assert!(!profile.udp); - } - - #[test] - fn metrics_snapshots_are_shared_across_clones() { - let metrics = InterceptionMetrics::default(); - let clone = metrics.clone(); - metrics.request_seen(); - clone.policy_denied(); - - assert_eq!( - metrics.snapshot(), - InterceptionSnapshot { - requests_seen: 1, - policy_denied: 1, - ..InterceptionSnapshot::default() - } - ); - } - - #[test] - fn every_metric_updates_only_its_own_counter() { - let metrics = InterceptionMetrics::default(); - metrics.request_seen(); - metrics.policy_allowed(); - metrics.policy_denied(); - metrics.connect_request(); - metrics.absolute_http_request(); - metrics.sink_request(); - metrics.failure(); - assert_eq!( - metrics.snapshot(), - InterceptionSnapshot { - requests_seen: 1, - policy_allowed: 1, - policy_denied: 1, - connect_requests: 1, - absolute_http_requests: 1, - sink_requests: 1, - failures: 1, - ..InterceptionSnapshot::default() - } - ); - } - - #[test] - fn concurrent_metric_updates_are_not_lost() { - let metrics = InterceptionMetrics::default(); - let workers = (0..8) - .map(|_| { - let metrics = metrics.clone(); - std::thread::spawn(move || { - for _ in 0..10_000 { - metrics.request_seen(); - metrics.policy_allowed(); - } - }) - }) - .collect::>(); - for worker in workers { - worker.join().unwrap(); - } - let snapshot = metrics.snapshot(); - assert_eq!(snapshot.requests_seen, 80_000); - assert_eq!(snapshot.policy_allowed, 80_000); - assert_eq!(snapshot.policy_denied, 0); - } -} diff --git a/crates/persisting-overlaynet/src/lib.rs b/crates/persisting-overlaynet/src/lib.rs deleted file mode 100644 index 3f1f783b3..000000000 --- a/crates/persisting-overlaynet/src/lib.rs +++ /dev/null @@ -1,27 +0,0 @@ -//! Network interception and egress policy drivers for trajectory capture. -//! -//! Host/container execution uses the cooperative HTTP proxy. libkrun VM -//! execution uses a non-bypassable virtio-net/smoltcp IPv4 TCP and DNS data -//! plane; unsupported protocols fail closed. - -mod bandwidth; -mod egress; -pub mod forward; -pub mod headers; -pub mod interception; -pub mod policy; -mod resolver; -pub mod server; -pub mod vm; - -pub use bandwidth::BandwidthRegistry; -pub use egress::{EgressContext, EgressRuntime}; - -pub use interception::{ - InterceptionDriver, InterceptionMetrics, InterceptionProfile, InterceptionSnapshot, - InterceptionStrength, -}; -pub use policy::{ - NetworkAccessRule, NetworkBandwidthLimit, NetworkConfig, NetworkMode, NetworkPolicy, -}; -pub use server::{OverlayRequestContext, OverlayServerState, OverlaySink}; diff --git a/crates/persisting-overlaynet/src/policy.rs b/crates/persisting-overlaynet/src/policy.rs deleted file mode 100644 index 56b7b2ffb..000000000 --- a/crates/persisting-overlaynet/src/policy.rs +++ /dev/null @@ -1,624 +0,0 @@ -//! Egress policy for the explicit HTTP proxy backend. - -use axum::http::StatusCode; -use persisting_agentctl::{ - ControlController, ControlMachine, ControlReason, ControlRequest, NetworkGuard, - PolicyControlController, -}; -use persisting_agentctl::{NetworkAccessRequest, NetworkCapability, NetworkDefaultAction}; -pub use persisting_agentctl::{NetworkAccessRule, NetworkBandwidthLimit}; -pub use persisting_agentctl::{ - NetworkRule as AllowedEntry, host_matches, normalize_host, - parse_network_rule as parse_allowed_entry, -}; -use serde::{Deserialize, Serialize}; -use std::net::SocketAddr; - -#[derive(Debug, Clone, Default, Deserialize, Serialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct NetworkConfig { - #[serde(default)] - pub mode: NetworkMode, - #[serde(default)] - pub allowed_hosts: Vec, - /// Structured grants. Prefer these when port, transport, or private-address - /// behavior must be constrained explicitly. - #[serde(default)] - pub rules: Vec, - /// Explicit deny rules. These take precedence over every allow rule. - #[serde(default)] - pub deny_rules: Vec, - /// Aggregate bandwidth constraints. Every matching constraint applies. - #[serde(default)] - pub limits: Vec, -} - -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize, Serialize)] -#[serde(rename_all = "kebab-case")] -pub enum NetworkMode { - #[default] - Public, - NoNetwork, - Allowlist, -} - -/// Minimal configuration view needed to compile an overlaynet policy. -pub trait PolicyConfig { - fn network(&self) -> &NetworkConfig; -} - -#[derive(Debug, Clone)] -pub struct NetworkPolicy { - mode: NetworkMode, - guard: NetworkGuard, - limits: Vec, -} - -#[derive(Debug, Clone)] -struct CompiledBandwidthLimit { - matcher: Option, - config: NetworkBandwidthLimit, -} - -impl NetworkPolicy { - pub fn compile(network: &NetworkConfig) -> anyhow::Result { - anyhow::ensure!( - network.mode == NetworkMode::Allowlist - || (network.allowed_hosts.is_empty() && network.rules.is_empty()), - "network allow entries require mode = \"allowlist\"" - ); - let capability = network_capability(network); - // Gateway-owned upstream routes and the listener itself are not Agent - // egress grants. Only explicit entries from `[network]` reach this guard. - let guard = NetworkGuard::compile(capability, Vec::new())?; - let limits = network - .limits - .iter() - .map(|config| { - anyhow::ensure!( - config.bytes_per_second > 0, - "network bandwidth limit must be greater than zero" - ); - anyhow::ensure!( - config.port != Some(0), - "network limit port must not be zero" - ); - let matcher = config - .host - .as_deref() - .map(parse_allowed_entry) - .transpose()?; - Ok(CompiledBandwidthLimit { - matcher, - config: config.clone(), - }) - }) - .collect::>>()?; - Ok(Self { - mode: network.mode, - guard, - limits, - }) - } - - pub fn from_config(config: &impl PolicyConfig) -> anyhow::Result { - Self::compile(config.network()) - } - - pub fn mode_str(&self) -> &'static str { - match self.mode { - NetworkMode::Public => "public", - NetworkMode::NoNetwork => "no-network", - NetworkMode::Allowlist => "allowlist", - } - } - - pub(crate) fn preflight(&self, request: &NetworkAccessRequest) -> Result<(), DenyReason> { - authorize_egress(&PolicyControlController, self, request) - } - - pub(crate) fn authorize( - &self, - controller: &dyn ControlController, - request: &NetworkAccessRequest, - ) -> Result<(), DenyReason> { - // The compiled policy is an invariant of the data plane. An injected - // controller may further restrict it, but must never be able to widen it. - self.preflight(request)?; - authorize_egress(controller, self, request) - } - - pub(crate) fn matching_limits( - &self, - host: &str, - port: Option, - resolved_addresses: &[SocketAddr], - ) -> Vec { - self.limits - .iter() - .filter(|limit| { - limit - .config - .port - .is_none_or(|expected| port == Some(expected)) - && limit.matcher.as_ref().is_none_or(|matcher| { - host_matches(host, std::slice::from_ref(matcher)) - || resolved_addresses.iter().any(|address| { - host_matches( - &address.ip().to_string(), - std::slice::from_ref(matcher), - ) - }) - }) - }) - .map(|limit| limit.config.clone()) - .collect() - } -} - -pub fn network_capability(network: &NetworkConfig) -> NetworkCapability { - if network.mode == NetworkMode::NoNetwork { - return NetworkCapability::Deny; - } - if network.deny_rules.is_empty() && network.limits.is_empty() { - return match network.mode { - NetworkMode::Public => NetworkCapability::Ambient, - NetworkMode::NoNetwork => unreachable!(), - NetworkMode::Allowlist => NetworkCapability::AllowList { - hosts: network.allowed_hosts.clone(), - rules: network.rules.clone(), - }, - }; - } - let mut allow = network.rules.clone(); - allow.extend( - network - .allowed_hosts - .iter() - .cloned() - .map(|host| NetworkAccessRule { - host, - ports: Vec::new(), - transports: Vec::new(), - allow_private_ips: false, - }), - ); - NetworkCapability::Policy { - default_action: match network.mode { - NetworkMode::Public => NetworkDefaultAction::Allow, - NetworkMode::Allowlist => NetworkDefaultAction::Deny, - NetworkMode::NoNetwork => unreachable!(), - }, - allow, - deny: network.deny_rules.clone(), - limits: network.limits.clone(), - } -} - -pub fn network_capability_from_config(config: &impl PolicyConfig) -> NetworkCapability { - network_capability(config.network()) -} - -pub fn validate_network_config(network: &NetworkConfig) -> anyhow::Result<()> { - NetworkPolicy::compile(network).map(|_| ()) -} - -pub fn host_from_authority(authority: &str) -> String { - let authority = authority.trim(); - if let Some(rest) = authority.strip_prefix('[') - && let Some(end) = rest.find(']') - { - return normalize_host(&rest[..end]); - } - if let Some((host, port)) = authority.rsplit_once(':') - && !host.is_empty() - && port.chars().all(|character| character.is_ascii_digit()) - { - return normalize_host(host); - } - normalize_host(authority) -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum DenyReason { - NoNetwork, - AllowlistEmpty, - NotInAllowlist, - PortNotAllowed, - TransportNotAllowed, - ResolvedAddressNotAllowed, - ExplicitDeny, -} - -impl DenyReason { - pub fn as_str(&self) -> &'static str { - match self { - Self::NoNetwork => "no-network", - Self::AllowlistEmpty => "allowlist-empty", - Self::NotInAllowlist => "not-in-allowlist", - Self::PortNotAllowed => "port-not-allowed", - Self::TransportNotAllowed => "transport-not-allowed", - Self::ResolvedAddressNotAllowed => "resolved-address-not-allowed", - Self::ExplicitDeny => "explicit-deny", - } - } -} - -pub fn authorize_egress( - controller: &dyn ControlController, - policy: &NetworkPolicy, - request: &NetworkAccessRequest, -) -> Result<(), DenyReason> { - let mut control = ControlMachine::new(); - let transition = control - .authorize( - controller, - ControlRequest::Network { - policy: &policy.guard, - request, - }, - ) - .expect("policy controller must return a valid authorization transition"); - let allowed = transition.is_allowed(); - let reason = transition.reason; - if allowed { - return Ok(()); - } - Err(match reason { - ControlReason::NetworkDenied => DenyReason::NoNetwork, - ControlReason::NetworkAllowListEmpty => DenyReason::AllowlistEmpty, - ControlReason::PortNotAllowed => DenyReason::PortNotAllowed, - ControlReason::TransportNotAllowed => DenyReason::TransportNotAllowed, - ControlReason::ResolvedAddressNotAllowed => DenyReason::ResolvedAddressNotAllowed, - ControlReason::ExplicitlyDenied => DenyReason::ExplicitDeny, - _ => DenyReason::NotInAllowlist, - }) -} - -pub fn forbidden_response(host: &str, reason: &DenyReason) -> (StatusCode, String) { - ( - StatusCode::FORBIDDEN, - format!( - "persisting-overlaynet: egress to `{host}` denied ({})", - reason.as_str() - ), - ) -} - -#[cfg(test)] -mod tests { - use super::*; - use persisting_agentctl::NetworkTransport; - use persisting_agentctl::PolicyControlController; - use proptest::prelude::*; - - fn host_strategy() -> impl Strategy { - proptest::string::string_regex("[a-z]{1,12}\\.example\\.com").unwrap() - } - - fn transport_strategy() -> impl Strategy { - prop_oneof![ - Just(NetworkTransport::Http), - Just(NetworkTransport::Https), - Just(NetworkTransport::TcpTunnel), - ] - } - - proptest! { - #[test] - fn allowlist_preserves_explicit_network_entries( - hosts in prop::collection::vec(host_strategy(), 0..8), - ) { - let capability = network_capability(&NetworkConfig { - mode: NetworkMode::Allowlist, - allowed_hosts: hosts.clone(), - ..NetworkConfig::default() - }); - prop_assert_eq!( - capability, - NetworkCapability::AllowList { - hosts, - rules: Vec::new(), - } - ); - } - - #[test] - fn no_network_denies_every_request( - host in host_strategy(), - port in prop::option::of(1u16..=u16::MAX), - transport in transport_strategy(), - ) { - let policy = NetworkPolicy::compile(&NetworkConfig { - mode: NetworkMode::NoNetwork, - ..NetworkConfig::default() - }).unwrap(); - let request = NetworkAccessRequest { - run_id: None, - attempt_id: None, - storyline_id: None, - host, - port, - transport, - resolved_ip: None, - }; - prop_assert_eq!( - authorize_egress(&PolicyControlController, &policy, &request), - Err(DenyReason::NoNetwork) - ); - } - - #[test] - fn port_scoped_deny_rejects_only_the_denied_port( - denied_port in 1u16..=u16::MAX, - ) { - let other_port = if denied_port == u16::MAX { - 1 - } else { - denied_port + 1 - }; - let policy = NetworkPolicy::compile(&NetworkConfig { - mode: NetworkMode::Public, - deny_rules: vec![NetworkAccessRule { - host: "api.example.com".into(), - ports: vec![denied_port], - transports: Vec::new(), - allow_private_ips: false, - }], - ..NetworkConfig::default() - }).unwrap(); - let request = |port| NetworkAccessRequest { - run_id: None, - attempt_id: None, - storyline_id: None, - host: "api.example.com".into(), - port: Some(port), - transport: NetworkTransport::TcpTunnel, - resolved_ip: None, - }; - prop_assert_eq!( - authorize_egress(&PolicyControlController, &policy, &request(denied_port)), - Err(DenyReason::ExplicitDeny) - ); - prop_assert!(authorize_egress( - &PolicyControlController, - &policy, - &request(other_port), - ).is_ok()); - } - - #[test] - fn wildcard_bandwidth_matches_subdomains_only( - subdomain in proptest::string::string_regex("[a-z]{1,12}").unwrap(), - port in 1u16..=u16::MAX, - ) { - let policy = NetworkPolicy::compile(&NetworkConfig { - limits: vec![NetworkBandwidthLimit { - host: Some("*.example.com".into()), - port: None, - bytes_per_second: 1_000, - }], - ..NetworkConfig::default() - }).unwrap(); - prop_assert_eq!( - policy.matching_limits(&format!("{subdomain}.example.com"), Some(port), &[]).len(), - 1, - ); - prop_assert!(policy.matching_limits("example.com", Some(port), &[]).is_empty()); - prop_assert!(policy.matching_limits("example.net", Some(port), &[]).is_empty()); - } - - #[test] - fn invalid_bandwidth_limits_are_rejected( - positive_rate in 1u64..=u64::MAX, - invalid_kind in 0u8..3, - ) { - let limit = match invalid_kind { - 0 => NetworkBandwidthLimit { - host: None, - port: None, - bytes_per_second: 0, - }, - 1 => NetworkBandwidthLimit { - host: Some("https://example.com".into()), - port: None, - bytes_per_second: positive_rate, - }, - _ => NetworkBandwidthLimit { - host: Some("example.com".into()), - port: Some(0), - bytes_per_second: positive_rate, - }, - }; - let result = NetworkPolicy::compile(&NetworkConfig { - limits: vec![limit], - ..NetworkConfig::default() - }); - prop_assert!(result.is_err()); - } - } - - #[test] - fn explicit_deny_precedes_allow_and_supports_default_allow() { - let deny = NetworkAccessRule { - host: "blocked.example.com".into(), - ports: Vec::new(), - transports: Vec::new(), - allow_private_ips: false, - }; - let policy = NetworkPolicy::compile(&NetworkConfig { - mode: NetworkMode::Public, - deny_rules: vec![deny.clone()], - ..NetworkConfig::default() - }) - .unwrap(); - let request = |host: &str| NetworkAccessRequest { - run_id: None, - attempt_id: None, - storyline_id: None, - host: host.into(), - port: Some(443), - transport: NetworkTransport::TcpTunnel, - resolved_ip: None, - }; - assert_eq!( - authorize_egress( - &PolicyControlController, - &policy, - &request("blocked.example.com") - ), - Err(DenyReason::ExplicitDeny) - ); - assert!( - authorize_egress( - &PolicyControlController, - &policy, - &request("allowed.example.com") - ) - .is_ok() - ); - - let policy = NetworkPolicy::compile(&NetworkConfig { - mode: NetworkMode::Allowlist, - rules: vec![deny.clone()], - deny_rules: vec![deny], - ..NetworkConfig::default() - }) - .unwrap(); - assert_eq!( - authorize_egress( - &PolicyControlController, - &policy, - &request("blocked.example.com") - ), - Err(DenyReason::ExplicitDeny) - ); - } - - #[test] - fn bandwidth_limits_stack_when_global_and_target_rules_match() { - let policy = NetworkPolicy::compile(&NetworkConfig { - limits: vec![ - NetworkBandwidthLimit { - host: None, - port: None, - bytes_per_second: 1_000_000, - }, - NetworkBandwidthLimit { - host: Some("api.example.com".into()), - port: Some(443), - bytes_per_second: 250_000, - }, - ], - ..NetworkConfig::default() - }) - .unwrap(); - let matched = policy.matching_limits("api.example.com", Some(443), &[]); - assert_eq!(matched.len(), 2); - assert_eq!( - policy - .matching_limits("api.example.com", Some(80), &[]) - .len(), - 1 - ); - assert_eq!( - policy - .matching_limits("other.example.com", Some(443), &[]) - .len(), - 1 - ); - } - - #[test] - fn cidr_deny_is_applied_after_hostname_resolution() { - let policy = NetworkPolicy::compile(&NetworkConfig { - mode: NetworkMode::Public, - deny_rules: vec![NetworkAccessRule { - host: "10.0.0.0/8".into(), - ports: Vec::new(), - transports: Vec::new(), - allow_private_ips: false, - }], - ..NetworkConfig::default() - }) - .unwrap(); - let mut request = NetworkAccessRequest { - run_id: None, - attempt_id: None, - storyline_id: None, - host: "service.example.com".into(), - port: Some(443), - transport: NetworkTransport::TcpTunnel, - resolved_ip: None, - }; - assert!(authorize_egress(&PolicyControlController, &policy, &request).is_ok()); - request.resolved_ip = Some("10.4.5.6".parse().unwrap()); - assert_eq!( - authorize_egress(&PolicyControlController, &policy, &request), - Err(DenyReason::ExplicitDeny) - ); - } - - #[test] - fn cidr_bandwidth_limit_matches_resolved_hostname_address() { - let policy = NetworkPolicy::compile(&NetworkConfig { - limits: vec![NetworkBandwidthLimit { - host: Some("10.0.0.0/8".into()), - port: Some(443), - bytes_per_second: 1_000, - }], - ..NetworkConfig::default() - }) - .unwrap(); - let resolved = ["10.4.5.6:443".parse().unwrap()]; - assert_eq!( - policy - .matching_limits("service.internal", Some(443), &resolved) - .len(), - 1 - ); - assert!( - policy - .matching_limits( - "service.internal", - Some(443), - &["192.168.1.2:443".parse().unwrap()], - ) - .is_empty() - ); - } - - #[test] - fn public_and_no_network_modes_reject_allow_entries() { - for mode in [NetworkMode::Public, NetworkMode::NoNetwork] { - assert!( - NetworkPolicy::compile(&NetworkConfig { - mode, - rules: vec![NetworkAccessRule { - host: "api.example.com".into(), - ports: vec![443], - transports: Vec::new(), - allow_private_ips: false, - }], - ..NetworkConfig::default() - }) - .is_err() - ); - } - } - - #[test] - fn public_validation_rejects_invalid_bandwidth_limits() { - assert!( - validate_network_config(&NetworkConfig { - limits: vec![NetworkBandwidthLimit { - host: None, - port: None, - bytes_per_second: 0, - }], - ..NetworkConfig::default() - }) - .is_err() - ); - } -} diff --git a/crates/persisting-overlaynet/src/resolver.rs b/crates/persisting-overlaynet/src/resolver.rs deleted file mode 100644 index 9a03b488c..000000000 --- a/crates/persisting-overlaynet/src/resolver.rs +++ /dev/null @@ -1,562 +0,0 @@ -//! Resolve a logical target once, authorize each concrete address, and return -//! only addresses that the connector is permitted to use. - -use std::collections::HashSet; -use std::net::{IpAddr, SocketAddr}; -use std::time::Duration; - -use persisting_agentctl::ControlController; -use persisting_agentctl::NetworkAccessRequest; -use tokio::net::lookup_host; -use tokio::time::timeout; - -use crate::policy::{DenyReason, NetworkPolicy}; - -const DNS_TIMEOUT: Duration = Duration::from_secs(10); - -#[derive(Debug, Clone)] -pub(crate) struct AuthorizedTarget { - pub host: String, - pub addresses: Vec, -} - -#[derive(Debug)] -pub(crate) enum TargetAuthorizationError { - Denied(DenyReason), - Resolve(anyhow::Error), -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) enum ResolvedAddressPolicy { - Strict, - /// Accept an opaque address returned by a host fake-IP DNS/TUN connector. - /// The logical hostname is re-authorized and IP literals never qualify. - HostConnectorAliases, -} - -pub(crate) async fn authorize_target( - controller: &dyn ControlController, - policy: &NetworkPolicy, - request: NetworkAccessRequest, -) -> Result { - authorize_target_with_policy(controller, policy, request, ResolvedAddressPolicy::Strict).await -} - -pub(crate) async fn authorize_target_with_policy( - controller: &dyn ControlController, - policy: &NetworkPolicy, - mut request: NetworkAccessRequest, - resolved_address_policy: ResolvedAddressPolicy, -) -> Result { - request.resolved_ip = None; - policy - .authorize(controller, &request) - .map_err(TargetAuthorizationError::Denied)?; - - let port = request.port.ok_or_else(|| { - TargetAuthorizationError::Resolve(anyhow::anyhow!( - "network target `{}` has no effective port", - request.host - )) - })?; - let resolved = timeout(DNS_TIMEOUT, lookup_host((request.host.as_str(), port))) - .await - .map_err(|_| { - TargetAuthorizationError::Resolve(anyhow::anyhow!( - "DNS resolution for `{}` timed out", - request.host - )) - })? - .map_err(|error| { - TargetAuthorizationError::Resolve(anyhow::anyhow!( - "resolve `{}`: {error}", - request.host - )) - })? - .collect::>(); - - authorize_resolved_target_with_policy( - controller, - policy, - request, - resolved, - resolved_address_policy, - ) -} - -fn authorize_resolved_target_with_policy( - controller: &dyn ControlController, - policy: &NetworkPolicy, - mut request: NetworkAccessRequest, - resolved: impl IntoIterator, - resolved_address_policy: ResolvedAddressPolicy, -) -> Result { - let mut seen = HashSet::new(); - let mut addresses = Vec::new(); - let mut denied = DenyReason::ResolvedAddressNotAllowed; - let mut resolved_any = false; - for address in resolved { - resolved_any = true; - if !seen.insert(address) { - continue; - } - request.resolved_ip = Some(address.ip()); - let authorization = policy.authorize(controller, &request).or_else(|reason| { - if reason != DenyReason::ResolvedAddressNotAllowed - || resolved_address_policy != ResolvedAddressPolicy::HostConnectorAliases - || !is_host_connector_alias(&request.host, address.ip()) - { - return Err(reason); - } - let mut logical_request = request.clone(); - logical_request.resolved_ip = None; - policy.authorize(controller, &logical_request) - }); - match authorization { - Ok(()) => addresses.push(address), - Err(reason) => denied = reason, - } - } - if !resolved_any { - return Err(TargetAuthorizationError::Resolve(anyhow::anyhow!( - "DNS resolution for `{}` returned no addresses", - request.host - ))); - } - if addresses.is_empty() { - return Err(TargetAuthorizationError::Denied(denied)); - } - Ok(AuthorizedTarget { - host: request.host, - addresses, - }) -} - -/// The benchmarking range is commonly used as an opaque fake-IP namespace by -/// host DNS/TUN connectors. It is never a connector alias for an IP-literal -/// request, so a guest cannot use this exception as direct egress. -pub(crate) fn is_host_connector_alias(host: &str, address: IpAddr) -> bool { - if host.parse::().is_ok() { - return false; - } - let IpAddr::V4(address) = address else { - return false; - }; - let octets = address.octets(); - octets[0] == 198 && (octets[1] == 18 || octets[1] == 19) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::policy::{NetworkConfig, NetworkMode}; - use persisting_agentctl::{ - ControlReason, ControlRequest, ControlTransition, PolicyControlController, - }; - use persisting_agentctl::{NetworkAccessRule, NetworkTransport}; - use proptest::prelude::*; - - fn request(host: &str) -> NetworkAccessRequest { - NetworkAccessRequest { - run_id: None, - attempt_id: None, - storyline_id: None, - host: host.into(), - port: Some(443), - transport: NetworkTransport::TcpTunnel, - resolved_ip: None, - } - } - - fn authorize_resolved_target( - controller: &dyn ControlController, - policy: &NetworkPolicy, - request: NetworkAccessRequest, - resolved: impl IntoIterator, - ) -> Result { - authorize_resolved_target_with_policy( - controller, - policy, - request, - resolved, - ResolvedAddressPolicy::Strict, - ) - } - - fn host_allowlist(host: &str) -> NetworkPolicy { - NetworkPolicy::compile(&NetworkConfig { - mode: NetworkMode::Allowlist, - rules: vec![NetworkAccessRule { - host: host.into(), - ports: vec![443], - transports: vec![NetworkTransport::TcpTunnel], - allow_private_ips: false, - }], - ..NetworkConfig::default() - }) - .unwrap() - } - - #[test] - fn filters_private_addresses_and_deduplicates_public_results() { - let policy = host_allowlist("api.example.com"); - let public: SocketAddr = "93.184.216.34:443".parse().unwrap(); - let private: SocketAddr = "127.0.0.1:443".parse().unwrap(); - let target = authorize_resolved_target( - &PolicyControlController, - &policy, - request("api.example.com"), - [private, public, public], - ) - .unwrap(); - assert_eq!(target.addresses, [public]); - } - - #[test] - fn rejects_when_every_resolved_address_is_private() { - let policy = host_allowlist("api.example.com"); - let result = authorize_resolved_target( - &PolicyControlController, - &policy, - request("api.example.com"), - [ - "127.0.0.1:443".parse().unwrap(), - "[::1]:443".parse().unwrap(), - ], - ); - assert!(matches!( - result, - Err(TargetAuthorizationError::Denied( - DenyReason::ResolvedAddressNotAllowed - )) - )); - } - - #[test] - fn strict_resolution_rejects_host_connector_aliases() { - let policy = host_allowlist("api.example.com"); - let result = authorize_resolved_target( - &PolicyControlController, - &policy, - request("api.example.com"), - ["198.18.0.42:443".parse().unwrap()], - ); - assert!(matches!( - result, - Err(TargetAuthorizationError::Denied( - DenyReason::ResolvedAddressNotAllowed - )) - )); - } - - #[test] - fn vm_resolution_accepts_an_authorized_host_connector_alias() { - let policy = host_allowlist("api.example.com"); - let alias: SocketAddr = "198.18.0.42:443".parse().unwrap(); - let target = authorize_resolved_target_with_policy( - &PolicyControlController, - &policy, - request("api.example.com"), - [alias], - ResolvedAddressPolicy::HostConnectorAliases, - ) - .unwrap(); - assert_eq!(target.addresses, [alias]); - } - - #[test] - fn explicit_connector_range_deny_is_not_bypassed() { - let policy = NetworkPolicy::compile(&NetworkConfig { - mode: NetworkMode::Allowlist, - rules: vec![NetworkAccessRule { - host: "api.example.com".into(), - ports: vec![443], - transports: vec![NetworkTransport::TcpTunnel], - allow_private_ips: false, - }], - deny_rules: vec![NetworkAccessRule { - host: "198.18.0.0/15".into(), - ports: Vec::new(), - transports: Vec::new(), - allow_private_ips: false, - }], - ..NetworkConfig::default() - }) - .unwrap(); - let result = authorize_resolved_target_with_policy( - &PolicyControlController, - &policy, - request("api.example.com"), - ["198.18.0.42:443".parse().unwrap()], - ResolvedAddressPolicy::HostConnectorAliases, - ); - assert!(matches!( - result, - Err(TargetAuthorizationError::Denied(DenyReason::ExplicitDeny)) - )); - } - - #[test] - fn connector_aliases_require_a_logical_hostname() { - assert!(is_host_connector_alias( - "api.example.com", - "198.18.0.42".parse().unwrap() - )); - assert!(!is_host_connector_alias( - "198.18.0.42", - "198.18.0.42".parse().unwrap() - )); - assert!(!is_host_connector_alias( - "api.example.com", - "93.184.216.34".parse().unwrap() - )); - } - - #[test] - fn explicit_cidr_deny_filters_only_matching_results() { - let policy = NetworkPolicy::compile(&NetworkConfig { - mode: NetworkMode::Public, - deny_rules: vec![NetworkAccessRule { - host: "10.0.0.0/8".into(), - ports: Vec::new(), - transports: Vec::new(), - allow_private_ips: false, - }], - ..NetworkConfig::default() - }) - .unwrap(); - let allowed: SocketAddr = "93.184.216.34:443".parse().unwrap(); - let target = authorize_resolved_target( - &PolicyControlController, - &policy, - request("api.example.com"), - ["10.0.0.1:443".parse().unwrap(), allowed], - ) - .unwrap(); - assert_eq!(target.addresses, [allowed]); - } - - #[test] - fn empty_resolution_is_an_upstream_error_not_a_policy_denial() { - let policy = host_allowlist("api.example.com"); - let result = authorize_resolved_target( - &PolicyControlController, - &policy, - request("api.example.com"), - [], - ); - assert!(matches!(result, Err(TargetAuthorizationError::Resolve(_)))); - } - - struct PerAddressController; - - impl ControlController for PerAddressController { - fn authorize(&self, request: ControlRequest<'_>) -> ControlTransition { - if let ControlRequest::Network { request, .. } = &request - && request.resolved_ip == Some("93.184.216.34".parse().unwrap()) - { - return ControlTransition::denied(ControlReason::ExplicitlyDenied); - } - PolicyControlController.authorize(request) - } - } - - #[test] - fn injected_controller_can_veto_one_concrete_address() { - let policy = host_allowlist("api.example.com"); - let retained: SocketAddr = "1.1.1.1:443".parse().unwrap(); - let target = authorize_resolved_target( - &PerAddressController, - &policy, - request("api.example.com"), - ["93.184.216.34:443".parse().unwrap(), retained], - ) - .unwrap(); - assert_eq!(target.addresses, [retained]); - } - - struct DenyHostController; - - impl ControlController for DenyHostController { - fn authorize(&self, request: ControlRequest<'_>) -> ControlTransition { - if let ControlRequest::Network { request, .. } = request - && request.host == "controller-denied.invalid" - { - return ControlTransition::denied(ControlReason::ExplicitlyDenied); - } - ControlTransition::allowed(ControlReason::AmbientNetwork) - } - } - - #[test] - fn connector_aliases_do_not_bypass_the_logical_controller_decision() { - let policy = host_allowlist("controller-denied.invalid"); - let result = authorize_resolved_target_with_policy( - &DenyHostController, - &policy, - request("controller-denied.invalid"), - ["198.18.0.42:443".parse().unwrap()], - ResolvedAddressPolicy::HostConnectorAliases, - ); - assert!(matches!( - result, - Err(TargetAuthorizationError::Denied(DenyReason::ExplicitDeny)) - )); - } - - #[tokio::test] - async fn injected_controller_denies_before_dns_resolution() { - let policy = NetworkPolicy::compile(&NetworkConfig::default()).unwrap(); - let result = authorize_target( - &DenyHostController, - &policy, - request("controller-denied.invalid"), - ) - .await; - assert!(matches!( - result, - Err(TargetAuthorizationError::Denied(DenyReason::ExplicitDeny)) - )); - } - - struct AllowEverythingController; - - impl ControlController for AllowEverythingController { - fn authorize(&self, _request: ControlRequest<'_>) -> ControlTransition { - ControlTransition::allowed(ControlReason::AmbientNetwork) - } - } - - #[tokio::test] - async fn injected_controller_cannot_widen_compiled_policy() { - let policy = NetworkPolicy::compile(&NetworkConfig { - mode: NetworkMode::NoNetwork, - ..NetworkConfig::default() - }) - .unwrap(); - let result = authorize_target( - &AllowEverythingController, - &policy, - request("must-not-resolve.invalid"), - ) - .await; - assert!(matches!( - result, - Err(TargetAuthorizationError::Denied(DenyReason::NoNetwork)) - )); - } - - #[tokio::test] - async fn static_denial_happens_before_dns_resolution() { - let policy = NetworkPolicy::compile(&NetworkConfig { - mode: NetworkMode::NoNetwork, - ..NetworkConfig::default() - }) - .unwrap(); - let result = authorize_target( - &PolicyControlController, - &policy, - request("must-not-resolve.invalid"), - ) - .await; - assert!(matches!( - result, - Err(TargetAuthorizationError::Denied(DenyReason::NoNetwork)) - )); - - let policy = host_allowlist("allowed.example.com"); - let result = authorize_target( - &PolicyControlController, - &policy, - request("unlisted.must-not-resolve.invalid"), - ) - .await; - assert!(matches!( - result, - Err(TargetAuthorizationError::Denied(DenyReason::NotInAllowlist)) - )); - } - - fn connector_alias_strategy() -> impl Strategy { - (18u8..=19, any::(), any::()) - .prop_map(|(second, third, fourth)| [198, second, third, fourth].into()) - } - - proptest! { - #[test] - fn connector_alias_range_is_recognized_for_logical_hosts_only( - address in connector_alias_strategy(), - ) { - prop_assert!(is_host_connector_alias("api.example.com", address)); - prop_assert!(!is_host_connector_alias(&address.to_string(), address)); - } - - #[test] - fn addresses_outside_connector_range_are_never_aliases( - first in any::(), - second in any::(), - third in any::(), - fourth in any::(), - ) { - prop_assume!(first != 198 || (second != 18 && second != 19)); - let address = IpAddr::from([first, second, third, fourth]); - prop_assert!(!is_host_connector_alias("api.example.com", address)); - } - - #[test] - fn authorized_results_are_public_deduplicated_and_first_seen_ordered( - indexes in prop::collection::vec(0usize..6, 1..32), - ) { - let candidates: [SocketAddr; 6] = [ - "1.1.1.1:443".parse().unwrap(), - "8.8.8.8:443".parse().unwrap(), - "9.9.9.9:443".parse().unwrap(), - "127.0.0.1:443".parse().unwrap(), - "10.0.0.1:443".parse().unwrap(), - "192.168.0.1:443".parse().unwrap(), - ]; - let resolved = indexes.iter().map(|index| candidates[*index]).collect::>(); - let mut expected = Vec::new(); - for address in &resolved { - if !address.ip().is_loopback() - && !matches!(address.ip(), IpAddr::V4(ip) if ip.is_private()) - && !expected.contains(address) - { - expected.push(*address); - } - } - - let policy = host_allowlist("api.example.com"); - let result = authorize_resolved_target( - &PolicyControlController, - &policy, - request("api.example.com"), - resolved, - ); - if expected.is_empty() { - prop_assert!(matches!( - result, - Err(TargetAuthorizationError::Denied(DenyReason::ResolvedAddressNotAllowed)) - )); - } else { - prop_assert_eq!(result.unwrap().addresses, expected); - } - } - - #[test] - fn alias_policy_accepts_the_entire_benchmarking_range( - address in connector_alias_strategy(), - ) { - let policy = host_allowlist("api.example.com"); - let socket = SocketAddr::new(address, 443); - let target = authorize_resolved_target_with_policy( - &PolicyControlController, - &policy, - request("api.example.com"), - [socket], - ResolvedAddressPolicy::HostConnectorAliases, - ).unwrap(); - prop_assert_eq!(target.addresses, [socket]); - } - } -} diff --git a/crates/persisting-overlaynet/src/server.rs b/crates/persisting-overlaynet/src/server.rs deleted file mode 100644 index 5c482d4ac..000000000 --- a/crates/persisting-overlaynet/src/server.rs +++ /dev/null @@ -1,544 +0,0 @@ -//! Generic explicit-proxy server and request dispatch. - -use std::net::SocketAddr; -use std::sync::Arc; -use std::sync::atomic::{AtomicUsize, Ordering}; - -use async_trait::async_trait; -use axum::Router; -use axum::extract::{ConnectInfo, Request, State}; -use axum::http::{Method, StatusCode}; -use axum::response::{IntoResponse, Response}; -use axum::routing::any; -use persisting_agentctl::ControlController; -use persisting_agentctl::{NetworkAccessRequest, NetworkTransport, RunId, StorylineId}; - -use crate::bandwidth::{BandwidthRegistry, BandwidthSession, throttle_body}; -use crate::forward::{ - handle_connect_authorized, is_forward_proxy_request, parse_connect_target, - transparent_forward_authorized, -}; -use crate::interception::InterceptionMetrics; -use crate::policy::{DenyReason, NetworkPolicy, forbidden_response}; -use crate::resolver::{AuthorizedTarget, TargetAuthorizationError, authorize_target}; -#[derive(Clone)] -pub struct OverlayRequestContext { - pub policy: NetworkPolicy, - pub run_id: Option, - pub attempt_id: Option, - pub storyline_id: Option, - pub session_id: String, - pub sink: T, -} - -/// Application sink attached to the generic OverlayNet proxy data plane. -/// -/// A sink decides which absolute-URI requests it consumes. Requests it does -/// not accept remain transparent proxy traffic. Relative gateway requests are -/// always delivered to the configured sink. OverlayNet is independent of any -/// concrete sink; an implementation may also compose several downstream sinks. -#[async_trait] -#[allow( - clippy::double_must_use, - reason = "async_trait adds must_use to futures" -)] -pub trait OverlaySink: Clone + Send + Sync + 'static { - type RequestContext: Clone + Send + Sync + 'static; - - fn request_context( - &self, - request: &Request, - ) -> anyhow::Result>; - - async fn handle( - &self, - request: Request, - peer: SocketAddr, - context: &OverlayRequestContext, - ) -> anyhow::Result; - - fn accepts(&self, _request: &Request) -> bool { - false - } - - fn on_denied( - &self, - _context: &OverlayRequestContext, - _host: &str, - _reason: &DenyReason, - ) { - } - - fn on_dispatch( - &self, - _context: &OverlayRequestContext, - _request: &Request, - _target: &str, - ) { - } -} - -#[derive(Clone)] -pub struct OverlayServerState { - control_controller: Arc, - sink: S, - active_requests: Arc, - interception_metrics: InterceptionMetrics, - bandwidth_registry: BandwidthRegistry, -} - -impl OverlayServerState -where - S: OverlaySink, -{ - pub fn new( - control_controller: Arc, - sink: S, - active_requests: Arc, - ) -> Self { - Self { - control_controller, - sink, - active_requests, - interception_metrics: InterceptionMetrics::default(), - bandwidth_registry: BandwidthRegistry::default(), - } - } - - pub fn with_interception_metrics(mut self, metrics: InterceptionMetrics) -> Self { - self.interception_metrics = metrics; - self - } - - pub fn with_bandwidth_registry(mut self, registry: BandwidthRegistry) -> Self { - self.bandwidth_registry = registry; - self - } -} - -pub fn build_router(state: OverlayServerState) -> Router -where - S: OverlaySink, -{ - Router::new() - .fallback(any(proxy_handler::)) - .with_state(state) -} - -async fn proxy_handler( - ConnectInfo(peer): ConnectInfo, - State(state): State>, - request: Request, -) -> Response -where - S: OverlaySink, -{ - let _active_request = ActiveRequestGuard::new(Arc::clone(&state.active_requests)); - state.interception_metrics.request_seen(); - let response = dispatch(&state, request, peer).await; - match response { - Ok(response) => response, - Err(error) => { - state.interception_metrics.failure(); - tracing::warn!("overlaynet proxy error: {error:#}"); - ( - StatusCode::BAD_GATEWAY, - format!("persisting-overlaynet: {error:#}"), - ) - .into_response() - } - } -} - -async fn dispatch( - state: &OverlayServerState, - request: Request, - peer: SocketAddr, -) -> anyhow::Result -where - S: OverlaySink, -{ - let context = state.sink.request_context(&request)?; - - if request.method() == Method::CONNECT { - state.interception_metrics.connect_request(); - let authority = request - .uri() - .authority() - .map(|authority| authority.to_string()) - .unwrap_or_default(); - let target = match parse_connect_target(&authority) { - Ok(target) => target, - Err(error) => { - state.interception_metrics.failure(); - return Ok(( - StatusCode::BAD_REQUEST, - format!("persisting-overlaynet: {error}"), - ) - .into_response()); - } - }; - let host = crate::policy::host_from_authority(&target.host); - let authorized = match authorize( - state, - &context, - &host, - Some(target.port), - NetworkTransport::TcpTunnel, - ) - .await - { - Ok(target) => target, - Err(TargetAuthorizationError::Denied(reason)) => { - state.interception_metrics.policy_denied(); - state.sink.on_denied(&context, &host, &reason); - let (status, message) = forbidden_response(&host, &reason); - return Ok((status, message).into_response()); - } - Err(TargetAuthorizationError::Resolve(error)) => return Err(error), - }; - state.interception_metrics.policy_allowed(); - state.sink.on_dispatch(&context, &request, "connect"); - let bandwidth = - bandwidth_session(state, &context, &host, Some(target.port), Some(&authorized)).await; - return handle_connect_authorized(request, target, &authorized, bandwidth).await; - } - - if is_forward_proxy_request(request.method(), request.uri()) { - state.interception_metrics.absolute_http_request(); - let host = request.uri().host().map(str::to_string).unwrap_or_default(); - let transport = if request.uri().scheme_str() == Some("https") { - NetworkTransport::Https - } else { - NetworkTransport::Http - }; - let port = request.uri().port_u16().or(match transport { - NetworkTransport::Https => Some(443), - NetworkTransport::Http => Some(80), - NetworkTransport::TcpTunnel => None, - }); - let authorized = match authorize(state, &context, &host, port, transport).await { - Ok(target) => target, - Err(TargetAuthorizationError::Denied(reason)) => { - state.interception_metrics.policy_denied(); - state.sink.on_denied(&context, &host, &reason); - let (status, message) = forbidden_response(&host, &reason); - return Ok((status, message).into_response()); - } - Err(TargetAuthorizationError::Resolve(error)) => return Err(error), - }; - state.interception_metrics.policy_allowed(); - let bandwidth = bandwidth_session(state, &context, &host, port, Some(&authorized)).await; - if state.sink.accepts(&request) { - state.interception_metrics.sink_request(); - state.sink.on_dispatch(&context, &request, "sink"); - let request = throttle_request(request, bandwidth.clone()); - let response = state.sink.handle(request, peer, &context).await?; - return Ok(throttle_response(response, bandwidth)); - } - state.sink.on_dispatch(&context, &request, "forward"); - return transparent_forward_authorized(request, &authorized, bandwidth) - .await - .map(IntoResponse::into_response); - } - - state.interception_metrics.sink_request(); - state.sink.on_dispatch(&context, &request, "sink"); - let bandwidth = bandwidth_session(state, &context, "", None, None).await; - let request = throttle_request(request, bandwidth.clone()); - let response = state.sink.handle(request, peer, &context).await?; - Ok(throttle_response(response, bandwidth)) -} - -async fn bandwidth_session( - state: &OverlayServerState, - context: &OverlayRequestContext, - host: &str, - port: Option, - authorized: Option<&AuthorizedTarget>, -) -> BandwidthSession -where - S: OverlaySink, -{ - state - .bandwidth_registry - .session(context.policy.matching_limits( - host, - port, - authorized.map_or(&[], |target| target.addresses.as_slice()), - )) - .await -} - -fn throttle_request(request: Request, bandwidth: BandwidthSession) -> Request { - let (parts, body) = request.into_parts(); - Request::from_parts(parts, throttle_body(body, bandwidth)) -} - -fn throttle_response(response: Response, bandwidth: BandwidthSession) -> Response { - let (parts, body) = response.into_parts(); - Response::from_parts(parts, throttle_body(body, bandwidth)) -} - -async fn authorize( - state: &OverlayServerState, - context: &OverlayRequestContext, - host: &str, - port: Option, - transport: NetworkTransport, -) -> Result -where - S: OverlaySink, -{ - authorize_target( - state.control_controller.as_ref(), - &context.policy, - NetworkAccessRequest { - run_id: context.run_id.clone().map(RunId), - attempt_id: context - .attempt_id - .clone() - .map(persisting_agentctl::AttemptId), - storyline_id: context.storyline_id.clone().map(StorylineId), - host: host.to_string(), - port, - transport, - resolved_ip: None, - }, - ) - .await -} - -struct ActiveRequestGuard { - counter: Arc, -} - -impl ActiveRequestGuard { - fn new(counter: Arc) -> Self { - counter.fetch_add(1, Ordering::Relaxed); - Self { counter } - } -} - -impl Drop for ActiveRequestGuard { - fn drop(&mut self) { - self.counter.fetch_sub(1, Ordering::Relaxed); - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::policy::{NetworkConfig, NetworkMode}; - use persisting_agentctl::PolicyControlController; - - #[derive(Clone)] - struct EventSink; - - #[async_trait] - impl OverlaySink for EventSink { - type RequestContext = (); - - fn request_context( - &self, - _request: &Request, - ) -> anyhow::Result> { - unreachable!("selection does not require a request context") - } - - async fn handle( - &self, - _request: Request, - _peer: SocketAddr, - _context: &OverlayRequestContext, - ) -> anyhow::Result { - unreachable!("selection does not dispatch the request") - } - - fn accepts(&self, request: &Request) -> bool { - request.uri().path().starts_with("/events/") - } - } - - #[test] - fn active_request_guard_decrements_on_drop() { - let counter = Arc::new(AtomicUsize::new(0)); - { - let _guard = ActiveRequestGuard::new(Arc::clone(&counter)); - assert_eq!(counter.load(Ordering::Relaxed), 1); - } - assert_eq!(counter.load(Ordering::Relaxed), 0); - } - - #[test] - fn overlaynet_accepts_non_gateway_sink_implementations() { - let accepted = Request::builder() - .uri("http://collector.example/events/agent") - .body(axum::body::Body::empty()) - .expect("valid request"); - let forwarded = Request::builder() - .uri("http://collector.example/health") - .body(axum::body::Body::empty()) - .expect("valid request"); - - assert!(EventSink.accepts(&accepted)); - assert!(!EventSink.accepts(&forwarded)); - } - - #[derive(Clone)] - struct PolicySink; - - #[async_trait] - impl OverlaySink for PolicySink { - type RequestContext = (); - - fn request_context( - &self, - _request: &Request, - ) -> anyhow::Result> { - Ok(OverlayRequestContext { - policy: NetworkPolicy::compile(&NetworkConfig { - mode: NetworkMode::NoNetwork, - allowed_hosts: Vec::new(), - rules: Vec::new(), - deny_rules: Vec::new(), - limits: Vec::new(), - })?, - run_id: Some("run-1".into()), - attempt_id: Some("attempt-1".into()), - storyline_id: None, - session_id: "session-1".into(), - sink: (), - }) - } - - async fn handle( - &self, - _request: Request, - _peer: SocketAddr, - _context: &OverlayRequestContext, - ) -> anyhow::Result { - unreachable!("denied requests do not reach the sink") - } - } - - #[test] - fn denied_proxy_requests_update_interception_metrics() { - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .unwrap(); - runtime.block_on(async { - let metrics = InterceptionMetrics::default(); - let state = OverlayServerState::new( - Arc::new(PolicyControlController), - PolicySink, - Arc::new(AtomicUsize::new(0)), - ) - .with_interception_metrics(metrics.clone()); - let request = Request::builder() - .method(Method::CONNECT) - .uri("example.com:443") - .body(axum::body::Body::empty()) - .unwrap(); - - metrics.request_seen(); - let response = dispatch(&state, request, "127.0.0.1:40000".parse().unwrap()) - .await - .unwrap(); - assert_eq!(response.status(), StatusCode::FORBIDDEN); - let snapshot = metrics.snapshot(); - assert_eq!(snapshot.requests_seen, 1); - assert_eq!(snapshot.connect_requests, 1); - assert_eq!(snapshot.policy_denied, 1); - assert_eq!(snapshot.policy_allowed, 0); - }); - } - - #[derive(Clone)] - struct FailingContextSink; - - #[async_trait] - impl OverlaySink for FailingContextSink { - type RequestContext = (); - - fn request_context( - &self, - _request: &Request, - ) -> anyhow::Result> { - anyhow::bail!("synthetic context failure") - } - - async fn handle( - &self, - _request: Request, - _peer: SocketAddr, - _context: &OverlayRequestContext, - ) -> anyhow::Result { - unreachable!() - } - } - - #[test] - fn handler_turns_internal_failures_into_502_and_releases_active_count() { - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .unwrap(); - runtime.block_on(async { - let active = Arc::new(AtomicUsize::new(0)); - let metrics = InterceptionMetrics::default(); - let state = OverlayServerState::new( - Arc::new(PolicyControlController), - FailingContextSink, - Arc::clone(&active), - ) - .with_interception_metrics(metrics.clone()); - let request = Request::builder() - .uri("/events") - .body(axum::body::Body::empty()) - .unwrap(); - let response = proxy_handler( - ConnectInfo("127.0.0.1:40000".parse().unwrap()), - State(state), - request, - ) - .await; - assert_eq!(response.status(), StatusCode::BAD_GATEWAY); - assert_eq!(active.load(Ordering::Relaxed), 0); - assert_eq!(metrics.snapshot().requests_seen, 1); - assert_eq!(metrics.snapshot().failures, 1); - }); - } - - #[test] - fn malformed_connect_is_400_and_never_reaches_policy() { - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .unwrap(); - runtime.block_on(async { - let metrics = InterceptionMetrics::default(); - let state = OverlayServerState::new( - Arc::new(PolicyControlController), - PolicySink, - Arc::new(AtomicUsize::new(0)), - ) - .with_interception_metrics(metrics.clone()); - let request = Request::builder() - .method(Method::CONNECT) - .uri("example.com:65536") - .body(axum::body::Body::empty()) - .unwrap(); - let response = dispatch(&state, request, "127.0.0.1:40000".parse().unwrap()) - .await - .unwrap(); - assert_eq!(response.status(), StatusCode::BAD_REQUEST); - let snapshot = metrics.snapshot(); - assert_eq!(snapshot.connect_requests, 1); - assert_eq!(snapshot.failures, 1); - assert_eq!(snapshot.policy_allowed, 0); - assert_eq!(snapshot.policy_denied, 0); - }); - } -} diff --git a/crates/persisting-overlaynet/src/vm.rs b/crates/persisting-overlaynet/src/vm.rs deleted file mode 100644 index e5e2f7ab6..000000000 --- a/crates/persisting-overlaynet/src/vm.rs +++ /dev/null @@ -1,1290 +0,0 @@ -//! Transparent IPv4/TCP data plane for libkrun virtio-net. -//! -//! libkrun's unix-stream backend carries one Ethernet frame at a time, prefixed -//! by a four-byte big-endian length. This module terminates that link in -//! smoltcp, serves DHCP and synthetic DNS locally, and opens policy-authorized -//! host TCP streams only after the guest SYN has been observed. - -use std::collections::{HashMap, VecDeque}; -use std::io; -use std::net::{IpAddr, Ipv4Addr, SocketAddr}; -use std::os::unix::net::UnixStream as StdUnixStream; -use std::thread; -use std::time::Duration as StdDuration; - -use anyhow::Context as _; -use smoltcp::iface::{Config as InterfaceConfig, Interface, SocketHandle, SocketSet}; -use smoltcp::phy::{Device, DeviceCapabilities, Medium, RxToken, TxToken}; -use smoltcp::socket::udp::UdpMetadata; -use smoltcp::socket::{tcp, udp}; -use smoltcp::time::{Duration, Instant}; -use smoltcp::wire::{ - DhcpMessageType, DhcpOption, DhcpPacket, DhcpRepr, EthernetAddress, EthernetFrame, - EthernetProtocol, HardwareAddress, IpAddress, IpCidr, IpEndpoint, IpProtocol, Ipv4Address, - Ipv4Packet, TcpPacket, -}; -use tokio::io::{AsyncReadExt, AsyncWriteExt}; -use tokio::net::{TcpStream, UnixStream}; -use tokio::sync::{mpsc, oneshot}; -use tokio::task::JoinHandle; - -use crate::egress::{ - CONNECT_TIMEOUT, EgressContext, EgressError, EgressRuntime, connect_tcp_addresses, -}; -use crate::interception::{InterceptionMetrics, InterceptionSnapshot}; -use crate::policy::DenyReason; -use crate::resolver::is_host_connector_alias; - -pub const ROUTER_IPV4: Ipv4Addr = Ipv4Addr::new(192, 0, 2, 1); -pub const GUEST_IPV4: Ipv4Addr = Ipv4Addr::new(192, 0, 2, 2); -pub const VM_MAC: [u8; 6] = [0x02, 0x50, 0x56, 0x00, 0x00, 0x02]; - -const ROUTER_MAC: EthernetAddress = EthernetAddress([0x02, 0x50, 0x56, 0x00, 0x00, 0x01]); -const FRAME_MTU: usize = 1514; -const MAX_FRAME_LEN: usize = 65_535; -const TCP_BUFFER_BYTES: usize = 64 * 1024; -const DEFAULT_MAX_FLOWS: usize = 256; -const TCP_CHANNEL_DEPTH: usize = 256; -const FLOW_BUFFER_CHUNKS: usize = TCP_BUFFER_BYTES / (16 * 1024); -const TCP_IDLE_TIMEOUT: Duration = Duration::from_secs(5 * 60); -const MAX_POLL_DELAY: StdDuration = StdDuration::from_millis(100); -const SYNTHETIC_DNS_CAPACITY: usize = 4096; -const DNS_MAX_MESSAGE: usize = 4096; - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct VmGatewayRoute { - /// Port exposed to the guest on [`ROUTER_IPV4`]. - pub guest_port: u16, - /// Attempt-local Gateway listener on the host. - pub host: SocketAddr, -} - -#[derive(Clone)] -pub struct VmNetworkConfig { - pub egress: EgressRuntime, - pub context: EgressContext, - pub gateway: Option, - pub max_flows: usize, - pub metrics: InterceptionMetrics, -} - -impl std::fmt::Debug for VmNetworkConfig { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("VmNetworkConfig") - .field("context", &self.context) - .field("gateway", &self.gateway) - .field("max_flows", &self.max_flows) - .finish_non_exhaustive() - } -} - -impl VmNetworkConfig { - pub fn new(egress: EgressRuntime, context: EgressContext) -> Self { - Self { - egress, - context, - gateway: None, - max_flows: DEFAULT_MAX_FLOWS, - metrics: InterceptionMetrics::default(), - } - } -} - -/// Attempt-scoped smoltcp backend. The returned peer stream is passed to -/// `krun_add_net_unixstream`; this side owns the policy and host sockets. -pub struct VmNetwork { - shutdown: Option>, - thread: Option>>, - metrics: InterceptionMetrics, -} - -impl VmNetwork { - pub fn start(config: VmNetworkConfig) -> anyhow::Result<(Self, StdUnixStream)> { - anyhow::ensure!( - config.max_flows > 0, - "VM network max_flows must be non-zero" - ); - let (backend, guest) = StdUnixStream::pair().context("create VM network socket pair")?; - backend - .set_nonblocking(true) - .context("make VM network backend nonblocking")?; - let (shutdown_tx, shutdown_rx) = oneshot::channel(); - let metrics = config.metrics.clone(); - let thread_metrics = metrics.clone(); - let thread = thread::Builder::new() - .name("pvisor-smoltcp".into()) - .spawn(move || { - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .context("build VM network runtime")?; - runtime.block_on(run_backend(backend, config, thread_metrics, shutdown_rx)) - }) - .context("spawn VM network backend")?; - Ok(( - Self { - shutdown: Some(shutdown_tx), - thread: Some(thread), - metrics, - }, - guest, - )) - } - - pub fn shutdown(mut self) -> anyhow::Result { - if let Some(shutdown) = self.shutdown.take() { - let _ = shutdown.send(()); - } - if let Some(thread) = self.thread.take() { - thread - .join() - .map_err(|_| anyhow::anyhow!("VM network backend panicked"))??; - } - Ok(self.metrics.snapshot()) - } -} - -impl Drop for VmNetwork { - fn drop(&mut self) { - if let Some(shutdown) = self.shutdown.take() { - let _ = shutdown.send(()); - } - if let Some(thread) = self.thread.take() - && thread.thread().id() != thread::current().id() - && let Err(error) = thread.join() - { - tracing::warn!(?error, "VM network backend panicked during drop"); - } - } -} - -#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)] -struct FlowKey { - guest_port: u16, - destination_addr: Ipv4Addr, - destination_port: u16, -} - -#[derive(Debug, Clone)] -enum FlowDestination { - Egress { host: String, port: u16 }, - Gateway(SocketAddr), - Dns, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum FlowPhase { - WaitingForSyn, - Connecting, - Connected, - LocalDns, -} - -struct Flow { - handle: SocketHandle, - destination: FlowDestination, - phase: FlowPhase, - upstream: Option>>, - upstream_task: Option>, - inbound: VecDeque>, - inbound_offset: usize, - dns_input: Vec, - remote_eof: bool, -} - -enum FlowEvent { - Connected { - key: FlowKey, - sender: mpsc::Sender>, - policy_authorized: bool, - }, - Data { - key: FlowKey, - bytes: Vec, - }, - Uploaded { - key: FlowKey, - bytes: usize, - }, - RemoteEof(FlowKey), - ConnectFailed { - key: FlowKey, - denied: bool, - error: String, - }, - BridgeFailed { - key: FlowKey, - error: String, - }, -} - -async fn run_backend( - backend: StdUnixStream, - config: VmNetworkConfig, - metrics: InterceptionMetrics, - mut shutdown: oneshot::Receiver<()>, -) -> anyhow::Result<()> { - let stream = UnixStream::from_std(backend).context("adopt VM network stream")?; - let (mut reader, mut writer) = stream.into_split(); - let mut frame_reader = FrameReader::default(); - let mut frame_device = FrameDevice::default(); - let clock = std::time::Instant::now(); - - let mut interface_config = InterfaceConfig::new(HardwareAddress::Ethernet(ROUTER_MAC)); - interface_config.random_seed = random_seed(); - let mut interface = Interface::new(interface_config, &mut frame_device, Instant::ZERO); - interface.update_ip_addrs(|addresses| { - addresses - .push(IpCidr::new(IpAddress::Ipv4(ROUTER_IPV4), 24)) - .expect("one interface address fits"); - }); - interface.set_any_ip(true); - interface - .routes_mut() - .add_default_ipv4_route(ROUTER_IPV4) - .expect("one default route fits"); - - let mut sockets = SocketSet::new(Vec::new()); - let dhcp_handle = sockets.add(udp_socket(4, 8192, 4, 8192, 67)); - let dns_handle = sockets.add(udp_socket(16, 65_535, 16, 65_535, 53)); - let mut flows = HashMap::::new(); - let mut dns = SyntheticDns::default(); - let (event_tx, mut event_rx) = mpsc::channel(TCP_CHANNEL_DEPTH); - let mut poll_delay = Box::pin(tokio::time::sleep(StdDuration::ZERO)); - - 'backend: loop { - tokio::select! { - biased; - _ = &mut shutdown => break, - event = event_rx.recv() => { - if let Some(event) = event { - apply_flow_event(event, &mut flows, &mut sockets, &metrics); - } - } - read = frame_reader.read_more(&mut reader) => { - match read { - Ok(0) => break, - Ok(_) => { - while let Some(frame) = frame_reader.next_frame()? { - if let Some(key) = initial_tcp_syn(&frame) { - ensure_listener( - key, - &mut flows, - &mut sockets, - &dns, - &config, - &metrics, - ); - } - if !supported_guest_frame(&frame) { - metrics.unsupported_packet(); - continue; - } - frame_device.receive_queue.push_back(frame); - } - } - Err(error) if error.kind() == io::ErrorKind::Interrupted => continue, - Err(error) => return Err(error).context("read VM Ethernet frame"), - } - } - _ = &mut poll_delay => {} - } - - let now = smol_now(clock); - let _ = interface.poll(now, &mut frame_device, &mut sockets); - service_dhcp(&mut sockets, dhcp_handle); - service_dns_udp(&mut sockets, dns_handle, &mut dns, &metrics); - drive_flows( - &mut sockets, - &mut flows, - &mut dns, - &config, - &event_tx, - &metrics, - ); - let _ = interface.poll(now, &mut frame_device, &mut sockets); - - while let Some(frame) = frame_device.transmit_queue.pop_front() { - let result = tokio::select! { - biased; - _ = &mut shutdown => break 'backend, - result = write_frame(&mut writer, &frame) => result, - }; - result.context("write VM Ethernet frame")?; - } - reap_closed_flows(&mut sockets, &mut flows, &metrics); - let delay = interface - .poll_delay(smol_now(clock), &sockets) - .map(|delay| StdDuration::from_micros(delay.total_micros())) - .unwrap_or(MAX_POLL_DELAY) - .min(MAX_POLL_DELAY); - poll_delay - .as_mut() - .reset(tokio::time::Instant::now() + delay); - } - - for (_, mut flow) in flows.drain() { - if let Some(task) = flow.upstream_task.take() { - task.abort(); - } - metrics.tcp_flow_closed(); - } - Ok(()) -} - -fn udp_socket( - rx_packets: usize, - rx_bytes: usize, - tx_packets: usize, - tx_bytes: usize, - port: u16, -) -> udp::Socket<'static> { - let rx = udp::PacketBuffer::new( - vec![udp::PacketMetadata::EMPTY; rx_packets], - vec![0; rx_bytes], - ); - let tx = udp::PacketBuffer::new( - vec![udp::PacketMetadata::EMPTY; tx_packets], - vec![0; tx_bytes], - ); - let mut socket = udp::Socket::new(rx, tx); - socket.bind(port).expect("non-zero UDP service port"); - socket -} - -fn random_seed() -> u64 { - let timestamp = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default(); - (timestamp.as_nanos() as u64) ^ (std::process::id() as u64).rotate_left(17) -} - -fn smol_now(clock: std::time::Instant) -> Instant { - Instant::from_micros(clock.elapsed().as_micros().min(i64::MAX as u128) as i64) -} - -fn ensure_listener( - key: FlowKey, - flows: &mut HashMap, - sockets: &mut SocketSet<'static>, - dns: &SyntheticDns, - config: &VmNetworkConfig, - metrics: &InterceptionMetrics, -) { - if flows.contains_key(&key) { - return; - } - if flows.len() >= config.max_flows { - metrics.tcp_flow_denied(); - return; - } - let destination = if key.destination_port == 53 { - FlowDestination::Dns - } else if key.destination_addr == ROUTER_IPV4 { - match config - .gateway - .filter(|gateway| gateway.guest_port == key.destination_port) - { - Some(gateway) => FlowDestination::Gateway(gateway.host), - None => { - metrics.tcp_flow_denied(); - return; - } - } - } else if synthetic_address(key.destination_addr) { - match dns.hostname(key.destination_addr) { - Some(host) => FlowDestination::Egress { - host: host.to_owned(), - port: key.destination_port, - }, - None => { - metrics.tcp_flow_denied(); - return; - } - } - } else if blocked_literal_destination(key.destination_addr) { - metrics.tcp_flow_denied(); - return; - } else { - FlowDestination::Egress { - host: key.destination_addr.to_string(), - port: key.destination_port, - } - }; - - let mut socket = tcp::Socket::new( - tcp::SocketBuffer::new(vec![0; TCP_BUFFER_BYTES]), - tcp::SocketBuffer::new(vec![0; TCP_BUFFER_BYTES]), - ); - socket.set_timeout(Some(TCP_IDLE_TIMEOUT)); - let local = IpEndpoint::new(IpAddress::Ipv4(key.destination_addr), key.destination_port); - if socket.listen(local).is_err() { - metrics.tcp_flow_denied(); - return; - } - let local_dns = matches!(destination, FlowDestination::Dns); - socket.pause_synack(!local_dns); - let handle = sockets.add(socket); - flows.insert( - key, - Flow { - handle, - destination, - phase: if local_dns { - FlowPhase::LocalDns - } else { - FlowPhase::WaitingForSyn - }, - upstream: None, - upstream_task: None, - inbound: VecDeque::new(), - inbound_offset: 0, - dns_input: Vec::new(), - remote_eof: false, - }, - ); - metrics.tcp_flow_opened(); -} - -fn drive_flows( - sockets: &mut SocketSet<'static>, - flows: &mut HashMap, - dns: &mut SyntheticDns, - config: &VmNetworkConfig, - event_tx: &mpsc::Sender, - metrics: &InterceptionMetrics, -) { - let keys = flows.keys().copied().collect::>(); - for key in keys { - let Some(flow) = flows.get_mut(&key) else { - continue; - }; - let socket = sockets.get_mut::(flow.handle); - - if flow.phase == FlowPhase::WaitingForSyn && socket.state() == tcp::State::SynReceived { - flow.phase = FlowPhase::Connecting; - let destination = flow.destination.clone(); - let egress = config.egress.clone(); - let context = config.context.clone(); - let events = event_tx.clone(); - flow.upstream_task = Some(tokio::spawn(async move { - connect_and_bridge(key, destination, egress, context, events).await; - })); - } - - if flow.phase == FlowPhase::LocalDns { - drive_dns_tcp(socket, flow, dns, metrics); - continue; - } - - if flow.phase == FlowPhase::Connected { - // The host connection is established before we release the paused - // SYN|ACK. Until the guest ACK arrives, `may_recv()` is false in - // smoltcp's SynReceived state; treating that as guest EOF would - // close the upstream write half before the first payload. - if guest_handshake_pending(socket.state()) { - continue; - } - if let Some(sender) = flow.upstream.as_ref() - && sender.capacity() > 0 - && socket.can_recv() - { - let mut bytes = vec![0; 16 * 1024]; - if let Ok(length) = socket.recv_slice(&mut bytes) { - bytes.truncate(length); - if length > 0 && sender.try_send(bytes).is_err() { - socket.abort(); - } - } - } - if !socket.may_recv() { - flow.upstream.take(); - } - flush_inbound(socket, flow); - if flow.remote_eof && flow.inbound.is_empty() && socket.may_send() { - socket.close(); - } - } - } -} - -fn guest_handshake_pending(state: tcp::State) -> bool { - state == tcp::State::SynReceived -} - -fn drive_dns_tcp( - socket: &mut tcp::Socket<'_>, - flow: &mut Flow, - dns: &mut SyntheticDns, - metrics: &InterceptionMetrics, -) { - if socket.can_recv() { - let mut bytes = [0; DNS_MAX_MESSAGE + 2]; - if let Ok(length) = socket.recv_slice(&mut bytes) { - flow.dns_input.extend_from_slice(&bytes[..length]); - } - } - loop { - if flow.dns_input.len() < 2 { - break; - } - let message_len = u16::from_be_bytes([flow.dns_input[0], flow.dns_input[1]]) as usize; - if message_len > DNS_MAX_MESSAGE { - socket.abort(); - return; - } - if flow.dns_input.len() < message_len + 2 { - break; - } - let query = flow.dns_input[2..message_len + 2].to_vec(); - flow.dns_input.drain(..message_len + 2); - metrics.dns_query(); - let response = dns.answer(&query); - metrics.dns_answer(); - let mut framed = Vec::with_capacity(response.len() + 2); - framed.extend_from_slice(&(response.len() as u16).to_be_bytes()); - framed.extend_from_slice(&response); - flow.inbound.push_back(framed); - } - flush_inbound(socket, flow); - if !socket.may_recv() && flow.inbound.is_empty() && socket.may_send() { - socket.close(); - } -} - -fn flush_inbound(socket: &mut tcp::Socket<'_>, flow: &mut Flow) { - while socket.can_send() { - let Some(front) = flow.inbound.front() else { - break; - }; - match socket.send_slice(&front[flow.inbound_offset..]) { - Ok(0) | Err(_) => break, - Ok(written) => { - flow.inbound_offset += written; - if flow.inbound_offset == front.len() { - flow.inbound.pop_front(); - flow.inbound_offset = 0; - } - } - } - } -} - -async fn connect_and_bridge( - key: FlowKey, - destination: FlowDestination, - egress: EgressRuntime, - context: EgressContext, - events: mpsc::Sender, -) { - let result = match destination { - FlowDestination::Egress { host, port } => connect_vm_egress(&egress, &context, &host, port) - .await - .map(|(stream, bandwidth)| (stream, bandwidth, true)), - FlowDestination::Gateway(address) => { - match tokio::time::timeout(CONNECT_TIMEOUT, TcpStream::connect(address)).await { - Ok(Ok(stream)) => Ok((stream, Default::default(), false)), - Ok(Err(error)) => Err(EgressError::Connect { - host: address.ip().to_string(), - port: address.port(), - source: error, - }), - Err(_) => Err(EgressError::ConnectTimeout { - host: address.ip().to_string(), - port: address.port(), - }), - } - } - FlowDestination::Dns => return, - }; - let (stream, bandwidth, policy_authorized) = match result { - Ok(connection) => connection, - Err(error) => { - let denied = matches!(error, EgressError::Denied(_)); - let _ = events - .send(FlowEvent::ConnectFailed { - key, - denied, - error: error.to_string(), - }) - .await; - return; - } - }; - let (sender, receiver) = mpsc::channel(FLOW_BUFFER_CHUNKS); - if events - .send(FlowEvent::Connected { - key, - sender, - policy_authorized, - }) - .await - .is_err() - { - return; - } - bridge_upstream(key, stream, receiver, bandwidth, events).await; -} - -async fn connect_vm_egress( - egress: &EgressRuntime, - context: &EgressContext, - host: &str, - port: u16, -) -> Result<(TcpStream, crate::bandwidth::BandwidthSession), EgressError> { - let (addresses, bandwidth) = egress.authorize_tcp(context, host, port).await?; - let addresses = addresses - .into_iter() - .filter(|address| { - !forbidden_host_address(address.ip()) || is_host_connector_alias(host, address.ip()) - }) - .collect::>(); - if addresses.is_empty() { - return Err(EgressError::Denied(DenyReason::ResolvedAddressNotAllowed)); - } - let stream = connect_tcp_addresses(&addresses, host, port).await?; - Ok((stream, bandwidth)) -} - -async fn bridge_upstream( - key: FlowKey, - stream: TcpStream, - mut outbound: mpsc::Receiver>, - bandwidth: crate::bandwidth::BandwidthSession, - events: mpsc::Sender, -) { - let (mut read, mut write) = stream.into_split(); - let upload_bandwidth = bandwidth.clone(); - let upload_events = events.clone(); - let upload = async move { - while let Some(bytes) = outbound.recv().await { - upload_bandwidth.throttle(bytes.len()).await; - write.write_all(&bytes).await?; - if upload_events - .send(FlowEvent::Uploaded { - key, - bytes: bytes.len(), - }) - .await - .is_err() - { - return Ok(()); - } - } - write.shutdown().await - }; - let download = async { - let mut buffer = vec![0; 16 * 1024]; - loop { - let length = read.read(&mut buffer).await?; - if length == 0 { - let _ = events.send(FlowEvent::RemoteEof(key)).await; - return Ok::<(), io::Error>(()); - } - bandwidth.throttle(length).await; - if events - .send(FlowEvent::Data { - key, - bytes: buffer[..length].to_vec(), - }) - .await - .is_err() - { - return Ok(()); - } - } - }; - let (upload_result, download_result) = tokio::join!(upload, download); - if let Err(error) = upload_result.or(download_result) { - let _ = events - .send(FlowEvent::BridgeFailed { - key, - error: error.to_string(), - }) - .await; - } -} - -fn apply_flow_event( - event: FlowEvent, - flows: &mut HashMap, - sockets: &mut SocketSet<'static>, - metrics: &InterceptionMetrics, -) { - match event { - FlowEvent::Connected { - key, - sender, - policy_authorized, - } => { - if let Some(flow) = flows.get_mut(&key) { - flow.upstream = Some(sender); - flow.phase = FlowPhase::Connected; - sockets - .get_mut::(flow.handle) - .pause_synack(false); - if policy_authorized { - metrics.policy_allowed(); - } - } - } - FlowEvent::Data { key, bytes } => { - if let Some(flow) = flows.get_mut(&key) { - let queued = flow - .inbound - .iter() - .map(Vec::len) - .sum::() - .saturating_sub(flow.inbound_offset); - if queued.saturating_add(bytes.len()) > TCP_BUFFER_BYTES { - sockets.get_mut::(flow.handle).abort(); - metrics.tcp_connect_failure(); - } else { - metrics.host_to_guest(bytes.len()); - flow.inbound.push_back(bytes); - } - } - } - FlowEvent::Uploaded { key, bytes } => { - if flows.contains_key(&key) { - metrics.guest_to_host(bytes); - } - } - FlowEvent::RemoteEof(key) => { - if let Some(flow) = flows.get_mut(&key) { - flow.remote_eof = true; - } - } - FlowEvent::ConnectFailed { key, denied, error } => { - if let Some(flow) = flows.get_mut(&key) { - tracing::debug!(?key, %error, "VM TCP flow failed closed"); - sockets.get_mut::(flow.handle).abort(); - if denied { - metrics.tcp_flow_denied(); - } else { - metrics.tcp_connect_failure(); - } - } - } - FlowEvent::BridgeFailed { key, error } => { - if let Some(flow) = flows.get_mut(&key) { - tracing::debug!(?key, %error, "VM TCP bridge failed closed"); - sockets.get_mut::(flow.handle).abort(); - } - } - } -} - -fn reap_closed_flows( - sockets: &mut SocketSet<'static>, - flows: &mut HashMap, - metrics: &InterceptionMetrics, -) { - let closed = flows - .iter() - .filter_map(|(key, flow)| { - (sockets.get::(flow.handle).state() == tcp::State::Closed).then_some(*key) - }) - .collect::>(); - for key in closed { - if let Some(mut flow) = flows.remove(&key) { - sockets.remove(flow.handle); - if let Some(task) = flow.upstream_task.take() { - task.abort(); - } - metrics.tcp_flow_closed(); - } - } -} - -fn service_dhcp(sockets: &mut SocketSet<'static>, handle: SocketHandle) { - let socket = sockets.get_mut::(handle); - while socket.can_recv() && socket.can_send() { - let Ok((request_bytes, _)) = socket.recv() else { - break; - }; - let Ok(packet) = DhcpPacket::new_checked(request_bytes) else { - continue; - }; - let Ok(request) = DhcpRepr::parse(&packet) else { - continue; - }; - let message_type = match request.message_type { - DhcpMessageType::Discover => DhcpMessageType::Offer, - DhcpMessageType::Request | DhcpMessageType::Inform => DhcpMessageType::Ack, - _ => continue, - }; - let mtu = 1500_u16.to_be_bytes(); - let options = [DhcpOption { - kind: 26, - data: &mtu, - }]; - let response = DhcpRepr { - message_type, - transaction_id: request.transaction_id, - secs: request.secs, - client_hardware_address: request.client_hardware_address, - client_ip: Ipv4Address::UNSPECIFIED, - your_ip: GUEST_IPV4, - server_ip: ROUTER_IPV4, - router: Some(ROUTER_IPV4), - subnet_mask: Some(Ipv4Address::new(255, 255, 255, 0)), - relay_agent_ip: Ipv4Address::UNSPECIFIED, - broadcast: true, - requested_ip: None, - client_identifier: request.client_identifier, - server_identifier: Some(ROUTER_IPV4), - parameter_request_list: None, - // libkrun's DHCP init writes resolv.conf when this option is set. - // pVisor injects the resolver file from its guest root instead. - dns_servers: None, - max_size: request.max_size, - lease_duration: Some(86_400), - renew_duration: None, - rebind_duration: None, - additional_options: &options, - }; - let mut bytes = vec![0; response.buffer_len()]; - let mut packet = DhcpPacket::new_unchecked(bytes.as_mut_slice()); - if response.emit(&mut packet).is_err() { - continue; - } - let mut metadata = - UdpMetadata::from(IpEndpoint::new(IpAddress::Ipv4(Ipv4Address::BROADCAST), 68)); - metadata.local_address = Some(IpAddress::Ipv4(ROUTER_IPV4)); - if let Ok(buffer) = socket.send(bytes.len(), metadata) { - buffer.copy_from_slice(&bytes); - } - } -} - -fn service_dns_udp( - sockets: &mut SocketSet<'static>, - handle: SocketHandle, - dns: &mut SyntheticDns, - metrics: &InterceptionMetrics, -) { - let socket = sockets.get_mut::(handle); - while socket.can_recv() && socket.can_send() { - let Ok((query, request_meta)) = socket.recv() else { - break; - }; - metrics.dns_query(); - let response = dns.answer(query); - let mut response_meta = UdpMetadata::from(request_meta.endpoint); - response_meta.local_address = request_meta - .local_address - .or(Some(IpAddress::Ipv4(ROUTER_IPV4))); - if let Ok(buffer) = socket.send(response.len(), response_meta) { - buffer.copy_from_slice(&response); - metrics.dns_answer(); - } - } -} - -#[derive(Default)] -struct SyntheticDns { - by_name: HashMap, - by_address: HashMap, -} - -impl SyntheticDns { - fn hostname(&self, address: Ipv4Addr) -> Option<&str> { - self.by_address.get(&address).map(String::as_str) - } - - fn allocate(&mut self, hostname: &str) -> Option { - if let Some(address) = self.by_name.get(hostname) { - return Some(*address); - } - if self.by_name.len() >= SYNTHETIC_DNS_CAPACITY { - return None; - } - let offset = self.by_name.len() as u32 + 1; - let address = Ipv4Addr::from(u32::from_be_bytes([198, 18, 0, 0]) + offset); - self.by_name.insert(hostname.to_owned(), address); - self.by_address.insert(address, hostname.to_owned()); - Some(address) - } - - fn answer(&mut self, query: &[u8]) -> Vec { - let id = query.get(..2).unwrap_or(&[0, 0]); - let id = [id[0], id[1]]; - let parsed = parse_dns_question(query); - let (question, rcode, answer) = match parsed { - Ok(question) if question.opcode != 0 => (Some(question), 4, None), - Ok(question) if question.class != 1 => (Some(question), 4, None), - Ok(question) if question.kind == 1 => { - let answer = self.allocate(&question.name); - let rcode = if answer.is_some() { 0 } else { 2 }; - (Some(question), rcode, answer) - } - Ok(question) if question.kind == 28 => (Some(question), 0, None), - Ok(question) => (Some(question), 4, None), - Err(()) => (None, 1, None), - }; - let mut response = Vec::with_capacity(64); - response.extend_from_slice(&id); - let request_flags = query - .get(2..4) - .map(|flags| u16::from_be_bytes([flags[0], flags[1]])) - .unwrap_or(0); - let flags = 0x8000 | (request_flags & 0x0100) | 0x0080 | rcode; - response.extend_from_slice(&flags.to_be_bytes()); - response.extend_from_slice(&(u16::from(question.is_some())).to_be_bytes()); - response.extend_from_slice(&(u16::from(answer.is_some())).to_be_bytes()); - response.extend_from_slice(&0_u16.to_be_bytes()); - response.extend_from_slice(&0_u16.to_be_bytes()); - if let Some(question) = question { - response.extend_from_slice(&query[12..question.end]); - if let Some(address) = answer { - response.extend_from_slice(&[0xc0, 0x0c]); - response.extend_from_slice(&1_u16.to_be_bytes()); - response.extend_from_slice(&1_u16.to_be_bytes()); - response.extend_from_slice(&60_u32.to_be_bytes()); - response.extend_from_slice(&4_u16.to_be_bytes()); - response.extend_from_slice(&address.octets()); - } - } - response - } -} - -struct DnsQuestion { - name: String, - kind: u16, - class: u16, - opcode: u8, - end: usize, -} - -fn parse_dns_question(query: &[u8]) -> Result { - if query.len() < 12 || u16::from_be_bytes([query[4], query[5]]) != 1 { - return Err(()); - } - let flags = u16::from_be_bytes([query[2], query[3]]); - if flags & 0x8000 != 0 { - return Err(()); - } - let mut position = 12; - let mut labels = Vec::new(); - let mut encoded_len = 0; - loop { - let length = *query.get(position).ok_or(())? as usize; - position += 1; - if length == 0 { - break; - } - if length > 63 || length & 0xc0 != 0 { - return Err(()); - } - encoded_len += length + 1; - if encoded_len > 255 || position + length > query.len() { - return Err(()); - } - let label = std::str::from_utf8(&query[position..position + length]).map_err(|_| ())?; - if !label - .bytes() - .all(|byte| byte.is_ascii_alphanumeric() || byte == b'-' || byte == b'_') - { - return Err(()); - } - labels.push(label.to_ascii_lowercase()); - position += length; - } - if labels.is_empty() || position + 4 > query.len() { - return Err(()); - } - let kind = u16::from_be_bytes([query[position], query[position + 1]]); - let class = u16::from_be_bytes([query[position + 2], query[position + 3]]); - position += 4; - Ok(DnsQuestion { - name: labels.join("."), - kind, - class, - opcode: ((flags >> 11) & 0x0f) as u8, - end: position, - }) -} - -fn synthetic_address(address: Ipv4Addr) -> bool { - let octets = address.octets(); - octets[0] == 198 && (octets[1] == 18 || octets[1] == 19) -} - -/// Hard VM destinations that cannot be enabled by Public mode or by a DNS -/// rebinding result. RFC1918 and loopback remain available intentionally for -/// explicit host/LAN services; the virtual router and special-purpose ranges -/// do not. -fn forbidden_host_address(address: IpAddr) -> bool { - let IpAddr::V4(address) = address else { - return true; - }; - let [a, b, c, _] = address.octets(); - address.is_unspecified() - || address.is_broadcast() - || address.is_multicast() - || address.is_link_local() - || a == 0 - || a >= 224 - || (a == 192 && b == 0 && c == 0) - || (a == 192 && b == 0 && c == 2) - || (a == 198 && (b == 18 || b == 19)) - || (a == 198 && b == 51 && c == 100) - || (a == 203 && b == 0 && c == 113) -} - -fn blocked_literal_destination(address: Ipv4Addr) -> bool { - forbidden_host_address(IpAddr::V4(address)) -} - -fn initial_tcp_syn(frame: &[u8]) -> Option { - let ethernet = EthernetFrame::new_checked(frame).ok()?; - if ethernet.ethertype() != EthernetProtocol::Ipv4 { - return None; - } - let ipv4 = Ipv4Packet::new_checked(ethernet.payload()).ok()?; - if !ipv4.verify_checksum() - || ipv4.src_addr() != GUEST_IPV4 - || ipv4.next_header() != IpProtocol::Tcp - || ipv4.more_frags() - || ipv4.frag_offset() != 0 - { - return None; - } - let tcp = TcpPacket::new_checked(ipv4.payload()).ok()?; - if !tcp.verify_checksum(&ipv4.src_addr().into(), &ipv4.dst_addr().into()) - || !tcp.syn() - || tcp.ack() - || tcp.rst() - { - return None; - } - Some(FlowKey { - guest_port: tcp.src_port(), - destination_addr: ipv4.dst_addr(), - destination_port: tcp.dst_port(), - }) -} - -fn supported_guest_frame(frame: &[u8]) -> bool { - let Ok(ethernet) = EthernetFrame::new_checked(frame) else { - return false; - }; - if ethernet.ethertype() == EthernetProtocol::Arp { - return true; - } - if ethernet.ethertype() != EthernetProtocol::Ipv4 { - return false; - } - let Ok(ipv4) = Ipv4Packet::new_checked(ethernet.payload()) else { - return false; - }; - if !ipv4.verify_checksum() - || (ipv4.src_addr() != GUEST_IPV4 && !ipv4.src_addr().is_unspecified()) - { - return false; - } - match ipv4.next_header() { - IpProtocol::Tcp => true, - IpProtocol::Udp => { - let payload = ipv4.payload(); - payload.len() >= 4 && matches!(u16::from_be_bytes([payload[2], payload[3]]), 53 | 67) - } - _ => false, - } -} - -#[derive(Default)] -struct FrameReader { - bytes: Vec, - offset: usize, -} - -impl FrameReader { - async fn read_more( - &mut self, - reader: &mut tokio::net::unix::OwnedReadHalf, - ) -> io::Result { - if self.offset > 0 && self.offset == self.bytes.len() { - self.bytes.clear(); - self.offset = 0; - } else if self.offset > 4096 { - self.bytes.drain(..self.offset); - self.offset = 0; - } - let mut buffer = [0; 16 * 1024]; - let read = reader.read(&mut buffer).await?; - self.bytes.extend_from_slice(&buffer[..read]); - Ok(read) - } - - fn next_frame(&mut self) -> anyhow::Result>> { - if self.bytes.len().saturating_sub(self.offset) < 4 { - return Ok(None); - } - let length = u32::from_be_bytes( - self.bytes[self.offset..self.offset + 4] - .try_into() - .expect("four byte prefix"), - ) as usize; - anyhow::ensure!( - (14..=MAX_FRAME_LEN).contains(&length), - "invalid libkrun Ethernet frame length {length}" - ); - if self.bytes.len() - self.offset < length + 4 { - return Ok(None); - } - let start = self.offset + 4; - let frame = self.bytes[start..start + length].to_vec(); - self.offset = start + length; - Ok(Some(frame)) - } -} - -async fn write_frame( - writer: &mut tokio::net::unix::OwnedWriteHalf, - frame: &[u8], -) -> io::Result<()> { - writer - .write_all(&(frame.len() as u32).to_be_bytes()) - .await?; - writer.write_all(frame).await -} - -#[derive(Default)] -struct FrameDevice { - receive_queue: VecDeque>, - transmit_queue: VecDeque>, -} - -struct FrameRxToken(Vec); -struct FrameTxToken<'a>(&'a mut VecDeque>); - -impl Device for FrameDevice { - type RxToken<'a> = FrameRxToken; - type TxToken<'a> = FrameTxToken<'a>; - - fn receive(&mut self, _timestamp: Instant) -> Option<(Self::RxToken<'_>, Self::TxToken<'_>)> { - let frame = self.receive_queue.pop_front()?; - Some((FrameRxToken(frame), FrameTxToken(&mut self.transmit_queue))) - } - - fn transmit(&mut self, _timestamp: Instant) -> Option> { - Some(FrameTxToken(&mut self.transmit_queue)) - } - - fn capabilities(&self) -> DeviceCapabilities { - let mut capabilities = DeviceCapabilities::default(); - capabilities.medium = Medium::Ethernet; - capabilities.max_transmission_unit = FRAME_MTU; - capabilities.max_burst_size = None; - capabilities - } -} - -impl RxToken for FrameRxToken { - fn consume(self, function: F) -> R - where - F: FnOnce(&[u8]) -> R, - { - function(&self.0) - } -} - -impl TxToken for FrameTxToken<'_> { - fn consume(self, length: usize, function: F) -> R - where - F: FnOnce(&mut [u8]) -> R, - { - let mut frame = vec![0; length]; - let result = function(&mut frame); - self.0.push_back(frame); - result - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn query(name: &str, kind: u16) -> Vec { - let mut bytes = vec![0x12, 0x34, 0x01, 0x00, 0x00, 0x01, 0, 0, 0, 0, 0, 0]; - for label in name.split('.') { - bytes.push(label.len() as u8); - bytes.extend_from_slice(label.as_bytes()); - } - bytes.push(0); - bytes.extend_from_slice(&kind.to_be_bytes()); - bytes.extend_from_slice(&1_u16.to_be_bytes()); - bytes - } - - #[test] - fn synthetic_dns_is_stable_and_does_not_resolve_on_the_host() { - let mut dns = SyntheticDns::default(); - let first = dns.answer(&query("Api.Example.COM", 1)); - let second = dns.answer(&query("api.example.com", 1)); - assert_eq!(&first[first.len() - 4..], &second[second.len() - 4..]); - assert_eq!(dns.by_name.len(), 1); - assert_eq!( - dns.hostname(Ipv4Addr::new(198, 18, 0, 1)), - Some("api.example.com") - ); - } - - #[test] - fn aaaa_returns_nodata_and_unknown_type_returns_notimp() { - let mut dns = SyntheticDns::default(); - let aaaa = dns.answer(&query("example.com", 28)); - assert_eq!(u16::from_be_bytes([aaaa[6], aaaa[7]]), 0); - assert_eq!(u16::from_be_bytes([aaaa[2], aaaa[3]]) & 0x0f, 0); - let txt = dns.answer(&query("example.com", 16)); - assert_eq!(u16::from_be_bytes([txt[2], txt[3]]) & 0x0f, 4); - } - - #[test] - fn malformed_dns_returns_formerr() { - let mut dns = SyntheticDns::default(); - let response = dns.answer(&[0xab, 0xcd]); - assert_eq!(&response[..2], &[0xab, 0xcd]); - assert_eq!(u16::from_be_bytes([response[2], response[3]]) & 0x0f, 1); - } - - #[test] - fn virtual_and_link_local_destinations_are_never_literal_egress() { - for address in [ - ROUTER_IPV4, - GUEST_IPV4, - Ipv4Addr::new(198, 18, 1, 1), - Ipv4Addr::new(169, 254, 169, 254), - Ipv4Addr::new(224, 0, 0, 1), - Ipv4Addr::BROADCAST, - ] { - assert!(blocked_literal_destination(address), "{address}"); - } - assert!(!blocked_literal_destination(Ipv4Addr::new(10, 0, 0, 1))); - assert!(!blocked_literal_destination(Ipv4Addr::new( - 93, 184, 216, 34 - ))); - } - - #[test] - fn frame_reader_handles_fragmented_and_coalesced_frames() { - let mut reader = FrameReader::default(); - reader.bytes.extend_from_slice(&14_u32.to_be_bytes()); - reader.bytes.extend_from_slice(&[1; 14]); - reader.bytes.extend_from_slice(&15_u32.to_be_bytes()); - reader.bytes.extend_from_slice(&[2; 15]); - assert_eq!(reader.next_frame().unwrap().unwrap(), vec![1; 14]); - assert_eq!(reader.next_frame().unwrap().unwrap(), vec![2; 15]); - assert!(reader.next_frame().unwrap().is_none()); - } - - #[test] - fn host_connection_does_not_imply_the_guest_handshake_is_complete() { - assert!(guest_handshake_pending(tcp::State::SynReceived)); - assert!(!guest_handshake_pending(tcp::State::Established)); - assert!(!guest_handshake_pending(tcp::State::CloseWait)); - } -} diff --git a/crates/persisting-pchronicle-cli/Cargo.toml b/crates/persisting-pchronicle-cli/Cargo.toml index ebb2dfe9b..1df4deba1 100644 --- a/crates/persisting-pchronicle-cli/Cargo.toml +++ b/crates/persisting-pchronicle-cli/Cargo.toml @@ -24,10 +24,8 @@ dirs.workspace = true futures.workspace = true include_dir = "=0.7.4" libc.workspace = true -persisting-gateway.workspace = true persisting-pchronicle = { workspace = true, features = ["lance-store", "s3-store"] } lance.workspace = true -persisting-events = { workspace = true, features = ["control"] } serde = { workspace = true, features = ["derive"] } serde_json.workspace = true tempfile.workspace = true @@ -43,6 +41,5 @@ uuid = { workspace = true, features = ["v4"] } [dev-dependencies] http-body-util.workspace = true -persisting-agentctl.workspace = true reqwest = { workspace = true, features = ["json", "rustls-tls"] } tower = { version = "0.5", features = ["util"] } diff --git a/crates/persisting-pchronicle-cli/README.md b/crates/persisting-pchronicle-cli/README.md index a908841e3..84f0d247f 100644 --- a/crates/persisting-pchronicle-cli/README.md +++ b/crates/persisting-pchronicle-cli/README.md @@ -3,9 +3,7 @@ **Standalone `pchronicle` CLI for onboarding, browsing, querying, importing, exporting, and serving trajectory Datasets.** -Owns the `pchronicle` binary, Warehouse HTTP, the write-capable -`--control` plane for local integrations, optional Gateway ingest/forwarding -flags, and the embed of staged `pchronicle-web` assets at build time. +Owns the `pchronicle` binary, loopback Warehouse HTTP, and embedded Web assets. Does not own trajectory models, Lance storage, Catalog, or the query engine — those live in [`persisting-pchronicle`](../persisting-pchronicle/README.md). @@ -15,7 +13,7 @@ Does not own the Web UI source — Current commands include `onboard`, `dataset` (pin/unpin/list/show/set/rename), `list`/`ls`, `stats`, bounded read-only `query`, built-in `stats` reports, assisted `agent` sessions, Source-local `find`, create/append/replace `import`, -destructive `drop`, complete-trajectory `export`, directory `sync`, `echo`, and +destructive `drop`, complete-trajectory `export`, directory `sync`, and `serve`. Import and export support ATIF, OpenAI Messages, ACTF, Storyline JSON, and record-level Compact JSONL. `sync --from SOURCE [--mirror MIRROR] [--to OUTPUT]` polls a source directory and, on each coalesced batch, @@ -24,10 +22,9 @@ Storyline Lance Dataset at `--to`. Provide at least one destination. With `--input-format compact-jsonl`, only `--mirror` is valid. Use `--once` for a finite run. -`pchronicle serve --control 127.0.0.1:0 URI` exposes the authenticated local -Control protocol for event producers. `serve --listen` is the read-only Warehouse. Public bind addresses are -rejected. Small deterministic Datasets live in -[`../../examples/data`](../../examples/data). +`pchronicle serve --listen 127.0.0.1:8080 URI` exposes the loopback Warehouse +API and Web UI. Public bind addresses are rejected. Deterministic Datasets live +in [`examples/data`](../../examples/data). ## Develop @@ -59,6 +56,5 @@ just chronicle-binary - [pChronicle get started](../../docs/src/en/pchronicle/get-started.md) - [pChronicle CLI reference](../../docs/src/en/pchronicle/reference/cli.md) - [Local read-only Dataset server](../../docs/src/en/pchronicle/guides/serve.md) -- [Gateway forwarding, rewriting, and capture](../../docs/src/en/pchronicle/guides/serve-gateway.md) - [`persisting-pchronicle`](../persisting-pchronicle/README.md) - [`pchronicle-web`](../../pchronicle-web/README.md) diff --git a/crates/persisting-pchronicle-cli/src/control.rs b/crates/persisting-pchronicle-cli/src/control.rs deleted file mode 100644 index 37039cff8..000000000 --- a/crates/persisting-pchronicle-cli/src/control.rs +++ /dev/null @@ -1,509 +0,0 @@ -//! pChronicle's write-capable control plane. -//! -//! The Warehouse HTTP server remains read-only. This long-lived JSONL/stdin -//! protocol is intended for trusted local integrations. - -use anyhow::{Context, Result}; -use fs2::FileExt; -use persisting_events::{ - AttemptRecord as ProtocolAttemptRecord, AttemptRecordState as ProtocolAttemptRecordState, - CHRONICLE_CONTROL_MAX_FRAME_BYTES, CHRONICLE_CONTROL_VERSION, ChronicleControlEnvelope, - ChronicleControlRequest, ChronicleControlResponse, ChronicleControlResponseEnvelope, - ChronicleServeControlReady, CommitRunOutcome, LeaseAcquireOutcome, TrajectoryAppendRequest, - TrajectoryAppendResponse, TrajectoryFormat, -}; -use persisting_pchronicle::storage::{ - AttemptRecord, AttemptRecordState, AttemptRegistry, DatasetLocation, RawEventLanceStore, - RunControlStore, StoryCoords, -}; -use std::fs::OpenOptions; -use std::io::Write; -use std::net::SocketAddr; -use std::sync::Arc; -use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader}; -use tokio::net::TcpStream; - -pub(super) struct PreparedControl { - listener: tokio::net::TcpListener, - endpoint: SocketAddr, - auth_token: String, - control: Arc, - attempts: Arc, -} - -impl PreparedControl { - pub(super) async fn bind(storage: &str, listen: SocketAddr) -> Result { - let control = Arc::new( - RunControlStore::open(storage) - .await - .context("open pChronicle Run control store")?, - ); - let attempts = Arc::new( - AttemptRegistry::open(storage) - .await - .context("open pChronicle Attempt registry")?, - ); - let listener = tokio::net::TcpListener::bind(listen) - .await - .context("bind pChronicle control listener")?; - let endpoint = listener - .local_addr() - .context("read pChronicle control listener address")?; - Ok(Self { - listener, - endpoint, - auth_token: uuid::Uuid::new_v4().simple().to_string(), - control, - attempts, - }) - } - - pub(super) fn ready(&self) -> ChronicleServeControlReady { - ChronicleServeControlReady { - endpoint: self.endpoint.to_string(), - auth_token: self.auth_token.clone(), - } - } - - pub(super) async fn serve(self, shutdown: impl std::future::Future) -> Result<()> { - tokio::pin!(shutdown); - loop { - tokio::select! { - _ = &mut shutdown => return Ok(()), - accepted = self.listener.accept() => { - let (stream, _) = accepted.context("accept pChronicle control client")?; - stream - .set_nodelay(true) - .context("configure pChronicle control socket")?; - let control = Arc::clone(&self.control); - let attempts = Arc::clone(&self.attempts); - let auth_token = self.auth_token.clone(); - tokio::spawn(async move { - if let Err(error) = - serve_connection(stream, control, attempts, auth_token).await - { - eprintln!("pChronicle control request failed: {error:#}"); - } - }); - } - } - } - } -} - -async fn serve_connection( - stream: TcpStream, - control: Arc, - attempts: Arc, - auth_token: String, -) -> Result<()> { - let (read, mut write) = stream.into_split(); - let mut input = BufReader::new(read.take((CHRONICLE_CONTROL_MAX_FRAME_BYTES + 1) as u64)); - let mut line = String::new(); - let bytes = input - .read_line(&mut line) - .await - .context("read control request")?; - anyhow::ensure!(bytes > 0, "empty pChronicle control request"); - let request_id = serde_json::from_str::(&line) - .map(|request| request.request_id) - .unwrap_or(0); - let response = match decode_request(&line, &auth_token) { - Ok(request) => handle_request(&control, &attempts, request).await, - Err(error) => Err(error), - }; - let response = ChronicleControlResponseEnvelope { - version: CHRONICLE_CONTROL_VERSION, - request_id, - response: match response { - Ok(response) => response, - Err(error) => ChronicleControlResponse::Error { - message: format!("{error:#}"), - }, - }, - }; - let mut encoded = serde_json::to_vec(&response).context("encode control response")?; - anyhow::ensure!( - encoded.len() <= CHRONICLE_CONTROL_MAX_FRAME_BYTES, - "pChronicle control response exceeds frame limit" - ); - encoded.push(b'\n'); - write - .write_all(&encoded) - .await - .context("write control response")?; - write.flush().await.context("flush control response")?; - Ok(()) -} - -fn decode_request(line: &str, auth_token: &str) -> Result { - anyhow::ensure!( - line.len() <= CHRONICLE_CONTROL_MAX_FRAME_BYTES, - "pChronicle control request exceeds frame limit" - ); - let request: ChronicleControlEnvelope = - serde_json::from_str(line).context("decode pChronicle control request")?; - anyhow::ensure!( - request.version == CHRONICLE_CONTROL_VERSION, - "unsupported pChronicle control version {}", - request.version - ); - anyhow::ensure!( - request.auth_token == auth_token, - "invalid pChronicle control token" - ); - Ok(request) -} - -async fn handle_request( - control: &RunControlStore, - attempts: &AttemptRegistry, - envelope: ChronicleControlEnvelope, -) -> Result { - use ChronicleControlRequest as Request; - use ChronicleControlResponse as Response; - - Ok(match envelope.request { - Request::Ping => Response::Pong, - Request::AcquireLease { - run_id, - task_id, - owner, - ttl_ms, - } => Response::LeaseAcquire(map_lease_outcome( - control - .acquire_lease(&run_id, task_id.as_deref(), &owner, ttl_ms) - .await?, - )), - Request::TakeoverLease { - run_id, - task_id, - owner, - ttl_ms, - } => Response::LeaseAcquire(map_lease_outcome( - control - .takeover_lease(&run_id, task_id.as_deref(), &owner, ttl_ms) - .await?, - )), - Request::BindAttempt { - run_id, - epoch, - attempt_id, - } => Response::Boolean(control.bind_attempt(&run_id, epoch, attempt_id).await?), - Request::RenewLease { - run_id, - epoch, - owner, - ttl_ms, - } => Response::Boolean(control.renew_lease(&run_id, epoch, &owner, ttl_ms).await?), - Request::CommitRun(request) => { - Response::CommitRun(map_commit_outcome(control.commit_run(request).await?)) - } - Request::GetRun { run_id } => Response::Run(control.get(&run_id).await?), - Request::ListRuns => Response::Runs(control.list().await?), - Request::GetAttempt { run_id } => { - Response::Attempt(attempts.get(&run_id).await?.map(map_attempt_record)) - } - Request::PublishAttemptActive { - run_id, - attempt_id, - lease_epoch, - ttl_ms, - } => Response::Boolean( - attempts - .publish_active(&run_id, &attempt_id, lease_epoch, ttl_ms) - .await?, - ), - Request::HeartbeatAttempt { - run_id, - attempt_id, - lease_epoch, - ttl_ms, - } => Response::Boolean( - attempts - .heartbeat(&run_id, &attempt_id, lease_epoch, ttl_ms) - .await?, - ), - Request::PublishAttemptTerminal { - run_id, - attempt_id, - lease_epoch, - result, - } => Response::Boolean( - attempts - .publish_terminal(&run_id, &attempt_id, lease_epoch, result) - .await?, - ), - Request::AppendTrajectory(request) => Response::TrajectoryAppend( - append_trajectory(request) - .await - .context("append trajectory")?, - ), - }) -} - -fn map_lease_outcome( - value: persisting_pchronicle::storage::LeaseAcquireOutcome, -) -> LeaseAcquireOutcome { - match value { - persisting_pchronicle::storage::LeaseAcquireOutcome::Acquired(value) => { - LeaseAcquireOutcome::Acquired(value) - } - persisting_pchronicle::storage::LeaseAcquireOutcome::Held(value) => { - LeaseAcquireOutcome::Held(value) - } - persisting_pchronicle::storage::LeaseAcquireOutcome::AlreadyCommitted(value) => { - LeaseAcquireOutcome::AlreadyCommitted(value) - } - } -} - -fn map_commit_outcome(value: persisting_pchronicle::storage::CommitRunOutcome) -> CommitRunOutcome { - match value { - persisting_pchronicle::storage::CommitRunOutcome::Committed(value) => { - CommitRunOutcome::Committed(value) - } - persisting_pchronicle::storage::CommitRunOutcome::AlreadyCommitted(value) => { - CommitRunOutcome::AlreadyCommitted(value) - } - persisting_pchronicle::storage::CommitRunOutcome::StaleLease { - supplied_epoch, - current_epoch, - } => CommitRunOutcome::StaleLease { - supplied_epoch, - current_epoch, - }, - persisting_pchronicle::storage::CommitRunOutcome::Conflict(value) => { - CommitRunOutcome::Conflict(value) - } - } -} - -fn map_attempt_record(value: AttemptRecord) -> ProtocolAttemptRecord { - ProtocolAttemptRecord { - revision: value.revision, - run_id: value.run_id, - attempt_id: value.attempt_id, - lease_epoch: value.lease_epoch, - state: match value.state { - AttemptRecordState::Active => ProtocolAttemptRecordState::Active, - AttemptRecordState::Terminal => ProtocolAttemptRecordState::Terminal, - }, - heartbeat_at_unix_ms: value.heartbeat_at_unix_ms, - expires_at_unix_ms: value.expires_at_unix_ms, - terminal_result: value.terminal_result, - } -} - -pub(crate) async fn append_trajectory( - request: TrajectoryAppendRequest, -) -> Result { - if request.format == TrajectoryFormat::Json { - return append_json_trajectory(request).await; - } - let session = StoryCoords::new( - &request.storage, - &request.agent_id, - &request.session_id, - request.root_session_id.clone(), - ); - let store = RawEventLanceStore; - let accepted_records = request.records.len(); - let note = if request.records.is_empty() { - "No non-empty records; storage unchanged.".to_string() - } else { - let outcome = store.append_events(&session, &request.records).await?; - format!("canonical Lance event log. {}", outcome.note) - }; - Ok(TrajectoryAppendResponse { - dataset: store.display_path(&session)?, - storage: request.storage, - agent_id: request.agent_id, - session_id: request.session_id, - accepted_records, - status: "ok".into(), - note, - }) -} - -async fn append_json_trajectory( - request: TrajectoryAppendRequest, -) -> Result { - let base = format!( - "{}/json/{}/{}", - request.storage.trim_end_matches('/'), - safe_segment(&request.agent_id), - safe_segment(&request.session_id) - ); - let location = DatasetLocation::parse(&base)?; - if let Some(path) = location.local_path() { - std::fs::create_dir_all(path)?; - let file_path = path.join("events.jsonl"); - let mut file = OpenOptions::new() - .create(true) - .append(true) - .open(&file_path)?; - // Serialize concurrent AppendTrajectory(Json) writers so lines are not - // interleaved across serve connections (and across processes). - file.lock_exclusive() - .with_context(|| format!("lock {}", file_path.display()))?; - for record in &request.records { - serde_json::to_writer(&mut file, record)?; - file.write_all(b"\n")?; - } - file.flush()?; - let _ = file.unlock(); - } else { - // Object stores do not provide an atomic append primitive. Keep each - // event immutable and independently recoverable under the warehouse - // prefix; the control worker serializes these writes. - for record in &request.records { - let uri = format!("{base}/events/{:020}.json", record.seq); - let location = DatasetLocation::parse(&uri)?; - location - .put_bytes(&serde_json::to_vec(record)?, false) - .await?; - } - } - Ok(TrajectoryAppendResponse { - dataset: base, - storage: request.storage, - agent_id: request.agent_id, - session_id: request.session_id, - accepted_records: request.records.len(), - status: "ok".into(), - note: "JSON EventRecord warehouse log".into(), - }) -} - -fn safe_segment(value: &str) -> String { - let mut result = value - .chars() - .map(|ch| { - if ch.is_ascii_alphanumeric() || matches!(ch, '-' | '_' | '.') { - ch - } else { - '_' - } - }) - .collect::(); - if result.is_empty() || result == "." || result == ".." { - result = "_unknown".into(); - } - result.truncate(128); - result -} - -#[cfg(test)] -mod tests { - use super::*; - use persisting_events::ChronicleServeControlReady; - use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; - - #[tokio::test] - async fn prepared_control_serves_ping_and_stops_on_shutdown() -> Result<()> { - let storage = tempfile::tempdir()?; - let prepared = PreparedControl::bind( - storage.path().to_str().unwrap(), - "127.0.0.1:0".parse().unwrap(), - ) - .await?; - let ChronicleServeControlReady { - endpoint, - auth_token, - } = prepared.ready(); - let (stop_tx, stop_rx) = tokio::sync::oneshot::channel::<()>(); - let server = tokio::spawn(async move { - prepared - .serve(async { - let _ = stop_rx.await; - }) - .await - }); - - let mut stream = TcpStream::connect(endpoint).await?; - let request = ChronicleControlEnvelope { - version: CHRONICLE_CONTROL_VERSION, - request_id: 7, - auth_token, - request: ChronicleControlRequest::Ping, - }; - let mut encoded = serde_json::to_vec(&request)?; - encoded.push(b'\n'); - stream.write_all(&encoded).await?; - stream.flush().await?; - - let mut response = String::new(); - BufReader::new(stream).read_line(&mut response).await?; - let response: ChronicleControlResponseEnvelope = serde_json::from_str(&response)?; - assert_eq!(response.request_id, 7); - assert!(matches!(response.response, ChronicleControlResponse::Pong)); - - stop_tx.send(()).unwrap(); - tokio::time::timeout(std::time::Duration::from_secs(5), server).await???; - Ok(()) - } - - #[tokio::test] - async fn concurrent_json_trajectory_appends_do_not_interleave_lines() -> Result<()> { - let storage = tempfile::tempdir()?; - let storage_path = storage.path().to_string_lossy().into_owned(); - let writers = 8usize; - let records_per_writer = 20usize; - let mut handles = Vec::new(); - for writer in 0..writers { - let storage_path = storage_path.clone(); - handles.push(tokio::spawn(async move { - let mut records = Vec::with_capacity(records_per_writer); - for index in 0..records_per_writer { - let seq = (writer * records_per_writer + index) as u64; - records.push(persisting_events::EventRecord { - identity: Default::default(), - seq, - source: "test".into(), - kind: "note".into(), - timestamp: None, - session_id: Some("session".into()), - agent_id: Some("agent".into()), - parent_uuid: None, - trace_id: None, - call_id: None, - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload: serde_json::json!({"writer": writer, "index": index}), - }); - } - append_json_trajectory(TrajectoryAppendRequest { - storage: storage_path, - agent_id: "agent".into(), - session_id: "session".into(), - format: TrajectoryFormat::Json, - root_session_id: Some("session".into()), - records, - }) - .await - })); - } - for handle in handles { - handle.await??; - } - let path = storage - .path() - .join("json") - .join("agent") - .join("session") - .join("events.jsonl"); - let text = std::fs::read_to_string(&path)?; - let mut lines = 0usize; - for line in text.lines() { - let value: serde_json::Value = serde_json::from_str(line) - .with_context(|| format!("invalid JSONL line: {line:?}"))?; - assert!(value.get("seq").is_some()); - lines += 1; - } - assert_eq!(lines, writers * records_per_writer); - Ok(()) - } -} diff --git a/crates/persisting-pchronicle-cli/src/exchange/import.rs b/crates/persisting-pchronicle-cli/src/exchange/import.rs index 554c3cef5..a61bb7794 100644 --- a/crates/persisting-pchronicle-cli/src/exchange/import.rs +++ b/crates/persisting-pchronicle-cli/src/exchange/import.rs @@ -172,17 +172,6 @@ pub(crate) async fn run_import( let from_location = (!args.stream) .then(|| DatasetLocation::parse(&args.from)) .transpose()?; - let canonical = if let Some(location) = &from_location { - let looks_like_store = location.is_object_store() - || location.local_path().is_some_and(std::path::Path::is_dir); - if looks_like_store { - probe_canonical_event_store(location.as_str()).await? - } else { - None - } - } else { - None - }; let output_arg = match args.output.as_deref() { Some(output) => expand_dataset_reference(output, settings_override, false)?, None => default_import_output(&args, settings_override)?, @@ -195,8 +184,7 @@ pub(crate) async fn run_import( .await; } let requested_destination = DatasetLocation::parse(&output_arg)?; - if canonical.is_none() - && requested_destination.is_object_store() + if requested_destination.is_object_store() && args.output_format != Some(ImportOutputFormat::Storyline) { anyhow::ensure!( @@ -208,21 +196,7 @@ pub(crate) async fn run_import( prepare_import_destination(&args, &output_arg, stdin_is_terminal, stdin, stderr).await?; let destination = prepared.location; let replace_existing = prepared.replace_existing; - if let Some(snapshot) = canonical { - anyhow::ensure!( - mode != ImportMode::Append, - "canonical event import does not support --append" - ); - return run_canonical_event_import( - args, - snapshot, - destination, - replace_existing, - stdout, - stderr, - ) - .await; - } + let mut progress = CliProgress::new(stderr_is_terminal); let _s3_throttle_ui = progress.attach_object_store_throttle(); let object_store_from = from_location @@ -2073,100 +2047,6 @@ pub(crate) async fn commit_storyline_import_batch( Ok(imported_total) } -pub(crate) async fn run_canonical_event_import( - args: ImportArgs, - _snapshot: EventFactSnapshot, - destination: DatasetLocation, - replace_existing: bool, - stdout: &mut dyn Write, - stderr: &mut dyn Write, -) -> Result<()> { - anyhow::ensure!( - args.format == ExchangeFormat::Auto, - "canonical event import does not accept a JSON exchange --format" - ); - anyhow::ensure!( - args.output_format != Some(ImportOutputFormat::Preserve), - "canonical event import cannot preserve an existing canonical event Store" - ); - if destination.exists().await? && !replace_existing { - return Err(cli_boundary_error( - BoundaryCode::Conflict, - "import output already exists", - )); - } - let output_uri = destination.as_str().to_string(); - - let (report, staged_path) = if replace_existing { - let output = destination - .local_path() - .context("replace import output must be a local Dataset path")?; - let parent = output - .parent() - .context("replace import output must have a parent directory")?; - let staging = tempfile::Builder::new() - .prefix(".pchronicle-import-") - .tempdir_in(parent) - .with_context(|| format!("create import staging directory in {}", parent.display()))?; - let staging_uri = staging.path().to_string_lossy().into_owned(); - let report = - match build_storyline_projection(&args.from, &staging_uri, "events.lance").await? { - StorylineProjectionBuildOutcome::Built(report) => report, - StorylineProjectionBuildOutcome::OutputNotEmpty => { - return Err(cli_boundary_error( - BoundaryCode::Conflict, - "import staging Dataset already exists", - )); - } - }; - std::fs::File::open(staging.path()) - .and_then(|directory| directory.sync_all()) - .context("sync import staging directory")?; - (report, Some((staging.keep(), output.to_path_buf()))) - } else { - let report = - match build_storyline_projection(&args.from, &output_uri, "events.lance").await? { - StorylineProjectionBuildOutcome::Built(report) => report, - StorylineProjectionBuildOutcome::OutputNotEmpty => { - return Err(cli_boundary_error( - BoundaryCode::Conflict, - "import output already exists", - )); - } - }; - (report, None) - }; - if let Some((staging_path, output)) = staged_path { - let mut cleanup = StagingPathGuard::new(staging_path.clone()); - publish_staged_dataset(&staging_path, &output, true, None).await?; - cleanup.disarm(); - } - let response = ImportResponse { - dataset_uri: output_uri, - source_path: Some("events.lance".into()), - format: Some("events".into()), - output_format: ImportOutputFormat::Storyline.response_name().into(), - sources: 1, - trajectories: report.storylines, - fact_rows: Some(report.fact_rows), - input_bytes: None, - on_disk_bytes: None, - }; - serde_json::to_writer_pretty(&mut *stdout, &response) - .context("encode canonical event import JSON")?; - writeln!(stdout).context("write canonical event import JSON")?; - writeln!( - stderr, - "dataset_uri={} source=events.lance format=events output_format={} trajectories={} fact_rows={}", - response.dataset_uri, - response.output_format, - response.trajectories, - report.fact_rows, - ) - .context("write canonical event import metadata")?; - Ok(()) -} - #[cfg(test)] mod tests { use super::*; diff --git a/crates/persisting-pchronicle-cli/src/exchange/progress.rs b/crates/persisting-pchronicle-cli/src/exchange/progress.rs index 5c0bcae5b..ab523a080 100644 --- a/crates/persisting-pchronicle-cli/src/exchange/progress.rs +++ b/crates/persisting-pchronicle-cli/src/exchange/progress.rs @@ -248,7 +248,6 @@ impl StageHandle { let _ = self.repaint(); } - #[allow(dead_code)] pub(crate) fn record_bytes(&self, bytes: u64) { if let Ok(mut state) = self.state.lock() { state.bytes = state.bytes.saturating_add(bytes); @@ -659,7 +658,6 @@ impl CliProgress { delete.repaint() } - #[allow(dead_code)] pub(crate) fn note_committed(&self, committed: u64, batch_bytes: u64) -> Result<()> { self.stage(StageId::Commit) .note_committed(committed, batch_bytes); diff --git a/crates/persisting-pchronicle-cli/src/gateway_capture.rs b/crates/persisting-pchronicle-cli/src/gateway_capture.rs deleted file mode 100644 index 4ff5fc8b1..000000000 --- a/crates/persisting-pchronicle-cli/src/gateway_capture.rs +++ /dev/null @@ -1,221 +0,0 @@ -//! Bridge Gateway's synchronous capture sink to pChronicle's async Lance writer. - -use std::sync::Arc; - -use persisting_gateway::record::EventRecord; -use persisting_gateway::session::storage::CaptureRoute; -use persisting_gateway::sink::{CallbackSink, CaptureEventSink}; -use persisting_pchronicle::storage::{ - RawEventAppendOutcome, RawEventAppendSender, RawEventAppendWorker, StoryCoords, - raw_event_append_queue, -}; - -use crate::gateway_partition::{GatewayPartitionRouter, GatewaySplitTemplate}; -use crate::{cli_boundary_error, server::problem::BoundaryCode}; - -pub(crate) struct GatewayCaptureWriter { - worker: RawEventAppendWorker, -} - -impl GatewayCaptureWriter { - pub(crate) fn finish(self) -> anyhow::Result<()> { - self.worker.finish() - } -} - -#[cfg(test)] -pub(crate) fn gateway_capture_sink( - dataset_uri: &str, - default_agent_id: &str, -) -> anyhow::Result<(Arc, GatewayCaptureWriter)> { - gateway_capture_sink_with_factory(dataset_uri, default_agent_id, None, raw_event_append_queue) -} - -pub(crate) fn gateway_capture_sink_with_split( - dataset_uri: &str, - default_agent_id: &str, - split: Option, -) -> anyhow::Result<(Arc, GatewayCaptureWriter)> { - gateway_capture_sink_with_factory(dataset_uri, default_agent_id, split, raw_event_append_queue) -} - -fn gateway_capture_sink_with_factory( - dataset_uri: &str, - default_agent_id: &str, - split: Option, - queue_factory: F, -) -> anyhow::Result<(Arc, GatewayCaptureWriter)> -where - F: FnOnce() -> anyhow::Result<(RawEventAppendSender, RawEventAppendWorker)>, -{ - let partitions = GatewayPartitionRouter::new(dataset_uri, split)?; - let (tx, worker) = queue_factory()?; - - let callback_tx = tx.clone(); - let callback = CallbackSink::new( - default_agent_id, - move |route: &CaptureRoute, agent_id, record: EventRecord| { - let user = record - .payload - .get("headers") - .and_then(|headers| headers.get("x-persisting-user-id")) - .and_then(|value| value.as_str()); - let dataset_uri = partitions.route(&route.seq_key(), user); - let outcome = callback_tx.append_durable( - StoryCoords::new( - dataset_uri, - agent_id, - route.storage_session_id.clone(), - route.append_root_session(), - ), - record, - )?; - map_append_outcome(outcome) - }, - ); - Ok((Arc::new(callback), GatewayCaptureWriter { worker })) -} - -fn map_append_outcome(outcome: RawEventAppendOutcome) -> anyhow::Result<()> { - match outcome { - RawEventAppendOutcome::Accepted => Ok(()), - RawEventAppendOutcome::Full => Err(cli_boundary_error( - BoundaryCode::ResourceExhausted, - "pChronicle append capacity exhausted", - )), - RawEventAppendOutcome::Unavailable => Err(cli_boundary_error( - BoundaryCode::Unavailable, - "service unavailable", - )), - } -} - -#[cfg(test)] -mod tests { - use persisting_gateway::record::EventRecord; - use persisting_pchronicle::storage::{RawEventAppendOutcome, RawEventLanceStore}; - - use super::*; - - #[test] - fn gateway_sink_returns_after_lance_event_is_visible() { - let dir = tempfile::tempdir().unwrap(); - let dataset_uri = dir.path().join("dataset").to_string_lossy().into_owned(); - let (sink, writer) = gateway_capture_sink(&dataset_uri, "agent").unwrap(); - let route = CaptureRoute { - root_session: Some("run".into()), - session_id: "session".into(), - storage_session_id: "session".into(), - subagent_id: None, - }; - let mut record = EventRecord { - identity: Default::default(), - seq: 0, - source: "gateway".into(), - kind: "test".into(), - timestamp: None, - session_id: None, - agent_id: None, - parent_uuid: None, - trace_id: None, - call_id: None, - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload: serde_json::Value::Null, - }; - - sink.append(&route, "agent", &mut record).unwrap(); - - let coords = StoryCoords::new(dataset_uri, "agent", "session", Some("run".into())); - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .unwrap(); - let replay = runtime - .block_on(RawEventLanceStore.replay(&coords, 0, None)) - .unwrap(); - assert_eq!(replay.records.len(), 1); - writer.finish().unwrap(); - } - - #[test] - fn gateway_maps_only_explicit_append_rejections() { - map_append_outcome(RawEventAppendOutcome::Accepted).unwrap(); - assert_eq!( - map_append_outcome(RawEventAppendOutcome::Full) - .unwrap_err() - .to_string(), - "resource_exhausted: pChronicle append capacity exhausted" - ); - assert_eq!( - map_append_outcome(RawEventAppendOutcome::Unavailable) - .unwrap_err() - .to_string(), - "unavailable: service unavailable" - ); - } - - #[test] - fn gateway_sink_preserves_append_source_chain_and_rejected_sequence() { - let dir = tempfile::tempdir().unwrap(); - let dataset_uri = dir.path().join("not-a-directory"); - std::fs::write(&dataset_uri, b"file").unwrap(); - let (sink, writer) = - gateway_capture_sink(dataset_uri.to_string_lossy().as_ref(), "agent").unwrap(); - let route = CaptureRoute { - root_session: Some("run".into()), - session_id: "session".into(), - storage_session_id: "session".into(), - subagent_id: None, - }; - let mut record = EventRecord { - identity: Default::default(), - seq: 99, - source: "gateway".into(), - kind: "test".into(), - timestamp: None, - session_id: None, - agent_id: None, - parent_uuid: None, - trace_id: None, - call_id: None, - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload: serde_json::Value::Null, - }; - - let error = sink.append(&route, "agent", &mut record).unwrap_err(); - - assert!( - error.chain().count() >= 2, - "missing source chain: {error:#}" - ); - assert_eq!(sink.peek_next_seq(&route), Some(0)); - writer.finish().unwrap(); - } - - #[test] - fn gateway_queue_start_failure_preserves_source_chain() { - let result = gateway_capture_sink_with_factory("memory://capture", "agent", None, || { - Err( - anyhow::Error::new(std::io::Error::other("spawn-failure-sentinel")) - .context("start injected append worker"), - ) - }); - let error = match result { - Ok(_) => panic!("injected queue startup failure unexpectedly succeeded"), - Err(error) => error, - }; - let rendered = format!("{error:#}"); - - assert!( - rendered.contains("start injected append worker"), - "{rendered}" - ); - assert!(rendered.contains("spawn-failure-sentinel"), "{rendered}"); - } -} diff --git a/crates/persisting-pchronicle-cli/src/gateway_ingest.rs b/crates/persisting-pchronicle-cli/src/gateway_ingest.rs deleted file mode 100644 index a5637fb19..000000000 --- a/crates/persisting-pchronicle-cli/src/gateway_ingest.rs +++ /dev/null @@ -1,660 +0,0 @@ -//! Config-free HTTP ingestion Gateway for canonical trajectory events. - -use std::collections::BTreeMap; -use std::sync::Arc; - -use anyhow::{Context, Result}; -use axum::body::Bytes; -use axum::extract::{DefaultBodyLimit, State}; -use axum::http::{HeaderMap, StatusCode, header}; -use axum::response::{IntoResponse, Response}; -use axum::routing::{get, post}; -use axum::{Json, Router}; -use persisting_events::{EventRecord, TrajectoryAppendResponse}; -use persisting_pchronicle::storage::{ - RawEventAppendOutcome, RawEventAppendSender, RawEventAppendWorker, StoryCoords, - raw_event_append_queue, -}; -use serde::{Deserialize, Serialize}; - -use crate::gateway_partition::{GatewayPartitionRouter, GatewaySplitTemplate}; - -const MAX_INGEST_BODY_BYTES: usize = 8 * 1024 * 1024; -const MAX_INGEST_RECORDS: usize = 256; -const USER_HEADER: &str = "x-persisting-user-id"; - -#[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -struct GatewayIngestRequest { - agent_id: String, - session_id: String, - #[serde(default)] - root_session_id: Option, - records: Vec, -} - -#[derive(Debug, Serialize)] -struct GatewayStatus { - status: &'static str, - mode: &'static str, - dataset: String, - #[serde(skip_serializing_if = "Option::is_none")] - split: Option, -} - -#[derive(Debug, Serialize)] -struct GatewayError { - error: &'static str, - message: String, -} - -struct IngestState { - partitions: GatewayPartitionRouter, - sender: RawEventAppendSender, -} - -pub(crate) struct PreparedIngestGateway { - listener: tokio::net::TcpListener, - endpoint: String, - state: Arc, - worker: RawEventAppendWorker, -} - -impl PreparedIngestGateway { - pub(crate) async fn bind( - listen: std::net::SocketAddr, - dataset_uri: String, - split: Option, - ) -> Result { - let listener = tokio::net::TcpListener::bind(listen) - .await - .with_context(|| format!("bind pChronicle ingest Gateway to {listen}"))?; - let endpoint = listener - .local_addr() - .context("read pChronicle ingest Gateway listen address")? - .to_string(); - let (sender, worker) = raw_event_append_queue()?; - Ok(Self { - listener, - endpoint, - state: Arc::new(IngestState { - partitions: GatewayPartitionRouter::new(dataset_uri, split)?, - sender, - }), - worker, - }) - } - - pub(crate) fn endpoint(&self) -> &str { - &self.endpoint - } - - pub(crate) fn dataset_uri(&self) -> &str { - self.state.partitions.dataset_uri() - } - - pub(crate) fn split_source(&self) -> Option<&str> { - self.state.partitions.split_source() - } - - pub(crate) async fn serve( - self, - shutdown: impl std::future::Future + Send + 'static, - ) -> Result<()> { - let Self { - listener, - state, - worker, - .. - } = self; - let app = Router::new() - .route("/healthz", get(health)) - .route("/admin/status", get(health)) - .route("/v1/events", post(append_events)) - .route("/api/public/otel/v1/traces", post(append_langfuse_otel)) - .route("/api/public/otel/v1/traces/", post(append_langfuse_otel)) - .layer(DefaultBodyLimit::max(MAX_INGEST_BODY_BYTES)) - .with_state(state); - let result = axum::serve(listener, app) - .with_graceful_shutdown(shutdown) - .await - .context("serve pChronicle ingest Gateway"); - worker - .finish() - .context("finish pChronicle ingest Gateway writer")?; - result - } -} - -/// Langfuse's current ingestion endpoint is OTLP/HTTP. Langfuse sends either -/// OTLP JSON or protobuf; both encodings are normalized into the same -/// loss-aware canonical event representation. -async fn append_langfuse_otel( - State(state): State>, - headers: HeaderMap, - body: Bytes, -) -> Response { - let content_type = headers - .get(header::CONTENT_TYPE) - .and_then(|value| value.to_str().ok()) - .and_then(|value| value.split(';').next()); - let is_json = content_type - .map(|value| value.eq_ignore_ascii_case("application/json") || value.ends_with("+json")) - .unwrap_or_else(|| body.first().is_some_and(|byte| *byte == b'{')); - let is_protobuf = content_type.is_some_and(|value| { - value.eq_ignore_ascii_case("application/x-protobuf") - || value.eq_ignore_ascii_case("application/protobuf") - || value.eq_ignore_ascii_case("application/octet-stream") - }); - if !is_json && !is_protobuf { - return ( - StatusCode::UNSUPPORTED_MEDIA_TYPE, - Json(GatewayError { - error: "unsupported_content_type", - message: "Langfuse OTLP Gateway accepts application/json or application/x-protobuf" - .into(), - }), - ) - .into_response(); - } - let document = if is_json { - match serde_json::from_slice(&body) { - Ok(document) => document, - Err(error) => { - return ( - StatusCode::BAD_REQUEST, - Json(GatewayError { - error: "invalid_otlp_json", - message: error.to_string(), - }), - ) - .into_response(); - } - } - } else { - match protobuf_otlp_to_json(&body) { - Ok(document) => document, - Err(error) => { - return ( - StatusCode::BAD_REQUEST, - Json(GatewayError { - error: "invalid_otlp_protobuf", - message: error, - }), - ) - .into_response(); - } - } - }; - let records = persisting_pchronicle::document::langfuse_otlp_json_to_events(&document); - match append_langfuse_inner(&state, &headers, records) { - Ok(_) => otlp_success_response(is_json), - Err((status, code, message)) => ( - status, - Json(GatewayError { - error: code, - message, - }), - ) - .into_response(), - } -} - -fn otlp_success_response(json_encoding: bool) -> Response { - // OTLP requires an ExportTraceServiceResponse with partial_success unset - // for full success. Its protobuf encoding is an empty message (zero bytes). - if json_encoding { - return ( - StatusCode::OK, - [(header::CONTENT_TYPE, "application/json")], - Json(serde_json::json!({})), - ) - .into_response(); - } - ( - StatusCode::OK, - [(header::CONTENT_TYPE, "application/x-protobuf")], - Bytes::new(), - ) - .into_response() -} - -fn append_langfuse_inner( - state: &IngestState, - headers: &HeaderMap, - records: Vec, -) -> std::result::Result { - let mut groups: BTreeMap<(String, String), Vec> = BTreeMap::new(); - for record in records { - let agent_id = record.agent_id.clone().unwrap_or_else(|| "langfuse".into()); - let session_id = record - .session_id - .clone() - .or_else(|| record.trace_id.clone()) - .unwrap_or_else(|| format!("batch-{}", groups.len())); - groups - .entry((agent_id, session_id)) - .or_default() - .push(record); - } - let mut accepted = 0; - for ((agent_id, session_id), records) in groups { - // The canonical endpoint deliberately caps one request at 256 records; - // an official Codex rollout can contain thousands of OTLP spans, so - // split one Langfuse batch into durable chunks without exposing that - // implementation limit to the OTLP client. - for chunk in records.chunks(MAX_INGEST_RECORDS) { - accepted += append_events_inner( - state, - headers, - GatewayIngestRequest { - agent_id: agent_id.clone(), - session_id: session_id.clone(), - root_session_id: None, - records: chunk.to_vec(), - }, - )? - .accepted_records; - } - } - Ok(accepted) -} - -#[derive(Debug)] -struct ProtoReader<'a> { - bytes: &'a [u8], - offset: usize, -} - -type ProtoField<'a> = (u32, u8, &'a [u8]); - -impl<'a> ProtoReader<'a> { - fn new(bytes: &'a [u8]) -> Self { - Self { bytes, offset: 0 } - } - - fn next(&mut self) -> Result>, String> { - if self.offset == self.bytes.len() { - return Ok(None); - } - let tag = self.varint()?; - let number = (tag >> 3) as u32; - let wire = (tag & 7) as u8; - if number == 0 { - return Err("protobuf field number is zero".into()); - } - let value = match wire { - 0 => { - let start = self.offset; - self.varint()?; - &self.bytes[start..self.offset] - } - 1 => self.take(8)?, - 2 => { - let length = self.varint()? as usize; - self.take(length)? - } - 5 => self.take(4)?, - _ => return Err(format!("unsupported protobuf wire type {wire}")), - }; - Ok(Some((number, wire, value))) - } - - fn varint(&mut self) -> Result { - let mut value = 0u64; - for shift in (0..70).step_by(7) { - let byte = *self - .bytes - .get(self.offset) - .ok_or_else(|| "truncated protobuf varint".to_string())?; - self.offset += 1; - value |= u64::from(byte & 0x7f) << shift; - if byte & 0x80 == 0 { - return Ok(value); - } - } - Err("protobuf varint is too long".into()) - } - - fn take(&mut self, length: usize) -> Result<&'a [u8], String> { - let end = self - .offset - .checked_add(length) - .ok_or_else(|| "protobuf length overflow".to_string())?; - let value = self - .bytes - .get(self.offset..end) - .ok_or_else(|| "truncated protobuf field".to_string())?; - self.offset = end; - Ok(value) - } -} - -fn protobuf_otlp_to_json(bytes: &[u8]) -> Result { - let mut resources = Vec::new(); - let mut reader = ProtoReader::new(bytes); - while let Some((number, wire, value)) = reader.next()? { - if number == 1 && wire == 2 { - resources.push(protobuf_resource_spans(value)?); - } - } - Ok(serde_json::json!({ "resourceSpans": resources })) -} - -fn protobuf_resource_spans(bytes: &[u8]) -> Result { - let mut resource = serde_json::json!({ "attributes": [] }); - let mut scopes = Vec::new(); - let mut reader = ProtoReader::new(bytes); - while let Some((number, wire, value)) = reader.next()? { - match (number, wire) { - (1, 2) => resource = protobuf_resource(value)?, - (2, 2) => scopes.push(protobuf_scope_spans(value)?), - _ => {} - } - } - Ok(serde_json::json!({ "resource": resource, "scopeSpans": scopes })) -} - -fn protobuf_resource(bytes: &[u8]) -> Result { - let mut attributes = Vec::new(); - let mut reader = ProtoReader::new(bytes); - while let Some((number, wire, value)) = reader.next()? { - if number == 1 && wire == 2 { - attributes.push(protobuf_key_value(value)?); - } - } - Ok(serde_json::json!({ "attributes": attributes })) -} - -fn protobuf_scope_spans(bytes: &[u8]) -> Result { - let mut spans = Vec::new(); - let mut reader = ProtoReader::new(bytes); - while let Some((number, wire, value)) = reader.next()? { - if number == 2 && wire == 2 { - spans.push(protobuf_span(value)?); - } - } - Ok(serde_json::json!({ "spans": spans })) -} - -fn protobuf_span(bytes: &[u8]) -> Result { - let mut span = serde_json::Map::new(); - let mut attributes = Vec::new(); - let mut reader = ProtoReader::new(bytes); - while let Some((number, wire, value)) = reader.next()? { - match (number, wire) { - (1, 2) => { - span.insert("traceId".into(), serde_json::Value::String(hex(value))); - } - (2, 2) => { - span.insert("spanId".into(), serde_json::Value::String(hex(value))); - } - (4, 2) => { - span.insert("parentSpanId".into(), serde_json::Value::String(hex(value))); - } - (5, 2) => { - span.insert("name".into(), serde_json::Value::String(text(value)?)); - } - (7, 1) => { - span.insert( - "startTimeUnixNano".into(), - serde_json::Value::String( - u64::from_le_bytes(value.try_into().unwrap()).to_string(), - ), - ); - } - (8, 1) => { - span.insert( - "endTimeUnixNano".into(), - serde_json::Value::String( - u64::from_le_bytes(value.try_into().unwrap()).to_string(), - ), - ); - } - (9, 2) => attributes.push(protobuf_key_value(value)?), - _ => {} - } - } - span.insert("attributes".into(), serde_json::Value::Array(attributes)); - Ok(serde_json::Value::Object(span)) -} - -fn protobuf_key_value(bytes: &[u8]) -> Result { - let mut key = String::new(); - let mut value = serde_json::Value::Null; - let mut reader = ProtoReader::new(bytes); - while let Some((number, wire, field)) = reader.next()? { - match (number, wire) { - (1, 2) => key = text(field)?, - (2, 2) => value = protobuf_any_value(field)?, - _ => {} - } - } - Ok(serde_json::json!({ "key": key, "value": value })) -} - -fn protobuf_any_value(bytes: &[u8]) -> Result { - let mut reader = ProtoReader::new(bytes); - while let Some((number, wire, value)) = reader.next()? { - let (name, json_value) = match (number, wire) { - (1, 2) => ("stringValue", serde_json::Value::String(text(value)?)), - (2, 0) => ( - "boolValue", - serde_json::Value::Bool(value.iter().any(|byte| *byte != 0)), - ), - (3, 0) => ( - "intValue", - serde_json::Value::String(decode_varint(value)?.to_string()), - ), - (4, 1) => ( - "doubleValue", - serde_json::json!(f64::from_le_bytes(value.try_into().unwrap())), - ), - (7, 2) => ("bytesValue", serde_json::Value::String(hex(value))), - _ => continue, - }; - return Ok(serde_json::json!({ name: json_value })); - } - Ok(serde_json::json!({ "stringValue": "" })) -} - -fn decode_varint(bytes: &[u8]) -> Result { - let mut reader = ProtoReader::new(bytes); - reader.varint() -} - -fn text(bytes: &[u8]) -> Result { - String::from_utf8(bytes.to_vec()).map_err(|_| "protobuf string is not valid UTF-8".into()) -} - -fn hex(bytes: &[u8]) -> String { - bytes.iter().map(|byte| format!("{byte:02x}")).collect() -} - -async fn health(State(state): State>) -> Json { - Json(GatewayStatus { - status: "ok", - mode: "ingest", - dataset: state.partitions.dataset_uri().to_string(), - split: state.partitions.split_source().map(str::to_string), - }) -} - -async fn append_events( - State(state): State>, - headers: HeaderMap, - Json(request): Json, -) -> Response { - match append_events_inner(&state, &headers, request) { - Ok(response) => (StatusCode::OK, Json(response)).into_response(), - Err((status, code, message)) => ( - status, - Json(GatewayError { - error: code, - message, - }), - ) - .into_response(), - } -} - -fn append_events_inner( - state: &IngestState, - headers: &HeaderMap, - request: GatewayIngestRequest, -) -> std::result::Result { - let agent_id = validate_identity("agent_id", request.agent_id)?; - let session_id = validate_identity("session_id", request.session_id)?; - let root_session_id = request - .root_session_id - .map(|value| validate_identity("root_session_id", value)) - .transpose()?; - if request.records.len() > MAX_INGEST_RECORDS { - return Err(( - StatusCode::PAYLOAD_TOO_LARGE, - "too_many_records", - format!("one ingest request may contain at most {MAX_INGEST_RECORDS} records"), - )); - } - for record in &request.records { - record - .validate() - .map_err(|error| (StatusCode::BAD_REQUEST, "invalid_event", error.to_string()))?; - } - - let route_key = format!( - "{agent_id}|{}", - root_session_id.as_deref().unwrap_or(&session_id) - ); - let user = headers - .get(USER_HEADER) - .and_then(|value| value.to_str().ok()); - let storage = state.partitions.route(&route_key, user); - let coords = StoryCoords::new( - storage.clone(), - agent_id.clone(), - session_id.clone(), - root_session_id, - ); - let accepted_records = request.records.len(); - let outcome = state - .sender - .append_durable_batch( - request - .records - .into_iter() - .map(|record| (coords.clone(), record)) - .collect(), - ) - .map_err(|error| { - ( - StatusCode::SERVICE_UNAVAILABLE, - "append_failed", - format!("append canonical event batch: {error:#}"), - ) - })?; - match outcome { - RawEventAppendOutcome::Accepted => {} - RawEventAppendOutcome::Full => { - return Err(( - StatusCode::TOO_MANY_REQUESTS, - "capacity_exhausted", - "pChronicle append capacity is exhausted".into(), - )); - } - RawEventAppendOutcome::Unavailable => { - return Err(( - StatusCode::SERVICE_UNAVAILABLE, - "unavailable", - "pChronicle append service is unavailable".into(), - )); - } - } - Ok(TrajectoryAppendResponse { - storage: storage.clone(), - agent_id, - session_id, - accepted_records, - dataset: storage, - status: "ok".into(), - note: "canonical events are durably visible".into(), - }) -} - -fn validate_identity( - field: &'static str, - value: String, -) -> std::result::Result { - let value = value.trim(); - if value.is_empty() - || value == "." - || value == ".." - || value.contains('/') - || value.contains('\\') - { - return Err(( - StatusCode::BAD_REQUEST, - "invalid_identity", - format!("{field} must be one non-empty path-safe segment"), - )); - } - Ok(value.to_string()) -} - -#[cfg(test)] -mod tests { - use super::*; - use axum::http::HeaderValue; - use persisting_events::EventIdentity; - - fn record() -> EventRecord { - EventRecord { - identity: EventIdentity::default(), - seq: 0, - source: "test".into(), - kind: "llm.request".into(), - timestamp: None, - session_id: None, - agent_id: None, - parent_uuid: None, - trace_id: None, - call_id: None, - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload: serde_json::Value::Null, - } - } - - #[test] - fn append_routes_to_user_partition_and_is_durable() { - let temporary = tempfile::tempdir().unwrap(); - let (sender, worker) = raw_event_append_queue().unwrap(); - let state = IngestState { - partitions: GatewayPartitionRouter::new( - temporary.path().to_string_lossy(), - Some(GatewaySplitTemplate::parse("{user}").unwrap()), - ) - .unwrap(), - sender, - }; - let mut headers = HeaderMap::new(); - headers.insert(USER_HEADER, HeaderValue::from_static("alice")); - let response = append_events_inner( - &state, - &headers, - GatewayIngestRequest { - agent_id: "agent".into(), - session_id: "session".into(), - root_session_id: None, - records: vec![record()], - }, - ) - .unwrap(); - assert!(response.storage.ends_with("/alice")); - assert_eq!(response.accepted_records, 1); - drop(state); - worker.finish().unwrap(); - } -} diff --git a/crates/persisting-pchronicle-cli/src/gateway_partition.rs b/crates/persisting-pchronicle-cli/src/gateway_partition.rs deleted file mode 100644 index bdf2b329a..000000000 --- a/crates/persisting-pchronicle-cli/src/gateway_partition.rs +++ /dev/null @@ -1,244 +0,0 @@ -//! Safe, bounded physical partitioning below one logical Gateway Dataset. - -use std::collections::HashMap; -use std::sync::{Mutex, MutexGuard}; - -use anyhow::{Context, Result}; -use chrono::{DateTime, Datelike, Timelike, Utc}; - -const MAX_TEMPLATE_BYTES: usize = 256; -const MAX_TEMPLATE_SEGMENTS: usize = 16; -const MAX_USER_SEGMENT_BYTES: usize = 80; - -fn lock_recover(mutex: &Mutex) -> MutexGuard<'_, T> { - mutex - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()) -} - -#[derive(Debug, Clone, PartialEq, Eq)] -enum Segment { - Literal(String), - User, - Date, - Hour, -} - -/// A deliberately small partition-template language. -/// -/// Templates are relative paths made from safe literal segments and the exact -/// placeholders `{user}`, `{date}`, and `{hour}`. -#[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) struct GatewaySplitTemplate { - source: String, - segments: Vec, -} - -impl GatewaySplitTemplate { - pub(crate) fn parse(source: &str) -> Result { - let source = source.trim(); - anyhow::ensure!( - !source.is_empty(), - "Gateway split template must not be empty" - ); - anyhow::ensure!( - source.len() <= MAX_TEMPLATE_BYTES, - "Gateway split template exceeds {MAX_TEMPLATE_BYTES} bytes" - ); - anyhow::ensure!( - !source.starts_with('/') && !source.ends_with('/'), - "Gateway split template must be relative and must not start or end with '/'" - ); - anyhow::ensure!( - !source.contains('\\'), - "Gateway split template must use '/' separators" - ); - - let raw_segments = source.split('/').collect::>(); - anyhow::ensure!( - raw_segments.len() <= MAX_TEMPLATE_SEGMENTS, - "Gateway split template exceeds {MAX_TEMPLATE_SEGMENTS} segments" - ); - let mut segments = Vec::with_capacity(raw_segments.len()); - for raw in raw_segments { - anyhow::ensure!( - !raw.is_empty(), - "Gateway split template contains an empty segment" - ); - let segment = match raw { - "{user}" => Segment::User, - "{date}" => Segment::Date, - "{hour}" => Segment::Hour, - "." | ".." => { - anyhow::bail!("Gateway split template must not contain '.' or '..' segments") - } - literal => { - anyhow::ensure!( - literal - .bytes() - .all(|byte| byte.is_ascii_alphanumeric() || b"-_.".contains(&byte)), - "Gateway split literal '{literal}' contains an unsupported character" - ); - anyhow::ensure!( - !literal.contains('{') && !literal.contains('}'), - "unknown Gateway split placeholder '{literal}'" - ); - Segment::Literal(literal.to_string()) - } - }; - segments.push(segment); - } - Ok(Self { - source: source.to_string(), - segments, - }) - } - - pub(crate) fn source(&self) -> &str { - &self.source - } - - fn render(&self, user: Option<&str>, started_at: DateTime) -> String { - let user = safe_user_segment(user); - self.segments - .iter() - .map(|segment| match segment { - Segment::Literal(value) => value.clone(), - Segment::User => user.clone(), - Segment::Date => format!( - "{:04}-{:02}-{:02}", - started_at.year(), - started_at.month(), - started_at.day() - ), - Segment::Hour => format!("{:02}", started_at.hour()), - }) - .collect::>() - .join("/") - } -} - -/// Pins each logical run/session to the first partition selected for it. -/// Request and response records therefore cannot land in different hourly -/// prefixes, and a long-lived session remains one canonical event source. -#[derive(Debug)] -pub(crate) struct GatewayPartitionRouter { - dataset_uri: String, - split: Option, - routes: Mutex>, -} - -impl GatewayPartitionRouter { - pub(crate) fn new( - dataset_uri: impl Into, - split: Option, - ) -> Result { - let dataset_uri = dataset_uri.into(); - persisting_pchronicle::storage::DatasetLocation::parse(&dataset_uri) - .context("validate Gateway Dataset URI")?; - Ok(Self { - dataset_uri: dataset_uri.trim_end_matches('/').to_string(), - split, - routes: Mutex::new(HashMap::new()), - }) - } - - pub(crate) fn dataset_uri(&self) -> &str { - &self.dataset_uri - } - - pub(crate) fn split_source(&self) -> Option<&str> { - self.split.as_ref().map(GatewaySplitTemplate::source) - } - - pub(crate) fn route(&self, route_key: &str, user: Option<&str>) -> String { - let Some(split) = &self.split else { - return self.dataset_uri.clone(); - }; - let mut routes = lock_recover(&self.routes); - routes - .entry(route_key.to_string()) - .or_insert_with(|| { - let prefix = split.render(user, Utc::now()); - format!("{}/{prefix}", self.dataset_uri) - }) - .clone() - } -} - -fn safe_user_segment(user: Option<&str>) -> String { - let Some(user) = user.map(str::trim).filter(|value| !value.is_empty()) else { - return "_unknown".to_string(); - }; - let mut safe = String::with_capacity(user.len().min(MAX_USER_SEGMENT_BYTES)); - let mut changed = false; - for byte in user.bytes() { - if safe.len() >= MAX_USER_SEGMENT_BYTES { - changed = true; - break; - } - if byte.is_ascii_alphanumeric() || b"-_".contains(&byte) { - safe.push(char::from(byte)); - } else { - safe.push('_'); - changed = true; - } - } - if safe.is_empty() || safe == "." || safe == ".." { - changed = true; - safe = "user".to_string(); - } - if changed { - let digest = blake3::hash(user.as_bytes()).to_hex(); - let keep = MAX_USER_SEGMENT_BYTES.saturating_sub(13).min(safe.len()); - safe.truncate(keep); - safe.push('-'); - safe.push_str(&digest[..12]); - } - safe -} - -#[cfg(test)] -mod tests { - use super::*; - use chrono::TimeZone; - - #[test] - fn template_renders_safe_relative_prefix() { - let template = GatewaySplitTemplate::parse("archive/{user}/{date}/{hour}").unwrap(); - let at = Utc.with_ymd_and_hms(2026, 8, 25, 7, 30, 0).unwrap(); - assert_eq!( - template.render(Some("alice"), at), - "archive/alice/2026-08-25/07" - ); - } - - #[test] - fn template_rejects_escape_and_unknown_placeholders() { - for invalid in ["/date", "date/", "date//hour", "../hour", "{month}"] { - assert!(GatewaySplitTemplate::parse(invalid).is_err(), "{invalid}"); - } - } - - #[test] - fn user_values_are_bounded_and_collision_resistant() { - let first = safe_user_segment(Some("alice@example.com")); - let second = safe_user_segment(Some("alice+example.com")); - assert!(first.starts_with("alice_example_com-")); - assert_ne!(first, second); - assert_eq!(safe_user_segment(None), "_unknown"); - } - - #[test] - fn router_pins_one_session_to_its_first_partition() { - let router = GatewayPartitionRouter::new( - "/tmp/captures", - Some(GatewaySplitTemplate::parse("{user}/{date}/{hour}").unwrap()), - ) - .unwrap(); - let first = router.route("agent|session", Some("alice")); - let second = router.route("agent|session", Some("bob")); - assert_eq!(first, second); - assert!(first.contains("/alice/")); - } -} diff --git a/crates/persisting-pchronicle-cli/src/lib.rs b/crates/persisting-pchronicle-cli/src/lib.rs index 2c2104dd9..9d55d50dd 100644 --- a/crates/persisting-pchronicle-cli/src/lib.rs +++ b/crates/persisting-pchronicle-cli/src/lib.rs @@ -1,14 +1,9 @@ #![recursion_limit = "256"] mod agent; -mod control; mod exchange; -mod gateway_capture; -mod gateway_ingest; -mod gateway_partition; mod onboard; mod output; -mod projection_supervisor; pub mod server; mod settings; mod sync; @@ -30,8 +25,7 @@ use std::time::Duration; use anyhow::{Context, Result, anyhow, bail}; use clap::{ArgGroup, Args, Parser, Subcommand, ValueEnum}; -use futures::{StreamExt, stream, stream::FuturesUnordered}; -use persisting_events::{CHRONICLE_SERVE_READY_VERSION, ChronicleServeReady}; +use futures::{StreamExt, stream}; use persisting_pchronicle::query::{ChronicleQueryEngine, ChronicleQueryExecutionOptions}; use persisting_pchronicle::search::{ FindExpr, FindJsonOperator, FindJsonPredicate, FindTextPredicate, combine_match_expressions, @@ -40,12 +34,9 @@ use persisting_pchronicle::search::{ #[cfg(test)] use persisting_pchronicle::storage::StorylineLanceStore; use persisting_pchronicle::storage::{ - AutomaticProjectionInspection, AutomaticProjectionState, CatalogErrorPolicy, - CatalogSnapshotOptions, CatalogSourceKind, CatalogSourceStatus, DEFAULT_DATASET_NAME, - DatasetCatalogSnapshot, DatasetLocation, DatasetMount, DiscoveredSource, EventFactSnapshot, - PathListKind, StorylineProjectionBuildOutcome, automatic_projection_inventory, - build_storyline_projection, inspect_automatic_storyline_projection, - probe_canonical_event_store, + CatalogErrorPolicy, CatalogSnapshotOptions, CatalogSourceKind, CatalogSourceStatus, + DEFAULT_DATASET_NAME, DatasetCatalogSnapshot, DatasetLocation, DatasetMount, DiscoveredSource, + PathListKind, }; use serde::{Deserialize, Serialize}; @@ -157,12 +148,7 @@ fn primary_dataset_reference(command: &Command) -> Option<&str> { Command::Export(args) => args.from.as_deref(), Command::Agent(args) => args.dataset_reference(), Command::Import(args) => args.output.as_deref(), - Command::Onboard(_) - | Command::Dataset(_) - | Command::Sync(_) - | Command::Echo(_) - | Command::Dev(_) - | Command::Serve(_) => None, + Command::Onboard(_) | Command::Dataset(_) | Command::Sync(_) | Command::Serve(_) => None, } } @@ -271,13 +257,7 @@ enum Command { /// `--mirror` replaces a Compact JSONL Lance Dataset; `--to` replaces a /// Storyline Lance Dataset. Provide either or both. Sync(sync::SyncArgs), - /// Run a deterministic local LLM upstream for Gateway testing. - #[command(hide = true)] - Echo(EchoArgs), - /// Unstable developer tools. - #[command(hide = true)] - Dev(DevArgs), - /// Run explicitly enabled Warehouse, Control, and Gateway services. + /// Serve the read-only Warehouse API and Web UI. Serve(Box), } @@ -917,22 +897,13 @@ struct ExportArgs { ArgGroup::new("dataset_source") .required(true) .multiple(true) - .args(["config", "storage", "positional_storage", "gateway_dataset", "catalog_config", "catalog_query_worker"]) + .args(["config", "storage", "positional_storage", "catalog_config", "catalog_query_worker"]) ), group( ArgGroup::new("storage_source") .multiple(true) .args(["storage", "positional_storage"]) - ), - group( - ArgGroup::new("serve_component") - .multiple(true) - .args(["listen", "control", "gateway", "gateway_config"]) - ), - group( - ArgGroup::new("gateway_mode") - .multiple(false) - .args(["gateway", "gateway_config"]) + ) )] struct ServeArgs { @@ -962,10 +933,6 @@ struct ServeArgs { #[arg(long)] listen: Option, - /// Loopback address for the authenticated Control protocol. - #[arg(long, requires = "storage_source", conflicts_with = "config")] - control: Option, - /// Open the Web UI in the system browser after the listener is ready. #[arg(long, requires = "listen")] open: bool, @@ -978,57 +945,12 @@ struct ServeArgs { )] home_links: Vec, - /// Start the config-free canonical event ingest Gateway. - /// `auto` selects loopback and an ephemeral port. - #[arg( - long, - value_name = "ADDRESS", - value_parser = parse_gateway_bind, - requires = "gateway_dataset" - )] - gateway: Option, - - /// Compatibility: start the forwarding LLM Gateway from a TOML file. - #[arg(long, value_name = "FILE", requires = "gateway_dataset")] - gateway_config: Option, - - /// Dataset path or object-store URI that receives Gateway events. - #[arg(long, value_name = "DATASET", requires = "gateway_mode")] - gateway_dataset: Option, - - /// Relative physical partition template using {user}, {date}, and {hour}. - #[arg(long, value_name = "TEMPLATE", requires = "gateway_mode")] - gateway_split: Option, - - /// Wait for this long without new Gateway events before refreshing an - /// existing Storyline projection. New sources are projected immediately. - #[arg( - long = "gateway-split-idle", - value_name = "DURATION", - value_parser = parse_duration_seconds, - default_value = "30m", - requires = "gateway" - )] - gateway_split_idle_seconds: u64, - - /// Local Gateway state directory; required for an object-store Dataset. - #[arg(long, value_name = "DIRECTORY", requires = "gateway_config")] - gateway_state: Option, - - /// Also maintain Gateway's live AgenticMD projection. - #[arg(long, requires = "gateway_config")] - gateway_stream_markdown: bool, - - /// Print Gateway diagnostics, including size-limited request/response bodies, to stderr. - #[arg(long = "gateway-debug", alias = "debug", requires = "gateway_config")] - debug: bool, - /// Directory ACL file. Authenticate API requests and execute them in /// bounded, user-scoped worker processes with explicit backend credentials. #[arg( long = "catalog-config", value_name = "FILE", - conflicts_with_all = ["config", "storage", "positional_storage", "gateway", "gateway_config", "gateway_dataset", "control"] + conflicts_with_all = ["config", "storage", "positional_storage"] )] catalog_config: Option, @@ -1162,45 +1084,6 @@ struct CatalogRevokeArgs { format: OutputFormat, } -fn parse_gateway_bind(value: &str) -> std::result::Result { - if value.eq_ignore_ascii_case("auto") { - return Ok(SocketAddr::from(([127, 0, 0, 1], 0))); - } - value - .parse::() - .map_err(|error| format!("invalid Gateway address '{value}': {error}")) -} - -#[derive(Debug, Args)] -struct EchoArgs { - /// Loopback address for the Echo server. - #[arg(long, default_value = "127.0.0.1:19080")] - listen: SocketAddr, - - /// Encode echoed text directly or as Base64. - #[arg(long, value_enum, default_value_t = EchoEncoding::Plain)] - encoding: EchoEncoding, -} - -#[derive(Debug, Args)] -struct DevArgs { - #[command(subcommand)] - command: DevCommand, -} - -#[derive(Debug, Subcommand)] -enum DevCommand { - /// Run a deterministic local LLM upstream for Gateway tests. - Echo(EchoArgs), -} - -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, ValueEnum)] -enum EchoEncoding { - #[default] - Plain, - Base64, -} - #[derive(Debug, Deserialize)] #[serde(deny_unknown_fields)] struct WarehouseFile { @@ -1308,7 +1191,6 @@ struct StatusCounts { trajectories: u64, steps: u64, tool_calls: u64, - events: u64, } impl std::ops::AddAssign for StatusCounts { @@ -1317,7 +1199,6 @@ impl std::ops::AddAssign for StatusCounts { self.trajectories = self.trajectories.saturating_add(other.trajectories); self.steps = self.steps.saturating_add(other.steps); self.tool_calls = self.tool_calls.saturating_add(other.tool_calls); - self.events = self.events.saturating_add(other.events); } } @@ -1330,75 +1211,9 @@ struct StatusResponse { counts_complete: bool, sources: StatusSources, counts: StatusCounts, - projections: Vec, source_errors: Vec, } -#[derive(Debug, Serialize)] -struct ProjectionStatusResponse { - source_path: String, - projection_path: String, - status: ProjectionStatusName, - #[serde(skip_serializing_if = "Option::is_none")] - generation: Option, - #[serde(skip_serializing_if = "Option::is_none")] - fact_version: Option, - #[serde(skip_serializing_if = "Option::is_none")] - fact_rows: Option, -} - -#[derive(Debug, Clone, Copy, Serialize)] -#[serde(rename_all = "snake_case")] -enum ProjectionStatusName { - Fresh, - Stale, - Missing, - Error, -} - -impl ProjectionStatusName { - fn as_str(self) -> &'static str { - match self { - Self::Fresh => "fresh", - Self::Stale => "stale", - Self::Missing => "missing", - Self::Error => "error", - } - } -} - -impl ProjectionStatusResponse { - fn from_inspection( - source_path: String, - projection_path: String, - inspection: AutomaticProjectionInspection, - ) -> Self { - Self { - source_path, - projection_path, - status: match inspection.state { - AutomaticProjectionState::Fresh => ProjectionStatusName::Fresh, - AutomaticProjectionState::Stale => ProjectionStatusName::Stale, - AutomaticProjectionState::Missing => ProjectionStatusName::Missing, - }, - generation: inspection.generation, - fact_version: Some(inspection.fact_version), - fact_rows: Some(inspection.fact_rows), - } - } - - fn error(source_path: String, projection_path: String) -> Self { - Self { - source_path, - projection_path, - status: ProjectionStatusName::Error, - generation: None, - fact_version: None, - fact_rows: None, - } - } -} - #[derive(Debug, Serialize)] struct StatusSources { total: usize, @@ -1412,8 +1227,6 @@ struct StatusSourceError { error: String, } -const STATUS_PROJECTION_CONCURRENCY: usize = 16; - #[derive(Debug, Serialize)] struct FindResponse { dataset_uri: String, @@ -1612,10 +1425,6 @@ pub async fn run_with_stdio( } Command::Export(args) => run_export(args, config, stdout, &mut diagnostics).await, Command::Sync(args) => sync::run(args, config, &mut diagnostics, stderr_is_terminal).await, - Command::Echo(args) => run_echo(args, &mut diagnostics).await, - Command::Dev(DevArgs { - command: DevCommand::Echo(args), - }) => run_echo(args, &mut diagnostics).await, Command::Serve(args) => { if args.catalog_query_worker { bail!("catalog worker must start before the async runtime"); @@ -1628,147 +1437,8 @@ pub async fn run_with_stdio( } } -struct PreparedProxyGateway { - config: persisting_gateway::config::ProxyConfig, - state_dir: PathBuf, - dataset_uri: String, - split: Option, - stream_markdown: bool, - listener: tokio::net::TcpListener, - admin_listener: tokio::net::TcpListener, - sink: Arc, - writer: gateway_capture::GatewayCaptureWriter, -} - -enum PreparedGateway { - Ingest(gateway_ingest::PreparedIngestGateway), - Proxy(Box), -} - -impl PreparedGateway { - fn endpoint(&self) -> &str { - match self { - Self::Ingest(gateway) => gateway.endpoint(), - Self::Proxy(gateway) => &gateway.config.listen, - } - } - - fn admin_endpoint(&self) -> Option<&str> { - match self { - Self::Ingest(_) => None, - Self::Proxy(gateway) => Some(&gateway.config.admin_listen), - } - } - - fn dataset_uri(&self) -> &str { - match self { - Self::Ingest(gateway) => gateway.dataset_uri(), - Self::Proxy(gateway) => &gateway.dataset_uri, - } - } - - fn split_source(&self) -> Option<&str> { - match self { - Self::Ingest(gateway) => gateway.split_source(), - Self::Proxy(gateway) => gateway.split.as_deref(), - } - } -} - const SERVE_STORAGE_DATASET_NAME: &str = "default"; -fn local_dataset_path(uri: &str) -> Result> { - Ok(DatasetLocation::parse(uri)? - .local_path() - .map(Path::to_path_buf)) -} - -fn parse_gateway_listener(value: &str, label: &str) -> Result { - value - .parse::() - .with_context(|| format!("parse {label} address '{value}'")) -} - -async fn prepare_gateway( - args: &ServeArgs, - dataset_uri: Option<&str>, -) -> Result> { - if args.gateway.is_none() && args.gateway_config.is_none() { - return Ok(None); - } - let dataset_uri = dataset_uri.context("Gateway requires --gateway-dataset DATASET")?; - let split = args - .gateway_split - .as_deref() - .map(gateway_partition::GatewaySplitTemplate::parse) - .transpose()?; - let local_dataset = local_dataset_path(dataset_uri)?; - - if let Some(listen) = args.gateway { - let gateway = - gateway_ingest::PreparedIngestGateway::bind(listen, dataset_uri.to_string(), split) - .await?; - return Ok(Some(PreparedGateway::Ingest(gateway))); - } - - let config_path = args - .gateway_config - .as_deref() - .context("missing Gateway config path")?; - let mut config = persisting_gateway::config::ProxyConfig::from_file(config_path) - .with_context(|| format!("load Gateway config {}", config_path.display()))?; - if args.debug { - config.debug = true; - persisting_gateway::runtime::debug::enable_debug_stderr(); - } - let state_dir = match args.gateway_state.clone() { - Some(path) => path, - None => local_dataset.clone().with_context(|| { - format!( - "Gateway capture Dataset '{dataset_uri}' uses object storage; provide --gateway-state DIRECTORY" - ) - })?, - }; - let listen = parse_gateway_listener(&config.listen, "Gateway")?; - let admin_listen = parse_gateway_listener(&config.admin_listen, "Gateway admin")?; - let listener = tokio::net::TcpListener::bind(listen) - .await - .with_context(|| format!("bind pChronicle Gateway to {listen}"))?; - let admin_listener = tokio::net::TcpListener::bind(admin_listen) - .await - .with_context(|| format!("bind pChronicle Gateway admin API to {admin_listen}"))?; - config.listen = listener - .local_addr() - .context("read pChronicle Gateway listen address")? - .to_string(); - config.admin_listen = admin_listener - .local_addr() - .context("read pChronicle Gateway admin listen address")? - .to_string(); - let (sink, writer) = gateway_capture::gateway_capture_sink_with_split( - dataset_uri, - &config.agent_id, - split.clone(), - )?; - Ok(Some(PreparedGateway::Proxy(Box::new( - PreparedProxyGateway { - config, - state_dir, - dataset_uri: dataset_uri.to_string(), - split: split.map(|template| template.source().to_string()), - stream_markdown: args.gateway_stream_markdown, - listener, - admin_listener, - sink, - writer, - }, - )))) -} - -async fn wait_for_stop(mut receiver: tokio::sync::watch::Receiver) { - let _ = receiver.wait_for(|stop| *stop).await; -} - async fn wait_for_termination() { #[cfg(unix)] { @@ -1791,219 +1461,6 @@ async fn wait_for_termination() { } } -#[cfg(test)] -async fn serve_warehouse_and_gateway( - warehouse_config: server::ChronicleServerConfig, - warehouse_listener: tokio::net::TcpListener, - gateway: PreparedGateway, - shutdown: impl std::future::Future + Send, -) -> Result<()> { - let (diagnostic_tx, diagnostic_rx) = tokio::sync::mpsc::channel(256); - let mut projections = projection_supervisor::ProjectionSupervisor::new( - warehouse_config.clone(), - None, - diagnostic_tx, - ); - projections.converge_before_readiness().await?; - let warehouse = server::PreparedWarehouse::prepare(warehouse_config).await?; - projections.set_warehouse(Some(warehouse.clone())); - let mut stderr = Vec::new(); - serve_components( - Some((warehouse, warehouse_listener)), - None, - Some(gateway), - projections, - diagnostic_rx, - &mut stderr, - shutdown, - ) - .await -} - -async fn serve_gateway_component( - gateway: PreparedGateway, - shutdown: impl std::future::Future + Send + 'static, -) -> Result<()> { - match gateway { - PreparedGateway::Ingest(gateway) => gateway.serve(shutdown).await, - PreparedGateway::Proxy(gateway) => { - let PreparedProxyGateway { - config, - state_dir, - listener, - admin_listener, - sink, - writer, - stream_markdown, - .. - } = *gateway; - let mut gateway_server = - Box::pin(persisting_gateway::serve_with_listeners_and_shutdown( - config, - state_dir, - sink, - stream_markdown, - listener, - admin_listener, - shutdown, - )); - let result = (&mut gateway_server).await; - drop(gateway_server); - writer - .finish() - .context("finish pChronicle Gateway capture")?; - result - } - } -} - -async fn serve_components( - warehouse: Option<(server::PreparedWarehouse, tokio::net::TcpListener)>, - control: Option, - gateway: Option, - projections: projection_supervisor::ProjectionSupervisor, - mut diagnostics: tokio::sync::mpsc::Receiver, - stderr: &mut W, - shutdown: impl std::future::Future + Send, -) -> Result<()> { - type ServiceFuture = - std::pin::Pin)> + Send>>; - - let (stop_tx, stop_rx) = tokio::sync::watch::channel(false); - let mut services = FuturesUnordered::::new(); - anyhow::ensure!( - warehouse.is_some() || control.is_some() || gateway.is_some(), - "pChronicle serve has no enabled service" - ); - if let Some((warehouse, listener)) = warehouse { - let stop = stop_rx.clone(); - services.push(Box::pin(async move { - ( - "Warehouse", - server::serve_prepared_warehouse_with_listener_and_shutdown( - warehouse, - listener, - wait_for_stop(stop), - ) - .await, - ) - })); - } - if let Some(control) = control { - let stop = stop_rx.clone(); - services.push(Box::pin(async move { - ("Control", control.serve(wait_for_stop(stop)).await) - })); - } - if let Some(gateway) = gateway { - let stop = stop_rx.clone(); - services.push(Box::pin(async move { - ( - "Gateway", - serve_gateway_component(gateway, wait_for_stop(stop)).await, - ) - })); - } - services.push(Box::pin(async move { - ("Projection", projections.run(stop_rx).await) - })); - - tokio::pin!(shutdown); - let mut diagnostics_open = true; - let mut diagnostic_error = None; - let first = loop { - tokio::select! { - _ = &mut shutdown => break None, - completed = services.next() => break completed, - diagnostic = diagnostics.recv(), if diagnostics_open => { - match diagnostic { - Some(diagnostic) => { - if let Err(error) = write_projection_diagnostic(stderr, &diagnostic) { - diagnostic_error = Some(error); - break None; - } - } - None => diagnostics_open = false, - } - } - } - }; - let _ = stop_tx.send(true); - - let mut sibling_error = None; - while !services.is_empty() { - tokio::select! { - completed = services.next() => { - if let Some((name, Err(error))) = completed { - sibling_error.get_or_insert_with(|| { - error.context(format!("stop pChronicle {name}")) - }); - } - } - diagnostic = diagnostics.recv(), if diagnostics_open => { - match diagnostic { - Some(diagnostic) => { - if diagnostic_error.is_none() - && let Err(error) = write_projection_diagnostic(stderr, &diagnostic) - { - diagnostic_error = Some(error); - } - } - None => diagnostics_open = false, - } - } - } - } - while let Ok(diagnostic) = diagnostics.try_recv() { - if diagnostic_error.is_none() - && let Err(error) = write_projection_diagnostic(stderr, &diagnostic) - { - diagnostic_error = Some(error); - } - } - - if let Some(error) = diagnostic_error { - return Err(error); - } - - match first { - Some((name, Err(error))) => Err(error.context(format!("pChronicle {name} stopped"))), - Some((name, Ok(()))) => bail!("pChronicle {name} stopped unexpectedly"), - None => match sibling_error { - Some(error) => Err(error), - None => Ok(()), - }, - } -} - -fn write_projection_diagnostic( - stderr: &mut W, - diagnostic: &projection_supervisor::ProjectionDiagnostic, -) -> Result<()> { - writeln!( - stderr, - "projection source={} output={} status={} retry_ms={}", - projection_supervisor::sanitize_log_field(&diagnostic.source_path), - projection_supervisor::sanitize_log_field(&diagnostic.projection_path), - diagnostic.status, - diagnostic.retry_ms, - ) - .context("write pChronicle projection diagnostic") -} - -fn warehouse_listen(args: &ServeArgs) -> Option { - match args.listen { - Some(listen) => Some(listen), - None if args.control.is_none() - && args.gateway.is_none() - && args.gateway_config.is_none() => - { - Some(SocketAddr::from(([127, 0, 0, 1], 0))) - } - None => None, - } -} - fn run_serve_catalog( command: ServeSubcommand, stdout_is_terminal: bool, @@ -2225,186 +1682,45 @@ async fn run_serve( stderr: &mut dyn Write, ) -> Result<()> { server::request_log::init_warehouse_tracing(log_level); - let gateway_dataset_uri = resolve_gateway_dataset_uri(&args, settings_override)?; - if let Some(uri) = gateway_dataset_uri.as_deref() { - prepare_local_gateway_dataset(uri).await?; - } let config = resolve_serve_config_with_settings(&args, settings_override)?; - let control_uri = args - .control - .is_some() - .then(|| control_storage_uri(&config).map(str::to_owned)) - .transpose()?; - if let Some(uri) = control_uri.as_deref() { - prepare_local_control_storage(uri).await?; - } - let (diagnostic_tx, diagnostic_rx) = tokio::sync::mpsc::channel(256); - let projection_idle = if args.gateway.is_some() { - Duration::from_secs(args.gateway_split_idle_seconds) + let listen = args + .listen + .unwrap_or_else(|| SocketAddr::from(([127, 0, 0, 1], 0))); + let listener = tokio::net::TcpListener::bind(listen) + .await + .with_context(|| format!("bind pChronicle Warehouse to {listen}"))?; + let warehouse = if let Some(path) = args.catalog_config.as_ref() { + let acl = server::catalog::CatalogAcl::load(path)?; + server::PreparedWarehouse::prepare_catalog(acl, config, Some(path.clone())).await? } else { - Duration::default() + server::PreparedWarehouse::prepare(config).await? }; - let mut projections = projection_supervisor::ProjectionSupervisor::with_projection_idle( - config.clone(), - None, - projection_idle, - diagnostic_tx, + let endpoint = listener.local_addr()?.to_string(); + let snapshot_id = warehouse.current_snapshot_id().await; + server::request_log::log_warehouse_startup( + &endpoint, + &warehouse.dataset_names(), + snapshot_id.as_deref(), ); - projections.converge_before_readiness().await?; - let warehouse = match warehouse_listen(&args) { - Some(listen) => { - let listener = tokio::net::TcpListener::bind(listen) - .await - .with_context(|| format!("bind pChronicle Warehouse to {listen}"))?; - let warehouse = if let Some(path) = args.catalog_config.as_ref() { - let acl = server::catalog::CatalogAcl::load(path)?; - server::PreparedWarehouse::prepare_catalog(acl, config.clone(), Some(path.clone())) - .await? - } else if args.gateway.is_some() { - server::PreparedWarehouse::prepare_live(config.clone()).await? - } else { - server::PreparedWarehouse::prepare(config.clone()).await? - }; - Some((warehouse, listener)) - } - None => None, - }; - projections.set_warehouse( - warehouse - .as_ref() - .map(|(warehouse, _listener)| warehouse.clone()), - ); - let control = match args.control { - Some(listen) => Some( - control::PreparedControl::bind( - control_uri.as_deref().context( - "pChronicle Control requires a Dataset named 'default'; pass default=DATASET", - )?, - listen, - ) - .await?, - ), - None => None, - }; - let gateway = prepare_gateway(&args, gateway_dataset_uri.as_deref()).await?; - - let warehouse_endpoint = warehouse - .as_ref() - .map(|(_, listener)| listener.local_addr().map(|addr| addr.to_string())) - .transpose() - .context("read pChronicle Warehouse listen address")?; - if let Some((warehouse, _)) = warehouse.as_ref() { - let snapshot_id = warehouse.current_snapshot_id().await; - server::request_log::log_warehouse_startup( - warehouse_endpoint.as_deref().unwrap_or_default(), - &warehouse.dataset_names(), - snapshot_id.as_deref(), - ); - } - let control_ready = control.as_ref().map(control::PreparedControl::ready); - let gateway_endpoint = gateway - .as_ref() - .map(|gateway| gateway.endpoint().to_string()); - let gateway_admin_endpoint = gateway - .as_ref() - .and_then(|gateway| gateway.admin_endpoint().map(str::to_string)); - let gateway_dataset = gateway - .as_ref() - .map(|gateway| gateway.dataset_uri().to_string()); - let gateway_split = gateway - .as_ref() - .and_then(|gateway| gateway.split_source().map(str::to_string)); - let ready = ChronicleServeReady { - version: CHRONICLE_SERVE_READY_VERSION, - warehouse_endpoint: warehouse_endpoint.clone(), - control: control_ready, - gateway_endpoint, - gateway_admin_endpoint, - gateway_dataset, - gateway_split, - }; - serde_json::to_writer(&mut *stdout, &ready).context("encode pChronicle serve readiness")?; - writeln!(stdout).context("write pChronicle serve readiness")?; - stdout.flush().context("flush pChronicle serve readiness")?; - - if let Some(endpoint) = &warehouse_endpoint { - writeln!(stderr, "pChronicle Warehouse: http://{endpoint}/") - .context("write pChronicle Warehouse address")?; - } - if let Some(ready) = &ready.control { - writeln!(stderr, "pChronicle Control: {}", ready.endpoint) - .context("write pChronicle Control address")?; - } - if let Some(gateway) = &gateway { - writeln!( - stderr, - "pChronicle Gateway: http://{}/ dataset={}", - gateway.endpoint(), - gateway.dataset_uri() - ) - .context("write pChronicle Gateway address")?; - if let Some(split) = gateway.split_source() { - writeln!(stderr, "pChronicle Gateway split: {split}") - .context("write pChronicle Gateway split")?; - } - if let Some(admin) = gateway.admin_endpoint() { - writeln!(stderr, "pChronicle Gateway admin: http://{admin}/") - .context("write pChronicle Gateway admin address")?; - } - if args.debug { - writeln!( - stderr, - "pChronicle Gateway debug: stderr (request/response bodies may be included)" - ) - .context("write pChronicle Gateway debug status")?; - } - } + serde_json::to_writer( + &mut *stdout, + &serde_json::json!({"version": 1, "warehouse_endpoint": endpoint}), + ) + .context("encode pChronicle serve readiness")?; + writeln!(stdout)?; + stdout.flush()?; + writeln!(stderr, "pChronicle Warehouse: http://{endpoint}/")?; if args.open { - let endpoint = warehouse_endpoint - .as_deref() - .context("--open requires --listen")?; open_browser(&format!("http://{endpoint}/"))?; } - serve_components( + server::serve_prepared_warehouse_with_listener_and_shutdown( warehouse, - control, - gateway, - projections, - diagnostic_rx, - stderr, + listener, wait_for_termination(), ) .await } -async fn prepare_local_control_storage(uri: &str) -> Result<()> { - let Some(path) = local_dataset_path(uri)? else { - return Ok(()); - }; - tokio::fs::create_dir_all(&path) - .await - .with_context(|| format!("create pChronicle Control storage root {}", path.display())) -} - -async fn prepare_local_gateway_dataset(uri: &str) -> Result<()> { - let Some(path) = local_dataset_path(uri)? else { - return Ok(()); - }; - tokio::fs::create_dir_all(&path) - .await - .with_context(|| format!("create pChronicle Gateway Dataset {}", path.display())) -} - -fn resolve_gateway_dataset_uri( - args: &ServeArgs, - settings_override: Option<&Path>, -) -> Result> { - args.gateway_dataset - .as_deref() - .map(|uri| expand_dataset_reference(uri, settings_override, false)) - .transpose() -} - fn serve_storage_uris(args: &ServeArgs) -> Vec { let mut storage = args.storage.clone(); storage.extend(args.positional_storage.iter().cloned()); @@ -2416,12 +1732,7 @@ fn resolve_serve_config_with_settings( settings_override: Option<&Path>, ) -> Result { let storage = serve_storage_uris(args); - let gateway_dataset = resolve_gateway_dataset_uri(args, settings_override)?; let mut config = if let Some(path) = args.catalog_config.as_deref() { - anyhow::ensure!( - gateway_dataset.is_none() && args.control.is_none(), - "catalog workers cannot share Gateway or Control listeners" - ); server::catalog::CatalogAcl::load(path)?; server::ChronicleServerConfig::front_only() } else { @@ -2447,43 +1758,14 @@ fn resolve_serve_config_with_settings( config.catalog_options.error_policy = CatalogErrorPolicy::Report; config } - (None, []) if gateway_dataset.is_some() => { - server::ChronicleServerConfig::mounted(vec![DatasetMount::new( - SERVE_STORAGE_DATASET_NAME, - gateway_dataset.as_deref().context("Gateway Dataset")?, - )?])? - } _ => bail!("serve requires at least one Dataset"), } }; - if let Some(uri) = gateway_dataset { - ensure_gateway_mount(&mut config, uri)?; - } + config.home_links = args.home_links.clone(); Ok(config) } -fn ensure_gateway_mount(config: &mut server::ChronicleServerConfig, uri: String) -> Result<()> { - if config.datasets.iter().any(|dataset| dataset.uri == uri) { - return Ok(()); - } - let name = if config.datasets.is_empty() { - SERVE_STORAGE_DATASET_NAME - } else { - "gateway" - }; - anyhow::ensure!( - !config.datasets.iter().any(|dataset| dataset.name == name), - "cannot auto-mount Gateway Dataset as '{name}'; that name is already mounted" - ); - config.datasets.push(DatasetMount::new(name, uri)?); - if config.default_dataset.is_none() && config.datasets.len() == 1 { - config.default_dataset = Some(name.to_string()); - } - config.catalog_options.error_policy = CatalogErrorPolicy::Report; - Ok(()) -} - #[cfg(test)] fn resolve_serve_config(args: &ServeArgs) -> Result { resolve_serve_config_with_settings(args, None) @@ -2585,39 +1867,6 @@ fn sanitize_derived_dataset_name(raw: &str) -> Result { }) } -fn control_storage_uri(config: &server::ChronicleServerConfig) -> Result<&str> { - config - .datasets - .iter() - .find(|dataset| dataset.name == SERVE_STORAGE_DATASET_NAME) - .map(|dataset| dataset.uri.as_str()) - .context("pChronicle Control requires a Dataset named 'default'; pass default=DATASET") -} - -async fn run_echo(args: EchoArgs, stderr: &mut dyn Write) -> Result<()> { - let listener = tokio::net::TcpListener::bind(args.listen) - .await - .with_context(|| format!("bind pChronicle Echo to {}", args.listen))?; - let addr = listener - .local_addr() - .context("read pChronicle Echo listen address")?; - let default_encoding = match args.encoding { - EchoEncoding::Plain => persisting_gateway::echo::EchoEncoding::Plain, - EchoEncoding::Base64 => persisting_gateway::echo::EchoEncoding::Base64, - }; - writeln!( - stderr, - "pChronicle Echo: http://{addr}/ encoding={default_encoding}" - ) - .context("write pChronicle Echo address")?; - persisting_gateway::echo::serve_with_shutdown( - listener, - persisting_gateway::echo::EchoServerConfig { default_encoding }, - wait_for_termination(), - ) - .await -} - fn open_browser(url: &str) -> Result<()> { #[cfg(target_os = "macos")] let mut command = ProcessCommand::new("open"); @@ -2876,37 +2125,6 @@ async fn run_status( let (dataset_uri, snapshot) = discover_snapshot(&dataset_uri, args.errors, args.max_files, args.max_entries).await?; let snapshot = Arc::new(snapshot); - let inventory = automatic_projection_inventory(snapshot.as_ref())?; - let mut projections = stream::iter(inventory.targets) - .map(|target| async move { - let source_path = target.source_path.clone(); - let projection_path = target.projection_path.clone(); - match inspect_automatic_storyline_projection(&target).await { - Ok(inspection) => ProjectionStatusResponse::from_inspection( - source_path, - projection_path, - inspection, - ), - Err(error) => { - tracing::error!( - error = ?error, - source = %source_path, - "pChronicle projection status inspection failed" - ); - ProjectionStatusResponse::error(source_path, projection_path) - } - } - }) - .buffered(STATUS_PROJECTION_CONCURRENCY) - .collect::>() - .await; - projections.extend( - inventory - .errors - .into_iter() - .map(|error| ProjectionStatusResponse::error(error.source_path, error.projection_path)), - ); - projections.sort_by(|left, right| left.source_path.cmp(&right.source_path)); let dataset = snapshot .dataset(DEFAULT_DATASET_NAME) .context("default Dataset missing from Snapshot")?; @@ -2971,7 +2189,6 @@ async fn run_status( error: error_sources, }, counts, - projections, source_errors, }; @@ -4151,8 +3368,7 @@ async fn query_status_counts( "SELECT \ (SELECT COUNT(*) FROM dataset.runs{predicate}) AS runs, \ (SELECT COUNT(*) FROM dataset.steps{predicate}) AS steps, \ - (SELECT COUNT(*) FROM dataset.tool_calls{predicate}) AS tool_calls, \ - (SELECT COUNT(*) FROM dataset.events{predicate}) AS events" + (SELECT COUNT(*) FROM dataset.tool_calls{predicate}) AS tool_calls" ); let remaining = deadline.saturating_duration_since(tokio::time::Instant::now()); anyhow::ensure!( @@ -4181,7 +3397,6 @@ async fn query_status_counts( runs: u64, steps: u64, tool_calls: u64, - events: u64, } let counts: QueryCounts = serde_json::from_str(line).context("decode Dataset status counts")?; Ok(StatusCounts { @@ -4189,7 +3404,6 @@ async fn query_status_counts( trajectories: counts.runs, steps: counts.steps, tool_calls: counts.tool_calls, - events: counts.events, }) } diff --git a/crates/persisting-pchronicle-cli/src/output.rs b/crates/persisting-pchronicle-cli/src/output.rs index 1d9cf9fea..b5a449842 100644 --- a/crates/persisting-pchronicle-cli/src/output.rs +++ b/crates/persisting-pchronicle-cli/src/output.rs @@ -263,39 +263,6 @@ pub(super) fn write_status_table(stdout: &mut dyn Write, response: &StatusRespon "tool_calls {} {accuracy}", response.counts.tool_calls )?; - writeln!( - stdout, - "events {} {accuracy}", - response.counts.events - )?; - if !response.projections.is_empty() { - writeln!(stdout)?; - writeln!( - stdout, - "PROJECTION STATUS FACT_VERSION FACT_ROWS GENERATION" - )?; - for projection in &response.projections { - let path = format!( - "{} -> {}", - projection.source_path, projection.projection_path - ); - writeln!( - stdout, - "{:<34} {:<8} {:<12} {:<9} {}", - truncate(&path, 34), - projection.status.as_str(), - projection - .fact_version - .map(|value| value.to_string()) - .unwrap_or_default(), - projection - .fact_rows - .map(|value| value.to_string()) - .unwrap_or_default(), - projection.generation.as_deref().unwrap_or_default(), - )?; - } - } for error in &response.source_errors { writeln!( stdout, diff --git a/crates/persisting-pchronicle-cli/src/projection_supervisor.rs b/crates/persisting-pchronicle-cli/src/projection_supervisor.rs deleted file mode 100644 index 1da95e952..000000000 --- a/crates/persisting-pchronicle-cli/src/projection_supervisor.rs +++ /dev/null @@ -1,682 +0,0 @@ -use super::*; - -use std::collections::BTreeMap; - -use persisting_pchronicle::storage::{ - automatic_projection_inventory, maintain_automatic_storyline_projection, -}; - -#[derive(Debug, Clone, Copy)] -pub(crate) struct ProjectionSupervisorOptions { - pub(crate) interval: Duration, - pub(crate) max_backoff: Duration, - pub(crate) max_concurrent: usize, - /// Delay runtime refreshes after an existing canonical source changes. - /// Startup projections and newly discovered sources are not delayed. - pub(crate) projection_idle: Duration, -} - -impl Default for ProjectionSupervisorOptions { - fn default() -> Self { - Self { - interval: Duration::from_secs(1), - max_backoff: Duration::from_secs(60), - max_concurrent: 16, - projection_idle: Duration::ZERO, - } - } -} - -#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] -pub(crate) struct ProjectionIterationReport { - pub(crate) succeeded: usize, - pub(crate) failed: usize, - pub(crate) publications: usize, - pub(crate) catalog_refreshes: usize, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) struct ProjectionDiagnostic { - pub(crate) source_path: String, - pub(crate) projection_path: String, - pub(crate) status: &'static str, - pub(crate) retry_ms: u64, -} - -#[derive(Debug, Clone, Copy)] -struct RetryState { - failures: u32, - next_attempt: tokio::time::Instant, -} - -pub(crate) struct ProjectionSupervisor { - config: server::ChronicleServerConfig, - warehouse: Option, - options: ProjectionSupervisorOptions, - diagnostics: tokio::sync::mpsc::Sender, - retries: BTreeMap, - source_watermarks: BTreeMap, - source_idle_since: BTreeMap, - catalog_retry: Option, - observed_snapshot_id: Option, - catalog_dirty: bool, -} - -impl ProjectionSupervisor { - pub(crate) fn new( - config: server::ChronicleServerConfig, - warehouse: Option, - diagnostics: tokio::sync::mpsc::Sender, - ) -> Self { - Self { - config, - warehouse, - options: ProjectionSupervisorOptions::default(), - diagnostics, - retries: BTreeMap::new(), - source_watermarks: BTreeMap::new(), - source_idle_since: BTreeMap::new(), - catalog_retry: None, - observed_snapshot_id: None, - catalog_dirty: false, - } - } - - pub(crate) fn with_projection_idle( - config: server::ChronicleServerConfig, - warehouse: Option, - projection_idle: Duration, - diagnostics: tokio::sync::mpsc::Sender, - ) -> Self { - let mut supervisor = Self::new(config, warehouse, diagnostics); - supervisor.options.projection_idle = projection_idle; - supervisor - } - - pub(crate) fn set_warehouse(&mut self, warehouse: Option) { - self.warehouse = warehouse; - self.catalog_dirty = false; - self.catalog_retry = None; - } - - async fn discover(&self) -> Result { - let mut options = self.config.catalog_options; - options.error_policy = CatalogErrorPolicy::Report; - DatasetCatalogSnapshot::discover( - self.config.datasets.clone(), - self.config.default_dataset.clone(), - options, - ) - .await - } - - pub(crate) async fn converge_before_readiness(&mut self) -> Result<()> { - if self.config.datasets.is_empty() { - return Ok(()); - } - let snapshot = self.discover().await?; - let inventory = automatic_projection_inventory(&snapshot)?; - let initial_watermarks = inventory - .targets - .iter() - .map(|target| { - ( - target.source_uri.clone(), - ( - target.source_snapshot.fact_version, - target.source_snapshot.fact_rows, - target.source_snapshot.layout_revision, - ), - ) - }) - .collect(); - let mut failures = inventory - .errors - .iter() - .map(|error| format!("{}: source discovery failed", error.source_path)) - .collect::>(); - let outcomes = stream::iter(inventory.targets) - .map(|target| async move { - let source_path = target.source_path.clone(); - maintain_automatic_storyline_projection(&target) - .await - .map(|_| ()) - .map_err(|error| (source_path, error)) - }) - .buffer_unordered(self.options.max_concurrent.max(1)) - .collect::>() - .await; - failures.extend(outcomes.into_iter().filter_map(|outcome| match outcome { - Ok(()) => None, - Err((source_path, error)) => { - Some(format!("{}: {error:#}", sanitize_log_field(&source_path))) - } - })); - anyhow::ensure!( - failures.is_empty(), - "automatic Storyline projection startup failed for {} source(s): {}", - failures.len(), - failures.join("; ") - ); - - let converged = self.discover().await?; - self.source_watermarks = initial_watermarks; - self.source_idle_since.clear(); - self.observed_snapshot_id = Some(converged.snapshot_id().to_string()); - self.catalog_dirty = false; - Ok(()) - } - - pub(crate) async fn run_iteration( - &mut self, - now: tokio::time::Instant, - ) -> ProjectionIterationReport { - let mut report = ProjectionIterationReport::default(); - let snapshot = match self.discover().await { - Ok(snapshot) => snapshot, - Err(_) => { - report.failed = 1; - self.send_diagnostic("catalog", "", "error", self.options.interval); - return report; - } - }; - if self.observed_snapshot_id.as_deref() != Some(snapshot.snapshot_id()) { - self.observed_snapshot_id = Some(snapshot.snapshot_id().to_string()); - } - let inventory = match automatic_projection_inventory(&snapshot) { - Ok(inventory) => inventory, - Err(_) => { - report.failed = 1; - self.send_diagnostic("catalog", "", "error", self.options.interval); - return report; - } - }; - - for error in inventory.errors { - let key = format!("error:{}/{}", error.dataset, error.source_path); - if self.retry_is_due(&key, now) { - report.failed = report.failed.saturating_add(1); - let delay = self.record_failure(key, now); - self.send_diagnostic(&error.source_path, &error.projection_path, "error", delay); - } - } - - let due = inventory - .targets - .into_iter() - .filter(|target| { - let key = target.source_uri.clone(); - let watermark = ( - target.source_snapshot.fact_version, - target.source_snapshot.fact_rows, - target.source_snapshot.layout_revision, - ); - let previous = self.source_watermarks.insert(key.clone(), watermark); - if let Some(previous) = previous { - if previous != watermark { - self.source_idle_since.entry(key.clone()).or_insert(now); - } - } else { - // A new canonical events.lance source is a structural - // Catalog change. Existing sources only update their - // watermark and remain queryable through live reads. - self.catalog_dirty = true; - } - let idle_elapsed = self - .source_idle_since - .get(&key) - .map(|since| now.saturating_duration_since(*since)) - .unwrap_or(self.options.projection_idle); - let idle_ready = self.options.projection_idle.is_zero() - || idle_elapsed >= self.options.projection_idle; - idle_ready && self.retry_is_due(&key, now) - }) - .collect::>(); - let outcomes = stream::iter(due) - .map(|target| async move { - let result = maintain_automatic_storyline_projection(&target).await; - (target, result) - }) - .buffer_unordered(self.options.max_concurrent.max(1)) - .collect::>() - .await; - for (target, outcome) in outcomes { - match outcome { - Ok(maintenance) => { - self.retries.remove(&target.source_uri); - self.source_idle_since.remove(&target.source_uri); - report.succeeded = report.succeeded.saturating_add(1); - if maintenance.published() { - report.publications = report.publications.saturating_add(1); - self.catalog_dirty = true; - } - } - Err(_) => { - report.failed = report.failed.saturating_add(1); - let delay = self.record_failure(target.source_uri, now); - self.send_diagnostic( - &target.source_path, - &target.projection_path, - "error", - delay, - ); - } - } - } - - let catalog_due = self - .catalog_retry - .is_none_or(|retry| retry.next_attempt <= now); - if self.catalog_dirty - && catalog_due - && let Some(warehouse) = self.warehouse.clone() - { - match warehouse.refresh_catalog().await { - Ok(snapshot_id) => { - self.observed_snapshot_id = Some(snapshot_id); - self.catalog_dirty = false; - self.catalog_retry = None; - report.catalog_refreshes = 1; - } - Err(_) => { - let failures = self - .catalog_retry - .map_or(1, |retry| retry.failures.saturating_add(1)); - let delay = self.retry_delay(failures); - self.catalog_retry = Some(RetryState { - failures, - next_attempt: now + delay, - }); - self.send_diagnostic("catalog", "", "error", delay); - } - } - } - report - } - - pub(crate) async fn run(mut self, mut stop: tokio::sync::watch::Receiver) -> Result<()> { - if self.config.datasets.is_empty() { - let _ = stop.changed().await; - return Ok(()); - } - loop { - tokio::select! { - changed = stop.changed() => { - if changed.is_err() || *stop.borrow() { - return Ok(()); - } - } - _ = tokio::time::sleep(self.options.interval) => { - self.run_iteration(tokio::time::Instant::now()).await; - } - } - } - } - - fn retry_is_due(&self, key: &str, now: tokio::time::Instant) -> bool { - self.retries - .get(key) - .is_none_or(|retry| retry.next_attempt <= now) - } - - fn record_failure(&mut self, key: String, now: tokio::time::Instant) -> Duration { - let failures = self - .retries - .get(&key) - .map_or(1, |retry| retry.failures.saturating_add(1)); - let delay = self.retry_delay(failures); - self.retries.insert( - key, - RetryState { - failures, - next_attempt: now + delay, - }, - ); - delay - } - - fn retry_delay(&self, failures: u32) -> Duration { - let exponent = failures.saturating_sub(1).min(20); - let multiplier = 1u32.checked_shl(exponent).unwrap_or(u32::MAX); - self.options - .interval - .saturating_mul(multiplier) - .min(self.options.max_backoff) - } - - fn send_diagnostic( - &self, - source_path: &str, - projection_path: &str, - status: &'static str, - retry: Duration, - ) { - let retry_ms = retry.as_millis().try_into().unwrap_or(u64::MAX); - let _ = self.diagnostics.try_send(ProjectionDiagnostic { - source_path: source_path.to_string(), - projection_path: projection_path.to_string(), - status, - retry_ms, - }); - } -} - -pub(crate) fn sanitize_log_field(value: &str) -> String { - value.chars().flat_map(char::escape_default).collect() -} - -#[cfg(test)] -mod tests { - use super::*; - use persisting_pchronicle::storage::{ - AutomaticProjectionState, RawEventLanceStore, StoryCoords, automatic_projection_inventory, - build_storyline_projection, inspect_automatic_storyline_projection, - }; - - async fn append_note(storage: &Path, run_id: &str, seq: u64) -> Result { - let coords = StoryCoords::new( - storage.to_string_lossy(), - "agent", - run_id, - Some(run_id.into()), - ); - RawEventLanceStore - .append_events( - &coords, - &[persisting_pchronicle::model::EventRecord { - identity: Default::default(), - seq, - source: "test".into(), - kind: "note".into(), - timestamp: None, - session_id: Some(run_id.into()), - agent_id: Some("agent".into()), - parent_uuid: None, - trace_id: None, - call_id: None, - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload: serde_json::json!({"content": format!("{run_id}-{seq}")}), - }], - ) - .await?; - persisting_pchronicle::storage::raw_event_lance_path(&coords) - } - - fn config(root: &Path) -> Result { - server::ChronicleServerConfig::mounted(vec![DatasetMount::default(root.to_string_lossy())?]) - } - - #[tokio::test] - async fn startup_converges_every_initial_projection_before_returning() -> Result<()> { - let temp = tempfile::tempdir()?; - let _suppress = persisting_pchronicle::storage::StorylineSearchIndexSuppressGuard::for_path( - temp.path(), - ); - let storage = temp.path().join("capture"); - append_note(&storage, "b", 0).await?; - append_note(&storage, "a", 0).await?; - let config = config(&storage.join("agent"))?; - let (diagnostics, _receiver) = tokio::sync::mpsc::channel(16); - let mut supervisor = ProjectionSupervisor::new(config.clone(), None, diagnostics); - - supervisor.converge_before_readiness().await?; - - let snapshot = DatasetCatalogSnapshot::discover( - config.datasets, - config.default_dataset, - config.catalog_options, - ) - .await?; - let inventory = automatic_projection_inventory(&snapshot)?; - assert_eq!(inventory.targets.len(), 2); - for target in inventory.targets { - assert_eq!( - inspect_automatic_storyline_projection(&target).await?.state, - AutomaticProjectionState::Fresh - ); - } - Ok(()) - } - - #[tokio::test] - async fn startup_rejects_a_foreign_deterministic_destination() -> Result<()> { - let temp = tempfile::tempdir()?; - let _suppress = persisting_pchronicle::storage::StorylineSearchIndexSuppressGuard::for_path( - temp.path(), - ); - let storage = temp.path().join("capture"); - let source_a = append_note(&storage, "a", 0).await?; - append_note(&storage, "b", 0).await?; - let projection_b = storage.join("agent/b/storyline"); - build_storyline_projection( - source_a.to_string_lossy(), - projection_b.to_string_lossy(), - "a/events.lance", - ) - .await?; - let before = std::fs::read(projection_b.join("CURRENT"))?; - let (diagnostics, _receiver) = tokio::sync::mpsc::channel(16); - let mut supervisor = - ProjectionSupervisor::new(config(&storage.join("agent"))?, None, diagnostics); - - let error = supervisor.converge_before_readiness().await.unwrap_err(); - assert!(error.to_string().contains("startup failed for 1 source")); - assert!(error.to_string().contains("b/events.lance")); - assert_eq!(std::fs::read(projection_b.join("CURRENT"))?, before); - Ok(()) - } - - #[tokio::test] - async fn runtime_discovers_sources_and_coalesces_catalog_refreshes() -> Result<()> { - let temp = tempfile::tempdir()?; - let _suppress = persisting_pchronicle::storage::StorylineSearchIndexSuppressGuard::for_path( - temp.path(), - ); - let root = temp.path().join("dataset"); - // Mount the agent leaf so shallow Directory discovery sees run/ - // events.lance Sources (not an unlabeled `agent/` Directory stub). - let agent = root.join("agent"); - std::fs::create_dir_all(&agent)?; - let config = config(&agent)?; - let (diagnostics, _receiver) = tokio::sync::mpsc::channel(16); - let mut supervisor = ProjectionSupervisor::new(config.clone(), None, diagnostics); - supervisor.converge_before_readiness().await?; - let warehouse = server::PreparedWarehouse::prepare(config).await?; - let initial_snapshot = warehouse.current_snapshot_id().await.unwrap(); - supervisor.set_warehouse(Some(warehouse.clone())); - - let source_a = append_note(&root, "a", 0).await?; - let source_b = append_note(&root, "b", 0).await?; - let report = supervisor.run_iteration(tokio::time::Instant::now()).await; - assert_eq!(report.succeeded, 2); - assert_eq!(report.failed, 0); - assert_eq!(report.publications, 2); - assert_eq!(report.catalog_refreshes, 1); - assert_ne!( - warehouse.current_snapshot_id().await.unwrap(), - initial_snapshot - ); - for source in [source_a, source_b] { - let snapshot = persisting_pchronicle::storage::probe_canonical_event_store( - source.to_string_lossy(), - ) - .await? - .unwrap(); - let target = persisting_pchronicle::storage::AutomaticProjectionTarget { - dataset: "dataset".into(), - source_path: format!( - "{}/events.lance", - source - .parent() - .unwrap() - .file_name() - .unwrap() - .to_string_lossy() - ), - source_uri: snapshot.source_uri.clone(), - projection_path: format!( - "{}/storyline", - source - .parent() - .unwrap() - .file_name() - .unwrap() - .to_string_lossy() - ), - projection_uri: source - .parent() - .unwrap() - .join("storyline") - .to_string_lossy() - .into_owned(), - source_snapshot: snapshot, - }; - assert_eq!( - inspect_automatic_storyline_projection(&target).await?.state, - AutomaticProjectionState::Fresh - ); - } - - append_note(&root, "a", 1).await?; - let report = supervisor.run_iteration(tokio::time::Instant::now()).await; - assert_eq!(report.publications, 1); - assert_eq!(report.catalog_refreshes, 1); - Ok(()) - } - - #[tokio::test] - async fn projection_idle_defers_existing_source_until_quiet_window() -> Result<()> { - let temp = tempfile::tempdir()?; - let _suppress = persisting_pchronicle::storage::StorylineSearchIndexSuppressGuard::for_path( - temp.path(), - ); - let root = temp.path().join("dataset"); - let agent = root.join("agent"); - std::fs::create_dir_all(&agent)?; - let config = config(&agent)?; - let (diagnostics, _receiver) = tokio::sync::mpsc::channel(16); - let mut supervisor = ProjectionSupervisor::with_projection_idle( - config, - None, - Duration::from_secs(30 * 60), - diagnostics, - ); - supervisor.converge_before_readiness().await?; - - append_note(&root, "run", 0).await?; - let initial = supervisor.run_iteration(tokio::time::Instant::now()).await; - assert_eq!(initial.publications, 1); - - append_note(&root, "run", 1).await?; - let changed_at = tokio::time::Instant::now(); - let deferred = supervisor.run_iteration(changed_at).await; - assert_eq!(deferred.publications, 0); - - let before_idle = supervisor - .run_iteration(changed_at + Duration::from_secs(30 * 60 - 1)) - .await; - assert_eq!(before_idle.publications, 0); - - let refreshed = supervisor - .run_iteration(changed_at + Duration::from_secs(30 * 60)) - .await; - assert_eq!(refreshed.publications, 1); - Ok(()) - } - - #[tokio::test] - async fn runtime_backoff_is_per_source_and_does_not_delay_healthy_sources() -> Result<()> { - let temp = tempfile::tempdir()?; - let _suppress = persisting_pchronicle::storage::StorylineSearchIndexSuppressGuard::for_path( - temp.path(), - ); - let root = temp.path().join("dataset"); - let good_source = append_note(&root, "good", 0).await?; - append_note(&root, "bad", 0).await?; - let bad_projection = root.join("agent/bad/storyline"); - build_storyline_projection( - good_source.to_string_lossy(), - bad_projection.to_string_lossy(), - "good/events.lance", - ) - .await?; - let (diagnostics, mut receiver) = tokio::sync::mpsc::channel(16); - let mut supervisor = - ProjectionSupervisor::new(config(&root.join("agent"))?, None, diagnostics); - supervisor.options.interval = Duration::from_millis(10); - supervisor.options.max_backoff = Duration::from_millis(40); - let now = tokio::time::Instant::now(); - - let first = supervisor.run_iteration(now).await; - assert_eq!(first.succeeded, 1); - assert_eq!(first.failed, 1); - let diagnostic = receiver.try_recv()?; - assert_eq!(diagnostic.source_path, "bad/events.lance"); - assert_eq!(diagnostic.retry_ms, 10); - - let immediate = supervisor.run_iteration(now).await; - assert_eq!(immediate.succeeded, 1); - assert_eq!(immediate.failed, 0); - let retried = supervisor - .run_iteration(now + Duration::from_millis(10)) - .await; - assert_eq!(retried.failed, 1); - assert_eq!(receiver.try_recv()?.retry_ms, 20); - assert_eq!(supervisor.retry_delay(20), Duration::from_millis(40)); - Ok(()) - } - - #[tokio::test] - async fn failed_catalog_refresh_stays_dirty_and_retries_independently() -> Result<()> { - let temp = tempfile::tempdir()?; - let _suppress = persisting_pchronicle::storage::StorylineSearchIndexSuppressGuard::for_path( - temp.path(), - ); - let root = temp.path().join("dataset"); - let agent = root.join("agent"); - std::fs::create_dir_all(&agent)?; - let config = config(&agent)?; - let (diagnostics, _receiver) = tokio::sync::mpsc::channel(16); - let mut supervisor = ProjectionSupervisor::new(config.clone(), None, diagnostics); - supervisor.options.interval = Duration::from_millis(10); - supervisor.options.max_backoff = Duration::from_millis(40); - supervisor.converge_before_readiness().await?; - let warehouse = server::PreparedWarehouse::prepare(config).await?; - supervisor.set_warehouse(Some(warehouse)); - - append_note(&root, "run", 0).await?; - // Place the broken Storyline marker beside run dirs under the agent - // mount so shallow discovery still sees `run/events.lance`. - let broken = agent.join("broken"); - std::fs::create_dir(&broken)?; - std::fs::write(broken.join("CURRENT"), "{")?; - let now = tokio::time::Instant::now(); - let failed = supervisor.run_iteration(now).await; - assert_eq!(failed.publications, 1); - assert_eq!(failed.catalog_refreshes, 0); - assert!(supervisor.catalog_dirty); - assert_eq!(supervisor.catalog_retry.unwrap().failures, 1); - - std::fs::remove_dir_all(&broken)?; - let deferred = supervisor.run_iteration(now).await; - assert_eq!(deferred.catalog_refreshes, 0); - assert!(supervisor.catalog_dirty); - let recovered = supervisor - .run_iteration(now + Duration::from_millis(10)) - .await; - assert_eq!(recovered.catalog_refreshes, 1); - assert!(!supervisor.catalog_dirty); - assert!(supervisor.catalog_retry.is_none()); - Ok(()) - } - - #[test] - fn diagnostics_escape_newlines_and_terminal_control_sequences() { - assert_eq!(sanitize_log_field("bad\n\u{1b}[31m"), "bad\\n\\u{1b}[31m"); - assert!(!sanitize_log_field("bad\n\u{1b}[31m").contains('\n')); - assert!(!sanitize_log_field("bad\n\u{1b}[31m").contains('\u{1b}')); - } -} diff --git a/crates/persisting-pchronicle-cli/src/server/acceleration.rs b/crates/persisting-pchronicle-cli/src/server/acceleration.rs index 498fc098a..4bbe252c7 100644 --- a/crates/persisting-pchronicle-cli/src/server/acceleration.rs +++ b/crates/persisting-pchronicle-cli/src/server/acceleration.rs @@ -39,8 +39,6 @@ const MAX_ROUTING_INDEX_ROWS: usize = 1_000_000; const MAX_ROUTING_INDEX_VALUES: usize = 1_000_000; const RUN_COLUMNS: &[&str] = &["run_id", "session_id", "agent_id", "agent_model_name"]; const STEP_COLUMNS: &[&str] = &["run_id", "session_id"]; -const EVENT_IDENTITY_COLUMNS: &[&str] = &["event_id", "trace_id"]; -const EVENT_PARTITION_COLUMNS: &[&str] = &["session_id", "agent_id"]; #[derive(Debug, Default)] pub(crate) struct ServerAcceleration { @@ -48,8 +46,6 @@ pub(crate) struct ServerAcceleration { run_summaries: OnceCell>, run_summaries_by_dataset: Mutex>, runs: OnceCell>, - event_identities: OnceCell>, - event_partitions: OnceCell>, } type CachedRunSummaries = Arc>; @@ -96,12 +92,6 @@ pub(crate) struct AccelerationStatus { pub(crate) run_index: Option, /// The generation's optional run index reached a terminal unavailable state. pub(crate) run_index_unavailable: bool, - pub(crate) event_identity_index: Option, - /// The generation's optional identity index reached a terminal unavailable state. - pub(crate) event_identity_index_unavailable: bool, - pub(crate) event_partition_index: Option, - /// The generation's optional partition index reached a terminal unavailable state. - pub(crate) event_partition_index_unavailable: bool, } #[derive(Debug, Clone, Serialize, PartialEq, Eq)] @@ -154,19 +144,11 @@ impl ServerAcceleration { let (run_summaries_ready, run_summaries_failed) = required_acceleration_status(&self.run_summaries); let (run_index, run_index_unavailable) = cached_index_status(&self.runs); - let (event_identity_index, event_identity_index_unavailable) = - cached_index_status(&self.event_identities); - let (event_partition_index, event_partition_index_unavailable) = - cached_index_status(&self.event_partitions); AccelerationStatus { run_summaries_ready, run_summaries_failed, run_index, run_index_unavailable, - event_identity_index, - event_identity_index_unavailable, - event_partition_index, - event_partition_index_unavailable, } } @@ -247,12 +229,6 @@ impl ServerAcceleration { self.route_sql_with(snapshot, sql, |kind| async move { match kind { RoutingIndexKind::Runs => build_run_index(snapshot, engine).await, - RoutingIndexKind::EventIdentities => { - build_event_identity_index(snapshot, engine).await - } - RoutingIndexKind::EventPartitions => { - build_event_partition_index(snapshot, engine).await - } } }) .await @@ -280,8 +256,6 @@ impl ServerAcceleration { let (cell, name) = match query.index_kind { RoutingIndexKind::Runs => (&self.runs, "run"), - RoutingIndexKind::EventIdentities => (&self.event_identities, "event_identity"), - RoutingIndexKind::EventPartitions => (&self.event_partitions, "event_partition"), }; let Some(index) = self .optional_index_with(cell, name, || build(query.index_kind)) @@ -371,8 +345,6 @@ fn cached_index_status( #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum RoutingIndexKind { Runs, - EventIdentities, - EventPartitions, } #[derive(Debug, Clone, PartialEq, Eq)] @@ -439,7 +411,6 @@ impl AnalyzedQuery { } let (dataset, table) = resolve_table(snapshot, name)?; let indexed_columns = match table.as_str() { - "events" => [EVENT_IDENTITY_COLUMNS, EVENT_PARTITION_COLUMNS].concat(), "runs" | "trajectories" => RUN_COLUMNS.to_vec(), "steps" | "tool_calls" => STEP_COLUMNS.to_vec(), _ => return None, @@ -457,22 +428,8 @@ impl AnalyzedQuery { collect_required_constraints(selection, &mut constraints); } constraints.retain(|constraint| indexed_columns.contains(&constraint.column.as_str())); - let index_kind = match table.as_str() { - "events" - if constraints.iter().any(|constraint| { - EVENT_IDENTITY_COLUMNS.contains(&constraint.column.as_str()) - }) => - { - RoutingIndexKind::EventIdentities - } - "events" => RoutingIndexKind::EventPartitions, - _ => RoutingIndexKind::Runs, - }; - let selected_columns = match index_kind { - RoutingIndexKind::Runs => indexed_columns.as_slice(), - RoutingIndexKind::EventIdentities => EVENT_IDENTITY_COLUMNS, - RoutingIndexKind::EventPartitions => EVENT_PARTITION_COLUMNS, - }; + let index_kind = RoutingIndexKind::Runs; + let selected_columns = indexed_columns.as_slice(); constraints.retain(|constraint| selected_columns.contains(&constraint.column.as_str())); Some(Self { statement, @@ -961,14 +918,12 @@ async fn build_run_summaries( root_session_id: None, path, row_count: 1, - duplicate_event_ids: 0, status: "record".into(), format: Some("compact-jsonl/v1".into()), }); } } } - let event_stats = build_event_stats(engine, name, &source.file).await?; let file = crate::sql_string(&source.file); let sql = format!( "WITH step_counts AS ( \ @@ -1033,13 +988,7 @@ async fn build_run_summaries( run_id.as_deref(), parent_session_id.as_deref(), ); - let status = event_stats - .get(&(file.clone(), session_id.clone())) - .map_or_else( - || run_status(&row), - |stats| stats.status.clone().unwrap_or_else(|| "active".into()), - ); - let event_stats = event_stats.get(&(file.clone(), session_id.clone())); + let status = run_status(&row); summaries.push(RunSummary { dataset: name.clone(), file, @@ -1050,15 +999,10 @@ async fn build_run_summaries( session_id, root_session_id, path, - row_count: event_stats.map_or_else( - || { - row.get("row_count") - .and_then(JsonValue::as_u64) - .unwrap_or(0) as usize - }, - |stats| stats.row_count, - ), - duplicate_event_ids: event_stats.map_or(0, |stats| stats.duplicate_event_ids), + row_count: row + .get("row_count") + .and_then(JsonValue::as_u64) + .unwrap_or(0) as usize, status, format: None, }); @@ -1107,95 +1051,6 @@ async fn bounded_summary_jsonl( String::from_utf8(output.bytes).context("decode run summary JSONL") } -#[derive(Debug, Clone)] -struct EventStats { - row_count: usize, - duplicate_event_ids: usize, - status: Option, -} - -async fn build_event_stats( - engine: &ChronicleQueryEngine, - dataset: &str, - file: &str, -) -> Result> { - let file = crate::sql_string(file); - let sql = format!( - "SELECT _file_, session_id, COUNT(*) AS row_count, \ - COUNT(event_id) - COUNT(DISTINCT event_id) AS duplicate_event_ids, \ - MAX(CASE \ - WHEN kind = 'run.failed' OR kind = 'run.cancelled' THEN 3 \ - WHEN kind = 'run.completed' THEN 2 \ - WHEN kind = 'session.ended' THEN 1 \ - ELSE 0 END) AS terminal_rank, \ - MAX(CASE WHEN kind = 'session.ended' THEN payload_json ELSE NULL END) \ - AS session_ended_payload_json, \ - SUM(CASE WHEN kind = 'llm.request' THEN 1 ELSE 0 END) \ - AS request_count, \ - SUM(CASE WHEN kind = 'llm.response' OR kind = 'llm.response.stream' \ - THEN 1 ELSE 0 END) AS response_count \ - FROM {dataset}.events WHERE _file_ = {file} GROUP BY _file_, session_id" - ); - let body = - bounded_summary_jsonl(engine, &sql, MAX_SUMMARY_ROWS as u64, MAX_SUMMARY_BYTES).await?; - body.lines() - .filter(|line| !line.trim().is_empty()) - .map(|line| { - let row: JsonValue = serde_json::from_str(line).context("decode event stats row")?; - let file = required_json_string(&row, "_file_")?.to_string(); - let session_id = required_json_string(&row, "session_id")?.to_string(); - let row_count = required_json_u64(&row, "row_count")? as usize; - let duplicate_event_ids = required_json_u64(&row, "duplicate_event_ids")? as usize; - let terminal_rank = required_json_u64(&row, "terminal_rank")?; - let request_count = required_json_u64(&row, "request_count")?; - let response_count = required_json_u64(&row, "response_count")?; - let status = terminal_event_status( - terminal_rank, - row.get("session_ended_payload_json") - .and_then(JsonValue::as_str), - ) - .or_else(|| capture_call_status(request_count, response_count)); - Ok(( - (file, session_id), - EventStats { - row_count, - duplicate_event_ids, - status, - }, - )) - }) - .collect() -} - -fn terminal_event_status(rank: u64, session_ended_payload_json: Option<&str>) -> Option { - match rank { - 3.. => Some("failed".into()), - 2 => Some("completed".into()), - 1 => { - let exit_code = session_ended_payload_json - .and_then(|raw| serde_json::from_str::(raw).ok()) - .and_then(|event| event.get("payload").cloned()) - .and_then(|payload| payload.get("exit_code").and_then(JsonValue::as_i64)); - Some(if exit_code.is_some_and(|code| code != 0) { - "failed".into() - } else { - "completed".into() - }) - } - _ => None, - } -} - -fn capture_call_status(request_count: u64, response_count: u64) -> Option { - if request_count == 0 && response_count == 0 { - None - } else if response_count >= request_count { - Some("completed".into()) - } else { - Some("active".into()) - } -} - fn run_status(row: &JsonValue) -> String { let values = [ "final_metrics", @@ -1337,31 +1192,6 @@ mod run_summary_tests { assert_eq!(run_status(&serde_json::json!({})), "active"); } - - #[test] - fn session_ended_status_honors_nonzero_exit_codes() { - let failed = serde_json::json!({"payload": {"exit_code": 7}}).to_string(); - let completed = serde_json::json!({"payload": {"exit_code": 0}}).to_string(); - - assert_eq!( - terminal_event_status(1, Some(&failed)).as_deref(), - Some("failed") - ); - assert_eq!( - terminal_event_status(1, Some(&completed)).as_deref(), - Some("completed") - ); - assert_eq!(terminal_event_status(3, None).as_deref(), Some("failed")); - assert_eq!(terminal_event_status(0, None), None); - } - - #[test] - fn gateway_call_pairs_close_runs_without_explicit_session_events() { - assert_eq!(capture_call_status(3, 3).as_deref(), Some("completed")); - assert_eq!(capture_call_status(3, 2).as_deref(), Some("active")); - assert_eq!(capture_call_status(0, 1).as_deref(), Some("completed")); - assert_eq!(capture_call_status(0, 0), None); - } } async fn build_run_index( @@ -1387,96 +1217,6 @@ async fn build_run_index( Ok(Arc::new(routes.finish())) } -async fn build_event_identity_index( - snapshot: &DatasetCatalogSnapshot, - engine: &ChronicleQueryEngine, -) -> Result> { - let mut routes = SourceRoutingIndexBuilder::new(EVENT_IDENTITY_COLUMNS); - for dataset in snapshot.datasets() { - let name = &dataset.mount.name; - routes.ensure_dataset(name); - let mut batches = engine - .dataframe(&format!( - "SELECT _file_, event_id, trace_id FROM {name}.events" - )) - .await? - .execute_stream() - .await?; - while let Some(batch) = batches.try_next().await? { - add_event_identity_batch(&mut routes, name, &batch)?; - routes.ensure_within_limits()?; - } - } - Ok(Arc::new(routes.finish())) -} - -async fn build_event_partition_index( - snapshot: &DatasetCatalogSnapshot, - engine: &ChronicleQueryEngine, -) -> Result> { - let mut routes = SourceRoutingIndexBuilder::new(EVENT_PARTITION_COLUMNS); - for dataset in snapshot.datasets() { - let name = &dataset.mount.name; - routes.ensure_dataset(name); - let mut batches = engine - .dataframe(&format!( - "SELECT _file_, session_id, agent_id FROM {name}.events" - )) - .await? - .execute_stream() - .await?; - while let Some(batch) = batches.try_next().await? { - add_event_partition_batch(&mut routes, name, &batch)?; - routes.ensure_within_limits()?; - } - } - Ok(Arc::new(routes.finish())) -} - -fn add_event_identity_batch( - routes: &mut SourceRoutingIndexBuilder, - dataset: &str, - batch: &RecordBatch, -) -> Result<()> { - let file = string_column(batch, "_file_")?; - let event_id = string_column(batch, "event_id")?; - let trace_id = string_column(batch, "trace_id")?; - for row in 0..batch.num_rows() { - anyhow::ensure!(!file.is_null(row), "event routing row has null _file_"); - routes.add( - dataset, - file.value(row), - [ - ("event_id", optional_string(event_id, row)), - ("trace_id", optional_string(trace_id, row)), - ], - ); - } - Ok(()) -} - -fn add_event_partition_batch( - routes: &mut SourceRoutingIndexBuilder, - dataset: &str, - batch: &RecordBatch, -) -> Result<()> { - let file = string_column(batch, "_file_")?; - let session_id = string_column(batch, "session_id")?; - let agent_id = string_column(batch, "agent_id")?; - for row in 0..batch.num_rows() { - anyhow::ensure!(!file.is_null(row), "event routing row has null _file_"); - routes.add( - dataset, - file.value(row), - [ - ("session_id", optional_string(session_id, row)), - ("agent_id", optional_string(agent_id, row)), - ], - ); - } - Ok(()) -} - fn add_run_batch( routes: &mut SourceRoutingIndexBuilder, dataset: &str, @@ -1525,12 +1265,6 @@ fn required_json_string<'a>(row: &'a JsonValue, field: &str) -> Result<&'a str> .with_context(|| format!("run index row is missing string field {field}")) } -fn required_json_u64(row: &JsonValue, field: &str) -> Result { - row.get(field) - .and_then(JsonValue::as_u64) - .with_context(|| format!("run index row is missing unsigned integer field {field}")) -} - #[cfg(test)] mod tests { use std::sync::atomic::{AtomicUsize, Ordering}; @@ -1710,8 +1444,6 @@ mod tests { let status = acceleration.status(); assert!(status.run_index.is_none()); assert!(status.run_index_unavailable); - assert!(!status.event_identity_index_unavailable); - assert!(!status.event_partition_index_unavailable); let status = serde_json::to_string(&status)?; assert!(!status.contains("optional-index-source-diagnostic")); assert!(!routed.iter().any(|query| { @@ -1743,40 +1475,13 @@ mod tests { Ok(()) } - fn event( - event_id: &str, - trace_id: &str, - agent_id: &str, - ) -> persisting_pchronicle::model::EventRecord { - persisting_pchronicle::model::EventRecord { - identity: persisting_pchronicle::model::EventIdentity { - event_id: Some(event_id.into()), - ..Default::default() - }, - seq: 1, - source: "server-routing-test".into(), - kind: "event".into(), - timestamp: None, - session_id: None, - agent_id: Some(agent_id.into()), - parent_uuid: None, - trace_id: Some(trace_id.into()), - call_id: None, - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload: serde_json::json!({"event_id": event_id}), - } - } - #[test] fn routing_index_intersects_values_without_duplicating_source_paths() { let mut builder = SourceRoutingIndexBuilder::new(&["event_id", "trace_id", "session_id", "agent_id"]); builder.add( "live", - "project-a/run-1/events.lance", + "project-a/run-1/storyline", [ ("event_id", Some("event-1")), ("trace_id", Some("trace-1")), @@ -1786,7 +1491,7 @@ mod tests { ); builder.add( "live", - "project-a/run-1/events.lance", + "project-a/run-1/storyline", [ ("event_id", Some("event-2")), ("trace_id", Some("trace-1")), @@ -1796,7 +1501,7 @@ mod tests { ); builder.add( "live", - "project-b/run-2/events.lance", + "project-b/run-2/storyline", [ ("event_id", Some("event-3")), ("trace_id", Some("trace-2")), @@ -1806,7 +1511,7 @@ mod tests { ); builder.add( "live", - "project-c/run-3/events.lance", + "project-c/run-3/storyline", [ ("event_id", Some("")), ("trace_id", None), @@ -1831,7 +1536,7 @@ mod tests { }, ] ), - Some(vec!["project-a/run-1/events.lance".into()]) + Some(vec!["project-a/run-1/storyline".into()]) ); assert_eq!( index.candidates( @@ -1851,7 +1556,7 @@ mod tests { values: vec![String::new()], }] ), - Some(vec!["project-c/run-3/events.lance".into()]) + Some(vec!["project-c/run-3/storyline".into()]) ); } @@ -1879,116 +1584,13 @@ mod tests { #[test] fn routing_index_limits_are_explicit() { - let mut builder = SourceRoutingIndexBuilder::new(EVENT_IDENTITY_COLUMNS); + let mut builder = SourceRoutingIndexBuilder::new(RUN_COLUMNS); builder.add( "live", - "one/events.lance", - [("event_id", Some("one")), ("trace_id", Some("trace"))], + "one/storyline", + [("session_id", Some("one")), ("agent_id", Some("agent"))], ); assert!(builder.ensure_limits(0, usize::MAX).is_err()); assert!(builder.ensure_limits(usize::MAX, 1).is_err()); } - - #[tokio::test] - async fn event_index_routes_to_one_catalog_source_without_changing_results() -> Result<()> { - use persisting_pchronicle::storage::{ - CatalogSnapshotOptions, DEFAULT_DATASET_NAME, DatasetMount, RawEventLanceAppender, - StoryCoords, - }; - - let root = std::env::temp_dir().join(format!( - "pchronicle-server-event-routing-{}-{}", - std::process::id(), - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH)? - .as_nanos() - )); - std::fs::create_dir_all(&root)?; - let project_a = StoryCoords::new( - root.to_string_lossy(), - "project-a", - "session-a", - Some("run-a".into()), - ); - let project_b = StoryCoords::new( - root.to_string_lossy(), - "project-b", - "session-b", - Some("run-b".into()), - ); - let mut appender = RawEventLanceAppender::default(); - appender - .append_event_batch(&[ - (project_a, event("event-a", "trace-a", "project-a")), - (project_b, event("event-b", "trace-b", "project-b")), - ]) - .await?; - appender.finish(); - - // Shallow Directory discovery only inspects mount children. Lift each - // events.lance beside the agent dir so both Sources stay in one Dataset - // while agent_id remains project-a / project-b. - for (agent, run) in [("project-a", "run-a"), ("project-b", "run-b")] { - let from = root.join(agent).join(run).join("events.lance"); - let to = root.join(agent).join("events.lance"); - std::fs::rename(&from, &to)?; - let _ = std::fs::remove_dir_all(root.join(agent).join(run)); - } - - let snapshot = Arc::new( - DatasetCatalogSnapshot::discover( - vec![DatasetMount::default(root.to_string_lossy())?], - Some(DEFAULT_DATASET_NAME.into()), - CatalogSnapshotOptions::default(), - ) - .await?, - ); - let engine = snapshot.clone().query_engine(Default::default()).await?; - let acceleration = ServerAcceleration::default(); - let sql = "SELECT _file_, event_id FROM events WHERE agent_id = 'project-a' AND event_id = 'event-a'"; - let routed = acceleration.route_sql(&snapshot, &engine, sql).await; - assert_eq!(routed.outcome, RoutingOutcome::Applied); - assert_eq!(routed.candidate_sources, Some(1)); - assert!( - routed.sql.contains("project-a/events.lance"), - "routed sql should prune to project-a events: {}", - routed.sql - ); - - let original = engine.query_jsonl(sql).await?; - let accelerated = engine.query_jsonl(&routed.sql).await?; - assert_eq!(accelerated, original); - assert_eq!(acceleration.status().event_identity_index.unwrap().rows, 2); - assert!(acceleration.status().event_partition_index.is_none()); - - let partition_acceleration = ServerAcceleration::default(); - let project_sql = - "SELECT event_id FROM events WHERE agent_id = 'project-a' ORDER BY event_id"; - let project_routed = partition_acceleration - .route_sql(&snapshot, &engine, project_sql) - .await; - assert_eq!(project_routed.outcome, RoutingOutcome::Applied); - assert_eq!(project_routed.candidate_sources, Some(1)); - assert_eq!( - engine.query_jsonl(&project_routed.sql).await?, - engine.query_jsonl(project_sql).await? - ); - assert!( - partition_acceleration - .status() - .event_identity_index - .is_none() - ); - assert_eq!( - partition_acceleration - .status() - .event_partition_index - .unwrap() - .rows, - 2 - ); - - std::fs::remove_dir_all(root)?; - Ok(()) - } } diff --git a/crates/persisting-pchronicle-cli/src/server/explorer.rs b/crates/persisting-pchronicle-cli/src/server/explorer.rs index a5be52612..b9d0dbc3b 100644 --- a/crates/persisting-pchronicle-cli/src/server/explorer.rs +++ b/crates/persisting-pchronicle-cli/src/server/explorer.rs @@ -1,9 +1,6 @@ -use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::collections::{BTreeMap, BTreeSet}; -use persisting_pchronicle::model::EventRecord; -use persisting_pchronicle::storage::{ - CatalogEventProvenance, DatasetMount, PathListEntry, PathListKind, -}; +use persisting_pchronicle::storage::{DatasetMount, PathListEntry, PathListKind}; use serde::{Deserialize, Serialize}; use serde_json::Value; @@ -298,8 +295,6 @@ pub(crate) struct HistogramBucket { #[derive(Clone, Debug, Serialize)] pub(crate) struct RunAnalysis { pub(crate) run: RunSummary, - pub(crate) event_provenance: CatalogEventProvenance, - pub(crate) event_count: usize, pub(crate) turn_count: usize, pub(crate) tool_call_count: usize, pub(crate) error_count: usize, @@ -336,7 +331,6 @@ pub(crate) struct TurnSummary { pub(crate) completion_tokens: Option, pub(crate) total_tokens: Option, pub(crate) tool_names: Vec, - pub(crate) event_seqs: Vec, pub(crate) has_error: bool, } @@ -345,8 +339,6 @@ pub(crate) struct TurnDetail { pub(crate) summary: TurnSummary, pub(crate) turn: persisting_pchronicle::model::StorylineTurn, pub(crate) wire_tool_calls: Vec, - pub(crate) event_provenance: CatalogEventProvenance, - pub(crate) events: Vec, } pub(crate) fn explorer_run_path( @@ -500,12 +492,7 @@ pub(crate) fn run_identity(run: &RunSummary) -> String { format!("{}\u{1f}{}\u{1f}{}", run.dataset, run.file, run.document_id) } -pub(crate) fn analyze( - run: RunSummary, - turns: &[TrajectoryTurnView], - events: &[EventRecord], - event_provenance: CatalogEventProvenance, -) -> RunAnalysis { +pub(crate) fn analyze(run: RunSummary, turns: &[TrajectoryTurnView]) -> RunAnalysis { let mut latencies = Vec::new(); let mut ttfts = Vec::new(); let mut prompt_tokens = 0u64; @@ -521,13 +508,11 @@ pub(crate) fn analyze( let mut kinds = BTreeMap::::new(); let mut model_groups = BTreeMap::::new(); let mut error_count = 0usize; - let index = EventIndex::new(events); for item in turns { if let Some(timestamp) = item.turn.timestamp.as_ref() { timestamps.push(timestamp.clone()); } - let linked = index.linked(&item.event_seqs, events); let mut values = Vec::new(); if let Some(value) = &item.turn.metrics { values.push(value); @@ -535,7 +520,6 @@ pub(crate) fn analyze( if let Some(value) = &item.turn.extra { values.push(value); } - values.extend(linked.iter().map(|event| &event.payload)); let (turn_prompt_tokens, turn_completion_tokens, turn_total_tokens) = token_counts(&values); if let Some(value) = turn_prompt_tokens { @@ -582,7 +566,7 @@ pub(crate) fn analyze( { models.insert(model.clone()); } - let has_error = turn_has_error(item, &linked); + let has_error = turn_has_error(item); if has_error { error_count += 1; } @@ -645,8 +629,6 @@ pub(crate) fn analyze( let latency_histogram = latency_histogram(&latencies); RunAnalysis { - event_provenance, - event_count: events.len(), turn_count: turns.len(), tool_call_count: turns .iter() @@ -683,13 +665,11 @@ pub(crate) fn analyze( pub(crate) fn turn_list_with_search( turns: &[TrajectoryTurnView], - events: &[EventRecord], q: Option<&str>, source: Option<&str>, search: TurnSearchStatus, ) -> TurnExplorerPage { let needle = q.unwrap_or_default().trim().to_ascii_lowercase(); - let index = EventIndex::new(events); let records = turns .iter() .filter(|item| { @@ -697,7 +677,7 @@ pub(crate) fn turn_list_with_search( source.is_empty() || source == "all" || item.turn.source == source }) && (needle.is_empty() || searchable_turn(item).contains(&needle)) }) - .map(|item| turn_summary(item, events, &index)) + .map(turn_summary) .collect::>(); let total = records.len(); TurnExplorerPage { @@ -716,67 +696,15 @@ pub(crate) fn turn_list_with_search( } } -/// Positions of every event sequence inside a trajectory's event list. -/// -/// Turns reference their events by sequence, so resolving them by scanning the -/// whole list once per turn costs `turns × events` and dominates long -/// trajectories. Repeated sequences are all kept: a source may emit an id more -/// than once, and collapsing them would change token and tool aggregates. -struct EventIndex(HashMap>); - -impl EventIndex { - fn new(events: &[EventRecord]) -> Self { - let mut positions = HashMap::>::with_capacity(events.len()); - for (position, event) in events.iter().enumerate() { - positions.entry(event.seq).or_default().push(position); - } - Self(positions) - } - - fn linked<'a>(&self, seqs: &[u64], events: &'a [EventRecord]) -> Vec<&'a EventRecord> { - let mut positions = seqs - .iter() - .filter_map(|seq| self.0.get(seq)) - .flatten() - .copied() - .collect::>(); - // A turn may list its sequences out of order or repeat one; callers - // expect each event once, in event-list order, as a scan produced them. - positions.sort_unstable(); - positions.dedup(); - positions - .into_iter() - .map(|position| &events[position]) - .collect() - } -} - -pub(crate) fn turn_detail( - item: &TrajectoryTurnView, - events: &[EventRecord], - event_provenance: CatalogEventProvenance, -) -> TurnDetail { - let index = EventIndex::new(events); - let linked = index - .linked(&item.event_seqs, events) - .into_iter() - .cloned() - .collect::>(); +pub(crate) fn turn_detail(item: &TrajectoryTurnView) -> TurnDetail { TurnDetail { - summary: turn_summary(item, events, &index), + summary: turn_summary(item), turn: item.turn.clone(), wire_tool_calls: item.wire_tool_calls.clone(), - event_provenance, - events: linked, } } -fn turn_summary( - item: &TrajectoryTurnView, - events: &[EventRecord], - index: &EventIndex, -) -> TurnSummary { - let linked = index.linked(&item.event_seqs, events); +fn turn_summary(item: &TrajectoryTurnView) -> TurnSummary { let mut values = Vec::new(); if let Some(value) = &item.turn.metrics { values.push(value); @@ -784,7 +712,6 @@ fn turn_summary( if let Some(value) = &item.turn.extra { values.push(value); } - values.extend(linked.iter().map(|event| &event.payload)); let (prompt_tokens, completion_tokens, total_tokens) = token_counts(&values); let tool_names = display_tool_calls(item) .into_iter() @@ -834,8 +761,7 @@ fn turn_summary( completion_tokens, total_tokens, tool_names, - event_seqs: item.event_seqs.clone(), - has_error: turn_has_error(item, &linked), + has_error: turn_has_error(item), } } @@ -1158,11 +1084,14 @@ fn compact(value: &str, limit: usize) -> String { } } -fn turn_has_error(item: &TrajectoryTurnView, events: &[&EventRecord]) -> bool { +fn turn_has_error(item: &TrajectoryTurnView) -> bool { item.turn.kind.as_deref().is_some_and(explicit_error_text) - || events.iter().any(|event| { - explicit_error_text(&event.kind) || value_has_explicit_error(&event.payload) - }) + || value_has_explicit_error(&item.turn.message) + || item + .turn + .extra + .as_ref() + .is_some_and(value_has_explicit_error) } fn explicit_error_text(value: &str) -> bool { @@ -1280,44 +1209,6 @@ mod tests { ); } - #[test] - fn event_index_matches_a_full_scan_for_duplicate_and_unordered_sequences() { - let events = [7u64, 3, 7, 5, 3, 9] - .into_iter() - .map(|seq| EventRecord { - identity: Default::default(), - seq, - source: "test".into(), - kind: "note".into(), - timestamp: None, - session_id: None, - agent_id: None, - parent_uuid: None, - trace_id: None, - call_id: None, - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload: serde_json::json!({ "seq": seq }), - }) - .collect::>(); - let index = EventIndex::new(&events); - for seqs in [ - vec![], - vec![3], - vec![9, 3, 7], - vec![5, 5], - vec![3, 4, 7, 11], - ] { - let scanned = events - .iter() - .filter(|event| seqs.contains(&event.seq)) - .collect::>(); - assert_eq!(index.linked(&seqs, &events), scanned, "{seqs:?}"); - } - } - #[test] fn percentiles_report_coverage_without_inventing_missing_samples() { let stats = metric_stats(vec![10.0, 20.0, 30.0, 40.0], 8); @@ -1418,7 +1309,6 @@ mod tests { root_session_id: None, path: format!("{dataset}/{file}/{session}"), row_count: 1, - duplicate_event_ids: 0, status: status.into(), format: Some("compact-jsonl/v1".into()), } @@ -1514,9 +1404,6 @@ mod tests { format: Some("compact-jsonl/v1".into()), kind: CatalogSourceKind::Store, revision: None, - projection_status: None, - projection_generation: None, - projection_candidates: 0, size_bytes: None, last_modified: None, status: CatalogSourceStatus::Ready, @@ -1532,9 +1419,6 @@ mod tests { format: None, kind: CatalogSourceKind::File, revision: None, - projection_status: None, - projection_generation: None, - projection_candidates: 0, size_bytes: None, last_modified: None, status: CatalogSourceStatus::Ready, @@ -1594,8 +1478,8 @@ mod tests { #[test] fn run_page_file_prefix_is_not_run_path() { let summaries = vec![ - sample_run("evals", "gsm8k/train/events.lance", "completed", "s1"), - sample_run("evals", "gsm8k/test/events.lance", "completed", "s2"), + sample_run("evals", "gsm8k/train/storyline", "completed", "s1"), + sample_run("evals", "gsm8k/test/storyline", "completed", "s2"), sample_run("evals", "mmlu.json", "completed", "s3"), ]; let page = run_page( diff --git a/crates/persisting-pchronicle-cli/src/server/mod.rs b/crates/persisting-pchronicle-cli/src/server/mod.rs index db27c1db6..28a0359bf 100644 --- a/crates/persisting-pchronicle-cli/src/server/mod.rs +++ b/crates/persisting-pchronicle-cli/src/server/mod.rs @@ -28,15 +28,12 @@ use persisting_pchronicle::analysis_compile::{ AnalysisSpec, CompileError, CompileScope, CompiledQuery, TableSchema, compile, }; use persisting_pchronicle::document::InputIssue; -use persisting_pchronicle::model::{EventRecord, StorylineTurn}; +use persisting_pchronicle::model::StorylineTurn; use persisting_pchronicle::query::{ChronicleQueryEngine, ChronicleQueryExecutionOptions}; use persisting_pchronicle::search::storyline_steps_fts_available; -#[cfg(test)] -use persisting_pchronicle::storage::StoryCoords; use persisting_pchronicle::storage::{ - CatalogConsistency, CatalogErrorPolicy, CatalogEventProvenance, CatalogSnapshotOptions, - CatalogStorylineKey, DEFAULT_DATASET_NAME, DatasetCatalogSnapshot, DatasetMount, - with_background_object_store_io, + CatalogConsistency, CatalogErrorPolicy, CatalogSnapshotOptions, CatalogStorylineKey, + DEFAULT_DATASET_NAME, DatasetCatalogSnapshot, DatasetMount, with_background_object_store_io, }; use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; @@ -67,9 +64,6 @@ struct AppState { catalog_refresh_interval: Duration, trajectory_cache: TrajectoryCache, trajectory_flights: Arc>>>, - /// Gateway-backed Warehouses read canonical events from the latest - /// manifest for single-trace observation, independent of projection idle. - live_reads: bool, catalog_acl: Option>, catalog_query_worker: bool, catalog_workers: Arc, @@ -209,7 +203,6 @@ pub(crate) struct RunSummary { pub(crate) root_session_id: Option, pub(crate) path: String, pub(crate) row_count: usize, - pub(crate) duplicate_event_ids: usize, pub(crate) status: String, #[serde(default, skip_serializing_if = "Option::is_none")] pub(crate) format: Option, @@ -223,8 +216,6 @@ pub(crate) struct SessionQuery { pub(crate) agent_id: String, pub(crate) session_id: String, pub(crate) root_session_id: Option, - pub(crate) offset: Option, - pub(crate) limit: Option, } /// Build the read-only Warehouse API and Web UI. @@ -248,7 +239,6 @@ fn app_state_with_catalog_refresh_interval( catalog_refresh_interval, trajectory_cache: Arc::new(tokio::sync::RwLock::new(None)), trajectory_flights: Arc::new(Mutex::new(HashMap::new())), - live_reads: false, catalog_acl: None, catalog_query_worker: false, catalog_workers: Arc::new(catalog_worker::WorkerPool::default()), @@ -274,15 +264,6 @@ impl PreparedWarehouse { Ok(warehouse) } - pub(crate) async fn prepare_live(config: ChronicleServerConfig) -> anyhow::Result { - let mut state = app_state(config); - state.live_reads = true; - let warehouse = Self { state }; - browse_coordinator(&warehouse.state).await; - warehouse.install_initial_runtime().await?; - Ok(warehouse) - } - #[cfg(test)] pub(crate) async fn prepare_catalog_front(acl: catalog::CatalogAcl) -> anyhow::Result { Self::prepare_catalog(acl, ChronicleServerConfig::front_only(), None).await @@ -331,6 +312,12 @@ impl PreparedWarehouse { snapshot_id } + #[cfg(test)] + pub(crate) async fn refresh_catalog(&self) -> anyhow::Result { + let runtime = self.refresh_runtime().await?; + Ok(runtime.snapshot.snapshot_id().to_string()) + } + async fn refresh_runtime(&self) -> anyhow::Result> { let _refresh = self.state.catalog_refresh.lock().await; let runtime = build_catalog_runtime(&self.state.config).await?; @@ -338,11 +325,6 @@ impl PreparedWarehouse { Ok(runtime) } - pub(crate) async fn refresh_catalog(&self) -> anyhow::Result { - let runtime = self.refresh_runtime().await?; - Ok(runtime.snapshot.snapshot_id().to_string()) - } - pub(crate) fn router(&self) -> Router { finish_routes(self.state.clone()) } @@ -385,7 +367,6 @@ fn api_routes() -> Router { .route("/explorer/record", get(explorer_record)) .route("/explorer/turns", get(explorer_turns)) .route("/explorer/turn", get(explorer_turn)) - .route("/events", get(events)) .route("/storyline", get(storyline)) .route("/trajectory-view", get(trajectory_view)) .route("/catalog", get(catalog).post(refresh_catalog)) @@ -626,7 +607,7 @@ async fn current_catalog_for_runs( return Ok(runtime); } // A zero interval requests synchronous freshness (used by embedded callers). - if state.catalog_refresh_interval.is_zero() || state.live_reads { + if state.catalog_refresh_interval.is_zero() { let _refresh = state.catalog_refresh.lock().await; return Ok(rebuild_catalog_for_runs(state, runtime).await); } @@ -769,7 +750,7 @@ async fn load_run_summaries( ApiError::not_found("dataset was not found").with_request_id(request_id.as_str()) })?; // Browse initialization and its disk projection are optional for Runs. - let cached_files = if file.is_none() && !state.live_reads { + let cached_files = if file.is_none() { match state.browse.get() { Some(browse) => browse.cached_source_paths(mount).await, None => Vec::new(), @@ -809,15 +790,7 @@ async fn load_run_summaries( Ok(summaries) }; let started = Instant::now(); - let result = if state.live_reads { - state - .scoped_queries - .run(scope, || execute(false)) - .await - .map(|summaries| (summaries, "summary_uncached")) - } else { - state.scoped_queries.cached(scope, execute).await - }; + let result = state.scoped_queries.cached(scope, execute).await; let result = result .map(|(summaries, cache_status)| { if let Some(metrics) = metrics { @@ -1036,7 +1009,6 @@ async fn try_compact_jsonl_runs_page( root_session_id: None, path, row_count: 1, - duplicate_event_ids: 0, status: "record".into(), format: Some("compact-jsonl/v1".into()), }, @@ -1191,7 +1163,6 @@ async fn try_on_demand_storyline_runs_page( root_session_id: None, path, row_count: 1, - duplicate_event_ids: 0, status: "completed".into(), format: Some("storyline-lance".into()), }, @@ -1723,18 +1694,17 @@ async fn try_resolve_on_demand_storyline_run( root_session_id: query.root_session_id.clone(), path, row_count: 1, - duplicate_event_ids: 0, status: "completed".into(), format: Some("storyline-lance".into()), })) } -async fn load_on_demand_storyline_bundle( +async fn load_on_demand_storyline_document( runtime: &CatalogRuntime, run: &RunSummary, request_id: &RequestId, op: &'static str, -) -> Result, ApiError> { +) -> Result, ApiError> { let Some(dataset) = runtime.snapshot.dataset(&run.dataset) else { return Ok(None); }; @@ -1772,43 +1742,29 @@ async fn load_on_demand_storyline_bundle( let Some(Some(storyline)) = stories.into_iter().next() else { return Ok(None); }; - let document = persisting_pchronicle::document::storyline_to_events(&storyline) - .map_err(|error| fail(request_id, op, error))?; - Ok(Some( - persisting_pchronicle::storage::CatalogTrajectoryBundle { - storyline, - event_view: persisting_pchronicle::storage::CatalogEventView { - provenance: CatalogEventProvenance::SyntheticFromStoryline, - document, - }, - }, - )) + Ok(Some(storyline)) } -async fn catalog_or_on_demand_trajectory_bundle( - state: &AppState, +async fn catalog_or_on_demand_storyline( + _state: &AppState, runtime: &CatalogRuntime, run: &RunSummary, request_id: &RequestId, op: &'static str, -) -> Result { +) -> Result { request_progress::phase("storage_read"); let key = catalog_storyline_key(run); - let catalog_result = if state.live_reads { - runtime.snapshot.load_live_trajectory_bundle(&key).await - } else { - runtime.snapshot.load_trajectory_bundle(&key).await - }; + let catalog_result = runtime.snapshot.load_storyline(&key).await; match catalog_result { - Ok(Some(bundle)) => Ok(bundle), - Ok(None) => load_on_demand_storyline_bundle(runtime, run, request_id, op) + Ok(Some(document)) => Ok(document), + Ok(None) => load_on_demand_storyline_document(runtime, run, request_id, op) .await? .ok_or_else(|| ApiError::not_found("run was not found")), Err(error) => { - if let Some(bundle) = - load_on_demand_storyline_bundle(runtime, run, request_id, op).await? + if let Some(document) = + load_on_demand_storyline_document(runtime, run, request_id, op).await? { - Ok(bundle) + Ok(document) } else { Err(fail(request_id, op, error)) } @@ -1825,101 +1781,6 @@ fn catalog_storyline_key(run: &RunSummary) -> CatalogStorylineKey { } } -#[cfg(test)] -fn event_uri_coords(uri: &str, run: &RunSummary) -> anyhow::Result { - let uri = uri.trim_end_matches('/'); - let run_uri = uri - .strip_suffix("/events.lance") - .or_else(|| (uri == "events.lance").then_some("")) - .with_context(|| format!("canonical event URI does not end in events.lance: {uri}"))?; - let (agent_uri, physical_run_id) = run_uri - .rsplit_once('/') - .context("canonical event URI has no Run directory")?; - let (storage, physical_agent_id) = agent_uri - .rsplit_once('/') - .map_or((".", agent_uri), |(storage, agent)| (storage, agent)); - anyhow::ensure!( - !physical_agent_id.is_empty() && !physical_run_id.is_empty(), - "canonical event URI has an invalid agent/Run hierarchy: {uri}" - ); - Ok(StoryCoords::new( - storage, - physical_agent_id, - run.session_id.clone(), - Some(physical_run_id.to_string()), - )) -} - -#[derive(Clone)] -struct LoadedEventView { - provenance: CatalogEventProvenance, - records: Vec, -} - -async fn load_events( - state: &AppState, - query: &SessionQuery, - request_id: &RequestId, -) -> Result { - let run = resolve_run_summary(state, query, request_id, None).await?; - let runtime = catalog_for_source(state, Some(&run.dataset), &run.file, request_id).await?; - let bundle = - catalog_or_on_demand_trajectory_bundle(state, &runtime, &run, request_id, "load_events") - .await?; - let document = bundle.event_view; - let offset = query - .offset - .unwrap_or(0) - .min(document.document.events.len()); - let end = query - .limit - .map(|limit| { - offset - .saturating_add(limit) - .min(document.document.events.len()) - }) - .unwrap_or(document.document.events.len()); - Ok(LoadedEventView { - provenance: document.provenance, - records: document.document.events[offset..end].to_vec(), - }) -} - -async fn events( - State(state): State, - request_id: RequestId, - query: Result, QueryRejection>, -) -> Result, ApiError> { - let query = api_query(query)?; - let offset = query.offset.unwrap_or(0); - let requested_limit = query.limit.unwrap_or(1000); - let full_query = SessionQuery { - offset: None, - limit: None, - ..query.clone() - }; - let event_view = load_events(&state, &full_query, &request_id).await?; - let total = event_view.records.len(); - let start = offset.min(total); - let end = start.saturating_add(requested_limit).min(total); - let records = event_view.records[start..end].to_vec(); - let next_offset = offset + records.len(); - Ok(Json(json!({ - "provenance": { - "kind": event_view.provenance, - "transform": event_view.provenance.transform() - }, - "snapshot": { - "offset": offset, - "next_offset": next_offset, - "total": total, - "has_more": next_offset < total && !records.is_empty(), - "limit": requested_limit - }, - "records": records - }))) -} - async fn storyline( State(state): State, request_id: RequestId, @@ -1928,11 +1789,10 @@ async fn storyline( let query = api_query(query)?; let run = resolve_run_summary(&state, &query, &request_id, None).await?; let runtime = catalog_for_source(&state, Some(&run.dataset), &run.file, &request_id).await?; - let bundle = - catalog_or_on_demand_trajectory_bundle(&state, &runtime, &run, &request_id, "storyline") - .await?; + let document = + catalog_or_on_demand_storyline(&state, &runtime, &run, &request_id, "storyline").await?; Ok(Json( - serde_json::to_value(bundle.storyline) + serde_json::to_value(document) .map_err(anyhow::Error::from) .map_err(|error| fail(&request_id, "storyline", error))?, )) @@ -1942,7 +1802,6 @@ async fn storyline( pub(crate) struct TrajectoryTurnView { pub(crate) turn: StorylineTurn, pub(crate) call_id: Option, - pub(crate) event_seqs: Vec, pub(crate) wire_tool_calls: Vec, } @@ -1958,8 +1817,6 @@ pub(crate) struct WireToolCall { #[derive(Debug, Serialize)] struct TrajectoryView { run: RunSummary, - event_provenance: CatalogEventProvenance, - event_kind_counts: BTreeMap, tool_call_count: usize, turns: Vec, } @@ -2037,30 +1894,10 @@ fn turn_call_id(turn: &StorylineTurn) -> Option { .map(str::to_string) } -fn turn_seq(turn: &StorylineTurn) -> Option { - turn.extra - .as_ref() - .and_then(|extra| extra.get("seq").or_else(|| extra.get("event_seq"))) - .and_then(Value::as_u64) -} - -fn event_seqs_for_turn(turn: &StorylineTurn, by_call: &BTreeMap>) -> Vec { - // Canonical Event -> Storyline projection records the authoritative source - // sequence on each turn. Prefer it over the broader call correlation: a - // call contains both request and response, and attaching both to both turns - // duplicates usage, tool calls, latency and TTFT in Explorer aggregates. - turn_seq(turn) - .map(|seq| vec![seq]) - .or_else(|| turn_call_id(turn).and_then(|id| by_call.get(&id).cloned())) - .unwrap_or_default() -} - #[derive(Clone)] struct LoadedTrajectory { runtime: Arc, run: RunSummary, - event_provenance: CatalogEventProvenance, - records: Vec, turns: Vec, } @@ -2119,13 +1956,11 @@ async fn load_trajectory( return Ok(Arc::new(LoadedTrajectory { runtime, run, - event_provenance: CatalogEventProvenance::SyntheticFromStoryline, - records: Vec::new(), turns: Vec::new(), })); } let phase = Instant::now(); - let bundle = catalog_or_on_demand_trajectory_bundle( + let document = catalog_or_on_demand_storyline( state, &runtime, &run, @@ -2134,21 +1969,10 @@ async fn load_trajectory( ) .await?; metrics.record("trajectory_read", phase); - let event_provenance = bundle.event_view.provenance; - let records = bundle.event_view.document.events; - let document = bundle.storyline; // ACTF step records carry the first user input at document level when it // is the baseline prompt. Preserve it on the first turn so Explorer can // render the user side of the conversation without changing storage. let document_prompt = document.prompt.clone(); - let mut by_call = BTreeMap::>::new(); - let mut by_seq = HashMap::>::with_capacity(records.len()); - for (position, event) in records.iter().enumerate() { - if let Some(call_id) = event.call_id.as_ref().filter(|id| !id.is_empty()) { - by_call.entry(call_id.clone()).or_default().push(event.seq); - } - by_seq.entry(event.seq).or_default().push(position); - } let turns = document .turns .into_iter() @@ -2158,21 +1982,10 @@ async fn load_trajectory( turn.prompt = document_prompt.clone(); } let call_id = turn_call_id(&turn); - let event_seqs = event_seqs_for_turn(&turn, &by_call); - // Resolve this turn's events through the sequence index. - // Scanning every record per turn is quadratic and dominates - // long trajectories. - let mut positions = event_seqs - .iter() - .filter_map(|seq| by_seq.get(seq)) - .flatten() - .copied() - .collect::>(); - positions.sort_unstable(); - positions.dedup(); let mut wire_tool_calls = Vec::new(); - for position in positions { - collect_wire_tool_calls(&records[position].payload, &mut wire_tool_calls); + collect_wire_tool_calls(&turn.message, &mut wire_tool_calls); + if let Some(extra) = &turn.extra { + collect_wire_tool_calls(extra, &mut wire_tool_calls); } let mut seen = BTreeSet::new(); wire_tool_calls.retain(|call| { @@ -2202,7 +2015,6 @@ async fn load_trajectory( TrajectoryTurnView { turn, call_id, - event_seqs, wire_tool_calls, } }) @@ -2210,8 +2022,6 @@ async fn load_trajectory( let loaded = Arc::new(LoadedTrajectory { runtime, run, - event_provenance, - records, turns, }); Ok::<_, ApiError>(loaded) @@ -2233,10 +2043,6 @@ async fn trajectory_view( ) -> Result, ApiError> { let query = api_query(query)?; let loaded = load_trajectory(&state, &query, &request_id, &metrics).await?; - let mut event_kind_counts = BTreeMap::new(); - for event in &loaded.records { - *event_kind_counts.entry(event.kind.clone()).or_insert(0) += 1; - } let tool_call_count = loaded .turns .iter() @@ -2245,8 +2051,6 @@ async fn trajectory_view( .sum(); Ok(Json(TrajectoryView { run: loaded.run.clone(), - event_provenance: loaded.event_provenance, - event_kind_counts, tool_call_count, turns: loaded.turns.clone(), })) @@ -2275,12 +2079,7 @@ async fn explorer_run( &metrics, )) .await?; - Ok(Json(explorer::analyze( - loaded.run.clone(), - &loaded.turns, - &loaded.records, - loaded.event_provenance, - ))) + Ok(Json(explorer::analyze(loaded.run.clone(), &loaded.turns))) }, ) .await @@ -2336,8 +2135,6 @@ impl TurnsQuery { agent_id: self.agent_id.clone(), session_id: self.session_id.clone(), root_session_id: self.root_session_id.clone(), - offset: None, - limit: None, } } } @@ -2486,7 +2283,6 @@ async fn explorer_turns_inner( let phase = Instant::now(); let mut page = explorer::turn_list_with_search( &turns, - &loaded.records, search_query, query.source.as_deref(), explorer::TurnSearchStatus { @@ -2496,12 +2292,7 @@ async fn explorer_turns_inner( }, ); if query.include_analysis { - page.analysis = Some(explorer::analyze( - loaded.run.clone(), - &loaded.turns, - &loaded.records, - loaded.event_provenance, - )); + page.analysis = Some(explorer::analyze(loaded.run.clone(), &loaded.turns)); } metrics.record("turn_projection", phase); Ok(Json(page)) @@ -2532,8 +2323,6 @@ async fn explorer_turn( agent_id: query.agent_id, session_id: query.session_id, root_session_id: query.root_session_id, - offset: None, - limit: None, }; let loaded = load_trajectory(&state, &session, &request_id, &metrics).await?; let item = loaded @@ -2541,11 +2330,7 @@ async fn explorer_turn( .iter() .find(|item| item.turn.id == query.turn_id) .ok_or_else(|| ApiError::not_found(format!("turn {} was not found", query.turn_id)))?; - Ok(Json(explorer::turn_detail( - item, - &loaded.records, - loaded.event_provenance, - ))) + Ok(Json(explorer::turn_detail(item))) } #[derive(Debug, Serialize)] @@ -2767,24 +2552,6 @@ fn source_query_fields() -> Vec { ] } -fn event_query_fields() -> Vec { - vec![ - field("_file_", "TEXT", "Dataset-relative recorded event source"), - field("seq", "BIGINT", "Canonical append sequence"), - field("event_id", "TEXT?", "Producer event identifier"), - field("timestamp", "TEXT?", "Captured timestamp"), - field("kind", "TEXT", "Canonical event kind"), - field("source", "TEXT", "Event producer"), - field("agent_id", "TEXT?", "Agent identifier"), - field("session_id", "TEXT?", "Session identifier"), - field("call_id", "TEXT?", "Call correlation identifier"), - field("trace_id", "TEXT?", "Trace correlation identifier"), - field("parent_call_id", "TEXT?", "Parent call identifier"), - field("model", "TEXT?", "Captured model"), - field("payload_json", "JSON", "Canonical event payload"), - ] -} - fn format_query_fields() -> Vec { vec![ field("id", "TEXT", "Stable record identifier"), @@ -2829,13 +2596,6 @@ fn query_table_summaries() -> Vec { grain: "complete run", fields: trajectory_query_fields(), }, - QueryTableSummary { - name: "events", - description: "Recorded events; empty for sources without recorded events", - kind: "table", - grain: "event", - fields: event_query_fields(), - }, QueryTableSummary { name: "atif", description: "ATIF documents exposed as id plus JSONB data", diff --git a/crates/persisting-pchronicle-cli/src/server/problem.rs b/crates/persisting-pchronicle-cli/src/server/problem.rs index b93e0fccb..b3ec1b9c1 100644 --- a/crates/persisting-pchronicle-cli/src/server/problem.rs +++ b/crates/persisting-pchronicle-cli/src/server/problem.rs @@ -221,7 +221,6 @@ impl ApiError { ) } - #[allow(dead_code)] pub(super) fn unavailable() -> Self { Self::public( StatusCode::SERVICE_UNAVAILABLE, diff --git a/crates/persisting-pchronicle-cli/src/server/tests.rs b/crates/persisting-pchronicle-cli/src/server/tests.rs index a1388abc3..92fc2f268 100644 --- a/crates/persisting-pchronicle-cli/src/server/tests.rs +++ b/crates/persisting-pchronicle-cli/src/server/tests.rs @@ -711,15 +711,15 @@ fn json_dataset_root() -> std::path::PathBuf { std::process::id() )); std::fs::create_dir_all(&root).unwrap(); - write_gateway_fixture(&root, "gateway.json", "json-session", "json-job"); + write_openai_fixture(&root, "gateway.json", "json-session", "json-job"); root } -fn write_gateway_fixture(root: &std::path::Path, file: &str, session_id: &str, job_id: &str) { - write_gateway_fixture_with_status(root, file, session_id, job_id, false); +fn write_openai_fixture(root: &std::path::Path, file: &str, session_id: &str, job_id: &str) { + write_openai_fixture_with_status(root, file, session_id, job_id, false); } -fn write_gateway_fixture_with_status( +fn write_openai_fixture_with_status( root: &std::path::Path, file: &str, session_id: &str, @@ -776,187 +776,6 @@ fn mounted_config_rejects_duplicate_dataset_names() { assert!(error.to_string().contains("unique")); } -#[test] -fn projected_turn_sequence_wins_over_call_wide_event_group() { - let turn = StorylineTurn { - id: 1, - kind: Some("llm.response".into()), - timestamp: None, - source: "agent".into(), - message: json!("done"), - reasoning_content: None, - reasoning_effort: None, - tool_calls: None, - observation: None, - metrics: None, - model_name: None, - llm_call_count: Some(1), - is_copied_context: None, - latency_ms: None, - ttft_ms: None, - extra: Some(json!({"call_id": "model-call", "seq": 11})), - env: None, - prompt: None, - finished_at: None, - }; - let by_call = BTreeMap::from([("model-call".into(), vec![10, 11])]); - - assert_eq!(event_seqs_for_turn(&turn, &by_call), vec![11]); -} - -#[test] -fn explorer_analysis_counts_usage_and_normalized_tools_once_per_call() { - use persisting_pchronicle::model::StorylineToolCall; - - let user = StorylineTurn { - id: 1, - kind: Some("llm.request".into()), - timestamp: Some( - persisting_pchronicle::model::StorylineTimestamp::from_rfc3339("2026-08-20T00:00:00Z") - .unwrap(), - ), - source: "user".into(), - message: json!("run tool"), - reasoning_content: None, - reasoning_effort: None, - tool_calls: None, - observation: None, - metrics: None, - model_name: None, - llm_call_count: None, - is_copied_context: None, - latency_ms: None, - ttft_ms: None, - extra: Some(json!({"call_id": "model-call", "seq": 0})), - env: None, - prompt: None, - finished_at: None, - }; - let agent = StorylineTurn { - id: 2, - kind: Some("llm.response".into()), - timestamp: Some( - persisting_pchronicle::model::StorylineTimestamp::from_rfc3339("2026-08-20T00:00:01Z") - .unwrap(), - ), - source: "agent".into(), - message: json!(""), - reasoning_content: None, - reasoning_effort: None, - tool_calls: Some(vec![StorylineToolCall { - tool_call_id: "tool-call-1".into(), - function_name: "lookup".into(), - arguments: json!({"q": "x"}), - result: None, - duration_ms: None, - extra: None, - kind: None, - response: None, - }]), - observation: None, - metrics: Some(json!({ - "prompt_tokens": 10, - "completion_tokens": 4, - "total_tokens": 14 - })), - model_name: Some("test-model".into()), - llm_call_count: Some(1), - is_copied_context: None, - latency_ms: Some(1000), - ttft_ms: Some(100), - extra: Some(json!({"call_id": "model-call", "seq": 1})), - env: None, - prompt: None, - finished_at: None, - }; - let event = |seq, kind: &str, payload| EventRecord { - identity: Default::default(), - seq, - source: "gateway".into(), - kind: kind.into(), - timestamp: None, - session_id: Some("session".into()), - agent_id: Some("agent".into()), - parent_uuid: None, - trace_id: None, - call_id: Some("model-call".into()), - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload, - }; - let events = vec![ - event(0, "llm.request", json!({"model": "test-model"})), - event( - 1, - "llm.response.stream", - json!({ - "usage": { - "prompt_tokens": 10, - "completion_tokens": 4, - "total_tokens": 14 - } - }), - ), - ]; - let turns = vec![ - TrajectoryTurnView { - turn: user, - call_id: Some("model-call".into()), - event_seqs: vec![0], - // A later request carries the prior assistant tool call as message - // history. It must not be counted as a new invocation. - wire_tool_calls: vec![WireToolCall { - id: Some("tool-call-1".into()), - name: "lookup".into(), - arguments: json!({"q": "x"}), - result: None, - }], - }, - TrajectoryTurnView { - turn: agent, - call_id: Some("model-call".into()), - event_seqs: vec![1], - wire_tool_calls: vec![WireToolCall { - id: Some("tool-call-1".into()), - name: "lookup".into(), - arguments: json!({"q": "x"}), - result: None, - }], - }, - ]; - let mut user_with_native_call = turns[0].clone(); - user_with_native_call.turn.tool_calls = turns[1].turn.tool_calls.clone(); - assert!(explorer::display_tool_calls(&user_with_native_call).is_empty()); - let run = RunSummary { - dataset: "dataset".into(), - file: "events.lance".into(), - document_id: "session".into(), - run_id: None, - agent_id: "agent".into(), - model_name: Some("test-model".into()), - session_id: "session".into(), - root_session_id: None, - path: "dataset/events.lance/session".into(), - row_count: 2, - duplicate_event_ids: 0, - status: "completed".into(), - format: None, - }; - - let analysis = explorer::analyze(run, &turns, &events, CatalogEventProvenance::Canonical); - assert_eq!(analysis.prompt_tokens, Some(10)); - assert_eq!(analysis.completion_tokens, Some(4)); - assert_eq!(analysis.total_tokens, Some(14)); - assert_eq!(analysis.tool_call_count, 1); - assert_eq!(analysis.tools.len(), 1); - assert_eq!(analysis.tools[0].name, "lookup"); - assert_eq!(analysis.tools[0].count, 1); - assert_eq!(analysis.latency_ms.sample_count, 1); - assert_eq!(analysis.ttft_ms.sample_count, 1); -} - #[test] fn evidence_queries_are_wrapped_with_a_server_side_row_bound() { assert_eq!( @@ -973,34 +792,6 @@ fn evidence_queries_are_wrapped_with_a_server_side_row_bound() { ); } -#[test] -fn canonical_event_uri_resolves_write_coordinates_independent_of_mount_root() { - let run = RunSummary { - dataset: "live".into(), - file: "agent/run-1/events.lance".into(), - document_id: "child".into(), - run_id: Some("child".into()), - agent_id: "agent".into(), - model_name: None, - session_id: "child".into(), - root_session_id: Some("run-1".into()), - path: "live/agent/run-1/events.lance/child".into(), - row_count: 1, - duplicate_event_ids: 0, - status: "active".into(), - format: None, - }; - let local = event_uri_coords("/tmp/capture/agent/run-1/events.lance", &run).unwrap(); - assert_eq!(local.storage, "/tmp/capture"); - assert_eq!(local.agent_id, "agent"); - assert_eq!(local.root_session_id.as_deref(), Some("run-1")); - - let remote = event_uri_coords("s3://bucket/prefix/agent/run-1/events.lance", &run).unwrap(); - assert_eq!(remote.storage, "s3://bucket/prefix"); - assert_eq!(remote.agent_id, "agent"); - assert_eq!(remote.session_id, "child"); -} - #[tokio::test] async fn json_datasets_expose_tables_and_support_read_only_sql() { use http_body_util::BodyExt; @@ -1028,7 +819,6 @@ async fn json_datasets_expose_tables_and_support_read_only_sql() { assert_eq!(tables["tables"][2]["name"], "steps"); assert_eq!(tables["tables"][3]["name"], "tool_calls"); assert_eq!(tables["tables"][4]["name"], "trajectories"); - assert_eq!(tables["tables"][5]["name"], "events"); assert_eq!(tables["tables"][4]["kind"], "view"); let view_names: Vec<_> = tables["tables"] .as_array() @@ -1107,7 +897,7 @@ async fn explorer_automatically_refreshes_new_dataset_sources() { serde_json::from_slice(&initial.into_body().collect().await.unwrap().to_bytes()).unwrap(); assert_eq!(initial["snapshot"]["total"], 1); - write_gateway_fixture(&root, "second.json", "second-session", "second-job"); + write_openai_fixture(&root, "second.json", "second-session", "second-job"); let tree = app .clone() .oneshot( @@ -1231,7 +1021,7 @@ async fn server_routing_index_prunes_point_queries_and_resets_on_refresh() -> an use tower::ServiceExt; let root = json_dataset_root(); - write_gateway_fixture(&root, "second.json", "second-session", "second-job"); + write_openai_fixture(&root, "second.json", "second-session", "second-job"); let app = router(root.to_string_lossy().to_string()); let routed = app @@ -1474,57 +1264,6 @@ async fn prepared_catalog_installs_refreshes_and_retains_the_last_good_runtime() Ok(()) } -#[tokio::test] -async fn live_warehouse_reads_new_events_without_catalog_refresh() -> anyhow::Result<()> { - use http_body_util::BodyExt; - use tower::ServiceExt; - - let temp = tempfile::tempdir()?; - let coords = StoryCoords::new(temp.path().to_string_lossy(), "agent", "session", None); - let event = |seq| EventRecord { - identity: persisting_pchronicle::model::EventIdentity::default(), - seq, - source: "test".into(), - kind: "note".into(), - timestamp: None, - session_id: Some("session".into()), - agent_id: Some("agent".into()), - parent_uuid: None, - trace_id: None, - call_id: None, - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload: json!({"seq": seq}), - }; - persisting_pchronicle::storage::RawEventLanceStore - .append_events(&coords, &[event(0)]) - .await?; - let config = ChronicleServerConfig::mounted(vec![DatasetMount::default( - temp.path().to_string_lossy().to_string(), - )?])?; - let prepared = PreparedWarehouse::prepare_live(config).await?; - - // Append after the Warehouse has pinned its initial Catalog snapshot. - persisting_pchronicle::storage::RawEventLanceStore - .append_events(&coords, &[event(1)]) - .await?; - let response = prepared - .router() - .oneshot( - axum::http::Request::builder() - .uri("/api/events?agent_id=agent&session_id=session") - .body(axum::body::Body::empty())?, - ) - .await?; - assert_eq!(response.status(), StatusCode::OK); - let body: Value = serde_json::from_slice(&response.into_body().collect().await?.to_bytes())?; - assert_eq!(body["snapshot"]["total"], 2); - assert_eq!(body["records"].as_array().map(Vec::len), Some(2)); - Ok(()) -} - #[tokio::test] async fn warehouse_keeps_api_v1_aliases_for_embedded_web_ui() { use http_body_util::BodyExt; @@ -1635,7 +1374,7 @@ async fn warehouse_does_not_expose_unused_har_or_revisions_routes() { #[tokio::test] async fn unscoped_runs_refresh_in_background_and_back_off_after_failure() { let root = tempfile::tempdir().unwrap(); - write_gateway_fixture(root.path(), "first.json", "first-session", "job"); + write_openai_fixture(root.path(), "first.json", "first-session", "job"); let config = ChronicleServerConfig::mounted(vec![ DatasetMount::default(root.path().to_string_lossy().to_string()).unwrap(), ]) @@ -1646,7 +1385,7 @@ async fn unscoped_runs_refresh_in_background_and_back_off_after_failure() { Arc::get_mut(&mut initial).unwrap().built_at = Instant::now() - Duration::from_secs(60); let old_id = initial.snapshot.snapshot_id().to_owned(); *state.catalog.write().await = Some(initial); - write_gateway_fixture(root.path(), "second.json", "second-session", "job"); + write_openai_fixture(root.path(), "second.json", "second-session", "job"); // An ongoing Catalog refresh must never make a warm UI request wait. let guard = state.catalog_refresh.lock().await; @@ -1711,14 +1450,14 @@ async fn explorer_runs_reuses_ui_summaries_until_explicit_catalog_refresh() { use tower::ServiceExt; let root = tempfile::tempdir().unwrap(); - write_gateway_fixture(root.path(), "first.json", "first-session", "job"); + write_openai_fixture(root.path(), "first.json", "first-session", "job"); let app = router(root.path().to_string_lossy().to_string()); let uri = "/api/explorer/runs?dataset=dataset&limit=1"; let (status, initial) = get_json(&app, uri).await; assert_eq!(status, StatusCode::OK, "{initial}"); assert_eq!(initial["snapshot"]["total"], 1); - write_gateway_fixture(root.path(), "second.json", "second-session", "job"); + write_openai_fixture(root.path(), "second.json", "second-session", "job"); let response = app .clone() .oneshot( @@ -1761,7 +1500,7 @@ async fn explorer_runs_prunes_unrelated_sources_before_querying() { let root = tempfile::tempdir().unwrap(); let prefix = "nested/a'_%"; std::fs::create_dir_all(root.path().join(prefix)).unwrap(); - write_gateway_fixture( + write_openai_fixture( root.path(), &format!("{prefix}/run.json"), "selected", @@ -1903,7 +1642,6 @@ async fn explorer_routes_page_runs_and_lazy_load_turn_evidence() { String::from_utf8_lossy(&analysis_body) ); let analysis: Value = serde_json::from_slice(&analysis_body).unwrap(); - assert_eq!(analysis["event_provenance"], "synthetic_from_storyline"); assert_eq!(analysis["turn_count"], 2); assert_eq!(analysis["latency_histogram"].as_array().unwrap().len(), 6); assert_eq!(analysis["source_breakdown"].as_array().unwrap().len(), 2); @@ -1947,28 +1685,7 @@ async fn explorer_routes_page_runs_and_lazy_load_turn_evidence() { let detail: Value = serde_json::from_slice(&detail_body).unwrap(); assert_eq!(detail["summary"]["id"], 1); assert_eq!(detail["turn"]["src"], "user"); - assert_eq!(detail["event_provenance"], "synthetic_from_storyline"); - - let events = app - .clone() - .oneshot( - axum::http::Request::builder() - .uri(format!("/api/events?{coordinates}")) - .body(axum::body::Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - assert_eq!(events.status(), StatusCode::OK); - let events = response_json(events).await; - assert_eq!( - events["provenance"], - json!({ - "kind": "synthetic_from_storyline", - "transform": "storyline_to_events_v1" - }) - ); - assert!(!events["records"].as_array().unwrap().is_empty()); + assert!(detail.get("events").is_none()); std::fs::remove_dir_all(root).unwrap(); } @@ -2037,7 +1754,7 @@ async fn explorer_uses_terminal_metadata_for_run_status() { use tower::ServiceExt; let root = json_dataset_root(); - write_gateway_fixture_with_status( + write_openai_fixture_with_status( &root, "completed.json", "completed-session", @@ -2233,7 +1950,7 @@ async fn explorer_tree_lists_mounted_datasets_by_run_count() -> anyhow::Result<( let live = json_dataset_root(); std::fs::create_dir_all(live.join("nested"))?; - write_gateway_fixture(&live, "nested/run.json", "nested-session", "nested-job"); + write_openai_fixture(&live, "nested/run.json", "nested-session", "nested-job"); let archive = json_dataset_root(); let config = ChronicleServerConfig::mounted(vec![ DatasetMount::new("live", live.to_string_lossy())?, @@ -2808,7 +2525,6 @@ async fn exact_trajectory_endpoints_do_not_wait_for_global_catalog() -> anyhow:: "explorer/turns", "explorer/turn", "trajectory-view", - "events", "storyline", ] { let (status, body) = tokio::time::timeout( @@ -2818,7 +2534,6 @@ async fn exact_trajectory_endpoints_do_not_wait_for_global_catalog() -> anyhow:: .await?; assert_eq!(status, StatusCode::OK, "{endpoint}: {body}"); if endpoint == "explorer/run" { - assert_eq!(body["event_provenance"], "synthetic_from_storyline"); assert!(body["turn_count"].as_u64().unwrap() > 0); } } @@ -2892,7 +2607,7 @@ async fn runs_unknown_dataset_is_a_structured_error_without_panicking() { #[tokio::test] async fn runs_deadline_includes_catalog_lock_and_releases_cancelled_work() { let root = tempfile::tempdir().unwrap(); - write_gateway_fixture(root.path(), "run.json", "session", "job"); + write_openai_fixture(root.path(), "run.json", "session", "job"); let config = ChronicleServerConfig::mounted(vec![ DatasetMount::default(root.path().to_string_lossy().to_string()).unwrap(), ]) @@ -3038,8 +2753,6 @@ async fn overlapping_trajectory_loads_share_immutable_result() { agent_id: run.agent_id.clone(), session_id: run.session_id.clone(), root_session_id: run.root_session_id.clone(), - offset: None, - limit: None, }; let (left, right) = tokio::join!( load_trajectory(&state, &query, &request, &metrics), diff --git a/crates/persisting-pchronicle-cli/src/tests.rs b/crates/persisting-pchronicle-cli/src/tests.rs index f1e4d2e71..6cd953848 100644 --- a/crates/persisting-pchronicle-cli/src/tests.rs +++ b/crates/persisting-pchronicle-cli/src/tests.rs @@ -1,75 +1,5 @@ use super::*; -#[tokio::test] -async fn trajectory_append_uses_persisting_events_protocol_directly() -> Result<()> { - let temporary = tempfile::tempdir()?; - let request = persisting_events::TrajectoryAppendRequest { - storage: temporary.path().to_string_lossy().into_owned(), - agent_id: "agent".into(), - session_id: "session".into(), - format: Default::default(), - root_session_id: None, - records: vec![persisting_events::EventRecord { - identity: persisting_events::EventIdentity::default(), - seq: 1, - source: "test".into(), - kind: "note".into(), - timestamp: None, - session_id: Some("session".into()), - agent_id: Some("agent".into()), - parent_uuid: None, - trace_id: None, - call_id: None, - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload: serde_json::json!({"content": "hello"}), - }], - }; - let response: persisting_events::TrajectoryAppendResponse = - crate::control::append_trajectory(request).await?; - assert_eq!(response.accepted_records, 1); - assert_eq!(response.status, "ok"); - Ok(()) -} - -#[tokio::test] -async fn trajectory_append_can_persist_json_in_the_warehouse() -> Result<()> { - let temporary = tempfile::tempdir()?; - let request = persisting_events::TrajectoryAppendRequest { - storage: temporary.path().to_string_lossy().into_owned(), - agent_id: "agent".into(), - session_id: "json-session".into(), - format: persisting_events::TrajectoryFormat::Json, - root_session_id: None, - records: vec![persisting_events::EventRecord { - identity: persisting_events::EventIdentity::default(), - seq: 7, - source: "gateway".into(), - kind: "llm.request".into(), - timestamp: None, - session_id: Some("json-session".into()), - agent_id: Some("agent".into()), - parent_uuid: None, - trace_id: None, - call_id: None, - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload: serde_json::json!({"http": {"request_body": {"raw": true}}}), - }], - }; - let response = crate::control::append_trajectory(request).await?; - assert_eq!(response.status, "ok"); - let path = temporary - .path() - .join("json/agent/json-session/events.jsonl"); - let contents = std::fs::read_to_string(path)?; - assert!(contents.contains("\"request_body\"")); - Ok(()) -} use clap::CommandFactory; use serde_json::Value; use std::ffi::{OsStr, OsString}; @@ -197,80 +127,6 @@ fn actf_with_unknown_step_fields(task_id: &str, users: &[&str]) -> Value { }) } -async fn append_canonical_note(storage: &std::path::Path) -> Result<()> { - let coords = persisting_pchronicle::storage::StoryCoords::new( - storage.to_string_lossy(), - "agent", - "session", - None, - ); - persisting_pchronicle::storage::RawEventLanceStore - .append_events( - &coords, - &[persisting_pchronicle::model::EventRecord { - identity: Default::default(), - seq: 0, - source: "test".into(), - kind: "note".into(), - timestamp: None, - session_id: None, - agent_id: None, - parent_uuid: None, - trace_id: None, - call_id: None, - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload: serde_json::json!({"content":"canonical"}), - }], - ) - .await?; - Ok(()) -} - -fn canonical_source(storage: &std::path::Path) -> Result { - let coords = persisting_pchronicle::storage::StoryCoords::new( - storage.to_string_lossy(), - "agent", - "session", - None, - ); - persisting_pchronicle::storage::raw_event_lance_path(&coords) -} - -async fn append_canonical_run(storage: &Path, run_id: &str, seq: u64) -> Result { - let coords = persisting_pchronicle::storage::StoryCoords::new( - storage.to_string_lossy(), - "agent", - run_id, - Some(run_id.into()), - ); - persisting_pchronicle::storage::RawEventLanceStore - .append_events( - &coords, - &[persisting_pchronicle::model::EventRecord { - identity: Default::default(), - seq, - source: "test".into(), - kind: "note".into(), - timestamp: None, - session_id: Some(run_id.into()), - agent_id: Some("agent".into()), - parent_uuid: None, - trace_id: None, - call_id: None, - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload: serde_json::json!({"content": format!("{run_id}-{seq}")}), - }], - ) - .await?; - persisting_pchronicle::storage::raw_event_lance_path(&coords) -} - #[test] fn command_tree_contains_the_product_commands() { let command = Cli::command(); @@ -282,7 +138,7 @@ fn command_tree_contains_the_product_commands() { names, [ "onboard", "dataset", "list", "stats", "query", "agent", "find", "import", "drop", - "export", "sync", "echo", "dev", "serve", + "export", "sync", "serve", ] ); let dataset = command @@ -520,20 +376,6 @@ fn canonical_parser_surface_matches_the_cli_guide() -> Result<()> { assert!(drop.yes); assert!(Cli::try_parse_from(["pchronicle", "export", "-t", "-", "-o", "storyline"]).is_err()); - assert!( - Cli::try_parse_from([ - "pchronicle", - "serve", - "--listen", - "127.0.0.1:8080", - "--gateway-config", - "gateway.toml", - "--gateway-dataset", - "evals", - "evals=./dataset" - ]) - .is_ok() - ); Ok(()) } @@ -1316,9 +1158,6 @@ fn list_source_status_does_not_serialize_catalog_diagnostics() -> Result<()> { format: None, kind: CatalogSourceKind::File, revision: None, - projection_status: None, - projection_generation: None, - projection_candidates: 0, size_bytes: None, last_modified: None, status: CatalogSourceStatus::Error, @@ -1358,169 +1197,12 @@ async fn status_reports_exact_counts_as_json() -> Result<()> { assert_eq!(value["counts"]["trajectories"], 1); assert_eq!(value["counts"]["steps"], 10); assert_eq!(value["counts"]["tool_calls"], 0); - assert_eq!(value["counts"]["events"], 0); + assert!(value["counts"].get("events").is_none()); assert!(value["source_errors"].as_array().unwrap().is_empty()); assert!(String::from_utf8(stderr)?.contains("counts_complete=true")); Ok(()) } -#[tokio::test] -async fn stats_health_and_reports_use_bounded_canonical_fallback() -> Result<()> { - let temp = tempfile::tempdir()?; - let storage = temp.path().join("capture"); - append_canonical_note(&storage).await?; - - let status = Cli::try_parse_from([ - "pchronicle", - "stats", - storage.to_str().unwrap(), - "--format", - "json", - "--errors", - "strict", - ])?; - let mut stdout = Vec::new(); - run(status, false, &mut stdout, &mut Vec::new()).await?; - let status: Value = serde_json::from_slice(&stdout)?; - assert_eq!(status["counts_complete"], true); - assert_eq!(status["counts"]["trajectories"], 1); - assert_eq!(status["counts"]["steps"], 1); - assert_eq!(status["counts"]["events"], 1); - - let analysis = Cli::try_parse_from([ - "pchronicle", - "stats", - "overview", - storage.to_str().unwrap(), - "--format", - "jsonl", - ])?; - let mut stdout = Vec::new(); - run(analysis, false, &mut stdout, &mut Vec::new()).await?; - let overview: Value = serde_json::from_slice(&stdout)?; - assert_eq!(overview["trajectories"], 1); - assert_eq!(overview["steps"], 1); - Ok(()) -} - -#[tokio::test] -async fn status_reports_projection_fresh_and_missing_in_source_order() -> Result<()> { - let temp = tempfile::tempdir()?; - let storage = temp.path().join("capture"); - let source_b = append_canonical_run(&storage, "b", 0).await?; - let source_a = append_canonical_run(&storage, "a", 0).await?; - let projection_a = storage.join("agent/a/storyline"); - persisting_pchronicle::storage::build_storyline_projection( - source_a.to_string_lossy(), - projection_a.to_string_lossy(), - "a/events.lance", - ) - .await?; - let projection_b = storage.join("agent/b/storyline"); - assert!(!projection_b.exists()); - - let cli = Cli::try_parse_from([ - "pchronicle", - "stats", - storage.join("agent").to_str().unwrap(), - "--format", - "json", - ])?; - let mut stdout = Vec::new(); - run(cli, false, &mut stdout, &mut Vec::new()).await?; - let response: Value = serde_json::from_slice(&stdout)?; - let projections = response["projections"].as_array().unwrap(); - assert_eq!(projections.len(), 2); - assert_eq!(projections[0]["source_path"], "a/events.lance"); - assert_eq!(projections[0]["projection_path"], "a/storyline"); - assert_eq!(projections[0]["status"], "fresh"); - assert_eq!(projections[0]["fact_version"], 1); - assert_eq!(projections[0]["fact_rows"], 1); - assert!(projections[0]["generation"].is_string()); - assert_eq!(projections[1]["source_path"], "b/events.lance"); - assert_eq!(projections[1]["projection_path"], "b/storyline"); - assert_eq!(projections[1]["status"], "missing"); - assert_eq!(projections[1]["fact_version"], 1); - assert_eq!(projections[1]["fact_rows"], 1); - assert!(projections[1].get("generation").is_none()); - assert_eq!(source_b.file_name().unwrap(), "events.lance"); - assert!(!projection_b.exists()); - - let cli = Cli::try_parse_from([ - "pchronicle", - "stats", - storage.join("agent").to_str().unwrap(), - "--format", - "table", - ])?; - let mut table = Vec::new(); - run(cli, true, &mut table, &mut Vec::new()).await?; - let table = String::from_utf8(table)?; - assert!(table.contains("PROJECTION")); - assert!(table.contains("a/events.lance -> a/storyline")); - assert!(table.contains("b/events.lance -> b/storyline")); - Ok(()) -} - -#[tokio::test] -async fn status_reports_projection_stale_and_safe_errors() -> Result<()> { - let temp = tempfile::tempdir()?; - let storage = temp.path().join("capture"); - for run_id in ["stale", "lineage-free", "malformed"] { - let source = append_canonical_run(&storage, run_id, 0).await?; - persisting_pchronicle::storage::build_storyline_projection( - source.to_string_lossy(), - storage - .join(format!("agent/{run_id}/storyline")) - .to_string_lossy(), - format!("{run_id}/events.lance"), - ) - .await?; - } - append_canonical_run(&storage, "stale", 1).await?; - - let lineage_free = storage.join("agent/lineage-free/storyline/CURRENT"); - let mut pointer: Value = serde_json::from_slice(&fs::read(&lineage_free)?)?; - let removed = pointer - .get_mut("committed") - .and_then(Value::as_object_mut) - .expect("CURRENT committed snapshot") - .remove("projection"); - assert!(removed.is_some(), "projection lineage fixture"); - fs::write(&lineage_free, serde_json::to_vec(&pointer)?)?; - let malformed = storage.join("agent/malformed/storyline/CURRENT"); - fs::write(&malformed, b"{broken")?; - let lineage_free_before = fs::read(&lineage_free)?; - let malformed_before = fs::read(&malformed)?; - - let cli = Cli::try_parse_from([ - "pchronicle", - "stats", - storage.join("agent").to_str().unwrap(), - "--format", - "json", - "--errors", - "report", - ])?; - let mut stdout = Vec::new(); - run(cli, false, &mut stdout, &mut Vec::new()).await?; - let response: Value = serde_json::from_slice(&stdout)?; - let projections = response["projections"].as_array().unwrap(); - assert_eq!(projections.len(), 3); - assert_eq!(projections[0]["source_path"], "lineage-free/events.lance"); - assert_eq!(projections[0]["status"], "error"); - assert!(projections[0].get("generation").is_none()); - assert!(projections[0].get("fact_version").is_none()); - assert!(projections[0].get("fact_rows").is_none()); - assert_eq!(projections[1]["source_path"], "malformed/events.lance"); - assert_eq!(projections[1]["status"], "error"); - assert_eq!(projections[2]["source_path"], "stale/events.lance"); - assert_eq!(projections[2]["status"], "stale"); - assert_eq!(fs::read(&lineage_free)?, lineage_free_before); - assert_eq!(fs::read(&malformed)?, malformed_before); - Ok(()) -} - #[tokio::test] async fn status_reports_partial_counts_for_bad_sources() -> Result<()> { let _tracing_guard = STATUS_REPORT_TRACING_LOCK.lock().await; @@ -2970,110 +2652,6 @@ async fn import_storyline_output_query_uses_squashed_source() -> Result<()> { Ok(()) } -#[tokio::test] -async fn canonical_event_import_auto_detects_and_is_create_only() -> Result<()> { - let temp = tempfile::tempdir()?; - let storage = temp.path().join("capture"); - append_canonical_note(&storage).await?; - let source = canonical_source(&storage)?; - let manifest = source.join("_manifest.json"); - let before = fs::read(&manifest)?; - let output = temp.path().join("storyline"); - - let cli = Cli::try_parse_from([ - "pchronicle", - "import", - "--from", - source.to_str().unwrap(), - "--output", - output.to_str().unwrap(), - ])?; - let mut stdout = Vec::new(); - let mut stderr = Vec::new(); - run(cli, false, &mut stdout, &mut stderr).await?; - - let response: Value = serde_json::from_slice(&stdout)?; - assert_eq!(response["format"], "events"); - assert_eq!(response["source_path"], "events.lance"); - assert_eq!(response["output_format"], "storyline-lance"); - assert_eq!(response["sources"], 1); - assert_eq!(response["trajectories"], 1); - assert_eq!(response["fact_rows"], 1); - assert!(response.get("input_bytes").is_none()); - assert_eq!(fs::read(&manifest)?, before); - assert!(output.join("CURRENT").is_file()); - let metadata = String::from_utf8(stderr)?; - assert!(metadata.contains("fact_rows=1")); - assert!(!metadata.contains("input_bytes=")); - - let query = Cli::try_parse_from([ - "pchronicle", - "query", - output.to_str().unwrap(), - "SELECT COUNT(*) AS runs FROM dataset.runs", - "--format", - "jsonl", - ])?; - let mut query_stdout = Vec::new(); - run(query, false, &mut query_stdout, &mut Vec::new()).await?; - let count: Value = serde_json::from_slice(&query_stdout)?; - assert_eq!(count["runs"], 1); - - let explicit = temp.path().join("explicit-storyline"); - let cli = Cli::try_parse_from([ - "pchronicle", - "import", - "--from", - source.to_str().unwrap(), - "--output", - explicit.to_str().unwrap(), - "--output-format", - "storyline", - ])?; - run(cli, false, &mut Vec::new(), &mut Vec::new()).await?; - assert!(explicit.join("CURRENT").is_file()); - - let preserved = temp.path().join("preserved"); - let cli = Cli::try_parse_from([ - "pchronicle", - "import", - "--from", - source.to_str().unwrap(), - "--output", - preserved.to_str().unwrap(), - "--output-format", - "preserve", - ])?; - let error = run(cli, false, &mut Vec::new(), &mut Vec::new()) - .await - .unwrap_err(); - assert!( - error - .to_string() - .contains("cannot preserve an existing canonical event Store") - ); - assert!(!preserved.exists()); - - let existing = temp.path().join("existing"); - fs::create_dir(&existing)?; - fs::write(existing.join("sentinel"), "keep")?; - let cli = Cli::try_parse_from([ - "pchronicle", - "import", - "--from", - source.to_str().unwrap(), - "--output", - existing.to_str().unwrap(), - ])?; - assert!( - run(cli, false, &mut Vec::new(), &mut Vec::new()) - .await - .is_err() - ); - assert_eq!(fs::read_to_string(existing.join("sentinel"))?, "keep"); - Ok(()) -} - #[tokio::test] async fn object_store_directory_import_recurses_json_files() -> Result<()> { let source = format!( @@ -3140,35 +2718,6 @@ async fn object_store_directory_import_recurses_json_files() -> Result<()> { Ok(()) } -#[tokio::test] -async fn canonical_event_import_supports_object_store_uris() -> Result<()> { - let temp = tempfile::tempdir()?; - let storage = temp.path().join("capture"); - append_canonical_note(&storage).await?; - let source = canonical_source(&storage)?; - let output = format!( - "shared-memory://pchronicle-canonical-import-{}/storyline", - uuid::Uuid::new_v4().simple() - ); - let cli = Cli::try_parse_from([ - "pchronicle", - "import", - "--from", - source.to_str().unwrap(), - "--output", - &output, - ])?; - let mut stdout = Vec::new(); - run(cli, false, &mut stdout, &mut Vec::new()).await?; - - let response: Value = serde_json::from_slice(&stdout)?; - assert_eq!(response["dataset_uri"], output); - assert_eq!(response["fact_rows"], 1); - let store = StorylineLanceStore::open_uri(&output).await?; - assert!(store.current_table_paths().await?.is_some()); - Ok(()) -} - #[tokio::test] async fn events_lance_suffix_without_manifest_remains_an_ordinary_directory_import() -> Result<()> { let temp = tempfile::tempdir()?; @@ -4643,17 +4192,8 @@ fn serve_args_with_storage(storage: Vec) -> ServeArgs { storage, positional_storage: Vec::new(), listen: None, - control: None, open: false, home_links: Vec::new(), - gateway: None, - gateway_config: None, - gateway_dataset: None, - gateway_split: None, - gateway_split_idle_seconds: 1800, - gateway_state: None, - gateway_stream_markdown: false, - debug: false, catalog_config: None, catalog_query_worker: false, } @@ -4811,31 +4351,6 @@ fn serve_storage_rejects_underivable_basename() { ); } -#[test] -fn serve_control_storage_uses_default_mount() -> Result<()> { - let config = resolve_serve_config(&serve_args_with_storage(vec![ - "default=./tmp".into(), - "evals=./data".into(), - ]))?; - assert_eq!(control_storage_uri(&config)?, "./tmp"); - Ok(()) -} - -#[test] -fn serve_control_storage_requires_default_mount() { - let config = resolve_serve_config(&serve_args_with_storage(vec![ - "./tmp".into(), - "./data".into(), - ])) - .unwrap(); - let error = control_storage_uri(&config).unwrap_err(); - let message = format!("{error:#}"); - assert!( - message.contains("default=DATASET"), - "unexpected error: {message}" - ); -} - #[test] fn warehouse_config_normalizes_mounts_and_selects_default() -> Result<()> { let temp = tempfile::tempdir()?; @@ -4954,128 +4469,6 @@ fn warehouse_config_rejects_unsafe_or_ambiguous_mounts() -> Result<()> { Ok(()) } -#[test] -fn serve_cli_requires_one_dataset_source_and_an_explicit_service() -> Result<()> { - for arguments in [ - vec![ - "pchronicle", - "serve", - "--storage", - "/tmp/data", - "--control", - "127.0.0.1:0", - ], - vec![ - "pchronicle", - "serve", - "--storage", - "/tmp/data", - "--listen", - "127.0.0.1:0", - ], - vec![ - "pchronicle", - "serve", - "--storage", - "/tmp/data", - "--gateway-config", - "gateway.toml", - "--gateway-dataset", - "/tmp/data", - ], - vec![ - "pchronicle", - "serve", - "--warehouse-config", - "warehouse.toml", - "--listen", - "127.0.0.1:0", - ], - vec![ - "pchronicle", - "serve", - "--storage", - "./tmp", - "--storage", - "./data", - "--listen", - "127.0.0.1:0", - ], - vec!["pchronicle", "serve", "--storage", "/tmp/data"], - vec![ - "pchronicle", - "serve", - "--warehouse-config", - "warehouse.toml", - ], - vec!["pchronicle", "serve", "s3://my-bucket/prefix"], - vec![ - "pchronicle", - "serve", - "s3://my-bucket/prefix", - "--listen", - "127.0.0.1:0", - ], - vec![ - "pchronicle", - "serve", - "s3://my-bucket/prefix", - "--control", - "127.0.0.1:0", - ], - vec![ - "pchronicle", - "serve", - "--storage", - "./tmp", - "s3://my-bucket/other", - "--listen", - "127.0.0.1:0", - ], - ] { - assert!( - Cli::try_parse_from(arguments.clone()).is_ok(), - "valid serve arguments rejected: {arguments:?}" - ); - } - - for arguments in [ - vec![ - "pchronicle", - "serve", - "--storage", - "a", - "--warehouse-config", - "b", - "--listen", - "127.0.0.1:0", - ], - vec![ - "pchronicle", - "serve", - "--warehouse-config", - "warehouse.toml", - "--control", - "127.0.0.1:0", - ], - vec![ - "pchronicle", - "serve", - "--storage", - "/tmp/data", - "--open", - "--control", - "127.0.0.1:0", - ], - ] { - assert!( - Cli::try_parse_from(arguments.clone()).is_err(), - "invalid serve arguments accepted: {arguments:?}" - ); - } - Ok(()) -} - #[test] fn serve_positional_uri_is_equivalent_to_storage() -> Result<()> { let cli = Cli::try_parse_from(["pchronicle", "serve", "s3://my-bucket/prefix"])?; @@ -5159,308 +4552,6 @@ uri = "{}" Ok(()) } -#[test] -fn serve_without_listen_defaults_warehouse_to_loopback_ephemeral_port() -> Result<()> { - let cli = Cli::try_parse_from(["pchronicle", "serve", "--storage", "/tmp/data"])?; - let Command::Serve(args) = cli.command else { - unreachable!("serve command parsed as another variant") - }; - assert_eq!(args.listen, None); - assert_eq!( - warehouse_listen(&args), - Some("127.0.0.1:0".parse::()?) - ); - - let control_only = Cli::try_parse_from([ - "pchronicle", - "serve", - "--storage", - "/tmp/data", - "--control", - "127.0.0.1:0", - ])?; - let Command::Serve(control_only) = control_only.command else { - unreachable!("serve command parsed as another variant") - }; - assert_eq!(warehouse_listen(&control_only), None); - Ok(()) -} - -#[test] -fn echo_cli_uses_a_normal_loopback_default() -> Result<()> { - let cli = Cli::try_parse_from(["pchronicle", "echo"])?; - let Command::Echo(args) = cli.command else { - unreachable!("echo command parsed as another variant") - }; - assert_eq!(args.listen, "127.0.0.1:19080".parse::()?); - assert_eq!(args.encoding, EchoEncoding::Plain); - Ok(()) -} - -#[test] -fn serve_gateway_options_are_explicit_and_scoped() -> Result<()> { - let ingest = Cli::try_parse_from([ - "pchronicle", - "serve", - "--gateway", - "auto", - "--gateway-dataset", - "/tmp/captures", - "--gateway-split", - "{user}/{date}/{hour}", - ])?; - let Command::Serve(ingest) = ingest.command else { - unreachable!("serve command parsed as another variant") - }; - assert_eq!(ingest.gateway, Some("127.0.0.1:0".parse::()?)); - assert_eq!(ingest.gateway_config, None); - assert_eq!( - ingest.gateway_split.as_deref(), - Some("{user}/{date}/{hour}") - ); - assert_eq!(ingest.gateway_split_idle_seconds, 30 * 60); - - let idle_override = Cli::try_parse_from([ - "pchronicle", - "serve", - "--gateway", - "auto", - "--gateway-dataset", - "/tmp/captures", - "--gateway-split-idle", - "45s", - ])?; - let Command::Serve(idle_override) = idle_override.command else { - unreachable!("serve command parsed as another variant") - }; - assert_eq!(idle_override.gateway_split_idle_seconds, 45); - - let cli = Cli::try_parse_from([ - "pchronicle", - "serve", - "--warehouse-config", - "warehouse.toml", - "--gateway-config", - "gateway.toml", - "--gateway-dataset", - "s3://capture-bucket/events", - "--gateway-state", - ".gateway-state", - "--gateway-stream-markdown", - "--gateway-debug", - ])?; - let Command::Serve(args) = cli.command else { - unreachable!("serve command parsed as another variant") - }; - assert_eq!(args.gateway, None); - assert_eq!(args.gateway_config, Some(PathBuf::from("gateway.toml"))); - assert_eq!( - args.gateway_dataset.as_deref(), - Some("s3://capture-bucket/events") - ); - assert_eq!(args.gateway_state, Some(PathBuf::from(".gateway-state"))); - assert!(args.gateway_stream_markdown); - assert!(args.debug); - - assert!( - Cli::try_parse_from([ - "pchronicle", - "serve", - "--warehouse-config", - "warehouse.toml", - "--gateway-dataset", - "captures", - ]) - .is_err() - ); - assert!( - Cli::try_parse_from([ - "pchronicle", - "serve", - "--warehouse-config", - "warehouse.toml", - "--gateway-debug", - ]) - .is_err() - ); - - let cli = Cli::try_parse_from([ - "pchronicle", - "serve", - "--warehouse-config", - "warehouse.toml", - "--gateway-config", - "gateway.toml", - "--gateway-dataset", - "/tmp/captures", - "--gateway-debug", - ])?; - let Command::Serve(args) = cli.command else { - unreachable!("serve command parsed as another variant") - }; - assert!(args.debug); - Ok(()) -} - -#[test] -fn gateway_dataset_uri_is_auto_mounted_and_deduplicated() -> Result<()> { - let mut config = - server::ChronicleServerConfig::mounted(vec![DatasetMount::new("evals", "/tmp/evals")?])?; - ensure_gateway_mount(&mut config, "/tmp/captures".into())?; - assert_eq!(config.datasets.len(), 2); - assert_eq!(config.datasets[1].name, "gateway"); - assert_eq!(config.datasets[1].uri, "/tmp/captures"); - ensure_gateway_mount(&mut config, "/tmp/captures".into())?; - assert_eq!(config.datasets.len(), 2); - Ok(()) -} - -#[test] -fn embedded_gateway_accepts_public_listeners() { - assert!(parse_gateway_listener("0.0.0.0:8787", "Gateway").is_ok()); - assert!(parse_gateway_listener("127.0.0.1:0", "Gateway").is_ok()); - assert!(parse_gateway_bind("0.0.0.0:0").is_ok()); - assert_eq!( - parse_gateway_bind("auto").unwrap(), - "127.0.0.1:0".parse::().unwrap() - ); -} - -#[tokio::test] -async fn embedded_gateway_forwards_and_persists_canonical_events() -> Result<()> { - let dataset = tempfile::tempdir()?; - let state = tempfile::tempdir()?; - let upstream_listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; - let upstream_addr = upstream_listener.local_addr()?; - let (upstream_stop_tx, upstream_stop_rx) = tokio::sync::oneshot::channel::<()>(); - let upstream = tokio::spawn(async move { - let app = axum::Router::new().route( - "/v1/chat/completions", - axum::routing::post(|| async { - axum::Json(serde_json::json!({ - "id": "chatcmpl-pchronicle", - "object": "chat.completion", - "choices": [{ - "index": 0, - "message": {"role": "assistant", "content": "stored"}, - "finish_reason": "stop" - }] - })) - }), - ); - axum::serve(upstream_listener, app) - .with_graceful_shutdown(async { - let _ = upstream_stop_rx.await; - }) - .await - }); - - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; - let gateway_addr = listener.local_addr()?; - let admin_listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; - let admin_addr = admin_listener.local_addr()?; - let mut config = persisting_gateway::config::ProxyConfig::from_toml_str(&format!( - r#" -listen = "127.0.0.1:0" -admin_listen = "127.0.0.1:0" -agent_id = "test-agent" - -[[models]] -name = "*" -upstream = "http://{upstream_addr}/v1" -"# - ))?; - config.listen = gateway_addr.to_string(); - config.admin_listen = admin_addr.to_string(); - let (sink, writer) = - gateway_capture::gateway_capture_sink(&dataset.path().to_string_lossy(), &config.agent_id)?; - let warehouse_listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; - let warehouse_addr = warehouse_listener.local_addr()?; - let warehouse_config = server::ChronicleServerConfig::mounted(vec![DatasetMount::default( - dataset.path().to_string_lossy(), - )?])?; - let prepared_gateway = PreparedGateway::Proxy(Box::new(PreparedProxyGateway { - config, - state_dir: state.path().to_path_buf(), - dataset_uri: dataset.path().to_string_lossy().into_owned(), - split: None, - stream_markdown: false, - listener, - admin_listener, - sink, - writer, - })); - let (serve_stop_tx, serve_stop_rx) = tokio::sync::oneshot::channel::<()>(); - let serve = tokio::spawn(async move { - serve_warehouse_and_gateway( - warehouse_config, - warehouse_listener, - prepared_gateway, - async { - let _ = serve_stop_rx.await; - }, - ) - .await - }); - - let client = reqwest::Client::builder() - .no_proxy() - .timeout(Duration::from_secs(10)) - .build()?; - let health = client - .get(format!("http://{warehouse_addr}/api/health")) - .send() - .await?; - assert_eq!(health.status(), reqwest::StatusCode::OK); - assert_eq!(health.json::().await?["mode"], "read_only"); - let response = client - .post(format!("http://{gateway_addr}/v1/chat/completions")) - .header("content-type", "application/json") - .header("x-persisting-session-id", "session-42") - .body(r#"{"model":"test","messages":[{"role":"user","content":"keep me"}]}"#) - .send() - .await?; - assert_eq!(response.status(), reqwest::StatusCode::OK); - assert_eq!( - response.json::().await?["choices"][0]["message"]["content"], - "stored" - ); - - let _ = serve_stop_tx.send(()); - tokio::time::timeout(Duration::from_secs(10), serve) - .await - .context("pChronicle serve shutdown timed out")???; - let _ = upstream_stop_tx.send(()); - tokio::time::timeout(Duration::from_secs(10), upstream) - .await - .context("mock upstream shutdown timed out")???; - - let snapshot = Arc::new( - DatasetCatalogSnapshot::discover( - vec![DatasetMount::default(dataset.path().to_string_lossy())?], - Some(DEFAULT_DATASET_NAME.into()), - CatalogSnapshotOptions::default(), - ) - .await?, - ); - let engine = snapshot.query_engine(Default::default()).await?; - let rows = engine - .query_jsonl( - "SELECT kind, COUNT(*) AS count FROM dataset.events GROUP BY kind ORDER BY kind", - ) - .await?; - let rows = rows - .lines() - .map(serde_json::from_str::) - .collect::, _>>()?; - assert_eq!(rows.len(), 2, "{rows:?}"); - assert_eq!(rows[0]["kind"], "llm.request"); - assert_eq!(rows[0]["count"], 1); - assert_eq!(rows[1]["kind"], "llm.response"); - assert_eq!(rows[1]["count"], 1); - Ok(()) -} - #[test] fn byte_units_are_stable() { assert_eq!(format_bytes(12), "12 B"); diff --git a/crates/persisting-pchronicle-cli/tests/binary_contract.rs b/crates/persisting-pchronicle-cli/tests/binary_contract.rs index e7e6a2b6e..cf2d7cd2f 100644 --- a/crates/persisting-pchronicle-cli/tests/binary_contract.rs +++ b/crates/persisting-pchronicle-cli/tests/binary_contract.rs @@ -106,20 +106,7 @@ fn serve_help_exposes_only_the_canonical_dataset_surface() -> Result<()> { ); assert!(stdout.contains("pchronicle serve catalog"), "{stdout}"); assert!(stdout.contains("catalog"), "{stdout}"); - for option in [ - "--listen", - "--control", - "--open", - "--home-link", - "--gateway", - "--gateway-config", - "--gateway-dataset", - "--gateway-split", - "--gateway-state", - "--gateway-stream-markdown", - "--gateway-debug", - "--catalog-config", - ] { + for option in ["--listen", "--open", "--home-link", "--catalog-config"] { assert!( stdout.contains(option), "serve help omits {option}: {stdout}" @@ -400,75 +387,6 @@ SELECT 1 Ok(()) } -#[tokio::test] -async fn canonical_event_import_is_queryable_in_release() -> Result<()> { - let temp = tempfile::tempdir()?; - let storage = temp.path().join("capture"); - let coords = persisting_pchronicle::storage::StoryCoords::new( - storage.to_string_lossy(), - "agent", - "run", - Some("run".into()), - ); - persisting_pchronicle::storage::RawEventLanceStore - .append_events( - &coords, - &[persisting_pchronicle::model::EventRecord { - identity: Default::default(), - seq: 0, - source: "test".into(), - kind: "note".into(), - timestamp: None, - session_id: Some("run".into()), - agent_id: Some("agent".into()), - parent_uuid: None, - trace_id: None, - call_id: None, - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload: serde_json::json!({"content":"release-smoke"}), - }], - ) - .await?; - let source = persisting_pchronicle::storage::raw_event_lance_path(&coords)?; - let output = temp.path().join("storyline"); - let imported = pchronicle(&[ - "import", - "--from", - source.to_str().unwrap(), - "--output", - output.to_str().unwrap(), - ])?; - assert!( - imported.status.success(), - "{}", - String::from_utf8_lossy(&imported.stderr) - ); - let response: Value = serde_json::from_slice(&imported.stdout)?; - assert_eq!(response["format"], "events"); - assert_eq!(response["output_format"], "storyline-lance"); - assert_eq!(response["fact_rows"], 1); - assert!(response.get("input_bytes").is_none()); - - let queried = pchronicle(&[ - "query", - output.to_str().unwrap(), - "SELECT COUNT(*) AS runs FROM dataset.runs", - "--format", - "jsonl", - ])?; - assert!( - queried.status.success(), - "{}", - String::from_utf8_lossy(&queried.stderr) - ); - let row: Value = serde_json::from_slice(&queried.stdout)?; - assert_eq!(row["runs"], 1); - Ok(()) -} - #[test] fn default_pin_is_persistent_across_cli_processes() -> Result<()> { let temp = tempfile::tempdir()?; @@ -1045,3 +963,17 @@ fn skill_bash_blocks(skill: &str) -> Result> { fn skill_example_needs_storyline(block: &str) -> bool { block.split_whitespace().any(|token| token == "find") } + +#[test] +fn removed_capture_commands_and_options_are_rejected() -> Result<()> { + for args in [ + vec!["echo"], + vec!["dev", "echo"], + vec!["serve", "data", "--gateway", "auto"], + vec!["serve", "data", "--control", "127.0.0.1:0"], + ] { + let output = pchronicle(&args)?; + assert!(!output.status.success(), "accepted removed CLI: {args:?}"); + } + Ok(()) +} diff --git a/crates/persisting-pchronicle-cli/tests/command_matrix.rs b/crates/persisting-pchronicle-cli/tests/command_matrix.rs index d4d4880a6..250047442 100644 --- a/crates/persisting-pchronicle-cli/tests/command_matrix.rs +++ b/crates/persisting-pchronicle-cli/tests/command_matrix.rs @@ -70,7 +70,6 @@ async fn catalog_command_matrix_reports_every_supported_format() -> Result<()> { "trajectories": fixture.trajectories, "steps": fixture.steps, "tool_calls": fixture.tool_calls, - "events": 0, }), "{fixture:?}" ); diff --git a/crates/persisting-pchronicle-cli/tests/control_process.rs b/crates/persisting-pchronicle-cli/tests/control_process.rs deleted file mode 100644 index 4faac0b05..000000000 --- a/crates/persisting-pchronicle-cli/tests/control_process.rs +++ /dev/null @@ -1,701 +0,0 @@ -use anyhow::{Context, Result}; -use persisting_agentctl::{AttemptId, RunCommitRequest, RunId, RunState}; -use persisting_events::{ - CHRONICLE_CONTROL_VERSION, CHRONICLE_SERVE_READY_VERSION, ChronicleControl, - ChronicleControlEnvelope, ChronicleControlRequest, ChronicleControlResponse, - ChronicleControlResponseEnvelope, ChronicleServeProcessClient, ChronicleServeReady, - CommitRunOutcome, LeaseAcquireOutcome, TrajectoryAppendRequest, -}; -use persisting_pchronicle::storage::{ - AutomaticProjectionState, AutomaticProjectionTarget, RawEventLanceStore, StoryCoords, - build_storyline_projection, inspect_automatic_storyline_projection, - probe_canonical_event_store, -}; -use std::future::Future; -use std::process::Stdio; -use std::time::Duration; -use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader}; -use tokio::net::TcpStream; -use tokio::process::Command; - -/// Serve prints readiness only after automatic Storyline projection converges, -/// which builds FTS indexes. Ten seconds is enough locally, but Linux CI -/// shards run hundreds of tests in parallel and the same projection can exceed -/// that budget. -const SERVE_PROJECTION_TIMEOUT: Duration = Duration::from_secs(60); -/// Per-poll HTTP bound. Catalog rebuild shares the serve Tokio runtime with -/// Warehouse HTTP, so a single GET can exceed this under Linux CI load. -/// Callers must treat a timeout as "not yet" and keep waiting until -/// [`SERVE_PROJECTION_TIMEOUT`]. -const CATALOG_POLL_TIMEOUT: Duration = Duration::from_secs(5); - -async fn wait_until(timeout: Duration, mut condition: F) -> Result<()> -where - F: FnMut() -> Fut, - Fut: Future>, -{ - tokio::time::timeout(timeout, async { - loop { - if condition().await? { - return Ok::<(), anyhow::Error>(()); - } - tokio::time::sleep(Duration::from_millis(25)).await; - } - }) - .await - .context("timed out waiting for pChronicle state")??; - Ok(()) -} - -async fn catalog_snapshot_id(client: &reqwest::Client, url: &str) -> Option { - let catalog = client - .get(url) - .send() - .await - .ok()? - .json::() - .await - .ok()?; - catalog["snapshot_id"].as_str().map(str::to_owned) -} - -async fn read_serve_ready(stdout: &mut BufReader) -> Result -where - R: tokio::io::AsyncRead + Unpin, -{ - let mut ready_line = String::new(); - let bytes = tokio::time::timeout(SERVE_PROJECTION_TIMEOUT, stdout.read_line(&mut ready_line)) - .await - .context("serve readiness timeout")??; - anyhow::ensure!(bytes > 0, "serve exited before writing readiness"); - serde_json::from_str(&ready_line) - .with_context(|| format!("decode serve readiness: {ready_line}")) -} - -async fn append_note( - storage: &std::path::Path, - run_id: &str, - seq: u64, -) -> Result { - let coords = StoryCoords::new( - storage.to_string_lossy(), - "agent", - run_id, - Some(run_id.into()), - ); - RawEventLanceStore - .append_events( - &coords, - &[persisting_pchronicle::model::EventRecord { - identity: Default::default(), - seq, - source: "test".into(), - kind: "note".into(), - timestamp: None, - session_id: Some(run_id.into()), - agent_id: Some("agent".into()), - parent_uuid: None, - trace_id: None, - call_id: None, - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload: serde_json::json!({"content": format!("{run_id}-{seq}")}), - }], - ) - .await?; - persisting_pchronicle::storage::raw_event_lance_path(&coords) -} - -async fn target_for(source: &std::path::Path) -> Result { - let snapshot = probe_canonical_event_store(source.to_string_lossy()) - .await? - .context("canonical test source")?; - let run_id = source - .parent() - .and_then(std::path::Path::file_name) - .and_then(|name| name.to_str()) - .context("canonical run path")?; - Ok(AutomaticProjectionTarget { - dataset: "default".into(), - source_path: format!("agent/{run_id}/events.lance"), - source_uri: snapshot.source_uri.clone(), - projection_path: format!("agent/{run_id}/storyline"), - projection_uri: source - .parent() - .context("canonical source parent")? - .join("storyline") - .to_string_lossy() - .into_owned(), - source_snapshot: snapshot, - }) -} - -async fn control_call( - ready: &persisting_events::ChronicleServeControlReady, - request_id: u64, - request: ChronicleControlRequest, -) -> Result { - let mut stream = TcpStream::connect(&ready.endpoint).await?; - let request = ChronicleControlEnvelope { - version: CHRONICLE_CONTROL_VERSION, - request_id, - auth_token: ready.auth_token.clone(), - request, - }; - let mut encoded = serde_json::to_vec(&request)?; - encoded.push(b'\n'); - stream.write_all(&encoded).await?; - stream.flush().await?; - let mut response_line = String::new(); - BufReader::new(stream).read_line(&mut response_line).await?; - let response: ChronicleControlResponseEnvelope = serde_json::from_str(&response_line)?; - anyhow::ensure!( - response.request_id == request_id, - "control response id mismatch" - ); - Ok(response.response) -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn serve_control_only_advertises_no_warehouse_and_accepts_ping() -> Result<()> { - let parent = tempfile::tempdir()?; - let root = parent.path().join("new-storage"); - assert!(!root.exists()); - let mut child = Command::new(env!("CARGO_BIN_EXE_pchronicle")) - .arg("serve") - .arg("--storage") - .arg(&root) - .arg("--control") - .arg("127.0.0.1:0") - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .kill_on_drop(true) - .spawn()?; - let mut stdout = BufReader::new(child.stdout.take().unwrap()); - let ready = read_serve_ready(&mut stdout).await?; - assert_eq!(ready.version, CHRONICLE_SERVE_READY_VERSION); - assert!(ready.warehouse_endpoint.is_none()); - assert!(ready.gateway_endpoint.is_none()); - assert!(ready.gateway_admin_endpoint.is_none()); - assert!(root.is_dir()); - let control = ready.control.unwrap(); - - let mut stream = TcpStream::connect(&control.endpoint).await?; - let request = ChronicleControlEnvelope { - version: CHRONICLE_CONTROL_VERSION, - request_id: 11, - auth_token: control.auth_token.clone(), - request: ChronicleControlRequest::Ping, - }; - let mut encoded = serde_json::to_vec(&request)?; - encoded.push(b'\n'); - stream.write_all(&encoded).await?; - stream.flush().await?; - let mut response_line = String::new(); - BufReader::new(stream).read_line(&mut response_line).await?; - let response: ChronicleControlResponseEnvelope = serde_json::from_str(&response_line)?; - assert_eq!(response.request_id, 11); - assert!(matches!(response.response, ChronicleControlResponse::Pong)); - - let mut stderr = child.stderr.take().unwrap(); - child.kill().await?; - let mut diagnostics = String::new(); - stderr.read_to_string(&mut diagnostics).await?; - assert!(!diagnostics.contains(&control.auth_token)); - Ok(()) -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn serve_ingest_gateway_only_advertises_no_warehouse_or_control() -> Result<()> { - let root = tempfile::tempdir()?; - let dataset = root.path().join("captures"); - let mut child = Command::new(env!("CARGO_BIN_EXE_pchronicle")) - .arg("serve") - .arg("--gateway") - .arg("auto") - .arg("--gateway-dataset") - .arg(&dataset) - .arg("--gateway-split") - .arg("{user}/{date}/{hour}") - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .kill_on_drop(true) - .spawn()?; - let ready = read_serve_ready(&mut BufReader::new(child.stdout.take().unwrap())).await?; - assert!(ready.warehouse_endpoint.is_none()); - assert!(ready.control.is_none()); - assert!(ready.gateway_endpoint.is_some()); - assert!(ready.gateway_admin_endpoint.is_none()); - - child.kill().await?; - Ok(()) -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn serve_can_host_warehouse_and_control_together() -> Result<()> { - let root = tempfile::tempdir()?; - let mut child = Command::new(env!("CARGO_BIN_EXE_pchronicle")) - .arg("serve") - .arg("--storage") - .arg(root.path()) - .arg("--listen") - .arg("127.0.0.1:0") - .arg("--control") - .arg("127.0.0.1:0") - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .kill_on_drop(true) - .spawn()?; - let ready = read_serve_ready(&mut BufReader::new(child.stdout.take().unwrap())).await?; - assert!(ready.warehouse_endpoint.is_some()); - assert!(ready.gateway_endpoint.is_none()); - assert!(ready.gateway_admin_endpoint.is_none()); - let control = ready.control.unwrap(); - - let mut stream = TcpStream::connect(&control.endpoint).await?; - let request = ChronicleControlEnvelope { - version: CHRONICLE_CONTROL_VERSION, - request_id: 12, - auth_token: control.auth_token, - request: ChronicleControlRequest::Ping, - }; - let mut encoded = serde_json::to_vec(&request)?; - encoded.push(b'\n'); - stream.write_all(&encoded).await?; - stream.flush().await?; - let mut response_line = String::new(); - BufReader::new(stream).read_line(&mut response_line).await?; - let response: ChronicleControlResponseEnvelope = serde_json::from_str(&response_line)?; - assert_eq!(response.request_id, 12); - assert!(matches!(response.response, ChronicleControlResponse::Pong)); - - child.kill().await?; - Ok(()) -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn control_process_owns_run_state_and_trajectory_append() -> Result<()> { - let root = tempfile::tempdir()?; - let client = ChronicleServeProcessClient::spawn( - env!("CARGO_BIN_EXE_pchronicle"), - root.path().to_string_lossy(), - ) - .await?; - let run_id = RunId::new("run-control-process"); - let lease = client - .acquire_lease(&run_id, Some("task-1"), "pilot-1", 30_000) - .await?; - let LeaseAcquireOutcome::Acquired(lease) = lease else { - panic!("first lease must be acquired") - }; - let attempt_id = AttemptId::new("attempt-1"); - assert!( - client - .bind_attempt(&run_id, lease.epoch, attempt_id.clone()) - .await? - ); - assert!( - client - .publish_attempt_active(run_id.as_str(), attempt_id.as_str(), lease.epoch, 30_000) - .await? - ); - assert!(client.get_attempt(run_id.as_str()).await?.is_some()); - - let committed = client - .commit_run(RunCommitRequest { - run_id: run_id.clone(), - task_id: Some("task-1".into()), - attempt_id, - lease_epoch: lease.epoch, - state: RunState::Completed, - event_high_watermark: None, - result_digest: "sha256:test".into(), - }) - .await?; - assert!(matches!(committed, CommitRunOutcome::Committed(_))); - assert!(client.get_run(&run_id).await?.unwrap().commit.is_some()); - assert_eq!(client.list_runs().await?.len(), 1); - - let appended = client - .append_trajectory(TrajectoryAppendRequest { - storage: root.path().join("traj").display().to_string(), - agent_id: "ppilot".into(), - session_id: "session-1".into(), - format: Default::default(), - root_session_id: None, - records: Vec::new(), - }) - .await?; - assert_eq!(appended.accepted_records, 0); - assert_eq!(appended.status, "ok"); - Ok(()) -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn serve_readiness_waits_for_projection_and_runtime_discovers_control_appends() -> Result<()> -{ - let root = tempfile::tempdir()?; - let initial_source = append_note(root.path(), "initial", 0).await?; - let mut child = Command::new(env!("CARGO_BIN_EXE_pchronicle")) - .arg("serve") - .arg("--storage") - .arg(root.path()) - .arg("--control") - .arg("127.0.0.1:0") - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .kill_on_drop(true) - .spawn()?; - let mut stdout = BufReader::new(child.stdout.take().unwrap()); - let ready = read_serve_ready(&mut stdout).await?; - let initial = target_for(&initial_source).await?; - assert_eq!( - inspect_automatic_storyline_projection(&initial) - .await? - .state, - AutomaticProjectionState::Fresh - ); - let control = ready.control.context("serve readiness omitted Control")?; - - let runtime_request = TrajectoryAppendRequest { - storage: root.path().to_string_lossy().into_owned(), - agent_id: "agent".into(), - session_id: "runtime".into(), - format: Default::default(), - root_session_id: Some("runtime".into()), - records: vec![persisting_events::EventRecord { - identity: Default::default(), - seq: 0, - source: "test".into(), - kind: "note".into(), - timestamp: None, - session_id: Some("runtime".into()), - agent_id: Some("agent".into()), - parent_uuid: None, - trace_id: None, - call_id: None, - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload: serde_json::json!({"content":"runtime"}), - }], - }; - assert!(matches!( - control_call( - &control, - 21, - ChronicleControlRequest::AppendTrajectory(runtime_request) - ) - .await?, - ChronicleControlResponse::TrajectoryAppend(_) - )); - let runtime_source = root.path().join("agent/runtime/events.lance"); - wait_until(SERVE_PROJECTION_TIMEOUT, || { - let runtime_source = runtime_source.clone(); - async move { - let Ok(target) = target_for(&runtime_source).await else { - return Ok(false); - }; - Ok(inspect_automatic_storyline_projection(&target) - .await - .is_ok_and(|inspection| inspection.state == AutomaticProjectionState::Fresh)) - } - }) - .await?; - assert!(matches!( - control_call(&control, 22, ChronicleControlRequest::Ping).await?, - ChronicleControlResponse::Pong - )); - - let mut stderr = child.stderr.take().unwrap(); - child.kill().await?; - let mut diagnostics = String::new(); - stderr.read_to_string(&mut diagnostics).await?; - assert!(!diagnostics.contains(&control.auth_token)); - Ok(()) -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn serve_startup_foreign_destination_exits_without_readiness_or_mutation() -> Result<()> { - let root = tempfile::tempdir()?; - let source_a = append_note(root.path(), "a", 0).await?; - append_note(root.path(), "b", 0).await?; - let projection_b = root.path().join("agent/b/storyline"); - build_storyline_projection( - source_a.to_string_lossy(), - projection_b.to_string_lossy(), - "agent/a/events.lance", - ) - .await?; - let before = std::fs::read(projection_b.join("CURRENT"))?; - let mut child = Command::new(env!("CARGO_BIN_EXE_pchronicle")) - .arg("serve") - .arg("--storage") - .arg(root.path()) - .arg("--control") - .arg("127.0.0.1:0") - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .kill_on_drop(true) - .spawn()?; - let mut ready_line = String::new(); - let bytes = tokio::time::timeout( - SERVE_PROJECTION_TIMEOUT, - BufReader::new(child.stdout.take().unwrap()).read_line(&mut ready_line), - ) - .await - .context("foreign startup did not exit")??; - assert_eq!(bytes, 0); - assert!(ready_line.is_empty()); - assert!(!child.wait().await?.success()); - assert_eq!(std::fs::read(projection_b.join("CURRENT"))?, before); - Ok(()) -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn two_serve_processes_accept_one_fresh_projection_winner() -> Result<()> { - let root = tempfile::tempdir()?; - let source = append_note(root.path(), "shared", 0).await?; - let mut children = Vec::new(); - for _ in 0..2 { - let child = Command::new(env!("CARGO_BIN_EXE_pchronicle")) - .arg("serve") - .arg("--storage") - .arg(root.path()) - .arg("--control") - .arg("127.0.0.1:0") - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .kill_on_drop(true) - .spawn()?; - children.push(child); - } - for child in &mut children { - let ready = read_serve_ready(&mut BufReader::new(child.stdout.take().unwrap())).await?; - assert!(ready.control.is_some()); - } - let target = target_for(&source).await?; - assert_eq!( - inspect_automatic_storyline_projection(&target).await?.state, - AutomaticProjectionState::Fresh - ); - for child in &mut children { - child.kill().await?; - } - Ok(()) -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn runtime_projection_failure_does_not_stop_control() -> Result<()> { - let root = tempfile::tempdir()?; - let owner_source = append_note(root.path(), "owner", 0).await?; - let mut child = Command::new(env!("CARGO_BIN_EXE_pchronicle")) - .arg("serve") - .arg("--storage") - .arg(root.path()) - .arg("--control") - .arg("127.0.0.1:0") - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .kill_on_drop(true) - .spawn()?; - let ready = read_serve_ready(&mut BufReader::new(child.stdout.take().unwrap())).await?; - let control = ready.control.context("serve readiness omitted Control")?; - - let foreign = root.path().join("agent/bad/storyline"); - build_storyline_projection( - owner_source.to_string_lossy(), - foreign.to_string_lossy(), - "agent/owner/events.lance", - ) - .await?; - let before = std::fs::read(foreign.join("CURRENT"))?; - let append = |run_id: &str| TrajectoryAppendRequest { - storage: root.path().to_string_lossy().into_owned(), - agent_id: "agent".into(), - session_id: run_id.into(), - format: Default::default(), - root_session_id: Some(run_id.into()), - records: vec![persisting_events::EventRecord { - identity: Default::default(), - seq: 0, - source: "test".into(), - kind: "note".into(), - timestamp: None, - session_id: Some(run_id.into()), - agent_id: Some("agent".into()), - parent_uuid: None, - trace_id: None, - call_id: None, - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload: serde_json::json!({"content": run_id}), - }], - }; - assert!(matches!( - control_call( - &control, - 31, - ChronicleControlRequest::AppendTrajectory(append("bad")) - ) - .await?, - ChronicleControlResponse::TrajectoryAppend(_) - )); - - let mut stderr = BufReader::new(child.stderr.take().unwrap()).lines(); - tokio::time::timeout(SERVE_PROJECTION_TIMEOUT, async { - while let Some(line) = stderr.next_line().await? { - if line.contains("projection source=agent/bad/events.lance") { - return Ok::<(), anyhow::Error>(()); - } - } - anyhow::bail!("serve stderr closed before projection diagnostic") - }) - .await - .context("runtime projection diagnostic timeout")??; - assert_eq!(std::fs::read(foreign.join("CURRENT"))?, before); - assert!(matches!( - control_call(&control, 32, ChronicleControlRequest::Ping).await?, - ChronicleControlResponse::Pong - )); - assert!(matches!( - control_call( - &control, - 33, - ChronicleControlRequest::AppendTrajectory(append("healthy")) - ) - .await?, - ChronicleControlResponse::TrajectoryAppend(_) - )); - let healthy = root.path().join("agent/healthy/events.lance"); - wait_until(SERVE_PROJECTION_TIMEOUT, || { - let healthy = healthy.clone(); - async move { - let Ok(target) = target_for(&healthy).await else { - return Ok(false); - }; - Ok(inspect_automatic_storyline_projection(&target) - .await - .is_ok_and(|inspection| inspection.state == AutomaticProjectionState::Fresh)) - } - }) - .await?; - child.kill().await?; - Ok(()) -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn warehouse_catalog_refreshes_after_control_append_projection() -> Result<()> { - let root = tempfile::tempdir()?; - append_note(root.path(), "initial", 0).await?; - let mut child = Command::new(env!("CARGO_BIN_EXE_pchronicle")) - .arg("serve") - .arg("--storage") - .arg(root.path()) - .arg("--listen") - .arg("127.0.0.1:0") - .arg("--control") - .arg("127.0.0.1:0") - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .kill_on_drop(true) - .spawn()?; - let ready = read_serve_ready(&mut BufReader::new(child.stdout.take().unwrap())).await?; - let control = ready.control.context("serve readiness omitted Control")?; - let warehouse = ready - .warehouse_endpoint - .context("serve readiness omitted Warehouse")?; - let client = reqwest::Client::builder() - .no_proxy() - .timeout(CATALOG_POLL_TIMEOUT) - .build()?; - let catalog_url = format!("http://{warehouse}/api/catalog"); - let initial_snapshot = tokio::time::timeout(SERVE_PROJECTION_TIMEOUT, async { - loop { - if let Some(snapshot) = catalog_snapshot_id(&client, &catalog_url).await { - break snapshot; - } - tokio::time::sleep(Duration::from_millis(25)).await; - } - }) - .await - .context("timed out waiting for initial catalog")?; - - let request = TrajectoryAppendRequest { - storage: root.path().to_string_lossy().into_owned(), - agent_id: "agent".into(), - session_id: "runtime".into(), - format: Default::default(), - root_session_id: Some("runtime".into()), - records: vec![persisting_events::EventRecord { - identity: Default::default(), - seq: 0, - source: "test".into(), - kind: "note".into(), - timestamp: None, - session_id: Some("runtime".into()), - agent_id: Some("agent".into()), - parent_uuid: None, - trace_id: None, - call_id: None, - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload: serde_json::json!({"content":"runtime"}), - }], - }; - assert!(matches!( - control_call( - &control, - 41, - ChronicleControlRequest::AppendTrajectory(request) - ) - .await?, - ChronicleControlResponse::TrajectoryAppend(_) - )); - let runtime_source = root.path().join("agent/runtime/events.lance"); - wait_until(SERVE_PROJECTION_TIMEOUT, || { - let client = client.clone(); - let catalog_url = catalog_url.clone(); - let initial_snapshot = initial_snapshot.clone(); - let runtime_source = runtime_source.clone(); - async move { - let Ok(target) = target_for(&runtime_source).await else { - return Ok(false); - }; - if !inspect_automatic_storyline_projection(&target) - .await - .is_ok_and(|inspection| inspection.state == AutomaticProjectionState::Fresh) - { - return Ok(false); - } - Ok(catalog_snapshot_id(&client, &catalog_url) - .await - .is_some_and(|snapshot| snapshot != initial_snapshot)) - } - }) - .await?; - let runtime = target_for(&runtime_source).await?; - assert_eq!( - inspect_automatic_storyline_projection(&runtime) - .await? - .state, - AutomaticProjectionState::Fresh - ); - child.kill().await?; - Ok(()) -} diff --git a/crates/persisting-pchronicle-cli/tests/langfuse_gateway_stress.rs b/crates/persisting-pchronicle-cli/tests/langfuse_gateway_stress.rs deleted file mode 100644 index 20808330d..000000000 --- a/crates/persisting-pchronicle-cli/tests/langfuse_gateway_stress.rs +++ /dev/null @@ -1,340 +0,0 @@ -//! Black-box Langfuse OTLP Gateway pressure benchmark. -//! -//! This is intentionally ignored: it starts a real `pchronicle serve` child, -//! performs concurrent HTTP requests, and writes a sizeable temporary dataset. -//! Run it on controlled hardware with: -//! -//! ```text -//! PCHRONICLE_LANGFUSE_STRESS_REQUESTS=128 \ -//! PCHRONICLE_LANGFUSE_STRESS_SPANS_PER_REQUEST=512 \ -//! PCHRONICLE_LANGFUSE_STRESS_CONCURRENCY=16 \ -//! cargo test -p persisting-pchronicle-cli --test langfuse_gateway_stress \ -//! langfuse_gateway_pressure -- --ignored --nocapture -//! ``` -//! -//! The benchmark validates more than throughput: every request must receive a -//! full-success OTLP response, every span must become one durable event, and -//! trace/span/parent relationships must survive the Gateway chunking boundary. - -use anyhow::{Context, Result, bail}; -use serde_json::{Value, json}; -use std::process::Stdio; -use std::sync::Arc; -use std::time::{Duration, Instant}; -use tokio::io::{AsyncBufReadExt, BufReader}; -use tokio::process::Command; -use tokio::sync::Semaphore; - -const DEFAULT_REQUESTS: usize = 32; -const DEFAULT_SPANS_PER_REQUEST: usize = 512; -const DEFAULT_CONCURRENCY: usize = 8; - -fn env_usize(name: &str, default: usize) -> Result { - let value = std::env::var(name).unwrap_or_else(|_| default.to_string()); - let parsed = value - .parse::() - .with_context(|| format!("{name} must be a positive integer"))?; - if parsed == 0 { - bail!("{name} must be greater than zero"); - } - Ok(parsed) -} - -fn attr(key: &str, value: impl Into) -> Value { - json!({"key": key, "value": {"stringValue": value.into()}}) -} - -fn trace_id(request: usize) -> String { - format!("{:032x}", request + 1) -} - -fn span_id(request: usize, span: usize) -> String { - format!("{:016x}", (request as u64) << 32 | (span as u64 + 1)) -} - -fn otlp_batch(request: usize, spans_per_request: usize) -> Result> { - let trace_id = trace_id(request); - let session_id = format!("langfuse-bench-session-{request}"); - let user_id = format!("langfuse-bench-user-{}", request % 16); - let base_ns = 1_800_000_000_000_000_000u64 + request as u64 * 1_000_000_000; - let spans = (0..spans_per_request) - .map(|span| { - let current_span_id = span_id(request, span); - let is_root = span == 0; - let observation_type = if is_root { - "agent" - } else if span % 7 == 0 { - "tool" - } else { - "generation" - }; - let mut attributes = vec![ - attr("user.id", user_id.clone()), - attr("session.id", session_id.clone()), - attr("langfuse.trace.name", "Gateway pressure benchmark"), - attr("langfuse.observation.type", observation_type), - attr( - "langfuse.observation.input", - format!("input-{request}-{span}"), - ), - attr( - "langfuse.observation.output", - format!("output-{request}-{span}"), - ), - ]; - if observation_type == "generation" { - attributes.push(attr("langfuse.observation.model.name", "bench-model")); - } - let mut span_json = json!({ - "traceId": trace_id, - "spanId": current_span_id, - "name": if is_root { "Codex Turn" } else { "LLM" }, - "kind": 1, - "startTimeUnixNano": base_ns + span as u64 * 1_000_000, - "endTimeUnixNano": base_ns + span as u64 * 1_000_000 + 500_000, - "attributes": attributes, - "events": [], - "links": [], - "status": {"code": 0} - }); - if !is_root { - span_json["parentSpanId"] = Value::String(span_id(request, 0)); - } - span_json - }) - .collect::>(); - serde_json::to_vec(&json!({ - "resourceSpans": [{ - "resource": {"attributes": [ - attr("service.name", "langfuse-gateway-benchmark"), - attr("telemetry.sdk.language", "rust") - ]}, - "scopeSpans": [{ - "scope": {"name": "langfuse-sdk", "version": "5.4.1"}, - "spans": spans - }] - }] - })) - .context("encode synthetic Langfuse OTLP batch") -} - -async fn query_jsonl(binary: &str, dataset: &std::path::Path, sql: &str) -> Result> { - let output = Command::new(binary) - .args([ - "query", - "--mount", - &format!("default={}", dataset.display()), - "--sql", - sql, - "--format", - "jsonl", - ]) - .output() - .await - .context("run pChronicle query after Gateway benchmark")?; - if !output.status.success() { - bail!( - "pChronicle query failed: {}", - String::from_utf8_lossy(&output.stderr) - ); - } - output - .stdout - .split(|byte| *byte == b'\n') - .filter(|line| !line.is_empty() && line.first() == Some(&b'{')) - .map(|line| serde_json::from_slice(line).context("decode pChronicle JSONL row")) - .collect() -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 4)] -#[ignore = "opt-in real HTTP pressure benchmark; tune PCHRONICLE_LANGFUSE_STRESS_*"] -async fn langfuse_gateway_pressure() -> Result<()> { - let requests = env_usize("PCHRONICLE_LANGFUSE_STRESS_REQUESTS", DEFAULT_REQUESTS)?; - let spans_per_request = env_usize( - "PCHRONICLE_LANGFUSE_STRESS_SPANS_PER_REQUEST", - DEFAULT_SPANS_PER_REQUEST, - )?; - let concurrency = env_usize( - "PCHRONICLE_LANGFUSE_STRESS_CONCURRENCY", - DEFAULT_CONCURRENCY, - )? - .min(requests); - - let temporary = tempfile::tempdir()?; - let dataset = temporary.path().join("captures"); - let binary = env!("CARGO_BIN_EXE_pchronicle"); - let mut child = Command::new(binary) - .args([ - "serve", - "--gateway", - "auto", - "--gateway-dataset", - dataset.to_str().context("dataset path is not UTF-8")?, - "--gateway-split", - "{user}/{date}/{hour}", - ]) - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::null()) - .kill_on_drop(true) - .spawn() - .context("start pChronicle Gateway benchmark child")?; - let stdout = child - .stdout - .take() - .context("capture Gateway readiness output")?; - let mut ready_line = String::new(); - tokio::time::timeout( - Duration::from_secs(30), - BufReader::new(stdout).read_line(&mut ready_line), - ) - .await - .context("wait for Gateway readiness timed out")??; - let ready: Value = serde_json::from_str(&ready_line).context("decode Gateway readiness")?; - let endpoint = ready["gateway_endpoint"] - .as_str() - .context("Gateway readiness did not advertise gateway_endpoint")?; - let url = format!("http://{endpoint}/api/public/otel/v1/traces"); - - let client = reqwest::Client::builder() - .no_proxy() - .pool_max_idle_per_host(concurrency) - .timeout(Duration::from_secs(120)) - .build()?; - let semaphore = Arc::new(Semaphore::new(concurrency)); - let started = Instant::now(); - let mut tasks = Vec::with_capacity(requests); - for request in 0..requests { - let permit = semaphore.clone().acquire_owned().await?; - let client = client.clone(); - let url = url.clone(); - let body = Arc::new(otlp_batch(request, spans_per_request)?); - tasks.push(tokio::spawn(async move { - let _permit = permit; - let started = Instant::now(); - let response = client - .post(url) - .header("content-type", "application/json") - .header("x-persisting-user-id", format!("langfuse-bench-user-{}", request % 16)) - .body(body.as_ref().clone()) - .send() - .await - .with_context(|| format!("send OTLP benchmark request {request}"))?; - let status = response.status(); - let content_type = response - .headers() - .get(reqwest::header::CONTENT_TYPE) - .and_then(|value| value.to_str().ok()) - .unwrap_or_default() - .to_string(); - let response_body = response.bytes().await?; - if status != reqwest::StatusCode::OK { - bail!("request {request} returned HTTP {status}: {response_body:?}"); - } - if !content_type.starts_with("application/json") || response_body.as_ref() != b"{}" { - bail!( - "request {request} did not receive full-success OTLP JSON response: content-type={content_type}, body={response_body:?}" - ); - } - Ok::(started.elapsed().as_millis()) - })); - } - let mut latencies_ms = Vec::with_capacity(requests); - for task in tasks { - latencies_ms.push(task.await.context("join Gateway benchmark request")??); - } - let elapsed = started.elapsed(); - - child.kill().await.context("stop Gateway benchmark child")?; - child.wait().await.context("reap Gateway benchmark child")?; - - let expected_events = requests - .checked_mul(spans_per_request) - .context("benchmark event count overflow")?; - let rows = query_jsonl( - binary, - &dataset, - "SELECT COUNT(*) AS events, COUNT(DISTINCT trace_id) AS traces, COUNT(DISTINCT call_id) AS spans, COUNT(event_id) AS ids, COUNT(DISTINCT event_id) AS unique_ids FROM default.events", - ) - .await?; - let counts = rows - .first() - .context("Gateway query returned no count row")?; - let events = counts["events"] - .as_u64() - .context("events count is not u64")?; - let traces = counts["traces"] - .as_u64() - .context("traces count is not u64")?; - let spans = counts["spans"].as_u64().context("spans count is not u64")?; - let ids = counts["ids"].as_u64().context("ids count is not u64")?; - let unique_ids = counts["unique_ids"] - .as_u64() - .context("unique_ids count is not u64")?; - if events != expected_events as u64 - || traces != requests as u64 - || spans != expected_events as u64 - || ids != expected_events as u64 - || unique_ids != expected_events as u64 - { - bail!( - "durability/integrity mismatch: expected events={expected_events}, traces={requests}, spans={expected_events}, ids={expected_events}; got {counts}" - ); - } - let kinds = query_jsonl( - binary, - &dataset, - "SELECT kind, COUNT(*) AS count FROM default.events GROUP BY kind ORDER BY kind", - ) - .await?; - let relationship_rows = query_jsonl( - binary, - &dataset, - "SELECT COUNT(*) AS missing_parent FROM default.events WHERE kind <> 'otel.agent' AND parent_call_id IS NULL", - ) - .await?; - let missing_parent = relationship_rows - .first() - .and_then(|row| row["missing_parent"].as_u64()) - .context("missing_parent count is not u64")?; - if missing_parent != 0 { - bail!("{missing_parent} non-root Langfuse spans lost their parent_call_id"); - } - let expected_tools = requests as u64 * spans_per_request.saturating_sub(1) as u64 / 7; - let expected_generations = expected_events as u64 - requests as u64 - expected_tools; - let modeled_rows = query_jsonl( - binary, - &dataset, - "SELECT COUNT(*) AS modeled FROM default.events WHERE kind = 'otel.generation' AND model = 'bench-model'", - ) - .await?; - let modeled = modeled_rows - .first() - .and_then(|row| row["modeled"].as_u64()) - .context("modeled count is not u64")?; - if modeled != expected_generations { - bail!( - "model promotion mismatch: expected {expected_generations} generation events, got {modeled}" - ); - } - let elapsed_secs = elapsed.as_secs_f64(); - latencies_ms.sort_unstable(); - let percentile = |numerator: usize, denominator: usize| -> u128 { - let index = ((latencies_ms.len() * numerator).saturating_add(denominator - 1) - / denominator) - .saturating_sub(1) - .min(latencies_ms.len().saturating_sub(1)); - latencies_ms[index] - }; - println!( - "langfuse_gateway_pressure: requests={requests} spans_per_request={spans_per_request} concurrency={concurrency} events={events} traces={traces} elapsed_ms={} requests_per_sec={:.1} spans_per_sec={:.1} latency_ms={{p50:{},p95:{},p99:{}}} kinds={}", - elapsed.as_millis(), - requests as f64 / elapsed_secs, - expected_events as f64 / elapsed_secs, - percentile(50, 100), - percentile(95, 100), - percentile(99, 100), - serde_json::to_string(&kinds)? - ); - Ok(()) -} diff --git a/crates/persisting-pchronicle-cli/tests/local_warehouse.rs b/crates/persisting-pchronicle-cli/tests/local_warehouse.rs index bd9c46d0b..e98a1f361 100644 --- a/crates/persisting-pchronicle-cli/tests/local_warehouse.rs +++ b/crates/persisting-pchronicle-cli/tests/local_warehouse.rs @@ -124,7 +124,6 @@ async fn default_pin_exercises_catalog_query_find_and_export_without_a_server() "trajectories": 4, "steps": 9, "tool_calls": 2, - "events": 0, }) ); diff --git a/crates/persisting-pchronicle-cli/tests/server_http_contract.rs b/crates/persisting-pchronicle-cli/tests/server_http_contract.rs index 35fbdf361..f85108db9 100644 --- a/crates/persisting-pchronicle-cli/tests/server_http_contract.rs +++ b/crates/persisting-pchronicle-cli/tests/server_http_contract.rs @@ -65,14 +65,7 @@ async fn warehouse_read_route_matrix_exposes_the_documented_surface() -> Result< .iter() .filter_map(|table| table["name"].as_str()) .collect(); - for name in [ - "sources", - "runs", - "steps", - "tool_calls", - "events", - "trajectories", - ] { + for name in ["sources", "runs", "steps", "tool_calls", "trajectories"] { assert!( names.contains(&name), "query tables omit documented relation {name}: {names:?}" @@ -92,6 +85,7 @@ async fn warehouse_write_route_matrix_never_exposes_dataset_mutations() -> Resul (Method::POST, "/api/maintain"), (Method::POST, "/api/query"), (Method::PUT, "/api/events"), + (Method::GET, "/api/events"), (Method::DELETE, "/api/catalog"), ] { let response = app diff --git a/crates/persisting-pchronicle/Cargo.toml b/crates/persisting-pchronicle/Cargo.toml index 146bc84d5..feebf82b6 100644 --- a/crates/persisting-pchronicle/Cargo.toml +++ b/crates/persisting-pchronicle/Cargo.toml @@ -61,8 +61,6 @@ lance-linalg = { workspace = true, optional = true } lance-table = { workspace = true, optional = true } opendal = { workspace = true, optional = true } object_store = { workspace = true, optional = true } -persisting-agentctl.workspace = true -persisting-events.workspace = true serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true, features = ["raw_value", "arbitrary_precision"] } serde_yaml.workspace = true @@ -96,14 +94,6 @@ harness = false name = "json_streaming" harness = false -[[bench]] -name = "projection_pipeline" -harness = false - -[[example]] -name = "langfuse_backend_feasibility" -required-features = ["lance-store"] - [[example]] name = "pchronicle_storage_query_benchmark" required-features = ["lance-store"] @@ -124,10 +114,6 @@ required-features = ["lance-store"] name = "atif_lance_corpus" required-features = ["lance-store"] -[[test]] -name = "capture_fixture_corpus" -required-features = ["lance-store"] - [[test]] name = "direct_file_query" required-features = ["lance-store"] @@ -140,14 +126,6 @@ required-features = ["lance-store"] name = "import_roundtrip_fixtures" required-features = ["lance-store"] -[[test]] -name = "langfuse_backend_faults" -required-features = ["lance-store"] - -[[test]] -name = "production_scale" -required-features = ["lance-store"] - [[test]] name = "query_engine" required-features = ["lance-store"] @@ -171,11 +149,6 @@ name = "proptest_storyline_codec" path = "tests/proptests/storyline_codec.rs" required-features = ["proptest"] -[[test]] -name = "proptest_events_projection" -path = "tests/proptests/events_projection.rs" -required-features = ["proptest"] - [[test]] name = "proptest_storage_layout" path = "tests/proptests/storage_layout.rs" @@ -186,11 +159,6 @@ name = "proptest_timestamp" path = "tests/proptests/timestamp.rs" required-features = ["proptest"] -[[test]] -name = "proptest_interop" -path = "tests/proptests/interop.rs" -required-features = ["proptest"] - [[test]] name = "proptest_format" path = "tests/proptests/format.rs" @@ -221,11 +189,6 @@ name = "proptest_llm" path = "tests/proptests/llm.rs" required-features = ["proptest"] -[[test]] -name = "proptest_events_model" -path = "tests/proptests/events_model.rs" -required-features = ["proptest"] - [[test]] name = "proptest_unknown_fields" path = "tests/proptests/unknown_fields.rs" @@ -241,11 +204,6 @@ name = "proptest_analysis_compile" path = "tests/proptests/analysis_compile.rs" required-features = ["proptest"] -[[test]] -name = "proptest_revision" -path = "tests/proptests/revision.rs" -required-features = ["proptest"] - [[test]] name = "proptest_storage_location" path = "tests/proptests/storage_location.rs" @@ -256,76 +214,21 @@ name = "proptest_catalog_identity" path = "tests/proptests/catalog_identity.rs" required-features = ["proptest"] -[[test]] -name = "proptest_attempt_registry" -path = "tests/proptests/attempt_registry.rs" -required-features = ["proptest"] - -[[test]] -name = "proptest_egress" -path = "tests/proptests/egress.rs" -required-features = ["proptest"] - -[[test]] -name = "proptest_run_control" -path = "tests/proptests/run_control.rs" -required-features = ["proptest"] - [[test]] name = "proptest_catalog_namespace" path = "tests/proptests/catalog_namespace.rs" required-features = ["proptest"] -[[test]] -name = "proptest_layout_resolve" -path = "tests/proptests/layout_resolve.rs" -required-features = ["proptest"] - [[test]] name = "proptest_agenticmd_layout" path = "tests/proptests/agenticmd_layout.rs" required-features = ["proptest"] -[[test]] -name = "proptest_event_manifest" -path = "tests/proptests/event_manifest.rs" -required-features = ["proptest"] - -[[test]] -name = "proptest_projection_automatic" -path = "tests/proptests/projection_automatic.rs" -required-features = ["proptest"] - [[test]] name = "proptest_storyline_store" path = "tests/proptests/storyline_store.rs" required-features = ["proptest"] -[[test]] -name = "proptest_discovery" -path = "tests/proptests/discovery.rs" -required-features = ["proptest"] - -[[test]] -name = "proptest_layout_coords" -path = "tests/proptests/layout_coords.rs" -required-features = ["proptest"] - -[[test]] -name = "proptest_event_snapshot" -path = "tests/proptests/event_snapshot.rs" -required-features = ["proptest"] - -[[test]] -name = "proptest_catalog_provenance" -path = "tests/proptests/catalog_provenance.rs" -required-features = ["proptest"] - -[[test]] -name = "proptest_storyline_metadata" -path = "tests/proptests/storyline_metadata.rs" -required-features = ["proptest"] - [[test]] name = "proptest_agenticmd_convert" path = "tests/proptests/agenticmd_convert.rs" @@ -341,11 +244,6 @@ name = "proptest_codex" path = "tests/proptests/codex.rs" required-features = ["proptest"] -[[test]] -name = "proptest_events_convert" -path = "tests/proptests/events_convert.rs" -required-features = ["proptest"] - [[test]] name = "proptest_storyline_datafusion" path = "tests/proptests/storyline_datafusion.rs" diff --git a/crates/persisting-pchronicle/README.md b/crates/persisting-pchronicle/README.md index fade90bdc..95f9005ed 100644 --- a/crates/persisting-pchronicle/README.md +++ b/crates/persisting-pchronicle/README.md @@ -2,20 +2,12 @@ **Persisting 的结构化轨迹与 Dataset 数据层。** -拥有轨迹领域模型、磁盘格式、Lance 持久化、数据源发现、DataFusion 查询、格式交换 -和可重建投影。其他 crate 可以生产或消费轨迹，但不应再定义第二套存储格式或轨迹 -协议 DTO。 +拥有 Storyline 轨迹模型、格式编解码、Lance 持久化、数据源发现、DataFusion 查询 +与格式交换。ATIF、ACTF、OpenAI Messages、Codex 和 Claude Code 等输入经 +Storyline 进入存储与查询管线，AgenticMD 是 Storyline 的 Markdown 编码。 -不拥有产品 CLI、只读 Warehouse HTTP、Web UI，也不拥有 Run 的启动与编排。 -[`persisting-pchronicle-cli`](../persisting-pchronicle-cli/README.md) 拥有 -`pchronicle` 命令、loopback API 与嵌入式静态资源。 -[`pchronicle-web`](../../pchronicle-web/README.md) 拥有浏览前端。 -Gateway 生产 canonical events。 - -Canonical Event 与 Storyline 分别在事实层和交换/分析层保持权威，关系是单向投影： -`events.lance` 是 append-only 运行时事实源；`StorylineDocument` 是与 ATIF v1.7 -对齐的权威轨迹模型和外围格式转换枢纽。AgenticMD 只是 Storyline 的人类可读 -Markdown 编码。 +CLI 和 Warehouse HTTP 由 [`persisting-pchronicle-cli`](../persisting-pchronicle-cli/README.md) +提供，浏览前端由 [`pchronicle-web`](../../pchronicle-web/README.md) 提供。 默认功能面通过四个模块组织：`model`、`document`、`storage`、`query`。外围 wire DTO、低层 parser、Arrow codec 和 DataFusion provider 保持私有。`search` 是独立 @@ -40,8 +32,7 @@ store.maintain(&LanceMaintenanceOptions { 预算按每张 Storyline 表、每次调用独立计算，默认不限制；它限制 compaction 输入， 不限制索引维护、GC、总内存或独立 Blob v2 文件的 I/O。Lance 不会拆开超预算的任务， 如果没有任务能放进预算，本次 compaction 可以不做任何合并，需要提高预算；字节预算 -要求源文件已有大小元数据。Raw Event 跨 segment 的维护使用不同路径，会明确拒绝 -这两个预算参数。每次成功维护仍通过 CURRENT 原子发布快照。 +要求源文件已有大小元数据。每次成功维护仍通过 CURRENT 原子发布快照。 ## Develop @@ -55,7 +46,7 @@ just proptest pchronicle - [pChronicle overview](../../docs/src/zh/pchronicle/index.md) - [产品架构](../../docs/src/zh/pchronicle/design/architecture.md) -- [记录数据、视图与版本](../../docs/src/zh/pchronicle/concepts/facts-and-projections.md) +- [轨迹数据与版本](../../docs/src/zh/pchronicle/concepts/facts-and-projections.md) - [pChronicle CLI](../../docs/src/zh/pchronicle/reference/cli.md) - [RFC-0003 ownership](../../docs/src/zh/rfcs/0003-pchronicle-ownership.md) - [`persisting-pchronicle-cli`](../persisting-pchronicle-cli/README.md) diff --git a/crates/persisting-pchronicle/benches/pchronicle_criterion.rs b/crates/persisting-pchronicle/benches/pchronicle_criterion.rs index 5c2fc3cce..97dc2ef61 100644 --- a/crates/persisting-pchronicle/benches/pchronicle_criterion.rs +++ b/crates/persisting-pchronicle/benches/pchronicle_criterion.rs @@ -9,8 +9,7 @@ use std::time::Duration; use criterion::{Criterion, Throughput, criterion_group, criterion_main}; use persisting_pchronicle::document::{ - DocumentFormat, decode_json_storylines, encode_json_storylines, project_event_records, - storyline_to_events, + DocumentFormat, decode_json_storylines, encode_json_storylines, }; use persisting_pchronicle::model::StorylineDocument; @@ -56,31 +55,6 @@ fn conversion_benchmarks(criterion: &mut Criterion) { group.finish(); } -fn projection_cpu_benchmarks(criterion: &mut Criterion) { - let fixtures = load_fixtures(); - let stories = fixtures - .iter() - .map(|raw| parse_atif(raw)) - .collect::>(); - let events = stories - .iter() - .map(|story| storyline_to_events(story).expect("project fixture to canonical events")) - .collect::>(); - let mut group = criterion.benchmark_group("projection_cpu"); - group.throughput(Throughput::Elements(stories.len() as u64)); - group.bench_function("events_to_storyline_corpus", |bencher| { - bencher.iter(|| { - for document in &events { - black_box( - project_event_records(black_box(&document.events)) - .expect("project canonical benchmark events"), - ); - } - }); - }); - group.finish(); -} - fn load_fixtures() -> Vec { fixture_paths() .into_iter() @@ -126,6 +100,6 @@ fn env_u64(name: &str, default: u64) -> u64 { criterion_group! { name = benches; config = criterion_config(); - targets = conversion_benchmarks, projection_cpu_benchmarks + targets = conversion_benchmarks } criterion_main!(benches); diff --git a/crates/persisting-pchronicle/benches/projection_pipeline.rs b/crates/persisting-pchronicle/benches/projection_pipeline.rs deleted file mode 100644 index f3ab9f6ba..000000000 --- a/crates/persisting-pchronicle/benches/projection_pipeline.rs +++ /dev/null @@ -1,202 +0,0 @@ -//! Canonical event append and events → Storyline materialization benchmark. - -use std::path::PathBuf; -use std::time::{Duration, Instant}; - -use anyhow::{Context, Result}; -use persisting_pchronicle::document::{ - DocumentFormat, decode_json_storylines, storyline_to_events, -}; -use persisting_pchronicle::model::{EventIdentity, EventRecord, StorylineDocument}; -use persisting_pchronicle::storage::{ - RawEventLanceAppender, StoryCoords, StorylineProjectionBuildOutcome, - StorylineProjectionSyncOutcome, build_storyline_projection, sync_storyline_projection, - verify_storyline_projection, -}; - -fn main() -> Result<()> { - let scale = env_usize("PCHRONICLE_BENCH_SCALE", 32); - anyhow::ensure!(scale > 0, "PCHRONICLE_BENCH_SCALE must be positive"); - let runtime = tokio::runtime::Builder::new_multi_thread() - .enable_all() - .build()?; - let result = runtime.block_on(run(scale))?; - - println!( - "canonical projection: {} Storylines, {} initial events", - result.storylines, result.events - ); - println!( - "RESULT benchmark=event_append rows={} initial_append_ms={:.3} rows_s={:.1}", - result.events, - milliseconds(result.initial_append), - result.events as f64 / result.initial_append.as_secs_f64(), - ); - println!( - "RESULT benchmark=projection_build documents={} build_ms={:.3} storylines_s={:.1}", - result.storylines, - milliseconds(result.projection_build), - result.storylines as f64 / result.projection_build.as_secs_f64(), - ); - println!( - "RESULT benchmark=projection_incremental suffix_append_ms={:.3} sync_ms={:.3}", - milliseconds(result.suffix_append), - milliseconds(result.projection_sync), - ); - println!( - "RESULT benchmark=projection_verify verify_ms={:.3}", - milliseconds(result.projection_verify), - ); - Ok(()) -} - -struct BenchmarkResult { - storylines: usize, - events: usize, - initial_append: Duration, - projection_build: Duration, - suffix_append: Duration, - projection_sync: Duration, - projection_verify: Duration, -} - -async fn run(scale: usize) -> Result { - let temporary = tempfile::tempdir()?; - let storage = temporary.path().join("canonical"); - let output = temporary.path().join("storyline"); - let stories = expand_stories(&load_base_stories()?, scale); - let root_session_id = "benchmark-run".to_string(); - let mut entries = Vec::new(); - for story in &stories { - let coords = StoryCoords::new( - storage.to_string_lossy(), - "benchmark-agent", - story.session_id.clone(), - Some(root_session_id.clone()), - ); - let events = storyline_to_events(story)?; - entries.extend( - events - .events - .into_iter() - .map(|event| (coords.clone(), event)), - ); - } - let events = entries.len(); - let source_uri = entries - .first() - .context("projection benchmark corpus is empty")? - .0 - .lance_event_path()? - .to_string_lossy() - .into_owned(); - let output_uri = output.to_string_lossy().into_owned(); - - let mut appender = RawEventLanceAppender::default(); - let started = Instant::now(); - let appended = appender.append_event_batch(&entries).await?; - let initial_append = started.elapsed(); - anyhow::ensure!(appended.accepted_records == events); - - let started = Instant::now(); - let StorylineProjectionBuildOutcome::Built(build) = - build_storyline_projection(&source_uri, &output_uri, "events.lance").await? - else { - anyhow::bail!("benchmark projection output was unexpectedly nonempty") - }; - let projection_build = started.elapsed(); - anyhow::ensure!(build.storylines == stories.len()); - - let affected = &entries[0].0; - let suffix = EventRecord { - identity: EventIdentity { - event_id: Some("benchmark-suffix".into()), - ..Default::default() - }, - seq: 1_000_000, - source: "benchmark".into(), - kind: "note".into(), - timestamp: None, - session_id: Some(affected.session_id.clone()), - agent_id: Some(affected.agent_id.clone()), - parent_uuid: None, - trace_id: None, - call_id: None, - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload: serde_json::json!({"content": "incremental projection benchmark"}), - }; - let started = Instant::now(); - appender - .append_event_batch(&[(affected.clone(), suffix)]) - .await?; - let suffix_append = started.elapsed(); - - let started = Instant::now(); - let StorylineProjectionSyncOutcome::Synced(sync) = - sync_storyline_projection(&source_uri, &output_uri).await? - else { - anyhow::bail!("benchmark projection unexpectedly required a rebuild") - }; - let projection_sync = started.elapsed(); - anyhow::ensure!(sync.affected_storylines == 1 && sync.suffix_rows_scanned == 1); - - let started = Instant::now(); - let verification = verify_storyline_projection(&source_uri, &output_uri).await?; - let projection_verify = started.elapsed(); - anyhow::ensure!(verification.fresh); - - Ok(BenchmarkResult { - storylines: stories.len(), - events, - initial_append, - projection_build, - suffix_append, - projection_sync, - projection_verify, - }) -} - -fn load_base_stories() -> Result> { - let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/atif"); - let mut paths = std::fs::read_dir(root)? - .map(|entry| entry.map(|entry| entry.path())) - .collect::>>()?; - paths.retain(|path| path.extension().and_then(|value| value.to_str()) == Some("json")); - paths.sort(); - paths - .into_iter() - .map(|path| { - let raw = std::fs::read_to_string(&path)?; - decode_json_storylines(DocumentFormat::Atif, &raw, &path)? - .pop() - .context("missing benchmark Storyline") - }) - .collect() -} - -fn expand_stories(base: &[StorylineDocument], scale: usize) -> Vec { - let mut stories = Vec::with_capacity(base.len() * scale); - for replica in 0..scale { - for (index, source) in base.iter().enumerate() { - let mut story = source.clone(); - story.session_id = format!("projection-{replica:06}-{index:02}"); - story.run_id = Some("benchmark-run".into()); - stories.push(story); - } - } - stories -} - -fn env_usize(name: &str, default: usize) -> usize { - std::env::var(name) - .ok() - .and_then(|value| value.parse().ok()) - .unwrap_or(default) -} - -fn milliseconds(duration: Duration) -> f64 { - duration.as_secs_f64() * 1_000.0 -} diff --git a/crates/persisting-pchronicle/examples/langfuse_backend_feasibility.rs b/crates/persisting-pchronicle/examples/langfuse_backend_feasibility.rs deleted file mode 100644 index cadab1e38..000000000 --- a/crates/persisting-pchronicle/examples/langfuse_backend_feasibility.rs +++ /dev/null @@ -1,1266 +0,0 @@ -//! Executable feasibility probe for using pChronicle as Langfuse's analytics backend. -//! -//! This is deliberately an adapter PoC, not a production integration. It keeps -//! Langfuse's public APIs out of scope and makes unsupported mutation semantics -//! explicit instead of emulating them inside pChronicle core. - -use std::collections::{BTreeMap, BTreeSet}; -use std::fs::{self, File}; -use std::io::{BufWriter, Write}; -use std::path::{Path, PathBuf}; -use std::sync::Arc; -use std::time::{Duration, Instant}; - -use anyhow::{Context, Result}; -use async_trait::async_trait; -use chrono::{Duration as ChronoDuration, TimeZone, Utc}; -use persisting_pchronicle::document::DocumentFormat; -use persisting_pchronicle::model::{EventIdentity, EventRecord}; -use persisting_pchronicle::query::{ChronicleQueryEngine, ChronicleQueryExecutionOptions}; -use persisting_pchronicle::storage::{ - CatalogSnapshotOptions, DatasetCatalogSnapshot, DatasetMount, LanceMaintenanceOptions, - RawEventLanceAppender, RawEventLanceStore, StoryCoords, raw_event_lance_path, -}; -use serde::Serialize; -use serde_json::{Value, json}; - -const DEFAULT_EVENTS: usize = 100_000; -const DEFAULT_SCORES: usize = 10_000; -const DEFAULT_DATASET_RUN_ITEMS: usize = 2_000; -const DEFAULT_BLOB_LOG_ROWS: usize = 1_000; -const DEFAULT_TRACES: usize = 200; -const DEFAULT_PRELOAD_BATCH_ROWS: usize = 5_000; -const DEFAULT_LOAD_SECONDS: usize = 3; -const LOAD_ROWS_PER_SECOND: usize = 10; - -#[derive(Debug, Clone, Serialize)] -struct LogicalRow { - row_kind: String, - project_id: String, - trace_id: String, - span_id: String, - parent_span_id: String, - logical_id: String, - event_ts: String, - event_ts_ms: u64, - start_time: String, - name: String, - event_type: String, - session_id: String, - user_id: String, - environment: String, - tags: Vec, - metadata_names: Vec, - metadata_values: Vec, - tool_names: Vec, - model: String, - input: String, - output: String, - usage_input: u64, - usage_output: u64, - total_cost: String, - version: u32, - is_deleted: bool, - bookmarked: bool, - public: bool, - dataset_id: String, - dataset_run_id: String, - storage_run_id: String, - seq: u64, -} - -impl LogicalRow { - fn to_event_record(&self) -> Result { - Ok(EventRecord { - identity: EventIdentity { - event_id: Some(self.logical_id.clone()), - timestamp_unix_ms: Some(self.event_ts_ms), - ..Default::default() - }, - seq: self.seq, - source: "langfuse-backend-feasibility".into(), - kind: self.row_kind.clone(), - timestamp: Some(self.event_ts.replace(' ', "T") + "Z"), - session_id: Some(self.storage_run_id.clone()), - agent_id: Some(self.project_id.clone()), - parent_uuid: None, - trace_id: (!self.trace_id.is_empty()).then(|| self.trace_id.clone()), - call_id: (!self.span_id.is_empty()).then(|| self.span_id.clone()), - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: (!self.parent_span_id.is_empty()).then(|| self.parent_span_id.clone()), - payload: serde_json::to_value(self).context("encode logical Langfuse row")?, - }) - } - - fn coords(&self, storage: &Path) -> StoryCoords { - StoryCoords::new( - storage.to_string_lossy(), - self.project_id.clone(), - self.storage_run_id.clone(), - Some(self.storage_run_id.clone()), - ) - } -} - -#[derive(Debug, Serialize)] -struct AppendReceipt { - acknowledged_rows: usize, - persisted_rows: usize, -} - -#[derive(Debug, Serialize)] -struct QueryMetric { - repetitions: usize, - result_rows: usize, - p50_ms: f64, - p95_ms: f64, - max_ms: f64, -} - -#[derive(Debug, Serialize)] -struct StreamMetric { - rows: usize, - bytes: usize, - first_byte_ms: Option, - elapsed_ms: f64, -} - -#[derive(Debug, Serialize)] -struct CapabilityProbe { - supported: bool, - error: Option, -} - -#[derive(Debug, Serialize)] -struct BackendHealth { - physical_rows: u64, - catalog_sources: usize, -} - -#[async_trait(?Send)] -#[allow( - clippy::double_must_use, - reason = "async_trait adds must_use to futures" -)] -trait LangfuseAnalyticsBackend { - async fn append(&mut self, rows: &[LogicalRow]) -> Result; - async fn point(&self, project_id: &str, logical_id: &str) -> Result>; - async fn list(&self, project_id: &str, limit: usize) -> Result>; - async fn aggregate(&self, project_id: &str) -> Result>; - async fn stream(&self, project_id: &str) -> Result; - async fn update_flags( - &mut self, - project_id: &str, - logical_id: &str, - bookmarked: bool, - public: bool, - ) -> Result<()>; - async fn delete_trace(&mut self, project_id: &str, trace_id: &str) -> Result<()>; - async fn delete_project(&mut self, project_id: &str) -> Result<()>; - async fn delete_older_than(&mut self, project_id: &str, cutoff: &str) -> Result<()>; - async fn flush(&mut self) -> Result<()>; - async fn health(&self) -> Result; -} - -struct PChronicleBackend { - storage: PathBuf, - writer: Option, - engine: Option, - catalog_sources: usize, -} - -impl PChronicleBackend { - fn new(storage: PathBuf) -> Self { - Self { - storage, - writer: Some(RawEventLanceAppender::default()), - engine: None, - catalog_sources: 0, - } - } - - fn engine(&self) -> Result<&ChronicleQueryEngine> { - self.engine - .as_ref() - .context("backend catalog has not been flushed/refreshed") - } - - async fn refresh_catalog(&mut self) -> Result { - let started = Instant::now(); - let snapshot = discover_catalog(&self.storage).await?; - self.catalog_sources = snapshot - .datasets() - .iter() - .map(|dataset| dataset.ready_source_count()) - .sum(); - self.engine = Some(snapshot.query_engine(Default::default()).await?); - Ok(started.elapsed()) - } -} - -#[async_trait(?Send)] -impl LangfuseAnalyticsBackend for PChronicleBackend { - async fn append(&mut self, rows: &[LogicalRow]) -> Result { - let entries = rows - .iter() - .map(|row| Ok((row.coords(&self.storage), row.to_event_record()?))) - .collect::>>()?; - let outcome = self - .writer - .get_or_insert_with(RawEventLanceAppender::default) - .append_event_batch(&entries) - .await?; - Ok(AppendReceipt { - acknowledged_rows: outcome.accepted_records, - persisted_rows: outcome.persisted_units, - }) - } - - async fn point(&self, project_id: &str, logical_id: &str) -> Result> { - query_values( - self.engine()?, - &format!( - "SELECT event_id, timestamp, kind, trace_id, call_id, payload_json \ - FROM events WHERE agent_id = {} AND event_id = {} \ - ORDER BY timestamp DESC LIMIT 1", - sql_string(project_id), - sql_string(logical_id) - ), - ) - .await - } - - async fn list(&self, project_id: &str, limit: usize) -> Result> { - query_values( - self.engine()?, - &format!( - "SELECT event_id, timestamp, kind, trace_id, call_id \ - FROM events WHERE agent_id = {} AND kind = 'event' \ - ORDER BY timestamp DESC LIMIT {limit}", - sql_string(project_id) - ), - ) - .await - } - - async fn aggregate(&self, project_id: &str) -> Result> { - query_values( - self.engine()?, - &format!( - "SELECT model, COUNT(*) AS row_count FROM events \ - WHERE agent_id = {} AND kind = 'event' GROUP BY model ORDER BY model", - sql_string(project_id) - ), - ) - .await - } - - async fn stream(&self, project_id: &str) -> Result { - let sql = format!( - "SELECT event_id, timestamp, kind, trace_id, call_id, payload_json \ - FROM events WHERE agent_id = {} AND kind = 'event' ORDER BY timestamp", - sql_string(project_id) - ); - let started = Instant::now(); - let mut writer = FirstByteWriter::new(started); - self.engine()?.write_query_jsonl(&sql, &mut writer).await?; - Ok(StreamMetric { - rows: writer.rows, - bytes: writer.bytes, - first_byte_ms: writer.first_byte.map(duration_ms), - elapsed_ms: duration_ms(started.elapsed()), - }) - } - - async fn update_flags( - &mut self, - _project_id: &str, - _logical_id: &str, - _bookmarked: bool, - _public: bool, - ) -> Result<()> { - anyhow::bail!( - "UNSUPPORTED_MUTATION: pChronicle is append-only and has no latest-version patch projection" - ) - } - - async fn delete_trace(&mut self, _project_id: &str, _trace_id: &str) -> Result<()> { - anyhow::bail!( - "UNSUPPORTED_DELETE: pChronicle has no trace tombstone projection or row delete contract" - ) - } - - async fn delete_project(&mut self, _project_id: &str) -> Result<()> { - anyhow::bail!( - "UNSUPPORTED_DELETE: pChronicle has no project deletion contract for a shared catalog" - ) - } - - async fn delete_older_than(&mut self, _project_id: &str, _cutoff: &str) -> Result<()> { - anyhow::bail!( - "UNSUPPORTED_RETENTION: pChronicle vacuum removes unreachable versions, not logical rows by tenant/time" - ) - } - - async fn flush(&mut self) -> Result<()> { - if let Some(writer) = self.writer.take() { - let _reports = writer.finish(); - } - self.refresh_catalog().await?; - Ok(()) - } - - async fn health(&self) -> Result { - let rows = count_query(self.engine()?, "SELECT COUNT(*) AS row_count FROM events").await?; - Ok(BackendHealth { - physical_rows: rows, - catalog_sources: self.catalog_sources, - }) - } -} - -struct FirstByteWriter { - started: Instant, - first_byte: Option, - bytes: usize, - rows: usize, -} - -impl FirstByteWriter { - fn new(started: Instant) -> Self { - Self { - started, - first_byte: None, - bytes: 0, - rows: 0, - } - } -} - -impl Write for FirstByteWriter { - fn write(&mut self, buffer: &[u8]) -> std::io::Result { - if self.first_byte.is_none() && !buffer.is_empty() { - self.first_byte = Some(self.started.elapsed()); - } - self.bytes += buffer.len(); - self.rows += buffer.iter().filter(|byte| **byte == b'\n').count(); - Ok(buffer.len()) - } - - fn flush(&mut self) -> std::io::Result<()> { - Ok(()) - } -} - -#[tokio::main] -async fn main() -> Result<()> { - let config = Config::from_env()?; - fs::create_dir_all(&config.workdir) - .with_context(|| format!("create workdir {}", config.workdir.display()))?; - let fixture_path = config.workdir.join("logical_rows.jsonl"); - let report_path = config.workdir.join("pchronicle-report.json"); - let storage = config.workdir.join("pchronicle-store"); - anyhow::ensure!( - !fixture_path.exists() && !report_path.exists() && !storage.exists(), - "workdir already contains feasibility-probe outputs; use an empty directory" - ); - - let fixture_started = Instant::now(); - let rows = generate_fixture(&config)?; - let expected_sources = rows - .iter() - .map(|row| (row.project_id.clone(), row.storage_run_id.clone())) - .collect::>() - .len(); - write_fixture(&fixture_path, &rows)?; - let fixture_elapsed = fixture_started.elapsed(); - - let mut backend = PChronicleBackend::new(storage.clone()); - let append_started = Instant::now(); - let mut acknowledged = 0usize; - for batch in rows.chunks(config.preload_batch_rows) { - let receipt = backend.append(batch).await?; - anyhow::ensure!( - receipt.acknowledged_rows == batch.len() && receipt.persisted_rows == batch.len(), - "append acknowledgement mismatch" - ); - acknowledged += receipt.acknowledged_rows; - } - let append_elapsed = append_started.elapsed(); - anyhow::ensure!( - acknowledged == rows.len(), - "not every fixture row was acknowledged" - ); - - let rss_before_catalog = current_rss_bytes(); - let catalog_started = Instant::now(); - backend.flush().await?; - let catalog_elapsed = catalog_started.elapsed(); - let rss_after_catalog = current_rss_bytes(); - let health = backend.health().await?; - anyhow::ensure!( - health.physical_rows == rows.len() as u64, - "physical row count differs from acknowledged fixture rows" - ); - anyhow::ensure!( - health.catalog_sources == expected_sources, - "catalog source count differs from generated run count" - ); - - let point_id = "project-a-event-00000000"; - let other_project_id = "project-b-event-00000500"; - let point_rows = backend.point("project-a", point_id).await?; - let cross_project_rows = backend.point("project-a", other_project_id).await?; - let list_rows = backend.list("project-a", 100).await?; - let aggregate_rows = backend.aggregate("project-a").await?; - let stream_metric = backend.stream("project-a").await?; - - let point_sql = format!( - "SELECT event_id, timestamp, kind FROM events WHERE agent_id = {} AND event_id = {} \ - ORDER BY timestamp DESC LIMIT 1", - sql_string("project-a"), - sql_string(point_id) - ); - let list_sql = format!( - "SELECT event_id, timestamp, kind, trace_id, call_id FROM events WHERE agent_id = {} \ - AND kind = 'event' ORDER BY timestamp DESC LIMIT 100", - sql_string("project-a") - ); - let aggregate_sql = format!( - "SELECT model, COUNT(*) AS row_count FROM events WHERE agent_id = {} AND kind = 'event' \ - GROUP BY model ORDER BY model", - sql_string("project-a") - ); - let point_metric = measure_query(backend.engine()?, &point_sql, 7).await?; - let list_metric = measure_query(backend.engine()?, &list_sql, 7).await?; - let aggregate_metric = measure_query(backend.engine()?, &aggregate_sql, 7).await?; - let rss_after_unpruned_queries = current_rss_bytes(); - - let fresh_global_point_setup_started = Instant::now(); - let fresh_global_point_engine = discover_catalog(&storage) - .await? - .query_engine(Default::default()) - .await?; - let fresh_global_point_setup_ms = duration_ms(fresh_global_point_setup_started.elapsed()); - let fresh_global_point_metric = - measure_query(&fresh_global_point_engine, &point_sql, 7).await?; - drop(fresh_global_point_engine); - - let fresh_global_list_setup_started = Instant::now(); - let fresh_global_list_engine = discover_catalog(&storage) - .await? - .query_engine(Default::default()) - .await?; - let fresh_global_list_setup_ms = duration_ms(fresh_global_list_setup_started.elapsed()); - let fresh_global_list_metric = measure_query(&fresh_global_list_engine, &list_sql, 7).await?; - drop(fresh_global_list_engine); - - let point_row = rows - .iter() - .find(|row| row.logical_id == point_id) - .context("fixture contains no point-query row")?; - let direct_open_started = Instant::now(); - let direct_engine = ChronicleQueryEngine::open( - DocumentFormat::CanonicalEvent, - raw_event_lance_path(&point_row.coords(&storage))?, - ChronicleQueryExecutionOptions::default(), - ) - .await?; - let direct_open_ms = duration_ms(direct_open_started.elapsed()); - let direct_point_metric = measure_query( - &direct_engine, - &format!( - "SELECT event_id, timestamp, kind, trace_id, call_id, payload_json \ - FROM events WHERE event_id = {} ORDER BY timestamp DESC LIMIT 1", - sql_string(point_id) - ), - 7, - ) - .await?; - drop(direct_engine); - - let exact_file_setup_started = Instant::now(); - let exact_file_engine = discover_catalog(&storage) - .await? - .query_engine(Default::default()) - .await?; - let exact_file_setup_ms = duration_ms(exact_file_setup_started.elapsed()); - let exact_file = format!( - "{}/{}/events.lance", - point_row.project_id, point_row.storage_run_id - ); - let exact_file_point_metric = measure_query( - &exact_file_engine, - &format!( - "SELECT event_id, timestamp, kind, trace_id, call_id, payload_json \ - FROM events WHERE _file_ = {} AND agent_id = {} AND event_id = {} \ - ORDER BY timestamp DESC LIMIT 1", - sql_string(&exact_file), - sql_string(&point_row.project_id), - sql_string(point_id) - ), - 7, - ) - .await?; - drop(exact_file_engine); - - let project_file_setup_started = Instant::now(); - let project_file_engine = discover_catalog(&storage) - .await? - .query_engine(Default::default()) - .await?; - let project_file_setup_ms = duration_ms(project_file_setup_started.elapsed()); - let project_file_list_metric = measure_query( - &project_file_engine, - "SELECT event_id, timestamp, kind, trace_id, call_id FROM events \ - WHERE _file_ LIKE 'project-a/%' AND agent_id = 'project-a' AND kind = 'event' \ - ORDER BY timestamp DESC LIMIT 100", - 7, - ) - .await?; - drop(project_file_engine); - - let event_physical_rows = count_query( - backend.engine()?, - "SELECT COUNT(*) AS row_count FROM events WHERE kind = 'event'", - ) - .await?; - let event_logical_ids = count_query( - backend.engine()?, - "SELECT COUNT(DISTINCT event_id) AS row_count FROM events WHERE kind = 'event'", - ) - .await?; - - let mutation_errors = json!({ - "update_flags": backend.update_flags("project-a", point_id, true, true).await.unwrap_err().to_string(), - "delete_trace": backend.delete_trace("project-a", "project-a-trace-0000").await.unwrap_err().to_string(), - "delete_project": backend.delete_project("project-a").await.unwrap_err().to_string(), - "delete_retention": backend.delete_older_than("project-a", "2026-01-01 00:00:00.000").await.unwrap_err().to_string(), - }); - - let capability_probes = run_capability_probes(backend.engine()?).await; - - let base_count = health.physical_rows; - let pinned_snapshot = discover_catalog(&storage).await?; - let pinned_engine = pinned_snapshot.query_engine(Default::default()).await?; - let load_rows = generate_load_rows(&config); - let load_query = async { - let mut query_samples = Vec::new(); - let iterations = config.load_seconds.saturating_mul(2).max(1); - for _ in 0..iterations { - let started = Instant::now(); - let count = count_query( - &pinned_engine, - "SELECT COUNT(*) AS row_count FROM events WHERE agent_id = 'project-a'", - ) - .await?; - query_samples.push(duration_ms(started.elapsed())); - tokio::time::sleep(Duration::from_millis(500)).await; - anyhow::ensure!(count > 0, "concurrent pinned query returned no rows"); - } - Result::>::Ok(query_samples) - }; - let load_append = async { - let phase_started = Instant::now(); - let mut ack_samples = Vec::new(); - let mut visibility_samples = Vec::new(); - for (index, row) in load_rows.iter().enumerate() { - let target = phase_started + Duration::from_millis((index as u64) * 100); - if Instant::now() < target { - tokio::time::sleep_until(tokio::time::Instant::from_std(target)).await; - } - let started = Instant::now(); - let receipt = backend.append(std::slice::from_ref(row)).await?; - anyhow::ensure!( - receipt.acknowledged_rows == 1, - "load append was not acknowledged" - ); - ack_samples.push(duration_ms(started.elapsed())); - let visibility_started = Instant::now(); - let path = raw_event_lance_path(&row.coords(&storage))?; - let fresh = ChronicleQueryEngine::open( - DocumentFormat::CanonicalEvent, - path, - ChronicleQueryExecutionOptions::default(), - ) - .await?; - let visible = count_query(&fresh, "SELECT COUNT(*) AS row_count FROM events").await?; - anyhow::ensure!( - visible == (index + 1) as u64, - "acknowledged load row {index} is not visible" - ); - visibility_samples.push(duration_ms(visibility_started.elapsed())); - } - Result::<(Vec, Vec, Duration)>::Ok(( - ack_samples, - visibility_samples, - phase_started.elapsed(), - )) - }; - let (query_samples, load_result) = tokio::join!(load_query, load_append); - let query_samples = query_samples?; - let (load_ack_samples, load_visibility_samples, load_elapsed) = load_result?; - let pinned_count_after_append = - count_query(&pinned_engine, "SELECT COUNT(*) AS row_count FROM events").await?; - backend.flush().await?; - let refreshed_count = backend.health().await?.physical_rows; - anyhow::ensure!( - pinned_count_after_append == base_count, - "pinned catalog snapshot changed while append continued" - ); - anyhow::ensure!( - refreshed_count == base_count + load_rows.len() as u64, - "catalog refresh did not expose every acknowledged load row" - ); - - let maintained_row = rows - .iter() - .find(|row| row.row_kind == "event") - .context("fixture contains no event row")?; - let maintenance_session = maintained_row.coords(&storage); - let maintenance_started = Instant::now(); - let maintenance_report = RawEventLanceStore - .maintain( - &maintenance_session, - &LanceMaintenanceOptions { - vacuum_older_than: None, - ..Default::default() - }, - ) - .await?; - let maintenance_elapsed = maintenance_started.elapsed(); - let mut restart_row = maintained_row.clone(); - restart_row.logical_id = "project-a-after-maintenance".into(); - restart_row.event_ts_ms += 10_000_000; - restart_row.event_ts = timestamp_string(restart_row.event_ts_ms); - restart_row.start_time = restart_row.event_ts.clone(); - restart_row.seq += 1_000_000; - let restart_started = Instant::now(); - backend.append(std::slice::from_ref(&restart_row)).await?; - backend.flush().await?; - let restart_append_and_catalog_elapsed = restart_started.elapsed(); - let restart_visibility_started = Instant::now(); - let restart_visible = backend - .point("project-a", "project-a-after-maintenance") - .await?; - let restart_visibility_elapsed = restart_visibility_started.elapsed(); - let restart_end_to_end_elapsed = restart_started.elapsed(); - - let report = json!({ - "probe": "langfuse-clickhouse-to-pchronicle-feasibility", - "status": "experimental", - "config": { - "events": config.events, - "scores": config.scores, - "dataset_run_items": config.dataset_run_items, - "blob_log_rows": config.blob_log_rows, - "traces": config.traces, - "preload_batch_rows": config.preload_batch_rows, - "load_rows_per_second": LOAD_ROWS_PER_SECOND, - "load_seconds": config.load_seconds, - }, - "artifacts": { - "fixture": fixture_path, - "storage": storage, - }, - "fixture": { - "rows": rows.len(), - "expected_sources": expected_sources, - "generation_ms": duration_ms(fixture_elapsed), - "jsonl_bytes": fs::metadata(&fixture_path)?.len(), - }, - "append": { - "acknowledged_rows": acknowledged, - "elapsed_ms": duration_ms(append_elapsed), - "rows_per_second": rows.len() as f64 / append_elapsed.as_secs_f64(), - "zero_acknowledged_loss": acknowledged as u64 == health.physical_rows, - }, - "catalog": { - "cold_start_ms": duration_ms(catalog_elapsed), - "sources": health.catalog_sources, - "rss_before_bytes": rss_before_catalog, - "rss_after_bytes": rss_after_catalog, - "rss_after_unpruned_queries_bytes": rss_after_unpruned_queries, - }, - "catalog_pruning": { - "fresh_global_point_catalog_setup_ms": fresh_global_point_setup_ms, - "fresh_global_point": fresh_global_point_metric, - "fresh_global_list_catalog_setup_ms": fresh_global_list_setup_ms, - "fresh_global_list": fresh_global_list_metric, - "direct_run_open_ms": direct_open_ms, - "direct_run_point": direct_point_metric, - "exact_file_catalog_setup_ms": exact_file_setup_ms, - "exact_file_point": exact_file_point_metric, - "project_file_catalog_setup_ms": project_file_setup_ms, - "project_file_list": project_file_list_metric, - }, - "semantics": { - "point_rows": point_rows.len(), - "list_rows": list_rows.len(), - "aggregate_rows": aggregate_rows, - "cross_project_point_rows": cross_project_rows.len(), - "event_physical_rows": event_physical_rows, - "event_distinct_logical_ids": event_logical_ids, - "duplicate_versions_remain_physical": event_physical_rows > event_logical_ids, - "mutations": mutation_errors, - "json_export": stream_metric, - "parquet_export": { - "supported": false, - "reason": "ChronicleQueryEngine exposes JSONL streaming only" - }, - }, - "query_latency": { - "point": point_metric, - "list": list_metric, - "aggregate": aggregate_metric, - }, - "sql_compatibility": capability_probes, - "load_phase": { - "rows": load_rows.len(), - "elapsed_ms": duration_ms(load_elapsed), - "effective_rows_per_second": load_rows.len() as f64 / load_elapsed.as_secs_f64(), - "ack_p50_ms": percentile(&load_ack_samples, 0.50), - "ack_p95_ms": percentile(&load_ack_samples, 0.95), - "visibility_p50_ms": percentile(&load_visibility_samples, 0.50), - "visibility_p95_ms": percentile(&load_visibility_samples, 0.95), - "visible_rows": refreshed_count - base_count, - "zero_acknowledged_loss": refreshed_count == base_count + load_rows.len() as u64, - "concurrent_query_p95_ms": percentile(&query_samples, 0.95), - "pinned_count_after_append": pinned_count_after_append, - "refreshed_count": refreshed_count, - }, - "maintenance_restart": { - "maintenance_ms": duration_ms(maintenance_elapsed), - "fragments_removed": maintenance_report.fragments_removed, - "restart_append_and_catalog_ms": duration_ms(restart_append_and_catalog_elapsed), - "restart_visibility_ms": duration_ms(restart_visibility_elapsed), - "restart_end_to_end_ms": duration_ms(restart_end_to_end_elapsed), - "restart_row_visible": restart_visible.len() == 1, - }, - "hard_failures": [ - "bookmarked/public updates require a mutable latest-version projection", - "trace/project/retention deletes have no logical tombstone contract", - "ClickHouse SQL and aggregate functions are not source-compatible with DataFusion", - "Parquet export is absent from the public query adapter", - "project-wide queries still resolve every per-Run source unless the adapter supplies catalog-owned _file_ pruning" - ], - "bounded_integration_verdict": "NO-GO", - }); - fs::write(&report_path, serde_json::to_vec_pretty(&report)?) - .with_context(|| format!("write report {}", report_path.display()))?; - println!("{}", serde_json::to_string_pretty(&report)?); - Ok(()) -} - -#[derive(Debug)] -struct Config { - workdir: PathBuf, - events: usize, - scores: usize, - dataset_run_items: usize, - blob_log_rows: usize, - traces: usize, - preload_batch_rows: usize, - load_seconds: usize, -} - -impl Config { - fn from_env() -> Result { - let workdir = std::env::var("PCHRONICLE_LANGFUSE_WORKDIR") - .map(PathBuf::from) - .context("PCHRONICLE_LANGFUSE_WORKDIR must point to an empty output directory")?; - let events = positive_env("PCHRONICLE_LANGFUSE_EVENTS", DEFAULT_EVENTS)?; - let traces = positive_env("PCHRONICLE_LANGFUSE_TRACES", DEFAULT_TRACES)?; - anyhow::ensure!(traces <= events, "trace count cannot exceed event count"); - Ok(Self { - workdir, - events, - scores: positive_env("PCHRONICLE_LANGFUSE_SCORES", DEFAULT_SCORES)?, - dataset_run_items: positive_env( - "PCHRONICLE_LANGFUSE_DATASET_RUN_ITEMS", - DEFAULT_DATASET_RUN_ITEMS, - )?, - blob_log_rows: positive_env( - "PCHRONICLE_LANGFUSE_BLOB_LOG_ROWS", - DEFAULT_BLOB_LOG_ROWS, - )?, - traces, - preload_batch_rows: positive_env( - "PCHRONICLE_LANGFUSE_PRELOAD_BATCH_ROWS", - DEFAULT_PRELOAD_BATCH_ROWS, - )?, - load_seconds: positive_env("PCHRONICLE_LANGFUSE_LOAD_SECONDS", DEFAULT_LOAD_SECONDS)?, - }) - } -} - -fn positive_env(name: &str, default: usize) -> Result { - let value = match std::env::var(name) { - Ok(value) => value - .parse::() - .with_context(|| format!("{name} must be a positive integer"))?, - Err(std::env::VarError::NotPresent) => default, - Err(error) => return Err(error).with_context(|| format!("read {name}")), - }; - anyhow::ensure!(value > 0, "{name} must be greater than zero"); - Ok(value) -} - -fn generate_fixture(config: &Config) -> Result> { - let total = config - .events - .checked_add(config.scores) - .and_then(|value| value.checked_add(config.dataset_run_items)) - .and_then(|value| value.checked_add(config.blob_log_rows)) - .context("fixture row count overflow")?; - let mut rows = Vec::with_capacity(total); - let base_ms = 1_767_225_600_000u64; - - for index in 0..config.events { - let trace_index = index.saturating_mul(config.traces) / config.events; - let project = project_for_trace(trace_index); - let trace_id = trace_id(project, trace_index / 2); - let trace_first = trace_index.saturating_mul(config.events) / config.traces; - let duplicate_base = if index > 0 && index % 997 == 0 { - index - 1 - } else { - index - }; - let logical_id = format!("{project}-event-{duplicate_base:08}"); - let span_id = format!("{project}-span-{duplicate_base:08}"); - let event_ts_ms = base_ms + index as u64; - let large_input = index % 1_000 == 0; - let input = if large_input { - format!("needle-token {}", "x".repeat(65_536)) - } else if index % 257 == 0 { - format!("needle-token prompt-{index}") - } else { - format!("prompt-{index}") - }; - rows.push(LogicalRow { - row_kind: "event".into(), - project_id: project.into(), - trace_id: trace_id.clone(), - span_id, - parent_span_id: if index == trace_first { - String::new() - } else { - format!("{project}-span-{trace_first:08}") - }, - logical_id, - event_ts: timestamp_string(event_ts_ms), - event_ts_ms, - start_time: timestamp_string(base_ms + trace_first as u64), - name: format!("operation-{}", index % 17), - event_type: if index == trace_first { - "TRACE".into() - } else { - ["SPAN", "GENERATION", "EVENT"][index % 3].into() - }, - session_id: format!("session-{}", trace_index % 31), - user_id: format!("user-{}", trace_index % 101), - environment: ["default", "staging", "production"][trace_index % 3].into(), - tags: vec![ - format!("tag-{}", index % 11), - format!("team-{}", trace_index % 7), - ], - metadata_names: vec!["region".into(), "request.class".into()], - metadata_values: vec![ - ["cn", "eu", "us"][trace_index % 3].into(), - format!("class-{}", index % 13), - ], - tool_names: vec![format!("tool-{}", index % 9)], - model: format!("model-{}", index % 5), - input, - output: format!("response-{index}-{}", "y".repeat(index % 257)), - usage_input: (index % 4_096) as u64, - usage_output: (index % 2_048) as u64, - total_cost: format!("{:.12}", (index % 10_000) as f64 / 1_000_000.0), - version: if duplicate_base == index { 1 } else { 2 }, - is_deleted: false, - bookmarked: index % 101 == 0, - public: index % 503 == 0, - dataset_id: String::new(), - dataset_run_id: String::new(), - storage_run_id: trace_id, - seq: 0, - }); - } - - for index in 0..config.scores { - let project = match index % 2 { - 0 => "project-a", - _ => "project-b", - }; - let attached = index % 3 != 0; - let trace_index = index % config.traces; - let attached_trace = trace_id(project, trace_index / 2); - let event_ts_ms = base_ms + config.events as u64 + index as u64; - rows.push(LogicalRow { - row_kind: "score".into(), - project_id: project.into(), - trace_id: if attached { - attached_trace.clone() - } else { - String::new() - }, - span_id: String::new(), - parent_span_id: String::new(), - logical_id: format!("{project}-score-{index:08}"), - event_ts: timestamp_string(event_ts_ms), - event_ts_ms, - start_time: timestamp_string(event_ts_ms), - name: format!("score-{}", index % 13), - event_type: "SCORE".into(), - session_id: String::new(), - user_id: String::new(), - environment: "default".into(), - tags: Vec::new(), - metadata_names: vec!["data_type".into()], - metadata_values: vec![["NUMERIC", "BOOLEAN", "CATEGORICAL"][index % 3].into()], - tool_names: Vec::new(), - model: String::new(), - input: String::new(), - output: format!("{}", index % 101), - usage_input: 0, - usage_output: 0, - total_cost: "0.000000000000".into(), - version: 1, - is_deleted: false, - bookmarked: false, - public: false, - dataset_id: String::new(), - dataset_run_id: String::new(), - storage_run_id: if attached { - attached_trace - } else { - format!("synthetic-scores-{}", index % 4) - }, - seq: 0, - }); - } - - for index in 0..config.dataset_run_items { - let project = if index % 2 == 0 { - "project-a" - } else { - "project-b" - }; - let event_ts_ms = base_ms + config.events as u64 + config.scores as u64 + index as u64; - rows.push(LogicalRow { - row_kind: "dataset_run_item".into(), - project_id: project.into(), - trace_id: String::new(), - span_id: String::new(), - parent_span_id: String::new(), - logical_id: format!("{project}-dataset-run-item-{index:08}"), - event_ts: timestamp_string(event_ts_ms), - event_ts_ms, - start_time: timestamp_string(event_ts_ms), - name: "dataset-run-item".into(), - event_type: "DATASET_RUN_ITEM".into(), - session_id: String::new(), - user_id: String::new(), - environment: "default".into(), - tags: Vec::new(), - metadata_names: vec!["status".into()], - metadata_values: vec![["complete", "error"][index % 2].into()], - tool_names: Vec::new(), - model: String::new(), - input: String::new(), - output: String::new(), - usage_input: 0, - usage_output: 0, - total_cost: "0.000000000000".into(), - version: 1, - is_deleted: false, - bookmarked: false, - public: false, - dataset_id: format!("dataset-{}", index % 17), - dataset_run_id: format!("dataset-run-{}", index % 41), - storage_run_id: format!("synthetic-dataset-items-{}", index % 4), - seq: 0, - }); - } - - for index in 0..config.blob_log_rows { - let project = if index % 2 == 0 { - "project-a" - } else { - "project-b" - }; - let event_ts_ms = base_ms - + config.events as u64 - + config.scores as u64 - + config.dataset_run_items as u64 - + index as u64; - rows.push(LogicalRow { - row_kind: "blob_storage_file_log".into(), - project_id: project.into(), - trace_id: String::new(), - span_id: String::new(), - parent_span_id: String::new(), - logical_id: format!("{project}-blob-log-{index:08}"), - event_ts: timestamp_string(event_ts_ms), - event_ts_ms, - start_time: timestamp_string(event_ts_ms), - name: "blob-storage-file-log".into(), - event_type: "BLOB_LOG".into(), - session_id: String::new(), - user_id: String::new(), - environment: "default".into(), - tags: Vec::new(), - metadata_names: vec!["bucket".into(), "path".into()], - metadata_values: vec![ - "review-bucket".into(), - format!("{project}/events/{index}.json"), - ], - tool_names: Vec::new(), - model: String::new(), - input: String::new(), - output: String::new(), - usage_input: 0, - usage_output: 0, - total_cost: "0.000000000000".into(), - version: 1, - is_deleted: index % 97 == 0, - bookmarked: false, - public: false, - dataset_id: String::new(), - dataset_run_id: String::new(), - storage_run_id: format!("synthetic-blob-log-{}", index % 2), - seq: 0, - }); - } - - let mut sequences = BTreeMap::<(String, String), u64>::new(); - for row in &mut rows { - let sequence = sequences - .entry((row.project_id.clone(), row.storage_run_id.clone())) - .or_default(); - row.seq = *sequence; - *sequence += 1; - } - Ok(rows) -} - -fn generate_load_rows(config: &Config) -> Vec { - let count = config.load_seconds * LOAD_ROWS_PER_SECOND; - let base_ms = 1_767_225_900_000u64; - (0..count) - .map(|index| { - let event_ts_ms = base_ms + (index as u64) * 100; - LogicalRow { - row_kind: "event".into(), - project_id: "project-a".into(), - trace_id: "project-a-load-trace".into(), - span_id: format!("project-a-load-span-{index:04}"), - parent_span_id: String::new(), - logical_id: format!("project-a-load-event-{index:04}"), - event_ts: timestamp_string(event_ts_ms), - event_ts_ms, - start_time: timestamp_string(event_ts_ms), - name: "load-phase".into(), - event_type: "SPAN".into(), - session_id: "load-session".into(), - user_id: "load-user".into(), - environment: "default".into(), - tags: vec!["load".into()], - metadata_names: vec!["rate".into()], - metadata_values: vec!["10eps".into()], - tool_names: Vec::new(), - model: "load-model".into(), - input: format!("load-input-{index}"), - output: format!("load-output-{index}"), - usage_input: 1, - usage_output: 1, - total_cost: "0.000001000000".into(), - version: 1, - is_deleted: false, - bookmarked: false, - public: false, - dataset_id: String::new(), - dataset_run_id: String::new(), - storage_run_id: "project-a-load-trace".into(), - seq: index as u64, - } - }) - .collect() -} - -fn write_fixture(path: &Path, rows: &[LogicalRow]) -> Result<()> { - let file = File::create(path).with_context(|| format!("create fixture {}", path.display()))?; - let mut writer = BufWriter::new(file); - for row in rows { - let mut value = serde_json::to_value(row)?; - let payload_json = serde_json::to_string(row)?; - value - .as_object_mut() - .context("logical row did not encode as an object")? - .insert("payload_json".into(), Value::String(payload_json)); - serde_json::to_writer(&mut writer, &value)?; - writer.write_all(b"\n")?; - } - writer.flush()?; - Ok(()) -} - -async fn discover_catalog(storage: &Path) -> Result> { - let mount = DatasetMount::default(storage.to_string_lossy())?; - Ok(Arc::new( - DatasetCatalogSnapshot::discover( - vec![mount], - Some("dataset".into()), - CatalogSnapshotOptions::default(), - ) - .await?, - )) -} - -async fn query_values(engine: &ChronicleQueryEngine, sql: &str) -> Result> { - engine - .query_jsonl(sql) - .await? - .lines() - .filter(|line| !line.trim().is_empty()) - .map(|line| serde_json::from_str(line).context("decode pChronicle query JSONL")) - .collect() -} - -async fn count_query(engine: &ChronicleQueryEngine, sql: &str) -> Result { - let rows = query_values(engine, sql).await?; - anyhow::ensure!(rows.len() == 1, "count query returned {} rows", rows.len()); - rows[0]["row_count"] - .as_u64() - .context("count query row_count is not an unsigned integer") -} - -async fn measure_query( - engine: &ChronicleQueryEngine, - sql: &str, - repetitions: usize, -) -> Result { - let mut samples = Vec::with_capacity(repetitions); - let mut result_rows = 0usize; - for _ in 0..repetitions { - let started = Instant::now(); - let batches = engine.query(sql).await?; - samples.push(duration_ms(started.elapsed())); - result_rows = batches.iter().map(|batch| batch.num_rows()).sum(); - } - Ok(QueryMetric { - repetitions, - result_rows, - p50_ms: percentile(&samples, 0.50), - p95_ms: percentile(&samples, 0.95), - max_ms: samples.iter().copied().fold(0.0, f64::max), - }) -} - -async fn run_capability_probes(engine: &ChronicleQueryEngine) -> BTreeMap { - let probes = [ - ("approx_top_k", "SELECT approx_top_k(5)(kind) FROM events"), - ( - "full_text_has_all_tokens", - "SELECT hasAllTokens(payload_json, 'needle-token') FROM events LIMIT 1", - ), - ( - "array_join", - "SELECT arrayJoin(['a', 'b']) FROM events LIMIT 1", - ), - ("sum_map", "SELECT sumMap(map('x', 1)) FROM events"), - ("arg_max", "SELECT argMax(kind, timestamp) FROM events"), - ("quantile_curried", "SELECT quantile(0.95)(seq) FROM events"), - ( - "with_fill", - "SELECT timestamp, count(*) FROM events GROUP BY timestamp ORDER BY timestamp WITH FILL", - ), - ( - "prewhere", - "SELECT * FROM events PREWHERE agent_id = 'project-a' LIMIT 1", - ), - ( - "limit_by", - "SELECT * FROM events ORDER BY timestamp DESC LIMIT 1 BY event_id", - ), - ("final", "SELECT COUNT(*) FROM events FINAL"), - ( - "json_extract_string", - "SELECT JSONExtractString(payload_json, 'kind') FROM events LIMIT 1", - ), - ]; - let mut results = BTreeMap::new(); - for (name, sql) in probes { - let result = match engine.query(sql).await { - Ok(_) => CapabilityProbe { - supported: true, - error: None, - }, - Err(error) => CapabilityProbe { - supported: false, - error: Some(redact_error(&error.to_string())), - }, - }; - results.insert(name.into(), result); - } - results -} - -fn timestamp_string(timestamp_ms: u64) -> String { - let datetime = Utc - .timestamp_millis_opt(timestamp_ms as i64) - .single() - .unwrap_or_else(|| Utc::now() + ChronoDuration::milliseconds(timestamp_ms as i64)); - datetime.format("%Y-%m-%d %H:%M:%S%.3f").to_string() -} - -fn project_for_trace(trace_index: usize) -> &'static str { - if trace_index.is_multiple_of(2) { - "project-a" - } else { - "project-b" - } -} - -fn trace_id(project: &str, local_index: usize) -> String { - format!("{project}-trace-{local_index:04}") -} - -fn sql_string(value: &str) -> String { - format!("'{}'", value.replace('\'', "''")) -} - -fn duration_ms(duration: Duration) -> f64 { - duration.as_secs_f64() * 1_000.0 -} - -fn percentile(samples: &[f64], quantile: f64) -> f64 { - if samples.is_empty() { - return 0.0; - } - let mut sorted = samples.to_vec(); - sorted.sort_by(f64::total_cmp); - let rank = ((sorted.len() - 1) as f64 * quantile).ceil() as usize; - sorted[rank.min(sorted.len() - 1)] -} - -fn current_rss_bytes() -> Option { - let output = std::process::Command::new("ps") - .args(["-o", "rss=", "-p", &std::process::id().to_string()]) - .output() - .ok()?; - let kib = String::from_utf8(output.stdout) - .ok()? - .trim() - .parse::() - .ok()?; - kib.checked_mul(1_024) -} - -fn redact_error(message: &str) -> String { - let mut compact = message.replace('\n', " "); - compact.truncate(compact.len().min(500)); - compact -} diff --git a/crates/persisting-pchronicle/src/agenticmd/codec.rs b/crates/persisting-pchronicle/src/agenticmd/codec.rs index 4c2d1a7c4..459899e05 100644 --- a/crates/persisting-pchronicle/src/agenticmd/codec.rs +++ b/crates/persisting-pchronicle/src/agenticmd/codec.rs @@ -168,7 +168,7 @@ fn escape_yaml_control_characters(json: &str) -> String { /// Encode a single agenticmd / capture TLV block (comment header + body). /// -/// Normative on-disk layout shared with `persisting-gateway` live write paths. +/// Normative on-disk layout for Storyline Markdown documents. pub fn encode_agenticmd_block(block: &MarkdownBlock) -> Result { let header = MarkdownHeader { type_name: block.header.type_name.clone(), diff --git a/crates/persisting-pchronicle/src/append_queue.rs b/crates/persisting-pchronicle/src/append_queue.rs deleted file mode 100644 index 25a96f93b..000000000 --- a/crates/persisting-pchronicle/src/append_queue.rs +++ /dev/null @@ -1,1073 +0,0 @@ -//! Bounded bridge from synchronous capture callbacks to the async Lance appender. - -use std::collections::BTreeMap; -use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; -use std::sync::{Arc, mpsc}; -use std::time::Duration; - -use anyhow::Context; - -use crate::formats::EventRecord; -use crate::layout::StoryCoords; -use crate::store::compact_sealed_event_segment; -use crate::store::{RawEventLanceAppender, raw_event_lance_path}; - -pub const DEFAULT_RAW_EVENT_QUEUE_CAPACITY: usize = 256; -pub const DEFAULT_RAW_EVENT_BATCH_SIZE: usize = 256; -pub const DEFAULT_RAW_EVENT_BATCH_DELAY: Duration = Duration::from_millis(2); -/// Bound tiny fragments in the live snapshot before fan-out reads exhaust -/// file descriptors. Compaction runs after durable waiters have been released. -pub const DEFAULT_RAW_EVENT_COMPACTION_THRESHOLD: usize = 8; -pub const DEFAULT_RAW_EVENT_TARGET_ROWS_PER_FRAGMENT: usize = - DEFAULT_RAW_EVENT_COMPACTION_THRESHOLD; -pub const DEFAULT_RAW_EVENT_HIERARCHY_FANOUT: usize = 8; -/// Maintenance is best-effort and recoverable through explicit `maintain`. -/// Bound its live backlog independently from durable capture admission. -pub const DEFAULT_RAW_EVENT_MAINTENANCE_CAPACITY: usize = 64; - -#[derive(Debug)] -struct RawEventAppendJob { - coords: StoryCoords, - record: EventRecord, - completion: Option>>, -} - -#[derive(Debug)] -enum WriterMessage { - Append(Box), - Finish, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum RawEventAppendOutcome { - Accepted, - Full, - Unavailable, -} - -#[derive(Debug)] -struct SenderState { - tx: mpsc::SyncSender, - accepting: AtomicBool, - in_flight: AtomicUsize, -} - -/// Cloneable, bounded sender suitable for synchronous capture callbacks. -/// -/// `try_append` never waits for queue capacity. A full queue is reported to -/// the caller so capture can apply an explicit failure policy instead of -/// blocking an Agent or Gateway thread. -#[derive(Clone, Debug)] -pub struct RawEventAppendSender { - state: Arc, -} - -impl RawEventAppendSender { - pub fn try_append(&self, coords: StoryCoords, record: EventRecord) -> RawEventAppendOutcome { - self.enqueue(coords, record, None) - } - - /// Append an event and wait until its Lance micro-batch is durably visible. - /// - /// Queue admission remains bounded and non-blocking. Once admitted, this - /// method waits for the writer so an upstream WAL can acknowledge the event - /// only after pChronicle has published the corresponding Lance segment. - pub fn append_durable( - &self, - coords: StoryCoords, - record: EventRecord, - ) -> anyhow::Result { - let (completion_tx, completion_rx) = mpsc::sync_channel(1); - match self.enqueue(coords, record, Some(completion_tx)) { - RawEventAppendOutcome::Accepted => completion_rx - .recv() - .context("await raw event append completion")? - .map(|()| RawEventAppendOutcome::Accepted), - rejection => Ok(rejection), - } - } - - /// Enqueue a batch before waiting for durability. This preserves the - /// worker's micro-batching behavior for high-volume OTLP requests; calling - /// `append_durable` once per record would serialize every Lance commit. - /// - /// Admission remains bounded and non-blocking. If the queue fills midway, - /// already-admitted records are drained to completion before `Full` is - /// returned, so callers never acknowledge an unfinished prefix. - pub fn append_durable_batch( - &self, - entries: Vec<(StoryCoords, EventRecord)>, - ) -> anyhow::Result { - let mut completions = Vec::with_capacity(entries.len()); - for (coords, record) in entries { - let (completion_tx, completion_rx) = mpsc::sync_channel(1); - match self.enqueue(coords, record, Some(completion_tx)) { - RawEventAppendOutcome::Accepted => completions.push(completion_rx), - rejection => { - for completion in completions { - let _ = completion.recv(); - } - return Ok(rejection); - } - } - } - for completion in completions { - completion - .recv() - .context("await raw event batch append completion")??; - } - Ok(RawEventAppendOutcome::Accepted) - } - - fn enqueue( - &self, - coords: StoryCoords, - record: EventRecord, - completion: Option>>, - ) -> RawEventAppendOutcome { - if !self.state.accepting.load(Ordering::SeqCst) { - return RawEventAppendOutcome::Unavailable; - } - - self.state.in_flight.fetch_add(1, Ordering::SeqCst); - if !self.state.accepting.load(Ordering::SeqCst) { - self.state.in_flight.fetch_sub(1, Ordering::SeqCst); - return RawEventAppendOutcome::Unavailable; - } - - let outcome = - match self - .state - .tx - .try_send(WriterMessage::Append(Box::new(RawEventAppendJob { - coords, - record, - completion, - }))) { - Ok(()) => RawEventAppendOutcome::Accepted, - Err(mpsc::TrySendError::Full(_)) => RawEventAppendOutcome::Full, - Err(mpsc::TrySendError::Disconnected(_)) => RawEventAppendOutcome::Unavailable, - }; - self.state.in_flight.fetch_sub(1, Ordering::SeqCst); - outcome - } -} - -/// Owner of the append worker. Call `finish` after all capture sinks have -/// stopped using their cloned senders. -pub struct RawEventAppendWorker { - state: Arc, - join: Option>>, -} - -impl RawEventAppendWorker { - pub fn finish(mut self) -> anyhow::Result<()> { - self.state.accepting.store(false, Ordering::SeqCst); - while self.state.in_flight.load(Ordering::SeqCst) != 0 { - std::thread::yield_now(); - } - - let finish_signal = self - .state - .tx - .send(WriterMessage::Finish) - .map_err(anyhow::Error::new) - .context("finish raw event append worker"); - let Some(join) = self.join.take() else { - return finish_signal; - }; - let worker_result = join - .join() - .map_err(|_| anyhow::anyhow!("pChronicle append worker thread panicked"))?; - match (finish_signal, worker_result) { - (_, Err(worker_error)) => Err(worker_error), - (Err(signal_error), Ok(())) => Err(signal_error), - (Ok(()), Ok(())) => Ok(()), - } - } -} - -pub fn raw_event_append_queue() -> anyhow::Result<(RawEventAppendSender, RawEventAppendWorker)> { - raw_event_append_queue_with_capacity(DEFAULT_RAW_EVENT_QUEUE_CAPACITY) -} - -pub fn raw_event_append_queue_with_capacity( - capacity: usize, -) -> anyhow::Result<(RawEventAppendSender, RawEventAppendWorker)> { - raw_event_append_queue_with_options( - capacity, - DEFAULT_RAW_EVENT_COMPACTION_THRESHOLD, - DEFAULT_RAW_EVENT_TARGET_ROWS_PER_FRAGMENT, - DEFAULT_RAW_EVENT_HIERARCHY_FANOUT, - ) -} - -fn raw_event_append_queue_with_options( - capacity: usize, - compaction_threshold: usize, - target_rows_per_fragment: usize, - hierarchy_fanout: usize, -) -> anyhow::Result<(RawEventAppendSender, RawEventAppendWorker)> { - if capacity == 0 { - anyhow::bail!("pChronicle append queue capacity must be greater than zero"); - } - anyhow::ensure!( - compaction_threshold > 1, - "pChronicle compaction threshold must be greater than one" - ); - anyhow::ensure!( - target_rows_per_fragment > 0, - "pChronicle compaction target rows must be greater than zero" - ); - anyhow::ensure!( - hierarchy_fanout > 1, - "pChronicle compaction hierarchy fanout must be greater than one" - ); - - let (tx, rx) = mpsc::sync_channel::(capacity); - let state = Arc::new(SenderState { - tx, - accepting: AtomicBool::new(true), - in_flight: AtomicUsize::new(0), - }); - let join = std::thread::Builder::new() - .name("pchronicle-append".to_string()) - .spawn({ - let worker_state = Arc::clone(&state); - move || { - run_append_worker( - rx, - worker_state, - compaction_threshold, - target_rows_per_fragment, - hierarchy_fanout, - ) - } - }) - .context("spawn pChronicle append worker")?; - - Ok(( - RawEventAppendSender { - state: Arc::clone(&state), - }, - RawEventAppendWorker { - state, - join: Some(join), - }, - )) -} - -fn run_append_worker( - rx: mpsc::Receiver, - state: Arc, - compaction_threshold: usize, - target_rows_per_fragment: usize, - hierarchy_fanout: usize, -) -> anyhow::Result<()> { - let runtime = tokio::runtime::Builder::new_multi_thread() - .worker_threads(2) - .enable_all() - .build() - .context("create pChronicle append worker runtime")?; - let mut appender = RawEventLanceAppender::default(); - let (maintenance_tx, mut maintenance_rx) = - tokio::sync::mpsc::channel(DEFAULT_RAW_EVENT_MAINTENANCE_CAPACITY); - let maintenance_task = runtime.spawn(async move { - while let Some(segment) = maintenance_rx.recv().await { - if let Err(error) = - compact_sealed_event_segment(segment, target_rows_per_fragment, hierarchy_fanout) - .await - { - tracing::warn!( - target: "persisting_pchronicle", - "background raw-event compaction failed: {error:#}" - ); - } - } - }); - let mut finishing = false; - - while !finishing { - let first = match rx.recv() { - Ok(WriterMessage::Append(job)) => *job, - Ok(WriterMessage::Finish) | Err(_) => break, - }; - let mut jobs = Vec::with_capacity(DEFAULT_RAW_EVENT_BATCH_SIZE); - jobs.push(first); - while jobs.len() < DEFAULT_RAW_EVENT_BATCH_SIZE { - match rx.recv_timeout(DEFAULT_RAW_EVENT_BATCH_DELAY) { - Ok(WriterMessage::Append(job)) => jobs.push(*job), - Ok(WriterMessage::Finish) => { - finishing = true; - break; - } - Err(mpsc::RecvTimeoutError::Timeout) => break, - Err(mpsc::RecvTimeoutError::Disconnected) => { - finishing = true; - break; - } - } - } - - let mut completions = BTreeMap::>>>::new(); - let mut entries = Vec::with_capacity(jobs.len()); - for job in jobs { - match raw_event_lance_path(&job.coords) { - Ok(path) => { - let uri = path.to_string_lossy().into_owned(); - if let Some(completion) = job.completion { - completions.entry(uri).or_default().push(completion); - } else { - completions.entry(uri).or_default(); - } - entries.push((job.coords, job.record)); - } - Err(error) => { - if let Some(error) = notify_completion(job.completion, Err(error)) { - tracing::warn!( - target: "persisting_pchronicle", - "non-durable raw-event append path failed: {error:#}" - ); - } - } - } - } - if entries.is_empty() { - continue; - } - let append_result = runtime - .block_on(appender.append_event_batch_partitioned(&entries)) - .context("append event batch to pChronicle"); - match append_result { - Ok(mut report) => { - for (uri, partition_completions) in completions { - let result = match report.take_outcome(&uri) { - Some(Ok(_)) => Ok(()), - Some(Err(error)) => Err(error.context("append raw event partition")), - None => Err(anyhow::anyhow!( - "pChronicle append returned no partition outcome" - ) - .context(format!("append raw event partition {uri}"))), - }; - if let Some(error) = - notify_partition_completions(&uri, partition_completions, result) - { - tracing::warn!( - target: "persisting_pchronicle", - "non-durable raw-event append failed for {uri}: {error:#}" - ); - } - } - } - Err(error) => { - return Err(complete_terminal_append_failure( - &rx, - &state, - completions, - error, - )); - } - } - - match appender.seal_fragmented_segments(compaction_threshold) { - Ok(segments) => { - for segment in segments { - enqueue_maintenance(&maintenance_tx, segment); - } - } - Err(error) => { - // The append and its manifest publication are already durable. - // Keep maintenance best-effort so a compaction failure cannot - // turn a successful capture into a retry. - tracing::warn!( - target: "persisting_pchronicle", - "background raw-event segment sealing failed: {error:#}" - ); - } - } - } - - // Seal the final partial segments after producers have stopped. This keeps - // post-shutdown readers from inheriting a tail of tiny live fragments. - if let Ok(segments) = appender.seal_fragmented_segments(2) { - for segment in segments { - enqueue_maintenance(&maintenance_tx, segment); - } - } - drop(maintenance_tx); - if let Err(error) = runtime.block_on(maintenance_task) { - tracing::warn!( - target: "persisting_pchronicle", - "raw-event maintenance worker failed: {error}" - ); - } - - // Full compaction across sealed segments, indexing, and vacuum remain - // explicit pChronicle maintenance operations. - let _reports = appender.finish(); - Ok(()) -} - -fn notify_completion( - completion: Option>>, - result: anyhow::Result<()>, -) -> Option { - match (completion, result) { - (Some(completion), Ok(())) => { - let _ = completion.send(Ok(())); - None - } - (Some(completion), Err(error)) => send_owned_completion_error(&completion, error), - (None, Ok(())) => None, - (None, Err(error)) => Some(error), - } -} - -fn notify_partition_completions( - uri: &str, - completions: Vec>>, - result: anyhow::Result<()>, -) -> Option { - match result { - Ok(()) => { - for completion in completions { - let _ = completion.send(Ok(())); - } - None - } - Err(error) => { - let mut original = Some(error); - for completion in completions { - if let Some(error) = original.take() { - original = send_owned_completion_error(&completion, error); - } else { - let _ = completion.send(Err(partition_append_failure(uri))); - } - } - original - } - } -} - -fn send_owned_completion_error( - completion: &mpsc::SyncSender>, - error: anyhow::Error, -) -> Option { - match completion.send(Err(error)) { - Ok(()) => None, - Err(send_error) => send_error.0.err(), - } -} - -fn partition_append_failure(uri: &str) -> anyhow::Error { - anyhow::anyhow!("partition micro-batch failed; another waiter owns the storage failure") - .context(format!("append raw event partition {uri}")) -} - -fn enqueue_maintenance( - sender: &tokio::sync::mpsc::Sender, - segment: crate::store::SealedEventSegment, -) { - match sender.try_send(segment) { - Ok(()) => {} - Err(tokio::sync::mpsc::error::TrySendError::Full(_)) => tracing::warn!( - target: "persisting_pchronicle", - "raw-event maintenance backlog is full; explicit maintenance will recover layout" - ), - Err(tokio::sync::mpsc::error::TrySendError::Closed(_)) => tracing::warn!( - target: "persisting_pchronicle", - "raw-event maintenance worker stopped unexpectedly" - ), - } -} - -fn complete_terminal_append_failure( - rx: &mpsc::Receiver, - state: &SenderState, - completions: BTreeMap>>>, - error: anyhow::Error, -) -> anyhow::Error { - stop_and_reject_pending(rx, state); - let canonical_uri = completions - .iter() - .find(|(_, waiters)| !waiters.is_empty()) - .map(|(uri, _)| uri.clone()); - let mut original = Some(error); - for (uri, partition_completions) in completions { - for completion in partition_completions { - if let Some(error) = original.take() { - original = send_owned_completion_error(&completion, error); - } else { - let _ = completion.send(Err(partition_append_failure(&uri))); - } - } - } - - original.unwrap_or_else(|| { - let worker_failure = - anyhow::anyhow!("pChronicle append worker stopped after a terminal storage failure"); - match canonical_uri { - Some(uri) => worker_failure.context(format!("append raw event partition {uri}")), - None => worker_failure, - } - }) -} - -fn stop_and_reject_pending(rx: &mpsc::Receiver, state: &SenderState) { - state.accepting.store(false, Ordering::SeqCst); - while state.in_flight.load(Ordering::SeqCst) != 0 { - std::thread::yield_now(); - } - while let Ok(message_in_queue) = rx.try_recv() { - if let WriterMessage::Append(job) = message_in_queue { - let _ = notify_completion( - job.completion, - Err(anyhow::anyhow!( - "pChronicle append worker stopped before acknowledging the event" - )), - ); - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use serde_json::Value; - use std::collections::BTreeMap; - - use crate::store::RawEventLanceStore; - - pub(super) fn event() -> EventRecord { - EventRecord { - identity: Default::default(), - seq: 1, - source: "test".into(), - kind: "test".into(), - timestamp: None, - session_id: None, - agent_id: None, - parent_uuid: None, - trace_id: None, - call_id: None, - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload: Value::Null, - } - } - - #[test] - fn full_and_closed_are_expected_append_outcomes() { - let (tx, _rx) = mpsc::sync_channel(1); - let state = Arc::new(SenderState { - tx, - accepting: AtomicBool::new(true), - in_flight: AtomicUsize::new(0), - }); - let sender = RawEventAppendSender { - state: Arc::clone(&state), - }; - let coords = StoryCoords::new("memory://queue", "agent", "session", None); - assert_eq!( - sender.try_append(coords.clone(), event()), - RawEventAppendOutcome::Accepted - ); - assert_eq!( - sender.try_append(coords.clone(), event()), - RawEventAppendOutcome::Full - ); - state.accepting.store(false, Ordering::SeqCst); - assert_eq!( - sender.try_append(coords, event()), - RawEventAppendOutcome::Unavailable - ); - } - - #[test] - fn disconnected_queue_is_an_unavailable_append_outcome() { - let (tx, rx) = mpsc::sync_channel(1); - drop(rx); - let state = Arc::new(SenderState { - tx, - accepting: AtomicBool::new(true), - in_flight: AtomicUsize::new(0), - }); - let sender = RawEventAppendSender { state }; - - assert_eq!( - sender.try_append( - StoryCoords::new("memory://queue", "agent", "session", None), - event(), - ), - RawEventAppendOutcome::Unavailable - ); - } - - #[test] - fn durable_append_returns_only_after_event_is_replayable() { - let dir = tempfile::tempdir().unwrap(); - let storage = dir.path().join("store"); - let coords = StoryCoords::new(storage.to_string_lossy(), "agent", "session", None); - let (sender, worker) = raw_event_append_queue_with_capacity(1).unwrap(); - - assert_eq!( - sender.append_durable(coords.clone(), event()).unwrap(), - RawEventAppendOutcome::Accepted - ); - - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .unwrap(); - let replay = runtime - .block_on(RawEventLanceStore.replay(&coords, 0, None)) - .unwrap(); - assert_eq!(replay.records.len(), 1); - worker.finish().unwrap(); - } - - #[test] - fn durable_append_isolates_one_partition_failure() { - let dir = tempfile::tempdir().unwrap(); - let invalid_storage = dir.path().join("not-a-directory"); - std::fs::write(&invalid_storage, b"file").unwrap(); - let invalid = StoryCoords::new(invalid_storage.to_string_lossy(), "agent", "session", None); - let valid = StoryCoords::new( - dir.path().join("valid").to_string_lossy(), - "agent", - "session", - None, - ); - let (sender, worker) = raw_event_append_queue_with_capacity(1).unwrap(); - - let error = sender.append_durable(invalid, event()).unwrap_err(); - assert!( - error.chain().count() >= 2, - "missing source chain: {error:#}" - ); - assert_eq!( - sender.append_durable(valid.clone(), event()).unwrap(), - RawEventAppendOutcome::Accepted - ); - worker.finish().unwrap(); - - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .unwrap(); - let replay = runtime - .block_on(RawEventLanceStore.replay(&valid, 0, None)) - .unwrap(); - assert_eq!(replay.records.len(), 1); - } - - #[test] - fn append_worker_bounds_live_small_fragments() { - let dir = tempfile::tempdir().unwrap(); - let storage = dir.path().join("store"); - let coords = StoryCoords::new(storage.to_string_lossy(), "agent", "session", None); - let (sender, worker) = raw_event_append_queue().unwrap(); - - for _ in 0..(DEFAULT_RAW_EVENT_COMPACTION_THRESHOLD * 3) { - sender.append_durable(coords.clone(), event()).unwrap(); - } - worker.finish().unwrap(); - - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .unwrap(); - let layout = runtime - .block_on(RawEventLanceStore.layout_stats(&coords)) - .unwrap(); - assert_eq!( - layout.visible_rows, - (DEFAULT_RAW_EVENT_COMPACTION_THRESHOLD * 3) as u64 - ); - assert_eq!( - layout.visible_segments, 3, - "unexpected sealed layout: {layout:?}" - ); - assert_eq!( - layout.visible_fragments, 3, - "sealed segments were not compacted: {layout:?}" - ); - } - - #[test] - fn append_worker_compacts_sealed_segments_across_multiple_levels() { - let dir = tempfile::tempdir().unwrap(); - let storage = dir.path().join("store"); - let coords = StoryCoords::new(storage.to_string_lossy(), "agent", "session", None); - let (sender, worker) = raw_event_append_queue_with_options(32, 2, 2, 2).unwrap(); - - // 8 rows become four L0 segments, two L1 segments, and finally one L2 - // segment. Each merge preserves append order and total row count. - for seq in 0..8 { - let mut record = event(); - record.seq = seq; - sender.append_durable(coords.clone(), record).unwrap(); - } - worker.finish().unwrap(); - - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .unwrap(); - let layout = runtime - .block_on(RawEventLanceStore.layout_stats(&coords)) - .unwrap(); - assert_eq!(layout.visible_rows, 8); - assert_eq!(layout.max_segment_level, 2); - assert_eq!(layout.sealed_segments, 1); - assert_eq!( - layout.visible_segments, 1, - "hierarchy did not carry: {layout:?}" - ); - assert_eq!(layout.visible_fragments, 1); - let replay = runtime - .block_on(RawEventLanceStore.replay(&coords, 0, None)) - .unwrap(); - let sequences = replay - .records - .iter() - .map(|record| record.seq) - .collect::>(); - assert_eq!(sequences, (0..8).collect::>()); - } - - #[test] - fn multiple_durable_waiters_preserve_same_partition_order() { - let dir = tempfile::tempdir().unwrap(); - let storage = dir.path().join("store"); - let coords = StoryCoords::new(storage.to_string_lossy(), "agent", "session", None); - let (tx, rx) = mpsc::sync_channel(4); - let state = Arc::new(SenderState { - tx, - accepting: AtomicBool::new(true), - in_flight: AtomicUsize::new(0), - }); - let sender = RawEventAppendSender { - state: Arc::clone(&state), - }; - let (first_tx, first_rx) = mpsc::sync_channel(1); - let (second_tx, second_rx) = mpsc::sync_channel(1); - for (seq, completion) in [(1, Some(first_tx)), (2, Some(second_tx)), (3, None)] { - let mut record = event(); - record.seq = seq; - assert_eq!( - sender.enqueue(coords.clone(), record, completion), - RawEventAppendOutcome::Accepted - ); - } - - let worker_state = Arc::clone(&state); - let join = std::thread::spawn(move || { - run_append_worker( - rx, - worker_state, - DEFAULT_RAW_EVENT_COMPACTION_THRESHOLD, - DEFAULT_RAW_EVENT_TARGET_ROWS_PER_FRAGMENT, - DEFAULT_RAW_EVENT_HIERARCHY_FANOUT, - ) - }); - let worker = RawEventAppendWorker { - state, - join: Some(join), - }; - - first_rx.recv().unwrap().unwrap(); - second_rx.recv().unwrap().unwrap(); - - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .unwrap(); - let layout = runtime - .block_on(RawEventLanceStore.layout_stats(&coords)) - .unwrap(); - assert_eq!(layout.visible_rows, 3); - assert_eq!( - layout.visible_fragments, 1, - "one same-partition queue micro-batch must make one Lance commit" - ); - worker.finish().unwrap(); - let replay = runtime - .block_on(RawEventLanceStore.replay(&coords, 0, None)) - .unwrap(); - let sequences = replay - .records - .iter() - .map(|record| record.seq) - .collect::>(); - assert_eq!(sequences, vec![1, 2, 3]); - } - - #[test] - fn failed_partition_batch_rejects_later_waiters_without_a_sequence_gap() { - let dir = tempfile::tempdir().unwrap(); - let storage = dir.path().join("store"); - let coords = StoryCoords::new(storage.to_string_lossy(), "agent", "session", None); - let (tx, rx) = mpsc::sync_channel(4); - let state = Arc::new(SenderState { - tx, - accepting: AtomicBool::new(true), - in_flight: AtomicUsize::new(0), - }); - let sender = RawEventAppendSender { - state: Arc::clone(&state), - }; - let (first_tx, first_rx) = mpsc::sync_channel(1); - let (second_tx, second_rx) = mpsc::sync_channel(1); - let mut invalid = event(); - invalid.seq = 1; - invalid.identity.timestamp_unix_ms = Some(1_000); - invalid.timestamp = Some("1970-01-01T00:00:02Z".into()); - let mut second = event(); - second.seq = 2; - assert_eq!( - sender.enqueue(coords.clone(), invalid, Some(first_tx)), - RawEventAppendOutcome::Accepted - ); - assert_eq!( - sender.enqueue(coords.clone(), second, Some(second_tx)), - RawEventAppendOutcome::Accepted - ); - - let worker_state = Arc::clone(&state); - let join = std::thread::spawn(move || { - run_append_worker( - rx, - worker_state, - DEFAULT_RAW_EVENT_COMPACTION_THRESHOLD, - DEFAULT_RAW_EVENT_TARGET_ROWS_PER_FRAGMENT, - DEFAULT_RAW_EVENT_HIERARCHY_FANOUT, - ) - }); - let worker = RawEventAppendWorker { - state, - join: Some(join), - }; - - let first_error = first_rx.recv().unwrap().unwrap_err(); - assert!( - format!("{first_error:#}").contains("timestamp"), - "{first_error:#}" - ); - let second_error = second_rx.recv().unwrap().unwrap_err(); - assert!( - format!("{second_error:#}").contains("event partition"), - "{second_error:#}" - ); - - let mut third = event(); - third.seq = 3; - assert_eq!( - sender.append_durable(coords.clone(), third).unwrap(), - RawEventAppendOutcome::Accepted - ); - worker.finish().unwrap(); - - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .unwrap(); - let replay = runtime - .block_on(RawEventLanceStore.replay(&coords, 0, None)) - .unwrap(); - let sequences = replay - .records - .iter() - .map(|record| record.seq) - .collect::>(); - assert_eq!(sequences, vec![3]); - } - - #[test] - fn partition_failure_transfers_original_error_after_selected_receiver_is_dropped() { - let (dropped_tx, dropped_rx) = mpsc::sync_channel(1); - drop(dropped_rx); - let (live_tx, live_rx) = mpsc::sync_channel(1); - let source_error = anyhow::Error::new(std::io::Error::other("partition-source-sentinel")) - .context("append event partition"); - - let unclaimed = notify_partition_completions( - "memory://partition/events.lance", - vec![dropped_tx, live_tx], - Err(source_error), - ); - - assert!(unclaimed.is_none()); - let error = live_rx.recv().unwrap().unwrap_err(); - let rendered = format!("{error:#}"); - assert!(rendered.contains("append event partition"), "{rendered}"); - assert!(rendered.contains("partition-source-sentinel"), "{rendered}"); - } - - #[test] - fn terminal_append_failure_reaches_waiter_and_worker_finish() { - let (tx, rx) = mpsc::sync_channel(2); - tx.send(WriterMessage::Finish).unwrap(); - let state = Arc::new(SenderState { - tx, - accepting: AtomicBool::new(true), - in_flight: AtomicUsize::new(0), - }); - let (completion_tx, completion_rx) = mpsc::sync_channel(1); - let uri = "memory://terminal/events.lance".to_string(); - let completions = BTreeMap::from([(uri.clone(), vec![completion_tx])]); - let worker_state = Arc::clone(&state); - let join = std::thread::spawn(move || -> anyhow::Result<()> { - Err(complete_terminal_append_failure( - &rx, - &worker_state, - completions, - anyhow::Error::new(std::io::Error::other("terminal-source-sentinel")) - .context("append raw event batch"), - )) - }); - let worker = RawEventAppendWorker { - state, - join: Some(join), - }; - - let waiter_error = completion_rx.recv().unwrap().unwrap_err(); - let waiter_rendered = format!("{waiter_error:#}"); - assert!(waiter_rendered.contains("terminal-source-sentinel")); - - let finish_error = worker.finish().unwrap_err(); - let finish_rendered = format!("{finish_error:#}"); - assert!(finish_rendered.contains("terminal storage failure")); - assert!(finish_rendered.contains(&uri)); - } - - #[test] - fn terminal_failure_returns_original_when_all_completion_receivers_are_dropped() { - let (tx, rx) = mpsc::sync_channel(2); - tx.send(WriterMessage::Finish).unwrap(); - let state = Arc::new(SenderState { - tx, - accepting: AtomicBool::new(true), - in_flight: AtomicUsize::new(0), - }); - let (completion_tx, completion_rx) = mpsc::sync_channel(1); - drop(completion_rx); - let completions = BTreeMap::from([( - "memory://terminal/events.lance".to_string(), - vec![completion_tx], - )]); - let worker_state = Arc::clone(&state); - let join = std::thread::spawn(move || -> anyhow::Result<()> { - Err(complete_terminal_append_failure( - &rx, - &worker_state, - completions, - anyhow::Error::new(std::io::Error::other("dropped-terminal-source-sentinel")) - .context("append raw event batch"), - )) - }); - let worker = RawEventAppendWorker { - state, - join: Some(join), - }; - - let error = worker.finish().unwrap_err(); - let rendered = format!("{error:#}"); - assert!(rendered.contains("append raw event batch"), "{rendered}"); - assert!( - rendered.contains("dropped-terminal-source-sentinel"), - "{rendered}" - ); - } - - #[test] - fn worker_panic_is_reported_by_finish() { - let (tx, rx) = mpsc::sync_channel(1); - let state = Arc::new(SenderState { - tx, - accepting: AtomicBool::new(true), - in_flight: AtomicUsize::new(0), - }); - let join = std::thread::spawn(move || -> anyhow::Result<()> { - drop(rx); - panic!("injected append worker panic"); - }); - let worker = RawEventAppendWorker { - state, - join: Some(join), - }; - - let error = worker.finish().unwrap_err(); - assert!(error.to_string().contains("worker thread panicked")); - } - - #[test] - fn worker_close_failure_keeps_the_channel_source() { - let (tx, rx) = mpsc::sync_channel(1); - let (closed_tx, closed_rx) = mpsc::sync_channel(1); - let state = Arc::new(SenderState { - tx, - accepting: AtomicBool::new(true), - in_flight: AtomicUsize::new(0), - }); - let join = std::thread::spawn(move || -> anyhow::Result<()> { - drop(rx); - closed_tx.send(()).unwrap(); - Ok(()) - }); - let worker = RawEventAppendWorker { - state, - join: Some(join), - }; - closed_rx.recv().unwrap(); - - let error = worker.finish().unwrap_err(); - assert!( - error.chain().count() >= 2, - "missing source chain: {error:#}" - ); - } -} - -#[cfg(all(test, feature = "proptest"))] -mod proptests { - use super::*; - use proptest::prelude::*; - - proptest! { - #[test] - fn bounded_sender_reports_full_after_capacity( - capacity in 1usize..16, - attempts in 1usize..32, - ) { - let (tx, _rx) = mpsc::sync_channel(capacity); - let state = Arc::new(SenderState { - tx, - accepting: AtomicBool::new(true), - in_flight: AtomicUsize::new(0), - }); - let sender = RawEventAppendSender { state }; - let outcomes = (0..attempts) - .map(|_| { - sender.try_append( - StoryCoords::new("memory://queue", "agent", "session", None), - super::tests::event(), - ) - }) - .collect::>(); - - let accepted = outcomes - .iter() - .filter(|outcome| **outcome == RawEventAppendOutcome::Accepted) - .count(); - let full = outcomes - .iter() - .filter(|outcome| **outcome == RawEventAppendOutcome::Full) - .count(); - prop_assert_eq!(accepted, attempts.min(capacity)); - prop_assert_eq!(full, attempts.saturating_sub(capacity)); - } - } -} diff --git a/crates/persisting-pchronicle/src/convert/mod.rs b/crates/persisting-pchronicle/src/convert/mod.rs index 6e6dfc4f4..fde09b210 100644 --- a/crates/persisting-pchronicle/src/convert/mod.rs +++ b/crates/persisting-pchronicle/src/convert/mod.rs @@ -1,15 +1,4 @@ -//! Hub conversions: every format converts **only** through [`StorylineDocument`]. -//! -//! ```text -//! events.lance (in-memory EventRecord) ──┐ -//! agenticmd ─────────────────────────────┤ -//! openai_msg ────────────────────────────┼──► storyline ──► … -//! actf ──────────────────────────────────┘ -//! ``` -//! -//! Canonical Event and Storyline Lance are accessed through typed storage APIs. -//! Physical format codecs and Event ↔ Storyline conversion live in `formats/`. -//! This module remains the public conversion hub. +//! Conversions through StorylineDocument. #[cfg(feature = "lance-store")] pub(crate) use crate::formats::actf::actf_to_storylines; @@ -17,6 +6,3 @@ pub(crate) use crate::formats::actf::actf_to_storylines; pub(crate) use crate::formats::atif::atif_collection_to_storylines; #[cfg(test)] pub(crate) use crate::formats::atif::{atif_to_storyline, storyline_to_atif, storylines_to_atif}; -#[cfg(feature = "lance-store")] -pub(crate) use crate::formats::events::event_storyline_key; -pub use crate::formats::events::{events_to_storyline, project_event_records, storyline_to_events}; diff --git a/crates/persisting-pchronicle/src/discovery.rs b/crates/persisting-pchronicle/src/discovery.rs deleted file mode 100644 index a4339d5f6..000000000 --- a/crates/persisting-pchronicle/src/discovery.rs +++ /dev/null @@ -1,116 +0,0 @@ -//! Discover logical story partitions inside pChronicle storage. - -use anyhow::Result; - -use crate::layout::StoryCoords; -use crate::store::distinct_session_ids_in_run; - -fn is_shared_lance_run_bucket(location: &StoryCoords) -> bool { - location - .root_session_id - .as_deref() - .is_some_and(|root| root == location.session_id) -} - -/// Expand capture run buckets into one coordinate per stored `session_id`. -pub async fn expand_story_locations(locations: Vec) -> Result> { - let mut expanded = Vec::new(); - for location in locations { - if !is_shared_lance_run_bucket(&location) { - expanded.push(location); - continue; - } - let session_ids = distinct_session_ids_in_run(&location).await?; - if session_ids.is_empty() { - expanded.push(location); - continue; - } - for session_id in session_ids { - expanded.push(StoryCoords::new( - location.storage.clone(), - location.agent_id.clone(), - session_id, - location.root_session_id.clone(), - )); - } - } - expanded.sort_by(|a, b| { - ( - a.storage.as_str(), - a.agent_id.as_str(), - a.root_session_id.as_deref().unwrap_or(""), - a.session_id.as_str(), - ) - .cmp(&( - b.storage.as_str(), - b.agent_id.as_str(), - b.root_session_id.as_deref().unwrap_or(""), - b.session_id.as_str(), - )) - }); - Ok(expanded) -} - -/// Drop run-id lifecycle partitions when the same run contains real story partitions. -pub fn drop_lifecycle_run_partitions(locations: Vec) -> Vec { - use std::collections::{HashMap, HashSet}; - - let mut groups: HashMap<(String, String, Option), Vec> = HashMap::new(); - for (index, location) in locations.iter().enumerate() { - groups - .entry(( - location.storage.clone(), - location.agent_id.clone(), - location.root_session_id.clone(), - )) - .or_default() - .push(index); - } - - let mut drop = HashSet::new(); - for indices in groups.values() { - if indices.len() <= 1 { - continue; - } - for &index in indices { - let location = &locations[index]; - if location - .root_session_id - .as_deref() - .is_some_and(|root| root == location.session_id) - { - drop.insert(index); - } - } - } - - locations - .into_iter() - .enumerate() - .filter(|(index, _)| !drop.contains(index)) - .map(|(_, location)| location) - .collect() -} - -pub fn expand_story_locations_blocking(locations: Vec) -> Result> { - tokio::runtime::Builder::new_multi_thread() - .enable_all() - .build()? - .block_on(expand_story_locations(locations)) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn drops_lifecycle_partition_when_story_partition_exists() { - let locations = vec![ - StoryCoords::new("store", "agent", "run-1", Some("run-1".into())), - StoryCoords::new("store", "agent", "story-1", Some("run-1".into())), - ]; - let kept = drop_lifecycle_run_partitions(locations); - assert_eq!(kept.len(), 1); - assert_eq!(kept[0].session_id, "story-1"); - } -} diff --git a/crates/persisting-pchronicle/src/document.rs b/crates/persisting-pchronicle/src/document.rs index 650fd7ffd..9d4a10b3b 100644 --- a/crates/persisting-pchronicle/src/document.rs +++ b/crates/persisting-pchronicle/src/document.rs @@ -12,11 +12,9 @@ pub use crate::agenticmd::{ encode_agenticmd, index_agenticmd_path, list_agenticmd_paths, rewrite_agenticmd_storyline_metadata, upsert_agenticmd_turn, write_agenticmd_storyline, }; -pub use crate::convert::{events_to_storyline, project_event_records, storyline_to_events}; pub use crate::format::DocumentFormat; pub use crate::formats::detect_format; pub use crate::input::{InputIssue, InputIssueKind, InputResult}; -pub use crate::interop::{events_to_har, langfuse_otlp_json_to_events, otlp_json_to_events}; pub type Result = anyhow::Result; @@ -126,7 +124,6 @@ pub enum FilterPushdown { #[derive(Debug, Clone, Copy, PartialEq, Eq)] #[cfg(feature = "lance-store")] pub enum QueryTables { - Events, Storyline, } diff --git a/crates/persisting-pchronicle/src/format.rs b/crates/persisting-pchronicle/src/format.rs index 56b723d23..fcc82bf1c 100644 --- a/crates/persisting-pchronicle/src/format.rs +++ b/crates/persisting-pchronicle/src/format.rs @@ -9,8 +9,6 @@ use std::str::FromStr; /// 枚举只描述物理表示，不暗示所有格式支持相同的读写操作。 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum DocumentFormat { - /// Lance 中 append-only 的 Canonical Event 事实。 - CanonicalEvent, /// 严格版本化的 Storyline JSON wire。 Storyline, /// Lance 中的 Storyline runs、steps、tool calls 和 objects。 @@ -31,7 +29,6 @@ pub enum DocumentFormat { impl DocumentFormat { pub const ALL: &[Self] = &[ - Self::CanonicalEvent, Self::Storyline, Self::StorylineLance, Self::AgenticMd, @@ -44,7 +41,6 @@ impl DocumentFormat { pub const fn as_str(self) -> &'static str { match self { - Self::CanonicalEvent => "canonical-event", Self::Storyline => "storyline", Self::StorylineLance => "storyline-lance", Self::AgenticMd => "agenticmd", @@ -68,7 +64,6 @@ impl FromStr for DocumentFormat { fn from_str(input: &str) -> InputResult { match input.trim().to_ascii_lowercase().as_str() { - "canonical-event" => Ok(Self::CanonicalEvent), "storyline" => Ok(Self::Storyline), "storyline-lance" => Ok(Self::StorylineLance), "agenticmd" => Ok(Self::AgenticMd), @@ -78,7 +73,7 @@ impl FromStr for DocumentFormat { "codex" => Ok(Self::Codex), "claude-code" => Ok(Self::ClaudeCode), other => Err(InputIssue::invalid(format!( - "unknown document format '{other}'; expected canonical-event|storyline|storyline-lance|agenticmd|atif|openai-msg|actf|codex|claude-code" + "unknown document format '{other}'; expected storyline|storyline-lance|agenticmd|atif|openai-msg|actf|codex|claude-code" ))), } } @@ -92,7 +87,6 @@ mod tests { #[test] fn document_format_names_are_canonical_only() { let cases = [ - ("canonical-event", DocumentFormat::CanonicalEvent), ("storyline", DocumentFormat::Storyline), ("storyline-lance", DocumentFormat::StorylineLance), ("agenticmd", DocumentFormat::AgenticMd), diff --git a/crates/persisting-pchronicle/src/formats/codec.rs b/crates/persisting-pchronicle/src/formats/codec.rs index 87f317bff..d13960a08 100644 --- a/crates/persisting-pchronicle/src/formats/codec.rs +++ b/crates/persisting-pchronicle/src/formats/codec.rs @@ -1,6 +1,6 @@ //! Document-codec contract for JSON / JSONL / Markdown trajectory files. //! -//! Storage backends (Canonical Event Lance, Storyline Lance) are not codecs +//! Storage backends (Storyline Lance) are not codecs //! and must not implement this trait. use std::io::{BufRead, Write}; diff --git a/crates/persisting-pchronicle/src/formats/detect.rs b/crates/persisting-pchronicle/src/formats/detect.rs index 388d50360..4f7531287 100644 --- a/crates/persisting-pchronicle/src/formats/detect.rs +++ b/crates/persisting-pchronicle/src/formats/detect.rs @@ -8,7 +8,6 @@ use crate::formats::registry; /// Detect format from a file path (extension / basename). /// -/// `events` is detected only as a Lance dataset path (`events.lance`), never as `.json` / `.jsonl`. pub fn detect_format_from_path(path: impl AsRef) -> Option { let path = path.as_ref(); let name = path @@ -16,9 +15,7 @@ pub fn detect_format_from_path(path: impl AsRef) -> Option .and_then(|s| s.to_str()) .unwrap_or("") .to_ascii_lowercase(); - if name == "events.lance" || (name.ends_with(".lance") && name.contains("event")) { - return Some(DocumentFormat::CanonicalEvent); - } + if name == "session_steps.json" { return Some(DocumentFormat::OpenaiMsg); } @@ -36,7 +33,6 @@ pub fn detect_format_from_path(path: impl AsRef) -> Option /// Detect format from document text when path is unavailable. /// -/// Does **not** classify EventRecord-shaped JSON as `events` (Lance-only). pub fn detect_format_from_content(input: &str) -> Result> { if let Some(format) = registry::detect(None, input.as_bytes())? { return Ok(Some(format)); diff --git a/crates/persisting-pchronicle/src/formats/events/convert.rs b/crates/persisting-pchronicle/src/formats/events/convert.rs deleted file mode 100644 index 9ba3f405a..000000000 --- a/crates/persisting-pchronicle/src/formats/events/convert.rs +++ /dev/null @@ -1,956 +0,0 @@ -//! events ⇄ storyline. - -use std::collections::BTreeMap; - -use chrono::{DateTime, SecondsFormat, Utc}; -use serde_json::{Value, json}; - -use super::{ChronicleEventRecordExt, EventIdentity, EventRecord, EventsDocument}; -use crate::Result; -use crate::format::DocumentFormat; -use crate::formats::llm::LlmContentPart; -use crate::formats::storyline::{ - STORYLINE_SCHEMA_VERSION, StoryLink, StorylineAgent, StorylineDocument, StorylineOrigin, - StorylineToolCall, StorylineTurn, -}; -use crate::formats::timestamp::StorylineTimestamp; - -/// Resolve and project exactly one canonical Storyline from append-ordered events. -/// -/// Identity-free input is invalid, but conflicting claims are resolved in -/// append order: the last non-empty value wins. Physical event replay overlays -/// its routing identity first, so canonical storage remains deterministically -/// session-scoped without rejecting producer metadata at ingest. -pub fn project_event_records(records: &[EventRecord]) -> Result { - records - .iter() - .try_for_each(EventRecord::validate) - .map_err(anyhow::Error::from)?; - records - .iter() - .try_for_each(|record| canonical_event_timestamp(record).map(|_| ()))?; - let session_id = records - .iter() - .rev() - .find_map(event_storyline_key) - .ok_or_else(|| { - anyhow::anyhow!("canonical event requires session_id, storyline_id, or run_id") - })? - .to_string(); - let run_id = records.iter().rev().find_map(|record| { - record - .identity - .run_id - .as_deref() - .filter(|id| !id.is_empty()) - .map(str::to_string) - }); - let mut story = events_to_storyline_unchecked(records)?; - story.session_id = session_id.clone(); - story.run_id = run_id.clone().filter(|run_id| run_id != &session_id); - if let Some(parent_session_id) = run_id.filter(|run_id| run_id != &session_id) { - story.parent = Some(StoryLink { - parent_session_id, - spawn_call_id: None, - spawn_id: None, - relation: "spawn".into(), - }); - } - Ok(story) -} - -pub(crate) fn event_storyline_key(record: &EventRecord) -> Option<&str> { - record - .session_id - .as_deref() - .filter(|id| !id.is_empty()) - .or_else(|| { - record - .identity - .storyline_id - .as_deref() - .filter(|id| !id.is_empty()) - }) - .or_else(|| { - record - .identity - .run_id - .as_deref() - .filter(|id| !id.is_empty()) - }) -} - -pub fn events_to_storyline(doc: &EventsDocument) -> Result { - project_event_records(&doc.events) -} - -fn events_to_storyline_unchecked(events: &[EventRecord]) -> Result { - let session_id = events - .iter() - .rev() - .find_map(|event| event.session_id.clone()) - .unwrap_or_else(|| "unknown".into()); - let agent_id = events - .iter() - .rev() - .find_map(|event| event.agent_id.clone()) - .unwrap_or_else(|| "unknown".into()); - - let tool_results = collect_tool_results(events)?; - let mut by_call: BTreeMap> = BTreeMap::new(); - let mut first_call_position = BTreeMap::::new(); - let mut orphans: Vec<&EventRecord> = Vec::new(); - for (position, ev) in events.iter().enumerate() { - match &ev.call_id { - Some(cid) if !cid.is_empty() => { - first_call_position.entry(cid.clone()).or_insert(position); - by_call.entry(cid.clone()).or_default().push(ev); - } - _ => orphans.push(ev), - } - } - - let mut turns = Vec::new(); - let mut next_id = 1i64; - - let mut call_order: Vec<(usize, String)> = first_call_position - .into_iter() - .map(|(cid, position)| (position, cid)) - .collect(); - call_order.sort_by_key(|(position, _)| *position); - - for (_, cid) in call_order { - let evs = &by_call[&cid]; - let tool_calls = collect_tool_calls(evs, &tool_results)?; - let first_ts = evs - .first() - .map(|event| canonical_event_timestamp(event)) - .transpose()? - .flatten(); - let mut model = None; - let mut user_text = None; - let mut asst_text = None; - let mut metrics = None; - let mut ttft_ms = None; - let mut latency_from_payload = None; - let mut req_ts = None; - let mut resp_ts = None; - let mut request_messages = None; - for ev in evs { - let timestamp = canonical_event_timestamp(ev)?; - match ev.kind.as_str() { - "llm.request" | "http.request" => { - req_ts = timestamp; - model = ev - .payload - .get("model") - .and_then(|v| v.as_str()) - .map(str::to_string); - user_text = extract_user(&ev.payload); - request_messages = ev.payload.get("messages").cloned(); - } - "llm.response" - | "llm.response.stream" - | "http.response" - | "http.response.stream" => { - resp_ts = timestamp; - asst_text = extract_assistant(&ev.payload); - if model.is_none() { - model = ev - .llm_response_payload()? - .and_then(|payload| payload.response.model); - } - if let Some(u) = ev.payload.get("usage") { - metrics = Some(u.clone()); - } - ttft_ms = ev - .payload - .get("ttft_ms") - .and_then(|v| v.as_i64()) - .or_else(|| { - metrics - .as_ref() - .and_then(|m| m.get("ttft_ms")) - .and_then(|v| v.as_i64()) - }); - if latency_from_payload.is_none() { - latency_from_payload = - ev.payload.get("latency_ms").and_then(|v| v.as_i64()); - } - } - _ => {} - } - } - - let latency_ms = - latency_from_payload.or_else(|| latency_between(req_ts.as_ref(), resp_ts.as_ref())); - - let req_seq = evs - .iter() - .find(|e| matches!(e.kind.as_str(), "llm.request" | "http.request")) - .map(|e| e.seq); - let resp_seq = evs - .iter() - .find(|e| { - matches!( - e.kind.as_str(), - "llm.response" - | "llm.response.stream" - | "http.response" - | "http.response.stream" - ) - }) - .map(|e| e.seq); - - if let Some(ut) = user_text.clone() - && asst_text.is_some() - { - let mut user_extra = json!({"call_id": cid}); - if let Some(seq) = req_seq { - user_extra["seq"] = json!(seq); - } - turns.push(StorylineTurn { - id: next_id, - kind: Some("llm.request".into()), - timestamp: req_ts.clone().or_else(|| first_ts.clone()), - source: "user".into(), - message: serde_json::Value::String(ut), - reasoning_content: None, - reasoning_effort: None, - tool_calls: None, - observation: None, - metrics: None, - model_name: None, - llm_call_count: None, - is_copied_context: None, - latency_ms: None, - ttft_ms: None, - extra: Some(user_extra), - env: None, - prompt: None, - finished_at: None, - }); - next_id += 1; - } - - let (source, message, turn_kind, turn_seq) = if let Some(a) = asst_text { - ( - "agent", - serde_json::Value::String(a), - "llm.response", - resp_seq, - ) - } else if let Some(u) = user_text { - ("user", serde_json::Value::String(u), "llm.request", req_seq) - } else { - ( - "agent", - serde_json::Value::String(String::new()), - "llm.response", - resp_seq.or(req_seq), - ) - }; - - let mut agent_extra = json!({ - "call_id": cid, - "request_messages": request_messages, - }); - if let Some(seq) = turn_seq { - agent_extra["seq"] = json!(seq); - } - - turns.push(StorylineTurn { - id: next_id, - kind: Some(turn_kind.into()), - timestamp: resp_ts.or(req_ts).or(first_ts), - source: source.into(), - message, - reasoning_content: None, - reasoning_effort: None, - tool_calls: (!tool_calls.is_empty()).then_some(tool_calls), - observation: None, - metrics, - model_name: model, - llm_call_count: if source == "agent" { Some(1) } else { None }, - is_copied_context: None, - latency_ms: if source == "agent" { latency_ms } else { None }, - ttft_ms: if source == "agent" { ttft_ms } else { None }, - extra: Some(agent_extra), - env: None, - prompt: None, - finished_at: None, - }); - next_id += 1; - } - - for ev in orphans { - if ev.kind.starts_with("session.") || ev.kind == "note" { - turns.push(StorylineTurn { - id: next_id, - kind: Some("internal".into()), - timestamp: canonical_event_timestamp(ev)?, - source: "system".into(), - message: json!({"kind": ev.kind, "payload": ev.payload}), - reasoning_content: None, - reasoning_effort: None, - tool_calls: None, - observation: None, - metrics: None, - model_name: None, - llm_call_count: None, - is_copied_context: None, - latency_ms: None, - ttft_ms: None, - extra: Some(json!({"seq": ev.seq, "event_seq": ev.seq})), - env: None, - prompt: None, - finished_at: None, - }); - next_id += 1; - } - } - - let agent_model_name = turns.iter().rev().find_map(|turn| turn.model_name.clone()); - - Ok(StorylineDocument { - schema_version: STORYLINE_SCHEMA_VERSION.into(), - origin: Some(StorylineOrigin { - format: DocumentFormat::CanonicalEvent.as_str().into(), - schema_version: None, - document_id: None, - }), - run_id: None, - trajectory_id: None, - attempt_id: None, - session_id, - agent: StorylineAgent { - id: agent_id.clone(), - name: Some(agent_id), - version: None, - model_name: agent_model_name, - tool_definitions: None, - extra: None, - }, - parent: None, - child_session_ids: None, - notes: None, - final_metrics: None, - continued_trajectory_ref: None, - extra: None, - meta: None, - task: None, - prompt: None, - started_at: None, - finished_at: None, - unknown_fields: Default::default(), - unknown_key_counts: Default::default(), - turns, - }) -} - -fn collect_tool_results(events: &[EventRecord]) -> Result> { - let mut results = BTreeMap::new(); - for event in events { - let Some(payload) = event.llm_request_payload()? else { - continue; - }; - for message in payload.request.messages { - for part in message.parts { - if let LlmContentPart::ToolResult { - call_id, content, .. - } = part - && !call_id.is_empty() - { - results.insert(call_id, content); - } - } - } - } - Ok(results) -} - -fn collect_tool_calls( - events: &[&EventRecord], - results: &BTreeMap, -) -> Result> { - let mut calls = Vec::::new(); - let mut positions = BTreeMap::::new(); - for event in events { - let Some(payload) = event.llm_response_payload()? else { - continue; - }; - for candidate in payload.response.candidates { - for part in candidate.message.parts { - let LlmContentPart::ToolCall { - id, - name, - arguments, - signature, - } = part - else { - continue; - }; - if id.is_empty() { - continue; - } - let result = results.get(&id).cloned(); - let extra = signature.map(|signature| json!({"signature": signature})); - if let Some(position) = positions.get(&id).copied() { - calls[position] = StorylineToolCall { - tool_call_id: id, - function_name: name, - arguments, - result, - duration_ms: None, - extra, - kind: None, - response: None, - }; - } else { - positions.insert(id.clone(), calls.len()); - calls.push(StorylineToolCall { - tool_call_id: id, - function_name: name, - arguments, - result, - duration_ms: None, - extra, - kind: None, - response: None, - }); - } - } - } - } - Ok(calls) -} - -pub fn storyline_to_events(story: &StorylineDocument) -> Result { - story.validate()?; - let mut events = Vec::new(); - let mut next_seq = 0u64; - let mut i = 0; - while i < story.turns.len() { - let turn = &story.turns[i]; - if turn.effective_kind() == "internal" { - let seq = take_seq(turn.extra.as_ref(), &mut next_seq); - events.push(EventRecord { - identity: EventIdentity::default(), - seq, - source: "pchronicle".into(), - kind: "note".into(), - timestamp: storyline_timestamp_text(turn.timestamp.as_ref()), - session_id: Some(story.session_id.clone()), - agent_id: Some(story.agent.id.clone()), - parent_uuid: None, - trace_id: None, - call_id: None, - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload: turn.message.clone(), - }); - i += 1; - continue; - } - - let explicit_call_id = turn - .extra - .as_ref() - .and_then(|e| e.get("call_id").and_then(|c| c.as_str())) - .filter(|s| !s.is_empty()) - .map(str::to_string); - - // Pair consecutive user → agent turns under one call_id so events→storyline - // can rebuild dialogue (orphans without call_id are otherwise dropped). - let paired_agent = if turn.source == "user" { - story - .turns - .get(i + 1) - .filter(|t| t.source == "agent" && t.effective_kind() != "internal") - } else { - None - }; - let call_id = explicit_call_id.or_else(|| { - paired_agent - .and_then(|a| { - a.extra - .as_ref() - .and_then(|e| e.get("call_id").and_then(|c| c.as_str())) - .filter(|s| !s.is_empty()) - .map(str::to_string) - }) - .or_else(|| Some(format!("turn-{}", turn.id))) - }); - - let text = message_text(&turn.message).unwrap_or_default(); - - match turn.source.as_str() { - "user" => { - let seq = take_seq(turn.extra.as_ref(), &mut next_seq); - let req_kind = turn - .kind - .as_deref() - .filter(|k| k.starts_with("llm.") || k.starts_with("http.")) - .unwrap_or("llm.request"); - events.push(EventRecord { - identity: EventIdentity::default(), - seq, - source: "pchronicle".into(), - kind: req_kind.into(), - timestamp: storyline_timestamp_text(turn.timestamp.as_ref()), - session_id: Some(story.session_id.clone()), - agent_id: Some(story.agent.id.clone()), - parent_uuid: None, - trace_id: None, - call_id: call_id.clone(), - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload: json!({ - "model": turn.model_name, - "messages": [{"role":"user","content": text}], - }), - }); - - if let Some(agent) = paired_agent { - let atext = message_text(&agent.message).unwrap_or_default(); - let seq = take_seq(agent.extra.as_ref(), &mut next_seq); - let resp_kind = agent - .kind - .as_deref() - .filter(|k| k.starts_with("llm.") || k.starts_with("http.")) - .unwrap_or("llm.response"); - events.push(EventRecord { - identity: EventIdentity::default(), - seq, - source: "pchronicle".into(), - kind: resp_kind.into(), - timestamp: storyline_timestamp_text(agent.timestamp.as_ref()), - session_id: Some(story.session_id.clone()), - agent_id: Some(story.agent.id.clone()), - parent_uuid: None, - trace_id: None, - call_id, - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload: json!({ - "content": atext, - "choices":[{"message":{"role":"assistant","content": atext}}], - "usage": agent.metrics, - "latency_ms": agent.latency_ms, - "ttft_ms": agent.ttft_ms, - }), - }); - i += 2; - continue; - } - } - _ => { - if let Some(msgs) = turn - .extra - .as_ref() - .and_then(|e| e.get("request_messages").cloned()) - { - let seq = take_seq(None, &mut next_seq); - events.push(EventRecord { - identity: EventIdentity::default(), - seq, - source: "pchronicle".into(), - kind: "llm.request".into(), - timestamp: storyline_timestamp_text(turn.timestamp.as_ref()), - session_id: Some(story.session_id.clone()), - agent_id: Some(story.agent.id.clone()), - parent_uuid: None, - trace_id: None, - call_id: call_id.clone(), - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload: json!({ - "model": turn.model_name, - "messages": msgs, - }), - }); - } - let seq = take_seq(turn.extra.as_ref(), &mut next_seq); - let resp_kind = turn - .kind - .as_deref() - .filter(|k| k.starts_with("llm.") || k.starts_with("http.")) - .unwrap_or("llm.response"); - events.push(EventRecord { - identity: EventIdentity::default(), - seq, - source: "pchronicle".into(), - kind: resp_kind.into(), - timestamp: storyline_timestamp_text(turn.timestamp.as_ref()), - session_id: Some(story.session_id.clone()), - agent_id: Some(story.agent.id.clone()), - parent_uuid: None, - trace_id: None, - call_id, - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload: json!({ - "content": text, - "choices":[{"message":{"role":"assistant","content": text}}], - "usage": turn.metrics, - "latency_ms": turn.latency_ms, - "ttft_ms": turn.ttft_ms, - }), - }); - } - } - i += 1; - } - - Ok(EventsDocument { - format: EventsDocument::FORMAT_NAME.into(), - session_id: Some(story.session_id.clone()), - agent_id: Some(story.agent.id.clone()), - events, - }) -} - -/// Prefer explicit `extra.seq`; otherwise allocate the next monotonic seq. -fn take_seq(extra: Option<&serde_json::Value>, next_seq: &mut u64) -> u64 { - if let Some(seq) = extra.and_then(|e| e.get("seq")).and_then(|v| v.as_u64()) { - if seq >= *next_seq { - *next_seq = seq + 1; - } - return seq; - } - let seq = *next_seq; - *next_seq += 1; - seq -} - -fn latency_between( - req: Option<&StorylineTimestamp>, - resp: Option<&StorylineTimestamp>, -) -> Option { - Some(resp?.instant().timestamp_millis() - req?.instant().timestamp_millis()) -} - -fn canonical_event_timestamp(record: &EventRecord) -> Result> { - let textual = record - .timestamp - .as_deref() - .map(StorylineTimestamp::from_rfc3339) - .transpose()?; - let textual_ms = textual - .as_ref() - .map(|timestamp| timestamp.instant().timestamp_millis()); - let canonical_ms = record - .identity - .timestamp_unix_ms - .map(|timestamp| { - i64::try_from(timestamp) - .map_err(|_| anyhow::anyhow!("event timestamp_unix_ms exceeds i64")) - }) - .transpose()?; - if let (Some(canonical), Some(textual)) = (canonical_ms, textual_ms) - && canonical != textual - { - anyhow::bail!( - "event timestamp conflict: timestamp_unix_ms={canonical}, RFC3339 timestamp={textual}" - ); - } - if let Some(timestamp) = textual { - return Ok(Some(timestamp)); - } - let Some(timestamp_ms) = canonical_ms else { - return Ok(None); - }; - let timestamp = DateTime::::from_timestamp_millis(timestamp_ms) - .ok_or_else(|| anyhow::anyhow!("event timestamp is outside the RFC3339 range"))?; - Ok(Some(StorylineTimestamp::from_rfc3339( - ×tamp.to_rfc3339_opts(SecondsFormat::Millis, true), - )?)) -} - -fn storyline_timestamp_text(timestamp: Option<&StorylineTimestamp>) -> Option { - timestamp.map(StorylineTimestamp::source_string_or_canonical) -} - -fn extract_user(payload: &serde_json::Value) -> Option { - payload - .get("user_content") - .and_then(|v| v.as_str()) - .map(str::to_string) - .or_else(|| { - payload - .get("messages") - .and_then(|m| m.as_array()) - .and_then(|arr| { - arr.iter().rev().find_map(|m| { - if m.get("role").and_then(|r| r.as_str()) == Some("user") { - m.get("content").and_then(|c| match c { - serde_json::Value::String(s) => Some(s.clone()), - _ => None, - }) - } else { - None - } - }) - }) - }) -} - -fn extract_assistant(payload: &serde_json::Value) -> Option { - payload - .get("assistant_content") - .and_then(|v| v.as_str()) - .map(str::to_string) - .or_else(|| { - payload - .pointer("/choices/0/message/content") - .and_then(|v| v.as_str()) - .map(str::to_string) - }) - .or_else(|| { - payload - .get("content") - .and_then(|v| v.as_str()) - .map(str::to_string) - }) -} - -fn message_text(message: &Value) -> Option { - match message { - Value::String(s) => Some(s.clone()), - Value::Array(parts) => { - let texts: Vec<_> = parts - .iter() - .filter_map(|p| p.get("text").and_then(|t| t.as_str())) - .collect(); - if texts.is_empty() { - None - } else { - Some(texts.join("")) - } - } - _ => None, - } -} - -#[cfg(test)] -mod projection_tests { - use super::*; - - fn response(session_id: Option<&str>, call_id: &str, seq: u64, content: &str) -> EventRecord { - EventRecord { - identity: EventIdentity::default(), - seq, - source: "test".into(), - kind: "llm.response".into(), - timestamp: None, - session_id: session_id.map(str::to_string), - agent_id: Some("agent".into()), - parent_uuid: None, - trace_id: None, - call_id: Some(call_id.into()), - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload: json!({"content": content}), - } - } - - #[test] - fn canonical_projection_rejects_missing_storyline_identity() { - let error = project_event_records(&[response(None, "call", 0, "text")]).unwrap_err(); - assert!(error.to_string().contains("requires session_id")); - } - - #[test] - fn canonical_projection_uses_append_order_before_producer_seq() { - let records = vec![ - response(Some("session"), "later-seq", 100, "first"), - response(Some("session"), "earlier-seq", 1, "second"), - ]; - let story = project_event_records(&records).unwrap(); - assert_eq!(story.turns[0].message, json!("first")); - assert_eq!(story.turns[1].message, json!("second")); - } - - #[test] - fn canonical_projection_uses_timestamp_unix_ms() { - let mut record = response(Some("session"), "call", 0, "text"); - record.identity.timestamp_unix_ms = Some(1_000); - let story = project_event_records(&[record]).unwrap(); - assert_eq!( - story.turns[0] - .timestamp - .as_ref() - .and_then(StorylineTimestamp::source_string), - Some("1970-01-01T00:00:01.000Z") - ); - } - - #[test] - fn canonical_projection_rejects_timestamp_conflicts() { - let mut record = response(Some("session"), "call", 0, "text"); - record.identity.timestamp_unix_ms = Some(1_000); - record.timestamp = Some("1970-01-01T00:00:02Z".into()); - let error = project_event_records(&[record]).unwrap_err(); - assert!(error.to_string().contains("timestamp conflict")); - } - - #[test] - fn canonical_projection_materializes_stream_tool_calls_and_followup_results() { - let mut response = response(Some("session"), "model-call-1", 0, ""); - response.kind = "llm.response.stream".into(); - response.payload = json!({ - "assistant_content": "", - "usage": {"input_tokens": 10, "output_tokens": 4, "total_tokens": 14}, - "llm_response": { - "output_format": "chat_completions", - "response": { - "model": "test-model", - "candidates": [{ - "index": 0, - "message": { - "role": "assistant", - "parts": [{ - "type": "tool_call", - "id": "tool-call-1", - "name": "lookup", - "arguments": {"query": "pchronicle"} - }] - }, - "finish_reason": "tool_calls" - }], - "usage": {"input_tokens": 10, "output_tokens": 4, "total_tokens": 14} - } - } - }); - let followup = EventRecord { - identity: EventIdentity::default(), - seq: 1, - source: "test".into(), - kind: "llm.request".into(), - timestamp: None, - session_id: Some("session".into()), - agent_id: Some("agent".into()), - parent_uuid: None, - trace_id: None, - call_id: Some("model-call-2".into()), - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload: json!({ - "user_content": "tool result", - "llm_request": { - "input_format": "chat_completions", - "request": { - "messages": [{ - "role": "tool", - "parts": [{ - "type": "tool_result", - "call_id": "tool-call-1", - "name": "lookup", - "content": {"ok": true} - }] - }], - "generation": {}, - "stream": true - } - } - }), - }; - - let story = project_event_records(&[response, followup]).unwrap(); - let calls = story.turns[0].tool_calls.as_ref().unwrap(); - assert_eq!(calls.len(), 1); - assert_eq!(calls[0].tool_call_id, "tool-call-1"); - assert_eq!(calls[0].function_name, "lookup"); - assert_eq!(calls[0].arguments, json!({"query": "pchronicle"})); - assert_eq!(calls[0].result.as_ref(), Some(&json!({"ok": true}))); - assert_eq!(story.agent.model_name.as_deref(), Some("test-model")); - } - - #[test] - fn public_projection_resolves_conflicting_identity_in_append_order() { - let mut first = response(Some("first"), "call-1", 0, "one"); - first.identity.run_id = Some("run-first".into()); - first.agent_id = Some("agent-first".into()); - let mut second = response(Some("second"), "call-2", 1, "two"); - second.identity.run_id = Some("run-second".into()); - second.agent_id = Some("agent-second".into()); - let story = events_to_storyline(&EventsDocument::new(vec![first, second])).unwrap(); - assert_eq!(story.session_id, "second"); - assert_eq!(story.run_id.as_deref(), Some("run-second")); - assert_eq!(story.agent.id, "agent-second"); - - let mut conflicting = response(Some("session"), "call", 0, "text"); - conflicting.identity.storyline_id = Some("other".into()); - let story = project_event_records(&[conflicting]).unwrap(); - assert_eq!(story.session_id, "session"); - } -} - -#[cfg(all(test, feature = "proptest"))] -mod proptests { - use proptest::prelude::*; - use serde_json::json; - - use super::*; - - fn id_strategy() -> impl Strategy { - proptest::string::string_regex("[a-zA-Z0-9._-]{1,24}").unwrap() - } - - fn response_event( - session: &str, - agent: &str, - call_id: &str, - seq: u64, - content: &str, - ) -> EventRecord { - EventRecord { - identity: EventIdentity::default(), - seq, - source: "test".into(), - kind: "llm.response".into(), - timestamp: None, - session_id: Some(session.into()), - agent_id: Some(agent.into()), - parent_uuid: None, - trace_id: None, - call_id: Some(call_id.into()), - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload: json!({"content": content}), - } - } - - proptest! { - #[test] - fn event_storyline_key_uses_session_then_storyline_then_run( - session in prop::option::of(id_strategy()), - storyline in prop::option::of(id_strategy()), - run in prop::option::of(id_strategy()), - ) { - prop_assume!(session.is_some() || storyline.is_some() || run.is_some()); - let mut record = response_event("fallback", "agent", "call", 1, "text"); - record.session_id = session.clone(); - record.identity.storyline_id = storyline.clone(); - record.identity.run_id = run.clone(); - let expected = session.as_deref().or(storyline.as_deref()).or(run.as_deref()); - prop_assert_eq!(event_storyline_key(&record), expected); - } - } -} diff --git a/crates/persisting-pchronicle/src/formats/events/mod.rs b/crates/persisting-pchronicle/src/formats/events/mod.rs deleted file mode 100644 index 418118e2b..000000000 --- a/crates/persisting-pchronicle/src/formats/events/mod.rs +++ /dev/null @@ -1,254 +0,0 @@ -//! `events` — Capture HTTP-first SoT, with Lance as the canonical query format. -//! -//! Production storage is `{run}/events.lance/` (Lance dataset). pChronicle does -//! The pChronicle control path may additionally archive complete EventRecords -//! as JSON in a warehouse prefix; conversion/query APIs below still operate on -//! Lance rows. -//! -//! What this module provides: -//! - [`EventRecord`] / [`EventsDocument`]: in-memory shape aligned with EventRecord -//! - [`events_to_storyline`] / [`storyline_to_events`] (programmatic, after you -//! already loaded rows from Lance) -//! - [`export_events_jsonl`]: test/debug export from canonical Lance rows -//! -//! Use the pChronicle APIs to extract Lance events for inspection. - -use serde::{Deserialize, Serialize}; - -use crate::Result; -pub use persisting_events::{EventIdentity, EventRecord}; - -mod convert; -#[cfg(feature = "lance-store")] -pub(crate) use convert::event_storyline_key; -pub use convert::{events_to_storyline, project_event_records, storyline_to_events}; - -/// In-memory batch of events (not a file format). -/// -/// Built after reading Lance rows or in unit tests. Do not treat serialized -/// [`EventsDocument`] JSON as a supported interchange format. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct EventsDocument { - pub format: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub session_id: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub agent_id: Option, - pub events: Vec, -} - -impl EventsDocument { - pub const FORMAT_NAME: &'static str = "events"; - - pub fn new(events: Vec) -> Self { - let session_id = events.iter().rev().find_map(|e| e.session_id.clone()); - let agent_id = events.iter().rev().find_map(|e| e.agent_id.clone()); - Self { - format: Self::FORMAT_NAME.into(), - session_id, - agent_id, - events, - } - } -} - -pub trait ChronicleEventRecordExt { - /// Decode the canonical typed semantics attached to an `llm.request` event. - /// Wire-only and legacy events legitimately return `None`. - fn llm_request_payload(&self) -> Result>; - - /// Decode the canonical typed semantics attached to an `llm.response` event. - fn llm_response_payload(&self) -> Result>; -} - -impl ChronicleEventRecordExt for EventRecord { - fn llm_request_payload(&self) -> Result> { - let Some(payload) = self.payload.get("llm_request") else { - return Ok(None); - }; - Ok(Some(serde_json::from_value(payload.clone())?)) - } - - fn llm_response_payload(&self) -> Result> { - let Some(payload) = self.payload.get("llm_response") else { - return Ok(None); - }; - Ok(Some(serde_json::from_value(payload.clone())?)) - } -} - -/// Error message shared by string-based convert APIs. -#[cfg(all(test, feature = "lance-store"))] -pub fn events_lance_only_message() -> &'static str { - "events is Lance-only (events.lance); JSON/JSONL is not a supported wire format. \ - Extract with traj/export tools, or call events_to_storyline / storyline_to_events \ - on in-memory EventRecord batches after loading Lance." -} - -/// Debug/export helper: serialize records as JSONL. **Not** a chronicle format. -#[cfg(all(test, feature = "lance-store"))] -pub fn export_events_jsonl(events: &[EventRecord]) -> Result { - let mut out = String::new(); - for event in events { - out.push_str(&serde_json::to_string(event)?); - out.push('\n'); - } - Ok(out) -} - -/// Debug/export helper: pretty JSON document. **Not** a chronicle format. -/// Test/fixture helper: parse JSONL into memory. **Not** part of the public format API surface -/// for `into_storyline` / `convert`. -#[cfg(all(test, feature = "lance-store"))] -pub fn parse_events_jsonl_for_test(input: &str) -> Result { - let mut events = Vec::new(); - for line in input.lines() { - let line = line.trim(); - if line.is_empty() { - continue; - } - let event = serde_json::from_str::(line)?; - events.push(event); - } - Ok(EventsDocument::new(events)) -} - -#[cfg(test)] -mod tests { - use super::*; - use serde_json::Value; - - #[test] - fn event_wire_has_no_schema_version() { - let value = serde_json::json!({ - "seq": 0, - "source": "capture", - "kind": "llm.request", - "payload": {} - }); - let record = serde_json::from_value::(value).unwrap(); - assert!( - serde_json::to_value(record) - .unwrap() - .get("schema_version") - .is_none() - ); - } - - #[test] - fn runtime_identity_is_flattened() { - let mut record = EventRecord { - identity: EventIdentity::default(), - seq: 0, - source: "capture".into(), - kind: "llm.request".into(), - timestamp: None, - session_id: None, - agent_id: None, - parent_uuid: None, - trace_id: None, - call_id: None, - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload: Value::Object(Default::default()), - }; - record.identity.event_id = Some("event-1".into()); - record.identity.run_id = Some("run-1".into()); - let encoded = serde_json::to_value(record).unwrap(); - assert_eq!(encoded["event_id"], "event-1"); - assert_eq!(encoded["run_id"], "run-1"); - assert!(encoded.get("identity").is_none()); - } - - #[test] - fn typed_llm_payload_decodes_from_event_envelope() { - let record = EventRecord { - identity: EventIdentity::default(), - seq: 1, - source: "gateway".into(), - kind: "llm.request".into(), - timestamp: None, - session_id: None, - agent_id: None, - parent_uuid: None, - trace_id: None, - call_id: None, - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload: serde_json::json!({ - "llm_request": { - "input_format":"chat_completions", - "request": { - "model":"m", - "system":[], - "messages":[], - "generation":{}, - "stream":false - } - } - }), - }; - let payload = record.llm_request_payload().unwrap().unwrap(); - assert_eq!(payload.request.model.as_deref(), Some("m")); - assert_eq!( - payload.input_format, - super::super::llm::LlmProtocol::ChatCompletions - ); - } - - #[cfg(feature = "proptest")] - mod proptests { - use proptest::prelude::*; - use serde_json::Value; - - use crate::formats::events::{EventIdentity, EventRecord, export_events_jsonl}; - - fn id_strategy() -> impl Strategy { - proptest::string::string_regex("[a-zA-Z0-9._-]{1,32}").unwrap() - } - - fn event_strategy() -> impl Strategy { - ( - any::(), - id_strategy(), - id_strategy(), - prop::option::of(id_strategy()), - prop::option::of(id_strategy()), - ) - .prop_map(|(seq, source, kind, session_id, agent_id)| EventRecord { - identity: EventIdentity::default(), - seq, - source, - kind, - timestamp: None, - session_id, - agent_id, - parent_uuid: None, - trace_id: None, - call_id: None, - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload: Value::Object(Default::default()), - }) - } - - proptest! { - #[test] - fn jsonl_export_emits_one_decodable_record_per_event(events in proptest::collection::vec(event_strategy(), 0..16)) { - let encoded = export_events_jsonl(&events).unwrap(); - let decoded = encoded - .lines() - .map(serde_json::from_str::) - .collect::, _>>() - .unwrap(); - prop_assert_eq!(decoded, events); - } - } - } -} diff --git a/crates/persisting-pchronicle/src/formats/mod.rs b/crates/persisting-pchronicle/src/formats/mod.rs index 83c98f36c..a0403391d 100644 --- a/crates/persisting-pchronicle/src/formats/mod.rs +++ b/crates/persisting-pchronicle/src/formats/mod.rs @@ -7,7 +7,6 @@ pub(crate) mod codec; pub(crate) mod codex; pub(crate) mod common; pub mod detect; -pub mod events; pub mod llm; pub mod openai_corpus; pub(crate) mod registry; @@ -16,7 +15,6 @@ pub mod timestamp; pub mod unknown_fields; pub use detect::detect_format; -pub use events::{EventIdentity, EventRecord}; #[cfg(any(test, feature = "lance-store"))] pub use openai_corpus::parse_openai_msg_corpus_value; pub use storyline::StorylineDocument; diff --git a/crates/persisting-pchronicle/src/interop.rs b/crates/persisting-pchronicle/src/interop.rs deleted file mode 100644 index 3e00f2580..000000000 --- a/crates/persisting-pchronicle/src/interop.rs +++ /dev/null @@ -1,364 +0,0 @@ -//! Loss-aware interoperability views for observability and HTTP tooling. - -use std::collections::BTreeMap; - -use serde_json::{Map, Value, json}; - -use crate::{EventIdentity, EventRecord}; - -fn wire(record: &EventRecord) -> &Value { - record.payload.get("http").unwrap_or(&record.payload) -} - -fn string(value: Option<&Value>) -> String { - value - .and_then(Value::as_str) - .unwrap_or_default() - .to_string() -} - -/// Export request/response pairs as HAR 1.2. Streaming and provenance details -/// live under the namespaced `_pchronicle` extension. -pub fn events_to_har(records: &[EventRecord]) -> Value { - let mut calls: BTreeMap, Option<&EventRecord>)> = BTreeMap::new(); - for record in records { - let key = record - .call_id - .clone() - .unwrap_or_else(|| format!("seq-{}", record.seq)); - let pair = calls.entry(key).or_default(); - if record.kind.contains("request") { - pair.0 = Some(record); - } else if record.kind.contains("response") { - pair.1 = Some(record); - } - } - let entries = calls - .into_iter() - .filter_map(|(call_id, (request, response))| { - let request = request?; - let req = wire(request); - let resp = response.map(wire); - Some(json!({ - "startedDateTime": request.timestamp, - "time": resp.and_then(|v| v.get("duration_ms")).and_then(Value::as_f64).unwrap_or(0.0), - "request": { - "method": string(req.get("method")), - "url": string(req.get("url").or_else(|| req.get("path"))), - "httpVersion": string(req.get("version")), - "headers": har_headers(req.get("headers")), - "queryString": [], - "postData": { "mimeType": "application/json", "text": body_text(req) }, - "headersSize": -1, - "bodySize": -1 - }, - "response": { - "status": resp.and_then(|v| v.get("status")).and_then(Value::as_u64).unwrap_or(0), - "statusText": "", - "httpVersion": resp.map(|v| string(v.get("version"))).unwrap_or_default(), - "headers": resp.map(|v| har_headers(v.get("headers"))).unwrap_or_default(), - "content": { "size": -1, "mimeType": "application/json", "text": resp.map(body_text).unwrap_or_default() }, - "redirectURL": "", "headersSize": -1, "bodySize": -1 - }, - "cache": {}, "timings": { "send": -1, "wait": -1, "receive": -1 }, - "_pchronicle": { "call_id": call_id, "trace_id": request.trace_id, "degraded": request.payload.get("degraded").and_then(Value::as_bool).unwrap_or(false) } - })) - }) - .collect::>(); - json!({"log":{"version":"1.2","creator":{"name":"pChronicle","version":env!("CARGO_PKG_VERSION")},"entries":entries}}) -} - -fn har_headers(value: Option<&Value>) -> Vec { - value - .and_then(Value::as_object) - .map(|headers| headers.iter().map(|(name, value)| json!({"name":name,"value":value.as_str().unwrap_or("")})).collect()) - .unwrap_or_default() -} - -fn body_text(value: &Value) -> String { - value - .get("body") - .or_else(|| value.get("request_body")) - .or_else(|| value.get("response_body")) - .map(|body| { - body.as_str() - .map(str::to_string) - .unwrap_or_else(|| body.to_string()) - }) - .unwrap_or_default() -} - -/// Import OTLP JSON spans as explicitly degraded, non-replayable events. -pub fn otlp_json_to_events(document: &Value) -> Vec { - let mut records = Vec::new(); - for resource in document - .get("resourceSpans") - .and_then(Value::as_array) - .into_iter() - .flatten() - { - for scope in resource - .get("scopeSpans") - .or_else(|| resource.get("scope_spans")) - .and_then(Value::as_array) - .into_iter() - .flatten() - { - for span in scope - .get("spans") - .and_then(Value::as_array) - .into_iter() - .flatten() - { - let attrs = otlp_attributes(span.get("attributes")); - records.push(EventRecord { - identity: EventIdentity { - producer: Some("otel-import".into()), - ..EventIdentity::default() - }, - seq: records.len() as u64, - source: "otel".into(), - kind: "otel.span".into(), - timestamp: None, - session_id: attrs.get("pchronicle.session_id").and_then(Value::as_str).map(str::to_string), - agent_id: None, - parent_uuid: None, - trace_id: span.get("traceId").or_else(|| span.get("trace_id")).and_then(Value::as_str).map(str::to_string), - call_id: span.get("spanId").or_else(|| span.get("span_id")).and_then(Value::as_str).map(str::to_string), - subagent_id: None, parent_agent_id: None, branch: None, parent_call_id: None, - payload: json!({"degraded":true,"replayable":false,"otel":{"name":span.get("name"),"attributes":attrs}}), - }); - } - } - } - records -} - -/// Import Langfuse's OTLP/HTTP JSON representation without discarding the -/// resource and span attributes used for trace/session/user correlation. -/// -/// The generic [`otlp_json_to_events`] importer intentionally marks records as -/// degraded for export tooling. Gateway ingestion uses this loss-aware view so -/// Langfuse fields remain queryable and replay consumers can inspect the exact -/// original span under `payload.otel`. -pub fn langfuse_otlp_json_to_events(document: &Value) -> Vec { - let mut records = Vec::new(); - for resource in document - .get("resourceSpans") - .or_else(|| document.get("resource_spans")) - .and_then(Value::as_array) - .into_iter() - .flatten() - { - let resource_attrs = otlp_attributes( - resource - .get("resource") - .and_then(|resource| resource.get("attributes")), - ); - let service_name = resource_attrs - .get("service.name") - .and_then(Value::as_str) - .unwrap_or("langfuse") - .to_string(); - for scope in resource - .get("scopeSpans") - .or_else(|| resource.get("scope_spans")) - .and_then(Value::as_array) - .into_iter() - .flatten() - { - for span in scope - .get("spans") - .and_then(Value::as_array) - .into_iter() - .flatten() - { - let attrs = otlp_attributes(span.get("attributes")); - let trace_id = span - .get("traceId") - .or_else(|| span.get("trace_id")) - .and_then(Value::as_str) - .map(str::to_string); - let call_id = span - .get("spanId") - .or_else(|| span.get("span_id")) - .and_then(Value::as_str) - .map(str::to_string); - let session_id = first_string_attr( - &attrs, - &["langfuse.session.id", "session.id", "pchronicle.session_id"], - ) - .or_else(|| trace_id.clone()); - let user_id = first_string_attr(&attrs, &["langfuse.user.id", "user.id"]); - let parent_call_id = span - .get("parentSpanId") - .or_else(|| span.get("parent_span_id")) - .and_then(Value::as_str) - .map(str::to_string); - let event_id = match (trace_id.as_deref(), call_id.as_deref()) { - (Some(trace_id), Some(call_id)) => Some(format!("otel:{trace_id}:{call_id}")), - _ => None, - }; - let kind = first_string_attr(&attrs, &["langfuse.observation.type"]) - .map(|value| format!("otel.{value}")) - .unwrap_or_else(|| "otel.span".into()); - let model = first_string_attr( - &attrs, - &[ - "langfuse.observation.model.name", - "gen_ai.request.model", - "gen_ai.response.model", - ], - ); - let timestamp = otlp_timestamp( - span.get("startTimeUnixNano") - .or_else(|| span.get("start_time_unix_nano")), - ); - records.push(EventRecord { - identity: EventIdentity { - event_id, - producer: Some("langfuse-otel".into()), - timestamp_unix_ms: timestamp.as_deref().and_then(timestamp_unix_ms), - ..EventIdentity::default() - }, - seq: records.len() as u64, - source: "langfuse.otel".into(), - kind, - timestamp, - session_id, - agent_id: Some(service_name.clone()), - parent_uuid: None, - trace_id, - call_id, - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id, - payload: json!({ - "model": model, - "langfuse": { - "user_id": user_id, - "attributes": attrs, - "resource": resource_attrs, - }, - "otel": { - "span": span, - "resource": resource, - "replayable": false, - } - }), - }); - } - } - } - records -} - -fn first_string_attr(attrs: &Value, keys: &[&str]) -> Option { - keys.iter() - .find_map(|key| attrs.get(*key).and_then(Value::as_str).map(str::to_string)) -} - -fn otlp_timestamp(value: Option<&Value>) -> Option { - let nanos = value.and_then(|value| { - value - .as_str() - .and_then(|value| value.parse::().ok()) - .or_else(|| value.as_u64().map(i128::from)) - })?; - let seconds = nanos.div_euclid(1_000_000_000); - let subsec = nanos.rem_euclid(1_000_000_000) as u32; - chrono::DateTime::::from_timestamp(seconds as i64, subsec) - .map(|value| value.to_rfc3339()) -} - -fn timestamp_unix_ms(value: &str) -> Option { - chrono::DateTime::parse_from_rfc3339(value) - .ok() - .map(|value| value.timestamp_millis().max(0) as u64) -} - -fn otlp_attributes(value: Option<&Value>) -> Value { - let mut out = Map::new(); - for attr in value.and_then(Value::as_array).into_iter().flatten() { - let Some(key) = attr.get("key").and_then(Value::as_str) else { - continue; - }; - let value = attr - .get("value") - .and_then(Value::as_object) - .and_then(|v| v.values().next()) - .cloned() - .unwrap_or(Value::Null); - out.insert(key.to_string(), value); - } - Value::Object(out) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn otlp_import_is_explicitly_degraded() { - let events = otlp_json_to_events( - &json!({"resourceSpans":[{"scopeSpans":[{"spans":[{"traceId":"t","spanId":"s","name":"chat","attributes":[]}]}]}]}), - ); - assert_eq!(events.len(), 1); - assert_eq!(events[0].payload["degraded"], true); - assert_eq!(events[0].trace_id.as_deref(), Some("t")); - } - - #[test] - fn langfuse_import_preserves_session_user_and_attributes() { - let events = langfuse_otlp_json_to_events(&json!({ - "resourceSpans": [{ - "resource": {"attributes": [{"key":"service.name","value":{"stringValue":"agent"}}]}, - "scopeSpans": [{"spans": [{ - "traceId":"t", "spanId":"s", "name":"chat", - "startTimeUnixNano":"1700000000000000000", - "attributes": [ - {"key":"langfuse.session.id","value":{"stringValue":"session"}}, - {"key":"langfuse.user.id","value":{"stringValue":"user"}}, - {"key":"langfuse.observation.model.name","value":{"stringValue":"gpt-test"}}, - {"key":"langfuse.observation.type","value":{"stringValue":"generation"}} - ] - }]}] - }] - })); - assert_eq!(events.len(), 1); - assert_eq!(events[0].session_id.as_deref(), Some("session")); - assert_eq!(events[0].agent_id.as_deref(), Some("agent")); - assert_eq!(events[0].kind, "otel.generation"); - assert_eq!(events[0].identity.event_id.as_deref(), Some("otel:t:s")); - assert_eq!(events[0].payload["model"], "gpt-test"); - assert_eq!(events[0].payload["langfuse"]["user_id"], "user"); - } - - #[cfg(feature = "proptest")] - mod proptests { - use proptest::prelude::*; - use serde_json::Value; - - use super::*; - - proptest! { - #[test] - fn otlp_timestamp_accepts_equivalent_string_and_numeric_nanos( - seconds in 0u64..=2_000_000_000, - subsec in 0u32..1_000_000_000, - ) { - let nanos = seconds * 1_000_000_000 + u64::from(subsec); - let string_value = Value::String(nanos.to_string()); - let numeric_value = Value::from(nanos); - let string_timestamp = otlp_timestamp(Some(&string_value)).unwrap(); - let numeric_timestamp = otlp_timestamp(Some(&numeric_value)).unwrap(); - prop_assert_eq!(&string_timestamp, &numeric_timestamp); - prop_assert_eq!( - timestamp_unix_ms(&string_timestamp), - Some(seconds * 1000 + u64::from(subsec / 1_000_000)), - ); - } - } - } -} diff --git a/crates/persisting-pchronicle/src/layout/coords.rs b/crates/persisting-pchronicle/src/layout/coords.rs deleted file mode 100644 index fa21d88e6..000000000 --- a/crates/persisting-pchronicle/src/layout/coords.rs +++ /dev/null @@ -1,155 +0,0 @@ -//! Run / Story coordinates for offline trajectory storage. -//! -//! - **Run** → `root_session_id` → `{storage}/{agent_id}/{run_id}/` -//! - **Story** → `session_id` → `{session_id}.md` under the run directory; Lance rows are filtered by session_id - -use std::path::{Path, PathBuf}; - -use anyhow::Result; - -/// Offline story coordinates shared by pChronicle clients. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct StoryCoords { - pub storage: String, - pub agent_id: String, - pub session_id: String, - pub root_session_id: Option, -} - -impl StoryCoords { - pub fn new( - storage: impl Into, - agent_id: impl Into, - session_id: impl Into, - root_session_id: Option, - ) -> Self { - Self { - storage: storage.into(), - agent_id: agent_id.into(), - session_id: session_id.into(), - root_session_id, - } - } - - pub fn run_dir(&self) -> Result { - story_run_dir( - &self.storage, - &self.agent_id, - &self.session_id, - self.root_session_id.as_deref(), - ) - } - - pub fn lance_event_path(&self) -> Result { - story_lance_event_path( - &self.storage, - &self.agent_id, - &self.session_id, - self.root_session_id.as_deref(), - ) - } -} - -fn validate_storage(storage: &str) -> Result<()> { - if storage.trim().is_empty() { - anyhow::bail!("storage path must not be empty"); - } - Ok(()) -} - -fn validate_path_segment(s: &str, field: &str) -> Result { - let t = s.trim(); - if t.is_empty() { - anyhow::bail!("{field} must not be empty"); - } - if t.contains('/') || t.contains('\\') { - anyhow::bail!("{field} must not contain '/' or '\\' (single path segment only)"); - } - if t == "." || t == ".." { - anyhow::bail!("{field} must not be '.' or '..'"); - } - Ok(t.to_string()) -} - -/// Run directory under `{storage}/{agent_id}/`. -pub fn story_run_dir( - storage: &str, - agent_id: &str, - session_id: &str, - root_session_id: Option<&str>, -) -> Result { - validate_storage(storage)?; - let a = validate_path_segment(agent_id, "agent_id")?; - match root_session_id { - Some(root) => { - let r = validate_path_segment(root, "root_session_id")?; - Ok(Path::new(storage).join(a).join(r)) - } - None => { - let s = validate_path_segment(session_id, "session_id")?; - Ok(Path::new(storage).join(a).join(s)) - } - } -} - -/// Epoch-fenced Lance event log root at `{run}/events.lance/`. -pub fn story_lance_event_path( - storage: &str, - agent_id: &str, - session_id: &str, - root_session_id: Option<&str>, -) -> Result { - let run = story_run_dir(storage, agent_id, session_id, root_session_id)?; - Ok(run.join("events.lance")) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn nested_sessions_share_run_level_lance_path() { - let root = story_lance_event_path("/store", "agent", "run-001", Some("run-001")).unwrap(); - let sub = story_lance_event_path("/store", "agent", "agent-sub", Some("run-001")).unwrap(); - assert_eq!(root, sub); - assert!(root.ends_with("agent/run-001/events.lance")); - } - - #[test] - fn flat_raw_event_lance_path_is_session_scoped() { - let path = story_lance_event_path("/store", "agent", "sess-flat", None).unwrap(); - assert!(path.ends_with("agent/sess-flat/events.lance")); - } - - #[test] - fn story_coords_lance_event_path_matches_helper() { - let coords = StoryCoords::new("/store", "agent", "run-x", Some("run-x".into())); - assert_eq!( - coords.lance_event_path().unwrap(), - story_lance_event_path("/store", "agent", "run-x", Some("run-x")).unwrap() - ); - } - - #[test] - fn object_store_uri_preserves_scheme_and_run_partitioning() { - let root = story_lance_event_path( - "s3://trajectory-bucket/prefix", - "agent", - "run-001", - Some("run-001"), - ) - .unwrap(); - let child = story_lance_event_path( - "s3://trajectory-bucket/prefix", - "agent", - "child-001", - Some("run-001"), - ) - .unwrap(); - assert_eq!(root, child); - assert_eq!( - root.to_string_lossy(), - "s3://trajectory-bucket/prefix/agent/run-001/events.lance" - ); - } -} diff --git a/crates/persisting-pchronicle/src/layout/mod.rs b/crates/persisting-pchronicle/src/layout/mod.rs deleted file mode 100644 index f91c66d12..000000000 --- a/crates/persisting-pchronicle/src/layout/mod.rs +++ /dev/null @@ -1,16 +0,0 @@ -//! On-disk layout for agent trajectory storage (run dirs, markdown, Lance paths). - -mod coords; -mod resolve; - -pub use crate::agenticmd::{ - is_subagent_session_storage_key, is_trajectory_markdown_path, locate_run_bucket_markdown, - locate_session_markdown, locate_session_markdown_for_key, sanitize_session_filename, - session_filename_stem, session_markdown_filename, session_markdown_path_for_key, - session_markdown_write_path_for_key, -}; -pub use coords::{StoryCoords, story_lance_event_path, story_run_dir}; -pub use resolve::{ - StoryLocationPartial, list_story_read_locations, merge_story_location, - resolve_story_read_location, try_infer_story_location, -}; diff --git a/crates/persisting-pchronicle/src/layout/resolve.rs b/crates/persisting-pchronicle/src/layout/resolve.rs deleted file mode 100644 index 0b94f6dee..000000000 --- a/crates/persisting-pchronicle/src/layout/resolve.rs +++ /dev/null @@ -1,847 +0,0 @@ -//! Resolve Run/Story paths from CLI storage arguments. -//! -//! Accepts a storage root (`store/`), agent directory (`store/{agent_id}/`), -//! or session directory (`store/{agent_id}/{session_id}/`, nested subagent paths). - -use std::path::{Component, Path}; - -use anyhow::{Context, Result}; - -use super::coords::StoryCoords; -use crate::agenticmd::{locate_session_markdown, session_markdown_path_for_key}; - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct StoryLocationPartial { - pub storage: String, - pub agent_id: Option, - pub session_id: Option, - pub root_session_id: Option, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -struct ParsedTrajPath { - storage: String, - agent_id: String, - session_id: String, - root_session_id: Option, -} - -fn path_components(path: &Path) -> Vec { - path.components() - .filter_map(|c| match c { - Component::Normal(s) => Some(s.to_string_lossy().into_owned()), - _ => None, - }) - .collect() -} - -fn join_storage_prefix(components: &[String], end: usize) -> String { - components[..end].join("/") -} - -fn normalized_path_arg(path_arg: &str) -> String { - path_arg.trim().trim_end_matches('/').to_string() -} - -/// Infer `{storage, agent_id, session_id[, root_session_id]}` from a session directory path. -fn infer_traj_location_from_path(path_arg: &str) -> Option { - let trimmed = normalized_path_arg(path_arg); - if trimmed.is_empty() { - return None; - } - let session_dir = Path::new(&trimmed); - if !looks_like_session_dir(session_dir) { - return None; - } - - let session_id = session_dir.file_name()?.to_string_lossy().into_owned(); - let parent = session_dir.parent()?; - - if parent.file_name().is_some_and(|n| n == "subagents") { - let root_dir = parent.parent()?; - let root_session_id = root_dir.file_name()?.to_string_lossy().into_owned(); - let agent_dir = root_dir.parent()?; - let agent_id = agent_dir.file_name()?.to_string_lossy().into_owned(); - let storage = agent_dir.parent()?.to_string_lossy().into_owned(); - return Some(ParsedTrajPath { - storage, - agent_id, - root_session_id: Some(root_session_id), - session_id, - }); - } - - let agent_dir = parent; - let agent_id = agent_dir.file_name()?.to_string_lossy().into_owned(); - let storage = agent_dir.parent()?.to_string_lossy().into_owned(); - Some(ParsedTrajPath { - storage, - agent_id, - session_id: session_id.clone(), - root_session_id: infer_root_session_id(session_dir, &session_id), - }) -} - -fn looks_like_session_dir(dir: &Path) -> bool { - dir.is_dir() - && (locate_session_markdown(dir).is_some() - || dir.join("events.lance").is_dir() - || dir.join("_versions").is_dir()) -} - -/// Capture run layout: `{storage}/{agent}/{run_id}/events.lance/` (dataset directory). -/// Flat layout: `{storage}/{agent}/{session_id}/events.lance/`. -fn infer_root_session_id(session_dir: &Path, session_id: &str) -> Option { - let is_run_dir = session_id.starts_with("run-"); - if session_dir.join("events.lance").is_dir() - || (is_run_dir && session_dir.join("_versions").is_dir()) - { - Some(session_id.to_string()) - } else { - None - } -} - -fn list_session_dirs(agent_dir: &Path) -> Option> { - let read_dir = std::fs::read_dir(agent_dir).ok()?; - let mut sessions = Vec::new(); - for entry in read_dir.flatten() { - let ft = entry.file_type().ok()?; - if !ft.is_dir() { - continue; - } - let name = entry.file_name().to_string_lossy().into_owned(); - if name.starts_with('.') || name == "subagents" { - continue; - } - if looks_like_session_dir(&entry.path()) { - sessions.push(name); - } - } - Some(sessions) -} - -/// `{storage}/{agent_id}` with a single session subdirectory. -fn infer_from_agent_dir(path_arg: &str) -> Option { - let trimmed = normalized_path_arg(path_arg); - let path = Path::new(&trimmed); - let components = path_components(path); - if components.len() < 2 || !path.is_dir() { - return None; - } - let n = components.len(); - let agent_id = components[n - 1].clone(); - let storage = join_storage_prefix(&components, n - 1); - let sessions = list_session_dirs(path)?; - if sessions.len() == 1 { - let session_id = sessions.into_iter().next()?; - let session_dir = path.join(&session_id); - return Some(ParsedTrajPath { - storage, - agent_id, - session_id: session_id.clone(), - root_session_id: infer_root_session_id(&session_dir, &session_id), - }); - } - None -} - -/// Storage root with exactly one session anywhere under `{storage}/{agent_id}/`. -fn infer_from_storage_root(path_arg: &str) -> Option { - let trimmed = normalized_path_arg(path_arg); - let path = Path::new(&trimmed); - if !path.is_dir() { - return None; - } - let read_dir = std::fs::read_dir(path).ok()?; - let mut found = Vec::new(); - for agent_entry in read_dir.flatten() { - if !agent_entry.file_type().ok()?.is_dir() { - continue; - } - let agent_id = agent_entry.file_name().to_string_lossy().into_owned(); - if agent_id.starts_with('.') { - continue; - } - let Some(sessions) = list_session_dirs(&agent_entry.path()) else { - continue; - }; - for session_id in sessions { - let session_dir = agent_entry.path().join(&session_id); - found.push(ParsedTrajPath { - storage: trimmed.clone(), - agent_id: agent_id.clone(), - session_id: session_id.clone(), - root_session_id: infer_root_session_id(&session_dir, &session_id), - }); - } - } - if found.len() == 1 { - return found.into_iter().next(); - } - None -} - -fn parsed_to_location(parsed: ParsedTrajPath) -> StoryCoords { - StoryCoords { - storage: parsed.storage, - agent_id: parsed.agent_id, - session_id: parsed.session_id, - root_session_id: parsed.root_session_id, - } -} - -/// Best-effort inference from a path (no CLI flags). -pub fn try_infer_story_location(path_arg: &str) -> Option { - infer_traj_location_from_path(path_arg) - .or_else(|| infer_from_agent_dir(path_arg)) - .or_else(|| infer_from_storage_root(path_arg)) - .map(parsed_to_location) -} - -/// Merge path inference with explicit flags; missing ids stay unset. -pub fn merge_story_location( - path_arg: String, - agent_id: Option, - session_id: Option, - root_session_id: Option, -) -> StoryLocationPartial { - let inferred = try_infer_story_location(&path_arg); - StoryLocationPartial { - storage: inferred - .as_ref() - .map(|p| p.storage.clone()) - .unwrap_or(path_arg), - agent_id: agent_id.or_else(|| inferred.as_ref().map(|p| p.agent_id.clone())), - session_id: session_id.or_else(|| inferred.as_ref().map(|p| p.session_id.clone())), - root_session_id: root_session_id - .or_else(|| inferred.as_ref().and_then(|p| p.root_session_id.clone())), - } -} - -/// When a capture run uses a header session id (UUID) for Lance/md but the run folder is `run-*`, -/// remap the story storage key from the primary markdown stem under the run directory. -fn remap_capture_run_story_session_id( - storage: &str, - agent_id: &str, - session_id: &str, - root_session_id: Option<&str>, -) -> String { - let Some(root) = root_session_id else { - return session_id.to_string(); - }; - if session_id != root { - return session_id.to_string(); - } - let run_dir = Path::new(storage).join(agent_id).join(root); - primary_story_storage_key(&run_dir, root).unwrap_or_else(|| session_id.to_string()) -} - -fn primary_story_storage_key(run_dir: &Path, run_id: &str) -> Option { - if session_markdown_path_for_key(run_dir, run_id).is_file() { - return None; - } - let md = locate_session_markdown(run_dir)?; - let stem = md.file_stem()?.to_str()?; - if stem == run_id { - return None; - } - Some(stem.to_string()) -} - -fn story_coords_from_run_bucket(storage: &str, agent_id: &str, run_id: &str) -> StoryCoords { - StoryCoords { - storage: storage.to_string(), - agent_id: agent_id.to_string(), - session_id: run_id.to_string(), - root_session_id: Some(run_id.to_string()), - } -} - -fn story_coords_from_parts( - storage: &str, - agent_id: &str, - session_id: &str, - root_session_id: Option, -) -> StoryCoords { - let root_session_id = root_session_id.or_else(|| { - infer_root_session_id( - Path::new(storage).join(agent_id).join(session_id).as_path(), - session_id, - ) - }); - let session_id = remap_capture_run_story_session_id( - storage, - agent_id, - session_id, - root_session_id.as_deref(), - ); - StoryCoords { - storage: storage.to_string(), - agent_id: agent_id.to_string(), - session_id, - root_session_id, - } -} - -/// Agent directory `{storage}/{agent_id}/` (has ≥1 session subdir, not itself a session dir). -fn parse_agent_dir(path_arg: &str) -> Option<(String, String)> { - let trimmed = normalized_path_arg(path_arg); - let path = Path::new(&trimmed); - if !path.is_dir() || looks_like_session_dir(path) { - return None; - } - let sessions = list_session_dirs(path)?; - if sessions.is_empty() { - return None; - } - let agent_id = path.file_name()?.to_string_lossy().into_owned(); - let storage = path.parent()?.to_string_lossy().into_owned(); - Some((storage, agent_id)) -} - -fn list_sessions_under_agent(storage: &str, agent_id: &str) -> Result> { - let agent_path = Path::new(storage).join(agent_id); - if !agent_path.is_dir() { - return Ok(Vec::new()); - } - let mut sessions = list_session_dirs(&agent_path).unwrap_or_default(); - sessions.sort(); - Ok(sessions - .into_iter() - .map(|run_id| story_coords_from_run_bucket(storage, agent_id, &run_id)) - .collect()) -} - -fn list_all_sessions(storage: &str) -> Result> { - let path = Path::new(storage); - if !path.is_dir() { - return Ok(Vec::new()); - } - let mut out = Vec::new(); - for agent_entry in std::fs::read_dir(path).with_context(|| format!("read_dir {storage}"))? { - let agent_entry = agent_entry?; - if !agent_entry.file_type()?.is_dir() { - continue; - } - let agent_id = agent_entry.file_name().to_string_lossy().into_owned(); - if agent_id.starts_with('.') { - continue; - } - out.extend(list_sessions_under_agent(storage, &agent_id)?); - } - out.sort_by(|a, b| (&a.agent_id, &a.session_id).cmp(&(&b.agent_id, &b.session_id))); - Ok(out) -} - -/// List trajectory sessions to read under `path_arg` (storage root, agent dir, or single session). -/// When `--session-id` is set, returns exactly one resolved session (same as [`resolve_story_read_location`]). -pub fn list_story_read_locations( - path_arg: String, - agent_id: Option, - session_id: Option, - root_session_id: Option, -) -> Result> { - if session_id.is_some() { - return Ok(vec![resolve_story_read_location( - "trajectory stats", - path_arg, - agent_id, - session_id, - root_session_id, - )?]); - } - - if let Some(parsed) = infer_traj_location_from_path(&path_arg) { - if let Some(ref want) = agent_id - && want != &parsed.agent_id - { - anyhow::bail!( - "trajectory stats: --agent-id {want} does not match path agent_id {}", - parsed.agent_id - ); - } - let run_dir = Path::new(&parsed.storage) - .join(&parsed.agent_id) - .join(&parsed.session_id); - let root = parsed - .root_session_id - .or_else(|| infer_root_session_id(run_dir.as_path(), &parsed.session_id)); - if root.as_deref() == Some(parsed.session_id.as_str()) { - return Ok(vec![story_coords_from_run_bucket( - &parsed.storage, - &parsed.agent_id, - &parsed.session_id, - )]); - } - return Ok(vec![story_coords_from_parts( - &parsed.storage, - &parsed.agent_id, - &parsed.session_id, - root.or(root_session_id), - )]); - } - - let partial = merge_story_location(path_arg.clone(), agent_id, None, root_session_id.clone()); - let storage = partial.storage; - - if let Some((stor, agent)) = parse_agent_dir(&path_arg) { - let sessions = list_sessions_under_agent(&stor, &agent)?; - if sessions.is_empty() { - anyhow::bail!( - "trajectory stats: no sessions under {stor}/{agent}/ (expected run dirs with events.lance or markdown)" - ); - } - return Ok(sessions); - } - - if let Some(agent) = partial.agent_id { - let sessions = list_sessions_under_agent(&storage, &agent)?; - if sessions.is_empty() { - anyhow::bail!( - "trajectory stats: no sessions under {storage}/{agent}/ (expected run dirs with events.lance or markdown)" - ); - } - return Ok(sessions); - } - - if Path::new(&storage).is_dir() { - let sessions = list_all_sessions(&storage)?; - if sessions.is_empty() { - anyhow::bail!( - "trajectory stats: no trajectory sessions under {storage}/ (expected {{agent_id}}/{{session_id}}/ layout)" - ); - } - return Ok(sessions); - } - - anyhow::bail!("trajectory stats: path not found or not a trajectory store: {path_arg}") -} - -/// Read commands (`replay`, `stats`) require a resolved session. -pub fn resolve_story_read_location( - op: &str, - path_arg: String, - agent_id: Option, - session_id: Option, - root_session_id: Option, -) -> Result { - let partial = merge_story_location(path_arg, agent_id, session_id, root_session_id); - match (partial.agent_id, partial.session_id) { - (Some(agent_id), Some(session_id)) => Ok(story_coords_from_parts( - &partial.storage, - &agent_id, - &session_id, - partial.root_session_id, - )), - _ => anyhow::bail!( - "{op}: 请指定 session 目录（如 `store/{{agent_id}}/{{session_id}}/`），或同时提供 --agent-id 与 --session-id" - ), - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn infer_flat_session_path() { - let base = - std::env::temp_dir().join(format!("persisting-traj-flat-{}", std::process::id())); - let _ = std::fs::remove_dir_all(&base); - let session = base - .join("store") - .join("deepseek-proxy") - .join("run-20260524-015351-928492000"); - std::fs::create_dir_all(&session).unwrap(); - std::fs::write(session.join("run-20260524-015351-928492000.md"), "# test\n").unwrap(); - - let p = infer_traj_location_from_path(session.to_str().unwrap()).unwrap(); - assert!(p.storage.ends_with("store")); - assert_eq!(p.agent_id, "deepseek-proxy"); - assert_eq!(p.session_id, "run-20260524-015351-928492000"); - - let _ = std::fs::remove_dir_all(&base); - } - - #[test] - fn infer_flat_session_path_string_only() { - let base = std::env::temp_dir().join(format!("persisting-traj-rel-{}", std::process::id())); - let _ = std::fs::remove_dir_all(&base); - let session = base.join("store").join("a").join("session-0001"); - std::fs::create_dir_all(&session).unwrap(); - std::fs::write(session.join("session-0001.md"), "# test\n").unwrap(); - assert!(infer_traj_location_from_path("store/a/session-0001").is_none()); - let p = infer_traj_location_from_path(session.to_str().unwrap()).unwrap(); - assert_eq!(p.agent_id, "a"); - assert_eq!(p.session_id, "session-0001"); - let _ = std::fs::remove_dir_all(&base); - } - - #[test] - fn infer_nested_subagent_path() { - let base = - std::env::temp_dir().join(format!("persisting-traj-nest-{}", std::process::id())); - let _ = std::fs::remove_dir_all(&base); - let session = base - .join("store") - .join("agent") - .join("root-run") - .join("subagents") - .join("sub-uuid"); - std::fs::create_dir_all(&session).unwrap(); - std::fs::write(session.join("sub-uuid.md"), "# test\n").unwrap(); - - let p = infer_traj_location_from_path(session.to_str().unwrap()).unwrap(); - assert!(p.storage.ends_with("store")); - assert_eq!(p.agent_id, "agent"); - assert_eq!(p.root_session_id.as_deref(), Some("root-run")); - assert_eq!(p.session_id, "sub-uuid"); - - let _ = std::fs::remove_dir_all(&base); - } - - #[test] - fn infer_storage_root_only_returns_none_without_scan() { - assert!(infer_traj_location_from_path("store").is_none()); - } - - #[test] - fn resolve_merges_explicit_flags_with_inferred_storage() { - let base = - std::env::temp_dir().join(format!("persisting-traj-merge-{}", std::process::id())); - let _ = std::fs::remove_dir_all(&base); - let session = base.join("store").join("deepseek-proxy").join("run-abc"); - std::fs::create_dir_all(&session).unwrap(); - std::fs::write(session.join("run-abc.md"), "# test\n").unwrap(); - - let loc = resolve_story_read_location( - "trajectory stats", - session.to_str().unwrap().into(), - None, - None, - None, - ) - .unwrap(); - assert!(loc.storage.ends_with("store")); - assert_eq!(loc.agent_id, "deepseek-proxy"); - assert_eq!(loc.session_id, "run-abc"); - - let _ = std::fs::remove_dir_all(&base); - } - - #[test] - fn resolve_infers_root_for_capture_run_layout() { - let base = - std::env::temp_dir().join(format!("persisting-traj-run-layout-{}", std::process::id())); - let _ = std::fs::remove_dir_all(&base); - let session = base.join("store").join("agent").join("run-1"); - std::fs::create_dir_all(&session).unwrap(); - std::fs::create_dir_all(session.join("events.lance")).unwrap(); - std::fs::write(session.join("run-1.md"), "# test\n").unwrap(); - - let loc = resolve_story_read_location( - "trajectory stats", - session.to_str().unwrap().into(), - None, - None, - None, - ) - .unwrap(); - assert!(loc.storage.ends_with("store")); - assert_eq!(loc.agent_id, "agent"); - assert_eq!(loc.session_id, "run-1"); - assert_eq!(loc.root_session_id.as_deref(), Some("run-1")); - - let loc = resolve_story_read_location( - "trajectory stats", - session.to_str().unwrap().into(), - Some("agent".into()), - Some("run-1".into()), - None, - ) - .unwrap(); - assert_eq!(loc.root_session_id.as_deref(), Some("run-1")); - - let _ = std::fs::remove_dir_all(&base); - } - - #[test] - fn resolve_capture_run_remaps_to_primary_story_stem() { - let base = - std::env::temp_dir().join(format!("persisting-traj-story-stem-{}", std::process::id())); - let _ = std::fs::remove_dir_all(&base); - let session = base.join("store").join("agent").join("run-1"); - std::fs::create_dir_all(&session).unwrap(); - std::fs::create_dir_all(session.join("events.lance")).unwrap(); - std::fs::write(session.join("uuid-story.md"), "# test\n").unwrap(); - - let loc = resolve_story_read_location( - "trajectory stats", - session.to_str().unwrap().into(), - None, - None, - None, - ) - .unwrap(); - assert_eq!(loc.session_id, "uuid-story"); - assert_eq!(loc.root_session_id.as_deref(), Some("run-1")); - - let _ = std::fs::remove_dir_all(&base); - } - - #[test] - fn resolve_real_capture_run_store_if_present() { - let session = Path::new("store/deepseek-proxy/run-20260529-020451-705391000"); - if !session.is_dir() { - return; - } - let loc = resolve_story_read_location( - "trajectory stats", - session.to_str().unwrap().into(), - None, - None, - None, - ) - .unwrap(); - assert_eq!( - loc.session_id, "5e0dfcdb-56ee-49d1-8921-4aeefeea3b17", - "expected header-session stem, got {}", - loc.session_id - ); - } - - #[test] - fn resolve_explicit_ids_with_storage_root() { - let loc = resolve_story_read_location( - "trajectory stats", - "store".into(), - Some("a".into()), - Some("s".into()), - None, - ) - .unwrap(); - assert_eq!(loc.storage, "store"); - assert_eq!(loc.agent_id, "a"); - assert_eq!(loc.session_id, "s"); - } - - #[test] - fn storage_root_with_run_session_is_not_session_dir() { - let base = std::env::temp_dir().join(format!( - "persisting-traj-capture-root-{}", - std::process::id() - )); - let _ = std::fs::remove_dir_all(&base); - let store = base.join("store"); - std::fs::create_dir_all(store.join(".capture")).unwrap(); - std::fs::write(store.join(".capture/run_session"), "run-20260101-000000").unwrap(); - let session = store.join("agent-a").join("run-20260101-000000"); - std::fs::create_dir_all(&session).unwrap(); - std::fs::create_dir_all(session.join("events.lance")).unwrap(); - - let loc = resolve_story_read_location( - "trajectory stats", - store.to_str().unwrap().into(), - Some("agent-a".into()), - Some("run-20260101-000000".into()), - None, - ) - .unwrap(); - assert_eq!(loc.storage, store.to_str().unwrap()); - assert_eq!(loc.agent_id, "agent-a"); - assert_eq!(loc.session_id, "run-20260101-000000"); - - let _ = std::fs::remove_dir_all(&base); - } - - #[test] - fn list_agent_dir_returns_all_sessions() { - let base = - std::env::temp_dir().join(format!("persisting-traj-list-agent-{}", std::process::id())); - let _ = std::fs::remove_dir_all(&base); - let agent = base.join("store").join("deepseek-proxy"); - for run in ["run-001", "run-002"] { - let session = agent.join(run); - std::fs::create_dir_all(&session).unwrap(); - std::fs::create_dir_all(session.join("events.lance")).unwrap(); - } - - let locs = - list_story_read_locations(agent.to_str().unwrap().into(), None, None, None).unwrap(); - assert_eq!(locs.len(), 2); - assert!(locs.iter().all(|l| l.agent_id == "deepseek-proxy")); - assert_eq!( - locs.iter() - .map(|l| l.session_id.as_str()) - .collect::>(), - vec!["run-001", "run-002"] - ); - assert!( - locs.iter() - .all(|l| l.root_session_id.as_deref() == Some(l.session_id.as_str())) - ); - - let _ = std::fs::remove_dir_all(&base); - } - - #[test] - fn resolve_run_path_remaps_primary_markdown_stem_for_single_session() { - let base = std::env::temp_dir().join(format!( - "persisting-traj-resolve-stem-{}", - std::process::id() - )); - let _ = std::fs::remove_dir_all(&base); - let run = base - .join("store") - .join("deepseek-proxy") - .join("run-with-md"); - std::fs::create_dir_all(&run).unwrap(); - std::fs::create_dir_all(run.join("events.lance")).unwrap(); - std::fs::write(run.join("header-session-uuid.md"), "# story\n").unwrap(); - - let resolved = resolve_story_read_location( - "trajectory stats", - run.to_str().unwrap().into(), - None, - None, - None, - ) - .unwrap(); - assert_eq!(resolved.session_id, "header-session-uuid"); - assert_eq!(resolved.root_session_id.as_deref(), Some("run-with-md")); - - let _ = std::fs::remove_dir_all(&base); - } - - #[test] - fn list_agent_dir_with_single_run_bucket() { - let base = std::env::temp_dir().join(format!( - "persisting-traj-list-single-run-{}", - std::process::id() - )); - let _ = std::fs::remove_dir_all(&base); - let agent = base.join("store").join("deepseek-proxy"); - let run = agent.join("run-only"); - std::fs::create_dir_all(&run).unwrap(); - std::fs::create_dir_all(run.join("events.lance")).unwrap(); - - let locs = - list_story_read_locations(agent.to_str().unwrap().into(), None, None, None).unwrap(); - assert_eq!(locs.len(), 1); - assert_eq!(locs[0].session_id, "run-only"); - assert_eq!(locs[0].root_session_id.as_deref(), Some("run-only")); - - let _ = std::fs::remove_dir_all(&base); - } - - #[test] - fn list_storage_root_returns_all_agents() { - let base = - std::env::temp_dir().join(format!("persisting-traj-list-root-{}", std::process::id())); - let _ = std::fs::remove_dir_all(&base); - let store = base.join("store"); - for (agent, run) in [("a1", "run-1"), ("a2", "run-2")] { - let session = store.join(agent).join(run); - std::fs::create_dir_all(&session).unwrap(); - std::fs::write(session.join(format!("{run}.md")), "# t\n").unwrap(); - } - - let locs = - list_story_read_locations(store.to_str().unwrap().into(), None, None, None).unwrap(); - assert_eq!(locs.len(), 2); - assert_eq!(locs[0].agent_id, "a1"); - assert_eq!(locs[1].agent_id, "a2"); - - let _ = std::fs::remove_dir_all(&base); - } - - #[test] - fn resolve_fails_without_ids_or_deep_path() { - assert!( - resolve_story_read_location("trajectory stats", "store".into(), None, None, None,) - .is_err() - ); - } - - #[test] - fn infer_from_agent_dir_single_session() { - let base = - std::env::temp_dir().join(format!("persisting-traj-path-{}", std::process::id())); - let _ = std::fs::remove_dir_all(&base); - let agent = base.join("store").join("deepseek-proxy"); - let session = agent.join("run-only"); - std::fs::create_dir_all(&session).unwrap(); - std::fs::write(session.join("run-only.md"), "# test\n").unwrap(); - - let p = infer_from_agent_dir(agent.to_str().unwrap()).unwrap(); - assert!(p.storage.ends_with("store")); - assert_eq!(p.agent_id, "deepseek-proxy"); - assert_eq!(p.session_id, "run-only"); - - let _ = std::fs::remove_dir_all(&base); - } - - #[test] - fn infer_from_storage_root_single_session() { - let base = - std::env::temp_dir().join(format!("persisting-traj-root-{}", std::process::id())); - let _ = std::fs::remove_dir_all(&base); - let store = base.join("store"); - let session = store.join("agent-a").join("run-one"); - std::fs::create_dir_all(&session).unwrap(); - std::fs::write(session.join("run-one.md"), "# test\n").unwrap(); - - let loc = try_infer_story_location(store.to_str().unwrap()).unwrap(); - assert!(loc.storage.ends_with("store")); - assert_eq!(loc.agent_id, "agent-a"); - assert_eq!(loc.session_id, "run-one"); - - let _ = std::fs::remove_dir_all(&base); - } - - #[test] - fn merge_partial_flags_with_inferred_path() { - let base = - std::env::temp_dir().join(format!("persisting-traj-partial-{}", std::process::id())); - let _ = std::fs::remove_dir_all(&base); - let session = base.join("store").join("a").join("session-0001"); - std::fs::create_dir_all(&session).unwrap(); - std::fs::write(session.join("session-0001.md"), "# test\n").unwrap(); - - let loc = merge_story_location( - session.to_str().unwrap().into(), - Some("override-agent".into()), - None, - None, - ); - assert!(loc.storage.ends_with("store")); - assert_eq!(loc.agent_id.as_deref(), Some("override-agent")); - assert_eq!(loc.session_id.as_deref(), Some("session-0001")); - - let _ = std::fs::remove_dir_all(&base); - } -} - -#[cfg(all(test, feature = "proptest"))] -mod proptests { - use proptest::prelude::*; - - use super::*; - - fn component_strategy() -> impl Strategy { - proptest::string::string_regex("[A-Za-z0-9._-]{1,32}").unwrap() - } - - proptest! { - #[test] - fn normalized_path_arguments_trim_outer_space_and_trailing_slashes( - component in component_strategy(), - trailing in proptest::collection::vec(Just('/'), 0..8), - ) { - let input = format!(" {component}{} ", trailing.into_iter().collect::()); - prop_assert_eq!(normalized_path_arg(&input), component.clone()); - prop_assert_eq!(normalized_path_arg(&normalized_path_arg(&input)), component); - } - } -} diff --git a/crates/persisting-pchronicle/src/lib.rs b/crates/persisting-pchronicle/src/lib.rs index 2d1aeae3c..566344a58 100644 --- a/crates/persisting-pchronicle/src/lib.rs +++ b/crates/persisting-pchronicle/src/lib.rs @@ -4,24 +4,14 @@ //! //! # 权威边界 //! -//! Canonical Event 是 append-only 运行时事实；[`model::StorylineDocument`] 是与 ATIF v1.7 -//! 对齐的交换和规范化查询模型。两者通过单向投影连接，Storyline 不承诺反建原始事件事实。 -//! ATIF、ACTF、OpenAI Msg 和 AgenticMD 只经 Storyline 互转；AgenticMD 是 Storyline 的 -//! Markdown 编码，不是另一套领域模型。 -//! -//! ```text -//! events.lance ──单向投影──► StorylineDocument ──► Storyline 三表 Lance -//! ├──◄──► AgenticMD -//! ├──◄──► ATIF -//! ├──◄──► OpenAI Msg -//! └──◄──► ACTF -//! ``` +//! [`model::StorylineDocument`] 是轨迹处理模型。ATIF、ACTF、OpenAI Msg 和 AgenticMD +//! 经 Storyline 互转；Storyline 可以直接写入三表 Lance 并通过 DataFusion 查询。 //! //! # 公共入口 //! -//! - [`model`]：Storyline、Canonical Event 与 LLM payload 权威类型； +//! - [`model`]：Storyline 与 LLM payload 权威类型； //! - [`document`]：磁盘格式、Storyline 语义 codec 与统一读取入口；Codex / Claude Code 会话 JSONL 为 decode-only； -//! - [`storage`]：Catalog、Lance store、append、投影和 revision； +//! - [`storage`]：Catalog、Lance store 与 append； //! - [`query`]：DataFusion 查询引擎与能力快照。 //! //! 外围 wire DTO、低层 parser、Markdown AST、Arrow codec、provider、manifest 与锁均不公开。 @@ -29,29 +19,19 @@ mod agenticmd; pub mod analysis_compile; -#[cfg(feature = "lance-store")] -mod append_queue; mod atif; mod convert; -#[cfg(feature = "lance-store")] -mod discovery; pub mod document; mod format; mod formats; mod input; -mod interop; -mod layout; #[cfg(feature = "search")] mod messages; pub mod model; #[cfg(feature = "search")] mod operations; -#[cfg(feature = "lance-store")] -mod projection; pub mod query; #[cfg(feature = "lance-store")] -mod revision; -#[cfg(feature = "lance-store")] pub mod search; pub mod storage; mod store; @@ -60,12 +40,12 @@ mod store; pub(crate) use document::{QueryCapabilities, QueryTables}; #[cfg(feature = "lance-store")] pub(crate) use format::DocumentFormat; +pub(crate) use formats::StorylineDocument; #[cfg(feature = "lance-store")] pub(crate) use formats::storyline::StorylineAgent; pub(crate) use formats::storyline::StorylineTurn; #[cfg(any(feature = "lance-store", test))] pub(crate) use formats::storyline::{StoryLink, StorylineToolCall}; -pub(crate) use formats::{EventIdentity, EventRecord, StorylineDocument}; pub(crate) use input::{InputIssue, InputResult}; pub type Result = anyhow::Result; #[cfg(feature = "search")] @@ -79,15 +59,6 @@ pub use operations::bridge::{ pub use operations::dispatch::invoke_request_body; #[cfg(feature = "search")] pub use search::agent as agent_search; -#[cfg(feature = "lance-store")] -pub(crate) use store::{ - EventRow, TRAJECTORY_AGENT_ID_COL, TRAJECTORY_CALL_ID_COL, TRAJECTORY_COLS, - TRAJECTORY_EVENT_ID_COL, TRAJECTORY_KIND_COL, TRAJECTORY_MODEL_COL, - TRAJECTORY_PARENT_CALL_ID_COL, TRAJECTORY_PAYLOAD_JSON_COL, TRAJECTORY_SEQ_COL, - TRAJECTORY_SESSION_ID_COL, TRAJECTORY_SOURCE_COL, TRAJECTORY_TIMESTAMP_COL, - TRAJECTORY_TRACE_ID_COL, event_record_to_event_row, event_row_to_event_record, - event_rows_from_batch, -}; #[cfg(feature = "search")] pub const PERSISTING_VECTOR_INDEX_NAME: &str = search::search_lance::PERSISTING_VECTOR_INDEX_NAME; #[cfg(feature = "search")] diff --git a/crates/persisting-pchronicle/src/model.rs b/crates/persisting-pchronicle/src/model.rs index ac83e1b07..5000548fe 100644 --- a/crates/persisting-pchronicle/src/model.rs +++ b/crates/persisting-pchronicle/src/model.rs @@ -1,8 +1,5 @@ //! pChronicle 的权威领域模型。 -pub use crate::formats::events::{ - ChronicleEventRecordExt, EventIdentity, EventRecord, EventsDocument, -}; pub use crate::formats::llm::{ LlmCandidate, LlmContentPart, LlmExtensions, LlmGenerationParams, LlmImageSource, LlmMessage, LlmProtocol, LlmRequest, LlmRequestEventPayload, LlmResponse, LlmResponseEventPayload, diff --git a/crates/persisting-pchronicle/src/projection/automatic.rs b/crates/persisting-pchronicle/src/projection/automatic.rs deleted file mode 100644 index cfdac34a3..000000000 --- a/crates/persisting-pchronicle/src/projection/automatic.rs +++ /dev/null @@ -1,828 +0,0 @@ -//! Automatic Storyline projections paired with canonical event stores. - -use std::path::Path; - -use anyhow::{Context, Result}; - -use super::storyline::{ - StorylineProjectionBuildOutcome, StorylineProjectionSyncMode, StorylineProjectionSyncOutcome, - build_storyline_projection, canonical_projection_lineage, projection_lineage_is_fresh, - rebuild_storyline_projection, storyline_projection_status, sync_storyline_projection, -}; -use crate::store::{ - CatalogSourceStatus, DatasetCatalogSnapshot, EventFactSnapshot, ProjectionSourceSnapshot, - RawEventDataSource, StorylineLanceStore, -}; - -const CANONICAL_EVENT_STORE_LEAF: &str = "events.lance"; -const AUTOMATIC_STORYLINE_PROJECTION_LEAF: &str = "storyline"; - -pub(crate) fn automatic_storyline_projection_uri(source_uri: &str) -> Result { - if source_uri.contains("://") { - let source_uri = source_uri.trim_end_matches('/'); - let (parent, leaf) = source_uri - .rsplit_once('/') - .with_context(|| format!("canonical event URI has no parent: {source_uri}"))?; - anyhow::ensure!( - leaf == CANONICAL_EVENT_STORE_LEAF, - "canonical event URI must end with /{CANONICAL_EVENT_STORE_LEAF}: {source_uri}" - ); - return Ok(format!("{parent}/{AUTOMATIC_STORYLINE_PROJECTION_LEAF}")); - } - - let source = Path::new(source_uri); - anyhow::ensure!( - source.file_name().and_then(|name| name.to_str()) == Some(CANONICAL_EVENT_STORE_LEAF), - "canonical event path must end with {CANONICAL_EVENT_STORE_LEAF}: {source_uri}" - ); - let parent = source - .parent() - .with_context(|| format!("canonical event path has no parent: {source_uri}"))?; - Ok(parent - .join(AUTOMATIC_STORYLINE_PROJECTION_LEAF) - .to_string_lossy() - .into_owned()) -} - -pub async fn probe_canonical_event_store( - uri: impl AsRef, -) -> Result> { - RawEventDataSource::probe_uri(uri).await -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct AutomaticProjectionTarget { - pub dataset: String, - pub source_path: String, - pub source_uri: String, - pub projection_path: String, - pub projection_uri: String, - pub source_snapshot: EventFactSnapshot, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct AutomaticProjectionInventoryError { - pub dataset: String, - pub source_path: String, - pub projection_path: String, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct AutomaticProjectionInventory { - pub snapshot_id: String, - pub targets: Vec, - pub errors: Vec, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize)] -#[serde(rename_all = "snake_case")] -pub enum AutomaticProjectionState { - Fresh, - Stale, - Missing, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct AutomaticProjectionInspection { - pub state: AutomaticProjectionState, - pub generation: Option, - pub fact_version: u64, - pub fact_rows: u64, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum AutomaticProjectionMaintenanceMode { - Unchanged, - Built, - Incremental, - Rebuilt, - ConcurrentWinner, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct AutomaticProjectionMaintenanceReport { - pub mode: AutomaticProjectionMaintenanceMode, - pub generation: String, - pub fact_version: u64, - pub fact_rows: u64, - pub trajectories: Option, -} - -impl AutomaticProjectionMaintenanceReport { - pub fn published(&self) -> bool { - matches!( - self.mode, - AutomaticProjectionMaintenanceMode::Built - | AutomaticProjectionMaintenanceMode::Incremental - | AutomaticProjectionMaintenanceMode::Rebuilt - ) - } -} - -pub fn automatic_projection_inventory( - snapshot: &DatasetCatalogSnapshot, -) -> Result { - let mut targets = snapshot - .canonical_event_sources() - .into_iter() - .map(|source| { - let source_path = display_source_path(&source.source_path); - Ok(AutomaticProjectionTarget { - dataset: source.dataset, - projection_path: display_projection_path(&source_path)?, - projection_uri: automatic_storyline_projection_uri(&source.source_uri)?, - source_path, - source_uri: source.source_uri, - source_snapshot: source.snapshot, - }) - }) - .collect::>>()?; - targets.sort_by(|left, right| { - (&left.dataset, &left.source_path).cmp(&(&right.dataset, &right.source_path)) - }); - - let mut errors = snapshot - .datasets() - .iter() - .flat_map(|dataset| { - dataset - .sources - .iter() - .filter(|source| { - source.status == CatalogSourceStatus::Error - && source.format.as_deref() - == Some(crate::DocumentFormat::CanonicalEvent.as_str()) - }) - .map(|source| { - let source_path = display_source_path(&source.file); - display_projection_path(&source_path).map(|projection_path| { - AutomaticProjectionInventoryError { - dataset: dataset.mount.name.clone(), - source_path, - projection_path, - } - }) - }) - }) - .collect::>>()?; - errors.sort_by(|left, right| { - (&left.dataset, &left.source_path).cmp(&(&right.dataset, &right.source_path)) - }); - - Ok(AutomaticProjectionInventory { - snapshot_id: snapshot.snapshot_id().to_string(), - targets, - errors, - }) -} - -pub async fn inspect_automatic_storyline_projection( - target: &AutomaticProjectionTarget, -) -> Result { - let status = storyline_projection_status(&target.projection_uri).await?; - let Some(generation) = status.generation else { - return Ok(inspection(target, AutomaticProjectionState::Missing, None)); - }; - let lineage = status - .lineage - .as_ref() - .context("automatic Storyline destination has no canonical lineage")?; - let ProjectionSourceSnapshot::CanonicalEvents { source_uri, .. } = &lineage.source else { - anyhow::bail!("automatic Storyline destination was not derived from canonical events"); - }; - anyhow::ensure!( - source_uri == &target.source_snapshot.source_uri, - "automatic Storyline destination does not have matching canonical source ownership" - ); - let state = if projection_lineage_is_fresh(&target.source_snapshot, lineage) { - AutomaticProjectionState::Fresh - } else { - AutomaticProjectionState::Stale - }; - Ok(inspection(target, state, Some(generation))) -} - -pub async fn maintain_automatic_storyline_projection( - target: &AutomaticProjectionTarget, -) -> Result { - let inspection = inspect_automatic_storyline_projection(target).await?; - match inspection.state { - AutomaticProjectionState::Missing => build_or_accept_concurrent_winner(target).await, - AutomaticProjectionState::Fresh => { - report_from_inspection(AutomaticProjectionMaintenanceMode::Unchanged, inspection) - } - AutomaticProjectionState::Stale => { - let outcome = - match sync_storyline_projection(&target.source_uri, &target.projection_uri).await { - Ok(outcome) => outcome, - Err(error) => return accept_fresh_concurrent_winner(target, error).await, - }; - match outcome { - StorylineProjectionSyncOutcome::Synced(report) => { - let mode = match report.mode { - StorylineProjectionSyncMode::Noop => { - AutomaticProjectionMaintenanceMode::ConcurrentWinner - } - StorylineProjectionSyncMode::Incremental => { - AutomaticProjectionMaintenanceMode::Incremental - } - StorylineProjectionSyncMode::Rebuild => { - AutomaticProjectionMaintenanceMode::Rebuilt - } - }; - Ok(AutomaticProjectionMaintenanceReport { - mode, - generation: report.generation, - fact_version: report.fact_version, - fact_rows: report.fact_rows, - trajectories: (mode - != AutomaticProjectionMaintenanceMode::ConcurrentWinner) - .then_some(report.affected_storylines), - }) - } - StorylineProjectionSyncOutcome::MissingProjection => { - build_or_accept_concurrent_winner(target).await - } - StorylineProjectionSyncOutcome::RequiresRebuild(_) => { - ensure_current_lineage_owns_target(target).await?; - match rebuild_storyline_projection( - &target.source_uri, - &target.projection_uri, - &target.source_path, - ) - .await - { - Ok(report) => Ok(AutomaticProjectionMaintenanceReport { - mode: AutomaticProjectionMaintenanceMode::Rebuilt, - generation: report.generation, - fact_version: report.fact_version, - fact_rows: report.fact_rows, - trajectories: Some(report.affected_storylines), - }), - Err(error) => accept_fresh_concurrent_winner(target, error).await, - } - } - } - } - } -} - -pub async fn storyline_projection_destination_exists(uri: impl AsRef) -> Result { - StorylineLanceStore::destination_exists(uri).await -} - -fn inspection( - target: &AutomaticProjectionTarget, - state: AutomaticProjectionState, - generation: Option, -) -> AutomaticProjectionInspection { - AutomaticProjectionInspection { - state, - generation, - fact_version: target.source_snapshot.fact_version, - fact_rows: target.source_snapshot.fact_rows, - } -} - -async fn build_or_accept_concurrent_winner( - target: &AutomaticProjectionTarget, -) -> Result { - match build_storyline_projection( - &target.source_uri, - &target.projection_uri, - &target.source_path, - ) - .await? - { - StorylineProjectionBuildOutcome::Built(report) => { - Ok(AutomaticProjectionMaintenanceReport { - mode: AutomaticProjectionMaintenanceMode::Built, - generation: report.generation, - fact_version: report.fact_version, - fact_rows: report.fact_rows, - trajectories: Some(report.storylines), - }) - } - StorylineProjectionBuildOutcome::OutputNotEmpty => { - let inspection = inspect_automatic_storyline_projection(target).await?; - anyhow::ensure!( - inspection.state == AutomaticProjectionState::Fresh, - "automatic Storyline destination became nonempty without a fresh matching projection" - ); - report_from_inspection( - AutomaticProjectionMaintenanceMode::ConcurrentWinner, - inspection, - ) - } - } -} - -async fn ensure_current_lineage_owns_target(target: &AutomaticProjectionTarget) -> Result<()> { - let status = storyline_projection_status(&target.projection_uri).await?; - let lineage = status - .lineage - .as_ref() - .context("automatic Storyline destination has no canonical lineage")?; - let ProjectionSourceSnapshot::CanonicalEvents { source_uri, .. } = &lineage.source else { - anyhow::bail!("automatic Storyline destination was not derived from canonical events"); - }; - let expected = canonical_projection_lineage(&target.source_snapshot, &target.source_path); - anyhow::ensure!( - source_uri == &target.source_snapshot.source_uri && lineage.source_id == expected.source_id, - "automatic Storyline destination does not have matching canonical source ownership" - ); - Ok(()) -} - -async fn accept_fresh_concurrent_winner( - target: &AutomaticProjectionTarget, - original: anyhow::Error, -) -> Result { - match inspect_automatic_storyline_projection(target).await { - Ok(inspection) if inspection.state == AutomaticProjectionState::Fresh => { - report_from_inspection( - AutomaticProjectionMaintenanceMode::ConcurrentWinner, - inspection, - ) - } - _ => Err(original), - } -} - -fn report_from_inspection( - mode: AutomaticProjectionMaintenanceMode, - inspection: AutomaticProjectionInspection, -) -> Result { - let generation = inspection - .generation - .context("fresh automatic Storyline projection has no generation")?; - Ok(AutomaticProjectionMaintenanceReport { - mode, - generation, - fact_version: inspection.fact_version, - fact_rows: inspection.fact_rows, - trajectories: None, - }) -} - -fn display_source_path(source_path: &str) -> String { - if source_path == "." { - CANONICAL_EVENT_STORE_LEAF.into() - } else { - source_path.into() - } -} - -fn display_projection_path(source_path: &str) -> Result { - let parent = source_path - .strip_suffix(CANONICAL_EVENT_STORE_LEAF) - .with_context(|| { - format!("canonical source path must end with {CANONICAL_EVENT_STORE_LEAF}") - })? - .trim_end_matches('/'); - Ok(if parent.is_empty() { - AUTOMATIC_STORYLINE_PROJECTION_LEAF.into() - } else { - format!("{parent}/{AUTOMATIC_STORYLINE_PROJECTION_LEAF}") - }) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::layout::StoryCoords; - use crate::projection::{StorylineProjectionBuildOutcome, build_storyline_projection}; - use crate::store::opendal_store::Store as OpendalStore; - use crate::store::{ - CatalogSnapshotOptions, DatasetCatalogSnapshot, DatasetMount, RawEventLanceStore, - raw_event_lance_path, - }; - use crate::{EventIdentity, EventRecord}; - - fn note(session_id: &str, seq: u64) -> EventRecord { - EventRecord { - identity: EventIdentity::default(), - seq, - source: "test".into(), - kind: "note".into(), - timestamp: None, - session_id: Some(session_id.into()), - agent_id: Some("agent".into()), - parent_uuid: None, - trace_id: None, - call_id: None, - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload: serde_json::json!({"content": session_id}), - } - } - - async fn append_note(storage: &Path, run_id: &str) -> Result { - append_note_at(storage, run_id, 0).await - } - - async fn append_note_at(storage: &Path, run_id: &str, seq: u64) -> Result { - let coords = StoryCoords::new( - storage.to_string_lossy(), - "agent", - run_id, - Some(run_id.into()), - ); - RawEventLanceStore - .append_events(&coords, &[note(run_id, seq)]) - .await?; - Ok(raw_event_lance_path(&coords)? - .to_string_lossy() - .into_owned()) - } - - async fn target_for(source_uri: &str, projection: &Path) -> Result { - let source_snapshot = probe_canonical_event_store(source_uri) - .await? - .expect("test source is canonical"); - Ok(AutomaticProjectionTarget { - dataset: "dataset".into(), - source_path: "events.lance".into(), - source_uri: source_snapshot.source_uri.clone(), - projection_path: "storyline".into(), - projection_uri: projection.to_string_lossy().into_owned(), - source_snapshot, - }) - } - - #[test] - fn automatic_projection_uri_requires_the_exact_canonical_leaf() { - assert_eq!( - automatic_storyline_projection_uri("/datasets/run/events.lance").unwrap(), - "/datasets/run/storyline" - ); - assert_eq!( - automatic_storyline_projection_uri("s3://bucket/runs/one/events.lance").unwrap(), - "s3://bucket/runs/one/storyline" - ); - assert_eq!( - automatic_storyline_projection_uri("memory://events.lance").unwrap(), - "memory://storyline" - ); - - for invalid in [ - "/datasets/run/not-events.lance", - "/datasets/run/events.lance/segment", - "s3://bucket/runs/events.lance-copy", - ] { - assert!( - automatic_storyline_projection_uri(invalid).is_err(), - "{invalid}" - ); - } - } - - #[cfg(feature = "proptest")] - mod proptests { - use proptest::prelude::*; - - use super::*; - - proptest! { - #[test] - fn local_projection_uris_replace_only_the_canonical_leaf( - parent in proptest::string::string_regex("[A-Za-z0-9_-]{1,24}").unwrap(), - nested in proptest::string::string_regex("[A-Za-z0-9_-]{1,24}").unwrap(), - ) { - let source = format!("/datasets/{parent}/{nested}/events.lance"); - let projection = automatic_storyline_projection_uri(&source).unwrap(); - prop_assert_eq!(projection, format!("/datasets/{parent}/{nested}/storyline")); - } - - #[test] - fn remote_projection_uris_preserve_scheme_and_bucket( - bucket in proptest::string::string_regex("[a-z0-9-]{1,16}").unwrap(), - prefix in proptest::string::string_regex("[a-z0-9/_-]{1,32}").unwrap(), - ) { - let source = format!("s3://{bucket}/{prefix}/events.lance"); - let projection = automatic_storyline_projection_uri(&source).unwrap(); - prop_assert_eq!(projection, format!("s3://{bucket}/{prefix}/storyline")); - } - } - } - - #[tokio::test] - async fn inventory_is_sorted_and_uses_pinned_event_snapshots() -> Result<()> { - let temp = tempfile::tempdir()?; - let storage = temp.path().join("capture"); - let source_b = append_note(&storage, "b").await?; - let source_a = append_note(&storage, "a").await?; - let projection_a = storage.join("agent/a/storyline"); - assert!(matches!( - build_storyline_projection(&source_a, projection_a.to_string_lossy(), "a/events.lance") - .await?, - StorylineProjectionBuildOutcome::Built(_) - )); - - let mount_root = storage.join("agent"); - let snapshot = DatasetCatalogSnapshot::discover( - vec![DatasetMount::default(mount_root.to_string_lossy())?], - Some("dataset".into()), - CatalogSnapshotOptions::default(), - ) - .await?; - let inventory = automatic_projection_inventory(&snapshot)?; - - assert_eq!(inventory.snapshot_id, snapshot.snapshot_id()); - assert!(inventory.errors.is_empty()); - assert_eq!( - inventory - .targets - .iter() - .map(|target| target.source_path.as_str()) - .collect::>(), - ["a/events.lance", "b/events.lance"] - ); - assert_eq!(inventory.targets[0].projection_path, "a/storyline"); - assert_eq!(inventory.targets[1].projection_path, "b/storyline"); - assert_eq!( - inventory.targets[1].source_uri, - std::fs::canonicalize(source_b)?.to_string_lossy() - ); - assert_eq!(inventory.targets[1].source_snapshot.fact_rows, 1); - let fresh = inspect_automatic_storyline_projection(&inventory.targets[0]).await?; - assert_eq!(fresh.state, AutomaticProjectionState::Fresh); - assert!(fresh.generation.is_some()); - - let direct = DatasetCatalogSnapshot::discover( - vec![DatasetMount::default(&source_a)?], - Some("dataset".into()), - CatalogSnapshotOptions::default(), - ) - .await?; - let direct = automatic_projection_inventory(&direct)?; - assert_eq!(direct.targets[0].source_path, "events.lance"); - assert_eq!(direct.targets[0].projection_path, "storyline"); - Ok(()) - } - - #[tokio::test] - async fn missing_inspection_is_observational() -> Result<()> { - let temp = tempfile::tempdir()?; - let storage = temp.path().join("capture"); - append_note(&storage, "missing").await?; - let mount_root = storage.join("agent"); - let snapshot = DatasetCatalogSnapshot::discover( - vec![DatasetMount::default(mount_root.to_string_lossy())?], - None, - CatalogSnapshotOptions::default(), - ) - .await?; - let target = automatic_projection_inventory(&snapshot)?.targets.remove(0); - - let inspection = inspect_automatic_storyline_projection(&target).await?; - assert_eq!(inspection.state, AutomaticProjectionState::Missing); - assert_eq!(inspection.generation, None); - assert_eq!(inspection.fact_rows, 1); - assert!(!Path::new(&target.projection_uri).exists()); - Ok(()) - } - - #[tokio::test] - async fn destination_existence_covers_local_and_object_stores() -> Result<()> { - let temp = tempfile::tempdir()?; - let missing = temp.path().join("missing"); - assert!(!storyline_projection_destination_exists(missing.to_string_lossy()).await?); - assert!(!missing.exists()); - - let empty = temp.path().join("empty"); - std::fs::create_dir(&empty)?; - assert!(storyline_projection_destination_exists(empty.to_string_lossy()).await?); - - let remote = format!( - "shared-memory://pchronicle-automatic-exists-{}/storyline", - uuid::Uuid::new_v4().simple() - ); - assert!(!storyline_projection_destination_exists(&remote).await?); - let store = OpendalStore::from_uri(&remote).await?; - store - .write_overwrite("sentinel", b"present".to_vec()) - .await?; - assert!(storyline_projection_destination_exists(&remote).await?); - Ok(()) - } - - #[tokio::test] - async fn inspection_refuses_foreign_lineage_free_and_malformed_destinations() -> Result<()> { - let temp = tempfile::tempdir()?; - let storage = temp.path().join("capture"); - let source_a = append_note(&storage, "a").await?; - let source_b = append_note(&storage, "b").await?; - let snapshot_b = probe_canonical_event_store(&source_b) - .await? - .expect("source b is canonical"); - let projection_b = storage.join("agent/b/storyline"); - assert!(matches!( - build_storyline_projection(&source_a, projection_b.to_string_lossy(), "a/events.lance") - .await?, - StorylineProjectionBuildOutcome::Built(_) - )); - let target_b = AutomaticProjectionTarget { - dataset: "dataset".into(), - source_path: "b/events.lance".into(), - source_uri: snapshot_b.source_uri.clone(), - projection_path: "b/storyline".into(), - projection_uri: projection_b.to_string_lossy().into_owned(), - source_snapshot: snapshot_b, - }; - let before = std::fs::read(projection_b.join("CURRENT"))?; - let error = inspect_automatic_storyline_projection(&target_b) - .await - .unwrap_err(); - assert!( - error - .to_string() - .contains("matching canonical source ownership") - ); - assert_eq!(std::fs::read(projection_b.join("CURRENT"))?, before); - - let projection_a = storage.join("agent/a/storyline"); - assert!(matches!( - build_storyline_projection(&source_a, projection_a.to_string_lossy(), "a/events.lance") - .await?, - StorylineProjectionBuildOutcome::Built(_) - )); - let snapshot_a = probe_canonical_event_store(&source_a) - .await? - .expect("source a is canonical"); - let target_a = AutomaticProjectionTarget { - dataset: "dataset".into(), - source_path: "a/events.lance".into(), - source_uri: snapshot_a.source_uri.clone(), - projection_path: "a/storyline".into(), - projection_uri: projection_a.to_string_lossy().into_owned(), - source_snapshot: snapshot_a, - }; - let current = projection_a.join("CURRENT"); - let mut pointer: serde_json::Value = serde_json::from_slice(&std::fs::read(¤t)?)?; - pointer["committed"] - .as_object_mut() - .expect("CURRENT pointer object") - .remove("projection"); - std::fs::write(¤t, serde_json::to_vec(&pointer)?)?; - let lineage_free = std::fs::read(¤t)?; - let error = inspect_automatic_storyline_projection(&target_a) - .await - .unwrap_err(); - assert!(error.to_string().contains("has no canonical lineage")); - assert_eq!(std::fs::read(¤t)?, lineage_free); - - std::fs::write(¤t, b"{broken")?; - let malformed = std::fs::read(¤t)?; - assert!( - inspect_automatic_storyline_projection(&target_a) - .await - .is_err() - ); - assert_eq!(std::fs::read(¤t)?, malformed); - Ok(()) - } - - #[tokio::test] - async fn inventory_reports_malformed_canonical_candidates_without_targeting_them() -> Result<()> - { - let temp = tempfile::tempdir()?; - let source = temp.path().join("bad/events.lance"); - std::fs::create_dir_all(&source)?; - std::fs::write(source.join("_manifest.json"), b"{broken")?; - let snapshot = DatasetCatalogSnapshot::discover( - vec![DatasetMount::default(temp.path().to_string_lossy())?], - None, - CatalogSnapshotOptions::default() - .with_error_policy(crate::store::CatalogErrorPolicy::Report), - ) - .await?; - - let inventory = automatic_projection_inventory(&snapshot)?; - assert!(inventory.targets.is_empty()); - assert_eq!( - inventory.errors, - [AutomaticProjectionInventoryError { - dataset: "dataset".into(), - source_path: "bad/events.lance".into(), - projection_path: "bad/storyline".into(), - }] - ); - Ok(()) - } - - #[tokio::test] - async fn maintenance_builds_syncs_and_noops() -> Result<()> { - let temp = tempfile::tempdir()?; - let storage = temp.path().join("capture"); - let source = append_note_at(&storage, "run", 0).await?; - let projection = storage.join("agent/run/storyline"); - let target = target_for(&source, &projection).await?; - - let built = maintain_automatic_storyline_projection(&target).await?; - assert_eq!(built.mode, AutomaticProjectionMaintenanceMode::Built); - assert!(built.published()); - assert_eq!(built.fact_rows, 1); - - let unchanged = maintain_automatic_storyline_projection(&target).await?; - assert_eq!( - unchanged.mode, - AutomaticProjectionMaintenanceMode::Unchanged - ); - assert!(!unchanged.published()); - assert_eq!(unchanged.generation, built.generation); - - append_note_at(&storage, "run", 1).await?; - let refreshed = target_for(&source, &projection).await?; - let incremental = maintain_automatic_storyline_projection(&refreshed).await?; - assert_eq!( - incremental.mode, - AutomaticProjectionMaintenanceMode::Incremental - ); - assert!(incremental.published()); - assert_eq!(incremental.fact_rows, 2); - assert_ne!(incremental.generation, built.generation); - Ok(()) - } - - #[tokio::test] - async fn maintenance_rebuilds_only_owned_outputs() -> Result<()> { - let temp = tempfile::tempdir()?; - let storage = temp.path().join("capture"); - let source = append_note(&storage, "run").await?; - let projection = storage.join("agent/run/storyline"); - let target = target_for(&source, &projection).await?; - assert_eq!( - maintain_automatic_storyline_projection(&target).await?.mode, - AutomaticProjectionMaintenanceMode::Built - ); - - let current = projection.join("CURRENT"); - let mut pointer: serde_json::Value = serde_json::from_slice(&std::fs::read(¤t)?)?; - pointer["committed"]["projection"]["recipe_hash"] = serde_json::json!("blake3:obsolete"); - std::fs::write(¤t, serde_json::to_vec(&pointer)?)?; - let rebuilt = maintain_automatic_storyline_projection(&target).await?; - assert_eq!(rebuilt.mode, AutomaticProjectionMaintenanceMode::Rebuilt); - assert!(rebuilt.published()); - - let mut pointer: serde_json::Value = serde_json::from_slice(&std::fs::read(¤t)?)?; - pointer["committed"]["projection"]["source"]["fact_version"] = serde_json::json!(999); - pointer["committed"]["projection"]["source"]["fact_rows"] = serde_json::json!(999); - std::fs::write(¤t, serde_json::to_vec(&pointer)?)?; - let rebuilt_non_monotonic = maintain_automatic_storyline_projection(&target).await?; - assert_eq!( - rebuilt_non_monotonic.mode, - AutomaticProjectionMaintenanceMode::Rebuilt - ); - - let mut pointer: serde_json::Value = serde_json::from_slice(&std::fs::read(¤t)?)?; - pointer["committed"]["projection"]["source"]["source_uri"] = - serde_json::json!("/foreign/events.lance"); - std::fs::write(¤t, serde_json::to_vec(&pointer)?)?; - let before = std::fs::read(¤t)?; - let error = maintain_automatic_storyline_projection(&target) - .await - .unwrap_err(); - assert!(error.to_string().contains("matching canonical source")); - assert_eq!(std::fs::read(¤t)?, before); - Ok(()) - } - - #[tokio::test] - async fn concurrent_maintenance_accepts_one_fresh_winner() -> Result<()> { - let temp = tempfile::tempdir()?; - let storage = temp.path().join("capture"); - let source = append_note(&storage, "run").await?; - let projection = storage.join("agent/run/storyline"); - let target = target_for(&source, &projection).await?; - - let (left, right) = tokio::join!( - maintain_automatic_storyline_projection(&target), - maintain_automatic_storyline_projection(&target), - ); - let reports = [left?, right?]; - assert_eq!( - reports - .iter() - .filter(|report| report.mode == AutomaticProjectionMaintenanceMode::Built) - .count(), - 1 - ); - assert_eq!( - reports - .iter() - .filter(|report| { - report.mode == AutomaticProjectionMaintenanceMode::ConcurrentWinner - }) - .count(), - 1 - ); - assert_eq!( - inspect_automatic_storyline_projection(&target).await?.state, - AutomaticProjectionState::Fresh - ); - Ok(()) - } -} diff --git a/crates/persisting-pchronicle/src/projection/mod.rs b/crates/persisting-pchronicle/src/projection/mod.rs deleted file mode 100644 index 25ecdde8b..000000000 --- a/crates/persisting-pchronicle/src/projection/mod.rs +++ /dev/null @@ -1,19 +0,0 @@ -//! Rebuildable views derived from canonical trajectory facts. - -mod automatic; -mod storyline; - -pub use automatic::{ - AutomaticProjectionInspection, AutomaticProjectionInventory, AutomaticProjectionInventoryError, - AutomaticProjectionMaintenanceMode, AutomaticProjectionMaintenanceReport, - AutomaticProjectionState, AutomaticProjectionTarget, automatic_projection_inventory, - inspect_automatic_storyline_projection, maintain_automatic_storyline_projection, - probe_canonical_event_store, storyline_projection_destination_exists, -}; -pub use storyline::{ - ProjectionRebuildReason, StorylineProjectionBuildOutcome, StorylineProjectionBuildReport, - StorylineProjectionStatus, StorylineProjectionSyncMode, StorylineProjectionSyncOutcome, - StorylineProjectionSyncReport, StorylineProjectionVerification, build_storyline_projection, - projection_lineage_is_fresh, rebuild_storyline_projection, storyline_projection_status, - sync_storyline_projection, verify_storyline_projection, -}; diff --git a/crates/persisting-pchronicle/src/projection/storyline.rs b/crates/persisting-pchronicle/src/projection/storyline.rs deleted file mode 100644 index cf219911a..000000000 --- a/crates/persisting-pchronicle/src/projection/storyline.rs +++ /dev/null @@ -1,878 +0,0 @@ -//! Rebuildable canonical-events → Storyline projection lifecycle. - -use std::collections::BTreeMap; - -use anyhow::{Context, Result}; -use serde::{Deserialize, Serialize}; - -use crate::convert::{event_storyline_key, project_event_records}; -use crate::formats::{EventRecord, StorylineDocument}; -use crate::store::{ - EventFactSnapshot, ProjectionSourceSnapshot, RawEventDataSource, StorylineLanceStore, - StorylineProjectionLineage, StorylineProjectionPublicationOutcome, -}; - -pub const STORYLINE_PROJECTOR_NAME: &str = "canonical-events-to-storyline"; -pub const STORYLINE_PROJECTION_COMPLETENESS: &str = "full"; -const STORYLINE_PROJECTION_RECIPE: &str = - "group=storyline_identity;order=manifest_append;projection=events-storyline"; - -#[cfg(test)] -#[derive(Clone)] -struct BuildPublicationBarrierHook { - output_uri: String, - barrier: std::sync::Arc, -} - -#[cfg(test)] -static BUILD_BEFORE_PUBLICATION_BARRIER: std::sync::Mutex> = - std::sync::Mutex::new(None); - -#[cfg(test)] -static PROJECT_SOURCE_READ_FAILURE: std::sync::Mutex> = std::sync::Mutex::new(None); - -#[cfg(test)] -async fn wait_before_build_publication(output_uri: &str) { - let barrier = BUILD_BEFORE_PUBLICATION_BARRIER - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .as_ref() - .filter(|hook| hook.output_uri == output_uri) - .map(|hook| hook.barrier.clone()); - if let Some(barrier) = barrier { - barrier.wait().await; - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct StorylineProjectionBuildReport { - pub source_uri: String, - pub output_uri: String, - pub source_id: String, - pub generation: String, - pub fact_version: u64, - pub fact_rows: u64, - pub storylines: usize, - pub steps: usize, - pub tool_calls: usize, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum StorylineProjectionBuildOutcome { - Built(StorylineProjectionBuildReport), - OutputNotEmpty, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct StorylineProjectionStatus { - pub output_uri: String, - pub generation: Option, - pub lineage: Option, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct StorylineProjectionVerification { - pub fresh: bool, - pub reason: String, - pub source: EventFactSnapshot, - pub projection: StorylineProjectionStatus, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum StorylineProjectionSyncMode { - Noop, - Incremental, - Rebuild, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct StorylineProjectionSyncReport { - pub mode: StorylineProjectionSyncMode, - pub source_uri: String, - pub output_uri: String, - pub generation: String, - pub previous_fact_version: Option, - pub fact_version: u64, - pub previous_fact_rows: Option, - pub fact_rows: u64, - pub affected_storylines: usize, - /// Newly appended canonical rows read to discover affected Storylines. - pub suffix_rows_scanned: u64, - /// Canonical history rows read for the affected Storylines. - pub history_rows_scanned: u64, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum ProjectionRebuildReason { - MissingLineage, - IncompatibleLineage, - NonMonotonicWatermark, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum StorylineProjectionSyncOutcome { - Synced(StorylineProjectionSyncReport), - MissingProjection, - RequiresRebuild(ProjectionRebuildReason), -} - -/// Build a complete, pinned Storyline projection into an empty output store. -pub async fn build_storyline_projection( - source_uri: impl AsRef, - output_uri: impl AsRef, - source_file: impl Into, -) -> Result { - let source = RawEventDataSource::open_uri(source_uri.as_ref()) - .await - .with_context(|| format!("open canonical event source {}", source_uri.as_ref()))?; - let output = StorylineLanceStore::open_uri(output_uri.as_ref()) - .await - .with_context(|| format!("open Storyline projection output {}", output_uri.as_ref()))?; - // Fast-path known conflicts before reading the full source. Publication - // still rechecks under the store write guard and object-store CURRENT CAS. - if output.current_table_paths().await?.is_some() { - return Ok(StorylineProjectionBuildOutcome::OutputNotEmpty); - } - let snapshot = source.fact_snapshot().clone(); - let lineage = canonical_projection_lineage(&snapshot, source_file.into()); - let stories = project_canonical_event_source(&source).await?; - #[cfg(test)] - wait_before_build_publication(output.root_uri()).await; - let report = match output - .create_projected_storyline_stream( - stories.into_iter().map(Ok::<_, anyhow::Error>), - lineage.clone(), - ) - .await - .context("publish Storyline projection")? - { - StorylineProjectionPublicationOutcome::Published(report) => report, - StorylineProjectionPublicationOutcome::OutputNotEmpty => { - return Ok(StorylineProjectionBuildOutcome::OutputNotEmpty); - } - }; - Ok(StorylineProjectionBuildOutcome::Built( - StorylineProjectionBuildReport { - source_uri: snapshot.source_uri, - output_uri: output.root_uri().to_string(), - source_id: lineage.source_id, - generation: report.generation, - fact_version: snapshot.fact_version, - fact_rows: snapshot.fact_rows, - storylines: report.storylines, - steps: report.steps, - tool_calls: report.tool_calls, - }, - )) -} - -pub async fn storyline_projection_status( - output_uri: impl AsRef, -) -> Result { - let output_uri = output_uri.as_ref(); - if !StorylineLanceStore::destination_exists(output_uri).await? { - return Ok(StorylineProjectionStatus { - output_uri: output_uri.to_string(), - generation: None, - lineage: None, - }); - } - let output = StorylineLanceStore::open_uri(output_uri).await?; - let paths = output.current_table_paths().await?; - Ok(StorylineProjectionStatus { - output_uri: output.root_uri().to_string(), - generation: paths.as_ref().map(|paths| paths.generation.clone()), - lineage: paths.and_then(|paths| paths.projection), - }) -} - -/// Replace every projection table from one pinned canonical fact snapshot. -pub async fn rebuild_storyline_projection( - source_uri: impl AsRef, - output_uri: impl AsRef, - source_file: impl Into, -) -> Result { - let source = RawEventDataSource::open_uri(source_uri.as_ref()).await?; - let output = StorylineLanceStore::open_uri(output_uri.as_ref()).await?; - let snapshot = source.fact_snapshot().clone(); - let previous = output - .current_table_paths() - .await? - .and_then(|paths| paths.projection); - let (previous_fact_version, previous_fact_rows) = lineage_fact_watermark(previous.as_ref()); - let lineage = canonical_projection_lineage(&snapshot, source_file.into()); - let stories = project_canonical_event_source(&source).await?; - let report = output - .rebuild_projected_storyline_stream( - stories.into_iter().map(Ok::<_, anyhow::Error>), - lineage, - ) - .await?; - Ok(StorylineProjectionSyncReport { - mode: StorylineProjectionSyncMode::Rebuild, - source_uri: snapshot.source_uri, - output_uri: output.root_uri().to_string(), - generation: report.generation, - previous_fact_version, - fact_version: snapshot.fact_version, - previous_fact_rows, - fact_rows: snapshot.fact_rows, - affected_storylines: report.storylines, - suffix_rows_scanned: 0, - history_rows_scanned: snapshot.fact_rows, - }) -} - -/// Advance a complete projection by replacing only Storylines touched by the -/// canonical append suffix. Incompatible or non-monotonic lineage requires an -/// explicit rebuild instead of guessing. -pub async fn sync_storyline_projection( - source_uri: impl AsRef, - output_uri: impl AsRef, -) -> Result { - let source = RawEventDataSource::open_uri(source_uri.as_ref()).await?; - let output = StorylineLanceStore::open_uri(output_uri.as_ref()).await?; - let snapshot = source.fact_snapshot().clone(); - let Some(paths) = output.current_table_paths().await? else { - return Ok(StorylineProjectionSyncOutcome::MissingProjection); - }; - let Some(previous) = paths.projection.as_ref() else { - return Ok(StorylineProjectionSyncOutcome::RequiresRebuild( - ProjectionRebuildReason::MissingLineage, - )); - }; - if !incremental_lineage_is_compatible(&snapshot, previous) { - return Ok(StorylineProjectionSyncOutcome::RequiresRebuild( - ProjectionRebuildReason::IncompatibleLineage, - )); - } - let ProjectionSourceSnapshot::CanonicalEvents { - fact_version: previous_fact_version, - fact_rows: previous_fact_rows, - .. - } = &previous.source - else { - anyhow::bail!( - "projection source is not canonical events; projection requires a complete rebuild" - ) - }; - - if projection_lineage_is_fresh(&snapshot, previous) { - return Ok(StorylineProjectionSyncOutcome::Synced( - StorylineProjectionSyncReport { - mode: StorylineProjectionSyncMode::Noop, - source_uri: snapshot.source_uri, - output_uri: output.root_uri().to_string(), - generation: paths.generation, - previous_fact_version: Some(*previous_fact_version), - fact_version: snapshot.fact_version, - previous_fact_rows: Some(*previous_fact_rows), - fact_rows: snapshot.fact_rows, - affected_storylines: 0, - suffix_rows_scanned: 0, - history_rows_scanned: 0, - }, - )); - } - - if *previous_fact_rows > snapshot.fact_rows || *previous_fact_version > snapshot.fact_version { - return Ok(StorylineProjectionSyncOutcome::RequiresRebuild( - ProjectionRebuildReason::NonMonotonicWatermark, - )); - } - let suffix = source - .read_records_range_in_append_order(*previous_fact_rows, snapshot.fact_rows) - .await?; - let suffix_rows_scanned = u64::try_from(suffix.len())?; - let affected = suffix - .iter() - .map(|record| { - event_storyline_key(record) - .context("new canonical event has no Storyline identity") - .map(str::to_string) - }) - .collect::>>()?; - anyhow::ensure!( - !affected.is_empty(), - "canonical fact watermark advanced without visible appended events" - ); - let affected_count = affected.len(); - let selected = source.read_records_for_storylines(&affected).await?; - let history_rows_scanned = u64::try_from(selected.len())?; - let stories = project_canonical_event_records(selected)?; - anyhow::ensure!( - stories.len() == affected_count, - "incremental projection did not cover every affected Storyline" - ); - let lineage = canonical_projection_lineage(&snapshot, previous.source_file.clone()); - output - .replace_projected_storylines(&stories, lineage) - .await?; - let generation = output - .current_table_paths() - .await? - .context("incremental projection committed no CURRENT generation")? - .generation; - Ok(StorylineProjectionSyncOutcome::Synced( - StorylineProjectionSyncReport { - mode: StorylineProjectionSyncMode::Incremental, - source_uri: snapshot.source_uri, - output_uri: output.root_uri().to_string(), - generation, - previous_fact_version: Some(*previous_fact_version), - fact_version: snapshot.fact_version, - previous_fact_rows: Some(*previous_fact_rows), - fact_rows: snapshot.fact_rows, - affected_storylines: affected_count, - suffix_rows_scanned, - history_rows_scanned, - }, - )) -} - -pub async fn verify_storyline_projection( - source_uri: impl AsRef, - output_uri: impl AsRef, -) -> Result { - let source = RawEventDataSource::open_uri(source_uri.as_ref()).await?; - let snapshot = source.fact_snapshot().clone(); - let projection = storyline_projection_status(output_uri).await?; - let (fresh, reason) = projection_lineage_freshness(&snapshot, projection.lineage.as_ref()); - Ok(StorylineProjectionVerification { - fresh, - reason, - source: snapshot, - projection, - }) -} - -pub fn canonical_projection_lineage( - snapshot: &EventFactSnapshot, - source_file: impl Into, -) -> StorylineProjectionLineage { - let source_id = format!( - "events:{}", - blake3::hash(snapshot.source_uri.as_bytes()).to_hex() - ); - StorylineProjectionLineage { - source_id, - source_file: source_file.into(), - source: ProjectionSourceSnapshot::CanonicalEvents { - source_uri: snapshot.source_uri.clone(), - fact_version: snapshot.fact_version, - fact_rows: snapshot.fact_rows, - layout_revision: snapshot.layout_revision, - }, - projector_name: STORYLINE_PROJECTOR_NAME.into(), - recipe_hash: blake3::hash(STORYLINE_PROJECTION_RECIPE.as_bytes()) - .to_hex() - .to_string(), - completeness: STORYLINE_PROJECTION_COMPLETENESS.into(), - } -} - -pub fn projection_lineage_is_fresh( - snapshot: &EventFactSnapshot, - lineage: &StorylineProjectionLineage, -) -> bool { - projection_lineage_freshness(snapshot, Some(lineage)).0 -} - -fn incremental_lineage_is_compatible( - snapshot: &EventFactSnapshot, - lineage: &StorylineProjectionLineage, -) -> bool { - let expected = canonical_projection_lineage(snapshot, lineage.source_file.clone()); - let ProjectionSourceSnapshot::CanonicalEvents { source_uri, .. } = &lineage.source else { - return false; - }; - source_uri == &snapshot.source_uri - && lineage.source_id == expected.source_id - && lineage.projector_name == expected.projector_name - && lineage.recipe_hash == expected.recipe_hash - && lineage.completeness == STORYLINE_PROJECTION_COMPLETENESS -} - -fn lineage_fact_watermark( - lineage: Option<&StorylineProjectionLineage>, -) -> (Option, Option) { - match lineage.map(|lineage| &lineage.source) { - Some(ProjectionSourceSnapshot::CanonicalEvents { - fact_version, - fact_rows, - .. - }) => (Some(*fact_version), Some(*fact_rows)), - _ => (None, None), - } -} - -fn projection_lineage_freshness( - snapshot: &EventFactSnapshot, - lineage: Option<&StorylineProjectionLineage>, -) -> (bool, String) { - let Some(lineage) = lineage else { - return (false, "projection has no canonical source lineage".into()); - }; - let ProjectionSourceSnapshot::CanonicalEvents { - source_uri, - fact_version, - fact_rows, - .. - } = &lineage.source - else { - return ( - false, - "projection was not derived from canonical events".into(), - ); - }; - if source_uri != &snapshot.source_uri - || fact_version != &snapshot.fact_version - || fact_rows != &snapshot.fact_rows - { - return ( - false, - "projection fact watermark does not match canonical events".into(), - ); - } - let mut expected = canonical_projection_lineage(snapshot, lineage.source_file.clone()); - // Layout-only compaction is not a fact change and must not stale a projection. - expected.source = lineage.source.clone(); - if lineage == &expected { - ( - true, - "projection matches the pinned canonical fact snapshot".into(), - ) - } else { - ( - false, - "projection lineage does not match the pinned canonical fact snapshot".into(), - ) - } -} - -async fn project_canonical_event_source( - source: &RawEventDataSource, -) -> Result> { - #[cfg(test)] - if PROJECT_SOURCE_READ_FAILURE - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .as_deref() - == Some(source.fact_snapshot().source_uri.as_str()) - { - anyhow::bail!("injected canonical source read failure"); - } - let records = source.read_records_in_append_order().await?; - project_canonical_event_records(records) -} - -fn project_canonical_event_records(records: Vec) -> Result> { - let mut groups = BTreeMap::>::new(); - for record in records { - let key = event_storyline_key(&record) - .context("canonical event requires session_id, storyline_id, or run_id")? - .to_string(); - groups.entry(key).or_default().push(record); - } - anyhow::ensure!(!groups.is_empty(), "canonical event source is empty"); - groups - .into_values() - .map(|records| project_event_records(&records)) - .collect() -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::{ - EventIdentity, - layout::StoryCoords, - store::{RawEventLanceStore, raw_event_lance_path}, - }; - use serde_json::json; - - struct BuildPublicationBarrier; - - struct ProjectSourceReadFailure; - - impl ProjectSourceReadFailure { - fn install(source_uri: impl Into) -> Self { - *PROJECT_SOURCE_READ_FAILURE - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(source_uri.into()); - Self - } - } - - impl Drop for ProjectSourceReadFailure { - fn drop(&mut self) { - *PROJECT_SOURCE_READ_FAILURE - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) = None; - } - } - - impl BuildPublicationBarrier { - fn install(output_uri: &str, parties: usize) -> Self { - *BUILD_BEFORE_PUBLICATION_BARRIER - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) = - Some(BuildPublicationBarrierHook { - output_uri: output_uri.to_string(), - barrier: std::sync::Arc::new(tokio::sync::Barrier::new(parties)), - }); - Self - } - } - - impl Drop for BuildPublicationBarrier { - fn drop(&mut self) { - *BUILD_BEFORE_PUBLICATION_BARRIER - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) = None; - } - } - - fn event(session_id: &str, seq: u64) -> EventRecord { - EventRecord { - identity: EventIdentity { - event_id: Some(format!("{session_id}-{seq}")), - ..Default::default() - }, - seq, - source: "test".into(), - kind: "note".into(), - timestamp: None, - session_id: Some(session_id.into()), - agent_id: Some("agent".into()), - parent_uuid: None, - trace_id: None, - call_id: None, - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload: json!({"content": format!("{session_id}-{seq}")}), - } - } - - async fn canonical_source(temp: &tempfile::TempDir) -> (String, String) { - let storage = temp.path().join("capture"); - let coords = StoryCoords::new( - storage.to_string_lossy(), - "agent", - "session", - Some("run".into()), - ); - RawEventLanceStore - .append_events(&coords, &[event("session", 1)]) - .await - .unwrap(); - let source = raw_event_lance_path(&coords) - .unwrap() - .to_string_lossy() - .into_owned(); - let output = temp.path().join("storyline").to_string_lossy().into_owned(); - (source, output) - } - - #[tokio::test] - async fn building_into_nonempty_output_is_an_explicit_conflict() { - let temp = tempfile::tempdir().unwrap(); - let (source, output) = canonical_source(&temp).await; - let StorylineProjectionBuildOutcome::Built(report) = - build_storyline_projection(&source, &output, "events.lance") - .await - .unwrap() - else { - panic!("initial build unexpectedly reported nonempty output") - }; - assert_eq!( - std::fs::canonicalize(&report.source_uri).unwrap(), - std::fs::canonicalize(&source).unwrap() - ); - assert_eq!( - std::fs::canonicalize(&report.output_uri).unwrap(), - std::fs::canonicalize(&output).unwrap() - ); - assert_eq!(report.fact_rows, 1); - assert_eq!(report.storylines, 1); - - let outcome = build_storyline_projection(&source, &output, "events.lance") - .await - .unwrap(); - - assert!(matches!( - outcome, - StorylineProjectionBuildOutcome::OutputNotEmpty - )); - } - - #[tokio::test] - async fn known_nonempty_output_skips_source_projection() { - let temp = tempfile::tempdir().unwrap(); - let (source, output) = canonical_source(&temp).await; - let initial = build_storyline_projection(&source, &output, "events.lance") - .await - .unwrap(); - assert!(matches!(initial, StorylineProjectionBuildOutcome::Built(_))); - let canonical_source = std::fs::canonicalize(&source) - .unwrap() - .to_string_lossy() - .into_owned(); - let _failure = ProjectSourceReadFailure::install(canonical_source); - - let outcome = build_storyline_projection(&source, &output, "events.lance") - .await - .unwrap(); - - assert!(matches!( - outcome, - StorylineProjectionBuildOutcome::OutputNotEmpty - )); - } - - #[tokio::test] - async fn concurrent_builds_publish_exactly_one_projection() { - let temp = tempfile::tempdir().unwrap(); - let (source, output) = canonical_source(&temp).await; - let _barrier = BuildPublicationBarrier::install(&output, 2); - - let (left, right) = tokio::join!( - build_storyline_projection(&source, &output, "left-events.lance"), - build_storyline_projection(&source, &output, "right-events.lance") - ); - let left = left.unwrap(); - let right = right.unwrap(); - - let built = usize::from(matches!(left, StorylineProjectionBuildOutcome::Built(_))) - + usize::from(matches!(right, StorylineProjectionBuildOutcome::Built(_))); - let conflicts = usize::from(matches!( - left, - StorylineProjectionBuildOutcome::OutputNotEmpty - )) + usize::from(matches!( - right, - StorylineProjectionBuildOutcome::OutputNotEmpty - )); - assert_eq!(built, 1); - assert_eq!(conflicts, 1); - - let (report, source_file) = match (&left, &right) { - (StorylineProjectionBuildOutcome::Built(report), _) => (report, "left-events.lance"), - (_, StorylineProjectionBuildOutcome::Built(report)) => (report, "right-events.lance"), - _ => unreachable!("exactly one build succeeded"), - }; - let store = StorylineLanceStore::open_uri(&output).await.unwrap(); - let current = store.current_table_paths().await.unwrap().unwrap(); - assert_eq!(current.generation, report.generation); - assert_eq!(current.projection.unwrap().source_file, source_file); - let story = store.get_storyline_full("session").await.unwrap().unwrap(); - assert_eq!(story.session_id, "session"); - assert_eq!(story.turns.len(), 1); - } - - #[tokio::test] - async fn empty_projection_is_an_explicit_missing_result() { - let temp = tempfile::tempdir().unwrap(); - let (source, output) = canonical_source(&temp).await; - - let outcome = sync_storyline_projection(&source, &output).await.unwrap(); - - assert!(matches!( - outcome, - StorylineProjectionSyncOutcome::MissingProjection - )); - } - - #[tokio::test] - async fn projection_without_lineage_requires_rebuild() { - let temp = tempfile::tempdir().unwrap(); - let (source, output) = canonical_source(&temp).await; - let output_store = StorylineLanceStore::open_uri(&output).await.unwrap(); - output_store - .replace_storyline_stream( - project_canonical_event_records(vec![event("session", 1)]) - .unwrap() - .into_iter() - .map(Ok), - ) - .await - .unwrap(); - - let outcome = sync_storyline_projection(&source, &output).await.unwrap(); - - assert!(matches!( - outcome, - StorylineProjectionSyncOutcome::RequiresRebuild( - ProjectionRebuildReason::MissingLineage - ) - )); - } - - #[tokio::test] - async fn incompatible_projection_lineage_requires_rebuild() { - let temp = tempfile::tempdir().unwrap(); - let (source, output) = canonical_source(&temp).await; - let source_store = RawEventDataSource::open_uri(&source).await.unwrap(); - let mut lineage = - canonical_projection_lineage(source_store.fact_snapshot(), "events.lance"); - lineage.recipe_hash = "different-recipe".into(); - let output_store = StorylineLanceStore::open_uri(&output).await.unwrap(); - output_store - .replace_projected_storyline_stream( - project_canonical_event_records(vec![event("session", 1)]) - .unwrap() - .into_iter() - .map(Ok), - lineage, - ) - .await - .unwrap(); - - let outcome = sync_storyline_projection(&source, &output).await.unwrap(); - - assert!(matches!( - outcome, - StorylineProjectionSyncOutcome::RequiresRebuild( - ProjectionRebuildReason::IncompatibleLineage - ) - )); - } - - #[tokio::test] - async fn non_monotonic_projection_watermark_requires_rebuild() { - let temp = tempfile::tempdir().unwrap(); - let (source, output) = canonical_source(&temp).await; - let source_store = RawEventDataSource::open_uri(&source).await.unwrap(); - let mut lineage = - canonical_projection_lineage(source_store.fact_snapshot(), "events.lance"); - let ProjectionSourceSnapshot::CanonicalEvents { - fact_version, - fact_rows, - .. - } = &mut lineage.source - else { - unreachable!() - }; - *fact_version += 1; - *fact_rows += 1; - let output_store = StorylineLanceStore::open_uri(&output).await.unwrap(); - output_store - .replace_projected_storyline_stream( - project_canonical_event_records(vec![event("session", 1)]) - .unwrap() - .into_iter() - .map(Ok), - lineage, - ) - .await - .unwrap(); - - let outcome = sync_storyline_projection(&source, &output).await.unwrap(); - - assert!(matches!( - outcome, - StorylineProjectionSyncOutcome::RequiresRebuild( - ProjectionRebuildReason::NonMonotonicWatermark - ) - )); - } - - #[test] - fn full_projection_groups_by_identity_without_reordering_each_group() { - let stories = - project_canonical_event_records(vec![event("b", 2), event("a", 7), event("b", 1)]) - .unwrap(); - assert_eq!( - stories - .iter() - .map(|story| story.session_id.as_str()) - .collect::>(), - vec!["a", "b"] - ); - assert_eq!(stories[1].turns[0].extra.as_ref().unwrap()["seq"], 2); - assert_eq!(stories[1].turns[1].extra.as_ref().unwrap()["seq"], 1); - } - - #[test] - fn verification_uses_fact_identity_not_layout_revision_alone() { - let snapshot = EventFactSnapshot { - source_uri: "events.lance".into(), - fact_version: 4, - fact_rows: 9, - layout_revision: 12, - }; - let lineage = canonical_projection_lineage(&snapshot, "events.lance"); - assert!(projection_lineage_freshness(&snapshot, Some(&lineage)).0); - let compacted = EventFactSnapshot { - layout_revision: 13, - ..snapshot.clone() - }; - assert!(projection_lineage_freshness(&compacted, Some(&lineage)).0); - let appended = EventFactSnapshot { - fact_version: 5, - fact_rows: 10, - ..compacted - }; - assert!(!projection_lineage_freshness(&appended, Some(&lineage)).0); - } -} - -#[cfg(all(test, feature = "proptest"))] -mod proptests { - use proptest::prelude::*; - - use super::*; - - fn source_uri_strategy() -> impl Strategy { - proptest::string::string_regex("[a-z0-9:/._-]{1,48}").unwrap() - } - - proptest! { - #[test] - fn canonical_lineage_is_fresh_for_the_same_fact_snapshot( - source_uri in source_uri_strategy(), - source_file in proptest::string::string_regex("[A-Za-z0-9._/-]{1,48}").unwrap(), - fact_version in 0u64..1_000_000, - fact_rows in 0u64..1_000_000, - layout_revision in 0u64..1_000_000, - ) { - let snapshot = EventFactSnapshot { source_uri, fact_version, fact_rows, layout_revision }; - let lineage = canonical_projection_lineage(&snapshot, source_file); - prop_assert!(projection_lineage_is_fresh(&snapshot, &lineage)); - - let mut compacted = snapshot.clone(); - compacted.layout_revision = compacted.layout_revision.saturating_add(1); - prop_assert!(projection_lineage_is_fresh(&compacted, &lineage)); - } - - #[test] - fn fact_watermark_or_source_changes_make_lineage_stale( - source_uri in source_uri_strategy(), - source_file in proptest::string::string_regex("[A-Za-z0-9._/-]{1,48}").unwrap(), - fact_version in 0u64..1_000_000, - fact_rows in 0u64..1_000_000, - ) { - let snapshot = EventFactSnapshot { - source_uri: source_uri.clone(), - fact_version, - fact_rows, - layout_revision: 0, - }; - let lineage = canonical_projection_lineage(&snapshot, source_file.clone()); - - let mut changed_facts = snapshot.clone(); - changed_facts.fact_version = changed_facts.fact_version.saturating_add(1); - prop_assert!(!projection_lineage_is_fresh(&changed_facts, &lineage)); - - let original_snapshot = snapshot.clone(); - let mut changed_source = snapshot; - changed_source.source_uri.push_str("-changed"); - prop_assert!(!projection_lineage_is_fresh(&changed_source, &lineage)); - - let changed_file = canonical_projection_lineage(&changed_source, source_file); - prop_assert!(!projection_lineage_is_fresh(&original_snapshot, &changed_file)); - } - } -} diff --git a/crates/persisting-pchronicle/src/query.rs b/crates/persisting-pchronicle/src/query.rs index 4cc0605b0..83f8b5dbc 100644 --- a/crates/persisting-pchronicle/src/query.rs +++ b/crates/persisting-pchronicle/src/query.rs @@ -7,9 +7,8 @@ pub use crate::document::{FilterPushdown, QueryCapabilities, QueryTables}; #[cfg(feature = "lance-store")] pub use crate::store::{ CATALOG_SOURCES_TABLE, CATALOG_TRAJECTORIES_TABLE, ChronicleQueryEngine, - ChronicleQueryExecutionOptions, DATAFUSION_EVENTS_TABLE, DATAFUSION_RUNS_TABLE, - DATAFUSION_STEPS_TABLE, DATAFUSION_TOOL_CALLS_TABLE, DEFAULT_QUERY_MEMORY_LIMIT_BYTES, - ExternalTableFormat, ExternalTableSpec, FileTrajectoryQueryMetricsSnapshot, IntrospectedField, - IntrospectedTable, QUERY_MEMORY_LIMIT_ENV, QueryBackendInfo, QuerySnapshot, QueryWriteOutcome, - SOURCE_FILE_COLUMN, + ChronicleQueryExecutionOptions, DATAFUSION_RUNS_TABLE, DATAFUSION_STEPS_TABLE, + DATAFUSION_TOOL_CALLS_TABLE, DEFAULT_QUERY_MEMORY_LIMIT_BYTES, ExternalTableFormat, + ExternalTableSpec, FileTrajectoryQueryMetricsSnapshot, IntrospectedField, IntrospectedTable, + QUERY_MEMORY_LIMIT_ENV, QueryBackendInfo, QuerySnapshot, QueryWriteOutcome, SOURCE_FILE_COLUMN, }; diff --git a/crates/persisting-pchronicle/src/revision.rs b/crates/persisting-pchronicle/src/revision.rs deleted file mode 100644 index 411e2d9f9..000000000 --- a/crates/persisting-pchronicle/src/revision.rs +++ /dev/null @@ -1,273 +0,0 @@ -//! Queryable lineage catalog for derived trajectory artifacts. - -use std::sync::Arc; - -use anyhow::{Context, Result}; -use futures::TryStreamExt; -use lance::dataset::{InsertBuilder, MergeInsertBuilder, WhenMatched, WhenNotMatched}; -use lance::deps::arrow_array::{Array, RecordBatch, RecordBatchIterator, StringArray}; -use lance::deps::arrow_schema::{DataType, Field, Schema}; -use serde::{Deserialize, Serialize}; -use serde_json::Value; - -use crate::layout::StoryCoords; - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct RevisionRow { - pub revision_id: String, - #[serde(default)] - pub parent_revision_ids: Vec, - pub kind: String, - pub canonical_snapshot: String, - pub recipe: Value, - pub status: String, - pub created_at: String, - #[serde(default)] - pub output_refs: Vec, -} - -pub fn revision_dataset_path(session: &StoryCoords) -> Result { - Ok(session - .run_dir()? - .join("revisions.lance") - .to_string_lossy() - .into_owned()) -} - -fn schema() -> Arc { - Arc::new(Schema::new( - [ - "revision_id", - "parent_revision_ids_json", - "kind", - "canonical_snapshot", - "recipe_json", - "status", - "created_at", - "output_refs_json", - ] - .into_iter() - .map(|name| Field::new(name, DataType::Utf8, false)) - .collect::>(), - )) -} - -fn batch(rows: &[RevisionRow]) -> Result { - let strings = |values: Vec| Arc::new(StringArray::from(values)) as _; - let parent_revision_ids = rows - .iter() - .map(|row| serde_json::to_string(&row.parent_revision_ids)) - .collect::>>()?; - let output_refs = rows - .iter() - .map(|row| serde_json::to_string(&row.output_refs)) - .collect::>>()?; - RecordBatch::try_new( - schema(), - vec![ - strings(rows.iter().map(|r| r.revision_id.clone()).collect()), - strings(parent_revision_ids), - strings(rows.iter().map(|r| r.kind.clone()).collect()), - strings(rows.iter().map(|r| r.canonical_snapshot.clone()).collect()), - strings(rows.iter().map(|r| r.recipe.to_string()).collect()), - strings(rows.iter().map(|r| r.status.clone()).collect()), - strings(rows.iter().map(|r| r.created_at.clone()).collect()), - strings(output_refs), - ], - ) - .context("build revision catalog batch") -} - -fn text(batch: &RecordBatch, column: &str, row: usize) -> Result { - let array = batch - .column(batch.schema().index_of(column)?) - .as_any() - .downcast_ref::() - .with_context(|| format!("revision column {column} must be Utf8"))?; - anyhow::ensure!( - !array.is_null(row), - "revision column {column} must be non-null" - ); - Ok(array.value(row).to_string()) -} - -pub async fn read_revisions(session: &StoryCoords) -> Result> { - let uri = revision_dataset_path(session)?; - let dataset = match crate::storage::open_lance_dataset(&uri).await { - Ok(dataset) => dataset, - Err(lance::Error::DatasetNotFound { .. }) => return Ok(Vec::new()), - Err(error) => return Err(anyhow::anyhow!(error)).context("open revisions.lance"), - }; - let batches: Vec = dataset - .scan() - .try_into_stream() - .await? - .try_collect() - .await?; - let mut rows = Vec::new(); - for batch in batches { - for row in 0..batch.num_rows() { - rows.push(RevisionRow { - revision_id: text(&batch, "revision_id", row)?, - parent_revision_ids: serde_json::from_str(&text( - &batch, - "parent_revision_ids_json", - row, - )?)?, - kind: text(&batch, "kind", row)?, - canonical_snapshot: text(&batch, "canonical_snapshot", row)?, - recipe: serde_json::from_str(&text(&batch, "recipe_json", row)?)?, - status: text(&batch, "status", row)?, - created_at: text(&batch, "created_at", row)?, - output_refs: serde_json::from_str(&text(&batch, "output_refs_json", row)?)?, - }); - } - } - rows.sort_by(|a, b| { - a.created_at - .cmp(&b.created_at) - .then_with(|| a.revision_id.cmp(&b.revision_id)) - }); - Ok(rows) -} - -pub async fn write_revisions(session: &StoryCoords, rows: &[RevisionRow]) -> Result { - let uri = revision_dataset_path(session)?; - if rows.is_empty() { - return Ok(uri); - } - let _guard = crate::store::dataset_write_lock::acquire(&uri).await?; - let batch = batch(rows)?; - match crate::storage::open_lance_dataset(&uri).await { - Ok(dataset) => { - let reader = Box::new(RecordBatchIterator::new(vec![Ok(batch)], schema())); - MergeInsertBuilder::try_new(Arc::new(dataset), vec!["revision_id".into()])? - .when_matched(WhenMatched::UpdateAll) - .when_not_matched(WhenNotMatched::InsertAll) - .try_build()? - .execute_reader(reader) - .await?; - } - Err(lance::Error::DatasetNotFound { .. }) => { - if let Some(parent) = std::path::Path::new(&uri).parent() { - tokio::fs::create_dir_all(parent).await?; - } - InsertBuilder::new(&uri).execute(vec![batch]).await?; - } - Err(error) => return Err(anyhow::anyhow!(error)).context("open revisions.lance"), - } - Ok(uri) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[tokio::test] - async fn revision_catalog_upserts_by_id() { - let dir = tempfile::tempdir().unwrap(); - let session = StoryCoords::new(dir.path().to_string_lossy(), "a", "s", None); - let mut row = RevisionRow { - revision_id: "r1".into(), - parent_revision_ids: vec![], - kind: "clean".into(), - canonical_snapshot: "manifest:1".into(), - recipe: serde_json::json!({"op":"redact"}), - status: "building".into(), - created_at: "2026-08-09T00:00:00Z".into(), - output_refs: vec![], - }; - write_revisions(&session, &[row.clone()]).await.unwrap(); - row.status = "ready".into(); - write_revisions(&session, &[row]).await.unwrap(); - let rows = read_revisions(&session).await.unwrap(); - assert_eq!(rows.len(), 1); - assert_eq!(rows[0].status, "ready"); - } - - #[cfg(feature = "proptest")] - mod proptests { - use proptest::prelude::*; - use serde_json::Value; - - use crate::revision::{RevisionRow, batch, text}; - - fn token_strategy() -> impl Strategy { - proptest::string::string_regex("[a-zA-Z0-9._:/-]{0,32}").unwrap() - } - - fn recipe_strategy() -> impl Strategy { - prop_oneof![ - Just(Value::Null), - any::().prop_map(Value::Bool), - (0u64..100_000).prop_map(Value::from), - token_strategy().prop_map(Value::String), - token_strategy().prop_map(|value| serde_json::json!({"op": value})), - ] - } - - fn revision_row_strategy() -> impl Strategy { - ( - token_strategy(), - proptest::collection::vec(token_strategy(), 0..4), - token_strategy(), - token_strategy(), - recipe_strategy(), - token_strategy(), - token_strategy(), - proptest::collection::vec(token_strategy(), 0..4), - ) - .prop_map( - |( - revision_id, - parent_revision_ids, - kind, - canonical_snapshot, - recipe, - status, - created_at, - output_refs, - )| RevisionRow { - revision_id, - parent_revision_ids, - kind, - canonical_snapshot, - recipe, - status, - created_at, - output_refs, - }, - ) - } - - proptest! { - #[test] - fn revision_batches_preserve_all_columns(rows in proptest::collection::vec(revision_row_strategy(), 0..12)) { - let record_batch = batch(&rows).unwrap(); - prop_assert_eq!(record_batch.num_rows(), rows.len()); - for (index, expected) in rows.iter().enumerate() { - prop_assert_eq!(text(&record_batch, "revision_id", index).unwrap(), expected.revision_id.clone()); - prop_assert_eq!(serde_json::from_str::>(&text(&record_batch, "parent_revision_ids_json", index).unwrap()).unwrap(), expected.parent_revision_ids.clone()); - prop_assert_eq!(text(&record_batch, "kind", index).unwrap(), expected.kind.clone()); - prop_assert_eq!(text(&record_batch, "canonical_snapshot", index).unwrap(), expected.canonical_snapshot.clone()); - prop_assert_eq!(serde_json::from_str::(&text(&record_batch, "recipe_json", index).unwrap()).unwrap(), expected.recipe.clone()); - prop_assert_eq!(text(&record_batch, "status", index).unwrap(), expected.status.clone()); - prop_assert_eq!(text(&record_batch, "created_at", index).unwrap(), expected.created_at.clone()); - prop_assert_eq!(serde_json::from_str::>(&text(&record_batch, "output_refs_json", index).unwrap()).unwrap(), expected.output_refs.clone()); - } - } - - #[test] - fn revision_batch_preserves_unicode_and_delimiters( - id in any::(), - snapshot in any::(), - status in any::(), - ) { - let row = RevisionRow { revision_id: id.clone(), parent_revision_ids: vec!["父/parent".into()], kind: "kind\nline".into(), canonical_snapshot: snapshot.clone(), recipe: serde_json::json!({"文本": status.clone()}), status, created_at: "2026-01-01T00:00:00Z".into(), output_refs: vec!["ref\nwith\nnewline".into()] }; - let record_batch = batch(&[row]).unwrap(); - prop_assert_eq!(text(&record_batch, "revision_id", 0).unwrap(), id); - prop_assert_eq!(text(&record_batch, "canonical_snapshot", 0).unwrap(), snapshot); - } - } - } -} diff --git a/crates/persisting-pchronicle/src/storage.rs b/crates/persisting-pchronicle/src/storage.rs index a1c93e03f..3dd381d20 100644 --- a/crates/persisting-pchronicle/src/storage.rs +++ b/crates/persisting-pchronicle/src/storage.rs @@ -31,26 +31,11 @@ pub fn parse_byte_size(value: &str) -> std::result::Result { .ok_or_else(|| "byte size must be greater than zero and fit in usize".to_owned()) } -#[cfg(feature = "lance-store")] -pub use crate::append_queue::{ - DEFAULT_RAW_EVENT_BATCH_DELAY, DEFAULT_RAW_EVENT_BATCH_SIZE, - DEFAULT_RAW_EVENT_COMPACTION_THRESHOLD, DEFAULT_RAW_EVENT_HIERARCHY_FANOUT, - DEFAULT_RAW_EVENT_MAINTENANCE_CAPACITY, DEFAULT_RAW_EVENT_QUEUE_CAPACITY, - DEFAULT_RAW_EVENT_TARGET_ROWS_PER_FRAGMENT, RawEventAppendOutcome, RawEventAppendSender, - RawEventAppendWorker, raw_event_append_queue, raw_event_append_queue_with_capacity, -}; -pub use crate::layout::{ - StoryCoords, StoryLocationPartial, is_subagent_session_storage_key, - is_trajectory_markdown_path, list_story_read_locations, locate_run_bucket_markdown, - locate_session_markdown, locate_session_markdown_for_key, merge_story_location, - resolve_story_read_location, sanitize_session_filename, session_filename_stem, - session_markdown_filename, session_markdown_path_for_key, session_markdown_write_path_for_key, - story_lance_event_path, story_run_dir, try_infer_story_location, -}; - -#[cfg(feature = "lance-store")] -pub use crate::discovery::{ - drop_lifecycle_run_partitions, expand_story_locations, expand_story_locations_blocking, +pub use crate::agenticmd::{ + is_subagent_session_storage_key, is_trajectory_markdown_path, locate_run_bucket_markdown, + locate_session_markdown, locate_session_markdown_for_key, sanitize_session_filename, + session_filename_stem, session_markdown_filename, session_markdown_path_for_key, + session_markdown_write_path_for_key, }; #[cfg(feature = "lance-store")] @@ -85,32 +70,25 @@ pub async fn open_lance_dataset(uri: &str) -> lance::Result { #[cfg(feature = "lance-store")] pub use crate::store::{ - AppendOutcome, AttemptRecord, AttemptRecordState, AttemptRegistry, CachedDataset, - CatalogConsistency, CatalogDataset, CatalogErrorPolicy, CatalogEventProvenance, - CatalogEventView, CatalogNamespace, CatalogPage, CatalogProjectionStatus, - CatalogSnapshotOptions, CatalogSourceDescription, CatalogSourceKind, CatalogSourceRevision, - CatalogSourceStatus, CatalogState, CatalogStatus, CatalogStorylineKey, CatalogTrajectoryBundle, - ChronicleManifest, CommitRunOutcome, CompactJsonlBuildPhase, CompactJsonlColumn, - CompactJsonlImportEvent, CompactJsonlOffload, CompactJsonlOptions, CompactJsonlRecord, - CompactJsonlStore, DEFAULT_CONTENT_OFFLOAD_THRESHOLD, DEFAULT_CONTENT_PREVIEW_BYTES, - DEFAULT_DATASET_NAME, DEFAULT_MAX_CHUNK_BYTES, DEFAULT_MAX_EVENT_FALLBACK_BYTES, - DEFAULT_MAX_EVENT_FALLBACK_ROWS, DEFAULT_PHYSICAL_PAGE_LIMIT, Dataset, DatasetCatalogSnapshot, + CachedDataset, CatalogConsistency, CatalogDataset, CatalogErrorPolicy, CatalogNamespace, + CatalogPage, CatalogSnapshotOptions, CatalogSourceDescription, CatalogSourceKind, + CatalogSourceRevision, CatalogSourceStatus, CatalogState, CatalogStatus, CatalogStorylineKey, + ChronicleManifest, CompactJsonlBuildPhase, CompactJsonlColumn, CompactJsonlImportEvent, + CompactJsonlOffload, CompactJsonlOptions, CompactJsonlRecord, CompactJsonlStore, + DEFAULT_CONTENT_OFFLOAD_THRESHOLD, DEFAULT_CONTENT_PREVIEW_BYTES, DEFAULT_DATASET_NAME, + DEFAULT_MAX_CHUNK_BYTES, DEFAULT_PHYSICAL_PAGE_LIMIT, Dataset, DatasetCatalogSnapshot, DatasetLocation, DatasetLocationKind, DatasetMount, DatasetResolver, DiscoveredSource, - EventFactSnapshot, EventLogLayoutStats, EventWriterFence, ExportOutcome, ImportableObjectEvent, - LanceMaintenanceOptions, LanceMaintenanceReport, LeaseAcquireOutcome, LocationSummary, + ImportableObjectEvent, LanceMaintenanceOptions, LanceMaintenanceReport, LocationSummary, ManifestCache, ManifestKind, ManifestListing, ManifestReadMode, ManifestRefreshReport, ManifestStats, NamespacePath, PathListEntry, PathListKind, PersistentCache, PhysicalColumn, PhysicalDataFile, PhysicalFileLayout, PhysicalFragment, PhysicalLayout, PhysicalPage, - PhysicalPagePreview, PhysicalPageQuery, PhysicalSource, PhysicalTable, - ProjectionSourceSnapshot, QueryScope, RawEventLanceAppender, RawEventLanceStore, ReplayOutcome, - ResolveMode, ResolveTarget, RunControlStore, ShallowNavEntry, StorylineContentOptions, - StorylineContentReadMode, StorylineDataSource, StorylineDataSourceOptions, StorylineLanceStore, - StorylineMaintenanceReport, StorylineProjectionLineage, StorylineSearchIndexSuppressGuard, - StorylineStreamImportReport, StorylineStreamOptions, StorylineTablePaths, TrajectoryStats, - attempt_registry_now_ms, distinct_session_ids_in_run, export_source_dirs, export_story_bundle, - inspect_physical_file, inspect_physical_layout, inspect_physical_page, list_physical_sources, - load_manifest, load_manifest_at_uri, raw_event_lance_path, write_compact_jsonl_manifest, - write_storyline_manifest, write_storyline_manifest_at_uri, + PhysicalPagePreview, PhysicalPageQuery, PhysicalSource, PhysicalTable, QueryScope, ResolveMode, + ResolveTarget, ShallowNavEntry, StorylineContentOptions, StorylineContentReadMode, + StorylineDataSource, StorylineDataSourceOptions, StorylineLanceStore, + StorylineMaintenanceReport, StorylineSearchIndexSuppressGuard, StorylineStreamImportReport, + StorylineStreamOptions, StorylineTablePaths, inspect_physical_file, inspect_physical_layout, + inspect_physical_page, list_physical_sources, load_manifest, load_manifest_at_uri, + write_compact_jsonl_manifest, write_storyline_manifest, write_storyline_manifest_at_uri, }; // Compatibility exports; new callers should use `crate::search`. @@ -122,23 +100,4 @@ pub use crate::search::{ }; #[cfg(feature = "lance-store")] -pub use crate::store::{ - DEFAULT_MAX_LOCAL_QUERY_ENTRIES, DEFAULT_MAX_LOCAL_QUERY_FILES, maintain_raw_events, -}; - -#[cfg(feature = "lance-store")] -pub use crate::projection::{ - AutomaticProjectionInspection, AutomaticProjectionInventory, AutomaticProjectionInventoryError, - AutomaticProjectionMaintenanceMode, AutomaticProjectionMaintenanceReport, - AutomaticProjectionState, AutomaticProjectionTarget, ProjectionRebuildReason, - StorylineProjectionBuildOutcome, StorylineProjectionBuildReport, StorylineProjectionStatus, - StorylineProjectionSyncMode, StorylineProjectionSyncOutcome, StorylineProjectionSyncReport, - StorylineProjectionVerification, automatic_projection_inventory, build_storyline_projection, - inspect_automatic_storyline_projection, maintain_automatic_storyline_projection, - probe_canonical_event_store, rebuild_storyline_projection, - storyline_projection_destination_exists, storyline_projection_status, - sync_storyline_projection, verify_storyline_projection, -}; - -#[cfg(feature = "lance-store")] -pub use crate::revision::{RevisionRow, read_revisions, revision_dataset_path, write_revisions}; +pub use crate::store::{DEFAULT_MAX_LOCAL_QUERY_ENTRIES, DEFAULT_MAX_LOCAL_QUERY_FILES}; diff --git a/crates/persisting-pchronicle/src/store/attempt_registry.rs b/crates/persisting-pchronicle/src/store/attempt_registry.rs deleted file mode 100644 index 4a1cb13fe..000000000 --- a/crates/persisting-pchronicle/src/store/attempt_registry.rs +++ /dev/null @@ -1,263 +0,0 @@ -//! Durable Attempt liveness and terminal-result registry. -//! -//! One CAS-managed record exists per Run. A newer lease epoch fences every -//! update from an older Attempt, while heartbeat expiry lets a coordinator distinguish -//! a live remote Attempt from an orphan after coordinator restart. - -use super::cas_store::{CasStore, Mutation, unix_now_ms}; -use anyhow::{Context, bail}; -use serde::{Deserialize, Serialize}; -use serde_json::Value; - -const ATTEMPT_DIR: &str = "attempt-registry"; - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum AttemptRecordState { - Active, - Terminal, -} - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct AttemptRecord { - pub revision: u64, - pub run_id: String, - pub attempt_id: String, - pub lease_epoch: u64, - pub state: AttemptRecordState, - pub heartbeat_at_unix_ms: u64, - pub expires_at_unix_ms: u64, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub terminal_result: Option, -} - -impl AttemptRecord { - pub fn is_live_at(&self, now_unix_ms: u64) -> bool { - self.state == AttemptRecordState::Active && self.expires_at_unix_ms > now_unix_ms - } -} - -pub struct AttemptRegistry { - store: CasStore, -} - -impl std::fmt::Debug for AttemptRegistry { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter - .debug_struct("AttemptRegistry") - .field("root_uri", &self.store.root_uri()) - .finish_non_exhaustive() - } -} - -impl AttemptRegistry { - pub async fn open(root: impl AsRef) -> anyhow::Result { - Ok(Self { - store: CasStore::open(root, ATTEMPT_DIR, "Attempt registry").await?, - }) - } - - pub fn root_uri(&self) -> &str { - self.store.root_uri() - } - - pub async fn publish_active( - &self, - run_id: &str, - attempt_id: &str, - lease_epoch: u64, - ttl_ms: u64, - ) -> anyhow::Result { - validate_identity(run_id, attempt_id, lease_epoch)?; - let run_id = run_id.to_string(); - let attempt_id = attempt_id.to_string(); - self.mutate(&run_id.clone(), move |current| { - if current.is_some_and(|record| { - record.lease_epoch > lease_epoch - || (record.lease_epoch == lease_epoch - && (record.attempt_id != attempt_id - || record.state == AttemptRecordState::Terminal)) - }) { - return Ok(Mutation::Unchanged(false)); - } - let now = unix_now_ms(); - let record = AttemptRecord { - revision: next_revision(current)?, - run_id: run_id.clone(), - attempt_id: attempt_id.clone(), - lease_epoch, - state: AttemptRecordState::Active, - heartbeat_at_unix_ms: now, - expires_at_unix_ms: now.saturating_add(ttl_ms.max(1)), - terminal_result: None, - }; - Ok(Mutation::Replace(record, true)) - }) - .await - } - - pub async fn heartbeat( - &self, - run_id: &str, - attempt_id: &str, - lease_epoch: u64, - ttl_ms: u64, - ) -> anyhow::Result { - validate_identity(run_id, attempt_id, lease_epoch)?; - let run_id = run_id.to_string(); - let attempt_id = attempt_id.to_string(); - self.mutate(&run_id.clone(), move |current| { - let Some(current) = current else { - return Ok(Mutation::Unchanged(false)); - }; - if current.run_id != run_id - || current.attempt_id != attempt_id - || current.lease_epoch != lease_epoch - || current.state != AttemptRecordState::Active - { - return Ok(Mutation::Unchanged(false)); - } - let now = unix_now_ms(); - let mut record = current.clone(); - record.revision = next_revision(Some(current))?; - record.heartbeat_at_unix_ms = now; - record.expires_at_unix_ms = now.saturating_add(ttl_ms.max(1)); - Ok(Mutation::Replace(record, true)) - }) - .await - } - - pub async fn publish_terminal( - &self, - run_id: &str, - attempt_id: &str, - lease_epoch: u64, - result: Value, - ) -> anyhow::Result { - validate_identity(run_id, attempt_id, lease_epoch)?; - let run_id = run_id.to_string(); - let attempt_id = attempt_id.to_string(); - self.mutate(&run_id.clone(), move |current| { - let Some(current) = current else { - return Ok(Mutation::Unchanged(false)); - }; - if current.run_id != run_id - || current.attempt_id != attempt_id - || current.lease_epoch != lease_epoch - { - return Ok(Mutation::Unchanged(false)); - } - if current.state == AttemptRecordState::Terminal { - return Ok(Mutation::Unchanged( - current.terminal_result.as_ref() == Some(&result), - )); - } - let now = unix_now_ms(); - let mut record = current.clone(); - record.revision = next_revision(Some(current))?; - record.state = AttemptRecordState::Terminal; - record.heartbeat_at_unix_ms = now; - record.expires_at_unix_ms = now; - record.terminal_result = Some(result.clone()); - Ok(Mutation::Replace(record, true)) - }) - .await - } - - pub async fn get(&self, run_id: &str) -> anyhow::Result> { - self.store.get(run_id).await - } - - async fn mutate(&self, run_id: &str, mutate: F) -> anyhow::Result - where - T: Send + 'static, - F: Fn(Option<&AttemptRecord>) -> anyhow::Result> - + Send - + Sync - + 'static, - { - self.store.mutate(run_id, mutate).await - } -} - -fn validate_identity(run_id: &str, attempt_id: &str, lease_epoch: u64) -> anyhow::Result<()> { - if run_id.trim().is_empty() || attempt_id.trim().is_empty() || lease_epoch == 0 { - bail!("Attempt registry requires run_id, attempt_id, and non-zero lease_epoch"); - } - Ok(()) -} - -fn next_revision(current: Option<&AttemptRecord>) -> anyhow::Result { - current - .map(|record| record.revision) - .unwrap_or(0) - .checked_add(1) - .context("Attempt record revision overflow") -} - -#[cfg(test)] -mod tests { - use super::*; - - #[tokio::test] - async fn newer_epoch_fences_old_attempt_heartbeats_and_terminal() { - let dir = tempfile::tempdir().unwrap(); - let registry = AttemptRegistry::open(dir.path().to_str().unwrap()) - .await - .unwrap(); - assert!( - registry - .publish_active("run-1", "attempt-1", 1, 10_000) - .await - .unwrap() - ); - assert!( - registry - .publish_active("run-1", "attempt-2", 2, 10_000) - .await - .unwrap() - ); - assert!( - !registry - .heartbeat("run-1", "attempt-1", 1, 10_000) - .await - .unwrap() - ); - assert!( - !registry - .publish_terminal("run-1", "attempt-1", 1, serde_json::json!({"old": true})) - .await - .unwrap() - ); - assert!( - registry - .publish_terminal("run-1", "attempt-2", 2, serde_json::json!({"ok": true})) - .await - .unwrap() - ); - let record = registry.get("run-1").await.unwrap().unwrap(); - assert_eq!(record.state, AttemptRecordState::Terminal); - assert_eq!(record.attempt_id, "attempt-2"); - } - - #[cfg(feature = "proptest")] - mod proptests { - use proptest::prelude::*; - - use super::*; - - proptest! { - #[test] - fn identity_validation_matches_the_registry_contract( - run_id in proptest::string::string_regex("[A-Za-z0-9 _-]{0,16}").unwrap(), - attempt_id in proptest::string::string_regex("[A-Za-z0-9 _-]{0,16}").unwrap(), - epoch in any::(), - ) { - let valid = !run_id.trim().is_empty() - && !attempt_id.trim().is_empty() - && epoch != 0; - prop_assert_eq!(validate_identity(&run_id, &attempt_id, epoch).is_ok(), valid); - } - } - } -} diff --git a/crates/persisting-pchronicle/src/store/cas_store.rs b/crates/persisting-pchronicle/src/store/cas_store.rs deleted file mode 100644 index b02916282..000000000 --- a/crates/persisting-pchronicle/src/store/cas_store.rs +++ /dev/null @@ -1,572 +0,0 @@ -//! Shared JSON CAS for per-key records. -//! -//! Local backend: exclusive flock, then tmp `create_new` + rename. -//! Object backend: conditional OpenDAL writes with retries. - -use super::opendal_store::{self, Store, Version}; -use anyhow::{Context, Result, bail}; -use fs2::FileExt; -use serde::Serialize; -use serde::de::DeserializeOwned; -use std::fs::{File, OpenOptions}; -use std::io::{Read, Write}; -use std::path::{Path, PathBuf}; - -pub(crate) const CAS_RETRIES: usize = 32; - -#[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) enum Mutation { - Unchanged(T), - Replace(Record, T), -} - -enum Backend { - Local(PathBuf), - Object { store: Store, root: String }, -} - -pub(crate) struct CasStore { - root_uri: String, - name: &'static str, - backend: Backend, -} - -impl std::fmt::Debug for CasStore { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter - .debug_struct("CasStore") - .field("root_uri", &self.root_uri) - .field("name", &self.name) - .finish_non_exhaustive() - } -} - -impl CasStore { - pub async fn open(root: impl AsRef, directory: &str, name: &'static str) -> Result { - let root = root.as_ref().trim(); - if root.is_empty() { - bail!("{name} root must not be empty"); - } - if !root.contains("://") { - let path = PathBuf::from(root).join(directory); - tokio::fs::create_dir_all(&path) - .await - .with_context(|| format!("create {name} root {}", path.display()))?; - return Ok(Self { - root_uri: root.to_string(), - name, - backend: Backend::Local(path), - }); - } - let store = Store::from_uri(root).await?; - Ok(Self { - root_uri: root.to_string(), - name, - backend: Backend::Object { - store, - root: directory.trim_matches('/').to_string(), - }, - }) - } - - pub fn root_uri(&self) -> &str { - &self.root_uri - } - - pub async fn get(&self, key: &str) -> Result> - where - Record: DeserializeOwned + Send + 'static, - { - match &self.backend { - Backend::Local(root) => { - let path = record_path(root, key); - let name = self.name; - tokio::task::spawn_blocking(move || read_local_record(&path, name)).await? - } - Backend::Object { store, root } => { - Ok( - read_object_record(store, &object_record_path(root, key), self.name) - .await? - .map(|(record, _)| record), - ) - } - } - } - - pub async fn list(&self) -> Result> - where - Record: DeserializeOwned + Send + 'static, - { - match &self.backend { - Backend::Local(root) => { - let root = root.clone(); - let name = self.name; - tokio::task::spawn_blocking(move || list_local_records(&root, name)).await? - } - Backend::Object { store, root } => { - let prefix = root.clone(); - let objects = store.list(&prefix).await?; - let mut out = Vec::new(); - for object in objects { - if !object.path.ends_with(".json") { - continue; - } - if let Some((record, _)) = - read_object_record(store, &object.path, self.name).await? - { - out.push(record); - } - } - Ok(out) - } - } - } - - pub async fn mutate(&self, key: &str, mutate: F) -> Result - where - Record: Serialize + DeserializeOwned + Send + 'static, - T: Send + 'static, - F: Fn(Option<&Record>) -> Result> + Send + Sync + 'static, - { - match &self.backend { - Backend::Local(root) => { - let path = record_path(root, key); - let lock_path = path.with_extension("lock"); - let name = self.name; - tokio::task::spawn_blocking(move || { - if let Some(parent) = path.parent() { - std::fs::create_dir_all(parent)?; - } - let lock = OpenOptions::new() - .create(true) - .truncate(false) - .read(true) - .write(true) - .open(&lock_path)?; - lock.lock_exclusive()?; - let current = read_local_record(&path, name)?; - let outcome = mutate(current.as_ref())?; - let value = match outcome { - Mutation::Unchanged(value) => value, - Mutation::Replace(record, value) => { - write_local_record(&path, &record, name)?; - value - } - }; - FileExt::unlock(&lock)?; - Ok(value) - }) - .await? - } - Backend::Object { store, root } => { - let fallback_lock = store.fallback_lock(); - let _fallback_guard = match fallback_lock.as_ref() { - Some(lock) => Some(lock.lock().await), - None => None, - }; - let path = object_record_path(root, key); - for _ in 0..CAS_RETRIES { - let current = read_object_record(store, &path, self.name).await?; - let outcome = mutate(current.as_ref().map(|(record, _)| record))?; - let (record, value) = match outcome { - Mutation::Unchanged(value) => return Ok(value), - Mutation::Replace(record, value) => (record, value), - }; - let version = current.as_ref().map(|(_, version)| version); - let bytes = serde_json::to_vec_pretty(&record)?; - let result = match version { - None => store.write_create(&path, bytes).await, - Some(version) => store.write_match(&path, bytes, version).await, - }; - match result { - Ok(_) => return Ok(value), - Err(error) - if error - .downcast_ref::() - .is_some_and(opendal_store::is_conflict) => - { - continue; - } - Err(error) => return Err(error), - } - } - bail!( - "{} CAS contention exceeded {CAS_RETRIES} retries", - self.name - ) - } - } - } -} - -pub use persisting_events::unix_now_ms; - -pub(crate) fn encoded_id(value: &str) -> String { - let mut encoded = String::with_capacity(value.len()); - for byte in value.bytes() { - match byte { - b'a'..=b'z' | b'A'..=b'Z' | b'0'..=b'9' | b'-' | b'_' | b'.' => { - encoded.push(char::from(byte)); - } - other => encoded.push_str(&format!("~{other:02x}")), - } - } - encoded -} - -fn record_path(root: &Path, key: &str) -> PathBuf { - root.join(format!("{}.json", encoded_id(key))) -} - -fn object_record_path(root: &str, key: &str) -> String { - format!("{}/{}.json", root.trim_matches('/'), encoded_id(key)) -} - -fn list_local_records( - root: &Path, - name: &'static str, -) -> Result> { - let mut out = Vec::new(); - for entry in std::fs::read_dir(root)? { - let entry = entry?; - if entry.path().extension().and_then(|value| value.to_str()) != Some("json") { - continue; - } - if let Some(record) = read_local_record(&entry.path(), name)? { - out.push(record); - } - } - Ok(out) -} - -fn read_local_record( - path: &Path, - name: &'static str, -) -> Result> { - let mut file = match File::open(path) { - Ok(file) => file, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), - Err(error) => return Err(error.into()), - }; - let mut bytes = Vec::new(); - file.read_to_end(&mut bytes)?; - Ok(Some(serde_json::from_slice(&bytes).with_context(|| { - format!("decode {name} record {}", path.display()) - })?)) -} - -fn write_local_record( - path: &Path, - record: &Record, - name: &'static str, -) -> Result<()> { - let parent = path - .parent() - .with_context(|| format!("{name} path has no parent"))?; - std::fs::create_dir_all(parent)?; - let temporary = parent.join(format!( - ".{}.{}.tmp", - path.file_name() - .and_then(|name| name.to_str()) - .unwrap_or("record"), - std::process::id() - )); - let mut file = OpenOptions::new() - .create_new(true) - .write(true) - .open(&temporary)?; - file.write_all(&serde_json::to_vec_pretty(record)?)?; - file.sync_all()?; - std::fs::rename(&temporary, path)?; - File::open(parent)?.sync_all()?; - Ok(()) -} - -async fn read_object_record( - store: &Store, - path: &str, - name: &'static str, -) -> Result> { - let Some((bytes, version)) = store.read(path).await? else { - return Ok(None); - }; - let record = - serde_json::from_slice(&bytes).with_context(|| format!("decode {name} object {path}"))?; - Ok(Some((record, version))) -} - -#[cfg(test)] -mod tests { - use super::*; - use serde::{Deserialize, Serialize}; - - #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] - struct Rec { - revision: u64, - key: String, - value: String, - } - - #[test] - fn encoded_id_keeps_safe_chars_and_escapes_the_rest() { - assert_eq!(encoded_id("abc-XYZ_1.2"), "abc-XYZ_1.2"); - assert_eq!(encoded_id("a/b"), "a~2fb"); - assert_eq!(encoded_id("a~b"), "a~7eb"); - assert_eq!(encoded_id(""), ""); - } - - #[test] - fn unix_now_ms_matches_system_clock_and_is_non_decreasing() { - let before = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map(|duration| duration.as_millis() as u64) - .unwrap_or(0); - let first = unix_now_ms(); - let second = unix_now_ms(); - let after = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map(|duration| duration.as_millis() as u64) - .unwrap_or(0); - assert!(first >= before, "got {first} before {before}"); - assert!(second >= first, "got {second} after {first}"); - assert!(after >= second, "got {after} after {second}"); - } - - #[tokio::test] - async fn local_mutate_replace_then_get_round_trips() { - let dir = tempfile::tempdir().unwrap(); - let store = CasStore::open(dir.path().to_str().unwrap(), "records", "Test store") - .await - .unwrap(); - assert!(store.get::("run/1").await.unwrap().is_none()); - store - .mutate::("run/1", |_| { - Ok(Mutation::Replace( - Rec { - revision: 1, - key: "run/1".into(), - value: "one".into(), - }, - (), - )) - }) - .await - .unwrap(); - assert_eq!( - store.get::("run/1").await.unwrap().unwrap(), - Rec { - revision: 1, - key: "run/1".into(), - value: "one".into(), - } - ); - let path = dir.path().join("records").join("run~2f1.json"); - assert!(path.is_file()); - } - - #[tokio::test] - async fn local_unchanged_does_not_rewrite() { - let dir = tempfile::tempdir().unwrap(); - let store = CasStore::open(dir.path().to_str().unwrap(), "records", "Test store") - .await - .unwrap(); - store - .mutate::("k", |_| { - Ok(Mutation::Replace( - Rec { - revision: 1, - key: "k".into(), - value: "v".into(), - }, - (), - )) - }) - .await - .unwrap(); - let path = dir.path().join("records").join("k.json"); - let before = std::fs::read(&path).unwrap(); - store - .mutate::("k", |_| Ok(Mutation::Unchanged(()))) - .await - .unwrap(); - assert_eq!(std::fs::read(&path).unwrap(), before); - } - - #[tokio::test] - async fn local_write_uses_create_new_and_does_not_truncate_existing_tmp() { - let dir = tempfile::tempdir().unwrap(); - let store = CasStore::open(dir.path().to_str().unwrap(), "records", "Test store") - .await - .unwrap(); - store - .mutate::("k", |_| { - Ok(Mutation::Replace( - Rec { - revision: 1, - key: "k".into(), - value: "v".into(), - }, - (), - )) - }) - .await - .unwrap(); - let tmp = dir - .path() - .join("records") - .join(format!(".k.json.{}.tmp", std::process::id())); - std::fs::write(&tmp, b"stale").unwrap(); - let err = store - .mutate::("k", |_| { - Ok(Mutation::Replace( - Rec { - revision: 2, - key: "k".into(), - value: "next".into(), - }, - (), - )) - }) - .await - .unwrap_err(); - assert!( - err.to_string().contains("File exists") - || err - .downcast_ref::() - .is_some_and(|error| error.kind() == std::io::ErrorKind::AlreadyExists) - || format!("{err:#}").contains("already exists"), - "{err:#}" - ); - assert_eq!( - store.get::("k").await.unwrap().unwrap().revision, - 1, - "failed create_new must not clobber the committed record" - ); - } - - #[tokio::test] - async fn local_list_returns_json_records() { - let dir = tempfile::tempdir().unwrap(); - let store = CasStore::open(dir.path().to_str().unwrap(), "records", "Test store") - .await - .unwrap(); - for key in ["b", "a"] { - let owned = key.to_string(); - store - .mutate::(key, move |_| { - Ok(Mutation::Replace( - Rec { - revision: 1, - key: owned.clone(), - value: owned.clone(), - }, - (), - )) - }) - .await - .unwrap(); - } - let mut records = store.list::().await.unwrap(); - records.sort_by(|left, right| left.key.cmp(&right.key)); - assert_eq!(records.len(), 2); - assert_eq!(records[0].key, "a"); - assert_eq!(records[1].key, "b"); - } - - #[tokio::test] - async fn object_store_backend_uses_conditional_create_and_update() { - let root = format!( - "shared-memory://cas-store-{}-{}/root", - std::process::id(), - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_millis() - ); - let store = CasStore::open(&root, "records", "Test store") - .await - .unwrap(); - store - .mutate::("run-object", |_| { - Ok(Mutation::Replace( - Rec { - revision: 1, - key: "run-object".into(), - value: "one".into(), - }, - (), - )) - }) - .await - .unwrap(); - store - .mutate::("run-object", |current| { - let mut record = current.cloned().unwrap(); - record.revision = 2; - record.value = "two".into(); - Ok(Mutation::Replace(record, ())) - }) - .await - .unwrap(); - let record = store.get::("run-object").await.unwrap().unwrap(); - assert_eq!(record.revision, 2); - assert_eq!(record.value, "two"); - let listed = store.list::().await.unwrap(); - assert_eq!(listed.len(), 1); - } - - #[cfg(feature = "proptest")] - mod proptests { - use proptest::prelude::*; - - use super::*; - - fn decode_id(encoded: &str) -> Vec { - let bytes = encoded.as_bytes(); - let mut decoded = Vec::new(); - let mut index = 0; - while index < bytes.len() { - if bytes[index] == b'~' { - let value = u8::from_str_radix( - std::str::from_utf8(&bytes[index + 1..index + 3]).unwrap(), - 16, - ) - .unwrap(); - decoded.push(value); - index += 3; - } else { - decoded.push(bytes[index]); - index += 1; - } - } - decoded - } - - proptest! { - #[test] - fn encoded_ids_roundtrip_arbitrary_utf8(value in any::()) { - let encoded = encoded_id(&value); - let decoded = String::from_utf8(decode_id(&encoded)).unwrap(); - prop_assert_eq!(decoded, value); - } - - #[test] - fn encoded_ids_use_only_safe_filename_alphabet(value in any::()) { - let encoded = encoded_id(&value); - let bytes = encoded.as_bytes(); - let mut index = 0; - while index < bytes.len() { - if bytes[index] == b'~' { - prop_assert!(index + 2 < bytes.len()); - prop_assert!(bytes[index + 1].is_ascii_hexdigit()); - prop_assert!(bytes[index + 2].is_ascii_hexdigit()); - index += 3; - } else { - prop_assert!(bytes[index].is_ascii_alphanumeric() || matches!(bytes[index], b'-' | b'_' | b'.')); - index += 1; - } - } - } - } - } -} diff --git a/crates/persisting-pchronicle/src/store/catalog/discovery.rs b/crates/persisting-pchronicle/src/store/catalog/discovery.rs index 5416cc999..c49c0c775 100644 --- a/crates/persisting-pchronicle/src/store/catalog/discovery.rs +++ b/crates/persisting-pchronicle/src/store/catalog/discovery.rs @@ -10,12 +10,7 @@ pub(super) enum Candidate { size_bytes: Option, last_modified: Option, }, - Events { - file: String, - uri: String, - size_bytes: Option, - last_modified: Option, - }, + Compact { file: String, uri: String, @@ -52,19 +47,6 @@ impl Candidate { last_modified.clone(), None, ), - Self::Events { - file, - size_bytes, - last_modified, - .. - } => ( - file.clone(), - Some(DocumentFormat::CanonicalEvent.as_str().to_string()), - CatalogSourceKind::Store, - *size_bytes, - last_modified.clone(), - None, - ), Self::Compact { file, uri, @@ -110,9 +92,6 @@ impl Candidate { format, kind, revision, - projection_status: None, - projection_generation: None, - projection_candidates: 0, size_bytes, last_modified, status: CatalogSourceStatus::Ready, @@ -157,31 +136,7 @@ pub(super) async fn freeze_candidate( )), )) } - Candidate::Events { file, uri, .. } => { - ensure_format_hint(mount, DocumentFormat::CanonicalEvent, &file)?; - let snapshot = RawEventDataSource::pin_uri(&uri) - .await - .with_context(|| format!("pin canonical event source {uri}"))?; - let fact = snapshot.fact_snapshot(); - source_row.revision = Some(CatalogSourceRevision::Events { - fact_version: fact.fact_version, - fact_rows: fact.fact_rows, - layout_revision: fact.layout_revision, - }); - Ok(( - source_row, - Arc::new(LazySource::new( - file, - LazySourceSpec::Events { - uri, - snapshot, - projection: None, - }, - options, - temporary_files, - )), - )) - } + Candidate::Compact { file, uri, .. } => { if let Some(manifest) = crate::store::catalog::manifest::try_load_manifest(Path::new(&uri)) @@ -262,122 +217,6 @@ pub(super) async fn freeze_candidate( } } -/// Collapse a canonical events source and its derived Storyline sidecar into -/// one Catalog identity. Fresh projections serve normalized tables; stale -/// projections remain hidden and the canonical events adapter is used instead. -pub(super) fn bind_canonical_storyline_projections( - source_rows: &mut Vec, - prepared_sources: &mut Vec>, -) -> Result<()> { - struct Binding { - projection_file: String, - event_file: String, - paths: StorylineTablePaths, - fresh: bool, - last_modified: Option, - } - - let mut bindings = BTreeMap::>::new(); - for projection in prepared_sources.iter() { - let LazySourceSpec::Storyline { paths } = &projection.spec else { - continue; - }; - let Some(lineage) = paths.projection.as_ref() else { - continue; - }; - let ProjectionSourceSnapshot::CanonicalEvents { source_uri, .. } = &lineage.source else { - continue; - }; - let Some(events) = prepared_sources.iter().find(|candidate| { - matches!( - &candidate.spec, - LazySourceSpec::Events { uri, .. } if uri == source_uri - ) - }) else { - continue; - }; - let LazySourceSpec::Events { snapshot, .. } = &events.spec else { - anyhow::bail!( - "catalog source '{}' matched canonical event URI but is not an events source", - events.file - ) - }; - let last_modified = source_rows - .iter() - .find(|source| source.file == projection.file) - .and_then(|source| source.last_modified.clone()); - bindings - .entry(events.file.clone()) - .or_default() - .push(Binding { - projection_file: projection.file.clone(), - event_file: events.file.clone(), - paths: paths.clone(), - fresh: projection_lineage_is_fresh(&snapshot.fact_snapshot(), lineage), - last_modified, - }); - } - - let mut projection_files = HashSet::new(); - for candidates in bindings.values_mut() { - candidates.sort_by(|left, right| { - ( - left.fresh, - left.last_modified.as_deref().unwrap_or(""), - left.paths.generation.as_str(), - left.projection_file.as_str(), - ) - .cmp(&( - right.fresh, - right.last_modified.as_deref().unwrap_or(""), - right.paths.generation.as_str(), - right.projection_file.as_str(), - )) - }); - let binding = candidates - .last() - .context("projection binding group is empty")?; - projection_files.extend( - candidates - .iter() - .map(|candidate| candidate.projection_file.clone()), - ); - let event_index = prepared_sources - .iter() - .position(|source| source.file == binding.event_file) - .context("bound canonical event source disappeared")?; - let event = &prepared_sources[event_index]; - let LazySourceSpec::Events { uri, snapshot, .. } = &event.spec else { - anyhow::bail!("bound Catalog source is not canonical events"); - }; - prepared_sources[event_index] = Arc::new(LazySource::new( - event.file.clone(), - LazySourceSpec::Events { - uri: uri.clone(), - snapshot: snapshot.clone(), - projection: binding.fresh.then(|| binding.paths.clone()), - }, - event.options, - event.temporary_files.clone(), - )); - let event_row = source_rows - .iter_mut() - .find(|source| source.file == binding.event_file) - .context("bound canonical event source row disappeared")?; - event_row.projection_status = Some(if binding.fresh { - CatalogProjectionStatus::Fresh - } else { - CatalogProjectionStatus::Stale - }); - event_row.projection_generation = Some(binding.paths.generation.clone()); - event_row.projection_candidates = candidates.len() as u64; - } - - source_rows.retain(|source| !projection_files.contains(&source.file)); - prepared_sources.retain(|source| !projection_files.contains(&source.file)); - Ok(()) -} - fn ensure_format_hint(mount: &DatasetMount, actual: DocumentFormat, file: &str) -> Result<()> { if let Some(expected) = mount.format_hint { anyhow::ensure!( @@ -394,121 +233,20 @@ mod proptests { use super::*; - fn token_strategy() -> impl Strategy { - proptest::string::string_regex("[A-Za-z0-9._/-]{1,32}").unwrap() - } - proptest! { - #[test] - fn storyline_and_event_candidates_expose_stable_source_stubs( - file in token_strategy(), - uri in token_strategy(), - size in prop::option::of(0u64..1_000_000), - modified in prop::option::of(token_strategy()), - is_events in any::(), - ) { - let candidate = if is_events { - Candidate::Events { file: file.clone(), uri, size_bytes: size, last_modified: modified.clone() } - } else { - Candidate::Storyline { file: file.clone(), uri, size_bytes: size, last_modified: modified.clone() } - }; - let source = candidate.source_stub(); - prop_assert_eq!(source.file, file); - prop_assert_eq!(source.size_bytes, size); - prop_assert_eq!(source.last_modified, modified); - prop_assert_eq!(source.kind, CatalogSourceKind::Store); - prop_assert_eq!(source.status, CatalogSourceStatus::Ready); - prop_assert_eq!(source.format.as_deref(), Some(if is_events { DocumentFormat::CanonicalEvent.as_str() } else { DocumentFormat::StorylineLance.as_str() })); - } - #[test] fn source_format_hints_must_match_the_candidate_format( name in proptest::string::string_regex("[A-Za-z_][A-Za-z0-9_]{0,19}").unwrap(), ) { let mount = DatasetMount::new(name, "memory://catalog/source").unwrap(); - prop_assert!(ensure_format_hint(&mount, DocumentFormat::CanonicalEvent, "events.lance").is_ok()); + prop_assert!(ensure_format_hint(&mount, DocumentFormat::Atif, "data.json").is_ok()); let hinted = mount.with_format_hint(DocumentFormat::StorylineLance); prop_assert!(ensure_format_hint(&hinted, DocumentFormat::StorylineLance, "storyline").is_ok()); - prop_assert!(ensure_format_hint(&hinted, DocumentFormat::CanonicalEvent, "events").is_err()); + prop_assert!(ensure_format_hint(&hinted, DocumentFormat::Atif, "data.json").is_err()); } } } -pub(super) async fn normalize_event_storylines( - source: &RawEventDataSource, - session_ids: Option<&BTreeSet>, - kind: CatalogTableKind, - max_rows: usize, - max_bytes: usize, -) -> Result> { - let records = match session_ids { - Some(session_ids) => { - source - .read_records_for_storylines_bounded(session_ids, max_rows, max_bytes) - .await? - } - None => source.read_records_bounded(max_rows, max_bytes).await?, - }; - normalize_event_records(records, kind, max_bytes) -} - -fn normalize_event_records( - records: Vec, - kind: CatalogTableKind, - max_bytes: usize, -) -> Result> { - let mut groups = BTreeMap::>::new(); - for record in records { - let key = event_storyline_key(&record) - .context("canonical event cannot be projected without a Storyline identity")?; - groups.entry(key.to_string()).or_default().push(record); - } - - let stories = groups.into_iter().map(|(group_key, records)| { - let story = project_event_records(&records)?; - anyhow::ensure!( - story.session_id == group_key, - "projected Storyline identity changed" - ); - Ok(story) - }); - - let (schema, batch) = match kind { - CatalogTableKind::Runs => { - let mut rows = Vec::::new(); - for story in stories { - rows.push(split_storyline(&story?)?.run); - } - (story_runs_arrow_schema(), story_runs_to_batch(&rows)?) - } - CatalogTableKind::Steps => { - let mut rows = Vec::::new(); - for story in stories { - rows.extend(split_storyline(&story?)?.steps); - } - (story_steps_arrow_schema(), story_steps_to_batch(&rows)?) - } - CatalogTableKind::ToolCalls => { - let mut rows = Vec::::new(); - for story in stories { - rows.extend(split_storyline(&story?)?.tool_calls); - } - ( - story_tool_calls_arrow_schema(), - story_tool_calls_to_batch(&rows)?, - ) - } - CatalogTableKind::Events => { - anyhow::bail!("canonical events do not require Storyline normalization") - } - }; - anyhow::ensure!( - batch.get_array_memory_size() <= max_bytes, - "normalized canonical event fallback exceeds max_event_fallback_bytes {max_bytes}; build or sync a Storyline projection" - ); - Ok(Arc::new(MemTable::try_new(schema, vec![vec![batch]])?)) -} - pub(super) async fn discover_candidates( mount: &DatasetMount, options: LocalQueryManifestOptions, @@ -614,7 +352,7 @@ pub(super) async fn discover_candidate_at( let parts: Vec<_> = file.split('/').collect(); for (index, part) in parts.iter().enumerate() { // Respect opaque Dataset ancestors. Directly jumping into a Lance - // interior or a linked projection would change the source namespace. + // interior would change the source namespace. match classify_local_dir(&root, ¤t).await? { LocalDirClass::Recurse => {} _ => return discover_candidates(mount, options).await, @@ -654,14 +392,6 @@ pub(super) async fn discover_candidate_at( LocalDirClass::Skip => Vec::new(), LocalDirClass::Recurse => collect_local_virtual(&root, ¤t, budget).await?, }; - // ponytail: linked projections can point outside this subtree. Discover the - // mount until discovery has an authoritative reverse lineage index. - if candidates - .iter() - .any(|candidate| matches!(candidate, Candidate::Storyline { .. })) - { - return discover_candidates(mount, options).await; - } Ok(candidates) } @@ -808,28 +538,6 @@ fn candidate_from_local_leaf_manifest( ) } -fn local_events_bundle(mount_root: &Path, path: &Path) -> Result> { - let events = path.join("events.lance"); - let metadata = fs::metadata(events.join("_manifest.json"))?; - let mut out = vec![Candidate::Events { - file: relative_catalog_path(mount_root, &events, true)?, - uri: canonical_local_uri(&events)?, - size_bytes: Some(metadata.len()), - last_modified: modified_string(&metadata), - }]; - let storyline = path.join("storyline"); - if storyline.join("CURRENT").is_file() { - let metadata = fs::metadata(storyline.join("CURRENT"))?; - out.push(Candidate::Storyline { - file: relative_catalog_path(mount_root, &storyline, true)?, - uri: canonical_local_uri(&storyline)?, - size_bytes: Some(metadata.len()), - last_modified: modified_string(&metadata), - }); - } - Ok(out) -} - async fn classify_local_dir(mount_root: &Path, path: &Path) -> Result { if let Some(manifest) = try_load_manifest(path) { return match manifest.kind { @@ -848,20 +556,6 @@ async fn classify_local_dir(mount_root: &Path, path: &Path) -> Result { - let sidecar = object_storyline_sidecar(&store, uri, &candidate).await?; - let mut candidates = vec![candidate]; - candidates.extend(sidecar); - // Both sides are needed for canonical projection binding before - // discover_impl filters to the requested source namespace. - for _ in &candidates { - budget.observe_source()?; - } + let candidates = vec![candidate]; + budget.observe_source()?; if !candidates.iter().any(|candidate| { let source = candidate.source_stub().file; source == file || source.starts_with(&format!("{file}/")) @@ -1116,14 +804,8 @@ async fn collect_object_virtual( .await? { Some(ObjectProbe::Source(candidate)) => { - let maybe_storyline = - object_storyline_sidecar(store, root_uri, &candidate).await?; budget.observe_source()?; candidates.push(candidate); - if let Some(storyline) = maybe_storyline { - budget.observe_source()?; - candidates.push(storyline); - } } Some(ObjectProbe::Branch) => stack.push(child_relative), None if child.ends_with(".lance") => {} @@ -1135,31 +817,6 @@ async fn collect_object_virtual( Ok(candidates) } -async fn object_storyline_sidecar( - store: &OpendalStore, - root_uri: &str, - candidate: &Candidate, -) -> Result> { - let storyline_rel = match candidate { - Candidate::Events { file, .. } if file.ends_with("/events.lance") => { - format!("{}/storyline", file.trim_end_matches("/events.lance")) - } - Candidate::Events { file, .. } if file == "events.lance" => "storyline".into(), - _ => return Ok(None), - }; - match probe_object_prefix( - store, - root_uri, - &storyline_rel, - root_source_path(&storyline_rel), - ) - .await? - { - Some(ObjectProbe::Source(storyline)) => Ok(Some(storyline)), - _ => Ok(None), - } -} - async fn probe_object_prefix( store: &OpendalStore, root_uri: &str, @@ -1237,37 +894,5 @@ async fn probe_object_prefix( }))); } - let events_manifest = if relative.is_empty() { - "_manifest.json".to_string() - } else if relative.trim_end_matches('/').ends_with("events.lance") { - join("_manifest.json") - } else { - join("events.lance/_manifest.json") - }; - if let Some(entry) = store.stat_file(&events_manifest).await? { - let meta = RemoteObjectMeta::from(entry); - let events_relative = if relative.is_empty() { - if root_uri.trim_end_matches('/').ends_with("events.lance") { - String::new() - } else { - "events.lance".to_string() - } - } else if relative.trim_end_matches('/').ends_with("events.lance") { - relative.to_string() - } else { - format!("{}/events.lance", relative.trim_end_matches('/')) - }; - return Ok(Some(ObjectProbe::Source(Candidate::Events { - file: if events_relative.is_empty() { - ".".into() - } else { - events_relative.clone() - }, - uri: child_uri(root_uri, &events_relative), - size_bytes: Some(meta.size), - last_modified: Some(meta.last_modified), - }))); - } - Ok(None) } diff --git a/crates/persisting-pchronicle/src/store/catalog/identity.rs b/crates/persisting-pchronicle/src/store/catalog/identity.rs index a4f2466c9..5bb5259d5 100644 --- a/crates/persisting-pchronicle/src/store/catalog/identity.rs +++ b/crates/persisting-pchronicle/src/store/catalog/identity.rs @@ -109,11 +109,7 @@ pub enum CatalogSourceRevision { Storyline { generation: String, }, - Events { - fact_version: u64, - fact_rows: u64, - layout_revision: u64, - }, + LocalFile { fingerprint: String, }, @@ -130,9 +126,6 @@ impl CatalogSourceRevision { pub fn snapshot_ref(&self) -> String { match self { Self::Storyline { generation } => generation.clone(), - Self::Events { - layout_revision, .. - } => format!("manifest-revision:{layout_revision}"), Self::LocalFile { fingerprint } => fingerprint.clone(), Self::Object { version: Some(version), diff --git a/crates/persisting-pchronicle/src/store/catalog/location.rs b/crates/persisting-pchronicle/src/store/catalog/location.rs index d93c6a952..d54ffc0ae 100644 --- a/crates/persisting-pchronicle/src/store/catalog/location.rs +++ b/crates/persisting-pchronicle/src/store/catalog/location.rs @@ -311,12 +311,7 @@ impl DatasetLocation { if dir.join("CURRENT").is_file() { return Ok(Some("storyline")); } - if dir.join("events.lance/_manifest.json").is_file() - || (dir.file_name().is_some_and(|name| name == "events.lance") - && dir.join("_manifest.json").is_file()) - { - return Ok(Some("other")); - } + return Ok(None); } @@ -351,15 +346,7 @@ impl DatasetLocation { if store.stat_file(&join("CURRENT")).await?.is_some() { return Ok(Some("storyline")); } - if store - .stat_file(&join("events.lance/_manifest.json")) - .await? - .is_some() - || (relative.ends_with("events.lance") - && store.stat_file(&join("_manifest.json")).await?.is_some()) - { - return Ok(Some("other")); - } + Ok(None) } diff --git a/crates/persisting-pchronicle/src/store/catalog/mod.rs b/crates/persisting-pchronicle/src/store/catalog/mod.rs index 41842a3c3..9a7449a80 100644 --- a/crates/persisting-pchronicle/src/store/catalog/mod.rs +++ b/crates/persisting-pchronicle/src/store/catalog/mod.rs @@ -40,11 +40,10 @@ use source::*; pub use status::{CatalogConsistency, CatalogState, CatalogStatus}; use discovery::{ - bind_canonical_storyline_projections, discover_cached_candidates, discover_candidate_at, - discover_candidates, freeze_candidate, normalize_event_storylines, + discover_cached_candidates, discover_candidate_at, discover_candidates, freeze_candidate, }; -use std::collections::{BTreeMap, BTreeSet, HashSet}; +use std::collections::{BTreeSet, HashSet}; use std::fs; use std::path::{Component, Path, PathBuf}; use std::sync::Arc; @@ -72,23 +71,18 @@ use serde::Serialize; use tokio::io::AsyncWriteExt; use tokio::sync::OnceCell; -use crate::convert::{event_storyline_key, project_event_records, storyline_to_events}; use crate::format::DocumentFormat; -use crate::formats::events::EventsDocument; -use crate::formats::{EventRecord, StorylineDocument}; -use crate::projection::projection_lineage_is_fresh; +use crate::formats::StorylineDocument; -use super::events::datafusion::{RawEventDataSource, RawEventDataSourceOptions, RawEventSnapshot}; use super::files::matches_file_filter; use super::opendal_store::Entry as OpendalEntry; use super::{ FileTrajectoryDataSource, FileTrajectoryDataSourceOptions, FileTrajectoryQueryMetrics, - LocalQueryInputFile, LocalQueryManifest, LocalQueryManifestOptions, ProjectionSourceSnapshot, - SOURCE_FILE_COLUMN, StoryRunRow, StoryStepRow, StoryToolCallRow, StorylineDataSource, - StorylineDataSourceOptions, StorylineTableKind, StorylineTablePaths, raw_event_arrow_schema, - reconstruct_storyline, split_storyline, story_runs_arrow_schema, story_runs_from_batch, - story_runs_to_batch, story_steps_arrow_schema, story_steps_from_batch, story_steps_to_batch, - story_tool_calls_arrow_schema, story_tool_calls_from_batch, story_tool_calls_to_batch, + LocalQueryInputFile, LocalQueryManifest, LocalQueryManifestOptions, SOURCE_FILE_COLUMN, + StorylineDataSource, StorylineDataSourceOptions, StorylineTableKind, StorylineTablePaths, + reconstruct_storyline, story_runs_arrow_schema, story_runs_from_batch, + story_steps_arrow_schema, story_steps_from_batch, story_tool_calls_arrow_schema, + story_tool_calls_from_batch, }; #[derive(Clone, Debug)] @@ -118,15 +112,8 @@ impl From for RemoteObjectMeta { pub const DEFAULT_DATASET_NAME: &str = "dataset"; -#[derive(Debug, Clone)] -pub(crate) enum PhysicalOpenTarget { - Events { uri: String }, - Storyline { paths: Box }, -} pub const CATALOG_SOURCES_TABLE: &str = "sources"; pub const CATALOG_TRAJECTORIES_TABLE: &str = "trajectories"; -pub const DEFAULT_MAX_EVENT_FALLBACK_ROWS: usize = 100_000; -pub const DEFAULT_MAX_EVENT_FALLBACK_BYTES: usize = 64 * 1024 * 1024; #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize)] #[serde(rename_all = "snake_case")] @@ -152,13 +139,6 @@ pub enum CatalogSourceStatus { Error, } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] -#[serde(rename_all = "snake_case")] -pub enum CatalogProjectionStatus { - Fresh, - Stale, -} - #[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub struct DiscoveredSource { /// Stable path relative to the Dataset mount. A source at the mount root @@ -167,11 +147,7 @@ pub struct DiscoveredSource { pub format: Option, pub kind: CatalogSourceKind, pub revision: Option, - pub projection_status: Option, - pub projection_generation: Option, - /// Number of lineage-linked Storyline projections considered for this - /// canonical source. Values greater than one are diagnostic, not fatal. - pub projection_candidates: u64, + /// Physical source size when available. pub size_bytes: Option, pub last_modified: Option, pub status: CatalogSourceStatus, @@ -197,52 +173,10 @@ pub struct CatalogStorylineKey { pub file: String, /// Stable identity of one document within `file`. pub document_id: String, - /// Session partition used when the source is Canonical Event storage. + /// Session identity within the document. pub session_id: String, } -/// Epistemic origin of the records in a Catalog event view. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] -#[serde(rename_all = "snake_case")] -pub enum CatalogEventProvenance { - Canonical, - SyntheticFromStoryline, -} - -impl CatalogEventProvenance { - pub const fn is_canonical(self) -> bool { - matches!(self, Self::Canonical) - } - - pub const fn transform(self) -> Option<&'static str> { - match self { - Self::Canonical => None, - Self::SyntheticFromStoryline => Some("storyline_to_events_v1"), - } - } -} - -/// Event-shaped records together with their epistemic origin. -/// -/// The wrapper is intentionally not interchangeable with `EventsDocument`: -/// callers must account for whether records were observed canonically or -/// synthesized from a normalized Storyline. -#[derive(Debug, Clone, PartialEq)] -pub struct CatalogEventView { - pub provenance: CatalogEventProvenance, - pub document: EventsDocument, -} - -/// One source-consistent trajectory materialization for Web/API consumers. -/// -/// Non-event sources normalize the Storyline once and derive the event view -/// from that same document, avoiding a second scan and content hydration pass. -#[derive(Debug, Clone, PartialEq)] -pub struct CatalogTrajectoryBundle { - pub storyline: StorylineDocument, - pub event_view: CatalogEventView, -} - #[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub struct CatalogDataset { pub mount: DatasetMount, @@ -286,11 +220,6 @@ pub struct CatalogSnapshotOptions { pub(crate) storyline: StorylineDataSourceOptions, /// Maximum physical Sources opened concurrently while planning one scan. pub max_concurrent_sources: usize, - /// Maximum selected canonical rows that may be normalized in memory when a - /// fresh Storyline projection is unavailable. - pub max_event_fallback_rows: usize, - /// Maximum Arrow bytes retained while normalizing selected canonical rows. - pub max_event_fallback_bytes: usize, } #[derive(Debug, Clone, PartialEq, Eq, Hash)] @@ -325,8 +254,6 @@ impl Default for CatalogSnapshotOptions { files: FileTrajectoryDataSourceOptions::default(), storyline: StorylineDataSourceOptions::default(), max_concurrent_sources, - max_event_fallback_rows: DEFAULT_MAX_EVENT_FALLBACK_ROWS, - max_event_fallback_bytes: DEFAULT_MAX_EVENT_FALLBACK_BYTES, } } } @@ -341,14 +268,6 @@ pub struct DatasetCatalogSnapshot { _temporary_files: Arc, } -#[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) struct CatalogCanonicalEventSource { - pub(crate) dataset: String, - pub(crate) source_path: String, - pub(crate) source_uri: String, - pub(crate) snapshot: crate::store::EventFactSnapshot, -} - impl DatasetCatalogSnapshot { /// Build a read-only query engine over this catalog snapshot. pub async fn query_engine( @@ -465,7 +384,6 @@ impl DatasetCatalogSnapshot { } } } - bind_canonical_storyline_projections(&mut source_rows, &mut prepared_sources)?; if let Some(prefix) = scope .as_ref() .and_then(|scope| scope.source_file.as_deref()) @@ -473,8 +391,7 @@ impl DatasetCatalogSnapshot { let prefix = prefix.trim().trim_matches('/'); let matches = |file: &str| file == prefix || file.starts_with(&format!("{prefix}/")); - // Filter after canonical/projection binding, preserving the same - // source identity as full discovery even on fallback paths. + // Preserve the requested source namespace on fallback paths. source_rows.retain(|source| matches(&source.file)); prepared_sources.retain(|source| matches(source.file())); } @@ -523,28 +440,6 @@ impl DatasetCatalogSnapshot { &self.datasets } - pub(crate) fn canonical_event_sources(&self) -> Vec { - self.prepared - .iter() - .flat_map(|dataset| { - dataset - .sources - .iter() - .filter_map(|source| match &source.spec { - LazySourceSpec::Events { uri, snapshot, .. } => { - Some(CatalogCanonicalEventSource { - dataset: dataset.name.clone(), - source_path: source.file.clone(), - source_uri: uri.clone(), - snapshot: snapshot.fact_snapshot(), - }) - } - _ => None, - }) - }) - .collect() - } - pub fn dataset(&self, name: &str) -> Option<&CatalogDataset> { self.datasets .iter() @@ -618,168 +513,8 @@ impl DatasetCatalogSnapshot { crate::store::CompactJsonlStore::read_record(uri, &key.document_id).await } - /// Resolve one canonical Storyline from the latest visible events - /// manifest. This intentionally bypasses the immutable Catalog snapshot - /// for live Gateway observation while retaining the Catalog's source - /// identity and path resolution. - pub async fn load_live_storyline( - &self, - key: &CatalogStorylineKey, - ) -> Result> { - let source = self.lazy_source(key)?.resolve().await?; - let ResolvedSource::Events(events) = source.as_ref() else { - return load_storyline_from_source(source.as_ref(), key).await; - }; - let uri = self - .lazy_source(key)? - .canonical_event_uri() - .context("live Storyline source is not canonical events")?; - let latest = RawEventDataSource::open_uri(uri).await?; - let session_ids = BTreeSet::from([key.session_id.clone()]); - let records = latest - .read_records_for_storylines_bounded( - &session_ids, - events.max_fallback_rows, - events.max_fallback_bytes, - ) - .await?; - (!records.is_empty()) - .then(|| project_event_records(&records)) - .transpose() - } - - /// Resolve the normalized Storyline and event view without normalizing a - /// non-event source twice. - pub async fn load_trajectory_bundle( - &self, - key: &CatalogStorylineKey, - ) -> Result> { - let source = self.lazy_source(key)?.resolve().await?; - if let ResolvedSource::Events(events) = source.as_ref() { - let Some(records) = events.records_for_storyline(&key.session_id).await? else { - return Ok(None); - }; - let storyline = load_storyline_from_source(source.as_ref(), key) - .await? - .context("canonical events resolved without a normalized Storyline")?; - return Ok(Some(CatalogTrajectoryBundle { - storyline, - event_view: CatalogEventView { - provenance: CatalogEventProvenance::Canonical, - document: EventsDocument::new(records), - }, - })); - } - - let Some(storyline) = load_storyline_from_source(source.as_ref(), key).await? else { - return Ok(None); - }; - let document = storyline_to_events(&storyline)?; - Ok(Some(CatalogTrajectoryBundle { - storyline, - event_view: CatalogEventView { - provenance: CatalogEventProvenance::SyntheticFromStoryline, - document, - }, - })) - } - - /// Resolve a live Gateway trajectory bundle from the latest canonical - /// events manifest. Projection freshness does not gate this read path. - pub async fn load_live_trajectory_bundle( - &self, - key: &CatalogStorylineKey, - ) -> Result> { - let source = self.lazy_source(key)?.resolve().await?; - let ResolvedSource::Events(events) = source.as_ref() else { - return self.load_trajectory_bundle(key).await; - }; - let uri = self - .lazy_source(key)? - .canonical_event_uri() - .context("live trajectory source is not canonical events")?; - let latest = RawEventDataSource::open_uri(uri).await?; - let session_ids = BTreeSet::from([key.session_id.clone()]); - let records = latest - .read_records_for_storylines_bounded( - &session_ids, - events.max_fallback_rows, - events.max_fallback_bytes, - ) - .await?; - if records.is_empty() { - return Ok(None); - } - let storyline = project_event_records(&records)?; - Ok(Some(CatalogTrajectoryBundle { - storyline, - event_view: CatalogEventView { - provenance: CatalogEventProvenance::Canonical, - document: EventsDocument::new(records), - }, - })) - } - - /// Return canonical records when the source is events.lance, otherwise a - /// deterministic synthetic event view of the normalized Storyline. - pub async fn load_events(&self, key: &CatalogStorylineKey) -> Result> { - let source = self.lazy_source(key)?.resolve().await?; - if let ResolvedSource::Events(source) = source.as_ref() { - return Ok(source - .records_for_storyline(&key.session_id) - .await? - .map(|records| CatalogEventView { - provenance: CatalogEventProvenance::Canonical, - document: EventsDocument::new(records), - })); - } - let Some(storyline) = load_storyline_from_source(source.as_ref(), key).await? else { - return Ok(None); - }; - Ok(Some(CatalogEventView { - provenance: CatalogEventProvenance::SyntheticFromStoryline, - document: storyline_to_events(&storyline)?, - })) - } - - /// Read the latest canonical events for live Gateway observation. - pub async fn load_live_events( - &self, - key: &CatalogStorylineKey, - ) -> Result> { - let source = self.lazy_source(key)?.resolve().await?; - let ResolvedSource::Events(events) = source.as_ref() else { - return self.load_events(key).await; - }; - let uri = self - .lazy_source(key)? - .canonical_event_uri() - .context("live event source is not canonical events")?; - let latest = RawEventDataSource::open_uri(uri).await?; - let session_ids = BTreeSet::from([key.session_id.clone()]); - let records = latest - .read_records_for_storylines_bounded( - &session_ids, - events.max_fallback_rows, - events.max_fallback_bytes, - ) - .await?; - Ok((!records.is_empty()).then_some(CatalogEventView { - provenance: CatalogEventProvenance::Canonical, - document: EventsDocument::new(records), - })) - } - - /// Physical canonical events URI for a Storyline source. Non-canonical sources - /// return `None`; callers must not infer write locations from Dataset mount - /// roots because a mount may start at any hierarchy level. - pub fn canonical_event_uri(&self, key: &CatalogStorylineKey) -> Result> { - Ok(self.lazy_source(key)?.canonical_event_uri()) - } - /// Return the normalized Storyline table paths for a physical Storyline - /// source. File and canonical-event sources return `None` and retain their - /// existing in-memory search behavior in callers. + /// source. Other source kinds return `None`. pub fn storyline_table_paths( &self, dataset: &str, @@ -802,33 +537,6 @@ impl DatasetCatalogSnapshot { }) } - pub(crate) fn physical_open_target( - &self, - dataset: &str, - file: &str, - ) -> Result { - let dataset_name = identity::normalize_sql_alias(dataset)?; - let prepared = self - .prepared - .iter() - .find(|candidate| candidate.name == dataset_name) - .with_context(|| format!("physical source not found: {dataset}/{file}"))?; - let source = prepared - .sources - .iter() - .find(|source| source.file() == file) - .with_context(|| format!("physical source not found: {dataset}/{file}"))?; - match &source.spec { - LazySourceSpec::Events { uri, .. } => { - Ok(PhysicalOpenTarget::Events { uri: uri.clone() }) - } - LazySourceSpec::Storyline { paths } => Ok(PhysicalOpenTarget::Storyline { - paths: Box::new(paths.clone()), - }), - _ => anyhow::bail!("physical source is not a Lance dataset: {dataset}/{file}"), - } - } - fn lazy_source(&self, key: &CatalogStorylineKey) -> Result<&LazySource> { let dataset = identity::normalize_sql_alias(&key.dataset)?; let prepared = self @@ -949,14 +657,6 @@ fn validate_catalog_options(options: CatalogSnapshotOptions) -> Result<()> { options.max_concurrent_sources > 0, "catalog max_concurrent_sources must be positive" ); - anyhow::ensure!( - options.max_event_fallback_rows > 0, - "catalog max_event_fallback_rows must be positive" - ); - anyhow::ensure!( - options.max_event_fallback_bytes > 0, - "catalog max_event_fallback_bytes must be positive" - ); Ok(()) } @@ -1087,15 +787,7 @@ fn catalog_snapshot_id(datasets: &[CatalogDataset]) -> String { hasher.update(b"\0"); hasher.update(snapshot_ref.as_bytes()); } - if let Some(projection_generation) = &source.projection_generation { - hasher.update(b"\0projection:"); - hasher.update(projection_generation.as_bytes()); - hasher.update(match source.projection_status { - Some(CatalogProjectionStatus::Fresh) => b":fresh", - Some(CatalogProjectionStatus::Stale) => b":stale", - None => b":none", - }); - } + if let Some(error) = &source.error { hasher.update(b"\0"); hasher.update(error.as_bytes()); diff --git a/crates/persisting-pchronicle/src/store/catalog/provider.rs b/crates/persisting-pchronicle/src/store/catalog/provider.rs index 5838f15ef..eac0c5861 100644 --- a/crates/persisting-pchronicle/src/store/catalog/provider.rs +++ b/crates/persisting-pchronicle/src/store/catalog/provider.rs @@ -109,18 +109,16 @@ pub(super) enum CatalogTableKind { Runs, Steps, ToolCalls, - Events, } impl CatalogTableKind { - pub(super) const ALL: [Self; 4] = [Self::Runs, Self::Steps, Self::ToolCalls, Self::Events]; + pub(super) const ALL: [Self; 3] = [Self::Runs, Self::Steps, Self::ToolCalls]; pub(super) fn table_name(self) -> &'static str { match self { Self::Runs => "runs", Self::Steps => "steps", Self::ToolCalls => "tool_calls", - Self::Events => "events", } } @@ -129,7 +127,6 @@ impl CatalogTableKind { Self::Runs => story_runs_arrow_schema(), Self::Steps => story_steps_arrow_schema(), Self::ToolCalls => story_tool_calls_arrow_schema(), - Self::Events => raw_event_arrow_schema(), } } } @@ -166,21 +163,17 @@ impl CatalogTableProvider { projection: Option<&Vec>, filters: &[Expr], limit: Option, - event_session_ids: Option<&BTreeSet>, ) -> datafusion::common::Result>> { let resolved = source.resolve().await.map_err(|error| { crate::store::datafusion_bridge::into_datafusion( error.context("resolve Dataset source"), ) })?; - let Some(table) = resolved - .table(self.kind, event_session_ids) - .await - .map_err(|error| { - crate::store::datafusion_bridge::into_datafusion( - error.context("prepare Dataset source table"), - ) - })? + let Some(table) = resolved.table(self.kind).await.map_err(|error| { + crate::store::datafusion_bridge::into_datafusion( + error.context("prepare Dataset source table"), + ) + })? else { return Ok(None); }; @@ -222,16 +215,13 @@ impl TableProvider for CatalogTableProvider { limit: Option, ) -> datafusion::common::Result> { let output_schema = projected_schema(&self.schema, projection)?; - let event_session_ids = required_string_values(filters, "session_id"); - let event_session_ids = event_session_ids.as_ref(); let selected = self .sources .iter() .filter(|source| { - source.supports(self.kind) - && filters - .iter() - .all(|filter| evaluate_file_filter(filter, source.file()).unwrap_or(true)) + filters + .iter() + .all(|filter| evaluate_file_filter(filter, source.file()).unwrap_or(true)) }) .cloned() .collect::>(); @@ -239,15 +229,8 @@ impl TableProvider for CatalogTableProvider { // during late resolution still fails the query rather than disappearing. let planned = stream::iter(selected) .map(|source| async move { - self.scan_source( - source.as_ref(), - state, - projection, - filters, - limit, - event_session_ids, - ) - .await + self.scan_source(source.as_ref(), state, projection, filters, limit) + .await }) .buffered(self.max_concurrent_sources) .try_collect::>() @@ -327,63 +310,6 @@ fn evaluate_file_filter(expr: &Expr, path: &str) -> Option { } } -fn required_string_values(filters: &[Expr], column: &str) -> Option> { - filters - .iter() - .filter_map(|filter| required_string_values_expr(filter, column)) - .reduce(|left, right| left.intersection(&right).cloned().collect()) -} - -fn required_string_values_expr(expr: &Expr, column: &str) -> Option> { - match expr { - Expr::BinaryExpr(binary) if binary.op == Operator::Eq => { - let value = if is_named_column(&binary.left, column) { - string_scalar(&binary.right) - } else if is_named_column(&binary.right, column) { - string_scalar(&binary.left) - } else { - None - }?; - Some(BTreeSet::from([value.to_string()])) - } - Expr::BinaryExpr(binary) if binary.op == Operator::And => { - match ( - required_string_values_expr(&binary.left, column), - required_string_values_expr(&binary.right, column), - ) { - (Some(left), Some(right)) => Some(left.intersection(&right).cloned().collect()), - (Some(values), None) | (None, Some(values)) => Some(values), - (None, None) => None, - } - } - Expr::BinaryExpr(binary) if binary.op == Operator::Or => { - let left = required_string_values_expr(&binary.left, column)?; - let right = required_string_values_expr(&binary.right, column)?; - Some(left.union(&right).cloned().collect()) - } - Expr::InList(list) if !list.negated && is_named_column(&list.expr, column) => list - .list - .iter() - .map(string_scalar) - .map(|value| value.map(str::to_string)) - .collect(), - _ => None, - } -} - -fn is_named_column(expr: &Expr, name: &str) -> bool { - matches!(expr, Expr::Column(column) if column.name == name) -} - -fn string_scalar(expr: &Expr) -> Option<&str> { - match expr { - Expr::Literal(ScalarValue::Utf8(Some(value)), _) - | Expr::Literal(ScalarValue::LargeUtf8(Some(value)), _) - | Expr::Literal(ScalarValue::Utf8View(Some(value)), _) => Some(value), - _ => None, - } -} - fn business_filters(filters: &[Expr]) -> Vec { let mut output = Vec::new(); for filter in filters { @@ -604,26 +530,6 @@ pub(super) fn sources_table_provider( .map(Option::as_deref) .collect::>(), )), - Arc::new(StringArray::from( - sources - .iter() - .map(|source| { - source.projection_status.map(|status| match status { - CatalogProjectionStatus::Fresh => "fresh", - CatalogProjectionStatus::Stale => "stale", - }) - }) - .collect::>(), - )), - Arc::new(StringArray::from( - sources - .iter() - .map(|source| source.projection_generation.as_deref()) - .collect::>(), - )), - Arc::new(UInt64Array::from_iter_values( - sources.iter().map(|source| source.projection_candidates), - )), Arc::new(UInt64Array::from( sources .iter() @@ -659,9 +565,6 @@ fn sources_schema() -> SchemaRef { Field::new("format", DataType::Utf8, true), Field::new("kind", DataType::Utf8, false), Field::new("snapshot_ref", DataType::Utf8, true), - Field::new("projection_status", DataType::Utf8, true), - Field::new("projection_generation", DataType::Utf8, true), - Field::new("projection_candidates", DataType::UInt64, false), Field::new("size_bytes", DataType::UInt64, true), Field::new("last_modified", DataType::Utf8, true), Field::new("status", DataType::Utf8, false), diff --git a/crates/persisting-pchronicle/src/store/catalog/source.rs b/crates/persisting-pchronicle/src/store/catalog/source.rs index 230ba2a06..a08e75f4b 100644 --- a/crates/persisting-pchronicle/src/store/catalog/source.rs +++ b/crates/persisting-pchronicle/src/store/catalog/source.rs @@ -5,14 +5,6 @@ pub(super) async fn load_storyline_from_source( source: &ResolvedSource, key: &CatalogStorylineKey, ) -> Result> { - if let ResolvedSource::Events(events) = source - && events.projection.is_none() - { - let Some(records) = events.records_for_storyline(&key.session_id).await? else { - return Ok(None); - }; - return Ok(Some(project_event_records(&records)?)); - } let context = SessionContext::new(); register_normalized_source(&context, source).await?; let document_predicate = sql_string(&key.document_id); @@ -148,32 +140,6 @@ mod proptests { prop_assert_eq!(failure.source().map(ToString::to_string), Some(message)); } - #[test] - fn storyline_lazy_sources_support_normalized_tables_but_not_events( - file in proptest::string::string_regex("[A-Za-z0-9._-]{1,24}").unwrap(), - generation in proptest::string::string_regex("gen-[A-Za-z0-9_-]{1,16}").unwrap(), - ) { - let temp = Arc::new(SnapshotTempDir::new().unwrap()); - let paths = StorylineTablePaths { - generation: generation.clone(), - table_generation: generation.clone(), - runs: PathBuf::from("runs.lance"), - steps: PathBuf::from("steps.lance"), - tool_calls: PathBuf::from("tool_calls.lance"), - objects: PathBuf::from("objects.lance"), - runs_version: 1, - steps_version: 1, - tool_calls_version: 1, - objects_version: 1, - projection: None, - }; - let source = LazySource::new(file, LazySourceSpec::Storyline { paths }, CatalogSnapshotOptions::default(), temp); - prop_assert!(source.supports(CatalogTableKind::Runs)); - prop_assert!(source.supports(CatalogTableKind::Steps)); - prop_assert!(source.supports(CatalogTableKind::ToolCalls)); - prop_assert!(!source.supports(CatalogTableKind::Events)); - prop_assert!(source.canonical_event_uri().is_none()); - } } } @@ -193,11 +159,7 @@ pub(super) enum LazySourceSpec { Storyline { paths: StorylineTablePaths, }, - Events { - uri: String, - snapshot: RawEventSnapshot, - projection: Option, - }, + Compact { uri: String, }, @@ -234,31 +196,15 @@ impl LazySource { &self.file } - pub(super) fn supports(&self, kind: CatalogTableKind) -> bool { - match (&self.spec, kind) { - (LazySourceSpec::Events { .. }, _) => true, - (_, CatalogTableKind::Events) => false, - _ => true, - } - } - pub(super) fn format_hint(&self) -> Option { match &self.spec { LazySourceSpec::Storyline { .. } => Some(DocumentFormat::StorylineLance), - LazySourceSpec::Events { .. } => Some(DocumentFormat::CanonicalEvent), LazySourceSpec::Compact { .. } => None, LazySourceSpec::LocalFile { format_hint, .. } | LazySourceSpec::RemoteFile { format_hint, .. } => *format_hint, } } - pub(super) fn canonical_event_uri(&self) -> Option<&str> { - match &self.spec { - LazySourceSpec::Events { uri, .. } => Some(uri), - _ => None, - } - } - pub(super) async fn resolve(&self) -> Result> { let result = self .resolved @@ -285,34 +231,7 @@ impl LazySource { ) .await?, )), - LazySourceSpec::Events { - snapshot, - projection, - .. - } => { - let source = RawEventDataSource::from_pinned_snapshot_with_options( - snapshot.clone(), - RawEventDataSourceOptions::default(), - ) - .await?; - let projection = match projection { - Some(paths) => Some( - StorylineDataSource::from_pinned_paths_with_options( - paths.clone(), - self.options.storyline, - ) - .await?, - ), - None => None, - }; - Ok(ResolvedSource::Events(ResolvedEventSource { - source, - projection, - max_fallback_rows: self.options.max_event_fallback_rows, - max_fallback_bytes: self.options.max_event_fallback_bytes, - normalization_count: AtomicUsize::new(0), - })) - } + LazySourceSpec::Compact { .. } => Ok(ResolvedSource::Compact), LazySourceSpec::LocalFile { root, @@ -385,54 +304,10 @@ impl LazySource { #[derive(Debug)] pub(super) enum ResolvedSource { Storyline(StorylineDataSource), - Events(ResolvedEventSource), File(FileTrajectoryDataSource), Compact, } -#[derive(Debug)] -pub(super) struct ResolvedEventSource { - source: RawEventDataSource, - projection: Option, - pub(super) max_fallback_rows: usize, - pub(super) max_fallback_bytes: usize, - pub(super) normalization_count: AtomicUsize, -} - -impl ResolvedEventSource { - async fn normalized_for( - &self, - session_ids: Option<&BTreeSet>, - kind: CatalogTableKind, - ) -> Result> { - self.normalization_count.fetch_add(1, Ordering::Relaxed); - normalize_event_storylines( - &self.source, - session_ids, - kind, - self.max_fallback_rows, - self.max_fallback_bytes, - ) - .await - } - - pub(super) async fn records_for_storyline( - &self, - session_id: &str, - ) -> Result>> { - let session_ids = BTreeSet::from([session_id.to_string()]); - let records = self - .source - .read_records_for_storylines_bounded( - &session_ids, - self.max_fallback_rows, - self.max_fallback_bytes, - ) - .await?; - Ok((!records.is_empty()).then_some(records)) - } -} - pub(super) struct ResolvedTable { pub(super) provider: Arc, pub(super) carries_file_column: bool, @@ -448,10 +323,7 @@ impl ResolvedSource { format: DocumentFormat, predicates: &[crate::store::virtual_document::NormalizedJsonPredicate], ) -> Result>> { - if matches!( - format, - DocumentFormat::CanonicalEvent | DocumentFormat::Codex | DocumentFormat::ClaudeCode - ) { + if matches!(format, DocumentFormat::Codex | DocumentFormat::ClaudeCode) { return Ok(None); } let mut candidates: Option> = None; @@ -467,7 +339,7 @@ impl ResolvedSource { CatalogTableKind::ToolCalls } }; - let Some(table) = self.table(kind, None).await? else { + let Some(table) = self.table(kind).await? else { return Ok(None); }; if table.provider.schema().index_of("document_id").is_err() { @@ -537,40 +409,17 @@ impl ResolvedSource { } Ok(rows) } - Self::Events(source) if format == DocumentFormat::CanonicalEvent => { - let context = SessionContext::new(); - source.source.register(&context)?; - let batches = context - .sql("SELECT * FROM events ORDER BY seq") - .await? - .collect() - .await?; - let mut rows = Vec::new(); - for batch in &batches { - for row in super::super::event_rows_from_batch(batch)? { - rows.push(( - row.event_id.unwrap_or_else(|| row.seq.to_string()), - row.payload_json, - )); - } - } - Ok(rows) - } + Self::Compact => Ok(Vec::new()), _ => Ok(Vec::new()), } } - pub(super) async fn table( - &self, - kind: CatalogTableKind, - event_session_ids: Option<&BTreeSet>, - ) -> Result> { + pub(super) async fn table(&self, kind: CatalogTableKind) -> Result> { let storyline_kind = || match kind { CatalogTableKind::Runs => Some(StorylineTableKind::Runs), CatalogTableKind::Steps => Some(StorylineTableKind::Steps), CatalogTableKind::ToolCalls => Some(StorylineTableKind::ToolCalls), - CatalogTableKind::Events => None, }; Ok(match self { Self::Storyline(source) => storyline_kind().map(|kind| ResolvedTable { @@ -581,23 +430,7 @@ impl ResolvedSource { provider: source.provider(kind), carries_file_column: true, }), - Self::Events(source) if kind == CatalogTableKind::Events => Some(ResolvedTable { - provider: source.source.provider(), - carries_file_column: false, - }), - Self::Events(source) => { - if let Some(projection) = &source.projection { - return Ok(storyline_kind().map(|kind| ResolvedTable { - provider: projection.provider(kind), - carries_file_column: false, - })); - } - let normalized = source.normalized_for(event_session_ids, kind).await?; - Some(ResolvedTable { - provider: normalized, - carries_file_column: false, - }) - } + Self::Compact => None, }) } @@ -610,14 +443,7 @@ async fn register_normalized_source( match source { ResolvedSource::Storyline(source) => source.register(context), ResolvedSource::File(source) => source.register(context), - ResolvedSource::Events(source) => { - if let Some(projection) = &source.projection { - return projection.register(context); - } - anyhow::bail!( - "registering all normalized canonical events requires a fresh Storyline projection" - ) - } + ResolvedSource::Compact => { anyhow::bail!("compact JSONL has no normalized trajectory tables") } diff --git a/crates/persisting-pchronicle/src/store/catalog/tests.rs b/crates/persisting-pchronicle/src/store/catalog/tests.rs index 754a36f10..b6901b7ca 100644 --- a/crates/persisting-pchronicle/src/store/catalog/tests.rs +++ b/crates/persisting-pchronicle/src/store/catalog/tests.rs @@ -1,12 +1,7 @@ use super::*; -use crate::layout::{StoryCoords, story_lance_event_path}; -use crate::projection::{ - StorylineProjectionBuildOutcome, StorylineProjectionSyncMode, StorylineProjectionSyncOutcome, - build_storyline_projection, rebuild_storyline_projection, sync_storyline_projection, -}; +use crate::store::StorylineLanceStore; use crate::store::opendal_store::Store as OpendalStore; -use crate::store::{RawEventLanceStore, StorylineLanceStore}; -use crate::{EventIdentity, StorylineAgent, StorylineTurn}; +use crate::{StorylineAgent, StorylineTurn}; fn write_openai_source(path: &Path, event_id: &str) -> Result<()> { fs::write( @@ -791,50 +786,6 @@ async fn catalog_prunes_storyline_sources_before_opening_lance() -> Result<()> { Ok(()) } -#[tokio::test] -async fn trajectory_bundle_derives_events_from_one_storyline_source_resolution() -> Result<()> { - let temp = tempfile::tempdir()?; - let expected = storyline("bundle-session", "bundle-run"); - StorylineLanceStore::open(temp.path()) - .await? - .replace_storyline(&expected) - .await?; - let snapshot = DatasetCatalogSnapshot::discover( - vec![DatasetMount::default(temp.path().to_string_lossy())?], - Some(DEFAULT_DATASET_NAME.into()), - CatalogSnapshotOptions::default(), - ) - .await?; - let key = CatalogStorylineKey { - dataset: DEFAULT_DATASET_NAME.into(), - file: ".".into(), - document_id: expected.session_id.clone(), - session_id: expected.session_id.clone(), - }; - - let bundle = snapshot - .load_trajectory_bundle(&key) - .await? - .context("trajectory bundle must exist")?; - - assert_eq!(bundle.storyline, expected); - assert_eq!( - bundle.event_view.provenance, - CatalogEventProvenance::SyntheticFromStoryline - ); - assert_eq!( - bundle.event_view.document, - storyline_to_events(&bundle.storyline)? - ); - assert_eq!( - snapshot.prepared[0].sources[0] - .resolution_count - .load(Ordering::Relaxed), - 1 - ); - Ok(()) -} - #[tokio::test] async fn one_source_keeps_storylines_with_a_shared_run_id_independent() -> Result<()> { let temp = tempfile::tempdir()?; @@ -974,338 +925,6 @@ async fn catalog_joins_require_file_keys_only_within_one_dataset() -> Result<()> Ok(()) } -#[tokio::test] -async fn canonical_event_source_exposes_and_loads_each_storyline_independently() -> Result<()> { - let temp = tempfile::tempdir()?; - let storage = temp.path().join("capture"); - for session_id in ["root", "child"] { - let coords = StoryCoords::new( - storage.to_string_lossy(), - "agent", - session_id, - Some("run-1".into()), - ); - RawEventLanceStore - .append_events( - &coords, - &[EventRecord { - identity: EventIdentity::default(), - seq: 0, - source: "test".into(), - kind: "note".into(), - timestamp: None, - session_id: Some(session_id.into()), - agent_id: Some("agent".into()), - parent_uuid: None, - trace_id: None, - call_id: None, - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload: serde_json::json!({"session": session_id}), - }], - ) - .await?; - } - - let events_uri = - story_lance_event_path(&storage.to_string_lossy(), "agent", "root", Some("run-1"))?; - let projection_uri = storage.join("agent/run-1/storyline"); - let StorylineProjectionBuildOutcome::Built(_) = build_storyline_projection( - events_uri.to_string_lossy(), - projection_uri.to_string_lossy(), - "agent/run-1/events.lance", - ) - .await? - else { - panic!("initial catalog projection build unexpectedly reported nonempty output") - }; - - let mount_root = storage.join("agent"); - let snapshot = Arc::new( - DatasetCatalogSnapshot::discover( - vec![DatasetMount::default(mount_root.to_string_lossy())?], - Some(DEFAULT_DATASET_NAME.into()), - CatalogSnapshotOptions::default(), - ) - .await?, - ); - assert_eq!(snapshot.datasets()[0].sources.len(), 1); - assert_eq!(snapshot.datasets()[0].sources[0].file, "run-1/events.lance"); - assert_eq!( - snapshot.datasets()[0].sources[0].projection_status, - Some(CatalogProjectionStatus::Fresh) - ); - let updated_coords = StoryCoords::new( - storage.to_string_lossy(), - "agent", - "root", - Some("run-1".into()), - ); - RawEventLanceStore - .append_events( - &updated_coords, - &[EventRecord { - identity: EventIdentity::default(), - seq: 1, - source: "test".into(), - kind: "note".into(), - timestamp: None, - session_id: Some("root".into()), - agent_id: Some("agent".into()), - parent_uuid: None, - trace_id: None, - call_id: None, - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload: serde_json::json!({"after": "snapshot"}), - }], - ) - .await?; - let live_key = CatalogStorylineKey { - dataset: DEFAULT_DATASET_NAME.into(), - file: "run-1/events.lance".into(), - document_id: "root".into(), - session_id: "root".into(), - }; - let live_events = snapshot - .load_live_events(&live_key) - .await? - .context("live canonical events must resolve after append")?; - assert_eq!(live_events.provenance, CatalogEventProvenance::Canonical); - assert_eq!(live_events.document.events.len(), 2); - let live_storyline = snapshot - .load_live_storyline(&live_key) - .await? - .context("live canonical Storyline must resolve after append")?; - assert_eq!(live_storyline.turns.len(), 2); - let lazy = &snapshot.prepared[0].sources[0]; - assert_eq!(lazy.resolution_count.load(Ordering::Relaxed), 1); - let engine = snapshot.clone().query_engine(Default::default()).await?; - assert_eq!(lazy.resolution_count.load(Ordering::Relaxed), 1); - let event_count = engine - .query_jsonl( - "SELECT COUNT(*) AS rows FROM dataset.events \ - WHERE _file_ = 'run-1/events.lance' AND seq = 0", - ) - .await?; - assert_eq!(event_count.trim(), r#"{"rows":2}"#); - assert_eq!(lazy.resolution_count.load(Ordering::Relaxed), 1); - let resolved = lazy.resolved.get().unwrap().as_ref().unwrap(); - let ResolvedSource::Events(events) = resolved.as_ref() else { - panic!("canonical event source resolved to the wrong adapter"); - }; - assert_eq!(events.normalization_count.load(Ordering::Relaxed), 0); - let runs = engine - .query_jsonl( - "SELECT _file_, document_id, run_id, session_id FROM dataset.runs ORDER BY session_id", - ) - .await?; - assert_eq!(events.normalization_count.load(Ordering::Relaxed), 0); - let keys = runs - .lines() - .map(serde_json::from_str::) - .collect::>>()?; - assert_eq!(keys.len(), 2); - for row in keys { - let events = snapshot - .load_events(&CatalogStorylineKey { - dataset: DEFAULT_DATASET_NAME.into(), - file: row["_file_"].as_str().unwrap().into(), - document_id: row["document_id"].as_str().unwrap().into(), - session_id: row["session_id"].as_str().unwrap().into(), - }) - .await? - .context("Catalog Storyline must resolve canonical events")?; - assert_eq!(events.provenance, CatalogEventProvenance::Canonical); - assert_eq!(events.document.events.len(), 1); - } - assert_eq!(events.normalization_count.load(Ordering::Relaxed), 0); - - let stale_snapshot = Arc::new( - DatasetCatalogSnapshot::discover( - vec![DatasetMount::default(mount_root.to_string_lossy())?], - Some(DEFAULT_DATASET_NAME.into()), - CatalogSnapshotOptions::default(), - ) - .await?, - ); - assert_eq!(stale_snapshot.datasets()[0].sources.len(), 1); - assert_eq!( - stale_snapshot.datasets()[0].sources[0].projection_status, - Some(CatalogProjectionStatus::Stale) - ); - let stale_engine = stale_snapshot - .clone() - .query_engine(Default::default()) - .await?; - let stale_resolved = stale_snapshot.prepared[0].sources[0].resolve().await?; - let ResolvedSource::Events(stale_events) = stale_resolved.as_ref() else { - panic!("stale projection did not fall back to canonical events"); - }; - stale_snapshot - .load_events(&CatalogStorylineKey { - dataset: DEFAULT_DATASET_NAME.into(), - file: "run-1/events.lance".into(), - document_id: "root".into(), - session_id: "root".into(), - }) - .await? - .context("point load must resolve the selected Storyline")?; - assert_eq!(stale_events.normalization_count.load(Ordering::Relaxed), 0); - stale_engine - .query("SELECT * FROM dataset.runs WHERE session_id = 'root'") - .await?; - assert_eq!(stale_events.normalization_count.load(Ordering::Relaxed), 1); - let broad_fallback = stale_engine - .query_jsonl("SELECT session_id FROM dataset.runs ORDER BY session_id") - .await?; - assert_eq!( - broad_fallback.lines().collect::>(), - [r#"{"session_id":"child"}"#, r#"{"session_id":"root"}"#] - ); - assert_eq!(stale_events.normalization_count.load(Ordering::Relaxed), 2); - - let limited_snapshot = Arc::new( - DatasetCatalogSnapshot::discover( - vec![DatasetMount::default(mount_root.to_string_lossy())?], - Some(DEFAULT_DATASET_NAME.into()), - CatalogSnapshotOptions { - max_event_fallback_rows: 1, - ..CatalogSnapshotOptions::default() - }, - ) - .await?, - ); - let limited_engine = limited_snapshot.query_engine(Default::default()).await?; - let limit_error = limited_engine - .query("SELECT * FROM dataset.runs") - .await - .unwrap_err(); - assert!(format!("{limit_error:#}").contains("max_event_fallback_rows 1")); - - let StorylineProjectionSyncOutcome::Synced(sync) = sync_storyline_projection( - events_uri.to_string_lossy(), - projection_uri.to_string_lossy(), - ) - .await? - else { - panic!("incremental sync returned a non-success outcome") - }; - assert_eq!(sync.mode, StorylineProjectionSyncMode::Incremental); - assert_eq!(sync.affected_storylines, 1); - assert_eq!(sync.suffix_rows_scanned, 1); - assert_eq!(sync.history_rows_scanned, 2); - let StorylineProjectionSyncOutcome::Synced(noop) = sync_storyline_projection( - events_uri.to_string_lossy(), - projection_uri.to_string_lossy(), - ) - .await? - else { - panic!("noop sync returned a non-success outcome") - }; - assert_eq!(noop.mode, StorylineProjectionSyncMode::Noop); - assert_eq!(noop.generation, sync.generation); - assert_eq!(noop.suffix_rows_scanned, 0); - assert_eq!(noop.history_rows_scanned, 0); - - let projection_store = StorylineLanceStore::open(&projection_uri).await?; - let before_rebuild = projection_store - .current_table_paths() - .await? - .context("synced projection must have CURRENT")?; - let rebuild = rebuild_storyline_projection( - events_uri.to_string_lossy(), - projection_uri.to_string_lossy(), - "agent/run-1/events.lance", - ) - .await?; - assert_eq!(rebuild.mode, StorylineProjectionSyncMode::Rebuild); - let after_rebuild = projection_store - .current_table_paths() - .await? - .context("rebuilt projection must have CURRENT")?; - assert_ne!( - before_rebuild.table_generation, - after_rebuild.table_generation - ); - assert_ne!(before_rebuild.generation, after_rebuild.generation); - Ok(()) -} - -#[tokio::test] -#[ignore = "known failure: canonical events binding with multiple fresh projections"] -async fn multiple_fresh_projections_choose_one_without_hiding_canonical_events() -> Result<()> { - let temp = tempfile::tempdir()?; - let storage = temp.path().join("capture"); - let coords = StoryCoords::new( - storage.to_string_lossy(), - "agent", - "root", - Some("run-1".into()), - ); - RawEventLanceStore - .append_events( - &coords, - &[EventRecord { - identity: EventIdentity::default(), - seq: 0, - source: "test".into(), - kind: "note".into(), - timestamp: None, - session_id: Some("root".into()), - agent_id: Some("agent".into()), - parent_uuid: None, - trace_id: None, - call_id: None, - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload: serde_json::json!({"content": "kept"}), - }], - ) - .await?; - let events_uri = - story_lance_event_path(&storage.to_string_lossy(), "agent", "root", Some("run-1"))?; - for name in ["storyline-a", "storyline-b"] { - let StorylineProjectionBuildOutcome::Built(_) = build_storyline_projection( - events_uri.to_string_lossy(), - storage.join("agent/run-1").join(name).to_string_lossy(), - "agent/run-1/events.lance", - ) - .await? - else { - panic!("catalog projection candidate build unexpectedly reported nonempty output") - }; - } - - let snapshot = DatasetCatalogSnapshot::discover( - vec![DatasetMount::default( - storage.join("agent").to_string_lossy(), - )?], - Some(DEFAULT_DATASET_NAME.into()), - CatalogSnapshotOptions::default(), - ) - .await?; - assert_eq!(snapshot.datasets()[0].sources.len(), 1); - let source = &snapshot.datasets()[0].sources[0]; - assert_eq!(source.file, "run-1/events.lance"); - assert_eq!( - source.projection_status, - Some(CatalogProjectionStatus::Fresh) - ); - assert_eq!(source.projection_candidates, 2); - assert!(matches!( - source.revision, - Some(CatalogSourceRevision::Events { .. }) - )); - Ok(()) -} - #[test] fn report_mode_source_status_does_not_serialize_operational_diagnostics() -> Result<()> { let stub = DiscoveredSource { @@ -1313,9 +932,6 @@ fn report_mode_source_status_does_not_serialize_operational_diagnostics() -> Res format: None, kind: CatalogSourceKind::File, revision: None, - projection_status: None, - projection_generation: None, - projection_candidates: 0, size_bytes: None, last_modified: None, status: CatalogSourceStatus::Ready, @@ -1369,9 +985,6 @@ mod proptests { format: None, kind: CatalogSourceKind::File, revision: None, - projection_status: None, - projection_generation: None, - projection_candidates: 0, size_bytes: None, last_modified: None, status: CatalogSourceStatus::Ready, @@ -1433,37 +1046,6 @@ async fn scoped_remote_discovery_includes_json_and_new_prefix_members() -> Resul Ok(()) } -#[tokio::test] -async fn scoped_remote_candidates_preserve_event_binding_and_opaque_ancestors() -> Result<()> { - let uri = format!( - "shared-memory://scoped-bundle-{}/root", - uuid::Uuid::new_v4().simple() - ); - let store = OpendalStore::from_uri(&uri).await?; - for path in [ - "agent/run/events.lance/_manifest.json", - "agent/run/storyline/CURRENT", - "opaque/CURRENT", - "opaque/interior.json", - ] { - store.write_overwrite(path, b"{}".to_vec()).await?; - } - let mount = DatasetMount::default(uri)?; - let options = CatalogSnapshotOptions::default().manifest; - for file in ["agent/run/events.lance", "agent/run/storyline"] { - let candidates = super::discovery::discover_candidate_at(&mount, file, options).await?; - let files: Vec<_> = candidates.iter().map(|c| c.source_stub().file).collect(); - assert_eq!(files, ["agent/run/events.lance", "agent/run/storyline"]); - } - let candidates = - super::discovery::discover_candidate_at(&mount, "opaque/interior.json", options).await?; - assert!( - candidates.is_empty(), - "opaque dataset contents are not separate sources" - ); - Ok(()) -} - #[tokio::test] async fn scoped_discovery_preserves_single_file_mount_identity() -> Result<()> { let temp = tempfile::tempdir()?; diff --git a/crates/persisting-pchronicle/src/store/dataset_write_lock.rs b/crates/persisting-pchronicle/src/store/dataset_write_lock.rs deleted file mode 100644 index 56a99a623..000000000 --- a/crates/persisting-pchronicle/src/store/dataset_write_lock.rs +++ /dev/null @@ -1,59 +0,0 @@ -//! Dataset-scoped write serialization. - -use std::fs::{File, OpenOptions}; -use std::path::Path; - -use anyhow::{Context, Result}; -use fs2::FileExt; - -use super::root_write_lock; - -pub(crate) struct DatasetWriteGuard { - _process: tokio::sync::OwnedMutexGuard<()>, - local_file: Option, -} - -impl Drop for DatasetWriteGuard { - fn drop(&mut self) { - if let Some(file) = &self.local_file { - let _ = FileExt::unlock(file); - } - } -} - -/// Serialize writers within the process and, for local datasets, across -/// processes. Object-store deployments rely on conditional publication of -/// the visibility manifest for cross-process safety; Lance transactions -/// provide atomic publication but not a global distributed mutex. -pub(crate) async fn acquire(uri: &str) -> Result { - let process = root_write_lock::for_root(uri).lock_owned().await; - let local_file = if super::events::is_object_store_uri(uri) { - None - } else { - let lock_path = Path::new(uri).with_extension("lance.write.lock"); - Some( - tokio::task::spawn_blocking(move || -> Result { - if let Some(parent) = lock_path.parent() { - std::fs::create_dir_all(parent) - .with_context(|| format!("create write-lock root {}", parent.display()))?; - } - let file = OpenOptions::new() - .create(true) - .truncate(false) - .read(true) - .write(true) - .open(&lock_path) - .with_context(|| format!("open dataset write lock {}", lock_path.display()))?; - file.lock_exclusive() - .with_context(|| format!("lock dataset {}", lock_path.display()))?; - Ok(file) - }) - .await - .context("join dataset write-lock task")??, - ) - }; - Ok(DatasetWriteGuard { - _process: process, - local_file, - }) -} diff --git a/crates/persisting-pchronicle/src/store/document_source.rs b/crates/persisting-pchronicle/src/store/document_source.rs index be285ab95..4829720fa 100644 --- a/crates/persisting-pchronicle/src/store/document_source.rs +++ b/crates/persisting-pchronicle/src/store/document_source.rs @@ -1,6 +1,5 @@ //! Private provider variants behind the public `DocumentSource` API. -use std::collections::BTreeSet; use std::path::Path; use anyhow::{Context as _, Result}; @@ -9,7 +8,7 @@ use datafusion::prelude::SessionContext; use futures::TryStreamExt; use crate::agenticmd::parse_agenticmd; -use crate::convert::{actf_to_storylines, project_event_records}; +use crate::convert::actf_to_storylines; use crate::document::encode_json_storylines; use crate::document::{ DEFAULT_DOCUMENT_MATERIALIZE_BYTES, DEFAULT_DOCUMENT_MATERIALIZE_ROWS, FilterPushdown, @@ -21,16 +20,12 @@ use crate::formats::{StorylineDocument, parse_openai_msg_corpus_value}; use super::files::DEFAULT_LOCAL_QUERY_MAX_RECORD_BYTES; use super::{ - AgenticMdDataSource, AtifReader, DEFAULT_MAX_EVENT_FALLBACK_BYTES, - DEFAULT_MAX_EVENT_FALLBACK_ROWS, FileTrajectoryDataSource, LocalQueryManifest, - RawEventDataSource, StorylineDataSource, datafusion_bridge::from_datafusion, + AgenticMdDataSource, AtifReader, FileTrajectoryDataSource, LocalQueryManifest, + StorylineDataSource, datafusion_bridge::from_datafusion, }; #[derive(Debug)] pub(crate) enum DocumentSourceImpl { - Events { - source: RawEventDataSource, - }, StorylineLance { source: Box, }, @@ -52,9 +47,6 @@ pub(crate) async fn open_document_source( ) -> Result { let path = path.to_path_buf(); match format { - DocumentFormat::CanonicalEvent => Ok(DocumentSourceImpl::Events { - source: RawEventDataSource::open(&path).await?, - }), DocumentFormat::StorylineLance => { let source = StorylineDataSource::open(&path).await?; Ok(DocumentSourceImpl::StorylineLance { @@ -95,7 +87,6 @@ pub(crate) async fn open_document_source( impl DocumentSourceImpl { pub(crate) fn format(&self) -> DocumentFormat { match self { - Self::Events { .. } => DocumentFormat::CanonicalEvent, Self::StorylineLance { .. } => DocumentFormat::StorylineLance, Self::AgenticMd { .. } => DocumentFormat::AgenticMd, Self::Files { format, .. } => *format, @@ -104,15 +95,6 @@ impl DocumentSourceImpl { pub(crate) fn capabilities(&self) -> QueryCapabilities { match self.format() { - DocumentFormat::CanonicalEvent => QueryCapabilities { - projection_pushdown: true, - filter_pushdown: FilterPushdown::Exact, - limit_pushdown: true, - scalar_indexes: true, - streaming_decode: true, - late_content_materialization: false, - snapshot_consistent: true, - }, DocumentFormat::StorylineLance => QueryCapabilities { projection_pushdown: true, filter_pushdown: FilterPushdown::ExpressionDependent, @@ -216,25 +198,7 @@ impl DocumentSourceImpl { } if *source_format == format => { virtual_rows_for_files(format, manifest, source.max_file_bytes(), None) } - Self::Events { source } if format == DocumentFormat::CanonicalEvent => { - let context = SessionContext::new(); - source.register(&context)?; - let mut stream = context - .sql("SELECT * FROM events ORDER BY seq") - .await? - .execute_stream() - .await?; - let mut rows = Vec::new(); - while let Some(batch) = stream.try_next().await? { - for row in super::event_rows_from_batch(&batch)? { - rows.push(( - row.event_id.unwrap_or_else(|| row.seq.to_string()), - row.payload_json, - )); - } - } - Ok(rows) - } + Self::AgenticMd { raw, story, .. } if format == DocumentFormat::AgenticMd => { let value = serde_json::json!({ "format": "agenticmd", @@ -330,38 +294,6 @@ impl DocumentSourceImpl { } Ok(()) } - Self::Events { source, .. } => { - let context = SessionContext::new(); - source.register(&context)?; - let mut batches = context - .sql( - "SELECT DISTINCT session_id FROM events \ - WHERE session_id IS NOT NULL ORDER BY session_id", - ) - .await - .map_err(|error| from_datafusion("plan event document scan", error))? - .execute_stream() - .await - .map_err(|error| from_datafusion("start event document scan", error))?; - while let Some(batch) = batches - .try_next() - .await - .map_err(|error| from_datafusion("stream event documents", error))? - { - for session_id in strings_from_batch(&batch, "session_id")? { - let requested = BTreeSet::from([session_id]); - let records = source - .read_records_for_storylines_bounded( - &requested, - DEFAULT_MAX_EVENT_FALLBACK_ROWS, - DEFAULT_MAX_EVENT_FALLBACK_BYTES, - ) - .await?; - on_storyline(project_event_records(&records)?)?; - } - } - Ok(()) - } } } @@ -379,13 +311,6 @@ impl DocumentSourceImpl { } } - pub(crate) fn event_snapshot(&self) -> Option<&super::EventFactSnapshot> { - match self { - Self::Events { source, .. } => Some(source.fact_snapshot()), - _ => None, - } - } - pub(crate) fn storyline_generation(&self) -> Option<&str> { match self { Self::StorylineLance { source, .. } => Some(source.generation()), @@ -394,15 +319,11 @@ impl DocumentSourceImpl { } fn tables(&self) -> QueryTables { - match self { - Self::Events { .. } => QueryTables::Events, - _ => QueryTables::Storyline, - } + QueryTables::Storyline } fn register(&self, context: &SessionContext) -> Result<()> { match self { - Self::Events { source, .. } => source.register(context), Self::StorylineLance { source, .. } => source.register(context), Self::AgenticMd { source, .. } => source.register(context), Self::Files { source, .. } => source.register(context), diff --git a/crates/persisting-pchronicle/src/store/egress.rs b/crates/persisting-pchronicle/src/store/egress.rs deleted file mode 100644 index 4fc5fcf39..000000000 --- a/crates/persisting-pchronicle/src/store/egress.rs +++ /dev/null @@ -1,134 +0,0 @@ -//! Offline trajectory bundle export. - -use std::path::{Path, PathBuf}; - -use anyhow::{Context, Result}; - -use crate::layout::StoryCoords; - -/// Result of copying story files to an export directory. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct ExportOutcome { - pub out_dir: String, - pub files_copied: usize, - pub source_paths: Vec, - pub note: String, -} - -/// Directories to copy for [`export_story_bundle`]. -pub fn export_source_dirs(coords: &StoryCoords, include_subagents: bool) -> Result> { - let run = coords.run_dir()?; - if coords.root_session_id.is_none() { - return Ok(vec![run]); - } - let root = coords.root_session_id.as_ref().context("root_session_id")?; - if include_subagents && coords.session_id == *root { - return Ok(vec![run]); - } - if coords.session_id == *root { - return Ok(vec![run]); - } - let sub = run.join("subagents").join(&coords.session_id); - if sub.is_dir() { - return Ok(vec![sub]); - } - Ok(vec![run]) -} - -/// Copy story tree(s) into `{out_dir}/{agent_id}/…` preserving relative layout under storage. -pub fn export_story_bundle( - coords: &StoryCoords, - out_dir: &Path, - include_subagents: bool, -) -> Result { - let sources = export_source_dirs(coords, include_subagents)?; - if sources.is_empty() { - anyhow::bail!("no export source directories for story"); - } - - let storage = Path::new(&coords.storage); - let mut files_copied = 0usize; - let mut source_paths = Vec::new(); - - for src in &sources { - source_paths.push(src.display().to_string()); - let rel = src - .strip_prefix(storage) - .unwrap_or(src.as_path()) - .to_path_buf(); - let dst = out_dir.join(rel); - files_copied += copy_dir_recursive(src, &dst)?; - } - - let note = if include_subagents && coords.root_session_id.is_some() { - format!( - "exported {} file(s) from {} source dir(s) (include_subagents={include_subagents})", - files_copied, - sources.len() - ) - } else { - format!( - "exported {} file(s) for story {} (agent {})", - files_copied, coords.session_id, coords.agent_id - ) - }; - - Ok(ExportOutcome { - out_dir: out_dir.display().to_string(), - files_copied, - source_paths, - note, - }) -} - -fn copy_dir_recursive(src: &Path, dst: &Path) -> Result { - if !src.is_dir() { - anyhow::bail!("export source is not a directory: {}", src.display()); - } - std::fs::create_dir_all(dst).with_context(|| format!("create_dir_all {}", dst.display()))?; - let mut n = 0usize; - for entry in std::fs::read_dir(src).with_context(|| format!("read_dir {}", src.display()))? { - let entry = entry?; - let from = entry.path(); - let to = dst.join(entry.file_name()); - if from.is_dir() { - n += copy_dir_recursive(&from, &to)?; - } else { - if let Some(parent) = to.parent() { - std::fs::create_dir_all(parent)?; - } - std::fs::copy(&from, &to) - .with_context(|| format!("copy {} -> {}", from.display(), to.display()))?; - n += 1; - } - } - Ok(n) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn export_flat_session() { - let base = std::env::temp_dir().join(format!("pchr-export-{}", std::process::id())); - let _ = std::fs::remove_dir_all(&base); - let store = base.join("store"); - let session = store.join("agent-a").join("sess-1"); - std::fs::create_dir_all(&session).unwrap(); - std::fs::write(session.join("sess-1.md"), "# x\n").unwrap(); - - let coords = StoryCoords::new(store.to_str().unwrap(), "agent-a", "sess-1", None); - let out = base.join("out"); - let r = export_story_bundle(&coords, &out, false).unwrap(); - assert!(r.files_copied >= 1); - assert!( - out.join("agent-a") - .join("sess-1") - .join("sess-1.md") - .exists() - ); - - let _ = std::fs::remove_dir_all(&base); - } -} diff --git a/crates/persisting-pchronicle/src/store/event_row.rs b/crates/persisting-pchronicle/src/store/event_row.rs deleted file mode 100644 index 4b612a031..000000000 --- a/crates/persisting-pchronicle/src/store/event_row.rs +++ /dev/null @@ -1,176 +0,0 @@ -//! Lance event row ↔ [`EventRecord`]. - -use anyhow::{Context, Result}; - -use crate::formats::events::EventRecord; - -/// One row in the Lance event log (canonical trajectory store). -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct EventRow { - /// Producer-defined Storyline sequence. Replay uses physical append order. - pub seq: i64, - /// Opaque business identity. Duplicates are valid appended facts. - pub event_id: Option, - /// Canonical event time in Unix epoch milliseconds. The Arrow physical - /// column is `Timestamp(Millisecond, UTC)`; the original textual timestamp - /// remains losslessly available in `payload_json`. - pub timestamp: i64, - pub kind: String, - pub source: String, - pub agent_id: Option, - pub session_id: Option, - pub call_id: Option, - pub trace_id: Option, - pub parent_call_id: Option, - pub model: Option, - /// Full [`EventRecord`] JSON; indexed columns are denormalized for filtering. - pub payload_json: String, -} - -fn index_model(rec: &EventRecord) -> Option { - rec.payload - .get("model") - .and_then(|v| v.as_str()) - .map(str::to_string) -} - -pub fn event_record_to_event_row(rec: &EventRecord) -> Result { - Ok(EventRow { - seq: i64::try_from(rec.seq).context("EventRecord seq exceeds i64")?, - event_id: rec.identity.event_id.clone(), - timestamp: canonical_timestamp_ms(rec)?, - kind: rec.kind.clone(), - source: rec.source.clone(), - agent_id: rec.agent_id.clone(), - session_id: rec.session_id.clone(), - call_id: rec.call_id.clone(), - trace_id: rec.trace_id.clone(), - parent_call_id: rec.parent_call_id.clone(), - model: index_model(rec), - payload_json: serde_json::to_string(rec).context("encode EventRecord JSON")?, - }) -} - -pub fn event_row_to_event_record(row: &EventRow) -> Result { - let mut record: EventRecord = - serde_json::from_str(&row.payload_json).context("decode EventRecord JSON")?; - record.validate().map_err(anyhow::Error::from)?; - let seq = i64::try_from(record.seq).context("EventRecord seq exceeds i64")?; - for (field, matches) in [ - ("seq", seq == row.seq), - ("event_id", record.identity.event_id == row.event_id), - ( - "timestamp", - canonical_timestamp_ms(&record)? == row.timestamp, - ), - ("kind", record.kind == row.kind), - ("source", record.source == row.source), - ("call_id", record.call_id == row.call_id), - ("trace_id", record.trace_id == row.trace_id), - ( - "parent_call_id", - record.parent_call_id == row.parent_call_id, - ), - ("model", index_model(&record) == row.model), - ] { - anyhow::ensure!( - matches, - "{field} mismatch between physical column and payload_json" - ); - } - // The append caller's physical routing coordinates are authoritative at - // replay time. payload_json still contains the producer's original claim, - // so admission remains lossless without adding conflict columns or indexes. - record.agent_id.clone_from(&row.agent_id); - record.session_id.clone_from(&row.session_id); - Ok(record) -} - -fn canonical_timestamp_ms(record: &EventRecord) -> Result { - record - .identity - .timestamp_unix_ms - .context("canonical event is missing timestamp_unix_ms") - .and_then(|timestamp| { - i64::try_from(timestamp).context("canonical event timestamp exceeds i64 milliseconds") - }) -} - -#[cfg(all(test, feature = "proptest"))] -mod proptests { - use super::*; - use proptest::prelude::*; - use serde_json::json; - - fn identifier() -> impl Strategy { - proptest::string::string_regex("[A-Za-z0-9._-]{1,24}").unwrap() - } - - proptest! { - #[test] - fn event_rows_roundtrip_canonical_records( - seq in 0i64..i64::MAX, - timestamp in 0u64..=i64::MAX as u64, - event_id in prop::option::of(identifier()), - source in identifier(), - kind in identifier(), - agent_id in prop::option::of(identifier()), - session_id in prop::option::of(identifier()), - call_id in prop::option::of(identifier()), - trace_id in prop::option::of(identifier()), - parent_call_id in prop::option::of(identifier()), - model in prop::option::of(identifier()), - ) { - let record = EventRecord { - identity: crate::formats::events::EventIdentity { - event_id, - timestamp_unix_ms: Some(timestamp), - ..Default::default() - }, - seq: seq as u64, - source, - kind, - timestamp: None, - session_id, - agent_id, - parent_uuid: None, - trace_id, - call_id, - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id, - payload: json!({"model": model, "nested": {"ok": true}}), - }; - let row = event_record_to_event_row(&record).unwrap(); - let decoded = event_row_to_event_record(&row).unwrap(); - prop_assert_eq!(decoded, record); - } - - #[test] - fn event_rows_require_a_canonical_timestamp( - seq in any::(), - source in identifier(), - kind in identifier(), - ) { - let record = EventRecord { - identity: Default::default(), - seq, - source, - kind, - timestamp: None, - session_id: None, - agent_id: None, - parent_uuid: None, - trace_id: None, - call_id: None, - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload: json!({}), - }; - prop_assert!(event_record_to_event_row(&record).is_err()); - } - } -} diff --git a/crates/persisting-pchronicle/src/store/events/datafusion.rs b/crates/persisting-pchronicle/src/store/events/datafusion.rs deleted file mode 100644 index fea4045aa..000000000 --- a/crates/persisting-pchronicle/src/store/events/datafusion.rs +++ /dev/null @@ -1,504 +0,0 @@ -//! DataFusion datasource for the canonical Lance event log. - -use std::collections::BTreeSet; -use std::path::Path; -use std::sync::Arc; - -use anyhow::{Context, Result}; -use async_trait::async_trait; -use datafusion::catalog::Session; -use datafusion::datasource::TableProvider; -use datafusion::logical_expr::{Expr, TableProviderFilterPushDown, TableType}; -use datafusion::physical_plan::ExecutionPlan; -use datafusion::physical_plan::limit::GlobalLimitExec; -use datafusion::physical_plan::union::UnionExec; -use datafusion::prelude::SessionContext; -use futures::TryStreamExt; -use lance::Dataset; -use lance::deps::arrow_schema::{Schema as ArrowSchema, SchemaRef}; - -use super::manifest::EventManifest; -use crate::store::datafusion_bridge::{from_datafusion, into_datafusion}; -use crate::{EventRecord, event_row_to_event_record, event_rows_from_batch}; - -pub const DATAFUSION_EVENTS_TABLE: &str = "events"; - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct RawEventDataSourceOptions { - pub use_scalar_indexes: bool, - pub scan_in_order: bool, -} - -impl Default for RawEventDataSourceOptions { - fn default() -> Self { - Self { - use_scalar_indexes: true, - scan_in_order: false, - } - } -} - -#[derive(Debug)] -pub struct RawEventTableProvider { - datasets: Vec>, - schema: SchemaRef, - options: RawEventDataSourceOptions, -} - -impl RawEventTableProvider { - fn new(datasets: Vec, options: RawEventDataSourceOptions) -> Result { - let first = datasets - .first() - .context("event manifest has no visible Lance segments")?; - let schema = Arc::new(ArrowSchema::from(first.schema())); - let datasets = datasets.into_iter().map(Arc::new).collect(); - Ok(Self { - datasets, - schema, - options, - }) - } - - pub fn datasets(&self) -> &[Arc] { - &self.datasets - } -} - -#[async_trait] -impl TableProvider for RawEventTableProvider { - fn schema(&self) -> SchemaRef { - self.schema.clone() - } - - fn table_type(&self) -> TableType { - TableType::Base - } - - async fn scan( - &self, - _state: &dyn Session, - projection: Option<&Vec>, - filters: &[Expr], - limit: Option, - ) -> datafusion::common::Result> { - let mut plans = Vec::with_capacity(self.datasets.len()); - for dataset in &self.datasets { - let mut scan = dataset.scan(); - match projection { - Some(projection) if projection.is_empty() => { - scan.empty_project() - .map_err(|error| into_datafusion(error.into()))?; - } - Some(projection) => { - let columns = projection - .iter() - .map(|index| self.schema.field(*index).name()) - .collect::>(); - scan.project(&columns) - .map_err(|error| into_datafusion(error.into()))?; - } - None => {} - } - if let Some(filter) = combine_filters(filters) { - scan.filter_expr(filter); - } - scan.scan_in_order(self.options.scan_in_order); - scan.use_scalar_index(self.options.use_scalar_indexes); - plans.push( - scan.create_plan() - .await - .map_err(|error| into_datafusion(error.into()))?, - ); - } - let union = UnionExec::try_new(plans)?; - Ok(match limit { - Some(limit) => Arc::new(GlobalLimitExec::new(union, 0, Some(limit))), - None => union, - }) - } - - fn supports_filters_pushdown( - &self, - filters: &[&Expr], - ) -> datafusion::common::Result> { - Ok(filters - .iter() - .map(|_| TableProviderFilterPushDown::Exact) - .collect()) - } -} - -#[derive(Debug)] -pub struct RawEventDataSource { - snapshot: EventFactSnapshot, - provider: Arc, - segment_rows: Vec, -} - -/// Stable logical and physical coordinates for one pinned canonical event view. -#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] -pub struct EventFactSnapshot { - pub source_uri: String, - pub fact_version: u64, - pub fact_rows: u64, - pub layout_revision: u64, -} - -/// A manifest-only canonical event snapshot. It pins visible segment versions -/// without opening any Lance dataset until a selected catalog scan needs it. -#[derive(Debug, Clone)] -pub(crate) struct RawEventSnapshot { - uri: String, - manifest: EventManifest, -} - -impl RawEventSnapshot { - pub(crate) fn uri(&self) -> &str { - &self.uri - } - - pub(crate) fn fact_snapshot(&self) -> EventFactSnapshot { - EventFactSnapshot { - source_uri: self.uri.clone(), - fact_version: self.manifest.fact_version, - fact_rows: self.manifest.fact_rows, - layout_revision: self.manifest.revision, - } - } -} - -impl RawEventDataSource { - pub async fn open(path: impl AsRef) -> Result { - let path = path.as_ref(); - let uri = path - .to_str() - .ok_or_else(|| anyhow::anyhow!("event Lance path is not valid UTF-8"))?; - Self::open_uri(uri).await - } - - pub async fn open_uri(uri: impl AsRef) -> Result { - Self::open_uri_with_options(uri, RawEventDataSourceOptions::default()).await - } - - pub async fn open_uri_with_options( - uri: impl AsRef, - options: RawEventDataSourceOptions, - ) -> Result { - let snapshot = Self::pin_uri(uri).await?; - Self::from_pinned_snapshot_with_options(snapshot, options).await - } - - /// Probe a URI using only the canonical visibility manifest. - /// - /// A path whose name merely ends in `events.lance` is not canonical. The - /// probe succeeds only when a valid, non-empty manifest is present, and it - /// does not open any Lance segment. - pub async fn probe_uri(uri: impl AsRef) -> Result> { - let uri = if uri.as_ref().contains("://") { - uri.as_ref().trim_end_matches('/').to_string() - } else { - match std::fs::canonicalize(uri.as_ref()) { - Ok(path) => path.to_string_lossy().into_owned(), - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), - Err(error) => { - return Err(error).with_context(|| { - format!("canonicalize canonical event source {}", uri.as_ref()) - }); - } - } - }; - let Some(manifest) = super::pin_visible_snapshot(&uri).await? else { - return Ok(None); - }; - anyhow::ensure!( - !manifest.segments.is_empty(), - "canonical event manifest has no visible segments at {uri}" - ); - Ok(Some(EventFactSnapshot { - source_uri: uri, - fact_version: manifest.fact_version, - fact_rows: manifest.fact_rows, - layout_revision: manifest.revision, - })) - } - - pub(crate) async fn pin_uri(uri: impl AsRef) -> Result { - let uri = if uri.as_ref().contains("://") { - uri.as_ref().to_string() - } else { - std::fs::canonicalize(uri.as_ref()) - .with_context(|| format!("canonicalize canonical event source {}", uri.as_ref()))? - .to_string_lossy() - .into_owned() - }; - let manifest = super::pin_visible_snapshot(&uri) - .await? - .with_context(|| format!("canonical event manifest does not exist at {uri}"))?; - anyhow::ensure!( - !manifest.segments.is_empty(), - "canonical event manifest has no visible segments at {uri}" - ); - Ok(RawEventSnapshot { uri, manifest }) - } - - pub(crate) async fn from_pinned_snapshot_with_options( - snapshot: RawEventSnapshot, - options: RawEventDataSourceOptions, - ) -> Result { - let segment_rows = snapshot - .manifest - .segments - .iter() - .map(|segment| segment.rows) - .collect::>(); - let datasets = super::open_pinned_snapshot(snapshot.uri(), &snapshot.manifest).await?; - anyhow::ensure!( - datasets.len() == segment_rows.len(), - "canonical event manifest segment count does not match opened datasets" - ); - let fact_snapshot = snapshot.fact_snapshot(); - Ok(Self { - snapshot: fact_snapshot, - provider: Arc::new(RawEventTableProvider::new(datasets, options)?), - segment_rows, - }) - } - - pub fn fact_snapshot(&self) -> &EventFactSnapshot { - &self.snapshot - } - - pub fn provider(&self) -> Arc { - self.provider.clone() - } - - pub fn register(&self, context: &SessionContext) -> Result<()> { - self.register_as(context, DATAFUSION_EVENTS_TABLE) - } - - pub fn register_as(&self, context: &SessionContext, table_name: &str) -> Result<()> { - anyhow::ensure!( - !table_name.trim().is_empty(), - "DataFusion event table name must not be empty" - ); - context - .register_table(table_name, self.provider.clone()) - .map_err(|error| from_datafusion("register canonical event table", error))?; - Ok(()) - } - - /// Read a pinned source in manifest segment and physical append order. - pub async fn read_records_in_append_order(&self) -> Result> { - self.read_records_range_in_append_order(0, self.snapshot.fact_rows) - .await - } - - /// Read the half-open logical append range `[start, end)` from one pinned - /// manifest without scanning preceding or following event rows. - pub async fn read_records_range_in_append_order( - &self, - start: u64, - end: u64, - ) -> Result> { - anyhow::ensure!(start <= end, "event append range start exceeds end"); - anyhow::ensure!( - end <= self.snapshot.fact_rows, - "event append range end {end} exceeds pinned fact rows {}", - self.snapshot.fact_rows - ); - if start == end { - return Ok(Vec::new()); - } - - let mut records = Vec::new(); - let mut segment_start = 0u64; - for (dataset, segment_rows) in self - .provider - .datasets() - .iter() - .zip(self.segment_rows.iter().copied()) - { - let segment_end = segment_start - .checked_add(segment_rows) - .context("event segment row range overflow")?; - let overlap_start = start.max(segment_start); - let overlap_end = end.min(segment_end); - if overlap_start >= overlap_end { - segment_start = segment_end; - continue; - } - let offset = overlap_start - segment_start; - let limit = overlap_end - overlap_start; - let mut scan = dataset.scan(); - scan.scan_in_order(true); - scan.limit( - Some(i64::try_from(limit).context("event range limit exceeds i64")?), - (offset > 0) - .then(|| i64::try_from(offset).context("event range offset exceeds i64")) - .transpose()?, - ) - .context("apply pinned event append range")?; - let batches = scan - .try_into_stream() - .await - .context("scan pinned canonical events in append order")? - .try_collect::>() - .await - .context("collect pinned canonical events in append order")?; - for batch in &batches { - for row in event_rows_from_batch(batch)? { - records.push(event_row_to_event_record(&row)?); - } - } - segment_start = segment_end; - } - anyhow::ensure!( - records.len() == usize::try_from(end - start)?, - "pinned event append range returned {} rows; expected {}", - records.len(), - end - start - ); - Ok(records) - } - - /// Read complete append-ordered histories for selected Storyline identities - /// from the same pinned manifest used for suffix discovery. - pub async fn read_records_for_storylines( - &self, - session_ids: &BTreeSet, - ) -> Result> { - self.read_records_inner(Some(session_ids), None, None).await - } - - /// Read selected Storyline histories while enforcing Catalog fallback - /// budgets before retaining decoded events in memory. - pub(crate) async fn read_records_for_storylines_bounded( - &self, - session_ids: &BTreeSet, - max_rows: usize, - max_bytes: usize, - ) -> Result> { - anyhow::ensure!(max_rows > 0, "event fallback max_rows must be positive"); - anyhow::ensure!(max_bytes > 0, "event fallback max_bytes must be positive"); - self.read_records_inner(Some(session_ids), Some(max_rows), Some(max_bytes)) - .await - } - - /// Read the complete pinned event snapshot with hard Catalog fallback - /// limits. - pub(crate) async fn read_records_bounded( - &self, - max_rows: usize, - max_bytes: usize, - ) -> Result> { - anyhow::ensure!(max_rows > 0, "event fallback max_rows must be positive"); - anyhow::ensure!(max_bytes > 0, "event fallback max_bytes must be positive"); - self.read_records_inner(None, Some(max_rows), Some(max_bytes)) - .await - } - - async fn read_records_inner( - &self, - session_ids: Option<&BTreeSet>, - max_rows: Option, - max_bytes: Option, - ) -> Result> { - if session_ids.is_some_and(BTreeSet::is_empty) { - return Ok(Vec::new()); - } - let predicate = session_ids.map(|session_ids| { - format!( - "session_id IN ({})", - session_ids - .iter() - .map(|id| format!("'{}'", id.replace('\'', "''"))) - .collect::>() - .join(",") - ) - }); - let mut records = Vec::new(); - let mut retained_bytes = 0usize; - for dataset in self.provider.datasets() { - let mut scan = dataset.scan(); - if let Some(predicate) = &predicate { - scan.filter(predicate) - .context("filter pinned events by Storyline identity")?; - } - scan.scan_in_order(true); - scan.use_scalar_index(true); - if let Some(max_bytes) = max_bytes { - scan.batch_size_bytes(u64::try_from(max_bytes.min(8 * 1024 * 1024))?); - } - let mut batches = scan - .try_into_stream() - .await - .context("scan pinned Storyline event histories")?; - while let Some(batch) = batches - .try_next() - .await - .context("read pinned Storyline event history batch")? - { - retained_bytes = retained_bytes - .checked_add(batch.get_array_memory_size()) - .context("event fallback retained byte count overflow")?; - if let Some(max_bytes) = max_bytes { - anyhow::ensure!( - retained_bytes <= max_bytes, - "canonical event fallback exceeds max_event_fallback_bytes {max_bytes}; build or sync a Storyline projection" - ); - } - let rows = event_rows_from_batch(&batch)?; - if let Some(max_rows) = max_rows { - anyhow::ensure!( - records.len().saturating_add(rows.len()) <= max_rows, - "canonical event fallback exceeds max_event_fallback_rows {max_rows}; build or sync a Storyline projection" - ); - } - for row in rows { - records.push(event_row_to_event_record(&row)?); - } - } - } - if let Some(session_ids) = session_ids { - anyhow::ensure!( - records.iter().all(|record| record - .session_id - .as_ref() - .is_some_and(|id| session_ids.contains(id))), - "Storyline event history scan returned an unrequested identity" - ); - } - Ok(records) - } -} - -fn combine_filters(filters: &[Expr]) -> Option { - let mut filters = filters.iter().cloned(); - let first = filters.next()?; - Some(filters.fold(first, Expr::and)) -} - -#[cfg(all(test, feature = "proptest"))] -mod proptests { - use super::*; - use datafusion::prelude::{col, lit}; - use proptest::prelude::*; - - proptest! { - #[test] - fn combining_filters_is_none_only_for_an_empty_slice(filter_count in 0usize..16) { - let filters = (0..filter_count) - .map(|index| col("session_id").eq(lit(index as i64))) - .collect::>(); - let combined = combine_filters(&filters); - prop_assert_eq!(combined.is_some(), filter_count > 0); - if let Some(expression) = combined { - let rendered = expression.to_string(); - for index in 0..filter_count { - prop_assert!(rendered.contains(&index.to_string()), "missing filter {index}: {rendered}"); - } - prop_assert_eq!(rendered.matches("AND").count(), filter_count.saturating_sub(1)); - } - } - } -} diff --git a/crates/persisting-pchronicle/src/store/events/manifest.rs b/crates/persisting-pchronicle/src/store/events/manifest.rs deleted file mode 100644 index 79b24b6e4..000000000 --- a/crates/persisting-pchronicle/src/store/events/manifest.rs +++ /dev/null @@ -1,951 +0,0 @@ -//! Epoch-fenced visibility manifest for the canonical event log. -//! -//! Writers append to a private Lance dataset and publish its immutable version -//! through this manifest. A newer epoch changes the active fence before it -//! writes data, so an older writer can create garbage but cannot make it -//! visible to readers. - -use std::fs::{File, OpenOptions}; -use std::io::{Read, Write}; -use std::path::{Path, PathBuf}; -use std::time::{Duration, SystemTime}; - -use anyhow::{Context, Result}; -use fs2::FileExt; -use serde::{Deserialize, Serialize}; - -use super::super::opendal_store::{self, Store, Version}; - -const MANIFEST_FILE: &str = "_manifest.json"; -const MANIFEST_LOCK_FILE: &str = "_manifest.lock"; -const CAS_RETRIES: usize = 64; - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct EventWriterFence { - pub epoch: u64, - pub writer_id: String, -} - -impl EventWriterFence { - pub fn new(epoch: u64, writer_id: impl Into) -> Result { - let writer_id = writer_id.into(); - anyhow::ensure!(epoch > 0, "event writer fence epoch must be non-zero"); - anyhow::ensure!( - !writer_id.trim().is_empty(), - "event writer fence writer_id must not be empty" - ); - Ok(Self { epoch, writer_id }) - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(super) enum EventWriterConflict { - StaleFence, - EpochAlreadyOwned, - PublicationChanged, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub(super) enum ManifestWriteOutcome { - Applied(T), - Conflict(EventWriterConflict), -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub(super) struct EventSegment { - pub id: String, - pub version: u64, - pub rows: u64, - pub level: u8, - pub sealed: bool, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub(super) struct EventManifest { - /// Physical manifest revision. Writer fencing, layout maintenance, and - /// fact publication may all advance this value. - pub revision: u64, - /// Logical fact revision. Only publishing newly visible rows advances it. - pub fact_version: u64, - /// Number of canonical rows visible at `fact_version`. - pub fact_rows: u64, - pub active_writer: EventWriterFence, - #[serde(default)] - pub segments: Vec, -} - -impl EventManifest { - pub fn total_rows(&self) -> u64 { - self.segments.iter().map(|segment| segment.rows).sum() - } -} - -pub(super) async fn read(root_uri: &str) -> Result> { - if is_object_store_uri(root_uri) { - let store = Store::from_uri(root_uri).await?; - Ok(read_object_manifest(&store, MANIFEST_FILE) - .await? - .map(|(manifest, _)| manifest)) - } else { - read_local_manifest(&Path::new(root_uri).join(MANIFEST_FILE)) - } -} - -pub(super) async fn activate( - root_uri: &str, - requested: Option<&EventWriterFence>, - auto_writer_id: &str, -) -> Result> { - let requested = requested.cloned(); - let auto_writer_id = auto_writer_id.to_string(); - mutate(root_uri, move |current| { - let next_fence = match (&requested, current) { - (Some(fence), Some(manifest)) if fence == &manifest.active_writer => { - return Ok(ManifestMutation::Unchanged(manifest.clone())); - } - (Some(fence), Some(manifest)) if fence.epoch < manifest.active_writer.epoch => { - return Ok(ManifestMutation::Conflict(EventWriterConflict::StaleFence)); - } - (Some(fence), Some(manifest)) if fence.epoch == manifest.active_writer.epoch => { - return Ok(ManifestMutation::Conflict( - EventWriterConflict::EpochAlreadyOwned, - )); - } - (Some(fence), _) => fence.clone(), - (None, Some(manifest)) => EventWriterFence { - epoch: manifest - .active_writer - .epoch - .checked_add(1) - .context("event writer epoch overflow")?, - writer_id: auto_writer_id.clone(), - }, - (None, None) => EventWriterFence { - epoch: 1, - writer_id: auto_writer_id.clone(), - }, - }; - let mut next = current.cloned().unwrap_or(EventManifest { - revision: 0, - fact_version: 0, - fact_rows: 0, - active_writer: next_fence.clone(), - segments: Vec::new(), - }); - next.revision = next - .revision - .checked_add(1) - .context("event manifest revision overflow")?; - next.active_writer = next_fence; - Ok(ManifestMutation::Replace(next.clone(), next)) - }) - .await -} - -pub(super) async fn publish_segment( - root_uri: &str, - fence: &EventWriterFence, - segment: EventSegment, -) -> Result> { - let fence = fence.clone(); - mutate(root_uri, move |current| { - let current = current.context("event manifest disappeared during publish")?; - if let Some(conflict) = active_writer_conflict(current, &fence) { - return Ok(ManifestMutation::Conflict(conflict)); - } - if let Some(existing) = current - .segments - .iter() - .find(|existing| existing.id == segment.id) - { - if existing == &segment { - return Ok(ManifestMutation::Unchanged(current.clone())); - } - anyhow::ensure!( - segment.version >= existing.version && segment.rows >= existing.rows, - "event segment publication cannot move version or row count backwards" - ); - anyhow::ensure!( - segment.level == existing.level, - "event segment publication cannot change compaction level" - ); - anyhow::ensure!( - !existing.sealed || segment.sealed, - "event segment publication cannot reopen a sealed segment" - ); - anyhow::ensure!( - segment.version > existing.version - || (segment.version == existing.version - && segment.rows == existing.rows - && !existing.sealed - && segment.sealed), - "event segment publication must advance version or seal the segment" - ); - } - let mut next = current.clone(); - next.revision = next - .revision - .checked_add(1) - .context("event manifest revision overflow")?; - match next - .segments - .iter_mut() - .find(|existing| existing.id == segment.id) - { - Some(existing) => *existing = segment.clone(), - None => next.segments.push(segment.clone()), - } - let visible_rows = next.total_rows(); - if visible_rows > current.fact_rows { - next.fact_version = current - .fact_version - .checked_add(1) - .context("event fact version overflow")?; - next.fact_rows = visible_rows; - } - Ok(ManifestMutation::Replace(next.clone(), next)) - }) - .await -} - -pub(super) async fn replace_segments( - root_uri: &str, - fence: &EventWriterFence, - segments: Vec, -) -> Result> { - let fence = fence.clone(); - mutate(root_uri, move |current| { - let current = current.context("event manifest disappeared during maintenance")?; - if let Some(conflict) = active_writer_conflict(current, &fence) { - return Ok(ManifestMutation::Conflict(conflict)); - } - let replacement_rows = segments.iter().try_fold(0_u64, |total, segment| { - total - .checked_add(segment.rows) - .context("event segment replacement row count overflow") - })?; - anyhow::ensure!( - replacement_rows == current.fact_rows, - "event maintenance must preserve visible fact rows" - ); - let mut next = current.clone(); - next.revision = next - .revision - .checked_add(1) - .context("event manifest revision overflow")?; - next.segments.clone_from(&segments); - Ok(ManifestMutation::Replace(next.clone(), next)) - }) - .await -} - -/// Atomically replace one contiguous immutable segment group while preserving -/// its position in append order. Exact descriptor matching prevents a stale -/// compactor from overwriting a segment version published by another task. -pub(super) async fn replace_segment_group( - root_uri: &str, - fence: &EventWriterFence, - expected: &[EventSegment], - replacement: EventSegment, -) -> Result> { - anyhow::ensure!(!expected.is_empty(), "segment replacement group is empty"); - let expected = expected.to_vec(); - let fence = fence.clone(); - mutate(root_uri, move |current| { - let current = current.context("event manifest disappeared during segment merge")?; - if let Some(conflict) = active_writer_conflict(current, &fence) { - return Ok(ManifestMutation::Conflict(conflict)); - } - let Some(start) = current - .segments - .windows(expected.len()) - .position(|window| window == expected.as_slice()) - else { - return Ok(ManifestMutation::Conflict( - EventWriterConflict::PublicationChanged, - )); - }; - let expected_rows = expected.iter().try_fold(0_u64, |total, segment| { - total - .checked_add(segment.rows) - .context("event segment merge row count overflow") - })?; - anyhow::ensure!( - replacement.rows == expected_rows, - "event segment merge must preserve row count" - ); - - let mut next = current.clone(); - next.revision = next - .revision - .checked_add(1) - .context("event manifest revision overflow")?; - next.segments.splice( - start..start + expected.len(), - std::iter::once(replacement.clone()), - ); - Ok(ManifestMutation::Replace(next.clone(), next)) - }) - .await -} - -pub(super) fn segment_uri(root_uri: &str, segment_id: &str) -> String { - format!( - "{}/segments/{segment_id}.lance", - root_uri.trim_end_matches('/') - ) -} - -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] -pub(super) struct SegmentCleanupStats { - pub segments_removed: u64, - pub bytes_removed: u64, -} - -pub(super) async fn cleanup_unreferenced_segments( - root_uri: &str, - visible_segments: &[EventSegment], - retention: Duration, -) -> Result { - let visible = visible_segments - .iter() - .map(|segment| format!("{}.lance", segment.id)) - .collect::>(); - if !is_object_store_uri(root_uri) { - let root = PathBuf::from(root_uri).join("segments"); - return tokio::task::spawn_blocking(move || { - cleanup_local_segments(&root, &visible, retention) - }) - .await?; - } - - let store = Store::from_uri(root_uri).await?; - let fallback_lock = store.fallback_lock(); - let _fallback_guard = match fallback_lock.as_ref() { - Some(lock) => Some(lock.lock().await), - None => None, - }; - let segments_root = "segments"; - let prefix = "segments/"; - let cutoff = SystemTime::now() - .checked_sub(retention) - .unwrap_or(SystemTime::UNIX_EPOCH); - let objects = store.list(segments_root).await?; - let mut candidates: std::collections::BTreeMap = - std::collections::BTreeMap::new(); - for object in objects { - let Some(relative) = object.path.strip_prefix(prefix) else { - continue; - }; - let Some(directory) = relative.split('/').next() else { - continue; - }; - if directory.is_empty() || visible.contains(directory) { - continue; - } - let entry = candidates.entry(directory.to_string()).or_insert((true, 0)); - let modified = object - .metadata - .last_modified() - .map(SystemTime::from) - .unwrap_or(SystemTime::UNIX_EPOCH); - entry.0 &= modified <= cutoff; - entry.1 = entry.1.saturating_add(object.metadata.content_length()); - } - let mut stats = SegmentCleanupStats::default(); - for (directory, (expired, bytes)) in candidates { - if !expired { - continue; - } - store - .remove(&format!("segments/{directory}")) - .await - .context("remove unreferenced event segment")?; - stats.segments_removed += 1; - stats.bytes_removed = stats.bytes_removed.saturating_add(bytes); - } - Ok(stats) -} - -fn cleanup_local_segments( - root: &Path, - visible: &std::collections::BTreeSet, - retention: Duration, -) -> Result { - let entries = match std::fs::read_dir(root) { - Ok(entries) => entries, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => { - return Ok(SegmentCleanupStats::default()); - } - Err(error) => return Err(error.into()), - }; - let cutoff = SystemTime::now() - .checked_sub(retention) - .unwrap_or(SystemTime::UNIX_EPOCH); - let mut stats = SegmentCleanupStats::default(); - for entry in entries { - let entry = entry?; - let name = entry.file_name().to_string_lossy().into_owned(); - if visible.contains(&name) || !name.ends_with(".lance") { - continue; - } - let metadata = entry.metadata()?; - if metadata.modified().unwrap_or(SystemTime::UNIX_EPOCH) > cutoff { - continue; - } - let bytes = directory_bytes(&entry.path())?; - std::fs::remove_dir_all(entry.path())?; - stats.segments_removed += 1; - stats.bytes_removed = stats.bytes_removed.saturating_add(bytes); - } - Ok(stats) -} - -fn directory_bytes(path: &Path) -> Result { - let mut bytes = 0u64; - for entry in std::fs::read_dir(path)? { - let entry = entry?; - let metadata = entry.metadata()?; - if metadata.is_dir() { - bytes = bytes.saturating_add(directory_bytes(&entry.path())?); - } else { - bytes = bytes.saturating_add(metadata.len()); - } - } - Ok(bytes) -} - -fn active_writer_conflict( - manifest: &EventManifest, - fence: &EventWriterFence, -) -> Option { - (&manifest.active_writer != fence).then_some(EventWriterConflict::StaleFence) -} - -enum ManifestMutation { - Unchanged(T), - Replace(EventManifest, T), - Conflict(EventWriterConflict), -} - -async fn mutate(root_uri: &str, mutation: F) -> Result> -where - T: Send + 'static, - F: Fn(Option<&EventManifest>) -> Result> + Send + Sync + 'static, -{ - if !is_object_store_uri(root_uri) { - let root = PathBuf::from(root_uri); - return tokio::task::spawn_blocking(move || { - std::fs::create_dir_all(&root)?; - let lock_path = root.join(MANIFEST_LOCK_FILE); - let lock = OpenOptions::new() - .create(true) - .truncate(false) - .read(true) - .write(true) - .open(&lock_path) - .with_context(|| format!("open event manifest lock {}", lock_path.display()))?; - lock.lock_exclusive() - .with_context(|| format!("lock event manifest {}", lock_path.display()))?; - let manifest_path = root.join(MANIFEST_FILE); - let current = read_local_manifest(&manifest_path)?; - let result = match mutation(current.as_ref())? { - ManifestMutation::Unchanged(value) => ManifestWriteOutcome::Applied(value), - ManifestMutation::Replace(manifest, value) => { - validate_manifest(&manifest)?; - write_local_manifest(&manifest_path, &manifest)?; - ManifestWriteOutcome::Applied(value) - } - ManifestMutation::Conflict(conflict) => ManifestWriteOutcome::Conflict(conflict), - }; - FileExt::unlock(&lock)?; - Ok(result) - }) - .await?; - } - - let store = Store::from_uri(root_uri).await?; - let fallback_lock = store.fallback_lock(); - let _fallback_guard = match fallback_lock.as_ref() { - Some(lock) => Some(lock.lock().await), - None => None, - }; - let path = MANIFEST_FILE; - for _ in 0..CAS_RETRIES { - let current = read_object_manifest(&store, path).await?; - let outcome = mutation(current.as_ref().map(|(manifest, _)| manifest))?; - let (manifest, value) = match outcome { - ManifestMutation::Unchanged(value) => { - return Ok(ManifestWriteOutcome::Applied(value)); - } - ManifestMutation::Replace(manifest, value) => (manifest, value), - ManifestMutation::Conflict(conflict) => { - return Ok(ManifestWriteOutcome::Conflict(conflict)); - } - }; - validate_manifest(&manifest)?; - let bytes = serde_json::to_vec_pretty(&manifest)?; - let result = match current.as_ref().map(|(_, version)| version) { - None => store.write_create(path, bytes).await, - Some(version) => store.write_match(path, bytes, version).await, - }; - match result { - Ok(_) => return Ok(ManifestWriteOutcome::Applied(value)), - Err(error) - if error - .downcast_ref::() - .is_some_and(opendal_store::is_conflict) => - { - continue; - } - Err(error) => return Err(error), - } - } - Ok(ManifestWriteOutcome::Conflict( - EventWriterConflict::PublicationChanged, - )) -} - -fn validate_manifest(manifest: &EventManifest) -> Result<()> { - EventWriterFence::new( - manifest.active_writer.epoch, - manifest.active_writer.writer_id.clone(), - )?; - anyhow::ensure!( - manifest.fact_rows == manifest.total_rows(), - "event manifest fact_rows {} does not match visible segment rows {}", - manifest.fact_rows, - manifest.total_rows() - ); - let mut ids = std::collections::BTreeSet::new(); - for segment in &manifest.segments { - anyhow::ensure!(!segment.id.is_empty(), "event segment id must not be empty"); - anyhow::ensure!( - segment.version > 0, - "event segment version must be non-zero" - ); - anyhow::ensure!(ids.insert(&segment.id), "duplicate event segment id"); - } - Ok(()) -} - -fn decode_manifest(bytes: &[u8], context: impl std::fmt::Display) -> Result { - let manifest: EventManifest = serde_json::from_slice(bytes) - .with_context(|| format!("decode event manifest {context}"))?; - validate_manifest(&manifest)?; - Ok(manifest) -} - -fn read_local_manifest(path: &Path) -> Result> { - let mut file = match File::open(path) { - Ok(file) => file, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), - Err(error) => return Err(error.into()), - }; - let mut bytes = Vec::new(); - file.read_to_end(&mut bytes)?; - let manifest = decode_manifest(&bytes, path.display())?; - Ok(Some(manifest)) -} - -fn write_local_manifest(path: &Path, manifest: &EventManifest) -> Result<()> { - let parent = path.parent().context("event manifest path has no parent")?; - std::fs::create_dir_all(parent)?; - let temporary = parent.join(format!(".{MANIFEST_FILE}.{}.tmp", uuid::Uuid::new_v4())); - let mut file = OpenOptions::new() - .create_new(true) - .write(true) - .open(&temporary)?; - file.write_all(&serde_json::to_vec_pretty(manifest)?)?; - file.sync_all()?; - std::fs::rename(&temporary, path)?; - File::open(parent)?.sync_all()?; - Ok(()) -} - -async fn read_object_manifest( - store: &Store, - path: &str, -) -> Result> { - let Some((bytes, version)) = store.read(path).await? else { - return Ok(None); - }; - let manifest = decode_manifest(&bytes, format_args!("object {path}"))?; - Ok(Some((manifest, version))) -} - -fn is_object_store_uri(uri: &str) -> bool { - uri.contains("://") -} - -#[cfg(test)] -mod tests { - use super::*; - - fn applied(outcome: ManifestWriteOutcome) -> T { - let ManifestWriteOutcome::Applied(value) = outcome else { - panic!("manifest write unexpectedly conflicted") - }; - value - } - - #[tokio::test] - async fn stale_fence_is_a_conflict_while_object_store_failure_is_an_error() { - let directory = tempfile::tempdir().unwrap(); - let root = directory.path().join("events.lance"); - let uri = root.to_str().unwrap(); - let old = EventWriterFence::new(1, "old").unwrap(); - let new = EventWriterFence::new(2, "new").unwrap(); - activate(uri, Some(&old), "unused").await.unwrap(); - publish_segment( - uri, - &old, - EventSegment { - id: "old-segment".into(), - version: 1, - rows: 10, - level: 0, - sealed: false, - }, - ) - .await - .unwrap(); - activate(uri, Some(&new), "unused").await.unwrap(); - let outcome = publish_segment( - uri, - &old, - EventSegment { - id: "old-segment".into(), - version: 2, - rows: 20, - level: 0, - sealed: false, - }, - ) - .await - .unwrap(); - assert_eq!( - outcome, - ManifestWriteOutcome::Conflict(EventWriterConflict::StaleFence) - ); - let manifest = read(uri).await.unwrap().unwrap(); - assert_eq!(manifest.active_writer, new); - assert_eq!(manifest.total_rows(), 10); - - assert!( - activate("unsupported-object-store://manifest", None, "writer") - .await - .is_err() - ); - } - - #[tokio::test] - async fn fact_version_changes_only_when_visible_rows_advance() { - let directory = tempfile::tempdir().unwrap(); - let root = directory.path().join("events.lance"); - let uri = root.to_str().unwrap(); - let first = EventWriterFence::new(1, "first").unwrap(); - let activated = applied(activate(uri, Some(&first), "unused").await.unwrap()); - assert_eq!((activated.fact_version, activated.fact_rows), (0, 0)); - - let published = applied( - publish_segment( - uri, - &first, - EventSegment { - id: "segment".into(), - version: 1, - rows: 3, - level: 0, - sealed: false, - }, - ) - .await - .unwrap(), - ); - assert_eq!((published.fact_version, published.fact_rows), (1, 3)); - - let second = EventWriterFence::new(2, "second").unwrap(); - let reactivated = applied(activate(uri, Some(&second), "unused").await.unwrap()); - assert!(reactivated.revision > published.revision); - assert_eq!(reactivated.fact_version, published.fact_version); - assert_eq!(reactivated.fact_rows, published.fact_rows); - - let maintained = applied( - replace_segments(uri, &second, reactivated.segments.clone()) - .await - .unwrap(), - ); - assert!(maintained.revision > reactivated.revision); - assert_eq!(maintained.fact_version, published.fact_version); - assert_eq!(maintained.fact_rows, published.fact_rows); - } - - #[tokio::test] - async fn same_epoch_cannot_be_claimed_by_another_writer() { - let uri = format!( - "shared-memory://event-manifest-fence-{}/events.lance", - uuid::Uuid::new_v4() - ); - let first = EventWriterFence::new(7, "first").unwrap(); - let second = EventWriterFence::new(7, "second").unwrap(); - activate(&uri, Some(&first), "unused").await.unwrap(); - assert_eq!( - activate(&uri, Some(&second), "unused").await.unwrap(), - ManifestWriteOutcome::Conflict(EventWriterConflict::EpochAlreadyOwned) - ); - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 4)] - async fn object_store_cas_serializes_concurrent_epoch_claims() { - const WRITERS: usize = 32; - let uri = format!( - "shared-memory://event-manifest-contention-{}/events.lance", - uuid::Uuid::new_v4() - ); - let mut claims = Vec::with_capacity(WRITERS); - for writer in 0..WRITERS { - let uri = uri.clone(); - claims.push(tokio::spawn(async move { - activate(&uri, None, &format!("writer-{writer}")) - .await - .map(|outcome| applied(outcome).active_writer.epoch) - })); - } - let mut epochs = Vec::with_capacity(WRITERS); - for claim in claims { - epochs.push(claim.await.unwrap().unwrap()); - } - epochs.sort_unstable(); - assert_eq!(epochs, (1..=WRITERS as u64).collect::>()); - let manifest = read(&uri).await.unwrap().unwrap(); - assert_eq!(manifest.active_writer.epoch, WRITERS as u64); - assert_eq!(manifest.revision, WRITERS as u64); - } - - #[tokio::test] - async fn manifest_metadata_is_constant_in_micro_batch_count_and_publish_is_idempotent() { - const PUBLISHES: u64 = 256; - let directory = tempfile::tempdir().unwrap(); - let root = directory.path().join("events.lance"); - let uri = root.to_str().unwrap(); - let fence = EventWriterFence::new(1, "writer").unwrap(); - activate(uri, Some(&fence), "unused").await.unwrap(); - publish_segment( - uri, - &fence, - EventSegment { - id: "one-writer-segment".into(), - version: 1, - rows: 1, - level: 0, - sealed: false, - }, - ) - .await - .unwrap(); - let initial_bytes = std::fs::metadata(root.join(MANIFEST_FILE)).unwrap().len(); - - for version in 2..=PUBLISHES { - publish_segment( - uri, - &fence, - EventSegment { - id: "one-writer-segment".into(), - version, - rows: version, - level: 0, - sealed: false, - }, - ) - .await - .unwrap(); - } - let before_retry = read(uri).await.unwrap().unwrap(); - let idempotent = applied( - publish_segment( - uri, - &fence, - EventSegment { - id: "one-writer-segment".into(), - version: PUBLISHES, - rows: PUBLISHES, - level: 0, - sealed: false, - }, - ) - .await - .unwrap(), - ); - assert_eq!(idempotent.revision, before_retry.revision); - assert_eq!(idempotent.segments.len(), 1); - assert_eq!(idempotent.total_rows(), PUBLISHES); - - let final_bytes = std::fs::metadata(root.join(MANIFEST_FILE)).unwrap().len(); - assert!( - final_bytes <= initial_bytes + 32, - "manifest bytes must grow only with integer digit width: initial={initial_bytes}, final={final_bytes}" - ); - } - - #[tokio::test] - async fn contiguous_segment_group_replacement_preserves_order_and_rows() { - let directory = tempfile::tempdir().unwrap(); - let root = directory.path().join("events.lance"); - let uri = root.to_str().unwrap(); - let fence = EventWriterFence::new(1, "writer").unwrap(); - activate(uri, Some(&fence), "unused").await.unwrap(); - for id in ["a", "b", "c"] { - publish_segment( - uri, - &fence, - EventSegment { - id: id.into(), - version: 1, - rows: 2, - level: 0, - sealed: true, - }, - ) - .await - .unwrap(); - } - let before = read(uri).await.unwrap().unwrap(); - let merged = applied( - replace_segment_group( - uri, - &fence, - &before.segments[..2], - EventSegment { - id: "ab".into(), - version: 1, - rows: 4, - level: 1, - sealed: true, - }, - ) - .await - .unwrap(), - ); - assert_eq!( - merged - .segments - .iter() - .map(|segment| segment.id.as_str()) - .collect::>(), - ["ab", "c"] - ); - assert_eq!(merged.total_rows(), before.total_rows()); - - let stale_outcome = replace_segment_group( - uri, - &fence, - &before.segments[..2], - EventSegment { - id: "stale".into(), - version: 1, - rows: 4, - level: 1, - sealed: true, - }, - ) - .await - .unwrap(); - assert_eq!( - stale_outcome, - ManifestWriteOutcome::Conflict(EventWriterConflict::PublicationChanged) - ); - } - - #[tokio::test] - async fn malformed_manifest_fails_closed() { - let directory = tempfile::tempdir().unwrap(); - let root = directory.path().join("events.lance"); - let uri = root.to_str().unwrap(); - activate(uri, None, "writer").await.unwrap(); - std::fs::write(root.join(MANIFEST_FILE), b"{truncated").unwrap(); - let error = read(uri).await.unwrap_err(); - assert!(format!("{error:#}").contains("decode event manifest")); - } - - #[test] - fn manifest_wire_has_no_schema_marker() { - let bytes = serde_json::to_vec(&serde_json::json!({ - "revision": 7, - "fact_version": 7, - "fact_rows": 0, - "active_writer": {"epoch": 1, "writer_id": "writer"}, - "segments": [] - })) - .unwrap(); - let manifest = decode_manifest(&bytes, "manifest fixture").unwrap(); - let value = serde_json::to_value(manifest).unwrap(); - assert!(value.get("schema_version").is_none()); - } - - #[tokio::test] - async fn object_store_cleanup_removes_only_unreferenced_segment_prefixes() { - let uri = format!( - "shared-memory://event-segment-cleanup-{}/events.lance", - uuid::Uuid::new_v4() - ); - let store = Store::from_uri(&uri).await.unwrap(); - let orphan = "segments/orphan.lance/data/file.lance"; - let visible = "segments/visible.lance/data/file.lance"; - store - .write_overwrite(orphan, b"orphan".to_vec()) - .await - .unwrap(); - store - .write_overwrite(visible, b"visible".to_vec()) - .await - .unwrap(); - - let stats = cleanup_unreferenced_segments( - &uri, - &[EventSegment { - id: "visible".into(), - version: 1, - rows: 1, - level: 0, - sealed: false, - }], - Duration::ZERO, - ) - .await - .unwrap(); - assert_eq!(stats.segments_removed, 1); - assert!(store.read(orphan).await.unwrap().is_none()); - assert!(store.read(visible).await.unwrap().is_some()); - } -} - -#[cfg(all(test, feature = "proptest"))] -mod proptests { - use proptest::prelude::*; - - use super::*; - - proptest! { - #[test] - fn manifest_total_rows_sums_segment_rows_without_reordering( - rows in proptest::collection::vec(0u64..1_000_000, 0..16), - ) { - let segments = rows.iter().enumerate().map(|(index, rows)| EventSegment { - id: format!("segment-{index}"), - version: index as u64 + 1, - rows: *rows, - level: (index % 4) as u8, - sealed: index % 2 == 0, - }).collect::>(); - let manifest = EventManifest { - revision: 1, - fact_version: 1, - fact_rows: rows.iter().sum(), - active_writer: EventWriterFence::new(1, "writer").unwrap(), - segments, - }; - prop_assert_eq!(manifest.total_rows(), rows.iter().sum::()); - } - } -} diff --git a/crates/persisting-pchronicle/src/store/events/mod.rs b/crates/persisting-pchronicle/src/store/events/mod.rs deleted file mode 100644 index fb388e05d..000000000 --- a/crates/persisting-pchronicle/src/store/events/mod.rs +++ /dev/null @@ -1,1075 +0,0 @@ -//! Lance event log backend for the canonical trajectory store. -//! -//! Capture runs use one run-level manifest at `{run}/events.lance`; each writer -//! epoch owns one private Lance segment and `session_id` filters rows when -//! replaying one story view. - -pub(super) mod datafusion; -mod manifest; -mod rows; - -use std::collections::BTreeMap; -use std::sync::Arc; -use std::time::Duration; - -use crate::EventRow; -use ::datafusion::error::DataFusionError; -use ::datafusion::physical_plan::SendableRecordBatchStream; -use ::datafusion::physical_plan::stream::RecordBatchStreamAdapter; -use anyhow::{Context, Result}; -use futures::{StreamExt, TryStreamExt}; -use lance::Dataset; -use lance::dataset::optimize::{CompactionOptions, compact_files}; -use lance::dataset::{InsertBuilder, WriteMode, WriteParams}; -use lance::deps::arrow_array::{Array, RecordBatch}; -use lance::index::DatasetIndexExt; -use lance_index::IndexType; -use lance_index::optimize::OptimizeOptions; -use lance_index::scalar::{BuiltinIndexType, ScalarIndexParams}; - -pub use self::datafusion::{DATAFUSION_EVENTS_TABLE, EventFactSnapshot, RawEventDataSource}; -use self::manifest as raw_event_manifest; -pub use self::manifest::EventWriterFence; -use self::manifest::{EventManifest, EventSegment, EventWriterConflict, ManifestWriteOutcome}; -pub use self::rows::{ - event_records_from_batch, event_rows_from_batch, event_rows_to_batch, raw_event_arrow_schema, -}; -use self::rows::{event_row_for_storage, schema_columns_note}; -use super::{ - AppendOutcome, ReplayOutcome, StoryCoords, TrajectoryStats, dataset_write_lock, - raw_event_lance_path, -}; - -const SESSION_INDEX_NAME: &str = "pchronicle_session_id_idx"; -const MAX_SEGMENT_OPEN_CONCURRENCY: usize = 16; -const MAX_APPEND_GROUP_CONCURRENCY: usize = 16; - -#[derive(Debug, Clone, PartialEq)] -pub struct LanceMaintenanceOptions { - pub compact: bool, - pub optimize_indices: bool, - pub vacuum_older_than: Option, - pub target_rows_per_fragment: usize, - /// Maximum live source rows compacted per Storyline data table - /// (runs/steps/tool_calls) per maintenance call. None leaves compaction - /// unlimited. An undersized budget can prevent progress; raise it if no - /// compaction task fits. Not supported by raw-event - /// maintenance, which rewrites across separate segment datasets. - pub max_compaction_source_rows: Option, - /// Maximum source data/overlay file bytes per Storyline table per call. - /// Excludes separate Blob v2 payloads and does not bound total memory or - /// index/GC work. An undersized budget can leave no eligible task; raise it - /// if compaction makes no progress. Requires recorded source file sizes. - /// None leaves compaction unlimited. Not supported by raw-event maintenance. - pub max_compaction_source_bytes: Option, -} - -impl Default for LanceMaintenanceOptions { - fn default() -> Self { - Self { - compact: true, - optimize_indices: true, - vacuum_older_than: Some(Duration::from_secs(7 * 24 * 60 * 60)), - target_rows_per_fragment: 1024 * 1024, - max_compaction_source_rows: None, - max_compaction_source_bytes: None, - } - } -} - -#[derive(Debug, Clone, Default, PartialEq, Eq)] -pub struct LanceMaintenanceReport { - pub fragments_removed: usize, - pub fragments_added: usize, - pub old_versions_removed: u64, - pub bytes_removed: u64, - pub final_version: Option, -} - -#[derive(Debug, Clone, Default, PartialEq, Eq)] -pub struct EventLogLayoutStats { - pub manifest_revision: Option, - pub active_epoch: Option, - pub visible_segments: usize, - pub visible_fragments: usize, - pub visible_rows: u64, - pub max_segment_level: u8, - pub sealed_segments: usize, -} - -#[derive(Debug)] -pub struct RawEventLanceAppender { - requested_fence: Option, - auto_writer_id: String, - datasets: BTreeMap, -} - -#[derive(Debug)] -struct CachedRawDataset { - fence: EventWriterFence, - segment_id: String, - segment_uri: String, - dataset: Option, - rows: u64, - pending_fragments: usize, - manifest_revision: u64, -} - -#[derive(Debug, Default)] -pub(crate) struct EventAppendBatchReport { - outcomes: BTreeMap>, -} - -impl EventAppendBatchReport { - #[cfg(test)] - pub(crate) fn outcome_for(&self, root_uri: &str) -> Option<&Result> { - self.outcomes.get(root_uri) - } - - pub(crate) fn take_outcome(&mut self, root_uri: &str) -> Option> { - self.outcomes.remove(root_uri) - } - - fn accepted_records(&self) -> usize { - self.outcomes - .values() - .filter_map(|outcome| outcome.as_ref().ok()) - .sum() - } - - fn take_failure(&mut self) -> Option<(String, anyhow::Error)> { - let uri = self - .outcomes - .iter() - .find(|(_, outcome)| outcome.is_err()) - .map(|(uri, _)| uri.clone())?; - match self.outcomes.remove(&uri)? { - Err(error) => Some((uri, error)), - Ok(records) => { - self.outcomes.insert(uri, Ok(records)); - None - } - } - } -} - -/// Immutable writer segment handed to background tiny-fragment maintenance. -#[derive(Debug)] -pub(crate) struct SealedEventSegment { - root_uri: String, - fence: EventWriterFence, - segment: EventSegment, -} - -impl Default for RawEventLanceAppender { - fn default() -> Self { - Self { - requested_fence: None, - auto_writer_id: format!("auto-{}", uuid::Uuid::new_v4()), - datasets: BTreeMap::new(), - } - } -} - -impl RawEventLanceAppender { - /// Construct a writer bound to an externally allocated lease epoch. - /// - /// `writer_id` must uniquely identify the lease holder or process attempt. - /// Another writer using the same epoch but a different id is rejected. - pub fn fenced(fence: EventWriterFence) -> Self { - Self { - auto_writer_id: fence.writer_id.clone(), - requested_fence: Some(fence), - datasets: BTreeMap::new(), - } - } - - /// Activate this writer before accepting data. A newer activation fences - /// every older appender at the manifest publication boundary. - pub async fn activate(&mut self, session: &StoryCoords) -> Result { - let uri = raw_event_lance_path(session)? - .to_string_lossy() - .into_owned(); - if let Some(state) = self.datasets.get(&uri) { - return Ok(state.fence.clone()); - } - let state = self.new_state(uri.as_str()).await?; - let fence = state.fence.clone(); - self.datasets.insert(uri, state); - Ok(fence) - } - - async fn new_state(&self, uri: &str) -> Result { - let manifest = manifest_write_applied( - raw_event_manifest::activate(uri, self.requested_fence.as_ref(), &self.auto_writer_id) - .await?, - )?; - let fence = manifest.active_writer.clone(); - let segment_id = format!("e{}-{}", fence.epoch, uuid::Uuid::new_v4()); - Ok(CachedRawDataset { - fence, - segment_uri: raw_event_manifest::segment_uri(uri, &segment_id), - segment_id, - dataset: None, - rows: 0, - pending_fragments: 0, - manifest_revision: manifest.revision, - }) - } - - /// Append one channel-sized micro-batch with no read-before-write work. - /// - /// This writer deliberately provides at-least-once, append-only semantics: - /// every input record becomes one physical row, including duplicate - /// `event_id` values. Visibility is published with an epoch-fenced manifest - /// CAS, so a stale writer can leave only an unreachable Lance version. - pub async fn append_event_batch( - &mut self, - entries: &[(StoryCoords, crate::EventRecord)], - ) -> Result { - let mut report = self.append_event_batch_partitioned(entries).await?; - if let Some((uri, error)) = report.take_failure() { - return Err(error.context(format!("append canonical event partition {uri}"))); - } - let accepted_records = report.accepted_records(); - Ok(AppendOutcome { - accepted_records, - persisted_units: accepted_records, - note: format!("Lance: committed {accepted_records} event(s) in a micro-batch"), - }) - } - - /// Append independent event-log roots concurrently and retain a result for - /// every root so capture acknowledgements do not share one failure domain. - pub(crate) async fn append_event_batch_partitioned( - &mut self, - entries: &[(StoryCoords, crate::EventRecord)], - ) -> Result { - if entries.is_empty() { - return Ok(EventAppendBatchReport::default()); - } - let mut groups: BTreeMap> = BTreeMap::new(); - for (session, record) in entries { - let uri = raw_event_lance_path(session)? - .to_string_lossy() - .into_owned(); - groups - .entry(uri) - .or_default() - .push((session.clone(), record.clone())); - } - - let mut report = EventAppendBatchReport::default(); - let mut pending = Vec::with_capacity(groups.len()); - for (uri, entries) in groups { - let rows = entries - .into_iter() - .map(|(session, record)| { - let record = super::canonicalize_event(&session, record)?; - event_row_for_storage(&session.session_id, &session.agent_id, &record) - }) - .collect::>>(); - let rows = match rows { - Ok(rows) => rows, - Err(error) => { - report.outcomes.insert(uri, Err(error)); - continue; - } - }; - let state = match self.datasets.remove(&uri) { - Some(state) => state, - None => match self.new_state(&uri).await { - Ok(state) => state, - Err(error) => { - report.outcomes.insert(uri, Err(error)); - continue; - } - }, - }; - pending.push((uri, state, rows)); - } - - // Every URI owns an independent Lance segment and manifest. Appending - // them sequentially made one slow session hold the completion for the - // entire cross-session micro-batch and indirectly starved Gateway's - // capture actors. Preserve per-URI ordering across batches while - // allowing independent sessions in this batch to progress concurrently. - let completed = futures::stream::iter(pending) - .map(|(uri, state, rows)| { - let outcome_uri = uri.clone(); - async move { (outcome_uri, append_event_group(uri, state, rows).await) } - }) - .buffer_unordered(MAX_APPEND_GROUP_CONCURRENCY) - .collect::>() - .await; - for (uri, outcome) in completed { - match outcome { - Ok((_returned_uri, state, appended_rows)) => { - self.datasets.insert(uri.clone(), state); - report.outcomes.insert(uri, Ok(appended_rows)); - } - Err(error) => { - report.outcomes.insert(uri, Err(error)); - } - } - } - Ok(report) - } - - /// Seal active segments that have accumulated enough micro-append - /// fragments. Future appends immediately move to a new private segment, so - /// the sealed segment can be compacted concurrently without commit races. - pub(crate) fn seal_fragmented_segments( - &mut self, - fragment_threshold: usize, - ) -> Result> { - anyhow::ensure!( - fragment_threshold > 1, - "fragment sealing threshold must be greater than one" - ); - - let mut sealed = Vec::new(); - for (uri, state) in &mut self.datasets { - if state.pending_fragments < fragment_threshold { - continue; - } - let dataset = state - .dataset - .take() - .context("fragmented event segment is missing its Lance dataset")?; - sealed.push(SealedEventSegment { - root_uri: uri.clone(), - fence: state.fence.clone(), - segment: EventSegment { - id: state.segment_id.clone(), - version: dataset.version_id(), - rows: state.rows, - level: 0, - sealed: false, - }, - }); - - state.segment_id = format!("e{}-{}", state.fence.epoch, uuid::Uuid::new_v4()); - state.segment_uri = raw_event_manifest::segment_uri(uri, &state.segment_id); - state.rows = 0; - state.pending_fragments = 0; - } - Ok(sealed) - } - - /// Close the writer without indexing, full compaction, vacuum, or any other - /// synchronous maintenance. The append worker already performs bounded - /// tiny-fragment compaction; heavy layout work remains available through - /// the explicit [`maintain`] API. - pub fn finish(self) -> Vec { - self.datasets - .into_values() - .map(|state| LanceMaintenanceReport { - final_version: Some(state.manifest_revision), - ..Default::default() - }) - .collect() - } -} - -async fn append_event_group( - uri: String, - mut state: CachedRawDataset, - rows: Vec, -) -> Result<(String, CachedRawDataset, usize)> { - let appended_rows = rows.len(); - let dataset = append_rows(&state.segment_uri, state.dataset.take(), rows).await?; - state.rows = state - .rows - .checked_add(appended_rows as u64) - .context("event segment row count overflow")?; - state.pending_fragments = state.pending_fragments.saturating_add(1); - let manifest = manifest_write_applied( - raw_event_manifest::publish_segment( - &uri, - &state.fence, - EventSegment { - id: state.segment_id.clone(), - version: dataset.version_id(), - rows: state.rows, - level: 0, - sealed: false, - }, - ) - .await?, - )?; - state.manifest_revision = manifest.revision; - state.dataset = Some(dataset); - Ok((uri, state, appended_rows)) -} - -/// Compact one immutable segment and atomically advance its outer visibility -/// pointer. A crash before publication leaves the original segment version -/// visible; a newer writer fence rejects the stale publication. -pub(crate) async fn compact_sealed_event_segment( - sealed: SealedEventSegment, - target_rows_per_fragment: usize, - hierarchy_fanout: usize, -) -> Result<()> { - anyhow::ensure!( - target_rows_per_fragment > 0, - "fragment compaction target rows must be greater than zero" - ); - anyhow::ensure!( - hierarchy_fanout > 1, - "fragment hierarchy fanout must be greater than one" - ); - let _guard = dataset_write_lock::acquire(&sealed.root_uri).await?; - let segment_uri = raw_event_manifest::segment_uri(&sealed.root_uri, &sealed.segment.id); - let mut dataset = crate::storage::open_lance_dataset(&segment_uri) - .await - .with_context(|| format!("open sealed event segment {segment_uri}"))?; - let metrics = compact_files( - &mut dataset, - CompactionOptions { - target_rows_per_fragment, - max_rows_per_group: target_rows_per_fragment.min(1024 * 1024), - num_threads: Some(1), - ..Default::default() - }, - None, - ) - .await - .with_context(|| format!("compact sealed trajectory segment {segment_uri}"))?; - let mut published_segment = sealed.segment; - if metrics.fragments_removed > 0 { - published_segment.version = dataset.version_id(); - } - published_segment.sealed = true; - manifest_write_applied( - raw_event_manifest::publish_segment(&sealed.root_uri, &sealed.fence, published_segment) - .await?, - )?; - compact_event_hierarchy_locked(&sealed.root_uri, &sealed.fence, hierarchy_fanout).await?; - Ok(()) -} - -fn next_hierarchy_group(manifest: &EventManifest, fanout: usize) -> Option> { - manifest - .segments - .windows(fanout) - .find(|segments| { - let level = segments[0].level; - segments - .iter() - .all(|segment| segment.sealed && segment.level == level) - }) - .map(<[EventSegment]>::to_vec) -} - -async fn compact_event_hierarchy_locked( - root_uri: &str, - fence: &EventWriterFence, - fanout: usize, -) -> Result<()> { - loop { - let manifest = raw_event_manifest::read(root_uri) - .await? - .context("event manifest disappeared during hierarchical compaction")?; - let Some(group) = next_hierarchy_group(&manifest, fanout) else { - return Ok(()); - }; - let next_level = group[0] - .level - .checked_add(1) - .context("event segment compaction level overflow")?; - let datasets = futures::stream::iter(group.iter().cloned()) - .map(|segment| async move { open_visible_segment(root_uri, &segment).await }) - .buffered(MAX_SEGMENT_OPEN_CONCURRENCY) - .try_collect::>() - .await?; - let rows = group.iter().try_fold(0_u64, |total, segment| { - total - .checked_add(segment.rows) - .context("hierarchical event segment row count overflow") - })?; - let target_rows = usize::try_from(rows) - .context("hierarchical event segment is too large for this platform")? - .max(1); - let (dataset, mut replacement) = - compact_visible_segments(root_uri, datasets, target_rows, false).await?; - replacement.version = dataset.version_id(); - replacement.level = next_level; - replacement.sealed = true; - manifest_write_applied( - raw_event_manifest::replace_segment_group(root_uri, fence, &group, replacement).await?, - )?; - } -} - -pub(super) fn validate_event_schema(dataset: &Dataset, uri: &str) -> Result<()> { - let actual = lance::deps::arrow_schema::Schema::from(dataset.schema()); - let expected = raw_event_arrow_schema(); - anyhow::ensure!( - actual.fields() == expected.fields(), - "trajectory dataset schema mismatch at {uri}: expected {expected:?}, found {actual:?}" - ); - Ok(()) -} - -async fn open_visible_segment(root_uri: &str, segment: &EventSegment) -> Result { - let uri = raw_event_manifest::segment_uri(root_uri, &segment.id); - let latest = crate::storage::open_lance_dataset(&uri) - .await - .with_context(|| format!("open event segment {uri}"))?; - let dataset = if latest.version_id() == segment.version { - latest - } else { - latest - .checkout_version(segment.version) - .await - .with_context(|| { - format!( - "open visible event segment version {} at {uri}", - segment.version - ) - })? - }; - validate_event_schema(&dataset, &uri)?; - Ok(dataset) -} - -async fn open_visible_snapshot(root_uri: &str) -> Result)>> { - let Some(manifest) = pin_visible_snapshot(root_uri).await? else { - return Ok(None); - }; - let datasets = open_pinned_snapshot(root_uri, &manifest).await?; - Ok(Some((manifest, datasets))) -} - -pub(crate) async fn inspect_visible_event_tables(uri: &str) -> Result> { - let Some((manifest, datasets)) = open_visible_snapshot(uri).await? else { - return Ok(Vec::new()); - }; - Ok(manifest - .segments - .iter() - .zip(datasets) - .map(|(segment, dataset)| (segment.id.clone(), dataset)) - .collect()) -} - -/// Read only the small visibility manifest. The returned segment versions are -/// the immutable catalog descriptor used for later lazy resolution. -async fn pin_visible_snapshot(root_uri: &str) -> Result> { - raw_event_manifest::read(root_uri).await -} - -/// Open exactly the segment versions captured by -/// [`pin_visible_snapshot`], without consulting the latest manifest again. -async fn open_pinned_snapshot(root_uri: &str, manifest: &EventManifest) -> Result> { - let datasets = futures::stream::iter(manifest.segments.iter().cloned()) - .map(|segment| async move { open_visible_segment(root_uri, &segment).await }) - .buffered(MAX_SEGMENT_OPEN_CONCURRENCY) - .try_collect::>() - .await?; - Ok(datasets) -} - -#[cfg(test)] -async fn visible_fragment_count(root_uri: &str) -> Result { - let Some((_, datasets)) = open_visible_snapshot(root_uri).await? else { - return Ok(0); - }; - Ok(datasets - .iter() - .map(|dataset| dataset.get_fragments().len()) - .sum()) -} - -async fn ensure_session_index(dataset: &mut Dataset) -> Result<()> { - if dataset - .load_indices_by_name(SESSION_INDEX_NAME) - .await - .context("load trajectory session index")? - .is_empty() - { - let _admission = super::index_build_gate::acquire().await; - dataset - .create_index( - &[crate::TRAJECTORY_SESSION_ID_COL], - IndexType::BTree, - Some(SESSION_INDEX_NAME.into()), - &ScalarIndexParams::for_builtin(BuiltinIndexType::BTree), - false, - ) - .await - .context("create trajectory session index")?; - } - Ok(()) -} - -async fn compact_visible_segments( - root_uri: &str, - datasets: Vec, - target_rows_per_fragment: usize, - optimize_indices: bool, -) -> Result<(Dataset, EventSegment)> { - anyhow::ensure!( - target_rows_per_fragment > 0, - "target_rows_per_fragment must be greater than zero" - ); - let segment_id = format!("compact-{}", uuid::Uuid::new_v4()); - let segment_uri = raw_event_manifest::segment_uri(root_uri, &segment_id); - let schema = raw_event_arrow_schema(); - let streams = futures::stream::iter(datasets).then(|dataset| async move { - dataset - .scan() - .scan_in_order(true) - .try_into_stream() - .await - .map(|stream| stream.map_err(|error| DataFusionError::External(Box::new(error)))) - .map_err(|error| DataFusionError::External(Box::new(error))) - }); - let flattened = streams.try_flatten(); - let stream: SendableRecordBatchStream = - Box::pin(RecordBatchStreamAdapter::new(schema.clone(), flattened)); - let params = WriteParams { - max_rows_per_file: target_rows_per_fragment, - max_rows_per_group: target_rows_per_fragment.min(1024 * 1024), - ..Default::default() - }; - let mut dataset = InsertBuilder::new(segment_uri.as_str()) - .with_params(¶ms) - .execute_stream(stream) - .await - .with_context(|| format!("write compacted event segment {segment_uri}"))?; - if optimize_indices { - ensure_session_index(&mut dataset).await?; - } - let rows = dataset - .count_rows(None) - .await - .context("count compacted event segment rows")? as u64; - Ok(( - dataset, - EventSegment { - id: segment_id, - version: 0, - rows, - level: 0, - sealed: true, - }, - )) -} - -pub async fn maintain( - session: &StoryCoords, - options: &LanceMaintenanceOptions, -) -> Result { - anyhow::ensure!( - options.max_compaction_source_rows.is_none() - && options.max_compaction_source_bytes.is_none(), - "compaction source budgets are supported only by Storyline maintenance" - ); - let uri = raw_event_lance_path(session)? - .to_string_lossy() - .into_owned(); - let _guard = dataset_write_lock::acquire(&uri).await?; - let Some((before_manifest, datasets)) = open_visible_snapshot(&uri).await? else { - return Ok(LanceMaintenanceReport::default()); - }; - if datasets.is_empty() { - return Ok(LanceMaintenanceReport { - final_version: Some(before_manifest.revision), - ..Default::default() - }); - } - let maintenance_writer_id = format!("maintenance-{}", uuid::Uuid::new_v4()); - let active = manifest_write_applied( - raw_event_manifest::activate(&uri, None, &maintenance_writer_id).await?, - )?; - let fence = active.active_writer.clone(); - let fragments_before = datasets - .iter() - .map(|dataset| dataset.get_fragments().len()) - .sum::(); - let mut report = LanceMaintenanceReport { - fragments_removed: 0, - fragments_added: fragments_before, - old_versions_removed: 0, - bytes_removed: 0, - final_version: Some(active.revision), - }; - if options.compact { - let (compacted, mut segment) = compact_visible_segments( - &uri, - datasets, - options.target_rows_per_fragment, - options.optimize_indices, - ) - .await?; - segment.version = compacted.version_id(); - let published = manifest_write_applied( - raw_event_manifest::replace_segments(&uri, &fence, vec![segment]).await?, - )?; - report.fragments_removed = fragments_before; - report.fragments_added = compacted.get_fragments().len(); - report.final_version = Some(published.revision); - if let Some(retention) = options.vacuum_older_than { - let retention = chrono::Duration::from_std(retention) - .context("trajectory Lance vacuum retention is too large")?; - let removed = compacted - .cleanup_old_versions(retention, Some(false), Some(true)) - .await - .context("vacuum compacted event segment versions")?; - report.old_versions_removed = removed.old_versions; - report.bytes_removed = removed.bytes_removed; - } - } else { - let mut updated_segments = Vec::with_capacity(datasets.len()); - for (mut dataset, mut segment) in datasets.into_iter().zip(before_manifest.segments) { - if options.optimize_indices { - ensure_session_index(&mut dataset).await?; - dataset - .optimize_indices(&OptimizeOptions::append()) - .await - .context("optimize trajectory Lance indices")?; - } - segment.version = dataset.version_id(); - updated_segments.push(segment); - } - let published = manifest_write_applied( - raw_event_manifest::replace_segments(&uri, &fence, updated_segments).await?, - )?; - report.final_version = Some(published.revision); - } - if let Some(retention) = options.vacuum_older_than { - let current = raw_event_manifest::read(&uri) - .await? - .context("event manifest disappeared after maintenance")?; - let removed = - raw_event_manifest::cleanup_unreferenced_segments(&uri, ¤t.segments, retention) - .await?; - report.old_versions_removed = report - .old_versions_removed - .saturating_add(removed.segments_removed); - report.bytes_removed = report.bytes_removed.saturating_add(removed.bytes_removed); - } - Ok(report) -} - -#[cfg(test)] -async fn read_all_rows(uri: &str) -> Result> { - let Some((_, datasets)) = open_visible_snapshot(uri).await? else { - return Ok(Vec::new()); - }; - let mut rows = Vec::new(); - for dataset in datasets { - let batches: Vec = dataset - .scan() - .try_into_stream() - .await - .with_context(|| format!("scan trajectory event segment under {uri}"))? - .try_collect() - .await - .with_context(|| format!("collect trajectory event segment under {uri}"))?; - for batch in &batches { - rows.extend(event_rows_from_batch(batch)?); - } - } - Ok(rows) -} - -async fn append_rows(uri: &str, dataset: Option, rows: Vec) -> Result { - let batch = event_rows_to_batch(raw_event_arrow_schema(), &rows)?; - if !is_object_store_uri(uri) - && let Some(parent) = std::path::Path::new(uri).parent() - { - tokio::fs::create_dir_all(parent) - .await - .with_context(|| format!("create_dir_all {}", parent.display()))?; - } - match dataset { - Some(dataset) => InsertBuilder::new(Arc::new(dataset)) - .with_params(&WriteParams { - mode: WriteMode::Append, - ..Default::default() - }) - .execute(vec![batch]) - .await - .with_context(|| format!("append trajectory Lance dataset {uri}")), - None => InsertBuilder::new(uri) - .execute(vec![batch]) - .await - .with_context(|| format!("create trajectory Lance dataset {uri}")), - } -} - -fn session_predicate(session_id: &str) -> String { - format!("session_id = '{}'", session_id.replace('\'', "''")) -} - -async fn read_session_rows( - dataset: &Dataset, - session_id: &str, - offset: usize, - limit: Option, -) -> Result> { - let mut scan = dataset.scan(); - scan.filter(&session_predicate(session_id)) - .with_context(|| format!("filter trajectory session {session_id}"))?; - scan.scan_in_order(true); - scan.limit( - limit.map(|value| value as i64), - (offset > 0).then_some(offset as i64), - ) - .context("apply trajectory replay offset/limit")?; - let batches: Vec = scan - .try_into_stream() - .await - .context("scan trajectory session rows")? - .try_collect() - .await - .context("collect trajectory session rows")?; - let mut rows = Vec::new(); - for batch in &batches { - rows.extend(event_rows_from_batch(batch)?); - } - // Lance scan order is the canonical immutable append order. Producer `seq` - // values may repeat or reset between logical Storylines and are never used - // to reorder stored facts. - Ok(rows) -} - -pub fn display_path(session: &StoryCoords) -> Result { - Ok(raw_event_lance_path(session)? - .to_string_lossy() - .into_owned()) -} - -pub async fn distinct_session_ids_in_run(run: &StoryCoords) -> Result> { - let path = raw_event_lance_path(run)?; - let uri = path.to_string_lossy().into_owned(); - let Some((_, datasets)) = open_visible_snapshot(&uri).await? else { - return Ok(Vec::new()); - }; - let mut ids = Vec::new(); - for dataset in datasets { - let mut scan = dataset.scan(); - scan.project(&[crate::TRAJECTORY_SESSION_ID_COL]) - .context("project trajectory session_id")?; - let batches: Vec = scan - .try_into_stream() - .await - .context("scan trajectory session ids")? - .try_collect() - .await - .context("collect trajectory session ids")?; - for batch in &batches { - let column = batch - .column_by_name(crate::TRAJECTORY_SESSION_ID_COL) - .context("trajectory scan missing session_id")?; - let values = column - .as_any() - .downcast_ref::() - .context("trajectory session_id must be Utf8")?; - ids.extend(values.iter().flatten().map(ToOwned::to_owned)); - } - } - ids.sort(); - ids.dedup(); - Ok(ids) -} - -pub async fn exists(session: &StoryCoords) -> Result { - let path = raw_event_lance_path(session)?; - let uri = path.to_string_lossy().into_owned(); - let Some((_, datasets)) = open_visible_snapshot(&uri).await? else { - return Ok(false); - }; - for dataset in datasets { - if dataset - .count_rows(Some(session_predicate(&session.session_id))) - .await - .context("count trajectory session rows")? - > 0 - { - return Ok(true); - } - } - Ok(false) -} - -pub async fn append_events( - session: &StoryCoords, - records: &[crate::EventRecord], -) -> Result { - let entries = records - .iter() - .cloned() - .map(|record| (session.clone(), record)) - .collect::>(); - RawEventLanceAppender::default() - .append_event_batch(&entries) - .await -} - -pub(super) fn is_object_store_uri(uri: &str) -> bool { - uri.contains("://") -} - -fn manifest_write_applied(outcome: ManifestWriteOutcome) -> Result { - match outcome { - ManifestWriteOutcome::Applied(value) => Ok(value), - ManifestWriteOutcome::Conflict(EventWriterConflict::StaleFence) => { - anyhow::bail!("stale event writer fence") - } - ManifestWriteOutcome::Conflict(EventWriterConflict::EpochAlreadyOwned) => { - anyhow::bail!("event writer epoch is already owned") - } - ManifestWriteOutcome::Conflict(EventWriterConflict::PublicationChanged) => { - anyhow::bail!("event manifest publication changed") - } - } -} - -pub async fn replay( - session: &StoryCoords, - offset: usize, - limit: Option, -) -> Result { - replay_available(session, offset, limit) - .await? - .ok_or_else(|| { - let uri = raw_event_lance_path(session) - .map(|path| path.to_string_lossy().into_owned()) - .unwrap_or_else(|_| "".into()); - anyhow::anyhow!("trajectory Lance dataset does not exist at {uri}") - }) -} - -/// Replay the currently committed rows for one Storyline, returning `None` -/// while its run-level Lance dataset has not been created yet. -/// -/// Each call reads one atomic manifest revision and checks out the exact Lance -/// versions it names. This makes follow reads immune to stale-writer versions. -pub async fn replay_available( - session: &StoryCoords, - offset: usize, - limit: Option, -) -> Result> { - let path = raw_event_lance_path(session)?; - let uri = path.to_string_lossy().into_owned(); - let Some((manifest, datasets)) = open_visible_snapshot(&uri).await? else { - return Ok(None); - }; - let mut remaining_offset = offset; - let mut remaining_limit = limit.unwrap_or(usize::MAX); - let mut rows = Vec::new(); - for dataset in datasets { - if remaining_limit == 0 { - break; - } - let segment_rows = dataset - .count_rows(Some(session_predicate(&session.session_id))) - .await - .context("count trajectory session rows in visible segment")?; - if remaining_offset >= segment_rows { - remaining_offset -= segment_rows; - continue; - } - let take = remaining_limit.min(segment_rows - remaining_offset); - rows.extend( - read_session_rows(&dataset, &session.session_id, remaining_offset, Some(take)).await?, - ); - remaining_offset = 0; - remaining_limit -= take; - } - let schema = raw_event_arrow_schema(); - let batch = event_rows_to_batch(schema, &rows)?; - let records = event_records_from_batch(&batch)?; - Ok(Some(ReplayOutcome { - records, - note: format!( - "Replay fenced Lance manifest revision {} at {uri}: session_id={}, ordered by immutable segment and append order, offset={offset}, limit={limit:?}.", - manifest.revision, session.session_id, - ), - })) -} - -pub async fn stats(session: &StoryCoords) -> Result { - let path = raw_event_lance_path(session)?; - let display = path.to_string_lossy().into_owned(); - let Some((manifest, datasets)) = open_visible_snapshot(&display).await? else { - return Ok(TrajectoryStats { - dataset: display, - row_count: 0, - manifest_revision: None, - status: "missing".to_string(), - note: "No Lance event log at this path yet; use trajectory add first.".to_string(), - }); - }; - let mut row_count = 0usize; - let mut fragment_count = 0usize; - let mut indexed_segments = 0usize; - for dataset in &datasets { - row_count += dataset - .count_rows(Some(session_predicate(&session.session_id))) - .await - .context("count trajectory session rows")?; - fragment_count += dataset.get_fragments().len(); - if !dataset - .load_indices_by_name(SESSION_INDEX_NAME) - .await - .context("inspect trajectory session index")? - .is_empty() - { - indexed_segments += 1; - } - } - Ok(TrajectoryStats { - dataset: display.clone(), - row_count, - manifest_revision: Some(manifest.revision), - status: "ok".to_string(), - note: format!( - "Lance epoch-fenced append-only [{}]; session_id={}; dataset={}; epoch={}; \ - visible_rows={}; segments={}; fragments={fragment_count}; indexed_segments={indexed_segments}", - schema_columns_note(), - session.session_id, - display, - manifest.active_writer.epoch, - manifest.total_rows(), - manifest.segments.len(), - ), - }) -} - -pub async fn layout_stats(session: &StoryCoords) -> Result { - let uri = raw_event_lance_path(session)? - .to_string_lossy() - .into_owned(); - let Some((manifest, datasets)) = open_visible_snapshot(&uri).await? else { - return Ok(EventLogLayoutStats::default()); - }; - Ok(EventLogLayoutStats { - manifest_revision: Some(manifest.revision), - active_epoch: Some(manifest.active_writer.epoch), - visible_segments: manifest.segments.len(), - visible_fragments: datasets - .iter() - .map(|dataset| dataset.get_fragments().len()) - .sum(), - visible_rows: manifest.total_rows(), - max_segment_level: manifest - .segments - .iter() - .map(|segment| segment.level) - .max() - .unwrap_or(0), - sealed_segments: manifest - .segments - .iter() - .filter(|segment| segment.sealed) - .count(), - }) -} - -#[cfg(test)] -mod tests; diff --git a/crates/persisting-pchronicle/src/store/events/rows.rs b/crates/persisting-pchronicle/src/store/events/rows.rs deleted file mode 100644 index dd570a812..000000000 --- a/crates/persisting-pchronicle/src/store/events/rows.rs +++ /dev/null @@ -1,444 +0,0 @@ -//! Shared Arrow row helpers for the canonical trajectory event log schema. - -use std::sync::Arc; - -use crate::{ - EventRecord, EventRow, TRAJECTORY_AGENT_ID_COL, TRAJECTORY_CALL_ID_COL, TRAJECTORY_COLS, - TRAJECTORY_EVENT_ID_COL, TRAJECTORY_KIND_COL, TRAJECTORY_MODEL_COL, - TRAJECTORY_PARENT_CALL_ID_COL, TRAJECTORY_PAYLOAD_JSON_COL, TRAJECTORY_SEQ_COL, - TRAJECTORY_SESSION_ID_COL, TRAJECTORY_SOURCE_COL, TRAJECTORY_TIMESTAMP_COL, - TRAJECTORY_TRACE_ID_COL, event_record_to_event_row, event_row_to_event_record, -}; -use anyhow::{Context, Result}; -use lance::deps::arrow_array::{ - Array, Int64Array, RecordBatch, StringArray, TimestampMillisecondArray, -}; -use lance::deps::arrow_schema::{DataType, Field, Schema as ArrowSchema, TimeUnit}; - -pub fn raw_event_arrow_schema() -> Arc { - Arc::new(ArrowSchema::new(vec![ - Field::new(TRAJECTORY_SEQ_COL, DataType::Int64, false), - Field::new(TRAJECTORY_EVENT_ID_COL, DataType::Utf8, true), - Field::new( - TRAJECTORY_TIMESTAMP_COL, - DataType::Timestamp(TimeUnit::Millisecond, Some("UTC".into())), - false, - ), - Field::new(TRAJECTORY_KIND_COL, DataType::Utf8, false), - Field::new(TRAJECTORY_SOURCE_COL, DataType::Utf8, false), - Field::new(TRAJECTORY_AGENT_ID_COL, DataType::Utf8, true), - Field::new(TRAJECTORY_SESSION_ID_COL, DataType::Utf8, true), - Field::new(TRAJECTORY_CALL_ID_COL, DataType::Utf8, true), - Field::new(TRAJECTORY_TRACE_ID_COL, DataType::Utf8, true), - Field::new(TRAJECTORY_PARENT_CALL_ID_COL, DataType::Utf8, true), - Field::new(TRAJECTORY_MODEL_COL, DataType::Utf8, true), - Field::new(TRAJECTORY_PAYLOAD_JSON_COL, DataType::Utf8, false), - ])) -} - -fn opt_utf8(values: &[Option]) -> StringArray { - StringArray::from(values.iter().map(|v| v.as_deref()).collect::>()) -} - -fn req_utf8(values: &[String]) -> StringArray { - StringArray::from(values.iter().map(|s| s.as_str()).collect::>()) -} - -pub fn event_rows_to_batch(schema: Arc, rows: &[EventRow]) -> Result { - RecordBatch::try_new( - schema, - vec![ - Arc::new(Int64Array::from( - rows.iter().map(|r| r.seq).collect::>(), - )), - Arc::new(opt_utf8( - &rows.iter().map(|r| r.event_id.clone()).collect::>(), - )), - Arc::new( - TimestampMillisecondArray::from( - rows.iter().map(|row| row.timestamp).collect::>(), - ) - .with_timezone("UTC"), - ), - Arc::new(req_utf8( - &rows.iter().map(|r| r.kind.clone()).collect::>(), - )), - Arc::new(req_utf8( - &rows.iter().map(|r| r.source.clone()).collect::>(), - )), - Arc::new(opt_utf8( - &rows.iter().map(|r| r.agent_id.clone()).collect::>(), - )), - Arc::new(opt_utf8( - &rows - .iter() - .map(|r| r.session_id.clone()) - .collect::>(), - )), - Arc::new(opt_utf8( - &rows.iter().map(|r| r.call_id.clone()).collect::>(), - )), - Arc::new(opt_utf8( - &rows.iter().map(|r| r.trace_id.clone()).collect::>(), - )), - Arc::new(opt_utf8( - &rows - .iter() - .map(|r| r.parent_call_id.clone()) - .collect::>(), - )), - Arc::new(opt_utf8( - &rows.iter().map(|r| r.model.clone()).collect::>(), - )), - Arc::new(req_utf8( - &rows - .iter() - .map(|r| r.payload_json.clone()) - .collect::>(), - )), - ], - ) - .context("build trajectory RecordBatch") -} - -pub(super) fn event_row_for_storage( - storage_session_id: &str, - storage_agent_id: &str, - record: &EventRecord, -) -> Result { - // Preserve the producer envelope in payload_json, but route and replay - // through the physical coordinates selected by the capture caller. - // Conflicting producer identity is evidence, not an append failure. - let mut row = event_record_to_event_row(record)?; - row.session_id = Some(storage_session_id.to_string()); - row.agent_id = Some(storage_agent_id.to_string()); - Ok(row) -} - -pub fn utf8_at(batch: &RecordBatch, name: &str, row: usize) -> Result> { - let col_idx = batch - .schema() - .fields() - .iter() - .position(|f| f.name() == name) - .ok_or_else(|| anyhow::anyhow!("batch missing column '{name}'"))?; - let col = batch.column(col_idx); - let Some(a) = col.as_any().downcast_ref::() else { - anyhow::bail!("expected Utf8 column {name}"); - }; - if a.is_null(row) { - Ok(None) - } else { - Ok(Some(a.value(row).to_string())) - } -} - -pub fn req_utf8_at(batch: &RecordBatch, name: &str, row: usize) -> Result { - utf8_at(batch, name, row)?.ok_or_else(|| anyhow::anyhow!("null required column {name}")) -} - -pub fn seq_at(batch: &RecordBatch, row: usize) -> Result { - let col_idx = batch - .schema() - .fields() - .iter() - .position(|f| f.name() == TRAJECTORY_SEQ_COL) - .ok_or_else(|| anyhow::anyhow!("batch missing {}", TRAJECTORY_SEQ_COL))?; - let col = batch.column(col_idx); - let Some(a) = col.as_any().downcast_ref::() else { - anyhow::bail!("expected Int64 column {}", TRAJECTORY_SEQ_COL); - }; - Ok(a.value(row)) -} - -fn timestamp_ms_at(batch: &RecordBatch, name: &str, row: usize) -> Result { - let col_idx = batch - .schema() - .fields() - .iter() - .position(|field| field.name() == name) - .ok_or_else(|| anyhow::anyhow!("batch missing column '{name}'"))?; - let column = batch.column(col_idx); - let array = column - .as_any() - .downcast_ref::() - .ok_or_else(|| anyhow::anyhow!("expected Timestamp(Millisecond, UTC) column {name}"))?; - anyhow::ensure!( - !array.is_null(row), - "null canonical timestamp in column {name}" - ); - Ok(array.value(row)) -} - -pub fn event_row_from_batch(batch: &RecordBatch, index: usize) -> Result { - let payload_json = req_utf8_at(batch, TRAJECTORY_PAYLOAD_JSON_COL, index)?; - let event_id = utf8_at(batch, TRAJECTORY_EVENT_ID_COL, index)?; - Ok(EventRow { - seq: seq_at(batch, index)?, - event_id, - timestamp: timestamp_ms_at(batch, TRAJECTORY_TIMESTAMP_COL, index)?, - kind: req_utf8_at(batch, TRAJECTORY_KIND_COL, index)?, - source: req_utf8_at(batch, TRAJECTORY_SOURCE_COL, index)?, - agent_id: utf8_at(batch, TRAJECTORY_AGENT_ID_COL, index)?, - session_id: utf8_at(batch, TRAJECTORY_SESSION_ID_COL, index)?, - call_id: utf8_at(batch, TRAJECTORY_CALL_ID_COL, index)?, - trace_id: utf8_at(batch, TRAJECTORY_TRACE_ID_COL, index)?, - parent_call_id: utf8_at(batch, TRAJECTORY_PARENT_CALL_ID_COL, index)?, - model: utf8_at(batch, TRAJECTORY_MODEL_COL, index)?, - payload_json, - }) -} - -pub fn event_records_from_batch(batch: &RecordBatch) -> Result> { - (0..batch.num_rows()) - .map(|i| { - let row = event_row_from_batch(batch, i)?; - event_row_to_event_record(&row) - }) - .collect() -} - -pub fn event_rows_from_batch(batch: &RecordBatch) -> Result> { - (0..batch.num_rows()) - .map(|i| event_row_from_batch(batch, i)) - .collect() -} - -pub fn schema_columns_note() -> String { - TRAJECTORY_COLS.join(", ") -} - -#[cfg(test)] -mod tests { - use super::*; - - const APPEND_CHUNK_ROWS: usize = 8192; - - fn record_batches_for_rows( - schema: Arc, - rows: &[EventRow], - ) -> Result> { - let mut batches = Vec::new(); - for chunk in rows.chunks(APPEND_CHUNK_ROWS) { - batches.push(event_rows_to_batch(schema.clone(), chunk)?); - } - Ok(batches) - } - - fn mk_row(seq: i64, session: &str, content: &str) -> EventRow { - EventRow { - seq, - event_id: Some(format!("event-{seq}")), - timestamp: 0, - kind: "note".into(), - source: "test".into(), - agent_id: Some("agent".into()), - session_id: Some(session.into()), - call_id: None, - trace_id: None, - parent_call_id: None, - model: None, - payload_json: format!(r#"{{"content":"{content}"}}"#), - } - } - - #[test] - fn event_row_for_storage_stamps_physical_route_and_preserves_producer_seq() { - let record = EventRecord { - identity: crate::EventIdentity { - event_id: Some("event-a".into()), - timestamp_unix_ms: Some(0), - ..Default::default() - }, - seq: 0, - source: "test".into(), - kind: "note".into(), - timestamp: None, - session_id: Some("sess-a".into()), - agent_id: None, - parent_uuid: None, - trace_id: None, - call_id: None, - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload: serde_json::json!({"content":"a"}), - }; - let row = event_row_for_storage("sess-a", "agent-a", &record).unwrap(); - assert_eq!(row.seq, 0); - assert_eq!(row.session_id.as_deref(), Some("sess-a")); - assert_eq!(row.agent_id.as_deref(), Some("agent-a")); - } - - #[test] - fn event_record_to_event_row_preserves_call_id() { - let resp = EventRecord { - identity: crate::EventIdentity { - event_id: Some("event-response".into()), - timestamp_unix_ms: Some(0), - ..Default::default() - }, - seq: 0, - source: "test".into(), - kind: "llm.response".into(), - timestamp: None, - session_id: Some("sess".into()), - agent_id: Some("agent".into()), - parent_uuid: None, - trace_id: Some("trace-a".into()), - call_id: Some("call-a".into()), - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload: serde_json::json!({ - "status": 200, - "choices":[{"message":{"role":"assistant","content":"你好！"}}], - }), - }; - let row = event_row_for_storage("sess", "agent", &resp).unwrap(); - assert_eq!(row.call_id.as_deref(), Some("call-a")); - assert_eq!(row.trace_id.as_deref(), Some("trace-a")); - assert_eq!(row.seq, 0); - - let back = crate::event_row_to_event_record(&row).unwrap(); - assert_eq!(back.call_id.as_deref(), Some("call-a")); - assert_eq!(back.seq, 0); - } - - #[test] - fn physical_routing_identity_wins_without_rewriting_payload_json() { - let record = EventRecord { - identity: crate::EventIdentity { - timestamp_unix_ms: Some(0), - ..Default::default() - }, - seq: 0, - source: "test".into(), - kind: "note".into(), - timestamp: None, - session_id: Some("payload-session".into()), - agent_id: Some("agent".into()), - parent_uuid: None, - trace_id: None, - call_id: None, - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload: serde_json::json!({"content":"test"}), - }; - let mut row = event_record_to_event_row(&record).unwrap(); - row.session_id = Some("physical-session".into()); - row.agent_id = Some("physical-agent".into()); - let replayed = crate::event_row_to_event_record(&row).unwrap(); - assert_eq!(replayed.session_id.as_deref(), Some("physical-session")); - assert_eq!(replayed.agent_id.as_deref(), Some("physical-agent")); - let preserved: EventRecord = serde_json::from_str(&row.payload_json).unwrap(); - assert_eq!(preserved.session_id.as_deref(), Some("payload-session")); - assert_eq!(preserved.agent_id.as_deref(), Some("agent")); - } - - #[test] - fn physical_seq_column_is_the_producer_storyline_seq() { - let record = EventRecord { - identity: crate::EventIdentity { - event_id: Some("event-storyline-seq".into()), - timestamp_unix_ms: Some(0), - ..Default::default() - }, - seq: 42, - source: "test".into(), - kind: "note".into(), - timestamp: None, - session_id: Some("story".into()), - agent_id: None, - parent_uuid: None, - trace_id: None, - call_id: None, - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload: serde_json::json!({"content":"preserve me"}), - }; - let row = event_record_to_event_row(&record).unwrap(); - assert_eq!(row.seq, 42); - assert_eq!(crate::event_row_to_event_record(&row).unwrap().seq, 42); - } - - #[test] - fn record_batches_split_at_append_chunk_boundary() { - let schema = raw_event_arrow_schema(); - let rows: Vec<_> = (0..APPEND_CHUNK_ROWS + 1) - .map(|i| mk_row(i as i64, "s", &format!("row-{i}"))) - .collect(); - let batches = record_batches_for_rows(schema, &rows).unwrap(); - assert_eq!(batches.len(), 2); - assert_eq!(batches[0].num_rows(), APPEND_CHUNK_ROWS); - assert_eq!(batches[1].num_rows(), 1); - } - - #[test] - fn roundtrip_batch_rows_preserves_payload() { - let schema = raw_event_arrow_schema(); - let rows = vec![mk_row(0, "s", "hello"), mk_row(1, "s", "world")]; - let batch = event_rows_to_batch(schema, &rows).unwrap(); - let back = event_rows_from_batch(&batch).unwrap(); - assert_eq!(back, rows); - } -} - -#[cfg(all(test, feature = "proptest"))] -mod proptests { - use std::sync::Arc; - - use proptest::prelude::*; - use serde_json::json; - - use crate::formats::events::EventIdentity; - - use super::*; - - fn token_strategy() -> impl Strategy { - proptest::string::string_regex("[A-Za-z0-9._-]{1,24}").unwrap() - } - - proptest! { - #[test] - fn event_arrow_batches_roundtrip_all_generated_rows( - rows in proptest::collection::vec( - (0u64..1_000_000, token_strategy(), proptest::string::string_regex("[A-Za-z0-9 .,!?]{0,48}").unwrap()), - 0..12, - ), - ) { - let records = rows.into_iter().map(|(seq, session, text)| { - let mut identity = EventIdentity::default(); - identity.event_id = Some(format!("event-{seq}")); - identity.timestamp_unix_ms = Some(seq + 1_000); - EventRecord { - identity, - seq, - source: "test".into(), - kind: "note".into(), - timestamp: None, - session_id: Some(session), - agent_id: Some("agent".into()), - parent_uuid: None, - trace_id: None, - call_id: None, - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload: json!({"text": text}), - } - }).collect::>(); - let event_rows = records.iter().map(event_record_to_event_row).collect::, _>>().unwrap(); - let batch = event_rows_to_batch(Arc::new(raw_event_arrow_schema().as_ref().clone()), &event_rows).unwrap(); - prop_assert_eq!(event_rows_from_batch(&batch).unwrap(), event_rows); - prop_assert_eq!(event_records_from_batch(&batch).unwrap(), records); - } - } -} diff --git a/crates/persisting-pchronicle/src/store/events/tests.rs b/crates/persisting-pchronicle/src/store/events/tests.rs deleted file mode 100644 index 830994e8f..000000000 --- a/crates/persisting-pchronicle/src/store/events/tests.rs +++ /dev/null @@ -1,937 +0,0 @@ -use super::*; -use crate::EventRecord; -use crate::layout::StoryCoords; -use std::sync::atomic::{AtomicU64, Ordering}; - -const CHUNK_ROWS: usize = 8192; -static NEXT_REMOTE_STORE: AtomicU64 = AtomicU64::new(1); - -fn remote_storage(label: &str) -> String { - format!( - "shared-memory://pchronicle-events-{}-{label}-{}/trajectories", - std::process::id(), - NEXT_REMOTE_STORE.fetch_add(1, Ordering::Relaxed) - ) -} - -fn note(content: &str) -> EventRecord { - EventRecord { - identity: crate::EventIdentity::default(), - seq: 0, - source: "test".into(), - kind: "note".into(), - timestamp: None, - session_id: None, - agent_id: None, - parent_uuid: None, - trace_id: None, - call_id: None, - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload: serde_json::json!({ "content": content }), - } -} - -fn identified_note(event_id: &str, seq: u64, content: &str) -> EventRecord { - EventRecord { - identity: crate::EventIdentity { - event_id: Some(event_id.into()), - ..Default::default() - }, - seq, - source: "test".into(), - kind: "note".into(), - timestamp: None, - session_id: None, - agent_id: None, - parent_uuid: None, - trace_id: None, - call_id: None, - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload: serde_json::json!({ "content": content }), - } -} - -fn run_session(storage: &str, agent: &str, session_id: &str, root: &str) -> StoryCoords { - StoryCoords::new(storage, agent, session_id, Some(root.to_string())) -} - -fn flat_session(storage: &str, agent: &str, session_id: &str) -> StoryCoords { - StoryCoords::new(storage, agent, session_id, None) -} - -fn payload_content(record: &EventRecord) -> String { - record.payload["content"].as_str().unwrap().to_string() -} - -#[test] -fn batch_report_failure_extraction_preserves_the_original_source() { - let mut report = EventAppendBatchReport::default(); - report.outcomes.insert( - "memory://failed-partition".into(), - Err(anyhow::Error::new(std::io::Error::other( - "partition failure sentinel", - ))), - ); - - let (uri, error) = report.take_failure().unwrap(); - - assert_eq!(uri, "memory://failed-partition"); - assert!(error.downcast_ref::().is_some()); - assert!(report.outcomes.is_empty()); -} - -#[tokio::test] -async fn append_creates_lance_dataset() { - let dir = tempfile::tempdir().unwrap(); - let storage = dir.path().join("store"); - std::fs::create_dir_all(&storage).unwrap(); - let storage_s = storage.to_string_lossy().to_string(); - let session = flat_session(&storage_s, "agent", "sess"); - - append_events(&session, &[note("one")]).await.unwrap(); - - let path = raw_event_lance_path(&session).unwrap(); - let (manifest, datasets) = open_visible_snapshot(&path.to_string_lossy()) - .await - .unwrap() - .unwrap(); - assert_eq!(manifest.total_rows(), 1); - assert_eq!(datasets.len(), 1); -} - -#[tokio::test] -async fn canonical_probe_requires_a_valid_nonempty_manifest() { - let dir = tempfile::tempdir().unwrap(); - assert_eq!( - RawEventDataSource::probe_uri( - dir.path() - .join("absent/events.lance") - .to_string_lossy() - .as_ref() - ) - .await - .unwrap(), - None - ); - let suffix_only = dir.path().join("suffix-only").join("events.lance"); - std::fs::create_dir_all(&suffix_only).unwrap(); - - assert_eq!( - RawEventDataSource::probe_uri(suffix_only.to_string_lossy().as_ref()) - .await - .unwrap(), - None - ); - - let storage = dir.path().join("store"); - std::fs::create_dir_all(&storage).unwrap(); - let session = flat_session(storage.to_str().unwrap(), "agent", "sess"); - append_events(&session, &[note("one")]).await.unwrap(); - let uri = raw_event_lance_path(&session) - .unwrap() - .to_string_lossy() - .into_owned(); - - let snapshot = RawEventDataSource::probe_uri(&uri) - .await - .unwrap() - .expect("valid canonical store"); - assert_eq!( - snapshot.source_uri, - std::fs::canonicalize(&uri).unwrap().to_string_lossy() - ); - assert_eq!(snapshot.fact_version, 1); - assert_eq!(snapshot.fact_rows, 1); - assert!(snapshot.layout_revision >= snapshot.fact_version); - - std::fs::write( - dir.path().join("suffix-only/events.lance/_manifest.json"), - b"{broken", - ) - .unwrap(); - let error = RawEventDataSource::probe_uri(suffix_only.to_string_lossy().as_ref()) - .await - .unwrap_err(); - assert!(format!("{error:#}").contains("decode event manifest")); -} - -#[tokio::test] -async fn canonical_timestamp_is_utc_milliseconds_and_preserves_original_text() { - let dir = tempfile::tempdir().unwrap(); - let storage = dir.path().join("store"); - std::fs::create_dir_all(&storage).unwrap(); - let session = flat_session(storage.to_str().unwrap(), "agent", "sess"); - let mut record = note("timestamped"); - record.timestamp = Some("2026-01-02T03:04:05.123456+08:00".into()); - - append_events(&session, std::slice::from_ref(&record)) - .await - .unwrap(); - - let uri = raw_event_lance_path(&session) - .unwrap() - .to_string_lossy() - .into_owned(); - let (_, datasets) = open_visible_snapshot(&uri).await.unwrap().unwrap(); - assert_eq!( - datasets[0] - .schema() - .field(crate::TRAJECTORY_TIMESTAMP_COL) - .unwrap() - .data_type(), - lance::deps::arrow_schema::DataType::Timestamp( - lance::deps::arrow_schema::TimeUnit::Millisecond, - Some("UTC".into()) - ) - ); - let rows = read_all_rows(&uri).await.unwrap(); - assert_eq!(rows[0].timestamp, 1_767_294_245_123); - let restored = replay(&session, 0, None).await.unwrap().records; - assert_eq!(restored[0].timestamp, record.timestamp); -} - -#[tokio::test] -async fn append_rejects_conflicting_canonical_timestamps() { - let dir = tempfile::tempdir().unwrap(); - let storage = dir.path().join("store"); - std::fs::create_dir_all(&storage).unwrap(); - let session = flat_session(storage.to_str().unwrap(), "agent", "sess"); - let mut record = note("timestamp-conflict"); - record.identity.timestamp_unix_ms = Some(1_000); - record.timestamp = Some("1970-01-01T00:00:02Z".into()); - - let error = append_events(&session, &[record]).await.unwrap_err(); - assert!(format!("{error:#}").contains("timestamp conflict")); -} - -#[tokio::test] -async fn bounded_storyline_fallback_rejects_row_and_byte_overflow() { - let dir = tempfile::tempdir().unwrap(); - let storage = dir.path().join("store"); - std::fs::create_dir_all(&storage).unwrap(); - let session = flat_session(storage.to_str().unwrap(), "agent", "sess"); - append_events(&session, &[note("one"), note("two")]) - .await - .unwrap(); - let source = RawEventDataSource::open(raw_event_lance_path(&session).unwrap()) - .await - .unwrap(); - let sessions = std::collections::BTreeSet::from(["sess".to_string()]); - - let row_error = source - .read_records_for_storylines_bounded(&sessions, 1, usize::MAX) - .await - .unwrap_err(); - assert!(row_error.to_string().contains("max_event_fallback_rows")); - - let byte_error = source - .read_records_for_storylines_bounded(&sessions, 10, 1) - .await - .unwrap_err(); - assert!(byte_error.to_string().contains("max_event_fallback_bytes")); -} - -#[tokio::test] -async fn append_then_append_preserves_rows() { - let dir = tempfile::tempdir().unwrap(); - let storage = dir.path().join("store"); - std::fs::create_dir_all(&storage).unwrap(); - let storage_s = storage.to_string_lossy().to_string(); - let session = flat_session(&storage_s, "agent", "sess"); - - append_events(&session, &[note("first")]).await.unwrap(); - append_events(&session, &[note("second")]).await.unwrap(); - - let replay = replay(&session, 0, None).await.unwrap(); - assert_eq!(replay.records.len(), 2); - assert_eq!(payload_content(&replay.records[0]), "first"); - assert_eq!(payload_content(&replay.records[1]), "second"); -} - -#[tokio::test] -async fn append_preserves_conflicting_claims_and_replays_physical_identity() { - let dir = tempfile::tempdir().unwrap(); - let storage = dir.path().join("store"); - std::fs::create_dir_all(&storage).unwrap(); - let session = run_session(storage.to_str().unwrap(), "agent", "child", "run"); - - type IdentityMutation = fn(&mut EventRecord); - let conflicts: [(&str, IdentityMutation); 4] = [ - ("session_id", |record: &mut EventRecord| { - record.session_id = Some("other".into()) - }), - ("storyline_id", |record: &mut EventRecord| { - record.identity.storyline_id = Some("other".into()) - }), - ("run_id", |record: &mut EventRecord| { - record.identity.run_id = Some("other".into()) - }), - ("agent_id", |record: &mut EventRecord| { - record.agent_id = Some("other".into()) - }), - ]; - for (field, mutate) in conflicts { - let mut record = note("conflict"); - mutate(&mut record); - record.payload["conflict_field"] = serde_json::Value::String(field.into()); - append_events(&session, &[record]).await.unwrap(); - } - let replayed = replay(&session, 0, None).await.unwrap().records; - assert_eq!(replayed.len(), conflicts.len()); - assert!(replayed.iter().all(|record| { - record.session_id.as_deref() == Some("child") && record.agent_id.as_deref() == Some("agent") - })); - - let uri = raw_event_lance_path(&session) - .unwrap() - .to_string_lossy() - .into_owned(); - let rows = read_all_rows(&uri).await.unwrap(); - let preserved = rows - .iter() - .map(|row| serde_json::from_str::(&row.payload_json).unwrap()) - .collect::>(); - assert_eq!(preserved[0].session_id.as_deref(), Some("other")); - assert_eq!(preserved[1].identity.storyline_id.as_deref(), Some("other")); - assert_eq!(preserved[2].identity.run_id.as_deref(), Some("other")); - assert_eq!(preserved[3].agent_id.as_deref(), Some("other")); -} - -#[tokio::test] -async fn append_fills_all_storage_identity_fields() { - let dir = tempfile::tempdir().unwrap(); - let storage = dir.path().join("store"); - std::fs::create_dir_all(&storage).unwrap(); - let session = run_session(storage.to_str().unwrap(), "agent", "child", "run"); - - append_events(&session, &[note("canonical")]).await.unwrap(); - let record = replay(&session, 0, None).await.unwrap().records.remove(0); - assert_eq!(record.session_id.as_deref(), Some("child")); - assert_eq!(record.identity.storyline_id.as_deref(), Some("child")); - assert_eq!(record.identity.run_id.as_deref(), Some("run")); - assert_eq!(record.agent_id.as_deref(), Some("agent")); -} - -#[tokio::test] -async fn pinned_source_reads_append_ranges_and_selected_storylines() { - let dir = tempfile::tempdir().unwrap(); - let storage = dir.path().join("store"); - std::fs::create_dir_all(&storage).unwrap(); - let root = run_session(storage.to_str().unwrap(), "agent", "root", "run"); - let child = run_session(storage.to_str().unwrap(), "agent", "child", "run"); - - append_events(&root, &[identified_note("root-0", 0, "root-zero")]) - .await - .unwrap(); - append_events(&child, &[identified_note("child-0", 0, "child-zero")]) - .await - .unwrap(); - append_events(&root, &[identified_note("root-1", 1, "root-one")]) - .await - .unwrap(); - - let source = RawEventDataSource::open(raw_event_lance_path(&root).unwrap()) - .await - .unwrap(); - let suffix = source - .read_records_range_in_append_order(1, 3) - .await - .unwrap(); - assert_eq!( - suffix.iter().map(payload_content).collect::>(), - ["child-zero", "root-one"] - ); - - let selected = source - .read_records_for_storylines(&["root".to_string()].into_iter().collect()) - .await - .unwrap(); - assert_eq!( - selected.iter().map(payload_content).collect::>(), - ["root-zero", "root-one"] - ); -} - -#[tokio::test] -async fn partitioned_append_reports_one_root_failure_without_losing_other_root() { - let dir = tempfile::tempdir().unwrap(); - let invalid_storage = dir.path().join("not-a-directory"); - std::fs::write(&invalid_storage, b"file").unwrap(); - let invalid = flat_session(invalid_storage.to_str().unwrap(), "agent", "bad"); - let valid = flat_session(dir.path().join("valid").to_str().unwrap(), "agent", "good"); - let invalid_uri = raw_event_lance_path(&invalid) - .unwrap() - .to_string_lossy() - .into_owned(); - let valid_uri = raw_event_lance_path(&valid) - .unwrap() - .to_string_lossy() - .into_owned(); - - let report = RawEventLanceAppender::default() - .append_event_batch_partitioned(&[(invalid, note("bad")), (valid.clone(), note("good"))]) - .await - .unwrap(); - assert!(report.outcome_for(&invalid_uri).unwrap().is_err()); - assert!(matches!(report.outcome_for(&valid_uri), Some(Ok(1)))); - assert_eq!(replay(&valid, 0, None).await.unwrap().records.len(), 1); -} - -#[tokio::test] -async fn typed_append_preserves_duplicates_and_storyline_seq() { - let dir = tempfile::tempdir().unwrap(); - let storage = dir.path().join("store"); - std::fs::create_dir_all(&storage).unwrap(); - let session = flat_session(storage.to_str().unwrap(), "agent", "sess"); - let record = identified_note("stable-event", 42, "once"); - - let first = append_events(&session, std::slice::from_ref(&record)) - .await - .unwrap(); - let duplicate = append_events(&session, &[record.clone(), record]) - .await - .unwrap(); - - assert_eq!(first.accepted_records, 1); - assert_eq!(duplicate.accepted_records, 2); - let replay = replay(&session, 0, None).await.unwrap(); - assert_eq!(replay.records.len(), 3); - assert_eq!( - replay.records[0].identity.event_id.as_deref(), - Some("stable-event") - ); - assert_eq!(replay.records[0].seq, 42); -} - -#[tokio::test] -async fn missing_event_id_is_stored_as_null_without_deduplication() { - let dir = tempfile::tempdir().unwrap(); - let storage = dir.path().join("store"); - std::fs::create_dir_all(&storage).unwrap(); - let session = flat_session(storage.to_str().unwrap(), "agent", "sess"); - let mut record = identified_note("temporary", 7, "retry-safe"); - record.identity.event_id = None; - - let first = append_events(&session, std::slice::from_ref(&record)) - .await - .unwrap(); - let retry = append_events(&session, std::slice::from_ref(&record)) - .await - .unwrap(); - assert_eq!(first.accepted_records, 1); - assert_eq!(retry.accepted_records, 1); - - let uri = raw_event_lance_path(&session) - .unwrap() - .to_string_lossy() - .into_owned(); - let (_, datasets) = open_visible_snapshot(&uri).await.unwrap().unwrap(); - assert!( - datasets[0] - .schema() - .field(crate::TRAJECTORY_EVENT_ID_COL) - .is_some() - ); - let rows = read_all_rows(&raw_event_lance_path(&session).unwrap().to_string_lossy()) - .await - .unwrap(); - assert_eq!(rows.len(), 2); - assert_eq!(rows[0].event_id, None); - assert_eq!(rows[1].event_id, None); - let restored = replay(&session, 0, None).await.unwrap().records; - assert!( - restored - .iter() - .all(|record| record.identity.event_id.is_none()) - ); -} - -#[tokio::test] -async fn event_schema_mismatch_is_rejected() { - let dir = tempfile::tempdir().unwrap(); - let storage = dir.path().join("store"); - std::fs::create_dir_all(&storage).unwrap(); - let session = flat_session(storage.to_str().unwrap(), "agent", "sess"); - append_events(&session, &[identified_note("event-1", 0, "first")]) - .await - .unwrap(); - - let uri = raw_event_lance_path(&session) - .unwrap() - .to_string_lossy() - .into_owned(); - let (manifest, _) = open_visible_snapshot(&uri).await.unwrap().unwrap(); - let mut segment = manifest.segments[0].clone(); - let mut dataset = Dataset::open(&raw_event_manifest::segment_uri(&uri, &segment.id)) - .await - .unwrap(); - dataset - .drop_columns(&[crate::TRAJECTORY_EVENT_ID_COL]) - .await - .unwrap(); - segment.version = dataset.version_id(); - raw_event_manifest::publish_segment(&uri, &manifest.active_writer, segment) - .await - .unwrap(); - - let replay_error = replay(&session, 0, None).await.unwrap_err(); - assert!(replay_error.to_string().contains("schema mismatch")); -} - -#[tokio::test] -async fn one_cached_appender_preserves_physical_append_order() { - let storage = remote_storage("cached-appender"); - let session = flat_session(&storage, "agent", "session"); - let mut writer = RawEventLanceAppender::default(); - - writer - .append_event_batch(&[(session.clone(), identified_note("first", 10, "first"))]) - .await - .unwrap(); - writer - .append_event_batch(&[(session.clone(), identified_note("second", 20, "second"))]) - .await - .unwrap(); - writer - .append_event_batch(&[(session.clone(), identified_note("third", 30, "third"))]) - .await - .unwrap(); - let reports = writer.finish(); - assert_eq!(reports.len(), 1); - assert_eq!(reports[0].fragments_removed, 0); - - let uri = raw_event_lance_path(&session) - .unwrap() - .to_string_lossy() - .into_owned(); - let (manifest, datasets) = open_visible_snapshot(&uri).await.unwrap().unwrap(); - assert_eq!(manifest.segments.len(), 1); - assert_eq!(datasets[0].get_fragments().len(), 3); - assert!( - datasets[0] - .load_indices_by_name(SESSION_INDEX_NAME) - .await - .unwrap() - .is_empty() - ); - - let restored = replay(&session, 0, None).await.unwrap(); - assert_eq!( - restored - .records - .iter() - .map(|record| record.seq) - .collect::>(), - [10, 20, 30] - ); -} - -#[tokio::test] -async fn replay_available_follows_committed_pages() { - let dir = tempfile::tempdir().unwrap(); - let storage = dir.path().join("store"); - std::fs::create_dir_all(&storage).unwrap(); - let storage_s = storage.to_string_lossy().to_string(); - let session = flat_session(&storage_s, "agent", "sess"); - - assert!( - replay_available(&session, 0, Some(2)) - .await - .unwrap() - .is_none() - ); - - append_events(&session, &[note("first"), note("second"), note("third")]) - .await - .unwrap(); - let first_page = replay_available(&session, 0, Some(2)) - .await - .unwrap() - .unwrap(); - assert_eq!( - first_page - .records - .iter() - .map(payload_content) - .collect::>(), - ["first", "second"] - ); - - append_events(&session, &[note("fourth")]).await.unwrap(); - let second_page = replay_available(&session, first_page.records.len(), Some(2)) - .await - .unwrap() - .unwrap(); - assert_eq!( - second_page - .records - .iter() - .map(payload_content) - .collect::>(), - ["third", "fourth"] - ); - assert!( - replay_available(&session, 4, Some(2)) - .await - .unwrap() - .unwrap() - .records - .is_empty() - ); -} - -#[tokio::test] -async fn object_store_uri_supports_append_only_replay() { - let storage = remote_storage("round-trip"); - let session = flat_session(&storage, "agent", "remote-session"); - assert!(display_path(&session).unwrap().starts_with(&storage)); - - append_events(&session, &[note("first"), note("second")]) - .await - .unwrap(); - assert_eq!(replay(&session, 0, None).await.unwrap().records.len(), 2); - - append_events(&session, &[note("third")]).await.unwrap(); - let replay = replay(&session, 0, None).await.unwrap(); - assert_eq!(replay.records.len(), 3); - assert_eq!(payload_content(&replay.records[2]), "third"); -} - -#[tokio::test] -async fn object_store_append_failure_preserves_committed_rows() { - let storage = remote_storage("failed-append"); - let session = flat_session(&storage, "agent", "session"); - append_events(&session, &[note("committed")]).await.unwrap(); - - append_events(&session, &[identified_note("invalid", u64::MAX, "invalid")]) - .await - .unwrap_err(); - - let replay = replay(&session, 0, None).await.unwrap(); - assert_eq!(replay.records.len(), 1); - assert_eq!(payload_content(&replay.records[0]), "committed"); -} - -#[tokio::test] -async fn session_partition_replay_isolates_stories_in_shared_run_dataset() { - let dir = tempfile::tempdir().unwrap(); - let storage = dir.path().join("store"); - std::fs::create_dir_all(&storage).unwrap(); - let storage_s = storage.to_string_lossy().to_string(); - let root = "run-20260101"; - - let main = run_session(&storage_s, "agent", root, root); - let sub_a = run_session(&storage_s, "agent", "agent-sub-a", root); - let sub_b = run_session(&storage_s, "agent", "agent-sub-b", root); - - append_events(&main, &[note("main-1"), note("main-2")]) - .await - .unwrap(); - append_events(&sub_a, &[note("sub-a-1")]).await.unwrap(); - append_events(&sub_b, &[note("sub-b-1"), note("sub-b-2")]) - .await - .unwrap(); - - let lance_path = raw_event_lance_path(&main).unwrap(); - assert_eq!( - raw_event_lance_path(&sub_a).unwrap(), - lance_path, - "run-level sessions share one events.lance" - ); - - let main_replay = replay(&main, 0, None).await.unwrap(); - assert_eq!(main_replay.records.len(), 2); - assert!( - main_replay - .records - .iter() - .all(|r| payload_content(r).starts_with("main-")) - ); - - let sub_a_replay = replay(&sub_a, 0, None).await.unwrap(); - assert_eq!(sub_a_replay.records.len(), 1); - assert_eq!(payload_content(&sub_a_replay.records[0]), "sub-a-1"); - - let sub_b_replay = replay(&sub_b, 1, Some(1)).await.unwrap(); - assert_eq!(sub_b_replay.records.len(), 1); - assert_eq!(payload_content(&sub_b_replay.records[0]), "sub-b-2"); -} - -#[tokio::test] -async fn session_partition_stats_and_exists_respect_session_id() { - let dir = tempfile::tempdir().unwrap(); - let storage = dir.path().join("store"); - std::fs::create_dir_all(&storage).unwrap(); - let storage_s = storage.to_string_lossy().to_string(); - let root = "run-partition"; - - let main = run_session(&storage_s, "agent", root, root); - let sub = run_session(&storage_s, "agent", "agent-worker", root); - let empty = run_session(&storage_s, "agent", "agent-never-written", root); - - append_events(&main, &[note("main")]).await.unwrap(); - append_events(&sub, &[note("sub-1"), note("sub-2")]) - .await - .unwrap(); - - assert!(exists(&main).await.unwrap()); - assert!(exists(&sub).await.unwrap()); - assert!(!exists(&empty).await.unwrap()); - - let main_stats = stats(&main).await.unwrap(); - assert_eq!(main_stats.row_count, 1); - - let sub_stats = stats(&sub).await.unwrap(); - assert_eq!(sub_stats.row_count, 2); -} - -#[tokio::test] -async fn append_keeps_producer_seq_independent_across_partitions() { - let dir = tempfile::tempdir().unwrap(); - let storage = dir.path().join("store"); - std::fs::create_dir_all(&storage).unwrap(); - let storage_s = storage.to_string_lossy().to_string(); - let root = "run-global-seq"; - - let main = run_session(&storage_s, "agent", root, root); - let sub = run_session(&storage_s, "agent", "agent-sub", root); - - append_events(&main, &[note("m1"), note("m2")]) - .await - .unwrap(); - append_events(&sub, &[note("s1")]).await.unwrap(); - - let rows = read_all_rows(&raw_event_lance_path(&main).unwrap().to_string_lossy()) - .await - .unwrap(); - assert_eq!(rows.len(), 3); - assert_eq!( - rows.iter().map(|r| r.seq).collect::>(), - vec![0, 0, 0] - ); -} - -#[tokio::test] -async fn routed_micro_batch_commits_multiple_stories_in_one_fragment() { - let dir = tempfile::tempdir().unwrap(); - let storage = dir.path().join("store"); - std::fs::create_dir_all(&storage).unwrap(); - let storage_s = storage.to_string_lossy().to_string(); - let main = run_session(&storage_s, "agent", "run", "run"); - let sub = run_session(&storage_s, "agent", "sub", "run"); - let records = [note("main"), note("sub")]; - RawEventLanceAppender::default() - .append_event_batch(&[ - (main.clone(), records[0].clone()), - (sub.clone(), records[1].clone()), - ]) - .await - .unwrap(); - - assert_eq!(replay(&main, 0, None).await.unwrap().records.len(), 1); - assert_eq!(replay(&sub, 0, None).await.unwrap().records.len(), 1); - let uri = raw_event_lance_path(&main) - .unwrap() - .to_string_lossy() - .into_owned(); - assert_eq!(visible_fragment_count(&uri).await.unwrap(), 1); -} - -#[tokio::test] -async fn routed_batch_creates_one_fragment_per_run_not_per_record() { - const RUNS: usize = 3; - const STORIES_PER_RUN: usize = 4; - const ROWS_PER_STORY: usize = 64; - - let dir = tempfile::tempdir().unwrap(); - let storage = dir.path().join("store"); - std::fs::create_dir_all(&storage).unwrap(); - let storage_s = storage.to_string_lossy().to_string(); - let mut entries = Vec::with_capacity(RUNS * STORIES_PER_RUN * ROWS_PER_STORY); - let mut run_roots = Vec::with_capacity(RUNS); - - for run_index in 0..RUNS { - let root = format!("run-{run_index}"); - let root_session = run_session(&storage_s, "agent", &root, &root); - run_roots.push(root_session); - for story_index in 0..STORIES_PER_RUN { - let session = run_session( - &storage_s, - "agent", - &format!("run-{run_index}-story-{story_index}"), - &root, - ); - for row_index in 0..ROWS_PER_STORY { - entries.push(( - session.clone(), - identified_note( - &format!("event-{run_index}-{story_index}-{row_index}"), - row_index as u64, - &format!("row-{row_index}"), - ), - )); - } - } - } - - let outcome = RawEventLanceAppender::default() - .append_event_batch(&entries) - .await - .unwrap(); - assert_eq!(outcome.accepted_records, entries.len()); - - for root in run_roots { - let uri = raw_event_lance_path(&root) - .unwrap() - .to_string_lossy() - .into_owned(); - let (manifest, datasets) = open_visible_snapshot(&uri).await.unwrap().unwrap(); - assert_eq!(manifest.segments.len(), 1); - assert_eq!( - datasets[0].get_fragments().len(), - 1, - "one routed call should create one fragment for each run dataset" - ); - assert_eq!( - datasets[0].count_rows(None).await.unwrap(), - STORIES_PER_RUN * ROWS_PER_STORY - ); - } -} - -#[tokio::test] -async fn raw_event_maintenance_rejects_storyline_budgets_before_writing() { - let dir = tempfile::tempdir().unwrap(); - let session = flat_session(dir.path().to_str().unwrap(), "agent", "run"); - let options = LanceMaintenanceOptions { - max_compaction_source_rows: Some(100), - ..Default::default() - }; - let error = maintain(&session, &options).await.unwrap_err(); - assert!(error.to_string().contains("only by Storyline maintenance")); - assert!(!raw_event_lance_path(&session).unwrap().exists()); -} - -#[tokio::test] -async fn explicit_maintenance_compacts_fragments_and_builds_session_index() { - let dir = tempfile::tempdir().unwrap(); - let storage = dir.path().join("store"); - std::fs::create_dir_all(&storage).unwrap(); - let storage_s = storage.to_string_lossy().to_string(); - let session = flat_session(&storage_s, "agent", "run"); - for index in 0..4 { - append_events(&session, &[note(&format!("event-{index}"))]) - .await - .unwrap(); - } - let report = maintain( - &session, - &LanceMaintenanceOptions { - vacuum_older_than: Some(Duration::ZERO), - ..Default::default() - }, - ) - .await - .unwrap(); - assert!(report.fragments_removed >= 4); - let uri = raw_event_lance_path(&session) - .unwrap() - .to_string_lossy() - .into_owned(); - let (manifest, datasets) = open_visible_snapshot(&uri).await.unwrap().unwrap(); - assert_eq!(manifest.segments.len(), 1); - assert_eq!(datasets[0].get_fragments().len(), 1); - assert!( - !datasets[0] - .load_indices_by_name(SESSION_INDEX_NAME) - .await - .unwrap() - .is_empty() - ); - assert!(report.old_versions_removed >= 4); - let segment_directories = - std::fs::read_dir(raw_event_lance_path(&session).unwrap().join("segments")) - .unwrap() - .filter_map(std::result::Result::ok) - .filter(|entry| entry.path().is_dir()) - .count(); - assert_eq!(segment_directories, 1); - assert_eq!(replay(&session, 0, None).await.unwrap().records.len(), 4); -} - -#[tokio::test] -async fn large_append_produces_valid_lance_dataset() { - let dir = tempfile::tempdir().unwrap(); - let storage = dir.path().join("store"); - std::fs::create_dir_all(&storage).unwrap(); - let storage_s = storage.to_string_lossy().to_string(); - let session = flat_session(&storage_s, "agent", "bulk"); - - let records: Vec = (0..CHUNK_ROWS + 50) - .map(|i| note(&format!("row-{i}"))) - .collect(); - let outcome = append_events(&session, &records).await.unwrap(); - assert_eq!(outcome.persisted_units, records.len()); - - let st = stats(&session).await.unwrap(); - assert_eq!(st.row_count, records.len()); - - let replay = replay(&session, CHUNK_ROWS, Some(10)).await.unwrap(); - assert_eq!(replay.records.len(), 10); - assert_eq!(payload_content(&replay.records[0]), "row-8192"); -} - -#[cfg(feature = "proptest")] -mod proptests { - use proptest::prelude::*; - - use super::*; - - proptest! { - #[test] - fn note_payload_content_roundtrips_arbitrary_text( - content in proptest::string::string_regex("[A-Za-z0-9 .,!?_:/-]{0,128}").unwrap(), - ) { - let record = note(&content); - prop_assert_eq!(payload_content(&record), content); - } - - #[test] - fn routed_session_coordinates_preserve_root_and_child_identity( - root in proptest::string::string_regex("[A-Za-z0-9_-]{1,24}").unwrap(), - child in proptest::string::string_regex("[A-Za-z0-9_-]{1,24}").unwrap(), - ) { - let coords = run_session("store", "agent", &child, &root); - prop_assert_eq!(&coords.session_id, &child); - prop_assert_eq!(coords.root_session_id.as_deref(), Some(root.as_str())); - } - - #[test] - fn session_predicates_escape_arbitrary_quotes_without_changing_the_value( - session_id in proptest::string::string_regex("[A-Za-z0-9_'-]{0,48}").unwrap(), - ) { - let predicate = session_predicate(&session_id); - prop_assert_eq!( - predicate, - format!("session_id = '{}'", session_id.replace('\'', "''")), - ); - } - - #[test] - fn session_predicates_keep_backslashes_and_quotes_literal( - session_id in proptest::string::string_regex("[A-Za-z0-9_'\\\\/-]{0,48}").unwrap(), - ) { - let predicate = session_predicate(&session_id); - prop_assert_eq!( - predicate, - format!("session_id = '{}'", session_id.replace('\'', "''")), - ); - } - } -} diff --git a/crates/persisting-pchronicle/src/store/inspect.rs b/crates/persisting-pchronicle/src/store/inspect.rs index 453a9e809..9f38a2770 100644 --- a/crates/persisting-pchronicle/src/store/inspect.rs +++ b/crates/persisting-pchronicle/src/store/inspect.rs @@ -15,8 +15,7 @@ use lance::deps::arrow_array::{ use lance::deps::arrow_schema::DataType; use serde::Serialize; -use super::catalog::{DatasetCatalogSnapshot, PhysicalOpenTarget}; -use super::events::inspect_visible_event_tables; +use super::catalog::DatasetCatalogSnapshot; use super::storyline::StorylineTablePaths; use super::{CatalogSourceStatus, DiscoveredSource}; use crate::format::DocumentFormat; @@ -181,10 +180,10 @@ pub async fn inspect_physical_layout( file: &str, ) -> Result { let format = require_lance_source(snapshot, dataset, file)?; - let tables = match snapshot.physical_open_target(dataset, file)? { - PhysicalOpenTarget::Events { uri } => inspect_event_tables(&uri).await?, - PhysicalOpenTarget::Storyline { paths } => inspect_storyline_tables(&paths).await?, - }; + let paths = snapshot + .storyline_table_paths(dataset, file)? + .with_context(|| format!("physical source is not a Lance dataset: {dataset}/{file}"))?; + let tables = inspect_storyline_tables(&paths).await?; Ok(PhysicalLayout { dataset: dataset.to_string(), file: file.to_string(), @@ -312,9 +311,7 @@ pub async fn inspect_physical_page( fn lance_format(source: &DiscoveredSource) -> Option<&str> { let format = source.format.as_deref()?; - (format == DocumentFormat::CanonicalEvent.as_str() - || format == DocumentFormat::StorylineLance.as_str()) - .then_some(format) + (format == DocumentFormat::StorylineLance.as_str()).then_some(format) } fn require_lance_source<'a>( @@ -349,27 +346,12 @@ async fn table_uri( file: &str, table: &str, ) -> Result { - match snapshot.physical_open_target(dataset, file)? { - PhysicalOpenTarget::Events { uri } => { - let tables = inspect_visible_event_tables(&uri).await?; - tables - .into_iter() - .find(|(name, _)| name == table) - .map(|(_, dataset)| dataset.uri().to_string()) - .with_context(|| format!("physical table not found: {table}")) - } - PhysicalOpenTarget::Storyline { paths } => storyline_table_path(&paths, table) - .map(|path| path.to_string_lossy().into_owned()) - .with_context(|| format!("physical table not found: {table}")), - } -} - -async fn inspect_event_tables(uri: &str) -> Result> { - let mut tables = Vec::new(); - for (name, dataset) in inspect_visible_event_tables(uri).await? { - tables.push(physical_table(&name, dataset).await?); - } - Ok(tables) + let paths = snapshot + .storyline_table_paths(dataset, file)? + .with_context(|| format!("physical source is not a Lance dataset: {dataset}/{file}"))?; + storyline_table_path(&paths, table) + .map(|path| path.to_string_lossy().into_owned()) + .with_context(|| format!("physical table not found: {table}")) } async fn inspect_storyline_tables(paths: &StorylineTablePaths) -> Result> { diff --git a/crates/persisting-pchronicle/src/store/mod.rs b/crates/persisting-pchronicle/src/store/mod.rs index 6e7c77d1e..e252fbacb 100644 --- a/crates/persisting-pchronicle/src/store/mod.rs +++ b/crates/persisting-pchronicle/src/store/mod.rs @@ -1,38 +1,21 @@ //! pChronicle storage, separated by logical data model. //! -//! - `events`: canonical append/replay storage for `EventRecord`. -//! - `storyline`: normalized three-table projection for `StorylineDocument`. +//! - `storyline`: normalized three-table storage for `StorylineDocument`. //! - `search`: document retrieval storage lives outside this module. -#[cfg(feature = "lance-store")] -use anyhow::Context as _; - #[cfg(feature = "lance-store")] mod agenticmd_datafusion; #[cfg(feature = "lance-store")] -mod attempt_registry; -#[cfg(feature = "lance-store")] pub(crate) mod blockcache; #[cfg(feature = "lance-store")] -mod cas_store; -#[cfg(feature = "lance-store")] mod catalog; #[cfg(feature = "lance-store")] -#[cfg(feature = "lance-store")] mod compact_jsonl; #[cfg(feature = "lance-store")] mod datafusion_bridge; #[cfg(feature = "lance-store")] -pub(crate) mod dataset_write_lock; -#[cfg(feature = "lance-store")] mod document_source; #[cfg(feature = "lance-store")] -mod egress; -#[cfg(feature = "lance-store")] -mod event_row; -#[cfg(feature = "lance-store")] -mod events; -#[cfg(feature = "lance-store")] mod files; #[cfg(feature = "lance-store")] pub(crate) mod index_build_gate; @@ -43,7 +26,6 @@ mod inspect; #[cfg(feature = "lance-store")] mod local_query_manifest; #[cfg(feature = "lance-store")] -#[cfg(feature = "lance-store")] pub(crate) mod object_store_io_gate; #[cfg(feature = "lance-store")] pub(crate) mod opendal_store; @@ -56,8 +38,6 @@ pub use persistent_cache::PersistentCache; #[cfg(feature = "lance-store")] mod root_write_lock; #[cfg(feature = "lance-store")] -mod run_control; -#[cfg(feature = "lance-store")] mod storyline; #[cfg(feature = "lance-store")] #[path = "storyline/model.rs"] @@ -68,10 +48,6 @@ mod virtual_document; #[cfg(feature = "lance-store")] pub(crate) use agenticmd_datafusion::AgenticMdDataSource; #[cfg(feature = "lance-store")] -pub use attempt_registry::{AttemptRecord, AttemptRecordState, AttemptRegistry}; -#[cfg(feature = "lance-store")] -pub use cas_store::unix_now_ms as attempt_registry_now_ms; -#[cfg(feature = "lance-store")] pub use catalog::location::{ DatasetLocation, DatasetLocationKind, ImportableObjectEvent, PathListEntry, PathListKind, ShallowNavEntry, @@ -87,11 +63,9 @@ pub use catalog::manifest::{ #[cfg(feature = "lance-store")] pub use catalog::{ CATALOG_SOURCES_TABLE, CATALOG_TRAJECTORIES_TABLE, CachedDataset, CatalogConsistency, - CatalogDataset, CatalogErrorPolicy, CatalogEventProvenance, CatalogEventView, CatalogNamespace, - CatalogPage, CatalogProjectionStatus, CatalogSnapshotOptions, CatalogSourceDescription, - CatalogSourceKind, CatalogSourceRevision, CatalogSourceStatus, CatalogState, CatalogStatus, - CatalogStorylineKey, CatalogTrajectoryBundle, DEFAULT_DATASET_NAME, - DEFAULT_MAX_EVENT_FALLBACK_BYTES, DEFAULT_MAX_EVENT_FALLBACK_ROWS, Dataset, + CatalogDataset, CatalogErrorPolicy, CatalogNamespace, CatalogPage, CatalogSnapshotOptions, + CatalogSourceDescription, CatalogSourceKind, CatalogSourceRevision, CatalogSourceStatus, + CatalogState, CatalogStatus, CatalogStorylineKey, DEFAULT_DATASET_NAME, Dataset, DatasetCatalogSnapshot, DatasetMount, DatasetResolver, DiscoveredSource, LocationSummary, ManifestCache, ManifestListing, ManifestReadMode, ManifestRefreshReport, NamespacePath, QueryScope, ResolveMode, ResolveTarget, @@ -104,19 +78,6 @@ pub use compact_jsonl::{ #[cfg(feature = "lance-store")] pub(crate) use document_source::{DocumentSourceImpl, open_document_source}; #[cfg(feature = "lance-store")] -pub use egress::{ExportOutcome, export_source_dirs, export_story_bundle}; -#[cfg(feature = "lance-store")] -pub use event_row::{EventRow, event_record_to_event_row, event_row_to_event_record}; -#[cfg(feature = "lance-store")] -pub use events::{ - DATAFUSION_EVENTS_TABLE, EventFactSnapshot, EventLogLayoutStats, EventWriterFence, - LanceMaintenanceOptions, LanceMaintenanceReport, RawEventDataSource, RawEventLanceAppender, - distinct_session_ids_in_run, event_rows_from_batch, maintain as maintain_raw_events, - raw_event_arrow_schema, -}; -#[cfg(feature = "lance-store")] -pub(crate) use events::{SealedEventSegment, compact_sealed_event_segment}; -#[cfg(feature = "lance-store")] pub(crate) use files::{ AtifReader, FileTrajectoryDataSource, FileTrajectoryDataSourceOptions, FileTrajectoryQueryMetrics, @@ -143,20 +104,16 @@ pub use query_engine::{ QUERY_MEMORY_LIMIT_ENV, QueryBackendInfo, QuerySnapshot, QueryWriteOutcome, }; #[cfg(feature = "lance-store")] -pub use run_control::{CommitRunOutcome, LeaseAcquireOutcome, RunControlStore}; -#[cfg(feature = "lance-store")] -pub(crate) use storyline::StorylineProjectionPublicationOutcome; -#[cfg(feature = "lance-store")] pub use storyline::{ DATAFUSION_RUNS_TABLE, DATAFUSION_STEPS_TABLE, DATAFUSION_TOOL_CALLS_TABLE, DEFAULT_CONTENT_OFFLOAD_THRESHOLD, DEFAULT_CONTENT_PREVIEW_BYTES, DEFAULT_MAX_CHUNK_BYTES, - ProjectionSourceSnapshot, StorylineContentOptions, StorylineContentReadMode, - StorylineDataFusionTableNames, StorylineDataSource, StorylineDataSourceOptions, - StorylineLanceStore, StorylineMaintenanceReport, StorylineProjectionLineage, - StorylineSearchIndexSuppressGuard, StorylineStreamImportReport, StorylineStreamOptions, - StorylineTableKind, StorylineTablePaths, story_runs_arrow_schema, story_runs_from_batch, - story_runs_to_batch, story_steps_arrow_schema, story_steps_from_batch, story_steps_to_batch, - story_tool_calls_arrow_schema, story_tool_calls_from_batch, story_tool_calls_to_batch, + StorylineContentOptions, StorylineContentReadMode, StorylineDataFusionTableNames, + StorylineDataSource, StorylineDataSourceOptions, StorylineLanceStore, + StorylineMaintenanceReport, StorylineSearchIndexSuppressGuard, StorylineStreamImportReport, + StorylineStreamOptions, StorylineTableKind, StorylineTablePaths, story_runs_arrow_schema, + story_runs_from_batch, story_runs_to_batch, story_steps_arrow_schema, story_steps_from_batch, + story_steps_to_batch, story_tool_calls_arrow_schema, story_tool_calls_from_batch, + story_tool_calls_to_batch, }; #[cfg(feature = "lance-store")] pub use storyline_model::{ @@ -165,224 +122,47 @@ pub use storyline_model::{ }; #[cfg(feature = "lance-store")] -use std::path::PathBuf; - -#[cfg(feature = "lance-store")] -use crate::formats::EventRecord; -#[cfg(feature = "lance-store")] -use crate::layout::{StoryCoords, story_lance_event_path}; - -/// Producer-defined Storyline sequence. Physical replay order is the immutable -/// Lance append order and does not require a read-before-write counter. -#[cfg(feature = "lance-store")] -pub const TRAJECTORY_SEQ_COL: &str = "seq"; -#[cfg(feature = "lance-store")] -pub const TRAJECTORY_EVENT_ID_COL: &str = "event_id"; -#[cfg(feature = "lance-store")] -pub const TRAJECTORY_TIMESTAMP_COL: &str = "timestamp"; -#[cfg(feature = "lance-store")] -pub const TRAJECTORY_SOURCE_COL: &str = "source"; -#[cfg(feature = "lance-store")] -pub const TRAJECTORY_KIND_COL: &str = "kind"; -#[cfg(feature = "lance-store")] -pub const TRAJECTORY_SESSION_ID_COL: &str = "session_id"; -#[cfg(feature = "lance-store")] -pub const TRAJECTORY_AGENT_ID_COL: &str = "agent_id"; -#[cfg(feature = "lance-store")] -pub const TRAJECTORY_CALL_ID_COL: &str = "call_id"; -#[cfg(feature = "lance-store")] -pub const TRAJECTORY_PARENT_CALL_ID_COL: &str = "parent_call_id"; -#[cfg(feature = "lance-store")] -pub const TRAJECTORY_MODEL_COL: &str = "model"; -#[cfg(feature = "lance-store")] -pub const TRAJECTORY_TRACE_ID_COL: &str = "trace_id"; -#[cfg(feature = "lance-store")] -pub const TRAJECTORY_PAYLOAD_JSON_COL: &str = "payload_json"; - -/// Canonical physical schema for the Lance event log. -#[cfg(feature = "lance-store")] -pub const TRAJECTORY_COLS: &[&str] = &[ - TRAJECTORY_SEQ_COL, - TRAJECTORY_EVENT_ID_COL, - TRAJECTORY_TIMESTAMP_COL, - TRAJECTORY_KIND_COL, - TRAJECTORY_SOURCE_COL, - TRAJECTORY_AGENT_ID_COL, - TRAJECTORY_SESSION_ID_COL, - TRAJECTORY_CALL_ID_COL, - TRAJECTORY_TRACE_ID_COL, - TRAJECTORY_PARENT_CALL_ID_COL, - TRAJECTORY_MODEL_COL, - TRAJECTORY_PAYLOAD_JSON_COL, -]; - -#[cfg(feature = "lance-store")] -fn canonicalize_event( - session: &StoryCoords, - mut record: EventRecord, -) -> anyhow::Result { - record.validate().map_err(anyhow::Error::from)?; - let run_id = session - .root_session_id - .as_deref() - .unwrap_or(&session.session_id); - fill_missing_identity(&mut record.identity.run_id, run_id); - fill_missing_identity(&mut record.identity.storyline_id, &session.session_id); - fill_missing_identity(&mut record.session_id, &session.session_id); - fill_missing_identity(&mut record.agent_id, &session.agent_id); - record - .identity - .producer - .get_or_insert_with(|| record.source.clone()); - // `event_id` is optional opaque producer/business data. pChronicle neither - // generates nor checks it and accepts duplicate IDs as appended facts. - let textual_timestamp_ms = record - .timestamp - .as_deref() - .map(|timestamp| { - u64::try_from( - chrono::DateTime::parse_from_rfc3339(timestamp) - .with_context(|| format!("parse event timestamp '{timestamp}' as RFC3339"))? - .timestamp_millis(), - ) - .context("event timestamp predates Unix epoch") - }) - .transpose()?; - match (record.identity.timestamp_unix_ms, textual_timestamp_ms) { - (Some(canonical), Some(textual)) => anyhow::ensure!( - canonical == textual, - "event timestamp conflict: timestamp_unix_ms={canonical}, RFC3339 timestamp={textual}" - ), - (None, textual) => { - record.identity.timestamp_unix_ms = - Some(textual.unwrap_or_else(attempt_registry_now_ms)); - } - (Some(_), None) => {} - } - Ok(record) -} - -#[cfg(feature = "lance-store")] -fn fill_missing_identity(actual: &mut Option, fallback: &str) { - if actual.is_none() { - *actual = Some(fallback.to_string()); - } -} - -#[cfg(feature = "lance-store")] -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct AppendOutcome { - pub accepted_records: usize, - pub persisted_units: usize, - pub note: String, -} - +use std::time::Duration; #[cfg(feature = "lance-store")] #[derive(Debug, Clone, PartialEq)] -pub struct ReplayOutcome { - pub records: Vec, - pub note: String, +pub struct LanceMaintenanceOptions { + pub compact: bool, + pub optimize_indices: bool, + pub vacuum_older_than: Option, + pub target_rows_per_fragment: usize, + /// Maximum live source rows compacted per Storyline data table + /// (runs/steps/tool_calls) per maintenance call. None leaves compaction + /// unlimited. An undersized budget can prevent progress; raise it if no + /// compaction task fits. + pub max_compaction_source_rows: Option, + /// Maximum source data/overlay file bytes per Storyline table per call. + /// Excludes separate Blob v2 payloads and does not bound total memory or + /// index/GC work. An undersized budget can leave no eligible task; raise it + /// if compaction makes no progress. Requires recorded source file sizes. + /// None leaves compaction unlimited. + pub max_compaction_source_bytes: Option, } #[cfg(feature = "lance-store")] -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct TrajectoryStats { - pub dataset: String, - pub row_count: usize, - pub manifest_revision: Option, - pub status: String, - pub note: String, -} - -#[cfg(feature = "lance-store")] -#[derive(Debug, Clone, Copy, Default)] -pub struct RawEventLanceStore; - -#[cfg(feature = "lance-store")] -impl RawEventLanceStore { - pub fn display_path(&self, session: &StoryCoords) -> anyhow::Result { - events::display_path(session) - } - - pub async fn exists(&self, session: &StoryCoords) -> anyhow::Result { - events::exists(session).await - } - - pub async fn append_events( - &self, - session: &StoryCoords, - records: &[EventRecord], - ) -> anyhow::Result { - events::append_events(session, records).await - } - - pub async fn replay( - &self, - session: &StoryCoords, - offset: usize, - limit: Option, - ) -> anyhow::Result { - events::replay(session, offset, limit).await - } - - pub async fn read_events( - &self, - session: &StoryCoords, - offset: usize, - limit: Option, - ) -> anyhow::Result> { - Ok(self.replay(session, offset, limit).await?.records) - } - - pub async fn stats(&self, session: &StoryCoords) -> anyhow::Result { - events::stats(session).await - } - - /// Append a channel-sized batch while preserving the Storyline identity of - /// each event. Entries sharing a run-level `events.lance` dataset are - /// committed together. - pub async fn append_event_batch( - &self, - entries: &[(StoryCoords, EventRecord)], - ) -> anyhow::Result { - RawEventLanceAppender::default() - .append_event_batch(entries) - .await - } - - pub async fn maintain( - &self, - session: &StoryCoords, - options: &LanceMaintenanceOptions, - ) -> anyhow::Result { - events::maintain(session, options).await - } - - pub async fn layout_stats(&self, session: &StoryCoords) -> anyhow::Result { - events::layout_stats(session).await - } - - /// Read the latest committed page for an append-only follow loop. - /// - /// `None` means the run-level dataset has not been created yet. Once it - /// exists, an empty `records` page means there are currently no rows after - /// `offset` for this Storyline. - pub async fn replay_available( - &self, - session: &StoryCoords, - offset: usize, - limit: Option, - ) -> anyhow::Result> { - events::replay_available(session, offset, limit).await +impl Default for LanceMaintenanceOptions { + fn default() -> Self { + Self { + compact: true, + optimize_indices: true, + vacuum_older_than: Some(Duration::from_secs(7 * 24 * 60 * 60)), + target_rows_per_fragment: 1024 * 1024, + max_compaction_source_rows: None, + max_compaction_source_bytes: None, + } } } #[cfg(feature = "lance-store")] -pub fn raw_event_lance_path(session: &StoryCoords) -> anyhow::Result { - story_lance_event_path( - &session.storage, - &session.agent_id, - &session.session_id, - session.root_session_id.as_deref(), - ) +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct LanceMaintenanceReport { + pub fragments_removed: usize, + pub fragments_added: usize, + pub old_versions_removed: u64, + pub bytes_removed: u64, + pub final_version: Option, } diff --git a/crates/persisting-pchronicle/src/store/opendal_store.rs b/crates/persisting-pchronicle/src/store/opendal_store.rs index a351521a1..7910daf15 100644 --- a/crates/persisting-pchronicle/src/store/opendal_store.rs +++ b/crates/persisting-pchronicle/src/store/opendal_store.rs @@ -507,10 +507,6 @@ impl Store { .await .map(|_| ()) } - - pub(crate) fn fallback_lock(&self) -> Option>> { - self.fallback_lock.clone() - } } fn operator_cache_key(uri: &str, normalized: &str) -> String { diff --git a/crates/persisting-pchronicle/src/store/query_engine.rs b/crates/persisting-pchronicle/src/store/query_engine.rs index 678e88ac7..d9a034f9b 100644 --- a/crates/persisting-pchronicle/src/store/query_engine.rs +++ b/crates/persisting-pchronicle/src/store/query_engine.rs @@ -34,15 +34,7 @@ pub struct QueryBackendInfo { #[derive(Debug, Clone, PartialEq, Eq)] pub enum QuerySnapshot { - CanonicalEvent { - format_version: u32, - fact_version: u64, - fact_rows: u64, - layout_revision: u64, - }, - Storyline { - generation: String, - }, + Storyline { generation: String }, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -168,22 +160,13 @@ impl ChronicleQueryEngine { source.inner.register_virtual_tables(&context).await?; let capabilities = source.capabilities(); let source_count = source.inner.source_count(); - let snapshot = if let Some(snapshot) = source.inner.event_snapshot() { - Some(QuerySnapshot::CanonicalEvent { - // Existing canonical manifests predate explicit format versioning. - format_version: 1, - fact_version: snapshot.fact_version, - fact_rows: snapshot.fact_rows, - layout_revision: snapshot.layout_revision, - }) - } else { + let snapshot = source .inner .storyline_generation() .map(|generation| QuerySnapshot::Storyline { generation: generation.to_string(), - }) - }; + }); let local_file_metrics = source.inner.file_metrics().into_iter().collect(); Ok(Self { context, diff --git a/crates/persisting-pchronicle/src/store/run_control.rs b/crates/persisting-pchronicle/src/store/run_control.rs deleted file mode 100644 index b01317318..000000000 --- a/crates/persisting-pchronicle/src/store/run_control.rs +++ /dev/null @@ -1,701 +0,0 @@ -//! CAS-managed Run leases and terminal commits. -//! -//! Lease acquisition and RunCommit share one per-Run control object. This is -//! important: checking a lease in one object and creating a commit in another -//! leaves a race where a newer lease can be issued between the two operations. - -use super::cas_store::{CasStore, Mutation, unix_now_ms}; -use anyhow::{Context, bail}; -use persisting_agentctl::{ - AttemptId, RunCommit, RunCommitRequest, RunControlRecord, RunId, RunLeaseRecord, -}; - -const CONTROL_DIR: &str = "run-control"; - -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum LeaseAcquireOutcome { - Acquired(RunLeaseRecord), - /// Another owner still holds an unexpired lease. - Held(RunLeaseRecord), - AlreadyCommitted(RunCommit), -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum CommitRunOutcome { - Committed(RunCommit), - AlreadyCommitted(RunCommit), - StaleLease { - supplied_epoch: u64, - current_epoch: Option, - }, - Conflict(RunCommit), -} - -/// A lightweight pChronicle control store backed by a local directory or an -/// object-store URI. Local updates use an advisory file lock plus atomic rename; -/// object stores use conditional create/update with ETag/version preconditions. -pub struct RunControlStore { - store: CasStore, -} - -impl std::fmt::Debug for RunControlStore { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter - .debug_struct("RunControlStore") - .field("root_uri", &self.store.root_uri()) - .finish_non_exhaustive() - } -} - -impl RunControlStore { - pub async fn open(root: impl AsRef) -> anyhow::Result { - Ok(Self { - store: CasStore::open(root, CONTROL_DIR, "Run control").await?, - }) - } - - pub fn root_uri(&self) -> &str { - self.store.root_uri() - } - - pub async fn acquire_lease( - &self, - run_id: &RunId, - task_id: Option<&str>, - owner: &str, - ttl_ms: u64, - ) -> anyhow::Result { - self.acquire_lease_inner(run_id, task_id, owner, ttl_ms, false) - .await - } - - /// Replace a lease only after the caller has established that its attempt - /// is absent or stale (for example during coordinator reconciliation). - pub async fn takeover_lease( - &self, - run_id: &RunId, - task_id: Option<&str>, - owner: &str, - ttl_ms: u64, - ) -> anyhow::Result { - self.acquire_lease_inner(run_id, task_id, owner, ttl_ms, true) - .await - } - - async fn acquire_lease_inner( - &self, - run_id: &RunId, - task_id: Option<&str>, - owner: &str, - ttl_ms: u64, - force: bool, - ) -> anyhow::Result { - if owner.trim().is_empty() { - bail!("Run lease owner must not be empty"); - } - let now = unix_now_ms(); - let owned_run_id = run_id.clone(); - let owned_task_id = task_id.map(str::to_owned); - let owned_owner = owner.to_string(); - let mutate = move |current: Option<&RunControlRecord>| -> anyhow::Result> { - if let Some(commit) = current.and_then(|record| record.commit.clone()) { - return Ok(Mutation::Unchanged(LeaseAcquireOutcome::AlreadyCommitted( - commit, - ))); - } - if let Some(existing) = current.and_then(|record| record.lease.as_ref()) { - if existing.owner != owned_owner && existing.expires_at_unix_ms > now && !force { - return Ok(Mutation::Unchanged(LeaseAcquireOutcome::Held( - existing.clone(), - ))); - } - if existing.owner == owned_owner && !force { - let mut renewed = existing.clone(); - renewed.expires_at_unix_ms = now.saturating_add(ttl_ms.max(1)); - let record = next_record(current, owned_run_id.clone(), |record| { - record.lease = Some(renewed.clone()); - })?; - return Ok(Mutation::Replace( - record, - LeaseAcquireOutcome::Acquired(renewed), - )); - } - } - let epoch = current - .and_then(|record| record.lease.as_ref().map(|lease| lease.epoch)) - .unwrap_or(0) - .checked_add(1) - .context("Run lease epoch overflow")?; - let lease = RunLeaseRecord { - run_id: owned_run_id.clone(), - task_id: owned_task_id.clone(), - epoch, - owner: owned_owner.clone(), - issued_at_unix_ms: now, - expires_at_unix_ms: now.saturating_add(ttl_ms.max(1)), - attempt_id: None, - }; - let record = next_record(current, owned_run_id.clone(), |record| { - record.lease = Some(lease.clone()); - })?; - Ok(Mutation::Replace( - record, - LeaseAcquireOutcome::Acquired(lease), - )) - }; - self.mutate(run_id, mutate).await - } - - pub async fn bind_attempt( - &self, - run_id: &RunId, - epoch: u64, - attempt_id: AttemptId, - ) -> anyhow::Result { - let owned_run_id = run_id.clone(); - self.mutate(run_id, move |current| { - let Some(current) = current else { - return Ok(Mutation::Unchanged(false)); - }; - let now = unix_now_ms(); - let Some(lease) = current.lease.as_ref() else { - return Ok(Mutation::Unchanged(false)); - }; - if current.commit.is_some() || lease.epoch != epoch || lease.expires_at_unix_ms <= now { - return Ok(Mutation::Unchanged(false)); - } - if lease.attempt_id.as_ref() == Some(&attempt_id) { - return Ok(Mutation::Unchanged(true)); - } - if lease.attempt_id.is_some() { - return Ok(Mutation::Unchanged(false)); - } - let record = next_record(Some(current), owned_run_id.clone(), |record| { - if let Some(lease) = record.lease.as_mut() { - lease.attempt_id = Some(attempt_id.clone()); - } - })?; - Ok(Mutation::Replace(record, true)) - }) - .await - } - - /// Extend a lease without changing its fencing epoch. Returns `false` if - /// ownership has moved or the Run is already committed. - pub async fn renew_lease( - &self, - run_id: &RunId, - epoch: u64, - owner: &str, - ttl_ms: u64, - ) -> anyhow::Result { - let owned_run_id = run_id.clone(); - let owned_owner = owner.to_string(); - self.mutate(run_id, move |current| { - let Some(current) = current else { - return Ok(Mutation::Unchanged(false)); - }; - let now = unix_now_ms(); - let matches = current.commit.is_none() - && current.lease.as_ref().is_some_and(|lease| { - lease.epoch == epoch - && lease.owner == owned_owner - && lease.expires_at_unix_ms > now - }); - if !matches { - return Ok(Mutation::Unchanged(false)); - } - let expires_at = now.saturating_add(ttl_ms.max(1)); - let record = next_record(Some(current), owned_run_id.clone(), |record| { - if let Some(lease) = record.lease.as_mut() { - lease.expires_at_unix_ms = expires_at; - } - })?; - Ok(Mutation::Replace(record, true)) - }) - .await - } - - pub async fn commit_run(&self, request: RunCommitRequest) -> anyhow::Result { - if !request.state.is_terminal() { - bail!("RunCommit requires a terminal Run state"); - } - if request.lease_epoch == 0 { - bail!("RunCommit requires a non-zero lease epoch"); - } - if request.result_digest.trim().is_empty() { - bail!("RunCommit result_digest must not be empty"); - } - let run_id = request.run_id.clone(); - let closure_run_id = run_id.clone(); - self.mutate(&run_id, move |current| { - if let Some(existing) = current.and_then(|record| record.commit.clone()) { - if existing.request == request { - return Ok(Mutation::Unchanged(CommitRunOutcome::AlreadyCommitted( - existing, - ))); - } - return Ok(Mutation::Unchanged(CommitRunOutcome::Conflict(existing))); - } - let now = unix_now_ms(); - let current_lease = current.and_then(|record| record.lease.as_ref()); - let current_epoch = current_lease.map(|lease| lease.epoch); - let lease_matches = current_lease.is_some_and(|lease| { - lease.epoch == request.lease_epoch - && lease.expires_at_unix_ms > now - && lease - .attempt_id - .as_ref() - .is_none_or(|attempt_id| attempt_id == &request.attempt_id) - }); - if !lease_matches { - return Ok(Mutation::Unchanged(CommitRunOutcome::StaleLease { - supplied_epoch: request.lease_epoch, - current_epoch, - })); - } - let commit = RunCommit { - request: request.clone(), - committed_at_unix_ms: unix_now_ms(), - }; - let record = next_record(current, closure_run_id.clone(), |record| { - if let Some(lease) = record.lease.as_mut() { - lease.attempt_id.get_or_insert(request.attempt_id.clone()); - } - record.commit = Some(commit.clone()); - })?; - Ok(Mutation::Replace( - record, - CommitRunOutcome::Committed(commit), - )) - }) - .await - } - - pub async fn get(&self, run_id: &RunId) -> anyhow::Result> { - self.store.get(run_id.as_str()).await - } - - pub async fn list(&self) -> anyhow::Result> { - let mut records = self.store.list::().await?; - records.sort_by(|left, right| left.run_id.cmp(&right.run_id)); - Ok(records) - } - - async fn mutate(&self, run_id: &RunId, mutate: F) -> anyhow::Result - where - T: Send + 'static, - F: Fn(Option<&RunControlRecord>) -> anyhow::Result> - + Send - + Sync - + 'static, - { - self.store.mutate(run_id.as_str(), mutate).await - } -} - -fn next_record( - current: Option<&RunControlRecord>, - run_id: RunId, - apply: impl FnOnce(&mut RunControlRecord), -) -> anyhow::Result { - let mut record = current.cloned().unwrap_or(RunControlRecord { - revision: 0, - run_id: run_id.clone(), - lease: None, - commit: None, - }); - if record.run_id != run_id { - bail!("invalid Run control record for {run_id}"); - } - record.revision = record - .revision - .checked_add(1) - .context("revision overflow")?; - apply(&mut record); - Ok(record) -} - -#[cfg(test)] -mod tests { - use super::*; - use persisting_agentctl::RunState; - - fn commit(run: &str, attempt: &str, epoch: u64, digest: &str) -> RunCommitRequest { - RunCommitRequest { - run_id: RunId::new(run), - task_id: Some("task-1".into()), - attempt_id: AttemptId::new(attempt), - lease_epoch: epoch, - state: RunState::Completed, - event_high_watermark: Some(9), - result_digest: digest.into(), - } - } - - #[tokio::test] - async fn stale_lease_cannot_win_terminal_commit() { - let dir = tempfile::tempdir().unwrap(); - let store = RunControlStore::open(dir.path().to_str().unwrap()) - .await - .unwrap(); - let run = RunId::new("run-1"); - let first = store - .acquire_lease(&run, Some("task-1"), "worker-a", 60_000) - .await - .unwrap(); - let LeaseAcquireOutcome::Acquired(first) = first else { - panic!() - }; - assert!(matches!( - store - .acquire_lease(&run, Some("task-1"), "worker-b", 60_000) - .await - .unwrap(), - LeaseAcquireOutcome::Held(_) - )); - let second = store - .takeover_lease(&run, Some("task-1"), "worker-b", 60_000) - .await - .unwrap(); - let LeaseAcquireOutcome::Acquired(second) = second else { - panic!() - }; - assert_eq!(second.epoch, first.epoch + 1); - assert!(matches!( - store.commit_run(commit("run-1", "attempt-a", first.epoch, "sha256:a")).await.unwrap(), - CommitRunOutcome::StaleLease { current_epoch: Some(epoch), .. } if epoch == second.epoch - )); - assert!(matches!( - store - .commit_run(commit("run-1", "attempt-b", second.epoch, "sha256:b")) - .await - .unwrap(), - CommitRunOutcome::Committed(_) - )); - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 4)] - async fn concurrent_commits_have_one_visible_winner() { - let dir = tempfile::tempdir().unwrap(); - let root = dir.path().to_str().unwrap().to_string(); - let store = RunControlStore::open(&root).await.unwrap(); - let run = RunId::new("run-race"); - let lease = store - .acquire_lease(&run, Some("task-1"), "worker", 60_000) - .await - .unwrap(); - let LeaseAcquireOutcome::Acquired(lease) = lease else { - panic!() - }; - let mut joins = Vec::new(); - for index in 0..8 { - let root = root.clone(); - let epoch = lease.epoch; - joins.push(tokio::spawn(async move { - let store = RunControlStore::open(root).await.unwrap(); - store - .commit_run(commit( - "run-race", - &format!("attempt-{index}"), - epoch, - &format!("sha256:{index}"), - )) - .await - .unwrap() - })); - } - let mut committed = 0; - for join in joins { - if matches!(join.await.unwrap(), CommitRunOutcome::Committed(_)) { - committed += 1; - } - } - assert_eq!(committed, 1); - assert!(store.get(&run).await.unwrap().unwrap().commit.is_some()); - } - - #[tokio::test] - async fn identical_commit_is_idempotent_but_different_result_conflicts() { - let dir = tempfile::tempdir().unwrap(); - let store = RunControlStore::open(dir.path().to_str().unwrap()) - .await - .unwrap(); - let run = RunId::new("run-idempotent"); - let lease = store - .acquire_lease(&run, None, "worker", 10_000) - .await - .unwrap(); - let LeaseAcquireOutcome::Acquired(lease) = lease else { - panic!() - }; - let request = commit("run-idempotent", "attempt-1", lease.epoch, "sha256:same"); - assert!(matches!( - store.commit_run(request.clone()).await.unwrap(), - CommitRunOutcome::Committed(_) - )); - assert!(matches!( - store.commit_run(request).await.unwrap(), - CommitRunOutcome::AlreadyCommitted(_) - )); - assert!(matches!( - store - .commit_run(commit( - "run-idempotent", - "attempt-2", - lease.epoch, - "sha256:other" - )) - .await - .unwrap(), - CommitRunOutcome::Conflict(_) - )); - assert!(matches!( - store - .acquire_lease(&run, None, "late-worker", 10_000) - .await - .unwrap(), - LeaseAcquireOutcome::AlreadyCommitted(_) - )); - } - - #[tokio::test] - async fn renewal_extends_same_epoch_and_rejects_wrong_owner() { - let dir = tempfile::tempdir().unwrap(); - let store = RunControlStore::open(dir.path().to_str().unwrap()) - .await - .unwrap(); - let run = RunId::new("run-renew"); - let lease = store - .acquire_lease(&run, Some("task-renew"), "owner-a", 1_000) - .await - .unwrap(); - let LeaseAcquireOutcome::Acquired(lease) = lease else { - panic!() - }; - assert!( - !store - .renew_lease(&run, lease.epoch, "owner-b", 2_000) - .await - .unwrap() - ); - assert!( - store - .renew_lease(&run, lease.epoch, "owner-a", 2_000) - .await - .unwrap() - ); - let renewed = store.get(&run).await.unwrap().unwrap().lease.unwrap(); - assert!(renewed.expires_at_unix_ms > lease.expires_at_unix_ms); - assert!(matches!( - store - .acquire_lease(&run, Some("task-renew"), "owner-b", 2_000) - .await - .unwrap(), - LeaseAcquireOutcome::Held(held) if held.epoch == lease.epoch - )); - } - - #[tokio::test] - async fn renewal_cannot_revive_an_expired_lease() { - let dir = tempfile::tempdir().unwrap(); - let store = RunControlStore::open(dir.path().to_str().unwrap()) - .await - .unwrap(); - let run = RunId::new("run-expired-renewal"); - let lease = store - .acquire_lease(&run, Some("task-expired"), "owner", 5) - .await - .unwrap(); - let LeaseAcquireOutcome::Acquired(lease) = lease else { - panic!() - }; - while unix_now_ms() <= lease.expires_at_unix_ms { - tokio::time::sleep(std::time::Duration::from_millis(1)).await; - } - - assert!( - !store - .renew_lease(&run, lease.epoch, "owner", 10_000) - .await - .unwrap() - ); - } - - #[tokio::test] - async fn expired_lease_cannot_commit_terminal_result() { - let dir = tempfile::tempdir().unwrap(); - let store = RunControlStore::open(dir.path().to_str().unwrap()) - .await - .unwrap(); - let run = RunId::new("run-expired-commit"); - let lease = store - .acquire_lease(&run, Some("task-1"), "owner", 5) - .await - .unwrap(); - let LeaseAcquireOutcome::Acquired(lease) = lease else { - panic!() - }; - while unix_now_ms() <= lease.expires_at_unix_ms { - tokio::time::sleep(std::time::Duration::from_millis(1)).await; - } - - assert!(matches!( - store - .commit_run(commit( - "run-expired-commit", - "attempt-expired", - lease.epoch, - "sha256:expired" - )) - .await - .unwrap(), - CommitRunOutcome::StaleLease { - supplied_epoch, - current_epoch: Some(current_epoch) - } if supplied_epoch == lease.epoch && current_epoch == lease.epoch - )); - assert!(store.get(&run).await.unwrap().unwrap().commit.is_none()); - } - - #[tokio::test] - async fn bound_attempt_cannot_be_replaced_within_same_epoch() { - let dir = tempfile::tempdir().unwrap(); - let store = RunControlStore::open(dir.path().to_str().unwrap()) - .await - .unwrap(); - let run = RunId::new("run-attempt-fence"); - let lease = store - .acquire_lease(&run, Some("task-1"), "owner", 10_000) - .await - .unwrap(); - let LeaseAcquireOutcome::Acquired(lease) = lease else { - panic!() - }; - - assert!( - store - .bind_attempt(&run, lease.epoch, AttemptId::new("attempt-owner")) - .await - .unwrap() - ); - assert!( - !store - .bind_attempt(&run, lease.epoch, AttemptId::new("attempt-stale")) - .await - .unwrap() - ); - assert!(matches!( - store - .commit_run(commit( - "run-attempt-fence", - "attempt-stale", - lease.epoch, - "sha256:stale" - )) - .await - .unwrap(), - CommitRunOutcome::StaleLease { .. } - )); - assert!(matches!( - store - .commit_run(commit( - "run-attempt-fence", - "attempt-owner", - lease.epoch, - "sha256:owner" - )) - .await - .unwrap(), - CommitRunOutcome::Committed(_) - )); - } - - #[tokio::test] - async fn object_store_backend_uses_conditional_create_and_update() { - let root = format!( - "shared-memory://run-control-{}-{}/root", - std::process::id(), - unix_now_ms() - ); - let store = RunControlStore::open(root).await.unwrap(); - let run = RunId::new("run-object-cas"); - let lease = store - .acquire_lease(&run, Some("task-object"), "owner", 10_000) - .await - .unwrap(); - let LeaseAcquireOutcome::Acquired(lease) = lease else { - panic!() - }; - assert!( - store - .renew_lease(&run, lease.epoch, "owner", 10_000) - .await - .unwrap() - ); - assert!(matches!( - store - .commit_run(commit( - "run-object-cas", - "attempt-object", - lease.epoch, - "sha256:object" - )) - .await - .unwrap(), - CommitRunOutcome::Committed(_) - )); - assert!(store.get(&run).await.unwrap().unwrap().commit.is_some()); - } -} - -#[cfg(all(test, feature = "proptest"))] -mod proptests { - use proptest::prelude::*; - - use super::*; - - proptest! { - #[test] - fn next_record_creates_or_advances_a_matching_run_control_record( - run in proptest::string::string_regex("[A-Za-z0-9_-]{1,24}").unwrap(), - revision in 0u64..1_000_000, - ) { - let run_id = RunId::new(run.clone()); - let current = RunControlRecord { - revision, - run_id: run_id.clone(), - lease: None, - commit: None, - }; - let next = next_record(Some(¤t), run_id.clone(), |_| {}).expect("matching run advances"); - prop_assert_eq!(next.revision, revision + 1); - prop_assert_eq!(next.run_id, run_id); - - let fresh = next_record(None, RunId::new(run), |_| {}).expect("new run creates record"); - prop_assert_eq!(fresh.revision, 1); - } - - #[test] - fn next_record_rejects_mismatched_or_overflowing_records( - run in proptest::string::string_regex("[A-Za-z0-9_-]{1,24}").unwrap(), - ) { - let current = RunControlRecord { - revision: 1, - run_id: RunId::new("other"), - lease: None, - commit: None, - }; - let mismatched = next_record(Some(¤t), RunId::new(run.clone()), |_| {}); - prop_assert!(mismatched.is_err()); - let overflowing = RunControlRecord { - revision: u64::MAX, - run_id: RunId::new(run.clone()), - lease: None, - commit: None, - }; - let overflow = next_record(Some(&overflowing), RunId::new(run), |_| {}); - prop_assert!(overflow.is_err()); - } - } -} diff --git a/crates/persisting-pchronicle/src/store/storyline/mod.rs b/crates/persisting-pchronicle/src/store/storyline/mod.rs index cb984ede1..7873dd608 100644 --- a/crates/persisting-pchronicle/src/store/storyline/mod.rs +++ b/crates/persisting-pchronicle/src/store/storyline/mod.rs @@ -127,68 +127,6 @@ pub struct StorylineTablePaths { pub steps_version: u64, pub tool_calls_version: u64, pub objects_version: u64, - /// Verified derivation metadata. `None` marks a directly-written store. - pub projection: Option, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(tag = "kind", rename_all = "snake_case")] -pub enum ProjectionSourceSnapshot { - CanonicalEvents { - source_uri: String, - fact_version: u64, - fact_rows: u64, - layout_revision: u64, - }, - Exchange { - source_uri: String, - snapshot_ref: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - content_digest: Option, - }, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct StorylineProjectionLineage { - pub source_id: String, - pub source_file: String, - pub source: ProjectionSourceSnapshot, - pub projector_name: String, - pub recipe_hash: String, - pub completeness: String, -} - -impl StorylineProjectionLineage { - fn validate(&self) -> Result<()> { - for (name, value) in [ - ("source_id", self.source_id.as_str()), - ("source_file", self.source_file.as_str()), - ("projector_name", self.projector_name.as_str()), - ("recipe_hash", self.recipe_hash.as_str()), - ("completeness", self.completeness.as_str()), - ] { - anyhow::ensure!( - !value.trim().is_empty(), - "projection {name} must not be empty" - ); - } - match &self.source { - ProjectionSourceSnapshot::CanonicalEvents { source_uri, .. } - | ProjectionSourceSnapshot::Exchange { source_uri, .. } => { - anyhow::ensure!( - !source_uri.trim().is_empty(), - "projection source_uri must not be empty" - ); - } - } - if let ProjectionSourceSnapshot::Exchange { snapshot_ref, .. } = &self.source { - anyhow::ensure!( - !snapshot_ref.trim().is_empty(), - "projection exchange snapshot_ref must not be empty" - ); - } - Ok(()) - } } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -202,8 +140,6 @@ struct StorylineSnapshotPointer { steps_version: u64, tool_calls_version: u64, objects_version: u64, - #[serde(default, skip_serializing_if = "Option::is_none")] - projection: Option, } #[derive(Debug, Clone)] @@ -251,27 +187,10 @@ pub struct StorylineStreamImportReport { pub tool_calls: usize, } -#[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) enum StorylineProjectionPublicationOutcome { - Published(StorylineStreamImportReport), - OutputNotEmpty, -} - -fn published_storyline_report( - outcome: StorylineProjectionPublicationOutcome, -) -> Result { - match outcome { - StorylineProjectionPublicationOutcome::Published(report) => Ok(report), - StorylineProjectionPublicationOutcome::OutputNotEmpty => { - anyhow::bail!("non-create Storyline publication reported nonempty output") - } - } -} - fn attach_stream_cleanup_failures( - result: Result, + result: Result, cleanup_failures: Vec, -) -> Result { +) -> Result { if cleanup_failures.is_empty() { return result; } @@ -285,26 +204,6 @@ fn attach_stream_cleanup_failures( } } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum StorylineStreamWriteMode { - Replace, - Rebuild, - CreateProjection, -} - -#[cfg(test)] -#[derive(Clone)] -struct CreateAfterEmptyReadBarrierHook { - root_uri: String, - barrier: Arc, - content_arrivals: Arc, - content_created: Arc, -} - -#[cfg(test)] -static CREATE_AFTER_EMPTY_READ_BARRIER: std::sync::Mutex> = - std::sync::Mutex::new(None); - #[cfg(test)] #[derive(Clone)] struct ReplacementAfterCurrentReadBarrierHook { @@ -391,19 +290,6 @@ impl Drop for StorylineSearchIndexSuppressGuard { } } -#[cfg(test)] -async fn wait_after_empty_current_read(root_uri: &str) { - let barrier = CREATE_AFTER_EMPTY_READ_BARRIER - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .as_ref() - .filter(|hook| hook.root_uri == root_uri) - .map(|hook| hook.barrier.clone()); - if let Some(barrier) = barrier { - barrier.wait().await; - } -} - #[cfg(test)] async fn wait_after_replacement_current_read(root_uri: &str) { let barrier = REPLACEMENT_AFTER_CURRENT_READ_BARRIER @@ -431,45 +317,6 @@ async fn wait_after_maintenance_publish(root_uri: &str) { } } -#[cfg(test)] -async fn wait_for_first_content_create(root_uri: &str) -> bool { - let hook = CREATE_AFTER_EMPTY_READ_BARRIER - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .as_ref() - .filter(|hook| hook.root_uri == root_uri) - .cloned(); - let Some(hook) = hook else { - return false; - }; - if hook - .content_arrivals - .fetch_add(1, std::sync::atomic::Ordering::SeqCst) - == 0 - { - true - } else { - hook.content_created.notified().await; - false - } -} - -#[cfg(test)] -fn release_waiting_content_create(root_uri: &str, first: bool) { - if !first { - return; - } - let notify = CREATE_AFTER_EMPTY_READ_BARRIER - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .as_ref() - .filter(|hook| hook.root_uri == root_uri) - .map(|hook| hook.content_created.clone()); - if let Some(notify) = notify { - notify.notify_one(); - } -} - /// Options for streaming Storyline writes. #[derive(Debug, Clone, Copy)] pub struct StorylineStreamOptions { @@ -749,7 +596,6 @@ impl StorylineLanceStore { paths.steps_version = pointer.steps_version; paths.tool_calls_version = pointer.tool_calls_version; paths.objects_version = pointer.objects_version; - paths.projection = pointer.projection; Ok(Some(paths)) } @@ -757,29 +603,6 @@ impl StorylineLanceStore { self.replace_storylines(std::slice::from_ref(story)).await } - /// Publish complete Storyline replacements with verifiable source lineage. - pub async fn replace_projected_storylines( - &self, - stories: &[StorylineDocument], - projection: StorylineProjectionLineage, - ) -> Result<()> { - if stories.is_empty() { - return Ok(()); - } - projection.validate()?; - let outcome = self - .replace_storyline_stream_with_projection( - stories.iter().cloned().map(Ok::<_, anyhow::Error>), - Some(projection), - StorylineStreamWriteMode::Replace, - None, - StorylineStreamOptions::default(), - ) - .await?; - published_storyline_report(outcome)?; - Ok(()) - } - /// Build an empty Storyline store directly from a replayable ATIF input. /// /// One producer normalizes each trajectory once and fans bounded Arrow @@ -809,16 +632,8 @@ impl StorylineLanceStore { where I: IntoIterator>, { - let outcome = self - .replace_storyline_stream_with_projection( - stories, - None, - StorylineStreamWriteMode::Replace, - None, - StorylineStreamOptions::default(), - ) - .await?; - published_storyline_report(outcome) + self.write_storyline_stream(stories, None, StorylineStreamOptions::default()) + .await } /// Like [`Self::replace_storyline_stream`], with explicit stream options. @@ -830,16 +645,7 @@ impl StorylineLanceStore { where I: IntoIterator>, { - let outcome = self - .replace_storyline_stream_with_projection( - stories, - None, - StorylineStreamWriteMode::Replace, - None, - options, - ) - .await?; - published_storyline_report(outcome) + self.write_storyline_stream(stories, None, options).await } /// Append pre-disambiguated Storylines only if the Dataset is still at @@ -870,90 +676,16 @@ impl StorylineLanceStore { where I: IntoIterator>, { - let outcome = self - .replace_storyline_stream_with_projection( - stories, - None, - StorylineStreamWriteMode::Replace, - Some(expected_generation), - options, - ) - .await?; - published_storyline_report(outcome) - } - - pub async fn replace_projected_storyline_stream( - &self, - stories: I, - projection: StorylineProjectionLineage, - ) -> Result - where - I: IntoIterator>, - { - projection.validate()?; - let outcome = self - .replace_storyline_stream_with_projection( - stories, - Some(projection), - StorylineStreamWriteMode::Replace, - None, - StorylineStreamOptions::default(), - ) - .await?; - published_storyline_report(outcome) - } - - pub(crate) async fn create_projected_storyline_stream( - &self, - stories: I, - projection: StorylineProjectionLineage, - ) -> Result - where - I: IntoIterator>, - { - projection.validate()?; - self.replace_storyline_stream_with_projection( - stories, - Some(projection), - StorylineStreamWriteMode::CreateProjection, - None, - StorylineStreamOptions::default(), - ) - .await - } - - /// Rebuild every normalized table into a new physical generation, then - /// atomically replace CURRENT. Existing readers remain pinned to the old - /// generation until publication succeeds. - pub async fn rebuild_projected_storyline_stream( - &self, - stories: I, - projection: StorylineProjectionLineage, - ) -> Result - where - I: IntoIterator>, - { - projection.validate()?; - let outcome = self - .replace_storyline_stream_with_projection( - stories, - Some(projection), - StorylineStreamWriteMode::Rebuild, - None, - StorylineStreamOptions::default(), - ) - .await?; - published_storyline_report(outcome) + self.write_storyline_stream(stories, Some(expected_generation), options) + .await } - async fn replace_storyline_stream_with_projection( + async fn write_storyline_stream( &self, stories: I, - projection: Option, - mode: StorylineStreamWriteMode, required_generation: Option<&str>, stream_options: StorylineStreamOptions, - ) -> Result + ) -> Result where I: IntoIterator>, { @@ -966,20 +698,12 @@ impl StorylineLanceStore { "Storyline append conflict: Dataset changed after duplicate-ID resolution" ); } - if mode == StorylineStreamWriteMode::CreateProjection && original.is_some() { - return Ok(StorylineProjectionPublicationOutcome::OutputNotEmpty); - } - #[cfg(test)] - if mode == StorylineStreamWriteMode::CreateProjection { - wait_after_empty_current_read(&self.root_uri).await; - } + let expected_generation = original.as_ref().map(|paths| paths.generation.clone()); #[cfg(test)] - if mode == StorylineStreamWriteMode::Replace { - wait_after_replacement_current_read(&self.root_uri).await; - } + wait_after_replacement_current_read(&self.root_uri).await; let writer_owner = next_generation(); - let writer_lease = if mode == StorylineStreamWriteMode::Replace && original.is_some() { + let writer_lease = if original.is_some() { Some( self.acquire_writer_lease_for_generation( &writer_owner, @@ -993,12 +717,11 @@ impl StorylineLanceStore { let mut writer_renewal = writer_lease .as_ref() .map(|lease| self.start_writer_lease_renewal(writer_owner.clone(), lease.lease.epoch)); - let rebuild = mode == StorylineStreamWriteMode::Rebuild; let takeover_generation = writer_lease .as_ref() .filter(|lease| lease.takeover) .map(|_| next_generation()); - let mut paths = if rebuild || takeover_generation.is_some() { + let mut paths = if takeover_generation.is_some() { None } else { original.clone() @@ -1006,9 +729,7 @@ impl StorylineLanceStore { let mut new_table_generation = takeover_generation.clone(); let mut iterator = stories.into_iter(); let mut chunk_state = StorylineChunkState::default(); - let mut next_storage_ordinal = if rebuild { - 0 - } else if let Some(paths) = &original { + let mut next_storage_ordinal = if let Some(paths) = &original { next_storage_ordinal(paths).await? } else { 0 @@ -1032,7 +753,7 @@ impl StorylineLanceStore { else { break; }; - if !rebuild && let Some(original) = &original { + if let Some(original) = &original { let existing = read_storage_ordinals_for_document_ids(original, &chunk.document_ids) .await?; @@ -1062,23 +783,14 @@ impl StorylineLanceStore { None => { let generation = next_generation(); let mut created = self.paths_for_generation(&generation); - #[cfg(test)] - let first_content_create = - if mode == StorylineStreamWriteMode::CreateProjection { - wait_for_first_content_create(&self.root_uri).await - } else { - false - }; let objects_result = commit_pending_content( &created.objects, original.as_ref().map(|paths| paths.objects_version), pending, - mode == StorylineStreamWriteMode::CreateProjection, + false, stream_options.optimize_indices, ) .await; - #[cfg(test)] - release_waiting_content_create(&self.root_uri, first_content_create); let objects_version = objects_result?; let (runs_version, steps_version, tool_calls_version) = join3_remote_aware( self.is_remote_object_store(), @@ -1228,9 +940,8 @@ impl StorylineLanceStore { steps_version, tool_calls_version, objects_version: current.objects_version, - projection, }; - let published = if let Some(lease) = &writer_lease { + if let Some(lease) = &writer_lease { let renewal = writer_renewal .take() .context("missing Storyline writer lease renewal")?; @@ -1243,20 +954,13 @@ impl StorylineLanceStore { "Storyline writer lease lost while publishing generation {}", snapshot.generation ); - true - } else if mode == StorylineStreamWriteMode::CreateProjection { - self.try_commit_snapshot(&snapshot, expected_generation.as_deref()) - .await? } else { self.commit_snapshot(&snapshot, expected_generation.as_deref()) .await?; - true }; - if !published { - return Ok(StorylineProjectionPublicationOutcome::OutputNotEmpty); - } + report.generation = generation; - Ok(StorylineProjectionPublicationOutcome::Published(report)) + Ok(report) } .await; @@ -1283,11 +987,7 @@ impl StorylineLanceStore { } } - if (result.is_err() - || matches!( - &result, - Ok(StorylineProjectionPublicationOutcome::OutputNotEmpty) - )) + if result.is_err() && let Some(generation) = new_table_generation && let Err(error) = self .control_store @@ -1402,7 +1102,6 @@ impl StorylineLanceStore { steps_version, tool_calls_version, objects_version, - projection: paths.projection.clone(), }; let published_snapshot = self .publish_writer_snapshot_retaining_lease( @@ -1498,15 +1197,13 @@ impl StorylineLanceStore { return Ok(()); } let outcome = self - .replace_storyline_stream_with_projection( + .write_storyline_stream( stories.iter().cloned().map(Ok::<_, anyhow::Error>), None, - StorylineStreamWriteMode::Replace, - None, StorylineStreamOptions::default(), ) .await?; - published_storyline_report(outcome)?; + let _ = outcome; Ok(()) } @@ -1637,7 +1334,6 @@ impl StorylineLanceStore { steps_version: 0, tool_calls_version: 0, objects_version: 0, - projection: None, } } @@ -1680,7 +1376,6 @@ impl StorylineLanceStore { cloned.steps_version = steps_version; cloned.tool_calls_version = tool_calls_version; cloned.objects_version = source.objects_version; - cloned.projection.clone_from(&source.projection); Ok(cloned) } @@ -1778,9 +1473,7 @@ fn validate_snapshot_pointer(pointer: &StorylineSnapshotPointer) -> Result<()> { pointer.schema_version, STORYLINE_LANCE_SCHEMA_VERSION ); - if let Some(projection) = &pointer.projection { - projection.validate()?; - } + validate_generation_name(&pointer.generation)?; if let Some(parent) = &pointer.parent_generation { validate_generation_name(parent)?; diff --git a/crates/persisting-pchronicle/src/store/storyline/model.rs b/crates/persisting-pchronicle/src/store/storyline/model.rs index 56ead51d1..727c568af 100644 --- a/crates/persisting-pchronicle/src/store/storyline/model.rs +++ b/crates/persisting-pchronicle/src/store/storyline/model.rs @@ -1,4 +1,4 @@ -//! Relational rows for the Storyline-native, three-table Lance projection. +//! Relational rows for the Storyline-native, three-table Lance storage. //! //! A Storyline document is normalized into one run row, ordered step rows, and //! tool-call rows. ATIF-compatible `observation.results[]` values are attached diff --git a/crates/persisting-pchronicle/src/store/storyline/tests.rs b/crates/persisting-pchronicle/src/store/storyline/tests.rs index fadb01538..7b3267487 100644 --- a/crates/persisting-pchronicle/src/store/storyline/tests.rs +++ b/crates/persisting-pchronicle/src/store/storyline/tests.rs @@ -11,50 +11,6 @@ fn remote_uri(label: &str) -> String { ) } -fn projection_lineage() -> StorylineProjectionLineage { - StorylineProjectionLineage { - source_id: "source-1".into(), - source_file: "agent/run/events.lance".into(), - source: ProjectionSourceSnapshot::CanonicalEvents { - source_uri: "/tmp/events.lance".into(), - fact_version: 7, - fact_rows: 42, - layout_revision: 9, - }, - projector_name: "events-storyline".into(), - recipe_hash: "blake3:recipe".into(), - completeness: "full".into(), - } -} - -#[test] -fn non_create_publication_mismatch_is_an_operational_error() { - let error = published_storyline_report(StorylineProjectionPublicationOutcome::OutputNotEmpty) - .unwrap_err(); - - assert!( - error - .to_string() - .contains("non-create Storyline publication reported nonempty output") - ); -} - -#[test] -fn create_projection_cleanup_failure_is_not_silently_discarded() { - let error = attach_stream_cleanup_failures( - Ok(StorylineProjectionPublicationOutcome::OutputNotEmpty), - vec!["remove staged generation: denied".into()], - ) - .unwrap_err(); - - assert!( - error - .to_string() - .contains("remove staged generation: denied"), - "{error:#}" - ); -} - async fn put_remote_object(uri: &str, relative: &str, contents: &[u8]) { let store = crate::store::opendal_store::Store::from_uri(uri) .await @@ -65,23 +21,6 @@ async fn put_remote_object(uri: &str, relative: &str, contents: &[u8]) { .unwrap(); } -struct CreateAfterEmptyReadBarrier; - -impl CreateAfterEmptyReadBarrier { - fn install(root_uri: &str, parties: usize) -> Self { - *CREATE_AFTER_EMPTY_READ_BARRIER - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) = - Some(CreateAfterEmptyReadBarrierHook { - root_uri: root_uri.to_string(), - barrier: Arc::new(tokio::sync::Barrier::new(parties)), - content_arrivals: Arc::new(std::sync::atomic::AtomicUsize::new(0)), - content_created: Arc::new(tokio::sync::Notify::new()), - }); - Self - } -} - struct ReplacementAfterCurrentReadBarrier; impl ReplacementAfterCurrentReadBarrier { @@ -105,14 +44,6 @@ impl Drop for ReplacementAfterCurrentReadBarrier { } } -impl Drop for CreateAfterEmptyReadBarrier { - fn drop(&mut self) { - *CREATE_AFTER_EMPTY_READ_BARRIER - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) = None; - } -} - struct MaintenanceAfterPublishPause { reached: Arc, resume: Arc, @@ -552,56 +483,6 @@ async fn persists_three_tables_and_round_trips_storyline() { ); } -#[tokio::test] -async fn projection_lineage_is_atomic_preserved_by_maintenance_and_cleared_by_direct_write() { - let dir = tempfile::tempdir().unwrap(); - let store = StorylineLanceStore::open(dir.path()).await.unwrap(); - let lineage = projection_lineage(); - store - .replace_projected_storylines(&[story("projected")], lineage.clone()) - .await - .unwrap(); - assert_eq!( - store - .current_table_paths() - .await - .unwrap() - .unwrap() - .projection, - Some(lineage.clone()) - ); - - store - .maintain(&LanceMaintenanceOptions { - compact: false, - optimize_indices: false, - vacuum_older_than: None, - ..Default::default() - }) - .await - .unwrap(); - assert_eq!( - store - .current_table_paths() - .await - .unwrap() - .unwrap() - .projection, - Some(lineage) - ); - - store.replace_storyline(&story("direct")).await.unwrap(); - assert!( - store - .current_table_paths() - .await - .unwrap() - .unwrap() - .projection - .is_none() - ); -} - #[tokio::test] async fn large_content_is_lossless_deduplicated_and_never_exposed_as_descriptor() { let dir = tempfile::tempdir().unwrap(); @@ -866,77 +747,6 @@ async fn remote_maintenance_preserves_shared_object_versions() { ); } -#[tokio::test] -async fn maintenance_prunes_expired_physical_generations() { - let dir = tempfile::tempdir().unwrap(); - let store = StorylineLanceStore::open(dir.path()).await.unwrap(); - store - .rebuild_projected_storyline_stream(vec![Ok(story("generation-one"))], projection_lineage()) - .await - .unwrap(); - let first_table_generation = store - .current_table_paths() - .await - .unwrap() - .unwrap() - .table_generation; - store - .rebuild_projected_storyline_stream(vec![Ok(story("generation-two"))], projection_lineage()) - .await - .unwrap(); - let current_table_generation = store - .current_table_paths() - .await - .unwrap() - .unwrap() - .table_generation; - assert_ne!(first_table_generation, current_table_generation); - - let malformed = dir - .path() - .join(GENERATIONS_DIR) - .join("not-owned-by-storyline"); - std::fs::create_dir_all(&malformed).unwrap(); - std::fs::write(malformed.join("keep"), b"not a Storyline generation").unwrap(); - - let no_vacuum = store - .maintain(&LanceMaintenanceOptions { - vacuum_older_than: None, - ..Default::default() - }) - .await - .unwrap(); - assert_eq!(no_vacuum.generations_removed, 0); - assert!( - dir.path() - .join(GENERATIONS_DIR) - .join(&first_table_generation) - .exists() - ); - - let vacuumed = store - .maintain(&LanceMaintenanceOptions { - vacuum_older_than: Some(std::time::Duration::ZERO), - ..Default::default() - }) - .await - .unwrap(); - assert_eq!(vacuumed.generations_removed, 1); - assert!( - !dir.path() - .join(GENERATIONS_DIR) - .join(first_table_generation) - .exists() - ); - assert!( - dir.path() - .join(GENERATIONS_DIR) - .join(current_table_generation) - .exists() - ); - assert!(malformed.exists()); -} - #[tokio::test] async fn content_descriptor_magic_in_user_text_round_trips_as_literal() { let dir = tempfile::tempdir().unwrap(); @@ -2082,7 +1892,6 @@ async fn stale_maintenance_cannot_prune_a_successor_generation() { steps_version: cloned.steps_version, tool_calls_version: cloned.tool_calls_version, objects_version: cloned.objects_version, - projection: cloned.projection, }; assert!( store @@ -2293,121 +2102,6 @@ async fn independent_replacements_conflict_at_current_cas_and_retry_cleanly() { ); } -async fn assert_independent_object_store_create_case( - label: &str, - content_options: StorylineContentOptions, - left_story: StorylineDocument, - right_story: StorylineDocument, - expect_offloaded_content: bool, -) { - let uri = remote_uri(label); - let mut left = StorylineLanceStore::open_uri_with_content_options(&uri, content_options) - .await - .unwrap(); - let mut right = StorylineLanceStore::open_uri_with_content_options(&uri, content_options) - .await - .unwrap(); - left.write_lock = Arc::new(tokio::sync::Mutex::new(())); - right.write_lock = Arc::new(tokio::sync::Mutex::new(())); - let _barrier = CreateAfterEmptyReadBarrier::install(&uri, 2); - let mut left_lineage = projection_lineage(); - left_lineage.source_file = format!("left-{label}.lance"); - let mut right_lineage = projection_lineage(); - right_lineage.source_file = format!("right-{label}.lance"); - - let (left_outcome, right_outcome) = tokio::join!( - left.create_projected_storyline_stream( - std::iter::once(Ok(left_story.clone())), - left_lineage.clone(), - ), - right.create_projected_storyline_stream( - std::iter::once(Ok(right_story.clone())), - right_lineage.clone(), - ) - ); - let left_outcome = left_outcome.unwrap(); - let right_outcome = right_outcome.unwrap(); - - let (report, winner_lineage, winner, loser_session_id) = match (&left_outcome, &right_outcome) { - (StorylineProjectionPublicationOutcome::Published(report), _) => ( - report, - &left_lineage, - &left_story, - right_story.session_id.as_str(), - ), - (_, StorylineProjectionPublicationOutcome::Published(report)) => ( - report, - &right_lineage, - &right_story, - left_story.session_id.as_str(), - ), - _ => panic!("exactly one independent create must publish for {label}"), - }; - assert!(matches!( - (&left_outcome, &right_outcome), - ( - StorylineProjectionPublicationOutcome::Published(_), - StorylineProjectionPublicationOutcome::OutputNotEmpty - ) | ( - StorylineProjectionPublicationOutcome::OutputNotEmpty, - StorylineProjectionPublicationOutcome::Published(_) - ) - )); - - let reopened = StorylineLanceStore::open_uri(&uri).await.unwrap(); - let current = reopened.current_table_paths().await.unwrap().unwrap(); - assert_eq!(current.generation, report.generation); - assert_eq!(current.projection.as_ref(), Some(winner_lineage)); - if expect_offloaded_content { - let objects = open_objects(¤t.objects, current.objects_version) - .await - .unwrap(); - assert!(objects.count_rows(None).await.unwrap() > 0); - } - assert_eq!( - reopened - .get_storyline_full(&winner.session_id) - .await - .unwrap(), - Some(winner.clone()) - ); - assert!( - reopened - .get_storyline_full(loser_session_id) - .await - .unwrap() - .is_none() - ); -} - -#[tokio::test] -async fn independent_object_store_creates_publish_one_inline_or_offloaded_projection() { - assert_independent_object_store_create_case( - "independent-concurrent-create-inline", - StorylineContentOptions::default(), - story("left-inline-winner"), - story("right-inline-winner"), - false, - ) - .await; - - let mut left_story = story("left-offloaded-winner"); - left_story.notes = Some("left projection content ".repeat(256)); - let mut right_story = story("right-offloaded-winner"); - right_story.notes = Some("right projection content ".repeat(256)); - assert_independent_object_store_create_case( - "independent-concurrent-create-offloaded", - StorylineContentOptions { - offload_threshold: 1, - ..Default::default() - }, - left_story, - right_story, - true, - ) - .await; -} - #[tokio::test] async fn stale_current_commit_is_rejected_without_moving_snapshot() { let uri = remote_uri("stale-current"); @@ -2429,7 +2123,6 @@ async fn stale_current_commit_is_rejected_without_moving_snapshot() { steps_version: stale.steps_version, tool_calls_version: stale.tool_calls_version, objects_version: stale.objects_version, - projection: stale.projection.clone(), }, Some(&stale.generation), ) diff --git a/crates/persisting-pchronicle/src/store/storyline/writer_control.rs b/crates/persisting-pchronicle/src/store/storyline/writer_control.rs index 600f223f6..243bc967f 100644 --- a/crates/persisting-pchronicle/src/store/storyline/writer_control.rs +++ b/crates/persisting-pchronicle/src/store/storyline/writer_control.rs @@ -3,13 +3,20 @@ use serde::{Deserialize, Serialize}; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; -pub(super) use super::super::cas_store::unix_now_ms; use super::super::opendal_store::{self, Version}; use super::{ CURRENT_FILE, StorylineLanceStore, StorylineSnapshotPointer, validate_current_control, write_local_current, }; +pub(super) fn unix_now_ms() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_millis() + .min(u64::MAX as u128) as u64 +} + const CONTROL_CAS_RETRIES: usize = 32; /// Brief retries when CURRENT still shows a held lease after a prior release. /// Object stores can lag on read-after-write for the control object. @@ -714,6 +721,23 @@ impl StorylineLanceStore { mod tests { use super::*; + #[test] + fn unix_now_ms_matches_system_clock_and_is_non_decreasing() { + let before = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|duration| duration.as_millis() as u64) + .unwrap_or(0); + let first = unix_now_ms(); + let second = unix_now_ms(); + let after = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|duration| duration.as_millis() as u64) + .unwrap_or(0); + assert!(first >= before, "got {first} before {before}"); + assert!(second >= first, "got {second} after {first}"); + assert!(after >= second, "got {after} after {second}"); + } + fn pointer(generation: &str) -> StorylineSnapshotPointer { StorylineSnapshotPointer { schema_version: crate::store::storyline::STORYLINE_LANCE_SCHEMA_VERSION, @@ -724,7 +748,6 @@ mod tests { steps_version: 1, tool_calls_version: 1, objects_version: 1, - projection: None, } } @@ -822,7 +845,6 @@ mod proptests { steps_version: 1, tool_calls_version: 1, objects_version: 1, - projection: None, } } diff --git a/crates/persisting-pchronicle/src/store/virtual_document.rs b/crates/persisting-pchronicle/src/store/virtual_document.rs index e72c36ad0..0dad5f19e 100644 --- a/crates/persisting-pchronicle/src/store/virtual_document.rs +++ b/crates/persisting-pchronicle/src/store/virtual_document.rs @@ -31,7 +31,6 @@ use crate::format::DocumentFormat; /// values would require quoting (`dataset.openai-msg`) in every query. pub(crate) fn table_name(format: DocumentFormat) -> Option<&'static str> { Some(match format { - DocumentFormat::CanonicalEvent => "events", DocumentFormat::Atif => "atif", DocumentFormat::Storyline => "storyline", DocumentFormat::Actf => "actf", @@ -44,9 +43,6 @@ pub(crate) fn table_name(format: DocumentFormat) -> Option<&'static str> { } pub(crate) fn formats() -> impl Iterator { - // Canonical Event already owns the stable `events` table (with its - // denormalized columns and payload_json); registering another provider - // under the same name would make catalog registration fail. [ DocumentFormat::Atif, DocumentFormat::Storyline, @@ -525,7 +521,6 @@ mod tests { #[test] fn virtual_table_names_use_query_friendly_aliases() { - assert_eq!(table_name(DocumentFormat::CanonicalEvent), Some("events")); assert_eq!(table_name(DocumentFormat::AgenticMd), Some("markdown")); assert_eq!(table_name(DocumentFormat::ClaudeCode), Some("claude")); assert_eq!(table_name(DocumentFormat::Atif), Some("atif")); diff --git a/crates/persisting-pchronicle/src/tests.rs b/crates/persisting-pchronicle/src/tests.rs index 0fc5d3df3..21e94c006 100644 --- a/crates/persisting-pchronicle/src/tests.rs +++ b/crates/persisting-pchronicle/src/tests.rs @@ -5,22 +5,14 @@ use crate::atif::{AtifAgent, AtifObservation, AtifStep, AtifToolCall, AtifTrajec #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum TestFormat { Storyline, - CanonicalEvent, AgenticMd, OpenaiMsg, Atif, } -impl TestFormat { - fn is_lance_only(self) -> bool { - self == Self::CanonicalEvent - } -} - fn into_storyline(format: TestFormat, input: &str) -> crate::Result { match format { TestFormat::Storyline => crate::formats::storyline::parse_storyline_document(input), - TestFormat::CanonicalEvent => Err(lance_only_error()), TestFormat::AgenticMd => Ok(crate::document::decode_agenticmd(input)?), TestFormat::OpenaiMsg => { let value = serde_json::from_str(input)?; @@ -43,7 +35,6 @@ fn into_storyline(format: TestFormat, input: &str) -> crate::Result crate::Result { match format { TestFormat::Storyline => story.to_json_string_pretty(), - TestFormat::CanonicalEvent => Err(lance_only_error()), TestFormat::AgenticMd => crate::document::encode_agenticmd(story), TestFormat::OpenaiMsg => Ok(serde_json::to_string_pretty( &crate::formats::openai_corpus::synthesize_openai_msg_corpus_value( @@ -58,19 +49,11 @@ fn from_storyline(format: TestFormat, story: &crate::StorylineDocument) -> crate fn convert(from: TestFormat, to: TestFormat, input: &str) -> crate::Result { if from == to { - return if from.is_lance_only() { - Err(lance_only_error()) - } else { - Ok(input.to_string()) - }; + return Ok(input.to_string()); } from_storyline(to, &into_storyline(from, input)?) } -fn lance_only_error() -> anyhow::Error { - anyhow::anyhow!(crate::formats::events::events_lance_only_message()) -} - fn sample_traj() -> AtifTrajectory { AtifTrajectory { schema_version: "ATIF-v1.7".into(), @@ -222,151 +205,6 @@ fn convert_peripheral_via_hub_only() { assert!(!traj.steps.is_empty()); } -#[test] -fn http_event_aliases_project_to_storyline() { - let doc = crate::model::EventsDocument::new(vec![ - crate::EventRecord { - identity: crate::EventIdentity::default(), - seq: 0, - source: "capture".into(), - kind: "http.request".into(), - timestamp: None, - session_id: Some("http-session".into()), - agent_id: Some("agent".into()), - parent_uuid: None, - trace_id: None, - call_id: Some("call-http".into()), - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload: json!({"user_content": "hello", "model": "m"}), - }, - crate::EventRecord { - identity: crate::EventIdentity::default(), - seq: 1, - source: "capture".into(), - kind: "http.response".into(), - timestamp: None, - session_id: Some("http-session".into()), - agent_id: Some("agent".into()), - parent_uuid: None, - trace_id: None, - call_id: Some("call-http".into()), - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload: json!({"assistant_content": "world"}), - }, - ]); - let story = crate::convert::events_to_storyline(&doc).unwrap(); - assert_eq!(story.turns.len(), 2); - assert_eq!(story.turns[0].message, json!("hello")); - assert_eq!(story.turns[1].message, json!("world")); -} - -#[test] -fn events_string_convert_is_lance_only_error() { - use crate::formats::events::events_lance_only_message; - let err = into_storyline(TestFormat::CanonicalEvent, "[]").unwrap_err(); - assert!( - err.to_string().contains("Lance-only") - || err - .to_string() - .contains(events_lance_only_message().split(';').next().unwrap()) - ); - let story = crate::StorylineDocument::new("s", "a"); - assert!(from_storyline(TestFormat::CanonicalEvent, &story).is_err()); - assert!(convert(TestFormat::Atif, TestFormat::CanonicalEvent, "{}").is_err()); - assert!(TestFormat::CanonicalEvent.is_lance_only()); -} - -#[test] -fn export_events_jsonl_debug_roundtrip_via_test_parser() { - use crate::formats::events::{ - EventRecord, EventsDocument, export_events_jsonl, parse_events_jsonl_for_test, - }; - use serde_json::json; - let events = vec![EventRecord { - identity: crate::EventIdentity::default(), - seq: 0, - source: "proxy".into(), - kind: "llm.request".into(), - timestamp: Some("2026-01-01T00:00:00Z".into()), - session_id: Some("s1".into()), - agent_id: Some("a1".into()), - parent_uuid: None, - trace_id: Some("t1".into()), - call_id: Some("c1".into()), - subagent_id: None, - parent_agent_id: None, - branch: None, - parent_call_id: None, - payload: json!({"model":"m","messages":[{"role":"user","content":"hi"}]}), - }]; - let text = export_events_jsonl(&events).unwrap(); - let doc = parse_events_jsonl_for_test(&text).unwrap(); - assert_eq!(doc.format, EventsDocument::FORMAT_NAME); - assert_eq!(doc.events.len(), 1); - assert_eq!(doc.events[0].kind, "llm.request"); - assert_eq!(doc.session_id.as_deref(), Some("s1")); -} - -#[test] -fn detect_format_from_content_and_path() { - use crate::formats::detect::{detect_format, detect_format_from_path}; - use std::path::Path; - assert_eq!( - detect_format_from_path(Path::new("/tmp/x/session_steps.json")), - Some(crate::DocumentFormat::OpenaiMsg) - ); - assert_eq!( - detect_format_from_path(Path::new("/tmp/x/events.lance")), - Some(crate::DocumentFormat::CanonicalEvent) - ); - assert_eq!( - detect_format_from_path(Path::new("/tmp/x/sess.md")), - Some(crate::DocumentFormat::AgenticMd) - ); - assert_eq!( - detect_format_from_path(Path::new("/tmp/x/storyline.json")), - None - ); - assert_eq!( - detect_format_from_path(Path::new("/tmp/x/task.actf.json")), - Some(crate::DocumentFormat::Actf) - ); - let atif = r#"{"schema_version":"ATIF-v1.7","session_id":"s","agent":{"name":"a","version":"1"},"steps":[]}"#; - assert_eq!( - detect_format(None, Some(atif)).unwrap(), - Some(crate::DocumentFormat::Atif) - ); - let atif_with_agenticmd_marker = r#"{"schema_version":"ATIF-v1.7","session_id":"s","agent":{"name":"a","version":"1"},"steps":[{"step_id":1,"source":"user","message":"source contains ` block structure. The reader also -accepts speaker-less blocks, the legacy `role/seq/session/agent` fields, -and ordinary Markdown body text. - -## 5. Writes and consistency - -1. An `EventRecord` is converted to Arrow rows before it enters Lance. One - bounded micro-batch is one Lance append to the current epoch segment, - then an exact version is published with a manifest CAS. -2. The hot path does not read old rows, row counts, or `event_id`. It does - not deduplicate, index, compact, or vacuum. -3. A conflict between producer identity and write coordinates still - appends. `payload_json` keeps the original claims. Physical - `session_id` / `agent_id` come from the caller coordinates and take - effect on replay. Projections take the last non-empty value for the - remaining identity claims in append order and do not add a - read-before-write, dedup, or index cost for the conflict. -4. `seq` is a producer-defined Storyline ordinal. The replay cursor uses - the immutable physical append order. -5. Distinct Stories in a Run bucket share the manifest and epoch segments, - but replay and stats isolate by `session_id`. -6. Live Markdown locates blocks by `call_id + source` (with a legacy-role - alias) so a streaming agent can update in place. -7. Canonical append and derived projections report results separately. A - projection failure must not be presented as a durable event write. -8. Segment/manifest publication for distinct root URIs in one micro-batch - is at most 16-way parallel. The same URI stays serial in batch order, - so a single Story's physical append order is not relaxed. - -The event fact layer provides at-least-once append. It does not provide -exactly-once delivery or ID uniqueness. Truncate, overwrite, and retry -dedup are not part of the fact write path. Conversion into an existing Run -fails; trimming should create a new Run or happen on a derived Storyline. -Compaction, a `session_id` index, and vacuum are explicit offline -maintenance. - -The upper Run lease produces a monotonic epoch. -`EventWriterFence(epoch, writer_id)` is activated before a new writer -writes data. Readers only see the segment versions pinned by the -manifest, so an underlying append finished by an old writer after takeover -is invisible. Another `writer_id` in the same epoch is rejected. The -protocol provides writer fencing. Concurrent multi-writer merge is not a -supported write mode. - -## 6. Format conversion - -Generic peripheral formats convert through the Storyline hub: - -```text -AgenticMD ─┐ -ATIF ──────┼── Storyline ── events / AgenticMD / ATIF / ACTF / OpenAI messages -ACTF ──────┤ -OpenAI msg ┘ -``` - -Paths that must keep the original payload read and write events directly -and must not go through a lossy Storyline roundtrip. Peripheral exchange -formats are handled by `pchronicle import/export`. - -## 7. Component boundaries - -| Component | Owns | Does not own | -|---|---|---| -| Gateway | protocol parsing, call lifecycle, capture order, live projection policy | generic store, format schema, offline conversion | -| pChronicle | formats, paths, durability, reads, conversion, and revision lineage | network forwarding, Agent lifecycle | - -## 8. Related documents - -- [Recorded data, views, and versions](../concepts/facts-and-projections.md) -- [Discover and query](../guides/discover-and-query.md) -- [Snapshot](catalog.md) -- [AgenticMD format](../reference/agenticmd.md) -- [Gateway capture](../guides/serve-gateway.md) -- [`pchronicle` Dataset commands](../reference/cli.md) +See [Storyline Lance](storyline-lance.md), [Snapshot](catalog.md), +[formats](../reference/formats/index.md), and [import/export](../guides/exchange.md). diff --git a/docs/src/en/pchronicle/guides/exchange.md b/docs/src/en/pchronicle/guides/exchange.md index 546a0a6da..e59105128 100644 --- a/docs/src/en/pchronicle/guides/exchange.md +++ b/docs/src/en/pchronicle/guides/exchange.md @@ -51,49 +51,6 @@ pchronicle import --from ./corpus --to ./normalized \ --output-format storyline ``` -A validated, non-empty canonical Event Store is detected before JSON scanning -and always creates Storyline Lance: - -```bash -pchronicle import --from ./run/events.lance --to ./run/storyline -``` - -This mode accepts local and object-store URIs, never mutates the source, and -supports create or confirmed replace (not append). Its JSON result reports `format: "events"`, -`output_format: "storyline-lance"`, and `fact_rows`; it omits `input_bytes`. -Explicit `--output-format preserve` and JSON exchange `--input-format` values are -invalid for canonical events. - -In the squashed Dataset, `_file_` is `.` for all normalized rows: - -```bash -pchronicle query ./normalized \ - --sql 'SELECT _file_, COUNT(*) AS runs FROM dataset.runs GROUP BY _file_' -``` - -`document_id` is globally unique in Storyline output. Collisions receive a -deterministic `#N` suffix; append can instead skip them with -`--on-duplicate skip`. Successful Storyline output does not retain source paths -as queryable information; use preserve output when file boundaries matter. - -ATIF `.jsonl` and `.ndjson` inputs decode every non-empty record. Symbolic -links found while walking a directory are skipped; an explicitly named link to -a regular file retains single-file behavior. The directory is published -atomically only after every input and the selected storage output succeed. -Stdin must be finite and explicit: - -```bash -cat input.json | pchronicle import --from - \ - --to ./imported --input-format openai-messages -``` - -After import, inspect the new boundary: - -```bash -pchronicle stats ./imported -pchronicle stats overview ./imported -``` - ## Export complete Runs ```bash @@ -115,5 +72,5 @@ Import/export is not a storage migration protocol and arbitrary SQL rows are not exportable Runs. For exact flags, see the [`pchronicle` CLI reference](../reference/cli.md). See [Run data formats](../reference/formats/index.md) for contracts and -[data contracts and revisions](../concepts/facts-and-projections.md) +[trajectory data and versions](../concepts/facts-and-projections.md) for the internal layer boundary. diff --git a/docs/src/en/pchronicle/guides/index.md b/docs/src/en/pchronicle/guides/index.md index 47053ecb1..401e5736b 100644 --- a/docs/src/en/pchronicle/guides/index.md +++ b/docs/src/en/pchronicle/guides/index.md @@ -7,7 +7,6 @@ task-oriented workflow. 2. [Import and export Runs](exchange.md). 3. [Serve Datasets locally](serve.md). 4. [Browse, drill down, and analyze in the local Web UI](ui.md). -5. [Forward, rewrite, and capture through the `serve` Gateway](serve-gateway.md). The guides explain decisions and complete workflows. Use the [`pchronicle` reference](../reference/cli.md) for exact flags, and diff --git a/docs/src/en/pchronicle/guides/serve-gateway.md b/docs/src/en/pchronicle/guides/serve-gateway.md deleted file mode 100644 index e6116d4c6..000000000 --- a/docs/src/en/pchronicle/guides/serve-gateway.md +++ /dev/null @@ -1,427 +0,0 @@ -# Gateway ingestion, forwarding, and capture for `pchronicle serve` - -`pchronicle serve` has two mutually exclusive Gateway modes: - -- `--gateway ADDRESS` starts a config-free HTTP ingestion endpoint for - canonical trajectory events; -- `--gateway-config FILE` keeps the compatibility LLM forwarding, protocol - rewriting, and capture mode described later in this guide. - -The config-free form controls storage entirely from the command line: - -```bash -pchronicle serve \ - --gateway auto \ - --gateway-dataset ./data/captures \ - --gateway-split '{user}/{date}/{hour}' -``` - -Send a batch to `POST /v1/events` with `agent_id`, `session_id`, optional -`root_session_id`, and `records`. The records use the shared canonical -`EventRecord` schema. `x-persisting-user-id` supplies `{user}`; missing users -use `_unknown`. A successful response means every accepted record is durably -visible. Canonical `/v1/events` requests accept at most 256 records and the -body limit is 8 MiB. Langfuse OTLP batches may contain more spans; the Gateway -splits those batches internally while keeping the client-facing request -boundary unchanged. Event IDs derived from `traceId` and `spanId` remain stable -across retries, which gives downstream compaction/deduplication a deterministic -key. - -Split templates are relative, contain at most 16 segments, and accept only -safe literal segments plus the exact placeholders `{user}`, `{date}`, and -`{hour}`. Time fields use UTC. A logical run/session is pinned to its first -partition for the life of the process. `--gateway-split-idle` controls how long -an existing canonical source must stay quiet before its Storyline projection is -refreshed; it defaults to `30m`. Newly discovered sources are still projected -immediately. This is a projection idle window, not a change to the physical -split path. - -The remaining sections describe the forwarding compatibility mode. - -## Langfuse OTLP pressure benchmark - -The repository includes a black-box pressure test. It starts a real -`pchronicle serve --gateway` child process, sends synthetic spans concurrently -through the Langfuse OTLP HTTP path, and queries the Dataset after shutdown. It -is ignored by default so regular CI stays bounded: - -```bash -PCHRONICLE_LANGFUSE_STRESS_REQUESTS=32 \ -PCHRONICLE_LANGFUSE_STRESS_SPANS_PER_REQUEST=512 \ -PCHRONICLE_LANGFUSE_STRESS_CONCURRENCY=8 \ -cargo test -p persisting-pchronicle-cli --test langfuse_gateway_stress \ - langfuse_gateway_pressure --offline -- --ignored --nocapture -``` - -The test checks HTTP 200 and the full-success OTLP response, internal chunking -for batches larger than 256 spans, durable event counts, trace/span/parent -relationships, and event-ID uniqueness. It prints requests/sec, spans/sec, and -p50/p95/p99 latency. Tune the three environment variables for the target -hardware; compare throughput only with fixed disk, filesystem, and concurrency, -and do not treat it as a cross-machine SLA. - -`pchronicle serve` can run a local LLM Gateway with or without the read-only Dataset server. -For each request, the Gateway selects an upstream, can rewrite the model and -wire protocol, returns a response in the client's protocol, and appends -canonical capture events to the CLI-selected output Dataset. The Dataset Web UI and API -remain read-only. - -Use this mode when an Agent or SDK already knows how to call an OpenAI-, -Anthropic-, or Gemini-compatible base URL and you want to capture that traffic -in a Dataset. - -## Configuration inputs - -Gateway mode deliberately keeps Dataset selection and forwarding configuration separate: - -| Input | Owns | -| --- | --- | -| `--gateway-dataset DATASET` | Output Dataset URI, mounted automatically | -| `gateway.toml` passed to `--gateway-config` | Gateway listeners, model routes, credentials, capture level, and network policy | -| Other CLI flags | Physical split, local Gateway state, live Markdown, and foreground debugging | - -Unknown Gateway TOML fields are rejected. The Gateway configuration must use -TOML; other file extensions are not accepted. - -## Minimal configuration - -Create `gateway.toml`: - -```toml -listen = "127.0.0.1:8787" -admin_listen = "127.0.0.1:8788" -agent_id = "local-agent" -capture_level = "dialogue" - -[[models]] -name = "deepseek-chat" -provider = "openai" -upstream = "https://api.deepseek.com/v1" -api_key_env = "DEEPSEEK_API_KEY" - -[[models]] -name = "*" -forward = "deepseek-chat" -``` - -Export the credential and start both services: - -```bash -export DEEPSEEK_API_KEY=sk-... - -pchronicle serve \ - --listen 127.0.0.1:8080 \ - --gateway-config gateway.toml \ - --gateway-dataset ./data/captures \ - --gateway-stream-markdown -``` - -This starts three loopback listeners: - -- `127.0.0.1:8080` — Dataset Web UI and read API; -- `127.0.0.1:8787` — LLM Gateway; -- `127.0.0.1:8788` — Gateway status and session API. - -Omit `--listen` to run only the Gateway listeners and capture sink; no -Dataset HTTP endpoint is created. - -Point the Agent or SDK at `http://127.0.0.1:8787/v1`. For example: - -```bash -curl http://127.0.0.1:8787/v1/chat/completions \ - -H 'content-type: application/json' \ - -H 'x-persisting-session-id: example-session' \ - -d '{ - "model": "deepseek-chat", - "messages": [{"role": "user", "content": "Hello"}] - }' -``` - -Inspect the Gateway independently of the Dataset server: - -```bash -curl http://127.0.0.1:8788/admin/status -curl http://127.0.0.1:8788/admin/sessions -``` - -## Request lifecycle - -The Gateway performs forwarding, rewriting, and capture as one request -pipeline: - -```text -client request - -> detect client protocol and model - -> select the first matching models[] route - -> authorize the configured model route - -> optionally rewrite the model and protocol - -> construct the upstream URL and credentials - -> forward to the upstream - -> optionally translate the response to the client protocol - -> record the request and completed or streaming response -``` - -Capture metadata distinguishes the model requested by the client from the -model sent upstream and records whether a model rewrite occurred. Capture does -not replace forwarding: an upstream error is returned to the client and also -closes the corresponding capture call. - -## Top-level Gateway fields - -| Field | Required | Default | Meaning | -| --- | --- | --- | --- | -| `listen` | Yes | — | LLM proxy listener. Embedded mode accepts loopback addresses only. Port `0` selects an available port. | -| `admin_listen` | No | `127.0.0.1:9876` | Listener for `/admin/status` and `/admin/sessions`; also loopback-only. | -| `agent_id` | No | `default` | Agent identity attached to capture records unless a more specific identity is derived from the request. | -| `session_header` | No | `x-persisting-session-id` | Request header used to group calls into a session. | -| `capture_level` | No | `dialogue` | Amount of request and response content retained: `summary`, `dialogue`, or `full`. | -| `debug` | No | `false` | Writes Gateway diagnostics to the state directory. Logs can include bounded request and response bodies. | -| `models` | Yes | — | Ordered list of model routing rules. | -| `network` | No | `mode = "public"` | Policy for explicit forward-proxy traffic. | - -The shared Gateway schema also accepts an `[overlay]` table, but -`pchronicle serve` does not create or apply a filesystem overlay. - -### Capture levels - -- `summary` stores protocol metadata and byte counts without user or assistant - message text. -- `dialogue` stores user and assistant dialogue and is the default. -- `full` additionally stores parsed request and response bodies. Use it only - when the additional content and secret exposure are acceptable. - -## Model routes - -Routes are evaluated in file order; the first matching `name` wins. A name may -be an exact model, `prefix*`, `*suffix`, or the catch-all `*`. - -| Field | Meaning | -| --- | --- | -| `name` | Required match pattern or exact target model name. Names must be unique. | -| `provider` | `openai`, `anthropic`, `gemini`, `vertex`, `bedrock`, `azure`, `copilot`, or `custom`; defaults to `openai`. | -| `upstream` | Upstream base URL, including its API prefix when applicable. Required unless `forward` is set. | -| `upstream_anthropic` | Optional Anthropic-compatible base used for `/v1/messages`; otherwise `upstream` is used. | -| `api_key_env` | Environment variable read by the `pchronicle` process. This is the recommended credential source. | -| `api_key` | Inline credential. Supported, but avoid committing secrets to configuration files. | -| `forward` | Exact name of another route. Rewrites the request model and uses that route's upstream. | - -A route cannot set both `upstream` and `forward`. Forwarding must target a -route with an upstream and cannot be chained. If neither configuration nor the -named environment variable supplies a key, the Gateway can use a compatible -client authentication header; otherwise the request fails before forwarding. - -Example with several providers: - -```toml -listen = "127.0.0.1:8787" -admin_listen = "127.0.0.1:8788" -capture_level = "dialogue" - -[[models]] -name = "claude*" -provider = "anthropic" -upstream = "https://api.anthropic.com/v1" -api_key_env = "ANTHROPIC_API_KEY" - -[[models]] -name = "gpt*" -provider = "openai" -upstream = "https://api.openai.com/v1" -api_key_env = "OPENAI_API_KEY" - -[[models]] -name = "gemini*" -provider = "gemini" -upstream = "https://generativelanguage.googleapis.com/v1beta" -api_key_env = "GEMINI_API_KEY" -``` - -## Forwarding and model rewriting - -After route selection, Gateway normalizes the effective request path against -the `upstream` base. An upstream ending in `/v1` and a client request to -`/v1/chat/completions` therefore produce one `/v1/chat/completions`, not -`/v1/v1/chat/completions`. Passthrough query parameters are retained. - -End-to-end headers are forwarded, while `Host`, `Content-Length`, hop-by-hop -headers, proxy authentication, and incoming LLM credentials are removed. -Gateway then applies the route credential as OpenAI Bearer, Anthropic -`x-api-key`, or Gemini `x-goog-api-key`. Redirects are returned to the client -instead of being followed inside Gateway. - -`forward` rewrites a client-visible model to one exact target route: - -```toml -[[models]] -name = "echo-upstream" -upstream = "http://127.0.0.1:19080/v1" - -[[models]] -name = "*" -forward = "echo-upstream" -``` - -A request for `client-model` now reaches the target with -`"model":"echo-upstream"`. Capture metadata retains both identities and marks -the rewrite. Forward targets must define `upstream`, cannot forward again, and -cannot be selected through another pattern match. Put specific patterns before -broad patterns because the first match wins. - -## Protocol rewriting - -Gateway selects a protocol bridge from the client path and target route. Both -regular responses and SSE streams are translated: - -| Client protocol | Target route | Upstream protocol | -| --- | --- | --- | -| Chat Completions | Non-Gemini | Chat Completions passthrough | -| Anthropic Messages | `upstream_anthropic` is set | Native Messages passthrough | -| Anthropic Messages | OpenAI-compatible without `upstream_anthropic` | Chat Completions | -| OpenAI Responses | Native OpenAI or Azure OpenAI | Responses passthrough | -| OpenAI Responses | Other OpenAI-compatible upstream | Chat Completions | -| Chat Completions, Messages, or Responses | `provider = "gemini"` | Gemini `generateContent` or `streamGenerateContent` | - -For a translated call, Gateway rewrites the request path and body and renders -the response or supported error envelope back into the client's protocol. The -bridge preserves common messages, tool calls, usage, reasoning, and streaming -events, but does not promise lossless preservation of every provider-specific -extension. Prefer passthrough when the client relies on such fields. - -## Test with the Echo upstream - -The repository includes a deterministic Rust Echo server for testing real HTTP -forwarding without an API key or model service. It supports `/echo`, Chat -Completions, Messages, Responses, Gemini, and their streaming forms. The last -user text controls the assistant output. - -Start it from a source checkout: - -```bash -just echo - -# Equivalent installed command: -pchronicle dev echo --listen 127.0.0.1:19080 --encoding plain -``` - -Point a route at it: - -```toml -listen = "127.0.0.1:8787" -admin_listen = "127.0.0.1:8788" -capture_level = "full" - -[[models]] -name = "echo-upstream" -provider = "openai" -upstream = "http://127.0.0.1:19080/v1" - -[[models]] -name = "*" -forward = "echo-upstream" -``` - -By default, the assistant returns the last user text directly. Override one -request to receive standard Base64 instead: - -```bash -curl http://127.0.0.1:8787/v1/messages \ - -H 'content-type: application/json' \ - -H 'x-persisting-echo-encoding: base64' \ - -d '{ - "model": "client-alias", - "max_tokens": 32, - "messages": [{"role": "user", "content": "hello"}] - }' -``` - -The Messages request is converted to Chat Completions, its model is rewritten -to `echo-upstream`, and the response is converted back to Messages with -`aGVsbG8=` as its text. Add `"stream": true` to exercise the same path with -SSE. The request header accepts `plain` or `base64`; `--encoding` sets the -server-wide default. The hidden testing command `pchronicle dev echo` only -binds to loopback and is intended for deterministic local Gateway tests. - -## Network policy - -`[network]` controls explicit proxy traffic such as `CONNECT` and absolute-URI -requests. Configured `models[].upstream` destinations are Gateway-owned routes, -not Agent egress grants; restrict the LLM surface with the model route list. - -The available modes are: - -```toml -[network] -mode = "no-network" -``` - -```toml -[network] -mode = "allowlist" -allowed_hosts = ["pypi.org", "files.pythonhosted.org", "*.github.com"] -``` - -`public` is the default. `no-network` denies explicit proxy egress. -`allowlist` requires a matching `allowed_hosts` entry or structured -`[[network.rules]]` rule. Explicit `[[network.deny_rules]]` entries take -precedence over allows. `pchronicle serve` only controls traffic that the client -sends through the Gateway; this does not provide process-wide network isolation. - -## Dataset and state selection - -`--gateway-dataset DATASET` is required and directly names the local path or -object-store URI receiving captures. pChronicle mounts it automatically. If -the same URI is already mounted positionally it is deduplicated. - -Canonical events are appended directly to that Dataset. Gateway runtime state -is separate and includes the session index, debug logs, and optional live -AgenticMD projection: - -- for a local Dataset, state defaults to the Dataset path; -- for an object-store Dataset such as `s3://...`, pass a writable local - `--gateway-state DIRECTORY`; -- pass `--gateway-stream-markdown` to maintain the live AgenticMD projection in - the state directory. - -Using an explicit state directory is also useful for keeping transient Gateway -files out of a local Dataset: - -```bash -pchronicle serve \ - --gateway-config gateway.toml \ - --gateway-dataset ./data/captures \ - --gateway-state ./.pchronicle-gateway \ - --gateway-stream-markdown -``` - -## CLI precedence and safety - -- `--listen` configures the Dataset server only. Gateway listeners always come from - `gateway.toml`. -- `--gateway-debug` enables Gateway debugging even - when `debug = false`; there is no CLI flag that forces configured debugging - off. -- `--gateway-dataset`, `--gateway-split`, `--gateway-state`, and - `--gateway-stream-markdown` are composition settings and are not Gateway TOML - fields. -- Dataset, Gateway, and admin listeners must all be loopback addresses. The - services do not provide an authentication or authorization boundary. -- Debug and `full` capture can retain sensitive request or response content. - Protect the state directory and capture Dataset accordingly. - -## Observe new captures - -Gateway events are durable after they have been flushed to the selected -Dataset. Appending to an existing `events.lance` source does not rebuild the -global Snapshot. Single-trace `/api/events`, `/api/storyline`, and -`/api/trajectory-view` requests reopen that source's latest manifest, so an -active trace can be observed in real time. The global Snapshot is refreshed when -a new canonical source (for example, a new split file) appears or a Storyline -projection is published. Projection or refresh failures use bounded retry and -retain the old queryable Snapshot; neither blocks durable capture writes. -`POST /api/catalog` remains available for an explicit manual refresh. On -`SIGINT` or `SIGTERM`, `pchronicle serve` stops both services and finishes the -Gateway capture writer before exiting. - -For exact command flags, see the [`pchronicle` CLI reference](../reference/cli.md). -For the storage model behind refresh, see -[Snapshot design](../design/catalog.md). diff --git a/docs/src/en/pchronicle/guides/serve.md b/docs/src/en/pchronicle/guides/serve.md index 7fa94be38..67593e090 100644 --- a/docs/src/en/pchronicle/guides/serve.md +++ b/docs/src/en/pchronicle/guides/serve.md @@ -1,165 +1,22 @@ # Serve Datasets locally -`pchronicle serve` mounts one or more Datasets in the bundled read-only Web UI -and API. It is a local inspection surface, not a public or multi-tenant data -service. - -## Command shape - -```text -pchronicle serve - [--listen LOOPBACK_ADDR] [--control LOOPBACK_ADDR] [--open] - [--home-link TEXT=PATH]... - [--gateway ADDRESS --gateway-dataset DATASET [--gateway-split TEMPLATE] - [--gateway-split-idle DURATION]] - [--gateway-config FILE --gateway-dataset DATASET [--gateway-state DIRECTORY]] - [--gateway-stream-markdown] [--gateway-debug] - [--catalog-config FILE] - [<[NAME=]DATASET> ...] -pchronicle serve catalog dataset add --catalog-config FILE NAME --uri URI [OPTIONS] -pchronicle serve catalog dataset remove --catalog-config FILE NAME... -pchronicle serve catalog dataset list --catalog-config FILE -pchronicle serve catalog issue --catalog-config FILE NAME -pchronicle serve catalog grant --catalog-config FILE NAME DATASET... -pchronicle serve catalog revoke --catalog-config FILE NAME DATASET... -``` - -Every listener must use a loopback address because the Web UI and read API do -not provide a public authentication boundary. - -## Open one Dataset - -```bash -pchronicle serve --open ./trajectory-data -``` - -A single bare Dataset is mounted as `default`. If no listener option is given, -the local Web UI uses an available loopback port. - -## Mount several Datasets - ```bash -pchronicle serve \ - --listen 127.0.0.1:8081 \ - evals=../data/atif archive=s3://example/archive -``` - -Mount names become SQL schema and API names. Use `NAME=DATASET` when a stable -name matters. -With several bare paths, pChronicle derives names from their last path -components. Those names can change with the paths, so reusable commands should -still set mount names explicitly. - -## Serve a path Directory - -```bash -pchronicle serve catalog dataset add \ - --catalog-config catalog.toml prod \ - --uri s3://bucket/prod \ - --endpoint http://127.0.0.1:9000 \ - --region us-west-2 \ - --access-key BACKEND_AK \ - --secret-key BACKEND_SK -pchronicle serve catalog issue --catalog-config catalog.toml alice -pchronicle serve catalog grant --catalog-config catalog.toml alice prod evals +pchronicle serve ./trajectory-data +pchronicle serve --listen 127.0.0.1:8080 train=./train eval=./eval pchronicle serve --catalog-config catalog.toml --listen 127.0.0.1:8081 ``` -`catalog.toml` lists libraries (`[datasets.*]`, each a path or `s3://` URI) and -users. `serve catalog dataset add|remove|list` rewrites libraries without -starting HTTP. `serve catalog issue` writes a user with empty grants and prints -the secret once on stdout; `grant` / `revoke` change which library names that -Use `*` as NAME to grant or revoke a dataset for every current user. New users do not inherit past wildcard grants; rerun the command after creating them. -user may open. Restart serve after editing the file. - -`pchronicle serve --catalog-config` mounts **every** library in the file into -Warehouse (same as positional mounts). It also enables Directory ticket routes -for `catalog://` pins. Do not combine `--catalog-config` with positional -Dataset mounts. Backend S3 endpoint, region, and keys from the file are applied -before stores open. The Web UI may send Directory user access/secret keys as -headers when you use catalog-authenticated flows. From another terminal: - -```bash -pchronicle dataset pin team catalog://127.0.0.1:8081 --ak USER_AK --sk USER_SK -pchronicle query @team/prod --sql 'SELECT 1' -``` - -`@team` is a Directory locator, not a Dataset. `@team/prod` fetches a ticket and -opens the ticket `uri` (a path). All `s3://` libraries in one Directory file must -share the same endpoint, region, and backend keys. The listener remains -loopback-only. Nested Dataset discovery may use `chronicle.manifest` sidecars -([RFC-0015](../../rfcs/0015-chronicle-manifest.md)). The Directory design is -specified in [RFC-0013](../../rfcs/0013-pchronicle-warehouse-catalog.md). - -## Enable Control or Gateway integration - -```bash -pchronicle serve \ - --control 127.0.0.1:0 \ - default=./trajectory-data - -pchronicle serve \ - --gateway auto \ - --gateway-dataset ./trajectory-data \ - --gateway-split '{user}/{date}/{hour}' -``` - -`--gateway` starts a config-free canonical event ingest endpoint. It accepts -`POST /v1/events`, uses `x-persisting-user-id` for `{user}`, and automatically -mounts the output Dataset. `{date}` and `{hour}` use UTC; one run/session is -pinned to its first partition so a streaming or long-lived trajectory is not -split across event sources. `auto` means `127.0.0.1:0`. Existing canonical -sources wait 30 minutes by default after their last event before Storyline -projection; override this with `--gateway-split-idle DURATION`. - -When a Warehouse listener is enabled, single-trace Gateway reads reopen the -latest canonical event manifest. Appending to an existing source therefore does -not wait for Snapshot refresh or Storyline projection; only new source files and -published projections require a global Snapshot update. - -Control requires a mount named `default`. `--control`, `--gateway`, or `--gateway-config` -without `--listen` starts the requested integration without also starting the -Web UI. The process writes one machine-readable readiness record to stdout; -Control credentials are not written to stderr. - -Mounted Datasets and HTTP operations are read-only. Import, export, maintenance, -and arbitrary filesystem access are not exposed through the API. Refreshes -replace the readable view only after the replacement is ready; a failed refresh -keeps the previous view available. - -Runs lists allow a short freshness delay. Successful summaries are reused by -Dataset and file scope. After 30 seconds, the next visit returns cached results -and starts a background refresh. At most one automatic Runs refresh runs per -process. Failures retain the previous result and wait at least 30 seconds before -retrying. The summary cache retains at most 32 scopes and 64 MiB in memory; -first access, process restart, or eviction still requires source discovery. -The existing Datasets browse cache remains persisted in local Lance storage. -Unscoped Runs requests also reuse their Catalog and refresh it in the background. -**Refresh** explicitly rebuilds the Catalog and invalidates scoped summaries on -success; an older in-flight build cannot republish an invalidated cache entry. -CLI `query`, query workers, and Gateway live reads bypass the summary cache. - -During startup the Web UI uses the lightweight `query/tables?ui=1` Catalog view. Dataset names and source counts come from the local Browse/manifest cache without triggering accurate Catalog discovery. It returns the `ui-cache` marker; Analysis compilation and SQL execution still bind to the current accurate Snapshot on the server. CLI and API requests without `ui=1` do not use this view. - -## Logs and failed requests - -`pchronicle serve` writes Warehouse request logs to stderr at `--log-level` -(default `info`), tracing target `pchronicle.serve`. Startup logs the listen -address, Dataset names, and Snapshot id. Each `/api` request logs method, -path, status, elapsed time, and a truncated query string. Query and compile -handlers also log truncated SQL. - -Failed responses include `code`, `message`, and `request_id`. The Web banner -shows the same `request_id`. Internal failures redact details in JSON; the -stderr ERROR line has `root_cause` and `chain` for that id. - -`--log-level error` keeps only internal failures. `--log-level` does not read -`RUST_LOG`. +The listener is loopback-only. Named mounts provide stable SQL aliases. +`--open` opens the Web UI; `--home-link TEXT=PATH` adds same-origin navigation +links. `--catalog-config` enables authenticated Directory and data operations +and cannot be combined with positional Dataset mounts. Configure datasets and +user grants with `pchronicle serve catalog`; storage credentials remain scoped +to each backend and worker. The server emits a readiness JSON record containing +`warehouse_endpoint`. -For Gateway behavior, continue with -[Gateway forwarding, rewriting, and capture](serve-gateway.md). For exact -flags, see the [`pchronicle` CLI reference](../reference/cli.md). Internal -refresh and versioning behavior belongs to [Snapshot design](../design/catalog.md). +Refresh builds a replacement Snapshot before switching readers. Failed refresh +keeps the previous view readable. Dataset writes remain CLI import operations. +Logs go to stderr; `--log-level` controls verbosity. Failed API requests include +`code`, `message`, and `request_id`; internal details remain in the server log. -Continue with the [local Web UI guide](ui.md) for task-oriented coverage of -Datasets, Runs, Analysis, Storage, and Assistant. +[CLI reference](../reference/cli.md) · [Web UI](ui.md) diff --git a/docs/src/en/pchronicle/reference/agenticmd.md b/docs/src/en/pchronicle/reference/agenticmd.md index 39d87e75f..8cfe194cd 100644 --- a/docs/src/en/pchronicle/reference/agenticmd.md +++ b/docs/src/en/pchronicle/reference/agenticmd.md @@ -67,9 +67,6 @@ frontmatter schema independent of Storyline. ## 4. Live updates -When live Markdown is enabled, Gateway projects visible dialogue into -AgenticMD while writing canonical Lance events: - 1. User blocks are written by `call_id`; 2. A streaming assistant updates in place with the same `call_id`; 3. Rewriting an assistant block must keep any following user blocks; @@ -83,13 +80,6 @@ does not change the canonical append result. ## 5. Relationship to Lance -Lance events own fidelity, replay, stats, and structured query. Internal -trajectory operations can rebuild AgenticMD from Lance, but the public -`pchronicle` CLI does not expose AgenticMD materialize or import subcommands. -AgenticMD does not compact or restore canonical events automatically. Public -exchange uses the formats supported by -[`pchronicle import/export`](cli.md). - ## 6. Examples and implementation - Lance/ATIF storage and analysis examples: `examples/pchronicle/` diff --git a/docs/src/en/pchronicle/reference/cases-platform.md b/docs/src/en/pchronicle/reference/cases-platform.md index 752498ec3..876ecd496 100644 --- a/docs/src/en/pchronicle/reference/cases-platform.md +++ b/docs/src/en/pchronicle/reference/cases-platform.md @@ -1,7 +1,7 @@ # pChronicle Directory and platform cases Platform-oriented Directory setup. The ACL file manages users, datasets -(libraries), and grants; Warehouse listen/Gateway options still come from +(libraries), and grants; Warehouse listen options still come from `pchronicle serve`. ## P01: Issue a Directory user from an empty config diff --git a/docs/src/en/pchronicle/reference/cli.md b/docs/src/en/pchronicle/reference/cli.md index 6ace554d3..fee6873d3 100644 --- a/docs/src/en/pchronicle/reference/cli.md +++ b/docs/src/en/pchronicle/reference/cli.md @@ -303,78 +303,24 @@ pchronicle agent claude @prod --ask 'Compare model latency' ### Serve -```text -pchronicle serve - [--listen LOOPBACK_ADDR] [--control LOOPBACK_ADDR] [--open] - [--home-link TEXT=PATH]... - [--gateway ADDRESS --gateway-dataset DATASET [--gateway-split TEMPLATE] - [--gateway-split-idle DURATION]] - [--gateway-config FILE --gateway-dataset DATASET [--gateway-state DIRECTORY]] - [--gateway-stream-markdown] [--gateway-debug] - [--catalog-config FILE] - [<[NAME=]DATASET> ...] -pchronicle serve catalog dataset add --catalog-config FILE NAME --uri URI [OPTIONS] -pchronicle serve catalog dataset remove --catalog-config FILE NAME... -pchronicle serve catalog dataset list --catalog-config FILE -pchronicle serve catalog issue --catalog-config FILE NAME -pchronicle serve catalog grant --catalog-config FILE NAME DATASET... -pchronicle serve catalog revoke --catalog-config FILE NAME DATASET... -``` - ```bash pchronicle serve ./trajectory-data -pchronicle serve \ - --gateway auto \ - --gateway-dataset ./trajectory-data \ - --gateway-split '{user}/{date}/{hour}' +pchronicle serve --listen 127.0.0.1:8080 train=./train eval=./eval +pchronicle serve --catalog-config catalog.toml --listen 127.0.0.1:8081 ``` -Every listener must use a loopback address. A bare single Dataset is mounted as -`default`; with several Datasets, use `NAME=DATASET` when a stable mount name is -needed. Control requires a mount named `default`. -Repeatable `--home-link TEXT=PATH` adds homepage nav capsules beside Warehouse. -`PATH` must be a same-origin relative path such as `/plugins`. -`--catalog-config FILE` authenticates every data API request and dispatches it to -an isolated exec worker containing only authorized mounts. It also enables -`catalog://` locators, and conflicts with positional mounts, Gateway, and Control. Pair Directory clients with -`dataset pin NAME catalog://127.0.0.1:PORT --ak --sk`. -`pchronicle serve catalog dataset add|remove|list` and -`issue|grant|revoke` rewrite that file and do not start HTTP; `issue` prints -the user secret once. Restart serve after changing libraries, users, or grants. -The pool reuses idle workers in the same authenticated scope and grows on demand -when they are busy, up to 4 workers per scope and 8 workers server-wide. At capacity, -requests wait for available capacity rather than a particular busy worker; at most -32 requests are admitted. Each worker still executes serially to keep IPC isolated. -A cleanup task runs every 30 seconds and reclaims workers idle for 120 seconds; -idle workers from other scopes may be reclaimed sooner when global capacity is full. -Queueing, startup and execution share a 60-second timeout; body reads have a -10-second timeout. Overload returns 503. Cancellation during execution or IPC -failure discards that worker; cancelling a queued request does not interrupt -other requests. Worker and disk -cache identity includes the user, grants and backend credential version. Caches -live under `PCHRONICLE_CACHE_DIR/workers/` or the system pchronicle cache directory. -Workers receive backend keys over private IPC before starting runtime threads; -they do not inherit AWS environment, profiles or the server's home configuration. -Login AK/SK authenticate the user; dataset AK/SK authenticate storage access. -Datasets with different S3 endpoints or credentials require explicit `dataset` -selection; cross-credential queries are currently rejected. Health, UI config -and static pages remain public; data APIs require authentication even for a -catalog with no users. Processes still use the server's OS identity: this is not -an OS privilege drop or a filesystem sandbox. - -`catalog` is a reserved `serve` subcommand; mount a path of that name as -`./catalog`. -See [RFC-0013](../../rfcs/0013-pchronicle-warehouse-catalog.md) and -[RFC-0015](../../rfcs/0015-chronicle-manifest.md) for nested discovery sidecars. -The config-free Gateway accepts canonical trajectory events at -`POST /v1/events`. `--gateway-dataset` is an output URI and is auto-mounted; -it is no longer a mounted Dataset name. Split templates accept the exact -placeholders `{user}`, `{date}`, and `{hour}`. Existing canonical sources wait -30 minutes by default after their last event before automatic Storyline -projection; override this with `--gateway-split-idle DURATION`. -In Gateway mode, the Warehouse's single-trace event, Storyline, and trajectory -endpoints read the latest canonical manifest for an already discovered source, -so active traces do not wait for projection or a global Snapshot refresh. +The listener is loopback-only. Named mounts provide stable SQL aliases. +`--open` opens the Web UI; `--home-link TEXT=PATH` adds same-origin navigation +links. `--catalog-config` enables authenticated Directory and data operations +and cannot be combined with positional Dataset mounts. Configure datasets and +user grants with `pchronicle serve catalog`; storage credentials remain scoped +to each backend and worker. The server emits a readiness JSON record containing +`warehouse_endpoint`. + +Refresh builds a replacement Snapshot before switching readers. Failed refresh +keeps the previous view readable. Dataset writes remain CLI import operations. +Logs go to stderr; `--log-level` controls verbosity. Failed API requests include +`code`, `message`, and `request_id`; internal details remain in the server log. ## Output and exit status diff --git a/docs/src/en/pchronicle/reference/formats/index.md b/docs/src/en/pchronicle/reference/formats/index.md index 9c18ebc97..e1e92e45b 100644 --- a/docs/src/en/pchronicle/reference/formats/index.md +++ b/docs/src/en/pchronicle/reference/formats/index.md @@ -2,7 +2,6 @@ | Format | Role | Specification | | --- | --- | --- | -| Events | Recorded HTTP-first event format | [RFC-0002](../../../rfcs/0002-events-format.md) | | Storyline | Internal normalized Run and tool-call model | [RFC-0001 § Wire schema](../../../rfcs/0001-storyline-format.md#wire-schema) | | ACTF | JSON Run interchange | [RFC-0004 § JSON Pointer mapping](../../../rfcs/0004-actf-format.md#actf-storyline-json-pointer-mapping) | | ATIF | Agent Run interchange | [RFC-0008 § JSON Pointer mapping](../../../rfcs/0008-atif-format.md#atif-storyline-json-pointer-mapping) | @@ -11,9 +10,7 @@ | Claude Code | Local Claude Code transcript JSONL (`~/.claude/projects/**/*.jsonl`). Decode-only. | — | | AgenticMD | Human-readable live and materialized view | [AgenticMD reference](../agenticmd.md) | -Recorded events and reconstructed projections have different fidelity and ownership. -See [Run storage design](../../design/trajectory-storage.md) before choosing a -format as a persistence boundary. +Storyline is the normalized trajectory model. Each format RFC is authoritative for that format's wire contract and its mapping to Storyline. Mapping tables elsewhere are non-normative. diff --git a/docs/src/en/pchronicle/reference/query-model.md b/docs/src/en/pchronicle/reference/query-model.md index e0c7dc169..8146b61f7 100644 --- a/docs/src/en/pchronicle/reference/query-model.md +++ b/docs/src/en/pchronicle/reference/query-model.md @@ -10,7 +10,6 @@ relations even when no compatible Source contributes rows. | `runs` | one normalized Run/session | every ready run data source | | `steps` | one normalized Step | every ready run data source | | `tool_calls` | one tool invocation and its linked result | every ready run data source | -| `events` | one canonical write-time fact | canonical event Sources only | | `trajectories` | one complete Run with ordered Step and tool aggregates | normalized run data sources | Use `DESCRIBE` for the exact columns exposed by the installed version: @@ -20,13 +19,12 @@ DESCRIBE dataset.sources; DESCRIBE dataset.runs; DESCRIBE dataset.steps; DESCRIBE dataset.tool_calls; -DESCRIBE dataset.events; DESCRIBE dataset.trajectories; ``` ## Source identity -Entity IDs are Source-local. `runs`, `steps`, `tool_calls`, and `events` retain +Entity IDs are Source-local. `runs`, `steps`, `tool_calls` retain `_file_`, the Dataset-relative `source_path`. A durable entity address includes the Dataset URI, `_file_`, entity kind, and original ID. @@ -54,9 +52,6 @@ namespace. | `format` | UTF-8, nullable | detected or declared representation | | `kind` | UTF-8, non-null | `store` or `file` | | `snapshot_ref` | UTF-8, nullable | generation, manifest revision, fingerprint, version, or ETag | -| `projection_status` | UTF-8, nullable | `fresh` or `stale` for a canonical events Source with a linked Storyline projection | -| `projection_generation` | UTF-8, nullable | generation selected as the read acceleration projection | -| `projection_candidates` | UInt64, non-null | number of linked projection candidates considered | | `size_bytes` | UInt64, nullable | candidate file or marker-object size | | `last_modified` | UTF-8, nullable | RFC 3339 timestamp when available | | `status` | UTF-8, non-null | `ready` or `error` | diff --git a/docs/src/en/pchronicle/reference/terminology.md b/docs/src/en/pchronicle/reference/terminology.md index 9fbac2b7f..01d0d1cc1 100644 --- a/docs/src/en/pchronicle/reference/terminology.md +++ b/docs/src/en/pchronicle/reference/terminology.md @@ -9,24 +9,18 @@ task-oriented documentation: | **Source file** | One input file or storage source inside a Dataset. | | **Run** | One stored Agent execution record. A Session ID identifies the record; an optional Run ID may group it with runtime activity. | | **Step** | One ordered user, agent, system, or tool-related entry in a Run. | -| **Event** | A low-level recorded or reconstructed item linked to a Step. | | **Result** | Rows returned by a query or analysis. | | **Snapshot** | The write/read sync protocol after a path is opened: which Sources exist and which version each is pinned to for one operation. | -The UI uses **Recorded events** for events captured directly from a runtime and -**Reconstructed events** for events derived from imported run data. It never -labels reconstructed events as raw data. - The following terms are reserved for technical documentation and APIs: - **Storyline** is the normalized storage and exchange model behind Runs and Steps. -- **Canonical Event** is the append-only recorded event model. - **Directory** is the optional platform locator that resolves a name to a path and issues a ticket. CLI flags and HTTP paths may still say `catalog`. - **Snapshot** (in APIs, sometimes still `DatasetCatalogSnapshot`) pins Source membership and versions. It is not the Directory listing. -- **projection**, **revision**, **fragment**, and **column page** describe +- **revision**, **fragment**, and **column page** describe storage and consistency mechanisms, not primary user workflows. Dataset pins (`@name`), Warehouse mount names, and Directory library names are diff --git a/docs/src/en/project/engineering.md b/docs/src/en/project/engineering.md index 4a1c484a3..39a687001 100644 --- a/docs/src/en/project/engineering.md +++ b/docs/src/en/project/engineering.md @@ -10,7 +10,7 @@ Run these from the repository root. `just --list` shows the full recipe set. | Command | What it does | |---|---| -| `just test` | Workspace Rust tests through `cargo nextest`, then the Python suite | +| `just test` | pChronicle and CLI tests through `cargo nextest`, then in-scope Python tests | | `just test ` | One crate or Cargo package (for example `pchronicle` or `persisting-pchronicle`) | | `just docs-sync` | Install the locked documentation environment | | `just docs-serve` | Local Zensical preview with automatic reload when files change | @@ -19,14 +19,16 @@ Run these from the repository root. `just --list` shows the full recipe set. | `just examples` | pChronicle product example suite | | `just dev` | Apply formatting, lint, then run Rust tests | | `just ci` | Check lint without rewriting files, run Rust/Python tests and property tests, then build | -| `just check-quick` | Check core runtime crates and pChronicle without default features | +| `just check-quick` | Check pChronicle CLI and pChronicle without default features | `just test` uses the debug nextest profile for faster iteration. Pass a Cargo package name or a short crate alias (`pchronicle`, -`pchronicle-cli`, `agentctl`, `capture`). `just test pchronicle` runs both +`pchronicle-cli`). `just test pchronicle` runs both `persisting-pchronicle` and `persisting-pchronicle-cli` (same as the CI pchronicle shard); use `just test pchronicle-cli` for the CLI crate alone. -The no-argument form also runs `just test-py`. +The no-argument form also runs `just test-py` for packaging, trajectory row +contracts, and benchmark reports. Python format/lint cover their tests and the +pChronicle build, documentation, and case scripts, following `AGENTS.md`. ## Current notes @@ -49,7 +51,7 @@ Local and ordinary CI builds use the platform's default linker. Linux wheels use the manylinux_2_28 image (glibc 2.28) so rustc libstd can link `statx` / `copy_file_range`. -`just check-quick` checks core runtime crates and pChronicle without default +`just check-quick` checks the pChronicle CLI and pChronicle without default features. `just dev` formats files before lint and Rust tests; `just ci` uses read-only lint checks and also runs Python and property tests. The GitHub Actions workflow additionally covers platform shards, Web builds, S3, and examples. diff --git a/docs/src/en/rfcs/0001-storyline-format.md b/docs/src/en/rfcs/0001-storyline-format.md index 3db28742f..dbacc029a 100644 --- a/docs/src/en/rfcs/0001-storyline-format.md +++ b/docs/src/en/rfcs/0001-storyline-format.md @@ -7,7 +7,6 @@ | **Date** | 2026-07-30 | | **Component** | pChronicle (`persisting-pchronicle`) | | **Implements** | `crates/persisting-pchronicle/src/formats/storyline.rs` | -| **Related** | [RFC-0002 Events](0002-events-format.md) · [RFC-0004 ACTF](0004-actf-format.md) · [RFC-0008 ATIF](0008-atif-format.md) · [RFC-0009 OpenAI Messages](0009-openai-messages-format.md) | --- @@ -15,16 +14,8 @@ **Storyline** 是 pChronicle 的**枢纽 interchange**：以 Harbor **ATIF-v1.7** 的 Trajectory / Step 折叠语义为基准，加上少量 hub 便利字段（短名 wire、性能顶栏、可选子会话外链）。 -捕获侧：`HTTP → events 流`（可 **记录** 到 `events.lance`，也可 **触发** handler）；handler 经 storyline 做格式转换与落盘。外围格式（`agenticmd` / `openai_msg` / `atif` / `actf`）**只与 storyline 互转**。 - -Storyline **不是**事实源。Canonical 事实仍是 `events.lance`。 - -```text -events ──┐ -agenticmd ┼──► storyline ──► … -openai_msg┤ -atif ─────┘ -``` +外围格式（`agenticmd` / `openai_msg` / `atif` / `actf`）只与 Storyline 互转。 +Storyline 是存储与查询管线使用的轨迹模型，可直接写入三表 Lance。 --- @@ -32,8 +23,7 @@ atif ─────┘ 1. **ATIF-first**：根对象 ≈ Trajectory，`turns[]` ≈ `steps[]`；能 1:1 的字段保持同义。 2. **Hub-only**：A→B MUST 经 storyline。 -3. **少叙事**：不引入 Capture Call / 相位 / event 回链等运行时读模型；那些留在 events。 -4. **Not SoT**：回放与审计以 events 为准。 +3. **来源可追溯**：保留 Source 与 Snapshot 身份，不推断输入中缺失的执行记录。 相对 ATIF 仅保留这些增量： @@ -243,7 +233,6 @@ from_storyline(format, story) → serialized convert(from, to, input) ≡ from_storyline(to, into_storyline(from, input)) ``` -`events` 为 Lance-only：字符串 convert API 对 `ChronicleFormat::Events` 报错；内存用 `events_to_storyline` / `storyline_to_events`。 --- diff --git a/docs/src/en/rfcs/0002-events-format.md b/docs/src/en/rfcs/0002-events-format.md deleted file mode 100644 index 35f98b050..000000000 --- a/docs/src/en/rfcs/0002-events-format.md +++ /dev/null @@ -1,585 +0,0 @@ -# RFC-0002: Events Format（`events`） - -| Field | Value | -|---|---| -| **Status** | Accepted | -| **Schema / format name** | `events`（逻辑文档）；物理存储常为 `events.lance` | -| **Date** | 2026-07-30 | -| **Component** | `persisting-events` + Gateway + pChronicle | -| **Implements** | `persisting-events::EventRecord` · `persisting-pchronicle` `formats/events.rs` / `EventRow` | -| **Related** | [RFC-0001 Storyline](0001-storyline-format.md) · [RFC-0007 Events/Sidecar 边界](0007-events-contract-pchronicle-sidecar.md) · [轨迹存储](../pchronicle/design/trajectory-storage.md) | - ---- - -## 摘要 - -> **Ownership amendment:** [RFC-0007](0007-events-contract-pchronicle-sidecar.md) -> 将存储无关的 `EventRecord` 契约迁至 `persisting-events`。本 RFC 继续规定 events 的 -> wire 内容与物理表示；pChronicle 继续独占 `EventRow` 和持久化实现。 - -**`events`** 是 Persisting 轨迹的 **canonical 事实流（SoT）**：按时间（`seq`）追加的底层记录，**优先刻画原始 HTTP（或等价传输层）交换**，而不是已经折叠好的对话轮次。 - -设计取向： - -> 先保住「线上真实发生过什么」（方法、URL、头、正文、状态、时序），再由上层（Storyline / Markdown / ATIF）去解释「这对人意味着什么」。 - -因此： - -| 层 | 角色 | 可否有损 | -|---|---|---| -| **`events`** | 原始交换日志（HTTP-first） | **SoT，目标可回放** | -| **`storyline`** | Normal / 互操作枢纽 | 从 events 投影时有损；自身存储必须无损 | -| `agenticmd` / `openai_msg` / `atif` / `actf` | 外围视图 / 训练交换 | 同源 Storyline 往返必须无损；跨格式仅承诺目标可表达语义 | - -`events` 进出其它 chronicle 格式时仍 **只经 storyline**；但 **回放、重放、审计、协议级恢复 MUST 读 events**，MUST NOT 依赖 storyline roundtrip。 - -本 RFC 规定： - -1. HTTP-first 的定位与分层职责； -2. 逻辑事件 `EventRecord`、信封 `EventsDocument`、Lance `EventRow`； -3. `payload` 的 wire 字段约定（request / response）； -4. 与 Storyline 的投影关系（摘要字段只是加速，不是真相）。 - ---- - -## 动机 - -Agent 轨迹若只存「user/assistant 文本 + tool_calls」，会丢掉： - -- 真实 URL / method / status / header（鉴权、路由、provider 切换）； -- 原始 body（含协议特有字段、tool schema、多模态 parts）； -- 流式分片、重试、取消、非 2xx、内部探测流量； -- 足以 **原样重放 HTTP** 或 **换协议解析器重解** 的材料。 - -Persisting Gateway 的主入口是代理流量。`events` 应对齐这一现实：**一行事件 ≈ 一次可定位的 HTTP 方向记录**（request 或 response），外加关联键（`session_id` / `call_id` / `trace_id`）。 - -对话摘要（`user_content` / `assistant_content`）可以写进 payload 作为 **可选索引/加速字段**，但： - -- MUST NOT 取代 `http.request_body` / `http.response_body`（或等价 wire 字段）成为唯一正文； -- 当摘要与 wire body 冲突时，**以 wire body 为准**。 - ---- - -## 设计目标 - -| 目标 | 说明 | -|---|---| -| **HTTP-first SoT** | 默认按原始请求/响应保存，使回放与再解析成为一等能力 | -| **Replayable** | 在凭证策略允许的前提下，应能从 events 重建「对同一 endpoint 再发一次等价请求」所需信息 | -| **Re-derivable views** | Storyline / Markdown / ATIF 视为可从 events **重新投影**的视图 | -| **Correlation envelope** | 顶栏保留 `session_id` / `call_id` / `trace_id` 等关联键，不把故事语义塞进 wire | -| **Append-only** | `seq` 单调；不原地改写既有 wire payload | -| **Hub via storyline** | 与其它外围格式互转 MUST 经 storyline；无损路径仍是读 events | - -非目标： - -- 不把 events 做成 ATIF step / Storyline Turn 的镜像； -- 不要求每条事件都是「对人友好的一句对话」； -- 不在本 RFC 规定 TLS 密钥、cookie 明文等敏感材料的强制落盘策略（见安全节）。 - ---- - -## 分层：原始交换 vs 故事投影 - -```text - ┌─────────────────────────────┐ - proxy / import → │ events 流（HTTP wire） │ - └──────────────┬──────────────┘ - │ - ┌──────────────┼──────────────┐ - ▼ ▼ - 记录 append 触发 handlers - │ │ - ▼ ▼ - events.lance storyline（hub） - （SoT / replay） │ - ┌─────────────┼─────────────┐ - ▼ ▼ ▼ - agenticmd openai_msg atif - └──────── 落盘 / 物化 ──────┘ -``` - -| 问题 | 应落在 | -|---|---| -| 「当时请求的 path / body 是什么？」 | **events** | -| 「这一轮对用户说了什么？」 | storyline / agenticmd | -| 「能否拿去 SFT？」 | atif / openai_msg（经 storyline） | -| 「能否按 call_id 重放 upstream？」 | **events** | - -与既有 Capture 表述的关系：协议差异可以在 **进入 Story 读模型之前**消化；但 **进入 events 之前不应过度消化**——解析器可以后置、可升级，原始字节/JSON body 应尽量留下。 - ---- - -## 术语 - -| 术语 | 含义 | -|---|---| -| **Wire record** | 一次 HTTP（或同类）方向的原始交换描述：method/url/headers/body/status/timing… | -| **EventRecord** | 逻辑事件 = 关联信封 + `kind` + wire-oriented `payload` | -| **EventRow** | Lance 列存行：索引列 + `payload_json` | -| **Summary fields** | `user_content` / `assistant_content` / `model` 等可选派生字段 | -| **Replay** | 用 wire 字段重建并再次发出（或模拟）等价交换 | - ---- - -## Schema：逻辑事件 `EventRecord` - -编码：UTF-8 JSON object。`EventRecord` 由存储无关的 `persisting-events` 唯一定义； -Gateway 与 pChronicle 直接使用同一类型。pChronicle 独占由该逻辑记录派生的 -物理 row schema 与存储实现。 -一行事件 MUST 包含 `seq`、`source`、`kind`、`payload`。 - -### 关联信封（顶栏） - -| Field | Type | Status | Description | -|---|---|---|---| -| `event_id` | string | Required for new writes | append 边界生成的稳定事件身份 | -| `run_id` | string | Required for new writes | 事件所属逻辑 Run；旧 capture row 读取时可缺省 | -| `attempt_id` | string | Optional | 生命周期事件所属 Attempt | -| `storyline_id` / `turn_id` | string | Optional | narrative 维度，与 Attempt 正交 | -| `timestamp_unix_ms` | integer | Required for new writes | 机器可比较的观测时间；与 `timestamp` 表示同一时刻 | -| `producer` | string | Required for new writes | 产生该记录的组件 | -| `seq` | integer | Required | dataset / 会话内单调序号 | -| `source` | string | Required | 如 `persisting-proxy`、`persisting-gateway` | -| `kind` | string | Required | 见 kind 节；HTTP 方向优先 | -| `timestamp` | string | Required for new writes | RFC3339 UTC（事件观测时间；与 `timestamp_unix_ms` 毫秒级一致） | -| `session_id` | string | Optional | ≈ Storyline `story_id` | -| `agent_id` | string | Optional | | -| `call_id` | string | Optional | 一次调用关联键（request/response 配对） | -| `trace_id` | string | Optional | | -| `parent_call_id` | string | Optional | 调用树 | -| `parent_uuid` | string | Optional | | -| `subagent_id` / `parent_agent_id` / `branch` | string | Optional | 子代理 / 分支 | - -顶栏 **SHOULD NOT** 承载完整对话文本；正文在 `payload` 的 wire 字段中。 - -新写入事件必须同时提供 `timestamp` 与 `timestamp_unix_ms`。Gateway capture sink 和 -事件 producer 负责提供时间字段；admission 仅为旧记录或兼容导入补齐缺失值。 -事件顺序仍由 `source + seq` 定义，时间戳用于关联和展示。 - -### `payload`：HTTP-first wire 对象 - -推荐统一放在 `payload.http`（或顶层扁平兼容键，见兼容节）。**完整回放**所需字段： - -#### Request 方向（`kind` = `http.request` 或兼容 `llm.request`） - -| Field | Type | Status | Description | -|---|---|---|---| -| `http.method` | string | Required | `GET` / `POST` / … | -| `http.url` | string | Required | 完整 URL 或 path+query；与 `host` 可拆分 | -| `http.path` | string | Recommended | 规范化 path（便于索引） | -| `http.query` | object / string | Optional | | -| `http.headers` | object \| array | **Required**（记录时） | **完整请求头**（可脱敏）；与 body 同为回放必需 | -| `http.request_body` | string \| object \| null | Recommended | **原始请求体**（JSON 对象或 base64/utf8 字符串） | -| `http.body_encoding` | string | Optional | `json` \| `utf8` \| `base64` \| `sse-wire` … | -| `http.content_type` | string | Optional | | -| `timing.started_at` | string | Optional | | -| `timing.duration_ms` | number | Optional | | - -#### Response 方向（`kind` = `http.response` / `http.response.stream` 或兼容 `llm.response*`） - -| Field | Type | Status | Description | -|---|---|---|---| -| `http.status` | integer | Required | HTTP 状态码 | -| `http.headers` | object \| array | **Required**（记录时） | **完整响应头**（可脱敏） | -| `http.response_body` | string \| object \| null | Recommended | **原始响应体**或流式拼接线 | -| `http.body_encoding` | string | Optional | 含 `sse` / `sse-wire` | -| `http.truncated` | boolean | Optional | 是否流式 | -| `timing.ttft_ms` / `timing.duration_ms` | number | Optional | | - - -#### 连接与客户端（长连接 / 对端身份） - -HTTP-first 回放不仅需要单次 request/response，还需要知道 **连接是否复用** 以及 **谁连上来**。记录时 MUST 尽量写入： - -| Field | Type | Status | Description | -|---|---|---|---| -| `connection.http_version` | string | Recommended | 如 `HTTP/1.1`、`HTTP/2.0` | -| `connection.persistent` | boolean | Recommended | 是否按协议/头判定为 keep-alive / 长连接 | -| `connection.connection_header` | string | Optional | 原始 `Connection` 头（如 `keep-alive` / `close`） | -| `connection.keep_alive` | string | Optional | 原始 `Keep-Alive` 头（超时/max 等） | -| `connection.upgrade` | string | Optional | `Upgrade`（如 `websocket`） | -| `client.peer` | string | **Required**（代理路径） | 客户端 `ip:port`（`ConnectInfo`） | -| `client.peer_ip` | string | Recommended | | -| `client.peer_port` | integer | Recommended | | -| `client.pid` | integer | Optional | 本机 peer 反查进程 PID（best-effort） | -| `client.command` | string | Optional | 进程命令行 | -| `client.machine_fp` | string | Optional | 机器指纹 | -| `client.user_agent` | string | Optional | 从请求头抽取，便于无 PID 时对照 | - -判定 `persistent` 的建议规则： - -1. 若 `Connection: close` → `false`； -2. 若 `Connection` 含 `keep-alive` → `true`； -3. 否则：`HTTP/1.1+` 默认 `true`，`HTTP/1.0` 默认 `false`； -4. `Upgrade: websocket` 等升级连接单独用 `upgrade` 标明，不假装成普通 keep-alive 复用。 - -说明： - -- 会话级 `session-meta.yaml`（`SessionClientMeta`）可继续存在；**事件级**仍 MUST 冗余写入 `client.*`，以便单条 event 可自描述回放。 -- hop-by-hop 头（`Connection` / `Keep-Alive`）转发上游前可能被剥离，但 **events 记录 MUST 保留客户端原始值**。 -- 上游侧长连接信息可从 response `headers` 的 `Connection`/`Keep-Alive` 再读；可另写 `connection.upstream_*`（Optional）。 - -#### 可选摘要 / 索引（非 SoT） - -| Field | Type | Status | Description | -|---|---|---|---| -| `model` | string | Optional | 从 body/header 抽取，便于 Lance 列 | -| `protocol` | string | Optional | 如 `chat.completions`、`anthropic.messages` | -| `provider` | string | Optional | | -| `user_content` | string | Optional | 可见 user 摘要；冲突时以 request_body 为准 | -| `assistant_content` | string | Optional | 可见 assistant 摘要；冲突时以 response_body 为准 | -| `usage` | object | Optional | token 等；能从 body 再解析则视为缓存 | -| `_tlv` | object | Optional | 自 Markdown compact 注入 | - -**Normative**： - -1. 记录 HTTP 交换时，实现 MUST **同时**持久化： - - **headers**（`payload.http.headers` 或兼容扁平键 `payload.headers`）； - - **body**（在 `capture_level` 允许完整 body 时：`request_body` / `response_body` 或等价 `body`）； - - **连接与客户端**：`connection.*`（至少能推导 `persistent`）与 `client.peer`（代理路径必填）。 -2. headers 与 body 同为回放材料：缺 headers 的事件与缺 body 一样，MUST 视为 **degraded**（`payload.degraded = true`），MUST NOT 宣称可完整 HTTP 回放。 -3. 敏感头（`Authorization` / `Cookie` / `Set-Cookie` / `X-Api-Key` 等）MUST 按红线脱敏或外置，并设置 `headers_redacted: true`；脱敏后仍 MUST 保留头名字面量（值为占位符），以便知道当时发过哪些头。 -4. Storyline / 训练格式导出 MUST 容忍从 wire body + headers 再解析；MUST NOT 假定摘要字段永远存在。 - -### 内存批 `EventsDocument`（非正式文件格式） - -| Field | Type | Status | Description | -|---|---|---|---| -| `format` | string | Required | `"events"` | -| `events` | array | Required | `EventRecord[]` | -| `session_id` / `agent_id` | string | Optional | 文档级默认 | - -### 物理行 `EventRow`（Lance） - -| Column | 说明 | -|---|---| -| `seq` / `timestamp` / `kind` / `source` | 索引 | -| `session_id` / `agent_id` / `call_id` / `trace_id` / `parent_call_id` | 关联过滤 | -| `model` | 可选反规范化 | -| `payload_json` | **完整 EventRecord JSON（含 wire）** | - -身份冲突采用 admission-first 合同：写入坐标不一致不会拒绝事实。`payload_json` 保留 -producer 原始声明；物理 `session_id` / `agent_id` 用于路由、过滤并在 replay 时覆盖同名 -声明。Storyline 投影对剩余冲突按物理 append order 采用最后一个非空值。该规则不需要 -读前写、去重表或额外索引。 - ---- - -## `kind` 约定 - -### 推荐（HTTP-first） - -| kind | 含义 | -|---|---| -| `http.request` | 一次 outbound/inbound HTTP 请求观测 | -| `http.response` | 对应响应（非流或已聚合） | -| `http.response.stream` | 流式响应片段或草稿聚合 | -| `http.cancel` | 取消 / 中断 | -| `session.started` / `session.ended` | 会话生命周期（可无 http 块） | -| `note` | 运维注解 | - -### LLM producer kinds - -| 现有 kind | 视为 | -|---|---| -| `llm.request` | 带 LLM 协议理解的 HTTP 请求（payload 仍应尽量含 path/body） | -| `llm.response` | `http.response` | -| `llm.response.stream` | `http.response.stream` | - -通用 HTTP producer 使用 `http.*`，Gateway LLM producer 可以使用 `llm.*`。 -`llm.*` **不**意味着可以只存对话摘要，仍按 HTTP-first 要求保存 wire。 - ---- - -## 回放与恢复（Replay） - -从一对共享 `call_id` 的 request/response 事件，消费者 SHOULD 能恢复： - -1. **HTTP 重放**：method + url + headers′ + body → 得到新的 response（headers′ 经红线剥离）； -2. **协议再解析**：换用新的 Anthropic/OpenAI/… 解析器从 body 重建 messages / tool_calls； -3. **视图重建**：重新投影 Storyline / Markdown / ATIF； -4. **时序重建**：按 `seq` / timestamp 恢复并发与流式顺序。 - -无法回放的常见原因（实现 MUST 在文档/元数据中可诊断）： - -- body 因 `capture_level` 被省略（`degraded`）； -- 敏感头被剥离且无替代凭证注入通道； -- 仅存 SSE 摘要未存 wire 帧； -- 只从 storyline 合成的「假 events」（无原始 wire）。 - ---- - -## 安全与红线 - -HTTP-first **不**等于无条件落盘密钥。 - -| 类别 | 默认建议 | -|---|---| -| `Authorization` / API key / cookie | 红线：哈希、删除或外置 secret store；回放时再注入 | -| 用户内容 / 工具结果 | 按产品保留策略 | -| 多模态像素 | 可旁路对象存储；events 内留引用与 hash | - -`payload.http.headers_redacted: true`（可选）表示头已经过红线处理。 - ---- - -## Wire 形态与探测 - -| 形态 | 地位 | -|---|---| -| **Lance `events.lance/`** | **唯一一等存储 / 采集格式（SoT）** | -| JSONL / JSON / `EventsDocument` 序列化 | **非正式**：仅调试导出或测试夹具；**不是** chronicle 互操作格式 | - -pChronicle：`into_storyline` / `convert` **不接受** events 的字符串输入；从 Lance 加载为内存 `EventRecord` 后再 `events_to_storyline`。 - -探测： - -| 方式 | 规则 | -|---|---| -| 路径 | 仅 `events.lance`（或 basename 含 `event` 的 `.lance`） | -| 内容 | **不**把 EventRecord 形 JSON 判定为 `events` | - -别名：`events` / `lance` / `bin` / `events.lance`。 - - ---- - -## 与 Storyline 的关系 - -```text -events (HTTP wire) ──interpret──► storyline ──synthesize──► events′ (often degraded) -``` - -| 方向 | 期望 | -|---|---| -| events → storyline | 从 wire（及可选摘要）折叠 Turn/Call；填写 `event_seqs` | -| storyline → events′ | 合成最小交换；**通常 degraded**，除非 storyline 显式携带完整 wire（不默认） | - -规范性： - -1. 互转其它外围格式 MUST 经 storyline。 -2. 宣称「可回放」时 MUST 基于非 degraded 的原始 events。 -3. 折叠器 SHOULD 优先解析 `http.request_body` / `http.response_body`，摘要字段仅作快路径。 - ---- - -## 兼容现有 Capture payload - -今日实现常见扁平键（仍合法）： - -| 现有键 | HTTP-first 对应 | -|---|---| -| `headers` | `http.headers`（**记录 MUST**） | -| `path` | `http.path` / `http.url` | -| `body` | request 的 `http.request_body` 或 response 的 `http.response_body` | -| `status` | `http.status` | -| `model` / `protocol` / `provider` | 摘要/索引 | -| `user_content` / `assistant_content` | 摘要 | - -读路径 MUST 接受扁平键；写路径 SHOULD 逐步迁移到 `payload.http.*`，或同时写两套直到迁移完成。 - ---- - -## 示例 A：HTTP-first 请求事件 - -```json -{ - "seq": 0, - "source": "persisting-proxy", - "kind": "http.request", - "timestamp": "2026-01-01T00:00:00Z", - "session_id": "sess-1", - "agent_id": "agent-a", - "trace_id": "tr-1", - "call_id": "c1", - "payload": { - "http": { - "method": "POST", - "url": "https://api.openai.com/v1/chat/completions", - "path": "/v1/chat/completions", - "headers": { - "content-type": "application/json", - "authorization": "", - "connection": "keep-alive", - "keep-alive": "timeout=60, max=1000", - "user-agent": "claude-cli/1.0" - }, - "headers_redacted": true, - "content_type": "application/json", - "body_encoding": "json", - "request_body": { - "model": "gpt-4o", - "messages": [{ "role": "user", "content": "ping" }] - } - }, - "connection": { - "http_version": "HTTP/1.1", - "persistent": true, - "connection_header": "keep-alive", - "keep_alive": "timeout=60, max=1000" - }, - "client": { - "peer": "127.0.0.1:54321", - "peer_ip": "127.0.0.1", - "peer_port": 54321, - "pid": 4242, - "command": "claude --print", - "user_agent": "claude-cli/1.0" - }, - "model": "gpt-4o", - "protocol": "chat.completions", - "user_content": "ping" - } -} -``` - ---- - -## 示例 B：Storyline → Events 映射（合成多为 degraded） - -key = events 字段，value = 在 Storyline 上求值的 JSONPath。 -**注意**：从 storyline 合成的 body 通常只是对话投影，不是原始 HTTP wire；应设 `payload.degraded = true`，除非另有 wire 旁路。 - -```json -{ - "format": "events", - "session_id": "$.story_id", - "agent_id": "$.agent.id", - "events": [ - { - "seq": null, - "source": null, - "kind": null, - "timestamp": "$.turns[0].timestamp", - "session_id": "$.story_id", - "agent_id": "$.agent.id", - "call_id": "$.turns[0].calls[0].call_id", - "trace_id": "$.turns[0].calls[0].trace_id", - "parent_call_id": "$.turns[0].calls[0].parent_call_id", - "payload": { - "degraded": null, - "http": { - "method": null, - "url": null, - "path": null, - "headers": null, - "request_body": "$.turns[0].calls[0].messages", - "body_encoding": null - }, - "model": "$.turns[0].calls[0].model", - "protocol": "$.turns[0].calls[0].protocol", - "user_content": "$.turns[0].user.text" - } - }, - { - "seq": null, - "source": null, - "kind": null, - "timestamp": "$.turns[0].timestamp", - "session_id": "$.story_id", - "agent_id": "$.agent.id", - "call_id": "$.turns[0].calls[0].call_id", - "trace_id": "$.turns[0].calls[0].trace_id", - "parent_call_id": "$.turns[0].calls[0].parent_call_id", - "payload": { - "degraded": null, - "http": { - "status": null, - "headers": null, - "response_body": "$.turns[0].assistant.content", - "body_encoding": null, - "truncated": null - }, - "model": "$.turns[0].calls[0].model", - "assistant_content": "$.turns[0].assistant.text", - "usage": "$.turns[0].calls[0].metrics" - } - } - ] -} -``` - -合成：`kind` ← `http.request` / `http.response`；`degraded` ← `true`（默认）；`seq`/`source` 由写入器分配。 - -口语对照： - -| events 字段 | Storyline 路径 | -|---|---| -| `events.0.call_id` | `turns.0.calls.0.call_id` | -| `events.0.payload.http.request_body` | `turns.0.calls.0.messages` | -| `events.1.payload.http.response_body` | `turns.0.assistant.content` | -| `events.0.payload.http.method` / `url` / `headers` | （无）→ null / 合成默认 | - ---- - -## 示例 C：Events → Storyline（从 wire 再解析） - -按 `call_id` 分组后，优先： - -| Storyline 字段 | 来源 | -|---|---| -| `calls[0].event_seqs` | `$[*].seq` | -| `calls[0].call_id` | `$[0].call_id` | -| `calls[0].protocol` | `$[0].payload.protocol` 或从 path 推断 | -| `calls[0].messages` | 解析 `$[?(@.kind=~/request/)].payload.http.request_body`（兼容 `.body`） | -| `user.text` | 摘要 `user_content` **或** 从 request_body 提取 | -| `assistant.text` | 摘要 `assistant_content` **或** 从 response_body / SSE 提取 | -| `calls[0].model` | `payload.model` 或 body.model | - ---- - -## 校验规则（Normative） - -实现 MUST： - -1. 拒绝缺少 `seq` / `source` / `kind` / `payload` 的事件； -2. 保持 `seq` 单调追加； -3. 与其它外围格式互转走 storyline hub； -4. 记录 HTTP 事件时持久化 **headers + body**（或显式标记 `degraded`）；缺一不可宣称可回放。 - -实现 SHOULD： - -1. 为同一 HTTP 交换的 request/response 共享 `call_id`； -1b. 代理路径写入 `client.peer` 与 `connection.persistent`（及原始 Connection/Keep-Alive）； -2. 写 `http.*` kind，同时读兼容 `llm.*`； -3. 从 events 投影 storyline 时填充 `event_seqs`； -4. 对红线头做可检测的脱敏标记。 - ---- - -## 与现有文档的关系 - -| 文档 | 关系 | -|---|---| -| [轨迹存储](../pchronicle/design/trajectory-storage.md) | Lance SoT；本 RFC 强调 SoT 内容应是 HTTP wire | -| Gateway 管线 (external repository) | 生产 events；Story 边界在 **之后** 解释 | -| [RFC-0001 Storyline](0001-storyline-format.md) | 有损 Normal 视图 / hub | -| [轨迹 Markdown 格式](../pchronicle/reference/agenticmd.md) | 人读投影，不是 SoT | - -实现现状：`EventRecord` 已具备 `path`/`body`/`status` 等扁平键；本 RFC 将其 **提升为明确的 HTTP-first 契约**，并引入 `http.*` kind 与 `payload.http` 嵌套作为演进目标。 - ---- - -## 已裁定的演进约束 - -1. 新 writer 使用嵌套 `payload.http`；reader 在磁盘迁移期继续接受历史扁平键。 -2. SSE 原始帧写为按 `seq` 排序的 `http.stream.frame` 事件，携带相对时间与原始字节；聚合文本只是投影。 -3. `seq` 由 producer 在 session 内单调分配，不是唯一键；at-least-once 重试可产生重复行。 -4. WebSocket、gRPC 等非 HTTP 传输使用独立 `transport.*` kind，不伪装成 HTTP。 -5. 默认 `capture_level` 为 `dialogue`。缺少完整 wire 的记录必须标记 `degraded`；只有 `full` capture 可声明 wire-replayable。 -6. Gateway 为客户端和 upstream 连接分配稳定 `connection.id`，用于关联 keep-alive 上的多次交换。 - ---- - -## Changelog - -| Version | Date | Notes | -|---|---|---| -| Draft | 2026-07-30 | 初稿：EventRecord / EventRow / wire、与 Storyline hub | -| Draft | 2026-07-30 | **定位修正为 HTTP-first SoT**：原始请求/响应可回放；摘要字段降为可选；引入 `http.*` kind 与 `payload.http` | -| Draft | 2026-07-30 | **headers 与 body 同为记录 MUST**；缺 headers 亦为 degraded | -| Draft | 2026-07-30 | 增补 **长连接 `connection.*` + 客户端 `client.*`（peer/pid/command）** 记录要求 | -| Draft | 2026-07-30 | **events 仅 Lance**：JSON/JSONL 降为调试导出，非一等 wire | -| Accepted amendment | 2026-08-16 | RFC-0007 将 `EventRecord` ownership 迁至 `persisting-events`；pChronicle 保留物理 schema 与持久化 ownership | -| Accepted | 2026-08-09 | 裁定 payload、SSE、seq、非 HTTP、capture 默认值与 connection identity；保持 at-least-once append-only。 | diff --git a/docs/src/en/rfcs/0003-pchronicle-ownership.md b/docs/src/en/rfcs/0003-pchronicle-ownership.md index 47dacccc2..f4fa9bd5a 100644 --- a/docs/src/en/rfcs/0003-pchronicle-ownership.md +++ b/docs/src/en/rfcs/0003-pchronicle-ownership.md @@ -1,114 +1,11 @@ -# RFC-0003: pChronicle 轨迹存储层所有权 +# RFC-0003: pChronicle ownership -| Field | Value | -|---|---| -| **Status** | Accepted | -| **Date** | 2026-07-31 | -| **Component** | `persisting-pchronicle` | -| **Amended by** | [RFC-0007 Events/Sidecar 边界](0007-events-contract-pchronicle-sidecar.md) | -| **Related** | [RFC-0001 Storyline](0001-storyline-format.md) · [RFC-0002 Events](0002-events-format.md) · [轨迹存储](../pchronicle/design/trajectory-storage.md) | +pChronicle owns Storyline models, peripheral format codecs, physical storage, +Source discovery, Snapshot construction, DataFusion queries, and revision +lineage. CLI and Web consume those contracts. They must not introduce a second +trajectory model or a parallel persistence format. -## 摘要 +The CLI owns commands, import/export orchestration, the loopback Warehouse +HTTP server, and static asset embedding. The Web owns the browsing interface. -`persisting-pchronicle` 是 Persisting 唯一的 **Agent 轨迹结构化存储与检索层**。轨迹的物理 schema、目录布局、物理落盘、读取、回放、格式转换和派生视图 MUST 收敛到 pChronicle。存储无关的逻辑事件信封由 `persisting-events` 定义，具体修订见 RFC-0007。 - -Gateway 和 CLI 可以分别生产事件和适配输入，但 MUST NOT 定义第二套通用轨迹存储实现。 - -## 决策 - -### 1. Canonical 记录 - -`persisting-events::EventRecord` 是 crate 边界和 Gateway 采集管线共同使用的 canonical 逻辑记录。 -provider/SSE payload 解释可作为 Gateway 扩展行为存在，但不得形成第二套可序列化轨迹类型。 - -`EventRow` 是 `events.lance` 的稳定物理行。其 Arrow schema、行转换、序号和 session 分区语义由 pChronicle 独占维护。 - -### 2. 物理层 - -| 层 | API / 实现 | 语义 | -|---|---|---| -| canonical event log | `RawEventLanceStore` | typed append、replay、stats；同一 run 按 `session_id` 分区 | -| 人读/调试视图 | `materialize_lance_to_markdown`, AgenticMD 文件 helpers | 从 canonical events 单向生成，可随时删除和重建 | -| 发现 | `expand_story_locations` | 发现 canonical Run/Story 分区；Markdown 不参与存储层选择 | -| 数据维护 | `RawEventLanceStore::maintain` | 显式离线 compaction、session 索引和 vacuum；事实层不支持 truncate/overwrite | -| Storyline 三表 | `StorylineLanceStore`, `StorylineDataSource` | 原子提交并查询 `runs` / `steps` / `tool_calls` | - -`events.lance` 是事实源。AgenticMD 和 Storyline 三表均可重建，不可被当作协议级审计或回放的事实源；ATIF 是互操作文档格式，不是独立存储模型。append、replay、stats 不得回退到 AgenticMD。 - -canonical event 写路径 MUST 是 at-least-once append-only：不得在 append 前扫描旧行或 -`event_id`，不得将 ID 唯一性、重试去重、truncate 或 overwrite 作为存储语义。一个微批 -只允许执行规范化、Arrow 编码、一次私有 Lance segment append 和一次 fencing manifest -CAS。索引、compaction 与 vacuum 必须由显式维护路径执行。 - -### 3. 格式层 - -pChronicle MUST 统一拥有除公共事件信封以外的轨迹格式与转换实现： - -- `EventRow` 以及 `storyline`、`agenticmd`、`openai_msg`、`atif`、`actf` 的 Rust 数据结构； -- `EventRecord` 到物理 row 与外围格式的映射； -- 格式识别、校验及转换； -- event 与物理行、Markdown block、Storyline turn 之间的映射； -- AgenticMD 的宽松解析、可视化生成、preamble 更新和调试索引； - -外围格式间的转换 MUST 经 Storyline hub。需要保真回放的路径 MUST 直接读取 events，不得依赖有损的 Storyline roundtrip。 - -### 4. 组件职责 - -| 组件 | MUST | MUST NOT | -|---|---|---| -| Gateway | 作为 OverlayNet sink 解释并转发 Agent/LLM 协议；维护采集顺序与调用生命周期；产出 `EventRecord`；实现与实时流状态有关的 live projection 策略 | 自有网络数据面或轨迹记录 schema；把 AgenticMD 当作可恢复事实源 | -| CLI / producer | 解析输入并产出公共事件；需要持久化时通过 pChronicle API 或 sidecar control 协议提交 | 自有 Markdown/ATIF parser、物理落盘协议或动态 ABI | -| pChronicle | 定义物理格式、存储、读取、转换和派生视图；消费公共 `EventRecord` | 依赖 Gateway 才能解释持久化数据；复制一套公共事件信封 | - -Gateway 的 live Markdown 行为可以保留 producer-specific 策略，例如流式 draft upsert；它必须同时写 canonical events。通用 batch materialize 归 pChronicle，不提供 AgenticMD → events 的隐式 compact。 - -### 5. 代码布局 - -- `persisting-pchronicle/src/operations/trajectory/` 是强类型轨迹操作适配。 -- `persisting-gateway/src/session/` 维护 session 身份、路由、client metadata、索引与 snapshot。 -- `persisting-gateway/src/projection/` 维护 Gateway 特有的可见文本解释、实时过滤、draft/upsert 和 reconcile。 -- `persisting-gateway/src/engine/` 维护采集 actor、WAL、顺序状态机和 egress；这里的 “engine” 是 Gateway 内部编排器，不是轨迹存储层。 -- `persisting-overlaynet` 是轻量显式代理网络层，负责 CONNECT、absolute-URI forward、header 规则和网络访问策略执行；Gateway 作为 `OverlaySink` 在其上解释并转发 LLM 流量、产出轨迹事件。OverlayNet 不依赖 Gateway，可配置其他 sink。 - -## 一致性与故障语义 - -1. canonical append 成功后，派生投影失败不得回滚或伪装成 canonical 写入失败；应报告或记录 projection failure。 -2. 状态机只能在 canonical append 成功后提交。 -3. WAL 重启后序号 MUST 单调延续；replay 成功后 MUST ack 原 WAL entry。 -4. `seq` 由 producer 定义；存储不得读取 row count 分配全局序号。writer epoch 的可见性 - 必须由 manifest CAS 串行化。 -5. Storyline 三表替换 MUST 对读者呈现单一提交点；`CURRENT` 指向的不可变 generation 是权威快照。ATIF 输入 MUST 先转换为 Storyline，不得维护第二套 normalized schema。 - -Run lease epoch MUST 通过 `EventWriterFence` 进入 canonical event 提交协议。新 epoch MUST -先以 compare-and-swap 激活 manifest；reader MUST 只读取 manifest 固定的 segment version， -不得直接打开某个 segment 的 latest version。失效 writer 的后续 Lance version 不得进入 -可见快照。相同 epoch、不同 writer_id 的激活 MUST 被拒绝。 - -## 收敛结果 - -- 原 Engine 中的 Trajectory 适配、Lance、Markdown 和 Arrow row 实现全部迁入 pChronicle；Engine crate 删除。 -- Gateway 与 pChronicle 直接使用 `persisting-events::EventRecord`；Gateway extension 仅承载实时 payload 解释。 -- Gateway 仅保留实时 payload 解释、live Markdown eligibility/upsert orchestration 与运行时 reconcile；格式解析、文件 I/O、frontmatter 契约与索引实现委托 pChronicle。 -- 只生产事件或调用 control 协议的组件依赖 `persisting-events`；需要存储、查询或格式转换的调用方依赖 pChronicle。 -- 旧 ATIF `sessions` / `steps` / `tool_calls`、`NormalizedStore`、内存联表视图及对应 Python 门面删除；ATIF 查询统一复用 Storyline 三表 schema。 -- append 边界直接传递 `EventRecord` 批次，不保留 RON/event-lines 字符串适配层。 - -新代码 SHOULD 按能力选择依赖：事件 producer 使用 `persisting-events`，存储和读取调用方 -使用 pChronicle。事件 producer 不得为了构造 `EventRecord` 依赖 pChronicle。 - -## 验收条件 - -- Workspace 不再包含 `persisting-engine` crate、动态库、C ABI 或 Engine RPC 信封。 -- append、replay、stats 等轨迹流程均由 pChronicle 提供。 -- CLI 不再实现 AgenticMD 到 event 的独立解析。 -- Gateway 不再定义与 `EventRecord` 同构的序列化 struct，也不再独立实现 AgenticMD 文档重写或索引。 -- pChronicle MUST 使用 Gateway 的真实 AgenticMD、request/response、provider snapshot 与 SSE fixture 验证 wire、Arrow、Lance 和投影兼容性。 -- ATIF 与 Lance 查询 MUST 注册相同的 `runs`、`steps`、`tool_calls` Arrow schema。 -- pChronicle 的物理后端、格式转换、WAL/投影相关边界行为有回归测试。 -- 文档和 crate metadata 均把 pChronicle 表述为结构化轨迹存储层。 - -## Amendment history - -| Date | RFC | Change | -|---|---|---| -| 2026-08-16 | [RFC-0007](0007-events-contract-pchronicle-sidecar.md) | 将公共 `EventRecord` 契约迁至 `persisting-events`；pChronicle 保留全部物理存储、格式转换与读取 ownership | +[Architecture](../pchronicle/design/architecture.md) diff --git a/docs/src/en/rfcs/0005-pchronicle-revision-lineage.md b/docs/src/en/rfcs/0005-pchronicle-revision-lineage.md deleted file mode 100644 index 9019fc73a..000000000 --- a/docs/src/en/rfcs/0005-pchronicle-revision-lineage.md +++ /dev/null @@ -1,40 +0,0 @@ -# RFC-0005: pChronicle 派生 Revision Lineage - -| Field | Value | -|---|---| -| **Status** | Accepted | -| **Date** | 2026-08-09 | -| **Component** | `persisting-pchronicle` | -| **Related** | [RFC-0002 Events](0002-events-format.md) · [RFC-0003 Ownership](0003-pchronicle-ownership.md) | - -## 摘要 - -clean、redact、augment 和格式化数据集是 canonical events 的派生产物。它们不得覆写或 -去重事实流，也不得把 catalog 语义塞入 Storyline `extra_json`。每个 Run 使用独立的 -`revisions.lance` 记录 lineage；canonical `events.lance` 继续维持 at-least-once、 -append-only 契约。 - -## 数据模型 - -每个 revision 以 `revision_id` 为 upsert key，包含： - -- `parent_revision_ids`：零个或多个父 revision； -- `kind`：`clean`、`redact`、`augment`、`export` 或扩展 kind； -- `canonical_snapshot`：输入 event manifest revision或 Storyline snapshot； -- `recipe`：可重放的程序、版本、参数和输入摘要 JSON； -- `status`：`building`、`ready` 或 `failed`； -- `created_at` 与 `output_refs`。 - -`revisions.lance` 是派生 catalog，不参与 canonical event 的提交、replay 或审计裁定。 -写入相同 `revision_id` 更新状态和产物引用；不同 revision 永不隐式合并。 - -## 一致性与回收 - -产物必须先持久化，再把 revision 切到 `ready`。失败记录可保留用于诊断。对象维护将 -`CURRENT` 以及保留的 `ready` revision 视为 GC roots；删除 revision 本身必须是显式维护操作。 - -## 验收 - -- 同一 revision 从 `building` 更新为 `ready` 只留下一个 catalog row。 -- lineage 写入不改变 canonical event 行数或 manifest。 -- catalog 可被 Web UI 和 DataFusion 消费，并能定位 recipe 与产物。 diff --git a/docs/src/en/rfcs/0006-pchronicle-vortex-backend.md b/docs/src/en/rfcs/0006-pchronicle-vortex-backend.md index f7d49866a..e77434b5b 100644 --- a/docs/src/en/rfcs/0006-pchronicle-vortex-backend.md +++ b/docs/src/en/rfcs/0006-pchronicle-vortex-backend.md @@ -1,12 +1,13 @@ # RFC-0006: pChronicle Vortex 轨迹后端提案 +> Historical design: the event format and event-to-Storyline projection have been removed. Current storage uses Storyline directly. + | Field | Value | |---|---| | **Status** | Proposed | | **Date** | 2026-08-16 | | **Component** | `persisting-pchronicle` / proposed `persisting-pchronicle-vortex` | | **Decision scope** | 可重建、读取优化的 Vortex trajectory projection；不替换 canonical facts | -| **Related** | [RFC-0001 Storyline](0001-storyline-format.md) · [RFC-0002 Events](0002-events-format.md) · [RFC-0003 Ownership](0003-pchronicle-ownership.md) · [RFC-0005 Revision Lineage](0005-pchronicle-revision-lineage.md) | | **External references** | [ATIF v1.7](https://github.com/harbor-framework/harbor/blob/main/rfcs/0001-trajectory-format.md) · [Vortex concepts](https://docs.vortex.dev/concepts/) · [Vortex file format](https://docs.vortex.dev/specs/file-format) | ## 1. 摘要与提议 diff --git a/docs/src/en/rfcs/0007-events-contract-pchronicle-sidecar.md b/docs/src/en/rfcs/0007-events-contract-pchronicle-sidecar.md deleted file mode 100644 index f7abb68f6..000000000 --- a/docs/src/en/rfcs/0007-events-contract-pchronicle-sidecar.md +++ /dev/null @@ -1,145 +0,0 @@ -# RFC-0007: 事件契约与 pChronicle Sidecar 边界 - -| Field | Value | -|---|---| -| **Status** | Accepted | -| **Date** | 2026-08-16 | -| **Components** | `persisting-events` · pChronicle · Gateway | -| **Amends** | [RFC-0002 Events](0002-events-format.md) · [RFC-0003 pChronicle ownership](0003-pchronicle-ownership.md) | -| **Related** | [端到端架构](../system-design/architecture.md) · [轨迹存储](../pchronicle/design/trajectory-storage.md) | - -## 摘要 - -Persisting 将运行时事件的逻辑契约从存储实现中拆出，由唯一新增 crate -`persisting-events` 拥有。Gateway、事件 producer 与 pChronicle 共享该契约，但只有 -pChronicle 拥有 Lance、DataFusion、对象存储、Catalog、查询与投影实现。 - -本地集成方可以启动 `pchronicle serve --control 127.0.0.1:0 DATASET`， -通过带版本和认证令牌的 Control 协议提交事件并等待 durable acknowledgement。 -存储实现由服务进程拥有，客户端只依赖逻辑事件与协议类型。 - -## 动机 - -逻辑事件描述“发生了什么”，物理存储实现决定“如何落盘、查询和投影”。将两者分离， -让 producer 可以发布事件而无需依赖具体存储后端，同时继续使用同一种公共事件类型。 -pChronicle 保留结构化轨迹的持久化和读取职责。 - -## 决策 - -### 1. `persisting-events` 拥有逻辑事件契约 - -`persisting-events` 是存储无关的公共契约包，拥有： - -- `EventIdentity`：`event_id`、`run_id`、`attempt_id`、Story/Turn identity、producer - 与观测时间； -- `EventRecord`：扁平 identity、`seq`、`source`、`kind`、关联字段与 JSON payload； -- 不涉及存储引擎的信封校验； -- 可选 `control` feature 下的 pChronicle control 消息、trait 与进程 client。 - -默认 feature 只提供事件数据模型。`control` feature 可以依赖 Agent control 类型和 Tokio， -但 MUST NOT 引入 Lance、Arrow、DataFusion、对象存储 SDK 或 pChronicle 实现。 - -`EventRecord` 是进程内和进程间共同使用的逻辑记录，不是物理存储 schema。事件所属的 -Run、Attempt 或 Session 与 producer 定义的 `seq` 一起形成排序上下文；wall-clock -timestamp 只用于关联和展示，不作为顺序真相。 - -### 2. pChronicle 拥有物理存储与持久解释 - -pChronicle 消费 `persisting-events::EventRecord`，并独占维护： - -- `EventRow`、Arrow schema 与逻辑记录到物理行的映射； -- Lance/Vortex 等物理后端、writer fencing、manifest、compaction 与 vacuum； -- Catalog、query、replay、格式转换、revision 和派生视图； -- `pchronicle serve` 内嵌的 Control 服务以及 append 成功的 durable acknowledgement。 - -RFC-0003 中“pChronicle 拥有轨迹格式”的约束仍适用于物理 schema、交换格式与转换。 -其中“pChronicle 唯一定义 `EventRecord`”以及“所有调用方直接依赖 pChronicle 类型”的部分 -由本 RFC 修订。 - -### 3. 本地 producer 通过 Control 服务提交 - -`pchronicle serve --control 127.0.0.1:0 DATASET` 提供本地持久化入口。 -`persisting-events` 的进程 client 负责启动子进程、握手与请求关联。 -子进程退出、握手失败或协议版本不兼容都使持久化请求显式失败。 - -使用此协议的 producer 提交 `EventRecord`；Lance row、DataFusion 查询与对象存储 -实现由 pChronicle 管理。 - -### 4. Control 协议随事件契约发布 - -不创建 `persisting-pchronicle-client`。control client、协议信封、Attempt/lease/commit -消息和内存测试实现位于 `persisting-events` 的可选 `control` feature 中。这些类型描述 -producer 到持久化服务的边界，本身不实现存储。 - -当前协议版本为 `2`，使用 loopback TCP 上的逐行 JSON frame。sidecar 在 stdout 发布 -一次 readiness 记录，包含 endpoint、协议版本和每进程随机认证令牌。每个请求携带版本、 -request ID 与令牌；client 校验响应版本和 request ID。frame 大小有明确上限。 - -这是本地子进程协议，不是面向不可信网络的远程公共 API。跨主机部署需要另行定义传输 -安全、身份和可用性契约。 - -### 5. ACK、背压与不确定性 - -Gateway 到存储的 append 队列是有界的。提交被拒绝与提交后的结果不确定必须区分。 -只有 pChronicle 完成 append 并返回成功时，事件才视为 durable。 - -连接中断、写入错误或 ACK 丢失无法证明事件未提交，必须分类为 unknown。此时 producer -消耗该序号，不能把不确定写入伪装为“肯定未写”。事实层仍允许 at-least-once 与重复 -`event_id`；exactly-once 和热路径去重不属于该协议。 - -同一 control sidecar 也承载 Run lease、Attempt active/heartbeat/terminal 和 Run commit。 -这些控制记录的 fencing/CAS 语义保持不变。 - -## 依赖边界 - -```text -Gateway / local event producers - │ - │ EventRecord + optional control protocol - ▼ - persisting-events - │ - │ versioned authenticated local IPC - ▼ - pchronicle serve process (Control enabled) - │ - ├── EventRow / Arrow - ├── Lance or another storage backend - └── Catalog / query / projection -``` - -`persisting-events` 的默认 feature 和可选 `control` feature 均不得引入 Lance、Arrow、 -DataFusion 或云对象存储 SDK。物理存储与查询实现只由 pChronicle 维护。 - -## 兼容与迁移 - -- `EventRecord` 的 JSON 顶层字段保持扁平，移动 crate 不改变既有 wire 形状； -- pChronicle 对外 re-export 公共事件类型，允许调用方渐进迁移 import； -- `persisting-pchronicle-client` 被删除，使用者改为 - `persisting-events = { features = ["control"] }`； -- pChronicle 的既有 Lance dataset、目录布局和 replay 语义不因这次 crate 拆分而变化。 - -## 被否决的方案 - -| 方案 | 原因 | -|---|---| -| producer 内嵌另一套 Lance adapter | producer 与存储引擎生命周期、feature 和依赖重新耦合 | -| `EventRecord` 继续由 pChronicle 定义 | producer 为使用基础事件信封被迫依赖存储产品 | -| 单独保留 `persisting-pchronicle-client` | 协议包过碎；control 契约可以作为事件边界的可选 feature | -| 各集成方分别实现 IPC client | 会产生协议漂移、重复认证与错误语义 | -| 仅依赖离线导入而无实时 Control 协议 | 缺少运行期 durable ACK、fencing 与统一 canonical append 语义 | - -## 验收条件 - -- Workspace 只新增 `persisting-events`，不存在 `persisting-pchronicle-client`； -- Gateway、producer、pChronicle 复用同一 `EventRecord`，没有同构公共事件 struct； -- `persisting-events` 不直接依赖 pChronicle 存储 API，也不链接 Lance/DataFusion； -- 进程 client 能够启动 Control 服务，持久写入有序事件并发布 Attempt 终态； -- 协议版本、认证、request correlation、frame limit、拒绝与 unknown 写入语义有测试； -- pChronicle 的原有 replay、query 与物理存储测试继续通过。 - -## Changelog - -| Version | Date | Notes | -|---|---|---| -| Accepted | 2026-08-16 | 拆出 `persisting-events`，移除 client crate，确立逻辑事件与 pChronicle 存储服务的边界 | diff --git a/docs/src/en/rfcs/0015-chronicle-manifest.md b/docs/src/en/rfcs/0015-chronicle-manifest.md index fc538183a..2067208dd 100644 --- a/docs/src/en/rfcs/0015-chronicle-manifest.md +++ b/docs/src/en/rfcs/0015-chronicle-manifest.md @@ -53,7 +53,7 @@ Goals: - `open(path)` builds a Snapshot. A leaf Dataset is one source. A plain directory MAY be opened as a **virtual dataset** that unions all nested Dataset leaves and peripheral JSON under that path. -- When the sidecar is missing, still classify Datasets via `CURRENT` / events / +- When the sidecar is missing, still classify Datasets via `CURRENT` / compact-jsonl markers. `list` MUST NOT recursively list an entire object-store prefix just to classify. `open` of a directory MAY recurse under the existing `max_entries` / `max_files` bounds. @@ -100,9 +100,7 @@ A path is a **leaf Dataset** when any of these holds, in this order: 1. `chronicle.manifest` with `kind = "leaf"` 2. `CURRENT` (Storyline) -3. `events.lance/_manifest.json` or a directory named `events.lance` with - `_manifest.json` -4. compact-jsonl Lance (`pchronicle.format = compact-jsonl/v1`, or a leaf +3. compact-jsonl Lance (`pchronicle.format = compact-jsonl/v1`, or a leaf sidecar with `format = "compact-jsonl/v1"`) Otherwise the path is a **Directory** (including `chronicle.manifest` diff --git a/docs/src/en/rfcs/index.md b/docs/src/en/rfcs/index.md index 58b2d422e..78b108263 100644 --- a/docs/src/en/rfcs/index.md +++ b/docs/src/en/rfcs/index.md @@ -13,12 +13,9 @@ each product's Reference and Guides. | RFC | Title | Status | |---|---|---| | [0001](0001-storyline-format.md) | Storyline format (`storyline/v1`) | Accepted | -| [0002](0002-events-format.md) | Events format (`events`) | Accepted | | [0003](0003-pchronicle-ownership.md) | pChronicle trajectory-store ownership | Accepted | | [0004](0004-actf-format.md) | ACTF v1.0 trajectory format | Accepted | -| [0005](0005-pchronicle-revision-lineage.md) | pChronicle derived Revision lineage | Accepted | | [0006](0006-pchronicle-vortex-backend.md) | pChronicle Vortex trajectory backend proposal | Proposed | -| [0007](0007-events-contract-pchronicle-sidecar.md) | Events contract and pChronicle Sidecar boundary | Accepted | | [0008](0008-atif-format.md) | ATIF v1.7 trajectory format | Accepted | | [0009](0009-openai-messages-format.md) | OpenAI Messages trajectory corpus format | Accepted | | [0010](0010-agent-corpus-lance-layout.md) | Layered Lance layout for Agent evaluation corpora | Proposed | diff --git a/docs/src/en/system-design/architecture.md b/docs/src/en/system-design/architecture.md index 1a9d3517f..5ab2fa20f 100644 --- a/docs/src/en/system-design/architecture.md +++ b/docs/src/en/system-design/architecture.md @@ -1,65 +1,12 @@ # System architecture -pChronicle captures, stores, queries, and exchanges Agent trajectory history. -This page defines the boundaries between its engine, interfaces, and supporting -libraries. Physical layouts are documented in [pChronicle Design](../pchronicle/design/index.md). +The pChronicle library owns trajectory models, format conversion, discovery, +Lance storage, snapshots, and bounded queries. The CLI owns import/export +orchestration and the loopback Warehouse API. The Web UI consumes those +models and API contracts. -## Component ownership +Storyline documents are written directly. Published table versions and writer +leases preserve storage consistency; queries read fixed source versions. +Dataset, Source, and revision identifiers keep transformed data traceable. -| Component | Responsibility | -|---|---| -| `persisting-pchronicle` | Trajectory models, storage, Source discovery, Dataset Snapshots, bounded queries, format conversion, and revision lineage | -| `persisting-pchronicle-cli` | The `pchronicle` command, local read-only Warehouse API, optional Control service, Gateway configuration, and embedded Web assets | -| `pchronicle-web` | Dataset browsing and query UI | -| `persisting-events` | Storage-independent `EventRecord` and optional versioned Control client/protocol | -| `persisting-gateway` | Model protocol adaptation, forwarding, session correlation, and trajectory capture | -| `persisting-overlaynet` | Proxy transport, request classification, and policy for intercepted traffic | -| `persisting-agentctl` | Shared control types, policy transitions, and cooperative client protocol | - -## Capture and import - -```text -Agent / SDK requests - → Gateway protocol handling and capture - → EventRecord - → pChronicle canonical event storage - → Dataset Snapshot → bounded queries → CLI / Web / export - -Supported files and object-store Sources - → discovery and pinned Source versions - → Storyline normalization and query projections - → Dataset Snapshot → bounded queries → CLI / Web / export -``` - -Gateway capture records traffic routed through it. Import reads supported -external formats directly. Both paths preserve available identities and source -provenance; neither supplies missing execution or isolation evidence. - -## Write and read boundaries - -The CLI may call the engine in process. Integrations can also use -`pchronicle serve --control 127.0.0.1:0 DATASET` through the authenticated, -versioned local protocol in `persisting-events`. Only a successful append ACK -confirms durability. A lost response leaves the outcome uncertain, and callers -must not treat it as proof that nothing was written. - -pChronicle owns the physical schema, writer fencing, manifest publication, and -maintenance. Producers submit logical records rather than defining parallel -storage layouts. See [RFC-0007](../rfcs/0007-events-contract-pchronicle-sidecar.md) -and [trajectory storage](../pchronicle/design/trajectory-storage.md). - -The Warehouse HTTP API and Web UI are read-only. The separately enabled Control -and Gateway capture paths can write. Public bind addresses are rejected by the -local server; the Control protocol is intended for trusted local processes. - -## Facts, views, and versions - -Canonical events retain their original logical payload. Storyline provides the -normalized model for supported exchange formats; projections support analysis -without replacing the original facts. Queries read a pinned Snapshot, and -revision lineage identifies derived outputs. - -Capture coverage is limited to the records a Source contains. Proxy policy -applies to traffic that reaches the proxy and does not establish process-wide -network isolation. See [Gateway capture](../pchronicle/guides/serve-gateway.md) -and [facts and projections](../pchronicle/concepts/facts-and-projections.md). +[pChronicle architecture](../pchronicle/design/architecture.md) diff --git a/docs/src/en/system-design/design-principles.md b/docs/src/en/system-design/design-principles.md index 843e037ae..f9b88ce6e 100644 --- a/docs/src/en/system-design/design-principles.md +++ b/docs/src/en/system-design/design-principles.md @@ -1,32 +1,8 @@ # Design principles -## Keep facts and projections distinct - -Canonical events preserve recorded facts. Normalized Storyline views and query -projections remain traceable to their Sources; missing records stay visible as -limits on the answer. - -## Make ownership explicit - -Shared event contracts describe logical records. pChronicle owns storage, -query, and exchange, while Gateway owns capture and protocol adaptation. -The CLI and Web UI use those boundaries rather than defining new data models. - -## Preserve provenance and versions - -An answer should identify the Dataset, Source, Snapshot, and query that produced -it. Revision lineage keeps derived outputs connected to their inputs across -normalization and export. - -## Bound the work - -Query budgets, bounded capture queues, and explicit append acknowledgements -make resource use and write outcomes inspectable. A failed projection must not -be reported as a successful durable append. - -## Keep data portable - -Documented formats and the CLI make Datasets usable without a particular -viewer. Local files and object-store Sources share the Dataset model. - -See the [system overview](index.md) and [roadmap](../roadmap.md). +Preserve input fidelity through documented codecs. Keep one normalized +Storyline model and one owner for storage and query semantics. Pin Source +versions for each operation, retain provenance for transformed output, and +bound parsing, I/O, and query work. Failed publication must leave the previous +committed data readable. Keep datasets portable across files, object storage, +CLI, and Web consumers. diff --git a/docs/src/en/system-design/index.md b/docs/src/en/system-design/index.md index 54bbdd6b7..7f124ca37 100644 --- a/docs/src/en/system-design/index.md +++ b/docs/src/en/system-design/index.md @@ -1,24 +1,9 @@ # System design -Persisting centers on [pChronicle](../pchronicle/index.md), an Agent trajectory -storage engine. Gateway captures model traffic, shared event contracts carry -records into storage, and the CLI and Web UI expose Datasets for inspection. +Persisting centers on the pChronicle trajectory processing pipeline: decode +supported inputs, write Storyline storage, pin Snapshot versions, query, and +export through CLI and Web. -```text -Gateway capture ── EventRecord ── pChronicle storage ─┐ -ATIF / ACTF / OpenAI Messages / Storyline Sources ────┴─> Dataset Snapshot - └─> query / exchange / Web UI -``` - -Canonical events preserve recorded facts. Storyline normalization and query -projections make supported Sources usable through common views. A Snapshot pins -the Source versions used to answer a query; importing data does not add facts -that the Source did not provide. - -## Continue by question - -- [Component ownership and data flow](architecture.md) +- [Architecture](architecture.md) - [Design principles](design-principles.md) -- [Storage and query implementation](../pchronicle/design/index.md) -- [Gateway capture](../pchronicle/guides/serve-gateway.md) -- [Project engineering notes](../project/engineering.md) +- [pChronicle implementation](../pchronicle/design/index.md) diff --git a/docs/src/zh/guides/using-persisting.md b/docs/src/zh/guides/using-persisting.md index 7692cf830..74651663e 100644 --- a/docs/src/zh/guides/using-persisting.md +++ b/docs/src/zh/guides/using-persisting.md @@ -7,11 +7,6 @@ 从[pChronicle 入门](../pchronicle/get-started.md)开始：打开 Dataset、查看汇总、提出一个 有界 SQL 问题，再定位答案背后的 Evidence。只读流程稳定后，再使用导入导出或服务指南。 -## 我需要捕获新的模型请求 - -使用 [Gateway](../pchronicle/guides/serve-gateway.md) 转发请求并持久化 canonical events。 -外部执行组件也可以通过 pChronicle Control 服务提交事件。 - ## 一套可靠的使用习惯 1. 从一个 Run 或一个 Dataset 开始。 diff --git a/docs/src/zh/installation.md b/docs/src/zh/installation.md index a549391bf..849585cc3 100644 --- a/docs/src/zh/installation.md +++ b/docs/src/zh/installation.md @@ -24,4 +24,3 @@ curl -fsSL https://raw.githubusercontent.com/DeepLink-org/Persisting/main/script 参见[工程说明](project/engineering.md)。 继续阅读[探索第一个 Dataset](pchronicle/get-started.md)或 -[Gateway 捕获](pchronicle/guides/serve-gateway.md)。 diff --git a/docs/src/zh/overview.md b/docs/src/zh/overview.md index e8c91fc57..3f5ed5128 100644 --- a/docs/src/zh/overview.md +++ b/docs/src/zh/overview.md @@ -35,5 +35,3 @@ pchronicle query ./trajectory-data \ ``` 继续阅读[探索第一个 Dataset](pchronicle/get-started.md)，学习 Dataset 健康检查、证据定位、格式、导入导出和只读 Web/API。 - -**完成本节后：**你可以把一个答案连接到产生它的 Dataset 和 Source。需要记录新的模型请求时，使用[Gateway 捕获](pchronicle/guides/serve-gateway.md)。 diff --git a/docs/src/zh/pchronicle/concepts/dataset-and-source.md b/docs/src/zh/pchronicle/concepts/dataset-and-source.md index b2eccf04b..c6463a6a2 100644 --- a/docs/src/zh/pchronicle/concepts/dataset-and-source.md +++ b/docs/src/zh/pchronicle/concepts/dataset-and-source.md @@ -14,8 +14,7 @@ Dataset 是 discovery、Snapshot、query 与 exchange 的边界。它不声称 ## Source -Source 是 Dataset 中能够独立发现和版本化的最小轨迹表示。它可以是 canonical event store、 -Storyline projection 或支持的交换文件。每一行规范化数据都保留 `source_path`，因此外部 ID +Source 是 Dataset 中能够独立发现和版本化的最小轨迹表示。它可以是 Storyline store 或支持的交换文件。每一行规范化数据都保留 `source_path`，因此外部 ID 仍是 Source-local，冲突不会被隐藏。 实体的完整地址是： diff --git a/docs/src/zh/pchronicle/concepts/facts-and-projections.md b/docs/src/zh/pchronicle/concepts/facts-and-projections.md index 04c57797c..142aa90d7 100644 --- a/docs/src/zh/pchronicle/concepts/facts-and-projections.md +++ b/docs/src/zh/pchronicle/concepts/facts-and-projections.md @@ -1,29 +1,9 @@ -# 记录数据、查询视图与派生版本 +# 轨迹数据与版本 -pChronicle 把“发生了什么”与“如何查看它”分开。 +Storyline 是数据处理管线的规范化轨迹模型。导入将支持的交换格式转为 Storyline， +存储与查询直接使用该模型。AgenticMD 是其可读 Markdown 编码。 -| 层次 | 职责 | 示例 | -| --- | --- | --- | -| 记录事实 | 持久的写入时事实 | lifecycle、模型、工具、输出和终态 event | -| 查询视图 | 规范化查询模型 | `runs`、`steps`、`tool_calls`、`trajectories` | -| 人读视图 | 便于诊断的输出 | AgenticMD | -| 交换格式 | 互操作边界 | ATIF、ACTF、OpenAI Messages、Storyline JSON | -| 派生版本 | 记录来源的转换数据 | 清洗、脱敏或增广 Run | +Snapshot 固定一次操作的 Source 成员与版本，不改变原始 Source 的身份。 -Canonical event 是 append-oriented 的事实。Projection 可以为会话或查询重新组织这些事实， -但不能静默成为第二个事实源。可重建视图需要记录输入 Snapshot 和 transform version。 - -Storyline 是会话导向的 projection。它的三表 Lance 布局为完整文档重建优化，不是时序数据库， -也不替代 canonical event 路径。 - -AgenticMD 是非权威的人读 projection。Markdown 视图缺失或过期不会改变 canonical event 结果。 - -对于 Gateway 配套的单 trace 观测，Warehouse 可以在 Catalog 已经定位 source 后重新打开最新的 -canonical event manifest。这样正在进行中的 trace 可以保持最新，而不要求每次追加事件都发布 -物化 Storyline projection。 - -存储 API 把派生版本称为 Revision。它指向 parent 和生成它的 transform。清洗、脱敏和增广因此创建新的历史分支， -而不是无痕改写历史。 - -所有权见[Run 存储设计](../design/trajectory-storage.md)，精确交换契约见 -[运行数据格式](../reference/formats/index.md)。 +参见 [Dataset、Source 与 Snapshot](dataset-and-source.md) 和 +[导入导出](../guides/exchange.md)。 diff --git a/docs/src/zh/pchronicle/design/architecture.md b/docs/src/zh/pchronicle/design/architecture.md index e5646977c..b52e66edd 100644 --- a/docs/src/zh/pchronicle/design/architecture.md +++ b/docs/src/zh/pchronicle/design/architecture.md @@ -1,164 +1,52 @@ # pChronicle 架构 -本文解释 pChronicle 如何存储 Agent 轨迹，并提供有资源限制的读取面。用户工作流属于 -[Guides](../guides/index.md)，精确命令属于[Reference](../reference/cli.md)，组件职责 -属于 [System Design](../../system-design/architecture.md)。 - -![pChronicle 产品边界](../../../assets/diagrams/persisting/pchronicle-product.svg) - -## 产品边界 - -pChronicle 是 Agent 轨迹**存储引擎**。它可以作为本地工具使用，也可以平台化部署在多条 -path 前面。CLI、Web、Agent 和 Warehouse 都是引擎的客户端，不是并列产品。 - -引擎 API 是： +pChronicle 专注于 Agent 轨迹数据处理管线： ```text -open(path) → pin Snapshot → 发现 / 定位 / 分析（写入路径上再 append） +ATIF / ACTF / OpenAI Messages / Codex / Claude Code + → 解码并校验 Storyline + → 直接写入 Storyline Lance + → 固定 Snapshot 并执行 DataFusion 查询 + → CLI / Warehouse API / Web UI / 导出 ``` -**Dataset 就是 path**：规范化的本地路径或对象存储 URI（`s3://`、`az://`、`gs://`）。 -mount 名、dataset pin（`@name`）、Directory 的 library 名都是 locator。解析完成后引擎只看见 path。 -凭据不得嵌入这条 path。 - -| 形态 | 用途 | 持久状态 | -| --- | --- | --- | -| 直接 Dataset | 检查本地路径或 S3 prefix | Dataset 外无状态 | -| 原生 Dataset | 接收 canonical event 或 create/append/replace import | Dataset manifest 与 version | -| 本地默认 Dataset | 为一个本地 root 省略 Dataset 参数 | 用户配置中的规范化路径 | -| 只读 Warehouse | 为 Web/API review 静态挂载 path；可选 Directory 做 path ACL | 配置与可重建 cache | - -pChronicle 不是 scheduler、Agent runtime、全局 Dataset 控制面、分布式 SQL 服务或时序数据库。 -Warehouse 只接受 loopback bind。未使用 `--catalog-config` 时没有用户鉴权；启用后 Directory -与数据面路由要求用户 ak/sk 请求头，这仍不是公网多租户服务。见 -[RFC-0013](../../rfcs/0013-pchronicle-warehouse-catalog.md)。 - -## 四层 - -| 层 | 职责 | 不是 | -| --- | --- | --- | -| **Path** | Dataset 身份。本地路径或对象存储 URI。 | mount 名、dataset pin（`@name`）、library 名、`catalog://` URI | -| **Directory**（可选） | 平台寻址：把名字解析成 path，并决定谁能打开。换票后客户端打开 path。 | 第三种 Dataset。不是 Snapshot。 | -| **Snapshot** | 一条 path 上写入与读取之间的同步协议：有哪些 Source、各钉在哪个版本。 | 名叫 Catalog 的产品。不是 Directory 列表。 | -| **Query 面** | 发现（`list`/`ls` / `sources`）、定位（`find`）、分析（`query`）。全部相对于已 pin 的 Snapshot。 | Web Explorer 的第四套语义 | - -代码里仍可能使用 `DatasetCatalogSnapshot`、`--catalog-config` 等名称。用户文档和 RFC 使用 -Path、Directory、Snapshot。 +AgenticMD 与 Storyline JSON 是 Storyline 的其他编码。Compact JSONL 保存任意 +JSON 记录，不赋予轨迹语义。 -Directory 见 [RFC-0013](../../rfcs/0013-pchronicle-warehouse-catalog.md)。Snapshot 构造见 -[Snapshot 设计](catalog.md)。 +## 职责 -## 数据层次与 Ownership - -```text -writers and importers - → canonical events and terminal facts - → logical Run / Step / ToolCall projections - → exchange representations - → lineage-bearing revisions -``` - -| 层次 | Ownership 规则 | +| 组件 | 职责 | | --- | --- | -| canonical event | 写入时事实；append-oriented 的事实源 | -| logical projection | 规范化、可重建查询视图 | -| exchange representation | import/export 契约；不会静默升级为全局事实 | -| revision | 带 parent Snapshot 和 transform lineage 的派生输出 | - -Storyline 是 session-oriented projection。它的 Lance 布局为完整文档重建优化，因此按 session -replace 不是 canonical 高频 append 路径。 - -## Dataset 寻址 - -规范化 path 是资源身份。Source path 标识其中一个能独立发现和版本化的表示。外部 ID -保持 Source-local： - -```text -(path, source_path, entity_kind, original_id) -``` - -Warehouse mount name 只是 SQL alias。移动到新 path 后就是不同 Dataset。`catalog://` 是 -Directory 解析用的 pin 类型，不是 `DatasetLocation` scheme。 - -## 读取路径 - -```text -path（经 Directory 换票或 pin 解析之后） - → resource-limited discovery - → pin Snapshot - → Source pruning and lazy open - → normalized DataFusion relations - → resource-limited CLI, API, or Web result -``` - -一次操作钉住每个 Source 的 version reference。本地文件使用 identity 与 fingerprint，Lance -使用发布的 generation 或 manifest，对象存储在可用时使用 version 或条件 ETag。Snapshot -不声称无关 Source 共享全局事务时间。 - -定位（`find`）与分析（`query`）共用这个已 pin 的 Snapshot。CLI 与 Web 共用 `find` 表达式、 -报告的 scope 和 `snapshot_id`。Web UI 可以对返回字段做高亮和可视截取；这层前端匹配不得 -改变命中集合。Web Explorer 是定位层的 UI，不是另一套查询语言。 - -支持时，规范化关系包括 `sources`、`runs`、`steps`、`tool_calls`、`events` 和 -`trajectories`。每个实体关系保留 `source_path`。SQL 只读，并拒绝 DDL、DML、网络函数和 -文件函数。 - -发现与裁剪算法见 [Snapshot 设计](catalog.md)。 - -## 写入与发布路径 - -Gateway 和 native writer 先发布 canonical event，再生成派生视图。Snapshot 引用的对象必须 -先持久化，最后才能发布可见 pointer。发布失败时，旧 Snapshot 保持可读。 - -```text -validate event - → persist payload or content-addressed object - → append canonical fact - → publish terminal fact or projection generation - → expose through a later Snapshot -``` - -Writer 并发由具体 store contract 定义。Snapshot compare-and-swap 本身不意味着 merge-and-retry。 -未发布 version 与不可达 object 需要显式维护路径。 - -Canonical/projection 边界见[运行存储](trajectory-storage.md)，Storyline 实现见 -[Storyline Lance](storyline-lance.md)。 +| `persisting-pchronicle` | 模型、编解码、数据源、Lance 存储、Snapshot 与查询 | +| `persisting-pchronicle-cli` | 命令、导入导出编排、loopback Warehouse HTTP、静态资源嵌入 | +| `pchronicle-web` | 浏览、分析、存储检查与 Assistant | -## 只读 Warehouse +Dataset 是规范化的本地路径或对象存储 URI。pin 与挂载名最终解析成路径。 +外部实体 ID 只在 Source 内有效，查询行保留来源路径。可选 Directory 负责认证 +与授权路径解析，不保存轨迹数据。 -Server 静态挂载命名 path。Refresh 先完整构造新 Snapshot，再切换 reader。 -Dataset table 先按 Source 裁剪，再打开命中的固定 version；cache 和 routing index 与 Snapshot -generation 绑定。 +## 读取 -使用 `--catalog-config` 时，父进程逐请求认证，并调度到有上限的 exec worker 池。 -每个 worker 的用户、授权及后端凭证范围固定，缓存独立；父进程不挂载数据集。 -后端凭证经私有 IPC 在 runtime 启动前传入。worker 仍使用服务端 OS 身份，进程隔离 -不等于文件系统沙箱。同时保留 `catalog://` 列表/换票路由。CLI 换票后打开票里的 `uri`（一条 path）并注入存储钥。这是 path 上的 -平台寻址，不是新的 Dataset 种类。 +发现阶段固定 Source 成员和版本。Dataset 与 `_file_` 裁剪之后才延迟打开物理 +Source。一次操作使用同一 Snapshot；独立 Source 不承诺全局事务时间。 +规范化表包括 `sources`、`runs`、`steps`、`tool_calls` 与 `trajectories`。 +Compact JSONL 暴露记录表。 -Web 与 API 是同一读取模型的 consumer，不形成新事实源。未知 API route 保持 error，不进入 -SPA fallback；只接受 loopback listener。 +CLI 与 Web 共享查询语义。SQL 有资源限制且只读，拒绝修改数据、网络函数和任意 +文件系统函数。 -用户配置见 [Warehouse 指南](../guides/serve.md)，精确 route 与 Gateway 组合见 -[`pchronicle` 参考](../reference/cli.md)。 +## 写入 -## 保证与明确不保证 +导入将支持的格式解码为 Storyline 后直接写入。Storyline store 在内容对象与 +表变更持久化之后，通过 `CURRENT` 发布精确表版本。发布失败保留上一 Snapshot。 +写入租约与 generation 校验防止并发替换静默丢失数据。维护按存储发布协议执行 +compaction 和不可达内容回收。 -| 领域 | 保证 | 不保证 | -| --- | --- | --- | -| identity | path + Source path + original ID 保持可见 | 外部 ID 全局唯一 | -| 读取一致性 | 一个 Snapshot 内固定 Source reference | 跨 Source 全局事务 | -| 发布 | 新发布成功前旧 Snapshot 保持可读 | 所有 writer 自动 merge retry | -| 查询 | 有资源限制的只读执行 | 任意 mutation 或无限制服务查询 | -| projection | 声明范围内的 lineage 与可重建性 | 没有 generation 记录的 freshness | -| 服务 | loopback-only 静态读取面；Directory 模式下数据面用用户钥鉴权 | 公网多租户 Warehouse | +## Warehouse -## 相关设计 +`pchronicle serve` 在 loopback 地址挂载命名路径。刷新先构建完整的新 Snapshot， +再切换读者。`--catalog-config` 启用 Directory 认证，并把数据操作交给限定授权 +挂载与后端凭证的有界 worker。普通模式通过 loopback API 读取本地挂载。 -- [Snapshot 设计](catalog.md)：discovery、Snapshot 构造、惰性 Source resolve 与裁剪。 -- [RFC-0013 path Directory](../../rfcs/0013-pchronicle-warehouse-catalog.md)：名字→path、ACL、换票。 -- [RFC-0015 `chronicle.manifest`](../../rfcs/0015-chronicle-manifest.md)：嵌套 Dataset 发现与聚合统计 sidecar。 -- [运行存储](trajectory-storage.md)：canonical fact、存储布局与写入 ownership。 -- [Storyline Lance](storyline-lance.md)：三表 projection、内容层、发布与维护。 -- [记录数据、视图与版本](../concepts/facts-and-projections.md)：这些层次的用户心智模型。 -- [pChronicle Reference](../reference/index.md)：精确 CLI 与格式契约。 +参见 [Snapshot](catalog.md)、[轨迹存储](trajectory-storage.md)、 +[Storyline Lance](storyline-lance.md) 与 [CLI 参考](../reference/cli.md)。 diff --git a/docs/src/zh/pchronicle/design/catalog.md b/docs/src/zh/pchronicle/design/catalog.md index 892fd33c2..de6ab2df1 100644 --- a/docs/src/zh/pchronicle/design/catalog.md +++ b/docs/src/zh/pchronicle/design/catalog.md @@ -1,653 +1,37 @@ -# pChronicle Snapshot 设计 +# Snapshot 与 Source 发现 -> 当前实现说明。代码与历史文档称 Dataset Catalog；产品口径称 **Snapshot**：打开 **path** -> 之后，写入与读取之间的同步协议。平台侧的名字→path 授权见 -> [RFC-0013 path Directory](../../rfcs/0013-pchronicle-warehouse-catalog.md)，不是本文。 -> -> Dataset 命令参数见 [`pchronicle` 命令参考](../reference/cli.md)；用户模型见 -> [Dataset、Source 与 Snapshot](../concepts/dataset-and-source.md)；查询工作流见 -> [发现并查询](../guides/discover-and-query.md)；轨迹物理格式见 -> [pChronicle 运行存储](trajectory-storage.md) 与 -> [Storyline 三表 Lance](storyline-lance.md)。 +`DatasetCatalogSnapshot` 固定一个或多个命名 Dataset 挂载下发现的 Source 与版本， +在存储和 DataFusion 之间提供读取边界，显式刷新时重建。它与负责授权、解析路径的 +可选 Directory 分离。 -格式 wire contract 与逐字段转换以 [RFC-0001 § Wire schema](../../rfcs/0001-storyline-format.md#wire-schema)、 -[RFC-0004 § ACTF 映射](../../rfcs/0004-actf-format.md#actf-storyline-json-pointer-mapping)、 -[RFC-0008 § ATIF 映射](../../rfcs/0008-atif-format.md#atif-storyline-json-pointer-mapping)和 -[RFC-0009 § OpenAI Messages 映射](../../rfcs/0009-openai-messages-format.md#openai-storyline-json-pointer-mapping)为准。 +## 发现与身份 -## 1. 定位 +Dataset 是本地路径或对象存储 URI。Source 包括 Storyline store、Compact JSONL +Dataset 和支持的交换文件。`chronicle.manifest` sidecar 分类嵌套 Dataset 并缓存 +聚合统计。递归发现不进入 Lance 内部文件。 -Snapshot 面向由**一条 path** 下多个存储位置和多种轨迹格式共同组成的查询空间（Warehouse -可同时 pin 多条 path，每条仍是独立 Dataset）。主要覆盖： +实体身份为 `(Dataset 路径, Source 路径, 实体类型, 原始 ID)`。不同 Source 的 +外部 ID 不自动合并。行保留 `_file_` 与 `document_id`，命名空间别名支持跨挂载 +联查。 -- 同时查询在线数据、历史归档和评测数据； -- 一条 path 下包含多层目录、多个 Run 级 `events.lance`，以及若干外围 JSON 文件； -- 不同 path 上存在相同的 `run_id`、`session_id` 或文件名； -- Web 服务需要在多次请求之间复用同一份发现结果，并且只在显式刷新后切换视图。 +发现限制文件数、字节量、解析并发并明确错误策略。本地文件固定身份与指纹，远程 +对象在可用时固定版本或 ETag。Storyline store 固定已发布 generation 和精确表版本。 +独立 Source 不构成全局事务。 -它位于 path 与 DataFusion SQL 之间。用户打开一条 path（或经 Directory 换票得到 path）； -pChronicle 递归发现轨迹源，将不同物理格式统一投影到稳定表，并在一次查询或一代 Web -Snapshot 内固定其成员与版本。这是写/读同步协议，不是一份需要长期维护的元数据数据库。 - -Snapshot **不是** Directory，也不复制源数据、不接管对象存储目录、不声明外围 JSON 已成为 -canonical 数据，也不要求后台同步任务。代码类型名 `DatasetCatalogSnapshot` 仍指本对象。 - -## 2. 目标与非目标 - -### 2.1 目标 - -1. **多 Dataset 联查**：一次 SQL 可以访问多个具名本地目录或对象存储前缀。 -2. **层级发现**：一个 Dataset 的 URI 可以指向存储根、Run 根、复合 store 或单个文件。 -3. **统一表模型**：Storyline、canonical events、ATIF、OpenAI messages 和 ACTF 使用相同 - 的查询表名。 -4. **稳定身份**：任何 Storyline 都能用 `(dataset, _file_, session_id)` 唯一定位到快照内的物理源。 -5. **快照一致性**：一条查询不会在执行中混入新发现的文件或新的 Lance generation。 -6. **稳定默认入口**：位置参数固定挂载为名为 `dataset` 的默认 Dataset。 -7. **有界失败**：发现、格式检测、单文件大小、解析并发和查询内存都有显式限制与错误策略。 -8. **安全写入**：命名挂载默认只读；服务端写操作只能落到显式选定的 canonical events - Dataset。 -9. **Catalog-aware 裁剪**：先用 Dataset 和 `_file_` 条件选出 source，再构造物理扫描计划。 -10. **惰性解析**：Catalog 快照只固定成员和版本描述；Lance dataset、远程对象和文件 - datasource 在查询确实需要时才打开，并在快照内 single-flight 复用。 - -### 2.2 非目标 - -- 不提供 Hive Metastore、Glue Catalog 一类持久化 catalog service。 -- 不在 Catalog 中建立跨文件索引、统计信息仓库或 materialized view。 -- 不把不同源中的同名 `run_id` 或 `session_id` 自动合并。 -- 不为多个独立物理源提供分布式事务或全局时间点读。 -- 不通过 Catalog 修改、搬运或转换外围 JSON；需要长期列式分析时仍应显式导入 Lance。 -- 不在 URI 参数中管理密钥；对象存储认证继续使用对应 SDK 的标准凭证链。 - -## 3. 核心模型 - -![Snapshot 查询路径](../../../assets/diagrams/persisting/dataset-catalog.svg) - -核心对象分为七层： - -| 对象 | 作用 | 生命周期 | -|---|---|---| -| `DatasetMount` | 保存层级 namespace、SQL alias、根 URI 和可选格式提示 | 配置期 | -| `CatalogDataset` | 一个 Dataset 及其 `DiscoveredSource` 列表 | 快照期 | -| `DiscoveredSource` | 描述一个复合 store 或外围文件的逻辑路径、格式、版本和状态 | 快照期 | -| `DatasetCatalogSnapshot` | 固定全部挂载的成员、源版本和临时对象文件 | 一条 CLI 查询或一代 Server Catalog | -| `LazySource` | 保存固定 source 描述，并并发安全地缓存首次解析结果或错误 | 与快照相同 | -| `CatalogTableProvider` | 在 DataFusion `scan` 边界执行 source 裁剪并组合命中的物理计划 | 每个 Dataset 稳定表 | -| `ChronicleQueryEngine` | 把快照注册为 DataFusion schema，并执行只读 SQL | 与快照相同 | - -### 3.1 Namespace、Dataset 与 SQL alias - -Dataset 身份是规范化 path，不等于物理 Lance dataset，也不等于 Warehouse mount 名。 -一条 path 可以包含多个 Storyline store、多个 `events.lance` 和多个外围文件。Warehouse -用 SQL alias 把已打开的 path 注册为 DataFusion schema，因此 `prod` 与 `staging` 可以同时 -存在；alias 不是 Dataset 身份。实现仍用 `NamespacePath` 表达挂载层级。 - -SQL alias 会去除首尾空白并转成小写，必须匹配 `[A-Za-z_][A-Za-z0-9_]*`。`public` 和 -`information_schema` 是保留名称。namespace component 允许字母、数字、`_`、`-`、`.`。 -完整 namespace 或 SQL alias 重名都会在发现前失败。 - -### 3.2 Source 与 `_file_` - -Source 是 Catalog 的最小发现单元： - -- Storyline `CURRENT` 根是一个 `store` source； -- canonical `events.lance` 根是一个 `store` source； -- 每个 JSON、JSONL 或 NDJSON 文件是一个 `file` source。 - -`_file_` 是 source 相对 Dataset 根的 UTF-8 逻辑路径，统一使用 `/` 分隔。挂载根自身作为 -source 时使用 `.`。它不是源表的持久字段，也不会写回 Lance。 - -### 3.3 Storyline 与 Run 身份 - -`session_id` 是 Storyline 的逻辑主键，但只在一个 source 内保证唯一；外围文件或不同归档 -中可以出现相同值。因此 Catalog 和 Server 使用以下复合键： - -```text -(dataset, _file_, session_id) -``` - -`run_id` 是 Run 分组键，一个物理 Run 可以包含主 Storyline 与多个 subagent Storyline， -所以同一 source 内多行可以共享一个 `run_id`。canonical events 规范化时按事件的 -Storyline/session 身份分组，并保留实际 `events.lance` URI，避免后续读写根据挂载根猜测 -物理位置。 - -### 3.4 Source revision - -内部用 `CatalogSourceRevision` 保存类型化 revision，而不是让一个字符串同时表示 Storyline -generation、event fact/layout 水位、本地文件指纹与对象版本。`sources.snapshot_ref` 仍是便于 -SQL 展示和筛选的字符串投影；一致性判断、快照摘要和 API 描述使用类型化 revision。 - -一个 canonical events source 可以关联多个派生 Storyline 投影。Catalog 不因此隐藏或拒绝 -canonical source，而按 `fresh → last_modified → generation → path` 的稳定顺序选出一个读取 -加速投影；`projection_candidates` 暴露候选数。所有候选都不新鲜时，查询回退到固定的 -canonical events 快照。 - -## 4. 挂载与默认 Dataset - -### 4.1 CLI 形式 - -`--mount NAME=DATASET` 可以重复： - -```bash -pchronicle query \ - --mount current=local:///srv/pchronicle/current \ - --mount archive=s3://trajectory-bucket/archive \ - --sql "SELECT * FROM current.runs" -``` - -`--mount` 与位置 Dataset 互斥。位置参数挂载为固定 schema `dataset`；只用 -`--mount` 时必须写 mount 名，没有隐式 `dataset` schema。用户配置文件（`-c`）只保存 -`[pins.]`（含保留名 `default`），不提供 query 挂载表。 - -```bash -pchronicle query --mount current=local:///srv/pchronicle/current \ - --mount archive=s3://trajectory-bucket/archive \ - --sql "SELECT table_schema, table_name FROM information_schema.tables" -``` - -### 4.2 默认选择规则 - -| CLI 输入 | 默认 Dataset | 不带 schema 的 `runs` 等表名 | -|---|---|---| -| 有位置参数 `INPUT` | 固定为 `dataset` | 指向 `dataset.runs` 等默认 view | -| 仅 `--mount`（一个或多个） | 无 | 必须写 `current.runs` 等限定名 | - -位置参数形式如下： - -```bash -pchronicle query ./capture --sql "SELECT * FROM dataset.runs" -``` - -等价于把 `./capture` 挂载为 `dataset`，并查询 `dataset.runs`。跨 Dataset 联查使用重复 -`--mount`，不要把位置 Dataset 和 `--mount` 写在同一条命令里。 - -## 5. 层级发现 - -### 5.1 示例 - -假设挂载目录如下： - -```text -capture-root/ -├── live/ -│ └── CURRENT -├── agents/ -│ └── codex/ -│ └── run-001/ -│ └── events.lance/ -│ └── _manifest.json -└── imports/ - ├── batch-a.atif.jsonl - └── nested/ - └── session.json -``` - -Catalog 在 **把 `capture-root` 当作 query 根打开**（虚拟 dataset）时产生四个 source。 -`_file_` 相对该根： - -| `_file_` | `kind` | 可能的 `format` | -|---|---|---| -| `live` | `store` | `storyline` | -| `agents/codex/run-001/events.lance` | `store` | `events` | -| `imports/batch-a.atif.jsonl` | `file` | `atif` | -| `imports/nested/session.json` | `file` | 按文件检测 | - -`live` 和 `events.lance` 的内部文件不会再次成为 source。这一“识别复合根后停止下探”的规则 -避免把 manifest、generation、segment 或 `objects.lance` 错当成用户输入。 - -`pchronicle ls capture-root` 不是上面这张表。它只列一层：`live/`（dataset）、 -`agents/`（directory）、`imports/`（directory）。再 `ls capture-root/imports` 才会看到 -`batch-a.atif.jsonl` 和 `nested/`。浏览成员是 [RFC-0015](../../rfcs/0015-chronicle-manifest.md) -的 `list`；query 成员是 `open`。 - -当目录含有 `chronicle.manifest` -（[RFC-0015](../../rfcs/0015-chronicle-manifest.md)）时，discovery 优先采用该 sidecar： -`leaf` 且 `format = compact-jsonl/v1` 时可不打开 Lance 即归类为 Compact source。 -对 branch 或纯目录做 `list` 只看一层 children；对两者做 `open` 则收集嵌套叶子以及 -叶子外的 JSON。写入方只更新 leaf manifest，不回写祖先。 - -### 5.2 本地发现 - -本地 URI 支持普通路径、`local://` 和 `file://`。**`open`**： - -1. 如果根是 `.json`、`.jsonl` 或 `.ndjson` 文件，直接建立单个 source。 -2. 如果根是叶子 Dataset（`chronicle.manifest` leaf、`CURRENT`、 - `events.lance/_manifest.json`、或 compact-jsonl Lance），整个根是一个 source - `_file_ = "."`。 -3. 否则根是 Directory：按稳定路径顺序递归，在叶子 Dataset 处停止，并把剩余支持的 - 外围 JSON 收为 file source（虚拟 dataset）。 -4. 符号链接不会跟随，避免循环、越界读取和同一物理文件的重复身份。 - -对同一 URI 的 **`list`** 只返回一层 children，见 RFC-0015。它不会打印虚拟 dataset 的 -source 表。 - -### 5.3 对象存储发现 - -对象 URI 通过 Lance/object-store 适配层解析。对对象前缀做 **`open`** 即下面的虚拟 -dataset 遍历；对同一 URI 做 **`list`** 仍只列一层（RFC-0015），MUST NOT 为浏览而列举 -整棵树。 - -Catalog 流式消费前缀 listing，并在读取 -`max_entries + 1` 个对象前失败；不会先把无界 listing 收进内存再检查。随后： - -1. 用 `CURRENT` 对象识别 Storyline 根； -2. 用 `events.lance/_manifest.json` 识别 canonical events 根； -3. 排除所有复合根内部对象； -4. 把剩余 `.json`、`.jsonl` 和 `.ndjson` 对象作为独立 source； -5. 按 Dataset 相对 object key 排序。 - -当前对象后端沿用 pChronicle/Lance 支持的 URI scheme，例如 `s3://`、`az://` 和 `gs://`。 -挂载或 listing 本身失败意味着无法建立可信成员集，即使使用 `report` 模式也会失败。 - -### 5.4 格式检测 - -每个外围文件独立检测格式，因此同一个 Dataset 可以混合 ATIF、OpenAI messages 与 ACTF。 -`pchronicle query` 不对 Dataset 施加格式约束。`find` 与 `export` 的 `--source` 只把查找 -收窄到一条 Dataset-relative Source path，不是格式提示。 - -本地和远程外围文件都不会为了自动检测而在 Catalog 构建期读取内容；如果没有显式格式 -提示，`sources.format` 可以是 `NULL`。Catalog 会先冻结本地文件指纹或远程对象版本，等 -`_file_` 裁剪选中该 source 后才做有界格式检测。检测结果与 datasource 解析结果一起缓存 -在快照的 `LazySource` 中。 - -## 6. SQL Provider - -Dataset 的稳定公共关系、精确 `sources` column、Source-local identity 与 join 规则属于 -[Query Model Reference](../reference/query-model.md)。本节只解释 Catalog 如何为这些关系 -构造执行计划。 - -外围文件不会生成伪造的原始 event 行；它们只能通过 Storyline 规范化关系查询。Catalog -为实体关系增加常量 `_file_`，把公开 Source identity 连接到惰性物理 Source。 - -### 6.1 Catalog-aware source pruning - -`runs`、`steps`、`tool_calls` 和 `events` 各自由一个 Dataset 级 -`CatalogTableProvider` 提供。DataFusion 把投影、过滤条件和 limit 交给 provider 后， -provider 按以下顺序构造物理计划： - -1. 先排除不能提供目标表的 source，例如 `events` 自动排除 Storyline 和外围文件； -2. 在每个 source 的常量 `_file_` 上求值可识别的过滤表达式； -3. 完全不可能匹配的 source 直接跳过，不调用 `LazySource::resolve`； -4. 以 `max_concurrent_sources` 为上限、按稳定 source 顺序解析候选，并把业务列投影、业务 - 谓词和 limit 继续交给其原生 provider； -5. 零个命中 source 生成 `EmptyExec`，一个命中 source 直接使用其计划，多个命中 source - 才生成 `UnionExec`，最后在需要时应用全局 limit。 - -可精确用于 source 裁剪的 `_file_` 谓词包括 `=`、`!=`、`IN`、`NOT IN`、大小写敏感的 -`LIKE`/`NOT LIKE`，以及由 `AND`、`OR`、`NOT` 组成且能安全求值的组合。对同时包含 source -条件和业务条件的表达式采取保守三值判断：只有能证明该 source 不可能匹配时才跳过。 -例如： - -```sql -SELECT run_id, session_id -FROM archive.runs -WHERE _file_ LIKE '2026/08/%' - AND session_id = 'session-42'; -``` - -这里 `LIKE` 在 Catalog 层裁剪 source，`session_id` 则下推到命中 source 的 Lance 或文件 -provider。没有 `_file_` 条件时，Catalog 没有跨 source 的 `run_id`/时间统计信息，必须把 -目标表的全部兼容 source 视为候选；业务谓词仍可在每个原生 provider 内下推。 - -`LazySource` 使用异步 `OnceCell` 缓存解析结果，多个并发查询命中同一个 source 时只执行 -一次打开、远程物化或格式解析。source 解析阶段的失败也会缓存，以保证同一快照内行为 -稳定。canonical events 的原始 `events` 表可以直接扫描固定 segment。没有 fresh 投影时， -带可证明 `session_id = ...` 或 `session_id IN (...)` 的查询只读取目标 Storyline 的完整历史； -宽查询读取固定 snapshot。两种 fallback 都受 `max_event_fallback_rows` 和 -`max_event_fallback_bytes` 约束，且只物化当前查询请求的关系表；超限时要求 build/sync -Storyline 投影。 -`load_events` 点查直接读取目标 session，不构造 DataFusion MemTable，但使用相同的行数和 -字节预算。 - -可以用 `EXPLAIN` 检查裁剪后的物理计划：精确命中一个 source 时，计划中不应出现 -`UnionExec`。 - -### 6.2 联查规则 - -同一 Dataset 可以包含多个物理 source，而 `run_id`/`session_id` 只保证在单个 source 中 -有效。两个内建轨迹表跨多个同 Dataset source 联接时，必须显式加入 `_file_` 等值： - -```sql -SELECT r.run_id, s.step_id, s.message_kind, s.message_value -FROM archive.runs r -JOIN archive.steps s - ON r._file_ = s._file_ - AND r.session_id = s.session_id; -``` - -遗漏 `_file_` 会在执行前被拒绝。跨 Dataset 联接不要求 `_file_` 相同，因为左右命名空间 -已经不同，通常也不会拥有相同的目录布局： - -```sql -SELECT c.run_id, a.run_id AS archived_run -FROM current.runs c -JOIN archive.runs a ON c.session_id = a.session_id; -``` - -校验作用于 `runs`、`steps` 和 `tool_calls` 的内建联接。查询引擎只接受单条只读 -`SELECT`、`VALUES`、`DESCRIBE` 或 -`EXPLAIN`，拒绝 DDL、DML、`COPY` 和多语句。 - -## 7. 快照与一致性 - -### 7.1 构建过程 - -一次 Catalog 构建按以下顺序完成： - -```text -解析并校验挂载 - → 冻结每个根的候选成员 - → 固定每个候选的 identity / CURRENT / manifest / object metadata - → 构造 sources 元数据 - → 计算 snapshot_id - → 注册 Dataset schema、CatalogTableProvider 与默认 view - → 发布给查询或 Server -``` - -只有完整构建成功的 `DatasetCatalogSnapshot` 才会交给查询引擎。构建过程不打开 Lance -dataset、不把远程 JSON 复制到本地，也不把 canonical events 规范化为 Storyline 三表。 - -### 7.2 不同源的固定方式 - -| 源 | 成员固定 | 内容/版本固定 | -|---|---|---| -| 本地外围文件 | 发现时冻结路径列表 | 记录路径、size、mtime，以及 Unix 上的 device/inode；命中后读取前后再次校验 | -| 远程外围对象 | 冻结 listing 的 `ObjectMeta` | 命中后按固定 version/ETag 条件读取，流式复制到快照临时目录，并校验最终大小 | -| Storyline store | 发现并读取 `CURRENT` 描述 | 冻结 generation 与三张表的精确版本；命中后才打开 Lance dataset | -| canonical events | 发现并读取 `_manifest.json` | 冻结 manifest revision 和可见 segment version；命中后才打开 segment | - -只有被查询选中的远程对象才会复制。复制按 chunk 写入受快照持有的临时文件，不把整个 -对象一次性读入内存；快照释放后临时目录随之清理。本地 fingerprint 是变化检测，不是 -内容哈希：如果攻击者保留相同文件身份、大小和修改时间进行原地改写，不在保证范围内。 - -### 7.3 一致性边界 - -快照保证： - -- 查询计划和执行看到相同的 source 成员集； -- Storyline/events 即使延迟打开，也只能打开快照已经固定的 generation、manifest 和 - segment version，不会重新读取最新指针； -- 后端提供 version/ETag 时，远程外围对象固定到 listing 时对应的版本； -- 新文件、新 generation 或新 manifest 只有下一条 CLI 查询或显式刷新后可见。 - -快照不保证多个彼此独立的 URI 来自同一个全局事务时刻，也不阻止源系统删除已经固定但 -尚未读取的数据。本地文件在发现与首次读取之间发生可检测变化时，查询会失败而不是混读。 -若对象后端既不提供 version 也不提供 ETag，Catalog 只能以 key、size 和修改时间描述 -`snapshot_ref` 并校验传输大小，不能提供相同强度的对象版本固定保证。 - -`snapshot_id` 是 Dataset 名称、URI、格式提示、source 相对路径、固定引用与候选错误的 -BLAKE3 摘要截断值，用于标识成员/版本视图；它不是内容校验和，也不代表业务提交 ID。 - -### 7.4 解析生命周期 - -快照中的每个 ready source 都持有一个固定描述和一个解析 cell。首次命中时： +## 延迟查询 ```text -CatalogTableProvider source pruning - → LazySource::resolve - → 打开固定 Lance 版本，或校验/物化固定文件 - → 创建原生 TableProvider - → 缓存 Result -``` - -因此“惰性”不改变快照边界：解析发生得晚，但解析目标在 Catalog 发布前已经固定。未被任何 -查询命中的 source 在整个快照生命周期中可以始终保持未打开状态。 - -## 8. 错误策略与资源边界 - -`list`/`ls` 和 `stats` 通过 `--errors` 提供两种策略： - -| 策略 | 单个候选无法固定描述或通过初始校验 | Dataset 根不存在、listing/遍历失败或超过全局限制 | -|---|---|---| -| `strict` | Catalog 构建失败 | Catalog 构建失败 | -| `report` | 写入 `.sources`，状态为 `error`，跳过数据表注册 | Catalog 构建失败 | - -`report` 的目标是容忍一个可信成员集中的坏文件，不是把不完整 listing 伪装成成功。候选 -错误写入公开 Catalog 前会去掉错误文本中 URI query 部分，避免反射可能存在的临时签名； -生产配置仍不应把凭证直接放进 URI。延迟到 SQL 扫描期才出现的 Lance 打开、远程条件 -读取、格式检测或记录解析错误在 `strict` 和 `report` 下都会让该查询失败，既不会静默 -漏掉 ready source，也不会追溯修改不可变快照中的 `sources.status`。 - -Catalog 复用直接文件查询的资源参数： - -- `max_files`：候选 source 数上限； -- `max_entries`：目录项或 object listing 数上限； -- `max_detection_bytes`：格式检测输入上限； -- `max_file_bytes`：外围文件/对象大小上限； -- 可选的 `max_record_bytes`、`max_concurrent_files`、cache 参数：解析期边界；默认不设置 - 单记录大小上限，由 `max_file_bytes` 限制 source； -- `max_concurrent_sources`：单次物理 scan 同时解析的 source 上限； -- `max_event_fallback_rows`、`max_event_fallback_bytes`：无 fresh 投影时单次定向 - canonical→Storyline fallback 的内存边界； -- DataFusion memory pool、spill path、spill bytes、timeout 与输出行数：查询期边界。 - -## 9. Server、刷新与 Web - -`pchronicle serve` 通过一个或多个位置参数 `[NAME=]DATASET` 挂载 Dataset。启动流程先收敛 -canonical Storyline 投影，再构建初始 Catalog，随后由所有 REST 和 SQL 请求共享。 - -| API | 语义 | -|---|---| -| `GET /api/catalog` | 返回当前 `snapshot_id`、创建时间、默认 Dataset、错误策略和 source 列表 | -| `POST /api/catalog` | 在锁外完整构建新快照，成功后原子替换，并清空轨迹缓存 | - -投影 supervisor 在运行期发现新的 canonical Store 或 projection 发布后，会标记 Catalog -dirty，随后在锁外完整重建并原子切换。已经发现的 `events.lance` 继续追加时只推进 source -watermark，不触发全局 Catalog 重建。Gateway 配套的单 trace 查询会先由 Catalog 定位 source， -再重新打开最新 canonical manifest，因此 Storyline projection 等待 idle 窗口时，正在进行的 -trace 仍然可见。刷新失败不会清空或部分更新旧 Catalog，而会保留 dirty 状态并有界重试；正在 -处理的请求持有旧快照的 `Arc`，可以继续完成。 -Web Explorer 从 Catalog 获取 Dataset 列表，服务端过滤、URL 状态和 Storyline 列表 -均携带完整 `(dataset, _file_, session_id)`；`run_id` 作为物理 Run 分组信息 -单独返回。Catalog 是不可变快照；`POST /api/catalog` 仍可显式刷新，但位置 Dataset 形式的 `serve` -维护的 canonical 更新会自动产生新快照。 - -### 9.1 Server source-routing 加速 - -Server 在每一代 `CatalogRuntime` 内持有可重建的内存加速结构，但不改变 -`DatasetCatalogSnapshot`、`CatalogDataset` 或 `DiscoveredSource` 的定义。索引按需从当前 -快照的稳定表派生： - -- `runs` 索引保存 `run_id`、`session_id`、`agent_id`、`agent_model_name` 到 source id 的 - 多值映射； -- `events` 使用两级 lazy index：identity 层保存 `event_id`、`trace_id`，partition 层保存 - `session_id`、`agent_id`；项目列表不会为高基数 event identity 付出内存成本； -- source 路径只在每个 Dataset 内保存一次，值键使用每代随机 keyed 64-bit fingerprint，单 source 命中 - 内联保存整数 source id；hash collision 只会扩大候选集，原 SQL 谓词仍负责最终过滤； -- Run 列表另行惰性缓存，不让 SQL point query 为 Explorer 的 `row_count` 聚合付费。 - -索引只在首个包含可路由条件的单表查询到达时构建，并由 async single-flight 防止并发重复 -扫描。构建使用 Arrow batch stream，不收集完整结果；单层索引最多接受 100 万行和 100 万 -distinct value，超过边界即丢弃未发布的临时索引并回退原查询。Server 只从顶层 `AND` 中 -提取必然成立的字符串等值或 `IN` 条件；联接、CTE、析取、 -复杂表达式、已有 `_file_` 条件、过多候选 source 或索引构建失败都保留原 SQL。命中时只向 -SQL 增加 `_file_ = ...` 或 `_file_ IN (...)`，原业务谓词仍由 DataFusion 执行。因此索引 -只能缩小物理 source 候选，不能改变结果语义。 - -`GET /api/catalog` 的 `acceleration` 字段报告索引是否已经构建及其行、source、distinct -value 数，并通过 `failed` 列出本 generation 已缓存的构建失败，避免每个请求重复全表扫描； -`POST /api/query/evidence` 用 `source_routing` 响应字段报告 `applied`、 -`already_pruned`、`not_applicable`、`not_selective` 或 `index_unavailable`。Catalog 刷新会把 -新快照、查询引擎和空加速结构作为同一个 runtime 原子发布；旧请求继续持有旧 runtime， -索引不会跨 `snapshot_id` 复用。 - -首次索引构建仍需扫描对应稳定表，主要收益来自同一 Server 生命周期内的后续 point/project -查询。CLI 的一次性 SQL 不使用这层状态，也不会让 Catalog 变成持久化元数据服务。 - -### 9.2 写入边界 - -`pchronicle serve` 只提供读取、Catalog 刷新和有界 evidence query，不暴露 maintenance、 -导入或任意 SQL 写接口。服务强制限制为 loopback；Gateway 和原生 writer -直接写 Dataset，不经过 Warehouse API。 - -## 10. Rust API 边界 - -核心 API 由 `persisting-pchronicle` 提供： - -```rust -use std::sync::Arc; -use persisting_pchronicle::{ - CatalogSnapshotOptions, ChronicleQueryEngine, DatasetCatalogSnapshot, DatasetMount, -}; - -let mounts = vec![ - DatasetMount::new("current", "local:///srv/pchronicle/current")?, - DatasetMount::new("archive", "s3://trajectory-bucket/archive")?, -]; -let snapshot = Arc::new( - DatasetCatalogSnapshot::discover(mounts, None, CatalogSnapshotOptions::default()).await?, -); -let engine = ChronicleQueryEngine::from_catalog_snapshot(snapshot).await?; -let rows = engine - .query_jsonl("SELECT COUNT(*) AS runs FROM archive.runs") - .await?; +挂载 → 发现并固定版本 → Dataset/_file_ 裁剪 → 延迟打开 Source + → 规范化关系表 → 有界只读 DataFusion 查询 ``` -需要按 Storyline 读取完整轨迹时使用 `CatalogStorylineKey` 调用快照的 `load_storyline`、`load_events` -或 `canonical_event_uri`。控制面可用 `list_namespaces`、`list_sources` 和 `describe_source` -分页浏览同一快照；page token 与 `snapshot_id` 绑定，刷新后不能复用。调用方不应绕过快照 -重新发现 source，否则可能把不同成员或版本拼进同一个响应。 - -## 11. 关键不变量 - -实现和后续扩展必须保持以下不变量： - -1. Namespace 是层级逻辑身份；SQL alias 是独立且唯一的小写 schema 名，二者不得混作一个字段。 -2. `_file_` 在一个快照内稳定、相对 Dataset 根，根 source 固定表示为 `.`。 -3. Catalog Storyline 的完整身份始终是 `(dataset, _file_, session_id)`；`run_id` 只用于 Run 分组。 -4. 识别复合 store 后不得继续把其内部文件注册为独立 source。 -5. 六张表即使为空也必须存在，并保持固定 schema。 -6. `events` 只能包含 canonical events，不能由有损 Storyline 反向伪造。 -7. 同 Dataset 多 source 的轨迹表联接必须携带 `_file_` 等值。 -8. 查询期 source 不能脱离持有它的快照生命周期。 -9. Server 只原子发布完整新快照；失败时继续提供旧快照。 -10. Warehouse Server 不得把任何 Dataset 或 source 作为写目标。 -11. `_file_` source pruning 必须发生在 `LazySource::resolve` 之前；不能为了判断是否命中而 - 打开 source。 -12. 延迟解析只能使用快照固定的版本描述，并在同一快照内 single-flight；不得在解析时 - 重新跟随 `CURRENT` 或最新 manifest。 -13. 多个投影关联同一 canonical events source 时必须保留 canonical source，并稳定选择最多 - 一个 fresh 投影；冲突只能形成诊断信息，不能阻断事实读取。 -14. Server routing index 必须与 snapshot 同代发布；构建或分析不确定时只能回退原查询， - 不得用不完整索引排除 source。 - -## 12. 取舍与备选方案 - -### 12.1 持久化元数据服务 - -持久化 Catalog 能缓存 listing 和统计信息，但会引入一致性协议、迁移、后台同步、权限与 -灾难恢复问题。当前工作负载更需要“对这一条查询看到什么”的确定边界，因此选择查询期 -快照。若未来 listing 成本成为主瓶颈，可以在不改变 SQL 模型的前提下增加可验证缓存。 - -### 12.2 把所有挂载平铺到 `public` - -平铺表无法区分在线与归档边界，也会让 `_file_` 必须编码 URI 或 Dataset 名称。使用 -DataFusion schema 保留用户给出的 Dataset 语义，并让跨 Dataset SQL 显式可审查。 - -### 12.3 用目录 basename 作为默认名称 - -basename 会受路径拼写、对象前缀和部署目录影响，不带 schema 的 SQL 也无法获得稳定解析结果。位置参数入口 -因此始终使用固定名称 `dataset`，而不是从 URI 猜名字。 - -### 12.4 发现时统一导入 Lance - -外围 JSON 的自动导入会改变查询的延迟、容量和失败语义，还会制造新的持久状态，因此 -Catalog 对它只做虚拟规范化。canonical `events.lance` 是例外：位置 Dataset 形式的 `serve` 在 Catalog -之外维护确定的同级 Storyline 投影；查询仍按 lineage 和 freshness 选择投影或固定快照 fallback。 - -### 12.5 只使用 `run_id` - -`run_id` 是分组键，不是 Storyline 主键；同一物理 Run 内的主 Agent 与 subagent 可以共享 -该值。用 `(dataset, _file_, session_id)` 定位既能保留物理来源，也能支持无歧义的读写路由。 - -## 13. 测试与演进 - -当前测试覆盖： - -- Dataset 名称规范化、保留名与重名拒绝； -- 本地混合格式递归发现、默认 view 和空 Dataset schema； -- `strict`/`report` 候选错误行为； -- Catalog 和查询引擎构建后 source 解析计数仍为零； -- `_file_` 与业务谓词组合只解析命中的本地、远程和 Storyline source； -- 单 source 物理计划没有 `UnionExec`，未命中的远程对象不会下载； -- 延迟错误不会在 `report` 下被静默跳过； -- canonical `events` 原始扫描和 session 点查不会触发全量 Storyline 规范化； -- 多 fresh 投影稳定选出一个且 canonical events 始终可见； -- 层级 namespace 分页、source describe 和跨快照 page-token 拒绝； -- 同 Dataset 危险联接拒绝与跨 Dataset 联接； -- 一个 canonical events source 中多个 Storyline 的独立读取； -- CLI 位置参数、单/多命名挂载、TOML 与帮助文本； -- Server 惰性 Catalog、Dataset 过滤、失败刷新保留旧快照和物理写入坐标； -- Server routing index 的多条件交集、单 source SQL 注入、结果等价、显式 `_file_` 保留与 - refresh 后清空； -- Web Dataset 选择与完整 Run 坐标编码。 - -后续扩展新格式或新后端时，应先定义它如何产生稳定 `_file_`、如何固定版本、能投影哪些 -表、是否允许写入，再接入发现器。不能固定成员或版本的后端必须显式降低一致性承诺，不能 -复用现有 `snapshot_ref` 暗示更强保证。 - -## 14. 相关实现 - -- `crates/persisting-pchronicle/src/store/catalog/`：发现、固定、惰性 source、Catalog - provider、source pruning 和 Run 路由； -- `crates/persisting-pchronicle/src/store/query_engine.rs`：Catalog DataFusion backend 与联接校验； -- `crates/persisting-pchronicle/src/store/storyline/datafusion.rs`：Storyline 描述固定与按固定 - generation 延迟打开； -- `crates/persisting-pchronicle/src/store/events/datafusion.rs`：canonical event manifest - 固定与按固定 segment 延迟打开； -- `crates/persisting-pchronicle-cli/src/lib.rs`：查询 CLI 挂载与默认 Dataset 解析； -- `crates/persisting-pchronicle-cli/src/server/mod.rs`：惰性构建、原子刷新、读写路由； -- `crates/persisting-pchronicle-cli/src/server/acceleration.rs`：同代内存 source-routing index、 - 保守 SQL 分析与 `_file_` 注入； -- `pchronicle-web/src/`：Dataset 选择和完整 Run identity。 - -## 浏览索引与查询一致性 - -Warehouse 的浏览索引与 `DatasetCatalogSnapshot` 独立。Tree 请求不构造查询 -引擎，也不等待查询来源的完整发现。响应保留原有字段，并增加 `browse`： -`consistency: best_effort`、目录视图摘要 `generation`、Unix 秒数 `observed_at`、 -`stale`、`refreshing`、`last_error`。根目录观察不完整时 `observed_at` 为零。 -视图摘要不代表 source 内容版本。 - -命中缓存立即返回；同一路径的冷请求共享一次刷新。serve 持有单个 worker， -统一执行请求刷新和周期遍历，最后一个服务器状态释放时终止。请求队列与同路径 -等待者分别最多 128 项。进程内浏览扫描串行，prefix list 启动间隔至少 100ms， -单次 list 超时为 20 秒；一次 list 可能包含多个存储请求，此限制不影响 SQL。 - -worker 每 30 秒从配置根目录开始或继续遍历浅层目录，在 prefix 之间优先处理 -页面请求，不展开 Dataset 叶节点，不补执行错过的定时器。每轮后台遍历最多 -10,000 个 prefix，更大目录仍可手动访问。后台遍历不提供完整性保证。刷新失败 -保留旧视图并退避，最长五分钟；只有完整成功的父目录 listing 才删除已消失子项 -的缓存视图。listing 本身不构成跨对象事务。 - -Lance 缓存位于 `PCHRONICLE_CACHE_DIR` 或系统 pChronicle 缓存目录，文件名为 -`catalog-<配置指纹>.lance`，行键另含 mount URI 指纹，避免同名不同路径串用。 -v2 暂保留 JSON payload，但加载时验证 schema version，全部成功后才发布内存 -索引。损坏缓存在独占文件锁保护下重建；另一个进程占用缓存或磁盘修复失败时 -退化为内存索引。未变化的观察只更新内存，避免每 30 秒产生 Lance 版本；重启后 -磁盘时间戳保守地标记缓存待刷新。旧 `catalog-ui.lance` 不再读取。 - -**准确查询不能从该缓存推断 source 集合。** 只检查缓存里已有 source 的 revision -仍可能漏掉新建来源。因此准确查询继续自行发现并逐 source pin。Runs 提供精确 -dataset 和 `_file_` 时，`QueryScope` 会完整发现该 source path，并将无关 mount -表示为空；浏览索引仍不决定 source 集合。`/api/catalog` 使用 -`per_source_pinned` 标明一般一致性，不承诺跨来源统一事务时间点。未限定范围的 -查询继续使用完整递归发现。 - -### 范围查询准入与摘要预算 - -同一 `QueryScope` 的重叠 Runs 摘要请求共享 discovery、pin、执行及结果或错误。 -每个 serve 最多同时执行两个不同 scope 的完整操作，最多保留 128 个活跃 scope。 -flight 表只持有弱引用：调用者结束后，下一次独立请求重新发现和 pin。初始化者 -取消时释放许可，已有等待者可以接手。不会永久保留每路径锁或旧查询运行时。 +Lazy source 在 Snapshot 内复用首次解析结果或失败。Storyline 提供 `runs`、 +`steps` 与 `tool_calls`；`trajectories` 是兼容视图，`sources` 描述物理来源。 +交换文件解码后使用同一规范化 schema，Compact JSONL 暴露记录关系。 +谓词与列下推遵守具体 provider 的语义。 -本地 scope 保留完整发现的叶节点边界、来源身份和符号链接规则。Storyline 子树 -退回完整 mount 发现，因为关联投影可能指向子树外的 canonical source。远端 scope 暂时 -先完整发现所选 mount，再于投影绑定之后过滤；缺少 manifest 不能证明普通目录 -或 JSON 对象不存在。远端子树剪枝仍待实现。 +Warehouse 先构建替换 Snapshot 再切换读者，摘要和路由缓存绑定 Snapshot + generation。刷新失败保留旧 Snapshot。CLI 查询与导出使用固定 Source 版本。 -摘要 SQL 保留逐 source 过滤和 join 前的 step 聚合。结果改成流式读取，避免先收集 -全部 Arrow batches 再创建完整 JSONL 副本。摘要集合最多 100,000 行；每个 source -的统计或 Runs JSONL 输出最多 64 MiB。超限返回要求缩小范围的错误，不返回部分 -成功结果。这些输出预算不限制 HashJoin 构建端，后者仍由 DataFusion 内存池约束。 -此次不改写任意 Analysis SQL。 +参见 [Dataset、Source 与 Snapshot](../concepts/dataset-and-source.md)、 +[查询模型](../reference/query-model.md) 与 [Storyline Lance](storyline-lance.md)。 diff --git a/docs/src/zh/pchronicle/design/storyline-lance.md b/docs/src/zh/pchronicle/design/storyline-lance.md index f0b1ac571..4f0580bcf 100644 --- a/docs/src/zh/pchronicle/design/storyline-lance.md +++ b/docs/src/zh/pchronicle/design/storyline-lance.md @@ -1,74 +1,7 @@ -# Storyline 三表 Lance 存储 +# Storyline Lance -`StorylineLanceStore` 是 pChronicle 的 Storyline-native 规范化存储表示。它与 -`events.lance` 原始事件日志并列存在，不替代后者。 - -逻辑 wire schema 以 [RFC-0001 § Wire schema](../../rfcs/0001-storyline-format.md#wire-schema) -为准；ACTF、ATIF 与 OpenAI Messages 的逐字段转换分别以 -[RFC-0004](../../rfcs/0004-actf-format.md#actf-storyline-json-pointer-mapping)、 -[RFC-0008](../../rfcs/0008-atif-format.md#atif-storyline-json-pointer-mapping)和 -[RFC-0009](../../rfcs/0009-openai-messages-format.md#openai-storyline-json-pointer-mapping) -的映射章节为准。本设计只定义 Storyline 的 Lance 物理投影。 - -## 投影合同与闭环 - -Storyline 同时保留 Hub 交换合同（A 路径），三表则是可从 canonical `events.lance` -重建的 silver projection（B 路径）。两种用途共享 schema，但写入身份严格分开：交换格式 -导入或直接 `replace_storyline` 不携带 canonical 血缘；只有 events projector 可以发布带 -projection lineage 的 `CURRENT`。 - -```text -events.lance (事实源) - ├─ serve 启动期/运行期 ─► runs + steps + tool_calls + objects - ├─ append-compatible sync ─► 只替换 append suffix 影响的 session - └─ Catalog fallback ──────► 投影缺失或 stale 时按固定快照即时转换 -``` - -`CURRENT` 除四张表的精确 Lance version 外，还记录 source URI / source id、 -`fact_version`、`fact_rows`、构建时的 layout revision、projector、recipe hash 和 -completeness。`fact_version` / `fact_rows` 是新鲜度水位；单纯 compaction 只 -改变 layout revision，不会使投影过期。直接文档写入会清除 lineage，维护则原样保留。 - -增量 sync 只有在 canonical manifest 强制校验 `fact_rows == total_rows()` 的前提下，才能把 -`[previous_fact_rows, fact_rows)` 当作 append 区间。布局维护还必须保持 replacement 前后行数 -和 segment 顺序不变，使 compaction 不会移动这个逻辑水位。区间读取完成后，projector 会再 -校验返回记录数严格等于区间长度；任一证明前提失效都会失败关闭，而不是静默漏事实。 - -常用运维命令： - -```bash -pchronicle serve --control 127.0.0.1:0 ./trajectory-data -pchronicle stats ./trajectory-data --format json -``` - -`serve` 在输出 readiness 前发现所有已验证且非空的 canonical Store，并把投影收敛到确定的 -同级 `storyline`。运行期间会继续发现新 Store，按条件执行增量 sync 或完整 rebuild；失败采用 -有界并发和重试，且不阻塞 canonical durable write。没有匹配 lineage 的目标属于外来数据， -绝不覆盖。`status` 报告 `fresh`、`stale`、`missing` 或 `error`，以及事实水位和 generation。 - -Catalog 会把 lineage 指向同一 canonical URI 的 sidecar 与 events Source 合并为一个逻辑 -Source。`sources.projection_status` 为 `fresh` 时，规范化查询使用三表；为 `stale` 时隐藏 -sidecar 并回退到固定 events snapshot 的确定性投影。`projection_generation` 公开实际命中的 -generation，便于监控与诊断。Catalog 不会把无血缘的 Storyline 文档库自动认作 canonical -events 的 projection。 - -Gateway 配套的 Warehouse 为单 trace 观测提供显式 live-read 路径：Catalog 定位已经发现的 -canonical source 后，`/api/events`、`/api/storyline` 和 `/api/trajectory-view` 会重新打开该 -source 最新可见的 events manifest。这不改变全局 SQL 查询的不可变快照语义，也不会让派生的 -Storyline sidecar 变成权威事实源。 - -投影 supervisor 已内置于 `serve`，使用有界并发和重试，并随进程一起关闭。它不进入 -Gateway 捕获写入热路径，因此 projection 或 Catalog refresh 故障不会阻塞 canonical -events 写入。 - -本文只负责三表物理 schema、内容层、Snapshot 发布、查询接入和维护语义。事实源与 -projection ownership 见[运行存储](trajectory-storage.md)，用户查询流程见 -[Dataset 查询指南](../guides/discover-and-query.md)。 - -这是 pChronicle 唯一的规范化三表模型。旧的 ATIF -`sessions` / `steps` / `tool_calls`、`NormalizedStore` 和内存联表视图已经删除。ATIF -仍作为输入输出格式存在，但查询时先转换为 Storyline，再投影到本页定义的 -`runs` / `steps` / `tool_calls` schema，不再维护第二套表结构。 +`StorylineLanceStore` 直接保存 Storyline 文档，通过 `CURRENT` 发布并固定 +runs、steps、tool-call 和内容表的 generation 与版本。 ## 表模型 @@ -309,8 +242,7 @@ Blob payload I/O；为避免把 preview 当成完整值产生错误结果，内 ## 统一查询引擎 -`ChronicleQueryEngine` 是对外的只读 SQL 门面。六种磁盘格式（Canonical Event、 -Storyline Lance、AgenticMD、ATIF、OpenAI Msg、ACTF）通过同一个入口 +`ChronicleQueryEngine` 是对外的只读 SQL 门面。五种磁盘格式（Storyline Lance、AgenticMD、ATIF、OpenAI Msg、ACTF）通过同一个入口 `ChronicleQueryEngine::open(format, path, options)` 打开，注册语义对应的查询表， 查询语句不随物理格式改变： @@ -337,11 +269,6 @@ let jsonl = atif.query_jsonl( ).await?; ``` -`DocumentFormat::CanonicalEvent` 注册 `events` 表；`runs`/`steps`/`tool_calls` -默认不实时注册，需要 Storyline 查询面时优先使用 lineage 新鲜的 Storyline Lance -投影，无投影时在行/字节预算内执行 bounded fallback（预算耗尽显式报错，不静默 -截断）。其余五种格式注册 `runs`/`steps`/`tool_calls`。 - `query` 返回 Arrow `RecordBatch`，适合服务端继续处理；`dataframe` 返回 lazy DataFrame， 适合追加 DataFusion 变换或查看计划；`query_jsonl` 用于 CLI/API 边界。调用者也可通过 `context()` 取得 `SessionContext` 注册 UDF 或额外表。`backend_info()` 返回 @@ -436,11 +363,6 @@ pchronicle query ./openai-data \ 这个门面执行 DDL/DML。Lance 引擎打开时固定 `CURRENT` 指向的三个版本，从而保证一次 查询会话内三张表来自同一快照。 -仓库使用 Criterion.rs + hyperfine 的统一 benchmark runner。Criterion 负责 CPU-bound -转换、events→Storyline 和三表 split/reconstruct 微基准；canonical event append、投影 -build/sync/verify、Lance/DataFusion 生命周期、JSON streaming 与 RSS 场景由 hyperfine -重复执行独立进程，最终生成统一 JSON、Markdown 和 HTML： - ```bash # PR/local smoke workload just benchmark-pchronicle @@ -473,9 +395,4 @@ benchmark 还单独输出 DataSource 冷打开并执行 SQL、`get_storyline_ful 索引；Lance 的主要优势仍是更小的物理体积、近乎常数的 datasource 打开时间，以及列 裁剪、并行扫描和选择性索引收益。 -## 相关文档 - -- [事实、Projection 与 Revision](../concepts/facts-and-projections.md)：解释 Storyline 为何是 projection。 -- [pChronicle 架构](architecture.md)：定义 publication 和 read consistency 保证。 -- [Snapshot](catalog.md)：说明 Source discovery 与固定 Snapshot 如何打开本 Store。 -- [`pchronicle` 参考](../reference/cli.md)：当前对外查询、导入导出与服务命令。 +## 相关文档 \ No newline at end of file diff --git a/docs/src/zh/pchronicle/design/trajectory-storage.md b/docs/src/zh/pchronicle/design/trajectory-storage.md index c776d3d35..316562aed 100644 --- a/docs/src/zh/pchronicle/design/trajectory-storage.md +++ b/docs/src/zh/pchronicle/design/trajectory-storage.md @@ -1,179 +1,19 @@ -# pChronicle 运行存储 +# 轨迹存储 -> 当前实现说明。规范性所有权见 [RFC-0003](../../rfcs/0003-pchronicle-ownership.md)与 -> [RFC-0007](../../rfcs/0007-events-contract-pchronicle-sidecar.md)， -> Dataset 命令见 [`pchronicle`](../reference/cli.md)。 +轨迹处理模型为 `StorylineDocument`。支持 ATIF、ACTF、OpenAI Messages、 +Storyline JSON 与 AgenticMD；Codex 和 Claude Code 会话 JSONL 只支持导入。 +编解码器继续解析并保留这些格式的时间戳。 -交换格式 wire contract 与逐字段转换以 [RFC-0001 § Wire schema](../../rfcs/0001-storyline-format.md#wire-schema)、 -[RFC-0004 § ACTF 映射](../../rfcs/0004-actf-format.md#actf-storyline-json-pointer-mapping)、 -[RFC-0008 § ATIF 映射](../../rfcs/0008-atif-format.md#atif-storyline-json-pointer-mapping)和 -[RFC-0009 § OpenAI Messages 映射](../../rfcs/0009-openai-messages-format.md#openai-storyline-json-pointer-mapping)为准。 +`StorylineLanceStore` 直接把文档写入 `runs.lance`、`steps.lance` 和 +`tool_calls.lance`，大字段使用内容层。`CURRENT` 固定已发布 generation 和精确 +Lance 版本。文档重建读取固定版本并校验引用内容。 -## 1. 定位 +导入支持创建、追加与替换。本地整 Dataset 替换先暂存输出再交换目录；对象存储 +整 Dataset 替换的失败语义不同，详见导入指南。store 内部发布使用写入租约与 + generation 校验。维护与删除遵守同样的发布与内容完整性保证。 -`persisting-pchronicle` 是 Agent 运行数据的结构化存储层，统一拥有： +Compact JSONL 是独立的记录存储路径，保留原始 JSON 字节，不推断 Storyline +Run、Step 或 ToolCall。 -- 公共 `persisting-events::EventRecord` 到 Lance `EventRow` 的映射与物理 schema； -- Run / Story 坐标、目录布局和发现规则； -- Lance canonical events 的读写、统计和维护； -- AgenticMD 人读/调试视图的生成与宽松解析； -- events、Storyline、ATIF、ACTF、OpenAI messages、AgenticMD 之间的格式转换； -- materialize、revision lineage 和标准查询视图。 - -`persisting-events` 拥有存储无关的逻辑事件信封。Gateway 负责产出事件；CLI -可以在进程内调用 pChronicle，本地 producer 也可以通过 `pchronicle serve` 的 Control 服务提交。 -这些 producer 都不定义第二套运行数据落盘格式。 - -## 2. 逻辑坐标 - -```text -Run -└── Storyline - └── Turn - └── Call - └── EventRecord -``` - -离线存储使用 `StoryCoords`： - -| 字段 | 含义 | -|---|---| -| `storage` | pChronicle 根目录 | -| `agent_id` | Agent 身份，单路径段 | -| `root_session_id` | 可选 Run 身份；主 Agent 与 subagent 共用 | -| `session_id` | Story 身份，也是 Lance 分区键 | - -有 `root_session_id` 时，多个 Story 共用一个 Run 级 `events.lance`；没有时， -`session_id` 自身就是目录边界。 - -## 3. 存储布局 - -### Lance events - -`events.lance/` 是完整事件表示的 Run 级容器：`_manifest.json` 保存 active writer fence -和可见 segment version，每个 writer epoch 使用独立 Lance segment。它保留 HTTP/模型调用、 -时间、身份、payload 和顺序。 -Gateway 的 durable 微批写入每累计 8 个小 fragment 就 seal 一个 L0 segment；后台维护将 -连续 8 个同层 sealed segment 合并并晋升到下一层。合并使用 manifest 中的 `level` 与 -`sealed` 元数据，只替换精确匹配的连续 segment 区间，active tail 永不参与。由此 visible -segment 数按层级增长而不是随事件线性增长；旧 version/file 仍按 maintenance 的保留期 -vacuum，避免破坏已经固定旧快照的 reader。 -物理 schema 把 `event_id` 提升为独立业务列，并把 `timestamp` 规范化为 UTC -`Timestamp(Millisecond)`。新写入的 Gateway `EventRecord` 会同时提供 RFC3339 -`timestamp` 和 `timestamp_unix_ms`；两者必须在毫秒级一致。admission 仍会为旧 producer -或兼容导入根据 RFC3339 `timestamp` 或接收时间补齐缺失值。Storyline 投影也从 -`timestamp_unix_ms` 生成 UTC 毫秒文本，输入文本时间戳保存在 `payload_json`。事实层不检查 -`event_id` 唯一性，也不为它维护索引； -重复 ID 和重试行是合法事实。完整 `EventRecord` 仍保存在 `payload_json`，因此回放不丢字段。 -需要审计保真度的工作流应使用 canonical events 层。 - -### AgenticMD - -AgenticMD 是面向人的 Markdown 调试视图。它保存可见对话块和会话摘要，适合实时查看、 -代码审阅与人工分析。它会省略协议噪声，字段也允许缺失或扩展，因此不是存储格式或 -原始 HTTP 事件的无损替代。 - -Gateway 捕获通过 pChronicle 写入 canonical events。Live AgenticMD 是诊断投影，Dataset -消费统一使用 pChronicle API 和 `pchronicle` 命令。 - -### Storyline 三表 Lance - -`StorylineLanceStore` 提供面向分析和 ATIF 互操作的规范化存储表示： -`runs.lance`、`steps.lance`、`tool_calls.lance`。它按 `source_call_id` 将 observation -result 归并到 tool call 行，并通过 `CURRENT` 中的三表版本元组保证原子切换。超过阈值的 -UTF-8/JSON cell 以 BLAKE3 内容地址外置到共享 `objects.lance`，跨运行复用；公开 schema -和 SQL 结果保持不变，查询只在真正引用内容列时延迟恢复 Blob。 - -ATIF object、array、pretty JSON 与 JSONL/NDJSON，以及 ACTF object/array 的 `steps` 临时查询还支持 -projection-aware 快路径：DataFusion 先传递所需列和安全谓词，reader 通过 seeded visitor -跳过未引用大字段并直接构造窄 Arrow batch；JSONL/NDJSON 按资源限制逐记录读取，array 通过 -结构扫描器逐 element 提取并使用 slice decoder，单 object 从 reader 流式解码。 -`SELECT *`、其他表和 OpenAI-message 回退到完整 Storyline 规范化。详细协议、发布顺序和执行边界见 -[Storyline 三表 Lance 存储](storyline-lance.md)。 - -## 4. 目录布局 - -扁平 Story： - -```text -storage/ -└── agent_id/ - └── session_id/ - ├── events.lance/ - │ ├── _manifest.json - │ └── segments/.lance/ - └── session_id.md -``` - -包含 subagent 的 Run： - -```text -storage/ -└── agent_id/ - └── root_session_id/ - ├── events.lance/ # manifest + writer segments，按 session_id 过滤 - ├── root_session_id.md - └── agent-.md -``` - -独立的 Storyline 分析 store 使用 `CURRENT`、`generations//{runs,steps,tool_calls}.lance` -和根级共享 `objects.lance`；`CURRENT` 的必需 `schema_version` 在打开表前验证整个四表物理 -布局。它不改变上面的 canonical event 目录。 - -系统生成的 AgenticMD 使用 `{session_id}.md` 文件名和 -`` 块结构；读取器同时接受无 speaker 的块、旧 -`role/seq/session/agent` 字段和普通 Markdown 正文。 - -## 5. 写入与一致性 - -1. `EventRecord` 进入 Lance 前转换为 Arrow 行，一个有界微批对应当前 epoch segment 的一次 - Lance append，随后以 manifest CAS 发布精确 version。 -2. 热路径不读取旧行、row count 或 `event_id`，不执行查重、索引、压缩或 vacuum。 -3. producer 身份与写入坐标冲突时仍然 append；`payload_json` 保留原始声明，物理 - `session_id` / `agent_id` 由调用方坐标决定并在 replay 时生效。投影对其余身份声明按 - append order 采用最后一个非空值，不为冲突增加读前写、查重或索引成本。 -4. `seq` 是 producer 定义的 Storyline 序号；replay cursor 使用不可变的物理 append 顺序。 -5. Run bucket 中不同 Story 共享 manifest 和 epoch segment，但 replay/stats 按 `session_id` 隔离。 -6. live Markdown 以 `call_id + source`（兼容旧 role）定位块，允许流式 agent 原地更新。 -7. canonical append 与派生投影分别报告结果；投影失败不能伪装成事件已持久化。 -8. 一个微批中不同 root URI 的 segment/manifest 最多 16 路并行发布；同一 URI 仍按 - batch 顺序串行，因此不放宽单 Story 的物理 append 顺序。 - -事件事实层提供 at-least-once append，不提供 exactly-once 或 ID 唯一性。truncate、overwrite -和 retry dedup 不属于事实写路径；转换到已有 Run 会失败，裁剪应创建新 Run 或在派生 -Storyline 上完成。compaction、`session_id` 索引和 vacuum 是显式离线维护。 - -上层 Run lease 产生单调 epoch；`EventWriterFence(epoch, writer_id)` 在新 writer 写数据前 -激活。reader 只读取 manifest 固定的 segment version，因此旧 writer 在 takeover 后完成的 -底层 append 不可见。相同 epoch 的另一个 writer_id 会被拒绝。该协议提供 writer fencing， -不把并发多 writer 合并定义为支持的写入模式。 - -## 6. 格式转换 - -通用外围格式通过 Storyline hub 转换： - -```text -AgenticMD ─┐ -ATIF ──────┼── Storyline ── events / AgenticMD / ATIF / ACTF / OpenAI messages -ACTF ──────┤ -OpenAI msg ┘ -``` - -需要保存原始 payload 的路径直接读写 events，不能经有损 Storyline roundtrip。 -外围交换格式由 `pchronicle import/export` 处理。 - -## 7. 组件边界 - -| 组件 | 负责 | 不负责 | -|---|---|---| -| Gateway | 协议解析、调用生命周期、采集顺序、live projection 策略 | 通用 store、格式 schema、离线转换 | -| pChronicle | 格式、路径、落盘、读取、转换与 revision lineage | 网络转发、Agent 生命周期 | - -## 8. 相关文档 - -- [记录数据、查询视图与派生版本](../concepts/facts-and-projections.md) -- [发现并查询](../guides/discover-and-query.md) -- [Snapshot](catalog.md) -- [AgenticMD 格式](../reference/agenticmd.md) -- [Gateway 捕获](../guides/serve-gateway.md) -- [`pchronicle` Dataset 命令](../reference/cli.md) +参见 [Storyline Lance](storyline-lance.md)、[Snapshot](catalog.md)、 +[格式](../reference/formats/index.md) 与 [导入导出](../guides/exchange.md)。 diff --git a/docs/src/zh/pchronicle/guides/exchange.md b/docs/src/zh/pchronicle/guides/exchange.md index bb30b7a7c..af27978f9 100644 --- a/docs/src/zh/pchronicle/guides/exchange.md +++ b/docs/src/zh/pchronicle/guides/exchange.md @@ -40,47 +40,6 @@ pchronicle import --from ./corpus --to ./normalized \ --output-format storyline ``` -经过验证且非空的 canonical Event Store 会在 JSON 扫描前被识别，并始终创建 -Storyline Lance： - -```bash -pchronicle import --from ./run/events.lance --to ./run/storyline -``` - -此模式接受本地和 object-store URI，不修改源，支持 create 或经确认的 replace（不支持 append）。JSON 结果报告 -`format: "events"`、`output_format: "storyline-lance"` 和 `fact_rows`，不包含 -`input_bytes`。canonical events 不接受显式 `--output-format preserve` 或 JSON exchange -`--input-format`。 - -squash 后，Dataset 所有规范化表中的 `_file_` 都是 `.`： - -```bash -pchronicle query ./normalized \ - --sql 'SELECT _file_, COUNT(*) AS runs FROM dataset.runs GROUP BY _file_' -``` - -Storyline 输出中的 `document_id` 全局唯一；冲突时会确定性地增加 `#N` 后缀，append 也可用 -`--on-duplicate skip` 跳过。成功的 Storyline Store 不把来源路径保存为可查询信息。需要保留 -文件边界时应使用 preserve 输出。 - -ATIF `.jsonl` 与 `.ndjson` 输入会逐条解码其中的非空记录。递归扫描目录时会跳过软链接； -若 `--from` 显式指定一个指向普通文件的软链接，则仍按单文件导入。只有所有输入和所选 -存储输出都成功后才会原子发布完整输出目录。单文件 preserve 导入会使用 -`trajectories.atif.jsonl`/`trajectories.atif.ndjson`，确保后续查询仍按行式容器读取。stdin 必须 -是有限且格式明确的输入： - -```bash -cat input.json | pchronicle import --from - \ - --to ./imported --input-format openai-messages -``` - -导入后检查新边界： - -```bash -pchronicle stats ./imported -pchronicle stats overview ./imported -``` - ## 导出完整 Run ```bash @@ -101,4 +60,4 @@ pchronicle export --from ./imported --to one.json --output-format actf \ Import/export 不是存储迁移协议，任意 SQL row 也不是可导出的完整 Run。精确参数见 [`pchronicle` 命令参考](../reference/cli.md)。格式契约见 [运行数据格式](../reference/formats/index.md)，层次边界见 -[数据契约与 Revision](../concepts/facts-and-projections.md)。 +[轨迹数据与版本](../concepts/facts-and-projections.md)。 diff --git a/docs/src/zh/pchronicle/guides/index.md b/docs/src/zh/pchronicle/guides/index.md index a67e69e73..eaca153a9 100644 --- a/docs/src/zh/pchronicle/guides/index.md +++ b/docs/src/zh/pchronicle/guides/index.md @@ -6,7 +6,6 @@ 2. [导入与导出 Run](exchange.md)。 3. [在本地服务 Dataset](serve.md)。 4. [使用本地 Web UI 浏览、下钻和分析](ui.md)。 -5. [通过 `serve` Gateway 转发、改写并捕获请求](serve-gateway.md)。 Guide 解释决策并完成工作流；精确参数查阅 [`pchronicle` 命令行指南](../reference/cli.md)，仓库内 fixture 见[项目示例](../../project/examples.md)。 diff --git a/docs/src/zh/pchronicle/guides/serve-gateway.md b/docs/src/zh/pchronicle/guides/serve-gateway.md deleted file mode 100644 index da39ae2e3..000000000 --- a/docs/src/zh/pchronicle/guides/serve-gateway.md +++ /dev/null @@ -1,382 +0,0 @@ -# `pchronicle serve` 的 Gateway 入库、转发与捕获 - -`pchronicle serve` 提供两个互斥的 Gateway 模式： - -- `--gateway ADDRESS` 启动无需配置文件的 canonical trajectory HTTP 入库端点； -- `--gateway-config FILE` 保留本文后续介绍的 LLM 转发、协议改写与 capture 兼容模式。 - -无需配置的形式全部由命令行控制： - -```bash -pchronicle serve \ - --gateway auto \ - --gateway-dataset ./data/captures \ - --gateway-split '{user}/{date}/{hour}' -``` - -客户端向 `POST /v1/events` 提交包含 `agent_id`、`session_id`、可选 -`root_session_id` 和 `records` 的批次；records 使用共享 canonical `EventRecord` schema。 -`x-persisting-user-id` 提供 `{user}`，缺失时使用 `_unknown`。成功响应表示所有 accepted -records 已经 durable visible。canonical `/v1/events` 单次最多 256 条记录， -请求体上限为 8 MiB。Langfuse OTLP 一批可能包含更多 spans，Gateway 会在内部 -切块，客户端看到的请求边界不变。由 `traceId` 和 `spanId` 派生的事件 ID 在重试 -时保持稳定，可作为下游压缩/去重的确定性键。 - -Split template 必须是相对路径，最多 16 段，只接受安全字面段以及精确占位符 -`{user}`、`{date}`、`{hour}`。时间字段使用 UTC；同一逻辑 run/session 在进程生命周期内 -固定到首次选择的分区。`--gateway-split-idle` 控制已有 canonical source 在最近一次 -事件后等待多久再刷新 Storyline projection，默认 `30m`；新发现的 source 首次仍会立即 -建立 projection。它是 projection 的空闲窗口，不改变物理 split 路径。 - -本文后续章节描述转发兼容模式。 - -## Langfuse OTLP 压力基准 - -仓库提供一个黑盒压力测试，会启动真实的 `pchronicle serve --gateway` 子进程， -通过 Langfuse OTLP HTTP 路径并发发送合成 spans，并在退出后查询 Dataset。它默认 -被 Cargo 忽略，不会拖慢普通 CI： - -```bash -PCHRONICLE_LANGFUSE_STRESS_REQUESTS=32 \ -PCHRONICLE_LANGFUSE_STRESS_SPANS_PER_REQUEST=512 \ -PCHRONICLE_LANGFUSE_STRESS_CONCURRENCY=8 \ -cargo test -p persisting-pchronicle-cli --test langfuse_gateway_stress \ - langfuse_gateway_pressure --offline -- --ignored --nocapture -``` - -测试同时验证 HTTP 200 和完整成功 OTLP 响应、超过 256 spans 时的内部切块、 -durable event 总数、trace/span/parent 关联以及 event ID 唯一性，并输出请求/秒、 -span/秒和 p50/p95/p99 延迟。三个环境变量可以按硬件调整规模；吞吐数字应在固定 -磁盘、文件系统和并发度的环境中比较，不作为跨机器的硬性 SLA。 - -`pchronicle serve` 可以启动本地 LLM Gateway，并可选择是否同时启动只读 Dataset 服务。Gateway 对每个请求 -选择 upstream，按需改写模型和 wire protocol，再以客户端协议返回响应，同时把 canonical -capture events 追加到 CLI 指定的输出 Dataset。Dataset Web UI 和 API 仍然保持只读。 - -当 Agent 或 SDK 已经能够调用 OpenAI、Anthropic 或 Gemini 兼容的 base URL，而你希望把 -这些流量捕获到 Dataset 时，可以使用这个模式。 - -## 配置输入 - -Gateway 模式刻意分离 Dataset 选择和转发配置： - -| 输入 | 负责的内容 | -| --- | --- | -| `--gateway-dataset DATASET` | 输出 Dataset URI，并自动挂载 | -| 传给 `--gateway-config` 的 `gateway.toml` | Gateway listener、模型路由、凭据、捕获级别和网络策略 | -| 其他 CLI 参数 | 物理 split、本地 Gateway 状态目录、实时 Markdown 和前台调试 | - -Gateway TOML 中的未知字段会被拒绝。Gateway 配置必须使用 TOML，不接受其他扩展名。 - -## 最小配置 - -创建 `gateway.toml`： - -```toml -listen = "127.0.0.1:8787" -admin_listen = "127.0.0.1:8788" -agent_id = "local-agent" -capture_level = "dialogue" - -[[models]] -name = "deepseek-chat" -provider = "openai" -upstream = "https://api.deepseek.com/v1" -api_key_env = "DEEPSEEK_API_KEY" - -[[models]] -name = "*" -forward = "deepseek-chat" -``` - -导出凭据，然后启动两个服务： - -```bash -export DEEPSEEK_API_KEY=sk-... - -pchronicle serve \ - --listen 127.0.0.1:8080 \ - --gateway-config gateway.toml \ - --gateway-dataset ./data/captures \ - --gateway-stream-markdown -``` - -该命令会启动三个 loopback listener： - -- `127.0.0.1:8080`：Dataset Web UI 和只读 API； -- `127.0.0.1:8787`：LLM Gateway； -- `127.0.0.1:8788`：Gateway 状态和 session API。 - -省略 `--listen` 时只启动 Gateway listener 与 capture sink，不会创建 Dataset HTTP -endpoint。 - -将 Agent 或 SDK 的 base URL 指向 `http://127.0.0.1:8787/v1`。例如： - -```bash -curl http://127.0.0.1:8787/v1/chat/completions \ - -H 'content-type: application/json' \ - -H 'x-persisting-session-id: example-session' \ - -d '{ - "model": "deepseek-chat", - "messages": [{"role": "user", "content": "Hello"}] - }' -``` - -可以独立于 Dataset 服务检查 Gateway： - -```bash -curl http://127.0.0.1:8788/admin/status -curl http://127.0.0.1:8788/admin/sessions -``` - -## 请求生命周期 - -Gateway 将转发、改写和捕获组成一条请求管线： - -```text -client request - -> 识别客户端协议和模型 - -> 选择第一个命中的 models[] 路由 - -> 对配置的模型路由执行授权 - -> 按需改写模型和协议 - -> 构造 upstream URL 和认证信息 - -> 转发到 upstream - -> 按需把响应翻译回客户端协议 - -> 记录请求以及完整或流式响应 -``` - -Capture metadata 会区分客户端请求模型和实际 upstream 模型，并记录是否发生模型改写。 -捕获不会取代转发：upstream error 会返回客户端，同时结束对应的 capture call。 - -## Gateway 顶层字段 - -| 字段 | 必填 | 默认值 | 含义 | -| --- | --- | --- | --- | -| `listen` | 是 | — | LLM proxy listener。内嵌模式只接受 loopback 地址；端口 `0` 表示选择可用端口。 | -| `admin_listen` | 否 | `127.0.0.1:9876` | `/admin/status` 和 `/admin/sessions` 的 listener，同样只允许 loopback。 | -| `agent_id` | 否 | `default` | 写入捕获记录的 Agent identity；请求能够提供更具体身份时使用请求身份。 | -| `session_header` | 否 | `x-persisting-session-id` | 用于把请求归入 session 的 header。 | -| `capture_level` | 否 | `dialogue` | 保留请求与响应内容的级别：`summary`、`dialogue` 或 `full`。 | -| `debug` | 否 | `false` | 向状态目录写入 Gateway 诊断；日志可能包含有界的请求和响应 body。 | -| `models` | 是 | — | 有序模型路由列表。 | -| `network` | 否 | `mode = "public"` | 显式 forward-proxy 流量的策略。 | - -共享 Gateway schema 还接受 `[overlay]`，但 `pchronicle serve` 不会创建或 apply 文件系统 -overlay。 - -### 捕获级别 - -- `summary` 保存协议元数据和字节数，不保存用户或 assistant 文本； -- `dialogue` 保存用户和 assistant 对话，是默认值； -- `full` 还会保存解析后的请求和响应 body。只有在能够接受额外存储量和敏感内容暴露时才应 - 使用。 - -## 模型路由 - -路由按文件顺序匹配，第一个命中的 `name` 生效。`name` 可以是精确模型名、`prefix*`、 -`*suffix` 或兜底规则 `*`。 - -| 字段 | 含义 | -| --- | --- | -| `name` | 必填的匹配规则或精确目标模型名；不能重名。 | -| `provider` | `openai`、`anthropic`、`gemini`、`vertex`、`bedrock`、`azure`、`copilot` 或 `custom`；默认 `openai`。 | -| `upstream` | Upstream base URL；适用时应包含 API prefix。未设置 `forward` 时必填。 | -| `upstream_anthropic` | `/v1/messages` 使用的可选 Anthropic-compatible base；未设置时使用 `upstream`。 | -| `api_key_env` | 由 `pchronicle` 进程读取的环境变量，推荐使用这种凭据来源。 | -| `api_key` | 内联凭据。实现支持，但不应把 secret 提交到配置文件。 | -| `forward` | 另一个路由的精确名称；重写请求模型并使用目标路由的 upstream。 | - -同一路由不能同时设置 `upstream` 和 `forward`。Forward 必须指向带有 upstream 的路由，且 -不能形成多级链。如果配置或指定环境变量没有提供 key，Gateway 可以使用兼容的客户端认证 -header；两者都没有时，请求会在转发前失败。 - -多 Provider 示例： - -```toml -listen = "127.0.0.1:8787" -admin_listen = "127.0.0.1:8788" -capture_level = "dialogue" - -[[models]] -name = "claude*" -provider = "anthropic" -upstream = "https://api.anthropic.com/v1" -api_key_env = "ANTHROPIC_API_KEY" - -[[models]] -name = "gpt*" -provider = "openai" -upstream = "https://api.openai.com/v1" -api_key_env = "OPENAI_API_KEY" - -[[models]] -name = "gemini*" -provider = "gemini" -upstream = "https://generativelanguage.googleapis.com/v1beta" -api_key_env = "GEMINI_API_KEY" -``` - -## 转发与模型改写 - -路由选定后，Gateway 会根据 `upstream` base 规范化有效请求路径。Upstream 以 `/v1` 结尾， -客户端请求 `/v1/chat/completions` 时，结果仍是一个 `/v1/chat/completions`，而不是 -`/v1/v1/chat/completions`。Passthrough 请求会保留 query parameter。 - -Gateway 转发端到端 header，但移除 `Host`、`Content-Length`、hop-by-hop header、proxy -authentication 和客户端 LLM credential，然后按 Provider 写入 OpenAI Bearer、Anthropic -`x-api-key` 或 Gemini `x-goog-api-key`。Redirect 会返回客户端，不会在 Gateway 内部继续跟随。 - -`forward` 可以把客户端模型改写到一个精确目标路由： - -```toml -[[models]] -name = "echo-upstream" -upstream = "http://127.0.0.1:19080/v1" - -[[models]] -name = "*" -forward = "echo-upstream" -``` - -此时对 `client-model` 的请求会以 `"model":"echo-upstream"` 到达目标。Capture metadata -保留两种 identity 并标记改写。Forward target 必须定义 `upstream`，不能继续 forward，也不会 -再次进行 pattern match。路由采用 first-match，因此具体 pattern 应放在宽泛 pattern 之前。 - -## 协议改写 - -Gateway 根据客户端 path 和目标路由选择 protocol bridge。普通响应和 SSE stream 都会转换： - -| 客户端协议 | 目标路由 | Upstream 协议 | -| --- | --- | --- | -| Chat Completions | 非 Gemini | Chat Completions passthrough | -| Anthropic Messages | 设置了 `upstream_anthropic` | 原生 Messages passthrough | -| Anthropic Messages | 未设置 `upstream_anthropic` 的 OpenAI-compatible route | Chat Completions | -| OpenAI Responses | 原生 OpenAI 或 Azure OpenAI | Responses passthrough | -| OpenAI Responses | 其他 OpenAI-compatible upstream | Chat Completions | -| Chat Completions、Messages 或 Responses | `provider = "gemini"` | Gemini `generateContent` 或 `streamGenerateContent` | - -需要转换时，Gateway 会改写 request path 和 body，再把 response 或受支持的 error envelope -渲染回客户端协议。Bridge 保留常见 message、tool call、usage、reasoning 和 streaming event， -但不承诺所有 Provider-specific extension 都能无损保留。客户端依赖这些字段时应使用 -passthrough。 - -## 使用 Echo upstream 测试 - -仓库提供了一个确定性的 Rust Echo server，可以在没有 API key 和真实模型服务的情况下测试 -真实 HTTP 转发。它支持 `/echo`、Chat Completions、Messages、Responses、Gemini 及对应的 -streaming 形式；最后一条用户文本决定 assistant 输出。 - -从源码仓库启动： - -```bash -just echo - -# 等价的已安装命令： -pchronicle dev echo --listen 127.0.0.1:19080 --encoding plain -``` - -将 Gateway route 指向 Echo server： - -```toml -listen = "127.0.0.1:8787" -admin_listen = "127.0.0.1:8788" -capture_level = "full" - -[[models]] -name = "echo-upstream" -provider = "openai" -upstream = "http://127.0.0.1:19080/v1" - -[[models]] -name = "*" -forward = "echo-upstream" -``` - -默认情况下，assistant 直接返回最后一条用户文本。可以让单个请求改为返回标准 Base64： - -```bash -curl http://127.0.0.1:8787/v1/messages \ - -H 'content-type: application/json' \ - -H 'x-persisting-echo-encoding: base64' \ - -d '{ - "model": "client-alias", - "max_tokens": 32, - "messages": [{"role": "user", "content": "hello"}] - }' -``` - -这条 Messages 请求会被转换成 Chat Completions，模型改写成 `echo-upstream`，响应再转换回 -Messages，其中的文本是 `aGVsbG8=`。加入 `"stream": true` 可以通过 SSE 测试同一条路径。 -请求 header 接受 `plain` 或 `base64`；`--encoding` 设置服务器默认值。隐藏测试命令 -`pchronicle dev echo` 只允许 loopback listener，用于确定性的本地 Gateway 测试。 - -## 网络策略 - -`[network]` 控制 `CONNECT` 和 absolute-URI request 等显式 proxy 流量。配置在 -`models[].upstream` 中的目标是 Gateway 自己拥有的路由，不是 Agent egress grant；应通过 -模型路由列表限制 LLM surface。 - -可用模式包括： - -```toml -[network] -mode = "no-network" -``` - -```toml -[network] -mode = "allowlist" -allowed_hosts = ["pypi.org", "files.pythonhosted.org", "*.github.com"] -``` - -`public` 是默认值；`no-network` 拒绝显式 proxy egress；`allowlist` 要求请求命中 -`allowed_hosts` 或结构化 `[[network.rules]]`。显式 `[[network.deny_rules]]` 的优先级高于 -allow。`pchronicle serve` 只能控制客户端主动发送到 Gateway 的流量，不提供进程级网络隔离。 - -## Dataset 与状态目录选择 - -`--gateway-dataset DATASET` 是必填的本地路径或 object-store URI。pChronicle 会自动挂载 -该输出；如果同一 URI 已经通过位置参数挂载，则自动去重。 - -Canonical events 直接追加到目标 Dataset。Gateway runtime state 与其分离，其中包括 session -index、debug log 和可选实时 AgenticMD projection： - -- 本地 Dataset 未显式配置时，state directory 默认使用 Dataset path； -- `s3://...` 等 object-store Dataset 必须提供可写的本地 `--gateway-state DIRECTORY`； -- `--gateway-stream-markdown` 会在 state directory 中维护实时 AgenticMD projection。 - -即使使用本地 Dataset，也可以显式分离暂态 Gateway 文件： - -```bash -pchronicle serve \ - --gateway-config gateway.toml \ - --gateway-dataset ./data/captures \ - --gateway-state ./.pchronicle-gateway \ - --gateway-stream-markdown -``` - -## CLI 优先级与安全边界 - -- `--listen` 只配置 Dataset 服务；Gateway listener 始终来自 `gateway.toml`； -- `--gateway-debug` 即使在 `debug = false` 时也会开启 Gateway 调试；没有 - CLI 参数能够强制关闭配置中已开启的 debug； -- `--gateway-dataset`、`--gateway-split`、`--gateway-state` 和 - `--gateway-stream-markdown` 是组合层配置，不是 - Gateway TOML 字段； -- Dataset、Gateway 和 admin listener 都必须是 loopback 地址；这些服务不提供 - authentication 或 authorization 边界； -- debug 和 `full` 捕获可能保留敏感的请求或响应内容，应相应保护状态目录和 Dataset。 - -## 查看新捕获 - -Gateway writer 排空后，events 会持久写入目标 Dataset。已有 `events.lance` 追加不会触发 -完整 Snapshot 重建；单 trace 的 `/api/events`、`/api/storyline` 和 `/api/trajectory-view` -会重新打开该文件的最新 manifest，因此可以实时观察正在进行中的 trace。只有发现新的 -canonical source（例如新的 split 文件）或 Storyline projection 发布时，才需要刷新全局 -Snapshot。projection 或 refresh 失败会有界重试并保留旧的可查询 Snapshot；两者都不阻塞 -durable capture write。`POST /api/catalog` 仍可用于显式手工刷新。 -收到 `SIGINT` 或 `SIGTERM` 时， -`pchronicle serve` 会停止两个服务，并在退出前完成 Gateway capture writer。 - -精确命令参数见 [`pchronicle` CLI 参考](../reference/cli.md)；刷新背后的存储模型见 -[Snapshot 设计](../design/catalog.md)。 diff --git a/docs/src/zh/pchronicle/guides/serve.md b/docs/src/zh/pchronicle/guides/serve.md index 52ef6ec76..bdb52f99f 100644 --- a/docs/src/zh/pchronicle/guides/serve.md +++ b/docs/src/zh/pchronicle/guides/serve.md @@ -1,145 +1,19 @@ # 在本地服务 Dataset -`pchronicle serve` 把一个或多个 Dataset 挂载到内建的只读 Web UI 和 API。它用于本地检查， -不是公开或多租户数据服务。 - -## 命令原型 - -```text -pchronicle serve - [--listen LOOPBACK_ADDR] [--control LOOPBACK_ADDR] [--open] - [--home-link TEXT=PATH]... - [--gateway ADDRESS --gateway-dataset DATASET [--gateway-split TEMPLATE] - [--gateway-split-idle DURATION]] - [--gateway-config FILE --gateway-dataset DATASET [--gateway-state DIRECTORY]] - [--gateway-stream-markdown] [--gateway-debug] - [--catalog-config FILE] - [<[NAME=]DATASET> ...] -pchronicle serve catalog dataset add --catalog-config FILE NAME --uri URI [OPTIONS] -pchronicle serve catalog dataset remove --catalog-config FILE NAME... -pchronicle serve catalog dataset list --catalog-config FILE -pchronicle serve catalog issue --catalog-config FILE NAME -pchronicle serve catalog grant --catalog-config FILE NAME DATASET... -pchronicle serve catalog revoke --catalog-config FILE NAME DATASET... -``` - -所有 listener 都必须使用 loopback 地址，因为 Web UI 和只读 API 不提供公开网络所需的认证边界。 - -## 打开一个 Dataset - -```bash -pchronicle serve --open ./trajectory-data -``` - -单个裸 Dataset 会挂载为 `default`。不提供 listener 参数时，本地 Web UI 使用一个可用的 -loopback 端口。 - -## 挂载多个 Dataset - ```bash -pchronicle serve \ - --listen 127.0.0.1:8081 \ - evals=../data/atif archive=s3://example/archive -``` - -Mount name 会成为 SQL schema 和 API 名称。需要稳定名称时使用 `NAME=DATASET`。 -挂载多个裸路径时，pChronicle 会从各路径末段生成名称；这些名称可能随路径变化，因此可复用的 -命令仍应显式指定 mount name。 - -## 启动 Directory - -```bash -pchronicle serve catalog dataset add \ - --catalog-config catalog.toml prod \ - --uri s3://bucket/prod \ - --endpoint http://127.0.0.1:9000 \ - --region us-west-2 \ - --access-key BACKEND_AK \ - --secret-key BACKEND_SK -pchronicle serve catalog issue --catalog-config catalog.toml alice -pchronicle serve catalog grant --catalog-config catalog.toml alice prod evals +pchronicle serve ./trajectory-data +pchronicle serve --listen 127.0.0.1:8080 train=./train eval=./eval pchronicle serve --catalog-config catalog.toml --listen 127.0.0.1:8081 ``` -`catalog.toml` 列出 libraries（`[datasets.*]`，本地 path 或 `s3://`）和 users。 -`serve catalog dataset add|remove|list` 改写 libraries，不启动 HTTP。 -`serve catalog issue` 写入一个无授权用户，并把 sk 只打印到这次 stdout； -`grant` / `revoke` 改该用户可打开的 library 名称。运行中的 serve 每 3 秒检查配置，无需重启。 -NAME 使用 `*` 可以为当前所有用户授予或撤销数据集。新建用户不会自动继承过去的通配授权，创建后请重新执行命令。 - -`pchronicle serve --catalog-config` 会把文件中的 **全部** library 挂进 Warehouse -（与位置参数挂载等价），并启用 `catalog://` 换票路由。不要与位置参数 Dataset -同时使用。文件里的 S3 endpoint / region / 后端密钥会在打开存储前写入进程环境。 -使用 Directory 用户钥时，Web UI 可通过请求头发送 ak/sk。另一终端： - -```bash -pchronicle dataset pin team catalog://127.0.0.1:8081 --ak USER_AK --sk USER_SK -pchronicle query @team/prod --sql 'SELECT 1' -``` - -`@team` 是 Directory locator，不是 Dataset。`@team/prod` 换票后打开票里的 `uri` -（一条 path）。同一 Directory 文件里所有 `s3://` 库必须共用同一组 endpoint、region -和后端密钥。listener 仍只允许 loopback。嵌套 Dataset 发现可使用 -`chronicle.manifest`（[RFC-0015](../../rfcs/0015-chronicle-manifest.md)）。 -Directory 设计见 [RFC-0013](../../rfcs/0013-pchronicle-warehouse-catalog.md)。 - -## 启用 Control 或 Gateway 集成 - -```bash -pchronicle serve \ - --control 127.0.0.1:0 \ - default=./trajectory-data - -pchronicle serve \ - --gateway auto \ - --gateway-dataset ./trajectory-data \ - --gateway-split '{user}/{date}/{hour}' -``` - -`--gateway` 启动无需配置文件的 canonical event HTTP 入库端点。它接受 -`POST /v1/events`，从 `x-persisting-user-id` 取得 `{user}`，并自动挂载输出 Dataset。 -`{date}` 和 `{hour}` 使用 UTC；同一个 run/session 会固定到首次选择的分区，避免流式响应或 -长会话被拆成多个 event source。`auto` 等价于 `127.0.0.1:0`。 -已有 canonical source 默认在最后一条事件后等待 30 分钟才执行 Storyline projection；可用 -`--gateway-split-idle DURATION` 覆盖。 - -启用 Warehouse listener 后，Gateway 模式的单 trace 查询会重新打开最新 canonical event -manifest。向已有 source 追加事件不需要等待 Snapshot 刷新或 Storyline projection；只有新建 -source 文件和 projection 发布才需要更新全局 Snapshot。 - -Control 要求存在名为 `default` 的挂载。只提供 `--control`、`--gateway` 或 -`--gateway-config`、不提供 -`--listen` 时，只启动所请求的集成，不同时启动 Web UI。进程向 stdout 写一条机器可读的 -readiness 记录；Control 凭据不会写入 stderr。 - -挂载的 Dataset 和 HTTP 操作均为只读。API 不暴露 import、export、maintenance 或任意文件 -访问。刷新只会在新视图准备完成后替换当前可读视图；刷新失败时，旧视图继续可用。 - -Runs 列表允许短暂滞后：按 Dataset 和文件范围复用最近成功的摘要，30 秒后再次访问时先 -返回旧结果，再触发后台刷新。全进程最多一个自动 Runs 刷新任务；失败保留旧结果，并至少 -等待 30 秒再重试。摘要缓存最多保留 32 个范围、64 MiB；它只驻留内存，首次访问、进程重启 -或缓存淘汰后仍需发现 source。已有的 Datasets 目录浏览缓存仍使用本地 Lance 持久化。 -未限定 Dataset 的 Runs 请求同样复用已有 Catalog，过期时后台更新。点击 **Refresh** 会 -强制刷新 Catalog，成功后使旧的分范围摘要失效；刷新前的任务不能重新发布旧缓存。 -命令行 `query`、查询 worker 和 Gateway 实时读取不使用这层摘要缓存。 - -Web UI 启动阶段使用 `query/tables?ui=1` 的轻量 Catalog 视图：Dataset 名称和 source 统计优先来自本地 Browse/manifest cache，不触发准确 Catalog discovery。它返回 `ui-cache` 标识；真正的 Analysis 编译和 SQL 执行仍在服务端绑定当前准确 Snapshot。CLI 和未带 `ui=1` 的 API 不使用此视图。 - -## 日志与失败请求 - -`pchronicle serve` 把 Warehouse 请求日志写到 stderr，级别由 `--log-level` 控制（默认 -`info`），tracing target 为 `pchronicle.serve`。启动时记录 listen 地址、Dataset 名和 -Snapshot id。每个 `/api` 请求记录 method、path、status、耗时，以及截断后的 query -string。query 和 compile handler 还会记录截断后的 SQL。 - -失败响应包含 `code`、`message` 和 `request_id`。Web 横幅展示同一个 `request_id`。 -内部失败在 JSON 中脱敏；stderr 上对应 id 的 ERROR 行带有 `root_cause` 和 `chain`。 - -`--log-level error` 只保留内部失败。`--log-level` 不读取 `RUST_LOG`。 +监听地址只允许 loopback。命名挂载提供稳定的 SQL 别名。 +`--open` 打开 Web UI，`--home-link TEXT=PATH` 增加同源导航链接。 +`--catalog-config` 启用 Directory 和数据操作认证，不能与位置参数挂载同时使用。 +通过 `pchronicle serve catalog` 配置 Dataset 和用户授权，存储凭证按后端与 worker +隔离。启动就绪 JSON 包含 `warehouse_endpoint`。 -Gateway 行为见 [Gateway 转发、改写与捕获](serve-gateway.md)，精确参数见 -[`pchronicle` 命令行参考](../reference/cli.md)。内部刷新和版本固定机制属于 -[Snapshot 设计](../design/catalog.md)。 +刷新先构建新 Snapshot 再切换读者，失败时保留旧视图。数据写入使用 CLI import。 +日志写入 stderr，`--log-level` 控制详细程度。失败 API 响应包含 `code`、`message` +和 `request_id`，内部错误细节保留在服务日志。 -要了解 Datasets、Runs、Analysis、Storage 和 Assistant 的实际操作，请继续阅读 -[本地 Web UI 使用指南](ui.md)。 +[CLI 参考](../reference/cli.md) · [Web UI](ui.md) diff --git a/docs/src/zh/pchronicle/reference/agenticmd.md b/docs/src/zh/pchronicle/reference/agenticmd.md index d027fc0b6..5f5907c18 100644 --- a/docs/src/zh/pchronicle/reference/agenticmd.md +++ b/docs/src/zh/pchronicle/reference/agenticmd.md @@ -60,8 +60,6 @@ pChronicle 定义并序列化 frontmatter，常用字段包括： ## 4. Live 更新 -启用 live Markdown 时，Gateway 在写入 canonical Lance 的同时将可见对话投影为 AgenticMD： - 1. user 块按 `call_id` 写入； 2. 流式 assistant 使用相同 `call_id` 原地更新； 3. 重写一个 assistant 块时必须保留其后的 user 块； @@ -74,7 +72,7 @@ pChronicle 定义并序列化 frontmatter，常用字段包括： Lance events 负责保真、replay、stats 和结构化查询。内部 trajectory operation 可以从 Lance 重建 AgenticMD，但当前公共 `pchronicle` CLI 不提供 AgenticMD materialize 或 import -子命令。AgenticMD 不会自动 compact 或恢复 canonical event；公共交换使用 +子命令。公共交换使用 [`pchronicle import/export`](cli.md) 支持的格式。 ## 6. 示例与实现 diff --git a/docs/src/zh/pchronicle/reference/cases-platform.md b/docs/src/zh/pchronicle/reference/cases-platform.md index e029bb8f8..bda90bd9f 100644 --- a/docs/src/zh/pchronicle/reference/cases-platform.md +++ b/docs/src/zh/pchronicle/reference/cases-platform.md @@ -1,7 +1,7 @@ # pChronicle Directory 与平台场景 面向平台部署的 Directory 配置。ACL 文件管理用户、datasets（libraries）和授权； -Warehouse 的 listen / Gateway 参数仍由 `pchronicle serve` 提供。 +Warehouse 的 listen 参数仍由 `pchronicle serve` 提供。 ## P01：从空配置签发 Directory 用户 diff --git a/docs/src/zh/pchronicle/reference/cli.md b/docs/src/zh/pchronicle/reference/cli.md index 050627f31..fdcc1a159 100644 --- a/docs/src/zh/pchronicle/reference/cli.md +++ b/docs/src/zh/pchronicle/reference/cli.md @@ -319,8 +319,7 @@ decode 与可跳过的 commit 失败会写入 WAL 与 `import.log`，进程继 | `claude-code` | 是 | 否 | | `compact-jsonl` | 是 | 是 | -Codex 和 Claude Code session 是 decode-only 输入格式。Canonical Event Store 会自动识别并投影为 -Storyline Dataset。默认创建要求目标不存在。`--append` 要求目标是已有 Storyline Dataset； +Codex 和 Claude Code session 是 decode-only 输入格式。默认创建要求目标不存在。`--append` 要求目标是已有 Storyline Dataset； 重复 `document_id` 默认增加 `#N` 后缀，也可用 `--on-duplicate skip` 跳过。`--replace` 会先将完整导入 写入临时路径，再将旧本地 Dataset rename 到备份路径、将新 Dataset rename 到正式路径，确认新路径 发布后才删除备份；因此必须交互确认或传入 `--yes`。对象存储 Dataset 的 replace 会先清空目标前缀再写入（非原子）。 @@ -409,66 +408,21 @@ Agent 注入是行为引导，不是 filesystem、network 或 tool permission ### 2.12 `serve` -```text -pchronicle serve - [--listen LOOPBACK_ADDR] [--control LOOPBACK_ADDR] [--open] - [--home-link TEXT=PATH]... - [--gateway ADDRESS --gateway-dataset DATASET [--gateway-split TEMPLATE] - [--gateway-split-idle DURATION]] - [--gateway-config FILE --gateway-dataset DATASET [--gateway-state DIRECTORY]] - [--gateway-stream-markdown] [--gateway-debug] - [--catalog-config FILE] - [<[NAME=]DATASET> ...] -pchronicle serve catalog dataset add --catalog-config FILE NAME --uri URI [OPTIONS] -pchronicle serve catalog dataset remove --catalog-config FILE NAME... -pchronicle serve catalog dataset list --catalog-config FILE -pchronicle serve catalog issue --catalog-config FILE NAME -pchronicle serve catalog grant --catalog-config FILE NAME DATASET... -pchronicle serve catalog revoke --catalog-config FILE NAME DATASET... -``` - ```bash pchronicle serve ./trajectory-data -pchronicle serve \ - --gateway auto \ - --gateway-dataset ./trajectory-data \ - --gateway-split '{user}/{date}/{hour}' -``` - -未指定服务 flag 时，只读 Web/API 默认监听 `127.0.0.1:0`。多个 Dataset 使用 -`NAME=DATASET` mount；Control 模式要求名为 `default` 的 mount。可重复的 -`--home-link TEXT=PATH` 会在首页 Warehouse 旁增加胶囊；`PATH` 必须是同源相对路径。 -`--catalog-config FILE` -启用逐请求 AK/SK 认证及 `catalog://` locator。父进程只监听、认证和调度，已授权数据集由 -独立 exec worker 读取；不能与位置参数 Dataset、Gateway 或 Control 同时使用。配合 `dataset pin NAME catalog://127.0.0.1:PORT --ak --sk`。 -`pchronicle serve catalog dataset add|remove|list` 与 `issue|grant|revoke` 只改该文件、 -不启动 HTTP；`issue` 把用户 sk 只打印一次。改 library、用户或授权后必须重启 serve。 -worker 池按并发压力创建进程：优先复用同一权限范围的空闲 worker，全部忙碌时按需扩容， -每个范围最多 4 个、整个服务最多 8 个进程。达到上限后等待任意可用容量， -最多接纳 32 个正在处理或排队的请求。每个 worker 内部仍串行处理，IPC 不交叉。 -空闲超过 120 秒的进程由每 30 秒执行一次的清理任务回收；全局容量不足时可提前回收 -其他范围的空闲进程。排队、启动和执行合计上限为 60 秒，请求体读取上限为 10 秒。 -过载返回 503；执行期间超时或 IPC 失败会淘汰对应进程，排队取消不会中断其他请求。 -同一用户、授权范围和后端凭证版本复用 worker 及独立磁盘缓存；修改授权后重启生效。 -缓存位于 `PCHRONICLE_CACHE_DIR/workers/`，未设置时使用系统 pchronicle 缓存目录。 -子进程不继承父进程的 AWS 环境、profile 或用户主目录配置；登录 AK/SK 用于认证， -数据集的 AK/SK 经私有 IPC 传入并用于对象存储访问。不同 S3 endpoint/凭证的数据集必须 -分别通过 `dataset` 参数选择，暂不支持这类跨凭证查询。健康检查、UI 配置和静态页面公开， -数据 API 需要认证。没有用户的 catalog 仍可管理，但数据 API 不接受匿名访问。 -这是进程与凭证上下文隔离，不是 OS 用户降权或文件系统沙箱；本地路径仍使用服务进程的 OS 身份。 - -`catalog` 是 `serve` 的保留子命令，挂载同名路径请用 `./catalog`。见 -[RFC-0013](../../rfcs/0013-pchronicle-warehouse-catalog.md) 与 -[RFC-0015](../../rfcs/0015-chronicle-manifest.md)。无需配置的 `--gateway` -在 `POST /v1/events` 接收 canonical trajectory events；`--gateway-dataset` 是自动挂载的 -输出 URI，不再是 mount name。`--gateway-split` 支持 `{user}`、`{date}`、`{hour}`。 -已有 canonical source 默认在最后一条事件后空闲 30 分钟才自动刷新 Storyline projection； -可用 `--gateway-split-idle DURATION` 覆盖。 -Gateway 模式启用 Warehouse 后，单 trace 的事件、Storyline 和 trajectory 接口会读取已经发现 -source 的最新 canonical manifest，正在进行中的 trace 不需要等待 projection 或全局 Catalog 刷新。 -旧式转发 Gateway 仍可使用 `--gateway-config`，对象存储 capture 必须提供本地 -`--gateway-state`。所有 listener 只允许 -loopback；服务准备完成后，stdout 输出一行版本化 readiness JSON，endpoint 和诊断写 stderr。 +pchronicle serve --listen 127.0.0.1:8080 train=./train eval=./eval +pchronicle serve --catalog-config catalog.toml --listen 127.0.0.1:8081 +``` + +监听地址只允许 loopback。命名挂载提供稳定的 SQL 别名。 +`--open` 打开 Web UI，`--home-link TEXT=PATH` 增加同源导航链接。 +`--catalog-config` 启用 Directory 和数据操作认证，不能与位置参数挂载同时使用。 +通过 `pchronicle serve catalog` 配置 Dataset 和用户授权，存储凭证按后端与 worker +隔离。启动就绪 JSON 包含 `warehouse_endpoint`。 + +刷新先构建新 Snapshot 再切换读者，失败时保留旧视图。数据写入使用 CLI import。 +日志写入 stderr，`--log-level` 控制详细程度。失败 API 响应包含 `code`、`message` +和 `request_id`，内部错误细节保留在服务日志。 #### Catalog 管理 diff --git a/docs/src/zh/pchronicle/reference/formats/index.md b/docs/src/zh/pchronicle/reference/formats/index.md index a0d2b6e81..c85bde7a7 100644 --- a/docs/src/zh/pchronicle/reference/formats/index.md +++ b/docs/src/zh/pchronicle/reference/formats/index.md @@ -2,7 +2,6 @@ | 格式 | 角色 | 规范 | | --- | --- | --- | -| Events | HTTP-first 记录事件格式 | [RFC-0002](../../../rfcs/0002-events-format.md) | | Storyline | 内部规范化 Run 与 tool-call 模型 | [RFC-0001 § Wire schema](../../../rfcs/0001-storyline-format.md#wire-schema) | | ACTF | JSON Run 交换格式 | [RFC-0004 § JSON Pointer 映射](../../../rfcs/0004-actf-format.md#actf-storyline-json-pointer-mapping) | | ATIF | Agent Run 交换格式 | [RFC-0008 § JSON Pointer 映射](../../../rfcs/0008-atif-format.md#atif-storyline-json-pointer-mapping) | diff --git a/docs/src/zh/pchronicle/reference/query-model.md b/docs/src/zh/pchronicle/reference/query-model.md index c80fd475c..1fee0dee8 100644 --- a/docs/src/zh/pchronicle/reference/query-model.md +++ b/docs/src/zh/pchronicle/reference/query-model.md @@ -9,7 +9,6 @@ | `runs` | 一个规范化 Run/session | 每个就绪的 Run 数据源 | | `steps` | 一个规范化 Step | 每个就绪的 Run 数据源 | | `tool_calls` | 一次工具调用及关联结果 | 每个就绪的 Run 数据源 | -| `events` | 一条 canonical 写入时事实 | 仅 canonical event Source | | `trajectories` | 一个带有序 Step 与工具汇总的完整 Run | 规范化 Run 数据源 | 使用 `DESCRIBE` 查询已安装版本的精确 column： @@ -19,7 +18,6 @@ DESCRIBE dataset.sources; DESCRIBE dataset.runs; DESCRIBE dataset.steps; DESCRIBE dataset.tool_calls; -DESCRIBE dataset.events; DESCRIBE dataset.trajectories; ``` @@ -50,9 +48,6 @@ JOIN dataset.steps s | `format` | UTF-8, nullable | 检测或声明的表示 | | `kind` | UTF-8, non-null | `store` 或 `file` | | `snapshot_ref` | UTF-8, nullable | generation、manifest revision、fingerprint、version 或 ETag | -| `projection_status` | UTF-8, nullable | canonical events Source 关联投影的 `fresh` 或 `stale` 状态 | -| `projection_generation` | UTF-8, nullable | 被选为读取加速投影的 generation | -| `projection_candidates` | UInt64, non-null | 参与选择的关联投影候选数 | | `size_bytes` | UInt64, nullable | 候选文件或 marker object 大小 | | `last_modified` | UTF-8, nullable | 可用时的 RFC 3339 timestamp | | `status` | UTF-8, non-null | `ready` 或 `error` | diff --git a/docs/src/zh/pchronicle/reference/terminology.md b/docs/src/zh/pchronicle/reference/terminology.md index 42b95d90a..f84e8d54d 100644 --- a/docs/src/zh/pchronicle/reference/terminology.md +++ b/docs/src/zh/pchronicle/reference/terminology.md @@ -8,21 +8,15 @@ pChronicle 在命令行、Web 界面和任务型文档中统一使用以下词 | **Source file（源文件）** | Dataset 中的一个输入文件或存储来源。 | | **Run（运行）** | 一条 Agent 执行记录。Session ID 标识这条记录；可选的 Run ID 用于关联运行时活动。 | | **Step（步骤）** | Run 中按顺序排列的一条用户、Agent、系统或工具相关记录。 | -| **Event（事件）** | 与 Step 关联的底层记录事件或重建事件。 | | **Result（结果）** | 查询或分析返回的数据行。 | | **Snapshot** | 打开 path 之后写入与读取之间的同步协议：有哪些 Source、一次操作各钉在哪个版本。 | -界面使用 **Recorded events（记录事件）** 表示运行时直接捕获的事件，使用 -**Reconstructed events（重建事件）** 表示从导入的运行数据推导出的事件。重建事件不会被标成 -原始数据。 - 以下词汇只用于技术文档和 API： - **Storyline** 是 Run 与 Step 背后的规范化存储和交换模型； -- **Canonical Event** 是只追加的记录事件模型； - **Directory** 是可选的平台 locator：把名字解析成 path 并换票。CLI 标志和 HTTP 路径仍可能写 `catalog`； - **Snapshot**（API 里有时仍叫 `DatasetCatalogSnapshot`）钉住 Source 成员与版本，不是 Directory 列表； -- **projection、revision、fragment、column page** 描述存储和一致性机制，不是日常操作入口。 +- **revision、fragment、column page** 描述存储和一致性机制，不是日常操作入口。 dataset pin（`@name`）、Warehouse mount 名和 Directory library 名都是 locator。解析完成后引擎只看见 path。 diff --git a/docs/src/zh/project/engineering.md b/docs/src/zh/project/engineering.md index 0ff476950..88ff5a49f 100644 --- a/docs/src/zh/project/engineering.md +++ b/docs/src/zh/project/engineering.md @@ -9,7 +9,7 @@ | 命令 | 作用 | |---|---| -| `just test` | 通过 `cargo nextest` 跑工作区 Rust 测试，再跑 Python 套件 | +| `just test` | 通过 `cargo nextest` 跑pChronicle 与 CLI 测试，再跑范围内的 Python 测试 | | `just test ` | 单个 crate 或 Cargo package（例如 `pchronicle` 或 `persisting-pchronicle`） | | `just docs-sync` | 安装锁定的文档环境 | | `just docs-serve` | 本地 Zensical 预览，文件修改时自动刷新 | @@ -18,14 +18,14 @@ | `just examples` | pChronicle 产品示例套件 | | `just dev` | 自动格式化、lint，然后运行 Rust 测试 | | `just ci` | 只读 lint 检查、Rust/Python 测试、性质测试，然后构建 | -| `just check-quick` | 核心 runtime crate 与无默认 feature 的 pChronicle 检查 | +| `just check-quick` | pChronicle CLI 与无默认 feature 的 pChronicle 检查 | `just test` 使用 debug nextest profile 以便更快迭代。传入 Cargo package 名 -或短 crate 别名（`pchronicle`、`pchronicle-cli`、 -`agentctl`、`capture`）。`just test pchronicle` 会同时跑 +或短 crate 别名（`pchronicle`、`pchronicle-cli`）。`just test pchronicle` 会同时跑 `persisting-pchronicle` 与 `persisting-pchronicle-cli`（与 CI 的 pchronicle shard 一致）；只要 CLI 时用 `just test pchronicle-cli`。无参数形式还会跑 -`just test-py`。 +`just test-py`，覆盖打包、轨迹行契约与基准报告。Python 格式化和 lint 检查这些 +测试及 pChronicle 构建、文档和用例脚本，遵循 `AGENTS.md` 的范围。 ## 当前笔记 @@ -47,7 +47,7 @@ Rust 测试用 `cargo nextest` 做进程隔离和并行执行；用 镜像（glibc 2.28），以便 rustc libstd 能链接 `statx` / `copy_file_range`。 -`just check-quick` 检查核心 runtime crate 以及无默认 feature 的 pChronicle。 +`just check-quick` 检查 pChronicle CLI 以及无默认 feature 的 pChronicle。 `just dev` 先格式化文件，再运行 lint 和 Rust 测试；`just ci` 使用只读 lint 检查， 并额外运行 Python 和性质测试。GitHub Actions 还覆盖平台分片、Web 构建、S3 与示例。 diff --git a/docs/src/zh/rfcs/0001-storyline-format.md b/docs/src/zh/rfcs/0001-storyline-format.md index 3db28742f..dbacc029a 100644 --- a/docs/src/zh/rfcs/0001-storyline-format.md +++ b/docs/src/zh/rfcs/0001-storyline-format.md @@ -7,7 +7,6 @@ | **Date** | 2026-07-30 | | **Component** | pChronicle (`persisting-pchronicle`) | | **Implements** | `crates/persisting-pchronicle/src/formats/storyline.rs` | -| **Related** | [RFC-0002 Events](0002-events-format.md) · [RFC-0004 ACTF](0004-actf-format.md) · [RFC-0008 ATIF](0008-atif-format.md) · [RFC-0009 OpenAI Messages](0009-openai-messages-format.md) | --- @@ -15,16 +14,8 @@ **Storyline** 是 pChronicle 的**枢纽 interchange**：以 Harbor **ATIF-v1.7** 的 Trajectory / Step 折叠语义为基准，加上少量 hub 便利字段（短名 wire、性能顶栏、可选子会话外链）。 -捕获侧：`HTTP → events 流`（可 **记录** 到 `events.lance`，也可 **触发** handler）；handler 经 storyline 做格式转换与落盘。外围格式（`agenticmd` / `openai_msg` / `atif` / `actf`）**只与 storyline 互转**。 - -Storyline **不是**事实源。Canonical 事实仍是 `events.lance`。 - -```text -events ──┐ -agenticmd ┼──► storyline ──► … -openai_msg┤ -atif ─────┘ -``` +外围格式（`agenticmd` / `openai_msg` / `atif` / `actf`）只与 Storyline 互转。 +Storyline 是存储与查询管线使用的轨迹模型，可直接写入三表 Lance。 --- @@ -32,8 +23,7 @@ atif ─────┘ 1. **ATIF-first**：根对象 ≈ Trajectory，`turns[]` ≈ `steps[]`；能 1:1 的字段保持同义。 2. **Hub-only**：A→B MUST 经 storyline。 -3. **少叙事**：不引入 Capture Call / 相位 / event 回链等运行时读模型；那些留在 events。 -4. **Not SoT**：回放与审计以 events 为准。 +3. **来源可追溯**：保留 Source 与 Snapshot 身份，不推断输入中缺失的执行记录。 相对 ATIF 仅保留这些增量： @@ -243,7 +233,6 @@ from_storyline(format, story) → serialized convert(from, to, input) ≡ from_storyline(to, into_storyline(from, input)) ``` -`events` 为 Lance-only：字符串 convert API 对 `ChronicleFormat::Events` 报错；内存用 `events_to_storyline` / `storyline_to_events`。 --- diff --git a/docs/src/zh/rfcs/0002-events-format.md b/docs/src/zh/rfcs/0002-events-format.md deleted file mode 100644 index b7c75a683..000000000 --- a/docs/src/zh/rfcs/0002-events-format.md +++ /dev/null @@ -1,585 +0,0 @@ -# RFC-0002: Events Format（`events`） - -| Field | Value | -|---|---| -| **Status** | Accepted | -| **Schema / format name** | `events`（逻辑文档）；物理存储常为 `events.lance` | -| **Date** | 2026-07-30 | -| **Component** | `persisting-events` + Gateway + pChronicle | -| **Implements** | `persisting-events::EventRecord` · `persisting-pchronicle` `formats/events.rs` / `EventRow` | -| **Related** | [RFC-0001 Storyline](0001-storyline-format.md) · [RFC-0007 Events/Sidecar 边界](0007-events-contract-pchronicle-sidecar.md) · [轨迹存储](../pchronicle/design/trajectory-storage.md) | - ---- - -## 摘要 - -> **Ownership amendment:** [RFC-0007](0007-events-contract-pchronicle-sidecar.md) -> 将存储无关的 `EventRecord` 契约迁至 `persisting-events`。本 RFC 继续规定 events 的 -> wire 内容与物理表示；pChronicle 继续独占 `EventRow` 和持久化实现。 - -**`events`** 是 Persisting 轨迹的 **canonical 事实流（SoT）**：按时间（`seq`）追加的底层记录，**优先刻画原始 HTTP（或等价传输层）交换**，而不是已经折叠好的对话轮次。 - -设计取向： - -> 先保住「线上真实发生过什么」（方法、URL、头、正文、状态、时序），再由上层（Storyline / Markdown / ATIF）去解释「这对人意味着什么」。 - -因此： - -| 层 | 角色 | 可否有损 | -|---|---|---| -| **`events`** | 原始交换日志（HTTP-first） | **SoT，目标可回放** | -| **`storyline`** | Normal / 互操作枢纽 | 从 events 投影时有损；自身存储必须无损 | -| `agenticmd` / `openai_msg` / `atif` / `actf` | 外围视图 / 训练交换 | 同源 Storyline 往返必须无损；跨格式仅承诺目标可表达语义 | - -`events` 进出其它 chronicle 格式时仍 **只经 storyline**；但 **回放、重放、审计、协议级恢复 MUST 读 events**，MUST NOT 依赖 storyline roundtrip。 - -本 RFC 规定： - -1. HTTP-first 的定位与分层职责； -2. 逻辑事件 `EventRecord`、信封 `EventsDocument`、Lance `EventRow`； -3. `payload` 的 wire 字段约定（request / response）； -4. 与 Storyline 的投影关系（摘要字段只是加速，不是真相）。 - ---- - -## 动机 - -Agent 轨迹若只存「user/assistant 文本 + tool_calls」，会丢掉： - -- 真实 URL / method / status / header（鉴权、路由、provider 切换）； -- 原始 body（含协议特有字段、tool schema、多模态 parts）； -- 流式分片、重试、取消、非 2xx、内部探测流量； -- 足以 **原样重放 HTTP** 或 **换协议解析器重解** 的材料。 - -Persisting Gateway 的主入口是代理流量。`events` 应对齐这一现实：**一行事件 ≈ 一次可定位的 HTTP 方向记录**（request 或 response），外加关联键（`session_id` / `call_id` / `trace_id`）。 - -对话摘要（`user_content` / `assistant_content`）可以写进 payload 作为 **可选索引/加速字段**，但： - -- MUST NOT 取代 `http.request_body` / `http.response_body`（或等价 wire 字段）成为唯一正文； -- 当摘要与 wire body 冲突时，**以 wire body 为准**。 - ---- - -## 设计目标 - -| 目标 | 说明 | -|---|---| -| **HTTP-first SoT** | 默认按原始请求/响应保存，使回放与再解析成为一等能力 | -| **Replayable** | 在凭证策略允许的前提下，应能从 events 重建「对同一 endpoint 再发一次等价请求」所需信息 | -| **Re-derivable views** | Storyline / Markdown / ATIF 视为可从 events **重新投影**的视图 | -| **Correlation envelope** | 顶栏保留 `session_id` / `call_id` / `trace_id` 等关联键，不把故事语义塞进 wire | -| **Append-only** | `seq` 单调；不原地改写既有 wire payload | -| **Hub via storyline** | 与其它外围格式互转 MUST 经 storyline；无损路径仍是读 events | - -非目标： - -- 不把 events 做成 ATIF step / Storyline Turn 的镜像； -- 不要求每条事件都是「对人友好的一句对话」； -- 不在本 RFC 规定 TLS 密钥、cookie 明文等敏感材料的强制落盘策略（见安全节）。 - ---- - -## 分层：原始交换 vs 故事投影 - -```text - ┌─────────────────────────────┐ - proxy / import → │ events 流（HTTP wire） │ - └──────────────┬──────────────┘ - │ - ┌──────────────┼──────────────┐ - ▼ ▼ - 记录 append 触发 handlers - │ │ - ▼ ▼ - events.lance storyline（hub） - （SoT / replay） │ - ┌─────────────┼─────────────┐ - ▼ ▼ ▼ - agenticmd openai_msg atif - └──────── 落盘 / 物化 ──────┘ -``` - -| 问题 | 应落在 | -|---|---| -| 「当时请求的 path / body 是什么？」 | **events** | -| 「这一轮对用户说了什么？」 | storyline / agenticmd | -| 「能否拿去 SFT？」 | atif / openai_msg（经 storyline） | -| 「能否按 call_id 重放 upstream？」 | **events** | - -与既有 Capture 表述的关系：协议差异可以在 **进入 Story 读模型之前**消化；但 **进入 events 之前不应过度消化**——解析器可以后置、可升级，原始字节/JSON body 应尽量留下。 - ---- - -## 术语 - -| 术语 | 含义 | -|---|---| -| **Wire record** | 一次 HTTP（或同类）方向的原始交换描述：method/url/headers/body/status/timing… | -| **EventRecord** | 逻辑事件 = 关联信封 + `kind` + wire-oriented `payload` | -| **EventRow** | Lance 列存行：索引列 + `payload_json` | -| **Summary fields** | `user_content` / `assistant_content` / `model` 等可选派生字段 | -| **Replay** | 用 wire 字段重建并再次发出（或模拟）等价交换 | - ---- - -## Schema：逻辑事件 `EventRecord` - -编码：UTF-8 JSON object。`EventRecord` 由存储无关的 `persisting-events` 唯一定义； -Gateway 与 pChronicle 直接使用同一类型。pChronicle 独占由该逻辑记录派生的 -物理 row schema 与存储实现。 -一行事件 MUST 包含 `seq`、`source`、`kind`、`payload`。 - -### 关联信封（顶栏） - -| Field | Type | Status | Description | -|---|---|---|---| -| `event_id` | string | Required for new writes | append 边界生成的稳定事件身份 | -| `run_id` | string | Required for new writes | 事件所属逻辑 Run；旧 capture row 读取时可缺省 | -| `attempt_id` | string | Optional | 生命周期事件所属 Attempt | -| `storyline_id` / `turn_id` | string | Optional | narrative 维度，与 Attempt 正交 | -| `timestamp_unix_ms` | integer | Required for new writes | 机器可比较的观测时间；与 `timestamp` 表示同一时刻 | -| `producer` | string | Required for new writes | 产生该记录的组件 | -| `seq` | integer | Required | dataset / 会话内单调序号 | -| `source` | string | Required | 如 `persisting-proxy`、`persisting-gateway` | -| `kind` | string | Required | 见 kind 节；HTTP 方向优先 | -| `timestamp` | string | Required for new writes | RFC3339 UTC（事件观测时间；与 `timestamp_unix_ms` 毫秒级一致） | -| `session_id` | string | Optional | ≈ Storyline `story_id` | -| `agent_id` | string | Optional | | -| `call_id` | string | Optional | 一次调用关联键（request/response 配对） | -| `trace_id` | string | Optional | | -| `parent_call_id` | string | Optional | 调用树 | -| `parent_uuid` | string | Optional | | -| `subagent_id` / `parent_agent_id` / `branch` | string | Optional | 子代理 / 分支 | - -顶栏 **SHOULD NOT** 承载完整对话文本；正文在 `payload` 的 wire 字段中。 - -新写入事件必须同时提供 `timestamp` 与 `timestamp_unix_ms`。Gateway capture sink 和 -事件 producer 负责提供时间字段；admission 仅为旧记录或兼容导入补齐缺失值。 -事件顺序仍由 `source + seq` 定义，时间戳用于关联和展示。 - -### `payload`：HTTP-first wire 对象 - -推荐统一放在 `payload.http`（或顶层扁平兼容键，见兼容节）。**完整回放**所需字段： - -#### Request 方向（`kind` = `http.request` 或兼容 `llm.request`） - -| Field | Type | Status | Description | -|---|---|---|---| -| `http.method` | string | Required | `GET` / `POST` / … | -| `http.url` | string | Required | 完整 URL 或 path+query；与 `host` 可拆分 | -| `http.path` | string | Recommended | 规范化 path（便于索引） | -| `http.query` | object / string | Optional | | -| `http.headers` | object \| array | **Required**（记录时） | **完整请求头**（可脱敏）；与 body 同为回放必需 | -| `http.request_body` | string \| object \| null | Recommended | **原始请求体**（JSON 对象或 base64/utf8 字符串） | -| `http.body_encoding` | string | Optional | `json` \| `utf8` \| `base64` \| `sse-wire` … | -| `http.content_type` | string | Optional | | -| `timing.started_at` | string | Optional | | -| `timing.duration_ms` | number | Optional | | - -#### Response 方向（`kind` = `http.response` / `http.response.stream` 或兼容 `llm.response*`） - -| Field | Type | Status | Description | -|---|---|---|---| -| `http.status` | integer | Required | HTTP 状态码 | -| `http.headers` | object \| array | **Required**（记录时） | **完整响应头**（可脱敏） | -| `http.response_body` | string \| object \| null | Recommended | **原始响应体**或流式拼接线 | -| `http.body_encoding` | string | Optional | 含 `sse` / `sse-wire` | -| `http.truncated` | boolean | Optional | 是否流式 | -| `timing.ttft_ms` / `timing.duration_ms` | number | Optional | | - - -#### 连接与客户端（长连接 / 对端身份） - -HTTP-first 回放不仅需要单次 request/response，还需要知道 **连接是否复用** 以及 **谁连上来**。记录时 MUST 尽量写入： - -| Field | Type | Status | Description | -|---|---|---|---| -| `connection.http_version` | string | Recommended | 如 `HTTP/1.1`、`HTTP/2.0` | -| `connection.persistent` | boolean | Recommended | 是否按协议/头判定为 keep-alive / 长连接 | -| `connection.connection_header` | string | Optional | 原始 `Connection` 头（如 `keep-alive` / `close`） | -| `connection.keep_alive` | string | Optional | 原始 `Keep-Alive` 头（超时/max 等） | -| `connection.upgrade` | string | Optional | `Upgrade`（如 `websocket`） | -| `client.peer` | string | **Required**（代理路径） | 客户端 `ip:port`（`ConnectInfo`） | -| `client.peer_ip` | string | Recommended | | -| `client.peer_port` | integer | Recommended | | -| `client.pid` | integer | Optional | 本机 peer 反查进程 PID（best-effort） | -| `client.command` | string | Optional | 进程命令行 | -| `client.machine_fp` | string | Optional | 机器指纹 | -| `client.user_agent` | string | Optional | 从请求头抽取，便于无 PID 时对照 | - -判定 `persistent` 的建议规则： - -1. 若 `Connection: close` → `false`； -2. 若 `Connection` 含 `keep-alive` → `true`； -3. 否则：`HTTP/1.1+` 默认 `true`，`HTTP/1.0` 默认 `false`； -4. `Upgrade: websocket` 等升级连接单独用 `upgrade` 标明，不假装成普通 keep-alive 复用。 - -说明： - -- 会话级 `session-meta.yaml`（`SessionClientMeta`）可继续存在；**事件级**仍 MUST 冗余写入 `client.*`，以便单条 event 可自描述回放。 -- hop-by-hop 头（`Connection` / `Keep-Alive`）转发上游前可能被剥离，但 **events 记录 MUST 保留客户端原始值**。 -- 上游侧长连接信息可从 response `headers` 的 `Connection`/`Keep-Alive` 再读；可另写 `connection.upstream_*`（Optional）。 - -#### 可选摘要 / 索引（非 SoT） - -| Field | Type | Status | Description | -|---|---|---|---| -| `model` | string | Optional | 从 body/header 抽取，便于 Lance 列 | -| `protocol` | string | Optional | 如 `chat.completions`、`anthropic.messages` | -| `provider` | string | Optional | | -| `user_content` | string | Optional | 可见 user 摘要；冲突时以 request_body 为准 | -| `assistant_content` | string | Optional | 可见 assistant 摘要；冲突时以 response_body 为准 | -| `usage` | object | Optional | token 等；能从 body 再解析则视为缓存 | -| `_tlv` | object | Optional | 自 Markdown compact 注入 | - -**Normative**： - -1. 记录 HTTP 交换时，实现 MUST **同时**持久化： - - **headers**（`payload.http.headers` 或兼容扁平键 `payload.headers`）； - - **body**（在 `capture_level` 允许完整 body 时：`request_body` / `response_body` 或等价 `body`）； - - **连接与客户端**：`connection.*`（至少能推导 `persistent`）与 `client.peer`（代理路径必填）。 -2. headers 与 body 同为回放材料：缺 headers 的事件与缺 body 一样，MUST 视为 **degraded**（`payload.degraded = true`），MUST NOT 宣称可完整 HTTP 回放。 -3. 敏感头（`Authorization` / `Cookie` / `Set-Cookie` / `X-Api-Key` 等）MUST 按红线脱敏或外置，并设置 `headers_redacted: true`；脱敏后仍 MUST 保留头名字面量（值为占位符），以便知道当时发过哪些头。 -4. Storyline / 训练格式导出 MUST 容忍从 wire body + headers 再解析；MUST NOT 假定摘要字段永远存在。 - -### 内存批 `EventsDocument`（非正式文件格式） - -| Field | Type | Status | Description | -|---|---|---|---| -| `format` | string | Required | `"events"` | -| `events` | array | Required | `EventRecord[]` | -| `session_id` / `agent_id` | string | Optional | 文档级默认 | - -### 物理行 `EventRow`（Lance） - -| Column | 说明 | -|---|---| -| `seq` / `timestamp` / `kind` / `source` | 索引 | -| `session_id` / `agent_id` / `call_id` / `trace_id` / `parent_call_id` | 关联过滤 | -| `model` | 可选反规范化 | -| `payload_json` | **完整 EventRecord JSON（含 wire）** | - -身份冲突采用 admission-first 合同：写入坐标不一致不会拒绝事实。`payload_json` 保留 -producer 原始声明；物理 `session_id` / `agent_id` 用于路由、过滤并在 replay 时覆盖同名 -声明。Storyline 投影对剩余冲突按物理 append order 采用最后一个非空值。该规则不需要 -读前写、去重表或额外索引。 - ---- - -## `kind` 约定 - -### 推荐（HTTP-first） - -| kind | 含义 | -|---|---| -| `http.request` | 一次 outbound/inbound HTTP 请求观测 | -| `http.response` | 对应响应（非流或已聚合） | -| `http.response.stream` | 流式响应片段或草稿聚合 | -| `http.cancel` | 取消 / 中断 | -| `session.started` / `session.ended` | 会话生命周期（可无 http 块） | -| `note` | 运维注解 | - -### LLM producer kinds - -| 现有 kind | 视为 | -|---|---| -| `llm.request` | 带 LLM 协议理解的 HTTP 请求（payload 仍应尽量含 path/body） | -| `llm.response` | `http.response` | -| `llm.response.stream` | `http.response.stream` | - -通用 HTTP producer 使用 `http.*`，Gateway LLM producer 可以使用 `llm.*`。 -`llm.*` **不**意味着可以只存对话摘要，仍按 HTTP-first 要求保存 wire。 - ---- - -## 回放与恢复（Replay） - -从一对共享 `call_id` 的 request/response 事件，消费者 SHOULD 能恢复： - -1. **HTTP 重放**：method + url + headers′ + body → 得到新的 response（headers′ 经红线剥离）； -2. **协议再解析**：换用新的 Anthropic/OpenAI/… 解析器从 body 重建 messages / tool_calls； -3. **视图重建**：重新投影 Storyline / Markdown / ATIF； -4. **时序重建**：按 `seq` / timestamp 恢复并发与流式顺序。 - -无法回放的常见原因（实现 MUST 在文档/元数据中可诊断）： - -- body 因 `capture_level` 被省略（`degraded`）； -- 敏感头被剥离且无替代凭证注入通道； -- 仅存 SSE 摘要未存 wire 帧； -- 只从 storyline 合成的「假 events」（无原始 wire）。 - ---- - -## 安全与红线 - -HTTP-first **不**等于无条件落盘密钥。 - -| 类别 | 默认建议 | -|---|---| -| `Authorization` / API key / cookie | 红线：哈希、删除或外置 secret store；回放时再注入 | -| 用户内容 / 工具结果 | 按产品保留策略 | -| 多模态像素 | 可旁路对象存储；events 内留引用与 hash | - -`payload.http.headers_redacted: true`（可选）表示头已经过红线处理。 - ---- - -## Wire 形态与探测 - -| 形态 | 地位 | -|---|---| -| **Lance `events.lance/`** | **唯一一等存储 / 采集格式（SoT）** | -| JSONL / JSON / `EventsDocument` 序列化 | **非正式**：仅调试导出或测试夹具；**不是** chronicle 互操作格式 | - -pChronicle：`into_storyline` / `convert` **不接受** events 的字符串输入；从 Lance 加载为内存 `EventRecord` 后再 `events_to_storyline`。 - -探测： - -| 方式 | 规则 | -|---|---| -| 路径 | 仅 `events.lance`（或 basename 含 `event` 的 `.lance`） | -| 内容 | **不**把 EventRecord 形 JSON 判定为 `events` | - -别名：`events` / `lance` / `bin` / `events.lance`。 - - ---- - -## 与 Storyline 的关系 - -```text -events (HTTP wire) ──interpret──► storyline ──synthesize──► events′ (often degraded) -``` - -| 方向 | 期望 | -|---|---| -| events → storyline | 从 wire（及可选摘要）折叠 Turn/Call；填写 `event_seqs` | -| storyline → events′ | 合成最小交换；**通常 degraded**，除非 storyline 显式携带完整 wire（不默认） | - -规范性： - -1. 互转其它外围格式 MUST 经 storyline。 -2. 宣称「可回放」时 MUST 基于非 degraded 的原始 events。 -3. 折叠器 SHOULD 优先解析 `http.request_body` / `http.response_body`，摘要字段仅作快路径。 - ---- - -## 兼容现有 Capture payload - -今日实现常见扁平键（仍合法）： - -| 现有键 | HTTP-first 对应 | -|---|---| -| `headers` | `http.headers`（**记录 MUST**） | -| `path` | `http.path` / `http.url` | -| `body` | request 的 `http.request_body` 或 response 的 `http.response_body` | -| `status` | `http.status` | -| `model` / `protocol` / `provider` | 摘要/索引 | -| `user_content` / `assistant_content` | 摘要 | - -读路径 MUST 接受扁平键；写路径 SHOULD 逐步迁移到 `payload.http.*`，或同时写两套直到迁移完成。 - ---- - -## 示例 A：HTTP-first 请求事件 - -```json -{ - "seq": 0, - "source": "persisting-proxy", - "kind": "http.request", - "timestamp": "2026-01-01T00:00:00Z", - "session_id": "sess-1", - "agent_id": "agent-a", - "trace_id": "tr-1", - "call_id": "c1", - "payload": { - "http": { - "method": "POST", - "url": "https://api.openai.com/v1/chat/completions", - "path": "/v1/chat/completions", - "headers": { - "content-type": "application/json", - "authorization": "", - "connection": "keep-alive", - "keep-alive": "timeout=60, max=1000", - "user-agent": "claude-cli/1.0" - }, - "headers_redacted": true, - "content_type": "application/json", - "body_encoding": "json", - "request_body": { - "model": "gpt-4o", - "messages": [{ "role": "user", "content": "ping" }] - } - }, - "connection": { - "http_version": "HTTP/1.1", - "persistent": true, - "connection_header": "keep-alive", - "keep_alive": "timeout=60, max=1000" - }, - "client": { - "peer": "127.0.0.1:54321", - "peer_ip": "127.0.0.1", - "peer_port": 54321, - "pid": 4242, - "command": "claude --print", - "user_agent": "claude-cli/1.0" - }, - "model": "gpt-4o", - "protocol": "chat.completions", - "user_content": "ping" - } -} -``` - ---- - -## 示例 B：Storyline → Events 映射（合成多为 degraded） - -key = events 字段，value = 在 Storyline 上求值的 JSONPath。 -**注意**：从 storyline 合成的 body 通常只是对话投影，不是原始 HTTP wire；应设 `payload.degraded = true`，除非另有 wire 旁路。 - -```json -{ - "format": "events", - "session_id": "$.story_id", - "agent_id": "$.agent.id", - "events": [ - { - "seq": null, - "source": null, - "kind": null, - "timestamp": "$.turns[0].timestamp", - "session_id": "$.story_id", - "agent_id": "$.agent.id", - "call_id": "$.turns[0].calls[0].call_id", - "trace_id": "$.turns[0].calls[0].trace_id", - "parent_call_id": "$.turns[0].calls[0].parent_call_id", - "payload": { - "degraded": null, - "http": { - "method": null, - "url": null, - "path": null, - "headers": null, - "request_body": "$.turns[0].calls[0].messages", - "body_encoding": null - }, - "model": "$.turns[0].calls[0].model", - "protocol": "$.turns[0].calls[0].protocol", - "user_content": "$.turns[0].user.text" - } - }, - { - "seq": null, - "source": null, - "kind": null, - "timestamp": "$.turns[0].timestamp", - "session_id": "$.story_id", - "agent_id": "$.agent.id", - "call_id": "$.turns[0].calls[0].call_id", - "trace_id": "$.turns[0].calls[0].trace_id", - "parent_call_id": "$.turns[0].calls[0].parent_call_id", - "payload": { - "degraded": null, - "http": { - "status": null, - "headers": null, - "response_body": "$.turns[0].assistant.content", - "body_encoding": null, - "truncated": null - }, - "model": "$.turns[0].calls[0].model", - "assistant_content": "$.turns[0].assistant.text", - "usage": "$.turns[0].calls[0].metrics" - } - } - ] -} -``` - -合成：`kind` ← `http.request` / `http.response`；`degraded` ← `true`（默认）；`seq`/`source` 由写入器分配。 - -口语对照： - -| events 字段 | Storyline 路径 | -|---|---| -| `events.0.call_id` | `turns.0.calls.0.call_id` | -| `events.0.payload.http.request_body` | `turns.0.calls.0.messages` | -| `events.1.payload.http.response_body` | `turns.0.assistant.content` | -| `events.0.payload.http.method` / `url` / `headers` | （无）→ null / 合成默认 | - ---- - -## 示例 C：Events → Storyline（从 wire 再解析） - -按 `call_id` 分组后，优先： - -| Storyline 字段 | 来源 | -|---|---| -| `calls[0].event_seqs` | `$[*].seq` | -| `calls[0].call_id` | `$[0].call_id` | -| `calls[0].protocol` | `$[0].payload.protocol` 或从 path 推断 | -| `calls[0].messages` | 解析 `$[?(@.kind=~/request/)].payload.http.request_body`（兼容 `.body`） | -| `user.text` | 摘要 `user_content` **或** 从 request_body 提取 | -| `assistant.text` | 摘要 `assistant_content` **或** 从 response_body / SSE 提取 | -| `calls[0].model` | `payload.model` 或 body.model | - ---- - -## 校验规则（Normative） - -实现 MUST： - -1. 拒绝缺少 `seq` / `source` / `kind` / `payload` 的事件； -2. 保持 `seq` 单调追加； -3. 与其它外围格式互转走 storyline hub； -4. 记录 HTTP 事件时持久化 **headers + body**（或显式标记 `degraded`）；缺一不可宣称可回放。 - -实现 SHOULD： - -1. 为同一 HTTP 交换的 request/response 共享 `call_id`； -1b. 代理路径写入 `client.peer` 与 `connection.persistent`（及原始 Connection/Keep-Alive）； -2. 写 `http.*` kind，同时读兼容 `llm.*`； -3. 从 events 投影 storyline 时填充 `event_seqs`； -4. 对红线头做可检测的脱敏标记。 - ---- - -## 与现有文档的关系 - -| 文档 | 关系 | -|---|---| -| [轨迹存储](../pchronicle/design/trajectory-storage.md) | Lance SoT；本 RFC 强调 SoT 内容应是 HTTP wire | -| Gateway 管线 （外部仓库） | 生产 events；Story 边界在 **之后** 解释 | -| [RFC-0001 Storyline](0001-storyline-format.md) | 有损 Normal 视图 / hub | -| [轨迹 Markdown 格式](../pchronicle/reference/agenticmd.md) | 人读投影，不是 SoT | - -实现现状：`EventRecord` 已具备 `path`/`body`/`status` 等扁平键；本 RFC 将其 **提升为明确的 HTTP-first 契约**，并引入 `http.*` kind 与 `payload.http` 嵌套作为演进目标。 - ---- - -## 已裁定的演进约束 - -1. 新 writer 使用嵌套 `payload.http`；reader 在磁盘迁移期继续接受历史扁平键。 -2. SSE 原始帧写为按 `seq` 排序的 `http.stream.frame` 事件，携带相对时间与原始字节；聚合文本只是投影。 -3. `seq` 由 producer 在 session 内单调分配，不是唯一键；at-least-once 重试可产生重复行。 -4. WebSocket、gRPC 等非 HTTP 传输使用独立 `transport.*` kind，不伪装成 HTTP。 -5. 默认 `capture_level` 为 `dialogue`。缺少完整 wire 的记录必须标记 `degraded`；只有 `full` capture 可声明 wire-replayable。 -6. Gateway 为客户端和 upstream 连接分配稳定 `connection.id`，用于关联 keep-alive 上的多次交换。 - ---- - -## Changelog - -| Version | Date | Notes | -|---|---|---| -| Draft | 2026-07-30 | 初稿：EventRecord / EventRow / wire、与 Storyline hub | -| Draft | 2026-07-30 | **定位修正为 HTTP-first SoT**：原始请求/响应可回放；摘要字段降为可选；引入 `http.*` kind 与 `payload.http` | -| Draft | 2026-07-30 | **headers 与 body 同为记录 MUST**；缺 headers 亦为 degraded | -| Draft | 2026-07-30 | 增补 **长连接 `connection.*` + 客户端 `client.*`（peer/pid/command）** 记录要求 | -| Draft | 2026-07-30 | **events 仅 Lance**：JSON/JSONL 降为调试导出，非一等 wire | -| Accepted amendment | 2026-08-16 | RFC-0007 将 `EventRecord` ownership 迁至 `persisting-events`；pChronicle 保留物理 schema 与持久化 ownership | -| Accepted | 2026-08-09 | 裁定 payload、SSE、seq、非 HTTP、capture 默认值与 connection identity；保持 at-least-once append-only。 | diff --git a/docs/src/zh/rfcs/0003-pchronicle-ownership.md b/docs/src/zh/rfcs/0003-pchronicle-ownership.md index 47dacccc2..aaefe046c 100644 --- a/docs/src/zh/rfcs/0003-pchronicle-ownership.md +++ b/docs/src/zh/rfcs/0003-pchronicle-ownership.md @@ -1,114 +1,10 @@ -# RFC-0003: pChronicle 轨迹存储层所有权 +# RFC-0003：pChronicle 职责 -| Field | Value | -|---|---| -| **Status** | Accepted | -| **Date** | 2026-07-31 | -| **Component** | `persisting-pchronicle` | -| **Amended by** | [RFC-0007 Events/Sidecar 边界](0007-events-contract-pchronicle-sidecar.md) | -| **Related** | [RFC-0001 Storyline](0001-storyline-format.md) · [RFC-0002 Events](0002-events-format.md) · [轨迹存储](../pchronicle/design/trajectory-storage.md) | +pChronicle 拥有 Storyline 模型、外围格式编解码、物理存储、Source 发现、 +Snapshot、DataFusion 查询与 revision lineage。CLI 和 Web 使用这些契约， +不得引入第二套通用轨迹模型或并行存储格式。 -## 摘要 +CLI 拥有命令、导入导出编排、loopback Warehouse HTTP 服务与静态资源嵌入。 +Web 拥有浏览界面。 -`persisting-pchronicle` 是 Persisting 唯一的 **Agent 轨迹结构化存储与检索层**。轨迹的物理 schema、目录布局、物理落盘、读取、回放、格式转换和派生视图 MUST 收敛到 pChronicle。存储无关的逻辑事件信封由 `persisting-events` 定义，具体修订见 RFC-0007。 - -Gateway 和 CLI 可以分别生产事件和适配输入，但 MUST NOT 定义第二套通用轨迹存储实现。 - -## 决策 - -### 1. Canonical 记录 - -`persisting-events::EventRecord` 是 crate 边界和 Gateway 采集管线共同使用的 canonical 逻辑记录。 -provider/SSE payload 解释可作为 Gateway 扩展行为存在，但不得形成第二套可序列化轨迹类型。 - -`EventRow` 是 `events.lance` 的稳定物理行。其 Arrow schema、行转换、序号和 session 分区语义由 pChronicle 独占维护。 - -### 2. 物理层 - -| 层 | API / 实现 | 语义 | -|---|---|---| -| canonical event log | `RawEventLanceStore` | typed append、replay、stats；同一 run 按 `session_id` 分区 | -| 人读/调试视图 | `materialize_lance_to_markdown`, AgenticMD 文件 helpers | 从 canonical events 单向生成，可随时删除和重建 | -| 发现 | `expand_story_locations` | 发现 canonical Run/Story 分区；Markdown 不参与存储层选择 | -| 数据维护 | `RawEventLanceStore::maintain` | 显式离线 compaction、session 索引和 vacuum；事实层不支持 truncate/overwrite | -| Storyline 三表 | `StorylineLanceStore`, `StorylineDataSource` | 原子提交并查询 `runs` / `steps` / `tool_calls` | - -`events.lance` 是事实源。AgenticMD 和 Storyline 三表均可重建，不可被当作协议级审计或回放的事实源；ATIF 是互操作文档格式，不是独立存储模型。append、replay、stats 不得回退到 AgenticMD。 - -canonical event 写路径 MUST 是 at-least-once append-only：不得在 append 前扫描旧行或 -`event_id`，不得将 ID 唯一性、重试去重、truncate 或 overwrite 作为存储语义。一个微批 -只允许执行规范化、Arrow 编码、一次私有 Lance segment append 和一次 fencing manifest -CAS。索引、compaction 与 vacuum 必须由显式维护路径执行。 - -### 3. 格式层 - -pChronicle MUST 统一拥有除公共事件信封以外的轨迹格式与转换实现： - -- `EventRow` 以及 `storyline`、`agenticmd`、`openai_msg`、`atif`、`actf` 的 Rust 数据结构； -- `EventRecord` 到物理 row 与外围格式的映射； -- 格式识别、校验及转换； -- event 与物理行、Markdown block、Storyline turn 之间的映射； -- AgenticMD 的宽松解析、可视化生成、preamble 更新和调试索引； - -外围格式间的转换 MUST 经 Storyline hub。需要保真回放的路径 MUST 直接读取 events，不得依赖有损的 Storyline roundtrip。 - -### 4. 组件职责 - -| 组件 | MUST | MUST NOT | -|---|---|---| -| Gateway | 作为 OverlayNet sink 解释并转发 Agent/LLM 协议；维护采集顺序与调用生命周期；产出 `EventRecord`；实现与实时流状态有关的 live projection 策略 | 自有网络数据面或轨迹记录 schema；把 AgenticMD 当作可恢复事实源 | -| CLI / producer | 解析输入并产出公共事件；需要持久化时通过 pChronicle API 或 sidecar control 协议提交 | 自有 Markdown/ATIF parser、物理落盘协议或动态 ABI | -| pChronicle | 定义物理格式、存储、读取、转换和派生视图；消费公共 `EventRecord` | 依赖 Gateway 才能解释持久化数据；复制一套公共事件信封 | - -Gateway 的 live Markdown 行为可以保留 producer-specific 策略，例如流式 draft upsert；它必须同时写 canonical events。通用 batch materialize 归 pChronicle，不提供 AgenticMD → events 的隐式 compact。 - -### 5. 代码布局 - -- `persisting-pchronicle/src/operations/trajectory/` 是强类型轨迹操作适配。 -- `persisting-gateway/src/session/` 维护 session 身份、路由、client metadata、索引与 snapshot。 -- `persisting-gateway/src/projection/` 维护 Gateway 特有的可见文本解释、实时过滤、draft/upsert 和 reconcile。 -- `persisting-gateway/src/engine/` 维护采集 actor、WAL、顺序状态机和 egress；这里的 “engine” 是 Gateway 内部编排器，不是轨迹存储层。 -- `persisting-overlaynet` 是轻量显式代理网络层，负责 CONNECT、absolute-URI forward、header 规则和网络访问策略执行；Gateway 作为 `OverlaySink` 在其上解释并转发 LLM 流量、产出轨迹事件。OverlayNet 不依赖 Gateway，可配置其他 sink。 - -## 一致性与故障语义 - -1. canonical append 成功后，派生投影失败不得回滚或伪装成 canonical 写入失败；应报告或记录 projection failure。 -2. 状态机只能在 canonical append 成功后提交。 -3. WAL 重启后序号 MUST 单调延续；replay 成功后 MUST ack 原 WAL entry。 -4. `seq` 由 producer 定义；存储不得读取 row count 分配全局序号。writer epoch 的可见性 - 必须由 manifest CAS 串行化。 -5. Storyline 三表替换 MUST 对读者呈现单一提交点；`CURRENT` 指向的不可变 generation 是权威快照。ATIF 输入 MUST 先转换为 Storyline，不得维护第二套 normalized schema。 - -Run lease epoch MUST 通过 `EventWriterFence` 进入 canonical event 提交协议。新 epoch MUST -先以 compare-and-swap 激活 manifest；reader MUST 只读取 manifest 固定的 segment version， -不得直接打开某个 segment 的 latest version。失效 writer 的后续 Lance version 不得进入 -可见快照。相同 epoch、不同 writer_id 的激活 MUST 被拒绝。 - -## 收敛结果 - -- 原 Engine 中的 Trajectory 适配、Lance、Markdown 和 Arrow row 实现全部迁入 pChronicle；Engine crate 删除。 -- Gateway 与 pChronicle 直接使用 `persisting-events::EventRecord`；Gateway extension 仅承载实时 payload 解释。 -- Gateway 仅保留实时 payload 解释、live Markdown eligibility/upsert orchestration 与运行时 reconcile；格式解析、文件 I/O、frontmatter 契约与索引实现委托 pChronicle。 -- 只生产事件或调用 control 协议的组件依赖 `persisting-events`；需要存储、查询或格式转换的调用方依赖 pChronicle。 -- 旧 ATIF `sessions` / `steps` / `tool_calls`、`NormalizedStore`、内存联表视图及对应 Python 门面删除；ATIF 查询统一复用 Storyline 三表 schema。 -- append 边界直接传递 `EventRecord` 批次，不保留 RON/event-lines 字符串适配层。 - -新代码 SHOULD 按能力选择依赖：事件 producer 使用 `persisting-events`，存储和读取调用方 -使用 pChronicle。事件 producer 不得为了构造 `EventRecord` 依赖 pChronicle。 - -## 验收条件 - -- Workspace 不再包含 `persisting-engine` crate、动态库、C ABI 或 Engine RPC 信封。 -- append、replay、stats 等轨迹流程均由 pChronicle 提供。 -- CLI 不再实现 AgenticMD 到 event 的独立解析。 -- Gateway 不再定义与 `EventRecord` 同构的序列化 struct，也不再独立实现 AgenticMD 文档重写或索引。 -- pChronicle MUST 使用 Gateway 的真实 AgenticMD、request/response、provider snapshot 与 SSE fixture 验证 wire、Arrow、Lance 和投影兼容性。 -- ATIF 与 Lance 查询 MUST 注册相同的 `runs`、`steps`、`tool_calls` Arrow schema。 -- pChronicle 的物理后端、格式转换、WAL/投影相关边界行为有回归测试。 -- 文档和 crate metadata 均把 pChronicle 表述为结构化轨迹存储层。 - -## Amendment history - -| Date | RFC | Change | -|---|---|---| -| 2026-08-16 | [RFC-0007](0007-events-contract-pchronicle-sidecar.md) | 将公共 `EventRecord` 契约迁至 `persisting-events`；pChronicle 保留全部物理存储、格式转换与读取 ownership | +[Architecture](../pchronicle/design/architecture.md) diff --git a/docs/src/zh/rfcs/0005-pchronicle-revision-lineage.md b/docs/src/zh/rfcs/0005-pchronicle-revision-lineage.md deleted file mode 100644 index 9019fc73a..000000000 --- a/docs/src/zh/rfcs/0005-pchronicle-revision-lineage.md +++ /dev/null @@ -1,40 +0,0 @@ -# RFC-0005: pChronicle 派生 Revision Lineage - -| Field | Value | -|---|---| -| **Status** | Accepted | -| **Date** | 2026-08-09 | -| **Component** | `persisting-pchronicle` | -| **Related** | [RFC-0002 Events](0002-events-format.md) · [RFC-0003 Ownership](0003-pchronicle-ownership.md) | - -## 摘要 - -clean、redact、augment 和格式化数据集是 canonical events 的派生产物。它们不得覆写或 -去重事实流，也不得把 catalog 语义塞入 Storyline `extra_json`。每个 Run 使用独立的 -`revisions.lance` 记录 lineage；canonical `events.lance` 继续维持 at-least-once、 -append-only 契约。 - -## 数据模型 - -每个 revision 以 `revision_id` 为 upsert key，包含： - -- `parent_revision_ids`：零个或多个父 revision； -- `kind`：`clean`、`redact`、`augment`、`export` 或扩展 kind； -- `canonical_snapshot`：输入 event manifest revision或 Storyline snapshot； -- `recipe`：可重放的程序、版本、参数和输入摘要 JSON； -- `status`：`building`、`ready` 或 `failed`； -- `created_at` 与 `output_refs`。 - -`revisions.lance` 是派生 catalog，不参与 canonical event 的提交、replay 或审计裁定。 -写入相同 `revision_id` 更新状态和产物引用；不同 revision 永不隐式合并。 - -## 一致性与回收 - -产物必须先持久化，再把 revision 切到 `ready`。失败记录可保留用于诊断。对象维护将 -`CURRENT` 以及保留的 `ready` revision 视为 GC roots；删除 revision 本身必须是显式维护操作。 - -## 验收 - -- 同一 revision 从 `building` 更新为 `ready` 只留下一个 catalog row。 -- lineage 写入不改变 canonical event 行数或 manifest。 -- catalog 可被 Web UI 和 DataFusion 消费，并能定位 recipe 与产物。 diff --git a/docs/src/zh/rfcs/0006-pchronicle-vortex-backend.md b/docs/src/zh/rfcs/0006-pchronicle-vortex-backend.md index f7d49866a..d4ec87618 100644 --- a/docs/src/zh/rfcs/0006-pchronicle-vortex-backend.md +++ b/docs/src/zh/rfcs/0006-pchronicle-vortex-backend.md @@ -1,12 +1,13 @@ # RFC-0006: pChronicle Vortex 轨迹后端提案 +> 历史设计：事件格式与事件到 Storyline 的投影已移除，当前存储直接使用 Storyline。 + | Field | Value | |---|---| | **Status** | Proposed | | **Date** | 2026-08-16 | | **Component** | `persisting-pchronicle` / proposed `persisting-pchronicle-vortex` | | **Decision scope** | 可重建、读取优化的 Vortex trajectory projection；不替换 canonical facts | -| **Related** | [RFC-0001 Storyline](0001-storyline-format.md) · [RFC-0002 Events](0002-events-format.md) · [RFC-0003 Ownership](0003-pchronicle-ownership.md) · [RFC-0005 Revision Lineage](0005-pchronicle-revision-lineage.md) | | **External references** | [ATIF v1.7](https://github.com/harbor-framework/harbor/blob/main/rfcs/0001-trajectory-format.md) · [Vortex concepts](https://docs.vortex.dev/concepts/) · [Vortex file format](https://docs.vortex.dev/specs/file-format) | ## 1. 摘要与提议 diff --git a/docs/src/zh/rfcs/0007-events-contract-pchronicle-sidecar.md b/docs/src/zh/rfcs/0007-events-contract-pchronicle-sidecar.md deleted file mode 100644 index f7abb68f6..000000000 --- a/docs/src/zh/rfcs/0007-events-contract-pchronicle-sidecar.md +++ /dev/null @@ -1,145 +0,0 @@ -# RFC-0007: 事件契约与 pChronicle Sidecar 边界 - -| Field | Value | -|---|---| -| **Status** | Accepted | -| **Date** | 2026-08-16 | -| **Components** | `persisting-events` · pChronicle · Gateway | -| **Amends** | [RFC-0002 Events](0002-events-format.md) · [RFC-0003 pChronicle ownership](0003-pchronicle-ownership.md) | -| **Related** | [端到端架构](../system-design/architecture.md) · [轨迹存储](../pchronicle/design/trajectory-storage.md) | - -## 摘要 - -Persisting 将运行时事件的逻辑契约从存储实现中拆出，由唯一新增 crate -`persisting-events` 拥有。Gateway、事件 producer 与 pChronicle 共享该契约，但只有 -pChronicle 拥有 Lance、DataFusion、对象存储、Catalog、查询与投影实现。 - -本地集成方可以启动 `pchronicle serve --control 127.0.0.1:0 DATASET`， -通过带版本和认证令牌的 Control 协议提交事件并等待 durable acknowledgement。 -存储实现由服务进程拥有，客户端只依赖逻辑事件与协议类型。 - -## 动机 - -逻辑事件描述“发生了什么”，物理存储实现决定“如何落盘、查询和投影”。将两者分离， -让 producer 可以发布事件而无需依赖具体存储后端，同时继续使用同一种公共事件类型。 -pChronicle 保留结构化轨迹的持久化和读取职责。 - -## 决策 - -### 1. `persisting-events` 拥有逻辑事件契约 - -`persisting-events` 是存储无关的公共契约包，拥有： - -- `EventIdentity`：`event_id`、`run_id`、`attempt_id`、Story/Turn identity、producer - 与观测时间； -- `EventRecord`：扁平 identity、`seq`、`source`、`kind`、关联字段与 JSON payload； -- 不涉及存储引擎的信封校验； -- 可选 `control` feature 下的 pChronicle control 消息、trait 与进程 client。 - -默认 feature 只提供事件数据模型。`control` feature 可以依赖 Agent control 类型和 Tokio， -但 MUST NOT 引入 Lance、Arrow、DataFusion、对象存储 SDK 或 pChronicle 实现。 - -`EventRecord` 是进程内和进程间共同使用的逻辑记录，不是物理存储 schema。事件所属的 -Run、Attempt 或 Session 与 producer 定义的 `seq` 一起形成排序上下文；wall-clock -timestamp 只用于关联和展示，不作为顺序真相。 - -### 2. pChronicle 拥有物理存储与持久解释 - -pChronicle 消费 `persisting-events::EventRecord`，并独占维护： - -- `EventRow`、Arrow schema 与逻辑记录到物理行的映射； -- Lance/Vortex 等物理后端、writer fencing、manifest、compaction 与 vacuum； -- Catalog、query、replay、格式转换、revision 和派生视图； -- `pchronicle serve` 内嵌的 Control 服务以及 append 成功的 durable acknowledgement。 - -RFC-0003 中“pChronicle 拥有轨迹格式”的约束仍适用于物理 schema、交换格式与转换。 -其中“pChronicle 唯一定义 `EventRecord`”以及“所有调用方直接依赖 pChronicle 类型”的部分 -由本 RFC 修订。 - -### 3. 本地 producer 通过 Control 服务提交 - -`pchronicle serve --control 127.0.0.1:0 DATASET` 提供本地持久化入口。 -`persisting-events` 的进程 client 负责启动子进程、握手与请求关联。 -子进程退出、握手失败或协议版本不兼容都使持久化请求显式失败。 - -使用此协议的 producer 提交 `EventRecord`；Lance row、DataFusion 查询与对象存储 -实现由 pChronicle 管理。 - -### 4. Control 协议随事件契约发布 - -不创建 `persisting-pchronicle-client`。control client、协议信封、Attempt/lease/commit -消息和内存测试实现位于 `persisting-events` 的可选 `control` feature 中。这些类型描述 -producer 到持久化服务的边界，本身不实现存储。 - -当前协议版本为 `2`，使用 loopback TCP 上的逐行 JSON frame。sidecar 在 stdout 发布 -一次 readiness 记录，包含 endpoint、协议版本和每进程随机认证令牌。每个请求携带版本、 -request ID 与令牌；client 校验响应版本和 request ID。frame 大小有明确上限。 - -这是本地子进程协议，不是面向不可信网络的远程公共 API。跨主机部署需要另行定义传输 -安全、身份和可用性契约。 - -### 5. ACK、背压与不确定性 - -Gateway 到存储的 append 队列是有界的。提交被拒绝与提交后的结果不确定必须区分。 -只有 pChronicle 完成 append 并返回成功时，事件才视为 durable。 - -连接中断、写入错误或 ACK 丢失无法证明事件未提交，必须分类为 unknown。此时 producer -消耗该序号，不能把不确定写入伪装为“肯定未写”。事实层仍允许 at-least-once 与重复 -`event_id`；exactly-once 和热路径去重不属于该协议。 - -同一 control sidecar 也承载 Run lease、Attempt active/heartbeat/terminal 和 Run commit。 -这些控制记录的 fencing/CAS 语义保持不变。 - -## 依赖边界 - -```text -Gateway / local event producers - │ - │ EventRecord + optional control protocol - ▼ - persisting-events - │ - │ versioned authenticated local IPC - ▼ - pchronicle serve process (Control enabled) - │ - ├── EventRow / Arrow - ├── Lance or another storage backend - └── Catalog / query / projection -``` - -`persisting-events` 的默认 feature 和可选 `control` feature 均不得引入 Lance、Arrow、 -DataFusion 或云对象存储 SDK。物理存储与查询实现只由 pChronicle 维护。 - -## 兼容与迁移 - -- `EventRecord` 的 JSON 顶层字段保持扁平，移动 crate 不改变既有 wire 形状； -- pChronicle 对外 re-export 公共事件类型，允许调用方渐进迁移 import； -- `persisting-pchronicle-client` 被删除，使用者改为 - `persisting-events = { features = ["control"] }`； -- pChronicle 的既有 Lance dataset、目录布局和 replay 语义不因这次 crate 拆分而变化。 - -## 被否决的方案 - -| 方案 | 原因 | -|---|---| -| producer 内嵌另一套 Lance adapter | producer 与存储引擎生命周期、feature 和依赖重新耦合 | -| `EventRecord` 继续由 pChronicle 定义 | producer 为使用基础事件信封被迫依赖存储产品 | -| 单独保留 `persisting-pchronicle-client` | 协议包过碎；control 契约可以作为事件边界的可选 feature | -| 各集成方分别实现 IPC client | 会产生协议漂移、重复认证与错误语义 | -| 仅依赖离线导入而无实时 Control 协议 | 缺少运行期 durable ACK、fencing 与统一 canonical append 语义 | - -## 验收条件 - -- Workspace 只新增 `persisting-events`，不存在 `persisting-pchronicle-client`； -- Gateway、producer、pChronicle 复用同一 `EventRecord`，没有同构公共事件 struct； -- `persisting-events` 不直接依赖 pChronicle 存储 API，也不链接 Lance/DataFusion； -- 进程 client 能够启动 Control 服务，持久写入有序事件并发布 Attempt 终态； -- 协议版本、认证、request correlation、frame limit、拒绝与 unknown 写入语义有测试； -- pChronicle 的原有 replay、query 与物理存储测试继续通过。 - -## Changelog - -| Version | Date | Notes | -|---|---|---| -| Accepted | 2026-08-16 | 拆出 `persisting-events`，移除 client crate，确立逻辑事件与 pChronicle 存储服务的边界 | diff --git a/docs/src/zh/rfcs/0015-chronicle-manifest.md b/docs/src/zh/rfcs/0015-chronicle-manifest.md index d2600a0b9..1405bce24 100644 --- a/docs/src/zh/rfcs/0015-chronicle-manifest.md +++ b/docs/src/zh/rfcs/0015-chronicle-manifest.md @@ -44,7 +44,7 @@ pChronicle 已对其它布局使用应用控制文件（Storyline 的 `CURRENT` - `list(path)` 与 shell `ls` 一致：只列一层 children，MUST NOT 把嵌套 source 摊平； - `open(path)` 生成 Snapshot。叶子 Dataset 是一个 source。纯目录 MAY 被当成 **虚拟 dataset** 打开：该路径下所有嵌套 Dataset 叶子与外围 JSON 合成一个查询空间。 -- 在 sidecar 缺失时，仍可用 `CURRENT` / events / compact-jsonl 标记做 Dataset +- 在 sidecar 缺失时，仍可用 `CURRENT` / compact-jsonl 标记做 Dataset 分类。`list` MUST NOT 为分类而全量递归列举对象存储前缀。对 Directory 做 `open` 时 MAY 在现有 `max_entries` / `max_files` 上限内递归。 @@ -83,7 +83,7 @@ pChronicle 已对其它布局使用应用控制文件（Storyline 的 `CURRENT` 1. `chronicle.manifest` 且 `kind = "leaf"` 2. `CURRENT`（Storyline） 3. `events.lance/_manifest.json`，或名为 `events.lance` 且含 `_manifest.json` 的目录 -4. compact-jsonl Lance（`pchronicle.format = compact-jsonl/v1`，或 leaf sidecar +3. compact-jsonl Lance（`pchronicle.format = compact-jsonl/v1`，或 leaf sidecar `format = "compact-jsonl/v1"`） 否则该路径是 **Directory**（包括 `chronicle.manifest` 的 `kind = "branch"`）。 diff --git a/docs/src/zh/rfcs/index.md b/docs/src/zh/rfcs/index.md index 73de532fd..74b852c87 100644 --- a/docs/src/zh/rfcs/index.md +++ b/docs/src/zh/rfcs/index.md @@ -9,12 +9,9 @@ RFC 正文保持英文，因为它们是历史决策快照。翻译会产生两 | RFC | 标题 | 状态 | |---|---|---| | [0001](0001-storyline-format.md) | Storyline 格式（`storyline/v1`） | Accepted | -| [0002](0002-events-format.md) | Events 格式（`events`） | Accepted | | [0003](0003-pchronicle-ownership.md) | pChronicle 所有权 | Accepted | | [0004](0004-actf-format.md) | ACTF 格式 | Accepted | -| [0005](0005-pchronicle-revision-lineage.md) | pChronicle 修订谱系 | Accepted | | [0006](0006-pchronicle-vortex-backend.md) | pChronicle Vortex 后端 | Proposed | -| [0007](0007-events-contract-pchronicle-sidecar.md) | 事件契约与 pChronicle Sidecar | Accepted | | [0008](0008-atif-format.md) | ATIF 格式 | Accepted | | [0009](0009-openai-messages-format.md) | OpenAI Messages 格式 | Accepted | | [0010](0010-agent-corpus-lance-layout.md) | Agent 评测语料的分层 Lance 布局 | Proposed | diff --git a/docs/src/zh/system-design/architecture.md b/docs/src/zh/system-design/architecture.md index bb4f4e6e2..6e0493334 100644 --- a/docs/src/zh/system-design/architecture.md +++ b/docs/src/zh/system-design/architecture.md @@ -1,57 +1,9 @@ # 系统架构 -pChronicle 负责 Agent 轨迹历史的捕获、存储、查询与交换。本文定义引擎、入口和内部库 -之间的职责；物理布局见 [pChronicle 设计](../pchronicle/design/index.md)。 +pChronicle 库拥有轨迹模型、格式转换、发现、Lance 存储、Snapshot 和有界查询。 +CLI 拥有导入导出编排与 loopback Warehouse API，Web UI 使用共享模型和 API 契约。 -## 组件职责 +Storyline 文档直接写入存储。发布表版本与写入租约保证一致性，查询读取固定来源版本。 +Dataset、Source 与 revision 身份让转换结果可追溯。 -| 组件 | 职责 | -|---|---| -| `persisting-pchronicle` | 轨迹模型、存储、Source 发现、Dataset Snapshot、有界查询、格式转换与版本血缘 | -| `persisting-pchronicle-cli` | `pchronicle` 命令、本地只读 Warehouse API、可选 Control 服务、Gateway 配置与内嵌 Web 资源 | -| `pchronicle-web` | Dataset 浏览与查询界面 | -| `persisting-events` | 存储无关的 `EventRecord` 及可选的带版本 Control 客户端与协议 | -| `persisting-gateway` | 模型协议适配、转发、会话关联与轨迹捕获 | -| `persisting-overlaynet` | 代理传输、请求分类与已拦截流量的策略执行 | -| `persisting-agentctl` | 共享控制类型、策略状态转换与协作式客户端协议 | - -## 捕获与导入 - -```text -Agent / SDK 请求 - → Gateway 协议处理与捕获 - → EventRecord - → pChronicle canonical event 存储 - → Dataset Snapshot → 有界查询 → CLI / Web / 导出 - -受支持的文件与对象存储 Source - → 发现并固定 Source 版本 - → Storyline 规范化与查询投影 - → Dataset Snapshot → 有界查询 → CLI / Web / 导出 -``` - -Gateway 捕获经过它的流量，导入则直接读取受支持的外部格式。两条路径保留可用身份与 -来源信息，不会补出源数据缺失的执行或隔离证据。 - -## 写入与读取边界 - -CLI 可以在进程内调用引擎。集成方也可以启动 -`pchronicle serve --control 127.0.0.1:0 DATASET`，使用 `persisting-events` 中带认证、 -带版本的本地协议。只有成功的 append ACK 才确认持久化；响应丢失意味着结果不确定， -调用方不能据此认定事件没有写入。 - -pChronicle 拥有物理 schema、writer fencing、manifest 发布与维护。Producer 提交逻辑 -记录，不另行定义存储布局。详见 [RFC-0007](../rfcs/0007-events-contract-pchronicle-sidecar.md) -与[轨迹存储](../pchronicle/design/trajectory-storage.md)。 - -Warehouse HTTP API 与 Web UI 只读，单独启用的 Control 和 Gateway 捕获路径可以写入。 -本地服务拒绝公共绑定地址，Control 协议面向可信本地进程。 - -## 事实、视图与版本 - -Canonical events 保留原始逻辑 payload。Storyline 为交换格式提供规范化模型，投影支持 -分析而不替代原始事实。查询读取固定的 Snapshot，版本血缘标识派生输出。 - -捕获覆盖范围取决于 Source 中实际存在的记录。代理策略仅作用于到达代理的流量，不能 -证明进程级网络隔离。详见 [Gateway 捕获](../pchronicle/guides/serve-gateway.md)和 -[事实与投影](../pchronicle/concepts/facts-and-projections.md)。 +[pChronicle architecture](../pchronicle/design/architecture.md) diff --git a/docs/src/zh/system-design/design-principles.md b/docs/src/zh/system-design/design-principles.md index f22a7d793..684f900c0 100644 --- a/docs/src/zh/system-design/design-principles.md +++ b/docs/src/zh/system-design/design-principles.md @@ -1,28 +1,6 @@ # 设计原则 -## 区分事实与投影 - -Canonical events 保留已记录的事实。规范化的 Storyline 视图和查询投影可以追溯到 -Source；记录缺失应作为答案的限制明确呈现。 - -## 明确职责 - -共享事件契约描述逻辑记录。pChronicle 拥有存储、查询和交换，Gateway 拥有捕获与协议 -适配。CLI 和 Web UI 使用这些边界，不另行定义数据模型。 - -## 保留来源与版本 - -答案应能定位到产生它的 Dataset、Source、Snapshot 和查询。版本血缘让派生输出在 -规范化和导出后仍然可以追溯到输入。 - -## 限制资源使用 - -查询预算、有界捕获队列和明确的 append ACK 让资源使用与写入结果可检查。投影失败 -不能被报告为已成功持久化。 - -## 保持数据可移植 - -公开格式与 CLI 让 Dataset 不依赖特定查看器。本地文件和对象存储 Source 使用统一的 -Dataset 模型。 - -继续阅读[系统概览](index.md)和[路线图](../roadmap.md)。 +通过明确的编解码契约保留输入信息。使用统一的 Storyline 模型，集中管理存储 +和查询语义。每次操作固定 Source 版本，转换结果保留来源，解析、I/O 与查询都有 +资源上限。发布失败时保留上一已提交数据。Dataset 可在文件、对象存储、CLI 与 Web +之间使用。 diff --git a/docs/src/zh/system-design/index.md b/docs/src/zh/system-design/index.md index d30761766..f1945123b 100644 --- a/docs/src/zh/system-design/index.md +++ b/docs/src/zh/system-design/index.md @@ -1,21 +1,8 @@ # 系统设计 -Persisting 围绕 [pChronicle](../pchronicle/index.md) 构建 Agent 轨迹存储引擎。 -Gateway 捕获模型流量，共享事件契约将记录交给存储，CLI 和 Web UI 提供 Dataset 浏览与分析入口。 +Persisting 专注于 pChronicle 轨迹数据处理管线：解码输入、写入 Storyline、 +固定 Snapshot 版本、查询，以及通过 CLI 和 Web 导出。 -```text -Gateway 捕获 ── EventRecord ── pChronicle 存储 ─┐ -ATIF / ACTF / OpenAI Messages / Storyline Source ┴─> Dataset Snapshot - └─> 查询 / 交换 / Web UI -``` - -Canonical events 保存已记录的事实；Storyline 规范化和查询投影将受支持的 Source -呈现为统一视图。Snapshot 固定查询使用的 Source 版本，导入不会补出源数据中不存在的事实。 - -## 按问题继续阅读 - -- [组件职责与数据流](architecture.md) -- [设计原则](design-principles.md) -- [存储与查询实现](../pchronicle/design/index.md) -- [Gateway 捕获](../pchronicle/guides/serve-gateway.md) -- [项目工程说明](../project/engineering.md) +- [Architecture](architecture.md) +- [Design principles](design-principles.md) +- [pChronicle implementation](../pchronicle/design/index.md) diff --git a/docs/static/img/diagrams/persisting/gateway-dataflow.svg b/docs/static/img/diagrams/persisting/gateway-dataflow.svg deleted file mode 100644 index 8833a5e67..000000000 --- a/docs/static/img/diagrams/persisting/gateway-dataflow.svg +++ /dev/null @@ -1,17 +0,0 @@ - - Gateway event dataflowThe HTTP capture proxy emits events to a non-blocking event flow. The flow appends canonical events to Lance and independently triggers handlers. Handlers reduce events into Storyline and write AgenticMD, ATIF, and other derived formats. Lance can replay events through the same handlers. - - Gateway dataflowRecord canonical events first; build human and exchange views through independent handlers - Capture proxyHTTP · SSE · protocol adapterforward without waiting on views - emit - Event flowordered append · bounded fan-outcallback never blocks - record - events.lancecanonical source of truthappend + replay - trigger - Handlersinterpret · reduce · retryfailure does not block HTTP - - Storyline hubsession + call normalizationhuman-readable semantics - - ViewsAgenticMDATIF · others - replay - diff --git a/docs/static/img/diagrams/persisting/gateway-request.svg b/docs/static/img/diagrams/persisting/gateway-request.svg deleted file mode 100644 index 081d2ba58..000000000 --- a/docs/static/img/diagrams/persisting/gateway-request.svg +++ /dev/null @@ -1,10 +0,0 @@ - - Gateway request capture sequenceAn Agent sends a request through the capture proxy. The proxy emits the request event and forwards upstream. Streaming drafts update handlers while SSE continues to the Agent. On cancellation Lance records cancel; on completion Lance records the final response and handlers finalize derived views. - - One captured conversation requestThe response path does not wait for derived-format persistence - AgentCapture proxyEvent flowevents.lanceHandlersUpstream - - REQUESTconversation requestemit request eventappend canonical requesttrigger Storyline / Markdownforward upstream - STREAMstreaming responseoptional draftlive view upsertforward SSE - TERMINALcancel or response completeappend terminal factfinalize views on completion - diff --git a/docs/static/img/diagrams/persisting/pchronicle-product.svg b/docs/static/img/diagrams/persisting/pchronicle-product.svg deleted file mode 100644 index 8f263c3ff..000000000 --- a/docs/static/img/diagrams/persisting/pchronicle-product.svg +++ /dev/null @@ -1,105 +0,0 @@ - - pChronicle product boundary - Canonical runtime event Sources and pinned local or S3 external trajectory Sources from external evaluators and Agent frameworks enter pChronicle through separate branches and converge at a Snapshot and normalized Dataset views. Supported external files include ATIF, ACTF, OpenAI Messages, and Storyline Sources; they do not become canonical runtime events. CLI import is a separate inbound path that accepts ATIF, ACTF, and OpenAI Messages into a new Dataset. Outbound surfaces provide query and analysis, export including Storyline, revisions, and a read-only Warehouse API and Web interface. - - - - - - - - - - - - - - - - pChronicle product boundary - Dataset-first trajectory data · direct Sources · directional read and exchange surfaces - - - ingest / write - - query / export / read - - - - Canonical event Sources - Gateway · event producers · native writers - - - - - Pinned external Sources - External evaluators · Agent frameworks - local/S3 ATIF · ACTF · OpenAI Messages - Storyline Sources · direct normalization - - - - - CLI import → new Dataset - ATIF · ACTF · OpenAI Messages - - - - - - - - - Snapshot - pin Source versions - preserve Source identity - normalize Dataset views - external files remain - non-event Sources - - - - - - - pChronicle Core - discover · pin Snapshot - normalize Sources · query - persist canonical runtime events - export · revision lineage - - - - - - - - - CLI query - query · analysis - - - - - Export + revisions - ATIF · ACTF · OpenAI - Storyline · revision lineage - - - - - Read-only Warehouse - API · Web - - diff --git a/docs/zensical.toml b/docs/zensical.toml index d150a75a5..c8b02b9f6 100644 --- a/docs/zensical.toml +++ b/docs/zensical.toml @@ -16,8 +16,8 @@ nav = [ { "Installation" = "en/installation.md" }, { "Recording and Analyzing Agent Trajectories" = "en/pchronicle/get-started.md" }, ] }, - { "pChronicle" = ["en/pchronicle/index.md", "en/pchronicle/get-started.md", { "Concepts" = ["en/pchronicle/concepts/index.md", "en/pchronicle/concepts/dataset-and-source.md", "en/pchronicle/concepts/facts-and-projections.md"] }, { "Guides" = ["en/pchronicle/guides/index.md", "en/pchronicle/guides/discover-and-query.md", "en/pchronicle/guides/exchange.md", "en/pchronicle/guides/serve.md", "en/pchronicle/guides/serve-gateway.md", "en/pchronicle/guides/ui.md", "en/pchronicle/guides/troubleshooting.md"] }, { "Design" = ["en/pchronicle/design/index.md", "en/pchronicle/design/architecture.md", "en/pchronicle/design/catalog.md", "en/pchronicle/design/trajectory-storage.md", "en/pchronicle/design/storyline-lance.md"] }, { "Reference" = ["en/pchronicle/reference/index.md", "en/pchronicle/reference/cli.md", "en/pchronicle/reference/query-model.md", "en/pchronicle/reference/terminology.md", "en/pchronicle/reference/agenticmd.md", "en/pchronicle/reference/formats/index.md", "en/pchronicle/reference/cases-self.md", "en/pchronicle/reference/cases-platform.md"] }] }, - { "Agent Infra Thinkings" = ["en/why-persisting.md", "en/roadmap.md", "en/guides/using-persisting.md", "en/system-design/index.md", "en/system-design/design-principles.md", "en/system-design/architecture.md", "en/project/index.md", "en/project/examples.md", "en/project/engineering.md", "en/project/releasing.md", { "RFCs" = ["en/rfcs/index.md", "en/rfcs/0001-storyline-format.md", "en/rfcs/0002-events-format.md", "en/rfcs/0003-pchronicle-ownership.md", "en/rfcs/0004-actf-format.md", "en/rfcs/0005-pchronicle-revision-lineage.md", "en/rfcs/0006-pchronicle-vortex-backend.md", "en/rfcs/0007-events-contract-pchronicle-sidecar.md", "en/rfcs/0008-atif-format.md", "en/rfcs/0009-openai-messages-format.md", "en/rfcs/0010-agent-corpus-lance-layout.md", "en/rfcs/0012-pchronicle-find-query-syntax.md", "en/rfcs/0013-pchronicle-warehouse-catalog.md", "en/rfcs/0014-compact-jsonl.md", "en/rfcs/0015-chronicle-manifest.md"] }] }, + { "pChronicle" = ["en/pchronicle/index.md", "en/pchronicle/get-started.md", { "Concepts" = ["en/pchronicle/concepts/index.md", "en/pchronicle/concepts/dataset-and-source.md", "en/pchronicle/concepts/facts-and-projections.md"] }, { "Guides" = ["en/pchronicle/guides/index.md", "en/pchronicle/guides/discover-and-query.md", "en/pchronicle/guides/exchange.md", "en/pchronicle/guides/serve.md", "en/pchronicle/guides/ui.md", "en/pchronicle/guides/troubleshooting.md"] }, { "Design" = ["en/pchronicle/design/index.md", "en/pchronicle/design/architecture.md", "en/pchronicle/design/catalog.md", "en/pchronicle/design/trajectory-storage.md", "en/pchronicle/design/storyline-lance.md"] }, { "Reference" = ["en/pchronicle/reference/index.md", "en/pchronicle/reference/cli.md", "en/pchronicle/reference/query-model.md", "en/pchronicle/reference/terminology.md", "en/pchronicle/reference/agenticmd.md", "en/pchronicle/reference/formats/index.md", "en/pchronicle/reference/cases-self.md", "en/pchronicle/reference/cases-platform.md"] }] }, + { "Agent Infra Thinkings" = ["en/why-persisting.md", "en/roadmap.md", "en/guides/using-persisting.md", "en/system-design/index.md", "en/system-design/design-principles.md", "en/system-design/architecture.md", "en/project/index.md", "en/project/examples.md", "en/project/engineering.md", "en/project/releasing.md", { "RFCs" = ["en/rfcs/index.md", "en/rfcs/0001-storyline-format.md", "en/rfcs/0003-pchronicle-ownership.md", "en/rfcs/0004-actf-format.md", "en/rfcs/0006-pchronicle-vortex-backend.md", "en/rfcs/0008-atif-format.md", "en/rfcs/0009-openai-messages-format.md", "en/rfcs/0010-agent-corpus-lance-layout.md", "en/rfcs/0012-pchronicle-find-query-syntax.md", "en/rfcs/0013-pchronicle-warehouse-catalog.md", "en/rfcs/0014-compact-jsonl.md", "en/rfcs/0015-chronicle-manifest.md"] }] }, ] [project.theme] diff --git a/justfile b/justfile index d28ca65c9..33444978d 100644 --- a/justfile +++ b/justfile @@ -5,14 +5,9 @@ repo := justfile_directory() docs_dir := repo / "docs" gen_py := repo / "scripts" / "generate_benchmark_data.py" -# Product CLI component set. Keep all Cargo build entry points below routed -# through `build-components` so package/bin changes have one source in just. -component_pchronicle := "-p persisting-pchronicle-cli --bin pchronicle" - -# Python 路径（ruff format） -ruff_paths := "persisting tests examples" -# lint 默认只扫包代码（与 CI 一致）；全量用 lint-py-all -ruff_lint_paths := "persisting" +# Python checks follow the active pChronicle scope in AGENTS.md. +python_tests := "tests/test_release_packaging.py tests/test_trajectory_dialogue.py benchmark/pchronicle" +ruff_paths := "scripts/packaging scripts/ci scripts/build-docs.py scripts/check-docs.py scripts/serve-docs.py scripts/run-pchronicle-cases.py " + python_tests # ── 帮助 ───────────────────────────────────────────────────────────────────── @@ -27,9 +22,6 @@ default: @echo " just py-dev # 同步纯 Python 开发环境" @echo " just install-cli # 安装 pchronicle" @echo " just chronicle-binary # 构建可直接测试的 pChronicle UI binary" - @echo " just echo # 启动确定性的本地 LLM Echo upstream" - @echo " just benchmark-gateway # Gateway 转发与持久化黑盒压测" - @echo " just benchmark-gateway-replay # 回放 examples/data 并生成人工 review bundle" @echo " just build-wheel # 打 release wheel → dist/" @echo " just docs-serve # 本地文档" @@ -48,12 +40,9 @@ test-list: just ci CI 近似 just test [package] 日常功能测试；可指定 Cargo 包 just proptest pchronicle pChronicle 全量性质测试回归 - just capture-test / test-py 其他定向测试入口 + just test-py 其他定向测试入口 （Rust 测试由 cargo nextest 执行；文档测试仍用 cargo test） just regression 大规模黑盒回归（按场景运行 tests/regression） - just gateway-fuzz 一分钟 Gateway 四类 fuzz 汇总 - just gateway-fuzz-formats / gateway-fuzz-forwarding - just gateway-fuzz-storage / gateway-fuzz-network just cases pchronicle|pchronicle-cluster 组件示例 @@ -66,7 +55,8 @@ test-list: examples-pchronicle: #!/usr/bin/env bash set -euo pipefail - just build-components release pchronicle-benchmark + just build release + cargo build --release --locked -p persisting-pchronicle --example pchronicle_storage_query_benchmark bash "{{ repo }}/examples/pchronicle/test.sh" --profile release bash "{{ repo }}/examples/pchronicle/output-contract.sh" >/dev/null @@ -79,78 +69,6 @@ examples: examples-pchronicle regression: bash tests/regression/run.sh -# Run the four Gateway fuzz contracts (about one minute total by default). -# Override duration, concurrency, and rate through PERSISTING_FUZZ_* variables. -[group('test')] -gateway-fuzz: - bash tests/regression/gateway-fuzz/run.sh - -[group('test')] -gateway-fuzz-formats: - bash tests/regression/gateway-fuzz/formats/run.sh - -[group('test')] -gateway-fuzz-forwarding: - bash tests/regression/gateway-fuzz/forwarding/run.sh - -[group('test')] -gateway-fuzz-storage: - bash tests/regression/gateway-fuzz/storage/run.sh - -[group('test')] -gateway-fuzz-network: - bash tests/regression/gateway-fuzz/network-policy/run.sh - -# Benchmark Gateway forwarding, typed capture, WAL, and durable Lance append -# against the deterministic local Echo upstream. -# Set PERSISTING_KEEP_TEST_ARTIFACTS=1 to retain the Dataset, WAL, and logs. -# Usage: `just benchmark-gateway`, or `just benchmark-gateway 30 32 1024 0 16 2048`. -[group('benchmark')] -benchmark-gateway duration="10" concurrency="16" payload_bytes="256" warmup="0" sessions="16" requests="1024": - #!/usr/bin/env bash - set -euo pipefail - extra_args=() - if [[ "${PERSISTING_KEEP_TEST_ARTIFACTS:-0}" == "1" ]]; then - extra_args+=(--keep-artifacts) - fi - bash benchmark/gateway/run.sh \ - --duration "{{ duration }}" \ - --concurrency "{{ concurrency }}" \ - --sessions "{{ sessions }}" \ - --requests "{{ requests }}" \ - --payload-bytes "{{ payload_bytes }}" \ - --warmup "{{ warmup }}" \ - "${extra_args[@]}" - -# Replay every supported trajectory example through Gateway and write a -# timestamped, human-reviewable bundle. Both arguments accept relative paths. -[group('benchmark')] -benchmark-gateway-replay data="examples/data" output="benchmark/gateway/results/replay-review": - bash benchmark/gateway/replay.sh \ - --data "{{ data }}" \ - --output "{{ output }}" - -# Find Gateway's saturation point with a closed-loop concurrency sweep. This -# consumes the existing release binary and deliberately skips the Echo baseline. -# Usage: `just benchmark-gateway-sweep`, or -# `just benchmark-gateway-sweep 10 "1 2 4 8 16 32 64" 256 0 16 256`. -[group('benchmark')] -benchmark-gateway-sweep duration="5" concurrencies="1 2 4 8 16 32" payload_bytes="256" warmup="0" sessions="16" requests="256": - #!/usr/bin/env bash - set -euo pipefail - for concurrency in {{ concurrencies }}; do - printf '\n==> Gateway concurrency %s\n' "$concurrency" - bash benchmark/gateway/run.sh \ - --duration "{{ duration }}" \ - --concurrency "$concurrency" \ - --sessions "{{ sessions }}" \ - --requests "{{ requests }}" \ - --payload-bytes "{{ payload_bytes }}" \ - --warmup "{{ warmup }}" \ - --skip-baseline \ - --output "benchmark/gateway/results/sweep-c${concurrency}.json" - done - # Run the unified Criterion + hyperfine pChronicle smoke benchmark and render # raw JSON, Markdown, HTML, and a Bencher-compatible metric projection. [group('benchmark')] @@ -169,13 +87,6 @@ benchmark-pchronicle-compare baseline candidate output="target/pchronicle-benchm # ── 构建 ───────────────────────────────────────────────────────────────────── -# Start the deterministic local LLM upstream used to test Gateway forwarding, -# model/protocol rewriting, streaming, and capture. -# Usage: `just echo`, or `just echo 127.0.0.1:19080 base64`. -[group('build')] -echo listen="127.0.0.1:19080" encoding="plain": - target/release/pchronicle echo --listen "{{ listen }}" --encoding "{{ encoding }}" - # Build the Dioxus trajectory workbench for compile-time embedding. chronicle-web-build: python3 scripts/packaging/stage_wheel_binaries.py --web-only @@ -198,21 +109,16 @@ chronicle-binary profile="debug": chronicle-web-build exit 2 ;; esac - just build-components "$profile" pchronicle + just build "$profile" binary="{{ repo }}/target/$profile/pchronicle" test -x "$binary" "$binary" serve --help >/dev/null printf 'Built pChronicle test binary: %s\n' "$binary" printf 'Run: %s serve --warehouse %s\n' "$binary" "{{ repo }}/data" -# Thin forward to `build-components` for the full product CLI set. +# Build the pChronicle CLI; use chronicle-binary to embed the Web UI. [group('build')] build profile="debug": - just build-components "{{ profile }}" all - -# Build pChronicle, optionally including its storage benchmark executable. -[group('build')] -build-components profile="debug" components="all": #!/usr/bin/env bash set -euo pipefail case "{{ profile }}" in @@ -220,17 +126,7 @@ build-components profile="debug" components="all": release) cargo_profile=release ;; *) echo "unsupported build profile: {{ profile }} (expected debug or release)" >&2; exit 2 ;; esac - case "{{ components }}" in - all|pchronicle) - cargo build --profile "$cargo_profile" --locked {{ component_pchronicle }} - ;; - pchronicle-benchmark) - cargo build --profile "$cargo_profile" --locked \ - {{ component_pchronicle }} \ - -p persisting-pchronicle --example pchronicle_storage_query_benchmark - ;; - *) echo "unsupported component set: {{ components }} (all|pchronicle|pchronicle-benchmark)" >&2; exit 2 ;; - esac + cargo build --profile "$cargo_profile" --locked -p persisting-pchronicle-cli --bin pchronicle # Install the pChronicle CLI. install-cli: @@ -238,7 +134,7 @@ install-cli: set -euo pipefail install_root="${CARGO_INSTALL_ROOT:-${CARGO_HOME:-$HOME/.cargo}}" cargo install --path crates/persisting-pchronicle-cli --locked --force --root "$install_root" - printf 'Installed Persisting component set in %s/bin\n' "$install_root" + printf 'Installed pchronicle in %s/bin\n' "$install_root" # PEP 517 release wheel（Python package + pchronicle）→ dist/ build-wheel: @@ -272,7 +168,7 @@ clean: fmt: fmt-rust fmt-py fmt-rust: - cargo fmt --all + cargo fmt -p persisting-pchronicle -p persisting-pchronicle-cli cargo fmt --manifest-path pchronicle-web/Cargo.toml fmt-py: @@ -282,7 +178,7 @@ fmt-py: fmt-check: fmt-check-rust fmt-check-py fmt-check-rust: - cargo fmt --all -- --check + cargo fmt -p persisting-pchronicle -p persisting-pchronicle-cli -- --check cargo fmt --manifest-path pchronicle-web/Cargo.toml -- --check fmt-check-py: @@ -291,26 +187,22 @@ fmt-check-py: # clippy + ruff（不改写） lint: lint-rust lint-py -lint-rust: clippy-deny clippy-pchronicle-web clippy-pchronicle-panics clippy-pchronicle-features +lint-rust: clippy-deny clippy-pchronicle-web clippy-pchronicle-features lint-py: - uvx ruff check {{ ruff_lint_paths }} - -# 含 tests/examples（较严，可能有存量告警） -lint-py-all: uvx ruff check {{ ruff_paths }} +# Compatibility alias; Python lint already covers in-scope scripts and tests. +lint-py-all: lint-py + clippy-deny: - cargo clippy --workspace --exclude persisting-dlcapt --all-targets --locked -- -D warnings + cargo clippy -p persisting-pchronicle -p persisting-pchronicle-cli --all-targets --locked -- -D warnings + cargo clippy -p persisting-pchronicle --lib --locked -- -D warnings -D clippy::unwrap_used -D clippy::expect_used -D clippy::unreachable -# pchronicle-web is a separate Cargo workspace and is not covered by the root -# workspace Clippy invocation above. +# pchronicle-web is a separate Cargo workspace and has its own Clippy check. clippy-pchronicle-web: cargo clippy --manifest-path pchronicle-web/Cargo.toml --all-targets --locked -- -D warnings -clippy-pchronicle-panics: - cargo clippy -p persisting-pchronicle --lib --locked -- -D warnings -D clippy::unwrap_used -D clippy::expect_used -D clippy::unreachable - clippy-pchronicle-features: cargo clippy -p persisting-pchronicle --lib --no-default-features --locked -- -D warnings -D clippy::unwrap_used -D clippy::expect_used -D clippy::unreachable cargo clippy -p persisting-pchronicle --lib --no-default-features --features lance-store --locked -- -D warnings -D clippy::unwrap_used -D clippy::expect_used -D clippy::unreachable @@ -354,18 +246,6 @@ ci: # ── Rust 测试 ───────────────────────────────────────────────────────────────── -# CI shard helper: `just ci-nextest persisting-gateway persisting-events …` -# Variadic args are interpolated by just (not passed as shebang $@). -[group('test')] -ci-nextest +packages: - #!/usr/bin/env bash - set -euo pipefail - args=() - for pkg in {{ packages }}; do - args+=(-p "$pkg") - done - cargo nextest run --locked "${args[@]}" - # 单 crate：pchronicle（含 CLI，对齐 CI pchronicle shard）| pchronicle-cli | … test-crate crate: #!/usr/bin/env bash @@ -377,10 +257,8 @@ test-crate crate: -p persisting-pchronicle-cli ;; pchronicle-cli) cargo nextest run -p persisting-pchronicle-cli --locked ;; - agentctl) cargo nextest run -p persisting-agentctl --locked ;; - capture) cargo nextest run -p persisting-gateway --locked ;; dlcapt) cargo test -p persisting-dlcapt ;; - *) echo "unknown crate: {{ crate }} (pchronicle|pchronicle-cli|agentctl|capture|dlcapt)" >&2; exit 2 ;; + *) echo "unknown crate: {{ crate }} (pchronicle|pchronicle-cli|dlcapt)" >&2; exit 2 ;; esac test-rust package="": @@ -390,22 +268,17 @@ test-rust package="": if [[ -n "$package" ]]; then cargo nextest run --locked -p "$package" else - cargo nextest run --workspace --exclude persisting-dlcapt --locked + just test-crate pchronicle fi [group('test')] smoke-pchronicle-cli: - just build-components debug pchronicle + just build debug target/debug/pchronicle query --help >/dev/null -# Shared EventRecord control-plane feature contract used by the Linux core shard. -[group('test')] -test-events-control: - cargo nextest run -p persisting-events --features control --locked - # Separate Cargo workspace covering the Dioxus trajectory workbench. [group('test')] -test-pchronicle-web: chronicle-web-build +test-pchronicle-web: cargo nextest run --manifest-path pchronicle-web/Cargo.toml --locked # Real S3/MinIO contract (ignored by default; requires PCHRONICLE_S3_TEST_URI). @@ -413,16 +286,6 @@ test-pchronicle-web: chronicle-web-build test-pchronicle-s3: cargo nextest run -p persisting-pchronicle --test s3_storage --locked --run-ignored all -test-capture-claude: - cargo nextest run -p persisting-gateway --test capture_apps_claude --locked - -test-capture-fixtures: - cargo nextest run -p persisting-gateway --locked --test llm_fixtures --test ag_fixture_tests - -test-capture-network: - cargo nextest run -p persisting-gateway --locked --lib network_policy - cargo nextest run -p persisting-gateway --locked --test network_policy_http - test-search-integration: cargo test -p persisting-pchronicle --test search_integration @@ -445,7 +308,7 @@ proptest package: # Rust + Python. Rust tests run debug-mode nextest for faster iteration; use # `just test-rust` with a package for targeted coverage. Passing a package runs # only that Rust package; `pchronicle` also runs persisting-pchronicle-cli. -# The full Python suite runs only for the no-argument repository-wide invocation. +# In-scope Python tests run only for the no-argument invocation. test package="": #!/usr/bin/env bash set -euo pipefail @@ -456,7 +319,7 @@ test package="": exit 0 fi case "$package" in - pchronicle|pchronicle-cli|agentctl|capture|dlcapt) + pchronicle|pchronicle-cli|dlcapt) just test-crate "$package" ;; *) @@ -471,10 +334,10 @@ py-dev: uv sync --all-extras test-py: - uv run pytest tests/ -q + uv run --extra dev pytest {{ python_tests }} -q test-py-v: - uv run pytest tests/ -v + uv run --extra dev pytest {{ python_tests }} -v # 安装本地 nightly 脚本自检（需已有 GitHub nightly release） install-nightly: @@ -508,28 +371,20 @@ generate-benchmark search_rows="100" traj_rows="50" seed="42" search_out="" traj python3 "$gen_py" "${args[@]}" check-quick: - cargo check \ - -p persisting-agentctl \ - -p persisting-events \ - -p persisting-gateway \ - --locked + cargo check -p persisting-pchronicle-cli --locked cargo check -p persisting-pchronicle --no-default-features --locked -# capture 相关 Rust 测试（Gateway 包测试已覆盖全部 capture targets）。 -capture-test: - just test-crate capture - # Execute pChronicle single-machine/self-service cases. [group('test')] test-pchronicle-cases: - cargo build --release -p persisting-pchronicle-cli --locked + just build release python3 scripts/run-pchronicle-cases.py --document docs/src/zh/pchronicle/reference/cases-self.md --pchronicle target/release/pchronicle --report target/pchronicle-self-case-report.md # List and execute pChronicle platform/Catalog cases. Server lifecycle cases are # reported as MANUAL unless explicitly selected with PCHRONICLE_CASE_MODE. [group('test')] test-pchronicle-cases-platform: - cargo build --release -p persisting-pchronicle-cli --locked + just build release python3 scripts/run-pchronicle-cases.py --document docs/src/zh/pchronicle/reference/cases-platform.md --pchronicle target/release/pchronicle --report target/pchronicle-platform-case-report.md # Run documented integration cases by component. @@ -549,11 +404,11 @@ cases target *args: fi case "{{target}}" in pchronicle) - just build-components release pchronicle + just build release python3 scripts/run-pchronicle-cases.py --document docs/src/zh/pchronicle/reference/cases-self.md --pchronicle "{{ repo }}/target/release/pchronicle" --report target/pchronicle-self-case-report.md "$@" ;; pchronicle-cluster) - just build-components release pchronicle + just build release python3 scripts/run-pchronicle-cases.py --document docs/src/zh/pchronicle/reference/cases-platform.md --pchronicle "{{ repo }}/target/release/pchronicle" --report target/pchronicle-platform-case-report.md "$@" ;; *) diff --git a/pchronicle-web/src/agent.rs b/pchronicle-web/src/agent.rs index cdbdb363c..5fe43595d 100644 --- a/pchronicle-web/src/agent.rs +++ b/pchronicle-web/src/agent.rs @@ -403,12 +403,10 @@ fn system_prompt( catalog_context: &str, ) -> String { let mut prompt = format!( - "You are pChronicle Assistant for local Agent run debugging. Use only data from the current run. Call tools when details are needed; do not invent facts. Missing measurements are not zero. Do not infer an error from arbitrary message text. Reconstructed events are derived from imported run data, not directly recorded events. Answer in the user's language, preferably in 3–7 concise bullets. Separate recorded facts from inference. Cite every inspected step with the internal marker [turn:ID]. Mention coverage or truncation when tool results report it.\n\nCurrent run analysis:\nsession={}\nstatus={}\nevent_origin={}\nstep_count={}\nevent_count={}\nerror_count={}\ntotal_tokens={}\nlatency_p95={}\nlatency_samples={}/{}\n\nquery_sql schema:\n{}", + "You are pChronicle Assistant for local Agent run debugging. Use only data from the current run. Call tools when details are needed; do not invent facts. Missing measurements are not zero. Do not infer an error from arbitrary message text. Answer in the user's language, preferably in 3–7 concise bullets. Separate recorded facts from inference. Cite every inspected step with the internal marker [turn:ID]. Mention coverage or truncation when tool results report it.\n\nCurrent run analysis:\nsession={}\nstatus={}\nstep_count={}\nerror_count={}\ntotal_tokens={}\nlatency_p95={}\nlatency_samples={}/{}\n\nquery_sql schema:\n{}", run.session_id, run.status, - analysis.event_provenance.as_str(), analysis.turn_count, - analysis.event_count, analysis.error_count, analysis .total_tokens @@ -1060,10 +1058,8 @@ pub fn unknown_tool_result(name: &str) -> String { pub fn format_analysis_result(analysis: &RunAnalysis) -> String { format!( - "event_provenance={} turns={} events={} tools={} explicit_errors={} tokens={} latency_p95={} latency_samples={}/{}\nsources={:?}\nkinds={:?}\nmodels={:?}\ntool_names={:?}", - analysis.event_provenance.as_str(), + "turns={} tools={} explicit_errors={} tokens={} latency_p95={} latency_samples={}/{}\nsources={:?}\nkinds={:?}\nmodels={:?}\ntool_names={:?}", analysis.turn_count, - analysis.event_count, analysis.tool_call_count, analysis.error_count, analysis @@ -1412,7 +1408,7 @@ mod tests { fn sample_run(session: &str, run_id: Option<&str>) -> RunSummary { RunSummary { dataset: "captures".into(), - file: "events.lance".into(), + file: "storyline".into(), run_id: run_id.map(str::to_string), agent_id: "agent".into(), model_name: None, @@ -1420,7 +1416,6 @@ mod tests { root_session_id: None, path: String::new(), row_count: 1, - duplicate_event_ids: 0, status: "completed".into(), format: None, } @@ -2100,8 +2095,6 @@ mod tests { fn sample_analysis() -> RunAnalysis { RunAnalysis { run: sample_run("s-a", Some("r1")), - event_provenance: crate::model::EventProvenance::Canonical, - event_count: 3, turn_count: 3, tool_call_count: 0, error_count: 0, @@ -2140,7 +2133,6 @@ mod tests { completion_tokens: None, total_tokens: None, tool_names: Vec::new(), - event_seqs: Vec::new(), has_error: false, }, turn: StorylineTurn { @@ -2159,8 +2151,6 @@ mod tests { extra: None, }, wire_tool_calls: Vec::new(), - event_provenance: crate::model::EventProvenance::Canonical, - events: Vec::new(), } } diff --git a/pchronicle-web/src/analysis.rs b/pchronicle-web/src/analysis.rs index 76cc5ef36..cf30eb387 100644 --- a/pchronicle-web/src/analysis.rs +++ b/pchronicle-web/src/analysis.rs @@ -2501,7 +2501,6 @@ mod tests { root_session_id: Some("root-a".into()), path: "agent/root-a/session-a".into(), row_count: 1, - duplicate_event_ids: 0, status: "ok".into(), format: None, }, @@ -2874,7 +2873,6 @@ mod tests { root_session_id: Some("root-a".into()), path: "agent/root-a/session-a".into(), row_count: 1, - duplicate_event_ids: 0, status: "ok".into(), format: None, }, diff --git a/pchronicle-web/src/analysis_agent.rs b/pchronicle-web/src/analysis_agent.rs index 656557820..1a8f6c3ac 100644 --- a/pchronicle-web/src/analysis_agent.rs +++ b/pchronicle-web/src/analysis_agent.rs @@ -1444,7 +1444,6 @@ mod tests { root_session_id: Some("root-1".into()), path: "private/internal/path".into(), row_count: 99, - duplicate_event_ids: 3, status: "private-status".into(), format: None, }, diff --git a/pchronicle-web/src/analysis_session.rs b/pchronicle-web/src/analysis_session.rs index b8e2c1f76..5beb0bf70 100644 --- a/pchronicle-web/src/analysis_session.rs +++ b/pchronicle-web/src/analysis_session.rs @@ -2451,7 +2451,6 @@ mod tests { root_session_id: Some("root-a".into()), path: format!("agent/root-a/{id}"), row_count: 1, - duplicate_event_ids: 0, status: "ok".into(), format: None, } diff --git a/pchronicle-web/src/chat_view.rs b/pchronicle-web/src/chat_view.rs index 14574d410..46a1e19dc 100644 --- a/pchronicle-web/src/chat_view.rs +++ b/pchronicle-web/src/chat_view.rs @@ -99,7 +99,6 @@ pub fn prompt_turn(turn: &TurnSummary) -> Option { user.completion_tokens = None; user.total_tokens = None; user.tool_names.clear(); - user.event_seqs.clear(); user.has_error = false; user.user_prompt = None; Some(user) @@ -167,7 +166,6 @@ mod tests { completion_tokens: None, total_tokens: None, tool_names: Vec::new(), - event_seqs: Vec::new(), has_error: false, } } diff --git a/pchronicle-web/src/components.rs b/pchronicle-web/src/components.rs index aa8b04a63..a27ba6f7d 100644 --- a/pchronicle-web/src/components.rs +++ b/pchronicle-web/src/components.rs @@ -10,8 +10,7 @@ use crate::chat_view::{ }; use crate::json_value::{JsonValue, is_structured_json}; use crate::model::{ - EventProvenance, QueryEvidence, StorylineTurn, TurnDetail, TurnSummary, WireToolCall, - extract_message_text, + QueryEvidence, StorylineTurn, TurnDetail, TurnSummary, WireToolCall, extract_message_text, }; #[component] @@ -283,22 +282,8 @@ fn span_from_entries( let call_id = entries .iter() .find_map(|turn| turn.call_id.clone().filter(|value| !value.is_empty())); - let seqs = entries - .iter() - .flat_map(|turn| turn.event_seqs.iter().copied()) - .collect::>(); - let (first_seq, last_seq) = if seqs.is_empty() { - let first = fallback_index as u64; - ( - first, - first.saturating_add(entries.len().saturating_sub(1) as u64), - ) - } else { - ( - seqs.iter().copied().min().unwrap_or(0), - seqs.iter().copied().max().unwrap_or(0), - ) - }; + let first_seq = fallback_index as u64; + let last_seq = first_seq.saturating_add(entries.len().saturating_sub(1) as u64); CompactSpanGroup { key, label, @@ -562,27 +547,11 @@ fn session_index_map(turns: &[TurnSummary]) -> HashMap { } fn session_axis_len(turns: &[TurnSummary]) -> usize { - if turns.iter().any(|turn| !turn.event_seqs.is_empty()) { - turns - .iter() - .flat_map(|turn| turn.event_seqs.iter().copied()) - .max() - .map(|seq| seq as usize + 1) - .unwrap_or(turns.len()) - .max(1) - } else { - turns.len().max(1) - } + turns.len().max(1) } -fn turn_session_span(turn: &TurnSummary, session_index: usize) -> (usize, usize) { - match ( - turn.event_seqs.iter().copied().min(), - turn.event_seqs.iter().copied().max(), - ) { - (Some(first), Some(last)) => (first as usize, last as usize), - _ => (session_index, session_index), - } +fn turn_session_span(_turn: &TurnSummary, session_index: usize) -> (usize, usize) { + (session_index, session_index) } fn seq_bars( @@ -807,10 +776,6 @@ pub fn TrajectoryView( let session_index = session_index_map(&turns); let axis_len = session_axis_len(&turns); let root_bars = seq_bars(&turns, &session_index, axis_len); - let total_refs = turns - .iter() - .map(|turn| turn.event_seqs.len()) - .sum::(); let total_tools = turns .iter() .map(|turn| turn.tool_names.len()) @@ -854,11 +819,11 @@ pub fn TrajectoryView( }); let root_caption = sequence_caption(0, axis_len.saturating_sub(1) as u64); rsx! { div { class, - div { class: "span-summary", span { strong { "{groups.len()} {noun}" } " · {total_refs} event references" } span { "Sequence window 0 — {axis_len.saturating_sub(1)}" } } + div { class: "span-summary", span { strong { "{groups.len()} {noun}" } " · {turns.len()} steps" } span { "Sequence window 0 — {axis_len.saturating_sub(1)}" } } div { class: "{table_class}", role: "tree", aria_label: "Run step hierarchy", div { class: "span-sticky-chrome", div { class: "span-table-head", div { "Structure" } div { "Overview" } div { class: "span-axis-head", span { "Sequence / coverage" } div { class: "span-axis-ticks", span { "0" } span { "25%" } span { "50%" } span { "75%" } span { "{axis_len.saturating_sub(1)}" } } } div { "Details" } } - div { class: "trace-root-summary", div { class: "span-structure root", div { div { class: "span-structure-title", strong { "run" } span { "{groups.len()} {noun}" } } div { class: "root-composition", if !root_meta.is_empty() { span { title: "{root_meta}", "{root_meta}" } } for modality in root_modalities { span { class: "modality-chip {modality}", "{modality}" } } } } } div { class: "span-row-copy root-copy" } OccupancyTrack { bars: root_bars.clone(), expose_range, focus_left, caption: root_caption, title: "Run coverage · {turns.len()} steps", exposed_ids: exposed_ids.clone(), expanded_turn_id, hovered_ids: hover_ids.clone() } div { class: "span-evidence-count", if total_refs > 0 { strong { "{total_refs} events" } } if total_tools > 0 { span { "{root_tool_label}" } } if !embedded { if let Some(id) = root_drawer_id { button { class: "pc2-conversation-drawer-button pc2-run-agenticmd-button", title: "Open run as AgenticMD", aria_label: "Open run as AgenticMD", onclick: move |event| { event.prevent_default(); event.stop_propagation(); on_open_drawer.call((id, "Run".to_string(), root_drawer_ids.clone())); }, "↗" } } } } } + div { class: "trace-root-summary", div { class: "span-structure root", div { div { class: "span-structure-title", strong { "run" } span { "{groups.len()} {noun}" } } div { class: "root-composition", if !root_meta.is_empty() { span { title: "{root_meta}", "{root_meta}" } } for modality in root_modalities { span { class: "modality-chip {modality}", "{modality}" } } } } } div { class: "span-row-copy root-copy" } OccupancyTrack { bars: root_bars.clone(), expose_range, focus_left, caption: root_caption, title: "Run coverage · {turns.len()} steps", exposed_ids: exposed_ids.clone(), expanded_turn_id, hovered_ids: hover_ids.clone() } div { class: "span-evidence-count", if total_tools > 0 { span { "{root_tool_label}" } } if !embedded { if let Some(id) = root_drawer_id { button { class: "pc2-conversation-drawer-button pc2-run-agenticmd-button", title: "Open run as AgenticMD", aria_label: "Open run as AgenticMD", onclick: move |event| { event.prevent_default(); event.stop_propagation(); on_open_drawer.call((id, "Run".to_string(), root_drawer_ids.clone())); }, "↗" } } } } } } div { class: "span-children", for group in groups { CompactSpanRow { @@ -906,11 +871,6 @@ fn CompactSpanRow( on_hover: EventHandler>, #[props(default)] query: String, ) -> Element { - let event_refs = group - .entries - .iter() - .map(|turn| turn.event_seqs.len()) - .sum::(); let preview = group.overview.clone(); let diagnostic = group_diagnostic(&group.entries); let modalities = summary_modalities(&group.entries); @@ -958,7 +918,7 @@ fn CompactSpanRow( if !composition_label(&group.entries).is_empty() { span { class: "summary-chip", "{composition_label(&group.entries)}" } } if !tool_summary.is_empty() { span { class: "summary-chip tool", title: "{tool_summary}", "{tool_summary}" } } for modality in modalities { span { class: "modality-chip {modality}", "{modality}" } } - if event_refs > 0 { span { class: "summary-chip event", "{event_refs} events" } } + } } } div { class: "span-row-copy", @@ -966,7 +926,7 @@ fn CompactSpanRow( if row_open { span { class: "span-row-diagnostic", title: "{diagnostic}", "{diagnostic}" } } } OccupancyTrack { bars, expose_range, focus_left, caption: caption.clone(), title: "{caption} · {meta}", exposed_ids, expanded_turn_id, hovered_ids } - div { class: "span-evidence-count", if event_refs > 0 { span { class: "span-count-chip event", "{event_refs} events" } } } + div { class: "span-evidence-count", } } if !embedded { if let Some(id) = drawer_id { @@ -1042,7 +1002,6 @@ fn CompactTurnRow( let expanded_facts = turn_expanded_facts(&turn); let tool_count = turn.tool_names.len(); let tool_label = format_tool_count(tool_count); - let event_count = turn.event_seqs.len(); if id < 0 { return rsx! { div { class: "compact-turn synthetic-prompt", @@ -1054,10 +1013,10 @@ fn CompactTurnRow( }; } if embedded { - return rsx! { button { class: "compact-turn pc2-embedded-turn", onclick: move |_| on_turn.call(id), span { class: "compact-turn-chevron" } span { class: "pc2-role {turn.source}", "{turn.source}" } code { "#{id}" } span { class: "compact-kind", "{kind}" } span { class: "compact-preview", title: "{preview}", HighlightedText { text: preview.clone(), query: query.clone() } } span { class: "compact-turn-stats", if tool_count > 0 { span { "{tool_label}" } } span { "{event_count} events" } } } }; + return rsx! { button { class: "compact-turn pc2-embedded-turn", onclick: move |_| on_turn.call(id), span { class: "compact-turn-chevron" } span { class: "pc2-role {turn.source}", "{turn.source}" } code { "#{id}" } span { class: "compact-kind", "{kind}" } span { class: "compact-preview", title: "{preview}", HighlightedText { text: preview.clone(), query: query.clone() } } span { class: "compact-turn-stats", if tool_count > 0 { span { "{tool_label}" } } } } }; } rsx! { details { class: if expanded { "compact-turn selected" } else { "compact-turn" }, open: expanded, - summary { aria_label: "Expand {turn.source} step {id}", onclick: move |event| { event.prevent_default(); on_turn.call(id); }, span { class: "compact-turn-chevron" } span { class: "pc2-role {turn.source}", "{turn.source}" } code { "#{id}" } if expanded { span { class: "compact-kind", "{expanded_facts}" } } else { span { class: "compact-kind", "{kind}" } span { class: "compact-preview", title: "{preview}", HighlightedText { text: preview.clone(), query: query.clone() } } span { class: "compact-turn-stats", if !collapsed_meta.is_empty() { span { "{collapsed_meta}" } } if tool_count > 0 { span { "{tool_label}" } } span { "{event_count} events" } } } } + summary { aria_label: "Expand {turn.source} step {id}", onclick: move |event| { event.prevent_default(); on_turn.call(id); }, span { class: "compact-turn-chevron" } span { class: "pc2-role {turn.source}", "{turn.source}" } code { "#{id}" } if expanded { span { class: "compact-kind", "{expanded_facts}" } } else { span { class: "compact-kind", "{kind}" } span { class: "compact-preview", title: "{preview}", HighlightedText { text: preview.clone(), query: query.clone() } } span { class: "compact-turn-stats", if !collapsed_meta.is_empty() { span { "{collapsed_meta}" } } if tool_count > 0 { span { "{tool_label}" } } } } } if expanded { div { class: "compact-turn-body pc2-inline-detail", if loading { div { class: "pc2-inline-loading", span { class: "spinner" } "Loading full step…" } } else if let Some(value) = detail.filter(|value| value.summary.id == id) { InlineTurnDetail { value, query: query.clone() } } else { div { class: "pc2-inline-unavailable", "Details are unavailable for this step." } } } } } } } @@ -1092,11 +1051,6 @@ fn InlineTurnDetail(value: TurnDetail, #[props(default)] query: String) -> Eleme } else { Vec::new() }; - let events = serde_json::to_value(&value.events).unwrap_or(Value::Array(Vec::new())); - let event_block_title = match value.event_provenance { - EventProvenance::Canonical => crate::terminology::RECORDED_EVENTS, - EventProvenance::SyntheticFromStoryline => crate::terminology::RECONSTRUCTED_EVENTS, - }; let has_any_tool_calls = !embedded_from_message.is_empty() || !deduped_wire_calls.is_empty() || !native_tool_calls.is_empty(); @@ -1113,9 +1067,7 @@ fn InlineTurnDetail(value: TurnDetail, #[props(default)] query: String) -> Eleme if value.summary.prompt_tokens.is_some() || value.summary.completion_tokens.is_some() { Fact { label: "Token split", value: format!("{} in · {} out", optional_u64(value.summary.prompt_tokens), optional_u64(value.summary.completion_tokens)) } } - if !value.events.is_empty() { - Fact { label: "Events", value: value.events.len().to_string() } - } + if !embedded_from_message.is_empty() { ToolCallCards { calls: embedded_from_message, observation: value.turn.observation.clone() } } else if !deduped_wire_calls.is_empty() { @@ -1156,9 +1108,7 @@ fn InlineTurnDetail(value: TurnDetail, #[props(default)] query: String) -> Eleme InlineSection { title: "Observation", ObservationChips { value: observation.clone() } ObservationBlock { value: observation, tone: "generic" } } } } - if !value.events.is_empty() { - InlineSection { title: event_block_title, JsonValue { value: events } } - } + if let Some(extra) = value.turn.extra.clone() { InlineSection { title: "Extra", JsonValue { value: extra } } } @@ -1697,7 +1647,7 @@ mod tests { ); } - fn turn(id: i64, source: &str, seqs: &[u64]) -> TurnSummary { + fn turn(id: i64, source: &str) -> TurnSummary { TurnSummary { id, source: source.into(), @@ -1715,18 +1665,13 @@ mod tests { completion_tokens: None, total_tokens: None, tool_names: Vec::new(), - event_seqs: seqs.to_vec(), has_error: false, } } #[test] fn chats_keep_one_timeline_row_for_a_user_and_following_agents() { - let turns = vec![ - turn(1, "user", &[0]), - turn(2, "agent", &[2, 5]), - turn(3, "agent", &[6]), - ]; + let turns = vec![turn(1, "user"), turn(2, "agent"), turn(3, "agent")]; let groups = chat_span_groups(&turns); assert_eq!(groups.len(), 1); assert_eq!(groups[0].label, "Conversation 1"); @@ -1739,7 +1684,7 @@ mod tests { .collect::>(), vec![1, 2, 3] ); - assert_eq!((groups[0].first_seq, groups[0].last_seq), (0, 6)); + assert_eq!((groups[0].first_seq, groups[0].last_seq), (0, 2)); let session_index = session_index_map(&turns); let bars = seq_bars(&groups[0].entries, &session_index, session_axis_len(&turns)); assert_eq!( @@ -1747,19 +1692,19 @@ mod tests { vec!["user", "agent", "agent"] ); assert!((bars[0].left - 0.0).abs() < 1e-9); - assert!((bars[0].width - 100.0 / 7.0).abs() < 1e-9); - assert!((bars[1].left - 200.0 / 7.0).abs() < 1e-9); - assert!((bars[1].width - 400.0 / 7.0).abs() < 1e-9); - assert!((bars[2].left - 600.0 / 7.0).abs() < 1e-9); + assert!((bars[0].width - 100.0 / 3.0).abs() < 1e-9); + assert!((bars[1].left - 100.0 / 3.0).abs() < 1e-9); + assert!((bars[1].width - 100.0 / 3.0).abs() < 1e-9); + assert!((bars[2].left - 200.0 / 3.0).abs() < 1e-9); } #[test] fn chat_structure_uses_user_chars_and_union_modalities() { - let mut user = turn(1, "user", &[0]); + let mut user = turn(1, "user"); user.preview = "Please continue".into(); user.char_count = 15; user.modalities = vec!["text".into()]; - let mut agent = turn(2, "agent", &[1]); + let mut agent = turn(2, "agent"); agent.preview = "ls".into(); agent.char_count = 80; agent.modalities = vec!["text".into(), "tool_call".into()]; @@ -1809,7 +1754,7 @@ mod tests { #[test] fn chat_without_user_keeps_chat_kind_and_no_user_overview() { - let mut agent = turn(2, "agent", &[0]); + let mut agent = turn(2, "agent"); agent.modalities = vec!["text".into()]; agent.char_count = 12; let groups = chat_span_groups(&[agent]); @@ -1836,7 +1781,7 @@ mod tests { #[test] fn steps_empty_preview_reads_as_no_text() { - let mut turn = turn(3, "agent", &[0]); + let mut turn = turn(3, "agent"); turn.preview.clear(); let groups = step_span_groups(&[turn]); assert_eq!(groups[0].kind_chip, "agent"); @@ -1845,10 +1790,10 @@ mod tests { #[test] fn expanded_chat_overview_is_diagnostics_not_user_text() { - let mut user = turn(1, "user", &[0]); + let mut user = turn(1, "user"); user.preview = "Please continue".into(); user.total_tokens = Some(100); - let mut agent = turn(2, "agent", &[1]); + let mut agent = turn(2, "agent"); agent.preview = "running ls".into(); agent.model_name = Some("glm".into()); agent.latency_ms = Some(2020.0); @@ -1874,10 +1819,10 @@ mod tests { #[test] fn occupancy_marks_split_exposed_range_from_expanded_turn() { let turns = vec![ - turn(1, "user", &[0]), - turn(2, "agent", &[1]), - turn(3, "user", &[2]), - turn(4, "agent", &[3]), + turn(1, "user"), + turn(2, "agent"), + turn(3, "user"), + turn(4, "agent"), ]; let session_index = session_index_map(&turns); let bars = seq_bars(&turns, &session_index, session_axis_len(&turns)); @@ -1901,10 +1846,10 @@ mod tests { #[test] fn sequence_axis_keeps_session_relative_position_and_type_colors() { let turns = vec![ - turn(1, "system", &[]), - turn(2, "user", &[]), - turn(3, "agent", &[]), - turn(4, "user", &[]), + turn(1, "system"), + turn(2, "user"), + turn(3, "agent"), + turn(4, "user"), ]; let session_index = session_index_map(&turns); let axis_len = session_axis_len(&turns); @@ -1924,11 +1869,7 @@ mod tests { #[test] fn source_filter_keeps_chat_members() { - let turns = vec![ - turn(1, "user", &[0]), - turn(2, "agent", &[2]), - turn(3, "system", &[4]), - ]; + let turns = vec![turn(1, "user"), turn(2, "agent"), turn(3, "system")]; let visible = chat_span_groups(&turns) .into_iter() .filter(|group| chat_row_visible(&group.entries, "agent", "")) @@ -1947,11 +1888,11 @@ mod tests { #[test] fn steps_keep_one_timeline_row_per_turn() { - let turns = vec![turn(1, "user", &[0]), turn(2, "agent", &[3])]; + let turns = vec![turn(1, "user"), turn(2, "agent")]; let groups = step_span_groups(&turns); assert_eq!(groups.len(), 2); assert_eq!(groups[0].label, "#1"); - assert_eq!(groups[1].first_seq, 3); + assert_eq!(groups[1].first_seq, 1); } #[test] diff --git a/pchronicle-web/src/copilot_sessions.rs b/pchronicle-web/src/copilot_sessions.rs index 44a534de0..3043c1807 100644 --- a/pchronicle-web/src/copilot_sessions.rs +++ b/pchronicle-web/src/copilot_sessions.rs @@ -294,7 +294,7 @@ mod tests { fn sample_run(session: &str) -> RunSummary { RunSummary { dataset: "captures".into(), - file: "events.lance".into(), + file: "storyline".into(), run_id: Some(session.into()), agent_id: "agent".into(), model_name: None, @@ -302,7 +302,6 @@ mod tests { root_session_id: None, path: String::new(), row_count: 1, - duplicate_event_ids: 0, status: "completed".into(), format: None, } diff --git a/pchronicle-web/src/model.rs b/pchronicle-web/src/model.rs index b63c2c258..3eec545b0 100644 --- a/pchronicle-web/src/model.rs +++ b/pchronicle-web/src/model.rs @@ -1,5 +1,3 @@ -use std::collections::BTreeMap; - use serde::{Deserialize, Deserializer, Serialize}; use serde_json::Value; @@ -51,7 +49,6 @@ pub struct RunSummary { pub root_session_id: Option, pub path: String, pub row_count: usize, - pub duplicate_event_ids: usize, pub status: String, #[serde(default)] pub format: Option, @@ -461,46 +458,6 @@ pub struct WireToolCall { pub result: Option, } -#[derive(Clone, Debug, PartialEq, Deserialize, Serialize)] -pub struct EventRecord { - pub seq: u64, - pub source: String, - pub kind: String, - #[serde(default, deserialize_with = "deserialize_optional_timestamp")] - pub timestamp: Option, - pub event_id: Option, - pub call_id: Option, - pub trace_id: Option, - pub producer: Option, - pub payload: Value, - #[serde(flatten)] - pub rest: BTreeMap, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Deserialize, Serialize)] -#[serde(rename_all = "snake_case")] -pub enum EventProvenance { - Canonical, - SyntheticFromStoryline, -} - -impl EventProvenance { - pub const fn as_str(self) -> &'static str { - match self { - Self::Canonical => "canonical", - Self::SyntheticFromStoryline => "synthetic_from_storyline", - } - } - - /// Plain-language label for user-facing summaries. - pub const fn display_label(self) -> &'static str { - match self { - Self::Canonical => crate::terminology::RECORDED_EVENTS, - Self::SyntheticFromStoryline => crate::terminology::RECONSTRUCTED_EVENTS, - } - } -} - #[derive(Clone, Debug, PartialEq, Deserialize)] pub struct MetricStats { pub sample_count: usize, @@ -542,8 +499,6 @@ pub struct HistogramBucket { #[derive(Clone, Debug, PartialEq, Deserialize)] pub struct RunAnalysis { pub run: RunSummary, - pub event_provenance: EventProvenance, - pub event_count: usize, pub turn_count: usize, pub tool_call_count: usize, pub error_count: usize, @@ -584,7 +539,6 @@ pub struct TurnSummary { pub completion_tokens: Option, pub total_tokens: Option, pub tool_names: Vec, - pub event_seqs: Vec, pub has_error: bool, } @@ -613,8 +567,6 @@ pub struct TurnDetail { pub summary: TurnSummary, pub turn: StorylineTurn, pub wire_tool_calls: Vec, - pub event_provenance: EventProvenance, - pub events: Vec, } #[derive(Clone, Debug, PartialEq, Deserialize, Serialize)] @@ -642,7 +594,6 @@ mod tests { root_session_id: Some("root+1".into()), path: "agent one/root+1/subagents/s-1".into(), row_count: 2, - duplicate_event_ids: 0, status: "ok".into(), format: None, }; @@ -664,7 +615,7 @@ mod tests { }, "records": [{ "dataset": "captures", - "file": "capture-comparison/events.lance", + "file": "capture-comparison/storyline", "document_id": "session-1", "run_id": null, "agent_id": "capture-comparison", @@ -673,18 +624,18 @@ mod tests { "root_session_id": null, "path": "captures/capture-comparison/session-1", "row_count": 6, - "duplicate_event_ids": 0, + "status": "completed", "model": "test-model" }], "path_index": [] })) - .expect("canonical Gateway runs may not have a run id"); + .expect("Storyline runs may not have a run id"); assert_eq!(page.records[0].run.run_id, None); assert_eq!( page.records[0].run.query(), - "dataset=captures&file=capture-comparison%2Fevents.lance&agent_id=capture-comparison&session_id=session-1" + "dataset=captures&file=capture-comparison%2Fstoryline&agent_id=capture-comparison&session_id=session-1" ); } @@ -695,7 +646,7 @@ mod tests { "dataset": "records", "file": "data.lance", "run_id": null, "agent_id": "compact-jsonl", "model_name": null, "session_id": "row-1", "root_session_id": null, "path": "records/data.lance/row-1", - "row_count": 1, "duplicate_event_ids": 0, "status": "record", + "row_count": 1, "status": "record", "format": "compact-jsonl/v1" }, "record": {"id": "row-1", "nested": {"ok": true}} @@ -705,37 +656,6 @@ mod tests { assert_eq!(detail.record["nested"]["ok"], true); } - #[test] - fn turn_detail_accepts_numeric_storyline_timestamps() { - let turn: StorylineTurn = serde_json::from_value(serde_json::json!({ - "id": 85, - "src": "user", - "msg": "hello", - "ts": 1785310111 - })) - .expect("explorer detail preserves numeric Storyline timestamps"); - assert_eq!(turn.timestamp.as_deref(), Some("1785310111")); - - let event: EventRecord = serde_json::from_value(serde_json::json!({ - "seq": 1, - "source": "gateway", - "kind": "llm.request", - "timestamp": 1785310111, - "payload": {} - })) - .expect("linked events may carry numeric timestamps"); - assert_eq!(event.timestamp.as_deref(), Some("1785310111")); - - let rfc3339: StorylineTurn = serde_json::from_value(serde_json::json!({ - "id": 1, - "src": "agent", - "msg": "ok", - "ts": "2026-07-29T00:00:00Z" - })) - .unwrap(); - assert_eq!(rfc3339.timestamp.as_deref(), Some("2026-07-29T00:00:00Z")); - } - #[test] fn extract_message_text_prefers_text_field_in_object() { let message = serde_json::json!({ diff --git a/pchronicle-web/src/result_explorer.rs b/pchronicle-web/src/result_explorer.rs index 1eac663ab..0a3e164eb 100644 --- a/pchronicle-web/src/result_explorer.rs +++ b/pchronicle-web/src/result_explorer.rs @@ -664,7 +664,7 @@ mod tests { fn nullable_run_and_root_coordinates_still_create_links() { let row = json!({ "dataset":"captures", - "_file_":"gateway/events.lance", + "_file_":"gateway/storyline", "agent_id":"gateway", "session_id":"session-a", "run_id": null, diff --git a/pchronicle-web/src/terminology.rs b/pchronicle-web/src/terminology.rs index a43bd9a8d..510797c5f 100644 --- a/pchronicle-web/src/terminology.rs +++ b/pchronicle-web/src/terminology.rs @@ -1,7 +1,6 @@ //! User-facing product terminology. //! -//! Keep storage and API names such as `Storyline`, `CanonicalEvent`, and -//! `CatalogEventProvenance` in their owning modules. The UI uses the simpler +//! Keep storage and API names such as `StorylineDocument` in their owning modules. The UI uses the simpler //! vocabulary below so implementation details do not leak into navigation and //! everyday workflows. @@ -12,5 +11,3 @@ pub const STORAGE: &str = "Storage"; pub const ASSISTANT: &str = "Assistant"; pub const TIMELINE: &str = "Timeline"; pub const STEPS: &str = "Steps"; -pub const RECORDED_EVENTS: &str = "Recorded events"; -pub const RECONSTRUCTED_EVENTS: &str = "Reconstructed events"; diff --git a/pchronicle-web/src/workspace.rs b/pchronicle-web/src/workspace.rs index 2f230594f..b1fc504c8 100644 --- a/pchronicle-web/src/workspace.rs +++ b/pchronicle-web/src/workspace.rs @@ -134,7 +134,6 @@ fn run_from_search(search: &str) -> Option { root_session_id: query_value(search, "root_session_id").filter(|value| !value.is_empty()), path: String::new(), row_count: 0, - duplicate_event_ids: 0, status: "loading".into(), format: None, }) @@ -230,7 +229,6 @@ pub fn App() -> Element { root_session_id: initial_root, path: String::new(), row_count: 0, - duplicate_event_ids: 0, status: "loading".into(), format: None, }); @@ -2128,7 +2126,7 @@ fn RunDetailWorkspace( #[component] fn MetricsStrip(analysis: RunAnalysis) -> Element { rsx! { div { class: "pc2-metrics", - Metric { label: "Steps", value: analysis.turn_count.to_string(), detail: format!("{} {}", analysis.event_count, analysis.event_provenance.display_label().to_ascii_lowercase()) } + Metric { label: "Steps", value: analysis.turn_count.to_string(), detail: "Storyline steps" } Metric { label: "Tools", value: analysis.tool_call_count.to_string(), detail: format!("{} tool names", analysis.tools.len()) } Metric { label: "Explicit errors", value: analysis.error_count.to_string(), detail: "Captured signals only" } Metric { label: "Tokens", value: analysis.total_tokens.map(|value| value.to_string()).unwrap_or_else(|| "—".into()), detail: format!("in {} · out {}", optional_u64(analysis.prompt_tokens), optional_u64(analysis.completion_tokens)) } @@ -2283,7 +2281,6 @@ fn OverviewAnalysis(analysis: RunAnalysis, turns: Vec) -> Element { code { "{elapsed}" } div { class: "pc2-run-span-meta", span { strong { "{analysis.models.len()}" } " models" } - span { strong { "{analysis.event_count}" } " {analysis.event_provenance.display_label().to_ascii_lowercase()}" } span { strong { "{analysis.error_count}" } " steps with explicit errors" } } } @@ -3442,7 +3439,7 @@ mod tests { "call_id": null, "preview": "ready-step-before-statistics", "model_name": null, "latency_ms": null, "ttft_ms": null, "prompt_tokens": null, "completion_tokens": null, "total_tokens": null, "tool_names": [], - "event_seqs": [], "has_error": false + "has_error": false })) .unwrap(); rsx! { RunDetailWorkspace { @@ -3579,7 +3576,6 @@ mod tests { root_session_id: Some("root".into()), path: path.into(), row_count: 1, - duplicate_event_ids: 0, status: "completed".into(), format: None, } diff --git a/pyproject.toml b/pyproject.toml index a62d30cbc..fa40ac96f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "persisting" version = "0.3.0" -description = "Durable Agent trajectory capture, storage, and query infrastructure" +description = "Durable Agent trajectory processing, storage, and query infrastructure" readme = "README.md" requires-python = ">=3.10" license = "Apache-2.0" diff --git a/scripts/build-docs.py b/scripts/build-docs.py index cfaaf58bd..ed7184f46 100644 --- a/scripts/build-docs.py +++ b/scripts/build-docs.py @@ -1,37 +1,49 @@ #!/usr/bin/env python3 """Build both native Zensical navigation trees into one bilingual site.""" -from pathlib import Path + import shutil import subprocess import sys import tempfile +from pathlib import Path -DOCS = Path(__file__).resolve().parents[1] / 'docs' +DOCS = Path(__file__).resolve().parents[1] / "docs" def build() -> None: - zensical = shutil.which('zensical') or str(Path(sys.executable).with_name('zensical')) - subprocess.run([zensical, 'build', '--strict'], cwd=DOCS, check=True) + zensical = shutil.which("zensical") or str(Path(sys.executable).with_name("zensical")) + subprocess.run([zensical, "build", "--strict"], cwd=DOCS, check=True) # Zensical has one canonical language/navigation per build. Reuse the same # content and search index, rendering Chinese HTML with its native zh theme. - config = (DOCS / 'zensical.toml').read_text() - before, nav = config.split('nav = [', 1) - nav, after = nav.split('\n[project.theme]', 1) + config = (DOCS / "zensical.toml").read_text() + before, nav = config.split("nav = [", 1) + nav, after = nav.split("\n[project.theme]", 1) nav = nav.replace('"en/', '"zh/') - for en, zh in {'Concepts': '概念', 'Guides': '指南', 'Design': '设计', 'Reference': '参考'}.items(): + for en, zh in { + "Concepts": "概念", + "Guides": "指南", + "Design": "设计", + "Reference": "参考", + }.items(): nav = nav.replace('"' + en + '"', '"' + zh + '"') - config = before + 'nav = [' + nav + '\n[project.theme]' + after - config = config.replace('language = "en"', 'language = "zh"').replace('homepage = "en/"', 'homepage = "zh/"') - with tempfile.TemporaryDirectory(prefix='.zensical-zh-', dir=DOCS) as temp: + config = before + "nav = [" + nav + "\n[project.theme]" + after + config = config.replace('language = "en"', 'language = "zh"').replace( + 'homepage = "en/"', 'homepage = "zh/"' + ) + with tempfile.TemporaryDirectory(prefix=".zensical-zh-", dir=DOCS) as temp: temp = Path(temp) config = config.replace('site_dir = "site"', f'site_dir = "{temp.name}/site"') - config = config.replace('[project]\n', f'[project]\ncache_dir = "{temp.name}/cache"\n', 1) - with tempfile.NamedTemporaryFile(mode='w', suffix='.toml', prefix='.zensical-zh-', dir=DOCS) as config_file: + config = config.replace("[project]\n", f'[project]\ncache_dir = "{temp.name}/cache"\n', 1) + with tempfile.NamedTemporaryFile( + mode="w", suffix=".toml", prefix=".zensical-zh-", dir=DOCS + ) as config_file: config_file.write(config) config_file.flush() - subprocess.run([zensical, 'build', '--strict', '-f', config_file.name], cwd=DOCS, check=True) - shutil.copytree(temp / 'site/zh', DOCS / 'site/zh', dirs_exist_ok=True) + subprocess.run( + [zensical, "build", "--strict", "-f", config_file.name], cwd=DOCS, check=True + ) + shutil.copytree(temp / "site/zh", DOCS / "site/zh", dirs_exist_ok=True) -if __name__ == '__main__': +if __name__ == "__main__": build() diff --git a/scripts/check-docs.py b/scripts/check-docs.py index 9da380ab2..8cbb70422 100644 --- a/scripts/check-docs.py +++ b/scripts/check-docs.py @@ -1,66 +1,90 @@ #!/usr/bin/env python3 """Check generated docs links, images, locale navigation and Markdown rendering.""" + from html.parser import HTMLParser from pathlib import Path from urllib.parse import unquote, urlsplit -ROOT = Path(__file__).resolve().parents[1] / 'docs/site' +ROOT = Path(__file__).resolve().parents[1] / "docs/site" + class Page(HTMLParser): def __init__(self, path): super().__init__(convert_charrefs=True) - self.ids, self.links, self.language = set(), [], '' + self.ids, self.links, self.language = set(), [], "" self.feed(path.read_text()) def handle_starttag(self, tag, attrs): attrs = dict(attrs) - if 'id' in attrs: - self.ids.add(attrs['id']) - if tag == 'html': - self.language = attrs.get('lang', '') - for key in ('href', 'src'): + if "id" in attrs: + self.ids.add(attrs["id"]) + if tag == "html": + self.language = attrs.get("lang", "") + for key in ("href", "src"): if key in attrs: - self.links.append((tag, attrs[key], attrs.get('class', ''))) + self.links.append((tag, attrs[key], attrs.get("class", ""))) def check(): - paths = [ROOT / 'index.html', ROOT / '404.html', *ROOT.glob('en/**/*.html'), *ROOT.glob('zh/**/*.html')] + paths = [ + ROOT / "index.html", + ROOT / "404.html", + *ROOT.glob("en/**/*.html"), + *ROOT.glob("zh/**/*.html"), + ] pages = {p.resolve(): Page(p) for p in paths} issues = [] for path, page in pages.items(): rel = path.relative_to(ROOT) - locale = rel.parts[0] if rel.parts[0] in ('en', 'zh') else None + locale = rel.parts[0] if rel.parts[0] in ("en", "zh") else None if locale and page.language != locale: - issues.append(f'{rel}: html lang={page.language}, expected {locale}') + issues.append(f"{rel}: html lang={page.language}, expected {locale}") for tag, href, classes in page.links: url = urlsplit(href) if url.scheme or url.netloc: continue target = unquote(url.path) - if target.startswith('/Persisting/'): - target = target[len('/Persisting'):] - dest = ((ROOT / target.lstrip('/')) if target.startswith('/') else (path.parent / target)).resolve() if target else path + if target.startswith("/Persisting/"): + target = target[len("/Persisting") :] + dest = ( + ( + (ROOT / target.lstrip("/")) + if target.startswith("/") + else (path.parent / target) + ).resolve() + if target + else path + ) if dest.is_dir(): - dest /= 'index.html' - if locale and 'md-select__link' in classes and dest.is_relative_to(ROOT): + dest /= "index.html" + if locale and "md-select__link" in classes and dest.is_relative_to(ROOT): target_rel = dest.relative_to(ROOT) if target_rel.parts[1:] != rel.parts[1:]: - issues.append(f'{rel}: language selector loses current article: {href}') + issues.append(f"{rel}: language selector loses current article: {href}") if not dest.exists(): - issues.append(f'{rel}: missing {tag} {href}') + issues.append(f"{rel}: missing {tag} {href}") elif url.fragment and dest in pages and unquote(url.fragment) not in pages[dest].ids: - issues.append(f'{rel}: missing anchor {href}') - if locale and ('md-tabs__link' in classes or 'md-nav__link' in classes or 'md-logo' in classes) and dest.is_relative_to(ROOT): + issues.append(f"{rel}: missing anchor {href}") + if ( + locale + and ( + "md-tabs__link" in classes or "md-nav__link" in classes or "md-logo" in classes + ) + and dest.is_relative_to(ROOT) + ): dest_locale = dest.relative_to(ROOT).parts[0] - if dest_locale in ('en', 'zh') and dest_locale != locale: - issues.append(f'{rel}: navigation changes language: {href}') - for locale in ('en', 'zh'): - product = (ROOT / locale / 'pchronicle/get-started/index.html').read_text() - if 'class="admonition tip"' not in product or ' None: prefix = "/Persisting" if self.path == prefix or self.path.startswith(prefix + "/"): - self.path = self.path[len(prefix):] or "/" + self.path = self.path[len(prefix) :] or "/" def do_GET(self) -> None: # noqa: N802 - stdlib handler API self._strip_deploy_prefix() @@ -33,7 +34,9 @@ def main() -> None: parser.add_argument("--host", default="127.0.0.1") parser.add_argument("--port", type=int, default=3000) parser.add_argument("--directory", type=Path, default=Path("site")) - parser.add_argument("--watch", action="store_true", help="Rebuild both languages when docs change") + parser.add_argument( + "--watch", action="store_true", help="Rebuild both languages when docs change" + ) args = parser.parse_args() directory = args.directory.resolve() @@ -42,21 +45,26 @@ def main() -> None: if args.watch: docs = Path(__file__).resolve().parents[1] / "docs" + def watch(): def snapshot(): files = [docs / "zensical.toml"] for folder in ("src", "overrides"): files.extend(p for p in (docs / folder).rglob("*") if p.is_file()) return {str(p): p.stat().st_mtime_ns for p in files} + previous = snapshot() tick = threading.Event() while not tick.wait(1): current = snapshot() if current != previous: - result = subprocess.run([sys.executable, str(docs.parent / "scripts/build-docs.py")]) + result = subprocess.run( + [sys.executable, str(docs.parent / "scripts/build-docs.py")] + ) if result.returncode: print("Docs rebuild failed; see errors above.", flush=True) previous = current + threading.Thread(target=watch, daemon=True).start() handler = partial(PreviewHandler, directory=str(directory)) diff --git a/tests/regression/README.md b/tests/regression/README.md index f26940139..5aaac01bc 100644 --- a/tests/regression/README.md +++ b/tests/regression/README.md @@ -1,37 +1,5 @@ -# Large-scale regression tests +# Regression scenarios -**Repository-level black-box regressions against prebuilt real component -binaries.** - -Each scenario has its own directory and one executable `run.sh` entry point. -Scenarios launch those binaries, open loopback listeners, and inspect durable -output, so they are intentionally separate from the quick unit-test gate. They -do not compile Rust implicitly. - -Does not own Gateway, pChronicle, or SDK implementations. Directories containing -`.long-running` are skipped by the aggregate runner. - -## Run - -```bash -just regression -tests/regression/gateway-echo/run.sh -just gateway-fuzz -``` - -The Gateway fuzz suite is split into `formats`, `forwarding`, `storage`, and -`network-policy`; each has its own nested `run.sh` and `just gateway-fuzz-*` -entry. See [`gateway-fuzz/README.md`](gateway-fuzz/README.md) for the contract -matrix. - -Set `PERSISTING_KEEP_TEST_ARTIFACTS=1` to retain a scenario's temporary files -for diagnosis. The `gateway-echo` scenario records Python SDK results, -pChronicle canonical events, and their per-session comparison as separate -JSONL logs. - -## Links - -- [`gateway-echo`](gateway-echo/README.md) -- [`gateway-fuzz`](gateway-fuzz/README.md) -- [Gateway architecture](../../docs/src/en/pchronicle/guides/serve-gateway.md) -- [`persisting-gateway`](../../crates/persisting-gateway/README.md) +Run available repository scenarios with `just regression`. Each scenario has a +`run.sh` entry point and uses prebuilt component binaries. Directories marked +`.long-running` are excluded from the aggregate runner. diff --git a/tests/regression/gateway-echo/README.md b/tests/regression/gateway-echo/README.md deleted file mode 100644 index 5384e0cf1..000000000 --- a/tests/regression/gateway-echo/README.md +++ /dev/null @@ -1,50 +0,0 @@ -# Gateway Python SDK regression - -**Starts the real Gateway, pChronicle Warehouse, and deterministic Echo -upstream, then exercises Gateway through pinned official Python SDKs.** - -Owns this black-box scenario and its per-session comparison logs. Does not -compile Rust and does not own Gateway or pChronicle. - -| Client | Client protocol | Upstream path | -|---|---|---| -| OpenAI | Chat Completions | OpenAI Chat Completions through wildcard forwarding | -| OpenAI | Responses streaming | OpenAI Chat Completions through protocol rewriting | -| Anthropic | Messages | Native Anthropic Messages with a Base64 Echo response | -| Google Gen AI | Gemini | Native `generateContent` | - -## Run - -```bash -cargo build --release --locked -p persisting-pchronicle-cli --bin pchronicle -tests/regression/gateway-echo/run.sh -``` - -`run.sh` is intentionally only the portable scenario entry point. Each server -binds port zero and publishes its selected loopback address. Configuration, -SDK calls, durable queries, cleanup, and assertions live in `regression.py` -and its focused Python modules; process lifecycle and readiness helpers are -shared with `gateway-fuzz` through `tests/regression/gateway_harness.py`. - -By default it uses `target/release/pchronicle` for both `pchronicle echo` and -`pchronicle serve`. Set `PERSISTING_PCHRONICLE_BIN` to test another prebuilt -binary. - -The SDKs are installed by `uv` in an isolated environment from -`requirements.txt`; they are not dependencies of the Persisting wheel. - -The scenario creates these logs before comparing them: - -- `logs/client-results.jsonl`: normalized results observed by each Python SDK. -- `logs/events.jsonl`: canonical events queried from the pChronicle dataset. -- `logs/comparison.jsonl`: the per-session join and every comparison result. -- `logs/python-clients.log`: Python SDK stdout and stderr. - -Logs are retained automatically on failure. Set -`PERSISTING_KEEP_TEST_ARTIFACTS=1` to retain them after a successful run. - -## Links - -- [Regression tests](../README.md) -- [Gateway architecture](../../../docs/src/en/pchronicle/guides/serve-gateway.md) -- [`persisting-gateway`](../../../crates/persisting-gateway/README.md) diff --git a/tests/regression/gateway-echo/compare_logs.py b/tests/regression/gateway-echo/compare_logs.py deleted file mode 100644 index dc473f128..000000000 --- a/tests/regression/gateway-echo/compare_logs.py +++ /dev/null @@ -1,130 +0,0 @@ -#!/usr/bin/env python3 -"""Join Python SDK results with canonical Gateway events and compare them.""" - -from __future__ import annotations - -import argparse -import json -from collections import defaultdict -from pathlib import Path -from typing import Any - - -def read_jsonl(path: Path) -> list[dict[str, Any]]: - return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line] - - -def parts_text(parts: list[dict[str, Any]]) -> str: - return "".join(part.get("text", "") for part in parts if part.get("type") == "text") - - -def canonical_request_text(payload: dict[str, Any]) -> str: - messages = payload["llm_request"]["request"]["messages"] - return "".join( - parts_text(message.get("parts", [])) - for message in messages - if message.get("role") == "user" - ) - - -def canonical_response_text(payload: dict[str, Any]) -> str: - candidates = payload["llm_response"]["response"].get("candidates", []) - return "".join( - parts_text(candidate.get("message", {}).get("parts", [])) for candidate in candidates - ) - - -def compare_logs(clients_path: Path, events_path: Path, output_path: Path) -> tuple[int, int]: - clients = read_jsonl(clients_path) - events = read_jsonl(events_path) - by_session: dict[str, list[dict[str, Any]]] = defaultdict(list) - for event in events: - event["record"] = json.loads(event["payload_json"]) - by_session[event["session_id"]].append(event) - - expected_sessions = {entry["session_id"] for entry in clients} - checks: list[dict[str, Any]] = [] - for entry in clients: - session_id = entry["session_id"] - contract = entry["expected_capture"] - session_events = by_session.get(session_id, []) - by_kind = {event["kind"]: event for event in session_events} - request = by_kind.get("llm.request") - response = by_kind.get(contract["response_kind"]) - - assertions = { - "two_events": len(session_events) == 2, - "request_present": request is not None, - "response_present": response is not None, - } - if request is not None and response is not None: - request_payload = request["record"]["payload"] - response_payload = response["record"]["payload"] - captured_input = canonical_request_text(request_payload) - captured_output = canonical_response_text(response_payload) - assertions.update( - { - "call_id": bool(request["call_id"]) - and request["call_id"] == response["call_id"], - "model": request["model"] == entry["model"], - "protocol": request_payload["protocol"] == entry["protocol"], - "provider": request_payload["provider"] == contract["provider"], - "input": captured_input == entry["input"], - "output": captured_output == entry["output"], - "status": response_payload["status"] == 200, - "upstream_path": response_payload["http"]["url"].endswith( - contract["upstream_path"] - ), - "forward_to": ( - request_payload.get("forward_to") == contract["forward_to"] - and response_payload.get("forward_to") == contract["forward_to"] - ), - } - ) - - checks.append( - { - "session_id": session_id, - "client": entry["client"], - "sdk_version": entry["sdk_version"], - "protocol": entry["protocol"], - "client_input": entry["input"], - "captured_input": ( - canonical_request_text(request["record"]["payload"]) if request else None - ), - "client_output": entry["output"], - "captured_output": ( - canonical_response_text(response["record"]["payload"]) if response else None - ), - "assertions": assertions, - "matched": all(assertions.values()), - } - ) - - unexpected_sessions = set(by_session) - expected_sessions - output_path.parent.mkdir(parents=True, exist_ok=True) - with output_path.open("w", encoding="utf-8") as handle: - for check in checks: - json.dump(check, handle, ensure_ascii=False, sort_keys=True) - handle.write("\n") - - failures = [check for check in checks if not check["matched"]] - assert len(clients) == 4, f"expected 4 SDK calls, got {len(clients)}" - assert len(events) == 8, f"expected 8 canonical events, got {len(events)}" - assert not unexpected_sessions, f"unexpected captured sessions: {sorted(unexpected_sessions)}" - assert not failures, json.dumps(failures, ensure_ascii=False, indent=2) - return len(clients), len(events) - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument("--clients", type=Path, required=True) - parser.add_argument("--events", type=Path, required=True) - parser.add_argument("--output", type=Path, required=True) - args = parser.parse_args() - client_count, event_count = compare_logs(args.clients, args.events, args.output) - print(f"compared {client_count} SDK calls with {event_count} canonical events") - - -if __name__ == "__main__": - main() diff --git a/tests/regression/gateway-echo/python_clients.py b/tests/regression/gateway-echo/python_clients.py deleted file mode 100644 index c34647c34..000000000 --- a/tests/regression/gateway-echo/python_clients.py +++ /dev/null @@ -1,209 +0,0 @@ -#!/usr/bin/env python3 -"""Exercise Gateway through the official OpenAI, Anthropic, and Gemini SDKs.""" - -from __future__ import annotations - -import argparse -import base64 -import json -from pathlib import Path -from typing import Any - -import anthropic -import openai -from anthropic import Anthropic -from google import genai -from google.genai import types -from openai import OpenAI - - -def append_result(output: Path, result: dict[str, Any]) -> None: - with output.open("a", encoding="utf-8") as handle: - json.dump(result, handle, ensure_ascii=False, sort_keys=True) - handle.write("\n") - - -def result( - *, - client: str, - sdk_version: str, - protocol: str, - session_id: str, - model: str, - input_text: str, - output_text: str, - response_kind: str, - upstream_path: str, - forward_to: str | None = None, - streaming: bool = False, - response_model: str | None = None, -) -> dict[str, Any]: - return { - "client": client, - "sdk_version": sdk_version, - "protocol": protocol, - "session_id": session_id, - "model": model, - "input": input_text, - "output": output_text, - "streaming": streaming, - "response_model": response_model, - "expected_capture": { - "provider": {"openai": "openai", "anthropic": "anthropic", "google-genai": "gemini"}[ - client - ], - "response_kind": response_kind, - "upstream_path": upstream_path, - "forward_to": forward_to, - }, - } - - -def run_openai(gateway: str, output: Path) -> None: - client = OpenAI(api_key="regression", base_url=f"{gateway}/v1", max_retries=0) - try: - chat_session = "sdk-openai-chat" - chat_input = 'OpenAI chat: "quoted" line\n第二行🙂' - chat = client.chat.completions.create( - model="sdk-openai-chat-model", - messages=[{"role": "user", "content": chat_input}], - extra_headers={"x-persisting-session-id": chat_session}, - ) - append_result( - output, - result( - client="openai", - sdk_version=openai.__version__, - protocol="chat_completions", - session_id=chat_session, - model="sdk-openai-chat-model", - input_text=chat_input, - output_text=chat.choices[0].message.content or "", - response_model=chat.model, - response_kind="llm.response", - upstream_path="/v1/chat/completions", - forward_to="echo-openai", - ), - ) - - responses_session = "sdk-openai-responses-stream" - responses_input = "OpenAI Responses streaming payload" - chunks: list[str] = [] - with client.responses.stream( - model="sdk-openai-responses-model", - input=responses_input, - extra_headers={"x-persisting-session-id": responses_session}, - ) as stream: - for event in stream: - if event.type == "response.output_text.delta": - chunks.append(event.delta) - final_response = stream.get_final_response() - append_result( - output, - result( - client="openai", - sdk_version=openai.__version__, - protocol="responses", - session_id=responses_session, - model="sdk-openai-responses-model", - input_text=responses_input, - output_text="".join(chunks), - response_model=final_response.model, - response_kind="llm.response.stream", - upstream_path="/v1/chat/completions", - forward_to="echo-openai", - streaming=True, - ), - ) - finally: - client.close() - - -def run_anthropic(gateway: str, output: Path) -> None: - session_id = "sdk-anthropic-messages-base64" - input_text = "Anthropic Messages 中文/Base64" - client = Anthropic(api_key="regression", base_url=gateway, max_retries=0) - try: - message = client.messages.create( - model="sdk-anthropic-model", - max_tokens=64, - messages=[{"role": "user", "content": input_text}], - extra_headers={ - "x-persisting-session-id": session_id, - "x-persisting-echo-encoding": "base64", - }, - ) - output_text = "".join(block.text for block in message.content if block.type == "text") - assert output_text == base64.b64encode(input_text.encode()).decode(), output_text - append_result( - output, - result( - client="anthropic", - sdk_version=anthropic.__version__, - protocol="messages", - session_id=session_id, - model="sdk-anthropic-model", - input_text=input_text, - output_text=output_text, - response_model=message.model, - response_kind="llm.response", - upstream_path="/v1/messages", - ), - ) - finally: - client.close() - - -def run_google_genai(gateway: str, output: Path) -> None: - session_id = "sdk-google-genai" - input_text = "Gemini native generateContent payload" - client = genai.Client( - api_key="regression", - http_options=types.HttpOptions( - base_url=gateway, - api_version="v1beta", - headers={"x-persisting-session-id": session_id}, - ), - ) - try: - chat = client.chats.create(model="sdk-google-genai-model") - response = chat.send_message(input_text) - append_result( - output, - result( - client="google-genai", - sdk_version=genai.__version__, - protocol="gemini", - session_id=session_id, - model="sdk-google-genai-model", - input_text=input_text, - output_text=response.text or "", - response_model=response.model_version, - response_kind="llm.response", - upstream_path="/v1beta/models/sdk-google-genai-model:generateContent", - ), - ) - finally: - client.close() - - -def run_all(gateway: str, output: Path) -> None: - output.parent.mkdir(parents=True, exist_ok=True) - output.unlink(missing_ok=True) - gateway = gateway.rstrip("/") - run_openai(gateway, output) - run_anthropic(gateway, output) - run_google_genai(gateway, output) - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument("--gateway", required=True) - parser.add_argument("--output", type=Path, required=True) - args = parser.parse_args() - - run_all(args.gateway, args.output) - - -if __name__ == "__main__": - main() diff --git a/tests/regression/gateway-echo/regression.py b/tests/regression/gateway-echo/regression.py deleted file mode 100644 index 3d5869a31..000000000 --- a/tests/regression/gateway-echo/regression.py +++ /dev/null @@ -1,243 +0,0 @@ -#!/usr/bin/env python3 -"""Black-box Gateway regression driven entirely from Python.""" - -from __future__ import annotations - -import contextlib -import os -import shutil -import signal -import subprocess -import sys -import tempfile -from pathlib import Path -from typing import TextIO - -from compare_logs import compare_logs -from python_clients import run_all - -SCENARIO_DIR = Path(__file__).resolve().parent -REPO_ROOT = SCENARIO_DIR.parents[2] -sys.path.insert(0, str(SCENARIO_DIR.parent)) - -from gateway_harness import ( # noqa: E402 - require_subcommand, - resolve_binary, - stop_process, - wait_http, - wait_logged_url, - without_proxy_environment, -) - -EVENT_QUERY = ( - "SELECT seq, kind, session_id, model, call_id, payload_json " - "FROM dataset.events ORDER BY session_id, seq" -) - - -def write_configs( - work_dir: Path, - *, - echo_url: str, - gateway_port: int, - admin_port: int, -) -> tuple[Path, Path]: - warehouse_config = work_dir / "warehouse.toml" - gateway_config = work_dir / "gateway.toml" - warehouse_config.write_text( - f'''default_dataset = "captures" - -[[datasets]] -name = "captures" -uri = "{work_dir / "dataset"}" -''', - encoding="utf-8", - ) - gateway_config.write_text( - f'''listen = "127.0.0.1:{gateway_port}" -admin_listen = "127.0.0.1:{admin_port}" -agent_id = "gateway-echo-regression" -capture_level = "full" - -[[models]] -name = "sdk-anthropic-model" -provider = "anthropic" -upstream = "{echo_url}/v1" -upstream_anthropic = "{echo_url}/v1" - -[[models]] -name = "sdk-google-genai-model" -provider = "gemini" -upstream = "{echo_url}/v1beta" - -[[models]] -name = "echo-openai" -provider = "openai" -upstream = "{echo_url}/v1" - -[[models]] -name = "*" -forward = "echo-openai" -''', - encoding="utf-8", - ) - return warehouse_config, gateway_config - - -def run_regression(work_dir: Path, pchronicle: Path) -> None: - logs_dir = work_dir / "logs" - dataset_dir = work_dir / "dataset" - state_dir = work_dir / "gateway-state" - logs_dir.mkdir(parents=True) - dataset_dir.mkdir() - state_dir.mkdir() - echo_process: subprocess.Popen[bytes] | None = None - serve_process: subprocess.Popen[bytes] | None = None - log_handles: list[TextIO] = [] - try: - echo_log = (logs_dir / "echo.log").open("w", encoding="utf-8") - log_handles.append(echo_log) - echo_process = subprocess.Popen( - [ - str(pchronicle), - "echo", - "--listen", - "127.0.0.1:0", - "--encoding", - "plain", - ], - cwd=REPO_ROOT, - stdout=echo_log, - stderr=subprocess.STDOUT, - ) - echo_url = wait_logged_url( - logs_dir / "echo.log", "pChronicle Echo: ", echo_process, "pChronicle Echo" - ) - wait_http(f"{echo_url}/health", echo_process, "pChronicle Echo") - - warehouse_config, gateway_config = write_configs( - work_dir, - echo_url=echo_url, - gateway_port=0, - admin_port=0, - ) - - serve_log = (logs_dir / "serve.log").open("w", encoding="utf-8") - log_handles.append(serve_log) - serve_process = subprocess.Popen( - [ - str(pchronicle), - "serve", - "--config", - str(warehouse_config), - "--listen", - "127.0.0.1:0", - "--gateway", - str(gateway_config), - "--gateway-dataset", - "captures", - "--gateway-state", - str(state_dir), - ], - cwd=REPO_ROOT, - stdout=serve_log, - stderr=subprocess.STDOUT, - ) - warehouse_url = wait_logged_url( - logs_dir / "serve.log", - "pChronicle Warehouse: ", - serve_process, - "pChronicle Warehouse", - ) - gateway_url = wait_logged_url( - logs_dir / "serve.log", - "pChronicle Gateway: ", - serve_process, - "pChronicle Gateway", - ) - admin_url = wait_logged_url( - logs_dir / "serve.log", - "pChronicle Gateway admin: ", - serve_process, - "pChronicle Gateway admin", - ) - wait_http(f"{warehouse_url}/api/health", serve_process, "pChronicle Warehouse") - wait_http(f"{admin_url}/admin/status", serve_process, "pChronicle Gateway admin") - - print("Running OpenAI, Anthropic, and Google Gen AI Python clients...", flush=True) - with (logs_dir / "python-clients.log").open("w", encoding="utf-8") as client_log: - with contextlib.redirect_stdout(client_log), contextlib.redirect_stderr(client_log): - with without_proxy_environment(): - run_all(gateway_url, logs_dir / "client-results.jsonl") - - # Graceful shutdown is part of the contract: pChronicle must drain the - # Gateway capture queue before the durable event query runs. - stop_process(serve_process, label="pChronicle serve", require_success=True) - serve_process = None - stop_process(echo_process, label="pChronicle Echo") - echo_process = None - - events_log = logs_dir / "events.jsonl" - subprocess.run( - [ - str(pchronicle), - "query", - str(dataset_dir), - EVENT_QUERY, - "--format", - "jsonl", - "--output", - str(events_log), - ], - cwd=REPO_ROOT, - check=True, - ) - client_count, event_count = compare_logs( - logs_dir / "client-results.jsonl", - events_log, - logs_dir / "comparison.jsonl", - ) - print( - f"compared {client_count} SDK calls with {event_count} canonical events", - flush=True, - ) - finally: - stop_process(serve_process, label="pChronicle serve") - stop_process(echo_process, label="pChronicle Echo") - for handle in log_handles: - handle.close() - - -def interrupt_on_sigterm(_signum: int, _frame: object) -> None: - raise KeyboardInterrupt - - -def main() -> None: - signal.signal(signal.SIGTERM, interrupt_on_sigterm) - pchronicle = resolve_binary( - "PERSISTING_PCHRONICLE_BIN", - REPO_ROOT / "target/release/pchronicle", - "cargo build --release --locked -p persisting-pchronicle-cli", - ) - require_subcommand( - pchronicle, - "echo", - "cargo build --release --locked -p persisting-pchronicle-cli", - ) - work_dir = Path(tempfile.mkdtemp(prefix="persisting-gateway-echo-regression.")) - success = False - try: - run_regression(work_dir, pchronicle) - success = True - finally: - keep_artifacts = os.environ.get("PERSISTING_KEEP_TEST_ARTIFACTS") == "1" - if keep_artifacts or not success: - print(f"Gateway Echo regression artifacts: {work_dir}", file=sys.stderr) - else: - shutil.rmtree(work_dir) - - print("PASS pchronicle echo: Python SDK protocols and durable capture logs match") - - -if __name__ == "__main__": - main() diff --git a/tests/regression/gateway-echo/requirements.txt b/tests/regression/gateway-echo/requirements.txt deleted file mode 100644 index 061e8bd46..000000000 --- a/tests/regression/gateway-echo/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -openai==3.1.0 -anthropic==0.122.0 -google-genai==2.18.1 diff --git a/tests/regression/gateway-echo/run.sh b/tests/regression/gateway-echo/run.sh deleted file mode 100755 index d919f66ed..000000000 --- a/tests/regression/gateway-echo/run.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -scenario_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" -exec uv run --isolated --no-project \ - --with-requirements "$scenario_dir/requirements.txt" \ - python "$scenario_dir/regression.py" diff --git a/tests/regression/gateway-fuzz/.long-running b/tests/regression/gateway-fuzz/.long-running deleted file mode 100644 index e149bda17..000000000 --- a/tests/regression/gateway-fuzz/.long-running +++ /dev/null @@ -1 +0,0 @@ -This scenario is intentionally excluded from the default regression sweep. diff --git a/tests/regression/gateway-fuzz/README.md b/tests/regression/gateway-fuzz/README.md deleted file mode 100644 index 3cab80e03..000000000 --- a/tests/regression/gateway-fuzz/README.md +++ /dev/null @@ -1,110 +0,0 @@ -# Gateway fuzz regressions - -**Contract-split Gateway fuzz: format, routing, persistence, and policy fail -independently.** - -Owns the four fuzz scenarios and their artifact layout. Every scenario uses -prebuilt `target/release/pchronicle`; none compiles Rust. Does not own Gateway -or pChronicle. - -| Scenario | Contract | Command | -|---|---|---| -| `formats` | OpenAI Chat Completions, OpenAI Responses, Anthropic Messages, and Gemini wire formats; streaming, valid JSON Unicode/whitespace controls, plain text, Base64, tool calls, reasoning, multimodal input, structured errors, and complete terminal Responses output | `just gateway-fuzz-formats` | -| `forwarding` | wildcard forwarding, target model rewrite, Responses-to-Chat conversion, and persisted `forward_to` evidence | `just gateway-fuzz-forwarding` | -| `storage` | graceful queue drain, session-level Lance durability, ordered lifecycle pairing, canonical tools/reasoning/multimodal/error bodies, and bounded multi-source queries | `just gateway-fuzz-storage` | -| `network-policy` | local-only allowlist and no-network decisions for absolute-URI HTTP, CONNECT, host/port denial, and model-owned relative LLM routes | `just gateway-fuzz-network` | - -## Run - -```bash -just gateway-fuzz -``` - -The aggregate keeps the historical one-minute default by assigning 15 seconds -to each scenario. A scenario run directly defaults to one minute. Override the -duration, rate, concurrency, or maximum message size without editing it: - -```bash -PERSISTING_FUZZ_DURATION_SECONDS=3600 \ -PERSISTING_FUZZ_REQUESTS_PER_SECOND=20 \ -PERSISTING_FUZZ_CONCURRENCY=32 \ -PERSISTING_FUZZ_MAX_MESSAGE_CHARS=16384 \ -just gateway-fuzz-storage -``` - -Each scenario chooses a random 64-bit seed. Replay one generated case with the -printed seed and case index: - -```bash -PERSISTING_FUZZ_SEED=123456789 \ -PERSISTING_FUZZ_REPLAY_CASE=231 \ -PERSISTING_KEEP_TEST_ARTIFACTS=1 \ -just gateway-fuzz-forwarding -``` - -Artifacts are always retained on failure. Format/forward/storage artifacts -contain the case plan, SDK results, and process logs; forwarding and storage -also contain canonical events and per-session comparisons. Network-policy -artifacts contain policy configurations, asserted denial reasons, and -status/body digests. All network -policy targets are loopback services created by the scenario—this test never -depends on public network access. - -成功时默认会删除临时产物。要人工检查 `formats` 数据，请保留它们： - -```bash -PERSISTING_KEEP_TEST_ARTIFACTS=1 just gateway-fuzz-formats -``` - -命令会打印 `Gateway formats fuzz artifacts: <目录>`。主要文件是： - -- `run.json`：seed 和运行参数； -- `logs/cases.jsonl`：每个生成用例的完整输入和协议参数； -- `logs/client-results.jsonl`：SDK 看到的结果摘要； -- `logs/format-contract-results.jsonl`：固定格式契约用例的实际结果； -- `dataset/`：Gateway 写入的 Lance capture Dataset； -- `gateway-state/`：WAL 和 Gateway 状态。 - -`formats` 不把持久化核对作为 pass/fail 契约，但保留后的 Dataset 仍可手工导出： - -```bash -fuzz_artifacts=/tmp/persisting-gateway-formats-fuzz.example -target/release/pchronicle query "$fuzz_artifacts/dataset" \ - 'SELECT seq, kind, session_id, model, call_id, payload_json FROM dataset.events ORDER BY session_id, seq' \ - --format jsonl \ - --output "$fuzz_artifacts/logs/events.jsonl" -``` - -如果希望测试本身自动核对持久化事件，请运行 -`PERSISTING_KEEP_TEST_ARTIFACTS=1 just gateway-fuzz-storage`；它还会生成 -`logs/events.jsonl` 和 `logs/comparison.jsonl`。 - -`formats` deliberately stops at the client/wire contract. It proves that the -Gateway can accept, forward/convert, and return each exercised shape, but it -does **not** prove that capture reached durable storage. A broken capture path -can therefore pass `formats`; `storage` and `forwarding` perform the durable -event reconciliation. Malformed JSON and transport-level invalid bytes are -also outside the random SDK alphabet and belong in explicit negative cases, -so an SDK rejection is not misreported as a Gateway defect. - -Storage verification queries only sources owned by the run in bounded groups, -which keeps file-descriptor use bounded under low macOS process limits. It -checks each session as an ordered lifecycle: exactly one request, zero -cancellations, zero or more explicitly marked drafts, exactly one terminal -response, stable `call_id`, and strictly increasing sequence numbers. It does -not assume that a session always contains exactly two physical events. -The controlled storage cases additionally reconcile canonical tool-call -arguments, reasoning parts, image data sources, structured upstream errors, -and the terminal Responses tool-call event. - -Every local service binds `127.0.0.1:0` itself and publishes the selected -address. The harness never reserves a port by bind-close-rebind. - -The parent directory contains `.long-running`, so `just regression` skips this -suite. - -## Links - -- [Regression tests](../README.md) -- [Gateway architecture](../../../docs/src/en/pchronicle/guides/serve-gateway.md) -- [`persisting-gateway`](../../../crates/persisting-gateway/README.md) diff --git a/tests/regression/gateway-fuzz/formats/run.sh b/tests/regression/gateway-fuzz/formats/run.sh deleted file mode 100755 index e75adaf45..000000000 --- a/tests/regression/gateway-fuzz/formats/run.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -scenario_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" -exec env PERSISTING_GATEWAY_FUZZ_SUITE=formats \ - uv run --isolated --no-project \ - --with-requirements "$scenario_dir/requirements.txt" \ - python "$scenario_dir/regression.py" diff --git a/tests/regression/gateway-fuzz/forwarding/run.sh b/tests/regression/gateway-fuzz/forwarding/run.sh deleted file mode 100755 index 0b5ce0b54..000000000 --- a/tests/regression/gateway-fuzz/forwarding/run.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -scenario_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" -exec env PERSISTING_GATEWAY_FUZZ_SUITE=forwarding \ - uv run --isolated --no-project \ - --with-requirements "$scenario_dir/requirements.txt" \ - python "$scenario_dir/regression.py" diff --git a/tests/regression/gateway-fuzz/network-policy/run.sh b/tests/regression/gateway-fuzz/network-policy/run.sh deleted file mode 100755 index 9aaae0c29..000000000 --- a/tests/regression/gateway-fuzz/network-policy/run.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -scenario_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" -exec env PERSISTING_GATEWAY_FUZZ_SUITE=network-policy \ - uv run --isolated --no-project \ - --with-requirements "$scenario_dir/requirements.txt" \ - python "$scenario_dir/regression.py" diff --git a/tests/regression/gateway-fuzz/regression.py b/tests/regression/gateway-fuzz/regression.py deleted file mode 100644 index 76e08083c..000000000 --- a/tests/regression/gateway-fuzz/regression.py +++ /dev/null @@ -1,1513 +0,0 @@ -#!/usr/bin/env python3 -"""Long-running randomized Gateway → Echo → pChronicle persistence fuzz test.""" - -from __future__ import annotations - -import base64 -import concurrent.futures -import hashlib -import http.client -import json -import os -import random -import secrets -import shutil -import signal -import socket -import string -import subprocess -import sys -import tempfile -import threading -import time -import urllib.parse -from dataclasses import asdict, dataclass -from pathlib import Path -from typing import Any, TextIO - -import anthropic -import openai -from anthropic import Anthropic -from google import genai -from google.genai import types -from openai import OpenAI - -SCENARIO_DIR = Path(__file__).resolve().parent -REPO_ROOT = SCENARIO_DIR.parents[2] -sys.path.insert(0, str(SCENARIO_DIR.parent)) - -from gateway_harness import ( # noqa: E402 - append_jsonl, - require_subcommand, - resolve_binary, - stop_process, - wait_http, - wait_logged_url, - without_proxy_environment, -) - -THREAD_CLIENTS = threading.local() -FUZZ_AGENT_ID = "gateway-random-fuzz" -QUERY_SOURCE_BATCH_SIZE = 32 -FUZZ_SUITES = {"formats", "forwarding", "storage", "network-policy"} -FORMAT_CONTRACT_SESSIONS = ( - "format-contract-tool", - "format-contract-reasoning", - "format-contract-multimodal", - "format-contract-responses-tool", - "format-contract-error", -) -ALPHABET = string.ascii_letters + string.digits + " \n\r\t\\\"'{}[],:/" + "中文日本語한국어🙂🚀éΩ" - - -@dataclass(frozen=True) -class FuzzCase: - index: int - case_seed: int - session_id: str - client: str - protocol: str - model: str - input: str - encoding: str - streaming: bool - provider: str - response_kind: str - upstream_path: str - forward_to: str | None - - -def env_int(name: str, default: int, *, minimum: int = 1) -> int: - raw = os.environ.get(name) - value = default if raw is None else int(raw) - if value < minimum: - raise ValueError(f"{name} must be >= {minimum}, got {value}") - return value - - -def resolve_pchronicle() -> Path: - build_hint = "cargo build --release --locked -p persisting-pchronicle-cli" - path = resolve_binary( - "PERSISTING_PCHRONICLE_BIN", - REPO_ROOT / "target/release/pchronicle", - build_hint, - ) - require_subcommand(path, "echo", build_hint) - return path - - -def random_text(rng: random.Random, max_chars: int) -> str: - bucket = rng.random() - if bucket < 0.65: - length = rng.randint(1, min(128, max_chars)) - elif bucket < 0.93: - length = rng.randint(129, min(2048, max_chars)) if max_chars >= 129 else max_chars - else: - floor = min(2049, max_chars) - length = rng.randint(floor, max_chars) - return "".join(rng.choice(ALPHABET) for _ in range(length)) - - -def make_case(index: int, master: random.Random, max_chars: int, suite: str) -> FuzzCase: - case_seed = master.getrandbits(64) - rng = random.Random(case_seed) - protocols = ( - ["chat_completions", "responses"] - if suite == "forwarding" - else ["chat_completions", "responses", "messages", "gemini"] - ) - protocol = ( - protocols[index % len(protocols)] if index < len(protocols) * 4 else rng.choice(protocols) - ) - matrix_round = index // len(protocols) - streaming = protocol != "gemini" and ( - bool(matrix_round & 1) if index < len(protocols) * 4 else rng.choice([True, False]) - ) - encoding = ( - ("base64" if matrix_round & 2 else "plain") - if index < len(protocols) * 4 - else rng.choice(["plain", "base64"]) - ) - session_id = f"fuzz-{index:08d}-{case_seed:016x}" - if protocol == "chat_completions": - client, model, provider = "openai", f"fuzz-openai-{rng.randrange(16)}", "openai" - upstream_path = "/v1/chat/completions" - forward_to = "echo-openai" if suite == "forwarding" else None - elif protocol == "responses": - client, model, provider = "openai", f"fuzz-responses-{rng.randrange(16)}", "openai" - upstream_path = "/v1/chat/completions" - forward_to = "echo-openai" if suite == "forwarding" else None - elif protocol == "messages": - client, model, provider = "anthropic", "fuzz-anthropic", "anthropic" - upstream_path, forward_to = "/v1/messages", None - else: - client, model, provider = "google-genai", "fuzz-gemini", "gemini" - upstream_path, forward_to = "/v1beta/models/fuzz-gemini:generateContent", None - return FuzzCase( - index=index, - case_seed=case_seed, - session_id=session_id, - client=client, - protocol=protocol, - model=model, - input=random_text(rng, max_chars), - encoding=encoding, - streaming=streaming, - provider=provider, - response_kind="llm.response.stream" if streaming else "llm.response", - upstream_path=upstream_path, - forward_to=forward_to, - ) - - -def openai_client(gateway: str) -> OpenAI: - client = getattr(THREAD_CLIENTS, "openai", None) - if client is None: - client = OpenAI(api_key="fuzz", base_url=f"{gateway}/v1", max_retries=0) - THREAD_CLIENTS.openai = client - return client - - -def anthropic_client(gateway: str) -> Anthropic: - client = getattr(THREAD_CLIENTS, "anthropic", None) - if client is None: - client = Anthropic(api_key="fuzz", base_url=gateway, max_retries=0) - THREAD_CLIENTS.anthropic = client - return client - - -def execute_case(case: FuzzCase, gateway: str) -> dict[str, Any]: - headers = { - "x-persisting-session-id": case.session_id, - "x-persisting-echo-encoding": case.encoding, - } - if case.protocol == "chat_completions": - client = openai_client(gateway) - response = client.chat.completions.create( - model=case.model, - messages=[{"role": "user", "content": case.input}], - stream=case.streaming, - extra_headers=headers, - ) - if case.streaming: - chunks: list[str] = [] - response_model = None - for chunk in response: - response_model = chunk.model or response_model - if chunk.choices: - chunks.append(chunk.choices[0].delta.content or "") - output = "".join(chunks) - else: - output = response.choices[0].message.content or "" - response_model = response.model - elif case.protocol == "responses": - client = openai_client(gateway) - if case.streaming: - chunks: list[str] = [] - with client.responses.stream( - model=case.model, - input=case.input, - extra_headers=headers, - ) as stream: - for event in stream: - if event.type == "response.output_text.delta": - chunks.append(event.delta) - final_response = stream.get_final_response() - output = "".join(chunks) - response_model = final_response.model - else: - response = client.responses.create( - model=case.model, - input=case.input, - extra_headers=headers, - ) - output = response.output_text - response_model = response.model - elif case.protocol == "messages": - client = anthropic_client(gateway) - if case.streaming: - with client.messages.stream( - model=case.model, - max_tokens=64, - messages=[{"role": "user", "content": case.input}], - extra_headers=headers, - ) as stream: - output = "".join(stream.text_stream) - response_model = stream.get_final_message().model - else: - response = client.messages.create( - model=case.model, - max_tokens=64, - messages=[{"role": "user", "content": case.input}], - extra_headers=headers, - ) - output = "".join(block.text for block in response.content if block.type == "text") - response_model = response.model - else: - client = genai.Client( - api_key="fuzz", - http_options=types.HttpOptions( - base_url=gateway, - api_version="v1beta", - headers=headers, - ), - ) - try: - response = client.chats.create(model=case.model).send_message(case.input) - output = response.text or "" - response_model = response.model_version - finally: - client.close() - - expected = ( - case.input if case.encoding == "plain" else base64.b64encode(case.input.encode()).decode() - ) - if output != expected: - raise AssertionError( - f"case {case.index} seed={case.case_seed}: client output mismatch " - f"{text_difference(expected, output)}" - ) - expected_response_model = ( - case.forward_to - if case.protocol == "chat_completions" and case.forward_to is not None - else case.model - ) - if response_model != expected_response_model: - raise AssertionError( - f"case {case.index} seed={case.case_seed}: response model mismatch " - f"expected={expected_response_model!r} actual={response_model!r}" - ) - return { - "index": case.index, - "session_id": case.session_id, - "output_chars": len(output), - "output_sha256": hashlib.sha256(output.encode("utf-8")).hexdigest(), - "response_model": response_model, - "sdk_version": { - "openai": openai.__version__, - "anthropic": anthropic.__version__, - "google-genai": genai.__version__, - }[case.client], - } - - -def text_difference(expected: str, actual: str) -> str: - """Return a bounded, reproducible diagnostic for a large text mismatch.""" - common = min(len(expected), len(actual)) - offset = next((index for index in range(common) if expected[index] != actual[index]), common) - expected_char = expected[offset] if offset < len(expected) else None - actual_char = actual[offset] if offset < len(actual) else None - start = max(0, offset - 32) - end = offset + 33 - - def digest(value: str) -> str: - return hashlib.sha256(value.encode("utf-8")).hexdigest() - - def codepoint(value: str | None) -> str: - return "EOF" if value is None else f"U+{ord(value):04X} {value!r}" - - return ( - f"first_difference={offset} " - f"expected_char={codepoint(expected_char)} actual_char={codepoint(actual_char)} " - f"expected_chars={len(expected)} actual_chars={len(actual)} " - f"expected_sha256={digest(expected)} actual_sha256={digest(actual)} " - f"expected_context={expected[start:end]!r} actual_context={actual[start:end]!r}" - ) - - -def parts_text(parts: list[dict[str, Any]]) -> str: - return "".join(part.get("text", "") for part in parts if part.get("type") == "text") - - -def canonical_request_text(payload: dict[str, Any]) -> str: - return "".join( - parts_text(message.get("parts", [])) - for message in payload["llm_request"]["request"]["messages"] - if message.get("role") == "user" - ) - - -def canonical_response_text(payload: dict[str, Any]) -> str: - return "".join( - parts_text(candidate.get("message", {}).get("parts", [])) - for candidate in payload["llm_response"]["response"].get("candidates", []) - ) - - -def load_jsonl_by_session(path: Path) -> dict[str, dict[str, Any]]: - records: dict[str, dict[str, Any]] = {} - for line in path.read_text(encoding="utf-8").splitlines(): - record = json.loads(line) - session_id = record["session_id"] - if session_id in records: - raise AssertionError(f"duplicate session {session_id!r} in {path}") - records[session_id] = record - return records - - -def is_draft_payload(payload: dict[str, Any]) -> bool: - return bool( - payload.get("draft") - or payload.get("llm_response", {}).get("draft") - or payload.get("llm_response", {}).get("response", {}).get("draft") - ) - - -def compare_events( - cases_path: Path, - results_path: Path, - events_path: Path, - comparison_path: Path, -) -> tuple[int, int]: - expected = load_jsonl_by_session(cases_path) - results = load_jsonl_by_session(results_path) - if set(results) != set(expected): - raise AssertionError( - "case/result session mismatch: " - f"missing={sorted(set(expected) - set(results))[:10]} " - f"unexpected={sorted(set(results) - set(expected))[:10]}" - ) - states = { - session_id: { - "request_count": 0, - "terminal_count": 0, - "draft_count": 0, - "cancel_count": 0, - "call_id": None, - "last_seq": None, - "kinds": [], - } - for session_id in expected - } - failures: list[dict[str, Any]] = [] - event_count = 0 - with events_path.open(encoding="utf-8") as events: - for line in events: - event_count += 1 - event = json.loads(line) - session_id = event["session_id"] - case = expected.get(session_id) - if case is None: - failures.append({"session_id": session_id, "failure": "unexpected session"}) - continue - payload = json.loads(event["payload_json"])["payload"] - state = states[session_id] - seq = event["seq"] - if state["last_seq"] is not None and seq <= state["last_seq"]: - failures.append( - { - "session_id": session_id, - "failure": "non-increasing seq", - "previous": state["last_seq"], - "actual": seq, - } - ) - state["last_seq"] = seq - state["kinds"].append(event["kind"]) - call_id = event.get("call_id") - if not call_id: - failures.append( - {"session_id": session_id, "kind": event["kind"], "failure": "missing call_id"} - ) - elif state["call_id"] is None: - state["call_id"] = call_id - elif state["call_id"] != call_id: - failures.append( - { - "session_id": session_id, - "kind": event["kind"], - "failure": "call_id changed within lifecycle", - "expected": state["call_id"], - "actual": call_id, - } - ) - if event["kind"] == "llm.request": - checks = { - "model": event["model"] == case["model"], - "protocol": payload["protocol"] == case["protocol"], - "provider": payload["provider"] == case["provider"], - "input": canonical_request_text(payload) == case["input"], - "forward_to": payload.get("forward_to") == case["forward_to"], - } - state["request_count"] += 1 - elif event["kind"] == "llm.call.cancelled": - checks = {"unexpected_cancel": False} - state["cancel_count"] += 1 - elif event["kind"] in { - "llm.response", - "llm.response.stream", - } and is_draft_payload(payload): - checks = {"draft_after_request": state["request_count"] == 1} - state["draft_count"] += 1 - elif event["kind"] in {"llm.response", "llm.response.stream"}: - expected_output = ( - case["input"] - if case["encoding"] == "plain" - else base64.b64encode(case["input"].encode()).decode() - ) - checks = { - "kind": event["kind"] == case["response_kind"], - "after_request": state["request_count"] == 1, - "output": canonical_response_text(payload) == expected_output, - "status": payload["status"] == 200, - "upstream_path": payload["http"]["url"].endswith(case["upstream_path"]), - "forward_to": payload.get("forward_to") == case["forward_to"], - } - state["terminal_count"] += 1 - else: - checks = {"recognized_lifecycle_kind": False} - if not all(checks.values()): - failures.append({"session_id": session_id, "kind": event["kind"], "checks": checks}) - - with comparison_path.open("w", encoding="utf-8") as comparisons: - for session_id, state in states.items(): - matched = bool( - state["request_count"] == 1 - and state["terminal_count"] == 1 - and state["cancel_count"] == 0 - ) - record = {"session_id": session_id, "matched": matched, **state} - json.dump(record, comparisons, sort_keys=True) - comparisons.write("\n") - if not matched: - failures.append({"session_id": session_id, "failure": "invalid lifecycle", **state}) - - if failures: - raise AssertionError(json.dumps(failures[:20], ensure_ascii=False, indent=2)) - return len(expected), event_count - - -def sql_string(value: str) -> str: - return "'" + value.replace("'", "''") + "'" - - -def query_captured_events( - pchronicle: Path, - dataset: Path, - results_path: Path, - events_path: Path, -) -> None: - """Read owned sources in bounded groups so low macOS FD limits are sufficient.""" - session_ids = [ - json.loads(line)["session_id"] - for line in results_path.read_text(encoding="utf-8").splitlines() - ] - query_session_events(pchronicle, dataset, session_ids, events_path) - - -def query_session_events( - pchronicle: Path, - dataset: Path, - session_ids: list[str], - events_path: Path, -) -> None: - events_path.write_bytes(b"") - for batch_index, start in enumerate(range(0, len(session_ids), QUERY_SOURCE_BATCH_SIZE)): - batch = session_ids[start : start + QUERY_SOURCE_BATCH_SIZE] - source_filter = ", ".join( - sql_string(f"{FUZZ_AGENT_ID}/{session_id}/events.lance") for session_id in batch - ) - part_path = events_path.with_name(f"{events_path.stem}-{batch_index:05d}.jsonl") - try: - query = subprocess.run( - [ - str(pchronicle), - "query", - str(dataset), - "SELECT seq, kind, session_id, model, call_id, payload_json " - f"FROM dataset.events WHERE _file_ IN ({source_filter}) " - "ORDER BY session_id, seq", - "--format", - "jsonl", - "--output", - str(part_path), - "--max-output-rows", - str(len(batch) * 64 + 100), - "--max-output-bytes", - str(1024 * 1024 * 1024), - "--timeout-seconds", - "300", - ], - cwd=REPO_ROOT, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True, - check=False, - ) - if query.returncode != 0: - raise RuntimeError( - f"pChronicle query batch {batch_index} failed " - f"for {len(batch)} sources (status {query.returncode}):\n{query.stdout}" - ) - with events_path.open("ab") as output, part_path.open("rb") as part: - shutil.copyfileobj(part, output) - finally: - part_path.unlink(missing_ok=True) - - -def write_configs( - work_dir: Path, - echo_url: str, - gateway_port: int, - admin_port: int, - suite: str, -) -> tuple[Path, Path]: - warehouse = work_dir / "warehouse.toml" - gateway = work_dir / "gateway.toml" - warehouse.write_text( - f'''default_dataset = "captures" - -[[datasets]] -name = "captures" -uri = "{work_dir / "dataset"}" -''', - encoding="utf-8", - ) - forwarding_routes = ( - f'''[[models]] -name = "echo-openai" -provider = "openai" -upstream = "{echo_url}/v1" - -[[models]] -name = "*" -forward = "echo-openai" -''' - if suite == "forwarding" - else f'''[[models]] -name = "fuzz-openai-*" -provider = "openai" -upstream = "{echo_url}/v1" - -[[models]] -name = "fuzz-responses-*" -provider = "openai" -upstream = "{echo_url}/v1" -''' - ) - gateway.write_text( - f'''listen = "127.0.0.1:{gateway_port}" -admin_listen = "127.0.0.1:{admin_port}" -agent_id = "{FUZZ_AGENT_ID}" -capture_level = "full" - -[[models]] -name = "fuzz-anthropic" -provider = "anthropic" -upstream = "{echo_url}/v1" -upstream_anthropic = "{echo_url}/v1" - -[[models]] -name = "fuzz-gemini" -provider = "gemini" -upstream = "{echo_url}/v1beta" - -{forwarding_routes} -''', - encoding="utf-8", - ) - return warehouse, gateway - - -def write_network_configs( - work_dir: Path, - name: str, - echo_url: str, - echo_port: int, - gateway_port: int, - admin_port: int, - mode: str, -) -> tuple[Path, Path, Path, Path]: - root = work_dir / name - dataset = root / "dataset" - state = root / "gateway-state" - dataset.mkdir(parents=True) - state.mkdir() - warehouse = root / "warehouse.toml" - gateway = root / "gateway.toml" - warehouse.write_text( - f'''default_dataset = "captures" - -[[datasets]] -name = "captures" -uri = "{dataset}" -''', - encoding="utf-8", - ) - if mode == "allowlist": - network = f"""[network] -mode = "allowlist" - -[[network.rules]] -host = "127.0.0.1" -ports = [{echo_port}] -transports = ["http", "tcp_tunnel"] -""" - else: - network = """[network] -mode = "no-network" -""" - gateway.write_text( - f'''listen = "127.0.0.1:{gateway_port}" -admin_listen = "127.0.0.1:{admin_port}" -agent_id = "gateway-network-fuzz-{name}" -capture_level = "full" - -{network} - -[[models]] -name = "network-model" -provider = "openai" -upstream = "{echo_url}/v1" -''', - encoding="utf-8", - ) - return warehouse, gateway, dataset, state - - -def start_pchronicle_serve( - pchronicle: Path, - warehouse: Path, - gateway: Path, - state: Path, - warehouse_port: int, - log: TextIO, -) -> subprocess.Popen[bytes]: - return subprocess.Popen( - [ - str(pchronicle), - "serve", - "--config", - str(warehouse), - "--listen", - f"127.0.0.1:{warehouse_port}", - "--gateway", - str(gateway), - "--gateway-dataset", - "captures", - "--gateway-state", - str(state), - ], - cwd=REPO_ROOT, - stdout=log, - stderr=subprocess.STDOUT, - ) - - -def proxy_absolute_get(gateway_port: int, target: str, target_host: str) -> tuple[int, str]: - connection = http.client.HTTPConnection("127.0.0.1", gateway_port, timeout=5) - try: - connection.request("GET", target, headers={"Host": target_host}) - response = connection.getresponse() - return response.status, response.read().decode("utf-8", errors="replace") - finally: - connection.close() - - -def proxy_connect(gateway_port: int, authority: str) -> tuple[int, str]: - with socket.create_connection(("127.0.0.1", gateway_port), timeout=5) as connection: - connection.sendall(f"CONNECT {authority} HTTP/1.1\r\nHost: {authority}\r\n\r\n".encode()) - response = bytearray() - while b"\r\n\r\n" not in response and len(response) < 64 * 1024: - chunk = connection.recv(4096) - if not chunk: - break - response.extend(chunk) - text = response.decode("utf-8", errors="replace") - first_line = text.splitlines()[0] if text.splitlines() else "" - fields = first_line.split() - status = int(fields[1]) if len(fields) >= 2 and fields[1].isdigit() else 0 - return status, text - - -def relative_llm_request(gateway_port: int, session_id: str, input_text: str) -> tuple[int, str]: - body = json.dumps( - { - "model": "network-model", - "messages": [{"role": "user", "content": input_text}], - }, - ensure_ascii=False, - ).encode() - connection = http.client.HTTPConnection("127.0.0.1", gateway_port, timeout=5) - try: - connection.request( - "POST", - "/v1/chat/completions", - body=body, - headers={ - "content-type": "application/json", - "x-persisting-session-id": session_id, - }, - ) - response = connection.getresponse() - response_body = response.read().decode("utf-8", errors="replace") - return response.status, response_body - finally: - connection.close() - - -def gateway_json_request( - gateway: str, - path: str, - payload: dict[str, Any], - *, - session_id: str, - mode: str, -) -> tuple[int, str, dict[str, str]]: - parsed = urllib.parse.urlparse(gateway) - assert parsed.hostname is not None and parsed.port is not None - connection = http.client.HTTPConnection(parsed.hostname, parsed.port, timeout=10) - try: - connection.request( - "POST", - path, - body=json.dumps(payload, ensure_ascii=False).encode("utf-8"), - headers={ - "content-type": "application/json", - "x-persisting-session-id": session_id, - "x-persisting-echo-mode": mode, - }, - ) - response = connection.getresponse() - body = response.read().decode("utf-8", errors="strict") - return response.status, body, {key.lower(): value for key, value in response.getheaders()} - finally: - connection.close() - - -def response_sse_events(body: str) -> list[dict[str, Any]]: - events: list[dict[str, Any]] = [] - for frame in body.replace("\r\n", "\n").split("\n\n"): - data = "\n".join( - line[5:].strip() for line in frame.splitlines() if line.startswith("data:") - ) - if data and data != "[DONE]": - events.append(json.loads(data)) - return events - - -def run_format_contract_cases(gateway: str, results_path: Path) -> int: - """Exercise non-text response contracts that random text round-trips cannot cover.""" - cases = 0 - - status, body, _ = gateway_json_request( - gateway, - "/v1/chat/completions", - {"model": "fuzz-openai-0", "messages": [{"role": "user", "content": "tool"}]}, - session_id="format-contract-tool", - mode="tool", - ) - tool = json.loads(body) - call = tool["choices"][0]["message"]["tool_calls"][0] - if status != 200 or call["function"] != { - "name": "weather", - "arguments": '{"city":"Paris"}', - }: - raise AssertionError( - f"Chat tool-call contract failed: status={status} body={body[:1000]!r}" - ) - append_jsonl(results_path, {"case": "chat-tool", "status": status, "call": call}) - cases += 1 - - status, body, _ = gateway_json_request( - gateway, - "/v1/chat/completions", - {"model": "fuzz-openai-0", "messages": [{"role": "user", "content": "reason"}]}, - session_id="format-contract-reasoning", - mode="reasoning", - ) - reasoning = json.loads(body) - message = reasoning["choices"][0]["message"] - if ( - status != 200 - or message.get("reasoning_content") != "echo-reasoning" - or message.get("content") != "reason" - ): - raise AssertionError( - f"Chat reasoning contract failed: status={status} body={body[:1000]!r}" - ) - append_jsonl( - results_path, - {"case": "chat-reasoning", "status": status, "reasoning": message["reasoning_content"]}, - ) - cases += 1 - - status, body, _ = gateway_json_request( - gateway, - "/v1/chat/completions", - { - "model": "fuzz-openai-0", - "messages": [ - { - "role": "user", - "content": [ - {"type": "text", "text": "describe"}, - { - "type": "image_url", - "image_url": {"url": "data:image/png;base64,iVBORw0KGgo="}, - }, - ], - } - ], - }, - session_id="format-contract-multimodal", - mode="inspect", - ) - multimodal = json.loads(body) - content = multimodal["choices"][0]["message"]["content"] - if status != 200 or content != "text=1 image=1": - raise AssertionError( - f"Chat multimodal contract failed: status={status} body={body[:1000]!r}" - ) - append_jsonl(results_path, {"case": "chat-multimodal", "status": status, "summary": content}) - cases += 1 - - status, body, _ = gateway_json_request( - gateway, - "/v1/responses", - { - "model": "fuzz-responses-0", - "input": "tool", - "stream": True, - "tools": [ - { - "type": "function", - "name": "weather", - "description": "weather lookup", - "parameters": {"type": "object", "properties": {"city": {"type": "string"}}}, - } - ], - }, - session_id="format-contract-responses-tool", - mode="tool", - ) - response_events = response_sse_events(body) - completed = next( - (event for event in response_events if event.get("type") == "response.completed"), - None, - ) - output = None if completed is None else completed["response"].get("output") - if ( - status != 200 - or not isinstance(output, list) - or len(output) != 1 - or output[0].get("type") != "function_call" - or output[0].get("name") != "weather" - ): - raise AssertionError( - "Responses terminal output contract failed: " - f"status={status} events={response_events[-5:]!r}" - ) - append_jsonl( - results_path, - {"case": "responses-terminal-tool", "status": status, "output": output}, - ) - cases += 1 - - status, body, _ = gateway_json_request( - gateway, - "/v1/chat/completions", - {"model": "fuzz-openai-0", "messages": [{"role": "user", "content": "fail"}]}, - session_id="format-contract-error", - mode="error", - ) - error = json.loads(body).get("error", {}) - if status != 429 or error != { - "message": "controlled echo failure", - "type": "echo_controlled_error", - "code": "echo_rate_limit", - }: - raise AssertionError(f"error-body contract failed: status={status} body={body[:1000]!r}") - append_jsonl(results_path, {"case": "error-body", "status": status, "error": error}) - cases += 1 - return cases - - -def canonical_candidate_parts(payload: dict[str, Any]) -> list[dict[str, Any]]: - return [ - part - for candidate in payload["llm_response"]["response"].get("candidates", []) - for part in candidate.get("message", {}).get("parts", []) - ] - - -def validate_durable_content_contracts(events_path: Path, comparison_path: Path) -> int: - grouped: dict[str, list[dict[str, Any]]] = { - session_id: [] for session_id in FORMAT_CONTRACT_SESSIONS - } - for line in events_path.read_text(encoding="utf-8").splitlines(): - event = json.loads(line) - if event["session_id"] not in grouped: - raise AssertionError(f"unexpected content-contract session {event['session_id']!r}") - event["payload"] = json.loads(event["payload_json"])["payload"] - grouped[event["session_id"]].append(event) - - comparisons: list[dict[str, Any]] = [] - failures: list[dict[str, Any]] = [] - for session_id, events in grouped.items(): - requests = [event for event in events if event["kind"] == "llm.request"] - drafts = [ - event - for event in events - if event["kind"] in {"llm.response", "llm.response.stream"} - and is_draft_payload(event["payload"]) - ] - terminals = [ - event - for event in events - if event["kind"] in {"llm.response", "llm.response.stream"} - and not is_draft_payload(event["payload"]) - ] - cancellations = [event for event in events if event["kind"] == "llm.call.cancelled"] - seqs = [event["seq"] for event in events] - call_ids = {event.get("call_id") for event in events} - checks: dict[str, bool] = { - "one_request": len(requests) == 1, - "one_terminal": len(terminals) == 1, - "no_cancel": not cancellations, - "ordered": all(left < right for left, right in zip(seqs, seqs[1:])), - "stable_call_id": len(call_ids) == 1 and None not in call_ids, - } - if len(requests) == 1 and len(terminals) == 1: - request = requests[0]["payload"] - terminal = terminals[0] - response = terminal["payload"] - request_parts = [ - part - for message in request["llm_request"]["request"].get("messages", []) - for part in message.get("parts", []) - ] - response_parts = canonical_candidate_parts(response) - if session_id == "format-contract-tool": - checks["tool_call"] = any( - part.get("type") == "tool_call" - and part.get("id") == "call_echo_weather" - and part.get("name") == "weather" - and part.get("arguments") == {"city": "Paris"} - for part in response_parts - ) - elif session_id == "format-contract-responses-tool": - checks["stream_terminal"] = terminal["kind"] == "llm.response.stream" - checks["tool_call"] = any( - part.get("type") == "tool_call" - and part.get("name") == "weather" - and part.get("arguments") == {"city": "Paris"} - for part in response_parts - ) - elif session_id == "format-contract-reasoning": - checks["reasoning"] = any( - part.get("type") == "reasoning" and part.get("text") == "echo-reasoning" - for part in response_parts - ) - checks["visible_text"] = any( - part.get("type") == "text" and part.get("text") == "reason" - for part in response_parts - ) - elif session_id == "format-contract-multimodal": - checks["request_text"] = any( - part.get("type") == "text" and part.get("text") == "describe" - for part in request_parts - ) - checks["request_image"] = any( - part.get("type") == "image" - and part.get("source", {}).get("type") == "data" - and part.get("source", {}).get("data") == "data:image/png;base64,iVBORw0KGgo=" - for part in request_parts - ) - checks["response_summary"] = canonical_response_text(response) == "text=1 image=1" - elif session_id == "format-contract-error": - checks["status"] = response.get("status") == 429 - checks["error_body"] = response.get("http", {}).get("response_body") == { - "error": { - "message": "controlled echo failure", - "type": "echo_controlled_error", - "code": "echo_rate_limit", - } - } - record = { - "session_id": session_id, - "matched": all(checks.values()), - "checks": checks, - "kinds": [event["kind"] for event in events], - "draft_count": len(drafts), - } - comparisons.append(record) - if not record["matched"]: - failures.append(record) - - with comparison_path.open("w", encoding="utf-8") as output: - for record in comparisons: - output.write(json.dumps(record, ensure_ascii=False, sort_keys=True) + "\n") - if failures: - raise AssertionError(json.dumps(failures, ensure_ascii=False, indent=2)) - return sum(len(events) for events in grouped.values()) - - -def run_network_fuzz(work_dir: Path, pchronicle: Path, seed: int) -> tuple[int, int]: - duration = env_int("PERSISTING_FUZZ_DURATION_SECONDS", 60) - rate = env_int("PERSISTING_FUZZ_REQUESTS_PER_SECOND", 5) - replay_raw = os.environ.get("PERSISTING_FUZZ_REPLAY_CASE") - replay_index = None if replay_raw is None else int(replay_raw) - if replay_index is not None and replay_index < 0: - raise ValueError("PERSISTING_FUZZ_REPLAY_CASE must be >= 0") - (work_dir / "run.json").write_text( - json.dumps( - { - "suite": "network-policy", - "seed": seed, - "duration_seconds": duration, - "requests_per_second": rate, - "replay_case": replay_index, - }, - indent=2, - sort_keys=True, - ) - + "\n", - encoding="utf-8", - ) - - logs = work_dir / "logs" - logs.mkdir() - handles: list[TextIO] = [] - processes: list[tuple[str, subprocess.Popen[bytes]]] = [] - try: - echo_log = (logs / "echo.log").open("w", encoding="utf-8") - handles.append(echo_log) - echo_process = subprocess.Popen( - [str(pchronicle), "echo", "--listen", "127.0.0.1:0"], - cwd=REPO_ROOT, - stdout=echo_log, - stderr=subprocess.STDOUT, - ) - processes.append(("pChronicle Echo", echo_process)) - echo_url = wait_logged_url( - logs / "echo.log", "pChronicle Echo: ", echo_process, "pChronicle Echo" - ) - echo_port = urllib.parse.urlparse(echo_url).port - assert echo_port is not None - wait_http(f"{echo_url}/health", echo_process, "pChronicle Echo") - - allow = write_network_configs(work_dir, "allowlist", echo_url, echo_port, 0, 0, "allowlist") - deny = write_network_configs( - work_dir, "no-network", echo_url, echo_port, 0, 0, "no-network" - ) - blocked_port = 1 if echo_port != 1 else 2 - listeners: dict[str, tuple[int, int]] = {} - - for name, config in [("allowlist", allow), ("no-network", deny)]: - warehouse, gateway, _dataset, state = config - serve_log = (logs / f"serve-{name}.log").open("w", encoding="utf-8") - handles.append(serve_log) - process = start_pchronicle_serve(pchronicle, warehouse, gateway, state, 0, serve_log) - processes.append((f"pChronicle serve {name}", process)) - warehouse_url = wait_logged_url( - logs / f"serve-{name}.log", - "pChronicle Warehouse: ", - process, - f"pChronicle Warehouse {name}", - ) - gateway_url = wait_logged_url( - logs / f"serve-{name}.log", - "pChronicle Gateway: ", - process, - f"pChronicle Gateway {name}", - ) - admin_url = wait_logged_url( - logs / f"serve-{name}.log", - "pChronicle Gateway admin: ", - process, - f"pChronicle Gateway admin {name}", - ) - gateway_port = urllib.parse.urlparse(gateway_url).port - admin_port = urllib.parse.urlparse(admin_url).port - assert gateway_port is not None and admin_port is not None - listeners[name] = (gateway_port, admin_port) - wait_http( - f"{warehouse_url}/api/health", - process, - f"pChronicle Warehouse {name}", - ) - wait_http( - f"{admin_url}/admin/status", - process, - f"pChronicle Gateway {name}", - ) - - allow_gateway_port, _allow_admin_port = listeners["allowlist"] - deny_gateway_port, _deny_admin_port = listeners["no-network"] - - case_kinds = [ - "allow-absolute", - "allow-connect", - "deny-hostname", - "deny-port", - "no-network-absolute", - "no-network-connect", - "allow-relative-llm", - "no-network-relative-llm", - ] - cases_path = logs / "cases.jsonl" - results_path = logs / "client-results.jsonl" - master = random.Random(seed) - deadline = time.monotonic() + duration - interval = 1 / rate - next_case = time.monotonic() - next_progress = next_case + 10 - index = 0 - while time.monotonic() < deadline: - if replay_index is not None and index > replay_index: - break - now = time.monotonic() - if replay_index is None and now < next_case: - time.sleep(min(0.05, next_case - now)) - continue - case_seed = master.getrandbits(64) - kind = ( - case_kinds[index] - if index < len(case_kinds) - else random.Random(case_seed).choice(case_kinds) - ) - if replay_index is not None and index < replay_index: - index += 1 - continue - case = {"index": index, "case_seed": case_seed, "kind": kind} - append_jsonl(cases_path, case) - authority = f"127.0.0.1:{echo_port}" - if kind == "allow-absolute": - status, body = proxy_absolute_get( - allow_gateway_port, f"{echo_url}/health", authority - ) - expected = 200 - elif kind == "allow-connect": - status, body = proxy_connect(allow_gateway_port, authority) - expected = 200 - elif kind == "deny-hostname": - status, body = proxy_absolute_get( - allow_gateway_port, - f"http://localhost:{echo_port}/health", - f"localhost:{echo_port}", - ) - expected = 403 - elif kind == "deny-port": - status, body = proxy_absolute_get( - allow_gateway_port, - f"http://127.0.0.1:{blocked_port}/health", - f"127.0.0.1:{blocked_port}", - ) - expected = 403 - elif kind == "no-network-absolute": - status, body = proxy_absolute_get( - deny_gateway_port, f"{echo_url}/health", authority - ) - expected = 403 - elif kind == "no-network-connect": - status, body = proxy_connect(deny_gateway_port, authority) - expected = 403 - else: - input_text = f"network policy relative LLM {index} 中文🙂" - gateway_port = ( - allow_gateway_port if kind == "allow-relative-llm" else deny_gateway_port - ) - status, body = relative_llm_request( - gateway_port, f"network-fuzz-{index:08d}", input_text - ) - expected = 200 - if status == 200: - payload = json.loads(body) - output = payload["choices"][0]["message"]["content"] - if output != input_text: - raise AssertionError( - f"network case {index} relative LLM mismatch: " - f"{text_difference(input_text, output)}" - ) - expected_reason = { - "deny-hostname": "not-in-allowlist", - "deny-port": "port-not-allowed", - "no-network-absolute": "no-network", - "no-network-connect": "no-network", - }.get(kind) - if status != expected: - raise AssertionError( - f"network case {index} seed={case_seed} kind={kind}: " - f"expected status {expected}, got {status}, body={body[:500]!r}" - ) - if expected_reason is not None and expected_reason not in body: - raise AssertionError( - f"network case {index} seed={case_seed} kind={kind}: " - f"expected denial reason {expected_reason!r}, body={body[:500]!r}" - ) - if kind == "allow-absolute": - health = json.loads(body) - if health != {"status": "ok", "service": "pchronicle-echo"}: - raise AssertionError(f"network case {index} allow body mismatch: {health!r}") - append_jsonl( - results_path, - { - **case, - "status": status, - "denial_reason": expected_reason, - "body_sha256": hashlib.sha256(body.encode()).hexdigest(), - }, - ) - index += 1 - next_case += interval - if now >= next_progress: - print( - "Gateway network-policy fuzz progress " - f"elapsed={int(duration - max(0, deadline - now))}s " - f"completed={index}", - flush=True, - ) - next_progress = now + 10 - - for label, process in reversed(processes): - stop_process(process, label=label, require_success=label.startswith("pChronicle serve")) - processes.clear() - return index, 0 - finally: - for label, process in reversed(processes): - stop_process(process, label=label) - for handle in handles: - handle.close() - - -def run_fuzz(work_dir: Path, pchronicle: Path, seed: int, suite: str) -> tuple[int, int]: - duration = env_int("PERSISTING_FUZZ_DURATION_SECONDS", 60) - concurrency = env_int("PERSISTING_FUZZ_CONCURRENCY", 8) - rate = env_int("PERSISTING_FUZZ_REQUESTS_PER_SECOND", 5) - max_chars = env_int("PERSISTING_FUZZ_MAX_MESSAGE_CHARS", 16_384) - replay_raw = os.environ.get("PERSISTING_FUZZ_REPLAY_CASE") - replay_index = None if replay_raw is None else int(replay_raw) - if replay_index is not None and replay_index < 0: - raise ValueError("PERSISTING_FUZZ_REPLAY_CASE must be >= 0") - (work_dir / "run.json").write_text( - json.dumps( - { - "seed": seed, - "suite": suite, - "duration_seconds": duration, - "concurrency": concurrency, - "requests_per_second": rate, - "max_message_chars": max_chars, - "replay_case": replay_index, - }, - indent=2, - sort_keys=True, - ) - + "\n", - encoding="utf-8", - ) - - logs = work_dir / "logs" - dataset = work_dir / "dataset" - state = work_dir / "gateway-state" - logs.mkdir() - dataset.mkdir() - state.mkdir() - echo_process: subprocess.Popen[bytes] | None = None - serve_process: subprocess.Popen[bytes] | None = None - handles: list[TextIO] = [] - try: - echo_log = (logs / "echo.log").open("w", encoding="utf-8") - handles.append(echo_log) - echo_process = subprocess.Popen( - [str(pchronicle), "echo", "--listen", "127.0.0.1:0"], - cwd=REPO_ROOT, - stdout=echo_log, - stderr=subprocess.STDOUT, - ) - echo_url = wait_logged_url( - logs / "echo.log", "pChronicle Echo: ", echo_process, "pChronicle Echo" - ) - wait_http(f"{echo_url}/health", echo_process, "pChronicle Echo") - - warehouse_config, gateway_config = write_configs(work_dir, echo_url, 0, 0, suite) - - serve_log = (logs / "serve.log").open("w", encoding="utf-8") - handles.append(serve_log) - serve_process = subprocess.Popen( - [ - str(pchronicle), - "serve", - "--config", - str(warehouse_config), - "--listen", - "127.0.0.1:0", - "--gateway", - str(gateway_config), - "--gateway-dataset", - "captures", - "--gateway-state", - str(state), - ], - cwd=REPO_ROOT, - stdout=serve_log, - stderr=subprocess.STDOUT, - ) - warehouse_url = wait_logged_url( - logs / "serve.log", - "pChronicle Warehouse: ", - serve_process, - "pChronicle Warehouse", - ) - gateway_url = wait_logged_url( - logs / "serve.log", - "pChronicle Gateway: ", - serve_process, - "pChronicle Gateway", - ) - admin_url = wait_logged_url( - logs / "serve.log", - "pChronicle Gateway admin: ", - serve_process, - "pChronicle Gateway admin", - ) - wait_http( - f"{warehouse_url}/api/health", - serve_process, - "pChronicle Warehouse", - ) - wait_http( - f"{admin_url}/admin/status", - serve_process, - "pChronicle Gateway admin", - ) - - plan_path = logs / "cases.jsonl" - results_path = logs / "client-results.jsonl" - contract_results_path = logs / "format-contract-results.jsonl" - failures_path = logs / "failures.jsonl" - master = random.Random(seed) - index = 0 - with without_proxy_environment(): - if replay_index is not None: - for index in range(replay_index + 1): - case = make_case(index, master, max_chars, suite) - append_jsonl(plan_path, asdict(case)) - try: - append_jsonl(results_path, execute_case(case, gateway_url)) - except Exception as error: - append_jsonl(failures_path, {**asdict(case), "error": str(error)}) - raise - index = 1 - else: - deadline = time.monotonic() + duration - started_at = time.monotonic() - interval = 1 / rate - next_submit = time.monotonic() - next_progress = started_at + 10 - completed = 0 - pending: dict[concurrent.futures.Future[dict[str, Any]], FuzzCase] = {} - with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as executor: - while time.monotonic() < deadline or pending: - now = time.monotonic() - while now < deadline and now >= next_submit and len(pending) < concurrency: - case = make_case(index, master, max_chars, suite) - append_jsonl(plan_path, asdict(case)) - pending[executor.submit(execute_case, case, gateway_url)] = case - index += 1 - next_submit += interval - if not pending: - time.sleep(min(0.05, max(0.0, next_submit - time.monotonic()))) - continue - done, _ = concurrent.futures.wait( - pending, - timeout=0.05, - return_when=concurrent.futures.FIRST_COMPLETED, - ) - for future in done: - case = pending.pop(future) - try: - append_jsonl(results_path, future.result()) - completed += 1 - except Exception as error: - append_jsonl( - failures_path, - {**asdict(case), "error": str(error)}, - ) - raise - now = time.monotonic() - if now >= next_progress: - print( - f"Gateway {suite} fuzz progress " - f"elapsed={int(now - started_at)}s " - f"submitted={index} completed={completed} pending={len(pending)}", - flush=True, - ) - next_progress = now + 10 - - contract_count = 0 - if suite in {"formats", "storage"}: - contract_count = run_format_contract_cases(gateway_url, contract_results_path) - - stop_process(serve_process, label="pChronicle serve", require_success=True) - serve_process = None - stop_process(echo_process, label="pChronicle Echo") - echo_process = None - - if suite == "formats": - return index + contract_count, 0 - - events_path = logs / "events.jsonl" - query_captured_events(pchronicle, dataset, results_path, events_path) - call_count, event_count = compare_events( - plan_path, results_path, events_path, logs / "comparison.jsonl" - ) - if suite == "storage": - contract_events_path = logs / "content-contract-events.jsonl" - query_session_events( - pchronicle, - dataset, - list(FORMAT_CONTRACT_SESSIONS), - contract_events_path, - ) - contract_event_count = validate_durable_content_contracts( - contract_events_path, - logs / "content-contract-comparison.jsonl", - ) - return call_count + contract_count, event_count + contract_event_count - return call_count, event_count - finally: - stop_process(serve_process, label="pChronicle serve") - stop_process(echo_process, label="pChronicle Echo") - for handle in handles: - handle.close() - - -def interrupt_on_sigterm(_signum: int, _frame: object) -> None: - raise KeyboardInterrupt - - -def main() -> None: - signal.signal(signal.SIGTERM, interrupt_on_sigterm) - pchronicle = resolve_pchronicle() - suite = os.environ.get("PERSISTING_GATEWAY_FUZZ_SUITE", "storage") - if suite not in FUZZ_SUITES: - raise ValueError( - f"PERSISTING_GATEWAY_FUZZ_SUITE must be one of {sorted(FUZZ_SUITES)}, got {suite!r}" - ) - seed = int(os.environ.get("PERSISTING_FUZZ_SEED", secrets.randbits(64))) - work_dir = Path(tempfile.mkdtemp(prefix=f"persisting-gateway-{suite}-fuzz-{seed}.")) - success = False - print(f"Gateway {suite} fuzz seed={seed} artifacts={work_dir}", flush=True) - try: - if suite == "network-policy": - call_count, event_count = run_network_fuzz(work_dir, pchronicle, seed) - else: - call_count, event_count = run_fuzz(work_dir, pchronicle, seed, suite) - success = True - finally: - keep = os.environ.get("PERSISTING_KEEP_TEST_ARTIFACTS") == "1" - if keep or not success: - print(f"Gateway {suite} fuzz artifacts: {work_dir}", file=sys.stderr) - else: - shutil.rmtree(work_dir) - if suite in {"formats", "network-policy"}: - detail = f"cases={call_count}" - else: - detail = f"calls={call_count} canonical_events={event_count}" - print(f"PASS gateway {suite} fuzz: seed={seed} {detail}", flush=True) - - -if __name__ == "__main__": - main() diff --git a/tests/regression/gateway-fuzz/requirements.txt b/tests/regression/gateway-fuzz/requirements.txt deleted file mode 100644 index 061e8bd46..000000000 --- a/tests/regression/gateway-fuzz/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -openai==3.1.0 -anthropic==0.122.0 -google-genai==2.18.1 diff --git a/tests/regression/gateway-fuzz/run.sh b/tests/regression/gateway-fuzz/run.sh deleted file mode 100755 index 6b45d600b..000000000 --- a/tests/regression/gateway-fuzz/run.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -scenario_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" -suite_duration="${PERSISTING_FUZZ_DURATION_SECONDS:-15}" - -for suite in formats forwarding storage network-policy; do - PERSISTING_FUZZ_DURATION_SECONDS="$suite_duration" \ - bash "$scenario_dir/$suite/run.sh" -done diff --git a/tests/regression/gateway-fuzz/storage/run.sh b/tests/regression/gateway-fuzz/storage/run.sh deleted file mode 100755 index 14af5ba63..000000000 --- a/tests/regression/gateway-fuzz/storage/run.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -scenario_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" -exec env PERSISTING_GATEWAY_FUZZ_SUITE=storage \ - uv run --isolated --no-project \ - --with-requirements "$scenario_dir/requirements.txt" \ - python "$scenario_dir/regression.py" diff --git a/tests/regression/gateway_harness.py b/tests/regression/gateway_harness.py deleted file mode 100644 index 6df97e1fa..000000000 --- a/tests/regression/gateway_harness.py +++ /dev/null @@ -1,124 +0,0 @@ -"""Shared process and filesystem helpers for Gateway black-box regressions.""" - -from __future__ import annotations - -import contextlib -import json -import os -import subprocess -import time -import urllib.error -import urllib.parse -import urllib.request -from pathlib import Path -from typing import Any - -PROXY_ENV_NAMES = {"all_proxy", "http_proxy", "https_proxy", "no_proxy"} - - -def append_jsonl(path: Path, value: dict[str, Any]) -> None: - with path.open("a", encoding="utf-8") as handle: - json.dump(value, handle, ensure_ascii=False, sort_keys=True) - handle.write("\n") - handle.flush() - - -def resolve_binary(env_name: str, default_path: Path, build_hint: str) -> Path: - path = Path(os.environ.get(env_name, default_path)).expanduser().resolve() - if not path.is_file() or not os.access(path, os.X_OK): - raise RuntimeError( - f"required binary is missing or not executable: {path}\n" - f"build it first with: {build_hint}\n" - f"or set {env_name} to an existing binary" - ) - return path - - -def require_subcommand(binary: Path, subcommand: str, build_hint: str) -> None: - result = subprocess.run( - [str(binary), subcommand, "--help"], - stdout=subprocess.DEVNULL, - stderr=subprocess.PIPE, - text=True, - check=False, - ) - if result.returncode != 0: - detail = result.stderr.strip().splitlines()[0] if result.stderr.strip() else "unsupported" - raise RuntimeError( - f"binary does not support {subcommand!r}: {binary}\n" - f"reported: {detail}\n" - f"rebuild it with: {build_hint}" - ) - - -def wait_http(url: str, process: subprocess.Popen[bytes], label: str) -> None: - deadline = time.monotonic() + 10 - while time.monotonic() < deadline: - if process.poll() is not None: - raise RuntimeError( - f"{label} exited before becoming ready (status {process.returncode})" - ) - try: - with urllib.request.urlopen(url, timeout=0.5) as response: - if 200 <= response.status < 300: - return - except (OSError, urllib.error.URLError, TimeoutError): - time.sleep(0.05) - raise TimeoutError(f"timed out waiting for {label} at {url}") - - -def wait_logged_url( - log_path: Path, - prefix: str, - process: subprocess.Popen[bytes], - label: str, -) -> str: - """Discover the address selected by a server that bound port zero.""" - deadline = time.monotonic() + 10 - while time.monotonic() < deadline: - if process.poll() is not None: - details = log_path.read_text(encoding="utf-8", errors="replace") - raise RuntimeError( - f"{label} exited before publishing its address " - f"(status {process.returncode}):\n{details}" - ) - content = log_path.read_text(encoding="utf-8", errors="replace") - for line in content.splitlines(): - if line.startswith(prefix): - candidate = line[len(prefix) :].split()[0].rstrip("/") - parsed = urllib.parse.urlparse(candidate) - if parsed.scheme == "http" and parsed.hostname and parsed.port: - return candidate - time.sleep(0.05) - raise TimeoutError(f"timed out waiting for {label} address in {log_path}") - - -def stop_process( - process: subprocess.Popen[bytes] | None, - *, - label: str, - require_success: bool = False, -) -> None: - if process is None: - return - if process.poll() is None: - process.terminate() - try: - process.wait(timeout=30) - except subprocess.TimeoutExpired: - process.kill() - process.wait(timeout=5) - raise RuntimeError(f"{label} did not stop after SIGTERM") - if require_success and process.returncode != 0: - raise RuntimeError(f"{label} exited with status {process.returncode}") - - -@contextlib.contextmanager -def without_proxy_environment(): - saved = {name: value for name, value in os.environ.items() if name.lower() in PROXY_ENV_NAMES} - for name in saved: - os.environ.pop(name, None) - try: - yield - finally: - os.environ.update(saved) diff --git a/uv.lock b/uv.lock index 78e3573a7..74e9ccc86 100644 --- a/uv.lock +++ b/uv.lock @@ -1217,7 +1217,7 @@ wheels = [ [[package]] name = "persisting" -version = "0.2.0" +version = "0.3.0" source = { editable = "." } dependencies = [ { name = "pulsing" },