diff --git a/Cargo.lock b/Cargo.lock index ff9de569..86894fa5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -199,9 +199,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.4.7" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54413ede23c2daf518f35156dfde027feb2374004d63bd497f983c8db9c0e313" +checksum = "86e055fa6e3e40ad6187ba25b6855eeb635d5271f8e0f0e1e9a5826b8d75bea0" dependencies = [ "find-msvc-tools", "shlex", @@ -271,9 +271,9 @@ dependencies = [ [[package]] name = "crc32fast" -version = "1.5.2" +version = "1.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78" +checksum = "0ba106e844045ddc02c20768ac4b0b7cb45f2cc2ecbd833ac9f5dacb183c2055" dependencies = [ "cfg-if", ] @@ -438,20 +438,20 @@ checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] name = "either" -version = "1.18.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" +checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" [[package]] name = "equivalent" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" [[package]] name = "errno" @@ -497,9 +497,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.13" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef25905e51abafe4dcea6c15fec58c57b601cdbd0ee53d22ea1d3016c587d39b" +checksum = "1c66915cc5391097de7a2b0b972e8505aa33c0df401fd3a6811dbdb32881ba48" [[package]] name = "flate2" @@ -563,7 +563,7 @@ checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -729,13 +729,12 @@ dependencies = [ [[package]] name = "hyper" -version = "1.11.1" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27b501faa50e7a26c3d3560ca625132f4078a17771f4810baf70475ae48cbe43" +checksum = "2c3e324da4c95177d6291d4c8730197c0d1822f8a9766814a4a44fa5ab797c9c" dependencies = [ "atomic-waker", "bytes", - "futures-channel", "futures-core", "http", "http-body", @@ -790,9 +789,9 @@ dependencies = [ [[package]] name = "icu_collections" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" +checksum = "fe090a42a0d3140fed8c63263ed1fecc02422cb2f4bfe55cd8de2de1b8e105dd" dependencies = [ "displaydoc", "potential_utf", @@ -804,9 +803,9 @@ dependencies = [ [[package]] name = "icu_locale_core" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" +checksum = "25d29b57c17c6528d0dc4c2141cfa2d08200465acb537a3909c8b9b8a4a984c0" dependencies = [ "displaydoc", "litemap", @@ -817,9 +816,9 @@ dependencies = [ [[package]] name = "icu_normalizer" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" +checksum = "404a3b8c486f205c87fe7f23ab6894d84a4ad76d6e4671ae1981ba7def87c599" dependencies = [ "icu_collections", "icu_normalizer_data", @@ -831,15 +830,15 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" +checksum = "20f0088bfb7441f7a37fa0c9e20cb1e9cba1cac6b9ce8cd2a6dc44294f8e8cbc" [[package]] name = "icu_properties" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" +checksum = "2ea83d991c56158f7cf898bf86436c1dfe7c07ee82fe88951f2695c9f37a379f" dependencies = [ "displaydoc", "icu_collections", @@ -852,15 +851,15 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" +checksum = "b98f29af82d02b163fe50161388350e743bb3107cc8e8c2f7314cb505faddfcf" [[package]] name = "icu_provider" -version = "2.3.1" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" +checksum = "413099315636c588ee8f011190b329b971a09a8fae9d157e01dcae8379468514" dependencies = [ "displaydoc", "icu_locale_core", @@ -968,9 +967,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.189" +version = "0.2.191" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" [[package]] name = "libloading" @@ -1042,9 +1041,9 @@ checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" [[package]] name = "memmap2" -version = "0.9.11" +version = "0.9.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0" +checksum = "6b0ca1155a9ecc25aa12e8313a119f32edf0255fea2f44839c4ac0a12b2ac157" dependencies = [ "libc", ] @@ -1083,9 +1082,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.2.3" +version = "1.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b18443e9c262bfe8fa82f51666e2642c53393f7e5c27b3e1aeab922cff5b9d8" +checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee" dependencies = [ "libc", "wasi", @@ -1245,7 +1244,7 @@ dependencies = [ "omni-runtime", "safetensors 0.8.0", "serde_json", - "sha2 0.11.0", + "sha2 0.11.1", "tempfile", "tokenizers 0.22.2", "tokio", @@ -1341,6 +1340,21 @@ dependencies = [ "tokio", ] +[[package]] +name = "omni-vela-native" +version = "0.1.0" +dependencies = [ + "anyhow", + "libm", + "omni-qwen3-5-native", + "onig", + "safetensors 0.8.0", + "serde", + "serde_json", + "sha2 0.11.1", + "tokenizers 0.22.2", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -1473,9 +1487,9 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.18" +version = "0.11.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9746dbde176634f4f2f1faf2404e30a31b2bc1e9cafb5329c95d8177a18c9fc" +checksum = "0e750cca55fe4f0439a15d0bb529da9651e79993e8e72c61a899a36d462befbe" dependencies = [ "bytes", "getrandom 0.4.3", @@ -1495,9 +1509,9 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.15" +version = "0.5.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" +checksum = "af66907df18639dcf4db56ca65490cabc4b27a97dbadd96f2926cca73298f016" dependencies = [ "cfg_aliases", "libc", @@ -1720,9 +1734,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.45" +version = "0.23.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" dependencies = [ "once_cell", "ring", @@ -1744,9 +1758,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.15" +version = "0.103.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" +checksum = "4443b1b6d39d7c663139942cc4558262562d2c84123bb6282ef177009ca8b281" dependencies = [ "ring", "rustls-pki-types", @@ -1815,15 +1829,16 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] name = "serde_json" -version = "1.0.151" +version = "1.0.154" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" dependencies = [ + "foldhash", "indexmap", "itoa", "memchr", @@ -1868,9 +1883,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.11.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +checksum = "47d7069beb7d6ac7b9acd1039986e73443f24234f41074da099d6f994ac9ad19" dependencies = [ "cfg-if", "cpufeatures 0.3.1", @@ -1907,9 +1922,9 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "smallvec" -version = "1.16.2" +version = "1.16.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9395f0f0eee849a9b707b2f06bb92a6a422090e2123bb2ef8e87a0e61892a8e" +checksum = "d0d5d4b1981b8bbbb7aed204cae5726d683d5d6ac7e74ec54122ea28f315e7c8" [[package]] name = "socket2" @@ -1970,9 +1985,9 @@ dependencies = [ [[package]] name = "syn" -version = "3.0.6" +version = "3.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" dependencies = [ "proc-macro2", "quote", @@ -1996,7 +2011,7 @@ checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -2029,14 +2044,14 @@ checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] name = "tinystr" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" +checksum = "abc840c238154d58f9e86b2792261edfde565a17696e1cda1a449de27979801a" dependencies = [ "displaydoc", "zerovec", @@ -2116,9 +2131,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.53.1" +version = "1.54.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +checksum = "b52fd3255258d26fac7264ab5a76b53c13367136dc011d3d6ad4f8ed070396a3" dependencies = [ "bytes", "libc", @@ -2138,14 +2153,14 @@ checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] name = "tokio-rustls" -version = "0.26.5" +version = "0.26.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0c85f2c3ef0b1cd58b36682f4b17aaa995f0e5db534d85692b4903abce21f67" +checksum = "c9cc2678c2cdd569ef8215e2afd7954ada2ae20b4fdd2c5fe6139a3b02d105db" dependencies = [ "rustls", "tokio", @@ -2153,9 +2168,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.19" +version = "0.7.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" +checksum = "e464cf451ba96ebfc6f9b6542f17ee8b8956e33f1e40d9690624e59d7a7f8a4b" dependencies = [ "bytes", "futures-core", @@ -2244,9 +2259,9 @@ checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" [[package]] name = "unicode-ident" -version = "1.0.26" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" [[package]] name = "unicode-normalization-alignments" @@ -2301,9 +2316,9 @@ checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] name = "want" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +checksum = "ec4cdd0dd910afe868b7ef477227d8d538b46b3075031afee8a9f2acb0a2ed0b" dependencies = [ "try-lock", ] @@ -2366,7 +2381,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", "wasm-bindgen-shared", ] @@ -2414,9 +2429,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "1.0.9" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" +checksum = "f5f280bbb2b287bc8e7de7e3852945a0ff50d1bbcb5ed42b6c7a568c546df171" dependencies = [ "rustls-pki-types", ] @@ -2523,9 +2538,9 @@ checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" [[package]] name = "yoke" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +checksum = "417ef3fcf6c300b7dbd25c85531a746931c02ad536f3cf4786ed1e88223340b3" dependencies = [ "stable_deref_trait", "yoke-derive", @@ -2534,30 +2549,30 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33811428bee40dbceb6d545e95754741d17a6aef9a4849f0fd62e2ba4f412a78" +checksum = "ec8ebde2db3681e8c9980cc27822030e68752690ddfa9473e739aeb4dbde6d71" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", "synstructure", ] [[package]] name = "zerocopy" -version = "0.8.59" +version = "0.8.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6df92bf3d9227be3d53173901ddbffac2babc27ae50f397776ffd6dc33f800cb" +checksum = "e5fe1f8f1b06191a00962174c61aa5005e0bb391a6d80d07e24d115c01a92ed8" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.59" +version = "0.8.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac4f328cf2f05d084e496c3e9c3f33ed0a183656a16e1fcec4d464d8373aec82" +checksum = "863ad3ac83293fb4d740aedbfdc9240dd8d1a50c1099acd76ce80ce7c7230c7f" dependencies = [ "proc-macro2", "quote", @@ -2581,15 +2596,15 @@ checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", "synstructure", ] [[package]] name = "zeroize" -version = "1.9.0" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" +checksum = "e13084392c5e4bc371903e2935a5eaeed24905a7511356b883835e18a78f6879" [[package]] name = "zerotrie" @@ -2604,9 +2619,9 @@ dependencies = [ [[package]] name = "zerovec" -version = "0.11.8" +version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" +checksum = "f58a2799cfe9a1670c7ea8edbd52ef8edb93cb37a7f83bf31ba9e35d9dae6a40" dependencies = [ "yoke", "zerofrom", @@ -2615,13 +2630,13 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.6" +version = "0.11.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" +checksum = "c3743ddbf47738d929cd1ae2e199d40d0162979dfe31aef8ae646d2bf00bf97a" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index d4d00a3d..13ec2aa8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,3 +1,3 @@ [workspace] -members = ["src/frontend", "src/runtime", "src/models/clm", "src/models/cua_s1/native", "src/models/qwen3_5/native", "src/models/open_jev/native", "src/models/jev_vl/native", "src/models/omnijev/native", "src/models/laya", "src/backends/cuda", "src/models/decider/native", "src/models/jemm/native", "src/models/startlux/native"] +members = ["src/frontend", "src/runtime", "src/models/clm", "src/models/cua_s1/native", "src/models/qwen3_5/native", "src/models/open_jev/native", "src/models/jev_vl/native", "src/models/omnijev/native", "src/models/laya", "src/backends/cuda", "src/models/decider/native", "src/models/jemm/native", "src/models/vela/native", "src/models/startlux/native"] resolver = "3" diff --git a/README.md b/README.md index 6e35c3ed..670cb82f 100644 --- a/README.md +++ b/README.md @@ -167,9 +167,10 @@ Python screenshot worker, Valen-Preview-0923 has a Python reference worker for text and one-image state, CLM has a stub-encoder contract recipe, and OmniJev-4B v1.1 runs on a native Rust/CUDA worker. D3 (Decision 3.0) has a Python reference worker that runs the checkpoint's own -pinned runtime for text, image and video state, and Vela 2.0 0.3B has one -that serves all five SystemOne question kinds (`choice`/`noul`/`score` plus -the `span`/`set` extensions) on the checkpoint's own pinned runtime. +pinned runtime for text, image and video state, and the Vela 2.0 members (0.3B and the 0.8B +hybrid) have one that serves all five +SystemOne question kinds (`choice`/`noul`/`score` plus the `span`/`set` +extensions) on each checkpoint's own pinned runtime. pplx-decider-v1.1-27b has a Python reference worker for the pinned Perplexity checkpoint. Public231 concurrency-1 latency for heads `1ba07d1`, `f968e576`, `ffae182`, `4b99b73`, and `fcf5e2b` is recorded in the recipe: @@ -191,6 +192,7 @@ checkpoint. Public231 concurrency-1 latency for heads `1ba07d1`, `f968e576`, | OmniJev-4B v1.1 | [Native Rust/CUDA worker](recipe/omnijev/native.md), eager: one image, Choice, Noul and Score; [reference comparison on sm_89](recipe/omnijev/native.md#comparison-with-the-reference) | | D3 (Decision 3.0) | [Python reference worker](recipe/d3/reference.md) on the checkpoint's pinned runtime; `choice`/`noul`/`score` over text, image and video state; author-run on A100 for text and media serving (see the integration and media PRs) | | Vela 2.0 0.3B | [Python reference worker](recipe/vela/reference.md) on the checkpoint's pinned runtime; all five kinds (`choice`/`noul`/`score`/`span`/`set`) for text state; CPU FP32 validated (author-run, evidence in the integration PR) | +| Vela 2.0 0.8B hybrid | [Python reference worker](recipe/vela/reference.md) on the checkpoint's pinned runtime; same surface plus `span_heads`; [native library](recipe/vela/native.md) (bundle export, wire contract, processing, F32 heads); CPU FP32 parity recorded in the integration PR | | Clef-flash (9B) and Clef (27B) | [Python reference worker](recipe/clef/reference.md) for text `choice`/`noul`/`score`; flash is the bring-up variant. Image, video and native CUDA are not served. L20X-labeled lab Public231 C1 primary is tip `594528a9`: p50 164.969 ms, p95 216.755 ms, `/health` `fla+causal_conv1d`, versus `9030065` run1 p50 247.518 ms, p95 413.025 ms (Δp50 −82.549 ms, Δp95 −196.270 ms) | | pplx-decider-v1.1-27b | [Python reference worker](recipe/pplx_decider/README.md) on the pinned checkpoint; `choice`/`noul`/`score` text plus up to four images. Lab L20X-labeled GPUs are the H200 target. Kept Public231 primary is `fcf5e2b`: 231/231, p50 87.782 ms, p95 291.183 ms, grouped-value FlashInfer with q and k packed before the L2 view (p50 −1.30%, p95 −1.56% versus `4b99b73` 88.942 / 295.797). `4b99b73` was p50 −4.20%, p95 −0.22% versus `ffae182` 92.844 / 296.458. Uncached all-sequence FlashInfer `3c72f02` is not the primary (p50 88.808 ms, p95 300.751 ms). Short-sequence FlashInfer `a50b9aa` missed its gate (p50 95.446 ms, p95 300.871 ms). cuDNN-first SDPA `0dcaa0ac` missed its gate (p50 138.501 ms, p95 332.859 ms). Fused gate-up `8b4ee326` missed its gate (p50 102.437 ms, p95 304.970 ms). Always-on dynamic compile `e36b32ed` missed the both-percentiles gate (p50 97.751 ms, p95 263.124 ms; p95 improved). The 512-token split `7e3b1be` missed its gate (p50 100.037 ms, p95 257.084 ms). Isolated-median compile `34ba5de` missed its gate (p50 97.084 ms, p95 256.679 ms; the 136-token probe was eager). The FlashInfer state-workspace ring `17a7b120` missed its gate (p50 89.165 ms, p95 295.723 ms). Always-on FLA residual RMSNorm `3afb472` missed its gate (p50 93.935 ms, p95 274.902 ms). Length-gated FLA residual RMSNorm `705ae089` missed its gate (p50 94.604 ms, p95 274.835 ms). `use_cp=auto` `a6f4d9a` missed its gate (p50 112.294 ms, p95 299.419 ms). Grouped-value `3ecdf5a` crashed in warmup (0/231, no p50 or p95). FLA fused gated RMSNorm `801ef7b` missed its gate (p50 89.172 ms, p95 274.790 ms). FLA `swiglu_fwd` `2bbbe4e` missed its gate (owner closeout p50 97.613 ms, p95 292.795 ms) and is not installed | diff --git a/docs/supported-models.md b/docs/supported-models.md index c5dbc4c2..4e6dc450 100644 --- a/docs/supported-models.md +++ b/docs/supported-models.md @@ -29,6 +29,7 @@ Models that are being added are also tracked in issues labeled [new model](https | OmniJev-4B v1.1 | [Native Rust/CUDA worker](../recipe/omnijev/native.md) on the shared Qwen3.5 vision and language path, eager | Contract, preparation, pixels, heads and finishing checked against the reference; no CPU inference | Compared with the reference on compute capability 8.9 for [140 answers](../recipe/omnijev/native.md#comparison-with-the-reference) | Not supported | Compute capability 8.0 or newer, CUDA toolkit to build, the export from `export.py` | | D3 (Decision 3.0) | [Python reference worker](../recipe/d3/reference.md) on the checkpoint's pinned runtime — text, image and video state over `choice`/`noul`/`score`, with the deliberate transport differences documented in the recipe; the native worker is in progress ([#136](https://github.com/ThinkFlowLab/system1-omni/issues/136)) | Unverified | Author-run on NVIDIA A100 (sm_80): serving smoke and reference parity for text and media (exact decisions, max delta 0.0); evidence in the integration and media PRs. Owner: @MrDongsls. | Unverified | Python 3.12, the [pinned runtime requirements](../recipe/d3/requirements.txt), the [pinned package](https://huggingface.co/vllm-sr/d3) (~52 GB); [media budgets and the 64 MiB total request cap](../recipe/d3/reference.md) | | Vela 2.0 0.3B | [Python reference worker](../recipe/vela/reference.md) running the checkpoint's pinned runtime | **CPU FP32 validated** (author-run, x86 WSL2, 24 cores): startup with all-kind warmup, the four example requests plus five synthetic rejections, and exact-match reference parity against the pinned upstream server on all nine (four 200s byte-equal incl. key order, agreed 422/413); byte-identical frontend forwarding; evidence in the integration PR. Owner: @MrDongsls. | Unverified (bf16-autocast is an untested deployment option) | Not supported | Python 3.12, the [pinned requirements](../recipe/vela/requirements.txt), the [pinned package](https://huggingface.co/vllm-sr/Vela-2.0-0.3B) (1.24 GB FP32 weights; ~4.4 GB with ONNX/assets) | +| Vela 2.0 0.8B hybrid | [Python reference worker](../recipe/vela/reference.md) running the checkpoint's pinned runtime (same wire layer as 0.3B); [native library](../recipe/vela/native.md) — bundle export, wire contract, processing, F32 heads and CUDA executor (no HTTP worker) | Same surface as 0.3B plus the `span_heads` field (router/broad dispatch); **CPU FP32 parity** (author-run; evidence in the integration PR) | Serving unverified; native executor parity author-run on sm_120 (evidence in the PR) | Not supported | Python 3.12, the [pinned requirements](../recipe/vela/requirements.txt), the [pinned package](https://huggingface.co/vllm-sr/Vela-2.0-0.8B) (2.02 GB shard + 4.5 MB broad head; ~3 GB FP32 parameters) | | Clef-flash (9B) | [Python reference worker](../recipe/clef/reference.md); text `choice`/`noul`/`score`; non-thinking prompt; `thinking.enabled: false` admitted and omitted from the prompt; 16384-token refusal | Contract tests only; no inference | Operator-reported Public231 C1 on the L20X-labeled lab host. Baseline `9030065` run1 p50 247.518 ms, p95 413.025 ms, 231/231. Tip `594528a9` remeasure, warmup 5, concurrency 1, stock Public231 with `thinking.enabled` false, metric `successful_latency`: 231/231, p50 164.969 ms, p95 216.755 ms, `/health` `fla+causal_conv1d`. Versus run1: Δp50 −82.549 ms (−33.35%), Δp95 −196.270 ms (−47.52%). Not re-run here | Not supported | Python 3.12, [card-tested torch 2.11 and transformers 5.10.2](../recipe/clef/requirements.txt), pinned `Cloudflare/clef-flash` snapshot (~19.1 GB) | | Clef (27B) | Same worker and text contract as Clef-flash, selected with `--variant clef` | Contract tests only; no inference | Not run | Not supported | Same stack; pinned `Cloudflare/clef` snapshot (~55.0 GB). Weights and joint-head config differ from flash | | pplx-decider-v1.1-27b | [Python reference worker](../recipe/pplx_decider/README.md) on the checkpoint's own runtime. Text and up to four images; `choice`/`noul`/`score`. Native CUDA executor is not this worker. Lab L20X-labeled GPUs are the H200 target. The causal-conv1d build keeps sm_90 and sm_89. Capability 8.9 is ready on FLA chunk. Capability 9.0 uses all-sequence FlashInfer. The kept primary rotates 48 `cu_seqlens` buffers per device and length, passes `use_cp` false, skips the q/k repeat, and packs q and k before the L2-norm view. `801ef7b` fused the 48 linear gated RMSNorms and missed the median (p50 89.172 ms, p95 274.790 ms), so that kernel stays off. `2bbbe4e` replaced `silu(gate) * up` with FLA `swiglu_fwd` and the owner closeout rejected it (p50 97.613 ms, p95 292.795 ms), so that kernel stays off. Gate and up GEMMs stay separate. Residual RMSNorm and the gated output norm stay eager. An unknown `PPLX_GDN_PREFILL` fails closed on every device, including capability 8.9. `3ecdf5a` crashed in warmup (0/231). Always-on fusion `3afb472` and the 512-token gate `705ae089` missed the median. The 48-slot FlashInfer state ring `17a7b120` missed the gate. `PPLX_GDN_PREFILL=fla` reproduces the prior `f968e576` chunk. `PPLX_GDN_PREFILL=short` is the rejected ≤256 dispatch. SDPA stays on the checkpoint order. The fused gate-up GEMM is refused. The forward stays eager. `PPLX_COMPILE=dynamic` and `PPLX_COMPILE=dynamic-tail` are rejected opt-ins | Contract checks only; no inference | Reported load on 1× lab L20X (sm_89, ~140 GiB) at head `1ba07d1`: BF16 resident ~50333 MiB, no OOM. Historical primary: 231/231, p50 98.777 ms, p95 341.202 ms. Prior primary is the `f968e576` remeasure on the lab L20X-labeled GPU, `compute_capability=9.0`, `fla-chunk`, `flashinfer` null: 231/231, p50 94.077 ms, p95 300.200 ms (p50 −4.76%, p95 −12.02%). Prior primary `ffae182`: 231/231, p50 92.844 ms, p95 296.458 ms (p50 −1.31%, p95 −1.25%). Prior primary `4b99b73`: 231/231, p50 88.942 ms, p95 295.797 ms (p50 −4.20%, p95 −0.22%), 48-slot `cu_seqlens`, quality `choice_acc=0.9065` (139), `noul_acc=0.8784` (74), `score_mae=0.2035` (18). Kept primary is `fcf5e2b`: 231/231, p50 87.782 ms, p95 291.183 ms (p50 −1.30%, p95 −1.56%), grouped-value FlashInfer with q and k packed, same quality digits. Uncached all-sequence FlashInfer `3c72f02`: p50 88.808 ms, p95 300.751 ms, gate failed. Short-sequence FlashInfer `a50b9aa`: p50 95.446 ms, p95 300.871 ms, gate failed. cuDNN-first SDPA `0dcaa0ac`: p50 138.501 ms, p95 332.859 ms, gate failed. Fused gate-up `8b4ee326`: p50 102.437 ms, p95 304.970 ms, gate failed. Always-on dynamic compile `e36b32ed`: p50 97.751 ms, p95 263.124 ms, gate failed (p95 improved). 512-token compile `7e3b1be`: p50 100.037 ms, p95 257.084 ms, gate failed. Isolated-median compile `34ba5de`: p50 97.084 ms, p95 256.679 ms, gate failed (median probe was eager). FlashInfer state-workspace ring `17a7b120`: p50 89.165 ms, p95 295.723 ms, gate failed. Always-on FLA residual RMSNorm `3afb472`: p50 93.935 ms, p95 274.902 ms, gate failed. Length-gated FLA residual RMSNorm `705ae089`: p50 94.604 ms, p95 274.835 ms, gate failed. `use_cp=auto` `a6f4d9a`: p50 112.294 ms, p95 299.419 ms, gate failed. Grouped-value `3ecdf5a` crashed in warmup (0/231, no p50 or p95). The fused linear gated-norm candidate has no sample. Does not enter the H800 board | Not supported | Python 3.12, pinned checkpoint revision `14c9d90137962d300f3de95633f7b6535e80c5ce`, checkpoint `requirements.txt`, `causal-conv1d` 1.7.0, `flash-linear-attention` 0.5.2, optional `flashinfer-python` 0.6.16.post3. Shard sizes in the release manifest sum with the readout to about 48.6 GiB before activations | @@ -39,14 +40,16 @@ Models that are being added are also tracked in issues labeled [new model](https - **Planned:** not implemented yet; the linked issue tracks it. The Cua-S1 workers answer `choice` questions only. +LAYA's English worker, Open-Jev and CLM support `choice`, `score`, and `noul` text questions. LAYA's English worker, Open-Jev, CLM, and StartLux Instant 4B support `choice`, `score`, and `noul` text questions. StartLux serves Instant only; AutoThink is not implemented, and the weights are CC BY-NC 4.0. D3's reference worker serves the same three kinds for text, image and video state (inline base64 media under the pinned runtime's per-item budgets); its media serving evidence is author-run on the A100 above. -Vela 2.0 0.3B's reference worker serves all five kinds (`choice`, `noul`, -`score`, plus the `span`/`set` extensions with their Noul views) for text -state on CPU FP32; the hybrid members (0.8B/4B/9B) are separate slices. +Vela 2.0's reference workers (0.3B and the 0.8B hybrid) serve all five +kinds (`choice`, `noul`, `score`, plus the `span`/`set` extensions with +their Noul views, and `span_heads` on the hybrid) for text state on CPU +FP32; the larger hybrid members (4B/9B) are separate slices. Clef-flash and Clef serve the same three kinds for text only. The worker refuses image and video inputs. The flash CUDA cell records two operator-reported Public231 concurrency-1 results on the L20X-labeled lab diff --git a/mkdocs.yml b/mkdocs.yml index 7e3be0ad..b4bc3858 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -84,7 +84,8 @@ nav: - CLM stub-encoder contract: recipe/clm/README.md - LFM2.5-350M choice worker: recipe/lfm2/README.md - D3 reference worker: recipe/d3/reference.md - - Vela 2.0 0.3B reference worker: recipe/vela/reference.md + - Vela 2.0 reference workers: recipe/vela/reference.md + - Vela 2.0 0.8B native library: recipe/vela/native.md - Clef reference worker: recipe/clef/reference.md - pplx-decider reference worker: recipe/pplx_decider/README.md - Contributing: CONTRIBUTING.md diff --git a/recipe/README.md b/recipe/README.md index 041c091d..c7a5c294 100644 --- a/recipe/README.md +++ b/recipe/README.md @@ -48,10 +48,14 @@ For a first real decision, follow the [complete CPU walkthrough](../docs/getting verify its runtime manifest, and serve `choice`/`noul`/`score` decisions over text, image and video state through the Python worker and the Rust frontend (text serving and parity validated on A100; media budgets and caps in the recipe). -- [Vela 2.0 0.3B reference worker](vela/reference.md): download the pinned +- [Vela 2.0 reference workers (0.3B and the 0.8B hybrid)](vela/reference.md): download the pinned vllm-sr/Vela-2.0-0.3B package, pass its full-hash package gate, and serve all five SystemOne question kinds (`choice`/`noul`/`score` plus the `span`/`set` - extensions) through the Python worker and the Rust frontend on CPU FP32. + extensions) through the Python worker and the Rust frontend on CPU FP32; + or vllm-sr/Vela-2.0-0.8B at revision `326b01d8` behind its own 10-file gate. +- [Vela 2.0 0.8B native library](vela/native.md): export the pinned-bundle + manifest gate, wire contract and token-level processing for the 0.8B hybrid, + checked against frozen fixtures; serving stays with the reference Python worker. - [Clef reference worker](clef/reference.md): load pinned Clef-flash (9B) or Clef (27B) and serve text `choice`/`noul`/`score`. Image, video and native CUDA are not served. Operator-reported flash Public231 concurrency-1 diff --git a/recipe/vela/compare_reference.py b/recipe/vela/compare_reference.py index fde4cd15..3588e06a 100644 --- a/recipe/vela/compare_reference.py +++ b/recipe/vela/compare_reference.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Compare the Vela 2.0 0.3B worker pipeline against the pinned upstream server. +"""Compare the Vela 2.0 worker pipeline against the pinned upstream server. Runs each fixture request through two paths that share one loaded pinned engine, so the comparison isolates this repository's admission and @@ -13,8 +13,8 @@ finishing). Both paths execute the same verified runtime, and the worker delegates -response construction to the same pinned code — so the criterion is the -charter's reference parity: exact match, including JSON key order, with +response construction to the same pinned code — so the criterion is +reference parity: exact match, including JSON key order, with rejections agreeing on status and body (``{"detail": [{"loc", "msg", "type"}]}``). ``--export`` additionally freezes the oracle fixture — request, SystemOne response and the predict()-level answers — for the @@ -24,11 +24,14 @@ The default fixture set is the recipe's four example requests plus five synthetic rejections: an invalid span question (whole-request -422), a schema that alone exceeds the 8,192-token window (413), and +422), a schema that alone exceeds the member's input window (413), and the three envelope 422s (missing field, extra field, wrong model name) that freeze the worker's preamble admission against the pinned -runtime's. CPU FP32 is the oracle setting; GPU bf16-autocast is a -deployment option, not this tool's default. +runtime's. The example files' ``model`` field is rewritten to the +served member's name, and the hybrid member's oracle fixtures export +under an ``oracle-hybrid-`` prefix, so both members share one tool. +CPU FP32 is the oracle setting; GPU bf16-autocast is a deployment +option, not this tool's default. """ from __future__ import annotations @@ -44,7 +47,7 @@ def parse_args(argv=None): parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) - parser.add_argument("--model", required=True, help="pinned Vela 2.0 0.3B package directory") + parser.add_argument("--model", required=True, help="pinned Vela 2.0 package directory") parser.add_argument("--device", default="cpu", help="torch device (default: the CPU FP32 oracle)") parser.add_argument( "--fixture", @@ -62,8 +65,8 @@ def parse_args(argv=None): return parser.parse_args(argv) -def over_limit_body(): - """A schema that alone cannot fit the 8,192-token window. +def over_limit_body(model=MODEL_NAME): + """A schema that alone cannot fit the member's input window. 255 choice options with ~300-character descriptions: far past ``max_length`` before any state is rendered, which is exactly the @@ -79,7 +82,7 @@ def over_limit_body(): for index in range(255) } return { - "model": MODEL_NAME, + "model": model, "state": "short", "questions": { "big": {"type": "choice", "instructions": "Pick one.", "criteria": criteria} @@ -87,11 +90,11 @@ def over_limit_body(): } -def invalid_span_body(): +def invalid_span_body(model=MODEL_NAME): """A span question whose criteria is a list, not a label mapping.""" return { - "model": MODEL_NAME, + "model": model, "state": "short", "questions": { "bad": { @@ -109,50 +112,63 @@ def missing_model_body(): return {"state": "short", "questions": {"q": {"type": "noul", "instructions": "any?"}}} -def extra_field_body(): +def extra_field_body(model=MODEL_NAME): """An envelope with a field the preamble does not permit ('extra_forbidden').""" return { - "model": MODEL_NAME, + "model": model, "state": "short", "questions": {"q": {"type": "noul", "instructions": "any?"}}, "images": [], } -def wrong_model_body(): +def wrong_model_body(served=MODEL_NAME): """Another member's name on this runtime ('value_error.model' 422).""" + other = "vllm-sr/Vela-2.0-0.3B" if served.endswith("0.8B") else "vllm-sr/Vela-2.0-0.8B" return { - "model": "vllm-sr/Vela-2.0-0.8B", + "model": other, "state": "short", "questions": {"q": {"type": "noul", "instructions": "any?"}}, } -def default_cases(): +def default_cases(served=MODEL_NAME): return [ ("example-request", HERE / "example-request.json"), ("example-request-set-threshold", HERE / "example-request-set-threshold.json"), ("example-request-cjk", HERE / "example-request-cjk.json"), ("example-request-score", HERE / "example-request-score.json"), - ("invalid-span-422", invalid_span_body()), - ("over-limit-413", over_limit_body()), + ("invalid-span-422", invalid_span_body(served)), + ("over-limit-413", over_limit_body(served)), ("missing-model-422", missing_model_body()), - ("extra-field-422", extra_field_body()), - ("wrong-model-422", wrong_model_body()), + ("extra-field-422", extra_field_body(served)), + ("wrong-model-422", wrong_model_body(served)), ] -def load_cases(paths): +def load_cases(paths, served=MODEL_NAME): if not paths: - cases = default_cases() + cases = default_cases(served) else: cases = [(Path(path).name, path) for path in paths] - return [ - (name, body if isinstance(body, dict) else json.loads(Path(body).read_text(encoding="utf-8"))) - for name, body in cases - ] + loaded = [] + for name, source in cases: + body = source if isinstance(source, dict) else json.loads(Path(source).read_text(encoding="utf-8")) + # The example files are member-agnostic apart from their model field; + # serving the other member rewrites it so the case exercises the 200 + # path it was written for, on both compared paths alike. Synthetic + # bodies (including the deliberate wrong-model one) are never touched. + if ( + not isinstance(source, dict) + and isinstance(body, dict) + and body.get("model") != served + and body.get("model") in (MODEL_NAME, "vllm-sr/Vela-2.0-0.8B") + ): + body = {**body, "model": served} + loaded.append((name, body)) + return loaded def reference_response(client, body): @@ -245,10 +261,10 @@ def main(argv=None): sys.path.insert(0, str(HERE.parents[1] / "src")) from fastapi.testclient import TestClient - from frontend.vela import decide - from models.vela.engine import Vela2Executor + from frontend.vela import decide, detect_member - executor = Vela2Executor(args.model, device=args.device) + executor_class, _default_name = detect_member(args.model) + executor = executor_class(args.model, device=args.device) # Both runtime imports stay inside a no-bytecode window: the package # gate refuses existing caches, and vela2_serve is a fresh import that # would otherwise write the __pycache__ the next worker start rejects. @@ -262,11 +278,15 @@ def main(argv=None): app = vela2_serve.create_app(executor.engine, model_name=executor.model_name, api_key=None) client = TestClient(app) + served = executor.model_name + # The hybrid member's oracle fixtures get their own prefix so both + # members' frozen baselines can live in one directory. + prefix = "oracle-hybrid-" if getattr(executor, "is_hybrid", False) else "oracle-" exported = 0 failed = 0 try: - for name, body in load_cases(args.fixture): + for name, body in load_cases(args.fixture, served): raw = json.dumps(body).encode("utf-8") worker = worker_response(raw, executor, decide) reference = reference_response(client, body) @@ -278,7 +298,7 @@ def main(argv=None): req, _plan = vela2_inference.system_one_to_predict(body["state"], body["questions"]) target = Path(args.export) if target.is_dir(): - target = target / f"oracle-{name}.json" + target = target / f"{prefix}{name}.json" export_oracle(name, body, worker[1], executor.engine.predict(req), target) exported += 1 print() diff --git a/recipe/vela/export_head_inputs.py b/recipe/vela/export_head_inputs.py new file mode 100644 index 00000000..625eadd0 --- /dev/null +++ b/recipe/vela/export_head_inputs.py @@ -0,0 +1,226 @@ +#!/usr/bin/env python3 +"""Freeze the head inputs/outputs of the pinned Vela 2.0 0.8B runtime. + +Records, per frozen case, exactly what +the two F32 heads consume and produce — CandidateHead (option-end +hidden states + the block query hidden) and SpanHeadV2 (word hidden +states + label-mean hiddens), on CPU FP32 with autocast off (the +heads' own setting, bit-deterministic) — so the Rust heads and the +scoring/decode mirror are checked against these without any GPU. Also +records the reference's calibrated predict answers for the same cases +(the scoring target). Hidden matrices go to a per-case safetensors +file; structure, small logits and answers go to JSON next to it. + + PYTHONPATH=src .venv/bin/python recipe/vela/export_head_inputs.py \ + --model weights/vela-2.0-0.8b --out tests/vela/fixtures/ +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +import numpy as np +from safetensors.numpy import save_file + +HERE = Path(__file__).resolve().parent +MODEL_NAME = "vllm-sr/Vela-2.0-0.8B" + + + +def parse_args(argv=None): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--model", required=True, help="pinned Vela 2.0 0.8B package directory") + parser.add_argument("--out", default=None, help="fixture output directory") + return parser.parse_args(argv) + + +def long_span_body(): + filler = " ".join(f"Sentence {i} reports a routine delivery event." for i in range(220)) + return { + "model": MODEL_NAME, + "state": {"request": filler}, + "questions": { + "claims": { + "type": "span", + "instructions": "Which spans mention a delivery?", + "criteria": {"delivery": "a delivery event"}, + "over": "request", + } + }, + } + + +def cases(): + named = [ + ("example-request", HERE / "example-request.json"), + ("example-request-set-threshold", HERE / "example-request-set-threshold.json"), + ("example-request-cjk", HERE / "example-request-cjk.json"), + ("example-request-score", HERE / "example-request-score.json"), + ] + loaded = [ + (name, {**json.loads(path.read_text(encoding="utf-8")), "model": MODEL_NAME}) + for name, path in named + ] + loaded.append(("long-span-windowed", long_span_body())) + return loaded + + +def plain(value): + if isinstance(value, dict): + return {key: plain(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [plain(item) for item in value] + if isinstance(value, np.ndarray): + return plain(value.tolist()) + if isinstance(value, (np.integer,)): + return int(value) + if isinstance(value, (np.floating,)): + return float(value) + if isinstance(value, (np.bool_,)): + return bool(value) + return value + + +class Recorder: + """Class-level wrappers around the two heads; records inputs/outputs.""" + + def __init__(self, module, model): + self.calls: list[dict] = [] + self.tensors: dict[str, np.ndarray] = {} + self._on = False + # Tensor names must never collide across rows: the counter keeps + # growing for the whole case. + self.next_index = 0 + self.module = module + self.model = model + original_head = module.CandidateHead.forward + original_span = module.SpanHeadV2.forward + recorder = self + + def head_forward(self, candidates, query): + out = original_head(self, candidates, query) + if recorder._on: + index = recorder.next_index + cname, qname = f"q{index}_cand", f"q{index}_query" + recorder.tensors[cname] = candidates.detach().cpu().numpy().astype(np.float32) + recorder.tensors[qname] = query.detach().cpu().numpy().astype(np.float32) + recorder.next_index = index + 1 + recorder.calls.append( + dict(kind="q", cand=cname, query=qname, + logits=[row.tolist() for row in out.detach().cpu().numpy()]) + ) + return out + + def span_forward(self, words, labels): + out = original_span(self, words, labels) + if recorder._on: + index = recorder.next_index + wname, lname = f"s{index}_words", f"s{index}_labels" + which = "broad" if self is getattr(recorder.model, "span_broad", None) else "router" + recorder.tensors[wname] = words.detach().cpu().numpy().astype(np.float32) + recorder.tensors[lname] = labels.detach().cpu().numpy().astype(np.float32) + recorder.next_index = index + 1 + recorder.calls.append( + dict(kind="span", head=which, words=wname, labels=lname, + logits=out.detach().cpu().numpy().tolist()) + ) + return out + + module.CandidateHead.forward = head_forward + module.SpanHeadV2.forward = span_forward + + +def rows_of(engine, body): + from vela2_inference import system_one_to_predict + + req, plan = system_one_to_predict(body["state"], body["questions"]) + parts, qs, _thr = engine._to_generic(req) + spans = [q for q in qs if q["type"] == "span"] + rest = [q for q in qs if q["type"] != "span"] + rows = [dict(parts=parts, questions=rest + spans[:1])] + [ + dict(parts=parts, questions=[s]) for s in spans[1:] + ] + rendered, preps, windows = [], [], [] + for row in rows: + if not row["questions"]: + continue + prepped = engine._prep(row) + rec = engine._render(prepped) + rec["head"] = prepped["head"] + rec["head_reason"] = prepped["head_reason"] + rec["label_alias"] = plain(prepped.get("label_alias")) + preps.append(prepped) + rendered.append(rec) + for wrow in engine._window_rows(prepped) or []: + wrec = engine._render(wrow) + wrec["head"] = prepped["head"] + windows.append(wrec) + return req, plan, rendered, windows + + +def main(argv=None) -> int: + args = parse_args(argv) + sys.path.insert(0, str(Path(args.model).resolve())) + from vela2_inference import Vela2 + + engine = Vela2.from_pretrained(args.model, backend="torch", device="cpu") + import modeling_vela2 + + recorder = Recorder(modeling_vela2, engine.run.model) + out = Path(args.out) if args.out else HERE.parents[1] / "tests" / "vela" / "fixtures" + out.mkdir(parents=True, exist_ok=True) + + for name, body in cases(): + req, plan, rows, windows = rows_of(engine, body) + # One row per backend call so every recorded head call is + # attributable; question blocks arrive stacked in row order. + calls = [] + tensors: dict[str, np.ndarray] = {} + recorder.calls = [] + recorder.tensors = {} + recorder.next_index = 0 + recorder._on = True + row_of_call: list[int] = [] + try: + for index, rec in enumerate(rows + windows): + # Windows render after the main rows (row index continues). + engine.run([rec]) + for call in recorder.calls: + row_of_call.append(index) + calls.extend(recorder.calls) + tensors.update(recorder.tensors) + recorder.calls = [] + # The reference's calibrated answers for the same request. + predict = plain(engine._predict(req)[0]) + finally: + recorder._on = False + for call, row in zip(calls, row_of_call): + call["row"] = row + save_file(tensors, str(out / f"head-input-{name}.safetensors")) + fixture = { + "case": name, + "request": body, + # The rows block carries the span-head dispatch and alias + # provenance for human inspection; the tests read the + # rendered rows from the native-plan fixtures. + "rows": [ + {key: rec[key] for key in ("head", "head_reason", "label_alias", "n_tokens", "seg_lens") + if key in rec} + for rec in rows + ], + "head_calls": calls, + "predict": predict, + } + (out / f"head-input-{name}.json").write_text( + json.dumps(fixture, ensure_ascii=False) + "\n", encoding="utf-8" + ) + span_calls = sum(1 for call in calls if call["kind"] == "span") + print(f"{name}: {len(rows)} row(s), {len(calls)} head call(s) ({span_calls} span), predict answers: {len(predict['answers'])}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/recipe/vela/export_native.py b/recipe/vela/export_native.py new file mode 100644 index 00000000..8cb5e592 --- /dev/null +++ b/recipe/vela/export_native.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python3 +"""Export the pinned Vela 2.0 0.8B package into a native bundle. + +The bundle re-keys the hybrid +backbone for the shared qwen3_5 native loader — ``backbone.X`` tensors +become ``model.X`` (the loader's prefix probe accepts ``model.``), with +**per-tensor dtypes preserved verbatim** (Linear/conv1d in BF16, +embeddings/norms/A_log/dt_bias in FP32 — the reference's own mixed +layout, no re-save rounding) — and copies the Vela-owned F32 heads and +the side-loaded broad span head as separate files, plus calibration and +tokenizer. ``config.json`` is the package's ``backbone_config`` written +at the root, which is exactly the shape the family ``Config::load`` +reads. The manifest pins every source file by SHA-256 (re-using the +worker's own gate table) and records the exported tensor inventory. + + PYTHONPATH=src .venv/bin/python recipe/vela/export_native.py \ + --package weights/vela-2.0-0.8b --out weights/vela-2.0-0.8b-native +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import shutil +import sys +from pathlib import Path + +BACKBONE_PREFIX = "backbone." +HEAD_PREFIXES = ("head.", "span2.", "set_bias") +FORMAT = "vela2-native-bundle/1" + + +def parse_args(argv=None): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--package", required=True, help="pinned Vela 2.0 0.8B package directory") + parser.add_argument("--out", required=True, help="output bundle directory (created)") + parser.add_argument( + "--bf16-out", + default=None, + help="also write a uniform-BF16 backbone directory for the shared qwen3_5 " + "loader (model.safetensors + config.json)", + ) + return parser.parse_args(argv) + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(8 * 1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def stage_into_place(target: Path, write_files) -> None: + """Stage the artifact set in a sibling directory and move it into place + with one rename, so the target only ever holds a complete export: a run + killed mid-write leaves at most the staging directory, which the next + run with the same target removes.""" + if target.exists() and any(target.iterdir()): + raise SystemExit(f"output directory {target} already exists and is not empty") + staging = target.with_name(target.name + ".export-partial") + if staging.is_dir(): + # Remains of a run killed mid-write with the same target directory. + shutil.rmtree(staging) + staging.mkdir(parents=True) + try: + write_files(staging) + if target.exists(): + # Empty, checked above; the rename replaces the directory whole. + target.rmdir() + os.replace(staging, target) + except BaseException: + shutil.rmtree(staging, ignore_errors=True) + raise + + +def main(argv=None) -> int: + args = parse_args(argv) + sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "src")) + import torch + from safetensors.torch import load_file, save_file + + from models.vela.hybrid import ( + HYBRID_REPO, + HYBRID_REVISION, + PINNED_FILES_SHA256, + verify_package, + ) + + package = Path(args.package) + out = Path(args.out) + verify_package(package) # the worker's own gate: provenance before export + config = json.loads((package / "config.json").read_text(encoding="utf-8")) + backbone_config = config["backbone_config"] + + # The shard map names every backbone and head tensor; broad_head is a + # separate side-loaded file. + index = json.loads((package / "model.safetensors.index.json").read_text(encoding="utf-8")) + weight_map: dict[str, str] = index["weight_map"] + shard_files = sorted(set(weight_map.values())) + tensors = {} + for name in shard_files: + tensors.update(load_file(str(package / name))) + broad = load_file(str(package / "broad_head.safetensors")) + + backbone, heads = {}, {} + dtypes: dict[str, dict[str, str]] = {"backbone": {}, "heads": {}} + for name, tensor in sorted(weight_map.items()): + value = tensors[name] + dtype = str(value.dtype).removeprefix("torch.") + if name.startswith(BACKBONE_PREFIX): + rekeyed = "model." + name[len(BACKBONE_PREFIX):] + backbone[rekeyed] = value + dtypes["backbone"][rekeyed] = f"{dtype}:{list(value.shape)}" + elif name.startswith(HEAD_PREFIXES): + heads[name] = value + dtypes["heads"][name] = f"{dtype}:{list(value.shape)}" + else: + raise SystemExit(f"unexpected tensor key {name!r}; update the export for the new prefix") + for name, value in sorted(broad.items()): + heads[name] = value + dtypes["heads"][name] = f"{str(value.dtype).removeprefix('torch.')}:{list(value.shape)}" + + if args.bf16_out and Path(args.bf16_out) != Path(args.out) / "bf16": + # The manifest pins the bf16 backbone as bf16/ inside the + # bundle and the bundle gate walks it there, so the only valid + # target is the bundle's own bf16 subdirectory. + raise SystemExit( + f"--bf16-out must be {Path(args.out) / 'bf16'} (inside --out); got {args.bf16_out}" + ) + + bf16 = sum(1 for d in dtypes["backbone"].values() if d.startswith("bfloat16")) + fp32 = len(dtypes["backbone"]) - bf16 + manifest = { + "format": FORMAT, + "model_id": "vllm-sr/Vela-2.0-0.8B", + "model_name": config["model_name"], + "max_length": config["max_length"], + "description": config.get("description", ""), + "source": { + "repo": HYBRID_REPO, + "revision": HYBRID_REVISION, + # The worker gate's own table, re-verified above. + "pinned_files_sha256": dict(PINNED_FILES_SHA256), + "shard_files": shard_files, + }, + "backbone": { + "tensors": len(backbone), + "bf16": bf16, + "fp32": fp32, + "note": "per-tensor dtypes preserved from the package; backbone.X re-keyed to model.X", + "inventory": dtypes["backbone"], + }, + "heads": { + "tensors": len(heads), + "inventory": dtypes["heads"], + "note": "Vela-owned F32 heads (CandidateHead, set_bias, span2 router, span_broad) in one file", + }, + "files_sha256": {}, + } + + def write_bundle(out: Path) -> None: + save_file(backbone, str(out / "backbone.safetensors")) + save_file(heads, str(out / "heads.safetensors")) + # config.json for the family loader: backbone_config at the root. + (out / "config.json").write_text(json.dumps(backbone_config, indent=1) + "\n", encoding="utf-8") + for name in ("calibration.json", "tokenizer.json", "tokenizer_config.json"): + shutil.copyfile(package / name, out / name) + if args.bf16_out: + # Uniform-BF16 re-save for the family crate's loader: every + # backbone tensor rounds to bfloat16 once, at export (the + # reference's own GPU setting runs the backbone under bf16 + # autocast too). Written inside the same staging tree so the + # whole bundle — bf16 included — lands with one rename and + # the manifest below pins the staged bytes. + bf16_dir = out / "bf16" + bf16_dir.mkdir() + uniform = {name: value.to(torch.bfloat16) for name, value in backbone.items()} + save_file(uniform, str(bf16_dir / "model.safetensors")) + (bf16_dir / "config.json").write_text(json.dumps(backbone_config, indent=1) + "\n", encoding="utf-8") + for name in [ + "backbone.safetensors", "heads.safetensors", "config.json", + "calibration.json", "tokenizer.json", "tokenizer_config.json", + ]: + manifest["files_sha256"][name] = sha256(out / name) + if args.bf16_out: + for name in ("model.safetensors", "config.json"): + manifest["files_sha256"][f"bf16/{name}"] = sha256(out / "bf16" / name) + (out / "vela_native_manifest.json").write_text( + json.dumps(manifest, indent=1, sort_keys=True) + "\n", encoding="utf-8" + ) + + stage_into_place(out, write_bundle) + + if args.bf16_out: + print(f"bf16 backbone: {len(backbone)} tensors -> {args.bf16_out}") + + print(f"export ok: {out}") + print(f" backbone: {len(backbone)} tensors ({bf16} bf16, {fp32} fp32), re-keyed {BACKBONE_PREFIX} -> model.") + print(f" heads: {len(heads)} tensors (F32, incl. span_broad)") + print(f" config: backbone_config at root, model_type={backbone_config.get('model_type')}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/recipe/vela/export_token_plan.py b/recipe/vela/export_token_plan.py new file mode 100644 index 00000000..39bfb2ea --- /dev/null +++ b/recipe/vela/export_token_plan.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +"""Freeze the token-level row plan of the pinned Vela 2.0 0.8B runtime. + +This tool records, for a set of request +bodies, exactly what the pinned runtime feeds the backbone — the +SystemOne → predict mapping, the per-row decomposition (parts pass + +question blocks; one span question per row), and the rendered records +(``p_ids``, block token ids with option/label end positions, query +positions, span word positions, truncation and segment lengths). The +native Rust processor must reproduce these byte-for-byte; the frozen +files under ``tests/vela/fixtures/native-plan-*.json`` are that parity +target. Model outputs are not recorded here — the oracle fixtures from +``compare_reference.py`` own the output side. + + PYTHONPATH=src .venv/bin/python recipe/vela/export_token_plan.py \ + --model weights/vela-2.0-0.8b --out tests/vela/fixtures/ +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +import numpy as np + +HERE = Path(__file__).resolve().parent +MODEL_NAME = "vllm-sr/Vela-2.0-0.8B" + + +def parse_args(argv=None): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--model", required=True, help="pinned Vela 2.0 0.8B package directory") + parser.add_argument("--out", default=None, help="fixture output directory") + return parser.parse_args(argv) + + +def long_span_body(): + """A span target beyond HYBRID_RMAX (2,048 tokens) to exercise the + windowed layout: the full-render span block is skipped and one row + per 1,800-token window (stride 1,536, cut at word starts) carries + the span question.""" + + filler = " ".join(f"Sentence {i} reports a routine delivery event." for i in range(220)) + return { + "model": MODEL_NAME, + "state": {"request": filler}, + "questions": { + "claims": { + "type": "span", + "instructions": "Which spans mention a delivery?", + "criteria": {"delivery": "a delivery event"}, + "over": "request", + } + }, + } + + +def cases(): + named = [ + ("example-request", HERE / "example-request.json"), + ("example-request-set-threshold", HERE / "example-request-set-threshold.json"), + ("example-request-cjk", HERE / "example-request-cjk.json"), + ("example-request-score", HERE / "example-request-score.json"), + ] + loaded = [ + ( + name, + {**json.loads(path.read_text(encoding="utf-8")), "model": MODEL_NAME}, + ) + for name, path in named + ] + loaded.append(("long-span-windowed", long_span_body())) + return loaded + + +def plain(value): + """numpy-free JSON for the rendered records.""" + + if isinstance(value, dict): + return {key: plain(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [plain(item) for item in value] + if isinstance(value, np.ndarray): + return plain(value.tolist()) + if isinstance(value, (np.integer,)): + return int(value) + if isinstance(value, (np.floating,)): + return float(value) + if isinstance(value, (np.bool_,)): + return bool(value) + return value + + +def rows_of(engine, body): + """Mirror ``Vela2._predict``'s row decomposition and render each row.""" + + from vela2_inference import system_one_to_predict + + req, plan = system_one_to_predict(body["state"], body["questions"]) + parts, qs, _thr = engine._to_generic(req) + spans = [q for q in qs if q["type"] == "span"] + rest = [q for q in qs if q["type"] != "span"] + rows = [dict(parts=parts, questions=rest + spans[:1])] + [ + dict(parts=parts, questions=[s]) for s in spans[1:] + ] + rendered, windows = [], [] + for row in rows: + if not row["questions"]: + continue + prepped = engine._prep(row) + rec = plain(engine._render(prepped)) + rec["head"] = prepped["head"] + rec["head_reason"] = prepped["head_reason"] + rendered.append(rec) + for wrow in engine._window_rows(prepped) or []: + wrec = plain(engine._render(wrow)) + wrec["head"] = prepped["head"] + windows.append(wrec) + return plain(req), plain(plan), rendered, windows + + +def main(argv=None) -> int: + args = parse_args(argv) + sys.path.insert(0, str(Path(args.model).resolve())) + from vela2_inference import Vela2 + + engine = Vela2.from_pretrained(args.model, backend="torch", device="cpu") + out = Path(args.out) if args.out else HERE.parents[1] / "tests" / "vela" / "fixtures" + out.mkdir(parents=True, exist_ok=True) + for name, body in cases(): + req, plan, rows, windows = rows_of(engine, body) + fixture = { + "case": name, + "request": body, + "predict_request": req, + "plan": plan, + "rows": rows, + "window_rows": windows, + } + target = out / f"native-plan-{name}.json" + target.write_text(json.dumps(fixture, ensure_ascii=False) + "\n", encoding="utf-8") + tokens = sum(r["n_tokens"] for r in rows) + sum(r["n_tokens"] for r in windows) + print( + f"{name}: {len(rows)} row(s) + {len(windows)} window row(s), " + f"{tokens} tokens -> {target.name}" + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/recipe/vela/native.md b/recipe/vela/native.md new file mode 100644 index 00000000..f5fa8f47 --- /dev/null +++ b/recipe/vela/native.md @@ -0,0 +1,119 @@ +# Vela 2.0 0.8B native library + +The native implementation for +[vllm-sr/Vela-2.0-0.8B](https://huggingface.co/vllm-sr/Vela-2.0-0.8B): +the exported-bundle gate, the wire contract, the token-level +processing, the F32 heads and scoring, and backbone execution on CUDA +through the shared qwen3_5 crate — all mirrored from the pinned +runtime and checked against frozen fixtures. This is a library; the +HTTP worker is the [reference recipe](reference.md)'s Python worker. + +Run the commands from the repository root with the same `.venv` and +pinned package as the reference recipe. The export needs about 6 GB of +free disk next to the package; the parity checks need no GPU except +the executor check. + +## Export the native bundle + +~~~sh +PYTHONPATH=src .venv/bin/python recipe/vela/export_native.py \ + --package weights/vela-2.0-0.8b --out weights/vela-2.0-0.8b-native \ + --bf16-out weights/vela-2.0-0.8b-native/bf16 +~~~ + +The backbone that executes on CUDA is a uniform-BF16 re-save (every +backbone tensor rounds once at export); the reference's own GPU setting +runs the backbone under bf16 autocast but keeps norm weights and GDN +gate math in FP32, so the uniform export is a slightly lower-precision +profile than either the CPU-FP32 recording or the reference GPU path. +The Vela heads stay F32 on the faithful tensors. + +The export re-runs the reference worker's package gate first +(provenance), then writes `weights/vela-2.0-0.8b-native/`: the backbone +re-keyed `backbone.` → `model.` with **per-tensor dtypes preserved** +(204 BF16 Linear/conv1d, 116 F32 embed/norms/A_log/dt_bias — the +reference's own mixed layout), the 37 F32 heads in `heads.safetensors` +(including the side-loaded broad span head), `backbone_config` at the +config root (the shared qwen3_5 `Config::load` shape), calibration and +tokenizer, and `vela_native_manifest.json` pinning the source +revision, the gate's file table, the tensor inventory and every file's +SHA-256. `--bf16-out` additionally writes the uniform-BF16 backbone +(`model.safetensors`) that the shared loader consumes, into the +bundle's own `bf16/` subdirectory. The whole set — bf16 included — is +staged in a sibling directory and moved into place with one rename, so +a run killed mid-write can no longer strand a partial bundle under +`--out`; a non-empty target directory is refused and the staging +remnant is cleared by the next run. + +## Freeze the fixtures (opt-in, needs the package) + +~~~sh +PYTHONPATH=src .venv/bin/python recipe/vela/export_token_plan.py \ + --model weights/vela-2.0-0.8b --out tests/vela/fixtures/ +PYTHONPATH=src .venv/bin/python recipe/vela/export_head_inputs.py \ + --model weights/vela-2.0-0.8b --out tests/vela/fixtures/ +~~~ + +The token plans record what the pinned runtime feeds the backbone (the +predict mapping, the per-row decomposition, the rendered `p_ids`, +blocks and readout positions, truncation, a windowed long-span case); +the head inputs record the hidden states its F32 heads consume, the +heads' logits and the calibrated predict answers. The example-request +token plan totals 1,041 tokens, matching the oracle fixture's +`usage.input_tokens`. + +## Checks + +~~~sh +cargo test -p omni-vela-native +cargo fmt --all --check +cargo clippy -p omni-vela-native --all-targets -- -D warnings +VELA_NATIVE_BUNDLE="$PWD/weights/vela-2.0-0.8b-native" \ + cargo test -p omni-vela-native --test vela_native_processing \ + --test vela_native_heads --test vela_native_response -- --ignored --nocapture +VELA_NATIVE_BUNDLE="$PWD/weights/vela-2.0-0.8b-native" \ +VELA_CUDA_LIB=/path/to/libqwen3_5_cuda.so \ + cargo test -p omni-vela-native --test vela_native_executor -- --ignored --nocapture +~~~ + +28 tests run everywhere with no weights and no GPU: the wire contract +against the pinned runtime's 422/400 bodies, the bundle manifest gate, +and the predict-mapping/plan parity on every frozen fixture. The four +opt-in checks (behind `-- --ignored`) replay the fixtures through the +Rust implementation: + +- render parity — the token plans match field-for-field (5/5); +- head parity — head-logit drift ≤ 3.8e-5, predict-answer drift + ≤ 7.3e-7, discrete picks and span structures equal (5/5); +- response parity — the finished SystemOne envelopes match the frozen + oracles within 1.5e-6 (4/4); +- executor parity — the backbone on CUDA replays the recorded + reference logits with a worst drift of 0.285 (the uniform-BF16 + backbone against the CPU-FP32 reference; gate 0.35, tighten-only via + `VELA_GPU_LOGIT_GATE`: a wider value clamps back down), 4/4 small + cases. + +For a field-level diff of one token-plan case: + +~~~sh +VELA_NATIVE_BUNDLE="$PWD/weights/vela-2.0-0.8b-native" \ + cargo run -p omni-vela-native --example parity -- example-request +~~~ + +## Recorded hardware evidence + +The executor check above is the first recorded execution of the shared +qwen3_5 CUDA kernels on sm_120 (RTX 5060 Laptop, CUDA 13.1; build with +`src/backends/cuda/qwen3_5/build.sh 120`). The bf16 backbone +takes about 3 GB of GPU memory; the executor check runs the four small +frozen cases in about three minutes on that device. + +## Scope and limits + +- Implemented: bundle export + manifest gate, wire admission, + token-level processing, F32 heads, scoring and span decoding, the + SystemOne response finishing, and backbone execution through the + shared crate (full-sequence per readout position, packed batches). +- Not implemented: the HTTP worker around the library, long-span GPU + parity on the windowed case, and measured serving latency. No + performance claims are made. diff --git a/recipe/vela/reference.md b/recipe/vela/reference.md index 98d30f9c..559a7afb 100644 --- a/recipe/vela/reference.md +++ b/recipe/vela/reference.md @@ -1,9 +1,13 @@ -# Vela 2.0 0.3B reference worker +# Vela 2.0 reference workers (0.3B, 0.8B hybrid) -This recipe serves Vela 2.0 0.3B through the Python reference worker and -the existing Rust frontend. The model-side implementation is under -[`src/models/vela/`](../../src/models/vela/), and the worker is -[`src/frontend/vela.py`](../../src/frontend/vela.py). +This recipe serves the Vela 2.0 members through the Python reference +worker and the existing Rust frontend: the 0.3B encoder and the 0.8B +hybrid, one pinned member per worker process. The model-side +implementation is under +[`src/models/vela/`](../../src/models/vela/) (0.3B in `engine.py`, the +0.8B hybrid in `hybrid.py`), and the worker is +[`src/frontend/vela.py`](../../src/frontend/vela.py), which picks the +executor from the package's declared `model_type`. Run the commands from the repository root with Python 3.12. The default and evidence-backed setting is CPU FP32 — the reference oracle; a CUDA @@ -14,33 +18,34 @@ description, not here. ## Scope -The slice is: +Scope: -- package: [`vllm-sr/Vela-2.0-0.3B`](https://huggingface.co/vllm-sr/Vela-2.0-0.3B) - at the pinned revision below; -- the Python reference worker behind the Rust frontend, executing the +- packages: [`vllm-sr/Vela-2.0-0.3B`](https://huggingface.co/vllm-sr/Vela-2.0-0.3B) + and [`vllm-sr/Vela-2.0-0.8B`](https://huggingface.co/vllm-sr/Vela-2.0-0.8B) + at the pinned revisions below; +- the Python reference worker behind the Rust frontend, executing each checkpoint's own pinned runtime (`vela2_inference.py` and siblings) after a full-hash package gate; - text/JSON state and all five typed kinds — `choice`, `noul`, `score`, `span`, `set` — in one request, with the span/set response extensions - (`spans`/`sets`/`thresholds` and their Noul views) served - forward-untouched; -- FP32 parameters, one bidirectional forward per assembled sequence, + (`spans`/`sets`/`thresholds`, `span_heads` on the hybrid, and their + Noul views) served forward-untouched; +- FP32 parameters, one bidirectional forward per assembled sequence + (0.3B) or one parts pass with per-question continuation (hybrid), calibrated readouts; zero generated tokens. -Out of scope for this slice: the hybrid members (0.8B/4B/9B — their -reference workers are a separate slice), the ONNX backend, any native -Rust/CUDA executor (the frozen RFC's ModernBERT-profile route), and -`noul_calibration` (a hybrid-only feature that does not exist on 0.3B). +Out of scope: the larger hybrid members (4B/9B), the ONNX backend of +the 0.3B, any native Rust/CUDA executor, and `noul_calibration` (off +by default; documented as a reference-server feature only). ## Architecture and ownership ~~~text Rust frontend - -> Python worker + -> Python worker (executor chosen by config.json model_type) -> admit: JSON transport + upstream envelope preamble (protocol) - -> verify: config identity pin, full SHA-256 of the 7-file table, - exact weights size, then import it (engine) + -> verify: config identity pin, full SHA-256 of the pinned table, + exact weights size, then import it (engine / hybrid) -> execute: the checkpoint's own vela2_inference system_one_request -> finish: response-order/key-set checks + formula mirrors (postprocess) ~~~ @@ -53,28 +58,37 @@ The ownership boundary follows triples; deeper validation (state parts, question semantics, the 413) is the pinned runtime's, verbatim; - `preprocess.py` maps the runtime's `SystemOneError` to 422/413; -- `engine.py` owns the pinned runtime and the package gate; -- `postprocess.py` owns the pure formulas and response finishing; -- `frontend.vela` owns HTTP lifecycle, the single-flight 503 lock, the - `VELA2_API_KEY` 401 gate and warmup-before-readiness; +- `engine.py` owns the 0.3B pinned runtime and its package gate; + `hybrid.py` mirrors it for the 0.8B (a 10-file pin table including + the shard index and the side-loaded `broad_head.safetensors`); +- `postprocess.py` owns the pure formulas and response finishing, + member-aware for the hybrid's `span_heads` field; +- `frontend.vela` owns HTTP lifecycle, member detection, the + single-flight 503 lock, the `VELA2_API_KEY` 401 gate and + warmup-before-readiness; - The Rust frontend forwards `/v1/systemone` and `/health` unchanged. ## Pinned artifacts -| Artifact | Value | -| --- | --- | -| Package | [vllm-sr/Vela-2.0-0.3B](https://huggingface.co/vllm-sr/Vela-2.0-0.3B) | -| Pinned revision | `d6f03aa9baca4f9017fa8f8e9c2764d3bed3f409` | -| Package gate | config structural identity + a 7-file SHA-256 table pinned per revision in `engine.py`, full hash before import, no unlisted importables, `__pycache__` or directory symlinks in the package root, exact 1,236,473,252-byte weights check | -| License | Apache-2.0; tokenizer material under Gemma Terms of Use | +| Artifact | 0.3B | 0.8B hybrid | +| --- | --- | --- | +| Package | [vllm-sr/Vela-2.0-0.3B](https://huggingface.co/vllm-sr/Vela-2.0-0.3B) | [vllm-sr/Vela-2.0-0.8B](https://huggingface.co/vllm-sr/Vela-2.0-0.8B) | +| Pinned revision | `d6f03aa9baca4f9017fa8f8e9c2764d3bed3f409` | `326b01d81f61fff7fd4ae570e54b87a49119e490` | +| Package gate | config structural identity + a 7-file SHA-256 table in `engine.py`; no unlisted importables, `__pycache__` or directory symlinks in the package root | config structural identity (incl. `model_name`) + a 10-file SHA-256 table in `hybrid.py`, cross-checked against the package's own SHA256SUMS | +| Weights check | exact 1,236,473,252 bytes | exact 2,022,189,488 bytes + hashed `broad_head.safetensors` | +| Served name | `vllm-sr/Vela-2.0-0.3B` | `vllm-sr/Vela-2.0-0.8B` (from `config.json`) | +| Input window | 8,192 tokens | 16,384 tokens | +| License | Apache-2.0; tokenizer material under Gemma Terms of Use | Apache-2.0; Qwen3.5-0.8B lineage notices in the package | -The gate's file table is checked in (not read from the package, which -ships no manifest), so a later snapshot — self-consistent or not — -cannot pass as the pinned revision. Re-check before changing the pin: +Each gate's file table is checked in (never read from the package), so +a later snapshot — self-consistent or not — cannot pass as the pinned +revision. Re-check before changing a pin: ~~~sh curl -sS "https://huggingface.co/api/models/vllm-sr/Vela-2.0-0.3B/revision/d6f03aa9baca4f9017fa8f8e9c2764d3bed3f409" | jq -r '.sha' sha256sum weights/vela-2.0-0.3b/{vela2_inference.py,modeling_vela2.py,configuration_vela2.py,vela2_serve.py,config.json,calibration.json,tokenizer.json} +curl -sS "https://huggingface.co/api/models/vllm-sr/Vela-2.0-0.8B/revision/326b01d81f61fff7fd4ae570e54b87a49119e490" | jq -r '.sha' +sha256sum weights/vela-2.0-0.8b/{vela2_inference.py,modeling_vela2.py,configuration_vela2.py,vela2_serve.py,config.json,calibration.json,tokenizer.json,tokenizer_config.json,model.safetensors.index.json,broad_head.safetensors} ~~~ ## Install and prepare the artifacts @@ -88,15 +102,21 @@ python3.12 -m venv .venv ~~~sh export VELA_PKG="$PWD/weights/vela-2.0-0.3b" export VELA_REV=d6f03aa9baca4f9017fa8f8e9c2764d3bed3f409 +export VELA_HYBRID_PKG="$PWD/weights/vela-2.0-0.8b" +export VELA_HYBRID_REV=326b01d81f61fff7fd4ae570e54b87a49119e490 mkdir -p weights .venv/bin/hf download vllm-sr/Vela-2.0-0.3B --revision "$VELA_REV" --local-dir "$VELA_PKG" +.venv/bin/hf download vllm-sr/Vela-2.0-0.8B --revision "$VELA_HYBRID_REV" --local-dir "$VELA_HYBRID_PKG" ~~~ -The full package is about 4.4 GB (the 1.24 GB FP32 safetensors, a 34 MB +The 0.3B package is about 4.4 GB (the 1.24 GB FP32 safetensors, a 34 MB tokenizer, the ONNX exports and doc assets); serving needs roughly 2 GB of RAM for the FP32 model plus working memory for the batch rows. -The worker refuses to start when the config is not the pinned member's, -when any of the 7 gated files fails its hash, when the package root +The 0.8B package is about 2.1 GB (a 2.02 GB shard plus the 4.5 MB +broad-head file); FP32 parameters take about 3 GB of RAM on CPU (about +3 GB of GPU memory on CUDA, where the backbone runs under bf16 +autocast). A worker refuses to start when the config is not its pinned +member's, when any gated file fails its hash, when the package root holds an importable entry the table does not name, an existing `__pycache__` or a directory symlink, or when the weights file is not the pinned size — that @@ -106,12 +126,15 @@ gate runs before any package code is imported. ~~~sh PYTHONPATH=src .venv/bin/python -m frontend.vela \ - --model "$VELA_PKG" --device cpu --port 8000 + --model "$VELA_PKG" --device cpu --port 8000 # 0.3B +PYTHONPATH=src .venv/bin/python -m frontend.vela \ + --model "$VELA_HYBRID_PKG" --device cpu --port 8001 # 0.8B hybrid ~~~ -The worker verifies the package, loads the FP32 model, runs a real -warmup request covering every question kind, and only then binds the -port: +The worker detects the member from the package's `model_type`, verifies +the package, loads the model, runs a real warmup request covering every +question kind (the hybrid's warmup also drives both span heads), and +only then binds the port: ~~~sh curl -sS -i http://127.0.0.1:8000/health # {"status": "ok"} (upstream verbatim) @@ -199,12 +222,13 @@ the [model contract](../../src/models/vela/README.md). In short: transport errors follow the shared worker discipline; every invalid field rejects the whole request with the upstream 422 `{"detail": [{"loc", "msg", "type"}]}`; 413 `too_long` fires only when -questions plus schema alone exceed 8,192 tokens; and over-limit state -text is **truncated as the reference truncates it** (shrink part first, -floor 64 then 1, then protected parts; windowed re-coverage of cut -labelled parts) — a deliberate, loudly documented divergence from the -repository's no-silent-truncation convention, with the guardrail risk -recorded in the model contract. +questions plus schema alone exceed the member's input window (8,192 on +0.3B, 16,384 on the hybrid); and over-limit state text is **truncated +as the reference truncates it** (shrink part first, floor 64 then 1, +then protected parts; windowed re-coverage of cut labelled parts) — a +deliberate, loudly documented divergence from the repository's +no-silent-truncation convention, with the guardrail risk recorded in +the model contract. ## Repository-side validation @@ -229,26 +253,32 @@ invalid-span 422, an over-limit 413, and the missing-field / extra-field / wrong-model envelope 422s) — through two paths that share one loaded engine: the pinned `vela2_serve.py` app via FastAPI's test client, and the production -`frontend.vela.decide` pipeline. The criterion is the charter's -reference parity: **exact match** including JSON key order for 200s, +`frontend.vela.decide` pipeline. The criterion is reference parity: +**exact match** including JSON key order for 200s, and agreement on status and body for rejections (needs `pip install fastapi uvicorn httpx` on top of the requirements file). `--export tests/vela/fixtures/oracle-.json` freezes each passing 200 case — request, SystemOne response and predict()-level answers — as -the frozen comparison baseline for the frozen RFC's native encoder -slices, and writes it only after a passing comparison. +the frozen comparison baseline, and writes it only after a passing +comparison. ~~~sh PYTHONPATH=src .venv/bin/python recipe/vela/compare_reference.py \ --model "$VELA_PKG" --device cpu \ --export tests/vela/fixtures/ +PYTHONPATH=src .venv/bin/python recipe/vela/compare_reference.py \ + --model "$VELA_HYBRID_PKG" --device cpu \ + --export tests/vela/fixtures/ ~~~ Run it by hand and record the verbatim output, host, package revision and dependency versions in the pull request description. CPU FP32 is the oracle setting; keep flash-linear-attention out of the fixture path -(it does not exist on this encoder member anyway) and do not substitute -GPU bf16-autocast numbers for the baseline. +(it changes the hybrid's numbers and its parity against the fallback +path is not documented upstream) and do not substitute GPU +bf16-autocast numbers for the baseline. The tool rewrites the example +files' `model` field to the served member and exports the hybrid's +oracle fixtures under an `oracle-hybrid-` prefix. ## Troubleshooting and limits @@ -257,15 +287,15 @@ GPU bf16-autocast numbers for the baseline. `__pycache__` bytecode caches, directory symlinks and wrong weights sizes at startup, before importing anything. - The 8 MiB body cap is slice-scoped (upstream sets no total-body - limit); a valid request bounded by the 8,192-token window is far - below it. -- `/health` and `/ready` are upstream's minimal bodies on this member, - not the richer shape of some other workers in this repository. + limit); a valid request bounded by either member's input window is + far below it. +- `/health` and `/ready` are upstream's minimal bodies on both + members, not the richer shape of some other workers in this + repository. - Long states are truncated (see Error behavior); if a consumer must never miss tail text, check `usage.input_tokens` against the expected budget or enforce limits upstream of the worker. - GPU bf16-autocast serving (`--device cuda`) is untested in this - slice's evidence; treat it as a deployment option to validate - separately. + evidence; treat it as a deployment option to validate separately. - GPU evidence for the supported-models table is recorded in the integration pull request, not in this recipe. diff --git a/recipe/vela/test_compare_reference.py b/recipe/vela/test_compare_reference.py index d97aaa45..f76fb69d 100644 --- a/recipe/vela/test_compare_reference.py +++ b/recipe/vela/test_compare_reference.py @@ -86,7 +86,7 @@ def check(name: str, condition: bool, detail: str = "") -> None: lines, ok = tool.compare_case("fixture", (200, response()), (422, rejection)) check("worker answering a rejected request fails", not ok, "\n".join(lines)) - # 3. The default case set carries the charter's fixture list shape. + # 3. The default case set carries the fixture list's shape. cases = tool.default_cases() names = [name for name, _ in cases] check( diff --git a/src/frontend/vela.py b/src/frontend/vela.py index c863a018..78f46631 100644 --- a/src/frontend/vela.py +++ b/src/frontend/vela.py @@ -1,10 +1,13 @@ -"""Python reference worker for Vela 2.0 0.3B. +"""Python reference worker for the Vela 2.0 members (0.3B, 0.8B hybrid). The Rust frontend remains the public transport layer. This worker owns only HTTP lifecycle and request orchestration; wire admission lives in models.vela.protocol, the pinned-runtime error mapping in -models.vela.preprocess, the pinned runtime and its package gate in -models.vela.engine, and response finishing in models.vela.postprocess. +models.vela.preprocess, the pinned runtimes and their package gates in +models.vela.engine (0.3B) and models.vela.hybrid (0.8B), and response +finishing in models.vela.postprocess. The executor is chosen by the +package's declared ``model_type``; one worker process serves one pinned +member. The error surface has two layers: transport errors follow the shared Python-worker discipline, and every request rejection the pinned runtime @@ -13,7 +16,8 @@ (or 413 ``too_long``), passed through verbatim. There are no per-question errors. Over-limit state text is truncated exactly as the pinned reference truncates it (see the model contract); only questions plus -schema that cannot fit the 8,192-token window are refused. +schema that cannot fit the member's input window (8,192 or 16,384 +tokens) are refused. """ from __future__ import annotations @@ -26,17 +30,21 @@ import socket import threading from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path from typing import Any from models.vela.engine import Vela2Executor +from models.vela.hybrid import EXPECTED_MODEL_NAME as HYBRID_MODEL_NAME, Vela2HybridExecutor from models.vela.postprocess import finish_response from models.vela.preprocess import map_runtime_error, to_request_error from models.vela.protocol import MODEL_NAME, MAX_BODY, RequestError, parse_body, admit_request LOG = logging.getLogger(__name__) -# Upstream vela2_serve.py /v1/models entry, kept verbatim so clients see -# the reference server's description and release date. +# Upstream vela2_serve.py /v1/models entry for the 0.3B member, kept +# verbatim so clients see the reference server's description and release +# date; the hybrid members source their entry from config.json, exactly +# as their vela2_serve.py does. MODELS_ENTRY = { "name": None, # the served model name, filled per request "release_date": "2026-09-29", @@ -48,7 +56,57 @@ UNAUTHORIZED_BODY = {"detail": "Missing or invalid API key"} -def decide(raw: bytes, executor: Vela2Executor) -> dict[str, Any]: +def detect_member(package: str | Path) -> tuple[type, str]: + """Pick the pinned executor for a package's declared member. + + Returns the executor class and the member's default served name; the + structural gate inside the executor is the real authority — this + only routes the right gate to the package. + """ + + config_path = Path(package) / "config.json" + if not config_path.is_file(): + raise ValueError(f"the package is missing config.json: {package}") + try: + config = json.loads(config_path.read_text(encoding="utf-8")) + except (OSError, ValueError) as exc: + raise ValueError(f"config.json in {package} is not readable JSON") from exc + if not isinstance(config, dict): + raise ValueError(f"config.json in {package} is not a JSON object") + model_type = config.get("model_type") + if model_type == "vela2-unified": + return Vela2Executor, MODEL_NAME + if model_type == "vela2-decoder": + return Vela2HybridExecutor, HYBRID_MODEL_NAME + raise ValueError( + f"config.json declares model_type {model_type!r}; this worker serves the " + "vela2-unified (0.3B) and vela2-decoder (0.8B) members of Vela 2.0" + ) + + +def models_entry(executor) -> dict[str, Any]: + """The /v1/models entry, verbatim per member's reference server.""" + + if getattr(executor, "is_hybrid", False): + # Upstream hybrid vela2_serve.py: name/description/release_date + # straight from config.json. + config = getattr(executor, "engine", None) + config = getattr(config, "config", None) if config is not None else None + config = config or {} + return { + "name": executor.model_name, + "release_date": config.get("release_date", ""), + "description": config.get( + "description", + "Vela 2.0: SystemOne Choice / Noul / Score with span and set extensions", + ), + } + entry = dict(MODELS_ENTRY) + entry["name"] = executor.model_name + return entry + + +def decide(raw: bytes, executor) -> dict[str, Any]: """Admit, execute, and finish one complete request.""" body = admit_request(parse_body(raw), executor.served) @@ -62,13 +120,13 @@ def decide(raw: bytes, executor: Vela2Executor) -> dict[str, Any]: # layer cannot foresee (state rendering, question semantics, the # window budget); its rejections are client errors, not failures. raise map_runtime_error(exc) from exc - return finish_response(questions, response) + return finish_response(questions, response, hybrid=getattr(executor, "is_hybrid", False)) class WorkerServer(ThreadingHTTPServer): daemon_threads = False - def __init__(self, address, executor: Vela2Executor, api_key: str | None = None): + def __init__(self, address, executor: Vela2Executor | Vela2HybridExecutor, api_key: str | None = None): self.executor = executor self.api_key = api_key self.inference_lock = threading.Lock() @@ -139,9 +197,7 @@ def do_GET(self): elif path == "/v1/models": if self._unauthorized(): return - entry = dict(MODELS_ENTRY) - entry["name"] = self.server.executor.model_name - self.send_json(200, {"models": [entry]}) + self.send_json(200, {"models": [models_entry(self.server.executor)]}) else: self.send_json(404, {"detail": "unknown route"}) @@ -190,12 +246,16 @@ def do_POST(self): def parse_args(argv=None): - parser = argparse.ArgumentParser(description="Python reference worker for Vela 2.0 0.3B") - parser.add_argument("--model", required=True, help="pinned Vela 2.0 0.3B package directory") - parser.add_argument("--device", default="cpu", help="torch device (reference contract: cpu)") + parser = argparse.ArgumentParser(description="Python reference worker for Vela 2.0 members") + parser.add_argument("--model", required=True, + help="pinned Vela 2.0 package directory (0.3B or the 0.8B hybrid)") + parser.add_argument("--device", default="cpu", + help="torch device (reference contract: cpu; cuda runs the hybrid " + "backbone under bf16 autocast with FP32 heads)") parser.add_argument("--batch-size", type=int, default=None) - parser.add_argument("--name", default=os.environ.get("VELA2_MODEL_NAME", MODEL_NAME), - help="model name accepted in the request's model field") + parser.add_argument("--name", default=os.environ.get("VELA2_MODEL_NAME"), + help="model name accepted in the request's model field " + "(default: the pinned member's served name)") parser.add_argument("--host", default="127.0.0.1") parser.add_argument("--port", type=int, default=8000) return parser.parse_args(argv) @@ -204,10 +264,11 @@ def parse_args(argv=None): def main(argv=None): args = parse_args(argv) logging.basicConfig(level=logging.INFO) - executor = Vela2Executor( + executor_class, default_name = detect_member(args.model) + executor = executor_class( args.model, device=args.device, - model_name=args.name, + model_name=args.name or default_name, batch_size=args.batch_size, ) server = None diff --git a/src/models/vela/README.md b/src/models/vela/README.md index 8c6c65f6..4274b740 100644 --- a/src/models/vela/README.md +++ b/src/models/vela/README.md @@ -1,48 +1,64 @@ -# Vela 2.0 0.3B reference contract +# Vela 2.0 reference contracts (0.3B, 0.8B hybrid) -This directory contains the System1-Omni integration slice for +This directory contains the System1-Omni integration slices for [Vela 2.0 0.3B](https://huggingface.co/vllm-sr/Vela-2.0-0.3B) -(`vela2-unified`). It is a reference-serving boundary only: a Python -worker that runs the checkpoint's own pinned runtime for text state and -all five typed question kinds (`choice`, `noul`, `score`, `span`, `set`) -on `/v1/systemone`. Native execution is not part of this slice; per the -frozen RFC this worker is the Phase-1 entry that freezes the -text/token_ids input, the position-level final-hidden readout contract, -the error/length boundaries and the parity baseline for later -ModernBERT-profile work. +(`vela2-unified`) and +[Vela 2.0 0.8B](https://huggingface.co/vllm-sr/Vela-2.0-0.8B) +(`vela2-decoder`, the first hybrid member): a Python worker that runs +the checkpoint's own pinned runtime for text state and all five typed +question kinds (`choice`, `noul`, `score`, `span`, `set`) on +`/v1/systemone`, one pinned member per worker process. The 0.3B +worker's contract freezes the text/token_ids input, the +position-level final-hidden readout contract, the error/length +boundaries and the parity baseline; see [Native path](#native-path-08b) +for the native library. The shared Vela 2.0 wire module is deliberately shaped for the family: the envelope admission (`protocol`), the pinned-runtime error mapping (`preprocess`) and response finishing (`postprocess`) are member-neutral; -this slice ships the 0.3B pinned executor (`engine`). The hybrid members -(0.8B/4B/9B) reuse the same wire layer with their own executors. +this directory ships the 0.3B pinned executor (`engine`) and the 0.8B +hybrid executor (`hybrid`). The worker picks the executor from the +package's declared `model_type`. The 4B/9B members are not implemented. ## Pinned artifacts -| Artifact | Value | -| --- | --- | -| Package | [vllm-sr/Vela-2.0-0.3B](https://huggingface.co/vllm-sr/Vela-2.0-0.3B) | -| Pinned revision | `d6f03aa9baca4f9017fa8f8e9c2764d3bed3f409` | -| Identity | `vela2-unified` / `Vela2Model`, 309,114,369 parameters, all FP32 (1,236,473,252-byte safetensors) | -| Runtime | the checkpoint's own flat files: `vela2_inference.py`, `modeling_vela2.py`, `configuration_vela2.py`, `vela2_serve.py` | -| Calibration | the package's own `calibration.json` (never cross-copied between members) | -| License | Apache-2.0; the inherited tokenizer material carries Gemma Terms of Use | -| Runtime dependencies | [recipe/vela/requirements.txt](../../recipe/vela/requirements.txt) (torch, transformers, safetensors, tokenizers, numpy) | - -The package ships no manifest, so the executor's startup gate checks the -config's structural identity (model type, architecture, `max_length` -8,192, the 8 marker ids), then hashes every small file the runtime reads -or executes — the four runtime modules, `config.json`, -`calibration.json` and `tokenizer.json` — against the table checked in -for the pinned revision, before anything is imported, and refuses -importable entries the table does not name plus any existing -`__pycache__` bytecode cache or directory symlink (`model.safetensors` is -checked by exact size; the runtime's strict safetensors load rejects -truncated or structurally mismatched weights, but not same-size -bit-level corruption). +| Artifact | 0.3B | 0.8B hybrid | +| --- | --- | --- | +| Package | [vllm-sr/Vela-2.0-0.3B](https://huggingface.co/vllm-sr/Vela-2.0-0.3B) | [vllm-sr/Vela-2.0-0.8B](https://huggingface.co/vllm-sr/Vela-2.0-0.8B) | +| Pinned revision | `d6f03aa9baca4f9017fa8f8e9c2764d3bed3f409` | `326b01d81f61fff7fd4ae570e54b87a49119e490` | +| Identity | `vela2-unified` / `Vela2Model`, 309,114,369 parameters, all FP32 (1,236,473,252-byte safetensors) | `vela2-decoder` / `Vela2DecoderModel`, 754,565,954 parameters (2,022,189,488-byte shard + hashed 4,480,172-byte `broad_head.safetensors`) | +| Input window | 8,192 tokens | 16,384 tokens | +| Runtime | the checkpoint's own flat files: `vela2_inference.py`, `modeling_vela2.py`, `configuration_vela2.py`, `vela2_serve.py` | the same four modules, decoder-specific | +| Calibration | the package's own `calibration.json` (never cross-copied between members — shipped thresholds differ) | same rule; the side-loaded broad head needs its `broad_head` block | +| License | Apache-2.0; the inherited tokenizer material carries Gemma Terms of Use | Apache-2.0; Qwen3.5-0.8B lineage notices ship in the package | +| Runtime dependencies | [recipe/vela/requirements.txt](../../recipe/vela/requirements.txt) (torch, transformers, safetensors, tokenizers, numpy) | same | + +The 0.3B package ships no manifest, so its gate checks the config's +structural identity (model type, architecture, `max_length` 8,192, the +8 marker ids), then hashes every small file the runtime reads or +executes — the four runtime modules, `config.json`, `calibration.json` +and `tokenizer.json` — against the table checked in for the pinned +revision, before anything is imported, and refuses importable entries +the table does not name plus any existing `__pycache__` bytecode cache +or directory symlink (`model.safetensors` is checked by exact size; the +runtime's strict safetensors load rejects truncated or structurally +mismatched weights, but not same-size bit-level corruption). The 0.8B +gate mirrors this with a 10-file table — the same four runtime modules, +`config.json` (identity includes the served `model_name`, so a 4B or 9B +package fails with its own name), `calibration.json`, `tokenizer.json`, +`tokenizer_config.json`, `model.safetensors.index.json` (the shard map) +and the hashed `broad_head.safetensors` — with the 2 GB shard checked +by exact size. A later self-consistent snapshot rewrites its own files and fails the checked-in pin. +On either member the runtime loads parameters in FP32; on `cuda` the +hybrid runs its backbone under bf16 autocast with FP32 heads (the +evaluated setting), and on `cpu` it runs FP32 throughout. The hybrid's +served name comes from `config.json` (`vllm-sr/Vela-2.0-0.8B`); the +runtime's family-generic `DEFAULT_MODEL_NAME` is never used as a +default. + ## Current serving scope - The request envelope is upstream's three-field form, exactly: @@ -103,8 +119,14 @@ The response envelope is the pinned runtime's, passed through verbatim answers echo the value without a rule field). Serving-time temperatures divide the logits per question type; the learned `tau`/`tau_span` live inside the model. - The hybrid-only fields (`span_heads`, `noul_calibration`) do not - exist on this member. + The hybrid-only `noul_calibration` block is default-off on every + member (env `VELA2_NOUL_CALIBRATION` or the reference server's flag + enable it); the worker serves the default path only. On the 0.3B the + `span_heads` field does not exist; on the hybrid it is emitted + together with `spans`, keyed by the span question ids + (`{"router"|"broad"}` per question, after the runtime's six-rule + dispatch: the `"head"` override, trained ids, exact router label + sets, PII subsets, hallucination aliases, else broad). - `usage.input_tokens` is the sum over every rendered sequence — span window rows and extra span-question rows included, nothing deduplicated; `usage.output_tokens` is always 0. @@ -128,7 +150,8 @@ per-question errors on this path: upstream server would answer 500); no such path is reachable on the pinned runtime's request path. 3. HTTP 413 (`too_long`) only when the questions plus schema alone - exceed the 8,192-token window. + exceed the member's input window (8,192 on 0.3B, 16,384 on the + hybrid). 4. HTTP 401 (`{"detail": "Missing or invalid API key"}`) only when `VELA2_API_KEY` is set and the request does not carry exactly `Authorization: Bearer ` (applies to `/v1/systemone` and @@ -166,14 +189,17 @@ every question kind), so answering at all means ready. - `preprocess.py` owns the mapping of the pinned runtime's `SystemOneError` to HTTP (422, or 413 for `too_long`) and of its other ValueErrors to client errors. -- `engine.py` owns the pinned runtime, the pre-import package gate, the - served name, device state and the all-kind warmup. +- `engine.py` owns the 0.3B pinned runtime, the pre-import package gate, the + served name, device state and the all-kind warmup; `hybrid.py` mirrors + it for the 0.8B (its warmup also drives both span heads). - `postprocess.py` owns the pure confidence/score formulas (mirrored for the CPU tests) and response finishing: answer-order and key-set - reconstruction including set views, usage accounting, and refusal of - hybrid-only fields on this member. -- `src/frontend/vela.py` owns HTTP lifecycle, the single-flight lock, - the API-key gate and orchestration. + reconstruction including set views, usage accounting, the + member-aware `span_heads` check, and refusal of hybrid-only fields + on the 0.3B. +- `src/frontend/vela.py` owns HTTP lifecycle, member detection from the + package's `model_type`, the per-member `/v1/models` entry, the + single-flight lock, the API-key gate and orchestration. - The Rust frontend owns transport and forwarding, not model state. The server-side error surface and envelope semantics are recorded as @@ -184,16 +210,81 @@ the shared worker-contract issue, and nothing is posted upstream. - CPU contract tests, no weights, no torch: `PYTHONPATH=src python -m pytest tests/vela -q` (envelope triples, - transport discipline, package gate, formula mirrors, response - finishing, the HTTP surface with a contract fake). + transport discipline, both package gates, formula mirrors, response + finishing including the hybrid `span_heads` surface, member + detection, the HTTP surface with a contract fake). - Reference parity is exact match on recorded fixtures under the pinned versions: `recipe/vela/compare_reference.py` drives the recipe fixtures plus five synthetic rejections (invalid-span 422, over-limit 413, and the missing-field / extra-field / wrong-model envelope 422s) through both the pinned `vela2_serve.py` app and the - production worker pipeline over one shared engine, and freezes the - oracle fixture for the native encoder slices. CPU FP32 (no + production worker pipeline over one shared engine, for either member, + and freezes the oracle fixture for the native slices (`oracle-` for + 0.3B, `oracle-hybrid-` for 0.8B). CPU FP32 (no flash-linear-attention) is the oracle setting; GPU bf16-autocast is - a deployment option outside this slice's evidence. + a deployment option outside these slices' evidence. - The serving procedure, hardware requirements and the recorded evidence pointer are [recipe/vela/reference.md](../../recipe/vela/reference.md). + +## Native path (0.8B) + +The native implementation lives in [`native/`](native/) as the +`omni-vela-native` crate, a library around the exported bundle: + +- [`recipe/vela/export_native.py`](../../recipe/vela/export_native.py) + exports the pinned package into a native bundle: the 320 backbone + tensors re-keyed `backbone.` → `model.` with **per-tensor dtypes + preserved** (204 BF16 Linear/conv1d + 116 F32 embed/norms/A_log/ + dt_bias — the reference's own layout), the 37 F32 head tensors + (CandidateHead, `set_bias`, span2 router, span_broad) in one file, + `backbone_config` at the config root, calibration and tokenizer + copied, and a manifest pinning the source revision, the worker + gate's own file table, the tensor inventory and every bundle file's + SHA-256. `--bf16-out` additionally writes the uniform-BF16 backbone + the shared qwen3_5 loader consumes. +- `native/src/bundle.rs` re-verifies that manifest (revision, + inventory counts, per-file hashes) before anything loads it. +- `native/src/contract.rs` is the wire admission, mirroring + `protocol.py` and the pinned runtime's preamble with byte-equal + 422/400 bodies. +- `native/src/{state,questions,words,render,processing}.rs` are the + token-level processing: the state → parts mapping, the SystemOne → + predict mapping with the pinned runtime's validation triples (Noul + folded into a two-option choice; choice renders `[ABS]`, the + reference's abstain default), the GLiNER2/CJK word splitter, the + six-rule span-head dispatch with the hallucination-alias label + replacement, the DEV2.0 prompt rendering (parts pass, question + blocks with option end and query positions, span layout a/c with + word positions, the truncation budget, the 16,384-window 413) and + the long-target window rows. All five frozen token plans + (`tests/vela/fixtures/native-plan-*.json`) match field-for-field. +- `native/src/{heads,scoring}.rs` are the F32 heads and the scoring + path: CandidateHead, both SpanHeadV2 heads, `set_bias`, the + per-type temperatures, set thresholds, the PII length rule and + sparse gate, `span_decode_r4` (word units, unit vote, fill, + edge/URL trimming), label aliasing, score normalization and the + window aggregation — then `system_one_response` finishes the + SystemOne envelope (Noul views with the range-filtered max word + probability and set fan-out, span clipping to the plan's field + ranges with re-trim, `span_heads`, thresholds, confidence formulas, + usage). Replaying the recorded head-input fixtures + (`tests/vela/fixtures/head-input-*`): head-logit drift ≤ 3.8e-5, + predict-answer drift ≤ 7.3e-7, and the finished responses match the + frozen oracles within 1.5e-6 on four cases. +- `native/src/executor.rs` runs the backbone on CUDA through the + shared qwen3_5 crate: the family `Model` exposes only last-position + hidden states, so every readout position runs as its own full + sequence — `[parts .. position]` — which causality makes identical + to the reference's in-tree readout, packed into `forward_batch`. + The backbone runs from the uniform-BF16 export (the shared loader's + format); the heads stay F32 on the faithful tensors. Recorded + execution on sm_120 (RTX 5060, CUDA 13.1): the four small frozen + cases replay the recorded reference logits with a worst drift of + 0.285 — the bf16-vs-FP32-reference scale (gate 0.35). + +The crate is a library; the HTTP worker around `Processor` + +`Executor` is not part of it. Long-span GPU parity and measured +latency are not recorded. Checks: `cargo test -p omni-vela-native` +(28 tests, no weights, no GPU; render, head and response parity are +opt-in with `VELA_NATIVE_BUNDLE`, executor parity additionally with +`VELA_CUDA_LIB`; see [recipe/vela/native.md](../../recipe/vela/native.md)). diff --git a/src/models/vela/engine.py b/src/models/vela/engine.py index 60e27d1c..ac2f316a 100644 --- a/src/models/vela/engine.py +++ b/src/models/vela/engine.py @@ -126,7 +126,7 @@ def verify_config(root: Path) -> dict[str, Any]: return config -def verify_files(root: Path, table: dict[str, str] | None = None) -> list[str]: +def verify_files(root: Path, table: dict[str, str] | None = None, revision: str = REFERENCE_REVISION) -> list[str]: """Hash every pinned file in full and return their names. Must run before any package code is imported: the table names the @@ -138,7 +138,9 @@ def verify_files(root: Path, table: dict[str, str] | None = None) -> list[str]: ``__init__`` in every module form, or a namespace package's nested modules — would shadow one of the pinned runtime's dependencies — and any ``__pycache__`` or directory symlink, whose contents the - walk cannot verify against the pin. + walk cannot verify against the pin. ``revision`` names the pin the + table belongs to in hash-failure messages (the hybrid member passes + its own). """ table = dict(table or PINNED_FILES_SHA256) @@ -151,7 +153,7 @@ def verify_files(root: Path, table: dict[str, str] | None = None) -> list[str]: problems.append(f"missing pinned file {name}") continue if _sha256_safe(target) != digest: - problems.append(f"{name}: SHA-256 differs from the pin for {REFERENCE_REVISION}") + problems.append(f"{name}: SHA-256 differs from the pin for {revision}") for entry in sorted(Path(root).iterdir()): if entry.name in table: continue @@ -239,6 +241,10 @@ def verify_package(root: Path, table: dict[str, str] | None = None) -> None: class Vela2Executor: """Load and drive one immutable pinned Vela 2.0 0.3B runtime instance.""" + # Response finishing uses this to refuse the hybrid-only ``span_heads`` + # field (hybrid.py's executor sets it True). + is_hybrid = False + def __init__( self, package: str | Path, diff --git a/src/models/vela/hybrid.py b/src/models/vela/hybrid.py new file mode 100644 index 00000000..e02fa6fb --- /dev/null +++ b/src/models/vela/hybrid.py @@ -0,0 +1,241 @@ +"""Pinned executor for Vela 2.0 0.8B (``vela2-decoder``), the first hybrid member. + +Mirrors :mod:`models.vela.engine` for the compact hybrid member. The +``vela2-decoder`` package ships the same flat runtime modules as the +0.3B (with decoder-specific contents), so the gate reuses the same +helpers and the same one-package-per-process discipline; what differs +is the pin table, the structural identity of ``config.json``, and the +weights layout — one 2 GB shard checked by exact size plus the +side-loaded second span head. + +The runtime loads parameters in FP32 either way; on ``cuda`` it runs +the backbone under bf16 autocast with FP32 heads (the evaluated +setting), on ``cpu`` it runs FP32 throughout. The served model name +comes from ``config.json`` ``model_name`` (upstream +``DEFAULT_MODEL_NAME`` is the family-generic ``vllm-sr/Vela-2.0`` and +must not be used as a default here). +""" + +from __future__ import annotations + +import json +import sys +import threading +from pathlib import Path +from typing import Any + +from .engine import ( + put_first_on_sys_path, + verify_files, + verify_no_foreign_runtime, +) + +HYBRID_REPO = "vllm-sr/Vela-2.0-0.8B" +HYBRID_REVISION = "326b01d81f61fff7fd4ae570e54b87a49119e490" +# Structural identity of the pinned config.json. model_name is unique +# per member, so a 4B or 9B package fails with its own name. +EXPECTED_MODEL_TYPE = "vela2-decoder" +EXPECTED_ARCHITECTURE = "Vela2DecoderModel" +EXPECTED_MAX_LENGTH = 16384 +EXPECTED_MODEL_NAME = "vllm-sr/Vela-2.0-0.8B" +WEIGHTS_NAME = "model-00001-of-00001.safetensors" +EXPECTED_WEIGHTS_BYTES = 2022189488 +# The files this gate hashes, transcribed from the pinned revision and +# cross-checked against the package's own SHA256SUMS; repinning means +# updating this table together with HYBRID_REVISION. Every file the +# runtime reads or executes is here: the four runtime modules, +# ``vela2_serve.py`` (never imported, part of the pinned surface), +# ``config.json``, ``calibration.json``, ``tokenizer.json`` (the loaded +# tokenizer), ``tokenizer_config.json`` (listed by the runtime's own +# download manifest), ``model.safetensors.index.json`` (the shard map +# the loader reads) and ``broad_head.safetensors`` (the side-loaded +# second span head; 4.5 MB hashes in milliseconds). The 2 GB shard is +# checked by exact size only, as on 0.3B. +PINNED_FILES_SHA256 = { + "vela2_serve.py": "68e1ef278561f452aa0ca6afd6aa288636cecdb0b37fd7f7146a3c9bf17aa787", + "vela2_inference.py": "11de1322788942153a22db164a3fb26abcaa4c5bb4b057cd572e277457c17dc2", + "modeling_vela2.py": "b7ecd8dcacfb24dbe8249c7b96829a737f4e2b493fc2157497f1b62d9e02727d", + "configuration_vela2.py": "c9a41e434a9d2cfeb3ab6581c00679d241b3468c04d160a811d6fe2c85cd071f", + "config.json": "0637ca193b9159b6ece25837913956477164d3392d9b5dd1e6b74b95a484004a", + "calibration.json": "d96e447962d0dc1d58fa8422dddbae8b99960e21fd75a10ef9b096f700a211f5", + "tokenizer.json": "06b9509352d2af50381ab2247e083b80d32d5c0aba91c272ca9ff729b6a0e523", + "tokenizer_config.json": "bee8eba30f0eb4af73c0fe2cd06d0f89b657d7819941c438157ec42f7c80ea87", + "model.safetensors.index.json": "b53d1c79219677289bdb1541a1373dfe53119884f248b23c94a27aef7338919d", + "broad_head.safetensors": "687d7a435d6de846796274e417b6333f83df3085a6e7c3bca4c01cb4d75a30f0", +} +# A warmup that touches every question kind's readout path on both span +# heads: the CandidateHead option rows (choice/set share it, noul and +# score ride the same candidate readout), the span decoder and the +# threshold echo, one router-head span (the unsupported label is a +# trained alias) and one broad-head span (an open label with no trained +# match). Model text only; the result is discarded. +WARMUP_REQUEST = { + "state": { + "request": "Hello, I would like to check the status of my order.", + "answer": "Your order was delivered yesterday.", + }, + "model": None, # filled with the served name at warmup time + "questions": { + "kind": { + "type": "choice", + "instructions": "What is this message about?", + "criteria": {"order": "an order's status", "other": "anything else"}, + }, + "claims": { + "type": "set", + "instructions": "Which topics does the message mention?", + "criteria": {"delivery": "delivery of an order", "billing": "payments or charges"}, + }, + "tone": { + "type": "score", + "instructions": "How formal is the message?", + "criteria": ["casual", "neutral", "formal"], + }, + "polite": { + "type": "noul", + "instructions": "Is the message polite?", + }, + "unsupported": { + "type": "span", + "instructions": "Which spans of the answer are not supported by the request?", + "criteria": {"unsupported": "a claim not supported by the request"}, + }, + "products": { + "type": "span", + "instructions": "Which products or services are mentioned?", + "criteria": {"product": "a product or a service"}, + "over": "request", + }, + }, +} + + +def verify_config(root: Path) -> dict[str, Any]: + """Read config.json and check the pinned member's structural identity.""" + + path = Path(root) / "config.json" + if not path.is_file(): + raise ValueError(f"the package is missing config.json; this worker only serves {HYBRID_REPO}") + config = json.loads(path.read_text(encoding="utf-8")) + if config.get("model_type") != EXPECTED_MODEL_TYPE: + raise ValueError( + f"config.json declares model_type {config.get('model_type')!r}; this worker only " + f"serves the {EXPECTED_MODEL_TYPE} member {HYBRID_REPO} at {HYBRID_REVISION}" + ) + if config.get("architectures") != [EXPECTED_ARCHITECTURE]: + raise ValueError(f"config.json architectures must be [{EXPECTED_ARCHITECTURE!r}]") + if config.get("max_length") != EXPECTED_MAX_LENGTH: + raise ValueError(f"config.json max_length must be {EXPECTED_MAX_LENGTH}") + if config.get("model_name") != EXPECTED_MODEL_NAME: + raise ValueError( + f"config.json model_name must be {EXPECTED_MODEL_NAME!r}; another Vela 2.0 member " + "needs its own pinned executor" + ) + return config + + +def verify_weights(root: Path) -> None: + """Check the pinned shard is present at its exact pinned size.""" + + target = Path(root) / WEIGHTS_NAME + if not target.is_file(): + raise ValueError(f"the package is missing {WEIGHTS_NAME}") + size = target.stat().st_size + if size != EXPECTED_WEIGHTS_BYTES: + raise ValueError( + f"{WEIGHTS_NAME} is {size} bytes; the pinned revision's file is {EXPECTED_WEIGHTS_BYTES}" + ) + + +def verify_package(root: Path, table: dict[str, str] | None = None) -> None: + """The full pre-import gate for one hybrid package directory.""" + + verify_config(root) + verify_files(root, table or PINNED_FILES_SHA256, HYBRID_REVISION) + verify_weights(root) + + +class Vela2HybridExecutor: + """Load and drive one immutable pinned Vela 2.0 0.8B runtime instance.""" + + is_hybrid = True + + def __init__( + self, + package: str | Path, + *, + device: str = "cpu", + model_name: str | None = None, + batch_size: int | None = None, + token_budget: int | None = None, + ) -> None: + root = Path(package) + if not root.is_dir(): + raise ValueError(f"the Vela 2.0 0.8B package directory does not exist: {root}") + verify_package(root) + + # Same discipline as the 0.3B executor: the just-verified package + # root goes first on sys.path so the flat runtime modules resolve + # to the checkpoint's own copies, and a runtime module already + # imported from another directory is refused — one worker process + # serves one package. + verify_no_foreign_runtime(root) + put_first_on_sys_path(root) + # Import with bytecode writing off: the gate refuses existing + # bytecode caches, so a __pycache__ this import wrote would fail + # every later gate run of the same package. + previous_dont_write_bytecode = sys.dont_write_bytecode + sys.dont_write_bytecode = True + try: + import vela2_inference + finally: + sys.dont_write_bytecode = previous_dont_write_bytecode + + self.SystemOneError = vela2_inference.SystemOneError + self._system_one_request = vela2_inference.system_one_request + self.engine = vela2_inference.Vela2.from_pretrained( + root, backend="torch", device=device, + **({"batch_size": batch_size} if batch_size is not None else {}), + **({"token_budget": token_budget} if token_budget is not None else {}), + ) + # The decoder members declare their served name in config.json; + # the runtime's DEFAULT_MODEL_NAME is family-generic. + self.model_name = model_name or self.engine.model_name + self.max_length = int(self.engine.max_len) + self._lock = threading.RLock() + self._closed = False + + @property + def served(self) -> set[str]: + """The model names this executor answers to (upstream: one).""" + + return {self.model_name} + + def system_one(self, body: dict[str, Any]) -> dict[str, Any]: + """Run one admitted request body; the response is upstream's, verbatim. + + Hybrid responses carry one extra top-level field, ``span_heads``, + alongside ``spans``; response finishing checks it in + :func:`models.vela.postprocess.finish_response`. + """ + + with self._lock: + if self._closed: + raise RuntimeError("Vela 2.0 hybrid executor is closed") + return self._system_one_request(self.engine, body, served=self.served) + + def warmup(self) -> None: + """Run one real request of every question kind on both span heads.""" + + request = dict(WARMUP_REQUEST) + request["model"] = self.model_name + response = self.system_one(request) + if not isinstance(response, dict) or "answers" not in response: + raise RuntimeError("warmup did not produce a SystemOne response") + + def close(self) -> None: + with self._lock: + if self._closed: + return + self._closed = True + self.engine = None diff --git a/src/models/vela/native/Cargo.toml b/src/models/vela/native/Cargo.toml new file mode 100644 index 00000000..f2c6547a --- /dev/null +++ b/src/models/vela/native/Cargo.toml @@ -0,0 +1,48 @@ +[package] +name = "omni-vela-native" +version = "0.1.0" +edition = "2024" +publish = false +description = "Native library for vllm-sr/Vela-2.0-0.8B: bundle gate, wire contract, processing, F32 heads, scoring and CUDA execution" + +[dependencies] +anyhow = "1.0.100" +libm = "0.2" +safetensors = "0.8.0" +onig = { package = "onig", version = "6", default-features = false } +omni-qwen3-5-native = { path = "../../qwen3_5/native" } +serde = { version = "1", features = ["derive"] } +serde_json = { version = "1.0.149", features = ["float_roundtrip", "preserve_order"] } +sha2 = "0.11" +tokenizers = { version = "=0.22.2", default-features = false, features = ["onig"] } + +[lib] +path = "src/lib.rs" + +[[test]] +name = "vela_native_contract" +path = "../../../../tests/vela/native_contract.rs" + +[[test]] +name = "vela_native_bundle" +path = "../../../../tests/vela/native_bundle.rs" + +[[test]] +name = "vela_native_processing" +path = "../../../../tests/vela/native_processing.rs" + +[[test]] +name = "vela_native_units" +path = "../../../../tests/vela/native_units.rs" + +[[test]] +name = "vela_native_heads" +path = "../../../../tests/vela/native_heads.rs" + +[[test]] +name = "vela_native_executor" +path = "../../../../tests/vela/native_executor.rs" + +[[test]] +name = "vela_native_response" +path = "../../../../tests/vela/native_response.rs" diff --git a/src/models/vela/native/examples/parity.rs b/src/models/vela/native/examples/parity.rs new file mode 100644 index 00000000..071d0f52 --- /dev/null +++ b/src/models/vela/native/examples/parity.rs @@ -0,0 +1,87 @@ +//! Debug helper: compare the rendered rows against one fixture and +//! print the first differing field. Run with: +//! +//! VELA_NATIVE_BUNDLE=/abs/path cargo run -p omni-vela-native --example parity -- example-request + +use std::path::PathBuf; + +use serde_json::Value; + +fn main() { + let case = std::env::args().nth(1).unwrap_or("example-request".into()); + let bundle = PathBuf::from( + std::env::var("VELA_NATIVE_BUNDLE").expect("VELA_NATIVE_BUNDLE points at the export"), + ); + let fixture: Value = serde_json::from_str( + &std::fs::read_to_string( + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../../../../tests/vela/fixtures") + .join(format!("native-plan-{case}.json")), + ) + .expect("fixture"), + ) + .expect("fixture json"); + + let processor = omni_vela_native::processing::Processor::load(&bundle).expect("load"); + let request = fixture["request"].as_object().expect("request"); + match processor.plan(request) { + Ok(planned) => { + let got = serde_json::json!({ + "rows": serde_json::to_value(&planned.rows).unwrap(), + "window_rows": serde_json::to_value(&planned.window_rows).unwrap(), + }); + compare( + "rows", + got["rows"].as_array().unwrap(), + fixture["rows"].as_array().unwrap(), + ); + if !fixture["window_rows"].is_null() { + compare( + "window_rows", + got["window_rows"].as_array().unwrap(), + fixture["window_rows"].as_array().unwrap(), + ); + } + println!("parity: no field differences found"); + } + Err(error) => println!( + "plan error: {}", + serde_json::to_string(&error.body).unwrap() + ), + } +} + +fn compare(name: &str, got: &[Value], want: &[Value]) { + println!("{name}: {} vs {} row(s)", got.len(), want.len()); + for (index, (a, b)) in got.iter().zip(want).enumerate() { + let (Some(a), Some(b)) = (a.as_object(), b.as_object()) else { + continue; + }; + for key in b.keys() { + if a.get(key) != b.get(key) { + println!("row {index} field {key}: differs"); + println!( + " want: {}", + serde_json::to_string(&b[key]) + .unwrap() + .chars() + .take(300) + .collect::() + ); + println!( + " got: {}", + serde_json::to_string(&a[key]) + .unwrap() + .chars() + .take(300) + .collect::() + ); + } + } + for key in a.keys() { + if !b.contains_key(key) { + println!("row {index} field {key}: extra in ours"); + } + } + } +} diff --git a/src/models/vela/native/src/bundle.rs b/src/models/vela/native/src/bundle.rs new file mode 100644 index 00000000..b467aa36 --- /dev/null +++ b/src/models/vela/native/src/bundle.rs @@ -0,0 +1,166 @@ +//! The exported native bundle's manifest gate. +//! +//! [`recipe/vela/export_native.py`](../../../recipe/vela/export_native.py) +//! produces the bundle from the pinned package; this module loads its +//! manifest, pins the source revision and the member's tensor +//! inventory, and re-hashes every bundle file before the (future) +//! executor touches it — the same provenance-before-import discipline +//! as the reference worker's package gate. + +use std::collections::HashSet; +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result, ensure}; +use serde_json::Value; + +pub const FORMAT: &str = "vela2-native-bundle/1"; +pub const MODEL_ID: &str = "vllm-sr/Vela-2.0-0.8B"; +pub const SOURCE_REVISION: &str = "326b01d81f61fff7fd4ae570e54b87a49119e490"; +/// This member's exported inventory, pinned from the pinned package: +/// 320 backbone tensors (204 BF16 Linear/conv1d + 116 F32) and 37 F32 +/// head tensors (CandidateHead, set_bias, span2 router, span_broad). +pub const BACKBONE_TENSORS: (usize, usize, usize) = (320, 204, 116); +pub const HEAD_TENSORS: usize = 37; + +#[derive(Debug)] +pub struct Bundle { + pub dir: PathBuf, + pub manifest: Value, +} + +impl Bundle { + /// Load and fully verify one exported bundle directory. + pub fn load(dir: &Path) -> Result { + let manifest: Value = serde_json::from_slice( + &std::fs::read(dir.join("vela_native_manifest.json")) + .context("export the pinned package first; see recipe/vela/export_native.py")?, + ) + .context("vela_native_manifest.json must be JSON")?; + ensure!( + manifest["format"] == FORMAT + && manifest["model_id"] == MODEL_ID + && manifest["source"]["revision"] == SOURCE_REVISION, + "expected a bundle exported from the pinned {MODEL_ID} package" + ); + let backbone = &manifest["backbone"]; + ensure!( + backbone["tensors"].as_u64() == Some(BACKBONE_TENSORS.0 as u64) + && backbone["bf16"].as_u64() == Some(BACKBONE_TENSORS.1 as u64) + && backbone["fp32"].as_u64() == Some(BACKBONE_TENSORS.2 as u64), + "unexpected backbone inventory; re-export from the pinned package" + ); + ensure!( + manifest["heads"]["tensors"].as_u64() == Some(HEAD_TENSORS as u64), + "unexpected head inventory; re-export from the pinned package" + ); + let files = manifest["files_sha256"] + .as_object() + .context("files_sha256 must be an object")?; + ensure!( + !files.is_empty(), + "files_sha256 pins no files; the gate must cover the whole bundle" + ); + for (name, digest) in files { + let actual = hash_file(&dir.join(name))?; + ensure!( + actual == digest.as_str().unwrap_or(""), + "{name} does not match the manifest SHA-256" + ); + } + // The uniform-BF16 backbone the executor runs is pinned too. + if manifest["files_sha256"] + .get("bf16/model.safetensors") + .is_some() + { + ensure!( + dir.join("bf16/model.safetensors").is_file(), + "the manifest pins bf16/model.safetensors but the file is missing; re-export" + ); + } + // Fail closed on the directory, as the reference gate walks the + // package root: a top-level entry must be a pinned file, this + // manifest, or a directory holding pinned paths (the export's + // bf16/ backbone) — anything else is unlisted, and inside an + // admitted directory every file must be pinned too. + let pinned: HashSet<&str> = files + .keys() + .map(String::as_str) + .chain(std::iter::once("vela_native_manifest.json")) + .collect(); + let mut unpinned: Vec = Vec::new(); + for entry in std::fs::read_dir(dir) + .with_context(|| format!("cannot list the bundle directory {}", dir.display()))? + { + let entry = entry.context("a readable bundle entry")?; + let name = entry.file_name().to_string_lossy().into_owned(); + if pinned.contains(name.as_str()) { + continue; + } + let prefix = format!("{name}/"); + if !files.keys().any(|key| key.starts_with(&prefix)) { + unpinned.push(name); + continue; + } + for inner in std::fs::read_dir(entry.path()) + .with_context(|| format!("cannot list the bundle subdirectory {name}"))? + { + let inner = inner.context("a readable bundle entry")?; + let inner_name = format!("{name}/{}", inner.file_name().to_string_lossy()); + if !pinned.contains(inner_name.as_str()) { + unpinned.push(inner_name); + } + } + } + unpinned.sort(); + ensure!( + unpinned.is_empty(), + "unpinned file(s) in the bundle directory: {unpinned:?}" + ); + ensure!( + manifest["max_length"].as_u64() == Some(16_384) + && manifest["model_name"].as_str() == Some(MODEL_ID), + "unexpected member identity in the manifest" + ); + Ok(Self { + dir: dir.to_owned(), + manifest, + }) + } + + /// The served model name (upstream: config.json `model_name`). + pub fn model_name(&self) -> &str { + self.manifest["model_name"].as_str().unwrap_or(MODEL_ID) + } + + /// The member's input window. + pub fn max_length(&self) -> usize { + self.manifest["max_length"].as_u64().unwrap_or(16_384) as usize + } +} + +fn hash_file(path: &Path) -> Result { + use anyhow::bail; + use sha2::{Digest, Sha256}; + use std::io::Read; + + let mut hasher = Sha256::new(); + let file = match std::fs::File::open(path) { + Ok(file) => file, + Err(error) => bail!("cannot read {}: {error}", path.display()), + }; + let mut reader = std::io::BufReader::new(file); + let mut chunk = vec![0u8; 8 * 1024 * 1024]; + loop { + let read = reader.read(&mut chunk)?; + if read == 0 { + break; + } + hasher.update(&chunk[..read]); + } + Ok(hex(&hasher.finalize())) +} + +/// Lower-hex of a digest; sha2 0.11's output has no `LowerHex` impl. +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|byte| format!("{byte:02x}")).collect() +} diff --git a/src/models/vela/native/src/contract.rs b/src/models/vela/native/src/contract.rs new file mode 100644 index 00000000..a2304730 --- /dev/null +++ b/src/models/vela/native/src/contract.rs @@ -0,0 +1,139 @@ +//! Vela 2.0 wire admission for the native worker, mirroring +//! `models.vela.protocol` and the pinned runtime's request preamble so +//! clients cannot tell this layer from the reference worker: the +//! three-field envelope with upstream's exact `{"detail": [{"loc", +//! "msg", "type"}]}` triples, and the whole-request error model. +//! +//! Deeper validation — state rendering, question semantics, the 413 — +//! lives in the processing modules and mirrors the pinned runtime +//! against the `native-plan-*` fixtures. One known residual +//! difference: a non-finite JSON number answers the generic 400 +//! message here, where the Python worker names the value; duplicate +//! keys are mirrored verbatim. + +use serde_json::{Map, Value, json}; + +/// Slice-scoped transport cap, as on the reference worker. +pub const MAX_BODY: usize = 8 * 1024 * 1024; +/// The envelope fields, in upstream's missing-field check order. +pub const REQUEST_FIELDS: [&str; 3] = ["state", "model", "questions"]; + +/// A request rejection: an HTTP status and the JSON body a client sees. +#[derive(Debug, PartialEq)] +pub struct RequestError { + pub status: u16, + pub body: Value, +} + +impl RequestError { + fn detail(status: u16, message: &str) -> Self { + Self { + status, + body: json!({"detail": message}), + } + } + + /// One upstream `SystemOneError` as a whole-request 422 triple. + pub fn envelope(loc: &[&str], msg: &str, type_: &str) -> Self { + let mut path = vec![Value::from("body")]; + path.extend(loc.iter().map(|part| Value::from(*part))); + Self { + status: 422, + body: json!({"detail": [{"loc": path, "msg": msg, "type": type_}]}), + } + } +} + +/// Decode a bounded JSON body and reject ambiguous JSON values. +pub fn parse_body(raw: &[u8]) -> Result, RequestError> { + if raw.len() > MAX_BODY { + return Err(RequestError::detail(413, "request body exceeds 8 MiB")); + } + let message = + omni_qwen3_5_native::json::parse(raw).map_err(|error| normalize_parse_error(&error))?; + Ok(message) +} + +/// Mirror the reference worker's 400 messages: duplicate keys and +/// non-object bodies verbatim, everything else the shared generic +/// message. +pub(crate) fn normalize_parse_error(error: &str) -> RequestError { + if let Some(rest) = error.split("duplicate key ").nth(1) { + // serde appends its position suffix after the quoted key. + if let Some(key) = rest + .strip_prefix('"') + .and_then(|tail| tail.split('"').next()) + { + return RequestError::detail(400, &format!("duplicate JSON key '{key}'")); + } + } + // The shared parser's exact non-object error, answered with the + // reference worker's wording. + if error == "request body must be a JSON object" { + return RequestError::detail(400, "request must be a JSON object"); + } + RequestError::detail(400, "request body must contain valid JSON and UTF-8 text") +} + +/// Admit the envelope exactly as upstream's request preamble does, for +/// one served model name: the three required fields present, no extra +/// field, `model` non-blank and equal to the served name. +pub fn admit_request(body: &Map, served: &str) -> Result<(), RequestError> { + for field in REQUEST_FIELDS { + if !body.contains_key(field) { + return Err(RequestError::envelope( + &[field], + "Field required", + "missing", + )); + } + } + let mut extra: Vec<&String> = body + .keys() + .filter(|key| !REQUEST_FIELDS.contains(&key.as_str())) + .collect(); + extra.sort(); + if let Some(first) = extra.first() { + return Err(RequestError::envelope( + &[first], + "Extra inputs are not permitted", + "extra_forbidden", + )); + } + let model = body["model"].as_str().unwrap_or(""); + if model.trim().is_empty() { + return Err(RequestError::envelope( + &["model"], + "model must not be empty or whitespace", + "value_error", + )); + } + if model != served { + return Err(RequestError::envelope( + &["model"], + "Model is not served by this runtime", + "value_error.model", + )); + } + Ok(()) +} + +/// Admit one complete raw request body: parse, then envelope-check. +pub fn compile(raw: &[u8], served: &str) -> Result, RequestError> { + let body = parse_body(raw)?; + admit_request(&body, served)?; + Ok(body) +} + +/// A pinned-runtime `ValueError` rejection as the reference worker maps +/// it (`map_runtime_error`): a whole-request 422 naming the runtime. +pub fn runtime_rejection(message: &str) -> RequestError { + RequestError::detail( + 422, + &format!("request rejected by the pinned runtime: {message}"), + ) +} + +#[cfg(test)] +#[path = "../../../../../tests/vela/native_contract_unit.rs"] +mod tests; diff --git a/src/models/vela/native/src/executor.rs b/src/models/vela/native/src/executor.rs new file mode 100644 index 00000000..4cd73266 --- /dev/null +++ b/src/models/vela/native/src/executor.rs @@ -0,0 +1,250 @@ +//! The native executor: the backbone through the shared qwen3_5 crate +//! on CUDA. The family `Model` exposes only last-position hidden +//! states, so every readout position runs as its own full sequence — +//! `[parts .. position]` — and causality makes that hidden identical +//! to the in-sequence one the reference reads inside its tree forward. +//! Sequences pack into `forward_batch`. +//! +//! The backbone runs from the bundle's uniform-BF16 export — the +//! shared loader's format — while the Vela heads stay F32 on the +//! bundle's faithful tensors. + +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result, ensure}; +use omni_qwen3_5_native::model::Model; + +use crate::bundle::Bundle; +use crate::heads::Heads; +use crate::render::{Block, RenderedRow}; + +/// Span-block bookkeeping while readouts accumulate. +struct SpanReadouts { + block_index: usize, + label_ranges: Vec<(usize, usize)>, +} + +/// One readout sequence: full token prefix, hidden at its last row. +struct Readout { + ids: Vec, + /// (block index, Option) attribution; None = the query. + tag: (usize, Option), +} + +pub struct Executor { + pub model: Model, + pub heads: Heads, + bundle: PathBuf, +} + +impl Executor { + pub fn load(bundle: &Path, library: &Path) -> Result { + Bundle::load(bundle).context("the exported bundle did not pass the manifest gate")?; + let model = Model::load(&bundle.join("bf16"), library) + .context("the shared qwen3_5 model did not load the uniform-BF16 backbone")?; + // heads.rs builds every projection at its private HIDDEN = 1024. + ensure!( + model.cfg.hidden == 1024, + "the backbone hidden size {} cannot feed the heads' 1024", + model.cfg.hidden + ); + let heads = Heads::load(bundle)?; + Ok(Self { + model, + heads, + bundle: bundle.to_owned(), + }) + } + + pub fn bundle(&self) -> &Path { + &self.bundle + } + + /// The candidate hiddens (per option end, `[ABS]` last when shown) + /// and the query hidden of every question block, plus the word + /// hiddens, label means and span-query hidden of the span block. + pub fn row_readouts(&mut self, row: &RenderedRow) -> Result { + let mut readouts: Vec = Vec::new(); + for (block_index, block) in row.blocks.iter().enumerate() { + let Block::Question { ids, ends, .. } = block else { + continue; + }; + for (end_index, end) in ends.iter().enumerate() { + readouts.push(Readout { + ids: [row.p_ids.as_slice(), &ids[..=*end]].concat(), + tag: (block_index, Some(end_index)), + }); + } + } + let mut span: Option = None; + if let Some(block) = row.blocks.iter().find(|b| matches!(b, Block::Span { .. })) { + let Block::Span { + ids, + label_ranges, + w_pos, + words_in_block, + .. + } = block + else { + unreachable!() + }; + let block_index = row + .blocks + .iter() + .position(|b| matches!(b, Block::Span { .. })) + .unwrap(); + for (word_index, position) in w_pos.iter().enumerate() { + // Layout c: w_pos is span-block-relative, the hidden + // rides [parts, block..=position]. Layout a: w_pos is + // parts-relative, the hidden is the parts pass itself. + let word_ids = if *words_in_block { + [row.p_ids.as_slice(), &ids[..=*position]].concat() + } else { + row.p_ids[..=*position].to_vec() + }; + readouts.push(Readout { + ids: word_ids, + tag: (block_index, Some(word_index)), + }); + } + span = Some(SpanReadouts { + block_index, + label_ranges: label_ranges.clone(), + }); + } + // Each block's query position is its last token: the full + // sequence's final hidden. + for (block_index, block) in row.blocks.iter().enumerate() { + let ids = match block { + Block::Question { ids, .. } | Block::Span { ids, .. } => ids, + }; + // The query is each block's last token by construction. + debug_assert!(ids.last().is_some()); + readouts.push(Readout { + ids: [row.p_ids.as_slice(), ids].concat(), + tag: (block_index, None), + }); + } + let hiddens = self.run_all(&readouts)?; + // Split per block. + /// Per block: (candidate/word hiddens, query hiddens). + type PerBlock = std::collections::HashMap>, Vec>)>; + let mut per_block: PerBlock = std::collections::HashMap::new(); + for (readout, hidden) in readouts.iter().zip(hiddens) { + let entry = per_block + .entry(readout.tag.0) + .or_insert_with(|| (Vec::new(), Vec::new())); + match readout.tag.1 { + Some(_) => entry.0.push(hidden), + None => entry.1.push(hidden), + } + } + let mut questions = Vec::new(); + for (block_index, block) in row.blocks.iter().enumerate() { + if let Block::Question { qid, .. } = block { + let (candidates, queries) = per_block + .remove(&block_index) + .unwrap_or_else(|| (Vec::new(), Vec::new())); + questions.push(QReadout { + block_index, + qid: qid.clone(), + candidates, + query: queries.into_iter().next(), + }); + } + } + let mut span_readouts = None; + if let Some(info) = span { + let (words, queries) = per_block + .remove(&info.block_index) + .unwrap_or_else(|| (Vec::new(), Vec::new())); + // The label mean hiddens: mean over each label range's + // option-block hiddens... the reference means the raw hidden + // states of the label tokens, so run those positions too. + let label_runs: Vec = info + .label_ranges + .iter() + .flat_map(|(start, end)| { + (*start..=*end).map(|position| Readout { + ids: [ + row.p_ids.as_slice(), + &row.span_block_ids(info.block_index)[..=position], + ] + .concat(), + tag: (info.block_index, None), + }) + }) + .collect(); + let label_hiddens = self.run_all(&label_runs)?; + let mut means = Vec::new(); + let mut cursor = 0usize; + for (start, end) in &info.label_ranges { + let count = end - start + 1; + let mut mean = vec![0.0f32; label_hiddens.first().map_or(0, |h| h.len())]; + for hidden in &label_hiddens[cursor..cursor + count] { + for (m, v) in mean.iter_mut().zip(hidden) { + *m += v; + } + } + for m in mean.iter_mut() { + *m /= count as f32; + } + means.push(mean); + cursor += count; + } + span_readouts = Some(SpanBlockReadouts { + words, + labels: means, + query: queries.into_iter().next(), + }); + } + Ok(RowReadouts { + questions, + span: span_readouts, + }) + } + + fn run_all(&mut self, readouts: &[Readout]) -> Result>> { + let mut hiddens = Vec::with_capacity(readouts.len()); + const CHUNK: usize = 16; + // A readout's rounding depends on which readouts share its + // packed chunk (the input-projection GEMM runs over the packed + // token count), so the recorded GPU evidence and CHUNK must + // move together. + for chunk in readouts.chunks(CHUNK) { + let inputs: Vec<&[u32]> = chunk.iter().map(|r| r.ids.as_slice()).collect(); + hiddens.extend(self.model.forward_batch(&inputs)?); + } + Ok(hiddens) + } +} + +/// Question-block readouts: candidate hiddens in option order +/// (`[ABS]` last) and the block's query hidden. +pub struct QReadout { + pub block_index: usize, + pub qid: String, + pub candidates: Vec>, + pub query: Option>, +} + +/// Span-block readouts: word hiddens, label-mean hiddens, query hidden. +pub struct SpanBlockReadouts { + pub words: Vec>, + pub labels: Vec>, + pub query: Option>, +} + +pub struct RowReadouts { + pub questions: Vec, + pub span: Option, +} + +impl RenderedRow { + fn span_block_ids(&self, block_index: usize) -> &[u32] { + match &self.blocks[block_index] { + Block::Span { ids, .. } => ids, + _ => &[], + } + } +} diff --git a/src/models/vela/native/src/heads.rs b/src/models/vela/native/src/heads.rs new file mode 100644 index 00000000..1381d555 --- /dev/null +++ b/src/models/vela/native/src/heads.rs @@ -0,0 +1,286 @@ +//! The Vela-owned F32 heads, mirrored from the checkpoint's +//! `CandidateHead` and `SpanHeadV2` (the reference computes both with +//! autocast disabled, in float32 — deterministic math). Tensors load +//! from the exported bundle's `heads.safetensors` (37 F32 tensors: +//! CandidateHead, `set_bias`, the router `span2.*` and the side-loaded +//! `span_broad.*`). + +use std::path::Path; + +use anyhow::{Context, Result, ensure}; +use safetensors::SafeTensors; + +const HIDDEN: usize = 1024; +const HEAD_DIM: usize = 256; + +fn gelu(x: f32) -> f32 { + 0.5 * x * (1.0 + libm::erff(x / std::f32::consts::SQRT_2)) +} + +/// One `Linear`: weights row-major `[out, in]`, optional bias. +struct Linear { + weight: Vec, + bias: Option>, + out: usize, + input: usize, +} + +impl Linear { + fn apply(&self, x: &[f32]) -> Vec { + debug_assert_eq!(x.len(), self.input); + let mut out = vec![0.0; self.out]; + for (row, slot) in out.iter_mut().enumerate() { + let mut sum = self.bias.as_ref().map_or(0.0, |b| b[row]); + let base = row * self.input; + for (index, value) in x.iter().enumerate() { + sum += self.weight[base + index] * value; + } + *slot = sum; + } + out + } +} + +/// One `LayerNorm` over the hidden vector (PyTorch defaults, eps 1e-5). +struct LayerNorm { + weight: Vec, + bias: Vec, +} + +impl LayerNorm { + fn apply(&self, x: &[f32]) -> Vec { + let n = x.len() as f32; + let mean = x.iter().sum::() / n; + let var = x.iter().map(|v| (v - mean) * (v - mean)).sum::() / n; + let inv = 1.0 / (var + 1e-5).sqrt(); + x.iter() + .enumerate() + .map(|(i, v)| (v - mean) * inv * self.weight[i] + self.bias[i]) + .collect() + } +} + +/// `SpanHeadV2` (router `span2.*` or broad `span_broad.*`). +pub struct SpanHead { + word_norm: LayerNorm, + label_norm: LayerNorm, + slot: Vec, // [slots, hidden] + slots: usize, + d: usize, + k: Linear, + q: Linear, + m: Linear, + n: Linear, + v: Linear, +} + +impl SpanHead { + /// score (W, L): words (W, H) raw states, labels (L, H) block means. + pub fn forward(&self, words: &[Vec], labels: &[Vec]) -> Vec> { + let mut w: Vec> = words.iter().map(|row| self.word_norm.apply(row)).collect(); + // Per-column mean over the words, subtracted. + if !w.is_empty() { + let mut mean = vec![0.0f32; HIDDEN]; + for row in &w { + for (i, value) in row.iter().enumerate() { + mean[i] += value; + } + } + for value in mean.iter_mut() { + *value /= w.len() as f32; + } + for row in w.iter_mut() { + for (i, value) in row.iter_mut().enumerate() { + *value -= mean[i]; + } + } + } + // Label states: normalized + slot embedding, mean-centered when + // two or more labels. + let mut e: Vec> = labels + .iter() + .enumerate() + .map(|(index, row)| { + let mut state = self.label_norm.apply(row); + let slot = index.min(self.slots - 1); + let base = slot * HIDDEN; + for (i, value) in state.iter_mut().enumerate() { + *value += self.slot[base + i]; + } + state + }) + .collect(); + if e.len() >= 2 { + let mut mean = vec![0.0f32; HIDDEN]; + for row in &e { + for (i, value) in row.iter().enumerate() { + mean[i] += value; + } + } + for value in mean.iter_mut() { + *value /= e.len() as f32; + } + for row in e.iter_mut() { + for (i, value) in row.iter_mut().enumerate() { + *value -= mean[i]; + } + } + } + let scale = 1.0 / (self.d as f32).sqrt(); + let k: Vec> = w.iter().map(|row| self.k.apply(row)).collect(); + let q: Vec> = e.iter().map(|row| self.q.apply(row)).collect(); + let m_rows: Vec> = w.iter().map(|row| self.m.apply(row)).collect(); + let n_rows: Vec> = e.iter().map(|row| self.n.apply(row)).collect(); + w.iter() + .enumerate() + .map(|(wi, _)| { + (0..labels.len()) + .map(|li| { + let mut bil = 0.0f32; + for (a, b) in k[wi].iter().zip(&q[li]) { + bil += a * b; + } + bil *= scale; + let mut summed = vec![0.0f32; HEAD_DIM]; + for (index, value) in summed.iter_mut().enumerate() { + *value = gelu(m_rows[wi][index] + n_rows[li][index]); + } + let mut score = 0.0f32; + for (index, value) in summed.iter().enumerate() { + score += self.v.weight[index] * value; + } + bil + score + }) + .collect() + }) + .collect() + } +} + +/// The two heads plus `set_bias`, loaded from one safetensors file. +pub struct Heads { + candidate_norm: LayerNorm, + query_norm: LayerNorm, + key: Linear, + query: Linear, + candidate_mlp: Linear, + query_mlp: Linear, + scalar: Linear, + pub set_bias: f32, + pub router: SpanHead, + pub broad: SpanHead, +} + +fn tensor<'a>(file: &'a SafeTensors<'a>, name: &str) -> Result> { + let view = file + .tensor(name) + .with_context(|| format!("heads.safetensors is missing {name}"))?; + ensure!( + view.dtype() == safetensors::Dtype::F32, + "{name} must be F32" + ); + Ok(view + .data() + .as_chunks::<4>() + .0 + .iter() + .map(|chunk| f32::from_le_bytes(*chunk)) + .collect()) +} + +fn linear(file: &SafeTensors, prefix: &str, bias: bool, input: usize) -> Result { + let weight = tensor(file, &format!("{prefix}.weight"))?; + let out = weight.len() / input; + ensure!( + weight.len() % input == 0 && out > 0, + "{prefix}.weight shape" + ); + Ok(Linear { + weight, + bias: if bias { + Some(tensor(file, &format!("{prefix}.bias"))?) + } else { + None + }, + out, + input, + }) +} + +fn norm(file: &SafeTensors, prefix: &str) -> Result { + Ok(LayerNorm { + weight: tensor(file, &format!("{prefix}.weight"))?, + bias: tensor(file, &format!("{prefix}.bias"))?, + }) +} + +fn span_head(file: &SafeTensors, prefix: &str, d: usize, slots: usize) -> Result { + let slot = tensor(file, &format!("{prefix}.slot.weight"))?; + Ok(SpanHead { + word_norm: norm(file, &format!("{prefix}.wn"))?, + label_norm: norm(file, &format!("{prefix}.ln"))?, + slots, + slot, + d, + k: linear(file, &format!("{prefix}.K"), false, HIDDEN)?, + q: linear(file, &format!("{prefix}.Q"), false, HIDDEN)?, + m: linear(file, &format!("{prefix}.M"), true, HIDDEN)?, + n: linear(file, &format!("{prefix}.N"), false, HIDDEN)?, + v: linear(file, &format!("{prefix}.v"), false, HEAD_DIM)?, + }) +} + +impl Heads { + pub fn load(bundle: &Path) -> Result { + let raw = std::fs::read(bundle.join("heads.safetensors")).context("heads.safetensors")?; + let file = SafeTensors::deserialize(&raw).context("heads.safetensors must parse")?; + let set_bias = tensor(&file, "set_bias")?; + ensure!(set_bias.len() == 1, "set_bias is a scalar"); + // The span-head geometry ships in the manifest's calibration + // (d 256, slots 64 for this member). + Ok(Self { + candidate_norm: norm(&file, "head.candidate_norm")?, + query_norm: norm(&file, "head.query_norm")?, + key: linear(&file, "head.key", false, HIDDEN)?, + query: linear(&file, "head.query", false, HIDDEN)?, + candidate_mlp: linear(&file, "head.candidate_mlp", true, HIDDEN)?, + query_mlp: linear(&file, "head.query_mlp", false, HIDDEN)?, + scalar: linear(&file, "head.scalar", false, HEAD_DIM)?, + set_bias: set_bias[0], + router: span_head(&file, "span2", HEAD_DIM, 64)?, + broad: span_head(&file, "span_broad", HEAD_DIM, 64)?, + }) + } + + /// CandidateHead: option-end hiddens (W, H) and the block's query + /// hidden (H) -> one logit per option (plus `[ABS]` last when the + /// row carries it — the caller slices). + pub fn forward_candidate(&self, candidates: &[Vec], query: &[f32]) -> Vec { + let global_query = self.query_norm.apply(query); + let q_proj = self.query.apply(&global_query); + let q_mlp = self.query_mlp.apply(&global_query); + let scale = 1.0 / (HEAD_DIM as f32).sqrt(); + candidates + .iter() + .map(|row| { + let candidate = self.candidate_norm.apply(row); + let key = self.key.apply(&candidate); + let mut bilinear = 0.0f32; + for (a, b) in key.iter().zip(&q_proj) { + bilinear += a * b; + } + bilinear *= scale; + let mlp = self.candidate_mlp.apply(&candidate); + let mut summed = vec![0.0f32; HEAD_DIM]; + for (index, value) in summed.iter_mut().enumerate() { + *value = gelu(mlp[index] + q_mlp[index]); + } + let mut nonlinear = 0.0f32; + for (index, value) in summed.iter().enumerate() { + nonlinear += self.scalar.weight[index] * value; + } + bilinear + nonlinear + }) + .collect() + } +} diff --git a/src/models/vela/native/src/lib.rs b/src/models/vela/native/src/lib.rs new file mode 100644 index 00000000..c30cea23 --- /dev/null +++ b/src/models/vela/native/src/lib.rs @@ -0,0 +1,19 @@ +//! Native library for vllm-sr/Vela-2.0-0.8B: the exported-bundle gate, +//! the wire contract, the token-level processing (state → parts, the +//! SystemOne → predict mapping with the pinned runtime's validation +//! triples, word units, the DEV2.0 prompt rendering — mirrored against +//! the frozen `native-plan-*` fixtures), the F32 heads and scoring +//! (checked against the recorded `head-input-*` fixtures and the +//! oracle responses), and backbone execution on CUDA through the +//! shared qwen3_5 crate. The HTTP worker is not part of this crate. + +pub mod bundle; +pub mod contract; +pub mod executor; +pub mod heads; +pub mod processing; +pub mod questions; +pub mod render; +pub mod scoring; +pub mod state; +pub mod words; diff --git a/src/models/vela/native/src/processing.rs b/src/models/vela/native/src/processing.rs new file mode 100644 index 00000000..12370856 --- /dev/null +++ b/src/models/vela/native/src/processing.rs @@ -0,0 +1,274 @@ +//! The processor: one admitted request body → predict mapping, plan, +//! and rendered rows, mirroring the reference path +//! (`system_one_to_predict` → `_to_generic` → `_predict`'s row split → +//! `_prep`/`_render`, plus the long-target window rows). Everything the +//! frozen `native-plan-*` fixtures record. + +use std::collections::HashSet; +use std::path::Path; + +use anyhow::{Context, Result}; +use serde_json::{Map, Value}; + +use crate::contract::RequestError; +use crate::questions::{PredictQuestion, system_one_to_predict}; +use crate::render::{ + Encoder, GenericQuestion, HALU_LABEL, PreparedPart, RMAX, RenderContext, RenderedRow, + window_part, +}; +use crate::state::system_one_state; + +pub struct Processor { + encoder: Encoder, + max_length: usize, + router_label_sets: Vec>, + has_broad: bool, + halu_desc: String, +} + +pub struct Planned { + pub predict_request: Value, + pub plan: Value, + pub rows: Vec, + pub window_rows: Vec, +} + +impl Processor { + /// Load from an exported native bundle (tokenizer + calibration; + /// the manifest gate runs first). + pub fn load(bundle: &Path) -> Result { + let manifest = crate::bundle::Bundle::load(bundle)?; + let calibration: Value = serde_json::from_slice( + &std::fs::read(bundle.join("calibration.json")).context("calibration.json")?, + ) + .context("calibration.json must be JSON")?; + let broad = calibration.get("broad_head").filter(|b| !b.is_null()); + let router_label_sets: Vec> = broad + .and_then(|b| b.get("router_label_sets")) + .and_then(Value::as_array) + .filter(|sets| !sets.is_empty()) + .map(|sets| { + sets.iter() + .filter_map(Value::as_array) + .map(|labels| { + labels + .iter() + .filter_map(Value::as_str) + .map(str::to_owned) + .collect() + }) + .collect() + }) + .unwrap_or_else(|| { + vec![ + calibration["pii_types"].as_array().map(|types| { + types + .iter() + .filter_map(Value::as_str) + .map(str::to_owned) + .collect() + }), + calibration["halu_schema"]["labels"] + .as_object() + .map(|labels| labels.keys().cloned().collect()), + ] + .into_iter() + .flatten() + .collect() + }); + let has_broad = broad.is_some(); // the bundle gate pinned the broad tensors + let halu_desc = calibration["halu_schema"]["labels"][HALU_LABEL] + .as_str() + .unwrap_or("a claim not supported by the context") + .to_owned(); + Ok(Self { + encoder: Encoder::load(&bundle.join("tokenizer.json"))?, + max_length: manifest.max_length(), + router_label_sets, + has_broad, + halu_desc, + }) + } + + /// The full planning path for one admitted request body. + pub fn plan(&self, body: &Map) -> Result { + let state = system_one_state(&body["state"])?; + let questions = body["questions"] + .as_object() + .ok_or_else(crate::questions::empty_questions_error)?; + let mapped = system_one_to_predict(state, questions)?; + let predict_request = serde_json::to_value(&mapped.request).expect("serializable"); + let plan = serde_json::to_value(&mapped.plan).expect("serializable"); + let state_parts = mapped.request.parts.clone(); + + // `_to_generic`: predict questions → generic render questions + // (Noul is already folded; choice always carries [ABS]). + let generic: Vec = + mapped.request.questions.iter().map(to_generic).collect(); + // `_predict`'s row split: non-span questions plus the first span + // in one row, every further span its own row. + let spans: Vec<&GenericQuestion> = generic.iter().filter(|q| q.kind == "span").collect(); + let rest: Vec = generic + .iter() + .filter(|q| q.kind != "span") + .cloned() + .collect(); + let mut rows: Vec> = Vec::new(); + if !rest.is_empty() || !spans.is_empty() { + let mut first = rest.clone(); + if let Some(span) = spans.first() { + first.push((*span).clone()); + } + rows.push(first); + for span in spans.iter().skip(1) { + rows.push(vec![(*span).clone()]); + } + } + let context = RenderContext { + encoder: &self.encoder, + max_length: self.max_length, + router_label_sets: &self.router_label_sets, + has_broad: self.has_broad, + halu_desc: self.halu_desc.clone(), + }; + let span_roles: HashSet = + spans.iter().flat_map(|q| q.over.iter().cloned()).collect(); + let mut rendered = Vec::new(); + let mut window_rows = Vec::new(); + let mut parts_by_role = std::collections::HashMap::new(); + for part in self.prepared_parts(&state_parts, &span_roles) { + parts_by_role.insert(part.role.clone(), part); + } + for row in rows { + if row.is_empty() { + continue; + } + let (adjusted, dispatch) = context.prepare_questions(&row)?; + let parts: Vec = state_parts + .iter() + .map(|part| { + parts_by_role + .get(&part.r#type) + .cloned() + .expect("prepared part") + }) + .collect(); + let record = context.render(&parts, &adjusted, &dispatch)?; + // `_window_rows`: a span target beyond RMAX becomes one row + // per window (the full-render span block stays on the row). + let span = row.iter().find(|q| q.kind == "span"); + if let Some(span) = span { + let index = parts.iter().position(|p| p.role == span.over[0]); + if let Some(index) = index { + let part = &parts[index]; + let has_words = part.words.is_some(); + if part.ids.len() > RMAX && has_words { + let (_, first_tokens) = part.words.as_ref().expect("words"); + for (a, b) in crate::render::span_windows(first_tokens, part.ids.len()) { + let mut windowed_parts = parts.clone(); + windowed_parts[index] = window_part(part, a, b); + let (w_adjusted, w_dispatch) = context.prepare_questions(&row)?; + let mut record = + context.render(&windowed_parts, &w_adjusted, &w_dispatch)?; + // The runtime's window rows carry the head only. + record.window = true; + window_rows.push(record); + } + } + } + } + rendered.push(record); + } + Ok(Planned { + predict_request, + plan, + rows: rendered, + window_rows, + }) + } + + /// The prepared parts for one request: tokens everywhere, offsets + /// and word units on span-target parts. + fn prepared_parts( + &self, + state_parts: &[crate::state::Part], + span_roles: &HashSet, + ) -> Vec { + state_parts + .iter() + .map(|part| { + let (ids, offsets) = self.encoder.ids_offsets(&part.text); + if span_roles.contains(&part.r#type) { + let words = crate::words::word_first_tokens(&part.text, &offsets); + PreparedPart { + role: part.r#type.clone(), + ids, + offsets: Some(offsets), + words: Some(words), + } + } else { + PreparedPart { + role: part.r#type.clone(), + ids, + offsets: None, + words: None, + } + } + }) + .collect() + } +} + +/// `_to_generic` for the SystemOne path: options/labels carry over in +/// order (score levels have empty descriptions), the target part is the +/// resolved `over`, choice always carries `[ABS]` (the reference's +/// `abstain` default is true and the mapper never sets it), and span +/// keeps its threshold and head override. +fn to_generic(question: &PredictQuestion) -> GenericQuestion { + let over: Vec = match &question.target_part { + Value::String(role) => vec![role.clone()], + Value::Array(roles) => roles + .iter() + .filter_map(Value::as_str) + .map(str::to_owned) + .collect(), + _ => Vec::new(), + }; + let (names, descriptions) = match (&question.labels, &question.options) { + (Some(labels), _) => ( + labels.iter().map(|n| n.name.clone()).collect(), + labels.iter().map(|n| n.description.clone()).collect(), + ), + (None, Some(Value::Array(items))) => { + // Objects carry name/description; score levels are strings. + let mut names = Vec::new(); + let mut descriptions = Vec::new(); + for item in items { + match item { + Value::String(name) => { + names.push(name.clone()); + descriptions.push(String::new()); + } + Value::Object(named) => { + names.push(named["name"].as_str().unwrap_or_default().to_owned()); + descriptions + .push(named["description"].as_str().unwrap_or_default().to_owned()); + } + _ => {} + } + } + (names, descriptions) + } + _ => (Vec::new(), Vec::new()), + }; + GenericQuestion { + id: question.id.clone(), + kind: question.kind.clone(), + over, + text: question.text.clone(), + names, + descriptions, + threshold: question.threshold, + head_override: question.head.clone(), + } +} diff --git a/src/models/vela/native/src/questions.rs b/src/models/vela/native/src/questions.rs new file mode 100644 index 00000000..4c095032 --- /dev/null +++ b/src/models/vela/native/src/questions.rs @@ -0,0 +1,441 @@ +//! The SystemOne (state, questions) → predict-request mapping, mirroring +//! the pinned runtime's `system_one_to_predict` exactly: the validation +//! walk and its whole-request error triples, the Noul folding into a +//! two-option choice, and the per-question plan used to build responses. + +use serde_json::{Map, Value}; + +use crate::contract::RequestError; +use crate::state::{Part, StateParts, content_text}; + +/// The `questions` container rejection shared by the mapping and the +/// processor's body walk. +pub fn empty_questions_error() -> RequestError { + qerror( + &["questions"], + "questions must be a non-empty object of named questions", + ) +} + +pub const SYSTEM_ONE_TYPES: [&str; 5] = ["noul", "choice", "score", "span", "set"]; +pub const NOUL_DEFAULT_NO: &str = "No. The statement or question is not satisfied."; +pub const NOUL_DEFAULT_YES: &str = "Yes. The statement or question is satisfied."; + +/// One option / label of a predict question (name, description). +#[derive(Debug, Clone, serde::Serialize, PartialEq)] +pub struct Named { + pub name: String, + pub description: String, +} + +/// The predict-level question, exactly as it serializes into the frozen +/// fixtures: Noul questions arrive here already folded into a choice. +#[derive(Debug, Clone, serde::Serialize)] +pub struct PredictQuestion { + pub id: String, + #[serde(rename = "type")] + pub kind: String, + pub text: String, + pub target_part: Value, + #[serde(skip_serializing_if = "Option::is_none")] + pub labels: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub options: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub threshold: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub head: Option, +} + +#[derive(Debug, Clone, serde::Serialize)] +pub struct PredictRequest { + pub parts: Vec, + pub questions: Vec, +} + +/// The response-side plan entry for one original question. +#[derive(Debug, Clone, serde::Serialize)] +pub struct PlanEntry { + pub qid: String, + #[serde(rename = "type")] + pub kind: String, + pub criteria: Value, + pub range: Option<(usize, usize)>, + pub part: Value, +} + +#[derive(Debug)] +pub struct Mapped { + pub request: PredictRequest, + pub plan: Vec, +} + +fn qerror(loc: &[&str], msg: &str) -> RequestError { + RequestError::envelope(loc, msg, "value_error") +} + +fn nonblank(value: &Value) -> bool { + match value { + Value::String(text) => !text.trim().is_empty(), + _ => true, + } +} + +/// `over` (state key | part name | list) → (target part(s), field range). +fn resolve_over( + over: &Value, + parts: &StateParts, + qloc: &[&str], +) -> Result<(Value, Option<(usize, usize)>), RequestError> { + let roles: Vec<&str> = parts.parts.iter().map(|p| p.r#type.as_str()).collect(); + let items: Vec<&Value> = match over { + Value::Array(list) => list.iter().collect(), + single => vec![single], + }; + let mut out: Vec<&str> = Vec::new(); + let mut range = None; + for item in items { + let Some(name) = item.as_str() else { + return Err(qerror( + &[qloc[0], qloc[1], "over"], + "over must be a state key, a part name or a list of them", + )); + }; + if let Some(field) = parts.fields.get(name) { + if !over.is_array() { + range = Some((field.start, field.end)); + } + if !out.contains(&field.role) { + out.push(field.role); + } + } else if roles.contains(&name) { + if !out.contains(&name) { + out.push(name); + } + } else { + let role = crate::state::key_role(name); + if roles.contains(&role) && !out.contains(&role) { + out.push(role); + } else { + return Err(qerror( + &[qloc[0], qloc[1], "over"], + &format!( + "'{name}' is not a field of the state (fields: [{fields}], parts: [{roles}])", + fields = parts + .fields + .keys() + .map(|k| format!("'{k}'")) + .collect::>() + .join(", "), + roles = roles.join(", "), + ), + )); + } + } + } + if out.is_empty() { + return Err(qerror( + &[qloc[0], qloc[1], "over"], + "over must not be empty", + )); + } + // Sequence order, so the pooled range covers exactly these parts. + let ordered: Vec<&str> = roles + .iter() + .filter(|role| out.contains(role)) + .copied() + .collect(); + let target = match ordered.len() { + 1 => Value::String(ordered[0].into()), + _ => Value::Array( + ordered + .into_iter() + .map(|r| Value::String(r.into())) + .collect(), + ), + }; + Ok((target, range)) +} + +/// The full mapping. `state` and `questions` are the admitted request's +/// fields; validation runs in the pinned runtime's order so error +/// precedence matches. +pub fn system_one_to_predict( + state_parts: StateParts, + questions: &Map, +) -> Result { + if questions.is_empty() { + return Err(qerror( + &["questions"], + "questions must be a non-empty object of named questions", + )); + } + let roles: Vec<&str> = state_parts + .parts + .iter() + .map(|p| p.r#type.as_str()) + .collect(); + let mut predict = Vec::new(); + let mut plan = Vec::new(); + for (qid, question) in questions { + if qid.trim().is_empty() { + return Err(qerror( + &["questions"], + "question IDs must not be empty or whitespace", + )); + } + let Some(question) = question.as_object() else { + return Err(qerror( + &["questions", qid], + "a question must be an object with a type", + )); + }; + let loc = ["questions", qid]; + let kind = question.get("type").and_then(Value::as_str).unwrap_or(""); + if !SYSTEM_ONE_TYPES.contains(&kind) { + return Err(RequestError::envelope( + &[loc[0], loc[1], "type"], + "type must be one of ['noul', 'choice', 'score', 'span', 'set']", + "literal_error", + )); + } + let mut allowed: Vec<&str> = vec!["type", "instructions", "criteria", "over"]; + if kind == "span" || kind == "set" { + allowed.push("threshold"); + } + if kind == "span" { + allowed.push("head"); + } + let extra: Vec<&String> = question + .keys() + .filter(|key| !allowed.contains(&key.as_str())) + .collect(); + if let Some(first) = extra.first() { + return Err(RequestError::envelope( + &[loc[0], loc[1], first], + "Extra inputs are not permitted", + "extra_forbidden", + )); + } + let instructions = question.get("instructions").unwrap_or(&Value::Null); + if !nonblank(instructions) { + return Err(qerror( + &[loc[0], loc[1], "instructions"], + "instructions must not be empty or whitespace", + )); + } + let text = match instructions { + Value::Null => String::new(), + other => content_text(other), + }; + let criteria = question.get("criteria").unwrap_or(&Value::Null); + // Target part: the resolved `over`, the span default, or all parts. + let (target_part, range) = match question.get("over") { + Some(Value::Null) | None => { + if kind == "span" { + let role = if roles.contains(&"answer") { + "answer" + } else if roles.contains(&"user") { + "user" + } else { + roles[0] + }; + (Value::String(role.into()), None) + } else if roles.len() == 1 { + (Value::String(roles[0].into()), None) + } else { + ( + Value::Array(roles.iter().map(|r| Value::String((*r).into())).collect()), + None, + ) + } + } + Some(over) => resolve_over(over, &state_parts, &loc)?, + }; + let mut predict_question = PredictQuestion { + id: qid.clone(), + kind: kind.to_string(), + text: text.clone(), + target_part: target_part.clone(), + labels: None, + options: None, + threshold: None, + head: None, + }; + let mut plan_criteria = criteria.clone(); + match kind { + "noul" => { + if !criteria.is_null() + && (!criteria.is_object() + || criteria + .as_object() + .is_some_and(|map| map.keys().any(|k| k != "true" && k != "false"))) + { + return Err(qerror( + &[loc[0], loc[1], "criteria"], + "noul criteria is an object with optional true / false", + )); + } + let map = criteria.as_object().cloned().unwrap_or_default(); + for key in ["true", "false"] { + if let Some(value) = map.get(key) + && !nonblank(value) + { + return Err(qerror( + &[loc[0], loc[1], "criteria", key], + "Noul criterion must not be empty or whitespace", + )); + } + } + let no = match map.get("false") { + Some(Value::Null) | None => NOUL_DEFAULT_NO.to_string(), + Some(value) => content_text(value), + }; + let yes = match map.get("true") { + Some(Value::Null) | None => NOUL_DEFAULT_YES.to_string(), + Some(value) => content_text(value), + }; + predict_question.kind = "choice".into(); + predict_question.options = Some(serde_json::json!([ + {"name": "no", "description": no}, + {"name": "yes", "description": yes}, + ])); + } + "choice" | "set" | "span" => { + let (lo, hi) = if kind == "choice" { (2, 255) } else { (1, 255) }; + let Some(map) = criteria.as_object() else { + return Err(qerror( + &[loc[0], loc[1], "criteria"], + &format!("{kind} criteria must be an object with {lo} to {hi} entries"), + )); + }; + if !(lo..=hi).contains(&map.len()) { + return Err(qerror( + &[loc[0], loc[1], "criteria"], + &format!("{kind} criteria must be an object with {lo} to {hi} entries"), + )); + } + let mut named = Vec::new(); + for (name, value) in map { + if name.trim().is_empty() || !nonblank(value) { + return Err(qerror( + &[loc[0], loc[1], "criteria", name], + "option names and descriptions must not be empty or whitespace", + )); + } + let description = match value { + Value::Null => String::new(), + other => content_text(other), + }; + named.push(Named { + name: name.clone(), + description, + }); + } + if kind == "span" { + predict_question.labels = Some(named); + } else { + predict_question.options = Some(serde_json::to_value(&named).unwrap()); + } + } + _ => { + // score + let Some(levels) = criteria.as_array() else { + return Err(qerror( + &[loc[0], loc[1], "criteria"], + "score criteria must be a list of 2 to 10 levels", + )); + }; + if !(2..=10).contains(&levels.len()) { + return Err(qerror( + &[loc[0], loc[1], "criteria"], + "score criteria must be a list of 2 to 10 levels", + )); + } + let mut names = Vec::new(); + for (index, value) in levels.iter().enumerate() { + if value.is_null() || !nonblank(value) { + return Err(qerror( + &[loc[0], loc[1], "criteria", &index.to_string()], + "Score criterion must not be empty or whitespace", + )); + } + names.push(content_text(value)); + } + if names.iter().collect::>().len() != names.len() { + return Err(qerror( + &[loc[0], loc[1], "criteria"], + "score levels must be distinct", + )); + } + predict_question.options = Some(serde_json::to_value(&names).unwrap()); + } + } + if let Some(threshold) = question.get("threshold") + && !threshold.is_null() + { + match threshold.as_f64().filter(|_| threshold.is_number()) { + Some(value) if (0.0..=1.0).contains(&value) => { + predict_question.threshold = Some(value); + } + _ => { + return Err(qerror( + &[loc[0], loc[1], "threshold"], + "threshold must be a number in [0, 1]", + )); + } + } + } + if kind == "span" + && let Some(head) = question.get("head") + && !head.is_null() + { + match head.as_str() { + Some(value @ ("router" | "broad")) => { + predict_question.head = Some(value.to_string()); + } + _ => { + return Err(RequestError::envelope( + &[loc[0], loc[1], "head"], + "head must be one of ['router', 'broad']", + "literal_error", + )); + } + } + } + // Python's plan keeps `crit`; noul normalized null -> {} earlier. + if kind == "noul" && plan_criteria.is_null() { + plan_criteria = serde_json::json!({}); + } + predict.push(predict_question); + plan.push(PlanEntry { + qid: qid.clone(), + kind: kind.to_string(), + criteria: plan_criteria, + range, + part: target_part, + }); + } + // The Noul views of set labels must not overwrite a question. + for entry in &plan { + if entry.kind == "set" + && let Value::Object(map) = &entry.criteria + { + for label in map.keys() { + if questions.contains_key(&format!("{}.{}", entry.qid, label)) { + return Err(qerror( + &["questions", &format!("{}.{}", entry.qid, label)], + "collides with the Noul view of a set label", + )); + } + } + } + } + Ok(Mapped { + request: PredictRequest { + parts: state_parts.parts, + questions: predict, + }, + plan, + }) +} diff --git a/src/models/vela/native/src/render.rs b/src/models/vela/native/src/render.rs new file mode 100644 index 00000000..ed3d11fd --- /dev/null +++ b/src/models/vela/native/src/render.rs @@ -0,0 +1,794 @@ +//! Prompt rendering, mirroring the pinned runtime's `_prep` + `_render` +//! (the DEV2.0 decoder layout): the parts pass, per-question blocks +//! with option end and query positions, the span block in layout a/c +//! with word positions, the truncation budget, and the long-target +//! windows. Output structs serialize exactly like the frozen +//! `native-plan-*` fixtures. + +use std::cell::RefCell; +use std::collections::{HashMap, HashSet}; +use std::path::Path; + +use std::collections::BTreeMap; +use tokenizers::Tokenizer; + +use crate::contract::RequestError; +use crate::state::canonical; + +pub const SUFFIX_CHOICE_SCORE: &str = + "\n\nSelect the single option best supported by the context and instructions.\nDecision:"; +pub const SUFFIX_SET: &str = + "\n\nSelect every option supported by the context and instructions.\nDecision:"; +pub const SUFFIX_SPAN: &str = + "\n\nMark every span of the target text that carries one of the labels.\nSpans:"; +pub const ABS_KEY: &str = "[ABS]"; +pub const ABS_DESC: &str = "None of the shown options applies."; +pub const PARTS_HEAD: &str = "Context:\n"; +pub const SEG_CLOSE: &str = "\n\n"; +pub const TEXT_HEAD: &str = "\n\nText:\n"; +pub const HYBRID_RMAX: usize = 2048; +pub const WIN: usize = 1800; +pub const STRIDE: usize = 1536; +pub const RMAX: usize = 2048; + +pub const ROUTER_SPAN_IDS: [&str; 3] = ["pii", "halu", "toxic"]; +pub const HALU_LABEL: &str = "unsupported"; +pub const HALU_ALIASES: [&str; 9] = [ + "unsupported", + "unsupportedclaim", + "hallucinated", + "hallucination", + "hallucinatedspan", + "notsupported", + "unsupportedspan", + "fabricated", + "fabrication", +]; +pub const PII_TYPES: [&str; 17] = [ + "AGE", + "CREDIT_CARD", + "DATE_TIME", + "DOMAIN_NAME", + "EMAIL_ADDRESS", + "GPE", + "IBAN_CODE", + "IP_ADDRESS", + "NRP", + "ORGANIZATION", + "PERSON", + "PHONE_NUMBER", + "STREET_ADDRESS", + "TITLE", + "US_DRIVER_LICENSE", + "US_SSN", + "ZIP_CODE", +]; + +fn seg_open(role: &str) -> String { + format!("\n") +} + +fn option_block(key: &str, desc: &str, tag: &str) -> String { + format!( + "\n<{tag}>\n{}\n", + canonical(&serde_json::json!({"key": key, "description": desc})) + ) +} + +fn q_head(kind: &str, over: &[String], text: &str, span: bool) -> String { + let target = over.join(", "); + format!( + "\n\nTask type: {kind}\nTarget: {target}\nQuestion:\n{text}\n{}", + if span { "Labels:" } else { "Options:" } + ) +} + +// ------------------------------------------------------------------ tokenizer + +/// The pinned tokenizer, with the reference's memoized plain-`ids` view +/// and char-offset encodings (the tokenizers crate reports byte +/// offsets; the reference works in Unicode code points). +/// Ids plus char offsets, the encoder cache entry. +type Encoded = (Vec, Vec<(usize, usize)>); + +pub struct Encoder { + tokenizer: Tokenizer, + cache: RefCell>, +} + +impl Encoder { + pub fn load(path: &Path) -> anyhow::Result { + let tokenizer = Tokenizer::from_file(path).map_err(anyhow::Error::msg)?; + Ok(Self { + tokenizer, + cache: RefCell::new(HashMap::new()), + }) + } + + fn encode(&self, text: &str) -> (Vec, Vec<(usize, usize)>) { + if let Some(hit) = self.cache.borrow().get(text) { + return hit.clone(); + } + let encoding = match self.tokenizer.encode(text, false) { + Ok(encoding) => encoding, + Err(error) => { + // The pinned tokenizer should never fail on valid UTF-8 + // input; surface the error instead of panicking so a + // wrapping worker can map it. + unreachable!("the pinned tokenizer failed on valid input: {error}") + } + }; + let byte_offsets = encoding.get_offsets(); + // Byte → code-point offsets. + let char_starts: Vec = text + .char_indices() + .map(|(byte, _)| byte) + .chain(std::iter::once(text.len())) + .collect(); + let to_char = |byte: usize| char_starts.partition_point(|&start| start < byte); + let offsets: Vec<(usize, usize)> = byte_offsets + .iter() + .map(|(start, end)| (to_char(*start), to_char(*end))) + .collect(); + let entry = (encoding.get_ids().to_vec(), offsets); + self.cache + .borrow_mut() + .insert(text.to_owned(), entry.clone()); + entry + } + + /// Memoized token ids of a whole string (the reference's `_ids`). + pub fn ids(&self, text: &str) -> Vec { + self.encode(text).0 + } + + /// Ids and char offsets (the reference's `_ids_offs`). + pub fn ids_offsets(&self, text: &str) -> (Vec, Vec<(usize, usize)>) { + self.encode(text) + } +} + +// ------------------------------------------------------------- span dispatch + +/// `[\s_\-]+` removed, lower-cased — the reference's `_norm_label`. +pub fn norm_label(name: &str) -> String { + let mut out = String::with_capacity(name.len()); + for ch in name.chars() { + if !ch.is_whitespace() && ch != '_' && ch != '-' { + out.extend(ch.to_lowercase()); + } + } + out +} + +/// The user's label when the span question has exactly one label that +/// is a hallucination alias, else none. +pub fn halu_alias(names: &[String]) -> Option { + if names.len() == 1 && HALU_ALIASES.contains(&norm_label(&names[0]).as_str()) { + Some(names[0].clone()) + } else { + None + } +} + +pub struct SpanDispatch { + pub head: &'static str, + pub reason: &'static str, +} + +/// The reference's six-rule dispatch, in order: `"head"` override, +/// trained ids, an exact router label set, a non-empty subset of the 17 +/// PII types, one hallucination alias, else broad (router when the +/// broad head is not loaded). +pub fn span_head_for( + id: &str, + labels: &HashSet, + head_override: Option<&str>, + router_label_sets: &[HashSet], + broad_available: bool, +) -> Result { + if let Some(head) = head_override { + if !matches!(head, "router" | "broad") { + return Err(format!( + "head must be one of ['router', 'broad'], got '{head:?}'" + )); + } + if head == "broad" && !broad_available { + return Err("the broad span head is not loaded (needs broad_head.safetensors)".into()); + } + return Ok(SpanDispatch { + head: if head == "broad" { "broad" } else { "router" }, + reason: "override", + }); + } + if ROUTER_SPAN_IDS.contains(&id) { + return Ok(SpanDispatch { + head: "router", + reason: "id", + }); + } + if router_label_sets.iter().any(|set| set == labels) { + return Ok(SpanDispatch { + head: "router", + reason: "label_set", + }); + } + let pii: HashSet<&str> = PII_TYPES.into_iter().collect(); + if !labels.is_empty() && labels.iter().all(|label| pii.contains(label.as_str())) { + return Ok(SpanDispatch { + head: "router", + reason: "pii_subset", + }); + } + if halu_alias(&labels.iter().cloned().collect::>()).is_some() { + return Ok(SpanDispatch { + head: "router", + reason: "halu_alias", + }); + } + if !broad_available { + return Ok(SpanDispatch { + head: "router", + reason: "broad_unavailable", + }); + } + Ok(SpanDispatch { + head: "broad", + reason: "default", + }) +} + +// ------------------------------------------------------------------ rendering + +/// One generic-contract question at render time (Noul is already folded +/// into a two-option choice; choice always carries `[ABS]` on the +/// SystemOne path — the reference's `_to_generic` defaults abstain to +/// true and the mapper never sets it). +#[derive(Clone)] +pub struct GenericQuestion { + pub id: String, + pub kind: String, + pub over: Vec, + pub text: String, + pub names: Vec, + pub descriptions: Vec, + pub threshold: Option, + pub head_override: Option, +} + +/// One prepared part: tokens, and for span-target parts also char +/// offsets and word units. +/// Word char offsets and each word's first token index. +pub type WordUnits = (Vec<(usize, usize)>, Vec); + +#[derive(Clone)] +pub struct PreparedPart { + pub role: String, + pub ids: Vec, + pub offsets: Option>, + pub words: Option, +} + +#[derive(Debug, Clone, serde::Serialize, PartialEq)] +pub struct TruncRecord { + pub shrink_cut: usize, + pub protected_cut: usize, +} + +#[derive(Debug, Clone, serde::Serialize, PartialEq)] +#[serde(tag = "kind")] +pub enum Block { + #[serde(rename = "q")] + Question { + ids: Vec, + ends: Vec, + query: usize, + qid: String, + qtype: String, + names: Vec, + has_abs: bool, + }, + #[serde(rename = "span")] + Span { + ends: Vec, + label_ranges: Vec<(usize, usize)>, + words_in_block: bool, + ids: Vec, + query: usize, + w_pos: Vec, + }, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct RenderedRow { + pub p_ids: Vec, + pub blocks: Vec, + pub trunc: TruncRecord, + pub seg_lens: BTreeMap, + pub layout: Option, + pub names: Option>, + pub w_offs: Option>, + pub w_pos: Option>, + pub span_over: Option, + pub n_tokens: usize, + pub head: String, + pub head_reason: Option, + /// The runtime's window rows carry the head only — no head_reason + /// key at all (predict_raw assigns head alone). + pub window: bool, +} + +/// Serialize exactly like the frozen fixtures: main rows always carry +/// `head_reason` (null when the row has no span); window rows omit it. +impl serde::Serialize for RenderedRow { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serialize_row(self, serializer) + } +} + +fn serialize_row(row: &RenderedRow, serializer: S) -> Result +where + S: serde::Serializer, +{ + use serde::ser::SerializeMap; + let mut map = serializer.serialize_map(None)?; + macro_rules! put { + ($key:literal, $value:expr) => { + map.serialize_entry($key, &$value)? + }; + } + put!("p_ids", row.p_ids); + put!("blocks", row.blocks); + put!("trunc", row.trunc); + put!("seg_lens", row.seg_lens); + put!("layout", row.layout); + if let Some(names) = &row.names { + put!("names", names); + } + if let Some(w_offs) = &row.w_offs { + put!("w_offs", w_offs); + } + if let Some(w_pos) = &row.w_pos { + put!("w_pos", w_pos); + } + if let Some(span_over) = &row.span_over { + put!("span_over", span_over); + } + put!("n_tokens", row.n_tokens); + put!("head", row.head); + if !row.window { + put!("head_reason", row.head_reason); + } + map.end() +} + +/// Everything the renderer needs beyond tokens. +pub struct RenderContext<'a> { + pub encoder: &'a Encoder, + pub max_length: usize, + pub router_label_sets: &'a [HashSet], + pub has_broad: bool, + pub halu_desc: String, +} + +/// The whole-request 413: questions plus schema alone do not fit. +pub fn too_long(max_length: usize) -> RequestError { + RequestError { + status: 413, + body: serde_json::json!({"detail": [{"loc": ["body", "questions"], + "msg": format!("the questions alone do not fit the {max_length}-token window"), + "type": "too_long"}]}), + } +} + +/// `schema_lib_r4.assemble`, eval path: the truncation order on the +/// decoder's token costs. Returns kept segment lengths and the record. +fn assemble_budget( + q_lens: &[usize], + el_lens: &[usize], + segs: &mut [(String, usize)], + max_len: usize, + protected: &HashSet, + shrink: Option<&str>, +) -> Result<(BTreeMap, TruncRecord), ()> { + let fixed = 2 + 1 + segs.len(); + fn over( + q_lens: &[usize], + el_lens: &[usize], + segs: &[(String, usize)], + fixed: usize, + max_len: usize, + ) -> i64 { + fixed as i64 + + q_lens.iter().sum::() as i64 + + el_lens.iter().sum::() as i64 + + segs.iter().map(|(_, n)| *n).sum::() as i64 + - max_len as i64 + } + let mut trunc = TruncRecord { + shrink_cut: 0, + protected_cut: 0, + }; + let shrink_index = shrink.and_then(|role| segs.iter().position(|(r, _)| r == role)); + for floor in [64, 1] { + if over(q_lens, el_lens, segs, fixed, max_len) > 0 + && let Some(index) = shrink_index + { + let full = segs[index].1; + let excess = over(q_lens, el_lens, segs, fixed, max_len).max(0) as usize; + let keep = full.min(floor.max(full.saturating_sub(excess))); + trunc.shrink_cut += full - keep; + segs[index].1 = keep; + } + } + if over(q_lens, el_lens, segs, fixed, max_len) > 0 { + for index in 0..segs.len() { + if over(q_lens, el_lens, segs, fixed, max_len) > 0 && protected.contains(&segs[index].0) + { + let full = segs[index].1; + let excess = over(q_lens, el_lens, segs, fixed, max_len).max(0) as usize; + let keep = 1.max(full.saturating_sub(excess)); + trunc.protected_cut += full - keep; + segs[index].1 = keep; + } + } + } + if over(q_lens, el_lens, segs, fixed, max_len) > 0 { + return Err(()); + } + let seg_lens: BTreeMap = + segs.iter().map(|(role, n)| (role.clone(), *n)).collect(); + Ok((seg_lens, trunc)) +} + +/// Token windows [a, b) over a part of n tokens, starting and cut at +/// word starts (`schema_lib_d3_20261001.windows`). +pub fn span_windows(first_tokens: &[usize], n: usize) -> Vec<(usize, usize)> { + let mut starts: Vec = first_tokens.to_vec(); + starts.sort_unstable(); + starts.dedup(); + if starts.is_empty() { + return vec![(0, n)]; + } + let mut out = Vec::new(); + let mut a = starts[0]; + loop { + let limit = a + WIN; + let next: Vec = starts + .iter() + .copied() + .filter(|&s| s > a && s < limit) + .collect(); + let b = if limit >= n { + n + } else if !next.is_empty() { + *starts + .iter() + .filter(|&&s| s < limit && s > a) + .max() + .unwrap() + } else { + n.min(limit) + }; + out.push((a, b)); + if b >= n { + break; + } + let candidate: Vec = starts + .iter() + .copied() + .filter(|&s| s >= a + STRIDE && s < b) + .collect(); + a = candidate.first().copied().unwrap_or(b); + } + out +} + +/// A windowed copy of a prepared part: tokens [a, b) and the words that +/// start inside them. +pub fn window_part(part: &PreparedPart, a: usize, b: usize) -> PreparedPart { + let (word_offsets, first_tokens) = part.words.as_ref().expect("windowed parts carry words"); + let selected: Vec = first_tokens + .iter() + .enumerate() + .filter(|&(_, &t)| t >= a && t < b) + .map(|(index, _)| index) + .collect(); + let kept_offsets: Vec<(usize, usize)> = selected.iter().map(|&i| word_offsets[i]).collect(); + PreparedPart { + role: part.role.clone(), + ids: part.ids[a..b].to_vec(), + offsets: part.offsets.as_ref().map(|offs| offs[a..b].to_vec()), + words: Some(( + kept_offsets, + selected.iter().map(|&i| first_tokens[i] - a).collect(), + )), + } +} + +impl RenderContext<'_> { + /// `_prep`'s question side: the span-head dispatch and the + /// hallucination-alias label replacement (parts arrive prepared — + /// word units need the original text and are the caller's job). + pub fn prepare_questions( + &self, + questions: &[GenericQuestion], + ) -> Result<(Vec, SpanDispatch), RequestError> { + let span = questions.iter().find(|q| q.kind == "span"); + let dispatch = match span { + Some(question) => { + let labels: HashSet = question.names.iter().cloned().collect(); + span_head_for( + &question.id, + &labels, + question.head_override.as_deref(), + self.router_label_sets, + self.has_broad, + ) + .map_err(|message| crate::contract::runtime_rejection(&message))? + } + None => SpanDispatch { + head: "router", + reason: "", + }, + }; + // The trained `unsupported` label renders in place of a single + // hallucination-alias label (the answer gets the user's name). + let mut adjusted = questions.to_vec(); + if let Some(question) = adjusted.iter_mut().find(|q| q.kind == "span") + && dispatch.head == "router" + && let Some(alias) = halu_alias(&question.names) + && alias != HALU_LABEL + { + question.names = vec![HALU_LABEL.to_string()]; + question.descriptions = vec![self.halu_desc.clone()]; + } + Ok((adjusted, dispatch)) + } + + /// `_render` on one prepared row: the parts pass and every block, + /// with the budget applied first. + pub fn render( + &self, + parts: &[PreparedPart], + questions: &[GenericQuestion], + dispatch: &SpanDispatch, + ) -> Result { + let ids_of = |text: &str| self.encoder.ids(text); + let roles: Vec = parts.iter().map(|p| p.role.clone()).collect(); + // Question blocks and their schema costs. + struct RenderedQ { + qid: String, + qtype: String, + names: Vec, + otoks: Vec>, + has_abs: bool, + head: Vec, + suffix: Vec, + abstain_tokens: Vec, + len: usize, + } + let mut rendered_qs: Vec = Vec::new(); + let mut el: Vec<(String, Vec)> = Vec::new(); // span labels + let mut span_question: Option<&GenericQuestion> = None; + for question in questions { + if question.kind == "span" { + span_question = Some(question); + for (name, desc) in question.names.iter().zip(&question.descriptions) { + let tokens = ids_of(&option_block(name, desc, "label")); + el.push((name.clone(), tokens.clone())); + } + continue; + } + let suffix = match question.kind.as_str() { + "set" => SUFFIX_SET, + _ => SUFFIX_CHOICE_SCORE, // choice and score share the wording + }; + let has_abs = question.kind == "choice"; // SystemOne: abstain defaults true + let head = ids_of(&q_head( + &question.kind, + &question.over, + &question.text, + false, + )); + let suffix = ids_of(suffix); + let abstain_tokens = if has_abs { + ids_of(&option_block(ABS_KEY, ABS_DESC, "option")) + } else { + Vec::new() + }; + let mut otoks = Vec::with_capacity(question.names.len()); + let mut len = + 1 + head.len() + suffix.len() + abstain_tokens.len() + usize::from(has_abs); + for (name, desc) in question.names.iter().zip(&question.descriptions) { + let tokens = ids_of(&option_block(name, desc, "option")); + len += 1 + tokens.len(); + otoks.push(tokens); + } + rendered_qs.push(RenderedQ { + qid: question.id.clone(), + qtype: question.kind.clone(), + names: question.names.clone(), + otoks, + has_abs, + head, + suffix, + abstain_tokens, + len, + }); + } + let span_over: Option = span_question.map(|question| question.over[0].clone()); + let mut overs: HashSet = HashSet::new(); + for question in questions { + overs.extend(question.over.iter().cloned()); + } + if let Some(over) = &span_over { + overs.insert(over.clone()); + } + let part_of = |role: &str| parts.iter().find(|p| p.role == role); + let shrink = roles.iter().find(|role| !overs.contains(*role)).cloned(); + let layout = match &span_over { + Some(over) if part_of(over).is_some_and(|p| p.ids.len() <= HYBRID_RMAX) => "c", + Some(_) => "a", + None => "", + }; + // The schema overhead the budget does not see. + let mut budget = ids_of(PARTS_HEAD).len(); + for role in &roles { + budget += ids_of(&seg_open(role)).len() + ids_of(SEG_CLOSE).len(); + } + if let Some(over) = &span_over { + let question = span_question.expect("span question"); + let head = ids_of(&q_head("span", &question.over, &question.text, true)); + let suffix = ids_of(SUFFIX_SPAN); + budget += head.len() + suffix.len(); + if layout == "c" { + let span_len = part_of(over).expect("span part").ids.len(); + budget += ids_of(TEXT_HEAD).len() + + ids_of(&seg_open(over)).len() + + ids_of(SEG_CLOSE).len(); + let headroom = self.max_length.saturating_sub(budget); + budget += span_len.min(headroom / 2); + } + } + let q_lens: Vec = rendered_qs.iter().map(|q| q.len).collect(); + let el_lens: Vec = el.iter().map(|(_, tokens)| 1 + tokens.len()).collect(); + let mut segs: Vec<(String, usize)> = parts + .iter() + .map(|p| (p.role.clone(), p.ids.len())) + .collect(); + let (seg_lens, trunc) = assemble_budget( + &q_lens, + &el_lens, + &mut segs, + 64.max(self.max_length - budget), + &overs, + shrink.as_deref(), + ) + .map_err(|()| too_long(self.max_length))?; + // The parts pass, in part order (not the sorted map). + let mut p_ids = ids_of(PARTS_HEAD); + let mut start: HashMap = HashMap::new(); + for part in parts { + let keep = seg_lens[&part.role]; + let open = ids_of(&seg_open(&part.role)); + start.insert(part.role.clone(), p_ids.len() + open.len()); + p_ids.extend(open); + p_ids.extend(part.ids.iter().take(keep)); + p_ids.extend(ids_of(SEG_CLOSE)); + } + // Question blocks. + let mut blocks: Vec = Vec::new(); + for q in &rendered_qs { + let mut ids = q.head.clone(); + let mut ends = Vec::with_capacity(q.names.len() + usize::from(q.has_abs)); + for tokens in &q.otoks { + ids.extend(tokens); + ends.push(ids.len() - 1); + } + if q.has_abs { + ids.extend(q.abstain_tokens.clone()); + ends.push(ids.len() - 1); + } + ids.extend(q.suffix.clone()); + let query = ids.len() - 1; + blocks.push(Block::Question { + ids, + ends, + query, + qid: q.qid.clone(), + qtype: q.qtype.clone(), + names: q.names.clone(), + has_abs: q.has_abs, + }); + } + // The span block. + let mut row_names = None; + let mut row_w_offs = None; + let mut row_w_pos = None; + if let Some(over) = &span_over + && !el.is_empty() + { + let question = span_question.expect("span question"); + let mut ids = ids_of(&q_head("span", &question.over, &question.text, true)); + let mut ends = Vec::new(); + let mut starts = Vec::new(); + for (_, tokens) in &el { + starts.push(ids.len()); + ids.extend(tokens); + ends.push(ids.len() - 1); + } + let part = part_of(over).expect("span part"); + let n = seg_lens[over]; + let (word_offsets, first_tokens) = part.words.as_ref().expect("span words"); + // Word indices whose first token survives the kept prefix; + // indices, not token values — two words can share one. + let keep: Vec = first_tokens + .iter() + .enumerate() + .filter(|&(_, &t)| t < n) + .map(|(index, _)| index) + .collect(); + let words_in_block = layout == "c"; + if words_in_block { + ids.extend(ids_of(TEXT_HEAD)); + ids.extend(ids_of(&seg_open(over))); + let c0 = ids.len(); + ids.extend(part.ids.iter().take(n)); + ids.extend(ids_of(SEG_CLOSE)); + row_w_pos = Some(keep.iter().map(|&i| c0 + first_tokens[i]).collect()); + } else { + let base = start[over]; + row_w_pos = Some(keep.iter().map(|&i| base + first_tokens[i]).collect()); + } + ids.extend(ids_of(SUFFIX_SPAN)); + let query = ids.len() - 1; + row_names = Some(el.iter().map(|(name, _)| name.clone()).collect()); + row_w_offs = Some(keep.iter().map(|&i| word_offsets[i]).collect()); + let label_ranges: Vec<(usize, usize)> = + starts.into_iter().zip(ends.iter().copied()).collect(); + blocks.push(Block::Span { + ends, + label_ranges, + words_in_block, + ids, + query, + w_pos: row_w_pos.clone().expect("w_pos"), + }); + } + let n_tokens = p_ids.len() + blocks.iter().map(block_len).sum::(); + Ok(RenderedRow { + p_ids, + blocks, + trunc, + seg_lens, + layout: if span_over.is_some() { + Some(layout.to_string()) + } else { + None + }, + names: row_names, + w_offs: row_w_offs, + w_pos: row_w_pos, + span_over, + n_tokens, + head: dispatch.head.to_string(), + head_reason: if dispatch.reason.is_empty() { + None + } else { + Some(dispatch.reason.to_string()) + }, + window: false, + }) + } +} + +fn block_len(block: &Block) -> usize { + match block { + Block::Question { ids, .. } => ids.len(), + Block::Span { ids, .. } => ids.len(), + } +} diff --git a/src/models/vela/native/src/scoring.rs b/src/models/vela/native/src/scoring.rs new file mode 100644 index 00000000..c8353bd3 --- /dev/null +++ b/src/models/vela/native/src/scoring.rs @@ -0,0 +1,840 @@ +//! The scoring and decode mirror of the pinned runtime's calibration +//! path (`calibrated`, `span_threshold`, the PII length rule and sparse +//! gate, `span_decode_r4.decode_v2`, `_trim`), turning head logits into +//! predict-level answers. Parity target: the `head-input-*.json` +//! fixtures' `predict.answers`. + +use serde_json::Value; + +/// The package's `calibration.json` as typed lookup. +pub struct Calibration { + pub data: Value, +} + +pub fn softmax(x: &[f64]) -> Vec { + let max = x.iter().copied().fold(f64::NEG_INFINITY, f64::max); + let e: Vec = x.iter().map(|v| (v - max).exp()).collect(); + let sum: f64 = e.iter().sum(); + e.iter().map(|v| v / sum).collect() +} + +pub fn sigmoid(x: f64) -> f64 { + 1.0 / (1.0 + (-x).exp()) +} + +impl Calibration { + pub fn load(bundle: &std::path::Path) -> anyhow::Result { + Ok(Self { + data: serde_json::from_slice(&std::fs::read(bundle.join("calibration.json"))?)?, + }) + } + + fn temperature(&self, qtype: &str) -> f64 { + self.data["temperature"] + .get(qtype) + .and_then(Value::as_f64) + .unwrap_or(1.0) + } + + pub fn span_temperature(&self, head: &str) -> f64 { + if head == "broad" { + self.data["broad_head"]["temperature"] + .as_f64() + .unwrap_or(1.0) + } else { + self.temperature("span") + } + } + + fn is_pii(&self, qid: &str, names: &[String]) -> bool { + if qid == "pii" { + return true; + } + let types: Vec<&str> = self.data["pii_types"] + .as_array() + .map(|a| a.iter().filter_map(Value::as_str).collect()) + .unwrap_or_default(); + !names.is_empty() && names.iter().all(|n| types.contains(&n.as_str())) + } + + /// log-space piecewise-linear between anchors, constant outside. + fn pii_length_threshold(&self, n_tokens: usize) -> f64 { + let anchors = self.data["pii_length_rule"]["anchors"] + .as_array() + .cloned() + .unwrap_or_default(); + let mut points: Vec<(f64, f64)> = anchors + .iter() + .map(|a| { + ( + a["n_tokens"].as_f64().unwrap_or(1.0).ln(), + a["threshold"].as_f64().unwrap_or(1.0).ln(), + ) + }) + .collect(); + points.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap()); + let x = (n_tokens.max(1) as f64).ln(); + let (xs, ys): (Vec, Vec) = points.into_iter().unzip(); + interp(&xs, &ys, x).exp() + } + + /// The span threshold and its rule string, before the sparse gate. + pub fn span_threshold( + &self, + qid: &str, + names: &[String], + n_tokens: usize, + head: &str, + ) -> (f64, String) { + if head == "broad" { + return ( + self.data["broad_head"]["threshold"].as_f64().unwrap_or(0.5), + "broad".into(), + ); + } + let th = &self.data["thresholds"]; + if self.is_pii(qid, names) { + if self + .data + .get("pii_length_rule") + .is_some_and(|v| !v.is_null()) + { + return (self.pii_length_threshold(n_tokens), "pii:length".into()); + } + for bucket in self.data["pii_buckets"] + .as_array() + .cloned() + .unwrap_or_default() + { + if n_tokens as f64 <= bucket["max_tokens"].as_f64().unwrap_or(f64::INFINITY) { + return ( + bucket["threshold"].as_f64().unwrap_or(0.5), + format!("pii:{}", bucket["name"].as_str().unwrap_or("")), + ); + } + } + } + for fam in ["halu", "toxic"] { + if qid == fam { + return ( + th[&format!("span:{fam}")].as_f64().unwrap_or(0.5), + fam.into(), + ); + } + } + let specific = format!("span:{qid}"); + if th.get(&specific).is_some() { + (th[&specific].as_f64().unwrap_or(0.5), specific) + } else { + (th["span:*"].as_f64().unwrap_or(0.5), "span:*".into()) + } + } + + pub fn set_threshold(&self, qid: &str) -> f64 { + let th = &self.data["thresholds"]; + th.get(format!("set:{qid}")) + .or_else(|| th.get("set:*")) + .and_then(Value::as_f64) + .unwrap_or(0.5) + } + + /// The PII sparse gate (disabled when K < 0; second decode pass + /// over the same probabilities). + pub fn sparse_gate( + &self, + probs: &[Vec], + offs: &[(usize, usize)], + names: &[String], + text: &str, + thr: f64, + rule: &str, + ) -> (f64, String) { + let gate = &self.data["pii_sparse_gate"]; + let k = gate["K"].as_i64().unwrap_or(-1); + if k < 0 || thr >= gate["t_sparse"].as_f64().unwrap_or(f64::INFINITY) || offs.is_empty() { + return (thr, rule.into()); + } + let probe = gate["probe_threshold"].as_f64().unwrap_or(0.5); + let decoded = decode_spans(probs, offs, names, text, probe); + if decoded.len() as i64 <= k { + ( + gate["t_sparse"].as_f64().unwrap_or(thr), + format!("{rule}+sparse"), + ) + } else { + (thr, rule.into()) + } + } +} + +/// numpy.interp: linear between points, constant outside. +fn interp(xs: &[f64], ys: &[f64], x: f64) -> f64 { + if xs.is_empty() { + return f64::NAN; + } + if x <= xs[0] { + return ys[0]; + } + if x >= xs[xs.len() - 1] { + return ys[ys.len() - 1]; + } + for i in 0..xs.len() - 1 { + if x >= xs[i] && x <= xs[i + 1] { + let t = (x - xs[i]) / (xs[i + 1] - xs[i]); + return ys[i] + t * (ys[i + 1] - ys[i]); + } + } + ys[ys.len() - 1] +} + +// ------------------------------------------------------------ span decoding + +const EDGE: &str = "[](){}<>「」『』【】《》〈》\"'“”‘’«»‹›"; +const URL_TRAIL: &str = ".,;:!?)"; + +fn is_cjk(ch: char) -> bool { + // The reference's CJK class; same ranges as the word splitter. + super::words::is_cjk_char(ch) +} + +fn trim_edges(text: &[char], s: &mut usize, e: &mut usize) { + while *s < *e && (text[*s].is_whitespace() || EDGE.contains(text[*s])) { + *s += 1; + } + while *e > *s && (text[*e - 1].is_whitespace() || EDGE.contains(text[*e - 1])) { + *e -= 1; + } +} + +fn trim_url(text: &[char], s: &mut usize, e: &mut usize) { + // (?:https?://|www\.) at s, case-insensitive. + // The reference regex matches against the full text from s; the + // lookahead is not bounded by the span end. + let head: String = text[*s..(*s + 8).min(text.len())].iter().collect(); + let is_url = { + let lower = head.to_lowercase(); + lower.starts_with("http://") || lower.starts_with("https://") || lower.starts_with("www.") + }; + if !is_url { + return; + } + if let Some(pos) = text[*s..*e].iter().position(|&c| is_cjk(c)) { + *e = *s + pos; + } + while *e > *s + && (URL_TRAIL.contains(text[*e - 1]) + || EDGE.contains(text[*e - 1]) + || text[*e - 1].is_whitespace()) + { + *e -= 1; + } +} + +fn trim(text: &[char], s: usize, e: usize) -> (usize, usize) { + let (mut s, mut e) = (s, e); + trim_edges(text, &mut s, &mut e); + trim_url(text, &mut s, &mut e); + (s, e) +} + +/// `span_decode_r4.decode_v2` with the scorer's arguments +/// (extend=false, trim, no bridge, unit vote + fill on). +#[allow(clippy::too_many_arguments)] +pub fn decode_spans( + probs: &[Vec], + offs: &[(usize, usize)], + names: &[String], + text: &str, + thr: f64, +) -> Vec { + let chars: Vec = text.chars().collect(); + let t_count = offs.len(); + if t_count == 0 { + return Vec::new(); + } + let valid: Vec = offs.iter().map(|(a, b)| b > a).collect(); + let mut lab = vec![-1i64; t_count]; + for t in 0..t_count { + let (k, pk) = argmax(&probs[t]); + if valid[t] && pk as f64 > thr { + lab[t] = k as i64; + } + } + // Unit ids over the text (the reference's `_UNIT`), then per-word + // first-nonspace lookup and alnum test. + let units = super::words::unit_ids(&chars); + let mut tu = vec![-1i64; t_count]; + let mut alnum = vec![false; t_count]; + for t in 0..t_count { + if !valid[t] { + continue; + } + let (a, b) = offs[t]; + let mut f = a; + while f < b && chars[f].is_whitespace() { + f += 1; + } + tu[t] = if f < b && f < chars.len() { + units[f] + } else { + -1 + }; + alnum[t] = chars[a..b].iter().any(|&c| is_alnum(c)); + } + // Unit vote + fill, per run of the same unit. + let mut t = 0; + while t < t_count { + if tu[t] < 0 { + t += 1; + continue; + } + let mut t2 = t; + while t2 + 1 < t_count && (tu[t2 + 1] == tu[t] || !valid[t2 + 1]) { + t2 += 1; + } + let idx: Vec = (t..=t2).filter(|&i| valid[i]).collect(); + let labeled: Vec = idx.iter().copied().filter(|&i| lab[i] >= 0).collect(); + if !labeled.is_empty() { + let distinct: Vec = { + let mut d: Vec = labeled.iter().map(|&i| lab[i]).collect(); + d.sort_unstable(); + d.dedup(); + d + }; + if distinct.len() > 1 { + // The reference picks the first strict maximum in + // labeled-token order; accumulate in a Vec keyed by + // first occurrence, not a HashMap (whose iteration + // order would make ties nondeterministic). + let mut order: Vec = Vec::new(); + let mut scores: Vec = Vec::new(); + for &i in &labeled { + let label = lab[i]; + match order.iter().position(|&l| l == label) { + Some(index) => scores[index] += probs[i][label as usize] as f64, + None => { + order.push(label); + scores.push(probs[i][label as usize] as f64); + } + } + } + let mut winner_index = 0usize; + for (index, score) in scores.iter().enumerate().skip(1) { + if *score > scores[winner_index] { + winner_index = index; + } + } + let winner = order[winner_index]; + for &i in &labeled { + lab[i] = winner; + } + } + let labels: Vec = { + let mut d: Vec = labeled.iter().map(|&i| lab[i]).collect(); + d.sort_unstable(); + d.dedup(); + d + }; + for c in labels { + let pos: Vec = labeled.iter().copied().filter(|&i| lab[i] == c).collect(); + for &i in &idx { + if pos[0] < i && i < pos[pos.len() - 1] && lab[i] < 0 { + lab[i] = c; + } + } + } + } + t = t2 + 1; + } + // Merge consecutive same-label words into spans, trim, mean prob. + let mut spans: Vec<(i64, usize, usize, Vec)> = Vec::new(); + for t in 0..t_count { + if !valid[t] { + continue; + } + let c = lab[t]; + if let Some(last) = spans.last_mut() + && c == last.0 + { + last.2 = offs[t].1; + last.3.push(t); + continue; + } + if c >= 0 { + spans.push((c, offs[t].0, offs[t].1, vec![t])); + } + } + let mut out = Vec::new(); + for (c, s, e, ts) in spans { + let (s, e) = trim(&chars, s, e); + if e <= s { + continue; + } + let mean = ts.iter().map(|&t| probs[t][c as usize] as f64).sum::() / ts.len() as f64; + out.push(serde_json::json!({ + "start": s, "end": e, + "label": names[c as usize], + "probability": mean, + })); + } + out +} + +fn argmax(row: &[f32]) -> (usize, f32) { + let mut best = 0usize; + let mut value = f32::NEG_INFINITY; + for (index, item) in row.iter().enumerate() { + if *item > value { + value = *item; + best = index; + } + } + (best, value) +} + +fn is_alnum(c: char) -> bool { + // The reference's `_ALNUM` = [^\W_]: word characters minus the + // underscore — Rust's alphanumeric already excludes it. + c.is_alphanumeric() +} + +// -------------------------------------------------------- answer assembly + +/// One scored question block: the reference's `raw["qs"]` entry. +pub struct QBlock { + pub qid: String, + pub qtype: String, + pub names: Vec, + pub has_abs: bool, + /// Logits over `names` (plus `[ABS]` last when `has_abs`), + /// **after** run_batch's `set_bias` addition for set blocks. + pub logits: Vec, +} + +/// One scored span: the reference's `raw["span"]` (post aggregation). +pub struct SpanOut { + pub qid: String, + pub over: String, + pub names: Vec, + pub offs: Vec<(usize, usize)>, + pub logits: Vec>, + pub head: String, + pub head_reason: Option, + pub label_alias: Option, + /// The span part's kept token count (the PII length-rule input). + pub n_tokens: usize, +} + +/// `calibrated` for one row: logits -> predict-level answers. +pub fn row_answers( + cal: &Calibration, + blocks: &[QBlock], + span: Option<&SpanOut>, + thresholds: &std::collections::HashMap, + span_text: Option<&str>, +) -> Vec { + let mut answers = Vec::new(); + for block in blocks { + let logits: Vec = block.logits.iter().map(|v| *v as f64).collect(); + let n = block.names.len(); + let temperature = cal.temperature(&block.qtype); + if block.qtype == "set" { + let p: Vec = logits[..n] + .iter() + .map(|v| sigmoid(v / temperature)) + .collect(); + let thr = thresholds + .get(&block.qid) + .copied() + .unwrap_or_else(|| cal.set_threshold(&block.qid)); + let probs: serde_json::Map = block + .names + .iter() + .zip(&p) + .map(|(name, value)| (name.clone(), json_f64(*value))) + .collect(); + let selected: Vec<&String> = block + .names + .iter() + .zip(&p) + .filter(|(_, value)| **value > thr) + .map(|(name, _)| name) + .collect(); + answers.push(serde_json::json!({ + "id": block.qid, "type": "set", + "answer": selected, + "probabilities": probs, + "threshold": thr, + })); + continue; + } + let p = softmax( + &logits[..n] + .iter() + .map(|v| v / temperature) + .collect::>(), + ); + let answer = block.names[argmax_usize(&logits[..n])].clone(); + let probs: serde_json::Map = block + .names + .iter() + .zip(&p) + .map(|(name, value)| (name.clone(), json_f64(*value))) + .collect(); + let mut entry = serde_json::json!({ + "id": block.qid, "type": block.qtype, + "answer": answer, + "probabilities": probs, + }); + if block.has_abs { + let full = softmax(&logits.iter().map(|v| v / temperature).collect::>()); + entry["abstain_probability"] = json_f64(full[full.len() - 1]); + } + if block.qtype == "score" { + let expected: f64 = p.iter().enumerate().map(|(i, v)| i as f64 * v).sum(); + entry["expected_index"] = json_f64(expected); + // The default values normalize the levels to 0..1 + // (i / (n-1)); the SystemOne path never passes explicit ones. + let n = p.len(); + let score: f64 = p + .iter() + .enumerate() + .map(|(i, v)| { + // The reference's default values: i/(n-1), or 0 for + // a single level (never NaN). + let value = if n > 1 { + i as f64 / (n - 1) as f64 + } else { + 0.0 + }; + value * v + }) + .sum(); + entry["score"] = json_f64(score); + } + answers.push(entry); + } + if let Some(span) = span { + let text = span_text.unwrap_or(""); + let head = span.head.as_str(); + // The per-question override wins over the calibrated rule. + let (mut thr, mut rule) = match thresholds.get(&span.qid) { + Some(given) => (*given, "given".to_string()), + None => cal.span_threshold(&span.qid, &span.names, span.n_tokens, head), + }; + let temperature = cal.span_temperature(head); + let probs: Vec> = span + .logits + .iter() + .map(|row| { + row.iter() + .map(|v| sigmoid(*v as f64 / temperature) as f32) + .collect() + }) + .collect(); + if rule.starts_with("pii:") { + let (gated, gated_rule) = + cal.sparse_gate(&probs, &span.offs, &span.names, text, thr, &rule); + thr = gated; + rule = gated_rule; + } + let mut decoded = if span.offs.is_empty() { + Vec::new() + } else { + decode_spans(&probs, &span.offs, &span.names, text, thr) + }; + let chars: Vec = text.chars().collect(); + for entry in decoded.iter_mut() { + let (s, e) = ( + entry["start"].as_u64().unwrap_or(0) as usize, + entry["end"].as_u64().unwrap_or(0) as usize, + ); + entry["text"] = Value::String(chars[s..e].iter().collect()); + if let Some(mapped) = span + .label_alias + .as_ref() + .and_then(Value::as_object) + .and_then(|alias| alias.get(entry["label"].as_str().unwrap_or(""))) + { + entry["label"] = mapped.clone(); + } + } + let mut answer = serde_json::json!({ + "id": span.qid, + "type": "span", + "target_part": span.over, + "spans": decoded, + "threshold": thr, + "threshold_rule": rule, + "head": head, + }); + if let Some(reason) = &span.head_reason { + answer["head_reason"] = Value::String(reason.clone()); + } + answers.push(answer); + } + answers +} + +fn argmax_usize(x: &[f64]) -> usize { + let mut best = 0usize; + let mut value = f64::NEG_INFINITY; + for (index, item) in x.iter().enumerate() { + if *item > value { + value = *item; + best = index; + } + } + best +} + +fn json_f64(value: f64) -> Value { + serde_json::Number::from_f64(value) + .map(Value::Number) + .unwrap_or(Value::Null) +} + +/// `predict_raw`'s window aggregation: sum the window rows' span logits +/// per original word (matched by char offsets), divide by coverage +/// count, uncovered words score -30. +pub fn aggregate_windows( + names: &[String], + original_offs: &[(usize, usize)], + window_logits: &[Vec>], + window_offs: &[Vec<(usize, usize)>], +) -> Vec> { + let labels = names.len(); + // The reference accumulates in float64; the result casts to f32. + let mut acc = vec![vec![0.0f64; labels]; original_offs.len()]; + let mut cnt = vec![0usize; original_offs.len()]; + for (logits, offs) in window_logits.iter().zip(window_offs) { + for ((s, e), row) in offs.iter().zip(logits) { + if let Some(u) = original_offs.iter().position(|(os, oe)| os == s && oe == e) { + for (l, value) in row.iter().enumerate() { + acc[u][l] += *value as f64; + } + cnt[u] += 1; + } + } + } + acc.iter() + .zip(&cnt) + .map(|(row, c)| { + if *c > 0 { + row.iter().map(|v| (*v / *c as f64) as f32).collect() + } else { + vec![-30.0; labels] + } + }) + .collect() +} + +// ---------------------------------------------------- SystemOne finishing + +/// `system_one_response`: predict answers + the response plan -> the +/// SystemOne envelope (model, answers, usage, spans/span_heads, +/// sets, thresholds), with the span Noul view = the max word +/// probability (pre-decode, range-filtered) and field-clipped spans. +#[allow(clippy::too_many_arguments)] +pub fn system_one_response( + model_name: &str, + plan: &[Value], + predict_answers: &[Value], + span_word_probabilities: &std::collections::HashMap>, + part_texts: &std::collections::HashMap, + input_tokens: usize, +) -> Value { + let by_id: std::collections::HashMap<&str, &Value> = predict_answers + .iter() + .map(|a| (a["id"].as_str().unwrap_or(""), a)) + .collect(); + let mut answers = serde_json::Map::new(); + let mut spans = serde_json::Map::new(); + let mut span_heads = serde_json::Map::new(); + let mut sets = serde_json::Map::new(); + let mut thresholds = serde_json::Map::new(); + for entry in plan { + let qid = entry["qid"].as_str().unwrap_or(""); + let Some(answer) = by_id.get(qid) else { + continue; + }; + match entry["type"].as_str().unwrap_or("") { + "noul" => { + answers.insert( + qid.to_string(), + serde_json::json!({"type": "noul", "noul": answer["probabilities"]["yes"]}), + ); + } + "choice" => { + let names: Vec<&str> = entry["criteria"] + .as_object() + .map(|c| c.keys().map(String::as_str).collect()) + .unwrap_or_default(); + let probabilities: serde_json::Map = names + .iter() + .map(|name| (name.to_string(), answer["probabilities"][*name].clone())) + .collect(); + let values: Vec = names + .iter() + .map(|name| answer["probabilities"][*name].as_f64().unwrap_or(0.0)) + .collect(); + answers.insert( + qid.to_string(), + serde_json::json!({ + "type": "choice", + "choice": answer["answer"], + "confidence": json_f64(choice_confidence(&values)), + "probabilities": probabilities, + }), + ); + } + "score" => { + let levels: Vec = entry["criteria"].as_array().cloned().unwrap_or_default(); + // The predict answer keys probabilities by the level's + // rendered name (content_text), not by index. + let names: Vec = levels + .iter() + .map(|level| match level { + Value::String(text) => text.clone(), + other => crate::state::canonical(other), + }) + .collect(); + let values: Vec = names + .iter() + .map(|name| { + answer["probabilities"][name.as_str()] + .as_f64() + .unwrap_or(0.0) + }) + .collect(); + let expected: f64 = values.iter().enumerate().map(|(i, v)| i as f64 * v).sum(); + let legend: serde_json::Map = levels + .iter() + .enumerate() + .map(|(i, level)| (i.to_string(), level.clone())) + .collect(); + let probabilities: serde_json::Map = values + .iter() + .enumerate() + .map(|(i, v)| (i.to_string(), json_f64(*v))) + .collect(); + answers.insert( + qid.to_string(), + serde_json::json!({ + "type": "score", + "score": json_f64(expected), + "confidence": json_f64(score_confidence(&values)), + "legend": legend, + "probabilities": probabilities, + }), + ); + } + "set" => { + let selected: Vec = answer["answer"].as_array().cloned().unwrap_or_default(); + sets.insert( + qid.to_string(), + serde_json::json!({ + "selected": selected, + "probabilities": answer["probabilities"], + }), + ); + thresholds.insert(qid.to_string(), answer["threshold"].clone()); + for (label, value) in answer["probabilities"] + .as_object() + .unwrap_or(&serde_json::Map::new()) + { + answers.insert( + format!("{qid}.{label}"), + serde_json::json!({"type": "noul", "noul": value}), + ); + } + } + _ => { + // span: clip to the plan's field range and re-trim. + let part = entry["part"].as_str().unwrap_or("user"); + let text = part_texts.get(part).cloned().unwrap_or_default(); + let chars: Vec = text.chars().collect(); + let mut out = Vec::new(); + match entry["range"].as_array() { + Some(range) if range.len() == 2 => { + let (lo, hi) = ( + range[0].as_u64().unwrap_or(0) as usize, + range[1].as_u64().unwrap_or(0) as usize, + ); + for z in answer["spans"].as_array().unwrap_or(&vec![]) { + let s = z["start"].as_u64().unwrap_or(0) as usize; + let e = z["end"].as_u64().unwrap_or(0) as usize; + let (s, e) = trim(&chars, s.max(lo), e.min(hi)); + if e > s { + out.push(serde_json::json!({ + "label": z["label"], + "start": s - lo, + "end": e - lo, + "text": chars[s..e].iter().collect::(), + "probability": z["probability"], + })); + } + } + } + _ => { + out = answer["spans"].as_array().cloned().unwrap_or_default(); + } + } + spans.insert(qid.to_string(), Value::Array(out)); + span_heads.insert(qid.to_string(), answer["head"].clone()); + thresholds.insert(qid.to_string(), answer["threshold"].clone()); + let max = span_word_probabilities + .get(qid) + .map(|probs| probs.iter().cloned().fold(0.0f32, f32::max)) + .unwrap_or(0.0); + answers.insert( + qid.to_string(), + serde_json::json!({"type": "noul", "noul": json_f64(max as f64)}), + ); + } + } + } + let mut response = serde_json::Map::new(); + response.insert("model".into(), Value::String(model_name.to_string())); + response.insert("answers".into(), Value::Object(answers)); + response.insert( + "usage".into(), + serde_json::json!({"input_tokens": input_tokens, "output_tokens": 0}), + ); + if !spans.is_empty() { + response.insert("spans".into(), Value::Object(spans)); + response.insert("span_heads".into(), Value::Object(span_heads)); + } + if !sets.is_empty() { + response.insert("sets".into(), Value::Object(sets)); + } + if !thresholds.is_empty() { + response.insert("thresholds".into(), Value::Object(thresholds)); + } + Value::Object(response) +} + +/// The top-two probability margin (the reference's `_choice_confidence`). +pub fn choice_confidence(values: &[f64]) -> f64 { + let mut sorted = values.to_vec(); + sorted.sort_by(|a, b| b.partial_cmp(a).unwrap()); + if sorted.len() == 1 { + return 1.0; + } + (sorted[0] - sorted[1]).clamp(0.0, 1.0) +} + +/// One minus the normalized level variance (`_score_confidence`). +pub fn score_confidence(values: &[f64]) -> f64 { + let n = values.len(); + if n == 1 { + return 1.0; + } + let mean: f64 = values.iter().enumerate().map(|(i, v)| i as f64 * v).sum(); + let var: f64 = values + .iter() + .enumerate() + .map(|(i, v)| v * (i as f64 - mean).powi(2)) + .sum(); + (1.0 - var / ((n * n - 1) as f64 / 12.0)).clamp(0.0, 1.0) +} diff --git a/src/models/vela/native/src/state.rs b/src/models/vela/native/src/state.rs new file mode 100644 index 00000000..d33e42bb --- /dev/null +++ b/src/models/vela/native/src/state.rs @@ -0,0 +1,235 @@ +//! DEV2.0 canonical JSON and the state → typed-parts mapping, mirroring +//! the pinned runtime's `canonical_json`, `content_text` and +//! `system_one_state` exactly (the frozen `native-plan-*` fixtures are +//! the parity target). + +use serde_json::Value; +use std::collections::BTreeMap; + +use crate::contract::RequestError; + +/// `json.dumps(ensure_ascii=False, sort_keys=True, separators=(",", ":"))` +/// — key-sorted compact JSON, non-ASCII literal. +pub fn canonical(value: &Value) -> String { + match value { + Value::Object(map) => { + let mut keys: Vec<&String> = map.keys().collect(); + keys.sort(); + let parts: Vec = keys + .iter() + .map(|key| { + format!( + "{}:{}", + serde_json::to_string(key).unwrap(), + canonical(&map[*key]) + ) + }) + .collect(); + format!("{{{}}}", parts.join(",")) + } + Value::Array(items) => { + let parts: Vec = items.iter().map(canonical).collect(); + format!("[{}]", parts.join(",")) + } + other => serde_json::to_string(other).unwrap(), + } +} + +/// JSON content → model text: strings as they are, anything else as +/// canonical JSON. +pub fn content_text(value: &Value) -> String { + match value { + Value::String(text) => text.clone(), + other => canonical(other), + } +} + +/// The state-key → part-role table (unknown keys land in `context`). +pub fn key_role(key: &str) -> &'static str { + match key.to_ascii_lowercase().as_str() { + "request" | "user" | "prompt" => "user", + "answer" | "response" => "answer", + _ => "context", + } +} + +/// One rendered state part, in user / context / answer order. +#[derive(Debug, Clone, serde::Serialize, PartialEq)] +pub struct Part { + pub r#type: String, + pub text: String, +} + +/// A named state field's span inside its part's text (Unicode code +/// points, as on the reference). +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct FieldRange { + pub role: &'static str, + pub start: usize, + pub end: usize, +} + +#[derive(Debug)] +pub struct StateParts { + pub parts: Vec, + pub fields: BTreeMap, +} + +fn state_error(msg: &str) -> RequestError { + RequestError::envelope(&["state"], msg, "value_error") +} + +/// The SystemOne state → (parts, fields) mapping, mirroring +/// `system_one_state`: a plain string is one user part; a JSON array is +/// canonicalized into one; an object groups its keys by role (request / +/// user / prompt → user; source / context / document / tools / +/// previous_answer → context; answer / response → answer; unknown keys +/// → context) in user / context / answer order — one known-key entry +/// per role renders bare, several (or an unknown key) render as +/// `"key:\n"` blocks joined by blank lines. +pub fn system_one_state(state: &Value) -> Result { + // A string holding a JSON object is read as that object. + let state = match state { + Value::String(text) => { + let stripped = text.trim_start(); + if stripped.starts_with('{') { + if let Value::Object(object) = + serde_json::from_str::(stripped).unwrap_or(Value::Null) + { + Value::Object(object) + } else { + state.clone() + } + } else { + state.clone() + } + } + other => other.clone(), + }; + match &state { + Value::String(text) => { + if text.trim().is_empty() { + return Err(state_error("state must not be empty or whitespace")); + } + let mut fields = BTreeMap::new(); + fields.insert( + "request".into(), + FieldRange { + role: "user", + start: 0, + end: text.chars().count(), + }, + ); + Ok(StateParts { + parts: vec![Part { + r#type: "user".into(), + text: text.clone(), + }], + fields, + }) + } + Value::Array(_) => { + let text = canonical(&state); + let mut fields = BTreeMap::new(); + fields.insert( + "request".into(), + FieldRange { + role: "user", + start: 0, + end: text.chars().count(), + }, + ); + Ok(StateParts { + parts: vec![Part { + r#type: "user".into(), + text, + }], + fields, + }) + } + Value::Object(object) if !object.is_empty() => { + // Group per role, keeping the object's insertion order inside + // each role and remembering which entries had a known key. + let mut by_role: [Vec<(&String, &Value, bool)>; 3] = + [Vec::new(), Vec::new(), Vec::new()]; + for (key, value) in object { + let lower = key.to_ascii_lowercase(); + let known = matches!( + lower.as_str(), + "request" + | "user" + | "prompt" + | "source" + | "context" + | "document" + | "tools" + | "previous_answer" + | "answer" + | "response" + ); + let slot = match key_role(&lower) { + "user" => 0, + "answer" => 2, + _ => 1, + }; + by_role[slot].push((key, value, known)); + } + let mut parts = Vec::new(); + let mut fields = BTreeMap::new(); + for (role, items) in [ + ("user", &by_role[0]), + ("context", &by_role[1]), + ("answer", &by_role[2]), + ] { + if items.is_empty() { + continue; + } + let mut text = String::new(); + let single = items.len() == 1 && items[0].2; + for (index, (key, value, _)) in items.iter().enumerate() { + let rendered = content_text(value); + if single { + fields.insert( + (*key).clone(), + FieldRange { + role, + start: 0, + end: rendered.chars().count(), + }, + ); + text = rendered; + } else { + if index > 0 { + text.push_str("\n\n"); + } + let head = format!("{key}:\n"); + let start = text.chars().count() + head.chars().count(); + fields.insert( + (*key).clone(), + FieldRange { + role, + start, + end: start + rendered.chars().count(), + }, + ); + text.push_str(&head); + text.push_str(&rendered); + } + } + if text.trim().is_empty() { + return Err(state_error(&format!( + "the {role} part of the state is empty" + ))); + } + parts.push(Part { + r#type: role.into(), + text, + }); + } + Ok(StateParts { parts, fields }) + } + _ => Err(state_error( + "state must be a non-empty string, JSON object or JSON array", + )), + } +} diff --git a/src/models/vela/native/src/words.rs b/src/models/vela/native/src/words.rs new file mode 100644 index 00000000..3b947fd3 --- /dev/null +++ b/src/models/vela/native/src/words.rs @@ -0,0 +1,129 @@ +//! Word units and first-token mapping, mirroring the pinned runtime's +//! `schema_lib_r4` `_WORD` splitter (GLiNER2 whitespace splitter + CJK +//! one unit per character + the r4 URL/email/handle rule) and +//! `word_first_tokens`. Offsets are Unicode code points everywhere, as +//! on the reference. + +use onig::Regex; + +/// The CJK ranges of the reference `_WORD` pattern, verbatim. +const CJK: &str = "⺀-⿿　-〿぀-ヿ㄀-ㇿ㐀-䶿一-鿿ꀀ-꓏가-힯豈-﫿︰-﹏＀-￯"; +/// Trailing punctuation the URL rule strips. +const TRAIL: &str = ".,;:!?)\\]}>\"'\u{201d}\u{2019}\u{bb}"; + +fn word_pattern() -> Regex { + let pattern = concat!( + "(?:https?://|www\\.)[^\\s{cjk}]+?(?=[{trail}]*(?![^\\s{cjk}]))", + "|[a-z0-9._%+-]+@[a-z0-9.-]+\\.[a-z]{2,}", + "|@[a-z0-9_]+", + "|[{cjk}]", + "|[^\\W{cjk}]+(?:[-_][^\\W{cjk}]+)*", + "|\\S", + ); + let pattern = pattern.replace("{cjk}", CJK).replace("{trail}", TRAIL); + Regex::with_options( + &pattern, + onig::RegexOptions::REGEX_OPTION_IGNORECASE, + onig::Syntax::ruby(), + ) + .expect("the reference word pattern must compile") +} + +/// Start-of-char byte offsets plus the end sentinel; index it with +/// `partition_point(|&start| start < byte)` to map a byte offset to its +/// code-point offset. +fn char_starts(text: &str) -> Vec { + text.char_indices() + .map(|(byte, _)| byte) + .chain(std::iter::once(text.len())) + .collect() +} + +/// The (start, end) word ranges of `text`, in code points. +pub fn split_words(text: &str) -> Vec<(usize, usize)> { + let regex = word_pattern(); + // Byte offsets → code-point offsets. + let char_of_byte = char_starts(text); + let to_char = |byte: usize| -> usize { char_of_byte.partition_point(|&start| start < byte) }; + regex + .find_iter(text) + .map(|(start, end)| (to_char(start), to_char(end))) + .collect() +} + +/// (word char offsets (W, 2), each word's first overlapping sub-word +/// token index), mirroring `word_first_tokens`: the first token whose +/// cumulative end passes the word start, skipping empty tokens. +pub fn word_first_tokens( + text: &str, + offsets: &[(usize, usize)], +) -> (Vec<(usize, usize)>, Vec) { + let words = split_words(text); + if words.is_empty() || offsets.is_empty() { + return (Vec::new(), Vec::new()); + } + let valid: Vec = offsets.iter().map(|(s, e)| e > s).collect(); + let mut cummax_end: Vec = Vec::with_capacity(offsets.len()); + let mut running: i64 = -1; + for (index, (_start, end)) in offsets.iter().enumerate() { + if valid[index] { + running = running.max(*end as i64); + } + cummax_end.push(running); + } + let mut word_offsets = Vec::new(); + let mut first_tokens = Vec::new(); + for (a, b) in words { + let mut t = cummax_end.partition_point(|&end| end <= a as i64); + while t < offsets.len() && !valid[t] { + t += 1; + } + if t < offsets.len() && offsets[t].0 < b { + word_offsets.push((a, b)); + first_tokens.push(t); + } + } + (word_offsets, first_tokens) +} + +/// The reference's CJK class as a range test (the same ranges the word +/// splitter carries). +pub fn is_cjk_char(ch: char) -> bool { + // The reference class has one contiguous run from U+8C48 to + // U+FAFF and ends the Yi block at U+A4CF (not U+A48F). + matches!(ch as u32, + 0x2E80..=0x2FFF | 0x3000..=0x303F | 0x3040..=0x30FF | 0x3100..=0x31FF + | 0x3400..=0x4DBF | 0x4E00..=0x9FFF | 0xA000..=0xA4CF | 0xAC00..=0xD7AF + | 0x8C48..=0xFAFF | 0xFE30..=0xFE4F | 0xFF00..=0xFFEF) +} + +/// The reference's `_UNIT` split (decode-time units): URL bodies, runs +/// of word characters (excluding CJK, `_` excluded by `[^\W_]`) plus +/// `@._-+`, or any non-space char. Returns each char's unit id, -1 +/// outside units — the decode's `u` array. +pub fn unit_ids(chars: &[char]) -> Vec { + let mut ids = vec![-1i64; chars.len()]; + let text: String = chars.iter().collect(); + let pattern = concat!( + "(?:https?://|www\\.)[^\\s\\]\\)\\}>「」『』\"'“”{cjk}]+?(?=[.,;:!?]*(?![^\\s\\]\\)\\}>「」『』\"'“”{cjk}]))", + "|(?:(?![{cjk}])[^\\W_]|[@._\\-+])+", + "|\\S", + ) + .replace("{cjk}", CJK); + let regex = Regex::with_options( + &pattern, + onig::RegexOptions::REGEX_OPTION_IGNORECASE, + onig::Syntax::ruby(), + ) + .expect("the reference unit pattern must compile"); + // find_iter yields byte offsets; ids indexes characters. + let char_of_byte = char_starts(&text); + let to_char = |byte: usize| -> usize { char_of_byte.partition_point(|&start| start < byte) }; + for (unit, (start, end)) in regex.find_iter(&text).enumerate() { + let (start, end) = (to_char(start), to_char(end)); + for id in ids.iter_mut().take(end).skip(start) { + *id = unit as i64; + } + } + ids +} diff --git a/src/models/vela/postprocess.py b/src/models/vela/postprocess.py index 141b6938..907acb9b 100644 --- a/src/models/vela/postprocess.py +++ b/src/models/vela/postprocess.py @@ -91,26 +91,35 @@ def _question_ids_of(questions: Mapping[str, Any], *kinds: str) -> list[str]: ] -def finish_response(questions: Mapping[str, Any], response: dict[str, Any]) -> dict[str, Any]: +def finish_response( + questions: Mapping[str, Any], + response: dict[str, Any], + *, + hybrid: bool = False, +) -> dict[str, Any]: """Check the runtime response against the admitted questions and pass it through. The pinned runtime owns every value in this response; these checks only refuse a response that could not be reconstructed for the - request that produced it (wrong key set or order, a hybrid-only + request that produced it (wrong key set or order, a member-invented field, broken usage accounting, an answer of unexpected shape). + ``hybrid`` selects the member: the hybrid executor's responses carry + ``span_heads`` alongside ``spans`` (upstream emits both together, + keyed by the span question ids); the 0.3B member's never do. """ if not isinstance(response, dict): raise RuntimeError("the pinned runtime returned a non-object response") keys = set(response) foreign = keys & HYBRID_ONLY_FIELDS - if foreign: + if foreign and not hybrid: raise RuntimeError( f"the pinned runtime returned hybrid-only field(s) {sorted(foreign)}; " "this worker serves the 0.3B member" ) missing = [field for field in REQUIRED_FIELDS if field not in response] - if missing or not keys <= TOP_LEVEL_FIELDS: + allowed = TOP_LEVEL_FIELDS | HYBRID_ONLY_FIELDS if hybrid else TOP_LEVEL_FIELDS + if missing or not keys <= allowed: raise RuntimeError("the pinned runtime returned an unexpected response envelope") answers = response["answers"] expected = expected_answer_order(questions) @@ -126,6 +135,14 @@ def finish_response(questions: Mapping[str, Any], response: dict[str, Any]) -> d raise RuntimeError(f"{field} do not match the request's {field} questions") elif field in response: raise RuntimeError(f"the pinned runtime returned {field} without {field} questions") + if hybrid: + span_ids = _question_ids_of(questions, "span") + if span_ids: + heads = response.get("span_heads") + if not isinstance(heads, dict) or list(heads) != span_ids: + raise RuntimeError("span_heads do not match the request's span questions") + elif "span_heads" in response: + raise RuntimeError("the pinned runtime returned span_heads without span questions") usage = response["usage"] if ( not isinstance(usage, dict) diff --git a/src/models/vela/protocol.py b/src/models/vela/protocol.py index de5dd2db..d9341206 100644 --- a/src/models/vela/protocol.py +++ b/src/models/vela/protocol.py @@ -24,10 +24,10 @@ # hybrid members serve their config.json "model_name" instead. MODEL_NAME = "vllm-sr/Vela-2.0-0.3B" # Slice-scoped transport cap. Upstream vela2_serve.py sets no total-body -# limit (its only size boundary is the 8,192-token input window); a text -# request bounded by that window is orders of magnitude smaller, so this -# cap never binds a valid request. A deliberate difference recorded with -# the integration. +# limit (its only size boundary is the member's token input window); a +# text request bounded by that window is orders of magnitude smaller, so +# this cap never binds a valid request. A deliberate difference recorded +# with the integration. MAX_BODY = 8 * 1024 * 1024 # The envelope fields, in upstream's missing-field check order. REQUEST_FIELDS = ("state", "model", "questions") diff --git a/tests/vela/fixtures/head-input-example-request-cjk.json b/tests/vela/fixtures/head-input-example-request-cjk.json new file mode 100644 index 00000000..3baaa645 --- /dev/null +++ b/tests/vela/fixtures/head-input-example-request-cjk.json @@ -0,0 +1 @@ +{"case": "example-request-cjk", "request": {"model": "vllm-sr/Vela-2.0-0.8B", "state": {"request": "客户张伟（zhang.wei@example.cn）于2026年10月1日在上海市浦东新区世纪大道100号办理了业务，联系电话13800138000。", "source": "客户的业务申请需要在三个工作日内审核完成。", "answer": "客户的业务已经即时完成了。"}, "questions": {"pii": {"type": "span", "instructions": "Which spans are personal information?", "criteria": {"PERSON": "a person's name", "EMAIL_ADDRESS": "an e-mail address", "PHONE_NUMBER": "a telephone number", "STREET_ADDRESS": "a street address or postal address"}, "over": "request"}, "unsupported": {"type": "span", "instructions": "Which spans of the answer are not supported by the context?", "criteria": {"unsupported": "a claim not supported by the context"}}, "billing": {"type": "noul", "instructions": "这个请求是否与账单有关？"}}}, "rows": [{"head": "router", "head_reason": "id", "label_alias": null, "n_tokens": 404, "seg_lens": {"user": 46, "context": 10, "answer": 6}}, {"head": "router", "head_reason": "label_set", "label_alias": null, "n_tokens": 193, "seg_lens": {"user": 46, "context": 10, "answer": 6}}], "head_calls": [{"kind": "q", "cand": "q0_cand", "query": "q0_query", "logits": [[-1.60933518409729, -3.7373664379119873, -10.640555381774902]], "row": 0}, {"kind": "span", "head": "router", "words": "s1_words", "labels": "s1_labels", "logits": [[-23.778533935546875, -35.67962646484375, -29.496707916259766, -35.02517318725586], [-23.778533935546875, -35.67962646484375, -29.496707916259766, -35.02517318725586], [-13.288297653198242, -17.101852416992188, -9.51349925994873, -13.732088088989258], [-13.288297653198242, -17.101852416992188, -9.51349925994873, -13.732088088989258], [-17.785783767700195, -24.783889770507812, -18.803138732910156, -24.5747013092041], [-5.519737243652344, -7.974113464355469, -2.0204527378082275, -4.032443523406982], [-19.710346221923828, -31.652324676513672, -24.4345703125, -30.253273010253906], [-22.11681365966797, -31.155961990356445, -25.493202209472656, -30.28239631652832], [-0.9185004830360413, -6.406106472015381, -0.692579448223114, -1.6303598880767822], [-4.95607852935791, -9.517541885375977, -2.2732596397399902, -4.304035663604736], [-0.19278430938720703, -7.234131336212158, -0.09104931354522705, -1.1003239154815674], [-2.164036750793457, -7.8626861572265625, -1.2940788269042969, -2.4866232872009277], [-1.3710556030273438, -6.950081825256348, -1.0965814590454102, -1.80461847782135], [-17.513790130615234, -17.537555694580078, -8.913171768188477, -11.932638168334961], [-21.893512725830078, -29.469112396240234, -24.093914031982422, -26.871994018554688], [-3.359339952468872, -7.748542308807373, -2.7091588973999023, -0.33820855617523193], [-3.359339952468872, -7.748542308807373, -2.7091588973999023, -0.33820855617523193], [-3.359339952468872, -7.748542308807373, -2.7091588973999023, -0.33820855617523193], [-3.4210779666900635, -7.483105659484863, -2.7869768142700195, -0.5679111480712891], [-3.4210779666900635, -7.483105659484863, -2.7869768142700195, -0.5679111480712891], [-3.4210779666900635, -7.483105659484863, -2.7869768142700195, -0.5679111480712891], [-3.4210779666900635, -7.483105659484863, -2.7869768142700195, -0.5679111480712891], [-3.6699564456939697, -8.56665325164795, -2.6497883796691895, 0.620187520980835], [-3.6699564456939697, -8.56665325164795, -2.6497883796691895, 0.620187520980835], [-3.4258086681365967, -9.092829704284668, -2.0032799243927, 0.25476884841918945], [-3.4258086681365967, -9.092829704284668, -2.0032799243927, 0.25476884841918945], [-1.0213323831558228, -8.279101371765137, -0.9210346341133118, 1.104820728302002], [-10.929340362548828, -11.960079193115234, -4.331625461578369, -3.7565321922302246], [-19.668752670288086, -21.596508026123047, -16.757234573364258, -19.447601318359375], [-19.668752670288086, -21.596508026123047, -16.757234573364258, -19.447601318359375], [-19.668752670288086, -21.596508026123047, -16.757234573364258, -19.447601318359375], [-19.40172004699707, -19.034996032714844, -14.057591438293457, -15.741283416748047], [-19.40172004699707, -19.034996032714844, -14.057591438293457, -15.741283416748047], [-19.49544906616211, -31.111461639404297, -23.796104431152344, -29.51728057861328], [-21.458332061767578, -33.868682861328125, -27.257335662841797, -32.84736633300781], [-21.458332061767578, -33.868682861328125, -27.257335662841797, -32.84736633300781], [-21.458332061767578, -33.868682861328125, -27.257335662841797, -32.84736633300781], [-21.458332061767578, -33.868682861328125, -27.257335662841797, -32.84736633300781], [-0.6149466633796692, -7.636533260345459, -0.18819284439086914, -0.1909337043762207], [-24.31658172607422, -34.86664581298828, -29.284488677978516, -32.92279052734375]], "row": 0}, {"kind": "span", "head": "router", "words": "s2_words", "labels": "s2_labels", "logits": [[-4.0871477127075195], [-4.0871477127075195], [-4.0871477127075195], [-2.413851261138916], [-2.413851261138916], [-1.4215611219406128], [-1.4215611219406128], [-0.6014259457588196], [-0.6014259457588196], [-1.4138461351394653], [-1.4138461351394653], [-1.4138461351394653], [-4.652787685394287]], "row": 1}], "predict": {"answers": [{"id": "pii", "type": "span", "target_part": "user", "spans": [{"start": 32, "end": 34, "label": "PHONE_NUMBER", "probability": 0.4658691883087158, "text": "10"}, {"start": 38, "end": 52, "label": "STREET_ADDRESS", "probability": 0.4821831285953522, "text": "上海市浦东新区世纪大道100"}, {"start": 63, "end": 74, "label": "PHONE_NUMBER", "probability": 0.4298103451728821, "text": "13800138000"}], "threshold": 0.2914, "threshold_rule": "pii:length", "head": "router", "head_reason": "id"}, {"id": "unsupported", "type": "span", "target_part": "answer", "spans": [], "threshold": 0.5, "threshold_rule": "span:*", "head": "router", "head_reason": "label_set"}, {"id": "billing", "type": "choice", "answer": "no", "probabilities": {"no": 0.8325233433461274, "yes": 0.1674766566538726}, "abstain_probability": 0.0009211774164155532}], "meta": {"tokens": {"user": 46, "context": 10, "answer": 6}, "windows": 0, "sequences": 2}}} diff --git a/tests/vela/fixtures/head-input-example-request-cjk.safetensors b/tests/vela/fixtures/head-input-example-request-cjk.safetensors new file mode 100644 index 00000000..bd7cb173 Binary files /dev/null and b/tests/vela/fixtures/head-input-example-request-cjk.safetensors differ diff --git a/tests/vela/fixtures/head-input-example-request-score.json b/tests/vela/fixtures/head-input-example-request-score.json new file mode 100644 index 00000000..1588ad68 --- /dev/null +++ b/tests/vela/fixtures/head-input-example-request-score.json @@ -0,0 +1 @@ +{"case": "example-request-score", "request": {"model": "vllm-sr/Vela-2.0-0.8B", "state": "Hello, I would like to check the status of my order. It has been five days and I have heard nothing back.", "questions": {"tone": {"type": "score", "instructions": "How formal is the message?", "criteria": ["casual", "neutral", "formal"]}, "urgency": {"type": "score", "instructions": "How urgently does the sender need a reply?", "criteria": ["no rush", "soon", "today", "immediately"]}}}, "rows": [{"head": "router", "head_reason": null, "label_alias": null, "n_tokens": 228, "seg_lens": {"user": 25}}], "head_calls": [{"kind": "q", "cand": "q0_cand", "query": "q0_query", "logits": [[-2.308352470397949, -1.095753788948059, -1.616711139678955, -0.3145672082901001], [-1.1832587718963623, -1.9728960990905762, -1.711325764656067, -3.1148641109466553]], "row": 0}], "predict": {"answers": [{"id": "tone", "type": "score", "answer": "neutral", "probabilities": {"casual": 0.16807713462834117, "neutral": 0.5139566439051507, "formal": 0.3179662214665082}, "expected_index": 1.1498890868381673, "score": 0.5749445434190836}, {"id": "urgency", "type": "score", "answer": "no rush", "probabilities": {"no rush": 0.44128118285871365, "soon": 0.21311503083546, "today": 0.2712213259289846, "immediately": 0.07438246037684161}, "expected_index": 0.9787050638239541, "score": 0.3262350212746513}], "meta": {"tokens": {"user": 25}, "windows": 0, "sequences": 1}}} diff --git a/tests/vela/fixtures/head-input-example-request-score.safetensors b/tests/vela/fixtures/head-input-example-request-score.safetensors new file mode 100644 index 00000000..1e580a6c Binary files /dev/null and b/tests/vela/fixtures/head-input-example-request-score.safetensors differ diff --git a/tests/vela/fixtures/head-input-example-request-set-threshold.json b/tests/vela/fixtures/head-input-example-request-set-threshold.json new file mode 100644 index 00000000..e8b374a2 --- /dev/null +++ b/tests/vela/fixtures/head-input-example-request-set-threshold.json @@ -0,0 +1 @@ +{"case": "example-request-set-threshold", "request": {"model": "vllm-sr/Vela-2.0-0.8B", "state": "My card was charged twice and the parcel never arrived.", "questions": {"issues": {"type": "set", "instructions": "Which issues does the customer report?", "criteria": {"billing": "payments, charges, refunds or invoices", "shipping": "delivery of an order or a parcel", "login": "signing in, passwords or account access"}, "threshold": 0.15}}}, "rows": [{"head": "router", "head_reason": null, "label_alias": null, "n_tokens": 133, "seg_lens": {"user": 11}}], "head_calls": [{"kind": "q", "cand": "q0_cand", "query": "q0_query", "logits": [[0.4933321475982666, 1.5158393383026123, -8.921335220336914]], "row": 0}], "predict": {"answers": [{"id": "issues", "type": "set", "answer": ["billing", "shipping"], "probabilities": {"billing": 0.6211446881608428, "shipping": 0.8205402483597802, "login": 0.00012988178108716601}, "threshold": 0.15}], "meta": {"tokens": {"user": 11}, "windows": 0, "sequences": 1}}} diff --git a/tests/vela/fixtures/head-input-example-request-set-threshold.safetensors b/tests/vela/fixtures/head-input-example-request-set-threshold.safetensors new file mode 100644 index 00000000..8f5ad489 Binary files /dev/null and b/tests/vela/fixtures/head-input-example-request-set-threshold.safetensors differ diff --git a/tests/vela/fixtures/head-input-example-request.json b/tests/vela/fixtures/head-input-example-request.json new file mode 100644 index 00000000..99263ddd --- /dev/null +++ b/tests/vela/fixtures/head-input-example-request.json @@ -0,0 +1 @@ +{"case": "example-request", "request": {"model": "vllm-sr/Vela-2.0-0.8B", "state": {"request": "Hi, I'm Tom Baker (tom.baker@example.com). What is the maximum daily dose of paracetamol for an adult?", "source": "For adults, the maximum dose of paracetamol is 4 grams in 24 hours, taken as 500 mg to 1 g every 4 to 6 hours.", "answer": "Adults can take up to 6 grams of paracetamol in 24 hours, in doses of 500 mg to 1 g every 4 to 6 hours."}, "questions": {"pii": {"type": "span", "instructions": "Which spans are personal information?", "criteria": {"AGE": "a person's age", "CREDIT_CARD": "a payment card number", "DATE_TIME": "a date, time or date of birth", "DOMAIN_NAME": "an internet domain name or website address", "EMAIL_ADDRESS": "an e-mail address", "GPE": "a country, city, state or other geopolitical place", "IBAN_CODE": "an international bank account number (IBAN)", "IP_ADDRESS": "an IPv4 or IPv6 address", "NRP": "a nationality, religious or political group", "ORGANIZATION": "the name of a company, institution or other organisation", "PERSON": "a person's name", "PHONE_NUMBER": "a telephone number", "STREET_ADDRESS": "a street address or postal address", "TITLE": "a personal or professional title such as Dr. or Mrs.", "US_DRIVER_LICENSE": "a US driver's licence number", "US_SSN": "a US social security number", "ZIP_CODE": "a postal or ZIP code"}, "over": "request"}, "halu": {"type": "span", "instructions": "Which spans of the answer are not supported by the context?", "criteria": {"unsupported": "a claim not supported by the context"}}, "domain": {"type": "choice", "instructions": "Which subject area is this request about?", "over": "request", "criteria": {"health": "medicine, clinical practice, nutrition, ageing or sexual health", "math": "arithmetic, algebra, geometry, statistics or other mathematics", "other": "a subject that fits none of the listed areas"}}}}, "rows": [{"head": "router", "head_reason": "id", "label_alias": null, "n_tokens": 772, "seg_lens": {"user": 27, "context": 39, "answer": 39}}, {"head": "router", "head_reason": "id", "label_alias": null, "n_tokens": 269, "seg_lens": {"user": 27, "context": 39, "answer": 39}}], "head_calls": [{"kind": "q", "cand": "q0_cand", "query": "q0_query", "logits": [[0.5404365062713623, -6.5386433601379395, -3.9254047870635986, -3.529416084289551]], "row": 0}, {"kind": "span", "head": "router", "words": "s1_words", "labels": "s1_labels", "logits": [[-9.851020812988281, -15.027624130249023, -14.757858276367188, -12.530649185180664, -3.660945415496826, -12.923664093017578, -17.943824768066406, -16.326257705688477, -5.620431900024414, -7.599363803863525, -11.234650611877441, -5.757130146026611, -5.652872085571289, -8.116801261901855, -9.238210678100586, -6.802857398986816, -6.324886322021484], [-16.28253936767578, -17.80463981628418, -18.329021453857422, -16.44207763671875, -10.134305953979492, -17.914127349853516, -20.482263565063477, -18.635469436645508, -10.971238136291504, -11.935432434082031, -16.362531661987305, -8.079880714416504, -9.401307106018066, -14.166604995727539, -16.919057846069336, -14.731834411621094, -12.536507606506348], [-16.975248336791992, -21.33708953857422, -22.336502075195312, -20.949878692626953, -12.478657722473145, -21.414112091064453, -24.373680114746094, -23.24188232421875, -13.325860023498535, -13.795920372009277, -16.87654685974121, -11.636231422424316, -11.92214584350586, -16.560941696166992, -20.60035514831543, -17.067073822021484, -14.519638061523438], [-16.7723445892334, -21.27077293395996, -22.190185546875, -21.27130889892578, -12.792131423950195, -22.414506912231445, -24.66496467590332, -23.94645118713379, -13.870166778564453, -14.444552421569824, -17.01270866394043, -11.867671012878418, -13.354513168334961, -16.251331329345703, -21.633752822875977, -18.205692291259766, -16.024608612060547], [-16.7723445892334, -21.27077293395996, -22.190185546875, -21.27130889892578, -12.792131423950195, -22.414506912231445, -24.66496467590332, -23.94645118713379, -13.870166778564453, -14.444552421569824, -17.01270866394043, -11.867671012878418, -13.354513168334961, -16.251331329345703, -21.633752822875977, -18.205692291259766, -16.024608612060547], [0.11913681030273438, -8.998804092407227, -8.105165481567383, -7.355640411376953, 0.8499364852905273, -5.159857273101807, -10.962669372558594, -10.69532585144043, -2.300189971923828, -2.142609119415283, 3.7711997032165527, -2.6488523483276367, -3.3644497394561768, 0.5961573123931885, -4.22589111328125, -4.46975040435791, -4.146326065063477], [-0.375388503074646, -8.638716697692871, -7.520653247833252, -6.3430376052856445, 0.8605620861053467, -4.922619819641113, -10.513166427612305, -10.039107322692871, -2.3470044136047363, -2.1044399738311768, 2.319047451019287, -3.095822811126709, -3.0459518432617188, 0.2876932621002197, -3.672348976135254, -4.247696399688721, -4.059322357177734], [-11.97199535369873, -15.74243450164795, -15.960700988769531, -12.123148918151855, -2.198594570159912, -14.888036727905273, -18.449371337890625, -13.958887100219727, -5.779473781585693, -7.8383917808532715, -11.294285774230957, -2.836668014526367, -3.8323404788970947, -9.61889362335205, -9.943918228149414, -6.213159561157227, -5.016597747802734], [-2.930969715118408, -8.456787109375, -8.0104341506958, -3.1965346336364746, 2.791138172149658, -5.6602864265441895, -9.054672241210938, -4.491463661193848, -3.634566068649292, -2.6940877437591553, 0.6949470043182373, -2.568450927734375, -3.0842747688293457, -2.6646759510040283, -5.099068641662598, -5.331076622009277, -4.802918434143066], [-17.96658706665039, -18.01045799255371, -19.822132110595703, -18.21443748474121, -12.168895721435547, -19.514663696289062, -20.82657241821289, -19.630020141601562, -13.630260467529297, -13.030271530151367, -18.71689224243164, -10.937429428100586, -11.503185272216797, -16.562448501586914, -20.173627853393555, -17.357961654663086, -14.826516151428223], [-17.96658706665039, -18.01045799255371, -19.822132110595703, -18.21443748474121, -12.168895721435547, -19.514663696289062, -20.82657241821289, -19.630020141601562, -13.630260467529297, -13.030271530151367, -18.71689224243164, -10.937429428100586, -11.503185272216797, -16.562448501586914, -20.173627853393555, -17.357961654663086, -14.826516151428223], [-16.90250587463379, -18.278526306152344, -18.84981346130371, -17.358551025390625, -11.352295875549316, -17.4305362701416, -20.804384231567383, -19.499237060546875, -12.09277629852295, -12.66122817993164, -17.810014724731445, -10.43729305267334, -9.796697616577148, -16.462594985961914, -17.877269744873047, -15.53689193725586, -12.608529090881348], [-17.571109771728516, -18.02204704284668, -18.949459075927734, -17.84019660949707, -12.319219589233398, -18.332033157348633, -20.70317840576172, -19.956119537353516, -13.069973945617676, -13.284128189086914, -18.666032791137695, -11.275001525878906, -11.1100492477417, -16.783119201660156, -19.03497314453125, -16.65223503112793, -14.116575241088867], [-17.20058250427246, -17.913480758666992, -19.152984619140625, -18.49874496459961, -12.374701499938965, -18.944435119628906, -20.980005264282227, -20.324554443359375, -13.318517684936523, -13.295575141906738, -18.446435928344727, -11.100875854492188, -11.361801147460938, -16.607181549072266, -19.333765029907227, -16.73701286315918, -14.240310668945312], [-10.385387420654297, -15.364139556884766, -15.511908531188965, -13.689546585083008, -3.9188613891601562, -13.724010467529297, -18.471012115478516, -16.611557006835938, -5.779675483703613, -7.293036460876465, -12.180856704711914, -4.711075782775879, -4.105204105377197, -10.222349166870117, -10.208038330078125, -5.902805805206299, -4.506038188934326], [-10.814464569091797, -14.951911926269531, -15.325828552246094, -12.80671501159668, -3.280482530593872, -13.21613883972168, -17.812551498413086, -15.572422981262207, -5.481819152832031, -7.064236164093018, -12.217747688293457, -4.450317859649658, -3.697108030319214, -10.052149772644043, -9.760396957397461, -5.52978515625, -4.171611309051514], [-14.402262687683105, -17.102127075195312, -17.262144088745117, -15.675093650817871, -8.45707893371582, -15.754737854003906, -19.572450637817383, -17.654773712158203, -9.236448287963867, -10.762482643127441, -15.542367935180664, -7.367783546447754, -6.410301208496094, -14.149118423461914, -13.918294906616211, -11.3816499710083, -8.444021224975586], [-12.651286125183105, -16.529403686523438, -16.55528450012207, -14.945316314697266, -6.345352649688721, -15.065194129943848, -18.96143341064453, -17.375417709350586, -7.089203834533691, -9.234535217285156, -13.864068031311035, -5.386238098144531, -4.79850959777832, -12.170395851135254, -11.348276138305664, -7.870039463043213, -5.878173828125], [-8.939207077026367, -13.950385093688965, -13.887860298156738, -11.350774765014648, -3.2517220973968506, -11.77290153503418, -16.509742736816406, -14.612834930419922, -6.24839973449707, -7.029216766357422, -10.663041114807129, -6.143183708190918, -6.020900249481201, -8.616498947143555, -8.805045127868652, -6.615787982940674, -6.1211934089660645], [-14.634203910827637, -16.696504592895508, -16.7551212310791, -15.17517375946045, -8.941736221313477, -15.509604454040527, -18.92809295654297, -17.477008819580078, -9.083824157714844, -10.468768119812012, -15.845718383789062, -7.56320333480835, -7.45651388168335, -14.0153226852417, -14.346378326416016, -11.850302696228027, -9.550122261047363], [-12.652865409851074, -16.130496978759766, -16.09650993347168, -14.029860496520996, -5.189237594604492, -14.40012264251709, -18.68414306640625, -16.8243408203125, -6.2160868644714355, -8.960195541381836, -13.997604370117188, -5.3901848793029785, -4.3825273513793945, -11.94284439086914, -10.51372241973877, -6.524692058563232, -5.048408508300781], [-15.082358360290527, -16.506101608276367, -16.36886215209961, -14.599099159240723, -8.522944450378418, -14.7816801071167, -18.691444396972656, -17.160917282104492, -9.380626678466797, -11.312457084655762, -16.12392234802246, -8.477106094360352, -7.283524036407471, -14.228918075561523, -13.477392196655273, -11.19268798828125, -8.870052337646484], [-18.80125617980957, -21.510345458984375, -22.83747100830078, -22.03952980041504, -15.80392837524414, -22.6486759185791, -24.405366897583008, -23.06026840209961, -17.308101654052734, -16.587818145751953, -19.37947654724121, -14.987780570983887, -15.041773796081543, -18.377851486206055, -22.79374122619629, -20.769546508789062, -18.1580810546875]], "row": 0}, {"kind": "span", "head": "router", "words": "s2_words", "labels": "s2_labels", "logits": [[-5.667586803436279], [-4.148900508880615], [-2.493729829788208], [-0.7364544868469238], [-0.2502114176750183], [3.92134428024292], [1.7255499362945557], [-2.568751811981201], [-3.1423110961914062], [-4.012135028839111], [-4.174318313598633], [-3.6643919944763184], [-4.65214729309082], [-5.806663990020752], [-5.589629650115967], [-5.320010185241699], [-3.489220142364502], [-4.177278518676758], [-4.293692111968994], [-4.44113302230835], [-4.547778129577637], [-5.308318138122559], [-6.041932106018066], [-6.042882442474365], [-5.594691753387451], [-4.543209552764893], [-4.973324298858643]], "row": 1}], "predict": {"answers": [{"id": "pii", "type": "span", "target_part": "user", "spans": [{"start": 8, "end": 17, "label": "PERSON", "probability": 0.983367919921875, "text": "Tom Baker"}, {"start": 19, "end": 40, "label": "EMAIL_ADDRESS", "probability": 0.9851056337356567, "text": "tom.baker@example.com"}], "threshold": 0.2914, "threshold_rule": "pii:length", "head": "router", "head_reason": "id"}, {"id": "halu", "type": "span", "target_part": "answer", "spans": [{"start": 19, "end": 29, "label": "unsupported", "probability": 0.7782317996025085, "text": "to 6 grams"}], "threshold": 0.35, "threshold_rule": "halu", "head": "router", "head_reason": "id"}, {"id": "domain", "type": "choice", "answer": "health", "probabilities": {"health": 0.9621185645193945, "math": 0.004639298028680393, "other": 0.03324213745192508}, "abstain_probability": 0.04287965117790319}], "meta": {"tokens": {"user": 27, "context": 39, "answer": 39}, "windows": 0, "sequences": 2}}} diff --git a/tests/vela/fixtures/head-input-example-request.safetensors b/tests/vela/fixtures/head-input-example-request.safetensors new file mode 100644 index 00000000..8e69195b Binary files /dev/null and b/tests/vela/fixtures/head-input-example-request.safetensors differ diff --git a/tests/vela/fixtures/head-input-long-span-windowed.json b/tests/vela/fixtures/head-input-long-span-windowed.json new file mode 100644 index 00000000..300076e2 --- /dev/null +++ b/tests/vela/fixtures/head-input-long-span-windowed.json @@ -0,0 +1 @@ +{"case": "long-span-windowed", "request": {"model": "vllm-sr/Vela-2.0-0.8B", "state": {"request": "Sentence 0 reports a routine delivery event. Sentence 1 reports a routine delivery event. Sentence 2 reports a routine delivery event. Sentence 3 reports a routine delivery event. Sentence 4 reports a routine delivery event. Sentence 5 reports a routine delivery event. Sentence 6 reports a routine delivery event. Sentence 7 reports a routine delivery event. Sentence 8 reports a routine delivery event. Sentence 9 reports a routine delivery event. Sentence 10 reports a routine delivery event. Sentence 11 reports a routine delivery event. Sentence 12 reports a routine delivery event. Sentence 13 reports a routine delivery event. Sentence 14 reports a routine delivery event. Sentence 15 reports a routine delivery event. Sentence 16 reports a routine delivery event. Sentence 17 reports a routine delivery event. Sentence 18 reports a routine delivery event. Sentence 19 reports a routine delivery event. Sentence 20 reports a routine delivery event. Sentence 21 reports a routine delivery event. Sentence 22 reports a routine delivery event. Sentence 23 reports a routine delivery event. Sentence 24 reports a routine delivery event. Sentence 25 reports a routine delivery event. Sentence 26 reports a routine delivery event. Sentence 27 reports a routine delivery event. Sentence 28 reports a routine delivery event. Sentence 29 reports a routine delivery event. Sentence 30 reports a routine delivery event. Sentence 31 reports a routine delivery event. Sentence 32 reports a routine delivery event. Sentence 33 reports a routine delivery event. Sentence 34 reports a routine delivery event. Sentence 35 reports a routine delivery event. Sentence 36 reports a routine delivery event. Sentence 37 reports a routine delivery event. Sentence 38 reports a routine delivery event. Sentence 39 reports a routine delivery event. Sentence 40 reports a routine delivery event. Sentence 41 reports a routine delivery event. Sentence 42 reports a routine delivery event. Sentence 43 reports a routine delivery event. Sentence 44 reports a routine delivery event. Sentence 45 reports a routine delivery event. Sentence 46 reports a routine delivery event. Sentence 47 reports a routine delivery event. Sentence 48 reports a routine delivery event. Sentence 49 reports a routine delivery event. Sentence 50 reports a routine delivery event. Sentence 51 reports a routine delivery event. Sentence 52 reports a routine delivery event. Sentence 53 reports a routine delivery event. Sentence 54 reports a routine delivery event. Sentence 55 reports a routine delivery event. Sentence 56 reports a routine delivery event. Sentence 57 reports a routine delivery event. Sentence 58 reports a routine delivery event. Sentence 59 reports a routine delivery event. Sentence 60 reports a routine delivery event. Sentence 61 reports a routine delivery event. Sentence 62 reports a routine delivery event. Sentence 63 reports a routine delivery event. Sentence 64 reports a routine delivery event. Sentence 65 reports a routine delivery event. Sentence 66 reports a routine delivery event. Sentence 67 reports a routine delivery event. Sentence 68 reports a routine delivery event. Sentence 69 reports a routine delivery event. Sentence 70 reports a routine delivery event. Sentence 71 reports a routine delivery event. Sentence 72 reports a routine delivery event. Sentence 73 reports a routine delivery event. Sentence 74 reports a routine delivery event. Sentence 75 reports a routine delivery event. Sentence 76 reports a routine delivery event. Sentence 77 reports a routine delivery event. Sentence 78 reports a routine delivery event. Sentence 79 reports a routine delivery event. Sentence 80 reports a routine delivery event. Sentence 81 reports a routine delivery event. Sentence 82 reports a routine delivery event. Sentence 83 reports a routine delivery event. Sentence 84 reports a routine delivery event. Sentence 85 reports a routine delivery event. Sentence 86 reports a routine delivery event. Sentence 87 reports a routine delivery event. Sentence 88 reports a routine delivery event. Sentence 89 reports a routine delivery event. Sentence 90 reports a routine delivery event. Sentence 91 reports a routine delivery event. Sentence 92 reports a routine delivery event. Sentence 93 reports a routine delivery event. Sentence 94 reports a routine delivery event. Sentence 95 reports a routine delivery event. Sentence 96 reports a routine delivery event. Sentence 97 reports a routine delivery event. Sentence 98 reports a routine delivery event. Sentence 99 reports a routine delivery event. Sentence 100 reports a routine delivery event. Sentence 101 reports a routine delivery event. Sentence 102 reports a routine delivery event. Sentence 103 reports a routine delivery event. Sentence 104 reports a routine delivery event. Sentence 105 reports a routine delivery event. Sentence 106 reports a routine delivery event. Sentence 107 reports a routine delivery event. Sentence 108 reports a routine delivery event. Sentence 109 reports a routine delivery event. Sentence 110 reports a routine delivery event. Sentence 111 reports a routine delivery event. Sentence 112 reports a routine delivery event. Sentence 113 reports a routine delivery event. Sentence 114 reports a routine delivery event. Sentence 115 reports a routine delivery event. Sentence 116 reports a routine delivery event. Sentence 117 reports a routine delivery event. Sentence 118 reports a routine delivery event. Sentence 119 reports a routine delivery event. Sentence 120 reports a routine delivery event. Sentence 121 reports a routine delivery event. Sentence 122 reports a routine delivery event. Sentence 123 reports a routine delivery event. Sentence 124 reports a routine delivery event. Sentence 125 reports a routine delivery event. Sentence 126 reports a routine delivery event. Sentence 127 reports a routine delivery event. Sentence 128 reports a routine delivery event. Sentence 129 reports a routine delivery event. Sentence 130 reports a routine delivery event. Sentence 131 reports a routine delivery event. Sentence 132 reports a routine delivery event. Sentence 133 reports a routine delivery event. Sentence 134 reports a routine delivery event. Sentence 135 reports a routine delivery event. Sentence 136 reports a routine delivery event. Sentence 137 reports a routine delivery event. Sentence 138 reports a routine delivery event. Sentence 139 reports a routine delivery event. Sentence 140 reports a routine delivery event. Sentence 141 reports a routine delivery event. Sentence 142 reports a routine delivery event. Sentence 143 reports a routine delivery event. Sentence 144 reports a routine delivery event. Sentence 145 reports a routine delivery event. Sentence 146 reports a routine delivery event. Sentence 147 reports a routine delivery event. Sentence 148 reports a routine delivery event. Sentence 149 reports a routine delivery event. Sentence 150 reports a routine delivery event. Sentence 151 reports a routine delivery event. Sentence 152 reports a routine delivery event. Sentence 153 reports a routine delivery event. Sentence 154 reports a routine delivery event. Sentence 155 reports a routine delivery event. Sentence 156 reports a routine delivery event. Sentence 157 reports a routine delivery event. Sentence 158 reports a routine delivery event. Sentence 159 reports a routine delivery event. Sentence 160 reports a routine delivery event. Sentence 161 reports a routine delivery event. Sentence 162 reports a routine delivery event. Sentence 163 reports a routine delivery event. Sentence 164 reports a routine delivery event. Sentence 165 reports a routine delivery event. Sentence 166 reports a routine delivery event. Sentence 167 reports a routine delivery event. Sentence 168 reports a routine delivery event. Sentence 169 reports a routine delivery event. Sentence 170 reports a routine delivery event. Sentence 171 reports a routine delivery event. Sentence 172 reports a routine delivery event. Sentence 173 reports a routine delivery event. Sentence 174 reports a routine delivery event. Sentence 175 reports a routine delivery event. Sentence 176 reports a routine delivery event. Sentence 177 reports a routine delivery event. Sentence 178 reports a routine delivery event. Sentence 179 reports a routine delivery event. Sentence 180 reports a routine delivery event. Sentence 181 reports a routine delivery event. Sentence 182 reports a routine delivery event. Sentence 183 reports a routine delivery event. Sentence 184 reports a routine delivery event. Sentence 185 reports a routine delivery event. Sentence 186 reports a routine delivery event. Sentence 187 reports a routine delivery event. Sentence 188 reports a routine delivery event. Sentence 189 reports a routine delivery event. Sentence 190 reports a routine delivery event. Sentence 191 reports a routine delivery event. Sentence 192 reports a routine delivery event. Sentence 193 reports a routine delivery event. Sentence 194 reports a routine delivery event. Sentence 195 reports a routine delivery event. Sentence 196 reports a routine delivery event. Sentence 197 reports a routine delivery event. Sentence 198 reports a routine delivery event. Sentence 199 reports a routine delivery event. Sentence 200 reports a routine delivery event. Sentence 201 reports a routine delivery event. Sentence 202 reports a routine delivery event. Sentence 203 reports a routine delivery event. Sentence 204 reports a routine delivery event. Sentence 205 reports a routine delivery event. Sentence 206 reports a routine delivery event. Sentence 207 reports a routine delivery event. Sentence 208 reports a routine delivery event. Sentence 209 reports a routine delivery event. Sentence 210 reports a routine delivery event. Sentence 211 reports a routine delivery event. Sentence 212 reports a routine delivery event. Sentence 213 reports a routine delivery event. Sentence 214 reports a routine delivery event. Sentence 215 reports a routine delivery event. Sentence 216 reports a routine delivery event. Sentence 217 reports a routine delivery event. Sentence 218 reports a routine delivery event. Sentence 219 reports a routine delivery event."}, "questions": {"claims": {"type": "span", "instructions": "Which spans mention a delivery?", "criteria": {"delivery": "a delivery event"}, "over": "request"}}}, "rows": [{"head": "broad", "head_reason": "default", "label_alias": null, "n_tokens": 2385, "seg_lens": {"user": 2310}}], "head_calls": [{"kind": "span", "head": "broad", "words": "s0_words", "labels": "s0_labels", "logits": [[-4.5031867027282715], [-5.110243797302246], [-4.144986152648926], [-3.987896680831909], [-3.73734712600708], [-3.6736156940460205], [-4.206828594207764], [-5.193665027618408], [-4.111611843109131], [-4.483518600463867], [-3.8180947303771973], [-3.600126266479492], [-3.552110195159912], [-3.0163097381591797], [-3.220355272293091], [-5.011423110961914], [-3.5230727195739746], [-3.642819404602051], [-3.5962486267089844], [-3.3500521183013916], [-2.7229509353637695], [-2.3767755031585693], [-2.9794607162475586], [-5.203192234039307], [-3.292698383331299], [-3.7594189643859863], [-3.732757329940796], [-3.4012789726257324], [-2.547316789627075], [-2.113595485687256], [-2.5140676498413086], [-5.272742748260498], [-3.1617307662963867], [-3.709965705871582], [-3.7790520191192627], [-3.5370266437530518], [-2.287034511566162], [-1.8732467889785767], [-2.1769633293151855], [-4.8842363357543945], [-3.1498923301696777], [-3.8360047340393066], [-3.7817606925964355], [-3.674208164215088], [-2.4299895763397217], [-1.9481273889541626], [-2.2568113803863525], [-4.809032440185547], [-3.203739643096924], [-3.894256114959717], [-3.7197253704071045], [-3.811279296875], [-2.5057950019836426], [-2.0324835777282715], [-2.2377896308898926], [-4.79273796081543], [-3.167496681213379], [-3.6729907989501953], [-3.7409868240356445], [-3.8900985717773438], [-2.517928123474121], [-2.085160255432129], [-2.204406499862671], [-4.476906776428223], [-3.0247135162353516], [-3.3603739738464355], [-3.733729362487793], [-4.003390312194824], [-2.5111379623413086], [-2.1227169036865234], [-2.1616055965423584], [-4.280056476593018], [-3.1476824283599854], [-3.284939765930176], [-3.5280537605285645], [-4.004054069519043], [-2.479815721511841], [-2.0832598209381104], [-2.1735243797302246], [-4.028883934020996], [-2.8546695709228516], [-5.135047912597656], [-3.472463607788086], [-4.031004905700684], [-2.346458673477173], [-2.040801525115967], [-2.221790075302124], [-4.339855194091797], [-3.024780511856079], [-3.8415279388427734], [-3.708129644393921], [-4.184869766235352], [-2.4905357360839844], [-2.205165147781372], [-2.2500195503234863], [-3.830564260482788], [-2.9054484367370605], [-3.8330211639404297], [-3.538437604904175], [-4.1763081550598145], [-2.4911670684814453], [-2.208170175552368], [-2.2881338596343994], [-3.502016544342041], [-2.874577522277832], [-3.443112850189209], [-3.5297534465789795], [-4.116558074951172], [-2.3881592750549316], [-2.120121717453003], [-2.2281429767608643], [-3.5904197692871094], [-2.7474117279052734], [-3.0000109672546387], [-3.4756243228912354], [-4.057490348815918], [-2.3550848960876465], [-2.1465511322021484], [-2.296060562133789], [-3.423558235168457], [-2.7227587699890137], [-2.926314115524292], [-3.155421018600464], [-3.8750150203704834], [-2.317171335220337], [-2.1981804370880127], [-2.2720916271209717], [-3.3031227588653564], [-2.679434299468994], [-2.67433762550354], [-3.415773391723633], [-4.033443927764893], [-2.472675323486328], [-2.3977558612823486], [-2.395137071609497], [-3.388845443725586], [-2.8307366371154785], [-3.1820380687713623], [-3.379044771194458], [-4.130788803100586], [-2.4755055904388428], [-2.3602936267852783], [-2.4117419719696045], [-3.2049059867858887], [-2.7125535011291504], [-2.8958773612976074], [-3.4741060733795166], [-4.144061088562012], [-2.7254254817962646], [-2.6585144996643066], [-2.6285369396209717], [-3.166553020477295], [-2.790513515472412], [-3.266775131225586], [-3.0983612537384033], [-3.816937208175659], [-2.5469412803649902], [-2.4206464290618896], [-2.4263386726379395], [-3.000765562057495], [-2.5916852951049805], [-4.455799579620361], [-3.155489444732666], [-3.771697998046875], [-2.603659152984619], [-2.567476272583008], [-2.458606719970703], [-3.4737415313720703], [-2.750251293182373], [-3.886612892150879], [-3.3631725311279297], [-4.0264105796813965], [-2.6563808917999268], [-2.6142210960388184], [-2.587613105773926], [-3.447003126144409], [-2.845402956008911], [-4.080188274383545], [-3.441471576690674], [-4.056282997131348], [-2.76839542388916], [-2.730438709259033], [-2.755331516265869], [-3.366610527038574], [-2.8473312854766846], [-3.438572406768799], [-3.302333354949951], [-4.012025356292725], [-2.7066521644592285], [-2.6642379760742188], [-2.733356475830078], [-3.3502445220947266], [-2.7474634647369385], [-2.9872095584869385], [-3.160595178604126], [-3.956681966781616], [-2.6496293544769287], [-2.6137173175811768], [-2.610538959503174], [-3.3916871547698975], [-2.761270046234131], [-3.066728115081787], [-3.053175687789917], [-3.845890998840332], [-2.5704703330993652], [-2.482295513153076], [-2.377286911010742], [-3.45330810546875], [-2.7584733963012695], [-2.7889020442962646], [-3.3100645542144775], [-4.016964912414551], [-2.7278318405151367], [-2.716367721557617], [-2.547883987426758], [-3.494657039642334], [-2.91843581199646], [-3.096928119659424], [-3.195547103881836], [-4.078264236450195], [-2.727675199508667], [-2.701345682144165], [-2.6138572692871094], [-3.2675936222076416], [-2.798149585723877], [-2.747892141342163], [-3.3961918354034424], [-4.055111885070801], [-2.8253610134124756], [-2.8022124767303467], [-2.7324352264404297], [-3.2563395500183105], [-2.9289698600769043], [-3.161409378051758], [-3.0892155170440674], [-3.8805861473083496], [-2.899251937866211], [-2.681178569793701], [-2.6384615898132324], [-3.1615285873413086], [-2.702582359313965], [-3.9901344776153564], [-3.0604772567749023], [-3.699664831161499], [-2.8672633171081543], [-2.5609076023101807], [-2.386392116546631], [-3.450453042984009], [-2.62668514251709], [-3.6577517986297607], [-3.306281328201294], [-3.9494171142578125], [-2.9576709270477295], [-2.840526819229126], [-2.739490032196045], [-3.390897035598755], [-2.76428484916687], [-3.4731738567352295], [-3.4074525833129883], [-4.002161026000977], [-3.1519720554351807], [-3.1033413410186768], [-2.934561252593994], [-3.2542884349823], [-2.830474853515625], [-3.329129695892334], [-3.327678680419922], [-3.939371347427368], [-3.1112637519836426], [-3.0389485359191895], [-2.9735100269317627], [-3.2114834785461426], [-2.697648286819458], [-2.960087299346924], [-3.382797956466675], [-3.8711726665496826], [-3.038339853286743], [-2.965508222579956], [-2.9370248317718506], [-3.130129814147949], [-2.655726432800293], [-2.828805923461914], [-3.200847864151001], [-3.733997344970703], [-2.9403762817382812], [-2.7187857627868652], [-2.602847099304199], [-3.108339548110962], [-2.5637216567993164], [-2.5126726627349854], [-3.274972915649414], [-3.8162355422973633], [-3.046719551086426], [-2.9648404121398926], [-2.751836061477661], [-3.1509900093078613], [-2.706282377243042], [-2.734219551086426], [-3.132106065750122], [-3.790893077850342], [-2.8981921672821045], [-2.8040385246276855], [-2.6840896606445312], [-2.9882516860961914], [-2.5822699069976807], [-2.5057430267333984], [-3.268061876296997], [-3.869846820831299], [-2.9899659156799316], [-2.91151762008667], [-2.740999937057495], [-2.9471120834350586], [-2.655282974243164], [-2.7110507488250732], [-3.1046581268310547], [-3.7542057037353516], [-3.0571537017822266], [-2.7020797729492188], [-2.583533763885498], [-2.841658115386963], [-2.4317500591278076], [-4.535123825073242], [-3.0092480182647705], [-3.6236672401428223], [-2.910571813583374], [-2.554750919342041], [-2.4116175174713135], [-2.9890851974487305], [-2.3250484466552734], [-3.066993236541748], [-3.2430224418640137], [-3.879951000213623], [-3.025676965713501], [-2.862931489944458], [-2.7611653804779053], [-3.0940093994140625], [-2.6080024242401123], [-3.0767292976379395], [-3.3543283939361572], [-3.919978618621826], [-3.2318620681762695], [-3.0806479454040527], [-2.905336856842041], [-2.9363017082214355], [-2.6144962310791016], [-3.106537342071533], [-3.303500175476074], [-3.8722317218780518], [-3.1726021766662598], [-2.9862594604492188], [-2.9007675647735596], [-2.8748927116394043], [-2.5799238681793213], [-2.9275498390197754], [-3.232797145843506], [-3.827728509902954], [-3.152682304382324], [-2.977900981903076], [-2.9310860633850098], [-2.8876113891601562], [-2.569924831390381], [-2.937041759490967], [-3.0091605186462402], [-3.7201836109161377], [-3.0235109329223633], [-2.6895289421081543], [-2.583458185195923], [-2.8790180683135986], [-2.4152817726135254], [-2.6710853576660156], [-3.0919342041015625], [-3.7484679222106934], [-3.072103977203369], [-2.8091025352478027], [-2.693146228790283], [-3.005251884460449], [-2.598769187927246], [-2.8888659477233887], [-3.0383265018463135], [-3.836486339569092], [-3.0531792640686035], [-2.892537832260132], [-2.8254306316375732], [-2.7800028324127197], [-2.6048214435577393], [-2.6699588298797607], [-3.1474015712738037], [-3.8490548133850098], [-3.0975375175476074], [-2.904897689819336], [-2.8561220169067383], [-2.8514044284820557], [-2.778257131576538], [-3.0959577560424805], [-3.0772719383239746], [-3.7788593769073486], [-3.11238169670105], [-2.7520251274108887], [-2.746412515640259], [-2.793661594390869], [-2.510711669921875], [-4.295222282409668], [-3.027052402496338], [-3.6460118293762207], [-3.016228199005127], [-2.6630256175994873], [-2.5573415756225586], [-3.402331829071045], [-2.674072742462158], [-3.0486929416656494], [-3.2769057750701904], [-3.9950859546661377], [-3.2196600437164307], [-3.1050453186035156], [-2.969278335571289], [-3.4232914447784424], [-2.7894840240478516], [-3.3626346588134766], [-3.4933228492736816], [-4.0912885665893555], [-3.417227268218994], [-3.370995044708252], [-3.2418053150177], [-3.2631843090057373], [-2.9207100868225098], [-3.2789716720581055], [-3.4469523429870605], [-4.027719020843506], [-3.39505672454834], [-3.289363145828247], [-3.229820966720581], [-3.270352363586426], [-2.843055486679077], [-3.1328699588775635], [-3.3853769302368164], [-3.9208431243896484], [-3.294778823852539], [-3.2019693851470947], [-3.1809754371643066], [-3.2545509338378906], [-2.8866958618164062], [-3.1352782249450684], [-3.108180522918701], [-3.753723621368408], [-3.1459057331085205], [-2.8914527893066406], [-2.8299739360809326], [-3.1975159645080566], [-2.6935720443725586], [-2.7886619567871094], [-3.180452823638916], [-3.7987427711486816], [-3.1926255226135254], [-3.072432518005371], [-2.8815739154815674], [-3.260772705078125], [-2.712655544281006], [-3.047394037246704], [-3.0763325691223145], [-3.803225040435791], [-3.1564512252807617], [-3.0740487575531006], [-2.962618350982666], [-3.112096071243286], [-2.7545487880706787], [-2.8254482746124268], [-3.3255155086517334], [-3.8312597274780273], [-3.1985819339752197], [-3.121063709259033], [-3.048802614212036], [-3.320197343826294], [-2.950960636138916], [-3.1894898414611816], [-3.101832866668701], [-3.756287097930908], [-3.243901491165161], [-2.9573025703430176], [-2.913924217224121], [-3.0539398193359375], [-2.553288459777832], [-3.984586238861084], [-3.009453773498535], [-3.5376837253570557], [-3.041944742202759], [-2.6505439281463623], [-2.4188127517700195], [-3.196794033050537], [-2.3315012454986572], [-3.0709123611450195], [-3.294649124145508], [-3.862866163253784], [-3.3014674186706543], [-3.0914626121520996], [-2.974423408508301], [-3.0324313640594482], [-2.518920660018921], [-3.0278146266937256], [-3.433835506439209], [-3.926903486251831], [-3.396409511566162], [-3.2318758964538574], [-3.107344627380371], [-2.9854376316070557], [-2.634305715560913], [-3.1417489051818848], [-3.371551990509033], [-3.8594484329223633], [-3.305750846862793], [-3.0833020210266113], [-3.0137805938720703], [-2.866870403289795], [-2.5058717727661133], [-2.899845838546753], [-3.3569135665893555], [-3.799060821533203], [-3.2622642517089844], [-3.0580050945281982], [-3.0193142890930176], [-2.922239303588867], [-2.5154945850372314], [-2.8869235515594482], [-3.2986326217651367], [-3.806964635848999], [-3.2242796421051025], [-2.9691548347473145], [-2.862969398498535], [-2.972041368484497], [-2.5057995319366455], [-2.7680773735046387], [-3.3304643630981445], [-3.926485538482666], [-3.3832781314849854], [-3.2387876510620117], [-3.0035510063171387], [-2.8867430686950684], [-2.461895227432251], [-2.874433994293213], [-3.3034188747406006], [-3.9354259967803955], [-3.381701946258545], [-3.2519869804382324], [-3.1549954414367676], [-2.804753303527832], [-2.5874733924865723], [-2.8049473762512207], [-3.5051255226135254], [-3.9563920497894287], [-3.393949031829834], [-3.3012351989746094], [-3.221285104751587], [-2.7689013481140137], [-2.730234146118164], [-3.238210678100586], [-3.308086395263672], [-3.8236443996429443], [-3.402456045150757], [-3.1107304096221924], [-3.0275890827178955], [-2.627351999282837], [-2.348923444747925], [-3.7539916038513184], [-3.229466438293457], [-3.6952037811279297], [-3.2762489318847656], [-3.024171829223633], [-2.834960460662842], [-2.8530304431915283], [-2.4097354412078857], [-3.1349966526031494], [-3.4111833572387695], [-3.9793858528137207], [-3.536642551422119], [-3.3145809173583984], [-3.2437994480133057], [-2.865528106689453], [-2.6916725635528564], [-3.1649131774902344], [-3.4530296325683594], [-3.9442942142486572], [-3.532402992248535], [-3.293041706085205], [-3.249469757080078], [-2.658794403076172], [-2.647170066833496], [-3.296597480773926], [-3.4606099128723145], [-3.902078628540039], [-3.47898006439209], [-3.0968685150146484], [-3.1101865768432617], [-2.6883492469787598], [-2.625246047973633], [-3.044886589050293], [-3.417605400085449], [-3.8265740871429443], [-3.3930106163024902], [-3.050856590270996], [-3.0786080360412598], [-2.7162671089172363], [-2.614358901977539], [-3.029118061065674], [-3.120817184448242], [-3.6690568923950195], [-3.1802303791046143], [-2.7332053184509277], [-2.6569743156433105], [-2.8592584133148193], [-2.5555052757263184], [-2.794926643371582], [-3.329676866531372], [-3.843104600906372], [-3.431717872619629], [-3.115694522857666], [-3.006934404373169], [-2.8927161693573], [-2.771969795227051], [-3.0985262393951416], [-3.3627560138702393], [-4.031410217285156], [-3.5628819465637207], [-3.2239112854003906], [-3.2190001010894775], [-2.8411803245544434], [-2.8179051876068115], [-2.8407933712005615], [-3.4907772541046143], [-3.9198992252349854], [-3.513185501098633], [-3.153759241104126], [-3.1190171241760254], [-2.955325126647949], [-3.0180397033691406], [-3.1650242805480957], [-3.3932175636291504], [-3.864010810852051], [-3.5058014392852783], [-3.060795783996582], [-3.019566774368286], [-2.8086724281311035], [-2.557786464691162], [-3.758640766143799], [-3.1801061630249023], [-3.586951971054077], [-3.2146682739257812], [-2.7669260501861572], [-2.5549402236938477], [-3.1608564853668213], [-2.6960854530334473], [-3.0531997680664062], [-3.4267263412475586], [-3.8595032691955566], [-3.4517011642456055], [-3.0779123306274414], [-2.9841978549957275], [-3.2594261169433594], [-2.7941360473632812], [-3.2054953575134277], [-3.5676653385162354], [-3.884894609451294], [-3.4993929862976074], [-3.1758859157562256], [-3.133627414703369], [-3.1515846252441406], [-2.8820602893829346], [-3.079322576522827], [-3.529384136199951], [-3.859815835952759], [-3.4623498916625977], [-3.101991891860962], [-3.114994764328003], [-2.927560806274414], [-2.7071175575256348], [-2.7987937927246094], [-3.44907808303833], [-3.7480454444885254], [-3.3471152782440186], [-3.0190157890319824], [-3.004462480545044], [-2.862771987915039], [-2.634998321533203], [-2.7141354084014893], [-3.160027265548706], [-3.5566444396972656], [-3.1041688919067383], [-2.692730188369751], [-2.61314058303833], [-2.9437198638916016], [-2.607759475708008], [-2.593719005584717], [-3.4252164363861084], [-3.802501678466797], [-3.3863446712493896], [-3.0950379371643066], [-2.9868483543395996], [-2.951575994491577], [-2.7338995933532715], [-2.80823016166687], [-3.424727439880371], [-3.9431984424591064], [-3.507297992706299], [-3.2219784259796143], [-3.2142210006713867], [-2.9190421104431152], [-2.769955635070801], [-2.6327760219573975], [-3.490687370300293], [-3.9137845039367676], [-3.5170719623565674], [-3.219968557357788], [-3.2301931381225586], [-2.9572811126708984], [-3.012561798095703], [-3.081015110015869], [-3.3550243377685547], [-3.7844724655151367], [-3.449817419052124], [-3.062757730484009], [-3.004728317260742], [-2.710850238800049], [-2.4290623664855957], [-3.6957039833068848], [-3.1508092880249023], [-3.5531063079833984], [-3.2513701915740967], [-2.8134853839874268], [-2.709958076477051], [-2.5848147869110107], [-2.291872501373291], [-2.8649346828460693], [-3.4370932579040527], [-3.8512067794799805], [-3.552619457244873], [-3.2169528007507324], [-3.2407848834991455], [-2.6746294498443604], [-2.599689483642578], [-2.9285759925842285], [-3.437777042388916], [-3.8265836238861084], [-3.537773370742798], [-3.2016634941101074], [-3.2524704933166504], [-2.658144474029541], [-2.762173652648926], [-3.0995121002197266], [-3.264481544494629], [-3.6513803005218506], [-3.368347406387329], [-2.944382905960083], [-3.054576873779297], [-2.4868855476379395], [-2.5668814182281494], [-2.783412456512451], [-3.181342124938965], [-3.5173301696777344], [-3.238799571990967], [-2.885099172592163], [-2.977668523788452], [-2.458744764328003], [-2.440518856048584], [-2.627070903778076], [-2.774517059326172], [-3.2667224407196045], [-2.907304286956787], [-2.479236602783203], [-2.4501259326934814], [-2.5030758380889893], [-2.359438419342041], [-2.4545862674713135], [-2.8487112522125244], [-3.347644567489624], [-3.076511859893799], [-2.7382187843322754], [-2.609323263168335], [-2.581723213195801], [-2.5250673294067383], [-2.7213134765625], [-3.008920192718506], [-3.6400961875915527], [-3.264674425125122], [-2.9725921154022217], [-2.9420738220214844], [-2.6470398902893066], [-2.631930112838745], [-2.5849618911743164], [-3.1815543174743652], [-3.645673990249634], [-3.3037900924682617], [-3.0025718212127686], [-2.8881304264068604], [-2.9212207794189453], [-2.870828151702881], [-3.036044120788574], [-3.0864479541778564], [-3.6391990184783936], [-3.3263533115386963], [-2.934431552886963], [-2.643834114074707], [-2.8082163333892822], [-2.2416751384735107], [-4.997744560241699], [-2.641695737838745], [-3.1480512619018555], [-2.7309954166412354], [-2.342461585998535], [-2.134186267852783], [-3.3599090576171875], [-2.389754295349121], [-3.8489699363708496], [-3.481875419616699], [-3.835409641265869], [-3.2867536544799805], [-3.135392189025879], [-3.1841211318969727], [-3.2435195446014404], [-2.6728482246398926], [-3.565577983856201], [-3.4907386302948], [-3.817366361618042], [-3.4380345344543457], [-3.2320961952209473], [-3.1732540130615234], [-3.00285005569458], [-2.7677855491638184], [-3.522904634475708], [-3.5236949920654297], [-3.7747085094451904], [-3.4220614433288574], [-3.1458094120025635], [-3.1818671226501465], [-2.9903762340545654], [-2.6828465461730957], [-3.038447856903076], [-3.425595760345459], [-3.6188178062438965], [-3.290921211242676], [-3.036555290222168], [-3.1029934883117676], [-2.7108778953552246], [-2.5454444885253906], [-2.9284615516662598], [-3.0249733924865723], [-3.4180800914764404], [-3.0599350929260254], [-2.679440975189209], [-2.603497266769409], [-2.686664581298828], [-2.444148302078247], [-2.622375011444092], [-3.2352237701416016], [-3.600078821182251], [-3.2915329933166504], [-3.012944221496582], [-2.9030768871307373], [-2.9213004112243652], [-2.5697474479675293], [-3.1269338130950928], [-3.297281265258789], [-3.792901039123535], [-3.4313840866088867], [-3.1427531242370605], [-3.144068479537964], [-2.790400981903076], [-2.6738078594207764], [-2.851374626159668], [-3.3939590454101562], [-3.8001742362976074], [-3.471911668777466], [-3.185490131378174], [-3.113187551498413], [-2.9338226318359375], [-2.7795844078063965], [-3.562433958053589], [-3.232546091079712], [-3.7366745471954346], [-3.478037118911743], [-3.048588991165161], [-2.967127799987793], [-2.7707386016845703], [-2.2879397869110107], [-3.0585551261901855], [-3.2217118740081787], [-3.562098741531372], [-3.3006858825683594], [-2.8595352172851562], [-2.712482213973999], [-3.0600295066833496], [-2.4890568256378174], [-3.295520782470703], [-3.5017483234405518], [-3.851046323776245], [-3.6015026569366455], [-3.247631311416626], [-3.267113208770752], [-2.940824270248413], [-2.527393341064453], [-3.2423627376556396], [-3.4514875411987305], [-3.772226572036743], [-3.578272819519043], [-3.2231717109680176], [-3.2506842613220215], [-2.684187412261963], [-2.6430392265319824], [-3.320096492767334], [-3.3535313606262207], [-3.661318778991699], [-3.4195094108581543], [-2.977952003479004], [-3.041194438934326], [-2.5273337364196777], [-2.498664379119873], [-2.8820204734802246], [-3.3365023136138916], [-3.584920644760132], [-3.37117338180542], [-2.966648817062378], [-3.056631565093994], [-2.4598677158355713], [-2.290769338607788], [-2.8910064697265625], [-2.918410301208496], [-3.323519706726074], [-3.0374629497528076], [-2.551300525665283], [-2.5643091201782227], [-2.3947341442108154], [-2.2270114421844482], [-2.5369584560394287], [-3.1112887859344482], [-3.514636754989624], [-3.2796175479888916], [-2.900486946105957], [-2.866894245147705], [-2.6816632747650146], [-2.5202133655548096], [-2.976116180419922], [-3.1536645889282227], [-3.724351406097412], [-3.412240505218506], [-3.0115339756011963], [-3.0364625453948975], [-2.5309927463531494], [-2.5785560607910156], [-2.7888803482055664], [-3.5357282161712646], [-3.8761041164398193], [-3.5989294052124023], [-3.22843074798584], [-3.239339590072632], [-2.9750654697418213], [-2.8486084938049316], [-3.531400203704834], [-3.2763261795043945], [-3.730445384979248], [-3.472618579864502], [-3.0076005458831787], [-2.977762222290039], [-2.5539000034332275], [-2.3810126781463623], [-3.1470108032226562], [-3.0243921279907227], [-3.3882980346679688], [-3.1547930240631104], [-2.6889665126800537], [-2.5054664611816406], [-2.591777801513672], [-2.3483340740203857], [-2.9539666175842285], [-3.4012956619262695], [-3.7214832305908203], [-3.4761898517608643], [-3.0995676517486572], [-3.0691843032836914], [-2.893742322921753], [-2.655484199523926], [-3.3345892429351807], [-3.4373271465301514], [-3.7741916179656982], [-3.54217529296875], [-3.1460137367248535], [-3.07498836517334], [-2.7824041843414307], [-2.734251022338867], [-3.4469966888427734], [-3.4238409996032715], [-3.684804677963257], [-3.426290512084961], [-2.985167980194092], [-2.9843270778656006], [-2.5652599334716797], [-2.5611166954040527], [-2.931624412536621], [-3.278148651123047], [-3.544569492340088], [-3.3001651763916016], [-2.9162063598632812], [-2.938485622406006], [-2.3827269077301025], [-2.4239583015441895], [-2.934877872467041], [-2.9046950340270996], [-3.320268392562866], [-3.002073287963867], [-2.5128841400146484], [-2.4146275520324707], [-2.4223287105560303], [-2.337623357772827], [-2.4254393577575684], [-3.146881103515625], [-3.481025218963623], [-3.182342052459717], [-2.804279327392578], [-2.708815574645996], [-2.638535976409912], [-2.4622440338134766], [-2.932757616043091], [-3.2871525287628174], [-3.7110848426818848], [-3.335937023162842], [-2.963754177093506], [-2.923792839050293], [-2.611741542816162], [-2.513537645339966], [-2.7864925861358643], [-3.4225645065307617], [-3.716189384460449], [-3.386918783187866], [-3.023972511291504], [-2.9621119499206543], [-2.74324369430542], [-2.672879457473755], [-3.4000864028930664], [-3.370421886444092], [-3.7394304275512695], [-3.4786009788513184], [-3.0315756797790527], [-2.902592658996582], [-2.6288487911224365], [-2.292703151702881], [-3.1351287364959717], [-3.217012405395508], [-3.4396896362304688], [-3.177994966506958], [-2.705684185028076], [-2.4454195499420166], [-2.8730030059814453], [-2.399569034576416], [-3.191361427307129], [-3.5410070419311523], [-3.7142417430877686], [-3.5119471549987793], [-3.1266396045684814], [-3.0673415660858154], [-3.0538177490234375], [-2.562906265258789], [-3.2854886054992676], [-3.495687484741211], [-3.7432548999786377], [-3.579080581665039], [-3.1908493041992188], [-3.098299026489258], [-2.9286046028137207], [-2.6609818935394287], [-3.617393970489502], [-3.4753806591033936], [-3.6897430419921875], [-3.483457088470459], [-3.0658950805664062], [-3.0838754177093506], [-2.6606664657592773], [-2.507230281829834], [-2.8288304805755615], [-3.4191102981567383], [-3.53617000579834], [-3.3194994926452637], [-2.94797420501709], [-2.9721262454986572], [-2.521807909011841], [-2.3236160278320312], [-2.742223024368286], [-2.9646549224853516], [-3.292872667312622], [-3.008470058441162], [-2.5324153900146484], [-2.458242654800415], [-2.449246883392334], [-2.3093013763427734], [-2.345308780670166], [-3.118659496307373], [-3.452319622039795], [-3.2350616455078125], [-2.8765339851379395], [-2.8064754009246826], [-2.757309675216675], [-2.4547152519226074], [-2.934093952178955], [-3.2687392234802246], [-3.6503047943115234], [-3.3517472743988037], [-2.9860727787017822], [-2.954586982727051], [-2.7319326400756836], [-2.5278379917144775], [-2.7103679180145264], [-3.3690080642700195], [-3.684565544128418], [-3.4038496017456055], [-3.0428578853607178], [-2.9786016941070557], [-2.848008871078491], [-2.7249393463134766], [-3.282515287399292], [-3.24224853515625], [-3.6659255027770996], [-3.431363105773926], [-2.9756247997283936], [-2.874397039413452], [-2.545549154281616], [-2.3420560359954834], [-3.1602134704589844], [-3.0126850605010986], [-3.371856212615967], [-3.1283516883850098], [-2.704622507095337], [-2.5314502716064453], [-2.3329994678497314], [-2.181795358657837], [-2.58194637298584], [-3.260038137435913], [-3.6390750408172607], [-3.443636655807495], [-3.073936939239502], [-2.9382288455963135], [-2.6380982398986816], [-2.5467429161071777], [-2.844155788421631], [-3.2590761184692383], [-3.596987724304199], [-3.397275447845459], [-3.023470878601074], [-2.9451024532318115], [-2.52231764793396], [-2.6014516353607178], [-2.926614761352539], [-3.247098445892334], [-3.567565679550171], [-3.323537826538086], [-2.9038686752319336], [-2.8909566402435303], [-2.351017951965332], [-2.464599609375], [-2.618572473526001], [-3.0008318424224854], [-3.3584275245666504], [-3.1483657360076904], [-2.755417823791504], [-2.769083023071289], [-2.257073163986206], [-2.2310619354248047], [-2.640090227127075], [-2.756155490875244], [-3.1554601192474365], [-2.88824462890625], [-2.422485828399658], [-2.361837863922119], [-2.1907122135162354], [-2.179175853729248], [-2.2744033336639404], [-2.80633807182312], [-3.259903907775879], [-3.0246200561523438], [-2.667954206466675], [-2.5753676891326904], [-2.435060977935791], [-2.3502426147460938], [-2.710244655609131], [-2.8579211235046387], [-3.442357063293457], [-3.114640712738037], [-2.7407212257385254], [-2.696715831756592], [-2.4022979736328125], [-2.521280288696289], [-2.604094982147217], [-3.130936861038208], [-3.5247814655303955], [-3.2162532806396484], [-2.8506386280059814], [-2.8146095275878906], [-2.454864740371704], [-2.6955432891845703], [-3.1879711151123047], [-3.0694684982299805], [-3.5853686332702637], [-3.3176541328430176], [-2.8888187408447266], [-2.7516441345214844], [-2.3814849853515625], [-2.3203933238983154], [-3.332015037536621], [-2.7918577194213867], [-3.221040964126587], [-2.9509520530700684], [-2.515359401702881], [-2.240274429321289], [-2.710294723510742], [-2.4265146255493164], [-2.883511543273926], [-3.2317566871643066], [-3.6661629676818848], [-3.5157132148742676], [-3.154888868331909], [-3.022998332977295], [-2.9311633110046387], [-2.751699924468994], [-3.177746295928955], [-3.331028938293457], [-3.720273971557617], [-3.587538719177246], [-3.22223162651062], [-3.1272711753845215], [-2.894808292388916], [-2.8703219890594482], [-3.270787000656128], [-3.496811866760254], [-3.8052453994750977], [-3.6398842334747314], [-3.223083972930908], [-3.16959810256958], [-2.747295379638672], [-2.7526063919067383], [-2.8468546867370605], [-3.4252841472625732], [-3.6472742557525635], [-3.495584011077881], [-3.138165235519409], [-3.1507816314697266], [-2.7097253799438477], [-2.659017562866211], [-3.0421667098999023], [-2.8085150718688965], [-3.2467291355133057], [-3.040405750274658], [-2.6211581230163574], [-2.5361087322235107], [-2.6175460815429688], [-2.581294536590576], [-2.397932529449463], [-2.9661824703216553], [-3.3943426609039307], [-3.243824005126953], [-2.9330332279205322], [-2.80828595161438], [-2.8558685779571533], [-2.6549510955810547], [-3.0997235774993896], [-3.0882811546325684], [-3.6176364421844482], [-3.3678598403930664], [-3.026418685913086], [-2.9394497871398926], [-2.830770969390869], [-2.734785556793213], [-2.902885675430298], [-3.2930002212524414], [-3.66241717338562], [-3.429262161254883], [-3.0735578536987305], [-2.9926297664642334], [-3.0805516242980957], [-2.9205784797668457], [-3.3658926486968994], [-3.105604887008667], [-3.6364636421203613], [-3.422640800476074], [-2.9676871299743652], [-2.8062093257904053], [-2.7586569786071777], [-2.4597091674804688], [-3.287386178970337], [-3.018235683441162], [-3.37151837348938], [-3.146873712539673], [-2.7184200286865234], [-2.4454243183135986], [-2.8317267894744873], [-2.5290777683258057], [-3.2094836235046387], [-3.467600107192993], [-3.767857789993286], [-3.556270122528076], [-3.2194480895996094], [-3.107192039489746], [-3.1031880378723145], [-2.7277469635009766], [-3.317214250564575], [-3.493439197540283], [-3.8041486740112305], [-3.6141438484191895], [-3.2926063537597656], [-3.185319662094116], [-3.009775161743164], [-2.8376364707946777], [-3.4733355045318604], [-3.4518654346466064], [-3.758373498916626], [-3.5416507720947266], [-3.151158571243286], [-3.1094655990600586], [-2.7530934810638428], [-2.712601661682129], [-2.882924795150757], [-3.415604829788208], [-3.637073040008545], [-3.41583251953125], [-3.0981369018554688], [-3.1043951511383057], [-2.6454334259033203], [-2.619405746459961], [-2.9913251399993896], [-3.238988161087036], [-3.572427988052368], [-3.2832419872283936], [-2.8783023357391357], [-2.7861523628234863], [-2.625786304473877], [-2.515207290649414], [-2.7463631629943848], [-3.264078140258789], [-3.646012306213379], [-3.442460775375366], [-3.14827823638916], [-3.039522886276245], [-2.7585182189941406], [-2.559166431427002], [-3.1382558345794678], [-3.3004353046417236], [-3.751485824584961], [-3.507065534591675], [-3.183985710144043], [-3.1453640460968018], [-2.795884609222412], [-2.618122100830078], [-2.8723905086517334], [-3.4296319484710693], [-3.7344579696655273], [-3.5080313682556152], [-3.1982951164245605], [-3.1656010150909424], [-2.798642635345459], [-2.762244462966919], [-3.383389949798584], [-3.3695316314697266], [-3.734748363494873], [-3.5395305156707764], [-3.1320111751556396], [-3.051684856414795], [-2.6936728954315186], [-2.4074368476867676], [-3.3344902992248535], [-3.378732919692993], [-3.669884204864502], [-3.4845118522644043], [-3.1243321895599365], [-2.9951229095458984], [-2.857006549835205], [-2.553025245666504], [-3.3101961612701416], [-3.5732264518737793], [-3.942887306213379], [-3.80142879486084], [-3.4761695861816406], [-3.379910469055176], [-3.03538179397583], [-2.753514528274536], [-3.60799503326416], [-3.595752477645874], [-3.8866052627563477], [-3.7812371253967285], [-3.44804310798645], [-3.3878116607666016], [-2.8401341438293457], [-2.8566417694091797], [-3.683255434036255], [-3.57861065864563], [-3.8425440788269043], [-3.7111306190490723], [-3.3018336296081543], [-3.2605128288269043], [-2.726048707962036], [-2.756875514984131], [-3.20404314994812], [-3.5732510089874268], [-3.794538974761963], [-3.668351888656616], [-3.2935688495635986], [-3.3062829971313477], [-2.6648151874542236], [-2.6641845703125], [-3.1933391094207764], [-3.196021556854248], [-3.561269521713257], [-3.377262592315674], [-2.9040286540985107], [-2.8485028743743896], [-2.5644679069519043], [-2.582756519317627], [-2.699636459350586], [-3.2496395111083984], [-3.633868932723999], [-3.520700693130493], [-3.156362771987915], [-3.066011667251587], [-2.8338332176208496], [-2.7110848426818848], [-3.353687286376953], [-3.2272744178771973], [-3.7708046436309814], [-3.6146883964538574], [-3.207472801208496], [-3.1550400257110596], [-2.812927722930908], [-2.7097978591918945], [-3.139752149581909], [-3.4301090240478516], [-3.7430801391601562], [-3.594717025756836], [-3.1768381595611572], [-3.1292457580566406], [-3.0190839767456055], [-2.91629695892334], [-3.683408498764038], [-3.314305543899536], [-3.774728298187256], [-3.642749786376953], [-3.16917085647583], [-3.0878570079803467], [-2.893242359161377], [-2.5743157863616943], [-3.8028323650360107], [-3.098750352859497], [-3.3611648082733154], [-3.240027666091919], [-2.735227584838867], [-2.503072738647461], [-2.9290966987609863], [-2.57271146774292], [-3.74971866607666], [-3.423412561416626], [-3.6844730377197266], [-3.6207494735717773], [-3.219633102416992], [-3.0850679874420166], [-3.2087368965148926], [-2.7718658447265625], [-3.8732666969299316], [-3.485156774520874], [-3.7062134742736816], [-3.646282196044922], [-3.2640552520751953], [-3.1448121070861816], [-3.0670127868652344], [-2.861415386199951], [-3.9854774475097656], [-3.397080421447754], [-3.652838945388794], [-3.554008960723877], [-3.1322507858276367], [-3.084904432296753], [-2.6851930618286133], [-2.702328681945801], [-3.2643210887908936], [-3.394766330718994], [-3.554903268814087], [-3.453718423843384], [-3.0947163105010986], [-3.1041760444641113], [-2.5907907485961914], [-2.5709614753723145], [-3.3679962158203125], [-3.025139331817627], [-3.296635627746582], [-3.118659734725952], [-2.6554973125457764], [-2.563690185546875], [-2.565920829772949], [-2.479855537414551], [-2.8879451751708984], [-3.2139058113098145], [-3.463848829269409], [-3.3464810848236084], [-2.982152223587036], [-2.894296646118164], [-2.7713301181793213], [-2.5889720916748047], [-3.588773250579834], [-3.3452858924865723], [-3.6651129722595215], [-3.4840786457061768], [-3.1079347133636475], [-3.0654337406158447], [-2.8054471015930176], [-2.6371593475341797], [-3.3767426013946533], [-3.4104013442993164], [-3.65047287940979], [-3.4797515869140625], [-3.106093645095825], [-3.064277172088623], [-2.870631456375122], [-2.808246374130249], [-3.8924951553344727], [-3.27691650390625], [-3.6088690757751465], [-3.4558262825012207], [-3.006011724472046], [-2.865206718444824], [-2.644439697265625], [-2.3443191051483154], [-3.8698482513427734], [-3.106802463531494], [-3.4090259075164795], [-3.270066499710083], [-2.792320489883423], [-2.6240859031677246], [-2.59503173828125], [-2.362605094909668], [-3.0311508178710938], [-3.367177724838257], [-3.664647102355957], [-3.599813938140869], [-3.2263283729553223], [-3.1302268505096436], [-2.7968785762786865], [-2.5684337615966797], [-3.0320584774017334], [-3.3292977809906006], [-3.6408352851867676], [-3.5774612426757812], [-3.1956794261932373], [-3.118408441543579], [-2.5956344604492188], [-2.6913490295410156], [-3.273282051086426], [-3.347306728363037], [-3.6383066177368164], [-3.5348868370056152], [-3.1098415851593018], [-3.0589096546173096], [-2.4412221908569336], [-2.612002372741699], [-2.6984872817993164], [-3.248682975769043], [-3.4714839458465576], [-3.371396541595459], [-3.0083765983581543], [-3.012059211730957], [-2.3238184452056885], [-2.4600045680999756], [-2.801676034927368], [-2.802344799041748], [-3.193342685699463], [-3.014861822128296], [-2.5161869525909424], [-2.401935577392578], [-2.209460735321045], [-2.346442699432373], [-2.3415822982788086], [-2.924604892730713], [-3.329289436340332], [-3.20627760887146], [-2.857482433319092], [-2.7188854217529297], [-2.546884536743164], [-2.500369071960449], [-3.142624616622925], [-3.0674219131469727], [-3.551912307739258], [-3.353278636932373], [-2.978517770767212], [-2.8710861206054688], [-2.5839295387268066], [-2.5982022285461426], [-2.926471710205078], [-3.2576489448547363], [-3.6080379486083984], [-3.4169139862060547], [-3.0052032470703125], [-2.878011703491211], [-2.8134853839874268], [-2.7970457077026367], [-3.495474338531494], [-3.2027883529663086], [-3.6364974975585938], [-3.4870433807373047], [-3.020742416381836], [-2.7501957416534424], [-2.6468324661254883], [-2.298635959625244], [-4.156946182250977], [-2.968641757965088], [-3.3462204933166504], [-3.127084970474243], [-2.6843204498291016], [-2.3027594089508057], [-2.9318113327026367], [-2.5740225315093994], [-3.423370838165283], [-3.61303973197937], [-3.8514952659606934], [-3.679452896118164], [-3.3802080154418945], [-3.2687602043151855], [-3.365943431854248], [-2.910097599029541], [-3.6455130577087402], [-3.637477159500122], [-3.847081422805786], [-3.7182812690734863], [-3.397934913635254], [-3.2744967937469482], [-3.2504637241363525], [-2.9266014099121094], [-3.2006959915161133], [-3.6920533180236816], [-3.8494811058044434], [-3.7047083377838135], [-3.3379201889038086], [-3.282905340194702], [-3.021963357925415], [-2.874441623687744], [-2.720808506011963], [-3.567777395248413], [-3.694612503051758], [-3.5549960136413574], [-3.209871292114258], [-3.210817337036133], [-2.743061065673828], [-2.67962646484375], [-2.606252670288086], [-3.210841655731201], [-3.421172618865967], [-3.2150726318359375], [-2.752065420150757], [-2.6300740242004395], [-2.668910503387451], [-2.6267058849334717], [-2.217494010925293], [-3.295365571975708], [-3.559401512145996], [-3.4092068672180176], [-3.0723962783813477], [-2.965762138366699], [-2.9024295806884766], [-2.7605128288269043], [-2.7977523803710938], [-3.3911468982696533], [-3.736747980117798], [-3.5499472618103027], [-3.1774675846099854], [-3.1003565788269043], [-2.877558708190918], [-2.741466999053955], [-2.5701205730438232], [-3.631467819213867], [-3.7947614192962646], [-3.6200127601623535], [-3.259577989578247], [-3.227219581604004], [-3.111893892288208], [-2.927877426147461], [-3.1118288040161133], [-3.4760541915893555], [-3.82798171043396], [-3.660681962966919], [-3.2026567459106445], [-3.06745982170105], [-2.9163174629211426], [-2.5107827186584473], [-2.7543580532073975], [-3.3247711658477783], [-3.575255870819092], [-3.4082813262939453], [-2.9560506343841553], [-2.7340991497039795], [-3.1581859588623047], [-2.6435978412628174], [-3.050903081893921], [-3.6423263549804688], [-3.855638265609741], [-3.780447483062744], [-3.420980453491211], [-3.3397090435028076], [-3.1515939235687256], [-2.776266574859619], [-3.2487196922302246], [-3.533071517944336], [-3.7864091396331787], [-3.7451727390289307], [-3.383924961090088], [-3.331754684448242], [-3.0292224884033203], [-2.916954517364502], [-3.3414742946624756], [-3.448403835296631], [-3.6725668907165527], [-3.590754508972168], [-3.1735265254974365], [-3.1659228801727295], [-2.853477716445923], [-2.8190603256225586], [-2.577502489089966], [-3.4731130599975586], [-3.618472099304199], [-3.5208663940429688], [-3.1676430702209473], [-3.2039833068847656], [-2.784921646118164], [-2.723785877227783], [-2.597787380218506], [-3.1204042434692383], [-3.379621982574463], [-3.2444405555725098], [-2.7961575984954834], [-2.737828493118286], [-2.7116189002990723], [-2.6735057830810547], [-2.406837224960327], [-3.253469944000244], [-3.5933496952056885], [-3.5163626670837402], [-3.1689529418945312], [-3.1269640922546387], [-2.9418206214904785], [-2.737407922744751], [-2.9498000144958496], [-3.344482660293579], [-3.7540316581726074], [-3.642815351486206], [-3.2756972312927246], [-3.2208633422851562], [-2.899384021759033], [-2.825432300567627], [-2.830540895462036], [-3.7058732509613037], [-3.9513602256774902], [-3.8286702632904053], [-3.4743030071258545], [-3.4488179683685303], [-3.3020009994506836], [-2.988687515258789], [-3.5051140785217285], [-3.5303125381469727], [-3.8839635848999023], [-3.780104160308838], [-3.3542416095733643], [-3.2982075214385986], [-2.9589450359344482]], "row": 0}, {"kind": "span", "head": "broad", "words": "s1_words", "labels": "s1_labels", "logits": [[-2.351999282836914], [-3.2861146926879883], [-2.071457624435425], [-2.4602627754211426], [-1.8625949621200562], [-2.047027826309204], [-2.3239541053771973], [-3.626887321472168], [-2.3483405113220215], [-2.9786715507507324], [-1.4222314357757568], [-1.844503402709961], [-1.3209636211395264], [-1.6392370462417603], [-1.9380539655685425], [-3.5257620811462402], [-2.2541913986206055], [-1.5854437351226807], [-1.782663106918335], [-2.515744924545288], [-1.4551222324371338], [-1.783217191696167], [-2.2827091217041016], [-3.5942647457122803], [-2.288733959197998], [-1.814456820487976], [-2.120316743850708], [-2.666901111602783], [-1.5888009071350098], [-1.8977081775665283], [-2.478454828262329], [-3.6913161277770996], [-2.448300361633301], [-2.0715279579162598], [-1.8338361978530884], [-2.383084297180176], [-1.6827189922332764], [-1.8510253429412842], [-2.4067676067352295], [-3.274144172668457], [-2.5227723121643066], [-2.184110164642334], [-2.219764471054077], [-2.5958662033081055], [-2.131826877593994], [-2.1779565811157227], [-2.542997121810913], [-3.3203463554382324], [-2.709588050842285], [-2.425050973892212], [-2.913311004638672], [-3.049304962158203], [-2.7997827529907227], [-2.8475773334503174], [-2.9129347801208496], [-3.5792112350463867], [-2.94014835357666], [-2.4511265754699707], [-2.6938092708587646], [-3.107053756713867], [-2.5412445068359375], [-2.543881893157959], [-2.8033976554870605], [-3.340123176574707], [-2.9487698078155518], [-2.258321762084961], [-2.7334959506988525], [-2.9328041076660156], [-2.574946165084839], [-2.386353015899658], [-2.7290751934051514], [-3.0835797786712646], [-2.7781600952148438], [-2.3306360244750977], [-2.894840717315674], [-3.1217055320739746], [-2.90492582321167], [-2.634967088699341], [-2.654693126678467], [-2.942322254180908], [-2.67533016204834], [-2.310062885284424], [-2.4764254093170166], [-2.982379913330078], [-2.726024866104126], [-2.5380630493164062], [-2.599724769592285], [-3.0781917572021484], [-2.67268443107605], [-2.4172849655151367], [-3.1995205879211426], [-3.387028694152832], [-3.1597583293914795], [-3.020578622817993], [-2.9183685779571533], [-3.3319602012634277], [-3.005150556564331], [-2.655348300933838], [-3.312901496887207], [-3.502896785736084], [-3.326502799987793], [-3.1171672344207764], [-2.977146863937378], [-3.0647552013397217], [-2.8119218349456787], [-2.6347837448120117], [-3.242929697036743], [-3.385951042175293], [-3.1226367950439453], [-2.92938232421875], [-2.8489975929260254], [-2.873330593109131], [-2.7924108505249023], [-2.5155136585235596], [-2.6101529598236084], [-2.8682923316955566], [-2.4759058952331543], [-2.32432222366333], [-2.509676456451416], [-2.738136053085327], [-2.5751655101776123], [-2.511366367340088], [-2.434668779373169], [-2.8093442916870117], [-2.4597370624542236], [-2.1925864219665527], [-2.265026569366455], [-2.606635808944702], [-2.569589614868164], [-2.4474518299102783], [-3.1315369606018066], [-3.330258846282959], [-3.1834590435028076], [-2.9461333751678467], [-2.7838711738586426], [-2.9982986450195312], [-2.8639140129089355], [-2.640132427215576], [-3.1092467308044434], [-3.345656394958496], [-3.0161149501800537], [-2.8642337322235107], [-2.882172107696533], [-2.9415836334228516], [-2.8425421714782715], [-2.5824174880981445], [-3.3017029762268066], [-3.4343185424804688], [-3.173388719558716], [-2.823185443878174], [-2.8955891132354736], [-2.9469070434570312], [-2.8801913261413574], [-2.73286771774292], [-3.1304006576538086], [-3.2778213024139404], [-3.086940050125122], [-2.7071034908294678], [-2.6154162883758545], [-2.465362310409546], [-2.4083869457244873], [-2.0807299613952637], [-3.0192887783050537], [-3.115584135055542], [-3.0074563026428223], [-2.6592860221862793], [-2.6600799560546875], [-2.6486566066741943], [-2.703080177307129], [-2.603400468826294], [-3.545090913772583], [-3.676462173461914], [-3.633878231048584], [-3.414682626724243], [-3.075254201889038], [-3.167475700378418], [-3.090211868286133], [-2.758925676345825], [-3.440603017807007], [-3.6364927291870117], [-3.616288900375366], [-3.5789742469787598], [-3.2785887718200684], [-3.303689956665039], [-3.055208683013916], [-2.8166756629943848], [-3.42702579498291], [-3.6088550090789795], [-3.3563108444213867], [-3.207009792327881], [-3.053516149520874], [-2.927600860595703], [-3.0033907890319824], [-2.7595555782318115], [-3.1866636276245117], [-3.2940454483032227], [-3.090822458267212], [-2.8461668491363525], [-2.8654515743255615], [-2.7394967079162598], [-2.736072540283203], [-2.7229604721069336], [-2.8041560649871826], [-3.0086841583251953], [-2.684821605682373], [-2.3853864669799805], [-2.4427380561828613], [-2.7070555686950684], [-2.6615591049194336], [-2.5376715660095215], [-3.3045644760131836], [-3.488103151321411], [-3.3025429248809814], [-3.088477849960327], [-2.889988899230957], [-3.0425522327423096], [-2.860687732696533], [-2.597816228866577], [-3.450432062149048], [-3.5520436763763428], [-3.3201825618743896], [-3.097409725189209], [-2.957402467727661], [-3.0354039669036865], [-2.813079595565796], [-2.44451904296875], [-3.3735146522521973], [-3.4592514038085938], [-3.2569403648376465], [-2.933631420135498], [-3.010178327560425], [-2.9987664222717285], [-2.9241271018981934], [-2.674637794494629], [-3.215228319168091], [-3.363966941833496], [-3.175990581512451], [-2.866244077682495], [-2.7509593963623047], [-2.674013137817383], [-2.572397232055664], [-2.283658981323242], [-3.0749685764312744], [-3.1150481700897217], [-2.978799343109131], [-2.6309356689453125], [-2.5329573154449463], [-2.73476243019104], [-2.6534435749053955], [-2.6418957710266113], [-3.441187858581543], [-3.460341453552246], [-3.332648754119873], [-2.9881787300109863], [-2.740382671356201], [-2.963399648666382], [-2.8248517513275146], [-2.6740872859954834], [-3.557119369506836], [-3.7387242317199707], [-3.677743911743164], [-3.3708038330078125], [-2.9666006565093994], [-3.1460883617401123], [-2.711873769760132], [-2.7425618171691895], [-3.531107187271118], [-3.722398042678833], [-3.4397730827331543], [-3.184204578399658], [-2.9603166580200195], [-3.077047109603882], [-3.134415864944458], [-2.9356689453125], [-3.5724692344665527], [-3.5326180458068848], [-3.233170509338379], [-2.9591586589813232], [-2.9584543704986572], [-2.974632501602173], [-2.9423813819885254], [-2.8333935737609863], [-3.0787997245788574], [-3.1498305797576904], [-2.8077139854431152], [-2.4857091903686523], [-2.46703839302063], [-2.836735248565674], [-2.764958381652832], [-2.547402858734131], [-3.6435322761535645], [-3.801168441772461], [-3.6398744583129883], [-3.389495372772217], [-3.0315918922424316], [-3.1958963871002197], [-3.0875203609466553], [-2.850008487701416], [-3.335598945617676], [-3.463289499282837], [-3.205141067504883], [-2.9718379974365234], [-2.910705327987671], [-2.921110153198242], [-2.839139461517334], [-2.5143721103668213], [-3.309785842895508], [-3.4399709701538086], [-3.2073137760162354], [-2.9310050010681152], [-2.9668822288513184], [-2.9172043800354004], [-2.955580711364746], [-2.693939685821533], [-3.0783305168151855], [-3.2795238494873047], [-3.0381486415863037], [-2.7074310779571533], [-2.557918071746826], [-2.4931492805480957], [-2.4969704151153564], [-2.3251161575317383], [-2.837054967880249], [-2.973512649536133], [-2.750908374786377], [-2.461597204208374], [-2.4562926292419434], [-2.539227247238159], [-2.573533535003662], [-2.724760055541992], [-3.291538715362549], [-3.3779118061065674], [-3.191070556640625], [-2.919306755065918], [-2.827911138534546], [-2.878228187561035], [-2.968708038330078], [-2.793642520904541], [-3.5484941005706787], [-3.569383144378662], [-3.4354677200317383], [-3.2140207290649414], [-3.2014548778533936], [-3.123699188232422], [-3.0571982860565186], [-2.901228427886963], [-3.4406657218933105], [-3.5101940631866455], [-3.1978225708007812], [-3.009692907333374], [-2.9862921237945557], [-2.8861544132232666], [-3.0008277893066406], [-2.766249895095825], [-3.2069902420043945], [-3.356257438659668], [-3.1127207279205322], [-2.8949034214019775], [-2.8696699142456055], [-2.690540313720703], [-2.7765419483184814], [-2.6917147636413574], [-2.7340762615203857], [-2.9686968326568604], [-2.6156744956970215], [-2.315448522567749], [-2.2582099437713623], [-2.394615411758423], [-2.486255645751953], [-2.343952178955078], [-3.055164098739624], [-3.2794339656829834], [-3.0376760959625244], [-2.8411521911621094], [-2.6768839359283447], [-2.6573009490966797], [-2.6563308238983154], [-2.4706344604492188], [-3.2290139198303223], [-3.3989450931549072], [-3.0206971168518066], [-2.849600315093994], [-2.7816994190216064], [-2.6770896911621094], [-2.661151170730591], [-2.394167900085449], [-3.1401875019073486], [-3.3224198818206787], [-3.045311450958252], [-2.839939832687378], [-2.8180136680603027], [-2.638383626937866], [-2.6572279930114746], [-2.4873173236846924], [-3.0559592247009277], [-3.2291769981384277], [-2.9053726196289062], [-2.619166851043701], [-2.4686074256896973], [-2.3695857524871826], [-2.337869882583618], [-2.242933750152588], [-2.7330517768859863], [-2.8996853828430176], [-2.6268019676208496], [-2.3521971702575684], [-2.3066904544830322], [-2.6438024044036865], [-2.53608775138855], [-2.5103135108947754], [-3.163695812225342], [-3.3452975749969482], [-3.0626981258392334], [-2.8006975650787354], [-2.7119393348693848], [-2.9658102989196777], [-2.805750846862793], [-2.6303815841674805], [-3.547795057296753], [-3.640038251876831], [-3.4043102264404297], [-3.1958556175231934], [-3.1924946308135986], [-3.1163721084594727], [-2.9519500732421875], [-2.875101089477539], [-3.4211292266845703], [-3.601597547531128], [-3.1529009342193604], [-3.010915994644165], [-2.983560085296631], [-3.046698570251465], [-2.965348720550537], [-2.825496196746826], [-3.3820505142211914], [-3.4926815032958984], [-3.196331024169922], [-3.004054307937622], [-2.985555410385132], [-2.8802576065063477], [-2.8922348022460938], [-2.76591157913208], [-2.930821180343628], [-3.125687599182129], [-2.8366730213165283], [-2.595963478088379], [-2.5801007747650146], [-2.6969470977783203], [-2.7062158584594727], [-2.5311808586120605], [-3.142136335372925], [-3.343869686126709], [-3.1298718452453613], [-2.9494383335113525], [-2.7738263607025146], [-2.8714072704315186], [-2.750135898590088], [-2.649747371673584], [-3.1227798461914062], [-3.373340606689453], [-3.063974618911743], [-2.8756344318389893], [-2.788553476333618], [-2.7882256507873535], [-2.6513912677764893], [-2.4259424209594727], [-3.170508861541748], [-3.378021240234375], [-3.057206869125366], [-2.862983226776123], [-2.926882743835449], [-2.9605343341827393], [-2.9230780601501465], [-2.6718642711639404], [-3.0482428073883057], [-3.2753374576568604], [-3.0337183475494385], [-2.752387046813965], [-2.640303611755371], [-2.592205047607422], [-2.532081127166748], [-2.4682106971740723], [-2.758899211883545], [-2.9040260314941406], [-2.738529682159424], [-2.456855058670044], [-2.3320372104644775], [-2.5157411098480225], [-2.503091335296631], [-2.7630975246429443], [-3.073822259902954], [-3.253864288330078], [-3.1154839992523193], [-2.8021180629730225], [-2.6192867755889893], [-2.6862831115722656], [-2.6489410400390625], [-2.689474582672119], [-3.2543535232543945], [-3.4074463844299316], [-3.316160202026367], [-3.0255043506622314], [-2.972233295440674], [-2.9618711471557617], [-2.938345193862915], [-2.903670310974121], [-3.2668039798736572], [-3.43176007270813], [-3.2525477409362793], [-2.9625205993652344], [-2.878673553466797], [-2.749999523162842], [-2.9318273067474365], [-2.7835464477539062], [-3.056758165359497], [-3.239743709564209], [-3.0655155181884766], [-2.822477340698242], [-2.818976879119873], [-2.6161270141601562], [-2.697136402130127], [-2.645996570587158], [-2.798264265060425], [-3.03578519821167], [-2.7547404766082764], [-2.4105870723724365], [-2.380244016647339], [-2.5030691623687744], [-2.4601926803588867], [-2.5408172607421875], [-2.875685453414917], [-3.149101972579956], [-2.99605655670166], [-2.710869789123535], [-2.5680429935455322], [-2.454983711242676], [-2.5392050743103027], [-2.6092569828033447], [-2.948263645172119], [-3.2537596225738525], [-3.072504758834839], [-2.8266031742095947], [-2.682720184326172], [-2.5137171745300293], [-2.481473684310913], [-2.427309036254883], [-3.036853313446045], [-3.301877498626709], [-3.0885753631591797], [-2.81510066986084], [-2.800964593887329], [-2.6109964847564697], [-2.633284568786621], [-2.6063051223754883], [-2.804241895675659], [-3.1039462089538574], [-2.9516549110412598], [-2.6421871185302734], [-2.458076238632202], [-2.2739038467407227], [-2.239731788635254], [-2.400282859802246], [-2.818373203277588], [-3.0382845401763916], [-2.8676645755767822], [-2.5819954872131348], [-2.4772021770477295], [-2.4666314125061035], [-2.3227696418762207], [-2.6698248386383057], [-2.7758383750915527], [-3.020473003387451], [-2.8708856105804443], [-2.7961652278900146], [-2.743922233581543], [-2.7564730644226074], [-2.6224565505981445], [-2.7182297706604004], [-2.540179491043091], [-2.580427885055542], [-2.4762520790100098], [-2.682948589324951], [-2.9018092155456543], [-3.229207992553711], [-2.704508066177368], [-2.8111114501953125], [-2.3438754081726074], [-2.074215888977051], [-1.7609851360321045], [-1.694972038269043], [-2.161740303039551], [-3.1421902179718018], [-2.8506879806518555], [-2.7757952213287354], [-3.2740731239318848], [-3.432013511657715], [-3.239187240600586], [-2.9951956272125244], [-2.9949803352355957], [-2.6462905406951904], [-2.603097438812256], [-2.615325689315796], [-2.867218494415283], [-3.1284313201904297], [-2.8734898567199707], [-2.519815444946289], [-2.474990129470825], [-2.4634287357330322], [-2.333002805709839], [-2.4827303886413574], [-3.09804105758667], [-3.348196268081665], [-3.1716220378875732], [-2.9144372940063477], [-2.811983346939087], [-2.714310646057129], [-2.542206048965454], [-2.721691131591797], [-3.020949602127075], [-3.3370487689971924], [-3.159771203994751], [-3.000319480895996], [-2.9210352897644043], [-2.667080879211426], [-2.589080810546875], [-2.5635828971862793], [-3.1472439765930176], [-3.413156032562256], [-3.233189821243286], [-2.9496846199035645], [-2.915106773376465], [-2.7010726928710938], [-2.725926160812378], [-2.7049307823181152], [-2.955781936645508], [-3.280496597290039], [-3.1415834426879883], [-2.859879732131958], [-2.7013497352600098], [-2.426579236984253], [-2.3576173782348633], [-2.3231685161590576], [-2.9085745811462402], [-3.0554733276367188], [-2.9098191261291504], [-2.5840976238250732], [-2.4693498611450195], [-2.4060256481170654], [-2.341966152191162], [-2.504096508026123], [-3.1242246627807617], [-3.313084363937378], [-3.1810097694396973], [-2.8603367805480957], [-2.6695847511291504], [-2.6352672576904297], [-2.5256776809692383], [-2.5389020442962646], [-3.083676815032959], [-3.176365613937378], [-2.954110622406006], [-2.932929277420044], [-3.1467394828796387], [-3.113424301147461], [-2.9522032737731934], [-2.8756842613220215], [-2.345126152038574], [-2.3596131801605225], [-2.212496519088745], [-2.396329164505005], [-2.7688472270965576], [-3.045576810836792], [-2.9444305896759033], [-2.6620492935180664], [-3.0960512161254883], [-3.2782466411590576], [-3.130591869354248], [-2.8250904083251953], [-2.779384136199951], [-2.3641958236694336], [-2.485758066177368], [-2.345268964767456], [-2.745203733444214], [-2.959909439086914], [-2.6722066402435303], [-2.319324016571045], [-2.2475087642669678], [-2.327101707458496], [-2.2959070205688477], [-2.2814407348632812], [-2.979106903076172], [-3.209562063217163], [-3.0627167224884033], [-2.7943520545959473], [-2.643641948699951], [-2.454519748687744], [-2.4948222637176514], [-2.460803985595703], [-2.9652655124664307], [-3.309950828552246], [-3.174729824066162], [-2.953270196914673], [-2.788142204284668], [-2.526062488555908], [-2.4166111946105957], [-2.3097267150878906], [-3.130882978439331], [-3.430004358291626], [-3.315007209777832], [-2.998142957687378], [-2.936711072921753], [-2.5791232585906982], [-2.6386170387268066], [-2.479102373123169], [-2.941988945007324], [-3.2616374492645264], [-3.154536724090576], [-2.835066318511963], [-2.6312832832336426], [-2.2997233867645264], [-2.2550675868988037], [-2.239375114440918], [-2.7764029502868652], [-2.9560139179229736], [-2.873814105987549], [-2.5257718563079834], [-2.4271397590637207], [-2.2562522888183594], [-2.282977819442749], [-2.3593454360961914], [-3.049097776412964], [-3.320639133453369], [-3.238558292388916], [-2.998518705368042], [-2.844893455505371], [-2.5405659675598145], [-2.5553884506225586], [-2.4810519218444824], [-3.007058620452881], [-3.2456393241882324], [-3.153097152709961], [-3.0481936931610107], [-3.1615326404571533], [-2.9623072147369385], [-2.841400146484375], [-2.728470802307129], [-2.5197596549987793], [-2.6783034801483154], [-2.599613666534424], [-2.748953104019165], [-2.872668743133545], [-2.7680835723876953], [-2.9152255058288574], [-2.64778208732605], [-3.0771117210388184], [-3.2744274139404297], [-3.176884651184082], [-2.924637794494629], [-2.8829445838928223], [-2.333259344100952], [-2.4465391635894775], [-2.287024736404419], [-2.602766275405884], [-2.914172649383545], [-2.7189218997955322], [-2.419809341430664], [-2.3045473098754883], [-2.153946876525879], [-2.250429391860962], [-2.2126922607421875], [-2.6668543815612793], [-2.974379062652588], [-2.8510489463806152], [-2.655942916870117], [-2.4559407234191895], [-2.0906460285186768], [-2.335200786590576], [-2.2912652492523193], [-2.730560779571533], [-3.138568162918091], [-2.9925386905670166], [-2.7873642444610596], [-2.5939583778381348], [-2.1422958374023438], [-2.2608861923217773], [-2.18302059173584], [-2.6923282146453857], [-3.0638697147369385], [-2.9225211143493652], [-2.6824331283569336], [-2.5205464363098145], [-2.2077133655548096], [-2.393080472946167], [-2.3106818199157715], [-2.5878660678863525], [-2.9778637886047363], [-2.8799009323120117], [-2.5659537315368652], [-2.251540422439575], [-1.9903203248977661], [-1.9211658239364624], [-2.175032138824463], [-2.254202365875244], [-2.645148754119873], [-2.506183624267578], [-2.2179973125457764], [-1.9312034845352173], [-1.8553577661514282], [-2.0040502548217773], [-2.410996913909912], [-2.7201805114746094], [-3.032827854156494], [-2.993107557296753], [-2.7464842796325684], [-2.6361961364746094], [-2.4633524417877197], [-2.3784213066101074], [-2.3772990703582764], [-2.900480031967163], [-3.142274856567383], [-3.0713653564453125], [-2.8135740756988525], [-2.7013323307037354], [-2.6796340942382812], [-2.6993415355682373], [-2.700137138366699], [-3.1568055152893066], [-3.357632875442505], [-3.247715473175049], [-2.9931654930114746], [-2.894641160964966], [-2.7290334701538086], [-2.8641397953033447], [-2.804729700088501], [-3.1559293270111084], [-3.2973389625549316], [-3.1835947036743164], [-2.9334912300109863], [-2.9324333667755127], [-2.689711570739746], [-2.7255988121032715], [-2.694202423095703], [-2.667616367340088], [-2.927004337310791], [-2.7290539741516113], [-2.454944133758545], [-2.354173183441162], [-2.336538314819336], [-2.383436679840088], [-2.25846791267395], [-2.784022808074951], [-3.028282642364502], [-2.908846616744995], [-2.6858503818511963], [-2.4537622928619385], [-2.3783297538757324], [-2.376957893371582], [-2.4654202461242676], [-2.9496243000030518], [-3.252835750579834], [-3.1011555194854736], [-2.8678696155548096], [-2.713365316390991], [-2.54046368598938], [-2.410975217819214], [-2.299098014831543], [-3.042114496231079], [-3.343519926071167], [-3.251695394515991], [-2.954981565475464], [-2.8058364391326904], [-2.450387954711914], [-2.493351936340332], [-2.63087797164917], [-2.861623525619507], [-3.243027448654175], [-3.1922378540039062], [-2.872384548187256], [-2.6351265907287598], [-2.341822624206543], [-2.091151475906372], [-2.0904688835144043], [-2.883458137512207], [-3.1585144996643066], [-3.122789144515991], [-2.817427635192871], [-2.602485418319702], [-2.5566043853759766], [-2.504387855529785], [-2.5496346950531006], [-3.068716049194336], [-3.2556240558624268], [-3.1893436908721924], [-3.0494117736816406], [-2.926018238067627], [-2.847855806350708], [-2.706827163696289], [-2.490861415863037], [-3.019075870513916], [-3.265737533569336], [-3.1922059059143066], [-3.1124954223632812], [-3.207737445831299], [-3.01149845123291], [-2.9144163131713867], [-2.8922958374023438], [-2.6639509201049805], [-2.7959272861480713], [-2.6809303760528564], [-2.877887725830078], [-2.9785239696502686], [-3.066838264465332], [-3.123457431793213], [-2.8507919311523438], [-3.217155933380127], [-3.373368263244629], [-3.2868094444274902], [-3.0716750621795654], [-3.0571341514587402], [-2.676396369934082], [-2.7524189949035645], [-2.693474769592285], [-2.820223808288574], [-3.079144239425659], [-2.8550643920898438], [-2.541846513748169], [-2.495490550994873], [-2.3962907791137695], [-2.4343321323394775], [-2.3244428634643555], [-3.0628795623779297], [-3.294231414794922], [-3.25191068649292], [-3.0285146236419678], [-2.8481171131134033], [-2.5827484130859375], [-2.648895740509033], [-2.5568490028381348], [-3.0211753845214844], [-3.373006582260132], [-3.3303232192993164], [-3.088895559310913], [-2.9598045349121094], [-2.610069513320923], [-2.6560440063476562], [-2.396069288253784], [-3.2740442752838135], [-3.5656328201293945], [-3.4729723930358887], [-3.2637131214141846], [-3.1937553882598877], [-2.939859390258789], [-2.9035396575927734], [-2.861393928527832], [-2.9746651649475098], [-3.310371160507202], [-3.293578624725342], [-3.058135509490967], [-2.9297385215759277], [-2.546355724334717], [-2.5080366134643555], [-2.2286901473999023], [-2.9363784790039062], [-3.1948180198669434], [-3.1226789951324463], [-2.857509136199951], [-2.814631938934326], [-2.535815477371216], [-2.4933090209960938], [-2.3852028846740723], [-3.1545052528381348], [-3.4121925830841064], [-3.442993402481079], [-3.2263243198394775], [-3.1130707263946533], [-2.8409018516540527], [-2.6968953609466553], [-2.4469404220581055], [-3.2102420330047607], [-3.4330759048461914], [-3.4071497917175293], [-3.194786310195923], [-3.2347428798675537], [-2.957003593444824], [-2.984161615371704], [-2.842207193374634], [-3.2506327629089355], [-3.495790958404541], [-3.4393022060394287], [-3.2849528789520264], [-3.217139482498169], [-2.832176685333252], [-3.069047212600708], [-2.7481095790863037], [-3.2063636779785156], [-3.4368910789489746], [-3.3446547985076904], [-3.1580238342285156], [-3.179271936416626], [-2.7854256629943848], [-2.8619601726531982], [-2.671905040740967], [-2.8195371627807617], [-3.117924690246582], [-2.8961129188537598], [-2.5322256088256836], [-2.5054516792297363], [-2.3488242626190186], [-2.459442615509033], [-2.2868847846984863], [-2.9232916831970215], [-3.2159690856933594], [-3.107124090194702], [-2.876591205596924], [-2.710221290588379], [-2.4587700366973877], [-2.520864963531494], [-2.512613296508789], [-3.0573654174804688], [-3.376857042312622], [-3.242140769958496], [-2.9929635524749756], [-2.853766918182373], [-2.559926748275757], [-2.4784939289093018], [-2.333082675933838], [-3.0364224910736084], [-3.316293954849243], [-3.202711582183838], [-3.013197422027588], [-2.9228618144989014], [-2.5664761066436768], [-2.6228418350219727], [-2.622124671936035], [-3.0295145511627197], [-3.385390281677246], [-3.3675432205200195], [-3.0718040466308594], [-2.8532028198242188], [-2.4637060165405273], [-2.2567315101623535], [-2.178554058074951], [-2.8641180992126465], [-3.0673532485961914], [-2.9984681606292725], [-2.708254814147949], [-2.667436361312866], [-2.4822702407836914], [-2.3963286876678467], [-2.4493699073791504], [-3.078834295272827], [-3.3008298873901367], [-3.3245160579681396], [-3.0579586029052734], [-2.9747660160064697], [-2.8196685314178467], [-2.4997963905334473], [-2.3308217525482178], [-3.0309648513793945], [-3.2555923461914062], [-3.2958436012268066], [-3.124056339263916], [-3.094397783279419], [-3.066838026046753], [-2.7443888187408447], [-2.7832279205322266], [-2.6597213745117188], [-2.912209987640381], [-2.794689178466797], [-2.8954601287841797], [-2.9549612998962402], [-2.971365451812744], [-2.9828970432281494], [-2.7144269943237305], [-3.1652488708496094], [-3.327113628387451], [-3.1839752197265625], [-2.961130380630493], [-2.995142936706543], [-2.7364885807037354], [-2.7187232971191406], [-2.611774444580078], [-2.8574867248535156], [-3.076359272003174], [-2.790888547897339], [-2.4545059204101562], [-2.475067138671875], [-2.484394073486328], [-2.4628701210021973], [-2.2570600509643555], [-2.999690294265747], [-3.2313926219940186], [-3.125297784805298], [-2.894630193710327], [-2.7308197021484375], [-2.559945583343506], [-2.5181708335876465], [-2.4492173194885254], [-3.063474655151367], [-3.3411641120910645], [-3.238541603088379], [-3.0376622676849365], [-2.920283317565918], [-2.7055182456970215], [-2.528989791870117], [-2.325770854949951], [-3.0692782402038574], [-3.3600010871887207], [-3.2416625022888184], [-2.9942514896392822], [-2.9464662075042725], [-2.6301493644714355], [-2.634970188140869], [-2.670470714569092], [-3.0464067459106445], [-3.376096725463867], [-3.3194875717163086], [-3.005401849746704], [-2.914647102355957], [-2.5263237953186035], [-2.465055227279663], [-2.167992115020752], [-2.8667776584625244], [-3.1001479625701904], [-2.9761009216308594], [-2.7023487091064453], [-2.6971004009246826], [-2.561209201812744], [-2.5551061630249023], [-2.4718005657196045], [-3.020256996154785], [-3.2604947090148926], [-3.240915298461914], [-2.9857659339904785], [-2.9445908069610596], [-2.6861729621887207], [-2.6573100090026855], [-2.377039909362793], [-3.035823106765747], [-3.242589235305786], [-3.1255836486816406], [-2.9536805152893066], [-2.971982955932617], [-2.775729179382324], [-2.820096254348755], [-2.6554417610168457], [-3.1136856079101562], [-3.279465675354004], [-2.980546236038208], [-2.9517898559570312], [-2.969308853149414], [-2.8359296321868896], [-2.909090995788574], [-2.5508179664611816], [-3.015681743621826], [-3.266671895980835], [-3.119802951812744], [-2.90954327583313], [-2.8704075813293457], [-2.4227004051208496], [-2.5674424171447754], [-2.4833407402038574], [-2.6753787994384766], [-2.9577648639678955], [-2.7060375213623047], [-2.4052817821502686], [-2.3828468322753906], [-2.253894567489624], [-2.4424073696136475], [-2.2601683139801025], [-2.824064254760742], [-3.1030771732330322], [-2.9477291107177734], [-2.764378070831299], [-2.635685443878174], [-2.3071577548980713], [-2.4193215370178223], [-2.35866379737854], [-2.9783496856689453], [-3.30366849899292], [-3.1179215908050537], [-2.923163890838623], [-2.763521194458008], [-2.4140233993530273], [-2.3577077388763428], [-2.2214789390563965], [-2.910573959350586], [-3.194722890853882], [-3.0029687881469727], [-2.826664447784424], [-2.7454328536987305], [-2.424675226211548], [-2.500023126602173], [-2.5249767303466797], [-2.822741985321045], [-3.186655282974243], [-3.0193355083465576], [-2.74031138420105], [-2.505479574203491], [-2.2266249656677246], [-2.2122607231140137], [-2.1319379806518555], [-2.6357879638671875], [-2.9547462463378906], [-2.8613715171813965], [-2.534148693084717], [-2.404193639755249], [-2.4572219848632812], [-2.4236738681793213], [-2.4375665187835693], [-2.956899404525757], [-3.293975591659546], [-3.247492790222168], [-2.997464179992676], [-2.8606691360473633], [-2.8152170181274414], [-2.5785436630249023], [-2.4531242847442627], [-3.1031525135040283], [-3.375092029571533], [-3.326935291290283], [-3.057490825653076], [-2.9813051223754883], [-2.898327350616455], [-2.877682685852051], [-2.749870777130127], [-2.9984898567199707], [-3.3474555015563965], [-3.181370973587036], [-2.997133255004883], [-2.905649423599243], [-2.8909149169921875], [-2.8653533458709717], [-2.7251169681549072], [-3.181175947189331], [-3.428089141845703], [-3.3224635124206543], [-3.1300604343414307], [-3.0589547157287598], [-2.840108871459961], [-2.7587461471557617], [-2.570492744445801], [-2.8331923484802246], [-3.1714682579040527], [-2.983741044998169], [-2.7386484146118164], [-2.656787872314453], [-2.5734524726867676], [-2.6192963123321533], [-2.310871124267578], [-3.026592493057251], [-3.321782112121582], [-3.2295570373535156], [-3.0583415031433105], [-2.8598928451538086], [-2.6666479110717773], [-2.7268521785736084], [-2.584979295730591], [-3.2139675617218018], [-3.526439905166626], [-3.3440001010894775], [-3.140411376953125], [-3.000943660736084], [-2.8205714225769043], [-2.574796199798584], [-2.4179720878601074], [-3.0405173301696777], [-3.3044703006744385], [-3.077873706817627], [-2.91772723197937], [-2.904515266418457], [-2.762028932571411], [-2.7087559700012207], [-2.5815865993499756], [-2.8814778327941895], [-3.2479941844940186], [-3.1403467655181885], [-2.887105941772461], [-2.6810429096221924], [-2.4756417274475098], [-2.2729218006134033], [-2.2346415519714355], [-2.7586607933044434], [-3.015138626098633], [-2.901251792907715], [-2.646851062774658], [-2.6060237884521484], [-2.4718024730682373], [-2.434689521789551], [-2.322321653366089], [-3.184056043624878], [-3.377044677734375], [-3.3315749168395996], [-3.0672407150268555], [-2.9395923614501953], [-2.9046735763549805], [-2.623380661010742], [-2.2832865715026855], [-3.1098337173461914], [-3.337916374206543], [-3.317861318588257], [-3.054816246032715], [-3.0161802768707275], [-2.938368320465088], [-2.8945131301879883], [-2.521782875061035], [-3.0664873123168945], [-3.347287893295288], [-3.302072048187256], [-3.078505039215088], [-2.9957385063171387], [-2.815464496612549], [-2.979480743408203], [-2.5527496337890625], [-3.0861780643463135], [-3.3144679069519043], [-3.2388126850128174], [-3.0771098136901855], [-3.126862049102783], [-2.7647900581359863], [-2.8685076236724854], [-2.4710919857025146], [-2.8230206966400146], [-3.0993430614471436], [-2.8889870643615723], [-2.601106882095337], [-2.614346981048584], [-2.436858654022217], [-2.539191484451294], [-2.157578706741333], [-2.838974952697754], [-3.1743738651275635], [-3.115579605102539], [-2.895904541015625], [-2.747342109680176], [-2.359203338623047], [-2.5894737243652344], [-2.4466700553894043], [-2.9429123401641846], [-3.289546489715576], [-3.2342543601989746], [-2.990096092224121], [-2.833496332168579], [-2.4530863761901855], [-2.4501521587371826], [-2.3559377193450928], [-2.9408693313598633], [-3.228602886199951], [-3.09515643119812], [-2.8467793464660645], [-2.7769808769226074], [-2.3958258628845215], [-2.532917022705078], [-2.6542210578918457], [-2.8319244384765625], [-3.175686836242676], [-3.1424663066864014], [-2.842029571533203], [-2.6392440795898438], [-2.339350461959839], [-2.212681531906128], [-2.176621198654175], [-2.932058811187744], [-3.180365562438965], [-3.1603591442108154], [-2.8462560176849365], [-2.709947109222412], [-2.6152749061584473], [-2.414862632751465], [-2.5309760570526123], [-2.519406318664551], [-2.3754560947418213], [-2.225719690322876], [-2.3433098793029785], [-2.4733331203460693], [-3.0858049392700195], [-2.738924503326416], [-2.762615919113159], [-2.309504508972168], [-2.2520127296447754], [-1.969367265701294], [-2.06306791305542], [-2.253488779067993], [-3.032094955444336], [-2.5401670932769775], [-3.949451208114624], [-3.0515480041503906]], "row": 1}, {"kind": "span", "head": "broad", "words": "s2_words", "labels": "s2_labels", "logits": [[-1.46242094039917], [-1.578751802444458], [-2.053593158721924], [-2.9410452842712402], [-2.306424379348755], [-3.757460117340088], [-1.6370573043823242], [-2.211214780807495], [-1.536679983139038], [-2.142828941345215], [-2.862086772918701], [-3.9242331981658936], [-2.2933921813964844], [-3.6780166625976562], [-1.6517142057418823], [-2.380481004714966], [-1.591235637664795], [-2.0094799995422363], [-2.404503107070923], [-3.467717170715332], [-2.3641114234924316], [-2.7695889472961426], [-2.0861878395080566], [-2.592149257659912], [-1.8654803037643433], [-1.9848178625106812], [-2.5245232582092285], [-3.3906359672546387], [-2.529547691345215], [-2.542813301086426], [-2.345221519470215], [-2.727003812789917], [-2.0476694107055664], [-2.133293628692627], [-2.5518147945404053], [-3.4651408195495605], [-2.7658395767211914], [-2.302833080291748], [-2.4353013038635254], [-2.463855028152466], [-1.9347522258758545], [-2.0442211627960205], [-2.3939530849456787], [-2.9283907413482666], [-2.678332567214966], [-2.4262123107910156], [-2.527204990386963], [-2.6600420475006104], [-2.270602226257324], [-2.151944875717163], [-2.376081705093384], [-2.854588747024536], [-2.913820266723633], [-2.5924062728881836], [-3.0764122009277344], [-3.290419101715088], [-2.9238505363464355], [-2.732651472091675], [-2.6838486194610596], [-3.12308931350708], [-3.3025271892547607], [-3.0790181159973145], [-2.9682908058166504], [-3.26625394821167], [-2.7204761505126953], [-2.7418296337127686], [-2.830167055130005], [-3.231773853302002], [-3.164748191833496], [-2.78770112991333], [-2.9771389961242676], [-3.122159957885742], [-2.6633999347686768], [-2.66452956199646], [-2.7854161262512207], [-2.9952034950256348], [-3.1623082160949707], [-2.983698844909668], [-3.0810179710388184], [-3.452760696411133], [-3.045166015625], [-2.861807346343994], [-2.781524419784546], [-2.89953875541687], [-3.0224063396453857], [-2.7129669189453125], [-3.128213405609131], [-3.2046072483062744], [-2.873152256011963], [-2.654444694519043], [-2.722339153289795], [-3.16219162940979], [-3.1910269260406494], [-3.2130908966064453], [-3.249825954437256], [-3.5398807525634766], [-3.2752647399902344], [-3.133925199508667], [-2.9835591316223145], [-3.4965100288391113], [-3.4956326484680176], [-3.2210333347320557], [-3.297624349594116], [-3.654934883117676], [-3.457042694091797], [-3.240978956222534], [-3.1543960571289062], [-3.4679675102233887], [-3.240448474884033], [-3.0153911113739014], [-3.108316421508789], [-3.2601749897003174], [-2.905198335647583], [-2.6711645126342773], [-2.5785229206085205], [-3.0109057426452637], [-3.114821434020996], [-2.7633297443389893], [-2.8154759407043457], [-2.977318525314331], [-2.675588369369507], [-2.6071386337280273], [-2.535313606262207], [-2.74051833152771], [-2.873736619949341], [-2.617774486541748], [-2.6441996097564697], [-2.862034559249878], [-2.439314365386963], [-2.2134766578674316], [-2.244021415710449], [-2.6685080528259277], [-2.8970210552215576], [-2.5514044761657715], [-3.276556968688965], [-3.4441306591033936], [-3.1439852714538574], [-2.9680213928222656], [-2.8223156929016113], [-2.8712239265441895], [-3.267129421234131], [-3.026742935180664], [-3.017608642578125], [-3.245150566101074], [-2.93282413482666], [-2.8432419300079346], [-2.716230630874634], [-2.7903053760528564], [-2.9822819232940674], [-2.541440963745117], [-3.149306058883667], [-3.331366777420044], [-2.9460644721984863], [-2.7934999465942383], [-2.7437655925750732], [-2.652984857559204], [-2.8247880935668945], [-2.552772045135498], [-3.106595516204834], [-3.343392848968506], [-2.9779348373413086], [-2.776317834854126], [-2.5749003887176514], [-2.606450080871582], [-2.7272493839263916], [-2.311065196990967], [-3.1335456371307373], [-3.2193636894226074], [-3.0292696952819824], [-2.8006536960601807], [-2.8089935779571533], [-3.030042886734009], [-2.967182159423828], [-2.8159594535827637], [-3.2302792072296143], [-3.697627305984497], [-3.40073823928833], [-3.2479445934295654], [-3.1798651218414307], [-3.505422592163086], [-3.3430838584899902], [-2.995479106903076], [-3.444399356842041], [-3.5439939498901367], [-3.367097854614258], [-3.18273663520813], [-3.2042036056518555], [-3.2875967025756836], [-3.007000684738159], [-2.924163818359375], [-3.5330796241760254], [-3.701493740081787], [-3.410532236099243], [-3.2107019424438477], [-3.1163318157196045], [-3.100166082382202], [-3.2035844326019287], [-2.903604745864868], [-3.251627206802368], [-3.3690693378448486], [-3.070988893508911], [-2.9405102729797363], [-2.9163568019866943], [-2.87406849861145], [-2.926581382751465], [-2.6954832077026367], [-2.802860975265503], [-3.016221523284912], [-2.7236180305480957], [-2.5148048400878906], [-2.5511057376861572], [-2.746978998184204], [-2.938279151916504], [-2.5906639099121094], [-3.4838156700134277], [-3.63032865524292], [-3.380119800567627], [-3.2677557468414307], [-3.1324727535247803], [-3.137328624725342], [-3.245553731918335], [-3.036865711212158], [-3.2577407360076904], [-3.413130521774292], [-3.230956792831421], [-3.172015428543091], [-3.007582426071167], [-2.81176495552063], [-2.9580307006835938], [-2.664283514022827], [-3.1377484798431396], [-3.2779111862182617], [-3.0554769039154053], [-2.977445602416992], [-2.917400360107422], [-2.8508565425872803], [-2.9309587478637695], [-2.787186622619629], [-3.110201358795166], [-3.2969672679901123], [-3.0543649196624756], [-2.9113950729370117], [-2.7102057933807373], [-2.5543832778930664], [-2.640897750854492], [-2.409438133239746], [-2.876380681991577], [-2.909414291381836], [-2.7171030044555664], [-2.4451940059661865], [-2.3132312297821045], [-2.532222270965576], [-2.669705629348755], [-2.735403537750244], [-3.0345163345336914], [-3.1155173778533936], [-3.02018404006958], [-2.830580949783325], [-2.645573616027832], [-2.866494655609131], [-2.8700008392333984], [-2.649355173110962], [-3.17244291305542], [-3.2559380531311035], [-3.1572906970977783], [-2.9559872150421143], [-2.8725802898406982], [-2.926013469696045], [-2.916090488433838], [-2.9111855030059814], [-3.191455364227295], [-3.2834763526916504], [-3.086082696914673], [-2.88552188873291], [-2.7162890434265137], [-2.541166067123413], [-3.002589464187622], [-2.706963062286377], [-3.024343967437744], [-3.1593198776245117], [-2.977766513824463], [-2.8756985664367676], [-2.824345588684082], [-2.6022756099700928], [-2.7312567234039307], [-2.5285208225250244], [-2.5240612030029297], [-2.7699081897735596], [-2.499464511871338], [-2.2411701679229736], [-2.109645366668701], [-2.3582091331481934], [-2.4978997707366943], [-2.373380184173584], [-3.2327699661254883], [-3.349247932434082], [-3.1022539138793945], [-2.94881534576416], [-2.713750123977661], [-2.7477924823760986], [-2.9743080139160156], [-2.8689956665039062], [-2.937528610229492], [-3.155108690261841], [-2.9831228256225586], [-2.886741876602173], [-2.6686954498291016], [-2.4850094318389893], [-2.4757680892944336], [-2.265232801437378], [-2.875843048095703], [-3.0870327949523926], [-2.8926165103912354], [-2.7715847492218018], [-2.64994215965271], [-2.447688102722168], [-2.675276517868042], [-2.650866746902466], [-2.872936725616455], [-3.069291591644287], [-2.862396478652954], [-2.6734023094177246], [-2.3874430656433105], [-2.1924843788146973], [-2.33781099319458], [-2.2302064895629883], [-2.282304286956787], [-2.587526321411133], [-2.5094544887542725], [-2.286508560180664], [-2.1020255088806152], [-2.100961923599243], [-2.319077968597412], [-2.4545364379882812], [-2.769298553466797], [-3.0072526931762695], [-2.9149882793426514], [-2.7805657386779785], [-2.54341983795166], [-2.530954360961914], [-2.506960868835449], [-2.441068172454834], [-2.8440146446228027], [-3.0782361030578613], [-2.9566822052001953], [-2.7972731590270996], [-2.7467286586761475], [-2.6086833477020264], [-2.663557767868042], [-2.7577033042907715], [-3.1096904277801514], [-3.2894845008850098], [-3.0419254302978516], [-2.8940842151641846], [-2.7279458045959473], [-2.4995057582855225], [-2.8857107162475586], [-2.6962525844573975], [-2.973830223083496], [-3.1677114963531494], [-2.985825777053833], [-2.9433088302612305], [-2.868805170059204], [-2.543186902999878], [-2.667307138442993], [-2.5054335594177246], [-2.384077548980713], [-2.760559320449829], [-2.5275940895080566], [-2.347568988800049], [-2.2192978858947754], [-2.2519147396087646], [-2.5080976486206055], [-2.438795566558838], [-2.9143314361572266], [-3.2368011474609375], [-2.991616725921631], [-2.947880268096924], [-2.665019989013672], [-2.6639909744262695], [-2.7605907917022705], [-2.770324230194092], [-2.872812271118164], [-3.159195899963379], [-2.9883627891540527], [-2.9722447395324707], [-2.749894142150879], [-2.546912670135498], [-2.5398712158203125], [-2.4099583625793457], [-2.828099012374878], [-3.097501277923584], [-2.8371920585632324], [-2.8131918907165527], [-2.693901300430298], [-2.520111322402954], [-2.6533384323120117], [-2.6465182304382324], [-2.7948946952819824], [-3.0514204502105713], [-2.8110618591308594], [-2.664250612258911], [-2.337146520614624], [-2.1378421783447266], [-2.166353464126587], [-2.075828790664673], [-2.438584804534912], [-2.748077869415283], [-2.548905849456787], [-2.357175588607788], [-2.1865313053131104], [-2.3369548320770264], [-2.483612060546875], [-2.5693202018737793], [-2.75522518157959], [-2.9844629764556885], [-2.830801248550415], [-2.6364073753356934], [-2.4932000637054443], [-2.869339942932129], [-2.6473758220672607], [-2.509058952331543], [-2.836165428161621], [-3.0454440116882324], [-2.894404888153076], [-2.7295479774475098], [-2.634716510772705], [-2.8711018562316895], [-2.8433566093444824], [-2.741441488265991], [-3.10032320022583], [-3.2780957221984863], [-3.0606303215026855], [-2.8737192153930664], [-2.7136144638061523], [-2.819831371307373], [-3.080127239227295], [-2.749569892883301], [-3.2098724842071533], [-3.3529460430145264], [-3.150506019592285], [-3.056368350982666], [-3.0094406604766846], [-2.8157286643981934], [-2.9148809909820557], [-2.579092025756836], [-2.696319341659546], [-2.9217865467071533], [-2.6827287673950195], [-2.465259552001953], [-2.361116409301758], [-2.532473087310791], [-2.7004036903381348], [-2.353888511657715], [-3.0381877422332764], [-3.2553250789642334], [-3.142531394958496], [-2.999772310256958], [-2.737884044647217], [-2.791410446166992], [-2.9174251556396484], [-2.6937365531921387], [-2.9756011962890625], [-3.225416660308838], [-3.069798469543457], [-2.947032928466797], [-2.7246832847595215], [-2.6679790019989014], [-2.645730972290039], [-2.4369804859161377], [-2.847442865371704], [-3.0735254287719727], [-2.936091661453247], [-2.836996078491211], [-2.753204107284546], [-2.627849578857422], [-2.8227999210357666], [-2.6518149375915527], [-2.7628166675567627], [-3.073761224746704], [-2.9348225593566895], [-2.7233333587646484], [-2.4637374877929688], [-2.402113914489746], [-2.384373664855957], [-2.3781328201293945], [-2.5474863052368164], [-2.7480883598327637], [-2.6128737926483154], [-2.355022430419922], [-2.209345579147339], [-2.4827587604522705], [-2.5864334106445312], [-2.7286441326141357], [-3.1218552589416504], [-3.231245994567871], [-3.119462013244629], [-2.942477226257324], [-2.74387526512146], [-2.889009475708008], [-2.8022103309631348], [-2.595551013946533], [-2.908257484436035], [-3.1511952877044678], [-3.0907046794891357], [-2.9123518466949463], [-2.858598232269287], [-2.877084493637085], [-2.945681095123291], [-2.853344440460205], [-2.9742584228515625], [-3.198514461517334], [-3.037407875061035], [-2.832915782928467], [-2.708615779876709], [-2.5856690406799316], [-2.9652225971221924], [-2.65535044670105], [-3.0538384914398193], [-3.2164549827575684], [-3.055551052093506], [-2.9343605041503906], [-2.9023802280426025], [-2.721137046813965], [-2.8291120529174805], [-2.584285259246826], [-2.5902562141418457], [-2.8442492485046387], [-2.585505485534668], [-2.3575143814086914], [-2.320420742034912], [-2.5341196060180664], [-2.697981357574463], [-2.397726058959961], [-3.0092453956604004], [-3.235905170440674], [-3.1099302768707275], [-2.962283134460449], [-2.822869300842285], [-2.7031607627868652], [-3.0261573791503906], [-2.7602977752685547], [-2.930368185043335], [-3.2076046466827393], [-3.0614378452301025], [-2.9320907592773438], [-2.8008909225463867], [-2.712317705154419], [-2.71592116355896], [-2.5747361183166504], [-2.934807777404785], [-3.2078824043273926], [-3.0015523433685303], [-2.8434958457946777], [-2.8725783824920654], [-2.828842878341675], [-3.002316474914551], [-2.95174503326416], [-2.674215793609619], [-3.0042810440063477], [-2.8610265254974365], [-2.6239986419677734], [-2.4865970611572266], [-2.248460292816162]], "row": 2}], "predict": {"answers": [{"id": "claims", "type": "span", "target_part": "user", "spans": [], "threshold": 0.35, "threshold_rule": "broad", "head": "broad", "head_reason": "default"}], "meta": {"tokens": {"user": 2310}, "windows": 2, "sequences": 1}}} diff --git a/tests/vela/fixtures/head-input-long-span-windowed.safetensors b/tests/vela/fixtures/head-input-long-span-windowed.safetensors new file mode 100644 index 00000000..c23ccacd Binary files /dev/null and b/tests/vela/fixtures/head-input-long-span-windowed.safetensors differ diff --git a/tests/vela/fixtures/native-plan-example-request-cjk.json b/tests/vela/fixtures/native-plan-example-request-cjk.json new file mode 100644 index 00000000..01973a26 --- /dev/null +++ b/tests/vela/fixtures/native-plan-example-request-cjk.json @@ -0,0 +1 @@ +{"case": "example-request-cjk", "request": {"model": "vllm-sr/Vela-2.0-0.8B", "state": {"request": "客户张伟（zhang.wei@example.cn）于2026年10月1日在上海市浦东新区世纪大道100号办理了业务，联系电话13800138000。", "source": "客户的业务申请需要在三个工作日内审核完成。", "answer": "客户的业务已经即时完成了。"}, "questions": {"pii": {"type": "span", "instructions": "Which spans are personal information?", "criteria": {"PERSON": "a person's name", "EMAIL_ADDRESS": "an e-mail address", "PHONE_NUMBER": "a telephone number", "STREET_ADDRESS": "a street address or postal address"}, "over": "request"}, "unsupported": {"type": "span", "instructions": "Which spans of the answer are not supported by the context?", "criteria": {"unsupported": "a claim not supported by the context"}}, "billing": {"type": "noul", "instructions": "这个请求是否与账单有关？"}}}, "predict_request": {"parts": [{"type": "user", "text": "客户张伟（zhang.wei@example.cn）于2026年10月1日在上海市浦东新区世纪大道100号办理了业务，联系电话13800138000。"}, {"type": "context", "text": "客户的业务申请需要在三个工作日内审核完成。"}, {"type": "answer", "text": "客户的业务已经即时完成了。"}], "questions": [{"id": "pii", "type": "span", "text": "Which spans are personal information?", "target_part": "user", "labels": [{"name": "PERSON", "description": "a person's name"}, {"name": "EMAIL_ADDRESS", "description": "an e-mail address"}, {"name": "PHONE_NUMBER", "description": "a telephone number"}, {"name": "STREET_ADDRESS", "description": "a street address or postal address"}]}, {"id": "unsupported", "type": "span", "text": "Which spans of the answer are not supported by the context?", "target_part": "answer", "labels": [{"name": "unsupported", "description": "a claim not supported by the context"}]}, {"id": "billing", "type": "choice", "text": "这个请求是否与账单有关？", "target_part": ["user", "context", "answer"], "options": [{"name": "no", "description": "No. The statement or question is not satisfied."}, {"name": "yes", "description": "Yes. The statement or question is satisfied."}]}]}, "plan": [{"qid": "pii", "type": "span", "criteria": {"PERSON": "a person's name", "EMAIL_ADDRESS": "an e-mail address", "PHONE_NUMBER": "a telephone number", "STREET_ADDRESS": "a street address or postal address"}, "range": [0, 75], "part": "user"}, {"qid": "unsupported", "type": "span", "criteria": {"unsupported": "a claim not supported by the context"}, "range": null, "part": "answer"}, {"qid": "billing", "type": "noul", "criteria": {}, "range": null, "part": ["user", "context", "answer"]}], "rows": [{"p_ids": [1905, 25, 198, 27, 22446, 3364, 420, 846, 731, 198, 96989, 138667, 9616, 89, 20032, 13, 26234, 34314, 21377, 7313, 95928, 17, 15, 17, 21, 95859, 16, 15, 96212, 16, 95971, 95772, 108914, 121458, 99524, 108611, 16, 15, 15, 96389, 134395, 97192, 3709, 108672, 16, 18, 23, 15, 15, 16, 18, 23, 15, 15, 15, 1710, 198, 510, 22446, 29, 198, 27, 22446, 3364, 420, 2078, 731, 198, 109480, 97192, 97379, 111341, 98868, 96069, 99671, 100616, 97235, 1710, 198, 510, 22446, 29, 198, 27, 22446, 3364, 420, 8944, 731, 198, 109480, 97192, 96790, 111267, 104154, 1710, 198, 510, 22446, 29, 198], "blocks": [{"kind": "q", "ids": [271, 6065, 913, 25, 5576, 198, 6196, 25, 1156, 11, 2193, 11, 4087, 198, 14162, 25, 198, 96488, 101729, 147343, 119526, 97495, 10992, 198, 3670, 25, 198, 13886, 29, 198, 4754, 4532, 3147, 2665, 13, 561, 4952, 466, 3296, 369, 524, 18933, 46292, 767, 3147, 2083, 8934, 198, 510, 1979, 29, 198, 13886, 29, 198, 4754, 4532, 3147, 9175, 13, 561, 4952, 466, 3296, 369, 18933, 46292, 767, 3147, 9405, 8934, 198, 510, 1979, 29, 198, 13886, 29, 198, 4754, 4532, 3147, 3928, 314, 279, 6625, 2519, 16298, 46292, 767, 3147, 58, 55553, 60, 8934, 198, 510, 1979, 29, 271, 3270, 279, 3074, 2904, 1786, 7021, 539, 279, 2193, 321, 10895, 13, 198, 72298, 25], "ends": [50, 74, 98], "query": 114, "qid": "billing", "qtype": "choice", "names": ["no", "yes"], "has_abs": true}, {"kind": "span", "ends": [41, 62, 82, 106], "label_ranges": [[22, 41], [42, 62], [63, 82], [83, 106]], "words_in_block": true, "ids": [271, 6065, 913, 25, 9111, 198, 6196, 25, 1156, 198, 14162, 25, 198, 22365, 42824, 513, 4204, 1928, 30, 198, 22933, 25, 198, 14057, 29, 198, 4754, 4532, 3147, 64, 1637, 579, 803, 2129, 767, 3147, 89974, 8934, 198, 510, 1448, 29, 198, 14057, 29, 198, 4754, 4532, 3147, 276, 378, 11139, 2534, 2129, 767, 3147, 34714, 19522, 8934, 198, 510, 1448, 29, 198, 14057, 29, 198, 4754, 4532, 3147, 64, 19919, 1324, 2129, 767, 3147, 72446, 18777, 8934, 198, 510, 1448, 29, 198, 14057, 29, 198, 4754, 4532, 3147, 64, 8329, 2534, 466, 38436, 2534, 2129, 767, 3147, 759, 54046, 19522, 8934, 198, 510, 1448, 29, 271, 1138, 25, 198, 27, 22446, 3364, 420, 846, 731, 198, 96989, 138667, 9616, 89, 20032, 13, 26234, 34314, 21377, 7313, 95928, 17, 15, 17, 21, 95859, 16, 15, 96212, 16, 95971, 95772, 108914, 121458, 99524, 108611, 16, 15, 15, 96389, 134395, 97192, 3709, 108672, 16, 18, 23, 15, 15, 16, 18, 23, 15, 15, 15, 1710, 198, 510, 22446, 29, 198, 271, 8679, 1396, 9111, 314, 279, 2100, 1414, 421, 22651, 799, 314, 279, 8928, 13, 198, 6205, 581, 25], "query": 187, "w_pos": [118, 118, 119, 119, 120, 121, 127, 128, 129, 133, 134, 136, 137, 138, 139, 140, 140, 140, 141, 141, 141, 141, 142, 142, 143, 143, 144, 147, 148, 148, 148, 149, 149, 150, 151, 151, 151, 151, 152, 163]}], "trunc": {"shrink_cut": 0, "protected_cut": 0}, "seg_lens": {"user": 46, "context": 10, "answer": 6}, "layout": "c", "names": ["PERSON", "EMAIL_ADDRESS", "PHONE_NUMBER", "STREET_ADDRESS"], "w_offs": [[0, 1], [1, 2], [2, 3], [3, 4], [4, 5], [5, 25], [25, 26], [26, 27], [27, 31], [31, 32], [32, 34], [34, 35], [35, 36], [36, 37], [37, 38], [38, 39], [39, 40], [40, 41], [41, 42], [42, 43], [43, 44], [44, 45], [45, 46], [46, 47], [47, 48], [48, 49], [49, 52], [52, 53], [53, 54], [54, 55], [55, 56], [56, 57], [57, 58], [58, 59], [59, 60], [60, 61], [61, 62], [62, 63], [63, 74], [74, 75]], "w_pos": [118, 118, 119, 119, 120, 121, 127, 128, 129, 133, 134, 136, 137, 138, 139, 140, 140, 140, 141, 141, 141, 141, 142, 142, 143, 143, 144, 147, 148, 148, 148, 149, 149, 150, 151, 151, 151, 151, 152, 163], "span_over": "user", "n_tokens": 404, "head": "router", "head_reason": "id"}, {"p_ids": [1905, 25, 198, 27, 22446, 3364, 420, 846, 731, 198, 96989, 138667, 9616, 89, 20032, 13, 26234, 34314, 21377, 7313, 95928, 17, 15, 17, 21, 95859, 16, 15, 96212, 16, 95971, 95772, 108914, 121458, 99524, 108611, 16, 15, 15, 96389, 134395, 97192, 3709, 108672, 16, 18, 23, 15, 15, 16, 18, 23, 15, 15, 15, 1710, 198, 510, 22446, 29, 198, 27, 22446, 3364, 420, 2078, 731, 198, 109480, 97192, 97379, 111341, 98868, 96069, 99671, 100616, 97235, 1710, 198, 510, 22446, 29, 198, 27, 22446, 3364, 420, 8944, 731, 198, 109480, 97192, 96790, 111267, 104154, 1710, 198, 510, 22446, 29, 198], "blocks": [{"kind": "span", "ends": [50], "label_ranges": [[28, 50]], "words_in_block": true, "ids": [271, 6065, 913, 25, 9111, 198, 6196, 25, 4087, 198, 14162, 25, 198, 22365, 42824, 314, 279, 4087, 513, 524, 7021, 539, 279, 2193, 30, 198, 22933, 25, 198, 14057, 29, 198, 4754, 4532, 3147, 64, 3591, 524, 7021, 539, 279, 2193, 2129, 767, 3147, 78349, 8934, 198, 510, 1448, 29, 271, 1138, 25, 198, 27, 22446, 3364, 420, 8944, 731, 198, 109480, 97192, 96790, 111267, 104154, 1710, 198, 510, 22446, 29, 198, 271, 8679, 1396, 9111, 314, 279, 2100, 1414, 421, 22651, 799, 314, 279, 8928, 13, 198, 6205, 581, 25], "query": 91, "w_pos": [62, 62, 62, 63, 63, 64, 64, 65, 65, 66, 66, 66, 67]}], "trunc": {"shrink_cut": 0, "protected_cut": 0}, "seg_lens": {"user": 46, "context": 10, "answer": 6}, "layout": "c", "names": ["unsupported"], "w_offs": [[0, 1], [1, 2], [2, 3], [3, 4], [4, 5], [5, 6], [6, 7], [7, 8], [8, 9], [9, 10], [10, 11], [11, 12], [12, 13]], "w_pos": [62, 62, 62, 63, 63, 64, 64, 65, 65, 66, 66, 66, 67], "span_over": "answer", "n_tokens": 193, "head": "router", "head_reason": "label_set"}], "window_rows": []} diff --git a/tests/vela/fixtures/native-plan-example-request-score.json b/tests/vela/fixtures/native-plan-example-request-score.json new file mode 100644 index 00000000..fc67c0f8 --- /dev/null +++ b/tests/vela/fixtures/native-plan-example-request-score.json @@ -0,0 +1 @@ +{"case": "example-request-score", "request": {"model": "vllm-sr/Vela-2.0-0.8B", "state": "Hello, I would like to check the status of my order. It has been five days and I have heard nothing back.", "questions": {"tone": {"type": "score", "instructions": "How formal is the message?", "criteria": ["casual", "neutral", "formal"]}, "urgency": {"type": "score", "instructions": "How urgently does the sender need a reply?", "criteria": ["no rush", "soon", "today", "immediately"]}}}, "predict_request": {"parts": [{"type": "user", "text": "Hello, I would like to check the status of my order. It has been five days and I have heard nothing back."}], "questions": [{"id": "tone", "type": "score", "text": "How formal is the message?", "target_part": "user", "options": ["casual", "neutral", "formal"]}, {"id": "urgency", "type": "score", "text": "How urgently does the sender need a reply?", "target_part": "user", "options": ["no rush", "soon", "today", "immediately"]}]}, "plan": [{"qid": "tone", "type": "score", "criteria": ["casual", "neutral", "formal"], "range": null, "part": "user"}, {"qid": "urgency", "type": "score", "criteria": ["no rush", "soon", "today", "immediately"], "range": null, "part": "user"}], "rows": [{"p_ids": [1905, 25, 198, 27, 22446, 3364, 420, 846, 731, 198, 9419, 11, 353, 1000, 1040, 310, 1716, 279, 2552, 314, 821, 1906, 13, 1049, 682, 978, 4097, 2756, 321, 353, 599, 6408, 4161, 1142, 13, 198, 510, 22446, 29, 198], "blocks": [{"kind": "q", "ids": [271, 6065, 913, 25, 5284, 198, 6196, 25, 1156, 198, 14162, 25, 198, 4199, 15440, 369, 279, 1876, 30, 198, 3670, 25, 198, 13886, 29, 198, 4754, 4532, 58719, 767, 3147, 23543, 899, 8934, 198, 510, 1979, 29, 198, 13886, 29, 198, 4754, 4532, 58719, 767, 3147, 57555, 8934, 198, 510, 1979, 29, 198, 13886, 29, 198, 4754, 4532, 58719, 767, 3147, 610, 278, 8934, 198, 510, 1979, 29, 271, 3270, 279, 3074, 2904, 1786, 7021, 539, 279, 2193, 321, 10895, 13, 198, 72298, 25], "ends": [37, 52, 68], "query": 84, "qid": "tone", "qtype": "score", "names": ["casual", "neutral", "formal"], "has_abs": false}, {"kind": "q", "ids": [271, 6065, 913, 25, 5284, 198, 6196, 25, 1156, 198, 14162, 25, 198, 4199, 74000, 1503, 279, 4495, 1144, 264, 9559, 30, 198, 3670, 25, 198, 13886, 29, 198, 4754, 4532, 58719, 767, 3147, 2083, 12601, 8934, 198, 510, 1979, 29, 198, 13886, 29, 198, 4754, 4532, 58719, 767, 3147, 63826, 8934, 198, 510, 1979, 29, 198, 13886, 29, 198, 4754, 4532, 58719, 767, 3147, 29129, 8934, 198, 510, 1979, 29, 198, 13886, 29, 198, 4754, 4532, 58719, 767, 3147, 316, 41763, 8934, 198, 510, 1979, 29, 271, 3270, 279, 3074, 2904, 1786, 7021, 539, 279, 2193, 321, 10895, 13, 198, 72298, 25], "ends": [40, 55, 70, 86], "query": 102, "qid": "urgency", "qtype": "score", "names": ["no rush", "soon", "today", "immediately"], "has_abs": false}], "trunc": {"shrink_cut": 0, "protected_cut": 0}, "seg_lens": {"user": 25}, "layout": null, "n_tokens": 228, "head": "router", "head_reason": null}], "window_rows": []} diff --git a/tests/vela/fixtures/native-plan-example-request-set-threshold.json b/tests/vela/fixtures/native-plan-example-request-set-threshold.json new file mode 100644 index 00000000..d0ea9b8f --- /dev/null +++ b/tests/vela/fixtures/native-plan-example-request-set-threshold.json @@ -0,0 +1 @@ +{"case": "example-request-set-threshold", "request": {"model": "vllm-sr/Vela-2.0-0.8B", "state": "My card was charged twice and the parcel never arrived.", "questions": {"issues": {"type": "set", "instructions": "Which issues does the customer report?", "criteria": {"billing": "payments, charges, refunds or invoices", "shipping": "delivery of an order or a parcel", "login": "signing in, passwords or account access"}, "threshold": 0.15}}}, "predict_request": {"parts": [{"type": "user", "text": "My card was charged twice and the parcel never arrived."}], "questions": [{"id": "issues", "type": "set", "text": "Which issues does the customer report?", "target_part": "user", "options": [{"name": "billing", "description": "payments, charges, refunds or invoices"}, {"name": "shipping", "description": "delivery of an order or a parcel"}, {"name": "login", "description": "signing in, passwords or account access"}], "threshold": 0.15}]}, "plan": [{"qid": "issues", "type": "set", "criteria": {"billing": "payments, charges, refunds or invoices", "shipping": "delivery of an order or a parcel", "login": "signing in, passwords or account access"}, "range": null, "part": "user"}], "rows": [{"p_ids": [1905, 25, 198, 27, 22446, 3364, 420, 846, 731, 198, 4888, 3575, 557, 11102, 10598, 321, 279, 28350, 2496, 11136, 13, 198, 510, 22446, 29, 198], "blocks": [{"kind": "q", "ids": [271, 6065, 913, 25, 716, 198, 6196, 25, 1156, 198, 14162, 25, 198, 22365, 4562, 1503, 279, 5813, 1830, 30, 198, 3670, 25, 198, 13886, 29, 198, 4754, 4532, 3147, 60917, 11, 9892, 11, 70057, 466, 61893, 2129, 767, 3147, 37355, 8934, 198, 510, 1979, 29, 198, 13886, 29, 198, 4754, 4532, 3147, 31320, 314, 449, 1906, 466, 264, 28350, 2129, 767, 3147, 25660, 8934, 198, 510, 1979, 29, 198, 13886, 29, 198, 4754, 4532, 3147, 7506, 286, 303, 11, 32599, 466, 2605, 2528, 2129, 767, 3147, 3547, 8934, 198, 510, 1979, 29, 271, 3270, 1396, 2904, 7021, 539, 279, 2193, 321, 10895, 13, 198, 72298, 25], "ends": [45, 68, 92], "query": 106, "qid": "issues", "qtype": "set", "names": ["billing", "shipping", "login"], "has_abs": false}], "trunc": {"shrink_cut": 0, "protected_cut": 0}, "seg_lens": {"user": 11}, "layout": null, "n_tokens": 133, "head": "router", "head_reason": null}], "window_rows": []} diff --git a/tests/vela/fixtures/native-plan-example-request.json b/tests/vela/fixtures/native-plan-example-request.json new file mode 100644 index 00000000..09bab119 --- /dev/null +++ b/tests/vela/fixtures/native-plan-example-request.json @@ -0,0 +1 @@ +{"case": "example-request", "request": {"model": "vllm-sr/Vela-2.0-0.8B", "state": {"request": "Hi, I'm Tom Baker (tom.baker@example.com). What is the maximum daily dose of paracetamol for an adult?", "source": "For adults, the maximum dose of paracetamol is 4 grams in 24 hours, taken as 500 mg to 1 g every 4 to 6 hours.", "answer": "Adults can take up to 6 grams of paracetamol in 24 hours, in doses of 500 mg to 1 g every 4 to 6 hours."}, "questions": {"pii": {"type": "span", "instructions": "Which spans are personal information?", "criteria": {"AGE": "a person's age", "CREDIT_CARD": "a payment card number", "DATE_TIME": "a date, time or date of birth", "DOMAIN_NAME": "an internet domain name or website address", "EMAIL_ADDRESS": "an e-mail address", "GPE": "a country, city, state or other geopolitical place", "IBAN_CODE": "an international bank account number (IBAN)", "IP_ADDRESS": "an IPv4 or IPv6 address", "NRP": "a nationality, religious or political group", "ORGANIZATION": "the name of a company, institution or other organisation", "PERSON": "a person's name", "PHONE_NUMBER": "a telephone number", "STREET_ADDRESS": "a street address or postal address", "TITLE": "a personal or professional title such as Dr. or Mrs.", "US_DRIVER_LICENSE": "a US driver's licence number", "US_SSN": "a US social security number", "ZIP_CODE": "a postal or ZIP code"}, "over": "request"}, "halu": {"type": "span", "instructions": "Which spans of the answer are not supported by the context?", "criteria": {"unsupported": "a claim not supported by the context"}}, "domain": {"type": "choice", "instructions": "Which subject area is this request about?", "over": "request", "criteria": {"health": "medicine, clinical practice, nutrition, ageing or sexual health", "math": "arithmetic, algebra, geometry, statistics or other mathematics", "other": "a subject that fits none of the listed areas"}}}}, "predict_request": {"parts": [{"type": "user", "text": "Hi, I'm Tom Baker (tom.baker@example.com). What is the maximum daily dose of paracetamol for an adult?"}, {"type": "context", "text": "For adults, the maximum dose of paracetamol is 4 grams in 24 hours, taken as 500 mg to 1 g every 4 to 6 hours."}, {"type": "answer", "text": "Adults can take up to 6 grams of paracetamol in 24 hours, in doses of 500 mg to 1 g every 4 to 6 hours."}], "questions": [{"id": "pii", "type": "span", "text": "Which spans are personal information?", "target_part": "user", "labels": [{"name": "AGE", "description": "a person's age"}, {"name": "CREDIT_CARD", "description": "a payment card number"}, {"name": "DATE_TIME", "description": "a date, time or date of birth"}, {"name": "DOMAIN_NAME", "description": "an internet domain name or website address"}, {"name": "EMAIL_ADDRESS", "description": "an e-mail address"}, {"name": "GPE", "description": "a country, city, state or other geopolitical place"}, {"name": "IBAN_CODE", "description": "an international bank account number (IBAN)"}, {"name": "IP_ADDRESS", "description": "an IPv4 or IPv6 address"}, {"name": "NRP", "description": "a nationality, religious or political group"}, {"name": "ORGANIZATION", "description": "the name of a company, institution or other organisation"}, {"name": "PERSON", "description": "a person's name"}, {"name": "PHONE_NUMBER", "description": "a telephone number"}, {"name": "STREET_ADDRESS", "description": "a street address or postal address"}, {"name": "TITLE", "description": "a personal or professional title such as Dr. or Mrs."}, {"name": "US_DRIVER_LICENSE", "description": "a US driver's licence number"}, {"name": "US_SSN", "description": "a US social security number"}, {"name": "ZIP_CODE", "description": "a postal or ZIP code"}]}, {"id": "halu", "type": "span", "text": "Which spans of the answer are not supported by the context?", "target_part": "answer", "labels": [{"name": "unsupported", "description": "a claim not supported by the context"}]}, {"id": "domain", "type": "choice", "text": "Which subject area is this request about?", "target_part": "user", "options": [{"name": "health", "description": "medicine, clinical practice, nutrition, ageing or sexual health"}, {"name": "math", "description": "arithmetic, algebra, geometry, statistics or other mathematics"}, {"name": "other", "description": "a subject that fits none of the listed areas"}]}]}, "plan": [{"qid": "pii", "type": "span", "criteria": {"AGE": "a person's age", "CREDIT_CARD": "a payment card number", "DATE_TIME": "a date, time or date of birth", "DOMAIN_NAME": "an internet domain name or website address", "EMAIL_ADDRESS": "an e-mail address", "GPE": "a country, city, state or other geopolitical place", "IBAN_CODE": "an international bank account number (IBAN)", "IP_ADDRESS": "an IPv4 or IPv6 address", "NRP": "a nationality, religious or political group", "ORGANIZATION": "the name of a company, institution or other organisation", "PERSON": "a person's name", "PHONE_NUMBER": "a telephone number", "STREET_ADDRESS": "a street address or postal address", "TITLE": "a personal or professional title such as Dr. or Mrs.", "US_DRIVER_LICENSE": "a US driver's licence number", "US_SSN": "a US social security number", "ZIP_CODE": "a postal or ZIP code"}, "range": [0, 102], "part": "user"}, {"qid": "halu", "type": "span", "criteria": {"unsupported": "a claim not supported by the context"}, "range": null, "part": "answer"}, {"qid": "domain", "type": "choice", "criteria": {"health": "medicine, clinical practice, nutrition, ageing or sexual health", "math": "arithmetic, algebra, geometry, statistics or other mathematics", "other": "a subject that fits none of the listed areas"}, "range": [0, 102], "part": "user"}], "rows": [{"p_ids": [1905, 25, 198, 27, 22446, 3364, 420, 846, 731, 198, 12675, 11, 353, 2688, 8105, 27466, 318, 36164, 918, 4263, 34314, 877, 553, 3437, 369, 279, 6966, 7071, 18537, 314, 1298, 64955, 235579, 364, 449, 6474, 30, 198, 510, 22446, 29, 198, 27, 22446, 3364, 420, 2078, 731, 198, 2381, 12236, 11, 279, 6966, 18537, 314, 1298, 64955, 235579, 369, 220, 19, 32216, 303, 220, 17, 19, 3979, 11, 4284, 430, 220, 20, 15, 15, 13348, 310, 220, 16, 338, 1396, 220, 19, 310, 220, 21, 3979, 13, 198, 510, 22446, 29, 198, 27, 22446, 3364, 420, 8944, 731, 198, 50281, 82, 628, 1831, 685, 310, 220, 21, 32216, 314, 1298, 64955, 235579, 303, 220, 17, 19, 3979, 11, 303, 32902, 314, 220, 20, 15, 15, 13348, 310, 220, 16, 338, 1396, 220, 19, 310, 220, 21, 3979, 13, 198, 510, 22446, 29, 198], "blocks": [{"kind": "q", "ids": [271, 6065, 913, 25, 5576, 198, 6196, 25, 1156, 198, 14162, 25, 198, 22365, 3704, 2982, 369, 411, 1622, 883, 30, 198, 3670, 25, 198, 13886, 29, 198, 4754, 4532, 3147, 79600, 11, 14075, 6380, 11, 24780, 11, 76242, 466, 7017, 2730, 2129, 767, 3147, 11776, 8934, 198, 510, 1979, 29, 198, 13886, 29, 198, 4754, 4532, 3147, 277, 25092, 11, 45309, 11, 16547, 11, 12760, 466, 975, 36352, 2129, 767, 3147, 10065, 8934, 198, 510, 1979, 29, 198, 13886, 29, 198, 4754, 4532, 3147, 64, 3704, 421, 17759, 6643, 314, 279, 9711, 5493, 2129, 767, 3147, 1520, 8934, 198, 510, 1979, 29, 198, 13886, 29, 198, 4754, 4532, 3147, 3928, 314, 279, 6625, 2519, 16298, 46292, 767, 3147, 58, 55553, 60, 8934, 198, 510, 1979, 29, 271, 3270, 279, 3074, 2904, 1786, 7021, 539, 279, 2193, 321, 10895, 13, 198, 72298, 25], "ends": [50, 77, 102, 126], "query": 142, "qid": "domain", "qtype": "choice", "names": ["health", "math", "other"], "has_abs": true}, {"kind": "span", "ends": [41, 63, 88, 112, 133, 160, 186, 210, 234, 262, 282, 302, 326, 353, 377, 400, 422], "label_ranges": [[22, 41], [42, 63], [64, 88], [89, 112], [113, 133], [134, 160], [161, 186], [187, 210], [211, 234], [235, 262], [263, 282], [283, 302], [303, 326], [327, 353], [354, 377], [378, 400], [401, 422]], "words_in_block": true, "ids": [271, 6065, 913, 25, 9111, 198, 6196, 25, 1156, 198, 14162, 25, 198, 22365, 42824, 513, 4204, 1928, 30, 198, 22933, 25, 198, 14057, 29, 198, 4754, 4532, 3147, 64, 1637, 579, 4092, 2129, 767, 3147, 3466, 8934, 198, 510, 1448, 29, 198, 14057, 29, 198, 4754, 4532, 3147, 64, 7903, 3575, 1324, 2129, 767, 3147, 34, 54569, 44012, 8934, 198, 510, 1448, 29, 198, 14057, 29, 198, 4754, 4532, 3147, 64, 2321, 11, 854, 466, 2321, 314, 6968, 2129, 767, 3147, 6876, 9754, 8934, 198, 510, 1448, 29, 198, 14057, 29, 198, 4754, 4532, 3147, 276, 7361, 7697, 803, 466, 3779, 2534, 2129, 767, 3147, 55214, 4556, 8934, 198, 510, 1448, 29, 198, 14057, 29, 198, 4754, 4532, 3147, 276, 378, 11139, 2534, 2129, 767, 3147, 34714, 19522, 8934, 198, 510, 1448, 29, 198, 14057, 29, 198, 4754, 4532, 3147, 64, 3046, 11, 3177, 11, 1528, 466, 975, 83899, 1925, 2129, 767, 3147, 38, 1679, 8934, 198, 510, 1448, 29, 198, 14057, 29, 198, 4754, 4532, 3147, 276, 6286, 5883, 2605, 1324, 318, 3151, 1058, 77039, 767, 3147, 3151, 1058, 9724, 8934, 198, 510, 1448, 29, 198, 14057, 29, 198, 4754, 4532, 3147, 276, 30521, 19, 466, 30521, 21, 2534, 2129, 767, 3147, 3191, 19522, 8934, 198, 510, 1448, 29, 198, 14057, 29, 198, 4754, 4532, 3147, 64, 56284, 11, 10072, 466, 4788, 1809, 2129, 767, 3147, 45, 20993, 8934, 198, 510, 1448, 29, 198, 14057, 29, 198, 4754, 4532, 3147, 1719, 803, 314, 264, 2724, 11, 14467, 466, 975, 20767, 2129, 767, 3147, 43433, 1058, 47383, 8934, 198, 510, 1448, 29, 198, 14057, 29, 198, 4754, 4532, 3147, 64, 1637, 579, 803, 2129, 767, 3147, 89974, 8934, 198, 510, 1448, 29, 198, 14057, 29, 198, 4754, 4532, 3147, 64, 19919, 1324, 2129, 767, 3147, 72446, 18777, 8934, 198, 510, 1448, 29, 198, 14057, 29, 198, 4754, 4532, 3147, 64, 8329, 2534, 466, 38436, 2534, 2129, 767, 3147, 759, 54046, 19522, 8934, 198, 510, 1448, 29, 198, 14057, 29, 198, 4754, 4532, 3147, 64, 4204, 466, 6376, 2192, 1680, 430, 2832, 13, 466, 17093, 46292, 767, 3147, 48639, 8934, 198, 510, 1448, 29, 198, 14057, 29, 198, 4754, 4532, 3147, 64, 2201, 5404, 579, 28995, 1324, 2129, 767, 3147, 1966, 42585, 55630, 8934, 198, 510, 1448, 29, 198, 14057, 29, 198, 4754, 4532, 3147, 64, 2201, 3471, 4610, 1324, 2129, 767, 3147, 1966, 1063, 18393, 8934, 198, 510, 1448, 29, 198, 14057, 29, 198, 4754, 4532, 3147, 64, 38436, 466, 54778, 1970, 2129, 767, 3147, 59506, 9724, 8934, 198, 510, 1448, 29, 271, 1138, 25, 198, 27, 22446, 3364, 420, 846, 731, 198, 12675, 11, 353, 2688, 8105, 27466, 318, 36164, 918, 4263, 34314, 877, 553, 3437, 369, 279, 6966, 7071, 18537, 314, 1298, 64955, 235579, 364, 449, 6474, 30, 198, 510, 22446, 29, 198, 271, 8679, 1396, 9111, 314, 279, 2100, 1414, 421, 22651, 799, 314, 279, 8928, 13, 198, 6205, 581, 25], "query": 484, "w_pos": [434, 435, 436, 437, 437, 438, 439, 440, 441, 446, 446, 447, 448, 449, 450, 451, 452, 453, 454, 457, 458, 459, 460]}], "trunc": {"shrink_cut": 0, "protected_cut": 0}, "seg_lens": {"user": 27, "context": 39, "answer": 39}, "layout": "c", "names": ["AGE", "CREDIT_CARD", "DATE_TIME", "DOMAIN_NAME", "EMAIL_ADDRESS", "GPE", "IBAN_CODE", "IP_ADDRESS", "NRP", "ORGANIZATION", "PERSON", "PHONE_NUMBER", "STREET_ADDRESS", "TITLE", "US_DRIVER_LICENSE", "US_SSN", "ZIP_CODE"], "w_offs": [[0, 2], [2, 3], [4, 5], [5, 6], [6, 7], [8, 11], [12, 17], [18, 19], [19, 40], [40, 41], [41, 42], [43, 47], [48, 50], [51, 54], [55, 62], [63, 68], [69, 73], [74, 76], [77, 88], [89, 92], [93, 95], [96, 101], [101, 102]], "w_pos": [434, 435, 436, 437, 437, 438, 439, 440, 441, 446, 446, 447, 448, 449, 450, 451, 452, 453, 454, 457, 458, 459, 460], "span_over": "user", "n_tokens": 772, "head": "router", "head_reason": "id"}, {"p_ids": [1905, 25, 198, 27, 22446, 3364, 420, 846, 731, 198, 12675, 11, 353, 2688, 8105, 27466, 318, 36164, 918, 4263, 34314, 877, 553, 3437, 369, 279, 6966, 7071, 18537, 314, 1298, 64955, 235579, 364, 449, 6474, 30, 198, 510, 22446, 29, 198, 27, 22446, 3364, 420, 2078, 731, 198, 2381, 12236, 11, 279, 6966, 18537, 314, 1298, 64955, 235579, 369, 220, 19, 32216, 303, 220, 17, 19, 3979, 11, 4284, 430, 220, 20, 15, 15, 13348, 310, 220, 16, 338, 1396, 220, 19, 310, 220, 21, 3979, 13, 198, 510, 22446, 29, 198, 27, 22446, 3364, 420, 8944, 731, 198, 50281, 82, 628, 1831, 685, 310, 220, 21, 32216, 314, 1298, 64955, 235579, 303, 220, 17, 19, 3979, 11, 303, 32902, 314, 220, 20, 15, 15, 13348, 310, 220, 16, 338, 1396, 220, 19, 310, 220, 21, 3979, 13, 198, 510, 22446, 29, 198], "blocks": [{"kind": "span", "ends": [50], "label_ranges": [[28, 50]], "words_in_block": true, "ids": [271, 6065, 913, 25, 9111, 198, 6196, 25, 4087, 198, 14162, 25, 198, 22365, 42824, 314, 279, 4087, 513, 524, 7021, 539, 279, 2193, 30, 198, 22933, 25, 198, 14057, 29, 198, 4754, 4532, 3147, 64, 3591, 524, 7021, 539, 279, 2193, 2129, 767, 3147, 78349, 8934, 198, 510, 1448, 29, 271, 1138, 25, 198, 27, 22446, 3364, 420, 8944, 731, 198, 50281, 82, 628, 1831, 685, 310, 220, 21, 32216, 314, 1298, 64955, 235579, 303, 220, 17, 19, 3979, 11, 303, 32902, 314, 220, 20, 15, 15, 13348, 310, 220, 16, 338, 1396, 220, 19, 310, 220, 21, 3979, 13, 198, 510, 22446, 29, 198, 271, 8679, 1396, 9111, 314, 279, 2100, 1414, 421, 22651, 799, 314, 279, 8928, 13, 198, 6205, 581, 25], "query": 124, "w_pos": [62, 64, 65, 66, 67, 69, 70, 71, 72, 75, 77, 79, 80, 81, 82, 83, 85, 88, 89, 91, 92, 93, 95, 96, 98, 99, 100]}], "trunc": {"shrink_cut": 0, "protected_cut": 0}, "seg_lens": {"user": 27, "context": 39, "answer": 39}, "layout": "c", "names": ["unsupported"], "w_offs": [[0, 6], [7, 10], [11, 15], [16, 18], [19, 21], [22, 23], [24, 29], [30, 32], [33, 44], [45, 47], [48, 50], [51, 56], [56, 57], [58, 60], [61, 66], [67, 69], [70, 73], [74, 76], [77, 79], [80, 81], [82, 83], [84, 89], [90, 91], [92, 94], [95, 96], [97, 102], [102, 103]], "w_pos": [62, 64, 65, 66, 67, 69, 70, 71, 72, 75, 77, 79, 80, 81, 82, 83, 85, 88, 89, 91, 92, 93, 95, 96, 98, 99, 100], "span_over": "answer", "n_tokens": 269, "head": "router", "head_reason": "id"}], "window_rows": []} diff --git a/tests/vela/fixtures/native-plan-long-span-windowed.json b/tests/vela/fixtures/native-plan-long-span-windowed.json new file mode 100644 index 00000000..0210fa02 --- /dev/null +++ b/tests/vela/fixtures/native-plan-long-span-windowed.json @@ -0,0 +1 @@ +{"case": "long-span-windowed", "request": {"model": "vllm-sr/Vela-2.0-0.8B", "state": {"request": "Sentence 0 reports a routine delivery event. Sentence 1 reports a routine delivery event. Sentence 2 reports a routine delivery event. Sentence 3 reports a routine delivery event. Sentence 4 reports a routine delivery event. Sentence 5 reports a routine delivery event. Sentence 6 reports a routine delivery event. Sentence 7 reports a routine delivery event. Sentence 8 reports a routine delivery event. Sentence 9 reports a routine delivery event. Sentence 10 reports a routine delivery event. Sentence 11 reports a routine delivery event. Sentence 12 reports a routine delivery event. Sentence 13 reports a routine delivery event. Sentence 14 reports a routine delivery event. Sentence 15 reports a routine delivery event. Sentence 16 reports a routine delivery event. Sentence 17 reports a routine delivery event. Sentence 18 reports a routine delivery event. Sentence 19 reports a routine delivery event. Sentence 20 reports a routine delivery event. Sentence 21 reports a routine delivery event. Sentence 22 reports a routine delivery event. Sentence 23 reports a routine delivery event. Sentence 24 reports a routine delivery event. Sentence 25 reports a routine delivery event. Sentence 26 reports a routine delivery event. Sentence 27 reports a routine delivery event. Sentence 28 reports a routine delivery event. Sentence 29 reports a routine delivery event. Sentence 30 reports a routine delivery event. Sentence 31 reports a routine delivery event. Sentence 32 reports a routine delivery event. Sentence 33 reports a routine delivery event. Sentence 34 reports a routine delivery event. Sentence 35 reports a routine delivery event. Sentence 36 reports a routine delivery event. Sentence 37 reports a routine delivery event. Sentence 38 reports a routine delivery event. Sentence 39 reports a routine delivery event. Sentence 40 reports a routine delivery event. Sentence 41 reports a routine delivery event. Sentence 42 reports a routine delivery event. Sentence 43 reports a routine delivery event. Sentence 44 reports a routine delivery event. Sentence 45 reports a routine delivery event. Sentence 46 reports a routine delivery event. Sentence 47 reports a routine delivery event. Sentence 48 reports a routine delivery event. Sentence 49 reports a routine delivery event. Sentence 50 reports a routine delivery event. Sentence 51 reports a routine delivery event. Sentence 52 reports a routine delivery event. Sentence 53 reports a routine delivery event. Sentence 54 reports a routine delivery event. Sentence 55 reports a routine delivery event. Sentence 56 reports a routine delivery event. Sentence 57 reports a routine delivery event. Sentence 58 reports a routine delivery event. Sentence 59 reports a routine delivery event. Sentence 60 reports a routine delivery event. Sentence 61 reports a routine delivery event. Sentence 62 reports a routine delivery event. Sentence 63 reports a routine delivery event. Sentence 64 reports a routine delivery event. Sentence 65 reports a routine delivery event. Sentence 66 reports a routine delivery event. Sentence 67 reports a routine delivery event. Sentence 68 reports a routine delivery event. Sentence 69 reports a routine delivery event. Sentence 70 reports a routine delivery event. Sentence 71 reports a routine delivery event. Sentence 72 reports a routine delivery event. Sentence 73 reports a routine delivery event. Sentence 74 reports a routine delivery event. Sentence 75 reports a routine delivery event. Sentence 76 reports a routine delivery event. Sentence 77 reports a routine delivery event. Sentence 78 reports a routine delivery event. Sentence 79 reports a routine delivery event. Sentence 80 reports a routine delivery event. Sentence 81 reports a routine delivery event. Sentence 82 reports a routine delivery event. Sentence 83 reports a routine delivery event. Sentence 84 reports a routine delivery event. Sentence 85 reports a routine delivery event. Sentence 86 reports a routine delivery event. Sentence 87 reports a routine delivery event. Sentence 88 reports a routine delivery event. Sentence 89 reports a routine delivery event. Sentence 90 reports a routine delivery event. Sentence 91 reports a routine delivery event. Sentence 92 reports a routine delivery event. Sentence 93 reports a routine delivery event. Sentence 94 reports a routine delivery event. Sentence 95 reports a routine delivery event. Sentence 96 reports a routine delivery event. Sentence 97 reports a routine delivery event. Sentence 98 reports a routine delivery event. Sentence 99 reports a routine delivery event. Sentence 100 reports a routine delivery event. Sentence 101 reports a routine delivery event. Sentence 102 reports a routine delivery event. Sentence 103 reports a routine delivery event. Sentence 104 reports a routine delivery event. Sentence 105 reports a routine delivery event. Sentence 106 reports a routine delivery event. Sentence 107 reports a routine delivery event. Sentence 108 reports a routine delivery event. Sentence 109 reports a routine delivery event. Sentence 110 reports a routine delivery event. Sentence 111 reports a routine delivery event. Sentence 112 reports a routine delivery event. Sentence 113 reports a routine delivery event. Sentence 114 reports a routine delivery event. Sentence 115 reports a routine delivery event. Sentence 116 reports a routine delivery event. Sentence 117 reports a routine delivery event. Sentence 118 reports a routine delivery event. Sentence 119 reports a routine delivery event. Sentence 120 reports a routine delivery event. Sentence 121 reports a routine delivery event. Sentence 122 reports a routine delivery event. Sentence 123 reports a routine delivery event. Sentence 124 reports a routine delivery event. Sentence 125 reports a routine delivery event. Sentence 126 reports a routine delivery event. Sentence 127 reports a routine delivery event. Sentence 128 reports a routine delivery event. Sentence 129 reports a routine delivery event. Sentence 130 reports a routine delivery event. Sentence 131 reports a routine delivery event. Sentence 132 reports a routine delivery event. Sentence 133 reports a routine delivery event. Sentence 134 reports a routine delivery event. Sentence 135 reports a routine delivery event. Sentence 136 reports a routine delivery event. Sentence 137 reports a routine delivery event. Sentence 138 reports a routine delivery event. Sentence 139 reports a routine delivery event. Sentence 140 reports a routine delivery event. Sentence 141 reports a routine delivery event. Sentence 142 reports a routine delivery event. Sentence 143 reports a routine delivery event. Sentence 144 reports a routine delivery event. Sentence 145 reports a routine delivery event. Sentence 146 reports a routine delivery event. Sentence 147 reports a routine delivery event. Sentence 148 reports a routine delivery event. Sentence 149 reports a routine delivery event. Sentence 150 reports a routine delivery event. Sentence 151 reports a routine delivery event. Sentence 152 reports a routine delivery event. Sentence 153 reports a routine delivery event. Sentence 154 reports a routine delivery event. Sentence 155 reports a routine delivery event. Sentence 156 reports a routine delivery event. Sentence 157 reports a routine delivery event. Sentence 158 reports a routine delivery event. Sentence 159 reports a routine delivery event. Sentence 160 reports a routine delivery event. Sentence 161 reports a routine delivery event. Sentence 162 reports a routine delivery event. Sentence 163 reports a routine delivery event. Sentence 164 reports a routine delivery event. Sentence 165 reports a routine delivery event. Sentence 166 reports a routine delivery event. Sentence 167 reports a routine delivery event. Sentence 168 reports a routine delivery event. Sentence 169 reports a routine delivery event. Sentence 170 reports a routine delivery event. Sentence 171 reports a routine delivery event. Sentence 172 reports a routine delivery event. Sentence 173 reports a routine delivery event. Sentence 174 reports a routine delivery event. Sentence 175 reports a routine delivery event. Sentence 176 reports a routine delivery event. Sentence 177 reports a routine delivery event. Sentence 178 reports a routine delivery event. Sentence 179 reports a routine delivery event. Sentence 180 reports a routine delivery event. Sentence 181 reports a routine delivery event. Sentence 182 reports a routine delivery event. Sentence 183 reports a routine delivery event. Sentence 184 reports a routine delivery event. Sentence 185 reports a routine delivery event. Sentence 186 reports a routine delivery event. Sentence 187 reports a routine delivery event. Sentence 188 reports a routine delivery event. Sentence 189 reports a routine delivery event. Sentence 190 reports a routine delivery event. Sentence 191 reports a routine delivery event. Sentence 192 reports a routine delivery event. Sentence 193 reports a routine delivery event. Sentence 194 reports a routine delivery event. Sentence 195 reports a routine delivery event. Sentence 196 reports a routine delivery event. Sentence 197 reports a routine delivery event. Sentence 198 reports a routine delivery event. Sentence 199 reports a routine delivery event. Sentence 200 reports a routine delivery event. Sentence 201 reports a routine delivery event. Sentence 202 reports a routine delivery event. Sentence 203 reports a routine delivery event. Sentence 204 reports a routine delivery event. Sentence 205 reports a routine delivery event. Sentence 206 reports a routine delivery event. Sentence 207 reports a routine delivery event. Sentence 208 reports a routine delivery event. Sentence 209 reports a routine delivery event. Sentence 210 reports a routine delivery event. Sentence 211 reports a routine delivery event. Sentence 212 reports a routine delivery event. Sentence 213 reports a routine delivery event. Sentence 214 reports a routine delivery event. Sentence 215 reports a routine delivery event. Sentence 216 reports a routine delivery event. Sentence 217 reports a routine delivery event. Sentence 218 reports a routine delivery event. Sentence 219 reports a routine delivery event."}, "questions": {"claims": {"type": "span", "instructions": "Which spans mention a delivery?", "criteria": {"delivery": "a delivery event"}, "over": "request"}}}, "predict_request": {"parts": [{"type": "user", "text": "Sentence 0 reports a routine delivery event. Sentence 1 reports a routine delivery event. Sentence 2 reports a routine delivery event. Sentence 3 reports a routine delivery event. Sentence 4 reports a routine delivery event. Sentence 5 reports a routine delivery event. Sentence 6 reports a routine delivery event. Sentence 7 reports a routine delivery event. Sentence 8 reports a routine delivery event. Sentence 9 reports a routine delivery event. Sentence 10 reports a routine delivery event. Sentence 11 reports a routine delivery event. Sentence 12 reports a routine delivery event. Sentence 13 reports a routine delivery event. Sentence 14 reports a routine delivery event. Sentence 15 reports a routine delivery event. Sentence 16 reports a routine delivery event. Sentence 17 reports a routine delivery event. Sentence 18 reports a routine delivery event. Sentence 19 reports a routine delivery event. Sentence 20 reports a routine delivery event. Sentence 21 reports a routine delivery event. Sentence 22 reports a routine delivery event. Sentence 23 reports a routine delivery event. Sentence 24 reports a routine delivery event. Sentence 25 reports a routine delivery event. Sentence 26 reports a routine delivery event. Sentence 27 reports a routine delivery event. Sentence 28 reports a routine delivery event. Sentence 29 reports a routine delivery event. Sentence 30 reports a routine delivery event. Sentence 31 reports a routine delivery event. Sentence 32 reports a routine delivery event. Sentence 33 reports a routine delivery event. Sentence 34 reports a routine delivery event. Sentence 35 reports a routine delivery event. Sentence 36 reports a routine delivery event. Sentence 37 reports a routine delivery event. Sentence 38 reports a routine delivery event. Sentence 39 reports a routine delivery event. Sentence 40 reports a routine delivery event. Sentence 41 reports a routine delivery event. Sentence 42 reports a routine delivery event. Sentence 43 reports a routine delivery event. Sentence 44 reports a routine delivery event. Sentence 45 reports a routine delivery event. Sentence 46 reports a routine delivery event. Sentence 47 reports a routine delivery event. Sentence 48 reports a routine delivery event. Sentence 49 reports a routine delivery event. Sentence 50 reports a routine delivery event. Sentence 51 reports a routine delivery event. Sentence 52 reports a routine delivery event. Sentence 53 reports a routine delivery event. Sentence 54 reports a routine delivery event. Sentence 55 reports a routine delivery event. Sentence 56 reports a routine delivery event. Sentence 57 reports a routine delivery event. Sentence 58 reports a routine delivery event. Sentence 59 reports a routine delivery event. Sentence 60 reports a routine delivery event. Sentence 61 reports a routine delivery event. Sentence 62 reports a routine delivery event. Sentence 63 reports a routine delivery event. Sentence 64 reports a routine delivery event. Sentence 65 reports a routine delivery event. Sentence 66 reports a routine delivery event. Sentence 67 reports a routine delivery event. Sentence 68 reports a routine delivery event. Sentence 69 reports a routine delivery event. Sentence 70 reports a routine delivery event. Sentence 71 reports a routine delivery event. Sentence 72 reports a routine delivery event. Sentence 73 reports a routine delivery event. Sentence 74 reports a routine delivery event. Sentence 75 reports a routine delivery event. Sentence 76 reports a routine delivery event. Sentence 77 reports a routine delivery event. Sentence 78 reports a routine delivery event. Sentence 79 reports a routine delivery event. Sentence 80 reports a routine delivery event. Sentence 81 reports a routine delivery event. Sentence 82 reports a routine delivery event. Sentence 83 reports a routine delivery event. Sentence 84 reports a routine delivery event. Sentence 85 reports a routine delivery event. Sentence 86 reports a routine delivery event. Sentence 87 reports a routine delivery event. Sentence 88 reports a routine delivery event. Sentence 89 reports a routine delivery event. Sentence 90 reports a routine delivery event. Sentence 91 reports a routine delivery event. Sentence 92 reports a routine delivery event. Sentence 93 reports a routine delivery event. Sentence 94 reports a routine delivery event. Sentence 95 reports a routine delivery event. Sentence 96 reports a routine delivery event. Sentence 97 reports a routine delivery event. Sentence 98 reports a routine delivery event. Sentence 99 reports a routine delivery event. Sentence 100 reports a routine delivery event. Sentence 101 reports a routine delivery event. Sentence 102 reports a routine delivery event. Sentence 103 reports a routine delivery event. Sentence 104 reports a routine delivery event. Sentence 105 reports a routine delivery event. Sentence 106 reports a routine delivery event. Sentence 107 reports a routine delivery event. Sentence 108 reports a routine delivery event. Sentence 109 reports a routine delivery event. Sentence 110 reports a routine delivery event. Sentence 111 reports a routine delivery event. Sentence 112 reports a routine delivery event. Sentence 113 reports a routine delivery event. Sentence 114 reports a routine delivery event. Sentence 115 reports a routine delivery event. Sentence 116 reports a routine delivery event. Sentence 117 reports a routine delivery event. Sentence 118 reports a routine delivery event. Sentence 119 reports a routine delivery event. Sentence 120 reports a routine delivery event. Sentence 121 reports a routine delivery event. Sentence 122 reports a routine delivery event. Sentence 123 reports a routine delivery event. Sentence 124 reports a routine delivery event. Sentence 125 reports a routine delivery event. Sentence 126 reports a routine delivery event. Sentence 127 reports a routine delivery event. Sentence 128 reports a routine delivery event. Sentence 129 reports a routine delivery event. Sentence 130 reports a routine delivery event. Sentence 131 reports a routine delivery event. Sentence 132 reports a routine delivery event. Sentence 133 reports a routine delivery event. Sentence 134 reports a routine delivery event. Sentence 135 reports a routine delivery event. Sentence 136 reports a routine delivery event. Sentence 137 reports a routine delivery event. Sentence 138 reports a routine delivery event. Sentence 139 reports a routine delivery event. Sentence 140 reports a routine delivery event. Sentence 141 reports a routine delivery event. Sentence 142 reports a routine delivery event. Sentence 143 reports a routine delivery event. Sentence 144 reports a routine delivery event. Sentence 145 reports a routine delivery event. Sentence 146 reports a routine delivery event. Sentence 147 reports a routine delivery event. Sentence 148 reports a routine delivery event. Sentence 149 reports a routine delivery event. Sentence 150 reports a routine delivery event. Sentence 151 reports a routine delivery event. Sentence 152 reports a routine delivery event. Sentence 153 reports a routine delivery event. Sentence 154 reports a routine delivery event. Sentence 155 reports a routine delivery event. Sentence 156 reports a routine delivery event. Sentence 157 reports a routine delivery event. Sentence 158 reports a routine delivery event. Sentence 159 reports a routine delivery event. Sentence 160 reports a routine delivery event. Sentence 161 reports a routine delivery event. Sentence 162 reports a routine delivery event. Sentence 163 reports a routine delivery event. Sentence 164 reports a routine delivery event. Sentence 165 reports a routine delivery event. Sentence 166 reports a routine delivery event. Sentence 167 reports a routine delivery event. Sentence 168 reports a routine delivery event. Sentence 169 reports a routine delivery event. Sentence 170 reports a routine delivery event. Sentence 171 reports a routine delivery event. Sentence 172 reports a routine delivery event. Sentence 173 reports a routine delivery event. Sentence 174 reports a routine delivery event. Sentence 175 reports a routine delivery event. Sentence 176 reports a routine delivery event. Sentence 177 reports a routine delivery event. Sentence 178 reports a routine delivery event. Sentence 179 reports a routine delivery event. Sentence 180 reports a routine delivery event. Sentence 181 reports a routine delivery event. Sentence 182 reports a routine delivery event. Sentence 183 reports a routine delivery event. Sentence 184 reports a routine delivery event. Sentence 185 reports a routine delivery event. Sentence 186 reports a routine delivery event. Sentence 187 reports a routine delivery event. Sentence 188 reports a routine delivery event. Sentence 189 reports a routine delivery event. Sentence 190 reports a routine delivery event. Sentence 191 reports a routine delivery event. Sentence 192 reports a routine delivery event. Sentence 193 reports a routine delivery event. Sentence 194 reports a routine delivery event. Sentence 195 reports a routine delivery event. Sentence 196 reports a routine delivery event. Sentence 197 reports a routine delivery event. Sentence 198 reports a routine delivery event. Sentence 199 reports a routine delivery event. Sentence 200 reports a routine delivery event. Sentence 201 reports a routine delivery event. Sentence 202 reports a routine delivery event. Sentence 203 reports a routine delivery event. Sentence 204 reports a routine delivery event. Sentence 205 reports a routine delivery event. Sentence 206 reports a routine delivery event. Sentence 207 reports a routine delivery event. Sentence 208 reports a routine delivery event. Sentence 209 reports a routine delivery event. Sentence 210 reports a routine delivery event. Sentence 211 reports a routine delivery event. Sentence 212 reports a routine delivery event. Sentence 213 reports a routine delivery event. Sentence 214 reports a routine delivery event. Sentence 215 reports a routine delivery event. Sentence 216 reports a routine delivery event. Sentence 217 reports a routine delivery event. Sentence 218 reports a routine delivery event. Sentence 219 reports a routine delivery event."}], "questions": [{"id": "claims", "type": "span", "text": "Which spans mention a delivery?", "target_part": "user", "labels": [{"name": "delivery", "description": "a delivery event"}]}]}, "plan": [{"qid": "claims", "type": "span", "criteria": {"delivery": "a delivery event"}, "range": [0, 10229], "part": "user"}], "rows": [{"p_ids": [1905, 25, 198, 27, 22446, 3364, 420, 846, 731, 198, 81646, 220, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 24, 6473, 264, 13618, 9403, 1484, 13, 198, 510, 22446, 29, 198], "blocks": [{"kind": "span", "ends": [40], "label_ranges": [[22, 40]], "words_in_block": false, "ids": [271, 6065, 913, 25, 9111, 198, 6196, 25, 1156, 198, 14162, 25, 198, 22365, 42824, 6088, 264, 9403, 30, 198, 22933, 25, 198, 14057, 29, 198, 4754, 4532, 3147, 64, 9403, 1484, 2129, 767, 3147, 31320, 8934, 198, 510, 1448, 29, 271, 8679, 1396, 9111, 314, 279, 2100, 1414, 421, 22651, 799, 314, 279, 8928, 13, 198, 6205, 581, 25], "query": 59, "w_pos": [10, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 23, 24, 25, 26, 27, 28, 30, 31, 32, 33, 34, 35, 36, 37, 39, 40, 41, 42, 43, 44, 45, 46, 48, 49, 50, 51, 52, 53, 54, 55, 57, 58, 59, 60, 61, 62, 63, 64, 66, 67, 68, 69, 70, 71, 72, 73, 75, 76, 77, 78, 79, 80, 81, 82, 84, 85, 86, 87, 88, 89, 90, 91, 93, 94, 95, 96, 97, 98, 99, 100, 102, 104, 105, 106, 107, 108, 109, 110, 112, 114, 115, 116, 117, 118, 119, 120, 122, 124, 125, 126, 127, 128, 129, 130, 132, 134, 135, 136, 137, 138, 139, 140, 142, 144, 145, 146, 147, 148, 149, 150, 152, 154, 155, 156, 157, 158, 159, 160, 162, 164, 165, 166, 167, 168, 169, 170, 172, 174, 175, 176, 177, 178, 179, 180, 182, 184, 185, 186, 187, 188, 189, 190, 192, 194, 195, 196, 197, 198, 199, 200, 202, 204, 205, 206, 207, 208, 209, 210, 212, 214, 215, 216, 217, 218, 219, 220, 222, 224, 225, 226, 227, 228, 229, 230, 232, 234, 235, 236, 237, 238, 239, 240, 242, 244, 245, 246, 247, 248, 249, 250, 252, 254, 255, 256, 257, 258, 259, 260, 262, 264, 265, 266, 267, 268, 269, 270, 272, 274, 275, 276, 277, 278, 279, 280, 282, 284, 285, 286, 287, 288, 289, 290, 292, 294, 295, 296, 297, 298, 299, 300, 302, 304, 305, 306, 307, 308, 309, 310, 312, 314, 315, 316, 317, 318, 319, 320, 322, 324, 325, 326, 327, 328, 329, 330, 332, 334, 335, 336, 337, 338, 339, 340, 342, 344, 345, 346, 347, 348, 349, 350, 352, 354, 355, 356, 357, 358, 359, 360, 362, 364, 365, 366, 367, 368, 369, 370, 372, 374, 375, 376, 377, 378, 379, 380, 382, 384, 385, 386, 387, 388, 389, 390, 392, 394, 395, 396, 397, 398, 399, 400, 402, 404, 405, 406, 407, 408, 409, 410, 412, 414, 415, 416, 417, 418, 419, 420, 422, 424, 425, 426, 427, 428, 429, 430, 432, 434, 435, 436, 437, 438, 439, 440, 442, 444, 445, 446, 447, 448, 449, 450, 452, 454, 455, 456, 457, 458, 459, 460, 462, 464, 465, 466, 467, 468, 469, 470, 472, 474, 475, 476, 477, 478, 479, 480, 482, 484, 485, 486, 487, 488, 489, 490, 492, 494, 495, 496, 497, 498, 499, 500, 502, 504, 505, 506, 507, 508, 509, 510, 512, 514, 515, 516, 517, 518, 519, 520, 522, 524, 525, 526, 527, 528, 529, 530, 532, 534, 535, 536, 537, 538, 539, 540, 542, 544, 545, 546, 547, 548, 549, 550, 552, 554, 555, 556, 557, 558, 559, 560, 562, 564, 565, 566, 567, 568, 569, 570, 572, 574, 575, 576, 577, 578, 579, 580, 582, 584, 585, 586, 587, 588, 589, 590, 592, 594, 595, 596, 597, 598, 599, 600, 602, 604, 605, 606, 607, 608, 609, 610, 612, 614, 615, 616, 617, 618, 619, 620, 622, 624, 625, 626, 627, 628, 629, 630, 632, 634, 635, 636, 637, 638, 639, 640, 642, 644, 645, 646, 647, 648, 649, 650, 652, 654, 655, 656, 657, 658, 659, 660, 662, 664, 665, 666, 667, 668, 669, 670, 672, 674, 675, 676, 677, 678, 679, 680, 682, 684, 685, 686, 687, 688, 689, 690, 692, 694, 695, 696, 697, 698, 699, 700, 702, 704, 705, 706, 707, 708, 709, 710, 712, 714, 715, 716, 717, 718, 719, 720, 722, 724, 725, 726, 727, 728, 729, 730, 732, 734, 735, 736, 737, 738, 739, 740, 742, 744, 745, 746, 747, 748, 749, 750, 752, 754, 755, 756, 757, 758, 759, 760, 762, 764, 765, 766, 767, 768, 769, 770, 772, 774, 775, 776, 777, 778, 779, 780, 782, 784, 785, 786, 787, 788, 789, 790, 792, 794, 795, 796, 797, 798, 799, 800, 802, 804, 805, 806, 807, 808, 809, 810, 812, 814, 815, 816, 817, 818, 819, 820, 822, 824, 825, 826, 827, 828, 829, 830, 832, 834, 835, 836, 837, 838, 839, 840, 842, 844, 845, 846, 847, 848, 849, 850, 852, 854, 855, 856, 857, 858, 859, 860, 862, 864, 865, 866, 867, 868, 869, 870, 872, 874, 875, 876, 877, 878, 879, 880, 882, 884, 885, 886, 887, 888, 889, 890, 892, 894, 895, 896, 897, 898, 899, 900, 902, 904, 905, 906, 907, 908, 909, 910, 912, 914, 915, 916, 917, 918, 919, 920, 922, 924, 925, 926, 927, 928, 929, 930, 932, 934, 935, 936, 937, 938, 939, 940, 942, 944, 945, 946, 947, 948, 949, 950, 952, 954, 955, 956, 957, 958, 959, 960, 962, 964, 965, 966, 967, 968, 969, 970, 972, 974, 975, 976, 977, 978, 979, 980, 982, 984, 985, 986, 987, 988, 989, 990, 992, 994, 995, 996, 997, 998, 999, 1000, 1002, 1005, 1006, 1007, 1008, 1009, 1010, 1011, 1013, 1016, 1017, 1018, 1019, 1020, 1021, 1022, 1024, 1027, 1028, 1029, 1030, 1031, 1032, 1033, 1035, 1038, 1039, 1040, 1041, 1042, 1043, 1044, 1046, 1049, 1050, 1051, 1052, 1053, 1054, 1055, 1057, 1060, 1061, 1062, 1063, 1064, 1065, 1066, 1068, 1071, 1072, 1073, 1074, 1075, 1076, 1077, 1079, 1082, 1083, 1084, 1085, 1086, 1087, 1088, 1090, 1093, 1094, 1095, 1096, 1097, 1098, 1099, 1101, 1104, 1105, 1106, 1107, 1108, 1109, 1110, 1112, 1115, 1116, 1117, 1118, 1119, 1120, 1121, 1123, 1126, 1127, 1128, 1129, 1130, 1131, 1132, 1134, 1137, 1138, 1139, 1140, 1141, 1142, 1143, 1145, 1148, 1149, 1150, 1151, 1152, 1153, 1154, 1156, 1159, 1160, 1161, 1162, 1163, 1164, 1165, 1167, 1170, 1171, 1172, 1173, 1174, 1175, 1176, 1178, 1181, 1182, 1183, 1184, 1185, 1186, 1187, 1189, 1192, 1193, 1194, 1195, 1196, 1197, 1198, 1200, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1211, 1214, 1215, 1216, 1217, 1218, 1219, 1220, 1222, 1225, 1226, 1227, 1228, 1229, 1230, 1231, 1233, 1236, 1237, 1238, 1239, 1240, 1241, 1242, 1244, 1247, 1248, 1249, 1250, 1251, 1252, 1253, 1255, 1258, 1259, 1260, 1261, 1262, 1263, 1264, 1266, 1269, 1270, 1271, 1272, 1273, 1274, 1275, 1277, 1280, 1281, 1282, 1283, 1284, 1285, 1286, 1288, 1291, 1292, 1293, 1294, 1295, 1296, 1297, 1299, 1302, 1303, 1304, 1305, 1306, 1307, 1308, 1310, 1313, 1314, 1315, 1316, 1317, 1318, 1319, 1321, 1324, 1325, 1326, 1327, 1328, 1329, 1330, 1332, 1335, 1336, 1337, 1338, 1339, 1340, 1341, 1343, 1346, 1347, 1348, 1349, 1350, 1351, 1352, 1354, 1357, 1358, 1359, 1360, 1361, 1362, 1363, 1365, 1368, 1369, 1370, 1371, 1372, 1373, 1374, 1376, 1379, 1380, 1381, 1382, 1383, 1384, 1385, 1387, 1390, 1391, 1392, 1393, 1394, 1395, 1396, 1398, 1401, 1402, 1403, 1404, 1405, 1406, 1407, 1409, 1412, 1413, 1414, 1415, 1416, 1417, 1418, 1420, 1423, 1424, 1425, 1426, 1427, 1428, 1429, 1431, 1434, 1435, 1436, 1437, 1438, 1439, 1440, 1442, 1445, 1446, 1447, 1448, 1449, 1450, 1451, 1453, 1456, 1457, 1458, 1459, 1460, 1461, 1462, 1464, 1467, 1468, 1469, 1470, 1471, 1472, 1473, 1475, 1478, 1479, 1480, 1481, 1482, 1483, 1484, 1486, 1489, 1490, 1491, 1492, 1493, 1494, 1495, 1497, 1500, 1501, 1502, 1503, 1504, 1505, 1506, 1508, 1511, 1512, 1513, 1514, 1515, 1516, 1517, 1519, 1522, 1523, 1524, 1525, 1526, 1527, 1528, 1530, 1533, 1534, 1535, 1536, 1537, 1538, 1539, 1541, 1544, 1545, 1546, 1547, 1548, 1549, 1550, 1552, 1555, 1556, 1557, 1558, 1559, 1560, 1561, 1563, 1566, 1567, 1568, 1569, 1570, 1571, 1572, 1574, 1577, 1578, 1579, 1580, 1581, 1582, 1583, 1585, 1588, 1589, 1590, 1591, 1592, 1593, 1594, 1596, 1599, 1600, 1601, 1602, 1603, 1604, 1605, 1607, 1610, 1611, 1612, 1613, 1614, 1615, 1616, 1618, 1621, 1622, 1623, 1624, 1625, 1626, 1627, 1629, 1632, 1633, 1634, 1635, 1636, 1637, 1638, 1640, 1643, 1644, 1645, 1646, 1647, 1648, 1649, 1651, 1654, 1655, 1656, 1657, 1658, 1659, 1660, 1662, 1665, 1666, 1667, 1668, 1669, 1670, 1671, 1673, 1676, 1677, 1678, 1679, 1680, 1681, 1682, 1684, 1687, 1688, 1689, 1690, 1691, 1692, 1693, 1695, 1698, 1699, 1700, 1701, 1702, 1703, 1704, 1706, 1709, 1710, 1711, 1712, 1713, 1714, 1715, 1717, 1720, 1721, 1722, 1723, 1724, 1725, 1726, 1728, 1731, 1732, 1733, 1734, 1735, 1736, 1737, 1739, 1742, 1743, 1744, 1745, 1746, 1747, 1748, 1750, 1753, 1754, 1755, 1756, 1757, 1758, 1759, 1761, 1764, 1765, 1766, 1767, 1768, 1769, 1770, 1772, 1775, 1776, 1777, 1778, 1779, 1780, 1781, 1783, 1786, 1787, 1788, 1789, 1790, 1791, 1792, 1794, 1797, 1798, 1799, 1800, 1801, 1802, 1803, 1805, 1808, 1809, 1810, 1811, 1812, 1813, 1814, 1816, 1819, 1820, 1821, 1822, 1823, 1824, 1825, 1827, 1830, 1831, 1832, 1833, 1834, 1835, 1836, 1838, 1841, 1842, 1843, 1844, 1845, 1846, 1847, 1849, 1852, 1853, 1854, 1855, 1856, 1857, 1858, 1860, 1863, 1864, 1865, 1866, 1867, 1868, 1869, 1871, 1874, 1875, 1876, 1877, 1878, 1879, 1880, 1882, 1885, 1886, 1887, 1888, 1889, 1890, 1891, 1893, 1896, 1897, 1898, 1899, 1900, 1901, 1902, 1904, 1907, 1908, 1909, 1910, 1911, 1912, 1913, 1915, 1918, 1919, 1920, 1921, 1922, 1923, 1924, 1926, 1929, 1930, 1931, 1932, 1933, 1934, 1935, 1937, 1940, 1941, 1942, 1943, 1944, 1945, 1946, 1948, 1951, 1952, 1953, 1954, 1955, 1956, 1957, 1959, 1962, 1963, 1964, 1965, 1966, 1967, 1968, 1970, 1973, 1974, 1975, 1976, 1977, 1978, 1979, 1981, 1984, 1985, 1986, 1987, 1988, 1989, 1990, 1992, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2003, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2014, 2017, 2018, 2019, 2020, 2021, 2022, 2023, 2025, 2028, 2029, 2030, 2031, 2032, 2033, 2034, 2036, 2039, 2040, 2041, 2042, 2043, 2044, 2045, 2047, 2050, 2051, 2052, 2053, 2054, 2055, 2056, 2058, 2061, 2062, 2063, 2064, 2065, 2066, 2067, 2069, 2072, 2073, 2074, 2075, 2076, 2077, 2078, 2080, 2083, 2084, 2085, 2086, 2087, 2088, 2089, 2091, 2094, 2095, 2096, 2097, 2098, 2099, 2100, 2102, 2105, 2106, 2107, 2108, 2109, 2110, 2111, 2113, 2116, 2117, 2118, 2119, 2120, 2121, 2122, 2124, 2127, 2128, 2129, 2130, 2131, 2132, 2133, 2135, 2138, 2139, 2140, 2141, 2142, 2143, 2144, 2146, 2149, 2150, 2151, 2152, 2153, 2154, 2155, 2157, 2160, 2161, 2162, 2163, 2164, 2165, 2166, 2168, 2171, 2172, 2173, 2174, 2175, 2176, 2177, 2179, 2182, 2183, 2184, 2185, 2186, 2187, 2188, 2190, 2193, 2194, 2195, 2196, 2197, 2198, 2199, 2201, 2204, 2205, 2206, 2207, 2208, 2209, 2210, 2212, 2215, 2216, 2217, 2218, 2219, 2220, 2221, 2223, 2226, 2227, 2228, 2229, 2230, 2231, 2232, 2234, 2237, 2238, 2239, 2240, 2241, 2242, 2243, 2245, 2248, 2249, 2250, 2251, 2252, 2253, 2254, 2256, 2259, 2260, 2261, 2262, 2263, 2264, 2265, 2267, 2270, 2271, 2272, 2273, 2274, 2275, 2276, 2278, 2281, 2282, 2283, 2284, 2285, 2286, 2287, 2289, 2292, 2293, 2294, 2295, 2296, 2297, 2298, 2300, 2303, 2304, 2305, 2306, 2307, 2308, 2309, 2311, 2314, 2315, 2316, 2317, 2318, 2319]}], "trunc": {"shrink_cut": 0, "protected_cut": 0}, "seg_lens": {"user": 2310}, "layout": "a", "names": ["delivery"], "w_offs": [[0, 8], [9, 10], [11, 18], [19, 20], [21, 28], [29, 37], [38, 43], [43, 44], [45, 53], [54, 55], [56, 63], [64, 65], [66, 73], [74, 82], [83, 88], [88, 89], [90, 98], [99, 100], [101, 108], [109, 110], [111, 118], [119, 127], [128, 133], [133, 134], [135, 143], [144, 145], [146, 153], [154, 155], [156, 163], [164, 172], [173, 178], [178, 179], [180, 188], [189, 190], [191, 198], [199, 200], [201, 208], [209, 217], [218, 223], [223, 224], [225, 233], [234, 235], [236, 243], [244, 245], [246, 253], [254, 262], [263, 268], [268, 269], [270, 278], [279, 280], [281, 288], [289, 290], [291, 298], [299, 307], [308, 313], [313, 314], [315, 323], [324, 325], [326, 333], [334, 335], [336, 343], [344, 352], [353, 358], [358, 359], [360, 368], [369, 370], [371, 378], [379, 380], [381, 388], [389, 397], [398, 403], [403, 404], [405, 413], [414, 415], [416, 423], [424, 425], [426, 433], [434, 442], [443, 448], [448, 449], [450, 458], [459, 461], [462, 469], [470, 471], [472, 479], [480, 488], [489, 494], [494, 495], [496, 504], [505, 507], [508, 515], [516, 517], [518, 525], [526, 534], [535, 540], [540, 541], [542, 550], [551, 553], [554, 561], [562, 563], [564, 571], [572, 580], [581, 586], [586, 587], [588, 596], [597, 599], [600, 607], [608, 609], [610, 617], [618, 626], [627, 632], [632, 633], [634, 642], [643, 645], [646, 653], [654, 655], [656, 663], [664, 672], [673, 678], [678, 679], [680, 688], [689, 691], [692, 699], [700, 701], [702, 709], [710, 718], [719, 724], [724, 725], [726, 734], [735, 737], [738, 745], [746, 747], [748, 755], [756, 764], [765, 770], [770, 771], [772, 780], [781, 783], [784, 791], [792, 793], [794, 801], [802, 810], [811, 816], [816, 817], [818, 826], [827, 829], [830, 837], [838, 839], [840, 847], [848, 856], [857, 862], [862, 863], [864, 872], [873, 875], [876, 883], [884, 885], [886, 893], [894, 902], [903, 908], [908, 909], [910, 918], [919, 921], [922, 929], [930, 931], [932, 939], [940, 948], [949, 954], [954, 955], [956, 964], [965, 967], [968, 975], [976, 977], [978, 985], [986, 994], [995, 1000], [1000, 1001], [1002, 1010], [1011, 1013], [1014, 1021], [1022, 1023], [1024, 1031], [1032, 1040], [1041, 1046], [1046, 1047], [1048, 1056], [1057, 1059], [1060, 1067], [1068, 1069], [1070, 1077], [1078, 1086], [1087, 1092], [1092, 1093], [1094, 1102], [1103, 1105], [1106, 1113], [1114, 1115], [1116, 1123], [1124, 1132], [1133, 1138], [1138, 1139], [1140, 1148], [1149, 1151], [1152, 1159], [1160, 1161], [1162, 1169], [1170, 1178], [1179, 1184], [1184, 1185], [1186, 1194], [1195, 1197], [1198, 1205], [1206, 1207], [1208, 1215], [1216, 1224], [1225, 1230], [1230, 1231], [1232, 1240], [1241, 1243], [1244, 1251], [1252, 1253], [1254, 1261], [1262, 1270], [1271, 1276], [1276, 1277], [1278, 1286], [1287, 1289], [1290, 1297], [1298, 1299], [1300, 1307], [1308, 1316], [1317, 1322], [1322, 1323], [1324, 1332], [1333, 1335], [1336, 1343], [1344, 1345], [1346, 1353], [1354, 1362], [1363, 1368], [1368, 1369], [1370, 1378], [1379, 1381], [1382, 1389], [1390, 1391], [1392, 1399], [1400, 1408], [1409, 1414], [1414, 1415], [1416, 1424], [1425, 1427], [1428, 1435], [1436, 1437], [1438, 1445], [1446, 1454], [1455, 1460], [1460, 1461], [1462, 1470], [1471, 1473], [1474, 1481], [1482, 1483], [1484, 1491], [1492, 1500], [1501, 1506], [1506, 1507], [1508, 1516], [1517, 1519], [1520, 1527], [1528, 1529], [1530, 1537], [1538, 1546], [1547, 1552], [1552, 1553], [1554, 1562], [1563, 1565], [1566, 1573], [1574, 1575], [1576, 1583], [1584, 1592], [1593, 1598], [1598, 1599], [1600, 1608], [1609, 1611], [1612, 1619], [1620, 1621], [1622, 1629], [1630, 1638], [1639, 1644], [1644, 1645], [1646, 1654], [1655, 1657], [1658, 1665], [1666, 1667], [1668, 1675], [1676, 1684], [1685, 1690], [1690, 1691], [1692, 1700], [1701, 1703], [1704, 1711], [1712, 1713], [1714, 1721], [1722, 1730], [1731, 1736], [1736, 1737], [1738, 1746], [1747, 1749], [1750, 1757], [1758, 1759], [1760, 1767], [1768, 1776], [1777, 1782], [1782, 1783], [1784, 1792], [1793, 1795], [1796, 1803], [1804, 1805], [1806, 1813], [1814, 1822], [1823, 1828], [1828, 1829], [1830, 1838], [1839, 1841], [1842, 1849], [1850, 1851], [1852, 1859], [1860, 1868], [1869, 1874], [1874, 1875], [1876, 1884], [1885, 1887], [1888, 1895], [1896, 1897], [1898, 1905], [1906, 1914], [1915, 1920], [1920, 1921], [1922, 1930], [1931, 1933], [1934, 1941], [1942, 1943], [1944, 1951], [1952, 1960], [1961, 1966], [1966, 1967], [1968, 1976], [1977, 1979], [1980, 1987], [1988, 1989], [1990, 1997], [1998, 2006], [2007, 2012], [2012, 2013], [2014, 2022], [2023, 2025], [2026, 2033], [2034, 2035], [2036, 2043], [2044, 2052], [2053, 2058], [2058, 2059], [2060, 2068], [2069, 2071], [2072, 2079], [2080, 2081], [2082, 2089], [2090, 2098], [2099, 2104], [2104, 2105], [2106, 2114], [2115, 2117], [2118, 2125], [2126, 2127], [2128, 2135], [2136, 2144], [2145, 2150], [2150, 2151], [2152, 2160], [2161, 2163], [2164, 2171], [2172, 2173], [2174, 2181], [2182, 2190], [2191, 2196], [2196, 2197], [2198, 2206], [2207, 2209], [2210, 2217], [2218, 2219], [2220, 2227], [2228, 2236], [2237, 2242], [2242, 2243], [2244, 2252], [2253, 2255], [2256, 2263], [2264, 2265], [2266, 2273], [2274, 2282], [2283, 2288], [2288, 2289], [2290, 2298], [2299, 2301], [2302, 2309], [2310, 2311], [2312, 2319], [2320, 2328], [2329, 2334], [2334, 2335], [2336, 2344], [2345, 2347], [2348, 2355], [2356, 2357], [2358, 2365], [2366, 2374], [2375, 2380], [2380, 2381], [2382, 2390], [2391, 2393], [2394, 2401], [2402, 2403], [2404, 2411], [2412, 2420], [2421, 2426], [2426, 2427], [2428, 2436], [2437, 2439], [2440, 2447], [2448, 2449], [2450, 2457], [2458, 2466], [2467, 2472], [2472, 2473], [2474, 2482], [2483, 2485], [2486, 2493], [2494, 2495], [2496, 2503], [2504, 2512], [2513, 2518], [2518, 2519], [2520, 2528], [2529, 2531], [2532, 2539], [2540, 2541], [2542, 2549], [2550, 2558], [2559, 2564], [2564, 2565], [2566, 2574], [2575, 2577], [2578, 2585], [2586, 2587], [2588, 2595], [2596, 2604], [2605, 2610], [2610, 2611], [2612, 2620], [2621, 2623], [2624, 2631], [2632, 2633], [2634, 2641], [2642, 2650], [2651, 2656], [2656, 2657], [2658, 2666], [2667, 2669], [2670, 2677], [2678, 2679], [2680, 2687], [2688, 2696], [2697, 2702], [2702, 2703], [2704, 2712], [2713, 2715], [2716, 2723], [2724, 2725], [2726, 2733], [2734, 2742], [2743, 2748], [2748, 2749], [2750, 2758], [2759, 2761], [2762, 2769], [2770, 2771], [2772, 2779], [2780, 2788], [2789, 2794], [2794, 2795], [2796, 2804], [2805, 2807], [2808, 2815], [2816, 2817], [2818, 2825], [2826, 2834], [2835, 2840], [2840, 2841], [2842, 2850], [2851, 2853], [2854, 2861], [2862, 2863], [2864, 2871], [2872, 2880], [2881, 2886], [2886, 2887], [2888, 2896], [2897, 2899], [2900, 2907], [2908, 2909], [2910, 2917], [2918, 2926], [2927, 2932], [2932, 2933], [2934, 2942], [2943, 2945], [2946, 2953], [2954, 2955], [2956, 2963], [2964, 2972], [2973, 2978], [2978, 2979], [2980, 2988], [2989, 2991], [2992, 2999], [3000, 3001], [3002, 3009], [3010, 3018], [3019, 3024], [3024, 3025], [3026, 3034], [3035, 3037], [3038, 3045], [3046, 3047], [3048, 3055], [3056, 3064], [3065, 3070], [3070, 3071], [3072, 3080], [3081, 3083], [3084, 3091], [3092, 3093], [3094, 3101], [3102, 3110], [3111, 3116], [3116, 3117], [3118, 3126], [3127, 3129], [3130, 3137], [3138, 3139], [3140, 3147], [3148, 3156], [3157, 3162], [3162, 3163], [3164, 3172], [3173, 3175], [3176, 3183], [3184, 3185], [3186, 3193], [3194, 3202], [3203, 3208], [3208, 3209], [3210, 3218], [3219, 3221], [3222, 3229], [3230, 3231], [3232, 3239], [3240, 3248], [3249, 3254], [3254, 3255], [3256, 3264], [3265, 3267], [3268, 3275], [3276, 3277], [3278, 3285], [3286, 3294], [3295, 3300], [3300, 3301], [3302, 3310], [3311, 3313], [3314, 3321], [3322, 3323], [3324, 3331], [3332, 3340], [3341, 3346], [3346, 3347], [3348, 3356], [3357, 3359], [3360, 3367], [3368, 3369], [3370, 3377], [3378, 3386], [3387, 3392], [3392, 3393], [3394, 3402], [3403, 3405], [3406, 3413], [3414, 3415], [3416, 3423], [3424, 3432], [3433, 3438], [3438, 3439], [3440, 3448], [3449, 3451], [3452, 3459], [3460, 3461], [3462, 3469], [3470, 3478], [3479, 3484], [3484, 3485], [3486, 3494], [3495, 3497], [3498, 3505], [3506, 3507], [3508, 3515], [3516, 3524], [3525, 3530], [3530, 3531], [3532, 3540], [3541, 3543], [3544, 3551], [3552, 3553], [3554, 3561], [3562, 3570], [3571, 3576], [3576, 3577], [3578, 3586], [3587, 3589], [3590, 3597], [3598, 3599], [3600, 3607], [3608, 3616], [3617, 3622], [3622, 3623], [3624, 3632], [3633, 3635], [3636, 3643], [3644, 3645], [3646, 3653], [3654, 3662], [3663, 3668], [3668, 3669], [3670, 3678], [3679, 3681], [3682, 3689], [3690, 3691], [3692, 3699], [3700, 3708], [3709, 3714], [3714, 3715], [3716, 3724], [3725, 3727], [3728, 3735], [3736, 3737], [3738, 3745], [3746, 3754], [3755, 3760], [3760, 3761], [3762, 3770], [3771, 3773], [3774, 3781], [3782, 3783], [3784, 3791], [3792, 3800], [3801, 3806], [3806, 3807], [3808, 3816], [3817, 3819], [3820, 3827], [3828, 3829], [3830, 3837], [3838, 3846], [3847, 3852], [3852, 3853], [3854, 3862], [3863, 3865], [3866, 3873], [3874, 3875], [3876, 3883], [3884, 3892], [3893, 3898], [3898, 3899], [3900, 3908], [3909, 3911], [3912, 3919], [3920, 3921], [3922, 3929], [3930, 3938], [3939, 3944], [3944, 3945], [3946, 3954], [3955, 3957], [3958, 3965], [3966, 3967], [3968, 3975], [3976, 3984], [3985, 3990], [3990, 3991], [3992, 4000], [4001, 4003], [4004, 4011], [4012, 4013], [4014, 4021], [4022, 4030], [4031, 4036], [4036, 4037], [4038, 4046], [4047, 4049], [4050, 4057], [4058, 4059], [4060, 4067], [4068, 4076], [4077, 4082], [4082, 4083], [4084, 4092], [4093, 4095], [4096, 4103], [4104, 4105], [4106, 4113], [4114, 4122], [4123, 4128], [4128, 4129], [4130, 4138], [4139, 4141], [4142, 4149], [4150, 4151], [4152, 4159], [4160, 4168], [4169, 4174], [4174, 4175], [4176, 4184], [4185, 4187], [4188, 4195], [4196, 4197], [4198, 4205], [4206, 4214], [4215, 4220], [4220, 4221], [4222, 4230], [4231, 4233], [4234, 4241], [4242, 4243], [4244, 4251], [4252, 4260], [4261, 4266], [4266, 4267], [4268, 4276], [4277, 4279], [4280, 4287], [4288, 4289], [4290, 4297], [4298, 4306], [4307, 4312], [4312, 4313], [4314, 4322], [4323, 4325], [4326, 4333], [4334, 4335], [4336, 4343], [4344, 4352], [4353, 4358], [4358, 4359], [4360, 4368], [4369, 4371], [4372, 4379], [4380, 4381], [4382, 4389], [4390, 4398], [4399, 4404], [4404, 4405], [4406, 4414], [4415, 4417], [4418, 4425], [4426, 4427], [4428, 4435], [4436, 4444], [4445, 4450], [4450, 4451], [4452, 4460], [4461, 4463], [4464, 4471], [4472, 4473], [4474, 4481], [4482, 4490], [4491, 4496], [4496, 4497], [4498, 4506], [4507, 4509], [4510, 4517], [4518, 4519], [4520, 4527], [4528, 4536], [4537, 4542], [4542, 4543], [4544, 4552], [4553, 4555], [4556, 4563], [4564, 4565], [4566, 4573], [4574, 4582], [4583, 4588], [4588, 4589], [4590, 4598], [4599, 4602], [4603, 4610], [4611, 4612], [4613, 4620], [4621, 4629], [4630, 4635], [4635, 4636], [4637, 4645], [4646, 4649], [4650, 4657], [4658, 4659], [4660, 4667], [4668, 4676], [4677, 4682], [4682, 4683], [4684, 4692], [4693, 4696], [4697, 4704], [4705, 4706], [4707, 4714], [4715, 4723], [4724, 4729], [4729, 4730], [4731, 4739], [4740, 4743], [4744, 4751], [4752, 4753], [4754, 4761], [4762, 4770], [4771, 4776], [4776, 4777], [4778, 4786], [4787, 4790], [4791, 4798], [4799, 4800], [4801, 4808], [4809, 4817], [4818, 4823], [4823, 4824], [4825, 4833], [4834, 4837], [4838, 4845], [4846, 4847], [4848, 4855], [4856, 4864], [4865, 4870], [4870, 4871], [4872, 4880], [4881, 4884], [4885, 4892], [4893, 4894], [4895, 4902], [4903, 4911], [4912, 4917], [4917, 4918], [4919, 4927], [4928, 4931], [4932, 4939], [4940, 4941], [4942, 4949], [4950, 4958], [4959, 4964], [4964, 4965], [4966, 4974], [4975, 4978], [4979, 4986], [4987, 4988], [4989, 4996], [4997, 5005], [5006, 5011], [5011, 5012], [5013, 5021], [5022, 5025], [5026, 5033], [5034, 5035], [5036, 5043], [5044, 5052], [5053, 5058], [5058, 5059], [5060, 5068], [5069, 5072], [5073, 5080], [5081, 5082], [5083, 5090], [5091, 5099], [5100, 5105], [5105, 5106], [5107, 5115], [5116, 5119], [5120, 5127], [5128, 5129], [5130, 5137], [5138, 5146], [5147, 5152], [5152, 5153], [5154, 5162], [5163, 5166], [5167, 5174], [5175, 5176], [5177, 5184], [5185, 5193], [5194, 5199], [5199, 5200], [5201, 5209], [5210, 5213], [5214, 5221], [5222, 5223], [5224, 5231], [5232, 5240], [5241, 5246], [5246, 5247], [5248, 5256], [5257, 5260], [5261, 5268], [5269, 5270], [5271, 5278], [5279, 5287], [5288, 5293], [5293, 5294], [5295, 5303], [5304, 5307], [5308, 5315], [5316, 5317], [5318, 5325], [5326, 5334], [5335, 5340], [5340, 5341], [5342, 5350], [5351, 5354], [5355, 5362], [5363, 5364], [5365, 5372], [5373, 5381], [5382, 5387], [5387, 5388], [5389, 5397], [5398, 5401], [5402, 5409], [5410, 5411], [5412, 5419], [5420, 5428], [5429, 5434], [5434, 5435], [5436, 5444], [5445, 5448], [5449, 5456], [5457, 5458], [5459, 5466], [5467, 5475], [5476, 5481], [5481, 5482], [5483, 5491], [5492, 5495], [5496, 5503], [5504, 5505], [5506, 5513], [5514, 5522], [5523, 5528], [5528, 5529], [5530, 5538], [5539, 5542], [5543, 5550], [5551, 5552], [5553, 5560], [5561, 5569], [5570, 5575], [5575, 5576], [5577, 5585], [5586, 5589], [5590, 5597], [5598, 5599], [5600, 5607], [5608, 5616], [5617, 5622], [5622, 5623], [5624, 5632], [5633, 5636], [5637, 5644], [5645, 5646], [5647, 5654], [5655, 5663], [5664, 5669], [5669, 5670], [5671, 5679], [5680, 5683], [5684, 5691], [5692, 5693], [5694, 5701], [5702, 5710], [5711, 5716], [5716, 5717], [5718, 5726], [5727, 5730], [5731, 5738], [5739, 5740], [5741, 5748], [5749, 5757], [5758, 5763], [5763, 5764], [5765, 5773], [5774, 5777], [5778, 5785], [5786, 5787], [5788, 5795], [5796, 5804], [5805, 5810], [5810, 5811], [5812, 5820], [5821, 5824], [5825, 5832], [5833, 5834], [5835, 5842], [5843, 5851], [5852, 5857], [5857, 5858], [5859, 5867], [5868, 5871], [5872, 5879], [5880, 5881], [5882, 5889], [5890, 5898], [5899, 5904], [5904, 5905], [5906, 5914], [5915, 5918], [5919, 5926], [5927, 5928], [5929, 5936], [5937, 5945], [5946, 5951], [5951, 5952], [5953, 5961], [5962, 5965], [5966, 5973], [5974, 5975], [5976, 5983], [5984, 5992], [5993, 5998], [5998, 5999], [6000, 6008], [6009, 6012], [6013, 6020], [6021, 6022], [6023, 6030], [6031, 6039], [6040, 6045], [6045, 6046], [6047, 6055], [6056, 6059], [6060, 6067], [6068, 6069], [6070, 6077], [6078, 6086], [6087, 6092], [6092, 6093], [6094, 6102], [6103, 6106], [6107, 6114], [6115, 6116], [6117, 6124], [6125, 6133], [6134, 6139], [6139, 6140], [6141, 6149], [6150, 6153], [6154, 6161], [6162, 6163], [6164, 6171], [6172, 6180], [6181, 6186], [6186, 6187], [6188, 6196], [6197, 6200], [6201, 6208], [6209, 6210], [6211, 6218], [6219, 6227], [6228, 6233], [6233, 6234], [6235, 6243], [6244, 6247], [6248, 6255], [6256, 6257], [6258, 6265], [6266, 6274], [6275, 6280], [6280, 6281], [6282, 6290], [6291, 6294], [6295, 6302], [6303, 6304], [6305, 6312], [6313, 6321], [6322, 6327], [6327, 6328], [6329, 6337], [6338, 6341], [6342, 6349], [6350, 6351], [6352, 6359], [6360, 6368], [6369, 6374], [6374, 6375], [6376, 6384], [6385, 6388], [6389, 6396], [6397, 6398], [6399, 6406], [6407, 6415], [6416, 6421], [6421, 6422], [6423, 6431], [6432, 6435], [6436, 6443], [6444, 6445], [6446, 6453], [6454, 6462], [6463, 6468], [6468, 6469], [6470, 6478], [6479, 6482], [6483, 6490], [6491, 6492], [6493, 6500], [6501, 6509], [6510, 6515], [6515, 6516], [6517, 6525], [6526, 6529], [6530, 6537], [6538, 6539], [6540, 6547], [6548, 6556], [6557, 6562], [6562, 6563], [6564, 6572], [6573, 6576], [6577, 6584], [6585, 6586], [6587, 6594], [6595, 6603], [6604, 6609], [6609, 6610], [6611, 6619], [6620, 6623], [6624, 6631], [6632, 6633], [6634, 6641], [6642, 6650], [6651, 6656], [6656, 6657], [6658, 6666], [6667, 6670], [6671, 6678], [6679, 6680], [6681, 6688], [6689, 6697], [6698, 6703], [6703, 6704], [6705, 6713], [6714, 6717], [6718, 6725], [6726, 6727], [6728, 6735], [6736, 6744], [6745, 6750], [6750, 6751], [6752, 6760], [6761, 6764], [6765, 6772], [6773, 6774], [6775, 6782], [6783, 6791], [6792, 6797], [6797, 6798], [6799, 6807], [6808, 6811], [6812, 6819], [6820, 6821], [6822, 6829], [6830, 6838], [6839, 6844], [6844, 6845], [6846, 6854], [6855, 6858], [6859, 6866], [6867, 6868], [6869, 6876], [6877, 6885], [6886, 6891], [6891, 6892], [6893, 6901], [6902, 6905], [6906, 6913], [6914, 6915], [6916, 6923], [6924, 6932], [6933, 6938], [6938, 6939], [6940, 6948], [6949, 6952], [6953, 6960], [6961, 6962], [6963, 6970], [6971, 6979], [6980, 6985], [6985, 6986], [6987, 6995], [6996, 6999], [7000, 7007], [7008, 7009], [7010, 7017], [7018, 7026], [7027, 7032], [7032, 7033], [7034, 7042], [7043, 7046], [7047, 7054], [7055, 7056], [7057, 7064], [7065, 7073], [7074, 7079], [7079, 7080], [7081, 7089], [7090, 7093], [7094, 7101], [7102, 7103], [7104, 7111], [7112, 7120], [7121, 7126], [7126, 7127], [7128, 7136], [7137, 7140], [7141, 7148], [7149, 7150], [7151, 7158], [7159, 7167], [7168, 7173], [7173, 7174], [7175, 7183], [7184, 7187], [7188, 7195], [7196, 7197], [7198, 7205], [7206, 7214], [7215, 7220], [7220, 7221], [7222, 7230], [7231, 7234], [7235, 7242], [7243, 7244], [7245, 7252], [7253, 7261], [7262, 7267], [7267, 7268], [7269, 7277], [7278, 7281], [7282, 7289], [7290, 7291], [7292, 7299], [7300, 7308], [7309, 7314], [7314, 7315], [7316, 7324], [7325, 7328], [7329, 7336], [7337, 7338], [7339, 7346], [7347, 7355], [7356, 7361], [7361, 7362], [7363, 7371], [7372, 7375], [7376, 7383], [7384, 7385], [7386, 7393], [7394, 7402], [7403, 7408], [7408, 7409], [7410, 7418], [7419, 7422], [7423, 7430], [7431, 7432], [7433, 7440], [7441, 7449], [7450, 7455], [7455, 7456], [7457, 7465], [7466, 7469], [7470, 7477], [7478, 7479], [7480, 7487], [7488, 7496], [7497, 7502], [7502, 7503], [7504, 7512], [7513, 7516], [7517, 7524], [7525, 7526], [7527, 7534], [7535, 7543], [7544, 7549], [7549, 7550], [7551, 7559], [7560, 7563], [7564, 7571], [7572, 7573], [7574, 7581], [7582, 7590], [7591, 7596], [7596, 7597], [7598, 7606], [7607, 7610], [7611, 7618], [7619, 7620], [7621, 7628], [7629, 7637], [7638, 7643], [7643, 7644], [7645, 7653], [7654, 7657], [7658, 7665], [7666, 7667], [7668, 7675], [7676, 7684], [7685, 7690], [7690, 7691], [7692, 7700], [7701, 7704], [7705, 7712], [7713, 7714], [7715, 7722], [7723, 7731], [7732, 7737], [7737, 7738], [7739, 7747], [7748, 7751], [7752, 7759], [7760, 7761], [7762, 7769], [7770, 7778], [7779, 7784], [7784, 7785], [7786, 7794], [7795, 7798], [7799, 7806], [7807, 7808], [7809, 7816], [7817, 7825], [7826, 7831], [7831, 7832], [7833, 7841], [7842, 7845], [7846, 7853], [7854, 7855], [7856, 7863], [7864, 7872], [7873, 7878], [7878, 7879], [7880, 7888], [7889, 7892], [7893, 7900], [7901, 7902], [7903, 7910], [7911, 7919], [7920, 7925], [7925, 7926], [7927, 7935], [7936, 7939], [7940, 7947], [7948, 7949], [7950, 7957], [7958, 7966], [7967, 7972], [7972, 7973], [7974, 7982], [7983, 7986], [7987, 7994], [7995, 7996], [7997, 8004], [8005, 8013], [8014, 8019], [8019, 8020], [8021, 8029], [8030, 8033], [8034, 8041], [8042, 8043], [8044, 8051], [8052, 8060], [8061, 8066], [8066, 8067], [8068, 8076], [8077, 8080], [8081, 8088], [8089, 8090], [8091, 8098], [8099, 8107], [8108, 8113], [8113, 8114], [8115, 8123], [8124, 8127], [8128, 8135], [8136, 8137], [8138, 8145], [8146, 8154], [8155, 8160], [8160, 8161], [8162, 8170], [8171, 8174], [8175, 8182], [8183, 8184], [8185, 8192], [8193, 8201], [8202, 8207], [8207, 8208], [8209, 8217], [8218, 8221], [8222, 8229], [8230, 8231], [8232, 8239], [8240, 8248], [8249, 8254], [8254, 8255], [8256, 8264], [8265, 8268], [8269, 8276], [8277, 8278], [8279, 8286], [8287, 8295], [8296, 8301], [8301, 8302], [8303, 8311], [8312, 8315], [8316, 8323], [8324, 8325], [8326, 8333], [8334, 8342], [8343, 8348], [8348, 8349], [8350, 8358], [8359, 8362], [8363, 8370], [8371, 8372], [8373, 8380], [8381, 8389], [8390, 8395], [8395, 8396], [8397, 8405], [8406, 8409], [8410, 8417], [8418, 8419], [8420, 8427], [8428, 8436], [8437, 8442], [8442, 8443], [8444, 8452], [8453, 8456], [8457, 8464], [8465, 8466], [8467, 8474], [8475, 8483], [8484, 8489], [8489, 8490], [8491, 8499], [8500, 8503], [8504, 8511], [8512, 8513], [8514, 8521], [8522, 8530], [8531, 8536], [8536, 8537], [8538, 8546], [8547, 8550], [8551, 8558], [8559, 8560], [8561, 8568], [8569, 8577], [8578, 8583], [8583, 8584], [8585, 8593], [8594, 8597], [8598, 8605], [8606, 8607], [8608, 8615], [8616, 8624], [8625, 8630], [8630, 8631], [8632, 8640], [8641, 8644], [8645, 8652], [8653, 8654], [8655, 8662], [8663, 8671], [8672, 8677], [8677, 8678], [8679, 8687], [8688, 8691], [8692, 8699], [8700, 8701], [8702, 8709], [8710, 8718], [8719, 8724], [8724, 8725], [8726, 8734], [8735, 8738], [8739, 8746], [8747, 8748], [8749, 8756], [8757, 8765], [8766, 8771], [8771, 8772], [8773, 8781], [8782, 8785], [8786, 8793], [8794, 8795], [8796, 8803], [8804, 8812], [8813, 8818], [8818, 8819], [8820, 8828], [8829, 8832], [8833, 8840], [8841, 8842], [8843, 8850], [8851, 8859], [8860, 8865], [8865, 8866], [8867, 8875], [8876, 8879], [8880, 8887], [8888, 8889], [8890, 8897], [8898, 8906], [8907, 8912], [8912, 8913], [8914, 8922], [8923, 8926], [8927, 8934], [8935, 8936], [8937, 8944], [8945, 8953], [8954, 8959], [8959, 8960], [8961, 8969], [8970, 8973], [8974, 8981], [8982, 8983], [8984, 8991], [8992, 9000], [9001, 9006], [9006, 9007], [9008, 9016], [9017, 9020], [9021, 9028], [9029, 9030], [9031, 9038], [9039, 9047], [9048, 9053], [9053, 9054], [9055, 9063], [9064, 9067], [9068, 9075], [9076, 9077], [9078, 9085], [9086, 9094], [9095, 9100], [9100, 9101], [9102, 9110], [9111, 9114], [9115, 9122], [9123, 9124], [9125, 9132], [9133, 9141], [9142, 9147], [9147, 9148], [9149, 9157], [9158, 9161], [9162, 9169], [9170, 9171], [9172, 9179], [9180, 9188], [9189, 9194], [9194, 9195], [9196, 9204], [9205, 9208], [9209, 9216], [9217, 9218], [9219, 9226], [9227, 9235], [9236, 9241], [9241, 9242], [9243, 9251], [9252, 9255], [9256, 9263], [9264, 9265], [9266, 9273], [9274, 9282], [9283, 9288], [9288, 9289], [9290, 9298], [9299, 9302], [9303, 9310], [9311, 9312], [9313, 9320], [9321, 9329], [9330, 9335], [9335, 9336], [9337, 9345], [9346, 9349], [9350, 9357], [9358, 9359], [9360, 9367], [9368, 9376], [9377, 9382], [9382, 9383], [9384, 9392], [9393, 9396], [9397, 9404], [9405, 9406], [9407, 9414], [9415, 9423], [9424, 9429], [9429, 9430], [9431, 9439], [9440, 9443], [9444, 9451], [9452, 9453], [9454, 9461], [9462, 9470], [9471, 9476], [9476, 9477], [9478, 9486], [9487, 9490], [9491, 9498], [9499, 9500], [9501, 9508], [9509, 9517], [9518, 9523], [9523, 9524], [9525, 9533], [9534, 9537], [9538, 9545], [9546, 9547], [9548, 9555], [9556, 9564], [9565, 9570], [9570, 9571], [9572, 9580], [9581, 9584], [9585, 9592], [9593, 9594], [9595, 9602], [9603, 9611], [9612, 9617], [9617, 9618], [9619, 9627], [9628, 9631], [9632, 9639], [9640, 9641], [9642, 9649], [9650, 9658], [9659, 9664], [9664, 9665], [9666, 9674], [9675, 9678], [9679, 9686], [9687, 9688], [9689, 9696], [9697, 9705], [9706, 9711], [9711, 9712], [9713, 9721], [9722, 9725], [9726, 9733], [9734, 9735], [9736, 9743], [9744, 9752], [9753, 9758], [9758, 9759], [9760, 9768], [9769, 9772], [9773, 9780], [9781, 9782], [9783, 9790], [9791, 9799], [9800, 9805], [9805, 9806], [9807, 9815], [9816, 9819], [9820, 9827], [9828, 9829], [9830, 9837], [9838, 9846], [9847, 9852], [9852, 9853], [9854, 9862], [9863, 9866], [9867, 9874], [9875, 9876], [9877, 9884], [9885, 9893], [9894, 9899], [9899, 9900], [9901, 9909], [9910, 9913], [9914, 9921], [9922, 9923], [9924, 9931], [9932, 9940], [9941, 9946], [9946, 9947], [9948, 9956], [9957, 9960], [9961, 9968], [9969, 9970], [9971, 9978], [9979, 9987], [9988, 9993], [9993, 9994], [9995, 10003], [10004, 10007], [10008, 10015], [10016, 10017], [10018, 10025], [10026, 10034], [10035, 10040], [10040, 10041], [10042, 10050], [10051, 10054], [10055, 10062], [10063, 10064], [10065, 10072], [10073, 10081], [10082, 10087], [10087, 10088], [10089, 10097], [10098, 10101], [10102, 10109], [10110, 10111], [10112, 10119], [10120, 10128], [10129, 10134], [10134, 10135], [10136, 10144], [10145, 10148], [10149, 10156], [10157, 10158], [10159, 10166], [10167, 10175], [10176, 10181], [10181, 10182], [10183, 10191], [10192, 10195], [10196, 10203], [10204, 10205], [10206, 10213], [10214, 10222], [10223, 10228], [10228, 10229]], "w_pos": [10, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 23, 24, 25, 26, 27, 28, 30, 31, 32, 33, 34, 35, 36, 37, 39, 40, 41, 42, 43, 44, 45, 46, 48, 49, 50, 51, 52, 53, 54, 55, 57, 58, 59, 60, 61, 62, 63, 64, 66, 67, 68, 69, 70, 71, 72, 73, 75, 76, 77, 78, 79, 80, 81, 82, 84, 85, 86, 87, 88, 89, 90, 91, 93, 94, 95, 96, 97, 98, 99, 100, 102, 104, 105, 106, 107, 108, 109, 110, 112, 114, 115, 116, 117, 118, 119, 120, 122, 124, 125, 126, 127, 128, 129, 130, 132, 134, 135, 136, 137, 138, 139, 140, 142, 144, 145, 146, 147, 148, 149, 150, 152, 154, 155, 156, 157, 158, 159, 160, 162, 164, 165, 166, 167, 168, 169, 170, 172, 174, 175, 176, 177, 178, 179, 180, 182, 184, 185, 186, 187, 188, 189, 190, 192, 194, 195, 196, 197, 198, 199, 200, 202, 204, 205, 206, 207, 208, 209, 210, 212, 214, 215, 216, 217, 218, 219, 220, 222, 224, 225, 226, 227, 228, 229, 230, 232, 234, 235, 236, 237, 238, 239, 240, 242, 244, 245, 246, 247, 248, 249, 250, 252, 254, 255, 256, 257, 258, 259, 260, 262, 264, 265, 266, 267, 268, 269, 270, 272, 274, 275, 276, 277, 278, 279, 280, 282, 284, 285, 286, 287, 288, 289, 290, 292, 294, 295, 296, 297, 298, 299, 300, 302, 304, 305, 306, 307, 308, 309, 310, 312, 314, 315, 316, 317, 318, 319, 320, 322, 324, 325, 326, 327, 328, 329, 330, 332, 334, 335, 336, 337, 338, 339, 340, 342, 344, 345, 346, 347, 348, 349, 350, 352, 354, 355, 356, 357, 358, 359, 360, 362, 364, 365, 366, 367, 368, 369, 370, 372, 374, 375, 376, 377, 378, 379, 380, 382, 384, 385, 386, 387, 388, 389, 390, 392, 394, 395, 396, 397, 398, 399, 400, 402, 404, 405, 406, 407, 408, 409, 410, 412, 414, 415, 416, 417, 418, 419, 420, 422, 424, 425, 426, 427, 428, 429, 430, 432, 434, 435, 436, 437, 438, 439, 440, 442, 444, 445, 446, 447, 448, 449, 450, 452, 454, 455, 456, 457, 458, 459, 460, 462, 464, 465, 466, 467, 468, 469, 470, 472, 474, 475, 476, 477, 478, 479, 480, 482, 484, 485, 486, 487, 488, 489, 490, 492, 494, 495, 496, 497, 498, 499, 500, 502, 504, 505, 506, 507, 508, 509, 510, 512, 514, 515, 516, 517, 518, 519, 520, 522, 524, 525, 526, 527, 528, 529, 530, 532, 534, 535, 536, 537, 538, 539, 540, 542, 544, 545, 546, 547, 548, 549, 550, 552, 554, 555, 556, 557, 558, 559, 560, 562, 564, 565, 566, 567, 568, 569, 570, 572, 574, 575, 576, 577, 578, 579, 580, 582, 584, 585, 586, 587, 588, 589, 590, 592, 594, 595, 596, 597, 598, 599, 600, 602, 604, 605, 606, 607, 608, 609, 610, 612, 614, 615, 616, 617, 618, 619, 620, 622, 624, 625, 626, 627, 628, 629, 630, 632, 634, 635, 636, 637, 638, 639, 640, 642, 644, 645, 646, 647, 648, 649, 650, 652, 654, 655, 656, 657, 658, 659, 660, 662, 664, 665, 666, 667, 668, 669, 670, 672, 674, 675, 676, 677, 678, 679, 680, 682, 684, 685, 686, 687, 688, 689, 690, 692, 694, 695, 696, 697, 698, 699, 700, 702, 704, 705, 706, 707, 708, 709, 710, 712, 714, 715, 716, 717, 718, 719, 720, 722, 724, 725, 726, 727, 728, 729, 730, 732, 734, 735, 736, 737, 738, 739, 740, 742, 744, 745, 746, 747, 748, 749, 750, 752, 754, 755, 756, 757, 758, 759, 760, 762, 764, 765, 766, 767, 768, 769, 770, 772, 774, 775, 776, 777, 778, 779, 780, 782, 784, 785, 786, 787, 788, 789, 790, 792, 794, 795, 796, 797, 798, 799, 800, 802, 804, 805, 806, 807, 808, 809, 810, 812, 814, 815, 816, 817, 818, 819, 820, 822, 824, 825, 826, 827, 828, 829, 830, 832, 834, 835, 836, 837, 838, 839, 840, 842, 844, 845, 846, 847, 848, 849, 850, 852, 854, 855, 856, 857, 858, 859, 860, 862, 864, 865, 866, 867, 868, 869, 870, 872, 874, 875, 876, 877, 878, 879, 880, 882, 884, 885, 886, 887, 888, 889, 890, 892, 894, 895, 896, 897, 898, 899, 900, 902, 904, 905, 906, 907, 908, 909, 910, 912, 914, 915, 916, 917, 918, 919, 920, 922, 924, 925, 926, 927, 928, 929, 930, 932, 934, 935, 936, 937, 938, 939, 940, 942, 944, 945, 946, 947, 948, 949, 950, 952, 954, 955, 956, 957, 958, 959, 960, 962, 964, 965, 966, 967, 968, 969, 970, 972, 974, 975, 976, 977, 978, 979, 980, 982, 984, 985, 986, 987, 988, 989, 990, 992, 994, 995, 996, 997, 998, 999, 1000, 1002, 1005, 1006, 1007, 1008, 1009, 1010, 1011, 1013, 1016, 1017, 1018, 1019, 1020, 1021, 1022, 1024, 1027, 1028, 1029, 1030, 1031, 1032, 1033, 1035, 1038, 1039, 1040, 1041, 1042, 1043, 1044, 1046, 1049, 1050, 1051, 1052, 1053, 1054, 1055, 1057, 1060, 1061, 1062, 1063, 1064, 1065, 1066, 1068, 1071, 1072, 1073, 1074, 1075, 1076, 1077, 1079, 1082, 1083, 1084, 1085, 1086, 1087, 1088, 1090, 1093, 1094, 1095, 1096, 1097, 1098, 1099, 1101, 1104, 1105, 1106, 1107, 1108, 1109, 1110, 1112, 1115, 1116, 1117, 1118, 1119, 1120, 1121, 1123, 1126, 1127, 1128, 1129, 1130, 1131, 1132, 1134, 1137, 1138, 1139, 1140, 1141, 1142, 1143, 1145, 1148, 1149, 1150, 1151, 1152, 1153, 1154, 1156, 1159, 1160, 1161, 1162, 1163, 1164, 1165, 1167, 1170, 1171, 1172, 1173, 1174, 1175, 1176, 1178, 1181, 1182, 1183, 1184, 1185, 1186, 1187, 1189, 1192, 1193, 1194, 1195, 1196, 1197, 1198, 1200, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1211, 1214, 1215, 1216, 1217, 1218, 1219, 1220, 1222, 1225, 1226, 1227, 1228, 1229, 1230, 1231, 1233, 1236, 1237, 1238, 1239, 1240, 1241, 1242, 1244, 1247, 1248, 1249, 1250, 1251, 1252, 1253, 1255, 1258, 1259, 1260, 1261, 1262, 1263, 1264, 1266, 1269, 1270, 1271, 1272, 1273, 1274, 1275, 1277, 1280, 1281, 1282, 1283, 1284, 1285, 1286, 1288, 1291, 1292, 1293, 1294, 1295, 1296, 1297, 1299, 1302, 1303, 1304, 1305, 1306, 1307, 1308, 1310, 1313, 1314, 1315, 1316, 1317, 1318, 1319, 1321, 1324, 1325, 1326, 1327, 1328, 1329, 1330, 1332, 1335, 1336, 1337, 1338, 1339, 1340, 1341, 1343, 1346, 1347, 1348, 1349, 1350, 1351, 1352, 1354, 1357, 1358, 1359, 1360, 1361, 1362, 1363, 1365, 1368, 1369, 1370, 1371, 1372, 1373, 1374, 1376, 1379, 1380, 1381, 1382, 1383, 1384, 1385, 1387, 1390, 1391, 1392, 1393, 1394, 1395, 1396, 1398, 1401, 1402, 1403, 1404, 1405, 1406, 1407, 1409, 1412, 1413, 1414, 1415, 1416, 1417, 1418, 1420, 1423, 1424, 1425, 1426, 1427, 1428, 1429, 1431, 1434, 1435, 1436, 1437, 1438, 1439, 1440, 1442, 1445, 1446, 1447, 1448, 1449, 1450, 1451, 1453, 1456, 1457, 1458, 1459, 1460, 1461, 1462, 1464, 1467, 1468, 1469, 1470, 1471, 1472, 1473, 1475, 1478, 1479, 1480, 1481, 1482, 1483, 1484, 1486, 1489, 1490, 1491, 1492, 1493, 1494, 1495, 1497, 1500, 1501, 1502, 1503, 1504, 1505, 1506, 1508, 1511, 1512, 1513, 1514, 1515, 1516, 1517, 1519, 1522, 1523, 1524, 1525, 1526, 1527, 1528, 1530, 1533, 1534, 1535, 1536, 1537, 1538, 1539, 1541, 1544, 1545, 1546, 1547, 1548, 1549, 1550, 1552, 1555, 1556, 1557, 1558, 1559, 1560, 1561, 1563, 1566, 1567, 1568, 1569, 1570, 1571, 1572, 1574, 1577, 1578, 1579, 1580, 1581, 1582, 1583, 1585, 1588, 1589, 1590, 1591, 1592, 1593, 1594, 1596, 1599, 1600, 1601, 1602, 1603, 1604, 1605, 1607, 1610, 1611, 1612, 1613, 1614, 1615, 1616, 1618, 1621, 1622, 1623, 1624, 1625, 1626, 1627, 1629, 1632, 1633, 1634, 1635, 1636, 1637, 1638, 1640, 1643, 1644, 1645, 1646, 1647, 1648, 1649, 1651, 1654, 1655, 1656, 1657, 1658, 1659, 1660, 1662, 1665, 1666, 1667, 1668, 1669, 1670, 1671, 1673, 1676, 1677, 1678, 1679, 1680, 1681, 1682, 1684, 1687, 1688, 1689, 1690, 1691, 1692, 1693, 1695, 1698, 1699, 1700, 1701, 1702, 1703, 1704, 1706, 1709, 1710, 1711, 1712, 1713, 1714, 1715, 1717, 1720, 1721, 1722, 1723, 1724, 1725, 1726, 1728, 1731, 1732, 1733, 1734, 1735, 1736, 1737, 1739, 1742, 1743, 1744, 1745, 1746, 1747, 1748, 1750, 1753, 1754, 1755, 1756, 1757, 1758, 1759, 1761, 1764, 1765, 1766, 1767, 1768, 1769, 1770, 1772, 1775, 1776, 1777, 1778, 1779, 1780, 1781, 1783, 1786, 1787, 1788, 1789, 1790, 1791, 1792, 1794, 1797, 1798, 1799, 1800, 1801, 1802, 1803, 1805, 1808, 1809, 1810, 1811, 1812, 1813, 1814, 1816, 1819, 1820, 1821, 1822, 1823, 1824, 1825, 1827, 1830, 1831, 1832, 1833, 1834, 1835, 1836, 1838, 1841, 1842, 1843, 1844, 1845, 1846, 1847, 1849, 1852, 1853, 1854, 1855, 1856, 1857, 1858, 1860, 1863, 1864, 1865, 1866, 1867, 1868, 1869, 1871, 1874, 1875, 1876, 1877, 1878, 1879, 1880, 1882, 1885, 1886, 1887, 1888, 1889, 1890, 1891, 1893, 1896, 1897, 1898, 1899, 1900, 1901, 1902, 1904, 1907, 1908, 1909, 1910, 1911, 1912, 1913, 1915, 1918, 1919, 1920, 1921, 1922, 1923, 1924, 1926, 1929, 1930, 1931, 1932, 1933, 1934, 1935, 1937, 1940, 1941, 1942, 1943, 1944, 1945, 1946, 1948, 1951, 1952, 1953, 1954, 1955, 1956, 1957, 1959, 1962, 1963, 1964, 1965, 1966, 1967, 1968, 1970, 1973, 1974, 1975, 1976, 1977, 1978, 1979, 1981, 1984, 1985, 1986, 1987, 1988, 1989, 1990, 1992, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2003, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2014, 2017, 2018, 2019, 2020, 2021, 2022, 2023, 2025, 2028, 2029, 2030, 2031, 2032, 2033, 2034, 2036, 2039, 2040, 2041, 2042, 2043, 2044, 2045, 2047, 2050, 2051, 2052, 2053, 2054, 2055, 2056, 2058, 2061, 2062, 2063, 2064, 2065, 2066, 2067, 2069, 2072, 2073, 2074, 2075, 2076, 2077, 2078, 2080, 2083, 2084, 2085, 2086, 2087, 2088, 2089, 2091, 2094, 2095, 2096, 2097, 2098, 2099, 2100, 2102, 2105, 2106, 2107, 2108, 2109, 2110, 2111, 2113, 2116, 2117, 2118, 2119, 2120, 2121, 2122, 2124, 2127, 2128, 2129, 2130, 2131, 2132, 2133, 2135, 2138, 2139, 2140, 2141, 2142, 2143, 2144, 2146, 2149, 2150, 2151, 2152, 2153, 2154, 2155, 2157, 2160, 2161, 2162, 2163, 2164, 2165, 2166, 2168, 2171, 2172, 2173, 2174, 2175, 2176, 2177, 2179, 2182, 2183, 2184, 2185, 2186, 2187, 2188, 2190, 2193, 2194, 2195, 2196, 2197, 2198, 2199, 2201, 2204, 2205, 2206, 2207, 2208, 2209, 2210, 2212, 2215, 2216, 2217, 2218, 2219, 2220, 2221, 2223, 2226, 2227, 2228, 2229, 2230, 2231, 2232, 2234, 2237, 2238, 2239, 2240, 2241, 2242, 2243, 2245, 2248, 2249, 2250, 2251, 2252, 2253, 2254, 2256, 2259, 2260, 2261, 2262, 2263, 2264, 2265, 2267, 2270, 2271, 2272, 2273, 2274, 2275, 2276, 2278, 2281, 2282, 2283, 2284, 2285, 2286, 2287, 2289, 2292, 2293, 2294, 2295, 2296, 2297, 2298, 2300, 2303, 2304, 2305, 2306, 2307, 2308, 2309, 2311, 2314, 2315, 2316, 2317, 2318, 2319], "span_over": "user", "n_tokens": 2385, "head": "broad", "head_reason": "default"}], "window_rows": [{"p_ids": [1905, 25, 198, 27, 22446, 3364, 420, 846, 731, 198, 81646, 220, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 18, 6473, 198, 510, 22446, 29, 198], "blocks": [{"kind": "span", "ends": [40], "label_ranges": [[22, 40]], "words_in_block": true, "ids": [271, 6065, 913, 25, 9111, 198, 6196, 25, 1156, 198, 14162, 25, 198, 22365, 42824, 6088, 264, 9403, 30, 198, 22933, 25, 198, 14057, 29, 198, 4754, 4532, 3147, 64, 9403, 1484, 2129, 767, 3147, 31320, 8934, 198, 510, 1448, 29, 271, 1138, 25, 198, 27, 22446, 3364, 420, 846, 731, 198, 81646, 220, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 18, 6473, 198, 510, 22446, 29, 198, 271, 8679, 1396, 9111, 314, 279, 2100, 1414, 421, 22651, 799, 314, 279, 8928, 13, 198, 6205, 581, 25], "query": 1874, "w_pos": [52, 54, 55, 56, 57, 58, 59, 60, 61, 63, 64, 65, 66, 67, 68, 69, 70, 72, 73, 74, 75, 76, 77, 78, 79, 81, 82, 83, 84, 85, 86, 87, 88, 90, 91, 92, 93, 94, 95, 96, 97, 99, 100, 101, 102, 103, 104, 105, 106, 108, 109, 110, 111, 112, 113, 114, 115, 117, 118, 119, 120, 121, 122, 123, 124, 126, 127, 128, 129, 130, 131, 132, 133, 135, 136, 137, 138, 139, 140, 141, 142, 144, 146, 147, 148, 149, 150, 151, 152, 154, 156, 157, 158, 159, 160, 161, 162, 164, 166, 167, 168, 169, 170, 171, 172, 174, 176, 177, 178, 179, 180, 181, 182, 184, 186, 187, 188, 189, 190, 191, 192, 194, 196, 197, 198, 199, 200, 201, 202, 204, 206, 207, 208, 209, 210, 211, 212, 214, 216, 217, 218, 219, 220, 221, 222, 224, 226, 227, 228, 229, 230, 231, 232, 234, 236, 237, 238, 239, 240, 241, 242, 244, 246, 247, 248, 249, 250, 251, 252, 254, 256, 257, 258, 259, 260, 261, 262, 264, 266, 267, 268, 269, 270, 271, 272, 274, 276, 277, 278, 279, 280, 281, 282, 284, 286, 287, 288, 289, 290, 291, 292, 294, 296, 297, 298, 299, 300, 301, 302, 304, 306, 307, 308, 309, 310, 311, 312, 314, 316, 317, 318, 319, 320, 321, 322, 324, 326, 327, 328, 329, 330, 331, 332, 334, 336, 337, 338, 339, 340, 341, 342, 344, 346, 347, 348, 349, 350, 351, 352, 354, 356, 357, 358, 359, 360, 361, 362, 364, 366, 367, 368, 369, 370, 371, 372, 374, 376, 377, 378, 379, 380, 381, 382, 384, 386, 387, 388, 389, 390, 391, 392, 394, 396, 397, 398, 399, 400, 401, 402, 404, 406, 407, 408, 409, 410, 411, 412, 414, 416, 417, 418, 419, 420, 421, 422, 424, 426, 427, 428, 429, 430, 431, 432, 434, 436, 437, 438, 439, 440, 441, 442, 444, 446, 447, 448, 449, 450, 451, 452, 454, 456, 457, 458, 459, 460, 461, 462, 464, 466, 467, 468, 469, 470, 471, 472, 474, 476, 477, 478, 479, 480, 481, 482, 484, 486, 487, 488, 489, 490, 491, 492, 494, 496, 497, 498, 499, 500, 501, 502, 504, 506, 507, 508, 509, 510, 511, 512, 514, 516, 517, 518, 519, 520, 521, 522, 524, 526, 527, 528, 529, 530, 531, 532, 534, 536, 537, 538, 539, 540, 541, 542, 544, 546, 547, 548, 549, 550, 551, 552, 554, 556, 557, 558, 559, 560, 561, 562, 564, 566, 567, 568, 569, 570, 571, 572, 574, 576, 577, 578, 579, 580, 581, 582, 584, 586, 587, 588, 589, 590, 591, 592, 594, 596, 597, 598, 599, 600, 601, 602, 604, 606, 607, 608, 609, 610, 611, 612, 614, 616, 617, 618, 619, 620, 621, 622, 624, 626, 627, 628, 629, 630, 631, 632, 634, 636, 637, 638, 639, 640, 641, 642, 644, 646, 647, 648, 649, 650, 651, 652, 654, 656, 657, 658, 659, 660, 661, 662, 664, 666, 667, 668, 669, 670, 671, 672, 674, 676, 677, 678, 679, 680, 681, 682, 684, 686, 687, 688, 689, 690, 691, 692, 694, 696, 697, 698, 699, 700, 701, 702, 704, 706, 707, 708, 709, 710, 711, 712, 714, 716, 717, 718, 719, 720, 721, 722, 724, 726, 727, 728, 729, 730, 731, 732, 734, 736, 737, 738, 739, 740, 741, 742, 744, 746, 747, 748, 749, 750, 751, 752, 754, 756, 757, 758, 759, 760, 761, 762, 764, 766, 767, 768, 769, 770, 771, 772, 774, 776, 777, 778, 779, 780, 781, 782, 784, 786, 787, 788, 789, 790, 791, 792, 794, 796, 797, 798, 799, 800, 801, 802, 804, 806, 807, 808, 809, 810, 811, 812, 814, 816, 817, 818, 819, 820, 821, 822, 824, 826, 827, 828, 829, 830, 831, 832, 834, 836, 837, 838, 839, 840, 841, 842, 844, 846, 847, 848, 849, 850, 851, 852, 854, 856, 857, 858, 859, 860, 861, 862, 864, 866, 867, 868, 869, 870, 871, 872, 874, 876, 877, 878, 879, 880, 881, 882, 884, 886, 887, 888, 889, 890, 891, 892, 894, 896, 897, 898, 899, 900, 901, 902, 904, 906, 907, 908, 909, 910, 911, 912, 914, 916, 917, 918, 919, 920, 921, 922, 924, 926, 927, 928, 929, 930, 931, 932, 934, 936, 937, 938, 939, 940, 941, 942, 944, 946, 947, 948, 949, 950, 951, 952, 954, 956, 957, 958, 959, 960, 961, 962, 964, 966, 967, 968, 969, 970, 971, 972, 974, 976, 977, 978, 979, 980, 981, 982, 984, 986, 987, 988, 989, 990, 991, 992, 994, 996, 997, 998, 999, 1000, 1001, 1002, 1004, 1006, 1007, 1008, 1009, 1010, 1011, 1012, 1014, 1016, 1017, 1018, 1019, 1020, 1021, 1022, 1024, 1026, 1027, 1028, 1029, 1030, 1031, 1032, 1034, 1036, 1037, 1038, 1039, 1040, 1041, 1042, 1044, 1047, 1048, 1049, 1050, 1051, 1052, 1053, 1055, 1058, 1059, 1060, 1061, 1062, 1063, 1064, 1066, 1069, 1070, 1071, 1072, 1073, 1074, 1075, 1077, 1080, 1081, 1082, 1083, 1084, 1085, 1086, 1088, 1091, 1092, 1093, 1094, 1095, 1096, 1097, 1099, 1102, 1103, 1104, 1105, 1106, 1107, 1108, 1110, 1113, 1114, 1115, 1116, 1117, 1118, 1119, 1121, 1124, 1125, 1126, 1127, 1128, 1129, 1130, 1132, 1135, 1136, 1137, 1138, 1139, 1140, 1141, 1143, 1146, 1147, 1148, 1149, 1150, 1151, 1152, 1154, 1157, 1158, 1159, 1160, 1161, 1162, 1163, 1165, 1168, 1169, 1170, 1171, 1172, 1173, 1174, 1176, 1179, 1180, 1181, 1182, 1183, 1184, 1185, 1187, 1190, 1191, 1192, 1193, 1194, 1195, 1196, 1198, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1209, 1212, 1213, 1214, 1215, 1216, 1217, 1218, 1220, 1223, 1224, 1225, 1226, 1227, 1228, 1229, 1231, 1234, 1235, 1236, 1237, 1238, 1239, 1240, 1242, 1245, 1246, 1247, 1248, 1249, 1250, 1251, 1253, 1256, 1257, 1258, 1259, 1260, 1261, 1262, 1264, 1267, 1268, 1269, 1270, 1271, 1272, 1273, 1275, 1278, 1279, 1280, 1281, 1282, 1283, 1284, 1286, 1289, 1290, 1291, 1292, 1293, 1294, 1295, 1297, 1300, 1301, 1302, 1303, 1304, 1305, 1306, 1308, 1311, 1312, 1313, 1314, 1315, 1316, 1317, 1319, 1322, 1323, 1324, 1325, 1326, 1327, 1328, 1330, 1333, 1334, 1335, 1336, 1337, 1338, 1339, 1341, 1344, 1345, 1346, 1347, 1348, 1349, 1350, 1352, 1355, 1356, 1357, 1358, 1359, 1360, 1361, 1363, 1366, 1367, 1368, 1369, 1370, 1371, 1372, 1374, 1377, 1378, 1379, 1380, 1381, 1382, 1383, 1385, 1388, 1389, 1390, 1391, 1392, 1393, 1394, 1396, 1399, 1400, 1401, 1402, 1403, 1404, 1405, 1407, 1410, 1411, 1412, 1413, 1414, 1415, 1416, 1418, 1421, 1422, 1423, 1424, 1425, 1426, 1427, 1429, 1432, 1433, 1434, 1435, 1436, 1437, 1438, 1440, 1443, 1444, 1445, 1446, 1447, 1448, 1449, 1451, 1454, 1455, 1456, 1457, 1458, 1459, 1460, 1462, 1465, 1466, 1467, 1468, 1469, 1470, 1471, 1473, 1476, 1477, 1478, 1479, 1480, 1481, 1482, 1484, 1487, 1488, 1489, 1490, 1491, 1492, 1493, 1495, 1498, 1499, 1500, 1501, 1502, 1503, 1504, 1506, 1509, 1510, 1511, 1512, 1513, 1514, 1515, 1517, 1520, 1521, 1522, 1523, 1524, 1525, 1526, 1528, 1531, 1532, 1533, 1534, 1535, 1536, 1537, 1539, 1542, 1543, 1544, 1545, 1546, 1547, 1548, 1550, 1553, 1554, 1555, 1556, 1557, 1558, 1559, 1561, 1564, 1565, 1566, 1567, 1568, 1569, 1570, 1572, 1575, 1576, 1577, 1578, 1579, 1580, 1581, 1583, 1586, 1587, 1588, 1589, 1590, 1591, 1592, 1594, 1597, 1598, 1599, 1600, 1601, 1602, 1603, 1605, 1608, 1609, 1610, 1611, 1612, 1613, 1614, 1616, 1619, 1620, 1621, 1622, 1623, 1624, 1625, 1627, 1630, 1631, 1632, 1633, 1634, 1635, 1636, 1638, 1641, 1642, 1643, 1644, 1645, 1646, 1647, 1649, 1652, 1653, 1654, 1655, 1656, 1657, 1658, 1660, 1663, 1664, 1665, 1666, 1667, 1668, 1669, 1671, 1674, 1675, 1676, 1677, 1678, 1679, 1680, 1682, 1685, 1686, 1687, 1688, 1689, 1690, 1691, 1693, 1696, 1697, 1698, 1699, 1700, 1701, 1702, 1704, 1707, 1708, 1709, 1710, 1711, 1712, 1713, 1715, 1718, 1719, 1720, 1721, 1722, 1723, 1724, 1726, 1729, 1730, 1731, 1732, 1733, 1734, 1735, 1737, 1740, 1741, 1742, 1743, 1744, 1745, 1746, 1748, 1751, 1752, 1753, 1754, 1755, 1756, 1757, 1759, 1762, 1763, 1764, 1765, 1766, 1767, 1768, 1770, 1773, 1774, 1775, 1776, 1777, 1778, 1779, 1781, 1784, 1785, 1786, 1787, 1788, 1789, 1790, 1792, 1795, 1796, 1797, 1798, 1799, 1800, 1801, 1803, 1806, 1807, 1808, 1809, 1810, 1811, 1812, 1814, 1817, 1818, 1819, 1820, 1821, 1822, 1823, 1825, 1828, 1829, 1830, 1831, 1832, 1833, 1834, 1836, 1839, 1840, 1841, 1842, 1843, 1844, 1845, 1847, 1850]}], "trunc": {"shrink_cut": 0, "protected_cut": 0}, "seg_lens": {"user": 1799}, "layout": "c", "names": ["delivery"], "w_offs": [[0, 8], [9, 10], [11, 18], [19, 20], [21, 28], [29, 37], [38, 43], [43, 44], [45, 53], [54, 55], [56, 63], [64, 65], [66, 73], [74, 82], [83, 88], [88, 89], [90, 98], [99, 100], [101, 108], [109, 110], [111, 118], [119, 127], [128, 133], [133, 134], [135, 143], [144, 145], [146, 153], [154, 155], [156, 163], [164, 172], [173, 178], [178, 179], [180, 188], [189, 190], [191, 198], [199, 200], [201, 208], [209, 217], [218, 223], [223, 224], [225, 233], [234, 235], [236, 243], [244, 245], [246, 253], [254, 262], [263, 268], [268, 269], [270, 278], [279, 280], [281, 288], [289, 290], [291, 298], [299, 307], [308, 313], [313, 314], [315, 323], [324, 325], [326, 333], [334, 335], [336, 343], [344, 352], [353, 358], [358, 359], [360, 368], [369, 370], [371, 378], [379, 380], [381, 388], [389, 397], [398, 403], [403, 404], [405, 413], [414, 415], [416, 423], [424, 425], [426, 433], [434, 442], [443, 448], [448, 449], [450, 458], [459, 461], [462, 469], [470, 471], [472, 479], [480, 488], [489, 494], [494, 495], [496, 504], [505, 507], [508, 515], [516, 517], [518, 525], [526, 534], [535, 540], [540, 541], [542, 550], [551, 553], [554, 561], [562, 563], [564, 571], [572, 580], [581, 586], [586, 587], [588, 596], [597, 599], [600, 607], [608, 609], [610, 617], [618, 626], [627, 632], [632, 633], [634, 642], [643, 645], [646, 653], [654, 655], [656, 663], [664, 672], [673, 678], [678, 679], [680, 688], [689, 691], [692, 699], [700, 701], [702, 709], [710, 718], [719, 724], [724, 725], [726, 734], [735, 737], [738, 745], [746, 747], [748, 755], [756, 764], [765, 770], [770, 771], [772, 780], [781, 783], [784, 791], [792, 793], [794, 801], [802, 810], [811, 816], [816, 817], [818, 826], [827, 829], [830, 837], [838, 839], [840, 847], [848, 856], [857, 862], [862, 863], [864, 872], [873, 875], [876, 883], [884, 885], [886, 893], [894, 902], [903, 908], [908, 909], [910, 918], [919, 921], [922, 929], [930, 931], [932, 939], [940, 948], [949, 954], [954, 955], [956, 964], [965, 967], [968, 975], [976, 977], [978, 985], [986, 994], [995, 1000], [1000, 1001], [1002, 1010], [1011, 1013], [1014, 1021], [1022, 1023], [1024, 1031], [1032, 1040], [1041, 1046], [1046, 1047], [1048, 1056], [1057, 1059], [1060, 1067], [1068, 1069], [1070, 1077], [1078, 1086], [1087, 1092], [1092, 1093], [1094, 1102], [1103, 1105], [1106, 1113], [1114, 1115], [1116, 1123], [1124, 1132], [1133, 1138], [1138, 1139], [1140, 1148], [1149, 1151], [1152, 1159], [1160, 1161], [1162, 1169], [1170, 1178], [1179, 1184], [1184, 1185], [1186, 1194], [1195, 1197], [1198, 1205], [1206, 1207], [1208, 1215], [1216, 1224], [1225, 1230], [1230, 1231], [1232, 1240], [1241, 1243], [1244, 1251], [1252, 1253], [1254, 1261], [1262, 1270], [1271, 1276], [1276, 1277], [1278, 1286], [1287, 1289], [1290, 1297], [1298, 1299], [1300, 1307], [1308, 1316], [1317, 1322], [1322, 1323], [1324, 1332], [1333, 1335], [1336, 1343], [1344, 1345], [1346, 1353], [1354, 1362], [1363, 1368], [1368, 1369], [1370, 1378], [1379, 1381], [1382, 1389], [1390, 1391], [1392, 1399], [1400, 1408], [1409, 1414], [1414, 1415], [1416, 1424], [1425, 1427], [1428, 1435], [1436, 1437], [1438, 1445], [1446, 1454], [1455, 1460], [1460, 1461], [1462, 1470], [1471, 1473], [1474, 1481], [1482, 1483], [1484, 1491], [1492, 1500], [1501, 1506], [1506, 1507], [1508, 1516], [1517, 1519], [1520, 1527], [1528, 1529], [1530, 1537], [1538, 1546], [1547, 1552], [1552, 1553], [1554, 1562], [1563, 1565], [1566, 1573], [1574, 1575], [1576, 1583], [1584, 1592], [1593, 1598], [1598, 1599], [1600, 1608], [1609, 1611], [1612, 1619], [1620, 1621], [1622, 1629], [1630, 1638], [1639, 1644], [1644, 1645], [1646, 1654], [1655, 1657], [1658, 1665], [1666, 1667], [1668, 1675], [1676, 1684], [1685, 1690], [1690, 1691], [1692, 1700], [1701, 1703], [1704, 1711], [1712, 1713], [1714, 1721], [1722, 1730], [1731, 1736], [1736, 1737], [1738, 1746], [1747, 1749], [1750, 1757], [1758, 1759], [1760, 1767], [1768, 1776], [1777, 1782], [1782, 1783], [1784, 1792], [1793, 1795], [1796, 1803], [1804, 1805], [1806, 1813], [1814, 1822], [1823, 1828], [1828, 1829], [1830, 1838], [1839, 1841], [1842, 1849], [1850, 1851], [1852, 1859], [1860, 1868], [1869, 1874], [1874, 1875], [1876, 1884], [1885, 1887], [1888, 1895], [1896, 1897], [1898, 1905], [1906, 1914], [1915, 1920], [1920, 1921], [1922, 1930], [1931, 1933], [1934, 1941], [1942, 1943], [1944, 1951], [1952, 1960], [1961, 1966], [1966, 1967], [1968, 1976], [1977, 1979], [1980, 1987], [1988, 1989], [1990, 1997], [1998, 2006], [2007, 2012], [2012, 2013], [2014, 2022], [2023, 2025], [2026, 2033], [2034, 2035], [2036, 2043], [2044, 2052], [2053, 2058], [2058, 2059], [2060, 2068], [2069, 2071], [2072, 2079], [2080, 2081], [2082, 2089], [2090, 2098], [2099, 2104], [2104, 2105], [2106, 2114], [2115, 2117], [2118, 2125], [2126, 2127], [2128, 2135], [2136, 2144], [2145, 2150], [2150, 2151], [2152, 2160], [2161, 2163], [2164, 2171], [2172, 2173], [2174, 2181], [2182, 2190], [2191, 2196], [2196, 2197], [2198, 2206], [2207, 2209], [2210, 2217], [2218, 2219], [2220, 2227], [2228, 2236], [2237, 2242], [2242, 2243], [2244, 2252], [2253, 2255], [2256, 2263], [2264, 2265], [2266, 2273], [2274, 2282], [2283, 2288], [2288, 2289], [2290, 2298], [2299, 2301], [2302, 2309], [2310, 2311], [2312, 2319], [2320, 2328], [2329, 2334], [2334, 2335], [2336, 2344], [2345, 2347], [2348, 2355], [2356, 2357], [2358, 2365], [2366, 2374], [2375, 2380], [2380, 2381], [2382, 2390], [2391, 2393], [2394, 2401], [2402, 2403], [2404, 2411], [2412, 2420], [2421, 2426], [2426, 2427], [2428, 2436], [2437, 2439], [2440, 2447], [2448, 2449], [2450, 2457], [2458, 2466], [2467, 2472], [2472, 2473], [2474, 2482], [2483, 2485], [2486, 2493], [2494, 2495], [2496, 2503], [2504, 2512], [2513, 2518], [2518, 2519], [2520, 2528], [2529, 2531], [2532, 2539], [2540, 2541], [2542, 2549], [2550, 2558], [2559, 2564], [2564, 2565], [2566, 2574], [2575, 2577], [2578, 2585], [2586, 2587], [2588, 2595], [2596, 2604], [2605, 2610], [2610, 2611], [2612, 2620], [2621, 2623], [2624, 2631], [2632, 2633], [2634, 2641], [2642, 2650], [2651, 2656], [2656, 2657], [2658, 2666], [2667, 2669], [2670, 2677], [2678, 2679], [2680, 2687], [2688, 2696], [2697, 2702], [2702, 2703], [2704, 2712], [2713, 2715], [2716, 2723], [2724, 2725], [2726, 2733], [2734, 2742], [2743, 2748], [2748, 2749], [2750, 2758], [2759, 2761], [2762, 2769], [2770, 2771], [2772, 2779], [2780, 2788], [2789, 2794], [2794, 2795], [2796, 2804], [2805, 2807], [2808, 2815], [2816, 2817], [2818, 2825], [2826, 2834], [2835, 2840], [2840, 2841], [2842, 2850], [2851, 2853], [2854, 2861], [2862, 2863], [2864, 2871], [2872, 2880], [2881, 2886], [2886, 2887], [2888, 2896], [2897, 2899], [2900, 2907], [2908, 2909], [2910, 2917], [2918, 2926], [2927, 2932], [2932, 2933], [2934, 2942], [2943, 2945], [2946, 2953], [2954, 2955], [2956, 2963], [2964, 2972], [2973, 2978], [2978, 2979], [2980, 2988], [2989, 2991], [2992, 2999], [3000, 3001], [3002, 3009], [3010, 3018], [3019, 3024], [3024, 3025], [3026, 3034], [3035, 3037], [3038, 3045], [3046, 3047], [3048, 3055], [3056, 3064], [3065, 3070], [3070, 3071], [3072, 3080], [3081, 3083], [3084, 3091], [3092, 3093], [3094, 3101], [3102, 3110], [3111, 3116], [3116, 3117], [3118, 3126], [3127, 3129], [3130, 3137], [3138, 3139], [3140, 3147], [3148, 3156], [3157, 3162], [3162, 3163], [3164, 3172], [3173, 3175], [3176, 3183], [3184, 3185], [3186, 3193], [3194, 3202], [3203, 3208], [3208, 3209], [3210, 3218], [3219, 3221], [3222, 3229], [3230, 3231], [3232, 3239], [3240, 3248], [3249, 3254], [3254, 3255], [3256, 3264], [3265, 3267], [3268, 3275], [3276, 3277], [3278, 3285], [3286, 3294], [3295, 3300], [3300, 3301], [3302, 3310], [3311, 3313], [3314, 3321], [3322, 3323], [3324, 3331], [3332, 3340], [3341, 3346], [3346, 3347], [3348, 3356], [3357, 3359], [3360, 3367], [3368, 3369], [3370, 3377], [3378, 3386], [3387, 3392], [3392, 3393], [3394, 3402], [3403, 3405], [3406, 3413], [3414, 3415], [3416, 3423], [3424, 3432], [3433, 3438], [3438, 3439], [3440, 3448], [3449, 3451], [3452, 3459], [3460, 3461], [3462, 3469], [3470, 3478], [3479, 3484], [3484, 3485], [3486, 3494], [3495, 3497], [3498, 3505], [3506, 3507], [3508, 3515], [3516, 3524], [3525, 3530], [3530, 3531], [3532, 3540], [3541, 3543], [3544, 3551], [3552, 3553], [3554, 3561], [3562, 3570], [3571, 3576], [3576, 3577], [3578, 3586], [3587, 3589], [3590, 3597], [3598, 3599], [3600, 3607], [3608, 3616], [3617, 3622], [3622, 3623], [3624, 3632], [3633, 3635], [3636, 3643], [3644, 3645], [3646, 3653], [3654, 3662], [3663, 3668], [3668, 3669], [3670, 3678], [3679, 3681], [3682, 3689], [3690, 3691], [3692, 3699], [3700, 3708], [3709, 3714], [3714, 3715], [3716, 3724], [3725, 3727], [3728, 3735], [3736, 3737], [3738, 3745], [3746, 3754], [3755, 3760], [3760, 3761], [3762, 3770], [3771, 3773], [3774, 3781], [3782, 3783], [3784, 3791], [3792, 3800], [3801, 3806], [3806, 3807], [3808, 3816], [3817, 3819], [3820, 3827], [3828, 3829], [3830, 3837], [3838, 3846], [3847, 3852], [3852, 3853], [3854, 3862], [3863, 3865], [3866, 3873], [3874, 3875], [3876, 3883], [3884, 3892], [3893, 3898], [3898, 3899], [3900, 3908], [3909, 3911], [3912, 3919], [3920, 3921], [3922, 3929], [3930, 3938], [3939, 3944], [3944, 3945], [3946, 3954], [3955, 3957], [3958, 3965], [3966, 3967], [3968, 3975], [3976, 3984], [3985, 3990], [3990, 3991], [3992, 4000], [4001, 4003], [4004, 4011], [4012, 4013], [4014, 4021], [4022, 4030], [4031, 4036], [4036, 4037], [4038, 4046], [4047, 4049], [4050, 4057], [4058, 4059], [4060, 4067], [4068, 4076], [4077, 4082], [4082, 4083], [4084, 4092], [4093, 4095], [4096, 4103], [4104, 4105], [4106, 4113], [4114, 4122], [4123, 4128], [4128, 4129], [4130, 4138], [4139, 4141], [4142, 4149], [4150, 4151], [4152, 4159], [4160, 4168], [4169, 4174], [4174, 4175], [4176, 4184], [4185, 4187], [4188, 4195], [4196, 4197], [4198, 4205], [4206, 4214], [4215, 4220], [4220, 4221], [4222, 4230], [4231, 4233], [4234, 4241], [4242, 4243], [4244, 4251], [4252, 4260], [4261, 4266], [4266, 4267], [4268, 4276], [4277, 4279], [4280, 4287], [4288, 4289], [4290, 4297], [4298, 4306], [4307, 4312], [4312, 4313], [4314, 4322], [4323, 4325], [4326, 4333], [4334, 4335], [4336, 4343], [4344, 4352], [4353, 4358], [4358, 4359], [4360, 4368], [4369, 4371], [4372, 4379], [4380, 4381], [4382, 4389], [4390, 4398], [4399, 4404], [4404, 4405], [4406, 4414], [4415, 4417], [4418, 4425], [4426, 4427], [4428, 4435], [4436, 4444], [4445, 4450], [4450, 4451], [4452, 4460], [4461, 4463], [4464, 4471], [4472, 4473], [4474, 4481], [4482, 4490], [4491, 4496], [4496, 4497], [4498, 4506], [4507, 4509], [4510, 4517], [4518, 4519], [4520, 4527], [4528, 4536], [4537, 4542], [4542, 4543], [4544, 4552], [4553, 4555], [4556, 4563], [4564, 4565], [4566, 4573], [4574, 4582], [4583, 4588], [4588, 4589], [4590, 4598], [4599, 4602], [4603, 4610], [4611, 4612], [4613, 4620], [4621, 4629], [4630, 4635], [4635, 4636], [4637, 4645], [4646, 4649], [4650, 4657], [4658, 4659], [4660, 4667], [4668, 4676], [4677, 4682], [4682, 4683], [4684, 4692], [4693, 4696], [4697, 4704], [4705, 4706], [4707, 4714], [4715, 4723], [4724, 4729], [4729, 4730], [4731, 4739], [4740, 4743], [4744, 4751], [4752, 4753], [4754, 4761], [4762, 4770], [4771, 4776], [4776, 4777], [4778, 4786], [4787, 4790], [4791, 4798], [4799, 4800], [4801, 4808], [4809, 4817], [4818, 4823], [4823, 4824], [4825, 4833], [4834, 4837], [4838, 4845], [4846, 4847], [4848, 4855], [4856, 4864], [4865, 4870], [4870, 4871], [4872, 4880], [4881, 4884], [4885, 4892], [4893, 4894], [4895, 4902], [4903, 4911], [4912, 4917], [4917, 4918], [4919, 4927], [4928, 4931], [4932, 4939], [4940, 4941], [4942, 4949], [4950, 4958], [4959, 4964], [4964, 4965], [4966, 4974], [4975, 4978], [4979, 4986], [4987, 4988], [4989, 4996], [4997, 5005], [5006, 5011], [5011, 5012], [5013, 5021], [5022, 5025], [5026, 5033], [5034, 5035], [5036, 5043], [5044, 5052], [5053, 5058], [5058, 5059], [5060, 5068], [5069, 5072], [5073, 5080], [5081, 5082], [5083, 5090], [5091, 5099], [5100, 5105], [5105, 5106], [5107, 5115], [5116, 5119], [5120, 5127], [5128, 5129], [5130, 5137], [5138, 5146], [5147, 5152], [5152, 5153], [5154, 5162], [5163, 5166], [5167, 5174], [5175, 5176], [5177, 5184], [5185, 5193], [5194, 5199], [5199, 5200], [5201, 5209], [5210, 5213], [5214, 5221], [5222, 5223], [5224, 5231], [5232, 5240], [5241, 5246], [5246, 5247], [5248, 5256], [5257, 5260], [5261, 5268], [5269, 5270], [5271, 5278], [5279, 5287], [5288, 5293], [5293, 5294], [5295, 5303], [5304, 5307], [5308, 5315], [5316, 5317], [5318, 5325], [5326, 5334], [5335, 5340], [5340, 5341], [5342, 5350], [5351, 5354], [5355, 5362], [5363, 5364], [5365, 5372], [5373, 5381], [5382, 5387], [5387, 5388], [5389, 5397], [5398, 5401], [5402, 5409], [5410, 5411], [5412, 5419], [5420, 5428], [5429, 5434], [5434, 5435], [5436, 5444], [5445, 5448], [5449, 5456], [5457, 5458], [5459, 5466], [5467, 5475], [5476, 5481], [5481, 5482], [5483, 5491], [5492, 5495], [5496, 5503], [5504, 5505], [5506, 5513], [5514, 5522], [5523, 5528], [5528, 5529], [5530, 5538], [5539, 5542], [5543, 5550], [5551, 5552], [5553, 5560], [5561, 5569], [5570, 5575], [5575, 5576], [5577, 5585], [5586, 5589], [5590, 5597], [5598, 5599], [5600, 5607], [5608, 5616], [5617, 5622], [5622, 5623], [5624, 5632], [5633, 5636], [5637, 5644], [5645, 5646], [5647, 5654], [5655, 5663], [5664, 5669], [5669, 5670], [5671, 5679], [5680, 5683], [5684, 5691], [5692, 5693], [5694, 5701], [5702, 5710], [5711, 5716], [5716, 5717], [5718, 5726], [5727, 5730], [5731, 5738], [5739, 5740], [5741, 5748], [5749, 5757], [5758, 5763], [5763, 5764], [5765, 5773], [5774, 5777], [5778, 5785], [5786, 5787], [5788, 5795], [5796, 5804], [5805, 5810], [5810, 5811], [5812, 5820], [5821, 5824], [5825, 5832], [5833, 5834], [5835, 5842], [5843, 5851], [5852, 5857], [5857, 5858], [5859, 5867], [5868, 5871], [5872, 5879], [5880, 5881], [5882, 5889], [5890, 5898], [5899, 5904], [5904, 5905], [5906, 5914], [5915, 5918], [5919, 5926], [5927, 5928], [5929, 5936], [5937, 5945], [5946, 5951], [5951, 5952], [5953, 5961], [5962, 5965], [5966, 5973], [5974, 5975], [5976, 5983], [5984, 5992], [5993, 5998], [5998, 5999], [6000, 6008], [6009, 6012], [6013, 6020], [6021, 6022], [6023, 6030], [6031, 6039], [6040, 6045], [6045, 6046], [6047, 6055], [6056, 6059], [6060, 6067], [6068, 6069], [6070, 6077], [6078, 6086], [6087, 6092], [6092, 6093], [6094, 6102], [6103, 6106], [6107, 6114], [6115, 6116], [6117, 6124], [6125, 6133], [6134, 6139], [6139, 6140], [6141, 6149], [6150, 6153], [6154, 6161], [6162, 6163], [6164, 6171], [6172, 6180], [6181, 6186], [6186, 6187], [6188, 6196], [6197, 6200], [6201, 6208], [6209, 6210], [6211, 6218], [6219, 6227], [6228, 6233], [6233, 6234], [6235, 6243], [6244, 6247], [6248, 6255], [6256, 6257], [6258, 6265], [6266, 6274], [6275, 6280], [6280, 6281], [6282, 6290], [6291, 6294], [6295, 6302], [6303, 6304], [6305, 6312], [6313, 6321], [6322, 6327], [6327, 6328], [6329, 6337], [6338, 6341], [6342, 6349], [6350, 6351], [6352, 6359], [6360, 6368], [6369, 6374], [6374, 6375], [6376, 6384], [6385, 6388], [6389, 6396], [6397, 6398], [6399, 6406], [6407, 6415], [6416, 6421], [6421, 6422], [6423, 6431], [6432, 6435], [6436, 6443], [6444, 6445], [6446, 6453], [6454, 6462], [6463, 6468], [6468, 6469], [6470, 6478], [6479, 6482], [6483, 6490], [6491, 6492], [6493, 6500], [6501, 6509], [6510, 6515], [6515, 6516], [6517, 6525], [6526, 6529], [6530, 6537], [6538, 6539], [6540, 6547], [6548, 6556], [6557, 6562], [6562, 6563], [6564, 6572], [6573, 6576], [6577, 6584], [6585, 6586], [6587, 6594], [6595, 6603], [6604, 6609], [6609, 6610], [6611, 6619], [6620, 6623], [6624, 6631], [6632, 6633], [6634, 6641], [6642, 6650], [6651, 6656], [6656, 6657], [6658, 6666], [6667, 6670], [6671, 6678], [6679, 6680], [6681, 6688], [6689, 6697], [6698, 6703], [6703, 6704], [6705, 6713], [6714, 6717], [6718, 6725], [6726, 6727], [6728, 6735], [6736, 6744], [6745, 6750], [6750, 6751], [6752, 6760], [6761, 6764], [6765, 6772], [6773, 6774], [6775, 6782], [6783, 6791], [6792, 6797], [6797, 6798], [6799, 6807], [6808, 6811], [6812, 6819], [6820, 6821], [6822, 6829], [6830, 6838], [6839, 6844], [6844, 6845], [6846, 6854], [6855, 6858], [6859, 6866], [6867, 6868], [6869, 6876], [6877, 6885], [6886, 6891], [6891, 6892], [6893, 6901], [6902, 6905], [6906, 6913], [6914, 6915], [6916, 6923], [6924, 6932], [6933, 6938], [6938, 6939], [6940, 6948], [6949, 6952], [6953, 6960], [6961, 6962], [6963, 6970], [6971, 6979], [6980, 6985], [6985, 6986], [6987, 6995], [6996, 6999], [7000, 7007], [7008, 7009], [7010, 7017], [7018, 7026], [7027, 7032], [7032, 7033], [7034, 7042], [7043, 7046], [7047, 7054], [7055, 7056], [7057, 7064], [7065, 7073], [7074, 7079], [7079, 7080], [7081, 7089], [7090, 7093], [7094, 7101], [7102, 7103], [7104, 7111], [7112, 7120], [7121, 7126], [7126, 7127], [7128, 7136], [7137, 7140], [7141, 7148], [7149, 7150], [7151, 7158], [7159, 7167], [7168, 7173], [7173, 7174], [7175, 7183], [7184, 7187], [7188, 7195], [7196, 7197], [7198, 7205], [7206, 7214], [7215, 7220], [7220, 7221], [7222, 7230], [7231, 7234], [7235, 7242], [7243, 7244], [7245, 7252], [7253, 7261], [7262, 7267], [7267, 7268], [7269, 7277], [7278, 7281], [7282, 7289], [7290, 7291], [7292, 7299], [7300, 7308], [7309, 7314], [7314, 7315], [7316, 7324], [7325, 7328], [7329, 7336], [7337, 7338], [7339, 7346], [7347, 7355], [7356, 7361], [7361, 7362], [7363, 7371], [7372, 7375], [7376, 7383], [7384, 7385], [7386, 7393], [7394, 7402], [7403, 7408], [7408, 7409], [7410, 7418], [7419, 7422], [7423, 7430], [7431, 7432], [7433, 7440], [7441, 7449], [7450, 7455], [7455, 7456], [7457, 7465], [7466, 7469], [7470, 7477], [7478, 7479], [7480, 7487], [7488, 7496], [7497, 7502], [7502, 7503], [7504, 7512], [7513, 7516], [7517, 7524], [7525, 7526], [7527, 7534], [7535, 7543], [7544, 7549], [7549, 7550], [7551, 7559], [7560, 7563], [7564, 7571], [7572, 7573], [7574, 7581], [7582, 7590], [7591, 7596], [7596, 7597], [7598, 7606], [7607, 7610], [7611, 7618], [7619, 7620], [7621, 7628], [7629, 7637], [7638, 7643], [7643, 7644], [7645, 7653], [7654, 7657], [7658, 7665], [7666, 7667], [7668, 7675], [7676, 7684], [7685, 7690], [7690, 7691], [7692, 7700], [7701, 7704], [7705, 7712], [7713, 7714], [7715, 7722], [7723, 7731], [7732, 7737], [7737, 7738], [7739, 7747], [7748, 7751], [7752, 7759], [7760, 7761], [7762, 7769], [7770, 7778], [7779, 7784], [7784, 7785], [7786, 7794], [7795, 7798], [7799, 7806], [7807, 7808], [7809, 7816], [7817, 7825], [7826, 7831], [7831, 7832], [7833, 7841], [7842, 7845], [7846, 7853], [7854, 7855], [7856, 7863], [7864, 7872], [7873, 7878], [7878, 7879], [7880, 7888], [7889, 7892], [7893, 7900], [7901, 7902], [7903, 7910], [7911, 7919], [7920, 7925], [7925, 7926], [7927, 7935], [7936, 7939], [7940, 7947], [7948, 7949], [7950, 7957], [7958, 7966], [7967, 7972], [7972, 7973], [7974, 7982], [7983, 7986], [7987, 7994], [7995, 7996], [7997, 8004], [8005, 8013], [8014, 8019], [8019, 8020], [8021, 8029], [8030, 8033], [8034, 8041]], "w_pos": [52, 54, 55, 56, 57, 58, 59, 60, 61, 63, 64, 65, 66, 67, 68, 69, 70, 72, 73, 74, 75, 76, 77, 78, 79, 81, 82, 83, 84, 85, 86, 87, 88, 90, 91, 92, 93, 94, 95, 96, 97, 99, 100, 101, 102, 103, 104, 105, 106, 108, 109, 110, 111, 112, 113, 114, 115, 117, 118, 119, 120, 121, 122, 123, 124, 126, 127, 128, 129, 130, 131, 132, 133, 135, 136, 137, 138, 139, 140, 141, 142, 144, 146, 147, 148, 149, 150, 151, 152, 154, 156, 157, 158, 159, 160, 161, 162, 164, 166, 167, 168, 169, 170, 171, 172, 174, 176, 177, 178, 179, 180, 181, 182, 184, 186, 187, 188, 189, 190, 191, 192, 194, 196, 197, 198, 199, 200, 201, 202, 204, 206, 207, 208, 209, 210, 211, 212, 214, 216, 217, 218, 219, 220, 221, 222, 224, 226, 227, 228, 229, 230, 231, 232, 234, 236, 237, 238, 239, 240, 241, 242, 244, 246, 247, 248, 249, 250, 251, 252, 254, 256, 257, 258, 259, 260, 261, 262, 264, 266, 267, 268, 269, 270, 271, 272, 274, 276, 277, 278, 279, 280, 281, 282, 284, 286, 287, 288, 289, 290, 291, 292, 294, 296, 297, 298, 299, 300, 301, 302, 304, 306, 307, 308, 309, 310, 311, 312, 314, 316, 317, 318, 319, 320, 321, 322, 324, 326, 327, 328, 329, 330, 331, 332, 334, 336, 337, 338, 339, 340, 341, 342, 344, 346, 347, 348, 349, 350, 351, 352, 354, 356, 357, 358, 359, 360, 361, 362, 364, 366, 367, 368, 369, 370, 371, 372, 374, 376, 377, 378, 379, 380, 381, 382, 384, 386, 387, 388, 389, 390, 391, 392, 394, 396, 397, 398, 399, 400, 401, 402, 404, 406, 407, 408, 409, 410, 411, 412, 414, 416, 417, 418, 419, 420, 421, 422, 424, 426, 427, 428, 429, 430, 431, 432, 434, 436, 437, 438, 439, 440, 441, 442, 444, 446, 447, 448, 449, 450, 451, 452, 454, 456, 457, 458, 459, 460, 461, 462, 464, 466, 467, 468, 469, 470, 471, 472, 474, 476, 477, 478, 479, 480, 481, 482, 484, 486, 487, 488, 489, 490, 491, 492, 494, 496, 497, 498, 499, 500, 501, 502, 504, 506, 507, 508, 509, 510, 511, 512, 514, 516, 517, 518, 519, 520, 521, 522, 524, 526, 527, 528, 529, 530, 531, 532, 534, 536, 537, 538, 539, 540, 541, 542, 544, 546, 547, 548, 549, 550, 551, 552, 554, 556, 557, 558, 559, 560, 561, 562, 564, 566, 567, 568, 569, 570, 571, 572, 574, 576, 577, 578, 579, 580, 581, 582, 584, 586, 587, 588, 589, 590, 591, 592, 594, 596, 597, 598, 599, 600, 601, 602, 604, 606, 607, 608, 609, 610, 611, 612, 614, 616, 617, 618, 619, 620, 621, 622, 624, 626, 627, 628, 629, 630, 631, 632, 634, 636, 637, 638, 639, 640, 641, 642, 644, 646, 647, 648, 649, 650, 651, 652, 654, 656, 657, 658, 659, 660, 661, 662, 664, 666, 667, 668, 669, 670, 671, 672, 674, 676, 677, 678, 679, 680, 681, 682, 684, 686, 687, 688, 689, 690, 691, 692, 694, 696, 697, 698, 699, 700, 701, 702, 704, 706, 707, 708, 709, 710, 711, 712, 714, 716, 717, 718, 719, 720, 721, 722, 724, 726, 727, 728, 729, 730, 731, 732, 734, 736, 737, 738, 739, 740, 741, 742, 744, 746, 747, 748, 749, 750, 751, 752, 754, 756, 757, 758, 759, 760, 761, 762, 764, 766, 767, 768, 769, 770, 771, 772, 774, 776, 777, 778, 779, 780, 781, 782, 784, 786, 787, 788, 789, 790, 791, 792, 794, 796, 797, 798, 799, 800, 801, 802, 804, 806, 807, 808, 809, 810, 811, 812, 814, 816, 817, 818, 819, 820, 821, 822, 824, 826, 827, 828, 829, 830, 831, 832, 834, 836, 837, 838, 839, 840, 841, 842, 844, 846, 847, 848, 849, 850, 851, 852, 854, 856, 857, 858, 859, 860, 861, 862, 864, 866, 867, 868, 869, 870, 871, 872, 874, 876, 877, 878, 879, 880, 881, 882, 884, 886, 887, 888, 889, 890, 891, 892, 894, 896, 897, 898, 899, 900, 901, 902, 904, 906, 907, 908, 909, 910, 911, 912, 914, 916, 917, 918, 919, 920, 921, 922, 924, 926, 927, 928, 929, 930, 931, 932, 934, 936, 937, 938, 939, 940, 941, 942, 944, 946, 947, 948, 949, 950, 951, 952, 954, 956, 957, 958, 959, 960, 961, 962, 964, 966, 967, 968, 969, 970, 971, 972, 974, 976, 977, 978, 979, 980, 981, 982, 984, 986, 987, 988, 989, 990, 991, 992, 994, 996, 997, 998, 999, 1000, 1001, 1002, 1004, 1006, 1007, 1008, 1009, 1010, 1011, 1012, 1014, 1016, 1017, 1018, 1019, 1020, 1021, 1022, 1024, 1026, 1027, 1028, 1029, 1030, 1031, 1032, 1034, 1036, 1037, 1038, 1039, 1040, 1041, 1042, 1044, 1047, 1048, 1049, 1050, 1051, 1052, 1053, 1055, 1058, 1059, 1060, 1061, 1062, 1063, 1064, 1066, 1069, 1070, 1071, 1072, 1073, 1074, 1075, 1077, 1080, 1081, 1082, 1083, 1084, 1085, 1086, 1088, 1091, 1092, 1093, 1094, 1095, 1096, 1097, 1099, 1102, 1103, 1104, 1105, 1106, 1107, 1108, 1110, 1113, 1114, 1115, 1116, 1117, 1118, 1119, 1121, 1124, 1125, 1126, 1127, 1128, 1129, 1130, 1132, 1135, 1136, 1137, 1138, 1139, 1140, 1141, 1143, 1146, 1147, 1148, 1149, 1150, 1151, 1152, 1154, 1157, 1158, 1159, 1160, 1161, 1162, 1163, 1165, 1168, 1169, 1170, 1171, 1172, 1173, 1174, 1176, 1179, 1180, 1181, 1182, 1183, 1184, 1185, 1187, 1190, 1191, 1192, 1193, 1194, 1195, 1196, 1198, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1209, 1212, 1213, 1214, 1215, 1216, 1217, 1218, 1220, 1223, 1224, 1225, 1226, 1227, 1228, 1229, 1231, 1234, 1235, 1236, 1237, 1238, 1239, 1240, 1242, 1245, 1246, 1247, 1248, 1249, 1250, 1251, 1253, 1256, 1257, 1258, 1259, 1260, 1261, 1262, 1264, 1267, 1268, 1269, 1270, 1271, 1272, 1273, 1275, 1278, 1279, 1280, 1281, 1282, 1283, 1284, 1286, 1289, 1290, 1291, 1292, 1293, 1294, 1295, 1297, 1300, 1301, 1302, 1303, 1304, 1305, 1306, 1308, 1311, 1312, 1313, 1314, 1315, 1316, 1317, 1319, 1322, 1323, 1324, 1325, 1326, 1327, 1328, 1330, 1333, 1334, 1335, 1336, 1337, 1338, 1339, 1341, 1344, 1345, 1346, 1347, 1348, 1349, 1350, 1352, 1355, 1356, 1357, 1358, 1359, 1360, 1361, 1363, 1366, 1367, 1368, 1369, 1370, 1371, 1372, 1374, 1377, 1378, 1379, 1380, 1381, 1382, 1383, 1385, 1388, 1389, 1390, 1391, 1392, 1393, 1394, 1396, 1399, 1400, 1401, 1402, 1403, 1404, 1405, 1407, 1410, 1411, 1412, 1413, 1414, 1415, 1416, 1418, 1421, 1422, 1423, 1424, 1425, 1426, 1427, 1429, 1432, 1433, 1434, 1435, 1436, 1437, 1438, 1440, 1443, 1444, 1445, 1446, 1447, 1448, 1449, 1451, 1454, 1455, 1456, 1457, 1458, 1459, 1460, 1462, 1465, 1466, 1467, 1468, 1469, 1470, 1471, 1473, 1476, 1477, 1478, 1479, 1480, 1481, 1482, 1484, 1487, 1488, 1489, 1490, 1491, 1492, 1493, 1495, 1498, 1499, 1500, 1501, 1502, 1503, 1504, 1506, 1509, 1510, 1511, 1512, 1513, 1514, 1515, 1517, 1520, 1521, 1522, 1523, 1524, 1525, 1526, 1528, 1531, 1532, 1533, 1534, 1535, 1536, 1537, 1539, 1542, 1543, 1544, 1545, 1546, 1547, 1548, 1550, 1553, 1554, 1555, 1556, 1557, 1558, 1559, 1561, 1564, 1565, 1566, 1567, 1568, 1569, 1570, 1572, 1575, 1576, 1577, 1578, 1579, 1580, 1581, 1583, 1586, 1587, 1588, 1589, 1590, 1591, 1592, 1594, 1597, 1598, 1599, 1600, 1601, 1602, 1603, 1605, 1608, 1609, 1610, 1611, 1612, 1613, 1614, 1616, 1619, 1620, 1621, 1622, 1623, 1624, 1625, 1627, 1630, 1631, 1632, 1633, 1634, 1635, 1636, 1638, 1641, 1642, 1643, 1644, 1645, 1646, 1647, 1649, 1652, 1653, 1654, 1655, 1656, 1657, 1658, 1660, 1663, 1664, 1665, 1666, 1667, 1668, 1669, 1671, 1674, 1675, 1676, 1677, 1678, 1679, 1680, 1682, 1685, 1686, 1687, 1688, 1689, 1690, 1691, 1693, 1696, 1697, 1698, 1699, 1700, 1701, 1702, 1704, 1707, 1708, 1709, 1710, 1711, 1712, 1713, 1715, 1718, 1719, 1720, 1721, 1722, 1723, 1724, 1726, 1729, 1730, 1731, 1732, 1733, 1734, 1735, 1737, 1740, 1741, 1742, 1743, 1744, 1745, 1746, 1748, 1751, 1752, 1753, 1754, 1755, 1756, 1757, 1759, 1762, 1763, 1764, 1765, 1766, 1767, 1768, 1770, 1773, 1774, 1775, 1776, 1777, 1778, 1779, 1781, 1784, 1785, 1786, 1787, 1788, 1789, 1790, 1792, 1795, 1796, 1797, 1798, 1799, 1800, 1801, 1803, 1806, 1807, 1808, 1809, 1810, 1811, 1812, 1814, 1817, 1818, 1819, 1820, 1821, 1822, 1823, 1825, 1828, 1829, 1830, 1831, 1832, 1833, 1834, 1836, 1839, 1840, 1841, 1842, 1843, 1844, 1845, 1847, 1850], "span_over": "user", "n_tokens": 3689, "head": "broad"}, {"p_ids": [1905, 25, 198, 27, 22446, 3364, 420, 846, 731, 198, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 24, 6473, 264, 13618, 9403, 1484, 13, 198, 510, 22446, 29, 198], "blocks": [{"kind": "span", "ends": [40], "label_ranges": [[22, 40]], "words_in_block": true, "ids": [271, 6065, 913, 25, 9111, 198, 6196, 25, 1156, 198, 14162, 25, 198, 22365, 42824, 6088, 264, 9403, 30, 198, 22933, 25, 198, 14057, 29, 198, 4754, 4532, 3147, 64, 9403, 1484, 2129, 767, 3147, 31320, 8934, 198, 510, 1448, 29, 271, 1138, 25, 198, 27, 22446, 3364, 420, 846, 731, 198, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 24, 6473, 264, 13618, 9403, 1484, 13, 198, 510, 22446, 29, 198, 271, 8679, 1396, 9111, 314, 279, 2100, 1414, 421, 22651, 799, 314, 279, 8928, 13, 198, 6205, 581, 25], "query": 849, "w_pos": [52, 53, 54, 55, 56, 58, 61, 62, 63, 64, 65, 66, 67, 69, 72, 73, 74, 75, 76, 77, 78, 80, 83, 84, 85, 86, 87, 88, 89, 91, 94, 95, 96, 97, 98, 99, 100, 102, 105, 106, 107, 108, 109, 110, 111, 113, 116, 117, 118, 119, 120, 121, 122, 124, 127, 128, 129, 130, 131, 132, 133, 135, 138, 139, 140, 141, 142, 143, 144, 146, 149, 150, 151, 152, 153, 154, 155, 157, 160, 161, 162, 163, 164, 165, 166, 168, 171, 172, 173, 174, 175, 176, 177, 179, 182, 183, 184, 185, 186, 187, 188, 190, 193, 194, 195, 196, 197, 198, 199, 201, 204, 205, 206, 207, 208, 209, 210, 212, 215, 216, 217, 218, 219, 220, 221, 223, 226, 227, 228, 229, 230, 231, 232, 234, 237, 238, 239, 240, 241, 242, 243, 245, 248, 249, 250, 251, 252, 253, 254, 256, 259, 260, 261, 262, 263, 264, 265, 267, 270, 271, 272, 273, 274, 275, 276, 278, 281, 282, 283, 284, 285, 286, 287, 289, 292, 293, 294, 295, 296, 297, 298, 300, 303, 304, 305, 306, 307, 308, 309, 311, 314, 315, 316, 317, 318, 319, 320, 322, 325, 326, 327, 328, 329, 330, 331, 333, 336, 337, 338, 339, 340, 341, 342, 344, 347, 348, 349, 350, 351, 352, 353, 355, 358, 359, 360, 361, 362, 363, 364, 366, 369, 370, 371, 372, 373, 374, 375, 377, 380, 381, 382, 383, 384, 385, 386, 388, 391, 392, 393, 394, 395, 396, 397, 399, 402, 403, 404, 405, 406, 407, 408, 410, 413, 414, 415, 416, 417, 418, 419, 421, 424, 425, 426, 427, 428, 429, 430, 432, 435, 436, 437, 438, 439, 440, 441, 443, 446, 447, 448, 449, 450, 451, 452, 454, 457, 458, 459, 460, 461, 462, 463, 465, 468, 469, 470, 471, 472, 473, 474, 476, 479, 480, 481, 482, 483, 484, 485, 487, 490, 491, 492, 493, 494, 495, 496, 498, 501, 502, 503, 504, 505, 506, 507, 509, 512, 513, 514, 515, 516, 517, 518, 520, 523, 524, 525, 526, 527, 528, 529, 531, 534, 535, 536, 537, 538, 539, 540, 542, 545, 546, 547, 548, 549, 550, 551, 553, 556, 557, 558, 559, 560, 561, 562, 564, 567, 568, 569, 570, 571, 572, 573, 575, 578, 579, 580, 581, 582, 583, 584, 586, 589, 590, 591, 592, 593, 594, 595, 597, 600, 601, 602, 603, 604, 605, 606, 608, 611, 612, 613, 614, 615, 616, 617, 619, 622, 623, 624, 625, 626, 627, 628, 630, 633, 634, 635, 636, 637, 638, 639, 641, 644, 645, 646, 647, 648, 649, 650, 652, 655, 656, 657, 658, 659, 660, 661, 663, 666, 667, 668, 669, 670, 671, 672, 674, 677, 678, 679, 680, 681, 682, 683, 685, 688, 689, 690, 691, 692, 693, 694, 696, 699, 700, 701, 702, 703, 704, 705, 707, 710, 711, 712, 713, 714, 715, 716, 718, 721, 722, 723, 724, 725, 726, 727, 729, 732, 733, 734, 735, 736, 737, 738, 740, 743, 744, 745, 746, 747, 748, 749, 751, 754, 755, 756, 757, 758, 759, 760, 762, 765, 766, 767, 768, 769, 770, 771, 773, 776, 777, 778, 779, 780, 781, 782, 784, 787, 788, 789, 790, 791, 792, 793, 795, 798, 799, 800, 801, 802, 803, 804, 806, 809, 810, 811, 812, 813, 814, 815, 817, 820, 821, 822, 823, 824, 825]}], "trunc": {"shrink_cut": 0, "protected_cut": 0}, "seg_lens": {"user": 774}, "layout": "c", "names": ["delivery"], "w_offs": [[6916, 6923], [6924, 6932], [6933, 6938], [6938, 6939], [6940, 6948], [6949, 6952], [6953, 6960], [6961, 6962], [6963, 6970], [6971, 6979], [6980, 6985], [6985, 6986], [6987, 6995], [6996, 6999], [7000, 7007], [7008, 7009], [7010, 7017], [7018, 7026], [7027, 7032], [7032, 7033], [7034, 7042], [7043, 7046], [7047, 7054], [7055, 7056], [7057, 7064], [7065, 7073], [7074, 7079], [7079, 7080], [7081, 7089], [7090, 7093], [7094, 7101], [7102, 7103], [7104, 7111], [7112, 7120], [7121, 7126], [7126, 7127], [7128, 7136], [7137, 7140], [7141, 7148], [7149, 7150], [7151, 7158], [7159, 7167], [7168, 7173], [7173, 7174], [7175, 7183], [7184, 7187], [7188, 7195], [7196, 7197], [7198, 7205], [7206, 7214], [7215, 7220], [7220, 7221], [7222, 7230], [7231, 7234], [7235, 7242], [7243, 7244], [7245, 7252], [7253, 7261], [7262, 7267], [7267, 7268], [7269, 7277], [7278, 7281], [7282, 7289], [7290, 7291], [7292, 7299], [7300, 7308], [7309, 7314], [7314, 7315], [7316, 7324], [7325, 7328], [7329, 7336], [7337, 7338], [7339, 7346], [7347, 7355], [7356, 7361], [7361, 7362], [7363, 7371], [7372, 7375], [7376, 7383], [7384, 7385], [7386, 7393], [7394, 7402], [7403, 7408], [7408, 7409], [7410, 7418], [7419, 7422], [7423, 7430], [7431, 7432], [7433, 7440], [7441, 7449], [7450, 7455], [7455, 7456], [7457, 7465], [7466, 7469], [7470, 7477], [7478, 7479], [7480, 7487], [7488, 7496], [7497, 7502], [7502, 7503], [7504, 7512], [7513, 7516], [7517, 7524], [7525, 7526], [7527, 7534], [7535, 7543], [7544, 7549], [7549, 7550], [7551, 7559], [7560, 7563], [7564, 7571], [7572, 7573], [7574, 7581], [7582, 7590], [7591, 7596], [7596, 7597], [7598, 7606], [7607, 7610], [7611, 7618], [7619, 7620], [7621, 7628], [7629, 7637], [7638, 7643], [7643, 7644], [7645, 7653], [7654, 7657], [7658, 7665], [7666, 7667], [7668, 7675], [7676, 7684], [7685, 7690], [7690, 7691], [7692, 7700], [7701, 7704], [7705, 7712], [7713, 7714], [7715, 7722], [7723, 7731], [7732, 7737], [7737, 7738], [7739, 7747], [7748, 7751], [7752, 7759], [7760, 7761], [7762, 7769], [7770, 7778], [7779, 7784], [7784, 7785], [7786, 7794], [7795, 7798], [7799, 7806], [7807, 7808], [7809, 7816], [7817, 7825], [7826, 7831], [7831, 7832], [7833, 7841], [7842, 7845], [7846, 7853], [7854, 7855], [7856, 7863], [7864, 7872], [7873, 7878], [7878, 7879], [7880, 7888], [7889, 7892], [7893, 7900], [7901, 7902], [7903, 7910], [7911, 7919], [7920, 7925], [7925, 7926], [7927, 7935], [7936, 7939], [7940, 7947], [7948, 7949], [7950, 7957], [7958, 7966], [7967, 7972], [7972, 7973], [7974, 7982], [7983, 7986], [7987, 7994], [7995, 7996], [7997, 8004], [8005, 8013], [8014, 8019], [8019, 8020], [8021, 8029], [8030, 8033], [8034, 8041], [8042, 8043], [8044, 8051], [8052, 8060], [8061, 8066], [8066, 8067], [8068, 8076], [8077, 8080], [8081, 8088], [8089, 8090], [8091, 8098], [8099, 8107], [8108, 8113], [8113, 8114], [8115, 8123], [8124, 8127], [8128, 8135], [8136, 8137], [8138, 8145], [8146, 8154], [8155, 8160], [8160, 8161], [8162, 8170], [8171, 8174], [8175, 8182], [8183, 8184], [8185, 8192], [8193, 8201], [8202, 8207], [8207, 8208], [8209, 8217], [8218, 8221], [8222, 8229], [8230, 8231], [8232, 8239], [8240, 8248], [8249, 8254], [8254, 8255], [8256, 8264], [8265, 8268], [8269, 8276], [8277, 8278], [8279, 8286], [8287, 8295], [8296, 8301], [8301, 8302], [8303, 8311], [8312, 8315], [8316, 8323], [8324, 8325], [8326, 8333], [8334, 8342], [8343, 8348], [8348, 8349], [8350, 8358], [8359, 8362], [8363, 8370], [8371, 8372], [8373, 8380], [8381, 8389], [8390, 8395], [8395, 8396], [8397, 8405], [8406, 8409], [8410, 8417], [8418, 8419], [8420, 8427], [8428, 8436], [8437, 8442], [8442, 8443], [8444, 8452], [8453, 8456], [8457, 8464], [8465, 8466], [8467, 8474], [8475, 8483], [8484, 8489], [8489, 8490], [8491, 8499], [8500, 8503], [8504, 8511], [8512, 8513], [8514, 8521], [8522, 8530], [8531, 8536], [8536, 8537], [8538, 8546], [8547, 8550], [8551, 8558], [8559, 8560], [8561, 8568], [8569, 8577], [8578, 8583], [8583, 8584], [8585, 8593], [8594, 8597], [8598, 8605], [8606, 8607], [8608, 8615], [8616, 8624], [8625, 8630], [8630, 8631], [8632, 8640], [8641, 8644], [8645, 8652], [8653, 8654], [8655, 8662], [8663, 8671], [8672, 8677], [8677, 8678], [8679, 8687], [8688, 8691], [8692, 8699], [8700, 8701], [8702, 8709], [8710, 8718], [8719, 8724], [8724, 8725], [8726, 8734], [8735, 8738], [8739, 8746], [8747, 8748], [8749, 8756], [8757, 8765], [8766, 8771], [8771, 8772], [8773, 8781], [8782, 8785], [8786, 8793], [8794, 8795], [8796, 8803], [8804, 8812], [8813, 8818], [8818, 8819], [8820, 8828], [8829, 8832], [8833, 8840], [8841, 8842], [8843, 8850], [8851, 8859], [8860, 8865], [8865, 8866], [8867, 8875], [8876, 8879], [8880, 8887], [8888, 8889], [8890, 8897], [8898, 8906], [8907, 8912], [8912, 8913], [8914, 8922], [8923, 8926], [8927, 8934], [8935, 8936], [8937, 8944], [8945, 8953], [8954, 8959], [8959, 8960], [8961, 8969], [8970, 8973], [8974, 8981], [8982, 8983], [8984, 8991], [8992, 9000], [9001, 9006], [9006, 9007], [9008, 9016], [9017, 9020], [9021, 9028], [9029, 9030], [9031, 9038], [9039, 9047], [9048, 9053], [9053, 9054], [9055, 9063], [9064, 9067], [9068, 9075], [9076, 9077], [9078, 9085], [9086, 9094], [9095, 9100], [9100, 9101], [9102, 9110], [9111, 9114], [9115, 9122], [9123, 9124], [9125, 9132], [9133, 9141], [9142, 9147], [9147, 9148], [9149, 9157], [9158, 9161], [9162, 9169], [9170, 9171], [9172, 9179], [9180, 9188], [9189, 9194], [9194, 9195], [9196, 9204], [9205, 9208], [9209, 9216], [9217, 9218], [9219, 9226], [9227, 9235], [9236, 9241], [9241, 9242], [9243, 9251], [9252, 9255], [9256, 9263], [9264, 9265], [9266, 9273], [9274, 9282], [9283, 9288], [9288, 9289], [9290, 9298], [9299, 9302], [9303, 9310], [9311, 9312], [9313, 9320], [9321, 9329], [9330, 9335], [9335, 9336], [9337, 9345], [9346, 9349], [9350, 9357], [9358, 9359], [9360, 9367], [9368, 9376], [9377, 9382], [9382, 9383], [9384, 9392], [9393, 9396], [9397, 9404], [9405, 9406], [9407, 9414], [9415, 9423], [9424, 9429], [9429, 9430], [9431, 9439], [9440, 9443], [9444, 9451], [9452, 9453], [9454, 9461], [9462, 9470], [9471, 9476], [9476, 9477], [9478, 9486], [9487, 9490], [9491, 9498], [9499, 9500], [9501, 9508], [9509, 9517], [9518, 9523], [9523, 9524], [9525, 9533], [9534, 9537], [9538, 9545], [9546, 9547], [9548, 9555], [9556, 9564], [9565, 9570], [9570, 9571], [9572, 9580], [9581, 9584], [9585, 9592], [9593, 9594], [9595, 9602], [9603, 9611], [9612, 9617], [9617, 9618], [9619, 9627], [9628, 9631], [9632, 9639], [9640, 9641], [9642, 9649], [9650, 9658], [9659, 9664], [9664, 9665], [9666, 9674], [9675, 9678], [9679, 9686], [9687, 9688], [9689, 9696], [9697, 9705], [9706, 9711], [9711, 9712], [9713, 9721], [9722, 9725], [9726, 9733], [9734, 9735], [9736, 9743], [9744, 9752], [9753, 9758], [9758, 9759], [9760, 9768], [9769, 9772], [9773, 9780], [9781, 9782], [9783, 9790], [9791, 9799], [9800, 9805], [9805, 9806], [9807, 9815], [9816, 9819], [9820, 9827], [9828, 9829], [9830, 9837], [9838, 9846], [9847, 9852], [9852, 9853], [9854, 9862], [9863, 9866], [9867, 9874], [9875, 9876], [9877, 9884], [9885, 9893], [9894, 9899], [9899, 9900], [9901, 9909], [9910, 9913], [9914, 9921], [9922, 9923], [9924, 9931], [9932, 9940], [9941, 9946], [9946, 9947], [9948, 9956], [9957, 9960], [9961, 9968], [9969, 9970], [9971, 9978], [9979, 9987], [9988, 9993], [9993, 9994], [9995, 10003], [10004, 10007], [10008, 10015], [10016, 10017], [10018, 10025], [10026, 10034], [10035, 10040], [10040, 10041], [10042, 10050], [10051, 10054], [10055, 10062], [10063, 10064], [10065, 10072], [10073, 10081], [10082, 10087], [10087, 10088], [10089, 10097], [10098, 10101], [10102, 10109], [10110, 10111], [10112, 10119], [10120, 10128], [10129, 10134], [10134, 10135], [10136, 10144], [10145, 10148], [10149, 10156], [10157, 10158], [10159, 10166], [10167, 10175], [10176, 10181], [10181, 10182], [10183, 10191], [10192, 10195], [10196, 10203], [10204, 10205], [10206, 10213], [10214, 10222], [10223, 10228], [10228, 10229]], "w_pos": [52, 53, 54, 55, 56, 58, 61, 62, 63, 64, 65, 66, 67, 69, 72, 73, 74, 75, 76, 77, 78, 80, 83, 84, 85, 86, 87, 88, 89, 91, 94, 95, 96, 97, 98, 99, 100, 102, 105, 106, 107, 108, 109, 110, 111, 113, 116, 117, 118, 119, 120, 121, 122, 124, 127, 128, 129, 130, 131, 132, 133, 135, 138, 139, 140, 141, 142, 143, 144, 146, 149, 150, 151, 152, 153, 154, 155, 157, 160, 161, 162, 163, 164, 165, 166, 168, 171, 172, 173, 174, 175, 176, 177, 179, 182, 183, 184, 185, 186, 187, 188, 190, 193, 194, 195, 196, 197, 198, 199, 201, 204, 205, 206, 207, 208, 209, 210, 212, 215, 216, 217, 218, 219, 220, 221, 223, 226, 227, 228, 229, 230, 231, 232, 234, 237, 238, 239, 240, 241, 242, 243, 245, 248, 249, 250, 251, 252, 253, 254, 256, 259, 260, 261, 262, 263, 264, 265, 267, 270, 271, 272, 273, 274, 275, 276, 278, 281, 282, 283, 284, 285, 286, 287, 289, 292, 293, 294, 295, 296, 297, 298, 300, 303, 304, 305, 306, 307, 308, 309, 311, 314, 315, 316, 317, 318, 319, 320, 322, 325, 326, 327, 328, 329, 330, 331, 333, 336, 337, 338, 339, 340, 341, 342, 344, 347, 348, 349, 350, 351, 352, 353, 355, 358, 359, 360, 361, 362, 363, 364, 366, 369, 370, 371, 372, 373, 374, 375, 377, 380, 381, 382, 383, 384, 385, 386, 388, 391, 392, 393, 394, 395, 396, 397, 399, 402, 403, 404, 405, 406, 407, 408, 410, 413, 414, 415, 416, 417, 418, 419, 421, 424, 425, 426, 427, 428, 429, 430, 432, 435, 436, 437, 438, 439, 440, 441, 443, 446, 447, 448, 449, 450, 451, 452, 454, 457, 458, 459, 460, 461, 462, 463, 465, 468, 469, 470, 471, 472, 473, 474, 476, 479, 480, 481, 482, 483, 484, 485, 487, 490, 491, 492, 493, 494, 495, 496, 498, 501, 502, 503, 504, 505, 506, 507, 509, 512, 513, 514, 515, 516, 517, 518, 520, 523, 524, 525, 526, 527, 528, 529, 531, 534, 535, 536, 537, 538, 539, 540, 542, 545, 546, 547, 548, 549, 550, 551, 553, 556, 557, 558, 559, 560, 561, 562, 564, 567, 568, 569, 570, 571, 572, 573, 575, 578, 579, 580, 581, 582, 583, 584, 586, 589, 590, 591, 592, 593, 594, 595, 597, 600, 601, 602, 603, 604, 605, 606, 608, 611, 612, 613, 614, 615, 616, 617, 619, 622, 623, 624, 625, 626, 627, 628, 630, 633, 634, 635, 636, 637, 638, 639, 641, 644, 645, 646, 647, 648, 649, 650, 652, 655, 656, 657, 658, 659, 660, 661, 663, 666, 667, 668, 669, 670, 671, 672, 674, 677, 678, 679, 680, 681, 682, 683, 685, 688, 689, 690, 691, 692, 693, 694, 696, 699, 700, 701, 702, 703, 704, 705, 707, 710, 711, 712, 713, 714, 715, 716, 718, 721, 722, 723, 724, 725, 726, 727, 729, 732, 733, 734, 735, 736, 737, 738, 740, 743, 744, 745, 746, 747, 748, 749, 751, 754, 755, 756, 757, 758, 759, 760, 762, 765, 766, 767, 768, 769, 770, 771, 773, 776, 777, 778, 779, 780, 781, 782, 784, 787, 788, 789, 790, 791, 792, 793, 795, 798, 799, 800, 801, 802, 803, 804, 806, 809, 810, 811, 812, 813, 814, 815, 817, 820, 821, 822, 823, 824, 825], "span_over": "user", "n_tokens": 1639, "head": "broad"}]} diff --git a/tests/vela/fixtures/oracle-hybrid-example-request-cjk.json b/tests/vela/fixtures/oracle-hybrid-example-request-cjk.json new file mode 100644 index 00000000..ddf6c71c --- /dev/null +++ b/tests/vela/fixtures/oracle-hybrid-example-request-cjk.json @@ -0,0 +1,155 @@ +{ + "case": "example-request-cjk", + "request": { + "model": "vllm-sr/Vela-2.0-0.8B", + "state": { + "request": "客户张伟（zhang.wei@example.cn）于2026年10月1日在上海市浦东新区世纪大道100号办理了业务，联系电话13800138000。", + "source": "客户的业务申请需要在三个工作日内审核完成。", + "answer": "客户的业务已经即时完成了。" + }, + "questions": { + "pii": { + "type": "span", + "instructions": "Which spans are personal information?", + "criteria": { + "PERSON": "a person's name", + "EMAIL_ADDRESS": "an e-mail address", + "PHONE_NUMBER": "a telephone number", + "STREET_ADDRESS": "a street address or postal address" + }, + "over": "request" + }, + "unsupported": { + "type": "span", + "instructions": "Which spans of the answer are not supported by the context?", + "criteria": { + "unsupported": "a claim not supported by the context" + } + }, + "billing": { + "type": "noul", + "instructions": "这个请求是否与账单有关？" + } + } + }, + "system_one": { + "model": "vllm-sr/Vela-2.0-0.8B", + "answers": { + "pii": { + "type": "noul", + "noul": 0.8401349388137276 + }, + "unsupported": { + "type": "noul", + "noul": 0.28838747148371713 + }, + "billing": { + "type": "noul", + "noul": 0.1674766566538726 + } + }, + "usage": { + "input_tokens": 597, + "output_tokens": 0 + }, + "spans": { + "pii": [ + { + "label": "PHONE_NUMBER", + "start": 32, + "end": 34, + "text": "10", + "probability": 0.4658691883087158 + }, + { + "label": "STREET_ADDRESS", + "start": 38, + "end": 52, + "text": "上海市浦东新区世纪大道100", + "probability": 0.4821831285953522 + }, + { + "label": "PHONE_NUMBER", + "start": 63, + "end": 74, + "text": "13800138000", + "probability": 0.4298103451728821 + } + ], + "unsupported": [] + }, + "span_heads": { + "pii": "router", + "unsupported": "router" + }, + "thresholds": { + "pii": 0.2914, + "unsupported": 0.5 + } + }, + "predict": { + "answers": [ + { + "id": "pii", + "type": "span", + "target_part": "user", + "spans": [ + { + "start": 32, + "end": 34, + "label": "PHONE_NUMBER", + "probability": 0.4658691883087158, + "text": "10" + }, + { + "start": 38, + "end": 52, + "label": "STREET_ADDRESS", + "probability": 0.4821831285953522, + "text": "上海市浦东新区世纪大道100" + }, + { + "start": 63, + "end": 74, + "label": "PHONE_NUMBER", + "probability": 0.4298103451728821, + "text": "13800138000" + } + ], + "threshold": 0.2914, + "threshold_rule": "pii:length", + "head": "router", + "head_reason": "id" + }, + { + "id": "unsupported", + "type": "span", + "target_part": "answer", + "spans": [], + "threshold": 0.5, + "threshold_rule": "span:*", + "head": "router", + "head_reason": "label_set" + }, + { + "id": "billing", + "type": "choice", + "answer": "no", + "probabilities": { + "no": 0.8325233433461274, + "yes": 0.1674766566538726 + }, + "abstain_probability": 0.0009211774164155532 + } + ], + "meta": { + "tokens": { + "user": 46, + "context": 10, + "answer": 6 + }, + "windows": 0, + "sequences": 2 + } + } +} diff --git a/tests/vela/fixtures/oracle-hybrid-example-request-score.json b/tests/vela/fixtures/oracle-hybrid-example-request-score.json new file mode 100644 index 00000000..9491e486 --- /dev/null +++ b/tests/vela/fixtures/oracle-hybrid-example-request-score.json @@ -0,0 +1,105 @@ +{ + "case": "example-request-score", + "request": { + "model": "vllm-sr/Vela-2.0-0.8B", + "state": "Hello, I would like to check the status of my order. It has been five days and I have heard nothing back.", + "questions": { + "tone": { + "type": "score", + "instructions": "How formal is the message?", + "criteria": [ + "casual", + "neutral", + "formal" + ] + }, + "urgency": { + "type": "score", + "instructions": "How urgently does the sender need a reply?", + "criteria": [ + "no rush", + "soon", + "today", + "immediately" + ] + } + } + }, + "system_one": { + "model": "vllm-sr/Vela-2.0-0.8B", + "answers": { + "tone": { + "type": "score", + "score": 1.1498890868381673, + "confidence": 0.3046350733874953, + "legend": { + "0": "casual", + "1": "neutral", + "2": "formal" + }, + "probabilities": { + "0": 0.16807713462834117, + "1": 0.5139566439051507, + "2": 0.3179662214665082 + } + }, + "urgency": { + "type": "score", + "score": 0.9787050638239541, + "confidence": 0.19233689920934172, + "legend": { + "0": "no rush", + "1": "soon", + "2": "today", + "3": "immediately" + }, + "probabilities": { + "0": 0.44128118285871365, + "1": 0.21311503083546, + "2": 0.2712213259289846, + "3": 0.07438246037684161 + } + } + }, + "usage": { + "input_tokens": 228, + "output_tokens": 0 + } + }, + "predict": { + "answers": [ + { + "id": "tone", + "type": "score", + "answer": "neutral", + "probabilities": { + "casual": 0.16807713462834117, + "neutral": 0.5139566439051507, + "formal": 0.3179662214665082 + }, + "expected_index": 1.1498890868381673, + "score": 0.5749445434190836 + }, + { + "id": "urgency", + "type": "score", + "answer": "no rush", + "probabilities": { + "no rush": 0.44128118285871365, + "soon": 0.21311503083546, + "today": 0.2712213259289846, + "immediately": 0.07438246037684161 + }, + "expected_index": 0.9787050638239541, + "score": 0.3262350212746513 + } + ], + "meta": { + "tokens": { + "user": 25 + }, + "windows": 0, + "sequences": 1 + } + } +} diff --git a/tests/vela/fixtures/oracle-hybrid-example-request-set-threshold.json b/tests/vela/fixtures/oracle-hybrid-example-request-set-threshold.json new file mode 100644 index 00000000..bab3d048 --- /dev/null +++ b/tests/vela/fixtures/oracle-hybrid-example-request-set-threshold.json @@ -0,0 +1,81 @@ +{ + "case": "example-request-set-threshold", + "request": { + "model": "vllm-sr/Vela-2.0-0.8B", + "state": "My card was charged twice and the parcel never arrived.", + "questions": { + "issues": { + "type": "set", + "instructions": "Which issues does the customer report?", + "criteria": { + "billing": "payments, charges, refunds or invoices", + "shipping": "delivery of an order or a parcel", + "login": "signing in, passwords or account access" + }, + "threshold": 0.15 + } + } + }, + "system_one": { + "model": "vllm-sr/Vela-2.0-0.8B", + "answers": { + "issues.billing": { + "type": "noul", + "noul": 0.6211446881608428 + }, + "issues.shipping": { + "type": "noul", + "noul": 0.8205402483597802 + }, + "issues.login": { + "type": "noul", + "noul": 0.00012988178108716601 + } + }, + "usage": { + "input_tokens": 133, + "output_tokens": 0 + }, + "sets": { + "issues": { + "selected": [ + "billing", + "shipping" + ], + "probabilities": { + "billing": 0.6211446881608428, + "shipping": 0.8205402483597802, + "login": 0.00012988178108716601 + } + } + }, + "thresholds": { + "issues": 0.15 + } + }, + "predict": { + "answers": [ + { + "id": "issues", + "type": "set", + "answer": [ + "billing", + "shipping" + ], + "probabilities": { + "billing": 0.6211446881608428, + "shipping": 0.8205402483597802, + "login": 0.00012988178108716601 + }, + "threshold": 0.15 + } + ], + "meta": { + "tokens": { + "user": 11 + }, + "windows": 0, + "sequences": 1 + } + } +} diff --git a/tests/vela/fixtures/oracle-hybrid-example-request.json b/tests/vela/fixtures/oracle-hybrid-example-request.json new file mode 100644 index 00000000..801ef7aa --- /dev/null +++ b/tests/vela/fixtures/oracle-hybrid-example-request.json @@ -0,0 +1,183 @@ +{ + "case": "example-request", + "request": { + "model": "vllm-sr/Vela-2.0-0.8B", + "state": { + "request": "Hi, I'm Tom Baker (tom.baker@example.com). What is the maximum daily dose of paracetamol for an adult?", + "source": "For adults, the maximum dose of paracetamol is 4 grams in 24 hours, taken as 500 mg to 1 g every 4 to 6 hours.", + "answer": "Adults can take up to 6 grams of paracetamol in 24 hours, in doses of 500 mg to 1 g every 4 to 6 hours." + }, + "questions": { + "pii": { + "type": "span", + "instructions": "Which spans are personal information?", + "criteria": { + "AGE": "a person's age", + "CREDIT_CARD": "a payment card number", + "DATE_TIME": "a date, time or date of birth", + "DOMAIN_NAME": "an internet domain name or website address", + "EMAIL_ADDRESS": "an e-mail address", + "GPE": "a country, city, state or other geopolitical place", + "IBAN_CODE": "an international bank account number (IBAN)", + "IP_ADDRESS": "an IPv4 or IPv6 address", + "NRP": "a nationality, religious or political group", + "ORGANIZATION": "the name of a company, institution or other organisation", + "PERSON": "a person's name", + "PHONE_NUMBER": "a telephone number", + "STREET_ADDRESS": "a street address or postal address", + "TITLE": "a personal or professional title such as Dr. or Mrs.", + "US_DRIVER_LICENSE": "a US driver's licence number", + "US_SSN": "a US social security number", + "ZIP_CODE": "a postal or ZIP code" + }, + "over": "request" + }, + "halu": { + "type": "span", + "instructions": "Which spans of the answer are not supported by the context?", + "criteria": { + "unsupported": "a claim not supported by the context" + } + }, + "domain": { + "type": "choice", + "instructions": "Which subject area is this request about?", + "over": "request", + "criteria": { + "health": "medicine, clinical practice, nutrition, ageing or sexual health", + "math": "arithmetic, algebra, geometry, statistics or other mathematics", + "other": "a subject that fits none of the listed areas" + } + } + } + }, + "system_one": { + "model": "vllm-sr/Vela-2.0-0.8B", + "answers": { + "pii": { + "type": "noul", + "noul": 0.9965421673633966 + }, + "halu": { + "type": "noul", + "noul": 0.9972382713052719 + }, + "domain": { + "type": "choice", + "choice": "health", + "confidence": 0.9288764270674694, + "probabilities": { + "health": 0.9621185645193945, + "math": 0.004639298028680393, + "other": 0.03324213745192508 + } + } + }, + "usage": { + "input_tokens": 1041, + "output_tokens": 0 + }, + "spans": { + "pii": [ + { + "label": "PERSON", + "start": 8, + "end": 17, + "text": "Tom Baker", + "probability": 0.983367919921875 + }, + { + "label": "EMAIL_ADDRESS", + "start": 19, + "end": 40, + "text": "tom.baker@example.com", + "probability": 0.9851056337356567 + } + ], + "halu": [ + { + "label": "unsupported", + "start": 19, + "end": 29, + "text": "to 6 grams", + "probability": 0.7782317996025085 + } + ] + }, + "span_heads": { + "pii": "router", + "halu": "router" + }, + "thresholds": { + "pii": 0.2914, + "halu": 0.35 + } + }, + "predict": { + "answers": [ + { + "id": "pii", + "type": "span", + "target_part": "user", + "spans": [ + { + "start": 8, + "end": 17, + "label": "PERSON", + "probability": 0.983367919921875, + "text": "Tom Baker" + }, + { + "start": 19, + "end": 40, + "label": "EMAIL_ADDRESS", + "probability": 0.9851056337356567, + "text": "tom.baker@example.com" + } + ], + "threshold": 0.2914, + "threshold_rule": "pii:length", + "head": "router", + "head_reason": "id" + }, + { + "id": "halu", + "type": "span", + "target_part": "answer", + "spans": [ + { + "start": 19, + "end": 29, + "label": "unsupported", + "probability": 0.7782317996025085, + "text": "to 6 grams" + } + ], + "threshold": 0.35, + "threshold_rule": "halu", + "head": "router", + "head_reason": "id" + }, + { + "id": "domain", + "type": "choice", + "answer": "health", + "probabilities": { + "health": 0.9621185645193945, + "math": 0.004639298028680393, + "other": 0.03324213745192508 + }, + "abstain_probability": 0.04287965117790319 + } + ], + "meta": { + "tokens": { + "user": 27, + "context": 39, + "answer": 39 + }, + "windows": 0, + "sequences": 2 + } + } +} diff --git a/tests/vela/native_bundle.rs b/tests/vela/native_bundle.rs new file mode 100644 index 00000000..9b0e9926 --- /dev/null +++ b/tests/vela/native_bundle.rs @@ -0,0 +1,152 @@ +//! Bundle-gate tests: a synthetic bundle built in a unique temp +//! directory (no weights, no network) drives the manifest gate — pins, +//! inventory counts and per-file hashes. + +use omni_vela_native::bundle::{self, Bundle}; +use serde_json::json; + +fn temp_dir(name: &str) -> std::path::PathBuf { + let dir = + std::env::temp_dir().join(format!("vela-native-bundle-{name}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).expect("tempdir"); + dir +} + +fn sha256(bytes: &[u8]) -> String { + use sha2::{Digest, Sha256}; + let digest = Sha256::new_with_prefix(bytes).finalize(); + digest.iter().map(|byte| format!("{byte:02x}")).collect() +} + +/// The minimal manifest of a correctly exported member; `files` are the +/// bundle files the gate hashes. +fn write_bundle(dir: &std::path::Path) { + let files: [(&str, &[u8]); 6] = [ + ("backbone.safetensors", b"backbone tensors"), + ("heads.safetensors", b"head tensors"), + ("config.json", b"{}"), + ("calibration.json", b"{}"), + ("tokenizer.json", b"{}"), + ("tokenizer_config.json", b"{}"), + ]; + let mut hashes = serde_json::Map::new(); + for (name, content) in files { + std::fs::write(dir.join(name), content).expect("write"); + hashes.insert(name.to_string(), json!(sha256(content))); + } + let manifest = json!({ + "format": bundle::FORMAT, + "model_id": bundle::MODEL_ID, + "model_name": "vllm-sr/Vela-2.0-0.8B", + "max_length": 16384, + "source": {"repo": "vllm-sr/Vela-2.0-0.8B", "revision": bundle::SOURCE_REVISION}, + "backbone": {"tensors": 320, "bf16": 204, "fp32": 116}, + "heads": {"tensors": 37}, + "files_sha256": hashes, + }); + std::fs::write( + dir.join("vela_native_manifest.json"), + serde_json::to_vec(&manifest).expect("manifest"), + ) + .expect("write"); +} + +#[test] +fn a_matching_bundle_loads_and_exposes_the_member() { + let dir = temp_dir("ok"); + write_bundle(&dir); + let loaded = Bundle::load(&dir).expect("loads"); + assert_eq!(loaded.model_name(), "vllm-sr/Vela-2.0-0.8B"); + assert_eq!(loaded.max_length(), 16384); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn a_tampered_file_fails_its_hash() { + let dir = temp_dir("tampered"); + write_bundle(&dir); + std::fs::write(dir.join("calibration.json"), b"{\"tampered\": true}").expect("write"); + let error = Bundle::load(&dir).unwrap_err(); + assert!(error.to_string().contains("calibration.json"), "{error}"); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn a_foreign_revision_is_refused() { + let dir = temp_dir("foreign"); + write_bundle(&dir); + let manifest_path = dir.join("vela_native_manifest.json"); + let mut manifest: serde_json::Value = + serde_json::from_slice(&std::fs::read(&manifest_path).expect("read")).expect("json"); + manifest["source"]["revision"] = json!("d692394127f6f87fc581d8634c9558fe4776efce"); + std::fs::write(&manifest_path, serde_json::to_vec(&manifest).expect("json")).expect("write"); + let error = Bundle::load(&dir).unwrap_err(); + assert!(error.to_string().contains("pinned"), "{error}"); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn an_unexpected_inventory_is_refused() { + let dir = temp_dir("inventory"); + write_bundle(&dir); + let manifest_path = dir.join("vela_native_manifest.json"); + let mut manifest: serde_json::Value = + serde_json::from_slice(&std::fs::read(&manifest_path).expect("read")).expect("json"); + manifest["backbone"]["bf16"] = json!(320); + manifest["backbone"]["fp32"] = json!(0); + std::fs::write(&manifest_path, serde_json::to_vec(&manifest).expect("json")).expect("write"); + let error = Bundle::load(&dir).unwrap_err(); + assert!(error.to_string().contains("inventory"), "{error}"); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn an_extra_unlisted_file_fails_the_gate() { + let dir = temp_dir("extra"); + write_bundle(&dir); + std::fs::write(dir.join("extra.bin"), b"not in the manifest").expect("write"); + let error = Bundle::load(&dir).unwrap_err(); + assert!(error.to_string().contains("extra.bin"), "{error}"); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn an_unlisted_file_inside_an_admitted_directory_fails_the_gate() { + let dir = temp_dir("bf16-extra"); + write_bundle(&dir); + let manifest_path = dir.join("vela_native_manifest.json"); + let mut manifest: serde_json::Value = + serde_json::from_slice(&std::fs::read(&manifest_path).expect("read")).expect("json"); + std::fs::create_dir(dir.join("bf16")).expect("mkdir"); + std::fs::write(dir.join("bf16/model.safetensors"), b"bf16 tensors").expect("write"); + std::fs::write(dir.join("bf16/extra.bin"), b"unlisted").expect("write"); + manifest["files_sha256"]["bf16/model.safetensors"] = json!(sha256(b"bf16 tensors")); + std::fs::write(&manifest_path, serde_json::to_vec(&manifest).expect("json")).expect("write"); + let error = Bundle::load(&dir).unwrap_err(); + assert!(error.to_string().contains("bf16/extra.bin"), "{error}"); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn an_empty_files_map_fails_the_gate() { + let dir = temp_dir("empty-files"); + write_bundle(&dir); + let manifest_path = dir.join("vela_native_manifest.json"); + let mut manifest: serde_json::Value = + serde_json::from_slice(&std::fs::read(&manifest_path).expect("read")).expect("json"); + manifest["files_sha256"] = json!({}); + std::fs::write(&manifest_path, serde_json::to_vec(&manifest).expect("json")).expect("write"); + let error = Bundle::load(&dir).unwrap_err(); + assert!(error.to_string().contains("files_sha256"), "{error}"); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn a_missing_manifest_names_the_export_step() { + let dir = temp_dir("empty"); + std::fs::create_dir_all(&dir).expect("tempdir"); + let error = Bundle::load(&dir).unwrap_err(); + assert!(error.to_string().contains("export_native.py"), "{error}"); + let _ = std::fs::remove_dir_all(&dir); +} diff --git a/tests/vela/native_contract.rs b/tests/vela/native_contract.rs new file mode 100644 index 00000000..ebe73454 --- /dev/null +++ b/tests/vela/native_contract.rs @@ -0,0 +1,126 @@ +//! Wire-contract tests for the native Vela 2.0 worker, mirroring the +//! reference worker's protocol tests: the envelope triples must be +//! byte-equal to the pinned runtime's rejections. + +use omni_vela_native::contract::{self, RequestError}; +use serde_json::json; + +fn body(raw: &str) -> serde_json::Map { + contract::parse_body(raw.as_bytes()).expect("valid body") +} + +fn detail(loc: &[&str], msg: &str, type_: &str) -> serde_json::Value { + let mut path = vec![json!("body")]; + path.extend(loc.iter().map(|part| json!(part))); + json!({"detail": [{"loc": path, "msg": msg, "type": type_}]}) +} + +const SERVED: &str = "vllm-sr/Vela-2.0-0.8B"; + +fn raw_body() -> String { + format!( + r#"{{"model": "{SERVED}", "state": "My card was charged twice.", "questions": {{"billing": {{"type": "noul", "instructions": "any?"}}}}}}"# + ) +} + +#[test] +fn the_envelope_passes_through_unchanged() { + let parsed = body(&raw_body()); + contract::admit_request(&parsed, SERVED).expect("admitted"); + assert_eq!(parsed.len(), 3); +} + +#[test] +fn a_missing_field_reports_upstreams_missing_triple() { + for drop in ["state", "model", "questions"] { + let mut parsed = body(&raw_body()); + parsed.shift_remove(drop); + let error = contract::admit_request(&parsed, SERVED).unwrap_err(); + assert_eq!( + error, + RequestError::envelope(&[drop], "Field required", "missing") + ); + assert_eq!(error.body, detail(&[drop], "Field required", "missing")); + } +} + +#[test] +fn an_extra_field_reports_upstreams_extra_forbidden() { + let mut parsed = body(&raw_body()); + parsed.insert("images".into(), json!([])); + let error = contract::admit_request(&parsed, SERVED).unwrap_err(); + assert_eq!(error.status, 422); + assert_eq!( + error.body, + detail( + &["images"], + "Extra inputs are not permitted", + "extra_forbidden" + ) + ); +} + +#[test] +fn a_blank_model_is_rejected_before_the_name_check() { + let mut parsed = body(&raw_body()); + parsed.insert("model".into(), json!(" ")); + let error = contract::admit_request(&parsed, SERVED).unwrap_err(); + assert_eq!( + error.body, + detail( + &["model"], + "model must not be empty or whitespace", + "value_error" + ) + ); +} + +#[test] +fn another_members_name_is_rejected_with_the_model_triple() { + let mut parsed = body(&raw_body()); + parsed.insert("model".into(), json!("vllm-sr/Vela-2.0-0.3B")); + let error = contract::admit_request(&parsed, SERVED).unwrap_err(); + assert_eq!( + error.body, + detail( + &["model"], + "Model is not served by this runtime", + "value_error.model" + ) + ); +} + +#[test] +fn duplicate_keys_are_rejected_with_the_reference_message() { + let error = + contract::parse_body(br#"{"model": "x", "model": "y", "state": "s", "questions": {}}"#) + .unwrap_err(); + assert_eq!(error.status, 400); + assert_eq!(error.body, json!({"detail": "duplicate JSON key 'model'"})); +} + +#[test] +fn invalid_json_is_a_generic_400() { + let error = contract::parse_body(b"{not json").unwrap_err(); + assert_eq!(error.status, 400); + assert_eq!( + error.body, + json!({"detail": "request body must contain valid JSON and UTF-8 text"}) + ); +} + +#[test] +fn a_non_object_body_is_rejected() { + let error = contract::parse_body(b"[1, 2]").unwrap_err(); + assert_eq!(error.status, 400); + assert_eq!( + error.body, + json!({"detail": "request must be a JSON object"}) + ); +} + +#[test] +fn compile_admits_and_returns_the_body() { + let parsed = contract::compile(raw_body().as_bytes(), SERVED).expect("admitted"); + assert!(parsed.contains_key("questions")); +} diff --git a/tests/vela/native_contract_unit.rs b/tests/vela/native_contract_unit.rs new file mode 100644 index 00000000..e2a60834 --- /dev/null +++ b/tests/vela/native_contract_unit.rs @@ -0,0 +1,14 @@ +//! Private-item tests for the native crate's wire contract, loaded by +//! the `#[path]` wiring in `src/models/vela/native/src/contract.rs`. + +use super::*; + +#[test] +fn duplicate_keys_mirror_the_reference_message() { + let error = normalize_parse_error("request body is not valid JSON: duplicate key \"state\""); + assert_eq!(error.status, 400); + assert_eq!( + error.body, + serde_json::json!({"detail": "duplicate JSON key 'state'"}) + ); +} diff --git a/tests/vela/native_executor.rs b/tests/vela/native_executor.rs new file mode 100644 index 00000000..c5eaae3e --- /dev/null +++ b/tests/vela/native_executor.rs @@ -0,0 +1,166 @@ +//! GPU executor check: the backbone through the shared qwen3_5 crate, +//! full-sequence per readout position, replayed into the Rust heads +//! and compared with the recorded reference logits. Opt-in: +//! +//! VELA_NATIVE_BUNDLE=.../vela-2.0-0.8b-native \ +//! VELA_CUDA_LIB=.../libqwen3_5_cuda.so \ +//! cargo test -p omni-vela-native --test vela_native_executor -- --ignored --nocapture +//! +//! The recorded reference ran CPU FP32; this path runs the uniform-BF16 +//! backbone, so the gates are backbone-level numerical tolerances, not +//! the reference worker's exact-match bar: logit drift is measured and +//! asserted against the gate. + +use std::path::PathBuf; + +use omni_vela_native::executor::Executor; +use serde_json::Value; + +const CASES: [&str; 4] = [ + "example-request", + "example-request-set-threshold", + "example-request-cjk", + "example-request-score", +]; + +fn fixtures_dir() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../../../tests/vela/fixtures") +} + +fn env_dir(name: &str) -> PathBuf { + let path = PathBuf::from(std::env::var(name).unwrap_or_else(|_| panic!("set {name}"))); + assert!(path.is_dir(), "{} is not a directory", path.display()); + path +} + +#[test] +#[ignore = "requires VELA_NATIVE_BUNDLE at an exported bundle and VELA_CUDA_LIB at the built CUDA library; GPU"] +fn the_gpu_backbone_replays_the_reference_readouts() { + let bundle = env_dir("VELA_NATIVE_BUNDLE"); + let library = PathBuf::from( + std::env::var("VELA_CUDA_LIB").expect("set VELA_CUDA_LIB to the built CUDA library"), + ); + assert!(library.exists(), "{} does not exist", library.display()); + let mut executor = Executor::load(&bundle, &library).expect("executor loads"); + // The env gate may only tighten the documented ceiling: an + // unparseable value is an error, and a wider one clamps back down. + let logit_gate = std::env::var("VELA_GPU_LOGIT_GATE") + .map(|v| { + v.parse::() + .expect("VELA_GPU_LOGIT_GATE must be a number") + }) + .unwrap_or(0.35) + .min(0.35); + + for case in CASES { + let head_fixture: Value = serde_json::from_str( + &std::fs::read_to_string(fixtures_dir().join(format!("head-input-{case}.json"))) + .expect("fixture"), + ) + .expect("json"); + let plan_fixture: Value = serde_json::from_str( + &std::fs::read_to_string(fixtures_dir().join(format!("native-plan-{case}.json"))) + .expect("fixture"), + ) + .expect("json"); + let request = plan_fixture["request"] + .as_object() + .expect("request") + .clone(); + let processor = + omni_vela_native::processing::Processor::load(&bundle).expect("processor loads"); + let planned = processor.plan(&request).expect("plan"); + let raw = std::fs::read(fixtures_dir().join(format!("head-input-{case}.safetensors"))) + .expect("safetensors fixture"); + let _file = safetensors::SafeTensors::deserialize(&raw).expect("safetensors"); + + // The recorded reference logits per call, in call order. + let calls: Vec<&Value> = head_fixture["head_calls"] + .as_array() + .unwrap() + .iter() + .collect(); + let mut q_call_index = 0usize; + let mut span_call_index = 0usize; + + let mut worst_logit = 0.0f32; + for row in &planned.rows { + let readouts = executor.row_readouts(row).expect("readouts"); + // One recorded q call carries the row's question blocks + // stacked; compare per block, matched by option count. + if !readouts.questions.is_empty() { + let call = next_call(&calls, &mut q_call_index, "q"); + let want: Vec> = call["logits"] + .as_array() + .unwrap() + .iter() + .map(|r| { + r.as_array() + .unwrap() + .iter() + .map(|v| v.as_f64().unwrap()) + .collect() + }) + .collect(); + // The stacked call's rows are in block order, padded to + // the widest block; compare each block's prefix. + for (question, ours) in readouts.questions.iter().zip(&want) { + let Some(query) = &question.query else { + continue; + }; + let got = executor + .heads + .forward_candidate(&question.candidates, query); + for (a, b) in got.iter().zip(ours.iter()) { + worst_logit = worst_logit.max((a - *b as f32).abs()); + } + } + } + if let Some(span) = &readouts.span { + let which = row.head.clone(); + let head = if which == "broad" { + &executor.heads.broad + } else { + &executor.heads.router + }; + let got = head.forward(&span.words, &span.labels); + let call = next_call(&calls, &mut span_call_index, "span"); + let want: Vec> = call["logits"] + .as_array() + .unwrap() + .iter() + .map(|row| { + row.as_array() + .unwrap() + .iter() + .map(|v| v.as_f64().unwrap()) + .collect() + }) + .collect(); + let mut worst_row = 0.0f32; + for (g, w) in got.iter().zip(&want) { + for (a, b) in g.iter().zip(w) { + worst_row = worst_row.max((a - *b as f32).abs()); + } + } + println!("{case}: span logits drift {worst_row:.4}"); + worst_logit = worst_logit.max(worst_row); + } + } + println!("{case}: worst logit drift {worst_logit:.4}"); + assert!( + worst_logit <= logit_gate, + "{case}: drift {worst_logit} > {logit_gate}" + ); + } +} + +fn next_call<'a>(calls: &[&'a Value], index: &mut usize, kind: &str) -> &'a Value { + let call = calls + .iter() + .filter(|c| c["kind"] == kind) + .nth(*index) + .unwrap_or_else(|| panic!("recorded {kind} call")); + *index += 1; + call +} diff --git a/tests/vela/native_heads.rs b/tests/vela/native_heads.rs new file mode 100644 index 00000000..4d293e7a --- /dev/null +++ b/tests/vela/native_heads.rs @@ -0,0 +1,412 @@ +//! Head and scoring parity against the recorded head-input +//! fixtures: the Rust heads must reproduce the reference's logits from +//! the same hidden states, and the scoring mirror must turn those +//! logits into the same predict-level answers. Needs the exported +//! bundle (heads + calibration) via `VELA_NATIVE_BUNDLE`; hidden states +//! load from the per-case safetensors fixtures. + +use std::collections::HashMap; +use std::path::PathBuf; + +use omni_vela_native::heads::Heads; +use omni_vela_native::scoring::{Calibration, SpanOut, aggregate_windows, row_answers}; +use serde_json::Value; + +const CASES: [&str; 5] = [ + "example-request", + "example-request-set-threshold", + "example-request-cjk", + "example-request-score", + "long-span-windowed", +]; + +/// One row's collected span calls: (head name, logits). +type RowSpanCalls = Vec<(String, Vec>)>; + +fn fixtures_dir() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../../../tests/vela/fixtures") +} + +fn load_json(name: &str) -> Value { + serde_json::from_str(&std::fs::read_to_string(fixtures_dir().join(name)).expect("fixture")) + .expect("fixture json") +} + +fn f32_matrix(raw: &[u8], name: &str) -> Vec> { + let file = safetensors::SafeTensors::deserialize(raw).expect("safetensors"); + let view = file.tensor(name).expect("tensor"); + assert_eq!(view.dtype(), safetensors::Dtype::F32); + let shape = view.shape(); + let data: Vec = view + .data() + .as_chunks::<4>() + .0 + .iter() + .map(|c| f32::from_le_bytes(*c)) + .collect(); + match shape.len() { + 1 => vec![data], + 2 => data.chunks(shape[1]).map(<[f32]>::to_vec).collect(), + 3 => data + .chunks(shape[1] * shape[2]) + .map(|block| block.chunks(shape[2]).map(<[f32]>::to_vec).collect()) + .next() + .unwrap_or_default(), + _ => panic!("unexpected shape {shape:?} for {name}"), + } +} + +fn f32_stack(raw: &[u8], name: &str) -> Vec>> { + let file = safetensors::SafeTensors::deserialize(raw).expect("safetensors"); + let view = file.tensor(name).expect("tensor"); + let shape = view.shape(); + assert_eq!(shape.len(), 3, "{name} must be 3D"); + let data: Vec = view + .data() + .as_chunks::<4>() + .0 + .iter() + .map(|c| f32::from_le_bytes(*c)) + .collect(); + data.chunks(shape[1] * shape[2]) + .map(|block| block.chunks(shape[2]).map(<[f32]>::to_vec).collect()) + .collect() +} + +fn bundle_dir() -> PathBuf { + PathBuf::from( + std::env::var("VELA_NATIVE_BUNDLE").expect("set VELA_NATIVE_BUNDLE to an exported bundle"), + ) +} + +/// Largest absolute difference between two nested JSON numbers; a +/// structural mismatch — an object key missing on either side, arrays +/// of unequal length, or a differing non-numeric value — scores the +/// full 1.0. +fn max_drift(a: &Value, b: &Value) -> f64 { + match (a, b) { + (Value::Number(x), Value::Number(y)) => { + (x.as_f64().unwrap_or(0.0) - y.as_f64().unwrap_or(0.0)).abs() + } + (Value::Array(x), Value::Array(y)) if x.len() == y.len() => x + .iter() + .zip(y) + .map(|(x, y)| max_drift(x, y)) + .fold(0.0, f64::max), + (Value::Object(x), Value::Object(y)) => x + .iter() + .map(|(k, v)| y.get(k).map(|w| max_drift(v, w)).unwrap_or(1.0)) + .chain(y.keys().filter(|k| !x.contains_key(*k)).map(|_| 1.0)) + .fold(0.0, f64::max), + (_, _) if a == b => 0.0, + _ => 1.0, + } +} + +#[test] +#[ignore = "requires VELA_NATIVE_BUNDLE at an exported bundle; CPU only"] +fn the_heads_reproduce_the_reference_logits_and_the_scoring_the_answers() { + let bundle = bundle_dir(); + let heads = Heads::load(&bundle).expect("heads load"); + let cal = Calibration::load(&bundle).expect("calibration loads"); + // The env gates may only tighten the documented ceilings: an + // unparseable value is an error, and a wider one clamps back down. + let logits_gate = std::env::var("VELA_HEADS_LOGIT_GATE") + .map(|v| { + v.parse::() + .expect("VELA_HEADS_LOGIT_GATE must be a number") + }) + .unwrap_or(4e-5) + .min(4e-5); + let answers_gate = std::env::var("VELA_HEADS_ANSWER_GATE") + .map(|v| { + v.parse::() + .expect("VELA_HEADS_ANSWER_GATE must be a number") + }) + .unwrap_or(1e-6) + .min(1e-6); + println!("gates: logit {logits_gate:.3e} answer {answers_gate:.3e}"); + + for case in CASES { + let fixture = load_json(&format!("head-input-{case}.json")); + let plan = load_json(&format!("native-plan-{case}.json")); + let raw = std::fs::read(fixtures_dir().join(format!("head-input-{case}.safetensors"))) + .expect("safetensors fixture"); + let request = fixture["request"].as_object().expect("request"); + + // Re-derive the per-row thresholds (the request's own overrides). + let mut thresholds: HashMap = HashMap::new(); + for (qid, question) in request["questions"].as_object().expect("questions") { + if let Some(t) = question.get("threshold").and_then(Value::as_f64) { + thresholds.insert(qid.clone(), t); + } + } + + // Drive every recorded head call through the Rust heads. + let mut q_calls_by_row: HashMap> = HashMap::new(); + let mut span_calls_by_row: HashMap = HashMap::new(); + let mut worst_logit = 0.0f64; + for call in fixture["head_calls"].as_array().expect("head calls") { + let row = call["row"].as_u64().expect("row") as usize; + match call["kind"].as_str().expect("kind") { + "q" => { + let cands = f32_stack(&raw, call["cand"].as_str().expect("cand")); + let queries = f32_matrix(&raw, call["query"].as_str().expect("query")); + let want = call["logits"].as_array().expect("logits"); + assert_eq!(cands.len(), want.len(), "{case}: block count"); + let mut got = Vec::new(); + for (cand, query) in cands.iter().zip(&queries) { + got.push(heads.forward_candidate(cand, query)); + } + for (g, w) in got.iter().zip(want) { + for (a, b) in g.iter().zip(w.as_array().unwrap()) { + worst_logit = worst_logit + .max((a - b.as_f64().unwrap_or(0.0) as f32).abs() as f64); + } + } + q_calls_by_row.entry(row).or_default().push(call.clone()); + } + "span" => { + let which = call["head"].as_str().expect("head"); + let words = f32_matrix(&raw, call["words"].as_str().expect("words")); + let labels = f32_matrix(&raw, call["labels"].as_str().expect("labels")); + let head = if which == "broad" { + &heads.broad + } else { + &heads.router + }; + let got = head.forward(&words, &labels); + let want = call["logits"].as_array().expect("logits"); + assert_eq!(got.len(), want.len(), "{case}: span words"); + for (g, w) in got.iter().zip(want) { + let w: Vec = w + .as_array() + .unwrap() + .iter() + .map(|v| v.as_f64().unwrap()) + .collect(); + for (a, b) in g.iter().zip(&w) { + worst_logit = worst_logit.max((a - *b as f32).abs() as f64); + } + } + span_calls_by_row + .entry(row) + .or_default() + .push((which.to_string(), got)); + } + _ => panic!("unknown head call kind"), + } + } + assert!( + worst_logit <= logits_gate, + "{case}: head logits drift {worst_logit} > {logits_gate}" + ); + + // Assemble predict answers row by row and compare. + let mut answers: Vec = Vec::new(); + for (row_index, row) in plan["rows"].as_array().expect("rows").iter().enumerate() { + let blocks: Vec = row["blocks"] + .as_array() + .expect("blocks") + .iter() + .filter(|b| b["kind"] == "q") + .cloned() + .collect(); + let mut qblocks = Vec::new(); + if let Some(calls) = q_calls_by_row.get(&row_index) { + // One stacked call carries all of the row's question + // blocks, in block order. + let call = calls.first().expect("q call"); + let cands = f32_stack(&raw, call["cand"].as_str().expect("cand")); + let queries = f32_matrix(&raw, call["query"].as_str().expect("query")); + for (block, (cand, query)) in blocks.iter().zip(cands.iter().zip(&queries)) { + let names: Vec = block["names"] + .as_array() + .unwrap() + .iter() + .map(|n| n.as_str().unwrap().to_string()) + .collect(); + let has_abs = block["has_abs"].as_bool().unwrap_or(false); + let mut logits = heads.forward_candidate(cand, query); + if block["qtype"] == "set" { + for value in logits.iter_mut() { + *value += heads.set_bias; + } + } + logits.truncate(names.len() + usize::from(has_abs)); + qblocks.push(omni_vela_native::scoring::QBlock { + qid: block["qid"].as_str().unwrap().to_string(), + qtype: block["qtype"].as_str().unwrap().to_string(), + names, + has_abs, + logits, + }); + } + } + // The span part of this row: original word offsets, the span + // block's names, and the part's kept token count. + let mut span_out = None; + let span_blocks: Vec<&Value> = row["blocks"] + .as_array() + .unwrap() + .iter() + .filter(|b| b["kind"] == "span") + .collect(); + let window_rows = plan["window_rows"].as_array().cloned().unwrap_or_default(); + let row_span_calls = span_calls_by_row + .get(&row_index) + .cloned() + .unwrap_or_default(); + let has_span = !span_blocks.is_empty() && !row_span_calls.is_empty(); + if has_span { + let names: Vec = span_block_names(row); + let w_offs: Vec<(usize, usize)> = row["w_offs"] + .as_array() + .map(|a| { + a.iter() + .map(|o| { + ( + o[0].as_u64().unwrap_or(0) as usize, + o[1].as_u64().unwrap_or(0) as usize, + ) + }) + .collect() + }) + .unwrap_or_default(); + let over = row["span_over"].as_str().unwrap_or("user").to_string(); + let n_tokens = row["seg_lens"][&over].as_u64().unwrap_or(0) as usize; + // Windowed rows: the full render is skipped (the first + // call) and the windows aggregate in order; else the + // single call is the answer's logits. + let (which, logits) = if window_rows.is_empty() { + row_span_calls.last().expect("span call").clone() + } else { + let which = row_span_calls[0].0.clone(); + let window_logits: Vec>> = + row_span_calls[1..].iter().map(|(_, l)| l.clone()).collect(); + let window_offs: Vec> = window_rows + .iter() + .map(|w| { + w["w_offs"] + .as_array() + .map(|a| { + a.iter() + .map(|o| { + ( + o[0].as_u64().unwrap_or(0) as usize, + o[1].as_u64().unwrap_or(0) as usize, + ) + }) + .collect() + }) + .unwrap_or_default() + }) + .collect(); + ( + which, + aggregate_windows(&names, &w_offs, &window_logits, &window_offs), + ) + }; + + span_out = Some(SpanOut { + qid: span_qid(&plan, row_index), // row 0 -> spans[0], row i -> spans[i] + over, + names, + offs: w_offs, + logits, + head: which.clone(), + head_reason: row["head_reason"].as_str().map(str::to_string), + label_alias: row.get("label_alias").filter(|v| !v.is_null()).cloned(), + n_tokens, + }); + } + // The span text: the targeted part's rendered text. + let span_text = row["span_over"].as_str().and_then(|role| { + plan["predict_request"]["parts"] + .as_array() + .unwrap() + .iter() + .find(|p| p["type"] == *role) + .map(|p| p["text"].as_str().unwrap().to_string()) + }); + let mut row_answers = row_answers( + &cal, + &qblocks, + span_out.as_ref(), + &thresholds, + span_text.as_deref(), + ); + // The span answer's id is filled by the caller above. + answers.append(&mut row_answers); + } + + // The reference orders answers by the predict request's + // question order, not by the row walk. + let order: Vec = plan["plan"] + .as_array() + .unwrap() + .iter() + .map(|p| p["qid"].as_str().unwrap().to_string()) + .collect(); + let mut answers = answers; + answers.sort_by_key(|a| { + order + .iter() + .position(|qid| qid == a["id"].as_str().unwrap_or("")) + .unwrap_or(usize::MAX) + }); + + let want = fixture["predict"]["answers"].as_array().expect("answers"); + let drift = max_drift(&Value::Array(answers.clone()), &Value::Array(want.clone())); + assert!( + drift <= answers_gate, + "{case}: answer drift {drift} > {answers_gate}\ngot: {}\nwant: {}", + serde_json::to_string_pretty(&answers).unwrap(), + serde_json::to_string_pretty(&want).unwrap(), + ); + // Structure: same answer count, same ids and discrete picks. + assert_eq!(answers.len(), want.len(), "{case}: answer count"); + for (got, expected) in answers.iter().zip(want) { + assert_eq!(got["id"], expected["id"], "{case}: answer order"); + if got["type"] != "span" { + assert_eq!(got["answer"], expected["answer"], "{case}: discrete pick"); + } else { + assert_eq!( + got["threshold_rule"], expected["threshold_rule"], + "{case}: rule" + ); + let got_spans = got["spans"].as_array().cloned().unwrap_or_default(); + let want_spans = expected["spans"].as_array().cloned().unwrap_or_default(); + assert_eq!(got_spans.len(), want_spans.len(), "{case}: span count"); + for (g, w) in got_spans.iter().zip(&want_spans) { + assert_eq!(g["label"], w["label"], "{case}: span label"); + assert_eq!(g["text"], w["text"], "{case}: span text"); + assert_eq!(g["start"], w["start"], "{case}: span start"); + assert_eq!(g["end"], w["end"], "{case}: span end"); + } + } + } + println!("{case}: logit drift {worst_logit:.3e}, answer drift {drift:.3e}"); + } +} + +fn span_block_names(row: &Value) -> Vec { + row["names"] + .as_array() + .map(|a| a.iter().map(|n| n.as_str().unwrap().to_string()).collect()) + .unwrap_or_default() +} + +fn span_qid(plan: &Value, row_index: usize) -> String { + // Row 0 carries the first span question; row i carries spans[i]. + let spans: Vec<&Value> = plan["plan"] + .as_array() + .unwrap() + .iter() + .filter(|p| p["type"] == "span") + .collect(); + let picked = spans.get(row_index).or_else(|| spans.first()); + picked + .map(|p| p["qid"].as_str().unwrap().to_string()) + .unwrap_or_else(|| format!("row{row_index}")) +} diff --git a/tests/vela/native_processing.rs b/tests/vela/native_processing.rs new file mode 100644 index 00000000..d87e1f6e --- /dev/null +++ b/tests/vela/native_processing.rs @@ -0,0 +1,137 @@ +//! Processing tests against the frozen token-plan fixtures. +//! +//! The mapping tests (state → parts, SystemOne → predict, plan) run +//! everywhere with no tokenizer and no weights. The full render parity +//! — rows and window rows byte-for-byte against the fixtures — needs +//! the exported bundle's tokenizer and is opt-in: +//! +//! VELA_NATIVE_BUNDLE=weights/vela-2.0-0.8b-native \ +//! cargo test -p omni-vela-native --test vela_native_processing -- --ignored --nocapture + +use std::path::PathBuf; + +use omni_vela_native::questions::system_one_to_predict; +use omni_vela_native::state::system_one_state; +use serde_json::Value; + +const CASES: [&str; 5] = [ + "example-request", + "example-request-set-threshold", + "example-request-cjk", + "example-request-score", + "long-span-windowed", +]; + +fn fixture(case: &str) -> Value { + let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../../../tests/vela/fixtures"); + serde_json::from_str( + &std::fs::read_to_string(root.join(format!("native-plan-{case}.json"))) + .expect("fixture present; record it with recipe/vela/export_token_plan.py"), + ) + .expect("fixture JSON") +} + +// ------------------------------------------------------- mapping (no tokens) + +fn mapped(case: &str) -> Value { + let fix = fixture(case); + let request = fix["request"].as_object().expect("request"); + let state = system_one_state(&request["state"]).expect("state maps"); + let questions = request["questions"].as_object().expect("questions"); + let mapped = system_one_to_predict(state, questions).expect("maps"); + serde_json::json!({ + "predict_request": serde_json::to_value(&mapped.request).unwrap(), + "plan": serde_json::to_value(&mapped.plan).unwrap(), + }) +} + +#[test] +fn the_predict_mapping_matches_every_fixture() { + for case in CASES { + let got = mapped(case); + let fix = fixture(case); + assert_eq!( + got["predict_request"], fix["predict_request"], + "predict_request differs for {case}" + ); + assert_eq!(got["plan"], fix["plan"], "plan differs for {case}"); + } +} + +#[test] +fn an_invalid_question_reports_the_pinned_triples() { + let state = system_one_state(&serde_json::json!("s")).expect("state"); + let questions = serde_json::json!({ + "q": {"type": "choice", "instructions": "ok?", "criteria": {"a": "x"}} + }); + let error = system_one_to_predict(state, questions.as_object().unwrap()).unwrap_err(); + assert_eq!( + error.body, + serde_json::json!({"detail": [{"loc": ["body", "questions", "q", "criteria"], + "msg": "choice criteria must be an object with 2 to 255 entries", "type": "value_error"}]}) + ); + + let state = system_one_state(&serde_json::json!("s")).expect("state"); + let questions = serde_json::json!({ + "q": {"type": "span", "instructions": "which?", "criteria": {"a": "x"}, + "head": "other", "over": "request"} + }); + let error = system_one_to_predict(state, questions.as_object().unwrap()).unwrap_err(); + assert_eq!( + error.body["detail"][0]["loc"], + serde_json::json!(["body", "questions", "q", "head"]) + ); + assert_eq!(error.body["detail"][0]["type"], "literal_error"); +} + +#[test] +fn a_set_label_collision_is_rejected_after_the_walk() { + let state = system_one_state(&serde_json::json!("s")).expect("state"); + let questions = serde_json::json!({ + "issues": {"type": "set", "instructions": "which?", "criteria": {"billing": "b"}}, + "issues.billing": {"type": "noul", "instructions": "which?"} + }); + let error = system_one_to_predict(state, questions.as_object().unwrap()).unwrap_err(); + assert_eq!( + error.body["detail"][0]["msg"], + "collides with the Noul view of a set label" + ); +} + +// ------------------------------------------------- full render (opt-in) + +fn bundle_dir() -> PathBuf { + let path = PathBuf::from( + std::env::var("VELA_NATIVE_BUNDLE").expect("set VELA_NATIVE_BUNDLE to an exported bundle"), + ); + assert!(path.is_dir(), "{} is not a directory", path.display()); + path +} + +#[test] +#[ignore = "requires VELA_NATIVE_BUNDLE at an exported bundle; CPU only"] +fn the_rendered_rows_match_every_fixture() { + let bundle = bundle_dir(); + let processor = omni_vela_native::processing::Processor::load(&bundle) + .expect("processor loads the exported bundle"); + for case in CASES { + let fix = fixture(case); + let request = fix["request"].as_object().expect("request"); + let planned = match processor.plan(request) { + Ok(planned) => planned, + Err(error) => panic!( + "plan failed for {case}: {}", + serde_json::to_string(&error.body).unwrap() + ), + }; + let got = serde_json::json!({ + "rows": serde_json::to_value(&planned.rows).unwrap(), + "window_rows": serde_json::to_value(&planned.window_rows).unwrap(), + }); + assert_eq!(got["rows"], fix["rows"], "rows differ for {case}"); + assert_eq!( + got["window_rows"], fix["window_rows"], + "window rows differ for {case}" + ); + } +} diff --git a/tests/vela/native_response.rs b/tests/vela/native_response.rs new file mode 100644 index 00000000..597cce5b --- /dev/null +++ b/tests/vela/native_response.rs @@ -0,0 +1,347 @@ +//! The SystemOne response finishing, replayed from the recorded +//! reference logits against the frozen oracle responses (the same +//! engine produced both on CPU FP32, so this checks the finishing +//! mirror exactly). Opt-in via `VELA_NATIVE_BUNDLE` behind +//! `-- --ignored` (calibration). + +use std::collections::HashMap; +use std::path::PathBuf; + +use omni_vela_native::scoring::{Calibration, SpanOut, row_answers, system_one_response}; +use serde_json::Value; + +const CASES: [&str; 4] = [ + "example-request", + "example-request-set-threshold", + "example-request-cjk", + "example-request-score", +]; + +fn fixtures_dir() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../../../tests/vela/fixtures") +} + +fn load(name: &str) -> Value { + serde_json::from_str(&std::fs::read_to_string(fixtures_dir().join(name)).expect("fixture")) + .expect("json") +} + +fn f32_matrix(raw: &[u8], name: &str) -> Vec> { + let file = safetensors::SafeTensors::deserialize(raw).expect("safetensors"); + let view = file.tensor(name).expect("tensor"); + let shape = view.shape(); + let data: Vec = view + .data() + .as_chunks::<4>() + .0 + .iter() + .map(|c| f32::from_le_bytes(*c)) + .collect(); + match shape.len() { + 1 => vec![data], + 2 => data.chunks(shape[1]).map(<[f32]>::to_vec).collect(), + _ => panic!("unexpected shape"), + } +} + +fn f32_stack(raw: &[u8], name: &str) -> Vec>> { + let file = safetensors::SafeTensors::deserialize(raw).expect("safetensors"); + let view = file.tensor(name).expect("tensor"); + let shape = view.shape(); + assert_eq!(shape.len(), 3); + let data: Vec = view + .data() + .as_chunks::<4>() + .0 + .iter() + .map(|c| f32::from_le_bytes(*c)) + .collect(); + data.chunks(shape[1] * shape[2]) + .map(|b| b.chunks(shape[2]).map(<[f32]>::to_vec).collect()) + .collect() +} + +#[test] +#[ignore = "requires VELA_NATIVE_BUNDLE at an exported bundle; CPU only"] +fn the_systemone_finishing_matches_the_frozen_oracles() { + let bundle = PathBuf::from( + std::env::var("VELA_NATIVE_BUNDLE").expect("set VELA_NATIVE_BUNDLE to an exported bundle"), + ); + let cal = Calibration::load(&bundle).expect("calibration"); + let heads = omni_vela_native::heads::Heads::load(&bundle).expect("heads"); + let gate = 2e-6; + + for case in CASES { + let head = load(&format!("head-input-{case}.json")); + let plan = load(&format!("native-plan-{case}.json")); + let oracle_name = if case == "example-request" { + "oracle-hybrid-example-request.json".to_string() + } else { + format!("oracle-hybrid-{case}.json") + }; + let oracle = load(&oracle_name); + let raw = std::fs::read(fixtures_dir().join(format!("head-input-{case}.safetensors"))) + .expect("safetensors"); + let request = plan["request"].as_object().expect("request").clone(); + + let mut thresholds: HashMap = HashMap::new(); + for (qid, question) in request["questions"].as_object().expect("questions") { + if let Some(t) = question.get("threshold").and_then(Value::as_f64) { + thresholds.insert(qid.clone(), t); + } + } + + // Drive the recorded calls through the Rust heads (as in the + // heads test) and assemble answers per row, in plan order. + let calls: Vec<&Value> = head["head_calls"].as_array().unwrap().iter().collect(); + let mut q_index = 0usize; + let mut span_index = 0usize; + let mut answers: Vec = Vec::new(); + let mut span_probs: HashMap> = HashMap::new(); + for (row_index, row) in plan["rows"].as_array().unwrap().iter().enumerate() { + let q_blocks: Vec<&Value> = row["blocks"] + .as_array() + .unwrap() + .iter() + .filter(|b| b["kind"] == "q") + .collect(); + let mut qblocks = Vec::new(); + if !q_blocks.is_empty() { + let call = calls + .iter() + .filter(|c| c["kind"] == "q") + .nth(q_index) + .expect("q call"); + q_index += 1; + let cands = f32_stack(&raw, call["cand"].as_str().unwrap()); + let queries = f32_matrix(&raw, call["query"].as_str().unwrap()); + for (block, (cand, query)) in q_blocks.iter().zip(cands.iter().zip(&queries)) { + let names: Vec = block["names"] + .as_array() + .unwrap() + .iter() + .map(|n| n.as_str().unwrap().to_string()) + .collect(); + let has_abs = block["has_abs"].as_bool().unwrap_or(false); + let mut logits = heads.forward_candidate(cand, query); + if block["qtype"] == "set" { + for value in logits.iter_mut() { + *value += heads.set_bias; + } + } + logits.truncate(names.len() + usize::from(has_abs)); + qblocks.push(omni_vela_native::scoring::QBlock { + qid: block["qid"].as_str().unwrap().to_string(), + qtype: block["qtype"].as_str().unwrap().to_string(), + names, + has_abs, + logits, + }); + } + } + // Span: use the recorded logits directly (the heads test + // covers head parity; here we exercise the finishing). + let span_blocks: Vec<&Value> = row["blocks"] + .as_array() + .unwrap() + .iter() + .filter(|b| b["kind"] == "span") + .collect(); + let mut span_out: Option = None; + if let Some(_block) = span_blocks.first() { + let call = calls + .iter() + .filter(|c| c["kind"] == "span") + .nth(span_index) + .expect("span call"); + span_index += 1; + let which = call["head"].as_str().unwrap_or("router").to_string(); + let logits: Vec> = call["logits"] + .as_array() + .unwrap() + .iter() + .map(|r| { + r.as_array() + .unwrap() + .iter() + .map(|v| v.as_f64().unwrap() as f32) + .collect() + }) + .collect(); + let names: Vec = row["names"] + .as_array() + .map(|a| a.iter().map(|n| n.as_str().unwrap().to_string()).collect()) + .unwrap_or_default(); + let w_offs: Vec<(usize, usize)> = row["w_offs"] + .as_array() + .map(|a| { + a.iter() + .map(|o| { + ( + o[0].as_u64().unwrap_or(0) as usize, + o[1].as_u64().unwrap_or(0) as usize, + ) + }) + .collect() + }) + .unwrap_or_default(); + let over = row["span_over"].as_str().unwrap_or("user").to_string(); + let n_tokens = row["seg_lens"][&over].as_u64().unwrap_or(0) as usize; + let qid = plan["plan"] + .as_array() + .unwrap() + .iter() + .filter(|p| p["type"] == "span") + .nth(row_index) + .or_else(|| { + plan["plan"] + .as_array() + .unwrap() + .iter() + .find(|p| p["type"] == "span") + }) + .map(|p| p["qid"].as_str().unwrap().to_string()) + .unwrap_or_default(); + // The Noul view needs the max word probability + // (pre-decode), range-filtered by the plan's range. + let temperature = cal.span_temperature(&which); + let probs: Vec = logits + .iter() + .map(|row| sigmoid_f32(row, temperature)) + .collect(); + let range = plan["plan"] + .as_array() + .unwrap() + .iter() + .find(|p| p["qid"] == qid.as_str()) + .and_then(|p| p["range"].as_array().cloned()); + let filtered: Vec = match &range { + Some(bounds) if bounds.len() == 2 => probs + .iter() + .zip(&w_offs) + .filter(|(_, (a, b))| { + *a >= bounds[0].as_u64().unwrap_or(0) as usize + && *b <= bounds[1].as_u64().unwrap_or(0) as usize + }) + .map(|(p, _)| *p) + .collect(), + _ => probs, + }; + span_probs.insert(qid.clone(), filtered); + span_out = Some(SpanOut { + qid: qid.clone(), + over, + names, + offs: w_offs, + logits, + head: which, + head_reason: row["head_reason"].as_str().map(str::to_string), + label_alias: row.get("label_alias").filter(|v| !v.is_null()).cloned(), + n_tokens, + }); + } + let span_text = row["span_over"].as_str().and_then(|role| { + plan["predict_request"]["parts"] + .as_array() + .unwrap() + .iter() + .find(|p| p["type"] == *role) + .map(|p| p["text"].as_str().unwrap().to_string()) + }); + let mut got = row_answers( + &cal, + &qblocks, + span_out.as_ref(), + &thresholds, + span_text.as_deref(), + ); + answers.append(&mut got); + } + // Plan-order the answers, then finish the envelope. + let order: Vec = plan["plan"] + .as_array() + .unwrap() + .iter() + .map(|p| p["qid"].as_str().unwrap().to_string()) + .collect(); + answers.sort_by_key(|a| { + order + .iter() + .position(|q| q == a["id"].as_str().unwrap_or("")) + .unwrap_or(usize::MAX) + }); + let part_texts: HashMap = plan["predict_request"]["parts"] + .as_array() + .unwrap() + .iter() + .map(|p| { + ( + p["type"].as_str().unwrap().to_string(), + p["text"].as_str().unwrap().to_string(), + ) + }) + .collect(); + let input_tokens = oracle["system_one"]["usage"]["input_tokens"] + .as_u64() + .unwrap_or(0) as usize; + let response = system_one_response( + "vllm-sr/Vela-2.0-0.8B", + plan["plan"].as_array().unwrap(), + &answers, + &span_probs, + &part_texts, + input_tokens, + ); + let want = &oracle["system_one"]; + let drift = max_drift(&response, want); + assert!( + drift <= gate, + "{case}: response drift {drift} > {gate}\ngot: {}\nwant: {}", + serde_json::to_string_pretty(&response).unwrap(), + serde_json::to_string_pretty(want).unwrap(), + ); + // Structure: same key sets and discrete fields. + for key in ["answers", "spans", "sets", "thresholds", "span_heads"] { + assert_eq!( + response + .get(key) + .and_then(Value::as_object) + .map(serde_json::Map::len), + want.get(key) + .and_then(Value::as_object) + .map(serde_json::Map::len), + "{case}: {key} key set" + ); + } + println!("{case}: response drift {drift:.3e}"); + } +} + +fn sigmoid_f32(row: &[f32], temperature: f64) -> f32 { + // The reference's per-word span view: sigmoid of every label logit + // (over the span temperature), max over the labels. + row.iter() + .map(|v| (1.0 / (1.0 + (-(*v as f64) / temperature).exp())) as f32) + .fold(0.0f32, |a, b| a.max(b)) +} + +fn max_drift(a: &Value, b: &Value) -> f64 { + match (a, b) { + (Value::Number(x), Value::Number(y)) => { + (x.as_f64().unwrap_or(0.0) - y.as_f64().unwrap_or(0.0)).abs() + } + (Value::Array(x), Value::Array(y)) if x.len() != y.len() => 1.0, + (Value::Array(x), Value::Array(y)) => x + .iter() + .zip(y) + .map(|(x, y)| max_drift(x, y)) + .fold(0.0, f64::max), + (Value::Object(x), Value::Object(y)) => x + .iter() + .map(|(k, v)| y.get(k).map(|w| max_drift(v, w)).unwrap_or(1.0)) + .chain(y.keys().filter(|k| !x.contains_key(*k)).map(|_| 1.0)) + .fold(0.0, f64::max), + (_, _) if a == b => 0.0, + _ => 1.0, + } +} diff --git a/tests/vela/native_units.rs b/tests/vela/native_units.rs new file mode 100644 index 00000000..ce695237 --- /dev/null +++ b/tests/vela/native_units.rs @@ -0,0 +1,101 @@ +//! Unit tests for the native crate's state mapping and word splitter, +//! extracted from the modules per the repository's test-placement +//! rule (the private contract test lives in native_contract_unit.rs, +//! loaded by the `#[path]` wiring in contract.rs). + +use omni_vela_native::state::system_one_state; +use omni_vela_native::words::{split_words, unit_ids, word_first_tokens}; +use serde_json::json; + +// ------------------------------------------------------------- state + +#[test] +fn a_plain_string_is_one_user_part() { + let parts = system_one_state(&json!("hello")).expect("parts"); + assert_eq!(parts.parts.len(), 1); + assert_eq!(parts.parts[0].r#type, "user"); + assert_eq!(parts.fields["request"].end, 5); +} + +#[test] +fn typed_keys_group_into_ordered_parts_with_field_ranges() { + let parts = system_one_state(&json!({ + "answer": "A", "request": "R", "source": "S", "extra": "E", + })) + .expect("parts"); + let roles: Vec<&str> = parts.parts.iter().map(|p| p.r#type.as_str()).collect(); + assert_eq!(roles, ["user", "context", "answer"]); + let context = &parts.parts[1]; + assert_eq!(context.text, "source:\nS\n\nextra:\nE"); + assert_eq!(parts.fields["source"].start, "source:\n".chars().count()); + assert_eq!( + parts.fields["extra"].start, + "source:\nS\n\nextra:\n".chars().count() + ); +} + +#[test] +fn an_empty_role_part_is_rejected() { + let error = system_one_state(&json!({"request": " "})).unwrap_err(); + assert_eq!( + error.body["detail"][0]["msg"], + serde_json::json!("the user part of the state is empty") + ); +} + +#[test] +fn a_string_holding_a_json_object_is_read_as_that_object() { + let parts = system_one_state(&json!("{\"request\": \"text\"}")).expect("parts"); + assert_eq!(parts.parts.len(), 1); + assert_eq!(parts.fields["request"].end, 4); +} + +// ------------------------------------------------------------- words + +#[test] +fn words_split_like_the_reference() { + let text = "Hi, I'm Tom Baker (tom.baker@example.com)."; + let words: Vec = split_words(text) + .iter() + .map(|(a, b)| text.chars().skip(*a).take(b - a).collect()) + .collect(); + assert_eq!( + words, + vec![ + "Hi", + ",", + "I", + "'", + "m", + "Tom", + "Baker", + "(", + "tom.baker@example.com", + ")", + "." + ] + ); +} + +#[test] +fn cjk_characters_are_one_unit_each() { + let text = "追踪 parcel 仓库"; + let words: Vec = split_words(text) + .iter() + .map(|(a, b)| text.chars().skip(*a).take(b - a).collect()) + .collect(); + assert_eq!(words, vec!["追", "踪", "parcel", "仓", "库"]); + // Mixed content: each CJK char its own unit, the ASCII word one + // unit, the space outside every unit. + let chars: Vec = "追踪 parcel".chars().collect(); + assert_eq!(unit_ids(&chars), vec![0, 1, -1, 2, 2, 2, 2, 2, 2]); +} + +#[test] +fn first_tokens_skip_empty_tokens() { + let text = "alpha beta"; + let offsets = vec![(0, 0), (0, 5), (6, 10)]; + let (word_offsets, first) = word_first_tokens(text, &offsets); + assert_eq!(word_offsets, vec![(0, 5), (6, 10)]); + assert_eq!(first, vec![1, 2]); +} diff --git a/tests/vela/test_hybrid.py b/tests/vela/test_hybrid.py new file mode 100644 index 00000000..ee3fb300 --- /dev/null +++ b/tests/vela/test_hybrid.py @@ -0,0 +1,310 @@ +"""Tests for the Vela 2.0 0.8B hybrid member: gate, worker dispatch, response finishing. + +Gate tests mirror test_engine_manifest: temp directories with an +injectable pin table and recomputed hashes — no weights, no torch, no +network. Worker tests drive FakeVela2Hybrid (from test_worker's fake, +plus the hybrid response surface) through decide/detect_member/ +models_entry: no weights, no torch, no GPU. + + PYTHONPATH=src python -m pytest tests/vela -q +""" + +from __future__ import annotations + +import hashlib +import json +import re +from pathlib import Path + +import pytest + +from frontend.vela import decide, detect_member, models_entry +from models.vela.engine import Vela2Executor, verify_files +from models.vela.hybrid import ( + EXPECTED_ARCHITECTURE, + EXPECTED_MAX_LENGTH, + EXPECTED_MODEL_NAME, + EXPECTED_MODEL_TYPE, + EXPECTED_WEIGHTS_BYTES, + HYBRID_REPO, + HYBRID_REVISION, + PINNED_FILES_SHA256, + WEIGHTS_NAME, + Vela2HybridExecutor, + verify_config, + verify_package, + verify_weights, +) +from models.vela.postprocess import finish_response + +from .test_protocol import body +from .test_worker import FakeVela2, five_kind_body + +CONFIG = { + "model_type": EXPECTED_MODEL_TYPE, + "architectures": [EXPECTED_ARCHITECTURE], + "max_length": EXPECTED_MAX_LENGTH, + "model_name": EXPECTED_MODEL_NAME, +} + + +def table_of(files: dict[str, bytes]) -> dict[str, str]: + return {name: hashlib.sha256(content).hexdigest() for name, content in files.items()} + + +def small_files(): + return { + "vela2_serve.py": b"upstream server stub\n", + "vela2_inference.py": b"upstream engine stub\n", + "modeling_vela2.py": b"upstream model stub\n", + "configuration_vela2.py": b"upstream config stub\n", + "calibration.json": b'{"temperature": {}}', + "tokenizer.json": b'{"version": "stub"}', + "tokenizer_config.json": b'{"model_type": "stub"}', + "model.safetensors.index.json": b'{"metadata": {"total_size": 1}}', + "broad_head.safetensors": b"second span head stub\n", + } + + +def build_package(root: Path, files: dict[str, bytes], config: dict | None = None) -> dict[str, str]: + for name, content in files.items(): + (root / name).write_bytes(content) + (root / "config.json").write_text( + json.dumps(config if config is not None else CONFIG), encoding="utf-8" + ) + with (root / WEIGHTS_NAME).open("wb") as stream: + stream.truncate(EXPECTED_WEIGHTS_BYTES) + files = dict(files) + files["config.json"] = (root / "config.json").read_bytes() + return table_of(files) + + +def hybrid_body(): + value = five_kind_body() + value["model"] = EXPECTED_MODEL_NAME + return value + + +class FakeVela2Hybrid(FakeVela2): + """The pinned hybrid runtime's contract, driven without its code.""" + + is_hybrid = True + + def __init__(self, model_name=EXPECTED_MODEL_NAME, too_long=False): + super().__init__(model_name, too_long) + self.max_length = 16384 + + class _Engine: + config = { + "model_name": EXPECTED_MODEL_NAME, + "description": ( + "Vela 2.0 0.8B: SystemOne Choice / Noul / Score " + "with span and set extensions, 16,384-token input" + ), + "release_date": "", + } + + self.engine = _Engine() + + def system_one(self, body): + response = super().system_one(body) + if "spans" in response: + # Upstream emits span_heads together with spans, keyed by the + # span question ids; router stands in for both heads here. + response["span_heads"] = {qid: "router" for qid in response["spans"]} + return response + + +# --------------------------------------------------------------------- gate + + +def test_the_checked_in_hybrid_pin_table_shape_is_locked(): + assert HYBRID_REPO == "vllm-sr/Vela-2.0-0.8B" + assert HYBRID_REVISION == "326b01d81f61fff7fd4ae570e54b87a49119e490" + assert set(PINNED_FILES_SHA256) == { + "vela2_serve.py", "vela2_inference.py", "modeling_vela2.py", + "configuration_vela2.py", "config.json", "calibration.json", "tokenizer.json", + "tokenizer_config.json", "model.safetensors.index.json", "broad_head.safetensors", + } + for digest in PINNED_FILES_SHA256.values(): + assert re.fullmatch(r"[0-9a-f]{64}", digest) + + +def test_verification_accepts_a_package_matching_its_table(tmp_path): + table = build_package(tmp_path, small_files()) + assert verify_files(tmp_path, table) == sorted(table) + verify_config(tmp_path) + verify_weights(tmp_path) + verify_package(tmp_path, table) + + +def test_a_unified_package_is_refused_with_its_own_name(tmp_path): + build_package( + tmp_path, small_files(), + config={**CONFIG, "model_type": "vela2-unified"}, + ) + with pytest.raises(ValueError, match="vela2-unified.*only serves the vela2-decoder member"): + verify_config(tmp_path) + + +@pytest.mark.parametrize( + ("override", "fragment"), + [ + ({"architectures": ["OtherModel"]}, "architectures"), + ({"max_length": 8192}, "max_length"), + ({"model_name": "vllm-sr/Vela-2.0-4B"}, "needs its own pinned executor"), + ], +) +def test_foreign_config_structures_are_refused(tmp_path, override, fragment): + build_package(tmp_path, small_files(), config={**CONFIG, **override}) + with pytest.raises(ValueError, match=fragment): + verify_config(tmp_path) + + +def test_tampered_files_are_rejected(tmp_path): + table = build_package(tmp_path, small_files()) + victim = tmp_path / "broad_head.safetensors" + victim.write_bytes(victim.read_bytes() + b"tampered\n") + # The shared gate must name the hybrid's pin, not the 0.3B's. + with pytest.raises( + ValueError, match=f"broad_head.safetensors: SHA-256 differs from the pin for {HYBRID_REVISION}" + ): + verify_files(tmp_path, table, HYBRID_REVISION) + + +def test_a_missing_broad_head_is_rejected(tmp_path): + table = build_package(tmp_path, small_files()) + (tmp_path / "broad_head.safetensors").unlink() + with pytest.raises(ValueError, match="missing pinned file broad_head.safetensors"): + verify_files(tmp_path, table) + + +def test_a_later_self_consistent_snapshot_is_refused_by_the_checked_in_pin(tmp_path): + build_package(tmp_path, small_files()) + with pytest.raises(ValueError, match="SHA-256 differs from the pin"): + verify_files(tmp_path, PINNED_FILES_SHA256) + + +def test_weights_are_checked_by_exact_pinned_size(tmp_path): + build_package(tmp_path, small_files()) + verify_weights(tmp_path) + + with (tmp_path / WEIGHTS_NAME).open("wb") as stream: + stream.truncate(EXPECTED_WEIGHTS_BYTES - 1) + with pytest.raises(ValueError, match="is 2022189487 bytes"): + verify_weights(tmp_path) + + (tmp_path / WEIGHTS_NAME).unlink() + with pytest.raises(ValueError, match=f"missing {WEIGHTS_NAME}"): + verify_weights(tmp_path) + + +# ------------------------------------------------------------- worker wiring + + +def test_detect_member_routes_each_declared_member(tmp_path): + unified = tmp_path / "unified" + unified.mkdir() + (unified / "config.json").write_text(json.dumps({"model_type": "vela2-unified"})) + decoder = tmp_path / "decoder" + decoder.mkdir() + (decoder / "config.json").write_text(json.dumps({"model_type": "vela2-decoder"})) + + cls, name = detect_member(unified) + assert cls is Vela2Executor and name == "vllm-sr/Vela-2.0-0.3B" + cls, name = detect_member(decoder) + assert cls is Vela2HybridExecutor and name == EXPECTED_MODEL_NAME + + +def test_detect_member_refuses_packages_without_a_usable_config(tmp_path): + empty = tmp_path / "empty" + empty.mkdir() + with pytest.raises(ValueError, match="missing config.json"): + detect_member(empty) + foreign = tmp_path / "foreign" + foreign.mkdir() + (foreign / "config.json").write_text(json.dumps({"model_type": "vela2-unified-xl"})) + with pytest.raises(ValueError, match="vela2-unified-xl"): + detect_member(foreign) + broken = tmp_path / "broken" + broken.mkdir() + (broken / "config.json").write_text("{not json") + with pytest.raises(ValueError, match="not readable JSON"): + detect_member(broken) + array = tmp_path / "array" + array.mkdir() + (array / "config.json").write_text("[]") + with pytest.raises(ValueError, match="not a JSON object"): + detect_member(array) + + +def test_models_entry_is_verbatim_per_member(): + hybrid = models_entry(FakeVela2Hybrid()) + assert hybrid == { + "name": EXPECTED_MODEL_NAME, + "release_date": "", + "description": ( + "Vela 2.0 0.8B: SystemOne Choice / Noul / Score " + "with span and set extensions, 16,384-token input" + ), + } + assert models_entry(FakeVela2())["release_date"] == "2026-09-29" + + +# -------------------------------------------------------- response finishing + + +def test_decide_answers_all_five_kinds_with_span_heads(): + response = decide(json.dumps(hybrid_body()).encode(), FakeVela2Hybrid()) + assert set(response) == {"model", "answers", "usage", "spans", "sets", "thresholds", "span_heads"} + assert list(response["span_heads"]) == ["unsupported"] + assert response["model"] == EXPECTED_MODEL_NAME + + +def test_decide_accepts_a_hybrid_response_without_span_questions(): + value = body(model=EXPECTED_MODEL_NAME) + response = decide(json.dumps(value).encode(), FakeVela2Hybrid()) + assert "span_heads" not in response and "spans" not in response + + +def test_a_hybrid_response_missing_span_heads_is_refused(): + class Headless(FakeVela2Hybrid): + def system_one(self, body): + response = super().system_one(body) + response.pop("span_heads", None) + return response + + with pytest.raises(RuntimeError, match="span_heads do not match"): + decide(json.dumps(hybrid_body()).encode(), Headless()) + + +def test_a_hybrid_response_with_stray_span_heads_is_refused(): + class Stray(FakeVela2Hybrid): + def system_one(self, body): + response = super().system_one(body) + response["span_heads"] = {"ghost": "router"} + return response + + with pytest.raises(RuntimeError, match="span_heads without span questions"): + decide(json.dumps(body(model=EXPECTED_MODEL_NAME)).encode(), Stray()) + + +def test_the_unified_member_still_refuses_span_heads(): + class UnifiedLeak(FakeVela2): + def system_one(self, body): + response = super().system_one(body) + if "spans" in response: + response["span_heads"] = {qid: "router" for qid in response["spans"]} + return response + + with pytest.raises(RuntimeError, match="this worker serves the 0.3B member"): + decide(json.dumps(five_kind_body()).encode(), UnifiedLeak()) + + +def test_finish_response_keeps_the_hybrid_envelope_tight(): + questions = hybrid_body()["questions"] + response = FakeVela2Hybrid().system_one(hybrid_body()) + assert finish_response(questions, response, hybrid=True) is response + del response["answers"] + with pytest.raises(RuntimeError, match="unexpected response envelope"): + finish_response(questions, response, hybrid=True) diff --git a/tests/vela/test_oracle_hybrid_fixtures.py b/tests/vela/test_oracle_hybrid_fixtures.py new file mode 100644 index 00000000..3114a904 --- /dev/null +++ b/tests/vela/test_oracle_hybrid_fixtures.py @@ -0,0 +1,63 @@ +"""Ties the frozen hybrid oracle fixtures to the recipe examples they came from. + +The 0.8B hybrid fixtures recorded by ``recipe/vela/compare_reference.py`` +under the ``oracle-hybrid-`` prefix keep the recipe's example requests +(with the ``model`` field rewritten to the hybrid's served name) and their +recorded ``system_one`` responses, which must pass the production +``finish_response`` gate on the hybrid setting; the 0.3B oracles have no +equivalent consumer yet. No weights, no torch. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from models.vela.hybrid import EXPECTED_MODEL_NAME +from models.vela.postprocess import finish_response + +TESTS = Path(__file__).parent +RECIPE = TESTS.parent.parent / "recipe" / "vela" + +CASES = [ + ("oracle-hybrid-example-request", "example-request.json"), + ("oracle-hybrid-example-request-cjk", "example-request-cjk.json"), + ("oracle-hybrid-example-request-set-threshold", "example-request-set-threshold.json"), + ("oracle-hybrid-example-request-score", "example-request-score.json"), +] + + +def oracle(case: str) -> dict: + return json.loads((TESTS / "fixtures" / f"{case}.json").read_text()) + + +def recipe_body(example: str) -> dict: + body = json.loads((RECIPE / example).read_text()) + # The tool's load_cases rewrite for file cases: the examples are + # member-agnostic apart from the model field. + return {**body, "model": EXPECTED_MODEL_NAME} + + +@pytest.mark.parametrize("case,example", CASES) +def test_the_oracle_request_is_the_recipe_example(case: str, example: str): + recorded = oracle(case) + served = recipe_body(example) + assert recorded["request"] == served + # Same insertion order, so byte-for-byte serialization holds too. + assert list(recorded["request"]) == list(served) + + +@pytest.mark.parametrize("case,example", CASES) +def test_the_recorded_response_passes_the_worker_gate(case: str, example: str): + recorded = oracle(case) + response = recorded["system_one"] + assert response["model"] == EXPECTED_MODEL_NAME + assert finish_response(recorded["request"]["questions"], response, hybrid=True) is response + if any(q.get("type") == "span" for q in recorded["request"]["questions"].values()): + assert set(response["span_heads"]) == { + qid for qid, q in recorded["request"]["questions"].items() if q.get("type") == "span" + } + # The frozen case label still names the recipe file it belongs to. + assert recorded["case"] == example.removesuffix(".json") diff --git a/tests/vela/test_worker_server.py b/tests/vela/test_worker_server.py index ce2d106b..9221a933 100644 --- a/tests/vela/test_worker_server.py +++ b/tests/vela/test_worker_server.py @@ -292,3 +292,51 @@ def test_the_correct_bearer_answers_normally(keyed_server): headers=(("Authorization", "Bearer secret-key"),), ) assert status == 200 and json.loads(raw)["answers"]["billing"] == {"type": "noul", "noul": 0.75} + + +@pytest.fixture() +def hybrid_server(): + from .test_hybrid import FakeVela2Hybrid, hybrid_body + + instance = start(FakeVela2Hybrid()) + yield instance, hybrid_body() + stop(instance) + + +def test_the_hybrid_member_serves_its_own_models_entry(hybrid_server): + server, _ = hybrid_server + status, _, raw = request(server, "GET", "/v1/models") + assert status == 200 + assert json.loads(raw) == { + "models": [ + { + "name": "vllm-sr/Vela-2.0-0.8B", + "release_date": "", + "description": ( + "Vela 2.0 0.8B: SystemOne Choice / Noul / Score " + "with span and set extensions, 16,384-token input" + ), + } + ] + } + + +def test_the_hybrid_member_answers_all_five_kinds_with_span_heads(hybrid_server): + server, hybrid_body = hybrid_server + status, ctype, raw = post(server, json.dumps(hybrid_body).encode("utf-8")) + assert status == 200 and ctype == "application/json" + response = json.loads(raw) + assert response["model"] == "vllm-sr/Vela-2.0-0.8B" + assert set(response) == { + "model", "answers", "usage", "spans", "sets", "thresholds", "span_heads" + } + assert list(response["span_heads"]) == ["unsupported"] + assert response["answers"]["unsupported"] == {"type": "noul", "noul": 0.9} + + +def test_the_hybrid_member_rejects_another_model_name(hybrid_server): + server, _ = hybrid_server + wrong = body(model="vllm-sr/Vela-2.0-0.3B") + status, _, raw = post(server, json.dumps(wrong).encode("utf-8")) + assert status == 422 + assert json.loads(raw)["detail"][0]["type"] == "value_error.model"