diff --git a/Cargo.lock b/Cargo.lock index 3dff4a6d..812e6d8b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -199,9 +199,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.4.7" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54413ede23c2daf518f35156dfde027feb2374004d63bd497f983c8db9c0e313" +checksum = "86e055fa6e3e40ad6187ba25b6855eeb635d5271f8e0f0e1e9a5826b8d75bea0" dependencies = [ "find-msvc-tools", "shlex", @@ -271,9 +271,9 @@ dependencies = [ [[package]] name = "crc32fast" -version = "1.5.2" +version = "1.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78" +checksum = "0ba106e844045ddc02c20768ac4b0b7cb45f2cc2ecbd833ac9f5dacb183c2055" dependencies = [ "cfg-if", ] @@ -438,20 +438,20 @@ checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] name = "either" -version = "1.18.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" +checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" [[package]] name = "equivalent" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" [[package]] name = "errno" @@ -497,9 +497,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.13" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef25905e51abafe4dcea6c15fec58c57b601cdbd0ee53d22ea1d3016c587d39b" +checksum = "1c66915cc5391097de7a2b0b972e8505aa33c0df401fd3a6811dbdb32881ba48" [[package]] name = "flate2" @@ -563,7 +563,7 @@ checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -729,13 +729,12 @@ dependencies = [ [[package]] name = "hyper" -version = "1.11.1" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27b501faa50e7a26c3d3560ca625132f4078a17771f4810baf70475ae48cbe43" +checksum = "2c3e324da4c95177d6291d4c8730197c0d1822f8a9766814a4a44fa5ab797c9c" dependencies = [ "atomic-waker", "bytes", - "futures-channel", "futures-core", "http", "http-body", @@ -790,9 +789,9 @@ dependencies = [ [[package]] name = "icu_collections" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" +checksum = "fe090a42a0d3140fed8c63263ed1fecc02422cb2f4bfe55cd8de2de1b8e105dd" dependencies = [ "displaydoc", "potential_utf", @@ -804,9 +803,9 @@ dependencies = [ [[package]] name = "icu_locale_core" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" +checksum = "25d29b57c17c6528d0dc4c2141cfa2d08200465acb537a3909c8b9b8a4a984c0" dependencies = [ "displaydoc", "litemap", @@ -817,9 +816,9 @@ dependencies = [ [[package]] name = "icu_normalizer" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" +checksum = "404a3b8c486f205c87fe7f23ab6894d84a4ad76d6e4671ae1981ba7def87c599" dependencies = [ "icu_collections", "icu_normalizer_data", @@ -831,15 +830,15 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" +checksum = "20f0088bfb7441f7a37fa0c9e20cb1e9cba1cac6b9ce8cd2a6dc44294f8e8cbc" [[package]] name = "icu_properties" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" +checksum = "2ea83d991c56158f7cf898bf86436c1dfe7c07ee82fe88951f2695c9f37a379f" dependencies = [ "displaydoc", "icu_collections", @@ -852,15 +851,15 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" +checksum = "b98f29af82d02b163fe50161388350e743bb3107cc8e8c2f7314cb505faddfcf" [[package]] name = "icu_provider" -version = "2.3.1" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" +checksum = "413099315636c588ee8f011190b329b971a09a8fae9d157e01dcae8379468514" dependencies = [ "displaydoc", "icu_locale_core", @@ -968,9 +967,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.189" +version = "0.2.191" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" [[package]] name = "libloading" @@ -1042,9 +1041,9 @@ checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" [[package]] name = "memmap2" -version = "0.9.11" +version = "0.9.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0" +checksum = "6b0ca1155a9ecc25aa12e8313a119f32edf0255fea2f44839c4ac0a12b2ac157" dependencies = [ "libc", ] @@ -1083,9 +1082,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.2.3" +version = "1.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b18443e9c262bfe8fa82f51666e2642c53393f7e5c27b3e1aeab922cff5b9d8" +checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee" dependencies = [ "libc", "wasi", @@ -1257,7 +1256,7 @@ dependencies = [ "omni-runtime", "safetensors 0.8.0", "serde_json", - "sha2 0.11.0", + "sha2 0.11.1", "tempfile", "tokenizers 0.22.2", "tokio", @@ -1353,6 +1352,21 @@ dependencies = [ "tokio", ] +[[package]] +name = "omni-vela-native" +version = "0.1.0" +dependencies = [ + "anyhow", + "libm", + "omni-qwen3-5-native", + "onig", + "safetensors 0.8.0", + "serde", + "serde_json", + "sha2 0.11.1", + "tokenizers 0.22.2", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -1485,9 +1499,9 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.18" +version = "0.11.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9746dbde176634f4f2f1faf2404e30a31b2bc1e9cafb5329c95d8177a18c9fc" +checksum = "0e750cca55fe4f0439a15d0bb529da9651e79993e8e72c61a899a36d462befbe" dependencies = [ "bytes", "getrandom 0.4.3", @@ -1507,9 +1521,9 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.15" +version = "0.5.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" +checksum = "af66907df18639dcf4db56ca65490cabc4b27a97dbadd96f2926cca73298f016" dependencies = [ "cfg_aliases", "libc", @@ -1732,9 +1746,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.45" +version = "0.23.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" dependencies = [ "once_cell", "ring", @@ -1756,9 +1770,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.15" +version = "0.103.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" +checksum = "4443b1b6d39d7c663139942cc4558262562d2c84123bb6282ef177009ca8b281" dependencies = [ "ring", "rustls-pki-types", @@ -1827,15 +1841,16 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] name = "serde_json" -version = "1.0.151" +version = "1.0.154" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" dependencies = [ + "foldhash", "indexmap", "itoa", "memchr", @@ -1880,9 +1895,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.11.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +checksum = "47d7069beb7d6ac7b9acd1039986e73443f24234f41074da099d6f994ac9ad19" dependencies = [ "cfg-if", "cpufeatures 0.3.1", @@ -1919,9 +1934,9 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "smallvec" -version = "1.16.2" +version = "1.16.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9395f0f0eee849a9b707b2f06bb92a6a422090e2123bb2ef8e87a0e61892a8e" +checksum = "d0d5d4b1981b8bbbb7aed204cae5726d683d5d6ac7e74ec54122ea28f315e7c8" [[package]] name = "socket2" @@ -1982,9 +1997,9 @@ dependencies = [ [[package]] name = "syn" -version = "3.0.6" +version = "3.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" dependencies = [ "proc-macro2", "quote", @@ -2008,7 +2023,7 @@ checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -2041,14 +2056,14 @@ checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] name = "tinystr" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" +checksum = "abc840c238154d58f9e86b2792261edfde565a17696e1cda1a449de27979801a" dependencies = [ "displaydoc", "zerovec", @@ -2128,9 +2143,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.53.1" +version = "1.54.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +checksum = "b52fd3255258d26fac7264ab5a76b53c13367136dc011d3d6ad4f8ed070396a3" dependencies = [ "bytes", "libc", @@ -2150,14 +2165,14 @@ checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] name = "tokio-rustls" -version = "0.26.5" +version = "0.26.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0c85f2c3ef0b1cd58b36682f4b17aaa995f0e5db534d85692b4903abce21f67" +checksum = "c9cc2678c2cdd569ef8215e2afd7954ada2ae20b4fdd2c5fe6139a3b02d105db" dependencies = [ "rustls", "tokio", @@ -2165,9 +2180,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.19" +version = "0.7.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" +checksum = "e464cf451ba96ebfc6f9b6542f17ee8b8956e33f1e40d9690624e59d7a7f8a4b" dependencies = [ "bytes", "futures-core", @@ -2256,9 +2271,9 @@ checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" [[package]] name = "unicode-ident" -version = "1.0.26" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" [[package]] name = "unicode-normalization-alignments" @@ -2313,9 +2328,9 @@ checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] name = "want" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +checksum = "ec4cdd0dd910afe868b7ef477227d8d538b46b3075031afee8a9f2acb0a2ed0b" dependencies = [ "try-lock", ] @@ -2378,7 +2393,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", "wasm-bindgen-shared", ] @@ -2426,9 +2441,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "1.0.9" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" +checksum = "f5f280bbb2b287bc8e7de7e3852945a0ff50d1bbcb5ed42b6c7a568c546df171" dependencies = [ "rustls-pki-types", ] @@ -2535,9 +2550,9 @@ checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" [[package]] name = "yoke" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +checksum = "417ef3fcf6c300b7dbd25c85531a746931c02ad536f3cf4786ed1e88223340b3" dependencies = [ "stable_deref_trait", "yoke-derive", @@ -2546,30 +2561,30 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33811428bee40dbceb6d545e95754741d17a6aef9a4849f0fd62e2ba4f412a78" +checksum = "ec8ebde2db3681e8c9980cc27822030e68752690ddfa9473e739aeb4dbde6d71" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", "synstructure", ] [[package]] name = "zerocopy" -version = "0.8.59" +version = "0.8.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6df92bf3d9227be3d53173901ddbffac2babc27ae50f397776ffd6dc33f800cb" +checksum = "e5fe1f8f1b06191a00962174c61aa5005e0bb391a6d80d07e24d115c01a92ed8" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.59" +version = "0.8.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac4f328cf2f05d084e496c3e9c3f33ed0a183656a16e1fcec4d464d8373aec82" +checksum = "863ad3ac83293fb4d740aedbfdc9240dd8d1a50c1099acd76ce80ce7c7230c7f" dependencies = [ "proc-macro2", "quote", @@ -2593,15 +2608,15 @@ checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", "synstructure", ] [[package]] name = "zeroize" -version = "1.9.0" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" +checksum = "e13084392c5e4bc371903e2935a5eaeed24905a7511356b883835e18a78f6879" [[package]] name = "zerotrie" @@ -2616,9 +2631,9 @@ dependencies = [ [[package]] name = "zerovec" -version = "0.11.8" +version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" +checksum = "f58a2799cfe9a1670c7ea8edbd52ef8edb93cb37a7f83bf31ba9e35d9dae6a40" dependencies = [ "yoke", "zerofrom", @@ -2627,13 +2642,13 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.6" +version = "0.11.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" +checksum = "c3743ddbf47738d929cd1ae2e199d40d0162979dfe31aef8ae646d2bf00bf97a" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 7910b3dd..cf1568f6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,3 +1,3 @@ [workspace] -members = ["src/frontend", "src/runtime", "src/models/clm", "src/models/cua_s1/native", "src/models/qwen3_5/native", "src/models/open_jev/native", "src/models/jev_vl/native", "src/models/omnijev/native", "src/models/laya", "src/backends/cuda", "src/models/decider/native", "src/models/jemm/native", "src/models/startlux/native", "src/models/clef/native"] +members = ["src/frontend", "src/runtime", "src/models/clm", "src/models/cua_s1/native", "src/models/qwen3_5/native", "src/models/open_jev/native", "src/models/jev_vl/native", "src/models/omnijev/native", "src/models/laya", "src/backends/cuda", "src/models/decider/native", "src/models/jemm/native", "src/models/vela/native", "src/models/startlux/native", "src/models/clef/native"] resolver = "3" diff --git a/README.md b/README.md index 1502a8fb..68a0874f 100644 --- a/README.md +++ b/README.md @@ -193,7 +193,7 @@ checkpoint. Public231 concurrency-1 latency for heads `1ba07d1`, `f968e576`, | OmniJev-4B v1.1 | [Native Rust/CUDA worker](recipe/omnijev/native.md), eager: one image, Choice, Noul and Score; [reference comparison on sm_89](recipe/omnijev/native.md#comparison-with-the-reference) | | D3 (Decision 3.0) | [Python reference worker](recipe/d3/reference.md) on the checkpoint's pinned runtime; `choice`/`noul`/`score` over text, image and video state; author-run on A100 for text and media serving (see the integration and media PRs) | | Vela 2.0 0.3B | [Python reference worker](recipe/vela/reference.md) on the checkpoint's pinned runtime; all five kinds (`choice`/`noul`/`score`/`span`/`set`) for text state; CPU FP32 validated (author-run, evidence in the integration PR) | -| Vela 2.0 0.8B hybrid | [Python reference worker](recipe/vela/reference.md) on the checkpoint's pinned runtime; same surface plus `span_heads`; CPU FP32 parity recorded in the integration PR | +| Vela 2.0 0.8B hybrid | [Python reference worker](recipe/vela/reference.md) on the checkpoint's pinned runtime; same surface plus `span_heads`; [native library](recipe/vela/native.md) (bundle export, wire contract, processing, F32 heads); CPU FP32 parity recorded in the integration PR | | Clef-flash (9B) and Clef (27B) | [Rust `omni-clef` server](recipe/clef/reference.md) plus the pinned Python joint-schema executor on one persistent thread; text `choice`/`noul`/`score`; flash is the bring-up variant. Image, video, and a native joint head are not served. The L20X-labeled lab Public231 C1 primary is the previous Python HTTP worker, tip `594528a9`: p50 164.969 ms, p95 216.755 ms, `/health` `fla+causal_conv1d`, versus `9030065` run1 p50 247.518 ms, p95 413.025 ms (Δp50 −82.549 ms, Δp95 −196.270 ms). Operator-reported `omni-clef` tip `f9127a0` on that host is a serve-path paste, not a latency KEEP: R1 p50 169.605 ms, p95 220.003 ms (231/231) does not pass the dual gate versus 164.969/216.755; R2 53.580/97.031 and R3 52.950/97.608 are not a new primary. Smoke 4/4, `/health` `fla+causal_conv1d`, quality matches the baseline (`choice_acc=0.8417`, `noul_acc=0.7973`, `score_mae=0.2488`) | | pplx-decider-v1.1-27b | [Python reference worker](recipe/pplx_decider/README.md) on the pinned checkpoint; `choice`/`noul`/`score` text plus up to four images. Lab L20X-labeled GPUs are the H200 target. Kept Public231 primary is `fcf5e2b`: 231/231, p50 87.782 ms, p95 291.183 ms, grouped-value FlashInfer with q and k packed before the L2 view (p50 −1.30%, p95 −1.56% versus `4b99b73` 88.942 / 295.797). `4b99b73` was p50 −4.20%, p95 −0.22% versus `ffae182` 92.844 / 296.458. Uncached all-sequence FlashInfer `3c72f02` is not the primary (p50 88.808 ms, p95 300.751 ms). Short-sequence FlashInfer `a50b9aa` missed its gate (p50 95.446 ms, p95 300.871 ms). cuDNN-first SDPA `0dcaa0ac` missed its gate (p50 138.501 ms, p95 332.859 ms). Fused gate-up `8b4ee326` missed its gate (p50 102.437 ms, p95 304.970 ms). Always-on dynamic compile `e36b32ed` missed the both-percentiles gate (p50 97.751 ms, p95 263.124 ms; p95 improved). The 512-token split `7e3b1be` missed its gate (p50 100.037 ms, p95 257.084 ms). Isolated-median compile `34ba5de` missed its gate (p50 97.084 ms, p95 256.679 ms; the 136-token probe was eager). The FlashInfer state-workspace ring `17a7b120` missed its gate (p50 89.165 ms, p95 295.723 ms). Always-on FLA residual RMSNorm `3afb472` missed its gate (p50 93.935 ms, p95 274.902 ms). Length-gated FLA residual RMSNorm `705ae089` missed its gate (p50 94.604 ms, p95 274.835 ms). `use_cp=auto` `a6f4d9a` missed its gate (p50 112.294 ms, p95 299.419 ms). Grouped-value `3ecdf5a` crashed in warmup (0/231, no p50 or p95). FLA fused gated RMSNorm `801ef7b` missed its gate (p50 89.172 ms, p95 274.790 ms). FLA `swiglu_fwd` `2bbbe4e` missed its gate (owner closeout p50 97.613 ms, p95 292.795 ms) and is not installed | diff --git a/docs/supported-models.md b/docs/supported-models.md index ca929b0e..0e92922d 100644 --- a/docs/supported-models.md +++ b/docs/supported-models.md @@ -29,7 +29,7 @@ Models that are being added are also tracked in issues labeled [new model](https | OmniJev-4B v1.1 | [Native Rust/CUDA worker](../recipe/omnijev/native.md) on the shared Qwen3.5 vision and language path, eager | Contract, preparation, pixels, heads and finishing checked against the reference; no CPU inference | Compared with the reference on compute capability 8.9 for [140 answers](../recipe/omnijev/native.md#comparison-with-the-reference) | Not supported | Compute capability 8.0 or newer, CUDA toolkit to build, the export from `export.py` | | D3 (Decision 3.0) | [Python reference worker](../recipe/d3/reference.md) on the checkpoint's pinned runtime — text, image and video state over `choice`/`noul`/`score`, with the deliberate transport differences documented in the recipe; the native worker is in progress ([#136](https://github.com/ThinkFlowLab/system1-omni/issues/136)) | Unverified | Author-run on NVIDIA A100 (sm_80): serving smoke and reference parity for text and media (exact decisions, max delta 0.0); evidence in the integration and media PRs. Owner: @MrDongsls. | Unverified | Python 3.12, the [pinned runtime requirements](../recipe/d3/requirements.txt), the [pinned package](https://huggingface.co/vllm-sr/d3) (~52 GB); [media budgets and the 64 MiB total request cap](../recipe/d3/reference.md) | | Vela 2.0 0.3B | [Python reference worker](../recipe/vela/reference.md) running the checkpoint's pinned runtime | **CPU FP32 validated** (author-run, x86 WSL2, 24 cores): startup with all-kind warmup, the four example requests plus five synthetic rejections, and exact-match reference parity against the pinned upstream server on all nine (four 200s byte-equal incl. key order, agreed 422/413); byte-identical frontend forwarding; evidence in the integration PR. Owner: @MrDongsls. | Unverified (bf16-autocast is an untested deployment option) | Not supported | Python 3.12, the [pinned requirements](../recipe/vela/requirements.txt), the [pinned package](https://huggingface.co/vllm-sr/Vela-2.0-0.3B) (1.24 GB FP32 weights; ~4.4 GB with ONNX/assets) | -| Vela 2.0 0.8B hybrid | [Python reference worker](../recipe/vela/reference.md) running the checkpoint's pinned runtime (same wire layer as 0.3B) | Same surface as 0.3B plus the `span_heads` field (router/broad dispatch); **CPU FP32 parity** (author-run; evidence in the integration PR) | Unverified (bf16-autocast is the evaluated upstream setting but untested here) | Not supported | Python 3.12, the [pinned requirements](../recipe/vela/requirements.txt), the [pinned package](https://huggingface.co/vllm-sr/Vela-2.0-0.8B) (2.02 GB shard + 4.5 MB broad head; ~3 GB FP32 parameters) | +| Vela 2.0 0.8B hybrid | [Python reference worker](../recipe/vela/reference.md) running the checkpoint's pinned runtime (same wire layer as 0.3B); [native library](../recipe/vela/native.md) — bundle export, wire contract, processing, F32 heads and scoring | Same surface as 0.3B plus the `span_heads` field (router/broad dispatch); **CPU FP32 parity** (author-run; evidence in the integration PR) | Unverified (bf16-autocast is the evaluated upstream setting but untested here) | Not supported | Python 3.12, the [pinned requirements](../recipe/vela/requirements.txt), the [pinned package](https://huggingface.co/vllm-sr/Vela-2.0-0.8B) (2.02 GB shard + 4.5 MB broad head; ~3 GB FP32 parameters) | | Clef-flash (9B) | [Rust `omni-clef` server](../recipe/clef/reference.md) with the pinned Python executor on one persistent thread; text `choice`/`noul`/`score`; non-thinking prompt; `thinking.enabled: false` admitted and omitted from the prompt; 16384-token refusal. The joint head is still the checkpoint module | Contract tests only, including the persistent-thread serve path; no inference | Kept Public231 primary on the L20X-labeled lab host is the previous `ThreadingHTTPServer` worker. Baseline `9030065` run1 p50 247.518 ms, p95 413.025 ms, 231/231. Tip `594528a9` remeasure, warmup 5, concurrency 1, stock Public231 with `thinking.enabled` false, metric `successful_latency`: 231/231, p50 164.969 ms, p95 216.755 ms, `/health` `fla+causal_conv1d`. Versus run1: Δp50 −82.549 ms (−33.35%), Δp95 −196.270 ms (−47.52%). Operator-reported `omni-clef` tip `f9127a0`: smoke 4/4, `/health` `fla+causal_conv1d`, R1 p50 169.605 ms, p95 220.003 ms (231/231), R2 53.580/97.031, R3 52.950/97.608, quality `choice_acc=0.8417` `noul_acc=0.7973` `score_mae=0.2488` (matches `594528a9`). Not a latency KEEP. R1 does not pass the dual gate versus 164.969/216.755. R2 and R3 are not a new primary. Not re-run here | Not supported | Python 3.12, [card-tested torch 2.11 and transformers 5.10.2](../recipe/clef/requirements.txt), a Rust build of `omni-clef`, pinned `Cloudflare/clef-flash` snapshot (~19.1 GB) | | Clef (27B) | Same `omni-clef` server and text contract as Clef-flash, selected with `--variant clef` | Contract tests only; no inference | Not run | Not supported | Same stack; pinned `Cloudflare/clef` snapshot (~55.0 GB). Weights and joint-head config differ from flash | | pplx-decider-v1.1-27b | [Python reference worker](../recipe/pplx_decider/README.md) on the checkpoint's own runtime. Text and up to four images; `choice`/`noul`/`score`. Native CUDA executor is not this worker. Lab L20X-labeled GPUs are the H200 target. The causal-conv1d build keeps sm_90 and sm_89. Capability 8.9 is ready on FLA chunk. Capability 9.0 uses all-sequence FlashInfer. The kept primary rotates 48 `cu_seqlens` buffers per device and length, passes `use_cp` false, skips the q/k repeat, and packs q and k before the L2-norm view. `801ef7b` fused the 48 linear gated RMSNorms and missed the median (p50 89.172 ms, p95 274.790 ms), so that kernel stays off. `2bbbe4e` replaced `silu(gate) * up` with FLA `swiglu_fwd` and the owner closeout rejected it (p50 97.613 ms, p95 292.795 ms), so that kernel stays off. Gate and up GEMMs stay separate. Residual RMSNorm and the gated output norm stay eager. An unknown `PPLX_GDN_PREFILL` fails closed on every device, including capability 8.9. `3ecdf5a` crashed in warmup (0/231). Always-on fusion `3afb472` and the 512-token gate `705ae089` missed the median. The 48-slot FlashInfer state ring `17a7b120` missed the gate. `PPLX_GDN_PREFILL=fla` reproduces the prior `f968e576` chunk. `PPLX_GDN_PREFILL=short` is the rejected ≤256 dispatch. SDPA stays on the checkpoint order. The fused gate-up GEMM is refused. The forward stays eager. `PPLX_COMPILE=dynamic` and `PPLX_COMPILE=dynamic-tail` are rejected opt-ins | Contract checks only; no inference | Reported load on 1× lab L20X (sm_89, ~140 GiB) at head `1ba07d1`: BF16 resident ~50333 MiB, no OOM. Historical primary: 231/231, p50 98.777 ms, p95 341.202 ms. Prior primary is the `f968e576` remeasure on the lab L20X-labeled GPU, `compute_capability=9.0`, `fla-chunk`, `flashinfer` null: 231/231, p50 94.077 ms, p95 300.200 ms (p50 −4.76%, p95 −12.02%). Prior primary `ffae182`: 231/231, p50 92.844 ms, p95 296.458 ms (p50 −1.31%, p95 −1.25%). Prior primary `4b99b73`: 231/231, p50 88.942 ms, p95 295.797 ms (p50 −4.20%, p95 −0.22%), 48-slot `cu_seqlens`, quality `choice_acc=0.9065` (139), `noul_acc=0.8784` (74), `score_mae=0.2035` (18). Kept primary is `fcf5e2b`: 231/231, p50 87.782 ms, p95 291.183 ms (p50 −1.30%, p95 −1.56%), grouped-value FlashInfer with q and k packed, same quality digits. Uncached all-sequence FlashInfer `3c72f02`: p50 88.808 ms, p95 300.751 ms, gate failed. Short-sequence FlashInfer `a50b9aa`: p50 95.446 ms, p95 300.871 ms, gate failed. cuDNN-first SDPA `0dcaa0ac`: p50 138.501 ms, p95 332.859 ms, gate failed. Fused gate-up `8b4ee326`: p50 102.437 ms, p95 304.970 ms, gate failed. Always-on dynamic compile `e36b32ed`: p50 97.751 ms, p95 263.124 ms, gate failed (p95 improved). 512-token compile `7e3b1be`: p50 100.037 ms, p95 257.084 ms, gate failed. Isolated-median compile `34ba5de`: p50 97.084 ms, p95 256.679 ms, gate failed (median probe was eager). FlashInfer state-workspace ring `17a7b120`: p50 89.165 ms, p95 295.723 ms, gate failed. Always-on FLA residual RMSNorm `3afb472`: p50 93.935 ms, p95 274.902 ms, gate failed. Length-gated FLA residual RMSNorm `705ae089`: p50 94.604 ms, p95 274.835 ms, gate failed. `use_cp=auto` `a6f4d9a`: p50 112.294 ms, p95 299.419 ms, gate failed. Grouped-value `3ecdf5a` crashed in warmup (0/231, no p50 or p95). The fused linear gated-norm candidate has no sample. Does not enter the H800 board | Not supported | Python 3.12, pinned checkpoint revision `14c9d90137962d300f3de95633f7b6535e80c5ce`, checkpoint `requirements.txt`, `causal-conv1d` 1.7.0, `flash-linear-attention` 0.5.2, optional `flashinfer-python` 0.6.16.post3. Shard sizes in the release manifest sum with the readout to about 48.6 GiB before activations | @@ -40,6 +40,7 @@ Models that are being added are also tracked in issues labeled [new model](https - **Planned:** not implemented yet; the linked issue tracks it. The Cua-S1 workers answer `choice` questions only. +LAYA's English worker, Open-Jev and CLM support `choice`, `score`, and `noul` text questions. LAYA's English worker, Open-Jev, CLM, and StartLux Instant 4B support `choice`, `score`, and `noul` text questions. StartLux serves Instant only; AutoThink is not implemented, and the weights are CC BY-NC 4.0. D3's reference worker serves the same three kinds for text, image and video diff --git a/mkdocs.yml b/mkdocs.yml index 86abd35a..da6fcd01 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -86,6 +86,7 @@ nav: - LFM2.5-350M choice worker: recipe/lfm2/README.md - D3 reference worker: recipe/d3/reference.md - Vela 2.0 reference workers: recipe/vela/reference.md + - Vela 2.0 0.8B native library: recipe/vela/native.md - Clef reference worker: recipe/clef/reference.md - pplx-decider reference worker: recipe/pplx_decider/README.md - Contributing: CONTRIBUTING.md diff --git a/recipe/README.md b/recipe/README.md index 22179d21..62f6bb3d 100644 --- a/recipe/README.md +++ b/recipe/README.md @@ -57,6 +57,9 @@ For a first real decision, follow the [complete CPU walkthrough](../docs/getting five SystemOne question kinds (`choice`/`noul`/`score` plus the `span`/`set` extensions) through the Python worker and the Rust frontend on CPU FP32; or vllm-sr/Vela-2.0-0.8B at revision `326b01d8` behind its own 10-file gate. +- [Vela 2.0 0.8B native library](vela/native.md): export the pinned-bundle + manifest gate, wire contract and token-level processing for the 0.8B hybrid, + checked against frozen fixtures; serving stays with the reference Python worker. - [Clef reference worker](clef/reference.md): build `omni-clef`, load pinned Clef-flash (9B) or Clef (27B), and serve text `choice`/`noul`/`score`. The joint head stays in the checkpoint's Python module, on one persistent diff --git a/recipe/vela/export_head_inputs.py b/recipe/vela/export_head_inputs.py new file mode 100644 index 00000000..b7c2ee82 --- /dev/null +++ b/recipe/vela/export_head_inputs.py @@ -0,0 +1,233 @@ +#!/usr/bin/env python3 +"""Freeze the head inputs/outputs of the pinned Vela 2.0 0.8B runtime. + +Records, per frozen case, exactly what +the two F32 heads consume and produce — CandidateHead (option-end +hidden states + the block query hidden) and SpanHeadV2 (word hidden +states + label-mean hiddens), on CPU FP32 with autocast off (the +heads' own setting, bit-deterministic) — so the Rust heads and the +scoring/decode mirror are checked against these without any GPU. Also +records the reference's calibrated predict answers for the same cases +(the scoring target). Hidden matrices go to a per-case safetensors +file; structure, small logits and answers go to JSON next to it. + + PYTHONPATH=src .venv/bin/python recipe/vela/export_head_inputs.py \ + --model weights/vela-2.0-0.8b --out tests/vela/fixtures/ +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +import numpy as np +from safetensors.numpy import save_file + +HERE = Path(__file__).resolve().parent +MODEL_NAME = "vllm-sr/Vela-2.0-0.8B" + + + +def parse_args(argv=None): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--model", required=True, help="pinned Vela 2.0 0.8B package directory") + parser.add_argument("--out", default=None, help="fixture output directory") + return parser.parse_args(argv) + + +def long_span_body(): + filler = " ".join(f"Sentence {i} reports a routine delivery event." for i in range(220)) + return { + "model": MODEL_NAME, + "state": {"request": filler}, + "questions": { + "claims": { + "type": "span", + "instructions": "Which spans mention a delivery?", + "criteria": {"delivery": "a delivery event"}, + "over": "request", + } + }, + } + + +def cases(): + named = [ + ("example-request", HERE / "example-request.json"), + ("example-request-set-threshold", HERE / "example-request-set-threshold.json"), + ("example-request-cjk", HERE / "example-request-cjk.json"), + ("example-request-score", HERE / "example-request-score.json"), + ] + loaded = [ + (name, {**json.loads(path.read_text(encoding="utf-8")), "model": MODEL_NAME}) + for name, path in named + ] + loaded.append(("long-span-windowed", long_span_body())) + return loaded + + +def plain(value): + if isinstance(value, dict): + return {key: plain(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [plain(item) for item in value] + if isinstance(value, np.ndarray): + return plain(value.tolist()) + if isinstance(value, (np.integer,)): + return int(value) + if isinstance(value, (np.floating,)): + return float(value) + if isinstance(value, (np.bool_,)): + return bool(value) + return value + + +class Recorder: + """Class-level wrappers around the two heads; records inputs/outputs.""" + + def __init__(self, module, model): + self.calls: list[dict] = [] + self.tensors: dict[str, np.ndarray] = {} + self._on = False + # Tensor names must never collide across rows: the counter keeps + # growing for the whole case. + self.next_index = 0 + self.module = module + self.model = model + original_head = module.CandidateHead.forward + original_span = module.SpanHeadV2.forward + recorder = self + + def head_forward(self, candidates, query): + out = original_head(self, candidates, query) + if recorder._on: + index = recorder.next_index + cname, qname = f"q{index}_cand", f"q{index}_query" + recorder.tensors[cname] = candidates.detach().cpu().numpy().astype(np.float32) + recorder.tensors[qname] = query.detach().cpu().numpy().astype(np.float32) + recorder.next_index = index + 1 + recorder.calls.append( + dict(kind="q", cand=cname, query=qname, + logits=[row.tolist() for row in out.detach().cpu().numpy()]) + ) + return out + + def span_forward(self, words, labels): + out = original_span(self, words, labels) + if recorder._on: + index = recorder.next_index + wname, lname = f"s{index}_words", f"s{index}_labels" + which = "broad" if self is getattr(recorder.model, "span_broad", None) else "router" + recorder.tensors[wname] = words.detach().cpu().numpy().astype(np.float32) + recorder.tensors[lname] = labels.detach().cpu().numpy().astype(np.float32) + recorder.next_index = index + 1 + recorder.calls.append( + dict(kind="span", head=which, words=wname, labels=lname, + logits=out.detach().cpu().numpy().tolist()) + ) + return out + + module.CandidateHead.forward = head_forward + module.SpanHeadV2.forward = span_forward + + +def rows_of(engine, body): + from vela2_inference import system_one_to_predict + + req, plan = system_one_to_predict(body["state"], body["questions"]) + parts, qs, _thr = engine._to_generic(req) + spans = [q for q in qs if q["type"] == "span"] + rest = [q for q in qs if q["type"] != "span"] + rows = [dict(parts=parts, questions=rest + spans[:1])] + [ + dict(parts=parts, questions=[s]) for s in spans[1:] + ] + rendered, preps, windows = [], [], [] + for row in rows: + if not row["questions"]: + continue + prepped = engine._prep(row) + rec = engine._render(prepped) + rec["head"] = prepped["head"] + rec["head_reason"] = prepped["head_reason"] + rec["label_alias"] = plain(prepped.get("label_alias")) + preps.append(prepped) + rendered.append(rec) + for wrow in engine._window_rows(prepped) or []: + wrec = engine._render(wrow) + wrec["head"] = prepped["head"] + windows.append(wrec) + return req, plan, rendered, windows + + +def main(argv=None) -> int: + args = parse_args(argv) + sys.path.insert(0, str(Path(args.model).resolve())) + # The package gate rejects bytecode caches. Cover initialization too: + # from_pretrained can import package modules lazily. + previous_dont_write_bytecode = sys.dont_write_bytecode + sys.dont_write_bytecode = True + try: + from vela2_inference import Vela2 + + engine = Vela2.from_pretrained(args.model, backend="torch", device="cpu") + import modeling_vela2 + + recorder = Recorder(modeling_vela2, engine.run.model) + finally: + sys.dont_write_bytecode = previous_dont_write_bytecode + out = Path(args.out) if args.out else HERE.parents[1] / "tests" / "vela" / "fixtures" + out.mkdir(parents=True, exist_ok=True) + + for name, body in cases(): + req, plan, rows, windows = rows_of(engine, body) + # One row per backend call so every recorded head call is + # attributable; question blocks arrive stacked in row order. + calls = [] + tensors: dict[str, np.ndarray] = {} + recorder.calls = [] + recorder.tensors = {} + recorder.next_index = 0 + recorder._on = True + row_of_call: list[int] = [] + try: + for index, rec in enumerate(rows + windows): + # Windows render after the main rows (row index continues). + engine.run([rec]) + for call in recorder.calls: + row_of_call.append(index) + calls.extend(recorder.calls) + tensors.update(recorder.tensors) + recorder.calls = [] + # The reference's calibrated answers for the same request. + predict = plain(engine._predict(req)[0]) + finally: + recorder._on = False + for call, row in zip(calls, row_of_call): + call["row"] = row + save_file(tensors, str(out / f"head-input-{name}.safetensors")) + fixture = { + "case": name, + "request": body, + # The rows block carries the span-head dispatch and alias + # provenance for human inspection; the tests read the + # rendered rows from the native-plan fixtures. + "rows": [ + {key: rec[key] for key in ("head", "head_reason", "label_alias", "n_tokens", "seg_lens") + if key in rec} + for rec in rows + ], + "head_calls": calls, + "predict": predict, + } + (out / f"head-input-{name}.json").write_text( + json.dumps(fixture, ensure_ascii=False) + "\n", encoding="utf-8" + ) + span_calls = sum(1 for call in calls if call["kind"] == "span") + print(f"{name}: {len(rows)} row(s), {len(calls)} head call(s) ({span_calls} span), predict answers: {len(predict['answers'])}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/recipe/vela/export_native.py b/recipe/vela/export_native.py new file mode 100644 index 00000000..9a3f3fb2 --- /dev/null +++ b/recipe/vela/export_native.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python3 +"""Export the pinned Vela 2.0 0.8B package into a native bundle. + +The bundle re-keys the hybrid +backbone for the shared qwen3_5 native loader — ``backbone.X`` tensors +become ``model.X`` (the loader's prefix probe accepts ``model.``), with +**per-tensor dtypes preserved verbatim** (Linear/conv1d in BF16, +embeddings/norms/A_log/dt_bias in FP32 — the reference's own mixed +layout, no re-save rounding) — and copies the Vela-owned F32 heads and +the side-loaded broad span head as separate files, plus calibration and +tokenizer. ``config.json`` is the package's ``backbone_config`` written +at the root, which is exactly the shape the family ``Config::load`` +reads. The manifest pins every source file by SHA-256 (re-using the +worker's own gate table) and records the exported tensor inventory. + + PYTHONPATH=src .venv/bin/python recipe/vela/export_native.py \ + --package weights/vela-2.0-0.8b --out weights/vela-2.0-0.8b-native +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import shutil +import sys +from pathlib import Path + +BACKBONE_PREFIX = "backbone." +HEAD_PREFIXES = ("head.", "span2.", "set_bias") +FORMAT = "vela2-native-bundle/1" + + +def parse_args(argv=None): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--package", required=True, help="pinned Vela 2.0 0.8B package directory") + parser.add_argument("--out", required=True, help="output bundle directory (created)") + parser.add_argument( + "--bf16-out", + default=None, + help="also write a uniform-BF16 backbone directory for the shared qwen3_5 " + "loader (model.safetensors + config.json)", + ) + return parser.parse_args(argv) + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(8 * 1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def stage_into_place(target: Path, write_files) -> None: + """Stage the artifact set in a sibling directory and move it into place + with one rename, so the target only ever holds a complete export: a run + killed mid-write leaves at most the staging directory, which the next + run with the same target removes.""" + if target.exists() and any(target.iterdir()): + raise SystemExit(f"output directory {target} already exists and is not empty") + staging = target.with_name(target.name + ".export-partial") + if staging.is_dir(): + # Remains of a run killed mid-write with the same target directory. + shutil.rmtree(staging) + staging.mkdir(parents=True) + try: + write_files(staging) + if target.exists(): + # Empty, checked above; the rename replaces the directory whole. + target.rmdir() + os.replace(staging, target) + except BaseException: + shutil.rmtree(staging, ignore_errors=True) + raise + + +def main(argv=None) -> int: + args = parse_args(argv) + sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "src")) + import torch + from safetensors.torch import load_file, save_file + + from models.vela.hybrid import ( + HYBRID_REPO, + HYBRID_REVISION, + PINNED_FILES_SHA256, + verify_package, + ) + + package = Path(args.package) + out = Path(args.out) + verify_package(package) # the worker's own gate: provenance before export + config = json.loads((package / "config.json").read_text(encoding="utf-8")) + backbone_config = config["backbone_config"] + + # The shard map names every backbone and head tensor; broad_head is a + # separate side-loaded file. + index = json.loads((package / "model.safetensors.index.json").read_text(encoding="utf-8")) + weight_map: dict[str, str] = index["weight_map"] + shard_files = sorted(set(weight_map.values())) + tensors = {} + for name in shard_files: + tensors.update(load_file(str(package / name))) + broad = load_file(str(package / "broad_head.safetensors")) + + backbone, heads = {}, {} + dtypes: dict[str, dict[str, str]] = {"backbone": {}, "heads": {}} + for name, tensor in sorted(weight_map.items()): + value = tensors[name] + dtype = str(value.dtype).removeprefix("torch.") + if name.startswith(BACKBONE_PREFIX): + rekeyed = "model." + name[len(BACKBONE_PREFIX):] + backbone[rekeyed] = value + dtypes["backbone"][rekeyed] = f"{dtype}:{list(value.shape)}" + elif name.startswith(HEAD_PREFIXES): + heads[name] = value + dtypes["heads"][name] = f"{dtype}:{list(value.shape)}" + else: + raise SystemExit(f"unexpected tensor key {name!r}; update the export for the new prefix") + for name, value in sorted(broad.items()): + heads[name] = value + dtypes["heads"][name] = f"{str(value.dtype).removeprefix('torch.')}:{list(value.shape)}" + + def write_bundle(out: Path) -> None: + save_file(backbone, str(out / "backbone.safetensors")) + save_file(heads, str(out / "heads.safetensors")) + # config.json for the family loader: backbone_config at the root. + (out / "config.json").write_text(json.dumps(backbone_config, indent=1) + "\n", encoding="utf-8") + for name in ("calibration.json", "tokenizer.json", "tokenizer_config.json"): + shutil.copyfile(package / name, out / name) + for name in [ + "backbone.safetensors", "heads.safetensors", "config.json", + "calibration.json", "tokenizer.json", "tokenizer_config.json", + ]: + manifest["files_sha256"][name] = sha256(out / name) + (out / "vela_native_manifest.json").write_text( + json.dumps(manifest, indent=1, sort_keys=True) + "\n", encoding="utf-8" + ) + + bf16 = sum(1 for d in dtypes["backbone"].values() if d.startswith("bfloat16")) + fp32 = len(dtypes["backbone"]) - bf16 + manifest = { + "format": FORMAT, + "model_id": "vllm-sr/Vela-2.0-0.8B", + "model_name": config["model_name"], + "max_length": config["max_length"], + "description": config.get("description", ""), + "source": { + "repo": HYBRID_REPO, + "revision": HYBRID_REVISION, + # The worker gate's own table, re-verified above. + "pinned_files_sha256": dict(PINNED_FILES_SHA256), + "shard_files": shard_files, + }, + "backbone": { + "tensors": len(backbone), + "bf16": bf16, + "fp32": fp32, + "note": "per-tensor dtypes preserved from the package; backbone.X re-keyed to model.X", + "inventory": dtypes["backbone"], + }, + "heads": { + "tensors": len(heads), + "inventory": dtypes["heads"], + "note": "Vela-owned F32 heads (CandidateHead, set_bias, span2 router, span_broad) in one file", + }, + "files_sha256": {}, + } + + stage_into_place(out, write_bundle) + + if args.bf16_out: + # Uniform-BF16 re-save for the family crate's loader: every + # backbone tensor rounds to bfloat16 once, at export (the + # reference's own GPU setting runs the backbone under bf16 + # autocast too). + uniform = {name: value.to(torch.bfloat16) for name, value in backbone.items()} + + def write_bf16(bf16_dir: Path) -> None: + save_file(uniform, str(bf16_dir / "model.safetensors")) + (bf16_dir / "config.json").write_text(json.dumps(backbone_config, indent=1) + "\n", encoding="utf-8") + + stage_into_place(Path(args.bf16_out), write_bf16) + print(f"bf16 backbone: {len(uniform)} tensors -> {args.bf16_out}") + + print(f"export ok: {out}") + print(f" backbone: {len(backbone)} tensors ({bf16} bf16, {fp32} fp32), re-keyed {BACKBONE_PREFIX} -> model.") + print(f" heads: {len(heads)} tensors (F32, incl. span_broad)") + print(f" config: backbone_config at root, model_type={backbone_config.get('model_type')}") + # The load-side gate anchors on the manifest's own digest, so the + # file table can no longer vouch for itself; the loader needs this + # value from outside the bundle directory. + manifest_sha = sha256(out / "vela_native_manifest.json") + print(f" manifest: SHA-256 {manifest_sha}") + print(f" loaders must pass it as the anchor, e.g.") + print(f" VELA_NATIVE_MANIFEST_SHA256={manifest_sha}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/recipe/vela/export_token_plan.py b/recipe/vela/export_token_plan.py new file mode 100644 index 00000000..7a75f9f8 --- /dev/null +++ b/recipe/vela/export_token_plan.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 +"""Freeze the token-level row plan of the pinned Vela 2.0 0.8B runtime. + +This tool records, for a set of request +bodies, exactly what the pinned runtime feeds the backbone — the +SystemOne → predict mapping, the per-row decomposition (parts pass + +question blocks; one span question per row), and the rendered records +(``p_ids``, block token ids with option/label end positions, query +positions, span word positions, truncation and segment lengths). The +native Rust processor must reproduce these byte-for-byte; the frozen +files under ``tests/vela/fixtures/native-plan-*.json`` are that parity +target. Model outputs are not recorded here — the oracle fixtures from +``compare_reference.py`` own the output side. + + PYTHONPATH=src .venv/bin/python recipe/vela/export_token_plan.py \ + --model weights/vela-2.0-0.8b --out tests/vela/fixtures/ +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +import numpy as np + +HERE = Path(__file__).resolve().parent +MODEL_NAME = "vllm-sr/Vela-2.0-0.8B" + + +def parse_args(argv=None): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--model", required=True, help="pinned Vela 2.0 0.8B package directory") + parser.add_argument("--out", default=None, help="fixture output directory") + return parser.parse_args(argv) + + +def long_span_body(): + """A span target beyond HYBRID_RMAX (2,048 tokens) to exercise the + windowed layout: the full-render span block is skipped and one row + per 1,800-token window (stride 1,536, cut at word starts) carries + the span question.""" + + filler = " ".join(f"Sentence {i} reports a routine delivery event." for i in range(220)) + return { + "model": MODEL_NAME, + "state": {"request": filler}, + "questions": { + "claims": { + "type": "span", + "instructions": "Which spans mention a delivery?", + "criteria": {"delivery": "a delivery event"}, + "over": "request", + } + }, + } + + +def cases(): + named = [ + ("example-request", HERE / "example-request.json"), + ("example-request-set-threshold", HERE / "example-request-set-threshold.json"), + ("example-request-cjk", HERE / "example-request-cjk.json"), + ("example-request-score", HERE / "example-request-score.json"), + ] + loaded = [ + ( + name, + {**json.loads(path.read_text(encoding="utf-8")), "model": MODEL_NAME}, + ) + for name, path in named + ] + loaded.append(("long-span-windowed", long_span_body())) + return loaded + + +def plain(value): + """numpy-free JSON for the rendered records.""" + + if isinstance(value, dict): + return {key: plain(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [plain(item) for item in value] + if isinstance(value, np.ndarray): + return plain(value.tolist()) + if isinstance(value, (np.integer,)): + return int(value) + if isinstance(value, (np.floating,)): + return float(value) + if isinstance(value, (np.bool_,)): + return bool(value) + return value + + +def rows_of(engine, body): + """Mirror ``Vela2._predict``'s row decomposition and render each row.""" + + from vela2_inference import system_one_to_predict + + req, plan = system_one_to_predict(body["state"], body["questions"]) + parts, qs, _thr = engine._to_generic(req) + spans = [q for q in qs if q["type"] == "span"] + rest = [q for q in qs if q["type"] != "span"] + rows = [dict(parts=parts, questions=rest + spans[:1])] + [ + dict(parts=parts, questions=[s]) for s in spans[1:] + ] + rendered, windows = [], [] + for row in rows: + if not row["questions"]: + continue + prepped = engine._prep(row) + rec = plain(engine._render(prepped)) + rec["head"] = prepped["head"] + rec["head_reason"] = prepped["head_reason"] + rendered.append(rec) + for wrow in engine._window_rows(prepped) or []: + wrec = plain(engine._render(wrow)) + wrec["head"] = prepped["head"] + windows.append(wrec) + return plain(req), plain(plan), rendered, windows + + +def main(argv=None) -> int: + args = parse_args(argv) + sys.path.insert(0, str(Path(args.model).resolve())) + # The package gate rejects bytecode caches. Cover initialization too: + # from_pretrained can import package modules lazily. + previous_dont_write_bytecode = sys.dont_write_bytecode + sys.dont_write_bytecode = True + try: + from vela2_inference import Vela2 + + engine = Vela2.from_pretrained(args.model, backend="torch", device="cpu") + finally: + sys.dont_write_bytecode = previous_dont_write_bytecode + out = Path(args.out) if args.out else HERE.parents[1] / "tests" / "vela" / "fixtures" + out.mkdir(parents=True, exist_ok=True) + for name, body in cases(): + req, plan, rows, windows = rows_of(engine, body) + fixture = { + "case": name, + "request": body, + "predict_request": req, + "plan": plan, + "rows": rows, + "window_rows": windows, + } + target = out / f"native-plan-{name}.json" + target.write_text(json.dumps(fixture, ensure_ascii=False) + "\n", encoding="utf-8") + tokens = sum(r["n_tokens"] for r in rows) + sum(r["n_tokens"] for r in windows) + print( + f"{name}: {len(rows)} row(s) + {len(windows)} window row(s), " + f"{tokens} tokens -> {target.name}" + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/recipe/vela/native.md b/recipe/vela/native.md new file mode 100644 index 00000000..4713aa4f --- /dev/null +++ b/recipe/vela/native.md @@ -0,0 +1,109 @@ +# Vela 2.0 0.8B native library + +The native implementation for +[vllm-sr/Vela-2.0-0.8B](https://huggingface.co/vllm-sr/Vela-2.0-0.8B): +the exported-bundle gate, the wire contract, and the token-level +processing — all mirrored from the pinned runtime and checked against +frozen fixtures. This is a library; serving runs through the +[reference recipe](reference.md)'s Python worker. + +Run the commands from the repository root with the same `.venv` and +pinned package as the reference recipe. The export needs about 6 GB of +free disk next to the package; the checks need no GPU. + +## Export the native bundle + +~~~sh +PYTHONPATH=src .venv/bin/python recipe/vela/export_native.py \ + --package weights/vela-2.0-0.8b --out weights/vela-2.0-0.8b-native +~~~ + +The backbone that executes on CUDA is a uniform-BF16 re-save (every +backbone tensor rounds once at export); the reference's own GPU setting +runs the backbone under bf16 autocast but keeps norm weights and GDN +gate math in FP32, so the uniform export is a slightly lower-precision +profile than either the CPU-FP32 recording or the reference GPU path. +The Vela heads stay F32 on the faithful tensors. + +The export re-runs the reference worker's package gate first +(provenance), then writes `weights/vela-2.0-0.8b-native/`: the backbone +re-keyed `backbone.` → `model.` with **per-tensor dtypes preserved** +(204 BF16 Linear/conv1d, 116 F32 embed/norms/A_log/dt_bias — the +reference's own mixed layout), the 37 F32 heads in `heads.safetensors` +(including the side-loaded broad span head), `backbone_config` at the +config root (the shared qwen3_5 `Config::load` shape), calibration and +tokenizer, and `vela_native_manifest.json` pinning the source +revision, the gate's file table, the tensor inventory and every file's +SHA-256. The set is staged in a sibling directory and moved into place +with one rename, so a run killed mid-write can no longer strand a +partial bundle under `--out`; a non-empty target directory is refused +and the staging remnant is cleared by the next run. + +The export ends by printing the manifest's own SHA-256. The load-side +gate requires it as an anchor (`Bundle::load`/`Processor::load` take it +as `pinned_manifest_sha256`; the opt-in checks read +`VELA_NATIVE_MANIFEST_SHA256`), so the manifest's file table is checked +against a digest fixed outside the bundle directory — rewriting the +manifest after swapping a file no longer passes the gate. `files_sha256` +keys must also be plain relative names inside the bundle and must not +cross a symlink. + +## Freeze the token-plan fixtures (opt-in, needs the package) + +~~~sh +PYTHONPATH=src .venv/bin/python recipe/vela/export_token_plan.py \ + --model weights/vela-2.0-0.8b --out tests/vela/fixtures/ +PYTHONPATH=src .venv/bin/python recipe/vela/export_head_inputs.py \ + --model weights/vela-2.0-0.8b --out tests/vela/fixtures/ +~~~ + +The token plans record what the pinned runtime feeds the backbone (the +predict mapping, the per-row decomposition, the rendered `p_ids`, +blocks and readout positions, truncation, a windowed long-span case). +The example-request token plan totals 1,041 tokens, matching the +oracle fixture's `usage.input_tokens`. The head inputs record the +hidden states the F32 heads consume, the heads' logits and the +calibrated predict answers. Both exporters import and initialize the +runtime with bytecode writing off, so the verified package stays +reusable: a later `verify_files()` gate run or worker startup is not +poisoned by a `__pycache__` an export wrote. + +## Checks + +~~~sh +cargo test -p omni-vela-native +cargo fmt --all --check +cargo clippy -p omni-vela-native --all-targets -- -D warnings +VELA_NATIVE_BUNDLE="$PWD/weights/vela-2.0-0.8b-native" \ + VELA_NATIVE_MANIFEST_SHA256= \ + cargo test -p omni-vela-native --test vela_native_processing -- --ignored --nocapture +VELA_NATIVE_BUNDLE="$PWD/weights/vela-2.0-0.8b-native" \ + cargo test -p omni-vela-native --test vela_native_heads -- --ignored --nocapture +~~~ + +31 tests run everywhere with no weights and no GPU: the wire contract +against the pinned runtime's 422/400 bodies, the bundle manifest gate, +and the predict-mapping/plan parity on every frozen fixture. The +`#[ignore]`d opt-in checks (run them with `-- --ignored`, as above) +replay the fixtures through the Rust implementation: + +- render parity — the token plans match field-for-field (5/5); +- head parity — head-logit drift <= 3.8e-5, predict-answer drift + <= 7.3e-7, discrete picks and span structures equal (5/5). For a field-level diff of one case: + +~~~sh +VELA_NATIVE_BUNDLE="$PWD/weights/vela-2.0-0.8b-native" \ + cargo run -p omni-vela-native --example parity -- example-request +~~~ + +## Scope and limits + +- Implemented: the bundle export + manifest gate, the wire admission, + the token-level processing (state mapping, predict mapping with the + validation triples, word units, span-head dispatch, the DEV2.0 + rendering, the truncation budget, window rows), and the F32 heads + with scoring and span decoding (CandidateHead, both SpanHeadV2 + heads, set_bias, temperatures, the PII length rule and sparse gate, + `span_decode_r4`, label aliasing, window aggregation). +- Not implemented: backbone execution and an HTTP worker. No + performance claims are made. diff --git a/src/models/vela/README.md b/src/models/vela/README.md index 74f2c4bb..7ea07edc 100644 --- a/src/models/vela/README.md +++ b/src/models/vela/README.md @@ -225,3 +225,50 @@ the shared worker-contract issue, and nothing is posted upstream. a deployment option outside these slices' evidence. - The serving procedure, hardware requirements and the recorded evidence pointer are [recipe/vela/reference.md](../../recipe/vela/reference.md). + +## Native path (0.8B) + +The native implementation lives in [`native/`](native/) as the +`omni-vela-native` crate, a library around the exported bundle: + +- [`recipe/vela/export_native.py`](../../recipe/vela/export_native.py) + exports the pinned package into a native bundle: the 320 backbone + tensors re-keyed `backbone.` → `model.` with **per-tensor dtypes + preserved** (204 BF16 Linear/conv1d + 116 F32 embed/norms/A_log/ + dt_bias — the reference's own layout), the 37 F32 head tensors + (CandidateHead, `set_bias`, span2 router, span_broad) in one file, + `backbone_config` at the config root, calibration and tokenizer + copied, and a manifest pinning the source revision, the worker + gate's own file table, the tensor inventory and every bundle file's + SHA-256. +- `native/src/bundle.rs` re-verifies that manifest (revision, + inventory counts, per-file hashes) before anything loads it. +- `native/src/contract.rs` is the wire admission, mirroring + `protocol.py` and the pinned runtime's preamble with byte-equal + 422/400 bodies. +- `native/src/{state,questions,words,render,processing}.rs` are the + token-level processing: the state → parts mapping, the SystemOne → + predict mapping with the pinned runtime's validation triples (Noul + folded into a two-option choice; choice renders `[ABS]`, the + reference's abstain default), the GLiNER2/CJK word splitter, the + six-rule span-head dispatch with the hallucination-alias label + replacement, the DEV2.0 prompt rendering (parts pass, question + blocks with option end and query positions, span layout a/c with + word positions, the truncation budget, the 16,384-window 413) and + the long-target window rows. All five frozen token plans + (`tests/vela/fixtures/native-plan-*.json`) match field-for-field. + +- `native/src/{heads,scoring}.rs` are the F32 heads and the scoring + path: CandidateHead, both SpanHeadV2 heads, `set_bias`, the + per-type temperatures, set thresholds, the PII length rule and + sparse gate, `span_decode_r4` (word units, unit vote, fill, + edge/URL trimming), label aliasing, score normalization and the + window aggregation. Replaying the recorded head-input fixtures + (`tests/vela/fixtures/head-input-*`): head-logit drift ≤ 3.8e-5, + predict-answer drift ≤ 7.3e-7, discrete picks and span structures + equal on all five cases. + +Backbone execution and an HTTP worker are not implemented. Checks: +`cargo test -p omni-vela-native` (27 tests, no weights, no GPU; +render and head parity opt-in with `VELA_NATIVE_BUNDLE`; see +[recipe/vela/native.md](../../recipe/vela/native.md)). diff --git a/src/models/vela/native/Cargo.toml b/src/models/vela/native/Cargo.toml new file mode 100644 index 00000000..91ff220b --- /dev/null +++ b/src/models/vela/native/Cargo.toml @@ -0,0 +1,40 @@ +[package] +name = "omni-vela-native" +version = "0.1.0" +edition = "2024" +publish = false +description = "Native library for vllm-sr/Vela-2.0-0.8B: bundle gate, wire contract, processing, F32 heads and scoring" + +[dependencies] +anyhow = "1.0.100" +libm = "0.2" +safetensors = "0.8.0" +onig = { package = "onig", version = "6", default-features = false } +omni-qwen3-5-native = { path = "../../qwen3_5/native" } +serde = { version = "1", features = ["derive"] } +tokenizers = { version = "=0.22.2", default-features = false, features = ["onig"] } +serde_json = { version = "1.0.149", features = ["float_roundtrip", "preserve_order"] } +sha2 = "0.11" + +[lib] +path = "src/lib.rs" + +[[test]] +name = "vela_native_contract" +path = "../../../../tests/vela/native_contract.rs" + +[[test]] +name = "vela_native_bundle" +path = "../../../../tests/vela/native_bundle.rs" + +[[test]] +name = "vela_native_processing" +path = "../../../../tests/vela/native_processing.rs" + +[[test]] +name = "vela_native_units" +path = "../../../../tests/vela/native_units.rs" + +[[test]] +name = "vela_native_heads" +path = "../../../../tests/vela/native_heads.rs" diff --git a/src/models/vela/native/examples/parity.rs b/src/models/vela/native/examples/parity.rs new file mode 100644 index 00000000..3c8103ab --- /dev/null +++ b/src/models/vela/native/examples/parity.rs @@ -0,0 +1,90 @@ +//! Debug helper: compare the rendered rows against one fixture and +//! print the first differing field. Run with: +//! +//! VELA_NATIVE_BUNDLE=/abs/path VELA_NATIVE_MANIFEST_SHA256= \ +//! cargo run -p omni-vela-native --example parity -- example-request + +use std::path::PathBuf; + +use serde_json::Value; + +fn main() { + let case = std::env::args().nth(1).unwrap_or("example-request".into()); + let bundle = PathBuf::from( + std::env::var("VELA_NATIVE_BUNDLE").expect("VELA_NATIVE_BUNDLE points at the export"), + ); + let anchor = std::env::var("VELA_NATIVE_MANIFEST_SHA256") + .expect("VELA_NATIVE_MANIFEST_SHA256 is the manifest digest the exporter prints"); + let fixture: Value = serde_json::from_str( + &std::fs::read_to_string( + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../../../../tests/vela/fixtures") + .join(format!("native-plan-{case}.json")), + ) + .expect("fixture"), + ) + .expect("fixture json"); + + let processor = omni_vela_native::processing::Processor::load(&bundle, &anchor).expect("load"); + let request = fixture["request"].as_object().expect("request"); + match processor.plan(request) { + Ok(planned) => { + let got = serde_json::json!({ + "rows": serde_json::to_value(&planned.rows).unwrap(), + "window_rows": serde_json::to_value(&planned.window_rows).unwrap(), + }); + compare( + "rows", + got["rows"].as_array().unwrap(), + fixture["rows"].as_array().unwrap(), + ); + if !fixture["window_rows"].is_null() { + compare( + "window_rows", + got["window_rows"].as_array().unwrap(), + fixture["window_rows"].as_array().unwrap(), + ); + } + println!("parity: no field differences found"); + } + Err(error) => println!( + "plan error: {}", + serde_json::to_string(&error.body).unwrap() + ), + } +} + +fn compare(name: &str, got: &[Value], want: &[Value]) { + println!("{name}: {} vs {} row(s)", got.len(), want.len()); + for (index, (a, b)) in got.iter().zip(want).enumerate() { + let (Some(a), Some(b)) = (a.as_object(), b.as_object()) else { + continue; + }; + for key in b.keys() { + if a.get(key) != b.get(key) { + println!("row {index} field {key}: differs"); + println!( + " want: {}", + serde_json::to_string(&b[key]) + .unwrap() + .chars() + .take(300) + .collect::() + ); + println!( + " got: {}", + serde_json::to_string(&a[key]) + .unwrap() + .chars() + .take(300) + .collect::() + ); + } + } + for key in a.keys() { + if !b.contains_key(key) { + println!("row {index} field {key}: extra in ours"); + } + } + } +} diff --git a/src/models/vela/native/src/bundle.rs b/src/models/vela/native/src/bundle.rs new file mode 100644 index 00000000..89addf83 --- /dev/null +++ b/src/models/vela/native/src/bundle.rs @@ -0,0 +1,185 @@ +//! The exported native bundle's manifest gate. +//! +//! [`recipe/vela/export_native.py`](../../../recipe/vela/export_native.py) +//! produces the bundle from the pinned package; this module loads its +//! manifest, pins the source revision and the member's tensor +//! inventory, and re-hashes every bundle file before the (future) +//! executor touches it — the same provenance-before-import discipline +//! as the reference worker's package gate. + +use std::collections::HashSet; +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result, ensure}; +use serde_json::Value; + +pub const FORMAT: &str = "vela2-native-bundle/1"; +pub const MODEL_ID: &str = "vllm-sr/Vela-2.0-0.8B"; +pub const SOURCE_REVISION: &str = "326b01d81f61fff7fd4ae570e54b87a49119e490"; +/// This member's exported inventory, pinned from the pinned package: +/// 320 backbone tensors (204 BF16 Linear/conv1d + 116 F32) and 37 F32 +/// head tensors (CandidateHead, set_bias, span2 router, span_broad). +pub const BACKBONE_TENSORS: (usize, usize, usize) = (320, 204, 116); +pub const HEAD_TENSORS: usize = 37; + +#[derive(Debug)] +pub struct Bundle { + pub dir: PathBuf, + pub manifest: Value, +} + +impl Bundle { + /// Load and fully verify one exported bundle directory. + /// + /// `pinned_manifest_sha256` anchors the whole integrity chain: the + /// manifest's own bytes must hash to it, so the `files_sha256` table + /// is checked against a value fixed outside the bundle directory + /// (the exporter prints it) instead of against itself. + pub fn load(dir: &Path, pinned_manifest_sha256: &str) -> Result { + let manifest_bytes = std::fs::read(dir.join("vela_native_manifest.json")) + .context("export the pinned package first; see recipe/vela/export_native.py")?; + ensure!( + hash_bytes(&manifest_bytes) == pinned_manifest_sha256, + "vela_native_manifest.json does not match the pinned SHA-256 anchor; \ + re-export the pinned package and re-pin the anchor it prints" + ); + let manifest: Value = serde_json::from_slice(&manifest_bytes) + .context("vela_native_manifest.json must be JSON")?; + ensure!( + manifest["format"] == FORMAT + && manifest["model_id"] == MODEL_ID + && manifest["source"]["revision"] == SOURCE_REVISION, + "expected a bundle exported from the pinned {MODEL_ID} package" + ); + let backbone = &manifest["backbone"]; + ensure!( + backbone["tensors"].as_u64() == Some(BACKBONE_TENSORS.0 as u64) + && backbone["bf16"].as_u64() == Some(BACKBONE_TENSORS.1 as u64) + && backbone["fp32"].as_u64() == Some(BACKBONE_TENSORS.2 as u64), + "unexpected backbone inventory; re-export from the pinned package" + ); + ensure!( + manifest["heads"]["tensors"].as_u64() == Some(HEAD_TENSORS as u64), + "unexpected head inventory; re-export from the pinned package" + ); + let files = manifest["files_sha256"] + .as_object() + .context("files_sha256 must be an object")?; + ensure!( + !files.is_empty(), + "files_sha256 pins no files; the gate must cover the whole bundle" + ); + for (name, digest) in files { + let path = pinned_path(dir, name)?; + let actual = hash_file(&path)?; + ensure!( + actual == digest.as_str().unwrap_or(""), + "{name} does not match the manifest SHA-256" + ); + } + // Fail closed on the directory, as the reference gate walks the + // package root: the export writes a flat bundle, so the + // directory must hold exactly the pinned files plus this + // manifest — nothing unlisted, at any depth. + let pinned: HashSet<&str> = files + .keys() + .map(String::as_str) + .chain(std::iter::once("vela_native_manifest.json")) + .collect(); + let mut unpinned: Vec = std::fs::read_dir(dir) + .with_context(|| format!("cannot list the bundle directory {}", dir.display()))? + .map(|entry| { + entry + .expect("a readable bundle entry") + .file_name() + .to_string_lossy() + .into_owned() + }) + .filter(|name| !pinned.contains(name.as_str())) + .collect(); + unpinned.sort(); + ensure!( + unpinned.is_empty(), + "unpinned file(s) in the bundle directory: {unpinned:?}" + ); + ensure!( + manifest["max_length"].as_u64() == Some(16_384) + && manifest["model_name"].as_str() == Some(MODEL_ID), + "unexpected member identity in the manifest" + ); + Ok(Self { + dir: dir.to_owned(), + manifest, + }) + } + + /// The served model name (upstream: config.json `model_name`). + pub fn model_name(&self) -> &str { + self.manifest["model_name"].as_str().unwrap_or(MODEL_ID) + } + + /// The member's input window. + pub fn max_length(&self) -> usize { + self.manifest["max_length"].as_u64().unwrap_or(16_384) as usize + } +} + +/// Resolve one `files_sha256` key inside the bundle directory: the key +/// must be a plain relative path (no absolute prefix, no `.`/`..` +/// components) and must not cross a symlink at any level — otherwise a +/// crafted manifest could make the gate hash files outside the bundle. +fn pinned_path(dir: &Path, name: &str) -> Result { + ensure!(!name.is_empty(), "files_sha256 pins an empty name"); + let relative = Path::new(name); + ensure!( + !relative.is_absolute(), + "files_sha256 key must be a plain relative name: {name:?}" + ); + let mut checked = dir.to_path_buf(); + for component in relative.components() { + let part = match component { + std::path::Component::Normal(part) => part, + _ => anyhow::bail!("files_sha256 key must be a plain relative name: {name:?}"), + }; + checked.push(part); + let meta = std::fs::symlink_metadata(&checked) + .with_context(|| format!("cannot stat {}", checked.display()))?; + ensure!( + !meta.file_type().is_symlink(), + "files_sha256 key must not cross a symlink: {name:?}" + ); + } + Ok(checked) +} + +fn hash_file(path: &Path) -> Result { + use anyhow::bail; + use sha2::{Digest, Sha256}; + use std::io::Read; + + let mut hasher = Sha256::new(); + let file = match std::fs::File::open(path) { + Ok(file) => file, + Err(error) => bail!("cannot read {}: {error}", path.display()), + }; + let mut reader = std::io::BufReader::new(file); + let mut chunk = vec![0u8; 8 * 1024 * 1024]; + loop { + let read = reader.read(&mut chunk)?; + if read == 0 { + break; + } + hasher.update(&chunk[..read]); + } + Ok(hex(&hasher.finalize())) +} + +/// Lower-hex of a digest; sha2 0.11's output has no `LowerHex` impl. +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|byte| format!("{byte:02x}")).collect() +} + +fn hash_bytes(bytes: &[u8]) -> String { + use sha2::{Digest, Sha256}; + hex(&Sha256::digest(bytes)) +} diff --git a/src/models/vela/native/src/contract.rs b/src/models/vela/native/src/contract.rs new file mode 100644 index 00000000..275f4cd1 --- /dev/null +++ b/src/models/vela/native/src/contract.rs @@ -0,0 +1,139 @@ +//! Vela 2.0 wire admission for the native worker, mirroring +//! `models.vela.protocol` and the pinned runtime's request preamble so +//! clients cannot tell this layer from the reference worker: the +//! three-field envelope with upstream's exact `{"detail": [{"loc", +//! "msg", "type"}]}` triples, and the whole-request error model. +//! +//! Deeper validation — state rendering, question semantics, the 413 — +//! lives in the processing modules and mirrors the pinned runtime +//! against the `native-plan-*` fixtures. One known residual +//! difference: a non-finite JSON number answers the generic 400 +//! message here, where the Python worker names the value; duplicate +//! keys are mirrored verbatim. + +use serde_json::{Map, Value, json}; + +/// Slice-scoped transport cap, as on the reference worker. +pub const MAX_BODY: usize = 8 * 1024 * 1024; +/// The envelope fields, in upstream's missing-field check order. +pub const REQUEST_FIELDS: [&str; 3] = ["state", "model", "questions"]; + +/// A request rejection: an HTTP status and the JSON body a client sees. +#[derive(Debug, PartialEq)] +pub struct RequestError { + pub status: u16, + pub body: Value, +} + +impl RequestError { + pub(crate) fn detail(status: u16, message: &str) -> Self { + Self { + status, + body: json!({"detail": message}), + } + } + + /// One upstream `SystemOneError` as a whole-request 422 triple. + pub fn envelope(loc: &[&str], msg: &str, type_: &str) -> Self { + let mut path = vec![Value::from("body")]; + path.extend(loc.iter().map(|part| Value::from(*part))); + Self { + status: 422, + body: json!({"detail": [{"loc": path, "msg": msg, "type": type_}]}), + } + } +} + +/// Decode a bounded JSON body and reject ambiguous JSON values. +pub fn parse_body(raw: &[u8]) -> Result, RequestError> { + if raw.len() > MAX_BODY { + return Err(RequestError::detail(413, "request body exceeds 8 MiB")); + } + let message = + omni_qwen3_5_native::json::parse(raw).map_err(|error| normalize_parse_error(&error))?; + Ok(message) +} + +/// Mirror the reference worker's 400 messages: duplicate keys and +/// non-object bodies verbatim, everything else the shared generic +/// message. +pub(crate) fn normalize_parse_error(error: &str) -> RequestError { + if let Some(rest) = error.split("duplicate key ").nth(1) { + // serde appends its position suffix after the quoted key. + if let Some(key) = rest + .strip_prefix('"') + .and_then(|tail| tail.split('"').next()) + { + return RequestError::detail(400, &format!("duplicate JSON key '{key}'")); + } + } + // The shared parser's exact non-object error, answered with the + // reference worker's wording. + if error == "request body must be a JSON object" { + return RequestError::detail(400, "request must be a JSON object"); + } + RequestError::detail(400, "request body must contain valid JSON and UTF-8 text") +} + +/// Admit the envelope exactly as upstream's request preamble does, for +/// one served model name: the three required fields present, no extra +/// field, `model` non-blank and equal to the served name. +pub fn admit_request(body: &Map, served: &str) -> Result<(), RequestError> { + for field in REQUEST_FIELDS { + if !body.contains_key(field) { + return Err(RequestError::envelope( + &[field], + "Field required", + "missing", + )); + } + } + let mut extra: Vec<&String> = body + .keys() + .filter(|key| !REQUEST_FIELDS.contains(&key.as_str())) + .collect(); + extra.sort(); + if let Some(first) = extra.first() { + return Err(RequestError::envelope( + &[first], + "Extra inputs are not permitted", + "extra_forbidden", + )); + } + let model = body["model"].as_str().unwrap_or(""); + if model.trim().is_empty() { + return Err(RequestError::envelope( + &["model"], + "model must not be empty or whitespace", + "value_error", + )); + } + if model != served { + return Err(RequestError::envelope( + &["model"], + "Model is not served by this runtime", + "value_error.model", + )); + } + Ok(()) +} + +/// Admit one complete raw request body: parse, then envelope-check. +pub fn compile(raw: &[u8], served: &str) -> Result, RequestError> { + let body = parse_body(raw)?; + admit_request(&body, served)?; + Ok(body) +} + +/// A pinned-runtime `ValueError` rejection as the reference worker maps +/// it (`map_runtime_error`): a whole-request 422 naming the runtime. +pub fn runtime_rejection(message: &str) -> RequestError { + RequestError::detail( + 422, + &format!("request rejected by the pinned runtime: {message}"), + ) +} + +#[cfg(test)] +#[path = "../../../../../tests/vela/native_contract_unit.rs"] +mod tests; diff --git a/src/models/vela/native/src/heads.rs b/src/models/vela/native/src/heads.rs new file mode 100644 index 00000000..3ec0e817 --- /dev/null +++ b/src/models/vela/native/src/heads.rs @@ -0,0 +1,291 @@ +//! The Vela-owned F32 heads, mirrored from the checkpoint's +//! `CandidateHead` and `SpanHeadV2` (the reference computes both with +//! autocast disabled, in float32 — deterministic math). Tensors load +//! from the exported bundle's `heads.safetensors` (37 F32 tensors: +//! CandidateHead, `set_bias`, the router `span2.*` and the side-loaded +//! `span_broad.*`). + +use std::path::Path; + +use anyhow::{Context, Result, ensure}; +use safetensors::SafeTensors; + +const HIDDEN: usize = 1024; +const HEAD_DIM: usize = 256; + +fn gelu(x: f32) -> f32 { + 0.5 * x * (1.0 + libm::erff(x / std::f32::consts::SQRT_2)) +} + +/// One `Linear`: weights row-major `[out, in]`, optional bias. +struct Linear { + weight: Vec, + bias: Option>, + out: usize, + input: usize, +} + +impl Linear { + fn apply(&self, x: &[f32]) -> Vec { + debug_assert_eq!(x.len(), self.input); + let mut out = vec![0.0; self.out]; + for (row, slot) in out.iter_mut().enumerate() { + let mut sum = self.bias.as_ref().map_or(0.0, |b| b[row]); + let base = row * self.input; + for (index, value) in x.iter().enumerate() { + sum += self.weight[base + index] * value; + } + *slot = sum; + } + out + } +} + +/// One `LayerNorm` over the hidden vector (PyTorch defaults, eps 1e-5). +struct LayerNorm { + weight: Vec, + bias: Vec, +} + +impl LayerNorm { + fn apply(&self, x: &[f32]) -> Vec { + let n = x.len() as f32; + let mean = x.iter().sum::() / n; + let var = x.iter().map(|v| (v - mean) * (v - mean)).sum::() / n; + let inv = 1.0 / (var + 1e-5).sqrt(); + x.iter() + .enumerate() + .map(|(i, v)| (v - mean) * inv * self.weight[i] + self.bias[i]) + .collect() + } +} + +/// `SpanHeadV2` (router `span2.*` or broad `span_broad.*`). +pub struct SpanHead { + word_norm: LayerNorm, + label_norm: LayerNorm, + slot: Vec, // [slots, hidden] + slots: usize, + d: usize, + k: Linear, + q: Linear, + m: Linear, + n: Linear, + v: Linear, +} + +impl SpanHead { + /// score (W, L): words (W, H) raw states, labels (L, H) block means. + pub fn forward(&self, words: &[Vec], labels: &[Vec]) -> Vec> { + let mut w: Vec> = words.iter().map(|row| self.word_norm.apply(row)).collect(); + // Per-column mean over the words, subtracted. + if !w.is_empty() { + let mut mean = vec![0.0f32; HIDDEN]; + for row in &w { + for (i, value) in row.iter().enumerate() { + mean[i] += value; + } + } + for value in mean.iter_mut() { + *value /= w.len() as f32; + } + for row in w.iter_mut() { + for (i, value) in row.iter_mut().enumerate() { + *value -= mean[i]; + } + } + } + // Label states: normalized + slot embedding, mean-centered when + // two or more labels. + let mut e: Vec> = labels + .iter() + .enumerate() + .map(|(index, row)| { + let mut state = self.label_norm.apply(row); + let slot = index.min(self.slots - 1); + let base = slot * HIDDEN; + for (i, value) in state.iter_mut().enumerate() { + *value += self.slot[base + i]; + } + state + }) + .collect(); + if e.len() >= 2 { + let mut mean = vec![0.0f32; HIDDEN]; + for row in &e { + for (i, value) in row.iter().enumerate() { + mean[i] += value; + } + } + for value in mean.iter_mut() { + *value /= e.len() as f32; + } + for row in e.iter_mut() { + for (i, value) in row.iter_mut().enumerate() { + *value -= mean[i]; + } + } + } + let scale = 1.0 / (self.d as f32).sqrt(); + let k: Vec> = w.iter().map(|row| self.k.apply(row)).collect(); + let q: Vec> = e.iter().map(|row| self.q.apply(row)).collect(); + let m_rows: Vec> = w.iter().map(|row| self.m.apply(row)).collect(); + let n_rows: Vec> = e.iter().map(|row| self.n.apply(row)).collect(); + w.iter() + .enumerate() + .map(|(wi, _)| { + (0..labels.len()) + .map(|li| { + let mut bil = 0.0f32; + for (a, b) in k[wi].iter().zip(&q[li]) { + bil += a * b; + } + bil *= scale; + let mut summed = vec![0.0f32; HEAD_DIM]; + for (index, value) in summed.iter_mut().enumerate() { + *value = gelu(m_rows[wi][index] + n_rows[li][index]); + } + let mut score = 0.0f32; + for (index, value) in summed.iter().enumerate() { + score += self.v.weight[index] * value; + } + bil + score + }) + .collect() + }) + .collect() + } +} + +/// The two heads plus `set_bias`, loaded from one safetensors file. +pub struct Heads { + candidate_norm: LayerNorm, + query_norm: LayerNorm, + key: Linear, + query: Linear, + candidate_mlp: Linear, + query_mlp: Linear, + scalar: Linear, + pub set_bias: f32, + pub router: SpanHead, + pub broad: SpanHead, +} + +fn tensor<'a>(file: &'a SafeTensors<'a>, name: &str) -> Result> { + let view = file + .tensor(name) + .with_context(|| format!("heads.safetensors is missing {name}"))?; + ensure!( + view.dtype() == safetensors::Dtype::F32, + "{name} must be F32" + ); + Ok(view + .data() + .as_chunks::<4>() + .0 + .iter() + .map(|chunk| f32::from_le_bytes(*chunk)) + .collect()) +} + +fn linear(file: &SafeTensors, prefix: &str, bias: bool, input: usize) -> Result { + let weight = tensor(file, &format!("{prefix}.weight"))?; + let out = weight.len() / input; + ensure!( + weight.len() % input == 0 && out > 0, + "{prefix}.weight shape" + ); + Ok(Linear { + weight, + bias: if bias { + Some(tensor(file, &format!("{prefix}.bias"))?) + } else { + None + }, + out, + input, + }) +} + +fn norm(file: &SafeTensors, prefix: &str) -> Result { + Ok(LayerNorm { + weight: tensor(file, &format!("{prefix}.weight"))?, + bias: tensor(file, &format!("{prefix}.bias"))?, + }) +} + +fn span_head(file: &SafeTensors, prefix: &str, d: usize, slots: usize) -> Result { + let slot = tensor(file, &format!("{prefix}.slot.weight"))?; + Ok(SpanHead { + word_norm: norm(file, &format!("{prefix}.wn"))?, + label_norm: norm(file, &format!("{prefix}.ln"))?, + slots, + slot, + d, + k: linear(file, &format!("{prefix}.K"), false, HIDDEN)?, + q: linear(file, &format!("{prefix}.Q"), false, HIDDEN)?, + m: linear(file, &format!("{prefix}.M"), true, HIDDEN)?, + n: linear(file, &format!("{prefix}.N"), false, HIDDEN)?, + v: linear(file, &format!("{prefix}.v"), false, HEAD_DIM)?, + }) +} + +impl Heads { + /// Load the head tensors from an exported bundle. The manifest gate + /// runs first — anchored by `pinned_manifest_sha256`, the digest the + /// exporter prints — so the tensors are only read after every file + /// in the directory has matched the anchored manifest. + pub fn load(bundle: &Path, pinned_manifest_sha256: &str) -> Result { + crate::bundle::Bundle::load(bundle, pinned_manifest_sha256)?; + let raw = std::fs::read(bundle.join("heads.safetensors")).context("heads.safetensors")?; + let file = SafeTensors::deserialize(&raw).context("heads.safetensors must parse")?; + let set_bias = tensor(&file, "set_bias")?; + ensure!(set_bias.len() == 1, "set_bias is a scalar"); + // The span-head geometry ships in the manifest's calibration + // (d 256, slots 64 for this member). + Ok(Self { + candidate_norm: norm(&file, "head.candidate_norm")?, + query_norm: norm(&file, "head.query_norm")?, + key: linear(&file, "head.key", false, HIDDEN)?, + query: linear(&file, "head.query", false, HIDDEN)?, + candidate_mlp: linear(&file, "head.candidate_mlp", true, HIDDEN)?, + query_mlp: linear(&file, "head.query_mlp", false, HIDDEN)?, + scalar: linear(&file, "head.scalar", false, HEAD_DIM)?, + set_bias: set_bias[0], + router: span_head(&file, "span2", HEAD_DIM, 64)?, + broad: span_head(&file, "span_broad", HEAD_DIM, 64)?, + }) + } + + /// CandidateHead: option-end hiddens (W, H) and the block's query + /// hidden (H) -> one logit per option (plus `[ABS]` last when the + /// row carries it — the caller slices). + pub fn forward_candidate(&self, candidates: &[Vec], query: &[f32]) -> Vec { + let global_query = self.query_norm.apply(query); + let q_proj = self.query.apply(&global_query); + let q_mlp = self.query_mlp.apply(&global_query); + let scale = 1.0 / (HEAD_DIM as f32).sqrt(); + candidates + .iter() + .map(|row| { + let candidate = self.candidate_norm.apply(row); + let key = self.key.apply(&candidate); + let mut bilinear = 0.0f32; + for (a, b) in key.iter().zip(&q_proj) { + bilinear += a * b; + } + bilinear *= scale; + let mlp = self.candidate_mlp.apply(&candidate); + let mut summed = vec![0.0f32; HEAD_DIM]; + for (index, value) in summed.iter_mut().enumerate() { + *value = gelu(mlp[index] + q_mlp[index]); + } + let mut nonlinear = 0.0f32; + for (index, value) in summed.iter().enumerate() { + nonlinear += self.scalar.weight[index] * value; + } + bilinear + nonlinear + }) + .collect() + } +} diff --git a/src/models/vela/native/src/lib.rs b/src/models/vela/native/src/lib.rs new file mode 100644 index 00000000..3221eeb1 --- /dev/null +++ b/src/models/vela/native/src/lib.rs @@ -0,0 +1,15 @@ +//! Native library for vllm-sr/Vela-2.0-0.8B: the exported-bundle gate, +//! the wire contract, and the token-level processing (state → parts, +//! the SystemOne → predict mapping with the pinned runtime's validation +//! triples, word units, the DEV2.0 prompt rendering — mirrored against +//! the frozen `native-plan-*` fixtures). + +pub mod bundle; +pub mod contract; +pub mod heads; +pub mod processing; +pub mod questions; +pub mod render; +pub mod scoring; +pub mod state; +pub mod words; diff --git a/src/models/vela/native/src/processing.rs b/src/models/vela/native/src/processing.rs new file mode 100644 index 00000000..ba4c5817 --- /dev/null +++ b/src/models/vela/native/src/processing.rs @@ -0,0 +1,276 @@ +//! The processor: one admitted request body → predict mapping, plan, +//! and rendered rows, mirroring the reference path +//! (`system_one_to_predict` → `_to_generic` → `_predict`'s row split → +//! `_prep`/`_render`, plus the long-target window rows). Everything the +//! frozen `native-plan-*` fixtures record. + +use std::collections::HashSet; +use std::path::Path; + +use anyhow::{Context, Result}; +use serde_json::{Map, Value}; + +use crate::contract::RequestError; +use crate::questions::{PredictQuestion, system_one_to_predict}; +use crate::render::{ + Encoder, GenericQuestion, HALU_LABEL, PreparedPart, RMAX, RenderContext, RenderedRow, + window_part, +}; +use crate::state::system_one_state; + +pub struct Processor { + encoder: Encoder, + max_length: usize, + router_label_sets: Vec>, + has_broad: bool, + halu_desc: String, +} + +pub struct Planned { + pub predict_request: Value, + pub plan: Value, + pub rows: Vec, + pub window_rows: Vec, +} + +impl Processor { + /// Load from an exported native bundle (tokenizer + calibration; + /// the manifest gate runs first, anchored by `pinned_manifest_sha256` + /// — the SHA-256 of `vela_native_manifest.json` that the exporter + /// prints, fixed outside the bundle directory). + pub fn load(bundle: &Path, pinned_manifest_sha256: &str) -> Result { + let manifest = crate::bundle::Bundle::load(bundle, pinned_manifest_sha256)?; + let calibration: Value = serde_json::from_slice( + &std::fs::read(bundle.join("calibration.json")).context("calibration.json")?, + ) + .context("calibration.json must be JSON")?; + let broad = calibration.get("broad_head").filter(|b| !b.is_null()); + let router_label_sets: Vec> = broad + .and_then(|b| b.get("router_label_sets")) + .and_then(Value::as_array) + .filter(|sets| !sets.is_empty()) + .map(|sets| { + sets.iter() + .filter_map(Value::as_array) + .map(|labels| { + labels + .iter() + .filter_map(Value::as_str) + .map(str::to_owned) + .collect() + }) + .collect() + }) + .unwrap_or_else(|| { + vec![ + calibration["pii_types"].as_array().map(|types| { + types + .iter() + .filter_map(Value::as_str) + .map(str::to_owned) + .collect() + }), + calibration["halu_schema"]["labels"] + .as_object() + .map(|labels| labels.keys().cloned().collect()), + ] + .into_iter() + .flatten() + .collect() + }); + let has_broad = broad.is_some(); // the bundle gate pinned the broad tensors + let halu_desc = calibration["halu_schema"]["labels"][HALU_LABEL] + .as_str() + .unwrap_or("a claim not supported by the context") + .to_owned(); + Ok(Self { + encoder: Encoder::load(&bundle.join("tokenizer.json"))?, + max_length: manifest.max_length(), + router_label_sets, + has_broad, + halu_desc, + }) + } + + /// The full planning path for one admitted request body. + pub fn plan(&self, body: &Map) -> Result { + let state = system_one_state(&body["state"])?; + let questions = body["questions"] + .as_object() + .ok_or_else(crate::questions::empty_questions_error)?; + let mapped = system_one_to_predict(state, questions)?; + let predict_request = serde_json::to_value(&mapped.request).expect("serializable"); + let plan = serde_json::to_value(&mapped.plan).expect("serializable"); + let state_parts = mapped.request.parts.clone(); + + // `_to_generic`: predict questions → generic render questions + // (Noul is already folded; choice always carries [ABS]). + let generic: Vec = + mapped.request.questions.iter().map(to_generic).collect(); + // `_predict`'s row split: non-span questions plus the first span + // in one row, every further span its own row. + let spans: Vec<&GenericQuestion> = generic.iter().filter(|q| q.kind == "span").collect(); + let rest: Vec = generic + .iter() + .filter(|q| q.kind != "span") + .cloned() + .collect(); + let mut rows: Vec> = Vec::new(); + if !rest.is_empty() || !spans.is_empty() { + let mut first = rest.clone(); + if let Some(span) = spans.first() { + first.push((*span).clone()); + } + rows.push(first); + for span in spans.iter().skip(1) { + rows.push(vec![(*span).clone()]); + } + } + let context = RenderContext { + encoder: &self.encoder, + max_length: self.max_length, + router_label_sets: &self.router_label_sets, + has_broad: self.has_broad, + halu_desc: self.halu_desc.clone(), + }; + let span_roles: HashSet = + spans.iter().flat_map(|q| q.over.iter().cloned()).collect(); + let mut rendered = Vec::new(); + let mut window_rows = Vec::new(); + let mut parts_by_role = std::collections::HashMap::new(); + for part in self.prepared_parts(&state_parts, &span_roles)? { + parts_by_role.insert(part.role.clone(), part); + } + for row in rows { + if row.is_empty() { + continue; + } + let (adjusted, dispatch) = context.prepare_questions(&row)?; + let parts: Vec = state_parts + .iter() + .map(|part| { + parts_by_role + .get(&part.r#type) + .cloned() + .expect("prepared part") + }) + .collect(); + let record = context.render(&parts, &adjusted, &dispatch)?; + // `_window_rows`: a span target beyond RMAX becomes one row + // per window (the full-render span block stays on the row). + let span = row.iter().find(|q| q.kind == "span"); + if let Some(span) = span { + let index = parts.iter().position(|p| p.role == span.over[0]); + if let Some(index) = index { + let part = &parts[index]; + let has_words = part.words.is_some(); + if part.ids.len() > RMAX && has_words { + let (_, first_tokens) = part.words.as_ref().expect("words"); + for (a, b) in crate::render::span_windows(first_tokens, part.ids.len()) { + let mut windowed_parts = parts.clone(); + windowed_parts[index] = window_part(part, a, b); + let (w_adjusted, w_dispatch) = context.prepare_questions(&row)?; + let mut record = + context.render(&windowed_parts, &w_adjusted, &w_dispatch)?; + // The runtime's window rows carry the head only. + record.window = true; + window_rows.push(record); + } + } + } + } + rendered.push(record); + } + Ok(Planned { + predict_request, + plan, + rows: rendered, + window_rows, + }) + } + + /// The prepared parts for one request: tokens everywhere, offsets + /// and word units on span-target parts. + fn prepared_parts( + &self, + state_parts: &[crate::state::Part], + span_roles: &HashSet, + ) -> Result, RequestError> { + state_parts + .iter() + .map(|part| { + let (ids, offsets) = self.encoder.ids_offsets(&part.text)?; + if span_roles.contains(&part.r#type) { + let words = crate::words::word_first_tokens(&part.text, &offsets); + Ok(PreparedPart { + role: part.r#type.clone(), + ids, + offsets: Some(offsets), + words: Some(words), + }) + } else { + Ok(PreparedPart { + role: part.r#type.clone(), + ids, + offsets: None, + words: None, + }) + } + }) + .collect() + } +} + +/// `_to_generic` for the SystemOne path: options/labels carry over in +/// order (score levels have empty descriptions), the target part is the +/// resolved `over`, choice always carries `[ABS]` (the reference's +/// `abstain` default is true and the mapper never sets it), and span +/// keeps its threshold and head override. +fn to_generic(question: &PredictQuestion) -> GenericQuestion { + let over: Vec = match &question.target_part { + Value::String(role) => vec![role.clone()], + Value::Array(roles) => roles + .iter() + .filter_map(Value::as_str) + .map(str::to_owned) + .collect(), + _ => Vec::new(), + }; + let (names, descriptions) = match (&question.labels, &question.options) { + (Some(labels), _) => ( + labels.iter().map(|n| n.name.clone()).collect(), + labels.iter().map(|n| n.description.clone()).collect(), + ), + (None, Some(Value::Array(items))) => { + // Objects carry name/description; score levels are strings. + let mut names = Vec::new(); + let mut descriptions = Vec::new(); + for item in items { + match item { + Value::String(name) => { + names.push(name.clone()); + descriptions.push(String::new()); + } + Value::Object(named) => { + names.push(named["name"].as_str().unwrap_or_default().to_owned()); + descriptions + .push(named["description"].as_str().unwrap_or_default().to_owned()); + } + _ => {} + } + } + (names, descriptions) + } + _ => (Vec::new(), Vec::new()), + }; + GenericQuestion { + id: question.id.clone(), + kind: question.kind.clone(), + over, + text: question.text.clone(), + names, + descriptions, + threshold: question.threshold, + head_override: question.head.clone(), + } +} diff --git a/src/models/vela/native/src/questions.rs b/src/models/vela/native/src/questions.rs new file mode 100644 index 00000000..4c095032 --- /dev/null +++ b/src/models/vela/native/src/questions.rs @@ -0,0 +1,441 @@ +//! The SystemOne (state, questions) → predict-request mapping, mirroring +//! the pinned runtime's `system_one_to_predict` exactly: the validation +//! walk and its whole-request error triples, the Noul folding into a +//! two-option choice, and the per-question plan used to build responses. + +use serde_json::{Map, Value}; + +use crate::contract::RequestError; +use crate::state::{Part, StateParts, content_text}; + +/// The `questions` container rejection shared by the mapping and the +/// processor's body walk. +pub fn empty_questions_error() -> RequestError { + qerror( + &["questions"], + "questions must be a non-empty object of named questions", + ) +} + +pub const SYSTEM_ONE_TYPES: [&str; 5] = ["noul", "choice", "score", "span", "set"]; +pub const NOUL_DEFAULT_NO: &str = "No. The statement or question is not satisfied."; +pub const NOUL_DEFAULT_YES: &str = "Yes. The statement or question is satisfied."; + +/// One option / label of a predict question (name, description). +#[derive(Debug, Clone, serde::Serialize, PartialEq)] +pub struct Named { + pub name: String, + pub description: String, +} + +/// The predict-level question, exactly as it serializes into the frozen +/// fixtures: Noul questions arrive here already folded into a choice. +#[derive(Debug, Clone, serde::Serialize)] +pub struct PredictQuestion { + pub id: String, + #[serde(rename = "type")] + pub kind: String, + pub text: String, + pub target_part: Value, + #[serde(skip_serializing_if = "Option::is_none")] + pub labels: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub options: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub threshold: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub head: Option, +} + +#[derive(Debug, Clone, serde::Serialize)] +pub struct PredictRequest { + pub parts: Vec, + pub questions: Vec, +} + +/// The response-side plan entry for one original question. +#[derive(Debug, Clone, serde::Serialize)] +pub struct PlanEntry { + pub qid: String, + #[serde(rename = "type")] + pub kind: String, + pub criteria: Value, + pub range: Option<(usize, usize)>, + pub part: Value, +} + +#[derive(Debug)] +pub struct Mapped { + pub request: PredictRequest, + pub plan: Vec, +} + +fn qerror(loc: &[&str], msg: &str) -> RequestError { + RequestError::envelope(loc, msg, "value_error") +} + +fn nonblank(value: &Value) -> bool { + match value { + Value::String(text) => !text.trim().is_empty(), + _ => true, + } +} + +/// `over` (state key | part name | list) → (target part(s), field range). +fn resolve_over( + over: &Value, + parts: &StateParts, + qloc: &[&str], +) -> Result<(Value, Option<(usize, usize)>), RequestError> { + let roles: Vec<&str> = parts.parts.iter().map(|p| p.r#type.as_str()).collect(); + let items: Vec<&Value> = match over { + Value::Array(list) => list.iter().collect(), + single => vec![single], + }; + let mut out: Vec<&str> = Vec::new(); + let mut range = None; + for item in items { + let Some(name) = item.as_str() else { + return Err(qerror( + &[qloc[0], qloc[1], "over"], + "over must be a state key, a part name or a list of them", + )); + }; + if let Some(field) = parts.fields.get(name) { + if !over.is_array() { + range = Some((field.start, field.end)); + } + if !out.contains(&field.role) { + out.push(field.role); + } + } else if roles.contains(&name) { + if !out.contains(&name) { + out.push(name); + } + } else { + let role = crate::state::key_role(name); + if roles.contains(&role) && !out.contains(&role) { + out.push(role); + } else { + return Err(qerror( + &[qloc[0], qloc[1], "over"], + &format!( + "'{name}' is not a field of the state (fields: [{fields}], parts: [{roles}])", + fields = parts + .fields + .keys() + .map(|k| format!("'{k}'")) + .collect::>() + .join(", "), + roles = roles.join(", "), + ), + )); + } + } + } + if out.is_empty() { + return Err(qerror( + &[qloc[0], qloc[1], "over"], + "over must not be empty", + )); + } + // Sequence order, so the pooled range covers exactly these parts. + let ordered: Vec<&str> = roles + .iter() + .filter(|role| out.contains(role)) + .copied() + .collect(); + let target = match ordered.len() { + 1 => Value::String(ordered[0].into()), + _ => Value::Array( + ordered + .into_iter() + .map(|r| Value::String(r.into())) + .collect(), + ), + }; + Ok((target, range)) +} + +/// The full mapping. `state` and `questions` are the admitted request's +/// fields; validation runs in the pinned runtime's order so error +/// precedence matches. +pub fn system_one_to_predict( + state_parts: StateParts, + questions: &Map, +) -> Result { + if questions.is_empty() { + return Err(qerror( + &["questions"], + "questions must be a non-empty object of named questions", + )); + } + let roles: Vec<&str> = state_parts + .parts + .iter() + .map(|p| p.r#type.as_str()) + .collect(); + let mut predict = Vec::new(); + let mut plan = Vec::new(); + for (qid, question) in questions { + if qid.trim().is_empty() { + return Err(qerror( + &["questions"], + "question IDs must not be empty or whitespace", + )); + } + let Some(question) = question.as_object() else { + return Err(qerror( + &["questions", qid], + "a question must be an object with a type", + )); + }; + let loc = ["questions", qid]; + let kind = question.get("type").and_then(Value::as_str).unwrap_or(""); + if !SYSTEM_ONE_TYPES.contains(&kind) { + return Err(RequestError::envelope( + &[loc[0], loc[1], "type"], + "type must be one of ['noul', 'choice', 'score', 'span', 'set']", + "literal_error", + )); + } + let mut allowed: Vec<&str> = vec!["type", "instructions", "criteria", "over"]; + if kind == "span" || kind == "set" { + allowed.push("threshold"); + } + if kind == "span" { + allowed.push("head"); + } + let extra: Vec<&String> = question + .keys() + .filter(|key| !allowed.contains(&key.as_str())) + .collect(); + if let Some(first) = extra.first() { + return Err(RequestError::envelope( + &[loc[0], loc[1], first], + "Extra inputs are not permitted", + "extra_forbidden", + )); + } + let instructions = question.get("instructions").unwrap_or(&Value::Null); + if !nonblank(instructions) { + return Err(qerror( + &[loc[0], loc[1], "instructions"], + "instructions must not be empty or whitespace", + )); + } + let text = match instructions { + Value::Null => String::new(), + other => content_text(other), + }; + let criteria = question.get("criteria").unwrap_or(&Value::Null); + // Target part: the resolved `over`, the span default, or all parts. + let (target_part, range) = match question.get("over") { + Some(Value::Null) | None => { + if kind == "span" { + let role = if roles.contains(&"answer") { + "answer" + } else if roles.contains(&"user") { + "user" + } else { + roles[0] + }; + (Value::String(role.into()), None) + } else if roles.len() == 1 { + (Value::String(roles[0].into()), None) + } else { + ( + Value::Array(roles.iter().map(|r| Value::String((*r).into())).collect()), + None, + ) + } + } + Some(over) => resolve_over(over, &state_parts, &loc)?, + }; + let mut predict_question = PredictQuestion { + id: qid.clone(), + kind: kind.to_string(), + text: text.clone(), + target_part: target_part.clone(), + labels: None, + options: None, + threshold: None, + head: None, + }; + let mut plan_criteria = criteria.clone(); + match kind { + "noul" => { + if !criteria.is_null() + && (!criteria.is_object() + || criteria + .as_object() + .is_some_and(|map| map.keys().any(|k| k != "true" && k != "false"))) + { + return Err(qerror( + &[loc[0], loc[1], "criteria"], + "noul criteria is an object with optional true / false", + )); + } + let map = criteria.as_object().cloned().unwrap_or_default(); + for key in ["true", "false"] { + if let Some(value) = map.get(key) + && !nonblank(value) + { + return Err(qerror( + &[loc[0], loc[1], "criteria", key], + "Noul criterion must not be empty or whitespace", + )); + } + } + let no = match map.get("false") { + Some(Value::Null) | None => NOUL_DEFAULT_NO.to_string(), + Some(value) => content_text(value), + }; + let yes = match map.get("true") { + Some(Value::Null) | None => NOUL_DEFAULT_YES.to_string(), + Some(value) => content_text(value), + }; + predict_question.kind = "choice".into(); + predict_question.options = Some(serde_json::json!([ + {"name": "no", "description": no}, + {"name": "yes", "description": yes}, + ])); + } + "choice" | "set" | "span" => { + let (lo, hi) = if kind == "choice" { (2, 255) } else { (1, 255) }; + let Some(map) = criteria.as_object() else { + return Err(qerror( + &[loc[0], loc[1], "criteria"], + &format!("{kind} criteria must be an object with {lo} to {hi} entries"), + )); + }; + if !(lo..=hi).contains(&map.len()) { + return Err(qerror( + &[loc[0], loc[1], "criteria"], + &format!("{kind} criteria must be an object with {lo} to {hi} entries"), + )); + } + let mut named = Vec::new(); + for (name, value) in map { + if name.trim().is_empty() || !nonblank(value) { + return Err(qerror( + &[loc[0], loc[1], "criteria", name], + "option names and descriptions must not be empty or whitespace", + )); + } + let description = match value { + Value::Null => String::new(), + other => content_text(other), + }; + named.push(Named { + name: name.clone(), + description, + }); + } + if kind == "span" { + predict_question.labels = Some(named); + } else { + predict_question.options = Some(serde_json::to_value(&named).unwrap()); + } + } + _ => { + // score + let Some(levels) = criteria.as_array() else { + return Err(qerror( + &[loc[0], loc[1], "criteria"], + "score criteria must be a list of 2 to 10 levels", + )); + }; + if !(2..=10).contains(&levels.len()) { + return Err(qerror( + &[loc[0], loc[1], "criteria"], + "score criteria must be a list of 2 to 10 levels", + )); + } + let mut names = Vec::new(); + for (index, value) in levels.iter().enumerate() { + if value.is_null() || !nonblank(value) { + return Err(qerror( + &[loc[0], loc[1], "criteria", &index.to_string()], + "Score criterion must not be empty or whitespace", + )); + } + names.push(content_text(value)); + } + if names.iter().collect::>().len() != names.len() { + return Err(qerror( + &[loc[0], loc[1], "criteria"], + "score levels must be distinct", + )); + } + predict_question.options = Some(serde_json::to_value(&names).unwrap()); + } + } + if let Some(threshold) = question.get("threshold") + && !threshold.is_null() + { + match threshold.as_f64().filter(|_| threshold.is_number()) { + Some(value) if (0.0..=1.0).contains(&value) => { + predict_question.threshold = Some(value); + } + _ => { + return Err(qerror( + &[loc[0], loc[1], "threshold"], + "threshold must be a number in [0, 1]", + )); + } + } + } + if kind == "span" + && let Some(head) = question.get("head") + && !head.is_null() + { + match head.as_str() { + Some(value @ ("router" | "broad")) => { + predict_question.head = Some(value.to_string()); + } + _ => { + return Err(RequestError::envelope( + &[loc[0], loc[1], "head"], + "head must be one of ['router', 'broad']", + "literal_error", + )); + } + } + } + // Python's plan keeps `crit`; noul normalized null -> {} earlier. + if kind == "noul" && plan_criteria.is_null() { + plan_criteria = serde_json::json!({}); + } + predict.push(predict_question); + plan.push(PlanEntry { + qid: qid.clone(), + kind: kind.to_string(), + criteria: plan_criteria, + range, + part: target_part, + }); + } + // The Noul views of set labels must not overwrite a question. + for entry in &plan { + if entry.kind == "set" + && let Value::Object(map) = &entry.criteria + { + for label in map.keys() { + if questions.contains_key(&format!("{}.{}", entry.qid, label)) { + return Err(qerror( + &["questions", &format!("{}.{}", entry.qid, label)], + "collides with the Noul view of a set label", + )); + } + } + } + } + Ok(Mapped { + request: PredictRequest { + parts: state_parts.parts, + questions: predict, + }, + plan, + }) +} diff --git a/src/models/vela/native/src/render.rs b/src/models/vela/native/src/render.rs new file mode 100644 index 00000000..2b8f17b5 --- /dev/null +++ b/src/models/vela/native/src/render.rs @@ -0,0 +1,801 @@ +//! Prompt rendering, mirroring the pinned runtime's `_prep` + `_render` +//! (the DEV2.0 decoder layout): the parts pass, per-question blocks +//! with option end and query positions, the span block in layout a/c +//! with word positions, the truncation budget, and the long-target +//! windows. Output structs serialize exactly like the frozen +//! `native-plan-*` fixtures. + +use std::cell::RefCell; +use std::collections::{HashMap, HashSet}; +use std::path::Path; + +use std::collections::BTreeMap; +use tokenizers::Tokenizer; + +use crate::contract::RequestError; +use crate::state::canonical; + +pub const SUFFIX_CHOICE_SCORE: &str = + "\n\nSelect the single option best supported by the context and instructions.\nDecision:"; +pub const SUFFIX_SET: &str = + "\n\nSelect every option supported by the context and instructions.\nDecision:"; +pub const SUFFIX_SPAN: &str = + "\n\nMark every span of the target text that carries one of the labels.\nSpans:"; +pub const ABS_KEY: &str = "[ABS]"; +pub const ABS_DESC: &str = "None of the shown options applies."; +pub const PARTS_HEAD: &str = "Context:\n"; +pub const SEG_CLOSE: &str = "\n\n"; +pub const TEXT_HEAD: &str = "\n\nText:\n"; +pub const HYBRID_RMAX: usize = 2048; +pub const WIN: usize = 1800; +pub const STRIDE: usize = 1536; +pub const RMAX: usize = 2048; + +pub const ROUTER_SPAN_IDS: [&str; 3] = ["pii", "halu", "toxic"]; +pub const HALU_LABEL: &str = "unsupported"; +pub const HALU_ALIASES: [&str; 9] = [ + "unsupported", + "unsupportedclaim", + "hallucinated", + "hallucination", + "hallucinatedspan", + "notsupported", + "unsupportedspan", + "fabricated", + "fabrication", +]; +pub const PII_TYPES: [&str; 17] = [ + "AGE", + "CREDIT_CARD", + "DATE_TIME", + "DOMAIN_NAME", + "EMAIL_ADDRESS", + "GPE", + "IBAN_CODE", + "IP_ADDRESS", + "NRP", + "ORGANIZATION", + "PERSON", + "PHONE_NUMBER", + "STREET_ADDRESS", + "TITLE", + "US_DRIVER_LICENSE", + "US_SSN", + "ZIP_CODE", +]; + +fn seg_open(role: &str) -> String { + format!("\n") +} + +fn option_block(key: &str, desc: &str, tag: &str) -> String { + format!( + "\n<{tag}>\n{}\n", + canonical(&serde_json::json!({"key": key, "description": desc})) + ) +} + +fn q_head(kind: &str, over: &[String], text: &str, span: bool) -> String { + let target = over.join(", "); + format!( + "\n\nTask type: {kind}\nTarget: {target}\nQuestion:\n{text}\n{}", + if span { "Labels:" } else { "Options:" } + ) +} + +// ------------------------------------------------------------------ tokenizer + +/// The pinned tokenizer, with the reference's memoized plain-`ids` view +/// and char-offset encodings (the tokenizers crate reports byte +/// offsets; the reference works in Unicode code points). +/// Ids plus char offsets, the encoder cache entry. +type Encoded = (Vec, Vec<(usize, usize)>); + +pub struct Encoder { + tokenizer: Tokenizer, + cache: RefCell>, +} + +/// The encode memo holds per-request text fragments; cap it so a +/// long-lived process cannot grow it without bound (whole-table reset, +/// not LRU — a miss just re-encodes). +const ENCODER_CACHE_LIMIT: usize = 4096; + +impl Encoder { + pub fn load(path: &Path) -> anyhow::Result { + let tokenizer = Tokenizer::from_file(path).map_err(anyhow::Error::msg)?; + Ok(Self { + tokenizer, + cache: RefCell::new(HashMap::new()), + }) + } + + fn encode(&self, text: &str) -> Result { + if let Some(hit) = self.cache.borrow().get(text) { + return Ok(hit.clone()); + } + let encoding = self.tokenizer.encode(text, false).map_err(|error| { + // The pinned tokenizer should never fail on valid UTF-8 + // input; surface the error instead of panicking so a + // wrapping worker can map it. + RequestError::detail( + 500, + &format!("the pinned tokenizer failed on valid input: {error}"), + ) + })?; + let byte_offsets = encoding.get_offsets(); + // Byte → code-point offsets. + let char_starts: Vec = text + .char_indices() + .map(|(byte, _)| byte) + .chain(std::iter::once(text.len())) + .collect(); + let to_char = |byte: usize| char_starts.partition_point(|&start| start < byte); + let offsets: Vec<(usize, usize)> = byte_offsets + .iter() + .map(|(start, end)| (to_char(*start), to_char(*end))) + .collect(); + let entry = (encoding.get_ids().to_vec(), offsets); + let mut cache = self.cache.borrow_mut(); + if cache.len() >= ENCODER_CACHE_LIMIT { + cache.clear(); + } + cache.insert(text.to_owned(), entry.clone()); + Ok(entry) + } + + /// Memoized token ids of a whole string (the reference's `_ids`). + pub fn ids(&self, text: &str) -> Result, RequestError> { + Ok(self.encode(text)?.0) + } + + /// Ids and char offsets (the reference's `_ids_offs`). + pub fn ids_offsets(&self, text: &str) -> Result { + self.encode(text) + } +} + +// ------------------------------------------------------------- span dispatch + +/// `[\s_\-]+` removed, lower-cased — the reference's `_norm_label`. +pub fn norm_label(name: &str) -> String { + let mut out = String::with_capacity(name.len()); + for ch in name.chars() { + if !ch.is_whitespace() && ch != '_' && ch != '-' { + out.extend(ch.to_lowercase()); + } + } + out +} + +/// The user's label when the span question has exactly one label that +/// is a hallucination alias, else none. +pub fn halu_alias(names: &[String]) -> Option { + if names.len() == 1 && HALU_ALIASES.contains(&norm_label(&names[0]).as_str()) { + Some(names[0].clone()) + } else { + None + } +} + +pub struct SpanDispatch { + pub head: &'static str, + pub reason: &'static str, +} + +/// The reference's six-rule dispatch, in order: `"head"` override, +/// trained ids, an exact router label set, a non-empty subset of the 17 +/// PII types, one hallucination alias, else broad (router when the +/// broad head is not loaded). +pub fn span_head_for( + id: &str, + labels: &HashSet, + head_override: Option<&str>, + router_label_sets: &[HashSet], + broad_available: bool, +) -> Result { + if let Some(head) = head_override { + if !matches!(head, "router" | "broad") { + return Err(format!( + "head must be one of ['router', 'broad'], got '{head:?}'" + )); + } + if head == "broad" && !broad_available { + return Err("the broad span head is not loaded (needs broad_head.safetensors)".into()); + } + return Ok(SpanDispatch { + head: if head == "broad" { "broad" } else { "router" }, + reason: "override", + }); + } + if ROUTER_SPAN_IDS.contains(&id) { + return Ok(SpanDispatch { + head: "router", + reason: "id", + }); + } + if router_label_sets.iter().any(|set| set == labels) { + return Ok(SpanDispatch { + head: "router", + reason: "label_set", + }); + } + let pii: HashSet<&str> = PII_TYPES.into_iter().collect(); + if !labels.is_empty() && labels.iter().all(|label| pii.contains(label.as_str())) { + return Ok(SpanDispatch { + head: "router", + reason: "pii_subset", + }); + } + if halu_alias(&labels.iter().cloned().collect::>()).is_some() { + return Ok(SpanDispatch { + head: "router", + reason: "halu_alias", + }); + } + if !broad_available { + return Ok(SpanDispatch { + head: "router", + reason: "broad_unavailable", + }); + } + Ok(SpanDispatch { + head: "broad", + reason: "default", + }) +} + +// ------------------------------------------------------------------ rendering + +/// One generic-contract question at render time (Noul is already folded +/// into a two-option choice; choice always carries `[ABS]` on the +/// SystemOne path — the reference's `_to_generic` defaults abstain to +/// true and the mapper never sets it). +#[derive(Clone)] +pub struct GenericQuestion { + pub id: String, + pub kind: String, + pub over: Vec, + pub text: String, + pub names: Vec, + pub descriptions: Vec, + pub threshold: Option, + pub head_override: Option, +} + +/// One prepared part: tokens, and for span-target parts also char +/// offsets and word units. +/// Word char offsets and each word's first token index. +pub type WordUnits = (Vec<(usize, usize)>, Vec); + +#[derive(Clone)] +pub struct PreparedPart { + pub role: String, + pub ids: Vec, + pub offsets: Option>, + pub words: Option, +} + +#[derive(Debug, Clone, serde::Serialize, PartialEq)] +pub struct TruncRecord { + pub shrink_cut: usize, + pub protected_cut: usize, +} + +#[derive(Debug, Clone, serde::Serialize, PartialEq)] +#[serde(tag = "kind")] +pub enum Block { + #[serde(rename = "q")] + Question { + ids: Vec, + ends: Vec, + query: usize, + qid: String, + qtype: String, + names: Vec, + has_abs: bool, + }, + #[serde(rename = "span")] + Span { + ends: Vec, + label_ranges: Vec<(usize, usize)>, + words_in_block: bool, + ids: Vec, + query: usize, + w_pos: Vec, + }, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct RenderedRow { + pub p_ids: Vec, + pub blocks: Vec, + pub trunc: TruncRecord, + pub seg_lens: BTreeMap, + pub layout: Option, + pub names: Option>, + pub w_offs: Option>, + pub w_pos: Option>, + pub span_over: Option, + pub n_tokens: usize, + pub head: String, + pub head_reason: Option, + /// The runtime's window rows carry the head only — no head_reason + /// key at all (predict_raw assigns head alone). + pub window: bool, +} + +/// Serialize exactly like the frozen fixtures: main rows always carry +/// `head_reason` (null when the row has no span); window rows omit it. +impl serde::Serialize for RenderedRow { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serialize_row(self, serializer) + } +} + +fn serialize_row(row: &RenderedRow, serializer: S) -> Result +where + S: serde::Serializer, +{ + use serde::ser::SerializeMap; + let mut map = serializer.serialize_map(None)?; + macro_rules! put { + ($key:literal, $value:expr) => { + map.serialize_entry($key, &$value)? + }; + } + put!("p_ids", row.p_ids); + put!("blocks", row.blocks); + put!("trunc", row.trunc); + put!("seg_lens", row.seg_lens); + put!("layout", row.layout); + if let Some(names) = &row.names { + put!("names", names); + } + if let Some(w_offs) = &row.w_offs { + put!("w_offs", w_offs); + } + if let Some(w_pos) = &row.w_pos { + put!("w_pos", w_pos); + } + if let Some(span_over) = &row.span_over { + put!("span_over", span_over); + } + put!("n_tokens", row.n_tokens); + put!("head", row.head); + if !row.window { + put!("head_reason", row.head_reason); + } + map.end() +} + +/// Everything the renderer needs beyond tokens. +pub struct RenderContext<'a> { + pub encoder: &'a Encoder, + pub max_length: usize, + pub router_label_sets: &'a [HashSet], + pub has_broad: bool, + pub halu_desc: String, +} + +/// The whole-request 413: questions plus schema alone do not fit. +pub fn too_long(max_length: usize) -> RequestError { + RequestError { + status: 413, + body: serde_json::json!({"detail": [{"loc": ["body", "questions"], + "msg": format!("the questions alone do not fit the {max_length}-token window"), + "type": "too_long"}]}), + } +} + +/// `schema_lib_r4.assemble`, eval path: the truncation order on the +/// decoder's token costs. Returns kept segment lengths and the record. +fn assemble_budget( + q_lens: &[usize], + el_lens: &[usize], + segs: &mut [(String, usize)], + max_len: usize, + protected: &HashSet, + shrink: Option<&str>, +) -> Result<(BTreeMap, TruncRecord), ()> { + let fixed = 2 + 1 + segs.len(); + fn over( + q_lens: &[usize], + el_lens: &[usize], + segs: &[(String, usize)], + fixed: usize, + max_len: usize, + ) -> i64 { + fixed as i64 + + q_lens.iter().sum::() as i64 + + el_lens.iter().sum::() as i64 + + segs.iter().map(|(_, n)| *n).sum::() as i64 + - max_len as i64 + } + let mut trunc = TruncRecord { + shrink_cut: 0, + protected_cut: 0, + }; + let shrink_index = shrink.and_then(|role| segs.iter().position(|(r, _)| r == role)); + for floor in [64, 1] { + if over(q_lens, el_lens, segs, fixed, max_len) > 0 + && let Some(index) = shrink_index + { + let full = segs[index].1; + let excess = over(q_lens, el_lens, segs, fixed, max_len).max(0) as usize; + let keep = full.min(floor.max(full.saturating_sub(excess))); + trunc.shrink_cut += full - keep; + segs[index].1 = keep; + } + } + if over(q_lens, el_lens, segs, fixed, max_len) > 0 { + for index in 0..segs.len() { + if over(q_lens, el_lens, segs, fixed, max_len) > 0 && protected.contains(&segs[index].0) + { + let full = segs[index].1; + let excess = over(q_lens, el_lens, segs, fixed, max_len).max(0) as usize; + let keep = 1.max(full.saturating_sub(excess)); + trunc.protected_cut += full - keep; + segs[index].1 = keep; + } + } + } + if over(q_lens, el_lens, segs, fixed, max_len) > 0 { + return Err(()); + } + let seg_lens: BTreeMap = + segs.iter().map(|(role, n)| (role.clone(), *n)).collect(); + Ok((seg_lens, trunc)) +} + +/// Token windows [a, b) over a part of n tokens, starting and cut at +/// word starts (`schema_lib_d3_20261001.windows`). +pub fn span_windows(first_tokens: &[usize], n: usize) -> Vec<(usize, usize)> { + let mut starts: Vec = first_tokens.to_vec(); + starts.sort_unstable(); + starts.dedup(); + if starts.is_empty() { + return vec![(0, n)]; + } + let mut out = Vec::new(); + let mut a = starts[0]; + loop { + let limit = a + WIN; + let next: Vec = starts + .iter() + .copied() + .filter(|&s| s > a && s < limit) + .collect(); + let b = if limit >= n { + n + } else if !next.is_empty() { + *starts + .iter() + .filter(|&&s| s < limit && s > a) + .max() + .unwrap() + } else { + n.min(limit) + }; + out.push((a, b)); + if b >= n { + break; + } + let candidate: Vec = starts + .iter() + .copied() + .filter(|&s| s >= a + STRIDE && s < b) + .collect(); + a = candidate.first().copied().unwrap_or(b); + } + out +} + +/// A windowed copy of a prepared part: tokens [a, b) and the words that +/// start inside them. +pub fn window_part(part: &PreparedPart, a: usize, b: usize) -> PreparedPart { + let (word_offsets, first_tokens) = part.words.as_ref().expect("windowed parts carry words"); + let selected: Vec = first_tokens + .iter() + .enumerate() + .filter(|&(_, &t)| t >= a && t < b) + .map(|(index, _)| index) + .collect(); + let kept_offsets: Vec<(usize, usize)> = selected.iter().map(|&i| word_offsets[i]).collect(); + PreparedPart { + role: part.role.clone(), + ids: part.ids[a..b].to_vec(), + offsets: part.offsets.as_ref().map(|offs| offs[a..b].to_vec()), + words: Some(( + kept_offsets, + selected.iter().map(|&i| first_tokens[i] - a).collect(), + )), + } +} + +impl RenderContext<'_> { + /// `_prep`'s question side: the span-head dispatch and the + /// hallucination-alias label replacement (parts arrive prepared — + /// word units need the original text and are the caller's job). + pub fn prepare_questions( + &self, + questions: &[GenericQuestion], + ) -> Result<(Vec, SpanDispatch), RequestError> { + let span = questions.iter().find(|q| q.kind == "span"); + let dispatch = match span { + Some(question) => { + let labels: HashSet = question.names.iter().cloned().collect(); + span_head_for( + &question.id, + &labels, + question.head_override.as_deref(), + self.router_label_sets, + self.has_broad, + ) + .map_err(|message| crate::contract::runtime_rejection(&message))? + } + None => SpanDispatch { + head: "router", + reason: "", + }, + }; + // The trained `unsupported` label renders in place of a single + // hallucination-alias label (the answer gets the user's name). + let mut adjusted = questions.to_vec(); + if let Some(question) = adjusted.iter_mut().find(|q| q.kind == "span") + && dispatch.head == "router" + && let Some(alias) = halu_alias(&question.names) + && alias != HALU_LABEL + { + question.names = vec![HALU_LABEL.to_string()]; + question.descriptions = vec![self.halu_desc.clone()]; + } + Ok((adjusted, dispatch)) + } + + /// `_render` on one prepared row: the parts pass and every block, + /// with the budget applied first. + pub fn render( + &self, + parts: &[PreparedPart], + questions: &[GenericQuestion], + dispatch: &SpanDispatch, + ) -> Result { + let ids_of = |text: &str| self.encoder.ids(text); + let roles: Vec = parts.iter().map(|p| p.role.clone()).collect(); + // Question blocks and their schema costs. + struct RenderedQ { + qid: String, + qtype: String, + names: Vec, + otoks: Vec>, + has_abs: bool, + head: Vec, + suffix: Vec, + abstain_tokens: Vec, + len: usize, + } + let mut rendered_qs: Vec = Vec::new(); + let mut el: Vec<(String, Vec)> = Vec::new(); // span labels + let mut span_question: Option<&GenericQuestion> = None; + for question in questions { + if question.kind == "span" { + span_question = Some(question); + for (name, desc) in question.names.iter().zip(&question.descriptions) { + let tokens = ids_of(&option_block(name, desc, "label"))?; + el.push((name.clone(), tokens.clone())); + } + continue; + } + let suffix = match question.kind.as_str() { + "set" => SUFFIX_SET, + _ => SUFFIX_CHOICE_SCORE, // choice and score share the wording + }; + let has_abs = question.kind == "choice"; // SystemOne: abstain defaults true + let head = ids_of(&q_head( + &question.kind, + &question.over, + &question.text, + false, + ))?; + let suffix = ids_of(suffix)?; + let abstain_tokens = if has_abs { + ids_of(&option_block(ABS_KEY, ABS_DESC, "option"))? + } else { + Vec::new() + }; + let mut otoks = Vec::with_capacity(question.names.len()); + let mut len = + 1 + head.len() + suffix.len() + abstain_tokens.len() + usize::from(has_abs); + for (name, desc) in question.names.iter().zip(&question.descriptions) { + let tokens = ids_of(&option_block(name, desc, "option"))?; + len += 1 + tokens.len(); + otoks.push(tokens); + } + rendered_qs.push(RenderedQ { + qid: question.id.clone(), + qtype: question.kind.clone(), + names: question.names.clone(), + otoks, + has_abs, + head, + suffix, + abstain_tokens, + len, + }); + } + let span_over: Option = span_question.map(|question| question.over[0].clone()); + let mut overs: HashSet = HashSet::new(); + for question in questions { + overs.extend(question.over.iter().cloned()); + } + if let Some(over) = &span_over { + overs.insert(over.clone()); + } + let part_of = |role: &str| parts.iter().find(|p| p.role == role); + let shrink = roles.iter().find(|role| !overs.contains(*role)).cloned(); + let layout = match &span_over { + Some(over) if part_of(over).is_some_and(|p| p.ids.len() <= HYBRID_RMAX) => "c", + Some(_) => "a", + None => "", + }; + // The schema overhead the budget does not see. + let mut budget = ids_of(PARTS_HEAD)?.len(); + for role in &roles { + budget += ids_of(&seg_open(role))?.len() + ids_of(SEG_CLOSE)?.len(); + } + if let Some(over) = &span_over { + let question = span_question.expect("span question"); + let head = ids_of(&q_head("span", &question.over, &question.text, true))?; + let suffix = ids_of(SUFFIX_SPAN)?; + budget += head.len() + suffix.len(); + if layout == "c" { + let span_len = part_of(over).expect("span part").ids.len(); + budget += ids_of(TEXT_HEAD)?.len() + + ids_of(&seg_open(over))?.len() + + ids_of(SEG_CLOSE)?.len(); + let headroom = self.max_length.saturating_sub(budget); + budget += span_len.min(headroom / 2); + } + } + let q_lens: Vec = rendered_qs.iter().map(|q| q.len).collect(); + let el_lens: Vec = el.iter().map(|(_, tokens)| 1 + tokens.len()).collect(); + let mut segs: Vec<(String, usize)> = parts + .iter() + .map(|p| (p.role.clone(), p.ids.len())) + .collect(); + let (seg_lens, trunc) = assemble_budget( + &q_lens, + &el_lens, + &mut segs, + 64.max(self.max_length - budget), + &overs, + shrink.as_deref(), + ) + .map_err(|()| too_long(self.max_length))?; + // The parts pass, in part order (not the sorted map). + let mut p_ids = ids_of(PARTS_HEAD)?; + let mut start: HashMap = HashMap::new(); + for part in parts { + let keep = seg_lens[&part.role]; + let open = ids_of(&seg_open(&part.role))?; + start.insert(part.role.clone(), p_ids.len() + open.len()); + p_ids.extend(open); + p_ids.extend(part.ids.iter().take(keep)); + p_ids.extend(ids_of(SEG_CLOSE)?); + } + // Question blocks. + let mut blocks: Vec = Vec::new(); + for q in &rendered_qs { + let mut ids = q.head.clone(); + let mut ends = Vec::with_capacity(q.names.len() + usize::from(q.has_abs)); + for tokens in &q.otoks { + ids.extend(tokens); + ends.push(ids.len() - 1); + } + if q.has_abs { + ids.extend(q.abstain_tokens.clone()); + ends.push(ids.len() - 1); + } + ids.extend(q.suffix.clone()); + let query = ids.len() - 1; + blocks.push(Block::Question { + ids, + ends, + query, + qid: q.qid.clone(), + qtype: q.qtype.clone(), + names: q.names.clone(), + has_abs: q.has_abs, + }); + } + // The span block. + let mut row_names = None; + let mut row_w_offs = None; + let mut row_w_pos = None; + if let Some(over) = &span_over + && !el.is_empty() + { + let question = span_question.expect("span question"); + let mut ids = ids_of(&q_head("span", &question.over, &question.text, true))?; + let mut ends = Vec::new(); + let mut starts = Vec::new(); + for (_, tokens) in &el { + starts.push(ids.len()); + ids.extend(tokens); + ends.push(ids.len() - 1); + } + let part = part_of(over).expect("span part"); + let n = seg_lens[over]; + let (word_offsets, first_tokens) = part.words.as_ref().expect("span words"); + // Word indices whose first token survives the kept prefix; + // indices, not token values — two words can share one. + let keep: Vec = first_tokens + .iter() + .enumerate() + .filter(|&(_, &t)| t < n) + .map(|(index, _)| index) + .collect(); + let words_in_block = layout == "c"; + if words_in_block { + ids.extend(ids_of(TEXT_HEAD)?); + ids.extend(ids_of(&seg_open(over))?); + let c0 = ids.len(); + ids.extend(part.ids.iter().take(n)); + ids.extend(ids_of(SEG_CLOSE)?); + row_w_pos = Some(keep.iter().map(|&i| c0 + first_tokens[i]).collect()); + } else { + let base = start[over]; + row_w_pos = Some(keep.iter().map(|&i| base + first_tokens[i]).collect()); + } + ids.extend(ids_of(SUFFIX_SPAN)?); + let query = ids.len() - 1; + row_names = Some(el.iter().map(|(name, _)| name.clone()).collect()); + row_w_offs = Some(keep.iter().map(|&i| word_offsets[i]).collect()); + let label_ranges: Vec<(usize, usize)> = + starts.into_iter().zip(ends.iter().copied()).collect(); + blocks.push(Block::Span { + ends, + label_ranges, + words_in_block, + ids, + query, + w_pos: row_w_pos.clone().expect("w_pos"), + }); + } + let n_tokens = p_ids.len() + blocks.iter().map(block_len).sum::(); + Ok(RenderedRow { + p_ids, + blocks, + trunc, + seg_lens, + layout: if span_over.is_some() { + Some(layout.to_string()) + } else { + None + }, + names: row_names, + w_offs: row_w_offs, + w_pos: row_w_pos, + span_over, + n_tokens, + head: dispatch.head.to_string(), + head_reason: if dispatch.reason.is_empty() { + None + } else { + Some(dispatch.reason.to_string()) + }, + window: false, + }) + } +} + +fn block_len(block: &Block) -> usize { + match block { + Block::Question { ids, .. } => ids.len(), + Block::Span { ids, .. } => ids.len(), + } +} diff --git a/src/models/vela/native/src/scoring.rs b/src/models/vela/native/src/scoring.rs new file mode 100644 index 00000000..eed4e315 --- /dev/null +++ b/src/models/vela/native/src/scoring.rs @@ -0,0 +1,636 @@ +//! The scoring and decode mirror of the pinned runtime's calibration +//! path (`calibrated`, `span_threshold`, the PII length rule and sparse +//! gate, `span_decode_r4.decode_v2`, `_trim`), turning head logits into +//! predict-level answers. Parity target: the `head-input-*.json` +//! fixtures' `predict.answers`. + +use serde_json::Value; + +/// The package's `calibration.json` as typed lookup. +pub struct Calibration { + pub data: Value, +} + +pub fn softmax(x: &[f64]) -> Vec { + let max = x.iter().copied().fold(f64::NEG_INFINITY, f64::max); + let e: Vec = x.iter().map(|v| (v - max).exp()).collect(); + let sum: f64 = e.iter().sum(); + e.iter().map(|v| v / sum).collect() +} + +pub fn sigmoid(x: f64) -> f64 { + 1.0 / (1.0 + (-x).exp()) +} + +impl Calibration { + /// Load the calibration from an exported bundle, after the anchored + /// manifest gate has verified every file in the directory. + pub fn load(bundle: &std::path::Path, pinned_manifest_sha256: &str) -> anyhow::Result { + crate::bundle::Bundle::load(bundle, pinned_manifest_sha256)?; + Ok(Self { + data: serde_json::from_slice(&std::fs::read(bundle.join("calibration.json"))?)?, + }) + } + + fn temperature(&self, qtype: &str) -> f64 { + self.data["temperature"] + .get(qtype) + .and_then(Value::as_f64) + .unwrap_or(1.0) + } + + pub fn span_temperature(&self, head: &str) -> f64 { + if head == "broad" { + self.data["broad_head"]["temperature"] + .as_f64() + .unwrap_or(1.0) + } else { + self.temperature("span") + } + } + + fn is_pii(&self, qid: &str, names: &[String]) -> bool { + if qid == "pii" { + return true; + } + let types: Vec<&str> = self.data["pii_types"] + .as_array() + .map(|a| a.iter().filter_map(Value::as_str).collect()) + .unwrap_or_default(); + !names.is_empty() && names.iter().all(|n| types.contains(&n.as_str())) + } + + /// log-space piecewise-linear between anchors, constant outside. + fn pii_length_threshold(&self, n_tokens: usize) -> f64 { + let anchors = self.data["pii_length_rule"]["anchors"] + .as_array() + .cloned() + .unwrap_or_default(); + let mut points: Vec<(f64, f64)> = anchors + .iter() + .map(|a| { + ( + a["n_tokens"].as_f64().unwrap_or(1.0).ln(), + a["threshold"].as_f64().unwrap_or(1.0).ln(), + ) + }) + .collect(); + points.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap()); + let x = (n_tokens.max(1) as f64).ln(); + let (xs, ys): (Vec, Vec) = points.into_iter().unzip(); + interp(&xs, &ys, x).exp() + } + + /// The span threshold and its rule string, before the sparse gate. + pub fn span_threshold( + &self, + qid: &str, + names: &[String], + n_tokens: usize, + head: &str, + ) -> (f64, String) { + if head == "broad" { + return ( + self.data["broad_head"]["threshold"].as_f64().unwrap_or(0.5), + "broad".into(), + ); + } + let th = &self.data["thresholds"]; + if self.is_pii(qid, names) { + if self + .data + .get("pii_length_rule") + .is_some_and(|v| !v.is_null()) + { + return (self.pii_length_threshold(n_tokens), "pii:length".into()); + } + for bucket in self.data["pii_buckets"] + .as_array() + .cloned() + .unwrap_or_default() + { + if n_tokens as f64 <= bucket["max_tokens"].as_f64().unwrap_or(f64::INFINITY) { + return ( + bucket["threshold"].as_f64().unwrap_or(0.5), + format!("pii:{}", bucket["name"].as_str().unwrap_or("")), + ); + } + } + } + for fam in ["halu", "toxic"] { + if qid == fam { + return ( + th[&format!("span:{fam}")].as_f64().unwrap_or(0.5), + fam.into(), + ); + } + } + let specific = format!("span:{qid}"); + if th.get(&specific).is_some() { + (th[&specific].as_f64().unwrap_or(0.5), specific) + } else { + (th["span:*"].as_f64().unwrap_or(0.5), "span:*".into()) + } + } + + pub fn set_threshold(&self, qid: &str) -> f64 { + let th = &self.data["thresholds"]; + th.get(format!("set:{qid}")) + .or_else(|| th.get("set:*")) + .and_then(Value::as_f64) + .unwrap_or(0.5) + } + + /// The PII sparse gate (disabled when K < 0; second decode pass + /// over the same probabilities). + pub fn sparse_gate( + &self, + probs: &[Vec], + offs: &[(usize, usize)], + names: &[String], + text: &str, + thr: f64, + rule: &str, + ) -> (f64, String) { + let gate = &self.data["pii_sparse_gate"]; + let k = gate["K"].as_i64().unwrap_or(-1); + if k < 0 || thr >= gate["t_sparse"].as_f64().unwrap_or(f64::INFINITY) || offs.is_empty() { + return (thr, rule.into()); + } + let probe = gate["probe_threshold"].as_f64().unwrap_or(0.5); + let decoded = decode_spans(probs, offs, names, text, probe); + if decoded.len() as i64 <= k { + ( + gate["t_sparse"].as_f64().unwrap_or(thr), + format!("{rule}+sparse"), + ) + } else { + (thr, rule.into()) + } + } +} + +/// numpy.interp: linear between points, constant outside. +fn interp(xs: &[f64], ys: &[f64], x: f64) -> f64 { + if xs.is_empty() { + return f64::NAN; + } + if x <= xs[0] { + return ys[0]; + } + if x >= xs[xs.len() - 1] { + return ys[ys.len() - 1]; + } + for i in 0..xs.len() - 1 { + if x >= xs[i] && x <= xs[i + 1] { + let t = (x - xs[i]) / (xs[i + 1] - xs[i]); + return ys[i] + t * (ys[i + 1] - ys[i]); + } + } + ys[ys.len() - 1] +} + +// ------------------------------------------------------------ span decoding + +const EDGE: &str = "[](){}<>「」『』【】《》〈》\"'“”‘’«»‹›"; +const URL_TRAIL: &str = ".,;:!?)"; + +fn is_cjk(ch: char) -> bool { + // The reference's CJK class; same ranges as the word splitter. + super::words::is_cjk_char(ch) +} + +fn trim_edges(text: &[char], s: &mut usize, e: &mut usize) { + while *s < *e && (text[*s].is_whitespace() || EDGE.contains(text[*s])) { + *s += 1; + } + while *e > *s && (text[*e - 1].is_whitespace() || EDGE.contains(text[*e - 1])) { + *e -= 1; + } +} + +fn trim_url(text: &[char], s: &mut usize, e: &mut usize) { + // (?:https?://|www\.) at s, case-insensitive. + // The reference regex matches against the full text from s; the + // lookahead is not bounded by the span end. + let head: String = text[*s..(*s + 8).min(text.len())].iter().collect(); + let is_url = { + let lower = head.to_lowercase(); + lower.starts_with("http://") || lower.starts_with("https://") || lower.starts_with("www.") + }; + if !is_url { + return; + } + if let Some(pos) = text[*s..*e].iter().position(|&c| is_cjk(c)) { + *e = *s + pos; + } + while *e > *s + && (URL_TRAIL.contains(text[*e - 1]) + || EDGE.contains(text[*e - 1]) + || text[*e - 1].is_whitespace()) + { + *e -= 1; + } +} + +fn trim(text: &[char], s: usize, e: usize) -> (usize, usize) { + let (mut s, mut e) = (s, e); + trim_edges(text, &mut s, &mut e); + trim_url(text, &mut s, &mut e); + (s, e) +} + +/// `span_decode_r4.decode_v2` with the scorer's arguments +/// (extend=false, trim, no bridge, unit vote + fill on). +#[allow(clippy::too_many_arguments)] +pub fn decode_spans( + probs: &[Vec], + offs: &[(usize, usize)], + names: &[String], + text: &str, + thr: f64, +) -> Vec { + let chars: Vec = text.chars().collect(); + let t_count = offs.len(); + if t_count == 0 { + return Vec::new(); + } + let valid: Vec = offs.iter().map(|(a, b)| b > a).collect(); + let mut lab = vec![-1i64; t_count]; + for t in 0..t_count { + let (k, pk) = argmax(&probs[t]); + if valid[t] && pk as f64 > thr { + lab[t] = k as i64; + } + } + // Unit ids over the text (the reference's `_UNIT`), then per-word + // first-nonspace lookup and alnum test. + let units = super::words::unit_ids(&chars); + let mut tu = vec![-1i64; t_count]; + let mut alnum = vec![false; t_count]; + for t in 0..t_count { + if !valid[t] { + continue; + } + let (a, b) = offs[t]; + let mut f = a; + while f < b && chars[f].is_whitespace() { + f += 1; + } + tu[t] = if f < b && f < chars.len() { + units[f] + } else { + -1 + }; + alnum[t] = chars[a..b].iter().any(|&c| is_alnum(c)); + } + // Unit vote + fill, per run of the same unit. + let mut t = 0; + while t < t_count { + if tu[t] < 0 { + t += 1; + continue; + } + let mut t2 = t; + while t2 + 1 < t_count && (tu[t2 + 1] == tu[t] || !valid[t2 + 1]) { + t2 += 1; + } + let idx: Vec = (t..=t2).filter(|&i| valid[i]).collect(); + let labeled: Vec = idx.iter().copied().filter(|&i| lab[i] >= 0).collect(); + if !labeled.is_empty() { + let distinct: Vec = { + let mut d: Vec = labeled.iter().map(|&i| lab[i]).collect(); + d.sort_unstable(); + d.dedup(); + d + }; + if distinct.len() > 1 { + // The reference picks the first strict maximum in + // labeled-token order; accumulate in a Vec keyed by + // first occurrence, not a HashMap (whose iteration + // order would make ties nondeterministic). + let mut order: Vec = Vec::new(); + let mut scores: Vec = Vec::new(); + for &i in &labeled { + let label = lab[i]; + match order.iter().position(|&l| l == label) { + Some(index) => scores[index] += probs[i][label as usize] as f64, + None => { + order.push(label); + scores.push(probs[i][label as usize] as f64); + } + } + } + let mut winner_index = 0usize; + for (index, score) in scores.iter().enumerate().skip(1) { + if *score > scores[winner_index] { + winner_index = index; + } + } + let winner = order[winner_index]; + for &i in &labeled { + lab[i] = winner; + } + } + let labels: Vec = { + let mut d: Vec = labeled.iter().map(|&i| lab[i]).collect(); + d.sort_unstable(); + d.dedup(); + d + }; + for c in labels { + let pos: Vec = labeled.iter().copied().filter(|&i| lab[i] == c).collect(); + for &i in &idx { + if pos[0] < i && i < pos[pos.len() - 1] && lab[i] < 0 { + lab[i] = c; + } + } + } + } + t = t2 + 1; + } + // Merge consecutive same-label words into spans, trim, mean prob. + let mut spans: Vec<(i64, usize, usize, Vec)> = Vec::new(); + let mut current_label = None; + for t in 0..t_count { + if !valid[t] { + continue; + } + let c = lab[t]; + if c < 0 { + current_label = None; + continue; + } + if current_label == Some(c) { + let last = spans.last_mut().expect("current span"); + last.2 = offs[t].1; + last.3.push(t); + } else { + spans.push((c, offs[t].0, offs[t].1, vec![t])); + } + current_label = Some(c); + } + let mut out = Vec::new(); + for (c, s, e, ts) in spans { + let (s, e) = trim(&chars, s, e); + if e <= s { + continue; + } + let mean = ts.iter().map(|&t| probs[t][c as usize] as f64).sum::() / ts.len() as f64; + out.push(serde_json::json!({ + "start": s, "end": e, + "label": names[c as usize], + "probability": mean, + })); + } + out +} + +fn argmax(row: &[f32]) -> (usize, f32) { + let mut best = 0usize; + let mut value = f32::NEG_INFINITY; + for (index, item) in row.iter().enumerate() { + if *item > value { + value = *item; + best = index; + } + } + (best, value) +} + +fn is_alnum(c: char) -> bool { + // The reference's `_ALNUM` = [^\W_]: word characters minus the + // underscore — Rust's alphanumeric already excludes it. + c.is_alphanumeric() +} + +// -------------------------------------------------------- answer assembly + +/// One scored question block: the reference's `raw["qs"]` entry. +pub struct QBlock { + pub qid: String, + pub qtype: String, + pub names: Vec, + pub has_abs: bool, + /// Logits over `names` (plus `[ABS]` last when `has_abs`), + /// **after** run_batch's `set_bias` addition for set blocks. + pub logits: Vec, +} + +/// One scored span: the reference's `raw["span"]` (post aggregation). +pub struct SpanOut { + pub qid: String, + pub over: String, + pub names: Vec, + pub offs: Vec<(usize, usize)>, + pub logits: Vec>, + pub head: String, + pub head_reason: Option, + pub label_alias: Option, + /// The span part's kept token count (the PII length-rule input). + pub n_tokens: usize, +} + +/// `calibrated` for one row: logits -> predict-level answers. +pub fn row_answers( + cal: &Calibration, + blocks: &[QBlock], + span: Option<&SpanOut>, + thresholds: &std::collections::HashMap, + span_text: Option<&str>, +) -> Vec { + let mut answers = Vec::new(); + for block in blocks { + let logits: Vec = block.logits.iter().map(|v| *v as f64).collect(); + let n = block.names.len(); + let temperature = cal.temperature(&block.qtype); + if block.qtype == "set" { + let p: Vec = logits[..n] + .iter() + .map(|v| sigmoid(v / temperature)) + .collect(); + let thr = thresholds + .get(&block.qid) + .copied() + .unwrap_or_else(|| cal.set_threshold(&block.qid)); + let probs: serde_json::Map = block + .names + .iter() + .zip(&p) + .map(|(name, value)| (name.clone(), json_f64(*value))) + .collect(); + let selected: Vec<&String> = block + .names + .iter() + .zip(&p) + .filter(|(_, value)| **value > thr) + .map(|(name, _)| name) + .collect(); + answers.push(serde_json::json!({ + "id": block.qid, "type": "set", + "answer": selected, + "probabilities": probs, + "threshold": thr, + })); + continue; + } + let p = softmax( + &logits[..n] + .iter() + .map(|v| v / temperature) + .collect::>(), + ); + let answer = block.names[argmax_usize(&logits[..n])].clone(); + let probs: serde_json::Map = block + .names + .iter() + .zip(&p) + .map(|(name, value)| (name.clone(), json_f64(*value))) + .collect(); + let mut entry = serde_json::json!({ + "id": block.qid, "type": block.qtype, + "answer": answer, + "probabilities": probs, + }); + if block.has_abs { + let full = softmax(&logits.iter().map(|v| v / temperature).collect::>()); + entry["abstain_probability"] = json_f64(full[full.len() - 1]); + } + if block.qtype == "score" { + let expected: f64 = p.iter().enumerate().map(|(i, v)| i as f64 * v).sum(); + entry["expected_index"] = json_f64(expected); + // The default values normalize the levels to 0..1 + // (i / (n-1)); the SystemOne path never passes explicit ones. + let n = p.len(); + let score: f64 = p + .iter() + .enumerate() + .map(|(i, v)| { + // The reference's default values: i/(n-1), or 0 for + // a single level (never NaN). + let value = if n > 1 { + i as f64 / (n - 1) as f64 + } else { + 0.0 + }; + value * v + }) + .sum(); + entry["score"] = json_f64(score); + } + answers.push(entry); + } + if let Some(span) = span { + let text = span_text.unwrap_or(""); + let head = span.head.as_str(); + // The per-question override wins over the calibrated rule. + let (mut thr, mut rule) = match thresholds.get(&span.qid) { + Some(given) => (*given, "given".to_string()), + None => cal.span_threshold(&span.qid, &span.names, span.n_tokens, head), + }; + let temperature = cal.span_temperature(head); + let probs: Vec> = span + .logits + .iter() + .map(|row| { + row.iter() + .map(|v| sigmoid(*v as f64 / temperature) as f32) + .collect() + }) + .collect(); + if rule.starts_with("pii:") { + let (gated, gated_rule) = + cal.sparse_gate(&probs, &span.offs, &span.names, text, thr, &rule); + thr = gated; + rule = gated_rule; + } + let mut decoded = if span.offs.is_empty() { + Vec::new() + } else { + decode_spans(&probs, &span.offs, &span.names, text, thr) + }; + let chars: Vec = text.chars().collect(); + for entry in decoded.iter_mut() { + let (s, e) = ( + entry["start"].as_u64().unwrap_or(0) as usize, + entry["end"].as_u64().unwrap_or(0) as usize, + ); + entry["text"] = Value::String(chars[s..e].iter().collect()); + if let Some(mapped) = span + .label_alias + .as_ref() + .and_then(Value::as_object) + .and_then(|alias| alias.get(entry["label"].as_str().unwrap_or(""))) + { + entry["label"] = mapped.clone(); + } + } + let mut answer = serde_json::json!({ + "id": span.qid, + "type": "span", + "target_part": span.over, + "spans": decoded, + "threshold": thr, + "threshold_rule": rule, + "head": head, + }); + if let Some(reason) = &span.head_reason { + answer["head_reason"] = Value::String(reason.clone()); + } + answers.push(answer); + } + answers +} + +fn argmax_usize(x: &[f64]) -> usize { + let mut best = 0usize; + let mut value = f64::NEG_INFINITY; + for (index, item) in x.iter().enumerate() { + if *item > value { + value = *item; + best = index; + } + } + best +} + +fn json_f64(value: f64) -> Value { + serde_json::Number::from_f64(value) + .map(Value::Number) + .unwrap_or(Value::Null) +} + +/// `predict_raw`'s window aggregation: sum the window rows' span logits +/// per original word (matched by char offsets), divide by coverage +/// count, uncovered words score -30. +pub fn aggregate_windows( + names: &[String], + original_offs: &[(usize, usize)], + window_logits: &[Vec>], + window_offs: &[Vec<(usize, usize)>], +) -> Vec> { + let labels = names.len(); + // The reference accumulates in float64; the result casts to f32. + let mut acc = vec![vec![0.0f64; labels]; original_offs.len()]; + let mut cnt = vec![0usize; original_offs.len()]; + for (logits, offs) in window_logits.iter().zip(window_offs) { + for ((s, e), row) in offs.iter().zip(logits) { + if let Some(u) = original_offs.iter().position(|(os, oe)| os == s && oe == e) { + for (l, value) in row.iter().enumerate() { + acc[u][l] += *value as f64; + } + cnt[u] += 1; + } + } + } + acc.iter() + .zip(&cnt) + .map(|(row, c)| { + if *c > 0 { + row.iter().map(|v| (*v / *c as f64) as f32).collect() + } else { + vec![-30.0; labels] + } + }) + .collect() +} diff --git a/src/models/vela/native/src/state.rs b/src/models/vela/native/src/state.rs new file mode 100644 index 00000000..d33e42bb --- /dev/null +++ b/src/models/vela/native/src/state.rs @@ -0,0 +1,235 @@ +//! DEV2.0 canonical JSON and the state → typed-parts mapping, mirroring +//! the pinned runtime's `canonical_json`, `content_text` and +//! `system_one_state` exactly (the frozen `native-plan-*` fixtures are +//! the parity target). + +use serde_json::Value; +use std::collections::BTreeMap; + +use crate::contract::RequestError; + +/// `json.dumps(ensure_ascii=False, sort_keys=True, separators=(",", ":"))` +/// — key-sorted compact JSON, non-ASCII literal. +pub fn canonical(value: &Value) -> String { + match value { + Value::Object(map) => { + let mut keys: Vec<&String> = map.keys().collect(); + keys.sort(); + let parts: Vec = keys + .iter() + .map(|key| { + format!( + "{}:{}", + serde_json::to_string(key).unwrap(), + canonical(&map[*key]) + ) + }) + .collect(); + format!("{{{}}}", parts.join(",")) + } + Value::Array(items) => { + let parts: Vec = items.iter().map(canonical).collect(); + format!("[{}]", parts.join(",")) + } + other => serde_json::to_string(other).unwrap(), + } +} + +/// JSON content → model text: strings as they are, anything else as +/// canonical JSON. +pub fn content_text(value: &Value) -> String { + match value { + Value::String(text) => text.clone(), + other => canonical(other), + } +} + +/// The state-key → part-role table (unknown keys land in `context`). +pub fn key_role(key: &str) -> &'static str { + match key.to_ascii_lowercase().as_str() { + "request" | "user" | "prompt" => "user", + "answer" | "response" => "answer", + _ => "context", + } +} + +/// One rendered state part, in user / context / answer order. +#[derive(Debug, Clone, serde::Serialize, PartialEq)] +pub struct Part { + pub r#type: String, + pub text: String, +} + +/// A named state field's span inside its part's text (Unicode code +/// points, as on the reference). +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct FieldRange { + pub role: &'static str, + pub start: usize, + pub end: usize, +} + +#[derive(Debug)] +pub struct StateParts { + pub parts: Vec, + pub fields: BTreeMap, +} + +fn state_error(msg: &str) -> RequestError { + RequestError::envelope(&["state"], msg, "value_error") +} + +/// The SystemOne state → (parts, fields) mapping, mirroring +/// `system_one_state`: a plain string is one user part; a JSON array is +/// canonicalized into one; an object groups its keys by role (request / +/// user / prompt → user; source / context / document / tools / +/// previous_answer → context; answer / response → answer; unknown keys +/// → context) in user / context / answer order — one known-key entry +/// per role renders bare, several (or an unknown key) render as +/// `"key:\n"` blocks joined by blank lines. +pub fn system_one_state(state: &Value) -> Result { + // A string holding a JSON object is read as that object. + let state = match state { + Value::String(text) => { + let stripped = text.trim_start(); + if stripped.starts_with('{') { + if let Value::Object(object) = + serde_json::from_str::(stripped).unwrap_or(Value::Null) + { + Value::Object(object) + } else { + state.clone() + } + } else { + state.clone() + } + } + other => other.clone(), + }; + match &state { + Value::String(text) => { + if text.trim().is_empty() { + return Err(state_error("state must not be empty or whitespace")); + } + let mut fields = BTreeMap::new(); + fields.insert( + "request".into(), + FieldRange { + role: "user", + start: 0, + end: text.chars().count(), + }, + ); + Ok(StateParts { + parts: vec![Part { + r#type: "user".into(), + text: text.clone(), + }], + fields, + }) + } + Value::Array(_) => { + let text = canonical(&state); + let mut fields = BTreeMap::new(); + fields.insert( + "request".into(), + FieldRange { + role: "user", + start: 0, + end: text.chars().count(), + }, + ); + Ok(StateParts { + parts: vec![Part { + r#type: "user".into(), + text, + }], + fields, + }) + } + Value::Object(object) if !object.is_empty() => { + // Group per role, keeping the object's insertion order inside + // each role and remembering which entries had a known key. + let mut by_role: [Vec<(&String, &Value, bool)>; 3] = + [Vec::new(), Vec::new(), Vec::new()]; + for (key, value) in object { + let lower = key.to_ascii_lowercase(); + let known = matches!( + lower.as_str(), + "request" + | "user" + | "prompt" + | "source" + | "context" + | "document" + | "tools" + | "previous_answer" + | "answer" + | "response" + ); + let slot = match key_role(&lower) { + "user" => 0, + "answer" => 2, + _ => 1, + }; + by_role[slot].push((key, value, known)); + } + let mut parts = Vec::new(); + let mut fields = BTreeMap::new(); + for (role, items) in [ + ("user", &by_role[0]), + ("context", &by_role[1]), + ("answer", &by_role[2]), + ] { + if items.is_empty() { + continue; + } + let mut text = String::new(); + let single = items.len() == 1 && items[0].2; + for (index, (key, value, _)) in items.iter().enumerate() { + let rendered = content_text(value); + if single { + fields.insert( + (*key).clone(), + FieldRange { + role, + start: 0, + end: rendered.chars().count(), + }, + ); + text = rendered; + } else { + if index > 0 { + text.push_str("\n\n"); + } + let head = format!("{key}:\n"); + let start = text.chars().count() + head.chars().count(); + fields.insert( + (*key).clone(), + FieldRange { + role, + start, + end: start + rendered.chars().count(), + }, + ); + text.push_str(&head); + text.push_str(&rendered); + } + } + if text.trim().is_empty() { + return Err(state_error(&format!( + "the {role} part of the state is empty" + ))); + } + parts.push(Part { + r#type: role.into(), + text, + }); + } + Ok(StateParts { parts, fields }) + } + _ => Err(state_error( + "state must be a non-empty string, JSON object or JSON array", + )), + } +} diff --git a/src/models/vela/native/src/words.rs b/src/models/vela/native/src/words.rs new file mode 100644 index 00000000..44fb40b8 --- /dev/null +++ b/src/models/vela/native/src/words.rs @@ -0,0 +1,135 @@ +//! Word units and first-token mapping, mirroring the pinned runtime's +//! `schema_lib_r4` `_WORD` splitter (GLiNER2 whitespace splitter + CJK +//! one unit per character + the r4 URL/email/handle rule) and +//! `word_first_tokens`. Offsets are Unicode code points everywhere, as +//! on the reference. + +use std::sync::LazyLock; + +use onig::Regex; + +/// The CJK ranges of the reference `_WORD` pattern, verbatim. +const CJK: &str = "⺀-⿿　-〿぀-ヿ㄀-ㇿ㐀-䶿一-鿿ꀀ-꓏가-힯豈-﫿︰-﹏＀-￯"; +/// Trailing punctuation the URL rule strips. +const TRAIL: &str = ".,;:!?)\\]}>\"'\u{201d}\u{2019}\u{bb}"; + +/// The reference `_WORD` pattern, compiled once per process (compiling +/// it per request recompiled the same pattern every time). +static WORD_PATTERN: LazyLock = LazyLock::new(|| { + let pattern = concat!( + "(?:https?://|www\\.)[^\\s{cjk}]+?(?=[{trail}]*(?![^\\s{cjk}]))", + "|[a-z0-9._%+-]+@[a-z0-9.-]+\\.[a-z]{2,}", + "|@[a-z0-9_]+", + "|[{cjk}]", + "|[^\\W{cjk}]+(?:[-_][^\\W{cjk}]+)*", + "|\\S", + ); + let pattern = pattern.replace("{cjk}", CJK).replace("{trail}", TRAIL); + Regex::with_options( + &pattern, + onig::RegexOptions::REGEX_OPTION_IGNORECASE, + onig::Syntax::ruby(), + ) + .expect("the reference word pattern must compile") +}); + +/// Start-of-char byte offsets plus the end sentinel; index it with +/// `partition_point(|&start| start < byte)` to map a byte offset to its +/// code-point offset. +fn char_starts(text: &str) -> Vec { + text.char_indices() + .map(|(byte, _)| byte) + .chain(std::iter::once(text.len())) + .collect() +} + +/// The (start, end) word ranges of `text`, in code points. +pub fn split_words(text: &str) -> Vec<(usize, usize)> { + // Byte offsets → code-point offsets. + let char_of_byte = char_starts(text); + let to_char = |byte: usize| -> usize { char_of_byte.partition_point(|&start| start < byte) }; + WORD_PATTERN + .find_iter(text) + .map(|(start, end)| (to_char(start), to_char(end))) + .collect() +} + +/// (word char offsets (W, 2), each word's first overlapping sub-word +/// token index), mirroring `word_first_tokens`: the first token whose +/// cumulative end passes the word start, skipping empty tokens. +pub fn word_first_tokens( + text: &str, + offsets: &[(usize, usize)], +) -> (Vec<(usize, usize)>, Vec) { + let words = split_words(text); + if words.is_empty() || offsets.is_empty() { + return (Vec::new(), Vec::new()); + } + let valid: Vec = offsets.iter().map(|(s, e)| e > s).collect(); + let mut cummax_end: Vec = Vec::with_capacity(offsets.len()); + let mut running: i64 = -1; + for (index, (_start, end)) in offsets.iter().enumerate() { + if valid[index] { + running = running.max(*end as i64); + } + cummax_end.push(running); + } + let mut word_offsets = Vec::new(); + let mut first_tokens = Vec::new(); + for (a, b) in words { + let mut t = cummax_end.partition_point(|&end| end <= a as i64); + while t < offsets.len() && !valid[t] { + t += 1; + } + if t < offsets.len() && offsets[t].0 < b { + word_offsets.push((a, b)); + first_tokens.push(t); + } + } + (word_offsets, first_tokens) +} + +/// The reference's CJK class as a range test (the same ranges the word +/// splitter carries). +pub fn is_cjk_char(ch: char) -> bool { + // The reference class has one contiguous run from U+8C48 to + // U+FAFF and ends the Yi block at U+A4CF (not U+A48F). + matches!(ch as u32, + 0x2E80..=0x2FFF | 0x3000..=0x303F | 0x3040..=0x30FF | 0x3100..=0x31FF + | 0x3400..=0x4DBF | 0x4E00..=0x9FFF | 0xA000..=0xA4CF | 0xAC00..=0xD7AF + | 0x8C48..=0xFAFF | 0xFE30..=0xFE4F | 0xFF00..=0xFFEF) +} + +/// The reference's `_UNIT` split (decode-time units): URL bodies, runs +/// of word characters (excluding CJK, `_` excluded by `[^\W_]`) plus +/// `@._-+`, or any non-space char. Returns each char's unit id, -1 +/// outside units — the decode's `u` array. +pub fn unit_ids(chars: &[char]) -> Vec { + static UNIT_PATTERN: LazyLock = LazyLock::new(|| { + let pattern = concat!( + "(?:https?://|www\\.)[^\\s\\]\\)\\}>「」『』\"'“”{cjk}]+?(?=[.,;:!?]*(?![^\\s\\]\\)\\}>「」『』\"'“”{cjk}]))", + "|(?:(?![{cjk}])[^\\W_]|[@._\\-+])+", + "|\\S", + ) + .replace("{cjk}", CJK); + Regex::with_options( + &pattern, + onig::RegexOptions::REGEX_OPTION_IGNORECASE, + onig::Syntax::ruby(), + ) + .expect("the reference unit pattern must compile") + }); + let mut ids = vec![-1i64; chars.len()]; + let text: String = chars.iter().collect(); + let regex = &*UNIT_PATTERN; + // find_iter yields byte offsets; ids indexes characters. + let char_of_byte = char_starts(&text); + let to_char = |byte: usize| -> usize { char_of_byte.partition_point(|&start| start < byte) }; + for (unit, (start, end)) in regex.find_iter(&text).enumerate() { + let (start, end) = (to_char(start), to_char(end)); + for id in ids.iter_mut().take(end).skip(start) { + *id = unit as i64; + } + } + ids +} diff --git a/tests/vela/fixtures/head-input-example-request-cjk.json b/tests/vela/fixtures/head-input-example-request-cjk.json new file mode 100644 index 00000000..3baaa645 --- /dev/null +++ b/tests/vela/fixtures/head-input-example-request-cjk.json @@ -0,0 +1 @@ +{"case": "example-request-cjk", "request": {"model": "vllm-sr/Vela-2.0-0.8B", "state": {"request": "客户张伟（zhang.wei@example.cn）于2026年10月1日在上海市浦东新区世纪大道100号办理了业务，联系电话13800138000。", "source": "客户的业务申请需要在三个工作日内审核完成。", "answer": "客户的业务已经即时完成了。"}, "questions": {"pii": {"type": "span", "instructions": "Which spans are personal information?", "criteria": {"PERSON": "a person's name", "EMAIL_ADDRESS": "an e-mail address", "PHONE_NUMBER": "a telephone number", "STREET_ADDRESS": "a street address or postal address"}, "over": "request"}, "unsupported": {"type": "span", "instructions": "Which spans of the answer are not supported by the context?", "criteria": {"unsupported": "a claim not supported by the context"}}, "billing": {"type": "noul", "instructions": "这个请求是否与账单有关？"}}}, "rows": [{"head": "router", "head_reason": "id", "label_alias": null, "n_tokens": 404, "seg_lens": {"user": 46, "context": 10, "answer": 6}}, {"head": "router", "head_reason": "label_set", "label_alias": null, "n_tokens": 193, "seg_lens": {"user": 46, "context": 10, "answer": 6}}], "head_calls": [{"kind": "q", "cand": "q0_cand", "query": "q0_query", "logits": [[-1.60933518409729, -3.7373664379119873, -10.640555381774902]], "row": 0}, {"kind": "span", "head": "router", "words": "s1_words", "labels": "s1_labels", "logits": [[-23.778533935546875, -35.67962646484375, -29.496707916259766, -35.02517318725586], [-23.778533935546875, -35.67962646484375, -29.496707916259766, -35.02517318725586], [-13.288297653198242, -17.101852416992188, -9.51349925994873, -13.732088088989258], [-13.288297653198242, -17.101852416992188, -9.51349925994873, -13.732088088989258], [-17.785783767700195, -24.783889770507812, -18.803138732910156, -24.5747013092041], [-5.519737243652344, -7.974113464355469, -2.0204527378082275, -4.032443523406982], [-19.710346221923828, -31.652324676513672, -24.4345703125, -30.253273010253906], [-22.11681365966797, -31.155961990356445, -25.493202209472656, -30.28239631652832], [-0.9185004830360413, -6.406106472015381, -0.692579448223114, -1.6303598880767822], [-4.95607852935791, -9.517541885375977, -2.2732596397399902, -4.304035663604736], [-0.19278430938720703, -7.234131336212158, -0.09104931354522705, -1.1003239154815674], [-2.164036750793457, -7.8626861572265625, -1.2940788269042969, -2.4866232872009277], [-1.3710556030273438, -6.950081825256348, -1.0965814590454102, -1.80461847782135], [-17.513790130615234, -17.537555694580078, -8.913171768188477, -11.932638168334961], [-21.893512725830078, -29.469112396240234, -24.093914031982422, -26.871994018554688], [-3.359339952468872, -7.748542308807373, -2.7091588973999023, -0.33820855617523193], [-3.359339952468872, -7.748542308807373, -2.7091588973999023, -0.33820855617523193], [-3.359339952468872, -7.748542308807373, -2.7091588973999023, -0.33820855617523193], [-3.4210779666900635, -7.483105659484863, -2.7869768142700195, -0.5679111480712891], [-3.4210779666900635, -7.483105659484863, -2.7869768142700195, -0.5679111480712891], [-3.4210779666900635, -7.483105659484863, -2.7869768142700195, -0.5679111480712891], [-3.4210779666900635, -7.483105659484863, -2.7869768142700195, -0.5679111480712891], [-3.6699564456939697, -8.56665325164795, -2.6497883796691895, 0.620187520980835], [-3.6699564456939697, -8.56665325164795, -2.6497883796691895, 0.620187520980835], [-3.4258086681365967, -9.092829704284668, -2.0032799243927, 0.25476884841918945], [-3.4258086681365967, -9.092829704284668, -2.0032799243927, 0.25476884841918945], [-1.0213323831558228, -8.279101371765137, -0.9210346341133118, 1.104820728302002], [-10.929340362548828, -11.960079193115234, -4.331625461578369, -3.7565321922302246], [-19.668752670288086, -21.596508026123047, -16.757234573364258, -19.447601318359375], [-19.668752670288086, -21.596508026123047, -16.757234573364258, -19.447601318359375], [-19.668752670288086, -21.596508026123047, -16.757234573364258, -19.447601318359375], [-19.40172004699707, -19.034996032714844, -14.057591438293457, -15.741283416748047], [-19.40172004699707, -19.034996032714844, -14.057591438293457, -15.741283416748047], [-19.49544906616211, -31.111461639404297, -23.796104431152344, -29.51728057861328], [-21.458332061767578, -33.868682861328125, -27.257335662841797, -32.84736633300781], [-21.458332061767578, -33.868682861328125, -27.257335662841797, -32.84736633300781], [-21.458332061767578, -33.868682861328125, -27.257335662841797, -32.84736633300781], [-21.458332061767578, -33.868682861328125, -27.257335662841797, -32.84736633300781], [-0.6149466633796692, -7.636533260345459, -0.18819284439086914, -0.1909337043762207], [-24.31658172607422, -34.86664581298828, -29.284488677978516, -32.92279052734375]], "row": 0}, {"kind": "span", "head": "router", "words": "s2_words", "labels": "s2_labels", "logits": [[-4.0871477127075195], [-4.0871477127075195], [-4.0871477127075195], [-2.413851261138916], [-2.413851261138916], [-1.4215611219406128], [-1.4215611219406128], [-0.6014259457588196], [-0.6014259457588196], [-1.4138461351394653], [-1.4138461351394653], [-1.4138461351394653], [-4.652787685394287]], "row": 1}], "predict": {"answers": [{"id": "pii", "type": "span", "target_part": "user", "spans": [{"start": 32, "end": 34, "label": "PHONE_NUMBER", "probability": 0.4658691883087158, "text": "10"}, {"start": 38, "end": 52, "label": "STREET_ADDRESS", "probability": 0.4821831285953522, "text": "上海市浦东新区世纪大道100"}, {"start": 63, "end": 74, "label": "PHONE_NUMBER", "probability": 0.4298103451728821, "text": "13800138000"}], "threshold": 0.2914, "threshold_rule": "pii:length", "head": "router", "head_reason": "id"}, {"id": "unsupported", "type": "span", "target_part": "answer", "spans": [], "threshold": 0.5, "threshold_rule": "span:*", "head": "router", "head_reason": "label_set"}, {"id": "billing", "type": "choice", "answer": "no", "probabilities": {"no": 0.8325233433461274, "yes": 0.1674766566538726}, "abstain_probability": 0.0009211774164155532}], "meta": {"tokens": {"user": 46, "context": 10, "answer": 6}, "windows": 0, "sequences": 2}}} diff --git a/tests/vela/fixtures/head-input-example-request-cjk.safetensors b/tests/vela/fixtures/head-input-example-request-cjk.safetensors new file mode 100644 index 00000000..bd7cb173 Binary files /dev/null and b/tests/vela/fixtures/head-input-example-request-cjk.safetensors differ diff --git a/tests/vela/fixtures/head-input-example-request-score.json b/tests/vela/fixtures/head-input-example-request-score.json new file mode 100644 index 00000000..1588ad68 --- /dev/null +++ b/tests/vela/fixtures/head-input-example-request-score.json @@ -0,0 +1 @@ +{"case": "example-request-score", "request": {"model": "vllm-sr/Vela-2.0-0.8B", "state": "Hello, I would like to check the status of my order. It has been five days and I have heard nothing back.", "questions": {"tone": {"type": "score", "instructions": "How formal is the message?", "criteria": ["casual", "neutral", "formal"]}, "urgency": {"type": "score", "instructions": "How urgently does the sender need a reply?", "criteria": ["no rush", "soon", "today", "immediately"]}}}, "rows": [{"head": "router", "head_reason": null, "label_alias": null, "n_tokens": 228, "seg_lens": {"user": 25}}], "head_calls": [{"kind": "q", "cand": "q0_cand", "query": "q0_query", "logits": [[-2.308352470397949, -1.095753788948059, -1.616711139678955, -0.3145672082901001], [-1.1832587718963623, -1.9728960990905762, -1.711325764656067, -3.1148641109466553]], "row": 0}], "predict": {"answers": [{"id": "tone", "type": "score", "answer": "neutral", "probabilities": {"casual": 0.16807713462834117, "neutral": 0.5139566439051507, "formal": 0.3179662214665082}, "expected_index": 1.1498890868381673, "score": 0.5749445434190836}, {"id": "urgency", "type": "score", "answer": "no rush", "probabilities": {"no rush": 0.44128118285871365, "soon": 0.21311503083546, "today": 0.2712213259289846, "immediately": 0.07438246037684161}, "expected_index": 0.9787050638239541, "score": 0.3262350212746513}], "meta": {"tokens": {"user": 25}, "windows": 0, "sequences": 1}}} diff --git a/tests/vela/fixtures/head-input-example-request-score.safetensors b/tests/vela/fixtures/head-input-example-request-score.safetensors new file mode 100644 index 00000000..1e580a6c Binary files /dev/null and b/tests/vela/fixtures/head-input-example-request-score.safetensors differ diff --git a/tests/vela/fixtures/head-input-example-request-set-threshold.json b/tests/vela/fixtures/head-input-example-request-set-threshold.json new file mode 100644 index 00000000..e8b374a2 --- /dev/null +++ b/tests/vela/fixtures/head-input-example-request-set-threshold.json @@ -0,0 +1 @@ +{"case": "example-request-set-threshold", "request": {"model": "vllm-sr/Vela-2.0-0.8B", "state": "My card was charged twice and the parcel never arrived.", "questions": {"issues": {"type": "set", "instructions": "Which issues does the customer report?", "criteria": {"billing": "payments, charges, refunds or invoices", "shipping": "delivery of an order or a parcel", "login": "signing in, passwords or account access"}, "threshold": 0.15}}}, "rows": [{"head": "router", "head_reason": null, "label_alias": null, "n_tokens": 133, "seg_lens": {"user": 11}}], "head_calls": [{"kind": "q", "cand": "q0_cand", "query": "q0_query", "logits": [[0.4933321475982666, 1.5158393383026123, -8.921335220336914]], "row": 0}], "predict": {"answers": [{"id": "issues", "type": "set", "answer": ["billing", "shipping"], "probabilities": {"billing": 0.6211446881608428, "shipping": 0.8205402483597802, "login": 0.00012988178108716601}, "threshold": 0.15}], "meta": {"tokens": {"user": 11}, "windows": 0, "sequences": 1}}} diff --git a/tests/vela/fixtures/head-input-example-request-set-threshold.safetensors b/tests/vela/fixtures/head-input-example-request-set-threshold.safetensors new file mode 100644 index 00000000..8f5ad489 Binary files /dev/null and b/tests/vela/fixtures/head-input-example-request-set-threshold.safetensors differ diff --git a/tests/vela/fixtures/head-input-example-request.json b/tests/vela/fixtures/head-input-example-request.json new file mode 100644 index 00000000..99263ddd --- /dev/null +++ b/tests/vela/fixtures/head-input-example-request.json @@ -0,0 +1 @@ +{"case": "example-request", "request": {"model": "vllm-sr/Vela-2.0-0.8B", "state": {"request": "Hi, I'm Tom Baker (tom.baker@example.com). What is the maximum daily dose of paracetamol for an adult?", "source": "For adults, the maximum dose of paracetamol is 4 grams in 24 hours, taken as 500 mg to 1 g every 4 to 6 hours.", "answer": "Adults can take up to 6 grams of paracetamol in 24 hours, in doses of 500 mg to 1 g every 4 to 6 hours."}, "questions": {"pii": {"type": "span", "instructions": "Which spans are personal information?", "criteria": {"AGE": "a person's age", "CREDIT_CARD": "a payment card number", "DATE_TIME": "a date, time or date of birth", "DOMAIN_NAME": "an internet domain name or website address", "EMAIL_ADDRESS": "an e-mail address", "GPE": "a country, city, state or other geopolitical place", "IBAN_CODE": "an international bank account number (IBAN)", "IP_ADDRESS": "an IPv4 or IPv6 address", "NRP": "a nationality, religious or political group", "ORGANIZATION": "the name of a company, institution or other organisation", "PERSON": "a person's name", "PHONE_NUMBER": "a telephone number", "STREET_ADDRESS": "a street address or postal address", "TITLE": "a personal or professional title such as Dr. or Mrs.", "US_DRIVER_LICENSE": "a US driver's licence number", "US_SSN": "a US social security number", "ZIP_CODE": "a postal or ZIP code"}, "over": "request"}, "halu": {"type": "span", "instructions": "Which spans of the answer are not supported by the context?", "criteria": {"unsupported": "a claim not supported by the context"}}, "domain": {"type": "choice", "instructions": "Which subject area is this request about?", "over": "request", "criteria": {"health": "medicine, clinical practice, nutrition, ageing or sexual health", "math": "arithmetic, algebra, geometry, statistics or other mathematics", "other": "a subject that fits none of the listed areas"}}}}, "rows": [{"head": "router", "head_reason": "id", "label_alias": null, "n_tokens": 772, "seg_lens": {"user": 27, "context": 39, "answer": 39}}, {"head": "router", "head_reason": "id", "label_alias": null, "n_tokens": 269, "seg_lens": {"user": 27, "context": 39, "answer": 39}}], "head_calls": [{"kind": "q", "cand": "q0_cand", "query": "q0_query", "logits": [[0.5404365062713623, -6.5386433601379395, -3.9254047870635986, -3.529416084289551]], "row": 0}, {"kind": "span", "head": "router", "words": "s1_words", "labels": "s1_labels", "logits": [[-9.851020812988281, -15.027624130249023, -14.757858276367188, -12.530649185180664, -3.660945415496826, -12.923664093017578, -17.943824768066406, -16.326257705688477, -5.620431900024414, -7.599363803863525, -11.234650611877441, -5.757130146026611, -5.652872085571289, -8.116801261901855, -9.238210678100586, -6.802857398986816, -6.324886322021484], [-16.28253936767578, -17.80463981628418, -18.329021453857422, -16.44207763671875, -10.134305953979492, -17.914127349853516, -20.482263565063477, -18.635469436645508, -10.971238136291504, -11.935432434082031, -16.362531661987305, -8.079880714416504, -9.401307106018066, -14.166604995727539, -16.919057846069336, -14.731834411621094, -12.536507606506348], [-16.975248336791992, -21.33708953857422, -22.336502075195312, -20.949878692626953, -12.478657722473145, -21.414112091064453, -24.373680114746094, -23.24188232421875, -13.325860023498535, -13.795920372009277, -16.87654685974121, -11.636231422424316, -11.92214584350586, -16.560941696166992, -20.60035514831543, -17.067073822021484, -14.519638061523438], [-16.7723445892334, -21.27077293395996, -22.190185546875, -21.27130889892578, -12.792131423950195, -22.414506912231445, -24.66496467590332, -23.94645118713379, -13.870166778564453, -14.444552421569824, -17.01270866394043, -11.867671012878418, -13.354513168334961, -16.251331329345703, -21.633752822875977, -18.205692291259766, -16.024608612060547], [-16.7723445892334, -21.27077293395996, -22.190185546875, -21.27130889892578, -12.792131423950195, -22.414506912231445, -24.66496467590332, -23.94645118713379, -13.870166778564453, -14.444552421569824, -17.01270866394043, -11.867671012878418, -13.354513168334961, -16.251331329345703, -21.633752822875977, -18.205692291259766, -16.024608612060547], [0.11913681030273438, -8.998804092407227, -8.105165481567383, -7.355640411376953, 0.8499364852905273, -5.159857273101807, -10.962669372558594, -10.69532585144043, -2.300189971923828, -2.142609119415283, 3.7711997032165527, -2.6488523483276367, -3.3644497394561768, 0.5961573123931885, -4.22589111328125, -4.46975040435791, -4.146326065063477], [-0.375388503074646, -8.638716697692871, -7.520653247833252, -6.3430376052856445, 0.8605620861053467, -4.922619819641113, -10.513166427612305, -10.039107322692871, -2.3470044136047363, -2.1044399738311768, 2.319047451019287, -3.095822811126709, -3.0459518432617188, 0.2876932621002197, -3.672348976135254, -4.247696399688721, -4.059322357177734], [-11.97199535369873, -15.74243450164795, -15.960700988769531, -12.123148918151855, -2.198594570159912, -14.888036727905273, -18.449371337890625, -13.958887100219727, -5.779473781585693, -7.8383917808532715, -11.294285774230957, -2.836668014526367, -3.8323404788970947, -9.61889362335205, -9.943918228149414, -6.213159561157227, -5.016597747802734], [-2.930969715118408, -8.456787109375, -8.0104341506958, -3.1965346336364746, 2.791138172149658, -5.6602864265441895, -9.054672241210938, -4.491463661193848, -3.634566068649292, -2.6940877437591553, 0.6949470043182373, -2.568450927734375, -3.0842747688293457, -2.6646759510040283, -5.099068641662598, -5.331076622009277, -4.802918434143066], [-17.96658706665039, -18.01045799255371, -19.822132110595703, -18.21443748474121, -12.168895721435547, -19.514663696289062, -20.82657241821289, -19.630020141601562, -13.630260467529297, -13.030271530151367, -18.71689224243164, -10.937429428100586, -11.503185272216797, -16.562448501586914, -20.173627853393555, -17.357961654663086, -14.826516151428223], [-17.96658706665039, -18.01045799255371, -19.822132110595703, -18.21443748474121, -12.168895721435547, -19.514663696289062, -20.82657241821289, -19.630020141601562, -13.630260467529297, -13.030271530151367, -18.71689224243164, -10.937429428100586, -11.503185272216797, -16.562448501586914, -20.173627853393555, -17.357961654663086, -14.826516151428223], [-16.90250587463379, -18.278526306152344, -18.84981346130371, -17.358551025390625, -11.352295875549316, -17.4305362701416, -20.804384231567383, -19.499237060546875, -12.09277629852295, -12.66122817993164, -17.810014724731445, -10.43729305267334, -9.796697616577148, -16.462594985961914, -17.877269744873047, -15.53689193725586, -12.608529090881348], [-17.571109771728516, -18.02204704284668, -18.949459075927734, -17.84019660949707, -12.319219589233398, -18.332033157348633, -20.70317840576172, -19.956119537353516, -13.069973945617676, -13.284128189086914, -18.666032791137695, -11.275001525878906, -11.1100492477417, -16.783119201660156, -19.03497314453125, -16.65223503112793, -14.116575241088867], [-17.20058250427246, -17.913480758666992, -19.152984619140625, -18.49874496459961, -12.374701499938965, -18.944435119628906, -20.980005264282227, -20.324554443359375, -13.318517684936523, -13.295575141906738, -18.446435928344727, -11.100875854492188, -11.361801147460938, -16.607181549072266, -19.333765029907227, -16.73701286315918, -14.240310668945312], [-10.385387420654297, -15.364139556884766, -15.511908531188965, -13.689546585083008, -3.9188613891601562, -13.724010467529297, -18.471012115478516, -16.611557006835938, -5.779675483703613, -7.293036460876465, -12.180856704711914, -4.711075782775879, -4.105204105377197, -10.222349166870117, -10.208038330078125, -5.902805805206299, -4.506038188934326], [-10.814464569091797, -14.951911926269531, -15.325828552246094, -12.80671501159668, -3.280482530593872, -13.21613883972168, -17.812551498413086, -15.572422981262207, -5.481819152832031, -7.064236164093018, -12.217747688293457, -4.450317859649658, -3.697108030319214, -10.052149772644043, -9.760396957397461, -5.52978515625, -4.171611309051514], [-14.402262687683105, -17.102127075195312, -17.262144088745117, -15.675093650817871, -8.45707893371582, -15.754737854003906, -19.572450637817383, -17.654773712158203, -9.236448287963867, -10.762482643127441, -15.542367935180664, -7.367783546447754, -6.410301208496094, -14.149118423461914, -13.918294906616211, -11.3816499710083, -8.444021224975586], [-12.651286125183105, -16.529403686523438, -16.55528450012207, -14.945316314697266, -6.345352649688721, -15.065194129943848, -18.96143341064453, -17.375417709350586, -7.089203834533691, -9.234535217285156, -13.864068031311035, -5.386238098144531, -4.79850959777832, -12.170395851135254, -11.348276138305664, -7.870039463043213, -5.878173828125], [-8.939207077026367, -13.950385093688965, -13.887860298156738, -11.350774765014648, -3.2517220973968506, -11.77290153503418, -16.509742736816406, -14.612834930419922, -6.24839973449707, -7.029216766357422, -10.663041114807129, -6.143183708190918, -6.020900249481201, -8.616498947143555, -8.805045127868652, -6.615787982940674, -6.1211934089660645], [-14.634203910827637, -16.696504592895508, -16.7551212310791, -15.17517375946045, -8.941736221313477, -15.509604454040527, -18.92809295654297, -17.477008819580078, -9.083824157714844, -10.468768119812012, -15.845718383789062, -7.56320333480835, -7.45651388168335, -14.0153226852417, -14.346378326416016, -11.850302696228027, -9.550122261047363], [-12.652865409851074, -16.130496978759766, -16.09650993347168, -14.029860496520996, -5.189237594604492, -14.40012264251709, -18.68414306640625, -16.8243408203125, -6.2160868644714355, -8.960195541381836, -13.997604370117188, -5.3901848793029785, -4.3825273513793945, -11.94284439086914, -10.51372241973877, -6.524692058563232, -5.048408508300781], [-15.082358360290527, -16.506101608276367, -16.36886215209961, -14.599099159240723, -8.522944450378418, -14.7816801071167, -18.691444396972656, -17.160917282104492, -9.380626678466797, -11.312457084655762, -16.12392234802246, -8.477106094360352, -7.283524036407471, -14.228918075561523, -13.477392196655273, -11.19268798828125, -8.870052337646484], [-18.80125617980957, -21.510345458984375, -22.83747100830078, -22.03952980041504, -15.80392837524414, -22.6486759185791, -24.405366897583008, -23.06026840209961, -17.308101654052734, -16.587818145751953, -19.37947654724121, -14.987780570983887, -15.041773796081543, -18.377851486206055, -22.79374122619629, -20.769546508789062, -18.1580810546875]], "row": 0}, {"kind": "span", "head": "router", "words": "s2_words", "labels": "s2_labels", "logits": [[-5.667586803436279], [-4.148900508880615], [-2.493729829788208], [-0.7364544868469238], [-0.2502114176750183], [3.92134428024292], [1.7255499362945557], [-2.568751811981201], [-3.1423110961914062], [-4.012135028839111], [-4.174318313598633], [-3.6643919944763184], [-4.65214729309082], [-5.806663990020752], [-5.589629650115967], [-5.320010185241699], [-3.489220142364502], [-4.177278518676758], [-4.293692111968994], [-4.44113302230835], [-4.547778129577637], [-5.308318138122559], [-6.041932106018066], [-6.042882442474365], [-5.594691753387451], [-4.543209552764893], [-4.973324298858643]], "row": 1}], "predict": {"answers": [{"id": "pii", "type": "span", "target_part": "user", "spans": [{"start": 8, "end": 17, "label": "PERSON", "probability": 0.983367919921875, "text": "Tom Baker"}, {"start": 19, "end": 40, "label": "EMAIL_ADDRESS", "probability": 0.9851056337356567, "text": "tom.baker@example.com"}], "threshold": 0.2914, "threshold_rule": "pii:length", "head": "router", "head_reason": "id"}, {"id": "halu", "type": "span", "target_part": "answer", "spans": [{"start": 19, "end": 29, "label": "unsupported", "probability": 0.7782317996025085, "text": "to 6 grams"}], "threshold": 0.35, "threshold_rule": "halu", "head": "router", "head_reason": "id"}, {"id": "domain", "type": "choice", "answer": "health", "probabilities": {"health": 0.9621185645193945, "math": 0.004639298028680393, "other": 0.03324213745192508}, "abstain_probability": 0.04287965117790319}], "meta": {"tokens": {"user": 27, "context": 39, "answer": 39}, "windows": 0, "sequences": 2}}} diff --git a/tests/vela/fixtures/head-input-example-request.safetensors b/tests/vela/fixtures/head-input-example-request.safetensors new file mode 100644 index 00000000..8e69195b Binary files /dev/null and b/tests/vela/fixtures/head-input-example-request.safetensors differ diff --git a/tests/vela/fixtures/head-input-long-span-windowed.json b/tests/vela/fixtures/head-input-long-span-windowed.json new file mode 100644 index 00000000..300076e2 --- /dev/null +++ b/tests/vela/fixtures/head-input-long-span-windowed.json @@ -0,0 +1 @@ +{"case": "long-span-windowed", "request": {"model": "vllm-sr/Vela-2.0-0.8B", "state": {"request": "Sentence 0 reports a routine delivery event. Sentence 1 reports a routine delivery event. Sentence 2 reports a routine delivery event. Sentence 3 reports a routine delivery event. Sentence 4 reports a routine delivery event. Sentence 5 reports a routine delivery event. Sentence 6 reports a routine delivery event. Sentence 7 reports a routine delivery event. Sentence 8 reports a routine delivery event. Sentence 9 reports a routine delivery event. Sentence 10 reports a routine delivery event. Sentence 11 reports a routine delivery event. Sentence 12 reports a routine delivery event. Sentence 13 reports a routine delivery event. Sentence 14 reports a routine delivery event. Sentence 15 reports a routine delivery event. Sentence 16 reports a routine delivery event. Sentence 17 reports a routine delivery event. Sentence 18 reports a routine delivery event. Sentence 19 reports a routine delivery event. Sentence 20 reports a routine delivery event. Sentence 21 reports a routine delivery event. Sentence 22 reports a routine delivery event. Sentence 23 reports a routine delivery event. Sentence 24 reports a routine delivery event. Sentence 25 reports a routine delivery event. Sentence 26 reports a routine delivery event. Sentence 27 reports a routine delivery event. Sentence 28 reports a routine delivery event. Sentence 29 reports a routine delivery event. Sentence 30 reports a routine delivery event. Sentence 31 reports a routine delivery event. Sentence 32 reports a routine delivery event. Sentence 33 reports a routine delivery event. Sentence 34 reports a routine delivery event. Sentence 35 reports a routine delivery event. Sentence 36 reports a routine delivery event. Sentence 37 reports a routine delivery event. Sentence 38 reports a routine delivery event. Sentence 39 reports a routine delivery event. Sentence 40 reports a routine delivery event. Sentence 41 reports a routine delivery event. Sentence 42 reports a routine delivery event. Sentence 43 reports a routine delivery event. Sentence 44 reports a routine delivery event. Sentence 45 reports a routine delivery event. Sentence 46 reports a routine delivery event. Sentence 47 reports a routine delivery event. Sentence 48 reports a routine delivery event. Sentence 49 reports a routine delivery event. Sentence 50 reports a routine delivery event. Sentence 51 reports a routine delivery event. Sentence 52 reports a routine delivery event. Sentence 53 reports a routine delivery event. Sentence 54 reports a routine delivery event. Sentence 55 reports a routine delivery event. Sentence 56 reports a routine delivery event. Sentence 57 reports a routine delivery event. Sentence 58 reports a routine delivery event. Sentence 59 reports a routine delivery event. Sentence 60 reports a routine delivery event. Sentence 61 reports a routine delivery event. Sentence 62 reports a routine delivery event. Sentence 63 reports a routine delivery event. Sentence 64 reports a routine delivery event. Sentence 65 reports a routine delivery event. Sentence 66 reports a routine delivery event. Sentence 67 reports a routine delivery event. Sentence 68 reports a routine delivery event. Sentence 69 reports a routine delivery event. Sentence 70 reports a routine delivery event. Sentence 71 reports a routine delivery event. Sentence 72 reports a routine delivery event. Sentence 73 reports a routine delivery event. Sentence 74 reports a routine delivery event. Sentence 75 reports a routine delivery event. Sentence 76 reports a routine delivery event. Sentence 77 reports a routine delivery event. Sentence 78 reports a routine delivery event. Sentence 79 reports a routine delivery event. Sentence 80 reports a routine delivery event. Sentence 81 reports a routine delivery event. Sentence 82 reports a routine delivery event. Sentence 83 reports a routine delivery event. Sentence 84 reports a routine delivery event. Sentence 85 reports a routine delivery event. Sentence 86 reports a routine delivery event. Sentence 87 reports a routine delivery event. Sentence 88 reports a routine delivery event. Sentence 89 reports a routine delivery event. Sentence 90 reports a routine delivery event. Sentence 91 reports a routine delivery event. Sentence 92 reports a routine delivery event. Sentence 93 reports a routine delivery event. Sentence 94 reports a routine delivery event. Sentence 95 reports a routine delivery event. Sentence 96 reports a routine delivery event. Sentence 97 reports a routine delivery event. Sentence 98 reports a routine delivery event. Sentence 99 reports a routine delivery event. Sentence 100 reports a routine delivery event. Sentence 101 reports a routine delivery event. Sentence 102 reports a routine delivery event. Sentence 103 reports a routine delivery event. Sentence 104 reports a routine delivery event. Sentence 105 reports a routine delivery event. Sentence 106 reports a routine delivery event. Sentence 107 reports a routine delivery event. Sentence 108 reports a routine delivery event. Sentence 109 reports a routine delivery event. Sentence 110 reports a routine delivery event. Sentence 111 reports a routine delivery event. Sentence 112 reports a routine delivery event. Sentence 113 reports a routine delivery event. Sentence 114 reports a routine delivery event. Sentence 115 reports a routine delivery event. Sentence 116 reports a routine delivery event. Sentence 117 reports a routine delivery event. Sentence 118 reports a routine delivery event. Sentence 119 reports a routine delivery event. Sentence 120 reports a routine delivery event. Sentence 121 reports a routine delivery event. Sentence 122 reports a routine delivery event. Sentence 123 reports a routine delivery event. Sentence 124 reports a routine delivery event. Sentence 125 reports a routine delivery event. Sentence 126 reports a routine delivery event. Sentence 127 reports a routine delivery event. Sentence 128 reports a routine delivery event. Sentence 129 reports a routine delivery event. Sentence 130 reports a routine delivery event. Sentence 131 reports a routine delivery event. Sentence 132 reports a routine delivery event. Sentence 133 reports a routine delivery event. Sentence 134 reports a routine delivery event. Sentence 135 reports a routine delivery event. Sentence 136 reports a routine delivery event. Sentence 137 reports a routine delivery event. Sentence 138 reports a routine delivery event. Sentence 139 reports a routine delivery event. Sentence 140 reports a routine delivery event. Sentence 141 reports a routine delivery event. Sentence 142 reports a routine delivery event. Sentence 143 reports a routine delivery event. Sentence 144 reports a routine delivery event. Sentence 145 reports a routine delivery event. Sentence 146 reports a routine delivery event. Sentence 147 reports a routine delivery event. Sentence 148 reports a routine delivery event. Sentence 149 reports a routine delivery event. Sentence 150 reports a routine delivery event. Sentence 151 reports a routine delivery event. Sentence 152 reports a routine delivery event. Sentence 153 reports a routine delivery event. Sentence 154 reports a routine delivery event. Sentence 155 reports a routine delivery event. Sentence 156 reports a routine delivery event. Sentence 157 reports a routine delivery event. Sentence 158 reports a routine delivery event. Sentence 159 reports a routine delivery event. Sentence 160 reports a routine delivery event. Sentence 161 reports a routine delivery event. Sentence 162 reports a routine delivery event. Sentence 163 reports a routine delivery event. Sentence 164 reports a routine delivery event. Sentence 165 reports a routine delivery event. Sentence 166 reports a routine delivery event. Sentence 167 reports a routine delivery event. Sentence 168 reports a routine delivery event. Sentence 169 reports a routine delivery event. Sentence 170 reports a routine delivery event. Sentence 171 reports a routine delivery event. Sentence 172 reports a routine delivery event. Sentence 173 reports a routine delivery event. Sentence 174 reports a routine delivery event. Sentence 175 reports a routine delivery event. Sentence 176 reports a routine delivery event. Sentence 177 reports a routine delivery event. Sentence 178 reports a routine delivery event. Sentence 179 reports a routine delivery event. Sentence 180 reports a routine delivery event. Sentence 181 reports a routine delivery event. Sentence 182 reports a routine delivery event. Sentence 183 reports a routine delivery event. Sentence 184 reports a routine delivery event. Sentence 185 reports a routine delivery event. Sentence 186 reports a routine delivery event. Sentence 187 reports a routine delivery event. Sentence 188 reports a routine delivery event. Sentence 189 reports a routine delivery event. Sentence 190 reports a routine delivery event. Sentence 191 reports a routine delivery event. Sentence 192 reports a routine delivery event. Sentence 193 reports a routine delivery event. Sentence 194 reports a routine delivery event. Sentence 195 reports a routine delivery event. Sentence 196 reports a routine delivery event. Sentence 197 reports a routine delivery event. Sentence 198 reports a routine delivery event. Sentence 199 reports a routine delivery event. Sentence 200 reports a routine delivery event. Sentence 201 reports a routine delivery event. Sentence 202 reports a routine delivery event. Sentence 203 reports a routine delivery event. Sentence 204 reports a routine delivery event. Sentence 205 reports a routine delivery event. Sentence 206 reports a routine delivery event. Sentence 207 reports a routine delivery event. Sentence 208 reports a routine delivery event. Sentence 209 reports a routine delivery event. Sentence 210 reports a routine delivery event. Sentence 211 reports a routine delivery event. Sentence 212 reports a routine delivery event. Sentence 213 reports a routine delivery event. Sentence 214 reports a routine delivery event. Sentence 215 reports a routine delivery event. Sentence 216 reports a routine delivery event. Sentence 217 reports a routine delivery event. Sentence 218 reports a routine delivery event. Sentence 219 reports a routine delivery event."}, "questions": {"claims": {"type": "span", "instructions": "Which spans mention a delivery?", "criteria": {"delivery": "a delivery event"}, "over": "request"}}}, "rows": [{"head": "broad", "head_reason": "default", "label_alias": null, "n_tokens": 2385, "seg_lens": {"user": 2310}}], "head_calls": [{"kind": "span", "head": "broad", "words": "s0_words", "labels": "s0_labels", "logits": [[-4.5031867027282715], [-5.110243797302246], [-4.144986152648926], [-3.987896680831909], [-3.73734712600708], [-3.6736156940460205], [-4.206828594207764], [-5.193665027618408], [-4.111611843109131], [-4.483518600463867], [-3.8180947303771973], [-3.600126266479492], [-3.552110195159912], [-3.0163097381591797], [-3.220355272293091], [-5.011423110961914], [-3.5230727195739746], [-3.642819404602051], [-3.5962486267089844], [-3.3500521183013916], [-2.7229509353637695], [-2.3767755031585693], [-2.9794607162475586], [-5.203192234039307], [-3.292698383331299], [-3.7594189643859863], [-3.732757329940796], [-3.4012789726257324], [-2.547316789627075], [-2.113595485687256], [-2.5140676498413086], [-5.272742748260498], [-3.1617307662963867], [-3.709965705871582], [-3.7790520191192627], [-3.5370266437530518], [-2.287034511566162], [-1.8732467889785767], [-2.1769633293151855], [-4.8842363357543945], [-3.1498923301696777], [-3.8360047340393066], [-3.7817606925964355], [-3.674208164215088], [-2.4299895763397217], [-1.9481273889541626], [-2.2568113803863525], [-4.809032440185547], [-3.203739643096924], [-3.894256114959717], [-3.7197253704071045], [-3.811279296875], [-2.5057950019836426], [-2.0324835777282715], [-2.2377896308898926], [-4.79273796081543], [-3.167496681213379], [-3.6729907989501953], [-3.7409868240356445], [-3.8900985717773438], [-2.517928123474121], [-2.085160255432129], [-2.204406499862671], [-4.476906776428223], [-3.0247135162353516], [-3.3603739738464355], [-3.733729362487793], [-4.003390312194824], [-2.5111379623413086], [-2.1227169036865234], [-2.1616055965423584], [-4.280056476593018], [-3.1476824283599854], [-3.284939765930176], [-3.5280537605285645], [-4.004054069519043], [-2.479815721511841], [-2.0832598209381104], [-2.1735243797302246], [-4.028883934020996], [-2.8546695709228516], [-5.135047912597656], [-3.472463607788086], [-4.031004905700684], [-2.346458673477173], [-2.040801525115967], [-2.221790075302124], [-4.339855194091797], [-3.024780511856079], [-3.8415279388427734], [-3.708129644393921], [-4.184869766235352], [-2.4905357360839844], [-2.205165147781372], [-2.2500195503234863], [-3.830564260482788], [-2.9054484367370605], [-3.8330211639404297], [-3.538437604904175], [-4.1763081550598145], [-2.4911670684814453], [-2.208170175552368], [-2.2881338596343994], [-3.502016544342041], [-2.874577522277832], [-3.443112850189209], [-3.5297534465789795], [-4.116558074951172], [-2.3881592750549316], [-2.120121717453003], [-2.2281429767608643], [-3.5904197692871094], [-2.7474117279052734], [-3.0000109672546387], [-3.4756243228912354], [-4.057490348815918], [-2.3550848960876465], [-2.1465511322021484], [-2.296060562133789], [-3.423558235168457], [-2.7227587699890137], [-2.926314115524292], [-3.155421018600464], [-3.8750150203704834], [-2.317171335220337], [-2.1981804370880127], [-2.2720916271209717], [-3.3031227588653564], [-2.679434299468994], [-2.67433762550354], [-3.415773391723633], [-4.033443927764893], [-2.472675323486328], [-2.3977558612823486], [-2.395137071609497], [-3.388845443725586], [-2.8307366371154785], [-3.1820380687713623], [-3.379044771194458], [-4.130788803100586], [-2.4755055904388428], [-2.3602936267852783], [-2.4117419719696045], [-3.2049059867858887], [-2.7125535011291504], [-2.8958773612976074], [-3.4741060733795166], [-4.144061088562012], [-2.7254254817962646], [-2.6585144996643066], [-2.6285369396209717], [-3.166553020477295], [-2.790513515472412], [-3.266775131225586], [-3.0983612537384033], [-3.816937208175659], [-2.5469412803649902], [-2.4206464290618896], [-2.4263386726379395], [-3.000765562057495], [-2.5916852951049805], [-4.455799579620361], [-3.155489444732666], [-3.771697998046875], [-2.603659152984619], [-2.567476272583008], [-2.458606719970703], [-3.4737415313720703], [-2.750251293182373], [-3.886612892150879], [-3.3631725311279297], [-4.0264105796813965], [-2.6563808917999268], [-2.6142210960388184], [-2.587613105773926], [-3.447003126144409], [-2.845402956008911], [-4.080188274383545], [-3.441471576690674], [-4.056282997131348], [-2.76839542388916], [-2.730438709259033], [-2.755331516265869], [-3.366610527038574], [-2.8473312854766846], [-3.438572406768799], [-3.302333354949951], [-4.012025356292725], [-2.7066521644592285], [-2.6642379760742188], [-2.733356475830078], [-3.3502445220947266], [-2.7474634647369385], [-2.9872095584869385], [-3.160595178604126], [-3.956681966781616], [-2.6496293544769287], [-2.6137173175811768], [-2.610538959503174], [-3.3916871547698975], [-2.761270046234131], [-3.066728115081787], [-3.053175687789917], [-3.845890998840332], [-2.5704703330993652], [-2.482295513153076], [-2.377286911010742], [-3.45330810546875], [-2.7584733963012695], [-2.7889020442962646], [-3.3100645542144775], [-4.016964912414551], [-2.7278318405151367], [-2.716367721557617], [-2.547883987426758], [-3.494657039642334], [-2.91843581199646], [-3.096928119659424], [-3.195547103881836], [-4.078264236450195], [-2.727675199508667], [-2.701345682144165], [-2.6138572692871094], [-3.2675936222076416], [-2.798149585723877], [-2.747892141342163], [-3.3961918354034424], [-4.055111885070801], [-2.8253610134124756], [-2.8022124767303467], [-2.7324352264404297], [-3.2563395500183105], [-2.9289698600769043], [-3.161409378051758], [-3.0892155170440674], [-3.8805861473083496], [-2.899251937866211], [-2.681178569793701], [-2.6384615898132324], [-3.1615285873413086], [-2.702582359313965], [-3.9901344776153564], [-3.0604772567749023], [-3.699664831161499], [-2.8672633171081543], [-2.5609076023101807], [-2.386392116546631], [-3.450453042984009], [-2.62668514251709], [-3.6577517986297607], [-3.306281328201294], [-3.9494171142578125], [-2.9576709270477295], [-2.840526819229126], [-2.739490032196045], [-3.390897035598755], [-2.76428484916687], [-3.4731738567352295], [-3.4074525833129883], [-4.002161026000977], [-3.1519720554351807], [-3.1033413410186768], [-2.934561252593994], [-3.2542884349823], [-2.830474853515625], [-3.329129695892334], [-3.327678680419922], [-3.939371347427368], [-3.1112637519836426], [-3.0389485359191895], [-2.9735100269317627], [-3.2114834785461426], [-2.697648286819458], [-2.960087299346924], [-3.382797956466675], [-3.8711726665496826], [-3.038339853286743], [-2.965508222579956], [-2.9370248317718506], [-3.130129814147949], [-2.655726432800293], [-2.828805923461914], [-3.200847864151001], [-3.733997344970703], [-2.9403762817382812], [-2.7187857627868652], [-2.602847099304199], [-3.108339548110962], [-2.5637216567993164], [-2.5126726627349854], [-3.274972915649414], [-3.8162355422973633], [-3.046719551086426], [-2.9648404121398926], [-2.751836061477661], [-3.1509900093078613], [-2.706282377243042], [-2.734219551086426], [-3.132106065750122], [-3.790893077850342], [-2.8981921672821045], [-2.8040385246276855], [-2.6840896606445312], [-2.9882516860961914], [-2.5822699069976807], [-2.5057430267333984], [-3.268061876296997], [-3.869846820831299], [-2.9899659156799316], [-2.91151762008667], [-2.740999937057495], [-2.9471120834350586], [-2.655282974243164], [-2.7110507488250732], [-3.1046581268310547], [-3.7542057037353516], [-3.0571537017822266], [-2.7020797729492188], [-2.583533763885498], [-2.841658115386963], [-2.4317500591278076], [-4.535123825073242], [-3.0092480182647705], [-3.6236672401428223], [-2.910571813583374], [-2.554750919342041], [-2.4116175174713135], [-2.9890851974487305], [-2.3250484466552734], [-3.066993236541748], [-3.2430224418640137], [-3.879951000213623], [-3.025676965713501], [-2.862931489944458], [-2.7611653804779053], [-3.0940093994140625], [-2.6080024242401123], [-3.0767292976379395], [-3.3543283939361572], [-3.919978618621826], [-3.2318620681762695], [-3.0806479454040527], [-2.905336856842041], [-2.9363017082214355], [-2.6144962310791016], [-3.106537342071533], [-3.303500175476074], [-3.8722317218780518], [-3.1726021766662598], [-2.9862594604492188], [-2.9007675647735596], [-2.8748927116394043], [-2.5799238681793213], [-2.9275498390197754], [-3.232797145843506], [-3.827728509902954], [-3.152682304382324], [-2.977900981903076], [-2.9310860633850098], [-2.8876113891601562], [-2.569924831390381], [-2.937041759490967], [-3.0091605186462402], [-3.7201836109161377], [-3.0235109329223633], [-2.6895289421081543], [-2.583458185195923], [-2.8790180683135986], [-2.4152817726135254], [-2.6710853576660156], [-3.0919342041015625], [-3.7484679222106934], [-3.072103977203369], [-2.8091025352478027], [-2.693146228790283], [-3.005251884460449], [-2.598769187927246], [-2.8888659477233887], [-3.0383265018463135], [-3.836486339569092], [-3.0531792640686035], [-2.892537832260132], [-2.8254306316375732], [-2.7800028324127197], [-2.6048214435577393], [-2.6699588298797607], [-3.1474015712738037], [-3.8490548133850098], [-3.0975375175476074], [-2.904897689819336], [-2.8561220169067383], [-2.8514044284820557], [-2.778257131576538], [-3.0959577560424805], [-3.0772719383239746], [-3.7788593769073486], [-3.11238169670105], [-2.7520251274108887], [-2.746412515640259], [-2.793661594390869], [-2.510711669921875], [-4.295222282409668], [-3.027052402496338], [-3.6460118293762207], [-3.016228199005127], [-2.6630256175994873], [-2.5573415756225586], [-3.402331829071045], [-2.674072742462158], [-3.0486929416656494], [-3.2769057750701904], [-3.9950859546661377], [-3.2196600437164307], [-3.1050453186035156], [-2.969278335571289], [-3.4232914447784424], [-2.7894840240478516], [-3.3626346588134766], [-3.4933228492736816], [-4.0912885665893555], [-3.417227268218994], [-3.370995044708252], [-3.2418053150177], [-3.2631843090057373], [-2.9207100868225098], [-3.2789716720581055], [-3.4469523429870605], [-4.027719020843506], [-3.39505672454834], [-3.289363145828247], [-3.229820966720581], [-3.270352363586426], [-2.843055486679077], [-3.1328699588775635], [-3.3853769302368164], [-3.9208431243896484], [-3.294778823852539], [-3.2019693851470947], [-3.1809754371643066], [-3.2545509338378906], [-2.8866958618164062], [-3.1352782249450684], [-3.108180522918701], [-3.753723621368408], [-3.1459057331085205], [-2.8914527893066406], [-2.8299739360809326], [-3.1975159645080566], [-2.6935720443725586], [-2.7886619567871094], [-3.180452823638916], [-3.7987427711486816], [-3.1926255226135254], [-3.072432518005371], [-2.8815739154815674], [-3.260772705078125], [-2.712655544281006], [-3.047394037246704], [-3.0763325691223145], [-3.803225040435791], [-3.1564512252807617], [-3.0740487575531006], [-2.962618350982666], [-3.112096071243286], [-2.7545487880706787], [-2.8254482746124268], [-3.3255155086517334], [-3.8312597274780273], [-3.1985819339752197], [-3.121063709259033], [-3.048802614212036], [-3.320197343826294], [-2.950960636138916], [-3.1894898414611816], [-3.101832866668701], [-3.756287097930908], [-3.243901491165161], [-2.9573025703430176], [-2.913924217224121], [-3.0539398193359375], [-2.553288459777832], [-3.984586238861084], [-3.009453773498535], [-3.5376837253570557], [-3.041944742202759], [-2.6505439281463623], [-2.4188127517700195], [-3.196794033050537], [-2.3315012454986572], [-3.0709123611450195], [-3.294649124145508], [-3.862866163253784], [-3.3014674186706543], [-3.0914626121520996], [-2.974423408508301], [-3.0324313640594482], [-2.518920660018921], [-3.0278146266937256], [-3.433835506439209], [-3.926903486251831], [-3.396409511566162], [-3.2318758964538574], [-3.107344627380371], [-2.9854376316070557], [-2.634305715560913], [-3.1417489051818848], [-3.371551990509033], [-3.8594484329223633], [-3.305750846862793], [-3.0833020210266113], [-3.0137805938720703], [-2.866870403289795], [-2.5058717727661133], [-2.899845838546753], [-3.3569135665893555], [-3.799060821533203], [-3.2622642517089844], [-3.0580050945281982], [-3.0193142890930176], [-2.922239303588867], [-2.5154945850372314], [-2.8869235515594482], [-3.2986326217651367], [-3.806964635848999], [-3.2242796421051025], [-2.9691548347473145], [-2.862969398498535], [-2.972041368484497], [-2.5057995319366455], [-2.7680773735046387], [-3.3304643630981445], [-3.926485538482666], [-3.3832781314849854], [-3.2387876510620117], [-3.0035510063171387], [-2.8867430686950684], [-2.461895227432251], [-2.874433994293213], [-3.3034188747406006], [-3.9354259967803955], [-3.381701946258545], [-3.2519869804382324], [-3.1549954414367676], [-2.804753303527832], [-2.5874733924865723], [-2.8049473762512207], [-3.5051255226135254], [-3.9563920497894287], [-3.393949031829834], [-3.3012351989746094], [-3.221285104751587], [-2.7689013481140137], [-2.730234146118164], [-3.238210678100586], [-3.308086395263672], [-3.8236443996429443], [-3.402456045150757], [-3.1107304096221924], [-3.0275890827178955], [-2.627351999282837], [-2.348923444747925], [-3.7539916038513184], [-3.229466438293457], [-3.6952037811279297], [-3.2762489318847656], [-3.024171829223633], [-2.834960460662842], [-2.8530304431915283], [-2.4097354412078857], [-3.1349966526031494], [-3.4111833572387695], [-3.9793858528137207], [-3.536642551422119], [-3.3145809173583984], [-3.2437994480133057], [-2.865528106689453], [-2.6916725635528564], [-3.1649131774902344], [-3.4530296325683594], [-3.9442942142486572], [-3.532402992248535], [-3.293041706085205], [-3.249469757080078], [-2.658794403076172], [-2.647170066833496], [-3.296597480773926], [-3.4606099128723145], [-3.902078628540039], [-3.47898006439209], [-3.0968685150146484], [-3.1101865768432617], [-2.6883492469787598], [-2.625246047973633], [-3.044886589050293], [-3.417605400085449], [-3.8265740871429443], [-3.3930106163024902], [-3.050856590270996], [-3.0786080360412598], [-2.7162671089172363], [-2.614358901977539], [-3.029118061065674], [-3.120817184448242], [-3.6690568923950195], [-3.1802303791046143], [-2.7332053184509277], [-2.6569743156433105], [-2.8592584133148193], [-2.5555052757263184], [-2.794926643371582], [-3.329676866531372], [-3.843104600906372], [-3.431717872619629], [-3.115694522857666], [-3.006934404373169], [-2.8927161693573], [-2.771969795227051], [-3.0985262393951416], [-3.3627560138702393], [-4.031410217285156], [-3.5628819465637207], [-3.2239112854003906], [-3.2190001010894775], [-2.8411803245544434], [-2.8179051876068115], [-2.8407933712005615], [-3.4907772541046143], [-3.9198992252349854], [-3.513185501098633], [-3.153759241104126], [-3.1190171241760254], [-2.955325126647949], [-3.0180397033691406], [-3.1650242805480957], [-3.3932175636291504], [-3.864010810852051], [-3.5058014392852783], [-3.060795783996582], [-3.019566774368286], [-2.8086724281311035], [-2.557786464691162], [-3.758640766143799], [-3.1801061630249023], [-3.586951971054077], [-3.2146682739257812], [-2.7669260501861572], [-2.5549402236938477], [-3.1608564853668213], [-2.6960854530334473], [-3.0531997680664062], [-3.4267263412475586], [-3.8595032691955566], [-3.4517011642456055], [-3.0779123306274414], [-2.9841978549957275], [-3.2594261169433594], [-2.7941360473632812], [-3.2054953575134277], [-3.5676653385162354], [-3.884894609451294], [-3.4993929862976074], [-3.1758859157562256], [-3.133627414703369], [-3.1515846252441406], [-2.8820602893829346], [-3.079322576522827], [-3.529384136199951], [-3.859815835952759], [-3.4623498916625977], [-3.101991891860962], [-3.114994764328003], [-2.927560806274414], [-2.7071175575256348], [-2.7987937927246094], [-3.44907808303833], [-3.7480454444885254], [-3.3471152782440186], [-3.0190157890319824], [-3.004462480545044], [-2.862771987915039], [-2.634998321533203], [-2.7141354084014893], [-3.160027265548706], [-3.5566444396972656], [-3.1041688919067383], [-2.692730188369751], [-2.61314058303833], [-2.9437198638916016], [-2.607759475708008], [-2.593719005584717], [-3.4252164363861084], [-3.802501678466797], [-3.3863446712493896], [-3.0950379371643066], [-2.9868483543395996], [-2.951575994491577], [-2.7338995933532715], [-2.80823016166687], [-3.424727439880371], [-3.9431984424591064], [-3.507297992706299], [-3.2219784259796143], [-3.2142210006713867], [-2.9190421104431152], [-2.769955635070801], [-2.6327760219573975], [-3.490687370300293], [-3.9137845039367676], [-3.5170719623565674], [-3.219968557357788], [-3.2301931381225586], [-2.9572811126708984], [-3.012561798095703], [-3.081015110015869], [-3.3550243377685547], [-3.7844724655151367], [-3.449817419052124], [-3.062757730484009], [-3.004728317260742], [-2.710850238800049], [-2.4290623664855957], [-3.6957039833068848], [-3.1508092880249023], [-3.5531063079833984], [-3.2513701915740967], [-2.8134853839874268], [-2.709958076477051], [-2.5848147869110107], [-2.291872501373291], [-2.8649346828460693], [-3.4370932579040527], [-3.8512067794799805], [-3.552619457244873], [-3.2169528007507324], [-3.2407848834991455], [-2.6746294498443604], [-2.599689483642578], [-2.9285759925842285], [-3.437777042388916], [-3.8265836238861084], [-3.537773370742798], [-3.2016634941101074], [-3.2524704933166504], [-2.658144474029541], [-2.762173652648926], [-3.0995121002197266], [-3.264481544494629], [-3.6513803005218506], [-3.368347406387329], [-2.944382905960083], [-3.054576873779297], [-2.4868855476379395], [-2.5668814182281494], [-2.783412456512451], [-3.181342124938965], [-3.5173301696777344], [-3.238799571990967], [-2.885099172592163], [-2.977668523788452], [-2.458744764328003], [-2.440518856048584], [-2.627070903778076], [-2.774517059326172], [-3.2667224407196045], [-2.907304286956787], [-2.479236602783203], [-2.4501259326934814], [-2.5030758380889893], [-2.359438419342041], [-2.4545862674713135], [-2.8487112522125244], [-3.347644567489624], [-3.076511859893799], [-2.7382187843322754], [-2.609323263168335], [-2.581723213195801], [-2.5250673294067383], [-2.7213134765625], [-3.008920192718506], [-3.6400961875915527], [-3.264674425125122], [-2.9725921154022217], [-2.9420738220214844], [-2.6470398902893066], [-2.631930112838745], [-2.5849618911743164], [-3.1815543174743652], [-3.645673990249634], [-3.3037900924682617], [-3.0025718212127686], [-2.8881304264068604], [-2.9212207794189453], [-2.870828151702881], [-3.036044120788574], [-3.0864479541778564], [-3.6391990184783936], [-3.3263533115386963], [-2.934431552886963], [-2.643834114074707], [-2.8082163333892822], [-2.2416751384735107], [-4.997744560241699], [-2.641695737838745], [-3.1480512619018555], [-2.7309954166412354], [-2.342461585998535], [-2.134186267852783], [-3.3599090576171875], [-2.389754295349121], [-3.8489699363708496], [-3.481875419616699], [-3.835409641265869], [-3.2867536544799805], [-3.135392189025879], [-3.1841211318969727], [-3.2435195446014404], [-2.6728482246398926], [-3.565577983856201], [-3.4907386302948], [-3.817366361618042], [-3.4380345344543457], [-3.2320961952209473], [-3.1732540130615234], [-3.00285005569458], [-2.7677855491638184], [-3.522904634475708], [-3.5236949920654297], [-3.7747085094451904], [-3.4220614433288574], [-3.1458094120025635], [-3.1818671226501465], [-2.9903762340545654], [-2.6828465461730957], [-3.038447856903076], [-3.425595760345459], [-3.6188178062438965], [-3.290921211242676], [-3.036555290222168], [-3.1029934883117676], [-2.7108778953552246], [-2.5454444885253906], [-2.9284615516662598], [-3.0249733924865723], [-3.4180800914764404], [-3.0599350929260254], [-2.679440975189209], [-2.603497266769409], [-2.686664581298828], [-2.444148302078247], [-2.622375011444092], [-3.2352237701416016], [-3.600078821182251], [-3.2915329933166504], [-3.012944221496582], [-2.9030768871307373], [-2.9213004112243652], [-2.5697474479675293], [-3.1269338130950928], [-3.297281265258789], [-3.792901039123535], [-3.4313840866088867], [-3.1427531242370605], [-3.144068479537964], [-2.790400981903076], [-2.6738078594207764], [-2.851374626159668], [-3.3939590454101562], [-3.8001742362976074], [-3.471911668777466], [-3.185490131378174], [-3.113187551498413], [-2.9338226318359375], [-2.7795844078063965], [-3.562433958053589], [-3.232546091079712], [-3.7366745471954346], [-3.478037118911743], [-3.048588991165161], [-2.967127799987793], [-2.7707386016845703], [-2.2879397869110107], [-3.0585551261901855], [-3.2217118740081787], [-3.562098741531372], [-3.3006858825683594], [-2.8595352172851562], [-2.712482213973999], [-3.0600295066833496], [-2.4890568256378174], [-3.295520782470703], [-3.5017483234405518], [-3.851046323776245], [-3.6015026569366455], [-3.247631311416626], [-3.267113208770752], [-2.940824270248413], [-2.527393341064453], [-3.2423627376556396], [-3.4514875411987305], [-3.772226572036743], [-3.578272819519043], [-3.2231717109680176], [-3.2506842613220215], [-2.684187412261963], [-2.6430392265319824], [-3.320096492767334], [-3.3535313606262207], [-3.661318778991699], [-3.4195094108581543], [-2.977952003479004], [-3.041194438934326], [-2.5273337364196777], [-2.498664379119873], [-2.8820204734802246], [-3.3365023136138916], [-3.584920644760132], [-3.37117338180542], [-2.966648817062378], [-3.056631565093994], [-2.4598677158355713], [-2.290769338607788], [-2.8910064697265625], [-2.918410301208496], [-3.323519706726074], [-3.0374629497528076], [-2.551300525665283], [-2.5643091201782227], [-2.3947341442108154], [-2.2270114421844482], [-2.5369584560394287], [-3.1112887859344482], [-3.514636754989624], [-3.2796175479888916], [-2.900486946105957], [-2.866894245147705], [-2.6816632747650146], [-2.5202133655548096], [-2.976116180419922], [-3.1536645889282227], [-3.724351406097412], [-3.412240505218506], [-3.0115339756011963], [-3.0364625453948975], [-2.5309927463531494], [-2.5785560607910156], [-2.7888803482055664], [-3.5357282161712646], [-3.8761041164398193], [-3.5989294052124023], [-3.22843074798584], [-3.239339590072632], [-2.9750654697418213], [-2.8486084938049316], [-3.531400203704834], [-3.2763261795043945], [-3.730445384979248], [-3.472618579864502], [-3.0076005458831787], [-2.977762222290039], [-2.5539000034332275], [-2.3810126781463623], [-3.1470108032226562], [-3.0243921279907227], [-3.3882980346679688], [-3.1547930240631104], [-2.6889665126800537], [-2.5054664611816406], [-2.591777801513672], [-2.3483340740203857], [-2.9539666175842285], [-3.4012956619262695], [-3.7214832305908203], [-3.4761898517608643], [-3.0995676517486572], [-3.0691843032836914], [-2.893742322921753], [-2.655484199523926], [-3.3345892429351807], [-3.4373271465301514], [-3.7741916179656982], [-3.54217529296875], [-3.1460137367248535], [-3.07498836517334], [-2.7824041843414307], [-2.734251022338867], [-3.4469966888427734], [-3.4238409996032715], [-3.684804677963257], [-3.426290512084961], [-2.985167980194092], [-2.9843270778656006], [-2.5652599334716797], [-2.5611166954040527], [-2.931624412536621], [-3.278148651123047], [-3.544569492340088], [-3.3001651763916016], [-2.9162063598632812], [-2.938485622406006], [-2.3827269077301025], [-2.4239583015441895], [-2.934877872467041], [-2.9046950340270996], [-3.320268392562866], [-3.002073287963867], [-2.5128841400146484], [-2.4146275520324707], [-2.4223287105560303], [-2.337623357772827], [-2.4254393577575684], [-3.146881103515625], [-3.481025218963623], [-3.182342052459717], [-2.804279327392578], [-2.708815574645996], [-2.638535976409912], [-2.4622440338134766], [-2.932757616043091], [-3.2871525287628174], [-3.7110848426818848], [-3.335937023162842], [-2.963754177093506], [-2.923792839050293], [-2.611741542816162], [-2.513537645339966], [-2.7864925861358643], [-3.4225645065307617], [-3.716189384460449], [-3.386918783187866], [-3.023972511291504], [-2.9621119499206543], [-2.74324369430542], [-2.672879457473755], [-3.4000864028930664], [-3.370421886444092], [-3.7394304275512695], [-3.4786009788513184], [-3.0315756797790527], [-2.902592658996582], [-2.6288487911224365], [-2.292703151702881], [-3.1351287364959717], [-3.217012405395508], [-3.4396896362304688], [-3.177994966506958], [-2.705684185028076], [-2.4454195499420166], [-2.8730030059814453], [-2.399569034576416], [-3.191361427307129], [-3.5410070419311523], [-3.7142417430877686], [-3.5119471549987793], [-3.1266396045684814], [-3.0673415660858154], [-3.0538177490234375], [-2.562906265258789], [-3.2854886054992676], [-3.495687484741211], [-3.7432548999786377], [-3.579080581665039], [-3.1908493041992188], [-3.098299026489258], [-2.9286046028137207], [-2.6609818935394287], [-3.617393970489502], [-3.4753806591033936], [-3.6897430419921875], [-3.483457088470459], [-3.0658950805664062], [-3.0838754177093506], [-2.6606664657592773], [-2.507230281829834], [-2.8288304805755615], [-3.4191102981567383], [-3.53617000579834], [-3.3194994926452637], [-2.94797420501709], [-2.9721262454986572], [-2.521807909011841], [-2.3236160278320312], [-2.742223024368286], [-2.9646549224853516], [-3.292872667312622], [-3.008470058441162], [-2.5324153900146484], [-2.458242654800415], [-2.449246883392334], [-2.3093013763427734], [-2.345308780670166], [-3.118659496307373], [-3.452319622039795], [-3.2350616455078125], [-2.8765339851379395], [-2.8064754009246826], [-2.757309675216675], [-2.4547152519226074], [-2.934093952178955], [-3.2687392234802246], [-3.6503047943115234], [-3.3517472743988037], [-2.9860727787017822], [-2.954586982727051], [-2.7319326400756836], [-2.5278379917144775], [-2.7103679180145264], [-3.3690080642700195], [-3.684565544128418], [-3.4038496017456055], [-3.0428578853607178], [-2.9786016941070557], [-2.848008871078491], [-2.7249393463134766], [-3.282515287399292], [-3.24224853515625], [-3.6659255027770996], [-3.431363105773926], [-2.9756247997283936], [-2.874397039413452], [-2.545549154281616], [-2.3420560359954834], [-3.1602134704589844], [-3.0126850605010986], [-3.371856212615967], [-3.1283516883850098], [-2.704622507095337], [-2.5314502716064453], [-2.3329994678497314], [-2.181795358657837], [-2.58194637298584], [-3.260038137435913], [-3.6390750408172607], [-3.443636655807495], [-3.073936939239502], [-2.9382288455963135], [-2.6380982398986816], [-2.5467429161071777], [-2.844155788421631], [-3.2590761184692383], [-3.596987724304199], [-3.397275447845459], [-3.023470878601074], [-2.9451024532318115], [-2.52231764793396], [-2.6014516353607178], [-2.926614761352539], [-3.247098445892334], [-3.567565679550171], [-3.323537826538086], [-2.9038686752319336], [-2.8909566402435303], [-2.351017951965332], [-2.464599609375], [-2.618572473526001], [-3.0008318424224854], [-3.3584275245666504], [-3.1483657360076904], [-2.755417823791504], [-2.769083023071289], [-2.257073163986206], [-2.2310619354248047], [-2.640090227127075], [-2.756155490875244], [-3.1554601192474365], [-2.88824462890625], [-2.422485828399658], [-2.361837863922119], [-2.1907122135162354], [-2.179175853729248], [-2.2744033336639404], [-2.80633807182312], [-3.259903907775879], [-3.0246200561523438], [-2.667954206466675], [-2.5753676891326904], [-2.435060977935791], [-2.3502426147460938], [-2.710244655609131], [-2.8579211235046387], [-3.442357063293457], [-3.114640712738037], [-2.7407212257385254], [-2.696715831756592], [-2.4022979736328125], [-2.521280288696289], [-2.604094982147217], [-3.130936861038208], [-3.5247814655303955], [-3.2162532806396484], [-2.8506386280059814], [-2.8146095275878906], [-2.454864740371704], [-2.6955432891845703], [-3.1879711151123047], [-3.0694684982299805], [-3.5853686332702637], [-3.3176541328430176], [-2.8888187408447266], [-2.7516441345214844], [-2.3814849853515625], [-2.3203933238983154], [-3.332015037536621], [-2.7918577194213867], [-3.221040964126587], [-2.9509520530700684], [-2.515359401702881], [-2.240274429321289], [-2.710294723510742], [-2.4265146255493164], [-2.883511543273926], [-3.2317566871643066], [-3.6661629676818848], [-3.5157132148742676], [-3.154888868331909], [-3.022998332977295], [-2.9311633110046387], [-2.751699924468994], [-3.177746295928955], [-3.331028938293457], [-3.720273971557617], [-3.587538719177246], [-3.22223162651062], [-3.1272711753845215], [-2.894808292388916], [-2.8703219890594482], [-3.270787000656128], [-3.496811866760254], [-3.8052453994750977], [-3.6398842334747314], [-3.223083972930908], [-3.16959810256958], [-2.747295379638672], [-2.7526063919067383], [-2.8468546867370605], [-3.4252841472625732], [-3.6472742557525635], [-3.495584011077881], [-3.138165235519409], [-3.1507816314697266], [-2.7097253799438477], [-2.659017562866211], [-3.0421667098999023], [-2.8085150718688965], [-3.2467291355133057], [-3.040405750274658], [-2.6211581230163574], [-2.5361087322235107], [-2.6175460815429688], [-2.581294536590576], [-2.397932529449463], [-2.9661824703216553], [-3.3943426609039307], [-3.243824005126953], [-2.9330332279205322], [-2.80828595161438], [-2.8558685779571533], [-2.6549510955810547], [-3.0997235774993896], [-3.0882811546325684], [-3.6176364421844482], [-3.3678598403930664], [-3.026418685913086], [-2.9394497871398926], [-2.830770969390869], [-2.734785556793213], [-2.902885675430298], [-3.2930002212524414], [-3.66241717338562], [-3.429262161254883], [-3.0735578536987305], [-2.9926297664642334], [-3.0805516242980957], [-2.9205784797668457], [-3.3658926486968994], [-3.105604887008667], [-3.6364636421203613], [-3.422640800476074], [-2.9676871299743652], [-2.8062093257904053], [-2.7586569786071777], [-2.4597091674804688], [-3.287386178970337], [-3.018235683441162], [-3.37151837348938], [-3.146873712539673], [-2.7184200286865234], [-2.4454243183135986], [-2.8317267894744873], [-2.5290777683258057], [-3.2094836235046387], [-3.467600107192993], [-3.767857789993286], [-3.556270122528076], [-3.2194480895996094], [-3.107192039489746], [-3.1031880378723145], [-2.7277469635009766], [-3.317214250564575], [-3.493439197540283], [-3.8041486740112305], [-3.6141438484191895], [-3.2926063537597656], [-3.185319662094116], [-3.009775161743164], [-2.8376364707946777], [-3.4733355045318604], [-3.4518654346466064], [-3.758373498916626], [-3.5416507720947266], [-3.151158571243286], [-3.1094655990600586], [-2.7530934810638428], [-2.712601661682129], [-2.882924795150757], [-3.415604829788208], [-3.637073040008545], [-3.41583251953125], [-3.0981369018554688], [-3.1043951511383057], [-2.6454334259033203], [-2.619405746459961], [-2.9913251399993896], [-3.238988161087036], [-3.572427988052368], [-3.2832419872283936], [-2.8783023357391357], [-2.7861523628234863], [-2.625786304473877], [-2.515207290649414], [-2.7463631629943848], [-3.264078140258789], [-3.646012306213379], [-3.442460775375366], [-3.14827823638916], [-3.039522886276245], [-2.7585182189941406], [-2.559166431427002], [-3.1382558345794678], [-3.3004353046417236], [-3.751485824584961], [-3.507065534591675], [-3.183985710144043], [-3.1453640460968018], [-2.795884609222412], [-2.618122100830078], [-2.8723905086517334], [-3.4296319484710693], [-3.7344579696655273], [-3.5080313682556152], [-3.1982951164245605], [-3.1656010150909424], [-2.798642635345459], [-2.762244462966919], [-3.383389949798584], [-3.3695316314697266], [-3.734748363494873], [-3.5395305156707764], [-3.1320111751556396], [-3.051684856414795], [-2.6936728954315186], [-2.4074368476867676], [-3.3344902992248535], [-3.378732919692993], [-3.669884204864502], [-3.4845118522644043], [-3.1243321895599365], [-2.9951229095458984], [-2.857006549835205], [-2.553025245666504], [-3.3101961612701416], [-3.5732264518737793], [-3.942887306213379], [-3.80142879486084], [-3.4761695861816406], [-3.379910469055176], [-3.03538179397583], [-2.753514528274536], [-3.60799503326416], [-3.595752477645874], [-3.8866052627563477], [-3.7812371253967285], [-3.44804310798645], [-3.3878116607666016], [-2.8401341438293457], [-2.8566417694091797], [-3.683255434036255], [-3.57861065864563], [-3.8425440788269043], [-3.7111306190490723], [-3.3018336296081543], [-3.2605128288269043], [-2.726048707962036], [-2.756875514984131], [-3.20404314994812], [-3.5732510089874268], [-3.794538974761963], [-3.668351888656616], [-3.2935688495635986], [-3.3062829971313477], [-2.6648151874542236], [-2.6641845703125], [-3.1933391094207764], [-3.196021556854248], [-3.561269521713257], [-3.377262592315674], [-2.9040286540985107], [-2.8485028743743896], [-2.5644679069519043], [-2.582756519317627], [-2.699636459350586], [-3.2496395111083984], [-3.633868932723999], [-3.520700693130493], [-3.156362771987915], [-3.066011667251587], [-2.8338332176208496], [-2.7110848426818848], [-3.353687286376953], [-3.2272744178771973], [-3.7708046436309814], [-3.6146883964538574], [-3.207472801208496], [-3.1550400257110596], [-2.812927722930908], [-2.7097978591918945], [-3.139752149581909], [-3.4301090240478516], [-3.7430801391601562], [-3.594717025756836], [-3.1768381595611572], [-3.1292457580566406], [-3.0190839767456055], [-2.91629695892334], [-3.683408498764038], [-3.314305543899536], [-3.774728298187256], [-3.642749786376953], [-3.16917085647583], [-3.0878570079803467], [-2.893242359161377], [-2.5743157863616943], [-3.8028323650360107], [-3.098750352859497], [-3.3611648082733154], [-3.240027666091919], [-2.735227584838867], [-2.503072738647461], [-2.9290966987609863], [-2.57271146774292], [-3.74971866607666], [-3.423412561416626], [-3.6844730377197266], [-3.6207494735717773], [-3.219633102416992], [-3.0850679874420166], [-3.2087368965148926], [-2.7718658447265625], [-3.8732666969299316], [-3.485156774520874], [-3.7062134742736816], [-3.646282196044922], [-3.2640552520751953], [-3.1448121070861816], [-3.0670127868652344], [-2.861415386199951], [-3.9854774475097656], [-3.397080421447754], [-3.652838945388794], [-3.554008960723877], [-3.1322507858276367], [-3.084904432296753], [-2.6851930618286133], [-2.702328681945801], [-3.2643210887908936], [-3.394766330718994], [-3.554903268814087], [-3.453718423843384], [-3.0947163105010986], [-3.1041760444641113], [-2.5907907485961914], [-2.5709614753723145], [-3.3679962158203125], [-3.025139331817627], [-3.296635627746582], [-3.118659734725952], [-2.6554973125457764], [-2.563690185546875], [-2.565920829772949], [-2.479855537414551], [-2.8879451751708984], [-3.2139058113098145], [-3.463848829269409], [-3.3464810848236084], [-2.982152223587036], [-2.894296646118164], [-2.7713301181793213], [-2.5889720916748047], [-3.588773250579834], [-3.3452858924865723], [-3.6651129722595215], [-3.4840786457061768], [-3.1079347133636475], [-3.0654337406158447], [-2.8054471015930176], [-2.6371593475341797], [-3.3767426013946533], [-3.4104013442993164], [-3.65047287940979], [-3.4797515869140625], [-3.106093645095825], [-3.064277172088623], [-2.870631456375122], [-2.808246374130249], [-3.8924951553344727], [-3.27691650390625], [-3.6088690757751465], [-3.4558262825012207], [-3.006011724472046], [-2.865206718444824], [-2.644439697265625], [-2.3443191051483154], [-3.8698482513427734], [-3.106802463531494], [-3.4090259075164795], [-3.270066499710083], [-2.792320489883423], [-2.6240859031677246], [-2.59503173828125], [-2.362605094909668], [-3.0311508178710938], [-3.367177724838257], [-3.664647102355957], [-3.599813938140869], [-3.2263283729553223], [-3.1302268505096436], [-2.7968785762786865], [-2.5684337615966797], [-3.0320584774017334], [-3.3292977809906006], [-3.6408352851867676], [-3.5774612426757812], [-3.1956794261932373], [-3.118408441543579], [-2.5956344604492188], [-2.6913490295410156], [-3.273282051086426], [-3.347306728363037], [-3.6383066177368164], [-3.5348868370056152], [-3.1098415851593018], [-3.0589096546173096], [-2.4412221908569336], [-2.612002372741699], [-2.6984872817993164], [-3.248682975769043], [-3.4714839458465576], [-3.371396541595459], [-3.0083765983581543], [-3.012059211730957], [-2.3238184452056885], [-2.4600045680999756], [-2.801676034927368], [-2.802344799041748], [-3.193342685699463], [-3.014861822128296], [-2.5161869525909424], [-2.401935577392578], [-2.209460735321045], [-2.346442699432373], [-2.3415822982788086], [-2.924604892730713], [-3.329289436340332], [-3.20627760887146], [-2.857482433319092], [-2.7188854217529297], [-2.546884536743164], [-2.500369071960449], [-3.142624616622925], [-3.0674219131469727], [-3.551912307739258], [-3.353278636932373], [-2.978517770767212], [-2.8710861206054688], [-2.5839295387268066], [-2.5982022285461426], [-2.926471710205078], [-3.2576489448547363], [-3.6080379486083984], [-3.4169139862060547], [-3.0052032470703125], [-2.878011703491211], [-2.8134853839874268], [-2.7970457077026367], [-3.495474338531494], [-3.2027883529663086], [-3.6364974975585938], [-3.4870433807373047], [-3.020742416381836], [-2.7501957416534424], [-2.6468324661254883], [-2.298635959625244], [-4.156946182250977], [-2.968641757965088], [-3.3462204933166504], [-3.127084970474243], [-2.6843204498291016], [-2.3027594089508057], [-2.9318113327026367], [-2.5740225315093994], [-3.423370838165283], [-3.61303973197937], [-3.8514952659606934], [-3.679452896118164], [-3.3802080154418945], [-3.2687602043151855], [-3.365943431854248], [-2.910097599029541], [-3.6455130577087402], [-3.637477159500122], [-3.847081422805786], [-3.7182812690734863], [-3.397934913635254], [-3.2744967937469482], [-3.2504637241363525], [-2.9266014099121094], [-3.2006959915161133], [-3.6920533180236816], [-3.8494811058044434], [-3.7047083377838135], [-3.3379201889038086], [-3.282905340194702], [-3.021963357925415], [-2.874441623687744], [-2.720808506011963], [-3.567777395248413], [-3.694612503051758], [-3.5549960136413574], [-3.209871292114258], [-3.210817337036133], [-2.743061065673828], [-2.67962646484375], [-2.606252670288086], [-3.210841655731201], [-3.421172618865967], [-3.2150726318359375], [-2.752065420150757], [-2.6300740242004395], [-2.668910503387451], [-2.6267058849334717], [-2.217494010925293], [-3.295365571975708], [-3.559401512145996], [-3.4092068672180176], [-3.0723962783813477], [-2.965762138366699], [-2.9024295806884766], [-2.7605128288269043], [-2.7977523803710938], [-3.3911468982696533], [-3.736747980117798], [-3.5499472618103027], [-3.1774675846099854], [-3.1003565788269043], [-2.877558708190918], [-2.741466999053955], [-2.5701205730438232], [-3.631467819213867], [-3.7947614192962646], [-3.6200127601623535], [-3.259577989578247], [-3.227219581604004], [-3.111893892288208], [-2.927877426147461], [-3.1118288040161133], [-3.4760541915893555], [-3.82798171043396], [-3.660681962966919], [-3.2026567459106445], [-3.06745982170105], [-2.9163174629211426], [-2.5107827186584473], [-2.7543580532073975], [-3.3247711658477783], [-3.575255870819092], [-3.4082813262939453], [-2.9560506343841553], [-2.7340991497039795], [-3.1581859588623047], [-2.6435978412628174], [-3.050903081893921], [-3.6423263549804688], [-3.855638265609741], [-3.780447483062744], [-3.420980453491211], [-3.3397090435028076], [-3.1515939235687256], [-2.776266574859619], [-3.2487196922302246], [-3.533071517944336], [-3.7864091396331787], [-3.7451727390289307], [-3.383924961090088], [-3.331754684448242], [-3.0292224884033203], [-2.916954517364502], [-3.3414742946624756], [-3.448403835296631], [-3.6725668907165527], [-3.590754508972168], [-3.1735265254974365], [-3.1659228801727295], [-2.853477716445923], [-2.8190603256225586], [-2.577502489089966], [-3.4731130599975586], [-3.618472099304199], [-3.5208663940429688], [-3.1676430702209473], [-3.2039833068847656], [-2.784921646118164], [-2.723785877227783], [-2.597787380218506], [-3.1204042434692383], [-3.379621982574463], [-3.2444405555725098], [-2.7961575984954834], [-2.737828493118286], [-2.7116189002990723], [-2.6735057830810547], [-2.406837224960327], [-3.253469944000244], [-3.5933496952056885], [-3.5163626670837402], [-3.1689529418945312], [-3.1269640922546387], [-2.9418206214904785], [-2.737407922744751], [-2.9498000144958496], [-3.344482660293579], [-3.7540316581726074], [-3.642815351486206], [-3.2756972312927246], [-3.2208633422851562], [-2.899384021759033], [-2.825432300567627], [-2.830540895462036], [-3.7058732509613037], [-3.9513602256774902], [-3.8286702632904053], [-3.4743030071258545], [-3.4488179683685303], [-3.3020009994506836], [-2.988687515258789], [-3.5051140785217285], [-3.5303125381469727], [-3.8839635848999023], [-3.780104160308838], [-3.3542416095733643], [-3.2982075214385986], [-2.9589450359344482]], "row": 0}, {"kind": "span", "head": "broad", "words": "s1_words", "labels": "s1_labels", "logits": [[-2.351999282836914], [-3.2861146926879883], [-2.071457624435425], [-2.4602627754211426], [-1.8625949621200562], [-2.047027826309204], [-2.3239541053771973], [-3.626887321472168], [-2.3483405113220215], [-2.9786715507507324], [-1.4222314357757568], [-1.844503402709961], [-1.3209636211395264], [-1.6392370462417603], [-1.9380539655685425], [-3.5257620811462402], [-2.2541913986206055], [-1.5854437351226807], [-1.782663106918335], [-2.515744924545288], [-1.4551222324371338], [-1.783217191696167], [-2.2827091217041016], [-3.5942647457122803], [-2.288733959197998], [-1.814456820487976], [-2.120316743850708], [-2.666901111602783], [-1.5888009071350098], [-1.8977081775665283], [-2.478454828262329], [-3.6913161277770996], [-2.448300361633301], [-2.0715279579162598], [-1.8338361978530884], [-2.383084297180176], [-1.6827189922332764], [-1.8510253429412842], [-2.4067676067352295], [-3.274144172668457], [-2.5227723121643066], [-2.184110164642334], [-2.219764471054077], [-2.5958662033081055], [-2.131826877593994], [-2.1779565811157227], [-2.542997121810913], [-3.3203463554382324], [-2.709588050842285], [-2.425050973892212], [-2.913311004638672], [-3.049304962158203], [-2.7997827529907227], [-2.8475773334503174], [-2.9129347801208496], [-3.5792112350463867], [-2.94014835357666], [-2.4511265754699707], [-2.6938092708587646], [-3.107053756713867], [-2.5412445068359375], [-2.543881893157959], [-2.8033976554870605], [-3.340123176574707], [-2.9487698078155518], [-2.258321762084961], [-2.7334959506988525], [-2.9328041076660156], [-2.574946165084839], [-2.386353015899658], [-2.7290751934051514], [-3.0835797786712646], [-2.7781600952148438], [-2.3306360244750977], [-2.894840717315674], [-3.1217055320739746], [-2.90492582321167], [-2.634967088699341], [-2.654693126678467], [-2.942322254180908], [-2.67533016204834], [-2.310062885284424], [-2.4764254093170166], [-2.982379913330078], [-2.726024866104126], [-2.5380630493164062], [-2.599724769592285], [-3.0781917572021484], [-2.67268443107605], [-2.4172849655151367], [-3.1995205879211426], [-3.387028694152832], [-3.1597583293914795], [-3.020578622817993], [-2.9183685779571533], [-3.3319602012634277], [-3.005150556564331], [-2.655348300933838], [-3.312901496887207], [-3.502896785736084], [-3.326502799987793], [-3.1171672344207764], [-2.977146863937378], [-3.0647552013397217], [-2.8119218349456787], [-2.6347837448120117], [-3.242929697036743], [-3.385951042175293], [-3.1226367950439453], [-2.92938232421875], [-2.8489975929260254], [-2.873330593109131], [-2.7924108505249023], [-2.5155136585235596], [-2.6101529598236084], [-2.8682923316955566], [-2.4759058952331543], [-2.32432222366333], [-2.509676456451416], [-2.738136053085327], [-2.5751655101776123], [-2.511366367340088], [-2.434668779373169], [-2.8093442916870117], [-2.4597370624542236], [-2.1925864219665527], [-2.265026569366455], [-2.606635808944702], [-2.569589614868164], [-2.4474518299102783], [-3.1315369606018066], [-3.330258846282959], [-3.1834590435028076], [-2.9461333751678467], [-2.7838711738586426], [-2.9982986450195312], [-2.8639140129089355], [-2.640132427215576], [-3.1092467308044434], [-3.345656394958496], [-3.0161149501800537], [-2.8642337322235107], [-2.882172107696533], [-2.9415836334228516], [-2.8425421714782715], [-2.5824174880981445], [-3.3017029762268066], [-3.4343185424804688], [-3.173388719558716], [-2.823185443878174], [-2.8955891132354736], [-2.9469070434570312], [-2.8801913261413574], [-2.73286771774292], [-3.1304006576538086], [-3.2778213024139404], [-3.086940050125122], [-2.7071034908294678], [-2.6154162883758545], [-2.465362310409546], [-2.4083869457244873], [-2.0807299613952637], [-3.0192887783050537], [-3.115584135055542], [-3.0074563026428223], [-2.6592860221862793], [-2.6600799560546875], [-2.6486566066741943], [-2.703080177307129], [-2.603400468826294], [-3.545090913772583], [-3.676462173461914], [-3.633878231048584], [-3.414682626724243], [-3.075254201889038], [-3.167475700378418], [-3.090211868286133], [-2.758925676345825], [-3.440603017807007], [-3.6364927291870117], [-3.616288900375366], [-3.5789742469787598], [-3.2785887718200684], [-3.303689956665039], [-3.055208683013916], [-2.8166756629943848], [-3.42702579498291], [-3.6088550090789795], [-3.3563108444213867], [-3.207009792327881], [-3.053516149520874], [-2.927600860595703], [-3.0033907890319824], [-2.7595555782318115], [-3.1866636276245117], [-3.2940454483032227], [-3.090822458267212], [-2.8461668491363525], [-2.8654515743255615], [-2.7394967079162598], [-2.736072540283203], [-2.7229604721069336], [-2.8041560649871826], [-3.0086841583251953], [-2.684821605682373], [-2.3853864669799805], [-2.4427380561828613], [-2.7070555686950684], [-2.6615591049194336], [-2.5376715660095215], [-3.3045644760131836], [-3.488103151321411], [-3.3025429248809814], [-3.088477849960327], [-2.889988899230957], [-3.0425522327423096], [-2.860687732696533], [-2.597816228866577], [-3.450432062149048], [-3.5520436763763428], [-3.3201825618743896], [-3.097409725189209], [-2.957402467727661], [-3.0354039669036865], [-2.813079595565796], [-2.44451904296875], [-3.3735146522521973], [-3.4592514038085938], [-3.2569403648376465], [-2.933631420135498], [-3.010178327560425], [-2.9987664222717285], [-2.9241271018981934], [-2.674637794494629], [-3.215228319168091], [-3.363966941833496], [-3.175990581512451], [-2.866244077682495], [-2.7509593963623047], [-2.674013137817383], [-2.572397232055664], [-2.283658981323242], [-3.0749685764312744], [-3.1150481700897217], [-2.978799343109131], [-2.6309356689453125], [-2.5329573154449463], [-2.73476243019104], [-2.6534435749053955], [-2.6418957710266113], [-3.441187858581543], [-3.460341453552246], [-3.332648754119873], [-2.9881787300109863], [-2.740382671356201], [-2.963399648666382], [-2.8248517513275146], [-2.6740872859954834], [-3.557119369506836], [-3.7387242317199707], [-3.677743911743164], [-3.3708038330078125], [-2.9666006565093994], [-3.1460883617401123], [-2.711873769760132], [-2.7425618171691895], [-3.531107187271118], [-3.722398042678833], [-3.4397730827331543], [-3.184204578399658], [-2.9603166580200195], [-3.077047109603882], [-3.134415864944458], [-2.9356689453125], [-3.5724692344665527], [-3.5326180458068848], [-3.233170509338379], [-2.9591586589813232], [-2.9584543704986572], [-2.974632501602173], [-2.9423813819885254], [-2.8333935737609863], [-3.0787997245788574], [-3.1498305797576904], [-2.8077139854431152], [-2.4857091903686523], [-2.46703839302063], [-2.836735248565674], [-2.764958381652832], [-2.547402858734131], [-3.6435322761535645], [-3.801168441772461], [-3.6398744583129883], [-3.389495372772217], [-3.0315918922424316], [-3.1958963871002197], [-3.0875203609466553], [-2.850008487701416], [-3.335598945617676], [-3.463289499282837], [-3.205141067504883], [-2.9718379974365234], [-2.910705327987671], [-2.921110153198242], [-2.839139461517334], [-2.5143721103668213], [-3.309785842895508], [-3.4399709701538086], [-3.2073137760162354], [-2.9310050010681152], [-2.9668822288513184], [-2.9172043800354004], [-2.955580711364746], [-2.693939685821533], [-3.0783305168151855], [-3.2795238494873047], [-3.0381486415863037], [-2.7074310779571533], [-2.557918071746826], [-2.4931492805480957], [-2.4969704151153564], [-2.3251161575317383], [-2.837054967880249], [-2.973512649536133], [-2.750908374786377], [-2.461597204208374], [-2.4562926292419434], [-2.539227247238159], [-2.573533535003662], [-2.724760055541992], [-3.291538715362549], [-3.3779118061065674], [-3.191070556640625], [-2.919306755065918], [-2.827911138534546], [-2.878228187561035], [-2.968708038330078], [-2.793642520904541], [-3.5484941005706787], [-3.569383144378662], [-3.4354677200317383], [-3.2140207290649414], [-3.2014548778533936], [-3.123699188232422], [-3.0571982860565186], [-2.901228427886963], [-3.4406657218933105], [-3.5101940631866455], [-3.1978225708007812], [-3.009692907333374], [-2.9862921237945557], [-2.8861544132232666], [-3.0008277893066406], [-2.766249895095825], [-3.2069902420043945], [-3.356257438659668], [-3.1127207279205322], [-2.8949034214019775], [-2.8696699142456055], [-2.690540313720703], [-2.7765419483184814], [-2.6917147636413574], [-2.7340762615203857], [-2.9686968326568604], [-2.6156744956970215], [-2.315448522567749], [-2.2582099437713623], [-2.394615411758423], [-2.486255645751953], [-2.343952178955078], [-3.055164098739624], [-3.2794339656829834], [-3.0376760959625244], [-2.8411521911621094], [-2.6768839359283447], [-2.6573009490966797], [-2.6563308238983154], [-2.4706344604492188], [-3.2290139198303223], [-3.3989450931549072], [-3.0206971168518066], [-2.849600315093994], [-2.7816994190216064], [-2.6770896911621094], [-2.661151170730591], [-2.394167900085449], [-3.1401875019073486], [-3.3224198818206787], [-3.045311450958252], [-2.839939832687378], [-2.8180136680603027], [-2.638383626937866], [-2.6572279930114746], [-2.4873173236846924], [-3.0559592247009277], [-3.2291769981384277], [-2.9053726196289062], [-2.619166851043701], [-2.4686074256896973], [-2.3695857524871826], [-2.337869882583618], [-2.242933750152588], [-2.7330517768859863], [-2.8996853828430176], [-2.6268019676208496], [-2.3521971702575684], [-2.3066904544830322], [-2.6438024044036865], [-2.53608775138855], [-2.5103135108947754], [-3.163695812225342], [-3.3452975749969482], [-3.0626981258392334], [-2.8006975650787354], [-2.7119393348693848], [-2.9658102989196777], [-2.805750846862793], [-2.6303815841674805], [-3.547795057296753], [-3.640038251876831], [-3.4043102264404297], [-3.1958556175231934], [-3.1924946308135986], [-3.1163721084594727], [-2.9519500732421875], [-2.875101089477539], [-3.4211292266845703], [-3.601597547531128], [-3.1529009342193604], [-3.010915994644165], [-2.983560085296631], [-3.046698570251465], [-2.965348720550537], [-2.825496196746826], [-3.3820505142211914], [-3.4926815032958984], [-3.196331024169922], [-3.004054307937622], [-2.985555410385132], [-2.8802576065063477], [-2.8922348022460938], [-2.76591157913208], [-2.930821180343628], [-3.125687599182129], [-2.8366730213165283], [-2.595963478088379], [-2.5801007747650146], [-2.6969470977783203], [-2.7062158584594727], [-2.5311808586120605], [-3.142136335372925], [-3.343869686126709], [-3.1298718452453613], [-2.9494383335113525], [-2.7738263607025146], [-2.8714072704315186], [-2.750135898590088], [-2.649747371673584], [-3.1227798461914062], [-3.373340606689453], [-3.063974618911743], [-2.8756344318389893], [-2.788553476333618], [-2.7882256507873535], [-2.6513912677764893], [-2.4259424209594727], [-3.170508861541748], [-3.378021240234375], [-3.057206869125366], [-2.862983226776123], [-2.926882743835449], [-2.9605343341827393], [-2.9230780601501465], [-2.6718642711639404], [-3.0482428073883057], [-3.2753374576568604], [-3.0337183475494385], [-2.752387046813965], [-2.640303611755371], [-2.592205047607422], [-2.532081127166748], [-2.4682106971740723], [-2.758899211883545], [-2.9040260314941406], [-2.738529682159424], [-2.456855058670044], [-2.3320372104644775], [-2.5157411098480225], [-2.503091335296631], [-2.7630975246429443], [-3.073822259902954], [-3.253864288330078], [-3.1154839992523193], [-2.8021180629730225], [-2.6192867755889893], [-2.6862831115722656], [-2.6489410400390625], [-2.689474582672119], [-3.2543535232543945], [-3.4074463844299316], [-3.316160202026367], [-3.0255043506622314], [-2.972233295440674], [-2.9618711471557617], [-2.938345193862915], [-2.903670310974121], [-3.2668039798736572], [-3.43176007270813], [-3.2525477409362793], [-2.9625205993652344], [-2.878673553466797], [-2.749999523162842], [-2.9318273067474365], [-2.7835464477539062], [-3.056758165359497], [-3.239743709564209], [-3.0655155181884766], [-2.822477340698242], [-2.818976879119873], [-2.6161270141601562], [-2.697136402130127], [-2.645996570587158], [-2.798264265060425], [-3.03578519821167], [-2.7547404766082764], [-2.4105870723724365], [-2.380244016647339], [-2.5030691623687744], [-2.4601926803588867], [-2.5408172607421875], [-2.875685453414917], [-3.149101972579956], [-2.99605655670166], [-2.710869789123535], [-2.5680429935455322], [-2.454983711242676], [-2.5392050743103027], [-2.6092569828033447], [-2.948263645172119], [-3.2537596225738525], [-3.072504758834839], [-2.8266031742095947], [-2.682720184326172], [-2.5137171745300293], [-2.481473684310913], [-2.427309036254883], [-3.036853313446045], [-3.301877498626709], [-3.0885753631591797], [-2.81510066986084], [-2.800964593887329], [-2.6109964847564697], [-2.633284568786621], [-2.6063051223754883], [-2.804241895675659], [-3.1039462089538574], [-2.9516549110412598], [-2.6421871185302734], [-2.458076238632202], [-2.2739038467407227], [-2.239731788635254], [-2.400282859802246], [-2.818373203277588], [-3.0382845401763916], [-2.8676645755767822], [-2.5819954872131348], [-2.4772021770477295], [-2.4666314125061035], [-2.3227696418762207], [-2.6698248386383057], [-2.7758383750915527], [-3.020473003387451], [-2.8708856105804443], [-2.7961652278900146], [-2.743922233581543], [-2.7564730644226074], [-2.6224565505981445], [-2.7182297706604004], [-2.540179491043091], [-2.580427885055542], [-2.4762520790100098], [-2.682948589324951], [-2.9018092155456543], [-3.229207992553711], [-2.704508066177368], [-2.8111114501953125], [-2.3438754081726074], [-2.074215888977051], [-1.7609851360321045], [-1.694972038269043], [-2.161740303039551], [-3.1421902179718018], [-2.8506879806518555], [-2.7757952213287354], [-3.2740731239318848], [-3.432013511657715], [-3.239187240600586], [-2.9951956272125244], [-2.9949803352355957], [-2.6462905406951904], [-2.603097438812256], [-2.615325689315796], [-2.867218494415283], [-3.1284313201904297], [-2.8734898567199707], [-2.519815444946289], [-2.474990129470825], [-2.4634287357330322], [-2.333002805709839], [-2.4827303886413574], [-3.09804105758667], [-3.348196268081665], [-3.1716220378875732], [-2.9144372940063477], [-2.811983346939087], [-2.714310646057129], [-2.542206048965454], [-2.721691131591797], [-3.020949602127075], [-3.3370487689971924], [-3.159771203994751], [-3.000319480895996], [-2.9210352897644043], [-2.667080879211426], [-2.589080810546875], [-2.5635828971862793], [-3.1472439765930176], [-3.413156032562256], [-3.233189821243286], [-2.9496846199035645], [-2.915106773376465], [-2.7010726928710938], [-2.725926160812378], [-2.7049307823181152], [-2.955781936645508], [-3.280496597290039], [-3.1415834426879883], [-2.859879732131958], [-2.7013497352600098], [-2.426579236984253], [-2.3576173782348633], [-2.3231685161590576], [-2.9085745811462402], [-3.0554733276367188], [-2.9098191261291504], [-2.5840976238250732], [-2.4693498611450195], [-2.4060256481170654], [-2.341966152191162], [-2.504096508026123], [-3.1242246627807617], [-3.313084363937378], [-3.1810097694396973], [-2.8603367805480957], [-2.6695847511291504], [-2.6352672576904297], [-2.5256776809692383], [-2.5389020442962646], [-3.083676815032959], [-3.176365613937378], [-2.954110622406006], [-2.932929277420044], [-3.1467394828796387], [-3.113424301147461], [-2.9522032737731934], [-2.8756842613220215], [-2.345126152038574], [-2.3596131801605225], [-2.212496519088745], [-2.396329164505005], [-2.7688472270965576], [-3.045576810836792], [-2.9444305896759033], [-2.6620492935180664], [-3.0960512161254883], [-3.2782466411590576], [-3.130591869354248], [-2.8250904083251953], [-2.779384136199951], [-2.3641958236694336], [-2.485758066177368], [-2.345268964767456], [-2.745203733444214], [-2.959909439086914], [-2.6722066402435303], [-2.319324016571045], [-2.2475087642669678], [-2.327101707458496], [-2.2959070205688477], [-2.2814407348632812], [-2.979106903076172], [-3.209562063217163], [-3.0627167224884033], [-2.7943520545959473], [-2.643641948699951], [-2.454519748687744], [-2.4948222637176514], [-2.460803985595703], [-2.9652655124664307], [-3.309950828552246], [-3.174729824066162], [-2.953270196914673], [-2.788142204284668], [-2.526062488555908], [-2.4166111946105957], [-2.3097267150878906], [-3.130882978439331], [-3.430004358291626], [-3.315007209777832], [-2.998142957687378], [-2.936711072921753], [-2.5791232585906982], [-2.6386170387268066], [-2.479102373123169], [-2.941988945007324], [-3.2616374492645264], [-3.154536724090576], [-2.835066318511963], [-2.6312832832336426], [-2.2997233867645264], [-2.2550675868988037], [-2.239375114440918], [-2.7764029502868652], [-2.9560139179229736], [-2.873814105987549], [-2.5257718563079834], [-2.4271397590637207], [-2.2562522888183594], [-2.282977819442749], [-2.3593454360961914], [-3.049097776412964], [-3.320639133453369], [-3.238558292388916], [-2.998518705368042], [-2.844893455505371], [-2.5405659675598145], [-2.5553884506225586], [-2.4810519218444824], [-3.007058620452881], [-3.2456393241882324], [-3.153097152709961], [-3.0481936931610107], [-3.1615326404571533], [-2.9623072147369385], [-2.841400146484375], [-2.728470802307129], [-2.5197596549987793], [-2.6783034801483154], [-2.599613666534424], [-2.748953104019165], [-2.872668743133545], [-2.7680835723876953], [-2.9152255058288574], [-2.64778208732605], [-3.0771117210388184], [-3.2744274139404297], [-3.176884651184082], [-2.924637794494629], [-2.8829445838928223], [-2.333259344100952], [-2.4465391635894775], [-2.287024736404419], [-2.602766275405884], [-2.914172649383545], [-2.7189218997955322], [-2.419809341430664], [-2.3045473098754883], [-2.153946876525879], [-2.250429391860962], [-2.2126922607421875], [-2.6668543815612793], [-2.974379062652588], [-2.8510489463806152], [-2.655942916870117], [-2.4559407234191895], [-2.0906460285186768], [-2.335200786590576], [-2.2912652492523193], [-2.730560779571533], [-3.138568162918091], [-2.9925386905670166], [-2.7873642444610596], [-2.5939583778381348], [-2.1422958374023438], [-2.2608861923217773], [-2.18302059173584], [-2.6923282146453857], [-3.0638697147369385], [-2.9225211143493652], [-2.6824331283569336], [-2.5205464363098145], [-2.2077133655548096], [-2.393080472946167], [-2.3106818199157715], [-2.5878660678863525], [-2.9778637886047363], [-2.8799009323120117], [-2.5659537315368652], [-2.251540422439575], [-1.9903203248977661], [-1.9211658239364624], [-2.175032138824463], [-2.254202365875244], [-2.645148754119873], [-2.506183624267578], [-2.2179973125457764], [-1.9312034845352173], [-1.8553577661514282], [-2.0040502548217773], [-2.410996913909912], [-2.7201805114746094], [-3.032827854156494], [-2.993107557296753], [-2.7464842796325684], [-2.6361961364746094], [-2.4633524417877197], [-2.3784213066101074], [-2.3772990703582764], [-2.900480031967163], [-3.142274856567383], [-3.0713653564453125], [-2.8135740756988525], [-2.7013323307037354], [-2.6796340942382812], [-2.6993415355682373], [-2.700137138366699], [-3.1568055152893066], [-3.357632875442505], [-3.247715473175049], [-2.9931654930114746], [-2.894641160964966], [-2.7290334701538086], [-2.8641397953033447], [-2.804729700088501], [-3.1559293270111084], [-3.2973389625549316], [-3.1835947036743164], [-2.9334912300109863], [-2.9324333667755127], [-2.689711570739746], [-2.7255988121032715], [-2.694202423095703], [-2.667616367340088], [-2.927004337310791], [-2.7290539741516113], [-2.454944133758545], [-2.354173183441162], [-2.336538314819336], [-2.383436679840088], [-2.25846791267395], [-2.784022808074951], [-3.028282642364502], [-2.908846616744995], [-2.6858503818511963], [-2.4537622928619385], [-2.3783297538757324], [-2.376957893371582], [-2.4654202461242676], [-2.9496243000030518], [-3.252835750579834], [-3.1011555194854736], [-2.8678696155548096], [-2.713365316390991], [-2.54046368598938], [-2.410975217819214], [-2.299098014831543], [-3.042114496231079], [-3.343519926071167], [-3.251695394515991], [-2.954981565475464], [-2.8058364391326904], [-2.450387954711914], [-2.493351936340332], [-2.63087797164917], [-2.861623525619507], [-3.243027448654175], [-3.1922378540039062], [-2.872384548187256], [-2.6351265907287598], [-2.341822624206543], [-2.091151475906372], [-2.0904688835144043], [-2.883458137512207], [-3.1585144996643066], [-3.122789144515991], [-2.817427635192871], [-2.602485418319702], [-2.5566043853759766], [-2.504387855529785], [-2.5496346950531006], [-3.068716049194336], [-3.2556240558624268], [-3.1893436908721924], [-3.0494117736816406], [-2.926018238067627], [-2.847855806350708], [-2.706827163696289], [-2.490861415863037], [-3.019075870513916], [-3.265737533569336], [-3.1922059059143066], [-3.1124954223632812], [-3.207737445831299], [-3.01149845123291], [-2.9144163131713867], [-2.8922958374023438], [-2.6639509201049805], [-2.7959272861480713], [-2.6809303760528564], [-2.877887725830078], [-2.9785239696502686], [-3.066838264465332], [-3.123457431793213], [-2.8507919311523438], [-3.217155933380127], [-3.373368263244629], [-3.2868094444274902], [-3.0716750621795654], [-3.0571341514587402], [-2.676396369934082], [-2.7524189949035645], [-2.693474769592285], [-2.820223808288574], [-3.079144239425659], [-2.8550643920898438], [-2.541846513748169], [-2.495490550994873], [-2.3962907791137695], [-2.4343321323394775], [-2.3244428634643555], [-3.0628795623779297], [-3.294231414794922], [-3.25191068649292], [-3.0285146236419678], [-2.8481171131134033], [-2.5827484130859375], [-2.648895740509033], [-2.5568490028381348], [-3.0211753845214844], [-3.373006582260132], [-3.3303232192993164], [-3.088895559310913], [-2.9598045349121094], [-2.610069513320923], [-2.6560440063476562], [-2.396069288253784], [-3.2740442752838135], [-3.5656328201293945], [-3.4729723930358887], [-3.2637131214141846], [-3.1937553882598877], [-2.939859390258789], [-2.9035396575927734], [-2.861393928527832], [-2.9746651649475098], [-3.310371160507202], [-3.293578624725342], [-3.058135509490967], [-2.9297385215759277], [-2.546355724334717], [-2.5080366134643555], [-2.2286901473999023], [-2.9363784790039062], [-3.1948180198669434], [-3.1226789951324463], [-2.857509136199951], [-2.814631938934326], [-2.535815477371216], [-2.4933090209960938], [-2.3852028846740723], [-3.1545052528381348], [-3.4121925830841064], [-3.442993402481079], [-3.2263243198394775], [-3.1130707263946533], [-2.8409018516540527], [-2.6968953609466553], [-2.4469404220581055], [-3.2102420330047607], [-3.4330759048461914], [-3.4071497917175293], [-3.194786310195923], [-3.2347428798675537], [-2.957003593444824], [-2.984161615371704], [-2.842207193374634], [-3.2506327629089355], [-3.495790958404541], [-3.4393022060394287], [-3.2849528789520264], [-3.217139482498169], [-2.832176685333252], [-3.069047212600708], [-2.7481095790863037], [-3.2063636779785156], [-3.4368910789489746], [-3.3446547985076904], [-3.1580238342285156], [-3.179271936416626], [-2.7854256629943848], [-2.8619601726531982], [-2.671905040740967], [-2.8195371627807617], [-3.117924690246582], [-2.8961129188537598], [-2.5322256088256836], [-2.5054516792297363], [-2.3488242626190186], [-2.459442615509033], [-2.2868847846984863], [-2.9232916831970215], [-3.2159690856933594], [-3.107124090194702], [-2.876591205596924], [-2.710221290588379], [-2.4587700366973877], [-2.520864963531494], [-2.512613296508789], [-3.0573654174804688], [-3.376857042312622], [-3.242140769958496], [-2.9929635524749756], [-2.853766918182373], [-2.559926748275757], [-2.4784939289093018], [-2.333082675933838], [-3.0364224910736084], [-3.316293954849243], [-3.202711582183838], [-3.013197422027588], [-2.9228618144989014], [-2.5664761066436768], [-2.6228418350219727], [-2.622124671936035], [-3.0295145511627197], [-3.385390281677246], [-3.3675432205200195], [-3.0718040466308594], [-2.8532028198242188], [-2.4637060165405273], [-2.2567315101623535], [-2.178554058074951], [-2.8641180992126465], [-3.0673532485961914], [-2.9984681606292725], [-2.708254814147949], [-2.667436361312866], [-2.4822702407836914], [-2.3963286876678467], [-2.4493699073791504], [-3.078834295272827], [-3.3008298873901367], [-3.3245160579681396], [-3.0579586029052734], [-2.9747660160064697], [-2.8196685314178467], [-2.4997963905334473], [-2.3308217525482178], [-3.0309648513793945], [-3.2555923461914062], [-3.2958436012268066], [-3.124056339263916], [-3.094397783279419], [-3.066838026046753], [-2.7443888187408447], [-2.7832279205322266], [-2.6597213745117188], [-2.912209987640381], [-2.794689178466797], [-2.8954601287841797], [-2.9549612998962402], [-2.971365451812744], [-2.9828970432281494], [-2.7144269943237305], [-3.1652488708496094], [-3.327113628387451], [-3.1839752197265625], [-2.961130380630493], [-2.995142936706543], [-2.7364885807037354], [-2.7187232971191406], [-2.611774444580078], [-2.8574867248535156], [-3.076359272003174], [-2.790888547897339], [-2.4545059204101562], [-2.475067138671875], [-2.484394073486328], [-2.4628701210021973], [-2.2570600509643555], [-2.999690294265747], [-3.2313926219940186], [-3.125297784805298], [-2.894630193710327], [-2.7308197021484375], [-2.559945583343506], [-2.5181708335876465], [-2.4492173194885254], [-3.063474655151367], [-3.3411641120910645], [-3.238541603088379], [-3.0376622676849365], [-2.920283317565918], [-2.7055182456970215], [-2.528989791870117], [-2.325770854949951], [-3.0692782402038574], [-3.3600010871887207], [-3.2416625022888184], [-2.9942514896392822], [-2.9464662075042725], [-2.6301493644714355], [-2.634970188140869], [-2.670470714569092], [-3.0464067459106445], [-3.376096725463867], [-3.3194875717163086], [-3.005401849746704], [-2.914647102355957], [-2.5263237953186035], [-2.465055227279663], [-2.167992115020752], [-2.8667776584625244], [-3.1001479625701904], [-2.9761009216308594], [-2.7023487091064453], [-2.6971004009246826], [-2.561209201812744], [-2.5551061630249023], [-2.4718005657196045], [-3.020256996154785], [-3.2604947090148926], [-3.240915298461914], [-2.9857659339904785], [-2.9445908069610596], [-2.6861729621887207], [-2.6573100090026855], [-2.377039909362793], [-3.035823106765747], [-3.242589235305786], [-3.1255836486816406], [-2.9536805152893066], [-2.971982955932617], [-2.775729179382324], [-2.820096254348755], [-2.6554417610168457], [-3.1136856079101562], [-3.279465675354004], [-2.980546236038208], [-2.9517898559570312], [-2.969308853149414], [-2.8359296321868896], [-2.909090995788574], [-2.5508179664611816], [-3.015681743621826], [-3.266671895980835], [-3.119802951812744], [-2.90954327583313], [-2.8704075813293457], [-2.4227004051208496], [-2.5674424171447754], [-2.4833407402038574], [-2.6753787994384766], [-2.9577648639678955], [-2.7060375213623047], [-2.4052817821502686], [-2.3828468322753906], [-2.253894567489624], [-2.4424073696136475], [-2.2601683139801025], [-2.824064254760742], [-3.1030771732330322], [-2.9477291107177734], [-2.764378070831299], [-2.635685443878174], [-2.3071577548980713], [-2.4193215370178223], [-2.35866379737854], [-2.9783496856689453], [-3.30366849899292], [-3.1179215908050537], [-2.923163890838623], [-2.763521194458008], [-2.4140233993530273], [-2.3577077388763428], [-2.2214789390563965], [-2.910573959350586], [-3.194722890853882], [-3.0029687881469727], [-2.826664447784424], [-2.7454328536987305], [-2.424675226211548], [-2.500023126602173], [-2.5249767303466797], [-2.822741985321045], [-3.186655282974243], [-3.0193355083465576], [-2.74031138420105], [-2.505479574203491], [-2.2266249656677246], [-2.2122607231140137], [-2.1319379806518555], [-2.6357879638671875], [-2.9547462463378906], [-2.8613715171813965], [-2.534148693084717], [-2.404193639755249], [-2.4572219848632812], [-2.4236738681793213], [-2.4375665187835693], [-2.956899404525757], [-3.293975591659546], [-3.247492790222168], [-2.997464179992676], [-2.8606691360473633], [-2.8152170181274414], [-2.5785436630249023], [-2.4531242847442627], [-3.1031525135040283], [-3.375092029571533], [-3.326935291290283], [-3.057490825653076], [-2.9813051223754883], [-2.898327350616455], [-2.877682685852051], [-2.749870777130127], [-2.9984898567199707], [-3.3474555015563965], [-3.181370973587036], [-2.997133255004883], [-2.905649423599243], [-2.8909149169921875], [-2.8653533458709717], [-2.7251169681549072], [-3.181175947189331], [-3.428089141845703], [-3.3224635124206543], [-3.1300604343414307], [-3.0589547157287598], [-2.840108871459961], [-2.7587461471557617], [-2.570492744445801], [-2.8331923484802246], [-3.1714682579040527], [-2.983741044998169], [-2.7386484146118164], [-2.656787872314453], [-2.5734524726867676], [-2.6192963123321533], [-2.310871124267578], [-3.026592493057251], [-3.321782112121582], [-3.2295570373535156], [-3.0583415031433105], [-2.8598928451538086], [-2.6666479110717773], [-2.7268521785736084], [-2.584979295730591], [-3.2139675617218018], [-3.526439905166626], [-3.3440001010894775], [-3.140411376953125], [-3.000943660736084], [-2.8205714225769043], [-2.574796199798584], [-2.4179720878601074], [-3.0405173301696777], [-3.3044703006744385], [-3.077873706817627], [-2.91772723197937], [-2.904515266418457], [-2.762028932571411], [-2.7087559700012207], [-2.5815865993499756], [-2.8814778327941895], [-3.2479941844940186], [-3.1403467655181885], [-2.887105941772461], [-2.6810429096221924], [-2.4756417274475098], [-2.2729218006134033], [-2.2346415519714355], [-2.7586607933044434], [-3.015138626098633], [-2.901251792907715], [-2.646851062774658], [-2.6060237884521484], [-2.4718024730682373], [-2.434689521789551], [-2.322321653366089], [-3.184056043624878], [-3.377044677734375], [-3.3315749168395996], [-3.0672407150268555], [-2.9395923614501953], [-2.9046735763549805], [-2.623380661010742], [-2.2832865715026855], [-3.1098337173461914], [-3.337916374206543], [-3.317861318588257], [-3.054816246032715], [-3.0161802768707275], [-2.938368320465088], [-2.8945131301879883], [-2.521782875061035], [-3.0664873123168945], [-3.347287893295288], [-3.302072048187256], [-3.078505039215088], [-2.9957385063171387], [-2.815464496612549], [-2.979480743408203], [-2.5527496337890625], [-3.0861780643463135], [-3.3144679069519043], [-3.2388126850128174], [-3.0771098136901855], [-3.126862049102783], [-2.7647900581359863], [-2.8685076236724854], [-2.4710919857025146], [-2.8230206966400146], [-3.0993430614471436], [-2.8889870643615723], [-2.601106882095337], [-2.614346981048584], [-2.436858654022217], [-2.539191484451294], [-2.157578706741333], [-2.838974952697754], [-3.1743738651275635], [-3.115579605102539], [-2.895904541015625], [-2.747342109680176], [-2.359203338623047], [-2.5894737243652344], [-2.4466700553894043], [-2.9429123401641846], [-3.289546489715576], [-3.2342543601989746], [-2.990096092224121], [-2.833496332168579], [-2.4530863761901855], [-2.4501521587371826], [-2.3559377193450928], [-2.9408693313598633], [-3.228602886199951], [-3.09515643119812], [-2.8467793464660645], [-2.7769808769226074], [-2.3958258628845215], [-2.532917022705078], [-2.6542210578918457], [-2.8319244384765625], [-3.175686836242676], [-3.1424663066864014], [-2.842029571533203], [-2.6392440795898438], [-2.339350461959839], [-2.212681531906128], [-2.176621198654175], [-2.932058811187744], [-3.180365562438965], [-3.1603591442108154], [-2.8462560176849365], [-2.709947109222412], [-2.6152749061584473], [-2.414862632751465], [-2.5309760570526123], [-2.519406318664551], [-2.3754560947418213], [-2.225719690322876], [-2.3433098793029785], [-2.4733331203460693], [-3.0858049392700195], [-2.738924503326416], [-2.762615919113159], [-2.309504508972168], [-2.2520127296447754], [-1.969367265701294], [-2.06306791305542], [-2.253488779067993], [-3.032094955444336], [-2.5401670932769775], [-3.949451208114624], [-3.0515480041503906]], "row": 1}, {"kind": "span", "head": "broad", "words": "s2_words", "labels": "s2_labels", "logits": [[-1.46242094039917], [-1.578751802444458], [-2.053593158721924], [-2.9410452842712402], [-2.306424379348755], [-3.757460117340088], [-1.6370573043823242], [-2.211214780807495], [-1.536679983139038], [-2.142828941345215], [-2.862086772918701], [-3.9242331981658936], [-2.2933921813964844], [-3.6780166625976562], [-1.6517142057418823], [-2.380481004714966], [-1.591235637664795], [-2.0094799995422363], [-2.404503107070923], [-3.467717170715332], [-2.3641114234924316], [-2.7695889472961426], [-2.0861878395080566], [-2.592149257659912], [-1.8654803037643433], [-1.9848178625106812], [-2.5245232582092285], [-3.3906359672546387], [-2.529547691345215], [-2.542813301086426], [-2.345221519470215], [-2.727003812789917], [-2.0476694107055664], [-2.133293628692627], [-2.5518147945404053], [-3.4651408195495605], [-2.7658395767211914], [-2.302833080291748], [-2.4353013038635254], [-2.463855028152466], [-1.9347522258758545], [-2.0442211627960205], [-2.3939530849456787], [-2.9283907413482666], [-2.678332567214966], [-2.4262123107910156], [-2.527204990386963], [-2.6600420475006104], [-2.270602226257324], [-2.151944875717163], [-2.376081705093384], [-2.854588747024536], [-2.913820266723633], [-2.5924062728881836], [-3.0764122009277344], [-3.290419101715088], [-2.9238505363464355], [-2.732651472091675], [-2.6838486194610596], [-3.12308931350708], [-3.3025271892547607], [-3.0790181159973145], [-2.9682908058166504], [-3.26625394821167], [-2.7204761505126953], [-2.7418296337127686], [-2.830167055130005], [-3.231773853302002], [-3.164748191833496], [-2.78770112991333], [-2.9771389961242676], [-3.122159957885742], [-2.6633999347686768], [-2.66452956199646], [-2.7854161262512207], [-2.9952034950256348], [-3.1623082160949707], [-2.983698844909668], [-3.0810179710388184], [-3.452760696411133], [-3.045166015625], [-2.861807346343994], [-2.781524419784546], [-2.89953875541687], [-3.0224063396453857], [-2.7129669189453125], [-3.128213405609131], [-3.2046072483062744], [-2.873152256011963], [-2.654444694519043], [-2.722339153289795], [-3.16219162940979], [-3.1910269260406494], [-3.2130908966064453], [-3.249825954437256], [-3.5398807525634766], [-3.2752647399902344], [-3.133925199508667], [-2.9835591316223145], [-3.4965100288391113], [-3.4956326484680176], [-3.2210333347320557], [-3.297624349594116], [-3.654934883117676], [-3.457042694091797], [-3.240978956222534], [-3.1543960571289062], [-3.4679675102233887], [-3.240448474884033], [-3.0153911113739014], [-3.108316421508789], [-3.2601749897003174], [-2.905198335647583], [-2.6711645126342773], [-2.5785229206085205], [-3.0109057426452637], [-3.114821434020996], [-2.7633297443389893], [-2.8154759407043457], [-2.977318525314331], [-2.675588369369507], [-2.6071386337280273], [-2.535313606262207], [-2.74051833152771], [-2.873736619949341], [-2.617774486541748], [-2.6441996097564697], [-2.862034559249878], [-2.439314365386963], [-2.2134766578674316], [-2.244021415710449], [-2.6685080528259277], [-2.8970210552215576], [-2.5514044761657715], [-3.276556968688965], [-3.4441306591033936], [-3.1439852714538574], [-2.9680213928222656], [-2.8223156929016113], [-2.8712239265441895], [-3.267129421234131], [-3.026742935180664], [-3.017608642578125], [-3.245150566101074], [-2.93282413482666], [-2.8432419300079346], [-2.716230630874634], [-2.7903053760528564], [-2.9822819232940674], [-2.541440963745117], [-3.149306058883667], [-3.331366777420044], [-2.9460644721984863], [-2.7934999465942383], [-2.7437655925750732], [-2.652984857559204], [-2.8247880935668945], [-2.552772045135498], [-3.106595516204834], [-3.343392848968506], [-2.9779348373413086], [-2.776317834854126], [-2.5749003887176514], [-2.606450080871582], [-2.7272493839263916], [-2.311065196990967], [-3.1335456371307373], [-3.2193636894226074], [-3.0292696952819824], [-2.8006536960601807], [-2.8089935779571533], [-3.030042886734009], [-2.967182159423828], [-2.8159594535827637], [-3.2302792072296143], [-3.697627305984497], [-3.40073823928833], [-3.2479445934295654], [-3.1798651218414307], [-3.505422592163086], [-3.3430838584899902], [-2.995479106903076], [-3.444399356842041], [-3.5439939498901367], [-3.367097854614258], [-3.18273663520813], [-3.2042036056518555], [-3.2875967025756836], [-3.007000684738159], [-2.924163818359375], [-3.5330796241760254], [-3.701493740081787], [-3.410532236099243], [-3.2107019424438477], [-3.1163318157196045], [-3.100166082382202], [-3.2035844326019287], [-2.903604745864868], [-3.251627206802368], [-3.3690693378448486], [-3.070988893508911], [-2.9405102729797363], [-2.9163568019866943], [-2.87406849861145], [-2.926581382751465], [-2.6954832077026367], [-2.802860975265503], [-3.016221523284912], [-2.7236180305480957], [-2.5148048400878906], [-2.5511057376861572], [-2.746978998184204], [-2.938279151916504], [-2.5906639099121094], [-3.4838156700134277], [-3.63032865524292], [-3.380119800567627], [-3.2677557468414307], [-3.1324727535247803], [-3.137328624725342], [-3.245553731918335], [-3.036865711212158], [-3.2577407360076904], [-3.413130521774292], [-3.230956792831421], [-3.172015428543091], [-3.007582426071167], [-2.81176495552063], [-2.9580307006835938], [-2.664283514022827], [-3.1377484798431396], [-3.2779111862182617], [-3.0554769039154053], [-2.977445602416992], [-2.917400360107422], [-2.8508565425872803], [-2.9309587478637695], [-2.787186622619629], [-3.110201358795166], [-3.2969672679901123], [-3.0543649196624756], [-2.9113950729370117], [-2.7102057933807373], [-2.5543832778930664], [-2.640897750854492], [-2.409438133239746], [-2.876380681991577], [-2.909414291381836], [-2.7171030044555664], [-2.4451940059661865], [-2.3132312297821045], [-2.532222270965576], [-2.669705629348755], [-2.735403537750244], [-3.0345163345336914], [-3.1155173778533936], [-3.02018404006958], [-2.830580949783325], [-2.645573616027832], [-2.866494655609131], [-2.8700008392333984], [-2.649355173110962], [-3.17244291305542], [-3.2559380531311035], [-3.1572906970977783], [-2.9559872150421143], [-2.8725802898406982], [-2.926013469696045], [-2.916090488433838], [-2.9111855030059814], [-3.191455364227295], [-3.2834763526916504], [-3.086082696914673], [-2.88552188873291], [-2.7162890434265137], [-2.541166067123413], [-3.002589464187622], [-2.706963062286377], [-3.024343967437744], [-3.1593198776245117], [-2.977766513824463], [-2.8756985664367676], [-2.824345588684082], [-2.6022756099700928], [-2.7312567234039307], [-2.5285208225250244], [-2.5240612030029297], [-2.7699081897735596], [-2.499464511871338], [-2.2411701679229736], [-2.109645366668701], [-2.3582091331481934], [-2.4978997707366943], [-2.373380184173584], [-3.2327699661254883], [-3.349247932434082], [-3.1022539138793945], [-2.94881534576416], [-2.713750123977661], [-2.7477924823760986], [-2.9743080139160156], [-2.8689956665039062], [-2.937528610229492], [-3.155108690261841], [-2.9831228256225586], [-2.886741876602173], [-2.6686954498291016], [-2.4850094318389893], [-2.4757680892944336], [-2.265232801437378], [-2.875843048095703], [-3.0870327949523926], [-2.8926165103912354], [-2.7715847492218018], [-2.64994215965271], [-2.447688102722168], [-2.675276517868042], [-2.650866746902466], [-2.872936725616455], [-3.069291591644287], [-2.862396478652954], [-2.6734023094177246], [-2.3874430656433105], [-2.1924843788146973], [-2.33781099319458], [-2.2302064895629883], [-2.282304286956787], [-2.587526321411133], [-2.5094544887542725], [-2.286508560180664], [-2.1020255088806152], [-2.100961923599243], [-2.319077968597412], [-2.4545364379882812], [-2.769298553466797], [-3.0072526931762695], [-2.9149882793426514], [-2.7805657386779785], [-2.54341983795166], [-2.530954360961914], [-2.506960868835449], [-2.441068172454834], [-2.8440146446228027], [-3.0782361030578613], [-2.9566822052001953], [-2.7972731590270996], [-2.7467286586761475], [-2.6086833477020264], [-2.663557767868042], [-2.7577033042907715], [-3.1096904277801514], [-3.2894845008850098], [-3.0419254302978516], [-2.8940842151641846], [-2.7279458045959473], [-2.4995057582855225], [-2.8857107162475586], [-2.6962525844573975], [-2.973830223083496], [-3.1677114963531494], [-2.985825777053833], [-2.9433088302612305], [-2.868805170059204], [-2.543186902999878], [-2.667307138442993], [-2.5054335594177246], [-2.384077548980713], [-2.760559320449829], [-2.5275940895080566], [-2.347568988800049], [-2.2192978858947754], [-2.2519147396087646], [-2.5080976486206055], [-2.438795566558838], [-2.9143314361572266], [-3.2368011474609375], [-2.991616725921631], [-2.947880268096924], [-2.665019989013672], [-2.6639909744262695], [-2.7605907917022705], [-2.770324230194092], [-2.872812271118164], [-3.159195899963379], [-2.9883627891540527], [-2.9722447395324707], [-2.749894142150879], [-2.546912670135498], [-2.5398712158203125], [-2.4099583625793457], [-2.828099012374878], [-3.097501277923584], [-2.8371920585632324], [-2.8131918907165527], [-2.693901300430298], [-2.520111322402954], [-2.6533384323120117], [-2.6465182304382324], [-2.7948946952819824], [-3.0514204502105713], [-2.8110618591308594], [-2.664250612258911], [-2.337146520614624], [-2.1378421783447266], [-2.166353464126587], [-2.075828790664673], [-2.438584804534912], [-2.748077869415283], [-2.548905849456787], [-2.357175588607788], [-2.1865313053131104], [-2.3369548320770264], [-2.483612060546875], [-2.5693202018737793], [-2.75522518157959], [-2.9844629764556885], [-2.830801248550415], [-2.6364073753356934], [-2.4932000637054443], [-2.869339942932129], [-2.6473758220672607], [-2.509058952331543], [-2.836165428161621], [-3.0454440116882324], [-2.894404888153076], [-2.7295479774475098], [-2.634716510772705], [-2.8711018562316895], [-2.8433566093444824], [-2.741441488265991], [-3.10032320022583], [-3.2780957221984863], [-3.0606303215026855], [-2.8737192153930664], [-2.7136144638061523], [-2.819831371307373], [-3.080127239227295], [-2.749569892883301], [-3.2098724842071533], [-3.3529460430145264], [-3.150506019592285], [-3.056368350982666], [-3.0094406604766846], [-2.8157286643981934], [-2.9148809909820557], [-2.579092025756836], [-2.696319341659546], [-2.9217865467071533], [-2.6827287673950195], [-2.465259552001953], [-2.361116409301758], [-2.532473087310791], [-2.7004036903381348], [-2.353888511657715], [-3.0381877422332764], [-3.2553250789642334], [-3.142531394958496], [-2.999772310256958], [-2.737884044647217], [-2.791410446166992], [-2.9174251556396484], [-2.6937365531921387], [-2.9756011962890625], [-3.225416660308838], [-3.069798469543457], [-2.947032928466797], [-2.7246832847595215], [-2.6679790019989014], [-2.645730972290039], [-2.4369804859161377], [-2.847442865371704], [-3.0735254287719727], [-2.936091661453247], [-2.836996078491211], [-2.753204107284546], [-2.627849578857422], [-2.8227999210357666], [-2.6518149375915527], [-2.7628166675567627], [-3.073761224746704], [-2.9348225593566895], [-2.7233333587646484], [-2.4637374877929688], [-2.402113914489746], [-2.384373664855957], [-2.3781328201293945], [-2.5474863052368164], [-2.7480883598327637], [-2.6128737926483154], [-2.355022430419922], [-2.209345579147339], [-2.4827587604522705], [-2.5864334106445312], [-2.7286441326141357], [-3.1218552589416504], [-3.231245994567871], [-3.119462013244629], [-2.942477226257324], [-2.74387526512146], [-2.889009475708008], [-2.8022103309631348], [-2.595551013946533], [-2.908257484436035], [-3.1511952877044678], [-3.0907046794891357], [-2.9123518466949463], [-2.858598232269287], [-2.877084493637085], [-2.945681095123291], [-2.853344440460205], [-2.9742584228515625], [-3.198514461517334], [-3.037407875061035], [-2.832915782928467], [-2.708615779876709], [-2.5856690406799316], [-2.9652225971221924], [-2.65535044670105], [-3.0538384914398193], [-3.2164549827575684], [-3.055551052093506], [-2.9343605041503906], [-2.9023802280426025], [-2.721137046813965], [-2.8291120529174805], [-2.584285259246826], [-2.5902562141418457], [-2.8442492485046387], [-2.585505485534668], [-2.3575143814086914], [-2.320420742034912], [-2.5341196060180664], [-2.697981357574463], [-2.397726058959961], [-3.0092453956604004], [-3.235905170440674], [-3.1099302768707275], [-2.962283134460449], [-2.822869300842285], [-2.7031607627868652], [-3.0261573791503906], [-2.7602977752685547], [-2.930368185043335], [-3.2076046466827393], [-3.0614378452301025], [-2.9320907592773438], [-2.8008909225463867], [-2.712317705154419], [-2.71592116355896], [-2.5747361183166504], [-2.934807777404785], [-3.2078824043273926], [-3.0015523433685303], [-2.8434958457946777], [-2.8725783824920654], [-2.828842878341675], [-3.002316474914551], [-2.95174503326416], [-2.674215793609619], [-3.0042810440063477], [-2.8610265254974365], [-2.6239986419677734], [-2.4865970611572266], [-2.248460292816162]], "row": 2}], "predict": {"answers": [{"id": "claims", "type": "span", "target_part": "user", "spans": [], "threshold": 0.35, "threshold_rule": "broad", "head": "broad", "head_reason": "default"}], "meta": {"tokens": {"user": 2310}, "windows": 2, "sequences": 1}}} diff --git a/tests/vela/fixtures/head-input-long-span-windowed.safetensors b/tests/vela/fixtures/head-input-long-span-windowed.safetensors new file mode 100644 index 00000000..c23ccacd Binary files /dev/null and b/tests/vela/fixtures/head-input-long-span-windowed.safetensors differ diff --git a/tests/vela/fixtures/native-plan-example-request-cjk.json b/tests/vela/fixtures/native-plan-example-request-cjk.json new file mode 100644 index 00000000..01973a26 --- /dev/null +++ b/tests/vela/fixtures/native-plan-example-request-cjk.json @@ -0,0 +1 @@ +{"case": "example-request-cjk", "request": {"model": "vllm-sr/Vela-2.0-0.8B", "state": {"request": "客户张伟（zhang.wei@example.cn）于2026年10月1日在上海市浦东新区世纪大道100号办理了业务，联系电话13800138000。", "source": "客户的业务申请需要在三个工作日内审核完成。", "answer": "客户的业务已经即时完成了。"}, "questions": {"pii": {"type": "span", "instructions": "Which spans are personal information?", "criteria": {"PERSON": "a person's name", "EMAIL_ADDRESS": "an e-mail address", "PHONE_NUMBER": "a telephone number", "STREET_ADDRESS": "a street address or postal address"}, "over": "request"}, "unsupported": {"type": "span", "instructions": "Which spans of the answer are not supported by the context?", "criteria": {"unsupported": "a claim not supported by the context"}}, "billing": {"type": "noul", "instructions": "这个请求是否与账单有关？"}}}, "predict_request": {"parts": [{"type": "user", "text": "客户张伟（zhang.wei@example.cn）于2026年10月1日在上海市浦东新区世纪大道100号办理了业务，联系电话13800138000。"}, {"type": "context", "text": "客户的业务申请需要在三个工作日内审核完成。"}, {"type": "answer", "text": "客户的业务已经即时完成了。"}], "questions": [{"id": "pii", "type": "span", "text": "Which spans are personal information?", "target_part": "user", "labels": [{"name": "PERSON", "description": "a person's name"}, {"name": "EMAIL_ADDRESS", "description": "an e-mail address"}, {"name": "PHONE_NUMBER", "description": "a telephone number"}, {"name": "STREET_ADDRESS", "description": "a street address or postal address"}]}, {"id": "unsupported", "type": "span", "text": "Which spans of the answer are not supported by the context?", "target_part": "answer", "labels": [{"name": "unsupported", "description": "a claim not supported by the context"}]}, {"id": "billing", "type": "choice", "text": "这个请求是否与账单有关？", "target_part": ["user", "context", "answer"], "options": [{"name": "no", "description": "No. The statement or question is not satisfied."}, {"name": "yes", "description": "Yes. The statement or question is satisfied."}]}]}, "plan": [{"qid": "pii", "type": "span", "criteria": {"PERSON": "a person's name", "EMAIL_ADDRESS": "an e-mail address", "PHONE_NUMBER": "a telephone number", "STREET_ADDRESS": "a street address or postal address"}, "range": [0, 75], "part": "user"}, {"qid": "unsupported", "type": "span", "criteria": {"unsupported": "a claim not supported by the context"}, "range": null, "part": "answer"}, {"qid": "billing", "type": "noul", "criteria": {}, "range": null, "part": ["user", "context", "answer"]}], "rows": [{"p_ids": [1905, 25, 198, 27, 22446, 3364, 420, 846, 731, 198, 96989, 138667, 9616, 89, 20032, 13, 26234, 34314, 21377, 7313, 95928, 17, 15, 17, 21, 95859, 16, 15, 96212, 16, 95971, 95772, 108914, 121458, 99524, 108611, 16, 15, 15, 96389, 134395, 97192, 3709, 108672, 16, 18, 23, 15, 15, 16, 18, 23, 15, 15, 15, 1710, 198, 510, 22446, 29, 198, 27, 22446, 3364, 420, 2078, 731, 198, 109480, 97192, 97379, 111341, 98868, 96069, 99671, 100616, 97235, 1710, 198, 510, 22446, 29, 198, 27, 22446, 3364, 420, 8944, 731, 198, 109480, 97192, 96790, 111267, 104154, 1710, 198, 510, 22446, 29, 198], "blocks": [{"kind": "q", "ids": [271, 6065, 913, 25, 5576, 198, 6196, 25, 1156, 11, 2193, 11, 4087, 198, 14162, 25, 198, 96488, 101729, 147343, 119526, 97495, 10992, 198, 3670, 25, 198, 13886, 29, 198, 4754, 4532, 3147, 2665, 13, 561, 4952, 466, 3296, 369, 524, 18933, 46292, 767, 3147, 2083, 8934, 198, 510, 1979, 29, 198, 13886, 29, 198, 4754, 4532, 3147, 9175, 13, 561, 4952, 466, 3296, 369, 18933, 46292, 767, 3147, 9405, 8934, 198, 510, 1979, 29, 198, 13886, 29, 198, 4754, 4532, 3147, 3928, 314, 279, 6625, 2519, 16298, 46292, 767, 3147, 58, 55553, 60, 8934, 198, 510, 1979, 29, 271, 3270, 279, 3074, 2904, 1786, 7021, 539, 279, 2193, 321, 10895, 13, 198, 72298, 25], "ends": [50, 74, 98], "query": 114, "qid": "billing", "qtype": "choice", "names": ["no", "yes"], "has_abs": true}, {"kind": "span", "ends": [41, 62, 82, 106], "label_ranges": [[22, 41], [42, 62], [63, 82], [83, 106]], "words_in_block": true, "ids": [271, 6065, 913, 25, 9111, 198, 6196, 25, 1156, 198, 14162, 25, 198, 22365, 42824, 513, 4204, 1928, 30, 198, 22933, 25, 198, 14057, 29, 198, 4754, 4532, 3147, 64, 1637, 579, 803, 2129, 767, 3147, 89974, 8934, 198, 510, 1448, 29, 198, 14057, 29, 198, 4754, 4532, 3147, 276, 378, 11139, 2534, 2129, 767, 3147, 34714, 19522, 8934, 198, 510, 1448, 29, 198, 14057, 29, 198, 4754, 4532, 3147, 64, 19919, 1324, 2129, 767, 3147, 72446, 18777, 8934, 198, 510, 1448, 29, 198, 14057, 29, 198, 4754, 4532, 3147, 64, 8329, 2534, 466, 38436, 2534, 2129, 767, 3147, 759, 54046, 19522, 8934, 198, 510, 1448, 29, 271, 1138, 25, 198, 27, 22446, 3364, 420, 846, 731, 198, 96989, 138667, 9616, 89, 20032, 13, 26234, 34314, 21377, 7313, 95928, 17, 15, 17, 21, 95859, 16, 15, 96212, 16, 95971, 95772, 108914, 121458, 99524, 108611, 16, 15, 15, 96389, 134395, 97192, 3709, 108672, 16, 18, 23, 15, 15, 16, 18, 23, 15, 15, 15, 1710, 198, 510, 22446, 29, 198, 271, 8679, 1396, 9111, 314, 279, 2100, 1414, 421, 22651, 799, 314, 279, 8928, 13, 198, 6205, 581, 25], "query": 187, "w_pos": [118, 118, 119, 119, 120, 121, 127, 128, 129, 133, 134, 136, 137, 138, 139, 140, 140, 140, 141, 141, 141, 141, 142, 142, 143, 143, 144, 147, 148, 148, 148, 149, 149, 150, 151, 151, 151, 151, 152, 163]}], "trunc": {"shrink_cut": 0, "protected_cut": 0}, "seg_lens": {"user": 46, "context": 10, "answer": 6}, "layout": "c", "names": ["PERSON", "EMAIL_ADDRESS", "PHONE_NUMBER", "STREET_ADDRESS"], "w_offs": [[0, 1], [1, 2], [2, 3], [3, 4], [4, 5], [5, 25], [25, 26], [26, 27], [27, 31], [31, 32], [32, 34], [34, 35], [35, 36], [36, 37], [37, 38], [38, 39], [39, 40], [40, 41], [41, 42], [42, 43], [43, 44], [44, 45], [45, 46], [46, 47], [47, 48], [48, 49], [49, 52], [52, 53], [53, 54], [54, 55], [55, 56], [56, 57], [57, 58], [58, 59], [59, 60], [60, 61], [61, 62], [62, 63], [63, 74], [74, 75]], "w_pos": [118, 118, 119, 119, 120, 121, 127, 128, 129, 133, 134, 136, 137, 138, 139, 140, 140, 140, 141, 141, 141, 141, 142, 142, 143, 143, 144, 147, 148, 148, 148, 149, 149, 150, 151, 151, 151, 151, 152, 163], "span_over": "user", "n_tokens": 404, "head": "router", "head_reason": "id"}, {"p_ids": [1905, 25, 198, 27, 22446, 3364, 420, 846, 731, 198, 96989, 138667, 9616, 89, 20032, 13, 26234, 34314, 21377, 7313, 95928, 17, 15, 17, 21, 95859, 16, 15, 96212, 16, 95971, 95772, 108914, 121458, 99524, 108611, 16, 15, 15, 96389, 134395, 97192, 3709, 108672, 16, 18, 23, 15, 15, 16, 18, 23, 15, 15, 15, 1710, 198, 510, 22446, 29, 198, 27, 22446, 3364, 420, 2078, 731, 198, 109480, 97192, 97379, 111341, 98868, 96069, 99671, 100616, 97235, 1710, 198, 510, 22446, 29, 198, 27, 22446, 3364, 420, 8944, 731, 198, 109480, 97192, 96790, 111267, 104154, 1710, 198, 510, 22446, 29, 198], "blocks": [{"kind": "span", "ends": [50], "label_ranges": [[28, 50]], "words_in_block": true, "ids": [271, 6065, 913, 25, 9111, 198, 6196, 25, 4087, 198, 14162, 25, 198, 22365, 42824, 314, 279, 4087, 513, 524, 7021, 539, 279, 2193, 30, 198, 22933, 25, 198, 14057, 29, 198, 4754, 4532, 3147, 64, 3591, 524, 7021, 539, 279, 2193, 2129, 767, 3147, 78349, 8934, 198, 510, 1448, 29, 271, 1138, 25, 198, 27, 22446, 3364, 420, 8944, 731, 198, 109480, 97192, 96790, 111267, 104154, 1710, 198, 510, 22446, 29, 198, 271, 8679, 1396, 9111, 314, 279, 2100, 1414, 421, 22651, 799, 314, 279, 8928, 13, 198, 6205, 581, 25], "query": 91, "w_pos": [62, 62, 62, 63, 63, 64, 64, 65, 65, 66, 66, 66, 67]}], "trunc": {"shrink_cut": 0, "protected_cut": 0}, "seg_lens": {"user": 46, "context": 10, "answer": 6}, "layout": "c", "names": ["unsupported"], "w_offs": [[0, 1], [1, 2], [2, 3], [3, 4], [4, 5], [5, 6], [6, 7], [7, 8], [8, 9], [9, 10], [10, 11], [11, 12], [12, 13]], "w_pos": [62, 62, 62, 63, 63, 64, 64, 65, 65, 66, 66, 66, 67], "span_over": "answer", "n_tokens": 193, "head": "router", "head_reason": "label_set"}], "window_rows": []} diff --git a/tests/vela/fixtures/native-plan-example-request-score.json b/tests/vela/fixtures/native-plan-example-request-score.json new file mode 100644 index 00000000..fc67c0f8 --- /dev/null +++ b/tests/vela/fixtures/native-plan-example-request-score.json @@ -0,0 +1 @@ +{"case": "example-request-score", "request": {"model": "vllm-sr/Vela-2.0-0.8B", "state": "Hello, I would like to check the status of my order. It has been five days and I have heard nothing back.", "questions": {"tone": {"type": "score", "instructions": "How formal is the message?", "criteria": ["casual", "neutral", "formal"]}, "urgency": {"type": "score", "instructions": "How urgently does the sender need a reply?", "criteria": ["no rush", "soon", "today", "immediately"]}}}, "predict_request": {"parts": [{"type": "user", "text": "Hello, I would like to check the status of my order. It has been five days and I have heard nothing back."}], "questions": [{"id": "tone", "type": "score", "text": "How formal is the message?", "target_part": "user", "options": ["casual", "neutral", "formal"]}, {"id": "urgency", "type": "score", "text": "How urgently does the sender need a reply?", "target_part": "user", "options": ["no rush", "soon", "today", "immediately"]}]}, "plan": [{"qid": "tone", "type": "score", "criteria": ["casual", "neutral", "formal"], "range": null, "part": "user"}, {"qid": "urgency", "type": "score", "criteria": ["no rush", "soon", "today", "immediately"], "range": null, "part": "user"}], "rows": [{"p_ids": [1905, 25, 198, 27, 22446, 3364, 420, 846, 731, 198, 9419, 11, 353, 1000, 1040, 310, 1716, 279, 2552, 314, 821, 1906, 13, 1049, 682, 978, 4097, 2756, 321, 353, 599, 6408, 4161, 1142, 13, 198, 510, 22446, 29, 198], "blocks": [{"kind": "q", "ids": [271, 6065, 913, 25, 5284, 198, 6196, 25, 1156, 198, 14162, 25, 198, 4199, 15440, 369, 279, 1876, 30, 198, 3670, 25, 198, 13886, 29, 198, 4754, 4532, 58719, 767, 3147, 23543, 899, 8934, 198, 510, 1979, 29, 198, 13886, 29, 198, 4754, 4532, 58719, 767, 3147, 57555, 8934, 198, 510, 1979, 29, 198, 13886, 29, 198, 4754, 4532, 58719, 767, 3147, 610, 278, 8934, 198, 510, 1979, 29, 271, 3270, 279, 3074, 2904, 1786, 7021, 539, 279, 2193, 321, 10895, 13, 198, 72298, 25], "ends": [37, 52, 68], "query": 84, "qid": "tone", "qtype": "score", "names": ["casual", "neutral", "formal"], "has_abs": false}, {"kind": "q", "ids": [271, 6065, 913, 25, 5284, 198, 6196, 25, 1156, 198, 14162, 25, 198, 4199, 74000, 1503, 279, 4495, 1144, 264, 9559, 30, 198, 3670, 25, 198, 13886, 29, 198, 4754, 4532, 58719, 767, 3147, 2083, 12601, 8934, 198, 510, 1979, 29, 198, 13886, 29, 198, 4754, 4532, 58719, 767, 3147, 63826, 8934, 198, 510, 1979, 29, 198, 13886, 29, 198, 4754, 4532, 58719, 767, 3147, 29129, 8934, 198, 510, 1979, 29, 198, 13886, 29, 198, 4754, 4532, 58719, 767, 3147, 316, 41763, 8934, 198, 510, 1979, 29, 271, 3270, 279, 3074, 2904, 1786, 7021, 539, 279, 2193, 321, 10895, 13, 198, 72298, 25], "ends": [40, 55, 70, 86], "query": 102, "qid": "urgency", "qtype": "score", "names": ["no rush", "soon", "today", "immediately"], "has_abs": false}], "trunc": {"shrink_cut": 0, "protected_cut": 0}, "seg_lens": {"user": 25}, "layout": null, "n_tokens": 228, "head": "router", "head_reason": null}], "window_rows": []} diff --git a/tests/vela/fixtures/native-plan-example-request-set-threshold.json b/tests/vela/fixtures/native-plan-example-request-set-threshold.json new file mode 100644 index 00000000..d0ea9b8f --- /dev/null +++ b/tests/vela/fixtures/native-plan-example-request-set-threshold.json @@ -0,0 +1 @@ +{"case": "example-request-set-threshold", "request": {"model": "vllm-sr/Vela-2.0-0.8B", "state": "My card was charged twice and the parcel never arrived.", "questions": {"issues": {"type": "set", "instructions": "Which issues does the customer report?", "criteria": {"billing": "payments, charges, refunds or invoices", "shipping": "delivery of an order or a parcel", "login": "signing in, passwords or account access"}, "threshold": 0.15}}}, "predict_request": {"parts": [{"type": "user", "text": "My card was charged twice and the parcel never arrived."}], "questions": [{"id": "issues", "type": "set", "text": "Which issues does the customer report?", "target_part": "user", "options": [{"name": "billing", "description": "payments, charges, refunds or invoices"}, {"name": "shipping", "description": "delivery of an order or a parcel"}, {"name": "login", "description": "signing in, passwords or account access"}], "threshold": 0.15}]}, "plan": [{"qid": "issues", "type": "set", "criteria": {"billing": "payments, charges, refunds or invoices", "shipping": "delivery of an order or a parcel", "login": "signing in, passwords or account access"}, "range": null, "part": "user"}], "rows": [{"p_ids": [1905, 25, 198, 27, 22446, 3364, 420, 846, 731, 198, 4888, 3575, 557, 11102, 10598, 321, 279, 28350, 2496, 11136, 13, 198, 510, 22446, 29, 198], "blocks": [{"kind": "q", "ids": [271, 6065, 913, 25, 716, 198, 6196, 25, 1156, 198, 14162, 25, 198, 22365, 4562, 1503, 279, 5813, 1830, 30, 198, 3670, 25, 198, 13886, 29, 198, 4754, 4532, 3147, 60917, 11, 9892, 11, 70057, 466, 61893, 2129, 767, 3147, 37355, 8934, 198, 510, 1979, 29, 198, 13886, 29, 198, 4754, 4532, 3147, 31320, 314, 449, 1906, 466, 264, 28350, 2129, 767, 3147, 25660, 8934, 198, 510, 1979, 29, 198, 13886, 29, 198, 4754, 4532, 3147, 7506, 286, 303, 11, 32599, 466, 2605, 2528, 2129, 767, 3147, 3547, 8934, 198, 510, 1979, 29, 271, 3270, 1396, 2904, 7021, 539, 279, 2193, 321, 10895, 13, 198, 72298, 25], "ends": [45, 68, 92], "query": 106, "qid": "issues", "qtype": "set", "names": ["billing", "shipping", "login"], "has_abs": false}], "trunc": {"shrink_cut": 0, "protected_cut": 0}, "seg_lens": {"user": 11}, "layout": null, "n_tokens": 133, "head": "router", "head_reason": null}], "window_rows": []} diff --git a/tests/vela/fixtures/native-plan-example-request.json b/tests/vela/fixtures/native-plan-example-request.json new file mode 100644 index 00000000..09bab119 --- /dev/null +++ b/tests/vela/fixtures/native-plan-example-request.json @@ -0,0 +1 @@ +{"case": "example-request", "request": {"model": "vllm-sr/Vela-2.0-0.8B", "state": {"request": "Hi, I'm Tom Baker (tom.baker@example.com). What is the maximum daily dose of paracetamol for an adult?", "source": "For adults, the maximum dose of paracetamol is 4 grams in 24 hours, taken as 500 mg to 1 g every 4 to 6 hours.", "answer": "Adults can take up to 6 grams of paracetamol in 24 hours, in doses of 500 mg to 1 g every 4 to 6 hours."}, "questions": {"pii": {"type": "span", "instructions": "Which spans are personal information?", "criteria": {"AGE": "a person's age", "CREDIT_CARD": "a payment card number", "DATE_TIME": "a date, time or date of birth", "DOMAIN_NAME": "an internet domain name or website address", "EMAIL_ADDRESS": "an e-mail address", "GPE": "a country, city, state or other geopolitical place", "IBAN_CODE": "an international bank account number (IBAN)", "IP_ADDRESS": "an IPv4 or IPv6 address", "NRP": "a nationality, religious or political group", "ORGANIZATION": "the name of a company, institution or other organisation", "PERSON": "a person's name", "PHONE_NUMBER": "a telephone number", "STREET_ADDRESS": "a street address or postal address", "TITLE": "a personal or professional title such as Dr. or Mrs.", "US_DRIVER_LICENSE": "a US driver's licence number", "US_SSN": "a US social security number", "ZIP_CODE": "a postal or ZIP code"}, "over": "request"}, "halu": {"type": "span", "instructions": "Which spans of the answer are not supported by the context?", "criteria": {"unsupported": "a claim not supported by the context"}}, "domain": {"type": "choice", "instructions": "Which subject area is this request about?", "over": "request", "criteria": {"health": "medicine, clinical practice, nutrition, ageing or sexual health", "math": "arithmetic, algebra, geometry, statistics or other mathematics", "other": "a subject that fits none of the listed areas"}}}}, "predict_request": {"parts": [{"type": "user", "text": "Hi, I'm Tom Baker (tom.baker@example.com). What is the maximum daily dose of paracetamol for an adult?"}, {"type": "context", "text": "For adults, the maximum dose of paracetamol is 4 grams in 24 hours, taken as 500 mg to 1 g every 4 to 6 hours."}, {"type": "answer", "text": "Adults can take up to 6 grams of paracetamol in 24 hours, in doses of 500 mg to 1 g every 4 to 6 hours."}], "questions": [{"id": "pii", "type": "span", "text": "Which spans are personal information?", "target_part": "user", "labels": [{"name": "AGE", "description": "a person's age"}, {"name": "CREDIT_CARD", "description": "a payment card number"}, {"name": "DATE_TIME", "description": "a date, time or date of birth"}, {"name": "DOMAIN_NAME", "description": "an internet domain name or website address"}, {"name": "EMAIL_ADDRESS", "description": "an e-mail address"}, {"name": "GPE", "description": "a country, city, state or other geopolitical place"}, {"name": "IBAN_CODE", "description": "an international bank account number (IBAN)"}, {"name": "IP_ADDRESS", "description": "an IPv4 or IPv6 address"}, {"name": "NRP", "description": "a nationality, religious or political group"}, {"name": "ORGANIZATION", "description": "the name of a company, institution or other organisation"}, {"name": "PERSON", "description": "a person's name"}, {"name": "PHONE_NUMBER", "description": "a telephone number"}, {"name": "STREET_ADDRESS", "description": "a street address or postal address"}, {"name": "TITLE", "description": "a personal or professional title such as Dr. or Mrs."}, {"name": "US_DRIVER_LICENSE", "description": "a US driver's licence number"}, {"name": "US_SSN", "description": "a US social security number"}, {"name": "ZIP_CODE", "description": "a postal or ZIP code"}]}, {"id": "halu", "type": "span", "text": "Which spans of the answer are not supported by the context?", "target_part": "answer", "labels": [{"name": "unsupported", "description": "a claim not supported by the context"}]}, {"id": "domain", "type": "choice", "text": "Which subject area is this request about?", "target_part": "user", "options": [{"name": "health", "description": "medicine, clinical practice, nutrition, ageing or sexual health"}, {"name": "math", "description": "arithmetic, algebra, geometry, statistics or other mathematics"}, {"name": "other", "description": "a subject that fits none of the listed areas"}]}]}, "plan": [{"qid": "pii", "type": "span", "criteria": {"AGE": "a person's age", "CREDIT_CARD": "a payment card number", "DATE_TIME": "a date, time or date of birth", "DOMAIN_NAME": "an internet domain name or website address", "EMAIL_ADDRESS": "an e-mail address", "GPE": "a country, city, state or other geopolitical place", "IBAN_CODE": "an international bank account number (IBAN)", "IP_ADDRESS": "an IPv4 or IPv6 address", "NRP": "a nationality, religious or political group", "ORGANIZATION": "the name of a company, institution or other organisation", "PERSON": "a person's name", "PHONE_NUMBER": "a telephone number", "STREET_ADDRESS": "a street address or postal address", "TITLE": "a personal or professional title such as Dr. or Mrs.", "US_DRIVER_LICENSE": "a US driver's licence number", "US_SSN": "a US social security number", "ZIP_CODE": "a postal or ZIP code"}, "range": [0, 102], "part": "user"}, {"qid": "halu", "type": "span", "criteria": {"unsupported": "a claim not supported by the context"}, "range": null, "part": "answer"}, {"qid": "domain", "type": "choice", "criteria": {"health": "medicine, clinical practice, nutrition, ageing or sexual health", "math": "arithmetic, algebra, geometry, statistics or other mathematics", "other": "a subject that fits none of the listed areas"}, "range": [0, 102], "part": "user"}], "rows": [{"p_ids": [1905, 25, 198, 27, 22446, 3364, 420, 846, 731, 198, 12675, 11, 353, 2688, 8105, 27466, 318, 36164, 918, 4263, 34314, 877, 553, 3437, 369, 279, 6966, 7071, 18537, 314, 1298, 64955, 235579, 364, 449, 6474, 30, 198, 510, 22446, 29, 198, 27, 22446, 3364, 420, 2078, 731, 198, 2381, 12236, 11, 279, 6966, 18537, 314, 1298, 64955, 235579, 369, 220, 19, 32216, 303, 220, 17, 19, 3979, 11, 4284, 430, 220, 20, 15, 15, 13348, 310, 220, 16, 338, 1396, 220, 19, 310, 220, 21, 3979, 13, 198, 510, 22446, 29, 198, 27, 22446, 3364, 420, 8944, 731, 198, 50281, 82, 628, 1831, 685, 310, 220, 21, 32216, 314, 1298, 64955, 235579, 303, 220, 17, 19, 3979, 11, 303, 32902, 314, 220, 20, 15, 15, 13348, 310, 220, 16, 338, 1396, 220, 19, 310, 220, 21, 3979, 13, 198, 510, 22446, 29, 198], "blocks": [{"kind": "q", "ids": [271, 6065, 913, 25, 5576, 198, 6196, 25, 1156, 198, 14162, 25, 198, 22365, 3704, 2982, 369, 411, 1622, 883, 30, 198, 3670, 25, 198, 13886, 29, 198, 4754, 4532, 3147, 79600, 11, 14075, 6380, 11, 24780, 11, 76242, 466, 7017, 2730, 2129, 767, 3147, 11776, 8934, 198, 510, 1979, 29, 198, 13886, 29, 198, 4754, 4532, 3147, 277, 25092, 11, 45309, 11, 16547, 11, 12760, 466, 975, 36352, 2129, 767, 3147, 10065, 8934, 198, 510, 1979, 29, 198, 13886, 29, 198, 4754, 4532, 3147, 64, 3704, 421, 17759, 6643, 314, 279, 9711, 5493, 2129, 767, 3147, 1520, 8934, 198, 510, 1979, 29, 198, 13886, 29, 198, 4754, 4532, 3147, 3928, 314, 279, 6625, 2519, 16298, 46292, 767, 3147, 58, 55553, 60, 8934, 198, 510, 1979, 29, 271, 3270, 279, 3074, 2904, 1786, 7021, 539, 279, 2193, 321, 10895, 13, 198, 72298, 25], "ends": [50, 77, 102, 126], "query": 142, "qid": "domain", "qtype": "choice", "names": ["health", "math", "other"], "has_abs": true}, {"kind": "span", "ends": [41, 63, 88, 112, 133, 160, 186, 210, 234, 262, 282, 302, 326, 353, 377, 400, 422], "label_ranges": [[22, 41], [42, 63], [64, 88], [89, 112], [113, 133], [134, 160], [161, 186], [187, 210], [211, 234], [235, 262], [263, 282], [283, 302], [303, 326], [327, 353], [354, 377], [378, 400], [401, 422]], "words_in_block": true, "ids": [271, 6065, 913, 25, 9111, 198, 6196, 25, 1156, 198, 14162, 25, 198, 22365, 42824, 513, 4204, 1928, 30, 198, 22933, 25, 198, 14057, 29, 198, 4754, 4532, 3147, 64, 1637, 579, 4092, 2129, 767, 3147, 3466, 8934, 198, 510, 1448, 29, 198, 14057, 29, 198, 4754, 4532, 3147, 64, 7903, 3575, 1324, 2129, 767, 3147, 34, 54569, 44012, 8934, 198, 510, 1448, 29, 198, 14057, 29, 198, 4754, 4532, 3147, 64, 2321, 11, 854, 466, 2321, 314, 6968, 2129, 767, 3147, 6876, 9754, 8934, 198, 510, 1448, 29, 198, 14057, 29, 198, 4754, 4532, 3147, 276, 7361, 7697, 803, 466, 3779, 2534, 2129, 767, 3147, 55214, 4556, 8934, 198, 510, 1448, 29, 198, 14057, 29, 198, 4754, 4532, 3147, 276, 378, 11139, 2534, 2129, 767, 3147, 34714, 19522, 8934, 198, 510, 1448, 29, 198, 14057, 29, 198, 4754, 4532, 3147, 64, 3046, 11, 3177, 11, 1528, 466, 975, 83899, 1925, 2129, 767, 3147, 38, 1679, 8934, 198, 510, 1448, 29, 198, 14057, 29, 198, 4754, 4532, 3147, 276, 6286, 5883, 2605, 1324, 318, 3151, 1058, 77039, 767, 3147, 3151, 1058, 9724, 8934, 198, 510, 1448, 29, 198, 14057, 29, 198, 4754, 4532, 3147, 276, 30521, 19, 466, 30521, 21, 2534, 2129, 767, 3147, 3191, 19522, 8934, 198, 510, 1448, 29, 198, 14057, 29, 198, 4754, 4532, 3147, 64, 56284, 11, 10072, 466, 4788, 1809, 2129, 767, 3147, 45, 20993, 8934, 198, 510, 1448, 29, 198, 14057, 29, 198, 4754, 4532, 3147, 1719, 803, 314, 264, 2724, 11, 14467, 466, 975, 20767, 2129, 767, 3147, 43433, 1058, 47383, 8934, 198, 510, 1448, 29, 198, 14057, 29, 198, 4754, 4532, 3147, 64, 1637, 579, 803, 2129, 767, 3147, 89974, 8934, 198, 510, 1448, 29, 198, 14057, 29, 198, 4754, 4532, 3147, 64, 19919, 1324, 2129, 767, 3147, 72446, 18777, 8934, 198, 510, 1448, 29, 198, 14057, 29, 198, 4754, 4532, 3147, 64, 8329, 2534, 466, 38436, 2534, 2129, 767, 3147, 759, 54046, 19522, 8934, 198, 510, 1448, 29, 198, 14057, 29, 198, 4754, 4532, 3147, 64, 4204, 466, 6376, 2192, 1680, 430, 2832, 13, 466, 17093, 46292, 767, 3147, 48639, 8934, 198, 510, 1448, 29, 198, 14057, 29, 198, 4754, 4532, 3147, 64, 2201, 5404, 579, 28995, 1324, 2129, 767, 3147, 1966, 42585, 55630, 8934, 198, 510, 1448, 29, 198, 14057, 29, 198, 4754, 4532, 3147, 64, 2201, 3471, 4610, 1324, 2129, 767, 3147, 1966, 1063, 18393, 8934, 198, 510, 1448, 29, 198, 14057, 29, 198, 4754, 4532, 3147, 64, 38436, 466, 54778, 1970, 2129, 767, 3147, 59506, 9724, 8934, 198, 510, 1448, 29, 271, 1138, 25, 198, 27, 22446, 3364, 420, 846, 731, 198, 12675, 11, 353, 2688, 8105, 27466, 318, 36164, 918, 4263, 34314, 877, 553, 3437, 369, 279, 6966, 7071, 18537, 314, 1298, 64955, 235579, 364, 449, 6474, 30, 198, 510, 22446, 29, 198, 271, 8679, 1396, 9111, 314, 279, 2100, 1414, 421, 22651, 799, 314, 279, 8928, 13, 198, 6205, 581, 25], "query": 484, "w_pos": [434, 435, 436, 437, 437, 438, 439, 440, 441, 446, 446, 447, 448, 449, 450, 451, 452, 453, 454, 457, 458, 459, 460]}], "trunc": {"shrink_cut": 0, "protected_cut": 0}, "seg_lens": {"user": 27, "context": 39, "answer": 39}, "layout": "c", "names": ["AGE", "CREDIT_CARD", "DATE_TIME", "DOMAIN_NAME", "EMAIL_ADDRESS", "GPE", "IBAN_CODE", "IP_ADDRESS", "NRP", "ORGANIZATION", "PERSON", "PHONE_NUMBER", "STREET_ADDRESS", "TITLE", "US_DRIVER_LICENSE", "US_SSN", "ZIP_CODE"], "w_offs": [[0, 2], [2, 3], [4, 5], [5, 6], [6, 7], [8, 11], [12, 17], [18, 19], [19, 40], [40, 41], [41, 42], [43, 47], [48, 50], [51, 54], [55, 62], [63, 68], [69, 73], [74, 76], [77, 88], [89, 92], [93, 95], [96, 101], [101, 102]], "w_pos": [434, 435, 436, 437, 437, 438, 439, 440, 441, 446, 446, 447, 448, 449, 450, 451, 452, 453, 454, 457, 458, 459, 460], "span_over": "user", "n_tokens": 772, "head": "router", "head_reason": "id"}, {"p_ids": [1905, 25, 198, 27, 22446, 3364, 420, 846, 731, 198, 12675, 11, 353, 2688, 8105, 27466, 318, 36164, 918, 4263, 34314, 877, 553, 3437, 369, 279, 6966, 7071, 18537, 314, 1298, 64955, 235579, 364, 449, 6474, 30, 198, 510, 22446, 29, 198, 27, 22446, 3364, 420, 2078, 731, 198, 2381, 12236, 11, 279, 6966, 18537, 314, 1298, 64955, 235579, 369, 220, 19, 32216, 303, 220, 17, 19, 3979, 11, 4284, 430, 220, 20, 15, 15, 13348, 310, 220, 16, 338, 1396, 220, 19, 310, 220, 21, 3979, 13, 198, 510, 22446, 29, 198, 27, 22446, 3364, 420, 8944, 731, 198, 50281, 82, 628, 1831, 685, 310, 220, 21, 32216, 314, 1298, 64955, 235579, 303, 220, 17, 19, 3979, 11, 303, 32902, 314, 220, 20, 15, 15, 13348, 310, 220, 16, 338, 1396, 220, 19, 310, 220, 21, 3979, 13, 198, 510, 22446, 29, 198], "blocks": [{"kind": "span", "ends": [50], "label_ranges": [[28, 50]], "words_in_block": true, "ids": [271, 6065, 913, 25, 9111, 198, 6196, 25, 4087, 198, 14162, 25, 198, 22365, 42824, 314, 279, 4087, 513, 524, 7021, 539, 279, 2193, 30, 198, 22933, 25, 198, 14057, 29, 198, 4754, 4532, 3147, 64, 3591, 524, 7021, 539, 279, 2193, 2129, 767, 3147, 78349, 8934, 198, 510, 1448, 29, 271, 1138, 25, 198, 27, 22446, 3364, 420, 8944, 731, 198, 50281, 82, 628, 1831, 685, 310, 220, 21, 32216, 314, 1298, 64955, 235579, 303, 220, 17, 19, 3979, 11, 303, 32902, 314, 220, 20, 15, 15, 13348, 310, 220, 16, 338, 1396, 220, 19, 310, 220, 21, 3979, 13, 198, 510, 22446, 29, 198, 271, 8679, 1396, 9111, 314, 279, 2100, 1414, 421, 22651, 799, 314, 279, 8928, 13, 198, 6205, 581, 25], "query": 124, "w_pos": [62, 64, 65, 66, 67, 69, 70, 71, 72, 75, 77, 79, 80, 81, 82, 83, 85, 88, 89, 91, 92, 93, 95, 96, 98, 99, 100]}], "trunc": {"shrink_cut": 0, "protected_cut": 0}, "seg_lens": {"user": 27, "context": 39, "answer": 39}, "layout": "c", "names": ["unsupported"], "w_offs": [[0, 6], [7, 10], [11, 15], [16, 18], [19, 21], [22, 23], [24, 29], [30, 32], [33, 44], [45, 47], [48, 50], [51, 56], [56, 57], [58, 60], [61, 66], [67, 69], [70, 73], [74, 76], [77, 79], [80, 81], [82, 83], [84, 89], [90, 91], [92, 94], [95, 96], [97, 102], [102, 103]], "w_pos": [62, 64, 65, 66, 67, 69, 70, 71, 72, 75, 77, 79, 80, 81, 82, 83, 85, 88, 89, 91, 92, 93, 95, 96, 98, 99, 100], "span_over": "answer", "n_tokens": 269, "head": "router", "head_reason": "id"}], "window_rows": []} diff --git a/tests/vela/fixtures/native-plan-long-span-windowed.json b/tests/vela/fixtures/native-plan-long-span-windowed.json new file mode 100644 index 00000000..0210fa02 --- /dev/null +++ b/tests/vela/fixtures/native-plan-long-span-windowed.json @@ -0,0 +1 @@ +{"case": "long-span-windowed", "request": {"model": "vllm-sr/Vela-2.0-0.8B", "state": {"request": "Sentence 0 reports a routine delivery event. Sentence 1 reports a routine delivery event. Sentence 2 reports a routine delivery event. Sentence 3 reports a routine delivery event. Sentence 4 reports a routine delivery event. Sentence 5 reports a routine delivery event. Sentence 6 reports a routine delivery event. Sentence 7 reports a routine delivery event. Sentence 8 reports a routine delivery event. Sentence 9 reports a routine delivery event. Sentence 10 reports a routine delivery event. Sentence 11 reports a routine delivery event. Sentence 12 reports a routine delivery event. Sentence 13 reports a routine delivery event. Sentence 14 reports a routine delivery event. Sentence 15 reports a routine delivery event. Sentence 16 reports a routine delivery event. Sentence 17 reports a routine delivery event. Sentence 18 reports a routine delivery event. Sentence 19 reports a routine delivery event. Sentence 20 reports a routine delivery event. Sentence 21 reports a routine delivery event. Sentence 22 reports a routine delivery event. Sentence 23 reports a routine delivery event. Sentence 24 reports a routine delivery event. Sentence 25 reports a routine delivery event. Sentence 26 reports a routine delivery event. Sentence 27 reports a routine delivery event. Sentence 28 reports a routine delivery event. Sentence 29 reports a routine delivery event. Sentence 30 reports a routine delivery event. Sentence 31 reports a routine delivery event. Sentence 32 reports a routine delivery event. Sentence 33 reports a routine delivery event. Sentence 34 reports a routine delivery event. Sentence 35 reports a routine delivery event. Sentence 36 reports a routine delivery event. Sentence 37 reports a routine delivery event. Sentence 38 reports a routine delivery event. Sentence 39 reports a routine delivery event. Sentence 40 reports a routine delivery event. Sentence 41 reports a routine delivery event. Sentence 42 reports a routine delivery event. Sentence 43 reports a routine delivery event. Sentence 44 reports a routine delivery event. Sentence 45 reports a routine delivery event. Sentence 46 reports a routine delivery event. Sentence 47 reports a routine delivery event. Sentence 48 reports a routine delivery event. Sentence 49 reports a routine delivery event. Sentence 50 reports a routine delivery event. Sentence 51 reports a routine delivery event. Sentence 52 reports a routine delivery event. Sentence 53 reports a routine delivery event. Sentence 54 reports a routine delivery event. Sentence 55 reports a routine delivery event. Sentence 56 reports a routine delivery event. Sentence 57 reports a routine delivery event. Sentence 58 reports a routine delivery event. Sentence 59 reports a routine delivery event. Sentence 60 reports a routine delivery event. Sentence 61 reports a routine delivery event. Sentence 62 reports a routine delivery event. Sentence 63 reports a routine delivery event. Sentence 64 reports a routine delivery event. Sentence 65 reports a routine delivery event. Sentence 66 reports a routine delivery event. Sentence 67 reports a routine delivery event. Sentence 68 reports a routine delivery event. Sentence 69 reports a routine delivery event. Sentence 70 reports a routine delivery event. Sentence 71 reports a routine delivery event. Sentence 72 reports a routine delivery event. Sentence 73 reports a routine delivery event. Sentence 74 reports a routine delivery event. Sentence 75 reports a routine delivery event. Sentence 76 reports a routine delivery event. Sentence 77 reports a routine delivery event. Sentence 78 reports a routine delivery event. Sentence 79 reports a routine delivery event. Sentence 80 reports a routine delivery event. Sentence 81 reports a routine delivery event. Sentence 82 reports a routine delivery event. Sentence 83 reports a routine delivery event. Sentence 84 reports a routine delivery event. Sentence 85 reports a routine delivery event. Sentence 86 reports a routine delivery event. Sentence 87 reports a routine delivery event. Sentence 88 reports a routine delivery event. Sentence 89 reports a routine delivery event. Sentence 90 reports a routine delivery event. Sentence 91 reports a routine delivery event. Sentence 92 reports a routine delivery event. Sentence 93 reports a routine delivery event. Sentence 94 reports a routine delivery event. Sentence 95 reports a routine delivery event. Sentence 96 reports a routine delivery event. Sentence 97 reports a routine delivery event. Sentence 98 reports a routine delivery event. Sentence 99 reports a routine delivery event. Sentence 100 reports a routine delivery event. Sentence 101 reports a routine delivery event. Sentence 102 reports a routine delivery event. Sentence 103 reports a routine delivery event. Sentence 104 reports a routine delivery event. Sentence 105 reports a routine delivery event. Sentence 106 reports a routine delivery event. Sentence 107 reports a routine delivery event. Sentence 108 reports a routine delivery event. Sentence 109 reports a routine delivery event. Sentence 110 reports a routine delivery event. Sentence 111 reports a routine delivery event. Sentence 112 reports a routine delivery event. Sentence 113 reports a routine delivery event. Sentence 114 reports a routine delivery event. Sentence 115 reports a routine delivery event. Sentence 116 reports a routine delivery event. Sentence 117 reports a routine delivery event. Sentence 118 reports a routine delivery event. Sentence 119 reports a routine delivery event. Sentence 120 reports a routine delivery event. Sentence 121 reports a routine delivery event. Sentence 122 reports a routine delivery event. Sentence 123 reports a routine delivery event. Sentence 124 reports a routine delivery event. Sentence 125 reports a routine delivery event. Sentence 126 reports a routine delivery event. Sentence 127 reports a routine delivery event. Sentence 128 reports a routine delivery event. Sentence 129 reports a routine delivery event. Sentence 130 reports a routine delivery event. Sentence 131 reports a routine delivery event. Sentence 132 reports a routine delivery event. Sentence 133 reports a routine delivery event. Sentence 134 reports a routine delivery event. Sentence 135 reports a routine delivery event. Sentence 136 reports a routine delivery event. Sentence 137 reports a routine delivery event. Sentence 138 reports a routine delivery event. Sentence 139 reports a routine delivery event. Sentence 140 reports a routine delivery event. Sentence 141 reports a routine delivery event. Sentence 142 reports a routine delivery event. Sentence 143 reports a routine delivery event. Sentence 144 reports a routine delivery event. Sentence 145 reports a routine delivery event. Sentence 146 reports a routine delivery event. Sentence 147 reports a routine delivery event. Sentence 148 reports a routine delivery event. Sentence 149 reports a routine delivery event. Sentence 150 reports a routine delivery event. Sentence 151 reports a routine delivery event. Sentence 152 reports a routine delivery event. Sentence 153 reports a routine delivery event. Sentence 154 reports a routine delivery event. Sentence 155 reports a routine delivery event. Sentence 156 reports a routine delivery event. Sentence 157 reports a routine delivery event. Sentence 158 reports a routine delivery event. Sentence 159 reports a routine delivery event. Sentence 160 reports a routine delivery event. Sentence 161 reports a routine delivery event. Sentence 162 reports a routine delivery event. Sentence 163 reports a routine delivery event. Sentence 164 reports a routine delivery event. Sentence 165 reports a routine delivery event. Sentence 166 reports a routine delivery event. Sentence 167 reports a routine delivery event. Sentence 168 reports a routine delivery event. Sentence 169 reports a routine delivery event. Sentence 170 reports a routine delivery event. Sentence 171 reports a routine delivery event. Sentence 172 reports a routine delivery event. Sentence 173 reports a routine delivery event. Sentence 174 reports a routine delivery event. Sentence 175 reports a routine delivery event. Sentence 176 reports a routine delivery event. Sentence 177 reports a routine delivery event. Sentence 178 reports a routine delivery event. Sentence 179 reports a routine delivery event. Sentence 180 reports a routine delivery event. Sentence 181 reports a routine delivery event. Sentence 182 reports a routine delivery event. Sentence 183 reports a routine delivery event. Sentence 184 reports a routine delivery event. Sentence 185 reports a routine delivery event. Sentence 186 reports a routine delivery event. Sentence 187 reports a routine delivery event. Sentence 188 reports a routine delivery event. Sentence 189 reports a routine delivery event. Sentence 190 reports a routine delivery event. Sentence 191 reports a routine delivery event. Sentence 192 reports a routine delivery event. Sentence 193 reports a routine delivery event. Sentence 194 reports a routine delivery event. Sentence 195 reports a routine delivery event. Sentence 196 reports a routine delivery event. Sentence 197 reports a routine delivery event. Sentence 198 reports a routine delivery event. Sentence 199 reports a routine delivery event. Sentence 200 reports a routine delivery event. Sentence 201 reports a routine delivery event. Sentence 202 reports a routine delivery event. Sentence 203 reports a routine delivery event. Sentence 204 reports a routine delivery event. Sentence 205 reports a routine delivery event. Sentence 206 reports a routine delivery event. Sentence 207 reports a routine delivery event. Sentence 208 reports a routine delivery event. Sentence 209 reports a routine delivery event. Sentence 210 reports a routine delivery event. Sentence 211 reports a routine delivery event. Sentence 212 reports a routine delivery event. Sentence 213 reports a routine delivery event. Sentence 214 reports a routine delivery event. Sentence 215 reports a routine delivery event. Sentence 216 reports a routine delivery event. Sentence 217 reports a routine delivery event. Sentence 218 reports a routine delivery event. Sentence 219 reports a routine delivery event."}, "questions": {"claims": {"type": "span", "instructions": "Which spans mention a delivery?", "criteria": {"delivery": "a delivery event"}, "over": "request"}}}, "predict_request": {"parts": [{"type": "user", "text": "Sentence 0 reports a routine delivery event. Sentence 1 reports a routine delivery event. Sentence 2 reports a routine delivery event. Sentence 3 reports a routine delivery event. Sentence 4 reports a routine delivery event. Sentence 5 reports a routine delivery event. Sentence 6 reports a routine delivery event. Sentence 7 reports a routine delivery event. Sentence 8 reports a routine delivery event. Sentence 9 reports a routine delivery event. Sentence 10 reports a routine delivery event. Sentence 11 reports a routine delivery event. Sentence 12 reports a routine delivery event. Sentence 13 reports a routine delivery event. Sentence 14 reports a routine delivery event. Sentence 15 reports a routine delivery event. Sentence 16 reports a routine delivery event. Sentence 17 reports a routine delivery event. Sentence 18 reports a routine delivery event. Sentence 19 reports a routine delivery event. Sentence 20 reports a routine delivery event. Sentence 21 reports a routine delivery event. Sentence 22 reports a routine delivery event. Sentence 23 reports a routine delivery event. Sentence 24 reports a routine delivery event. Sentence 25 reports a routine delivery event. Sentence 26 reports a routine delivery event. Sentence 27 reports a routine delivery event. Sentence 28 reports a routine delivery event. Sentence 29 reports a routine delivery event. Sentence 30 reports a routine delivery event. Sentence 31 reports a routine delivery event. Sentence 32 reports a routine delivery event. Sentence 33 reports a routine delivery event. Sentence 34 reports a routine delivery event. Sentence 35 reports a routine delivery event. Sentence 36 reports a routine delivery event. Sentence 37 reports a routine delivery event. Sentence 38 reports a routine delivery event. Sentence 39 reports a routine delivery event. Sentence 40 reports a routine delivery event. Sentence 41 reports a routine delivery event. Sentence 42 reports a routine delivery event. Sentence 43 reports a routine delivery event. Sentence 44 reports a routine delivery event. Sentence 45 reports a routine delivery event. Sentence 46 reports a routine delivery event. Sentence 47 reports a routine delivery event. Sentence 48 reports a routine delivery event. Sentence 49 reports a routine delivery event. Sentence 50 reports a routine delivery event. Sentence 51 reports a routine delivery event. Sentence 52 reports a routine delivery event. Sentence 53 reports a routine delivery event. Sentence 54 reports a routine delivery event. Sentence 55 reports a routine delivery event. Sentence 56 reports a routine delivery event. Sentence 57 reports a routine delivery event. Sentence 58 reports a routine delivery event. Sentence 59 reports a routine delivery event. Sentence 60 reports a routine delivery event. Sentence 61 reports a routine delivery event. Sentence 62 reports a routine delivery event. Sentence 63 reports a routine delivery event. Sentence 64 reports a routine delivery event. Sentence 65 reports a routine delivery event. Sentence 66 reports a routine delivery event. Sentence 67 reports a routine delivery event. Sentence 68 reports a routine delivery event. Sentence 69 reports a routine delivery event. Sentence 70 reports a routine delivery event. Sentence 71 reports a routine delivery event. Sentence 72 reports a routine delivery event. Sentence 73 reports a routine delivery event. Sentence 74 reports a routine delivery event. Sentence 75 reports a routine delivery event. Sentence 76 reports a routine delivery event. Sentence 77 reports a routine delivery event. Sentence 78 reports a routine delivery event. Sentence 79 reports a routine delivery event. Sentence 80 reports a routine delivery event. Sentence 81 reports a routine delivery event. Sentence 82 reports a routine delivery event. Sentence 83 reports a routine delivery event. Sentence 84 reports a routine delivery event. Sentence 85 reports a routine delivery event. Sentence 86 reports a routine delivery event. Sentence 87 reports a routine delivery event. Sentence 88 reports a routine delivery event. Sentence 89 reports a routine delivery event. Sentence 90 reports a routine delivery event. Sentence 91 reports a routine delivery event. Sentence 92 reports a routine delivery event. Sentence 93 reports a routine delivery event. Sentence 94 reports a routine delivery event. Sentence 95 reports a routine delivery event. Sentence 96 reports a routine delivery event. Sentence 97 reports a routine delivery event. Sentence 98 reports a routine delivery event. Sentence 99 reports a routine delivery event. Sentence 100 reports a routine delivery event. Sentence 101 reports a routine delivery event. Sentence 102 reports a routine delivery event. Sentence 103 reports a routine delivery event. Sentence 104 reports a routine delivery event. Sentence 105 reports a routine delivery event. Sentence 106 reports a routine delivery event. Sentence 107 reports a routine delivery event. Sentence 108 reports a routine delivery event. Sentence 109 reports a routine delivery event. Sentence 110 reports a routine delivery event. Sentence 111 reports a routine delivery event. Sentence 112 reports a routine delivery event. Sentence 113 reports a routine delivery event. Sentence 114 reports a routine delivery event. Sentence 115 reports a routine delivery event. Sentence 116 reports a routine delivery event. Sentence 117 reports a routine delivery event. Sentence 118 reports a routine delivery event. Sentence 119 reports a routine delivery event. Sentence 120 reports a routine delivery event. Sentence 121 reports a routine delivery event. Sentence 122 reports a routine delivery event. Sentence 123 reports a routine delivery event. Sentence 124 reports a routine delivery event. Sentence 125 reports a routine delivery event. Sentence 126 reports a routine delivery event. Sentence 127 reports a routine delivery event. Sentence 128 reports a routine delivery event. Sentence 129 reports a routine delivery event. Sentence 130 reports a routine delivery event. Sentence 131 reports a routine delivery event. Sentence 132 reports a routine delivery event. Sentence 133 reports a routine delivery event. Sentence 134 reports a routine delivery event. Sentence 135 reports a routine delivery event. Sentence 136 reports a routine delivery event. Sentence 137 reports a routine delivery event. Sentence 138 reports a routine delivery event. Sentence 139 reports a routine delivery event. Sentence 140 reports a routine delivery event. Sentence 141 reports a routine delivery event. Sentence 142 reports a routine delivery event. Sentence 143 reports a routine delivery event. Sentence 144 reports a routine delivery event. Sentence 145 reports a routine delivery event. Sentence 146 reports a routine delivery event. Sentence 147 reports a routine delivery event. Sentence 148 reports a routine delivery event. Sentence 149 reports a routine delivery event. Sentence 150 reports a routine delivery event. Sentence 151 reports a routine delivery event. Sentence 152 reports a routine delivery event. Sentence 153 reports a routine delivery event. Sentence 154 reports a routine delivery event. Sentence 155 reports a routine delivery event. Sentence 156 reports a routine delivery event. Sentence 157 reports a routine delivery event. Sentence 158 reports a routine delivery event. Sentence 159 reports a routine delivery event. Sentence 160 reports a routine delivery event. Sentence 161 reports a routine delivery event. Sentence 162 reports a routine delivery event. Sentence 163 reports a routine delivery event. Sentence 164 reports a routine delivery event. Sentence 165 reports a routine delivery event. Sentence 166 reports a routine delivery event. Sentence 167 reports a routine delivery event. Sentence 168 reports a routine delivery event. Sentence 169 reports a routine delivery event. Sentence 170 reports a routine delivery event. Sentence 171 reports a routine delivery event. Sentence 172 reports a routine delivery event. Sentence 173 reports a routine delivery event. Sentence 174 reports a routine delivery event. Sentence 175 reports a routine delivery event. Sentence 176 reports a routine delivery event. Sentence 177 reports a routine delivery event. Sentence 178 reports a routine delivery event. Sentence 179 reports a routine delivery event. Sentence 180 reports a routine delivery event. Sentence 181 reports a routine delivery event. Sentence 182 reports a routine delivery event. Sentence 183 reports a routine delivery event. Sentence 184 reports a routine delivery event. Sentence 185 reports a routine delivery event. Sentence 186 reports a routine delivery event. Sentence 187 reports a routine delivery event. Sentence 188 reports a routine delivery event. Sentence 189 reports a routine delivery event. Sentence 190 reports a routine delivery event. Sentence 191 reports a routine delivery event. Sentence 192 reports a routine delivery event. Sentence 193 reports a routine delivery event. Sentence 194 reports a routine delivery event. Sentence 195 reports a routine delivery event. Sentence 196 reports a routine delivery event. Sentence 197 reports a routine delivery event. Sentence 198 reports a routine delivery event. Sentence 199 reports a routine delivery event. Sentence 200 reports a routine delivery event. Sentence 201 reports a routine delivery event. Sentence 202 reports a routine delivery event. Sentence 203 reports a routine delivery event. Sentence 204 reports a routine delivery event. Sentence 205 reports a routine delivery event. Sentence 206 reports a routine delivery event. Sentence 207 reports a routine delivery event. Sentence 208 reports a routine delivery event. Sentence 209 reports a routine delivery event. Sentence 210 reports a routine delivery event. Sentence 211 reports a routine delivery event. Sentence 212 reports a routine delivery event. Sentence 213 reports a routine delivery event. Sentence 214 reports a routine delivery event. Sentence 215 reports a routine delivery event. Sentence 216 reports a routine delivery event. Sentence 217 reports a routine delivery event. Sentence 218 reports a routine delivery event. Sentence 219 reports a routine delivery event."}], "questions": [{"id": "claims", "type": "span", "text": "Which spans mention a delivery?", "target_part": "user", "labels": [{"name": "delivery", "description": "a delivery event"}]}]}, "plan": [{"qid": "claims", "type": "span", "criteria": {"delivery": "a delivery event"}, "range": [0, 10229], "part": "user"}], "rows": [{"p_ids": [1905, 25, 198, 27, 22446, 3364, 420, 846, 731, 198, 81646, 220, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 24, 6473, 264, 13618, 9403, 1484, 13, 198, 510, 22446, 29, 198], "blocks": [{"kind": "span", "ends": [40], "label_ranges": [[22, 40]], "words_in_block": false, "ids": [271, 6065, 913, 25, 9111, 198, 6196, 25, 1156, 198, 14162, 25, 198, 22365, 42824, 6088, 264, 9403, 30, 198, 22933, 25, 198, 14057, 29, 198, 4754, 4532, 3147, 64, 9403, 1484, 2129, 767, 3147, 31320, 8934, 198, 510, 1448, 29, 271, 8679, 1396, 9111, 314, 279, 2100, 1414, 421, 22651, 799, 314, 279, 8928, 13, 198, 6205, 581, 25], "query": 59, "w_pos": [10, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 23, 24, 25, 26, 27, 28, 30, 31, 32, 33, 34, 35, 36, 37, 39, 40, 41, 42, 43, 44, 45, 46, 48, 49, 50, 51, 52, 53, 54, 55, 57, 58, 59, 60, 61, 62, 63, 64, 66, 67, 68, 69, 70, 71, 72, 73, 75, 76, 77, 78, 79, 80, 81, 82, 84, 85, 86, 87, 88, 89, 90, 91, 93, 94, 95, 96, 97, 98, 99, 100, 102, 104, 105, 106, 107, 108, 109, 110, 112, 114, 115, 116, 117, 118, 119, 120, 122, 124, 125, 126, 127, 128, 129, 130, 132, 134, 135, 136, 137, 138, 139, 140, 142, 144, 145, 146, 147, 148, 149, 150, 152, 154, 155, 156, 157, 158, 159, 160, 162, 164, 165, 166, 167, 168, 169, 170, 172, 174, 175, 176, 177, 178, 179, 180, 182, 184, 185, 186, 187, 188, 189, 190, 192, 194, 195, 196, 197, 198, 199, 200, 202, 204, 205, 206, 207, 208, 209, 210, 212, 214, 215, 216, 217, 218, 219, 220, 222, 224, 225, 226, 227, 228, 229, 230, 232, 234, 235, 236, 237, 238, 239, 240, 242, 244, 245, 246, 247, 248, 249, 250, 252, 254, 255, 256, 257, 258, 259, 260, 262, 264, 265, 266, 267, 268, 269, 270, 272, 274, 275, 276, 277, 278, 279, 280, 282, 284, 285, 286, 287, 288, 289, 290, 292, 294, 295, 296, 297, 298, 299, 300, 302, 304, 305, 306, 307, 308, 309, 310, 312, 314, 315, 316, 317, 318, 319, 320, 322, 324, 325, 326, 327, 328, 329, 330, 332, 334, 335, 336, 337, 338, 339, 340, 342, 344, 345, 346, 347, 348, 349, 350, 352, 354, 355, 356, 357, 358, 359, 360, 362, 364, 365, 366, 367, 368, 369, 370, 372, 374, 375, 376, 377, 378, 379, 380, 382, 384, 385, 386, 387, 388, 389, 390, 392, 394, 395, 396, 397, 398, 399, 400, 402, 404, 405, 406, 407, 408, 409, 410, 412, 414, 415, 416, 417, 418, 419, 420, 422, 424, 425, 426, 427, 428, 429, 430, 432, 434, 435, 436, 437, 438, 439, 440, 442, 444, 445, 446, 447, 448, 449, 450, 452, 454, 455, 456, 457, 458, 459, 460, 462, 464, 465, 466, 467, 468, 469, 470, 472, 474, 475, 476, 477, 478, 479, 480, 482, 484, 485, 486, 487, 488, 489, 490, 492, 494, 495, 496, 497, 498, 499, 500, 502, 504, 505, 506, 507, 508, 509, 510, 512, 514, 515, 516, 517, 518, 519, 520, 522, 524, 525, 526, 527, 528, 529, 530, 532, 534, 535, 536, 537, 538, 539, 540, 542, 544, 545, 546, 547, 548, 549, 550, 552, 554, 555, 556, 557, 558, 559, 560, 562, 564, 565, 566, 567, 568, 569, 570, 572, 574, 575, 576, 577, 578, 579, 580, 582, 584, 585, 586, 587, 588, 589, 590, 592, 594, 595, 596, 597, 598, 599, 600, 602, 604, 605, 606, 607, 608, 609, 610, 612, 614, 615, 616, 617, 618, 619, 620, 622, 624, 625, 626, 627, 628, 629, 630, 632, 634, 635, 636, 637, 638, 639, 640, 642, 644, 645, 646, 647, 648, 649, 650, 652, 654, 655, 656, 657, 658, 659, 660, 662, 664, 665, 666, 667, 668, 669, 670, 672, 674, 675, 676, 677, 678, 679, 680, 682, 684, 685, 686, 687, 688, 689, 690, 692, 694, 695, 696, 697, 698, 699, 700, 702, 704, 705, 706, 707, 708, 709, 710, 712, 714, 715, 716, 717, 718, 719, 720, 722, 724, 725, 726, 727, 728, 729, 730, 732, 734, 735, 736, 737, 738, 739, 740, 742, 744, 745, 746, 747, 748, 749, 750, 752, 754, 755, 756, 757, 758, 759, 760, 762, 764, 765, 766, 767, 768, 769, 770, 772, 774, 775, 776, 777, 778, 779, 780, 782, 784, 785, 786, 787, 788, 789, 790, 792, 794, 795, 796, 797, 798, 799, 800, 802, 804, 805, 806, 807, 808, 809, 810, 812, 814, 815, 816, 817, 818, 819, 820, 822, 824, 825, 826, 827, 828, 829, 830, 832, 834, 835, 836, 837, 838, 839, 840, 842, 844, 845, 846, 847, 848, 849, 850, 852, 854, 855, 856, 857, 858, 859, 860, 862, 864, 865, 866, 867, 868, 869, 870, 872, 874, 875, 876, 877, 878, 879, 880, 882, 884, 885, 886, 887, 888, 889, 890, 892, 894, 895, 896, 897, 898, 899, 900, 902, 904, 905, 906, 907, 908, 909, 910, 912, 914, 915, 916, 917, 918, 919, 920, 922, 924, 925, 926, 927, 928, 929, 930, 932, 934, 935, 936, 937, 938, 939, 940, 942, 944, 945, 946, 947, 948, 949, 950, 952, 954, 955, 956, 957, 958, 959, 960, 962, 964, 965, 966, 967, 968, 969, 970, 972, 974, 975, 976, 977, 978, 979, 980, 982, 984, 985, 986, 987, 988, 989, 990, 992, 994, 995, 996, 997, 998, 999, 1000, 1002, 1005, 1006, 1007, 1008, 1009, 1010, 1011, 1013, 1016, 1017, 1018, 1019, 1020, 1021, 1022, 1024, 1027, 1028, 1029, 1030, 1031, 1032, 1033, 1035, 1038, 1039, 1040, 1041, 1042, 1043, 1044, 1046, 1049, 1050, 1051, 1052, 1053, 1054, 1055, 1057, 1060, 1061, 1062, 1063, 1064, 1065, 1066, 1068, 1071, 1072, 1073, 1074, 1075, 1076, 1077, 1079, 1082, 1083, 1084, 1085, 1086, 1087, 1088, 1090, 1093, 1094, 1095, 1096, 1097, 1098, 1099, 1101, 1104, 1105, 1106, 1107, 1108, 1109, 1110, 1112, 1115, 1116, 1117, 1118, 1119, 1120, 1121, 1123, 1126, 1127, 1128, 1129, 1130, 1131, 1132, 1134, 1137, 1138, 1139, 1140, 1141, 1142, 1143, 1145, 1148, 1149, 1150, 1151, 1152, 1153, 1154, 1156, 1159, 1160, 1161, 1162, 1163, 1164, 1165, 1167, 1170, 1171, 1172, 1173, 1174, 1175, 1176, 1178, 1181, 1182, 1183, 1184, 1185, 1186, 1187, 1189, 1192, 1193, 1194, 1195, 1196, 1197, 1198, 1200, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1211, 1214, 1215, 1216, 1217, 1218, 1219, 1220, 1222, 1225, 1226, 1227, 1228, 1229, 1230, 1231, 1233, 1236, 1237, 1238, 1239, 1240, 1241, 1242, 1244, 1247, 1248, 1249, 1250, 1251, 1252, 1253, 1255, 1258, 1259, 1260, 1261, 1262, 1263, 1264, 1266, 1269, 1270, 1271, 1272, 1273, 1274, 1275, 1277, 1280, 1281, 1282, 1283, 1284, 1285, 1286, 1288, 1291, 1292, 1293, 1294, 1295, 1296, 1297, 1299, 1302, 1303, 1304, 1305, 1306, 1307, 1308, 1310, 1313, 1314, 1315, 1316, 1317, 1318, 1319, 1321, 1324, 1325, 1326, 1327, 1328, 1329, 1330, 1332, 1335, 1336, 1337, 1338, 1339, 1340, 1341, 1343, 1346, 1347, 1348, 1349, 1350, 1351, 1352, 1354, 1357, 1358, 1359, 1360, 1361, 1362, 1363, 1365, 1368, 1369, 1370, 1371, 1372, 1373, 1374, 1376, 1379, 1380, 1381, 1382, 1383, 1384, 1385, 1387, 1390, 1391, 1392, 1393, 1394, 1395, 1396, 1398, 1401, 1402, 1403, 1404, 1405, 1406, 1407, 1409, 1412, 1413, 1414, 1415, 1416, 1417, 1418, 1420, 1423, 1424, 1425, 1426, 1427, 1428, 1429, 1431, 1434, 1435, 1436, 1437, 1438, 1439, 1440, 1442, 1445, 1446, 1447, 1448, 1449, 1450, 1451, 1453, 1456, 1457, 1458, 1459, 1460, 1461, 1462, 1464, 1467, 1468, 1469, 1470, 1471, 1472, 1473, 1475, 1478, 1479, 1480, 1481, 1482, 1483, 1484, 1486, 1489, 1490, 1491, 1492, 1493, 1494, 1495, 1497, 1500, 1501, 1502, 1503, 1504, 1505, 1506, 1508, 1511, 1512, 1513, 1514, 1515, 1516, 1517, 1519, 1522, 1523, 1524, 1525, 1526, 1527, 1528, 1530, 1533, 1534, 1535, 1536, 1537, 1538, 1539, 1541, 1544, 1545, 1546, 1547, 1548, 1549, 1550, 1552, 1555, 1556, 1557, 1558, 1559, 1560, 1561, 1563, 1566, 1567, 1568, 1569, 1570, 1571, 1572, 1574, 1577, 1578, 1579, 1580, 1581, 1582, 1583, 1585, 1588, 1589, 1590, 1591, 1592, 1593, 1594, 1596, 1599, 1600, 1601, 1602, 1603, 1604, 1605, 1607, 1610, 1611, 1612, 1613, 1614, 1615, 1616, 1618, 1621, 1622, 1623, 1624, 1625, 1626, 1627, 1629, 1632, 1633, 1634, 1635, 1636, 1637, 1638, 1640, 1643, 1644, 1645, 1646, 1647, 1648, 1649, 1651, 1654, 1655, 1656, 1657, 1658, 1659, 1660, 1662, 1665, 1666, 1667, 1668, 1669, 1670, 1671, 1673, 1676, 1677, 1678, 1679, 1680, 1681, 1682, 1684, 1687, 1688, 1689, 1690, 1691, 1692, 1693, 1695, 1698, 1699, 1700, 1701, 1702, 1703, 1704, 1706, 1709, 1710, 1711, 1712, 1713, 1714, 1715, 1717, 1720, 1721, 1722, 1723, 1724, 1725, 1726, 1728, 1731, 1732, 1733, 1734, 1735, 1736, 1737, 1739, 1742, 1743, 1744, 1745, 1746, 1747, 1748, 1750, 1753, 1754, 1755, 1756, 1757, 1758, 1759, 1761, 1764, 1765, 1766, 1767, 1768, 1769, 1770, 1772, 1775, 1776, 1777, 1778, 1779, 1780, 1781, 1783, 1786, 1787, 1788, 1789, 1790, 1791, 1792, 1794, 1797, 1798, 1799, 1800, 1801, 1802, 1803, 1805, 1808, 1809, 1810, 1811, 1812, 1813, 1814, 1816, 1819, 1820, 1821, 1822, 1823, 1824, 1825, 1827, 1830, 1831, 1832, 1833, 1834, 1835, 1836, 1838, 1841, 1842, 1843, 1844, 1845, 1846, 1847, 1849, 1852, 1853, 1854, 1855, 1856, 1857, 1858, 1860, 1863, 1864, 1865, 1866, 1867, 1868, 1869, 1871, 1874, 1875, 1876, 1877, 1878, 1879, 1880, 1882, 1885, 1886, 1887, 1888, 1889, 1890, 1891, 1893, 1896, 1897, 1898, 1899, 1900, 1901, 1902, 1904, 1907, 1908, 1909, 1910, 1911, 1912, 1913, 1915, 1918, 1919, 1920, 1921, 1922, 1923, 1924, 1926, 1929, 1930, 1931, 1932, 1933, 1934, 1935, 1937, 1940, 1941, 1942, 1943, 1944, 1945, 1946, 1948, 1951, 1952, 1953, 1954, 1955, 1956, 1957, 1959, 1962, 1963, 1964, 1965, 1966, 1967, 1968, 1970, 1973, 1974, 1975, 1976, 1977, 1978, 1979, 1981, 1984, 1985, 1986, 1987, 1988, 1989, 1990, 1992, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2003, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2014, 2017, 2018, 2019, 2020, 2021, 2022, 2023, 2025, 2028, 2029, 2030, 2031, 2032, 2033, 2034, 2036, 2039, 2040, 2041, 2042, 2043, 2044, 2045, 2047, 2050, 2051, 2052, 2053, 2054, 2055, 2056, 2058, 2061, 2062, 2063, 2064, 2065, 2066, 2067, 2069, 2072, 2073, 2074, 2075, 2076, 2077, 2078, 2080, 2083, 2084, 2085, 2086, 2087, 2088, 2089, 2091, 2094, 2095, 2096, 2097, 2098, 2099, 2100, 2102, 2105, 2106, 2107, 2108, 2109, 2110, 2111, 2113, 2116, 2117, 2118, 2119, 2120, 2121, 2122, 2124, 2127, 2128, 2129, 2130, 2131, 2132, 2133, 2135, 2138, 2139, 2140, 2141, 2142, 2143, 2144, 2146, 2149, 2150, 2151, 2152, 2153, 2154, 2155, 2157, 2160, 2161, 2162, 2163, 2164, 2165, 2166, 2168, 2171, 2172, 2173, 2174, 2175, 2176, 2177, 2179, 2182, 2183, 2184, 2185, 2186, 2187, 2188, 2190, 2193, 2194, 2195, 2196, 2197, 2198, 2199, 2201, 2204, 2205, 2206, 2207, 2208, 2209, 2210, 2212, 2215, 2216, 2217, 2218, 2219, 2220, 2221, 2223, 2226, 2227, 2228, 2229, 2230, 2231, 2232, 2234, 2237, 2238, 2239, 2240, 2241, 2242, 2243, 2245, 2248, 2249, 2250, 2251, 2252, 2253, 2254, 2256, 2259, 2260, 2261, 2262, 2263, 2264, 2265, 2267, 2270, 2271, 2272, 2273, 2274, 2275, 2276, 2278, 2281, 2282, 2283, 2284, 2285, 2286, 2287, 2289, 2292, 2293, 2294, 2295, 2296, 2297, 2298, 2300, 2303, 2304, 2305, 2306, 2307, 2308, 2309, 2311, 2314, 2315, 2316, 2317, 2318, 2319]}], "trunc": {"shrink_cut": 0, "protected_cut": 0}, "seg_lens": {"user": 2310}, "layout": "a", "names": ["delivery"], "w_offs": [[0, 8], [9, 10], [11, 18], [19, 20], [21, 28], [29, 37], [38, 43], [43, 44], [45, 53], [54, 55], [56, 63], [64, 65], [66, 73], [74, 82], [83, 88], [88, 89], [90, 98], [99, 100], [101, 108], [109, 110], [111, 118], [119, 127], [128, 133], [133, 134], [135, 143], [144, 145], [146, 153], [154, 155], [156, 163], [164, 172], [173, 178], [178, 179], [180, 188], [189, 190], [191, 198], [199, 200], [201, 208], [209, 217], [218, 223], [223, 224], [225, 233], [234, 235], [236, 243], [244, 245], [246, 253], [254, 262], [263, 268], [268, 269], [270, 278], [279, 280], [281, 288], [289, 290], [291, 298], [299, 307], [308, 313], [313, 314], [315, 323], [324, 325], [326, 333], [334, 335], [336, 343], [344, 352], [353, 358], [358, 359], [360, 368], [369, 370], [371, 378], [379, 380], [381, 388], [389, 397], [398, 403], [403, 404], [405, 413], [414, 415], [416, 423], [424, 425], [426, 433], [434, 442], [443, 448], [448, 449], [450, 458], [459, 461], [462, 469], [470, 471], [472, 479], [480, 488], [489, 494], [494, 495], [496, 504], [505, 507], [508, 515], [516, 517], [518, 525], [526, 534], [535, 540], [540, 541], [542, 550], [551, 553], [554, 561], [562, 563], [564, 571], [572, 580], [581, 586], [586, 587], [588, 596], [597, 599], [600, 607], [608, 609], [610, 617], [618, 626], [627, 632], [632, 633], [634, 642], [643, 645], [646, 653], [654, 655], [656, 663], [664, 672], [673, 678], [678, 679], [680, 688], [689, 691], [692, 699], [700, 701], [702, 709], [710, 718], [719, 724], [724, 725], [726, 734], [735, 737], [738, 745], [746, 747], [748, 755], [756, 764], [765, 770], [770, 771], [772, 780], [781, 783], [784, 791], [792, 793], [794, 801], [802, 810], [811, 816], [816, 817], [818, 826], [827, 829], [830, 837], [838, 839], [840, 847], [848, 856], [857, 862], [862, 863], [864, 872], [873, 875], [876, 883], [884, 885], [886, 893], [894, 902], [903, 908], [908, 909], [910, 918], [919, 921], [922, 929], [930, 931], [932, 939], [940, 948], [949, 954], [954, 955], [956, 964], [965, 967], [968, 975], [976, 977], [978, 985], [986, 994], [995, 1000], [1000, 1001], [1002, 1010], [1011, 1013], [1014, 1021], [1022, 1023], [1024, 1031], [1032, 1040], [1041, 1046], [1046, 1047], [1048, 1056], [1057, 1059], [1060, 1067], [1068, 1069], [1070, 1077], [1078, 1086], [1087, 1092], [1092, 1093], [1094, 1102], [1103, 1105], [1106, 1113], [1114, 1115], [1116, 1123], [1124, 1132], [1133, 1138], [1138, 1139], [1140, 1148], [1149, 1151], [1152, 1159], [1160, 1161], [1162, 1169], [1170, 1178], [1179, 1184], [1184, 1185], [1186, 1194], [1195, 1197], [1198, 1205], [1206, 1207], [1208, 1215], [1216, 1224], [1225, 1230], [1230, 1231], [1232, 1240], [1241, 1243], [1244, 1251], [1252, 1253], [1254, 1261], [1262, 1270], [1271, 1276], [1276, 1277], [1278, 1286], [1287, 1289], [1290, 1297], [1298, 1299], [1300, 1307], [1308, 1316], [1317, 1322], [1322, 1323], [1324, 1332], [1333, 1335], [1336, 1343], [1344, 1345], [1346, 1353], [1354, 1362], [1363, 1368], [1368, 1369], [1370, 1378], [1379, 1381], [1382, 1389], [1390, 1391], [1392, 1399], [1400, 1408], [1409, 1414], [1414, 1415], [1416, 1424], [1425, 1427], [1428, 1435], [1436, 1437], [1438, 1445], [1446, 1454], [1455, 1460], [1460, 1461], [1462, 1470], [1471, 1473], [1474, 1481], [1482, 1483], [1484, 1491], [1492, 1500], [1501, 1506], [1506, 1507], [1508, 1516], [1517, 1519], [1520, 1527], [1528, 1529], [1530, 1537], [1538, 1546], [1547, 1552], [1552, 1553], [1554, 1562], [1563, 1565], [1566, 1573], [1574, 1575], [1576, 1583], [1584, 1592], [1593, 1598], [1598, 1599], [1600, 1608], [1609, 1611], [1612, 1619], [1620, 1621], [1622, 1629], [1630, 1638], [1639, 1644], [1644, 1645], [1646, 1654], [1655, 1657], [1658, 1665], [1666, 1667], [1668, 1675], [1676, 1684], [1685, 1690], [1690, 1691], [1692, 1700], [1701, 1703], [1704, 1711], [1712, 1713], [1714, 1721], [1722, 1730], [1731, 1736], [1736, 1737], [1738, 1746], [1747, 1749], [1750, 1757], [1758, 1759], [1760, 1767], [1768, 1776], [1777, 1782], [1782, 1783], [1784, 1792], [1793, 1795], [1796, 1803], [1804, 1805], [1806, 1813], [1814, 1822], [1823, 1828], [1828, 1829], [1830, 1838], [1839, 1841], [1842, 1849], [1850, 1851], [1852, 1859], [1860, 1868], [1869, 1874], [1874, 1875], [1876, 1884], [1885, 1887], [1888, 1895], [1896, 1897], [1898, 1905], [1906, 1914], [1915, 1920], [1920, 1921], [1922, 1930], [1931, 1933], [1934, 1941], [1942, 1943], [1944, 1951], [1952, 1960], [1961, 1966], [1966, 1967], [1968, 1976], [1977, 1979], [1980, 1987], [1988, 1989], [1990, 1997], [1998, 2006], [2007, 2012], [2012, 2013], [2014, 2022], [2023, 2025], [2026, 2033], [2034, 2035], [2036, 2043], [2044, 2052], [2053, 2058], [2058, 2059], [2060, 2068], [2069, 2071], [2072, 2079], [2080, 2081], [2082, 2089], [2090, 2098], [2099, 2104], [2104, 2105], [2106, 2114], [2115, 2117], [2118, 2125], [2126, 2127], [2128, 2135], [2136, 2144], [2145, 2150], [2150, 2151], [2152, 2160], [2161, 2163], [2164, 2171], [2172, 2173], [2174, 2181], [2182, 2190], [2191, 2196], [2196, 2197], [2198, 2206], [2207, 2209], [2210, 2217], [2218, 2219], [2220, 2227], [2228, 2236], [2237, 2242], [2242, 2243], [2244, 2252], [2253, 2255], [2256, 2263], [2264, 2265], [2266, 2273], [2274, 2282], [2283, 2288], [2288, 2289], [2290, 2298], [2299, 2301], [2302, 2309], [2310, 2311], [2312, 2319], [2320, 2328], [2329, 2334], [2334, 2335], [2336, 2344], [2345, 2347], [2348, 2355], [2356, 2357], [2358, 2365], [2366, 2374], [2375, 2380], [2380, 2381], [2382, 2390], [2391, 2393], [2394, 2401], [2402, 2403], [2404, 2411], [2412, 2420], [2421, 2426], [2426, 2427], [2428, 2436], [2437, 2439], [2440, 2447], [2448, 2449], [2450, 2457], [2458, 2466], [2467, 2472], [2472, 2473], [2474, 2482], [2483, 2485], [2486, 2493], [2494, 2495], [2496, 2503], [2504, 2512], [2513, 2518], [2518, 2519], [2520, 2528], [2529, 2531], [2532, 2539], [2540, 2541], [2542, 2549], [2550, 2558], [2559, 2564], [2564, 2565], [2566, 2574], [2575, 2577], [2578, 2585], [2586, 2587], [2588, 2595], [2596, 2604], [2605, 2610], [2610, 2611], [2612, 2620], [2621, 2623], [2624, 2631], [2632, 2633], [2634, 2641], [2642, 2650], [2651, 2656], [2656, 2657], [2658, 2666], [2667, 2669], [2670, 2677], [2678, 2679], [2680, 2687], [2688, 2696], [2697, 2702], [2702, 2703], [2704, 2712], [2713, 2715], [2716, 2723], [2724, 2725], [2726, 2733], [2734, 2742], [2743, 2748], [2748, 2749], [2750, 2758], [2759, 2761], [2762, 2769], [2770, 2771], [2772, 2779], [2780, 2788], [2789, 2794], [2794, 2795], [2796, 2804], [2805, 2807], [2808, 2815], [2816, 2817], [2818, 2825], [2826, 2834], [2835, 2840], [2840, 2841], [2842, 2850], [2851, 2853], [2854, 2861], [2862, 2863], [2864, 2871], [2872, 2880], [2881, 2886], [2886, 2887], [2888, 2896], [2897, 2899], [2900, 2907], [2908, 2909], [2910, 2917], [2918, 2926], [2927, 2932], [2932, 2933], [2934, 2942], [2943, 2945], [2946, 2953], [2954, 2955], [2956, 2963], [2964, 2972], [2973, 2978], [2978, 2979], [2980, 2988], [2989, 2991], [2992, 2999], [3000, 3001], [3002, 3009], [3010, 3018], [3019, 3024], [3024, 3025], [3026, 3034], [3035, 3037], [3038, 3045], [3046, 3047], [3048, 3055], [3056, 3064], [3065, 3070], [3070, 3071], [3072, 3080], [3081, 3083], [3084, 3091], [3092, 3093], [3094, 3101], [3102, 3110], [3111, 3116], [3116, 3117], [3118, 3126], [3127, 3129], [3130, 3137], [3138, 3139], [3140, 3147], [3148, 3156], [3157, 3162], [3162, 3163], [3164, 3172], [3173, 3175], [3176, 3183], [3184, 3185], [3186, 3193], [3194, 3202], [3203, 3208], [3208, 3209], [3210, 3218], [3219, 3221], [3222, 3229], [3230, 3231], [3232, 3239], [3240, 3248], [3249, 3254], [3254, 3255], [3256, 3264], [3265, 3267], [3268, 3275], [3276, 3277], [3278, 3285], [3286, 3294], [3295, 3300], [3300, 3301], [3302, 3310], [3311, 3313], [3314, 3321], [3322, 3323], [3324, 3331], [3332, 3340], [3341, 3346], [3346, 3347], [3348, 3356], [3357, 3359], [3360, 3367], [3368, 3369], [3370, 3377], [3378, 3386], [3387, 3392], [3392, 3393], [3394, 3402], [3403, 3405], [3406, 3413], [3414, 3415], [3416, 3423], [3424, 3432], [3433, 3438], [3438, 3439], [3440, 3448], [3449, 3451], [3452, 3459], [3460, 3461], [3462, 3469], [3470, 3478], [3479, 3484], [3484, 3485], [3486, 3494], [3495, 3497], [3498, 3505], [3506, 3507], [3508, 3515], [3516, 3524], [3525, 3530], [3530, 3531], [3532, 3540], [3541, 3543], [3544, 3551], [3552, 3553], [3554, 3561], [3562, 3570], [3571, 3576], [3576, 3577], [3578, 3586], [3587, 3589], [3590, 3597], [3598, 3599], [3600, 3607], [3608, 3616], [3617, 3622], [3622, 3623], [3624, 3632], [3633, 3635], [3636, 3643], [3644, 3645], [3646, 3653], [3654, 3662], [3663, 3668], [3668, 3669], [3670, 3678], [3679, 3681], [3682, 3689], [3690, 3691], [3692, 3699], [3700, 3708], [3709, 3714], [3714, 3715], [3716, 3724], [3725, 3727], [3728, 3735], [3736, 3737], [3738, 3745], [3746, 3754], [3755, 3760], [3760, 3761], [3762, 3770], [3771, 3773], [3774, 3781], [3782, 3783], [3784, 3791], [3792, 3800], [3801, 3806], [3806, 3807], [3808, 3816], [3817, 3819], [3820, 3827], [3828, 3829], [3830, 3837], [3838, 3846], [3847, 3852], [3852, 3853], [3854, 3862], [3863, 3865], [3866, 3873], [3874, 3875], [3876, 3883], [3884, 3892], [3893, 3898], [3898, 3899], [3900, 3908], [3909, 3911], [3912, 3919], [3920, 3921], [3922, 3929], [3930, 3938], [3939, 3944], [3944, 3945], [3946, 3954], [3955, 3957], [3958, 3965], [3966, 3967], [3968, 3975], [3976, 3984], [3985, 3990], [3990, 3991], [3992, 4000], [4001, 4003], [4004, 4011], [4012, 4013], [4014, 4021], [4022, 4030], [4031, 4036], [4036, 4037], [4038, 4046], [4047, 4049], [4050, 4057], [4058, 4059], [4060, 4067], [4068, 4076], [4077, 4082], [4082, 4083], [4084, 4092], [4093, 4095], [4096, 4103], [4104, 4105], [4106, 4113], [4114, 4122], [4123, 4128], [4128, 4129], [4130, 4138], [4139, 4141], [4142, 4149], [4150, 4151], [4152, 4159], [4160, 4168], [4169, 4174], [4174, 4175], [4176, 4184], [4185, 4187], [4188, 4195], [4196, 4197], [4198, 4205], [4206, 4214], [4215, 4220], [4220, 4221], [4222, 4230], [4231, 4233], [4234, 4241], [4242, 4243], [4244, 4251], [4252, 4260], [4261, 4266], [4266, 4267], [4268, 4276], [4277, 4279], [4280, 4287], [4288, 4289], [4290, 4297], [4298, 4306], [4307, 4312], [4312, 4313], [4314, 4322], [4323, 4325], [4326, 4333], [4334, 4335], [4336, 4343], [4344, 4352], [4353, 4358], [4358, 4359], [4360, 4368], [4369, 4371], [4372, 4379], [4380, 4381], [4382, 4389], [4390, 4398], [4399, 4404], [4404, 4405], [4406, 4414], [4415, 4417], [4418, 4425], [4426, 4427], [4428, 4435], [4436, 4444], [4445, 4450], [4450, 4451], [4452, 4460], [4461, 4463], [4464, 4471], [4472, 4473], [4474, 4481], [4482, 4490], [4491, 4496], [4496, 4497], [4498, 4506], [4507, 4509], [4510, 4517], [4518, 4519], [4520, 4527], [4528, 4536], [4537, 4542], [4542, 4543], [4544, 4552], [4553, 4555], [4556, 4563], [4564, 4565], [4566, 4573], [4574, 4582], [4583, 4588], [4588, 4589], [4590, 4598], [4599, 4602], [4603, 4610], [4611, 4612], [4613, 4620], [4621, 4629], [4630, 4635], [4635, 4636], [4637, 4645], [4646, 4649], [4650, 4657], [4658, 4659], [4660, 4667], [4668, 4676], [4677, 4682], [4682, 4683], [4684, 4692], [4693, 4696], [4697, 4704], [4705, 4706], [4707, 4714], [4715, 4723], [4724, 4729], [4729, 4730], [4731, 4739], [4740, 4743], [4744, 4751], [4752, 4753], [4754, 4761], [4762, 4770], [4771, 4776], [4776, 4777], [4778, 4786], [4787, 4790], [4791, 4798], [4799, 4800], [4801, 4808], [4809, 4817], [4818, 4823], [4823, 4824], [4825, 4833], [4834, 4837], [4838, 4845], [4846, 4847], [4848, 4855], [4856, 4864], [4865, 4870], [4870, 4871], [4872, 4880], [4881, 4884], [4885, 4892], [4893, 4894], [4895, 4902], [4903, 4911], [4912, 4917], [4917, 4918], [4919, 4927], [4928, 4931], [4932, 4939], [4940, 4941], [4942, 4949], [4950, 4958], [4959, 4964], [4964, 4965], [4966, 4974], [4975, 4978], [4979, 4986], [4987, 4988], [4989, 4996], [4997, 5005], [5006, 5011], [5011, 5012], [5013, 5021], [5022, 5025], [5026, 5033], [5034, 5035], [5036, 5043], [5044, 5052], [5053, 5058], [5058, 5059], [5060, 5068], [5069, 5072], [5073, 5080], [5081, 5082], [5083, 5090], [5091, 5099], [5100, 5105], [5105, 5106], [5107, 5115], [5116, 5119], [5120, 5127], [5128, 5129], [5130, 5137], [5138, 5146], [5147, 5152], [5152, 5153], [5154, 5162], [5163, 5166], [5167, 5174], [5175, 5176], [5177, 5184], [5185, 5193], [5194, 5199], [5199, 5200], [5201, 5209], [5210, 5213], [5214, 5221], [5222, 5223], [5224, 5231], [5232, 5240], [5241, 5246], [5246, 5247], [5248, 5256], [5257, 5260], [5261, 5268], [5269, 5270], [5271, 5278], [5279, 5287], [5288, 5293], [5293, 5294], [5295, 5303], [5304, 5307], [5308, 5315], [5316, 5317], [5318, 5325], [5326, 5334], [5335, 5340], [5340, 5341], [5342, 5350], [5351, 5354], [5355, 5362], [5363, 5364], [5365, 5372], [5373, 5381], [5382, 5387], [5387, 5388], [5389, 5397], [5398, 5401], [5402, 5409], [5410, 5411], [5412, 5419], [5420, 5428], [5429, 5434], [5434, 5435], [5436, 5444], [5445, 5448], [5449, 5456], [5457, 5458], [5459, 5466], [5467, 5475], [5476, 5481], [5481, 5482], [5483, 5491], [5492, 5495], [5496, 5503], [5504, 5505], [5506, 5513], [5514, 5522], [5523, 5528], [5528, 5529], [5530, 5538], [5539, 5542], [5543, 5550], [5551, 5552], [5553, 5560], [5561, 5569], [5570, 5575], [5575, 5576], [5577, 5585], [5586, 5589], [5590, 5597], [5598, 5599], [5600, 5607], [5608, 5616], [5617, 5622], [5622, 5623], [5624, 5632], [5633, 5636], [5637, 5644], [5645, 5646], [5647, 5654], [5655, 5663], [5664, 5669], [5669, 5670], [5671, 5679], [5680, 5683], [5684, 5691], [5692, 5693], [5694, 5701], [5702, 5710], [5711, 5716], [5716, 5717], [5718, 5726], [5727, 5730], [5731, 5738], [5739, 5740], [5741, 5748], [5749, 5757], [5758, 5763], [5763, 5764], [5765, 5773], [5774, 5777], [5778, 5785], [5786, 5787], [5788, 5795], [5796, 5804], [5805, 5810], [5810, 5811], [5812, 5820], [5821, 5824], [5825, 5832], [5833, 5834], [5835, 5842], [5843, 5851], [5852, 5857], [5857, 5858], [5859, 5867], [5868, 5871], [5872, 5879], [5880, 5881], [5882, 5889], [5890, 5898], [5899, 5904], [5904, 5905], [5906, 5914], [5915, 5918], [5919, 5926], [5927, 5928], [5929, 5936], [5937, 5945], [5946, 5951], [5951, 5952], [5953, 5961], [5962, 5965], [5966, 5973], [5974, 5975], [5976, 5983], [5984, 5992], [5993, 5998], [5998, 5999], [6000, 6008], [6009, 6012], [6013, 6020], [6021, 6022], [6023, 6030], [6031, 6039], [6040, 6045], [6045, 6046], [6047, 6055], [6056, 6059], [6060, 6067], [6068, 6069], [6070, 6077], [6078, 6086], [6087, 6092], [6092, 6093], [6094, 6102], [6103, 6106], [6107, 6114], [6115, 6116], [6117, 6124], [6125, 6133], [6134, 6139], [6139, 6140], [6141, 6149], [6150, 6153], [6154, 6161], [6162, 6163], [6164, 6171], [6172, 6180], [6181, 6186], [6186, 6187], [6188, 6196], [6197, 6200], [6201, 6208], [6209, 6210], [6211, 6218], [6219, 6227], [6228, 6233], [6233, 6234], [6235, 6243], [6244, 6247], [6248, 6255], [6256, 6257], [6258, 6265], [6266, 6274], [6275, 6280], [6280, 6281], [6282, 6290], [6291, 6294], [6295, 6302], [6303, 6304], [6305, 6312], [6313, 6321], [6322, 6327], [6327, 6328], [6329, 6337], [6338, 6341], [6342, 6349], [6350, 6351], [6352, 6359], [6360, 6368], [6369, 6374], [6374, 6375], [6376, 6384], [6385, 6388], [6389, 6396], [6397, 6398], [6399, 6406], [6407, 6415], [6416, 6421], [6421, 6422], [6423, 6431], [6432, 6435], [6436, 6443], [6444, 6445], [6446, 6453], [6454, 6462], [6463, 6468], [6468, 6469], [6470, 6478], [6479, 6482], [6483, 6490], [6491, 6492], [6493, 6500], [6501, 6509], [6510, 6515], [6515, 6516], [6517, 6525], [6526, 6529], [6530, 6537], [6538, 6539], [6540, 6547], [6548, 6556], [6557, 6562], [6562, 6563], [6564, 6572], [6573, 6576], [6577, 6584], [6585, 6586], [6587, 6594], [6595, 6603], [6604, 6609], [6609, 6610], [6611, 6619], [6620, 6623], [6624, 6631], [6632, 6633], [6634, 6641], [6642, 6650], [6651, 6656], [6656, 6657], [6658, 6666], [6667, 6670], [6671, 6678], [6679, 6680], [6681, 6688], [6689, 6697], [6698, 6703], [6703, 6704], [6705, 6713], [6714, 6717], [6718, 6725], [6726, 6727], [6728, 6735], [6736, 6744], [6745, 6750], [6750, 6751], [6752, 6760], [6761, 6764], [6765, 6772], [6773, 6774], [6775, 6782], [6783, 6791], [6792, 6797], [6797, 6798], [6799, 6807], [6808, 6811], [6812, 6819], [6820, 6821], [6822, 6829], [6830, 6838], [6839, 6844], [6844, 6845], [6846, 6854], [6855, 6858], [6859, 6866], [6867, 6868], [6869, 6876], [6877, 6885], [6886, 6891], [6891, 6892], [6893, 6901], [6902, 6905], [6906, 6913], [6914, 6915], [6916, 6923], [6924, 6932], [6933, 6938], [6938, 6939], [6940, 6948], [6949, 6952], [6953, 6960], [6961, 6962], [6963, 6970], [6971, 6979], [6980, 6985], [6985, 6986], [6987, 6995], [6996, 6999], [7000, 7007], [7008, 7009], [7010, 7017], [7018, 7026], [7027, 7032], [7032, 7033], [7034, 7042], [7043, 7046], [7047, 7054], [7055, 7056], [7057, 7064], [7065, 7073], [7074, 7079], [7079, 7080], [7081, 7089], [7090, 7093], [7094, 7101], [7102, 7103], [7104, 7111], [7112, 7120], [7121, 7126], [7126, 7127], [7128, 7136], [7137, 7140], [7141, 7148], [7149, 7150], [7151, 7158], [7159, 7167], [7168, 7173], [7173, 7174], [7175, 7183], [7184, 7187], [7188, 7195], [7196, 7197], [7198, 7205], [7206, 7214], [7215, 7220], [7220, 7221], [7222, 7230], [7231, 7234], [7235, 7242], [7243, 7244], [7245, 7252], [7253, 7261], [7262, 7267], [7267, 7268], [7269, 7277], [7278, 7281], [7282, 7289], [7290, 7291], [7292, 7299], [7300, 7308], [7309, 7314], [7314, 7315], [7316, 7324], [7325, 7328], [7329, 7336], [7337, 7338], [7339, 7346], [7347, 7355], [7356, 7361], [7361, 7362], [7363, 7371], [7372, 7375], [7376, 7383], [7384, 7385], [7386, 7393], [7394, 7402], [7403, 7408], [7408, 7409], [7410, 7418], [7419, 7422], [7423, 7430], [7431, 7432], [7433, 7440], [7441, 7449], [7450, 7455], [7455, 7456], [7457, 7465], [7466, 7469], [7470, 7477], [7478, 7479], [7480, 7487], [7488, 7496], [7497, 7502], [7502, 7503], [7504, 7512], [7513, 7516], [7517, 7524], [7525, 7526], [7527, 7534], [7535, 7543], [7544, 7549], [7549, 7550], [7551, 7559], [7560, 7563], [7564, 7571], [7572, 7573], [7574, 7581], [7582, 7590], [7591, 7596], [7596, 7597], [7598, 7606], [7607, 7610], [7611, 7618], [7619, 7620], [7621, 7628], [7629, 7637], [7638, 7643], [7643, 7644], [7645, 7653], [7654, 7657], [7658, 7665], [7666, 7667], [7668, 7675], [7676, 7684], [7685, 7690], [7690, 7691], [7692, 7700], [7701, 7704], [7705, 7712], [7713, 7714], [7715, 7722], [7723, 7731], [7732, 7737], [7737, 7738], [7739, 7747], [7748, 7751], [7752, 7759], [7760, 7761], [7762, 7769], [7770, 7778], [7779, 7784], [7784, 7785], [7786, 7794], [7795, 7798], [7799, 7806], [7807, 7808], [7809, 7816], [7817, 7825], [7826, 7831], [7831, 7832], [7833, 7841], [7842, 7845], [7846, 7853], [7854, 7855], [7856, 7863], [7864, 7872], [7873, 7878], [7878, 7879], [7880, 7888], [7889, 7892], [7893, 7900], [7901, 7902], [7903, 7910], [7911, 7919], [7920, 7925], [7925, 7926], [7927, 7935], [7936, 7939], [7940, 7947], [7948, 7949], [7950, 7957], [7958, 7966], [7967, 7972], [7972, 7973], [7974, 7982], [7983, 7986], [7987, 7994], [7995, 7996], [7997, 8004], [8005, 8013], [8014, 8019], [8019, 8020], [8021, 8029], [8030, 8033], [8034, 8041], [8042, 8043], [8044, 8051], [8052, 8060], [8061, 8066], [8066, 8067], [8068, 8076], [8077, 8080], [8081, 8088], [8089, 8090], [8091, 8098], [8099, 8107], [8108, 8113], [8113, 8114], [8115, 8123], [8124, 8127], [8128, 8135], [8136, 8137], [8138, 8145], [8146, 8154], [8155, 8160], [8160, 8161], [8162, 8170], [8171, 8174], [8175, 8182], [8183, 8184], [8185, 8192], [8193, 8201], [8202, 8207], [8207, 8208], [8209, 8217], [8218, 8221], [8222, 8229], [8230, 8231], [8232, 8239], [8240, 8248], [8249, 8254], [8254, 8255], [8256, 8264], [8265, 8268], [8269, 8276], [8277, 8278], [8279, 8286], [8287, 8295], [8296, 8301], [8301, 8302], [8303, 8311], [8312, 8315], [8316, 8323], [8324, 8325], [8326, 8333], [8334, 8342], [8343, 8348], [8348, 8349], [8350, 8358], [8359, 8362], [8363, 8370], [8371, 8372], [8373, 8380], [8381, 8389], [8390, 8395], [8395, 8396], [8397, 8405], [8406, 8409], [8410, 8417], [8418, 8419], [8420, 8427], [8428, 8436], [8437, 8442], [8442, 8443], [8444, 8452], [8453, 8456], [8457, 8464], [8465, 8466], [8467, 8474], [8475, 8483], [8484, 8489], [8489, 8490], [8491, 8499], [8500, 8503], [8504, 8511], [8512, 8513], [8514, 8521], [8522, 8530], [8531, 8536], [8536, 8537], [8538, 8546], [8547, 8550], [8551, 8558], [8559, 8560], [8561, 8568], [8569, 8577], [8578, 8583], [8583, 8584], [8585, 8593], [8594, 8597], [8598, 8605], [8606, 8607], [8608, 8615], [8616, 8624], [8625, 8630], [8630, 8631], [8632, 8640], [8641, 8644], [8645, 8652], [8653, 8654], [8655, 8662], [8663, 8671], [8672, 8677], [8677, 8678], [8679, 8687], [8688, 8691], [8692, 8699], [8700, 8701], [8702, 8709], [8710, 8718], [8719, 8724], [8724, 8725], [8726, 8734], [8735, 8738], [8739, 8746], [8747, 8748], [8749, 8756], [8757, 8765], [8766, 8771], [8771, 8772], [8773, 8781], [8782, 8785], [8786, 8793], [8794, 8795], [8796, 8803], [8804, 8812], [8813, 8818], [8818, 8819], [8820, 8828], [8829, 8832], [8833, 8840], [8841, 8842], [8843, 8850], [8851, 8859], [8860, 8865], [8865, 8866], [8867, 8875], [8876, 8879], [8880, 8887], [8888, 8889], [8890, 8897], [8898, 8906], [8907, 8912], [8912, 8913], [8914, 8922], [8923, 8926], [8927, 8934], [8935, 8936], [8937, 8944], [8945, 8953], [8954, 8959], [8959, 8960], [8961, 8969], [8970, 8973], [8974, 8981], [8982, 8983], [8984, 8991], [8992, 9000], [9001, 9006], [9006, 9007], [9008, 9016], [9017, 9020], [9021, 9028], [9029, 9030], [9031, 9038], [9039, 9047], [9048, 9053], [9053, 9054], [9055, 9063], [9064, 9067], [9068, 9075], [9076, 9077], [9078, 9085], [9086, 9094], [9095, 9100], [9100, 9101], [9102, 9110], [9111, 9114], [9115, 9122], [9123, 9124], [9125, 9132], [9133, 9141], [9142, 9147], [9147, 9148], [9149, 9157], [9158, 9161], [9162, 9169], [9170, 9171], [9172, 9179], [9180, 9188], [9189, 9194], [9194, 9195], [9196, 9204], [9205, 9208], [9209, 9216], [9217, 9218], [9219, 9226], [9227, 9235], [9236, 9241], [9241, 9242], [9243, 9251], [9252, 9255], [9256, 9263], [9264, 9265], [9266, 9273], [9274, 9282], [9283, 9288], [9288, 9289], [9290, 9298], [9299, 9302], [9303, 9310], [9311, 9312], [9313, 9320], [9321, 9329], [9330, 9335], [9335, 9336], [9337, 9345], [9346, 9349], [9350, 9357], [9358, 9359], [9360, 9367], [9368, 9376], [9377, 9382], [9382, 9383], [9384, 9392], [9393, 9396], [9397, 9404], [9405, 9406], [9407, 9414], [9415, 9423], [9424, 9429], [9429, 9430], [9431, 9439], [9440, 9443], [9444, 9451], [9452, 9453], [9454, 9461], [9462, 9470], [9471, 9476], [9476, 9477], [9478, 9486], [9487, 9490], [9491, 9498], [9499, 9500], [9501, 9508], [9509, 9517], [9518, 9523], [9523, 9524], [9525, 9533], [9534, 9537], [9538, 9545], [9546, 9547], [9548, 9555], [9556, 9564], [9565, 9570], [9570, 9571], [9572, 9580], [9581, 9584], [9585, 9592], [9593, 9594], [9595, 9602], [9603, 9611], [9612, 9617], [9617, 9618], [9619, 9627], [9628, 9631], [9632, 9639], [9640, 9641], [9642, 9649], [9650, 9658], [9659, 9664], [9664, 9665], [9666, 9674], [9675, 9678], [9679, 9686], [9687, 9688], [9689, 9696], [9697, 9705], [9706, 9711], [9711, 9712], [9713, 9721], [9722, 9725], [9726, 9733], [9734, 9735], [9736, 9743], [9744, 9752], [9753, 9758], [9758, 9759], [9760, 9768], [9769, 9772], [9773, 9780], [9781, 9782], [9783, 9790], [9791, 9799], [9800, 9805], [9805, 9806], [9807, 9815], [9816, 9819], [9820, 9827], [9828, 9829], [9830, 9837], [9838, 9846], [9847, 9852], [9852, 9853], [9854, 9862], [9863, 9866], [9867, 9874], [9875, 9876], [9877, 9884], [9885, 9893], [9894, 9899], [9899, 9900], [9901, 9909], [9910, 9913], [9914, 9921], [9922, 9923], [9924, 9931], [9932, 9940], [9941, 9946], [9946, 9947], [9948, 9956], [9957, 9960], [9961, 9968], [9969, 9970], [9971, 9978], [9979, 9987], [9988, 9993], [9993, 9994], [9995, 10003], [10004, 10007], [10008, 10015], [10016, 10017], [10018, 10025], [10026, 10034], [10035, 10040], [10040, 10041], [10042, 10050], [10051, 10054], [10055, 10062], [10063, 10064], [10065, 10072], [10073, 10081], [10082, 10087], [10087, 10088], [10089, 10097], [10098, 10101], [10102, 10109], [10110, 10111], [10112, 10119], [10120, 10128], [10129, 10134], [10134, 10135], [10136, 10144], [10145, 10148], [10149, 10156], [10157, 10158], [10159, 10166], [10167, 10175], [10176, 10181], [10181, 10182], [10183, 10191], [10192, 10195], [10196, 10203], [10204, 10205], [10206, 10213], [10214, 10222], [10223, 10228], [10228, 10229]], "w_pos": [10, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 23, 24, 25, 26, 27, 28, 30, 31, 32, 33, 34, 35, 36, 37, 39, 40, 41, 42, 43, 44, 45, 46, 48, 49, 50, 51, 52, 53, 54, 55, 57, 58, 59, 60, 61, 62, 63, 64, 66, 67, 68, 69, 70, 71, 72, 73, 75, 76, 77, 78, 79, 80, 81, 82, 84, 85, 86, 87, 88, 89, 90, 91, 93, 94, 95, 96, 97, 98, 99, 100, 102, 104, 105, 106, 107, 108, 109, 110, 112, 114, 115, 116, 117, 118, 119, 120, 122, 124, 125, 126, 127, 128, 129, 130, 132, 134, 135, 136, 137, 138, 139, 140, 142, 144, 145, 146, 147, 148, 149, 150, 152, 154, 155, 156, 157, 158, 159, 160, 162, 164, 165, 166, 167, 168, 169, 170, 172, 174, 175, 176, 177, 178, 179, 180, 182, 184, 185, 186, 187, 188, 189, 190, 192, 194, 195, 196, 197, 198, 199, 200, 202, 204, 205, 206, 207, 208, 209, 210, 212, 214, 215, 216, 217, 218, 219, 220, 222, 224, 225, 226, 227, 228, 229, 230, 232, 234, 235, 236, 237, 238, 239, 240, 242, 244, 245, 246, 247, 248, 249, 250, 252, 254, 255, 256, 257, 258, 259, 260, 262, 264, 265, 266, 267, 268, 269, 270, 272, 274, 275, 276, 277, 278, 279, 280, 282, 284, 285, 286, 287, 288, 289, 290, 292, 294, 295, 296, 297, 298, 299, 300, 302, 304, 305, 306, 307, 308, 309, 310, 312, 314, 315, 316, 317, 318, 319, 320, 322, 324, 325, 326, 327, 328, 329, 330, 332, 334, 335, 336, 337, 338, 339, 340, 342, 344, 345, 346, 347, 348, 349, 350, 352, 354, 355, 356, 357, 358, 359, 360, 362, 364, 365, 366, 367, 368, 369, 370, 372, 374, 375, 376, 377, 378, 379, 380, 382, 384, 385, 386, 387, 388, 389, 390, 392, 394, 395, 396, 397, 398, 399, 400, 402, 404, 405, 406, 407, 408, 409, 410, 412, 414, 415, 416, 417, 418, 419, 420, 422, 424, 425, 426, 427, 428, 429, 430, 432, 434, 435, 436, 437, 438, 439, 440, 442, 444, 445, 446, 447, 448, 449, 450, 452, 454, 455, 456, 457, 458, 459, 460, 462, 464, 465, 466, 467, 468, 469, 470, 472, 474, 475, 476, 477, 478, 479, 480, 482, 484, 485, 486, 487, 488, 489, 490, 492, 494, 495, 496, 497, 498, 499, 500, 502, 504, 505, 506, 507, 508, 509, 510, 512, 514, 515, 516, 517, 518, 519, 520, 522, 524, 525, 526, 527, 528, 529, 530, 532, 534, 535, 536, 537, 538, 539, 540, 542, 544, 545, 546, 547, 548, 549, 550, 552, 554, 555, 556, 557, 558, 559, 560, 562, 564, 565, 566, 567, 568, 569, 570, 572, 574, 575, 576, 577, 578, 579, 580, 582, 584, 585, 586, 587, 588, 589, 590, 592, 594, 595, 596, 597, 598, 599, 600, 602, 604, 605, 606, 607, 608, 609, 610, 612, 614, 615, 616, 617, 618, 619, 620, 622, 624, 625, 626, 627, 628, 629, 630, 632, 634, 635, 636, 637, 638, 639, 640, 642, 644, 645, 646, 647, 648, 649, 650, 652, 654, 655, 656, 657, 658, 659, 660, 662, 664, 665, 666, 667, 668, 669, 670, 672, 674, 675, 676, 677, 678, 679, 680, 682, 684, 685, 686, 687, 688, 689, 690, 692, 694, 695, 696, 697, 698, 699, 700, 702, 704, 705, 706, 707, 708, 709, 710, 712, 714, 715, 716, 717, 718, 719, 720, 722, 724, 725, 726, 727, 728, 729, 730, 732, 734, 735, 736, 737, 738, 739, 740, 742, 744, 745, 746, 747, 748, 749, 750, 752, 754, 755, 756, 757, 758, 759, 760, 762, 764, 765, 766, 767, 768, 769, 770, 772, 774, 775, 776, 777, 778, 779, 780, 782, 784, 785, 786, 787, 788, 789, 790, 792, 794, 795, 796, 797, 798, 799, 800, 802, 804, 805, 806, 807, 808, 809, 810, 812, 814, 815, 816, 817, 818, 819, 820, 822, 824, 825, 826, 827, 828, 829, 830, 832, 834, 835, 836, 837, 838, 839, 840, 842, 844, 845, 846, 847, 848, 849, 850, 852, 854, 855, 856, 857, 858, 859, 860, 862, 864, 865, 866, 867, 868, 869, 870, 872, 874, 875, 876, 877, 878, 879, 880, 882, 884, 885, 886, 887, 888, 889, 890, 892, 894, 895, 896, 897, 898, 899, 900, 902, 904, 905, 906, 907, 908, 909, 910, 912, 914, 915, 916, 917, 918, 919, 920, 922, 924, 925, 926, 927, 928, 929, 930, 932, 934, 935, 936, 937, 938, 939, 940, 942, 944, 945, 946, 947, 948, 949, 950, 952, 954, 955, 956, 957, 958, 959, 960, 962, 964, 965, 966, 967, 968, 969, 970, 972, 974, 975, 976, 977, 978, 979, 980, 982, 984, 985, 986, 987, 988, 989, 990, 992, 994, 995, 996, 997, 998, 999, 1000, 1002, 1005, 1006, 1007, 1008, 1009, 1010, 1011, 1013, 1016, 1017, 1018, 1019, 1020, 1021, 1022, 1024, 1027, 1028, 1029, 1030, 1031, 1032, 1033, 1035, 1038, 1039, 1040, 1041, 1042, 1043, 1044, 1046, 1049, 1050, 1051, 1052, 1053, 1054, 1055, 1057, 1060, 1061, 1062, 1063, 1064, 1065, 1066, 1068, 1071, 1072, 1073, 1074, 1075, 1076, 1077, 1079, 1082, 1083, 1084, 1085, 1086, 1087, 1088, 1090, 1093, 1094, 1095, 1096, 1097, 1098, 1099, 1101, 1104, 1105, 1106, 1107, 1108, 1109, 1110, 1112, 1115, 1116, 1117, 1118, 1119, 1120, 1121, 1123, 1126, 1127, 1128, 1129, 1130, 1131, 1132, 1134, 1137, 1138, 1139, 1140, 1141, 1142, 1143, 1145, 1148, 1149, 1150, 1151, 1152, 1153, 1154, 1156, 1159, 1160, 1161, 1162, 1163, 1164, 1165, 1167, 1170, 1171, 1172, 1173, 1174, 1175, 1176, 1178, 1181, 1182, 1183, 1184, 1185, 1186, 1187, 1189, 1192, 1193, 1194, 1195, 1196, 1197, 1198, 1200, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1211, 1214, 1215, 1216, 1217, 1218, 1219, 1220, 1222, 1225, 1226, 1227, 1228, 1229, 1230, 1231, 1233, 1236, 1237, 1238, 1239, 1240, 1241, 1242, 1244, 1247, 1248, 1249, 1250, 1251, 1252, 1253, 1255, 1258, 1259, 1260, 1261, 1262, 1263, 1264, 1266, 1269, 1270, 1271, 1272, 1273, 1274, 1275, 1277, 1280, 1281, 1282, 1283, 1284, 1285, 1286, 1288, 1291, 1292, 1293, 1294, 1295, 1296, 1297, 1299, 1302, 1303, 1304, 1305, 1306, 1307, 1308, 1310, 1313, 1314, 1315, 1316, 1317, 1318, 1319, 1321, 1324, 1325, 1326, 1327, 1328, 1329, 1330, 1332, 1335, 1336, 1337, 1338, 1339, 1340, 1341, 1343, 1346, 1347, 1348, 1349, 1350, 1351, 1352, 1354, 1357, 1358, 1359, 1360, 1361, 1362, 1363, 1365, 1368, 1369, 1370, 1371, 1372, 1373, 1374, 1376, 1379, 1380, 1381, 1382, 1383, 1384, 1385, 1387, 1390, 1391, 1392, 1393, 1394, 1395, 1396, 1398, 1401, 1402, 1403, 1404, 1405, 1406, 1407, 1409, 1412, 1413, 1414, 1415, 1416, 1417, 1418, 1420, 1423, 1424, 1425, 1426, 1427, 1428, 1429, 1431, 1434, 1435, 1436, 1437, 1438, 1439, 1440, 1442, 1445, 1446, 1447, 1448, 1449, 1450, 1451, 1453, 1456, 1457, 1458, 1459, 1460, 1461, 1462, 1464, 1467, 1468, 1469, 1470, 1471, 1472, 1473, 1475, 1478, 1479, 1480, 1481, 1482, 1483, 1484, 1486, 1489, 1490, 1491, 1492, 1493, 1494, 1495, 1497, 1500, 1501, 1502, 1503, 1504, 1505, 1506, 1508, 1511, 1512, 1513, 1514, 1515, 1516, 1517, 1519, 1522, 1523, 1524, 1525, 1526, 1527, 1528, 1530, 1533, 1534, 1535, 1536, 1537, 1538, 1539, 1541, 1544, 1545, 1546, 1547, 1548, 1549, 1550, 1552, 1555, 1556, 1557, 1558, 1559, 1560, 1561, 1563, 1566, 1567, 1568, 1569, 1570, 1571, 1572, 1574, 1577, 1578, 1579, 1580, 1581, 1582, 1583, 1585, 1588, 1589, 1590, 1591, 1592, 1593, 1594, 1596, 1599, 1600, 1601, 1602, 1603, 1604, 1605, 1607, 1610, 1611, 1612, 1613, 1614, 1615, 1616, 1618, 1621, 1622, 1623, 1624, 1625, 1626, 1627, 1629, 1632, 1633, 1634, 1635, 1636, 1637, 1638, 1640, 1643, 1644, 1645, 1646, 1647, 1648, 1649, 1651, 1654, 1655, 1656, 1657, 1658, 1659, 1660, 1662, 1665, 1666, 1667, 1668, 1669, 1670, 1671, 1673, 1676, 1677, 1678, 1679, 1680, 1681, 1682, 1684, 1687, 1688, 1689, 1690, 1691, 1692, 1693, 1695, 1698, 1699, 1700, 1701, 1702, 1703, 1704, 1706, 1709, 1710, 1711, 1712, 1713, 1714, 1715, 1717, 1720, 1721, 1722, 1723, 1724, 1725, 1726, 1728, 1731, 1732, 1733, 1734, 1735, 1736, 1737, 1739, 1742, 1743, 1744, 1745, 1746, 1747, 1748, 1750, 1753, 1754, 1755, 1756, 1757, 1758, 1759, 1761, 1764, 1765, 1766, 1767, 1768, 1769, 1770, 1772, 1775, 1776, 1777, 1778, 1779, 1780, 1781, 1783, 1786, 1787, 1788, 1789, 1790, 1791, 1792, 1794, 1797, 1798, 1799, 1800, 1801, 1802, 1803, 1805, 1808, 1809, 1810, 1811, 1812, 1813, 1814, 1816, 1819, 1820, 1821, 1822, 1823, 1824, 1825, 1827, 1830, 1831, 1832, 1833, 1834, 1835, 1836, 1838, 1841, 1842, 1843, 1844, 1845, 1846, 1847, 1849, 1852, 1853, 1854, 1855, 1856, 1857, 1858, 1860, 1863, 1864, 1865, 1866, 1867, 1868, 1869, 1871, 1874, 1875, 1876, 1877, 1878, 1879, 1880, 1882, 1885, 1886, 1887, 1888, 1889, 1890, 1891, 1893, 1896, 1897, 1898, 1899, 1900, 1901, 1902, 1904, 1907, 1908, 1909, 1910, 1911, 1912, 1913, 1915, 1918, 1919, 1920, 1921, 1922, 1923, 1924, 1926, 1929, 1930, 1931, 1932, 1933, 1934, 1935, 1937, 1940, 1941, 1942, 1943, 1944, 1945, 1946, 1948, 1951, 1952, 1953, 1954, 1955, 1956, 1957, 1959, 1962, 1963, 1964, 1965, 1966, 1967, 1968, 1970, 1973, 1974, 1975, 1976, 1977, 1978, 1979, 1981, 1984, 1985, 1986, 1987, 1988, 1989, 1990, 1992, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2003, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2014, 2017, 2018, 2019, 2020, 2021, 2022, 2023, 2025, 2028, 2029, 2030, 2031, 2032, 2033, 2034, 2036, 2039, 2040, 2041, 2042, 2043, 2044, 2045, 2047, 2050, 2051, 2052, 2053, 2054, 2055, 2056, 2058, 2061, 2062, 2063, 2064, 2065, 2066, 2067, 2069, 2072, 2073, 2074, 2075, 2076, 2077, 2078, 2080, 2083, 2084, 2085, 2086, 2087, 2088, 2089, 2091, 2094, 2095, 2096, 2097, 2098, 2099, 2100, 2102, 2105, 2106, 2107, 2108, 2109, 2110, 2111, 2113, 2116, 2117, 2118, 2119, 2120, 2121, 2122, 2124, 2127, 2128, 2129, 2130, 2131, 2132, 2133, 2135, 2138, 2139, 2140, 2141, 2142, 2143, 2144, 2146, 2149, 2150, 2151, 2152, 2153, 2154, 2155, 2157, 2160, 2161, 2162, 2163, 2164, 2165, 2166, 2168, 2171, 2172, 2173, 2174, 2175, 2176, 2177, 2179, 2182, 2183, 2184, 2185, 2186, 2187, 2188, 2190, 2193, 2194, 2195, 2196, 2197, 2198, 2199, 2201, 2204, 2205, 2206, 2207, 2208, 2209, 2210, 2212, 2215, 2216, 2217, 2218, 2219, 2220, 2221, 2223, 2226, 2227, 2228, 2229, 2230, 2231, 2232, 2234, 2237, 2238, 2239, 2240, 2241, 2242, 2243, 2245, 2248, 2249, 2250, 2251, 2252, 2253, 2254, 2256, 2259, 2260, 2261, 2262, 2263, 2264, 2265, 2267, 2270, 2271, 2272, 2273, 2274, 2275, 2276, 2278, 2281, 2282, 2283, 2284, 2285, 2286, 2287, 2289, 2292, 2293, 2294, 2295, 2296, 2297, 2298, 2300, 2303, 2304, 2305, 2306, 2307, 2308, 2309, 2311, 2314, 2315, 2316, 2317, 2318, 2319], "span_over": "user", "n_tokens": 2385, "head": "broad", "head_reason": "default"}], "window_rows": [{"p_ids": [1905, 25, 198, 27, 22446, 3364, 420, 846, 731, 198, 81646, 220, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 18, 6473, 198, 510, 22446, 29, 198], "blocks": [{"kind": "span", "ends": [40], "label_ranges": [[22, 40]], "words_in_block": true, "ids": [271, 6065, 913, 25, 9111, 198, 6196, 25, 1156, 198, 14162, 25, 198, 22365, 42824, 6088, 264, 9403, 30, 198, 22933, 25, 198, 14057, 29, 198, 4754, 4532, 3147, 64, 9403, 1484, 2129, 767, 3147, 31320, 8934, 198, 510, 1448, 29, 271, 1138, 25, 198, 27, 22446, 3364, 420, 846, 731, 198, 81646, 220, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 18, 6473, 198, 510, 22446, 29, 198, 271, 8679, 1396, 9111, 314, 279, 2100, 1414, 421, 22651, 799, 314, 279, 8928, 13, 198, 6205, 581, 25], "query": 1874, "w_pos": [52, 54, 55, 56, 57, 58, 59, 60, 61, 63, 64, 65, 66, 67, 68, 69, 70, 72, 73, 74, 75, 76, 77, 78, 79, 81, 82, 83, 84, 85, 86, 87, 88, 90, 91, 92, 93, 94, 95, 96, 97, 99, 100, 101, 102, 103, 104, 105, 106, 108, 109, 110, 111, 112, 113, 114, 115, 117, 118, 119, 120, 121, 122, 123, 124, 126, 127, 128, 129, 130, 131, 132, 133, 135, 136, 137, 138, 139, 140, 141, 142, 144, 146, 147, 148, 149, 150, 151, 152, 154, 156, 157, 158, 159, 160, 161, 162, 164, 166, 167, 168, 169, 170, 171, 172, 174, 176, 177, 178, 179, 180, 181, 182, 184, 186, 187, 188, 189, 190, 191, 192, 194, 196, 197, 198, 199, 200, 201, 202, 204, 206, 207, 208, 209, 210, 211, 212, 214, 216, 217, 218, 219, 220, 221, 222, 224, 226, 227, 228, 229, 230, 231, 232, 234, 236, 237, 238, 239, 240, 241, 242, 244, 246, 247, 248, 249, 250, 251, 252, 254, 256, 257, 258, 259, 260, 261, 262, 264, 266, 267, 268, 269, 270, 271, 272, 274, 276, 277, 278, 279, 280, 281, 282, 284, 286, 287, 288, 289, 290, 291, 292, 294, 296, 297, 298, 299, 300, 301, 302, 304, 306, 307, 308, 309, 310, 311, 312, 314, 316, 317, 318, 319, 320, 321, 322, 324, 326, 327, 328, 329, 330, 331, 332, 334, 336, 337, 338, 339, 340, 341, 342, 344, 346, 347, 348, 349, 350, 351, 352, 354, 356, 357, 358, 359, 360, 361, 362, 364, 366, 367, 368, 369, 370, 371, 372, 374, 376, 377, 378, 379, 380, 381, 382, 384, 386, 387, 388, 389, 390, 391, 392, 394, 396, 397, 398, 399, 400, 401, 402, 404, 406, 407, 408, 409, 410, 411, 412, 414, 416, 417, 418, 419, 420, 421, 422, 424, 426, 427, 428, 429, 430, 431, 432, 434, 436, 437, 438, 439, 440, 441, 442, 444, 446, 447, 448, 449, 450, 451, 452, 454, 456, 457, 458, 459, 460, 461, 462, 464, 466, 467, 468, 469, 470, 471, 472, 474, 476, 477, 478, 479, 480, 481, 482, 484, 486, 487, 488, 489, 490, 491, 492, 494, 496, 497, 498, 499, 500, 501, 502, 504, 506, 507, 508, 509, 510, 511, 512, 514, 516, 517, 518, 519, 520, 521, 522, 524, 526, 527, 528, 529, 530, 531, 532, 534, 536, 537, 538, 539, 540, 541, 542, 544, 546, 547, 548, 549, 550, 551, 552, 554, 556, 557, 558, 559, 560, 561, 562, 564, 566, 567, 568, 569, 570, 571, 572, 574, 576, 577, 578, 579, 580, 581, 582, 584, 586, 587, 588, 589, 590, 591, 592, 594, 596, 597, 598, 599, 600, 601, 602, 604, 606, 607, 608, 609, 610, 611, 612, 614, 616, 617, 618, 619, 620, 621, 622, 624, 626, 627, 628, 629, 630, 631, 632, 634, 636, 637, 638, 639, 640, 641, 642, 644, 646, 647, 648, 649, 650, 651, 652, 654, 656, 657, 658, 659, 660, 661, 662, 664, 666, 667, 668, 669, 670, 671, 672, 674, 676, 677, 678, 679, 680, 681, 682, 684, 686, 687, 688, 689, 690, 691, 692, 694, 696, 697, 698, 699, 700, 701, 702, 704, 706, 707, 708, 709, 710, 711, 712, 714, 716, 717, 718, 719, 720, 721, 722, 724, 726, 727, 728, 729, 730, 731, 732, 734, 736, 737, 738, 739, 740, 741, 742, 744, 746, 747, 748, 749, 750, 751, 752, 754, 756, 757, 758, 759, 760, 761, 762, 764, 766, 767, 768, 769, 770, 771, 772, 774, 776, 777, 778, 779, 780, 781, 782, 784, 786, 787, 788, 789, 790, 791, 792, 794, 796, 797, 798, 799, 800, 801, 802, 804, 806, 807, 808, 809, 810, 811, 812, 814, 816, 817, 818, 819, 820, 821, 822, 824, 826, 827, 828, 829, 830, 831, 832, 834, 836, 837, 838, 839, 840, 841, 842, 844, 846, 847, 848, 849, 850, 851, 852, 854, 856, 857, 858, 859, 860, 861, 862, 864, 866, 867, 868, 869, 870, 871, 872, 874, 876, 877, 878, 879, 880, 881, 882, 884, 886, 887, 888, 889, 890, 891, 892, 894, 896, 897, 898, 899, 900, 901, 902, 904, 906, 907, 908, 909, 910, 911, 912, 914, 916, 917, 918, 919, 920, 921, 922, 924, 926, 927, 928, 929, 930, 931, 932, 934, 936, 937, 938, 939, 940, 941, 942, 944, 946, 947, 948, 949, 950, 951, 952, 954, 956, 957, 958, 959, 960, 961, 962, 964, 966, 967, 968, 969, 970, 971, 972, 974, 976, 977, 978, 979, 980, 981, 982, 984, 986, 987, 988, 989, 990, 991, 992, 994, 996, 997, 998, 999, 1000, 1001, 1002, 1004, 1006, 1007, 1008, 1009, 1010, 1011, 1012, 1014, 1016, 1017, 1018, 1019, 1020, 1021, 1022, 1024, 1026, 1027, 1028, 1029, 1030, 1031, 1032, 1034, 1036, 1037, 1038, 1039, 1040, 1041, 1042, 1044, 1047, 1048, 1049, 1050, 1051, 1052, 1053, 1055, 1058, 1059, 1060, 1061, 1062, 1063, 1064, 1066, 1069, 1070, 1071, 1072, 1073, 1074, 1075, 1077, 1080, 1081, 1082, 1083, 1084, 1085, 1086, 1088, 1091, 1092, 1093, 1094, 1095, 1096, 1097, 1099, 1102, 1103, 1104, 1105, 1106, 1107, 1108, 1110, 1113, 1114, 1115, 1116, 1117, 1118, 1119, 1121, 1124, 1125, 1126, 1127, 1128, 1129, 1130, 1132, 1135, 1136, 1137, 1138, 1139, 1140, 1141, 1143, 1146, 1147, 1148, 1149, 1150, 1151, 1152, 1154, 1157, 1158, 1159, 1160, 1161, 1162, 1163, 1165, 1168, 1169, 1170, 1171, 1172, 1173, 1174, 1176, 1179, 1180, 1181, 1182, 1183, 1184, 1185, 1187, 1190, 1191, 1192, 1193, 1194, 1195, 1196, 1198, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1209, 1212, 1213, 1214, 1215, 1216, 1217, 1218, 1220, 1223, 1224, 1225, 1226, 1227, 1228, 1229, 1231, 1234, 1235, 1236, 1237, 1238, 1239, 1240, 1242, 1245, 1246, 1247, 1248, 1249, 1250, 1251, 1253, 1256, 1257, 1258, 1259, 1260, 1261, 1262, 1264, 1267, 1268, 1269, 1270, 1271, 1272, 1273, 1275, 1278, 1279, 1280, 1281, 1282, 1283, 1284, 1286, 1289, 1290, 1291, 1292, 1293, 1294, 1295, 1297, 1300, 1301, 1302, 1303, 1304, 1305, 1306, 1308, 1311, 1312, 1313, 1314, 1315, 1316, 1317, 1319, 1322, 1323, 1324, 1325, 1326, 1327, 1328, 1330, 1333, 1334, 1335, 1336, 1337, 1338, 1339, 1341, 1344, 1345, 1346, 1347, 1348, 1349, 1350, 1352, 1355, 1356, 1357, 1358, 1359, 1360, 1361, 1363, 1366, 1367, 1368, 1369, 1370, 1371, 1372, 1374, 1377, 1378, 1379, 1380, 1381, 1382, 1383, 1385, 1388, 1389, 1390, 1391, 1392, 1393, 1394, 1396, 1399, 1400, 1401, 1402, 1403, 1404, 1405, 1407, 1410, 1411, 1412, 1413, 1414, 1415, 1416, 1418, 1421, 1422, 1423, 1424, 1425, 1426, 1427, 1429, 1432, 1433, 1434, 1435, 1436, 1437, 1438, 1440, 1443, 1444, 1445, 1446, 1447, 1448, 1449, 1451, 1454, 1455, 1456, 1457, 1458, 1459, 1460, 1462, 1465, 1466, 1467, 1468, 1469, 1470, 1471, 1473, 1476, 1477, 1478, 1479, 1480, 1481, 1482, 1484, 1487, 1488, 1489, 1490, 1491, 1492, 1493, 1495, 1498, 1499, 1500, 1501, 1502, 1503, 1504, 1506, 1509, 1510, 1511, 1512, 1513, 1514, 1515, 1517, 1520, 1521, 1522, 1523, 1524, 1525, 1526, 1528, 1531, 1532, 1533, 1534, 1535, 1536, 1537, 1539, 1542, 1543, 1544, 1545, 1546, 1547, 1548, 1550, 1553, 1554, 1555, 1556, 1557, 1558, 1559, 1561, 1564, 1565, 1566, 1567, 1568, 1569, 1570, 1572, 1575, 1576, 1577, 1578, 1579, 1580, 1581, 1583, 1586, 1587, 1588, 1589, 1590, 1591, 1592, 1594, 1597, 1598, 1599, 1600, 1601, 1602, 1603, 1605, 1608, 1609, 1610, 1611, 1612, 1613, 1614, 1616, 1619, 1620, 1621, 1622, 1623, 1624, 1625, 1627, 1630, 1631, 1632, 1633, 1634, 1635, 1636, 1638, 1641, 1642, 1643, 1644, 1645, 1646, 1647, 1649, 1652, 1653, 1654, 1655, 1656, 1657, 1658, 1660, 1663, 1664, 1665, 1666, 1667, 1668, 1669, 1671, 1674, 1675, 1676, 1677, 1678, 1679, 1680, 1682, 1685, 1686, 1687, 1688, 1689, 1690, 1691, 1693, 1696, 1697, 1698, 1699, 1700, 1701, 1702, 1704, 1707, 1708, 1709, 1710, 1711, 1712, 1713, 1715, 1718, 1719, 1720, 1721, 1722, 1723, 1724, 1726, 1729, 1730, 1731, 1732, 1733, 1734, 1735, 1737, 1740, 1741, 1742, 1743, 1744, 1745, 1746, 1748, 1751, 1752, 1753, 1754, 1755, 1756, 1757, 1759, 1762, 1763, 1764, 1765, 1766, 1767, 1768, 1770, 1773, 1774, 1775, 1776, 1777, 1778, 1779, 1781, 1784, 1785, 1786, 1787, 1788, 1789, 1790, 1792, 1795, 1796, 1797, 1798, 1799, 1800, 1801, 1803, 1806, 1807, 1808, 1809, 1810, 1811, 1812, 1814, 1817, 1818, 1819, 1820, 1821, 1822, 1823, 1825, 1828, 1829, 1830, 1831, 1832, 1833, 1834, 1836, 1839, 1840, 1841, 1842, 1843, 1844, 1845, 1847, 1850]}], "trunc": {"shrink_cut": 0, "protected_cut": 0}, "seg_lens": {"user": 1799}, "layout": "c", "names": ["delivery"], "w_offs": [[0, 8], [9, 10], [11, 18], [19, 20], [21, 28], [29, 37], [38, 43], [43, 44], [45, 53], [54, 55], [56, 63], [64, 65], [66, 73], [74, 82], [83, 88], [88, 89], [90, 98], [99, 100], [101, 108], [109, 110], [111, 118], [119, 127], [128, 133], [133, 134], [135, 143], [144, 145], [146, 153], [154, 155], [156, 163], [164, 172], [173, 178], [178, 179], [180, 188], [189, 190], [191, 198], [199, 200], [201, 208], [209, 217], [218, 223], [223, 224], [225, 233], [234, 235], [236, 243], [244, 245], [246, 253], [254, 262], [263, 268], [268, 269], [270, 278], [279, 280], [281, 288], [289, 290], [291, 298], [299, 307], [308, 313], [313, 314], [315, 323], [324, 325], [326, 333], [334, 335], [336, 343], [344, 352], [353, 358], [358, 359], [360, 368], [369, 370], [371, 378], [379, 380], [381, 388], [389, 397], [398, 403], [403, 404], [405, 413], [414, 415], [416, 423], [424, 425], [426, 433], [434, 442], [443, 448], [448, 449], [450, 458], [459, 461], [462, 469], [470, 471], [472, 479], [480, 488], [489, 494], [494, 495], [496, 504], [505, 507], [508, 515], [516, 517], [518, 525], [526, 534], [535, 540], [540, 541], [542, 550], [551, 553], [554, 561], [562, 563], [564, 571], [572, 580], [581, 586], [586, 587], [588, 596], [597, 599], [600, 607], [608, 609], [610, 617], [618, 626], [627, 632], [632, 633], [634, 642], [643, 645], [646, 653], [654, 655], [656, 663], [664, 672], [673, 678], [678, 679], [680, 688], [689, 691], [692, 699], [700, 701], [702, 709], [710, 718], [719, 724], [724, 725], [726, 734], [735, 737], [738, 745], [746, 747], [748, 755], [756, 764], [765, 770], [770, 771], [772, 780], [781, 783], [784, 791], [792, 793], [794, 801], [802, 810], [811, 816], [816, 817], [818, 826], [827, 829], [830, 837], [838, 839], [840, 847], [848, 856], [857, 862], [862, 863], [864, 872], [873, 875], [876, 883], [884, 885], [886, 893], [894, 902], [903, 908], [908, 909], [910, 918], [919, 921], [922, 929], [930, 931], [932, 939], [940, 948], [949, 954], [954, 955], [956, 964], [965, 967], [968, 975], [976, 977], [978, 985], [986, 994], [995, 1000], [1000, 1001], [1002, 1010], [1011, 1013], [1014, 1021], [1022, 1023], [1024, 1031], [1032, 1040], [1041, 1046], [1046, 1047], [1048, 1056], [1057, 1059], [1060, 1067], [1068, 1069], [1070, 1077], [1078, 1086], [1087, 1092], [1092, 1093], [1094, 1102], [1103, 1105], [1106, 1113], [1114, 1115], [1116, 1123], [1124, 1132], [1133, 1138], [1138, 1139], [1140, 1148], [1149, 1151], [1152, 1159], [1160, 1161], [1162, 1169], [1170, 1178], [1179, 1184], [1184, 1185], [1186, 1194], [1195, 1197], [1198, 1205], [1206, 1207], [1208, 1215], [1216, 1224], [1225, 1230], [1230, 1231], [1232, 1240], [1241, 1243], [1244, 1251], [1252, 1253], [1254, 1261], [1262, 1270], [1271, 1276], [1276, 1277], [1278, 1286], [1287, 1289], [1290, 1297], [1298, 1299], [1300, 1307], [1308, 1316], [1317, 1322], [1322, 1323], [1324, 1332], [1333, 1335], [1336, 1343], [1344, 1345], [1346, 1353], [1354, 1362], [1363, 1368], [1368, 1369], [1370, 1378], [1379, 1381], [1382, 1389], [1390, 1391], [1392, 1399], [1400, 1408], [1409, 1414], [1414, 1415], [1416, 1424], [1425, 1427], [1428, 1435], [1436, 1437], [1438, 1445], [1446, 1454], [1455, 1460], [1460, 1461], [1462, 1470], [1471, 1473], [1474, 1481], [1482, 1483], [1484, 1491], [1492, 1500], [1501, 1506], [1506, 1507], [1508, 1516], [1517, 1519], [1520, 1527], [1528, 1529], [1530, 1537], [1538, 1546], [1547, 1552], [1552, 1553], [1554, 1562], [1563, 1565], [1566, 1573], [1574, 1575], [1576, 1583], [1584, 1592], [1593, 1598], [1598, 1599], [1600, 1608], [1609, 1611], [1612, 1619], [1620, 1621], [1622, 1629], [1630, 1638], [1639, 1644], [1644, 1645], [1646, 1654], [1655, 1657], [1658, 1665], [1666, 1667], [1668, 1675], [1676, 1684], [1685, 1690], [1690, 1691], [1692, 1700], [1701, 1703], [1704, 1711], [1712, 1713], [1714, 1721], [1722, 1730], [1731, 1736], [1736, 1737], [1738, 1746], [1747, 1749], [1750, 1757], [1758, 1759], [1760, 1767], [1768, 1776], [1777, 1782], [1782, 1783], [1784, 1792], [1793, 1795], [1796, 1803], [1804, 1805], [1806, 1813], [1814, 1822], [1823, 1828], [1828, 1829], [1830, 1838], [1839, 1841], [1842, 1849], [1850, 1851], [1852, 1859], [1860, 1868], [1869, 1874], [1874, 1875], [1876, 1884], [1885, 1887], [1888, 1895], [1896, 1897], [1898, 1905], [1906, 1914], [1915, 1920], [1920, 1921], [1922, 1930], [1931, 1933], [1934, 1941], [1942, 1943], [1944, 1951], [1952, 1960], [1961, 1966], [1966, 1967], [1968, 1976], [1977, 1979], [1980, 1987], [1988, 1989], [1990, 1997], [1998, 2006], [2007, 2012], [2012, 2013], [2014, 2022], [2023, 2025], [2026, 2033], [2034, 2035], [2036, 2043], [2044, 2052], [2053, 2058], [2058, 2059], [2060, 2068], [2069, 2071], [2072, 2079], [2080, 2081], [2082, 2089], [2090, 2098], [2099, 2104], [2104, 2105], [2106, 2114], [2115, 2117], [2118, 2125], [2126, 2127], [2128, 2135], [2136, 2144], [2145, 2150], [2150, 2151], [2152, 2160], [2161, 2163], [2164, 2171], [2172, 2173], [2174, 2181], [2182, 2190], [2191, 2196], [2196, 2197], [2198, 2206], [2207, 2209], [2210, 2217], [2218, 2219], [2220, 2227], [2228, 2236], [2237, 2242], [2242, 2243], [2244, 2252], [2253, 2255], [2256, 2263], [2264, 2265], [2266, 2273], [2274, 2282], [2283, 2288], [2288, 2289], [2290, 2298], [2299, 2301], [2302, 2309], [2310, 2311], [2312, 2319], [2320, 2328], [2329, 2334], [2334, 2335], [2336, 2344], [2345, 2347], [2348, 2355], [2356, 2357], [2358, 2365], [2366, 2374], [2375, 2380], [2380, 2381], [2382, 2390], [2391, 2393], [2394, 2401], [2402, 2403], [2404, 2411], [2412, 2420], [2421, 2426], [2426, 2427], [2428, 2436], [2437, 2439], [2440, 2447], [2448, 2449], [2450, 2457], [2458, 2466], [2467, 2472], [2472, 2473], [2474, 2482], [2483, 2485], [2486, 2493], [2494, 2495], [2496, 2503], [2504, 2512], [2513, 2518], [2518, 2519], [2520, 2528], [2529, 2531], [2532, 2539], [2540, 2541], [2542, 2549], [2550, 2558], [2559, 2564], [2564, 2565], [2566, 2574], [2575, 2577], [2578, 2585], [2586, 2587], [2588, 2595], [2596, 2604], [2605, 2610], [2610, 2611], [2612, 2620], [2621, 2623], [2624, 2631], [2632, 2633], [2634, 2641], [2642, 2650], [2651, 2656], [2656, 2657], [2658, 2666], [2667, 2669], [2670, 2677], [2678, 2679], [2680, 2687], [2688, 2696], [2697, 2702], [2702, 2703], [2704, 2712], [2713, 2715], [2716, 2723], [2724, 2725], [2726, 2733], [2734, 2742], [2743, 2748], [2748, 2749], [2750, 2758], [2759, 2761], [2762, 2769], [2770, 2771], [2772, 2779], [2780, 2788], [2789, 2794], [2794, 2795], [2796, 2804], [2805, 2807], [2808, 2815], [2816, 2817], [2818, 2825], [2826, 2834], [2835, 2840], [2840, 2841], [2842, 2850], [2851, 2853], [2854, 2861], [2862, 2863], [2864, 2871], [2872, 2880], [2881, 2886], [2886, 2887], [2888, 2896], [2897, 2899], [2900, 2907], [2908, 2909], [2910, 2917], [2918, 2926], [2927, 2932], [2932, 2933], [2934, 2942], [2943, 2945], [2946, 2953], [2954, 2955], [2956, 2963], [2964, 2972], [2973, 2978], [2978, 2979], [2980, 2988], [2989, 2991], [2992, 2999], [3000, 3001], [3002, 3009], [3010, 3018], [3019, 3024], [3024, 3025], [3026, 3034], [3035, 3037], [3038, 3045], [3046, 3047], [3048, 3055], [3056, 3064], [3065, 3070], [3070, 3071], [3072, 3080], [3081, 3083], [3084, 3091], [3092, 3093], [3094, 3101], [3102, 3110], [3111, 3116], [3116, 3117], [3118, 3126], [3127, 3129], [3130, 3137], [3138, 3139], [3140, 3147], [3148, 3156], [3157, 3162], [3162, 3163], [3164, 3172], [3173, 3175], [3176, 3183], [3184, 3185], [3186, 3193], [3194, 3202], [3203, 3208], [3208, 3209], [3210, 3218], [3219, 3221], [3222, 3229], [3230, 3231], [3232, 3239], [3240, 3248], [3249, 3254], [3254, 3255], [3256, 3264], [3265, 3267], [3268, 3275], [3276, 3277], [3278, 3285], [3286, 3294], [3295, 3300], [3300, 3301], [3302, 3310], [3311, 3313], [3314, 3321], [3322, 3323], [3324, 3331], [3332, 3340], [3341, 3346], [3346, 3347], [3348, 3356], [3357, 3359], [3360, 3367], [3368, 3369], [3370, 3377], [3378, 3386], [3387, 3392], [3392, 3393], [3394, 3402], [3403, 3405], [3406, 3413], [3414, 3415], [3416, 3423], [3424, 3432], [3433, 3438], [3438, 3439], [3440, 3448], [3449, 3451], [3452, 3459], [3460, 3461], [3462, 3469], [3470, 3478], [3479, 3484], [3484, 3485], [3486, 3494], [3495, 3497], [3498, 3505], [3506, 3507], [3508, 3515], [3516, 3524], [3525, 3530], [3530, 3531], [3532, 3540], [3541, 3543], [3544, 3551], [3552, 3553], [3554, 3561], [3562, 3570], [3571, 3576], [3576, 3577], [3578, 3586], [3587, 3589], [3590, 3597], [3598, 3599], [3600, 3607], [3608, 3616], [3617, 3622], [3622, 3623], [3624, 3632], [3633, 3635], [3636, 3643], [3644, 3645], [3646, 3653], [3654, 3662], [3663, 3668], [3668, 3669], [3670, 3678], [3679, 3681], [3682, 3689], [3690, 3691], [3692, 3699], [3700, 3708], [3709, 3714], [3714, 3715], [3716, 3724], [3725, 3727], [3728, 3735], [3736, 3737], [3738, 3745], [3746, 3754], [3755, 3760], [3760, 3761], [3762, 3770], [3771, 3773], [3774, 3781], [3782, 3783], [3784, 3791], [3792, 3800], [3801, 3806], [3806, 3807], [3808, 3816], [3817, 3819], [3820, 3827], [3828, 3829], [3830, 3837], [3838, 3846], [3847, 3852], [3852, 3853], [3854, 3862], [3863, 3865], [3866, 3873], [3874, 3875], [3876, 3883], [3884, 3892], [3893, 3898], [3898, 3899], [3900, 3908], [3909, 3911], [3912, 3919], [3920, 3921], [3922, 3929], [3930, 3938], [3939, 3944], [3944, 3945], [3946, 3954], [3955, 3957], [3958, 3965], [3966, 3967], [3968, 3975], [3976, 3984], [3985, 3990], [3990, 3991], [3992, 4000], [4001, 4003], [4004, 4011], [4012, 4013], [4014, 4021], [4022, 4030], [4031, 4036], [4036, 4037], [4038, 4046], [4047, 4049], [4050, 4057], [4058, 4059], [4060, 4067], [4068, 4076], [4077, 4082], [4082, 4083], [4084, 4092], [4093, 4095], [4096, 4103], [4104, 4105], [4106, 4113], [4114, 4122], [4123, 4128], [4128, 4129], [4130, 4138], [4139, 4141], [4142, 4149], [4150, 4151], [4152, 4159], [4160, 4168], [4169, 4174], [4174, 4175], [4176, 4184], [4185, 4187], [4188, 4195], [4196, 4197], [4198, 4205], [4206, 4214], [4215, 4220], [4220, 4221], [4222, 4230], [4231, 4233], [4234, 4241], [4242, 4243], [4244, 4251], [4252, 4260], [4261, 4266], [4266, 4267], [4268, 4276], [4277, 4279], [4280, 4287], [4288, 4289], [4290, 4297], [4298, 4306], [4307, 4312], [4312, 4313], [4314, 4322], [4323, 4325], [4326, 4333], [4334, 4335], [4336, 4343], [4344, 4352], [4353, 4358], [4358, 4359], [4360, 4368], [4369, 4371], [4372, 4379], [4380, 4381], [4382, 4389], [4390, 4398], [4399, 4404], [4404, 4405], [4406, 4414], [4415, 4417], [4418, 4425], [4426, 4427], [4428, 4435], [4436, 4444], [4445, 4450], [4450, 4451], [4452, 4460], [4461, 4463], [4464, 4471], [4472, 4473], [4474, 4481], [4482, 4490], [4491, 4496], [4496, 4497], [4498, 4506], [4507, 4509], [4510, 4517], [4518, 4519], [4520, 4527], [4528, 4536], [4537, 4542], [4542, 4543], [4544, 4552], [4553, 4555], [4556, 4563], [4564, 4565], [4566, 4573], [4574, 4582], [4583, 4588], [4588, 4589], [4590, 4598], [4599, 4602], [4603, 4610], [4611, 4612], [4613, 4620], [4621, 4629], [4630, 4635], [4635, 4636], [4637, 4645], [4646, 4649], [4650, 4657], [4658, 4659], [4660, 4667], [4668, 4676], [4677, 4682], [4682, 4683], [4684, 4692], [4693, 4696], [4697, 4704], [4705, 4706], [4707, 4714], [4715, 4723], [4724, 4729], [4729, 4730], [4731, 4739], [4740, 4743], [4744, 4751], [4752, 4753], [4754, 4761], [4762, 4770], [4771, 4776], [4776, 4777], [4778, 4786], [4787, 4790], [4791, 4798], [4799, 4800], [4801, 4808], [4809, 4817], [4818, 4823], [4823, 4824], [4825, 4833], [4834, 4837], [4838, 4845], [4846, 4847], [4848, 4855], [4856, 4864], [4865, 4870], [4870, 4871], [4872, 4880], [4881, 4884], [4885, 4892], [4893, 4894], [4895, 4902], [4903, 4911], [4912, 4917], [4917, 4918], [4919, 4927], [4928, 4931], [4932, 4939], [4940, 4941], [4942, 4949], [4950, 4958], [4959, 4964], [4964, 4965], [4966, 4974], [4975, 4978], [4979, 4986], [4987, 4988], [4989, 4996], [4997, 5005], [5006, 5011], [5011, 5012], [5013, 5021], [5022, 5025], [5026, 5033], [5034, 5035], [5036, 5043], [5044, 5052], [5053, 5058], [5058, 5059], [5060, 5068], [5069, 5072], [5073, 5080], [5081, 5082], [5083, 5090], [5091, 5099], [5100, 5105], [5105, 5106], [5107, 5115], [5116, 5119], [5120, 5127], [5128, 5129], [5130, 5137], [5138, 5146], [5147, 5152], [5152, 5153], [5154, 5162], [5163, 5166], [5167, 5174], [5175, 5176], [5177, 5184], [5185, 5193], [5194, 5199], [5199, 5200], [5201, 5209], [5210, 5213], [5214, 5221], [5222, 5223], [5224, 5231], [5232, 5240], [5241, 5246], [5246, 5247], [5248, 5256], [5257, 5260], [5261, 5268], [5269, 5270], [5271, 5278], [5279, 5287], [5288, 5293], [5293, 5294], [5295, 5303], [5304, 5307], [5308, 5315], [5316, 5317], [5318, 5325], [5326, 5334], [5335, 5340], [5340, 5341], [5342, 5350], [5351, 5354], [5355, 5362], [5363, 5364], [5365, 5372], [5373, 5381], [5382, 5387], [5387, 5388], [5389, 5397], [5398, 5401], [5402, 5409], [5410, 5411], [5412, 5419], [5420, 5428], [5429, 5434], [5434, 5435], [5436, 5444], [5445, 5448], [5449, 5456], [5457, 5458], [5459, 5466], [5467, 5475], [5476, 5481], [5481, 5482], [5483, 5491], [5492, 5495], [5496, 5503], [5504, 5505], [5506, 5513], [5514, 5522], [5523, 5528], [5528, 5529], [5530, 5538], [5539, 5542], [5543, 5550], [5551, 5552], [5553, 5560], [5561, 5569], [5570, 5575], [5575, 5576], [5577, 5585], [5586, 5589], [5590, 5597], [5598, 5599], [5600, 5607], [5608, 5616], [5617, 5622], [5622, 5623], [5624, 5632], [5633, 5636], [5637, 5644], [5645, 5646], [5647, 5654], [5655, 5663], [5664, 5669], [5669, 5670], [5671, 5679], [5680, 5683], [5684, 5691], [5692, 5693], [5694, 5701], [5702, 5710], [5711, 5716], [5716, 5717], [5718, 5726], [5727, 5730], [5731, 5738], [5739, 5740], [5741, 5748], [5749, 5757], [5758, 5763], [5763, 5764], [5765, 5773], [5774, 5777], [5778, 5785], [5786, 5787], [5788, 5795], [5796, 5804], [5805, 5810], [5810, 5811], [5812, 5820], [5821, 5824], [5825, 5832], [5833, 5834], [5835, 5842], [5843, 5851], [5852, 5857], [5857, 5858], [5859, 5867], [5868, 5871], [5872, 5879], [5880, 5881], [5882, 5889], [5890, 5898], [5899, 5904], [5904, 5905], [5906, 5914], [5915, 5918], [5919, 5926], [5927, 5928], [5929, 5936], [5937, 5945], [5946, 5951], [5951, 5952], [5953, 5961], [5962, 5965], [5966, 5973], [5974, 5975], [5976, 5983], [5984, 5992], [5993, 5998], [5998, 5999], [6000, 6008], [6009, 6012], [6013, 6020], [6021, 6022], [6023, 6030], [6031, 6039], [6040, 6045], [6045, 6046], [6047, 6055], [6056, 6059], [6060, 6067], [6068, 6069], [6070, 6077], [6078, 6086], [6087, 6092], [6092, 6093], [6094, 6102], [6103, 6106], [6107, 6114], [6115, 6116], [6117, 6124], [6125, 6133], [6134, 6139], [6139, 6140], [6141, 6149], [6150, 6153], [6154, 6161], [6162, 6163], [6164, 6171], [6172, 6180], [6181, 6186], [6186, 6187], [6188, 6196], [6197, 6200], [6201, 6208], [6209, 6210], [6211, 6218], [6219, 6227], [6228, 6233], [6233, 6234], [6235, 6243], [6244, 6247], [6248, 6255], [6256, 6257], [6258, 6265], [6266, 6274], [6275, 6280], [6280, 6281], [6282, 6290], [6291, 6294], [6295, 6302], [6303, 6304], [6305, 6312], [6313, 6321], [6322, 6327], [6327, 6328], [6329, 6337], [6338, 6341], [6342, 6349], [6350, 6351], [6352, 6359], [6360, 6368], [6369, 6374], [6374, 6375], [6376, 6384], [6385, 6388], [6389, 6396], [6397, 6398], [6399, 6406], [6407, 6415], [6416, 6421], [6421, 6422], [6423, 6431], [6432, 6435], [6436, 6443], [6444, 6445], [6446, 6453], [6454, 6462], [6463, 6468], [6468, 6469], [6470, 6478], [6479, 6482], [6483, 6490], [6491, 6492], [6493, 6500], [6501, 6509], [6510, 6515], [6515, 6516], [6517, 6525], [6526, 6529], [6530, 6537], [6538, 6539], [6540, 6547], [6548, 6556], [6557, 6562], [6562, 6563], [6564, 6572], [6573, 6576], [6577, 6584], [6585, 6586], [6587, 6594], [6595, 6603], [6604, 6609], [6609, 6610], [6611, 6619], [6620, 6623], [6624, 6631], [6632, 6633], [6634, 6641], [6642, 6650], [6651, 6656], [6656, 6657], [6658, 6666], [6667, 6670], [6671, 6678], [6679, 6680], [6681, 6688], [6689, 6697], [6698, 6703], [6703, 6704], [6705, 6713], [6714, 6717], [6718, 6725], [6726, 6727], [6728, 6735], [6736, 6744], [6745, 6750], [6750, 6751], [6752, 6760], [6761, 6764], [6765, 6772], [6773, 6774], [6775, 6782], [6783, 6791], [6792, 6797], [6797, 6798], [6799, 6807], [6808, 6811], [6812, 6819], [6820, 6821], [6822, 6829], [6830, 6838], [6839, 6844], [6844, 6845], [6846, 6854], [6855, 6858], [6859, 6866], [6867, 6868], [6869, 6876], [6877, 6885], [6886, 6891], [6891, 6892], [6893, 6901], [6902, 6905], [6906, 6913], [6914, 6915], [6916, 6923], [6924, 6932], [6933, 6938], [6938, 6939], [6940, 6948], [6949, 6952], [6953, 6960], [6961, 6962], [6963, 6970], [6971, 6979], [6980, 6985], [6985, 6986], [6987, 6995], [6996, 6999], [7000, 7007], [7008, 7009], [7010, 7017], [7018, 7026], [7027, 7032], [7032, 7033], [7034, 7042], [7043, 7046], [7047, 7054], [7055, 7056], [7057, 7064], [7065, 7073], [7074, 7079], [7079, 7080], [7081, 7089], [7090, 7093], [7094, 7101], [7102, 7103], [7104, 7111], [7112, 7120], [7121, 7126], [7126, 7127], [7128, 7136], [7137, 7140], [7141, 7148], [7149, 7150], [7151, 7158], [7159, 7167], [7168, 7173], [7173, 7174], [7175, 7183], [7184, 7187], [7188, 7195], [7196, 7197], [7198, 7205], [7206, 7214], [7215, 7220], [7220, 7221], [7222, 7230], [7231, 7234], [7235, 7242], [7243, 7244], [7245, 7252], [7253, 7261], [7262, 7267], [7267, 7268], [7269, 7277], [7278, 7281], [7282, 7289], [7290, 7291], [7292, 7299], [7300, 7308], [7309, 7314], [7314, 7315], [7316, 7324], [7325, 7328], [7329, 7336], [7337, 7338], [7339, 7346], [7347, 7355], [7356, 7361], [7361, 7362], [7363, 7371], [7372, 7375], [7376, 7383], [7384, 7385], [7386, 7393], [7394, 7402], [7403, 7408], [7408, 7409], [7410, 7418], [7419, 7422], [7423, 7430], [7431, 7432], [7433, 7440], [7441, 7449], [7450, 7455], [7455, 7456], [7457, 7465], [7466, 7469], [7470, 7477], [7478, 7479], [7480, 7487], [7488, 7496], [7497, 7502], [7502, 7503], [7504, 7512], [7513, 7516], [7517, 7524], [7525, 7526], [7527, 7534], [7535, 7543], [7544, 7549], [7549, 7550], [7551, 7559], [7560, 7563], [7564, 7571], [7572, 7573], [7574, 7581], [7582, 7590], [7591, 7596], [7596, 7597], [7598, 7606], [7607, 7610], [7611, 7618], [7619, 7620], [7621, 7628], [7629, 7637], [7638, 7643], [7643, 7644], [7645, 7653], [7654, 7657], [7658, 7665], [7666, 7667], [7668, 7675], [7676, 7684], [7685, 7690], [7690, 7691], [7692, 7700], [7701, 7704], [7705, 7712], [7713, 7714], [7715, 7722], [7723, 7731], [7732, 7737], [7737, 7738], [7739, 7747], [7748, 7751], [7752, 7759], [7760, 7761], [7762, 7769], [7770, 7778], [7779, 7784], [7784, 7785], [7786, 7794], [7795, 7798], [7799, 7806], [7807, 7808], [7809, 7816], [7817, 7825], [7826, 7831], [7831, 7832], [7833, 7841], [7842, 7845], [7846, 7853], [7854, 7855], [7856, 7863], [7864, 7872], [7873, 7878], [7878, 7879], [7880, 7888], [7889, 7892], [7893, 7900], [7901, 7902], [7903, 7910], [7911, 7919], [7920, 7925], [7925, 7926], [7927, 7935], [7936, 7939], [7940, 7947], [7948, 7949], [7950, 7957], [7958, 7966], [7967, 7972], [7972, 7973], [7974, 7982], [7983, 7986], [7987, 7994], [7995, 7996], [7997, 8004], [8005, 8013], [8014, 8019], [8019, 8020], [8021, 8029], [8030, 8033], [8034, 8041]], "w_pos": [52, 54, 55, 56, 57, 58, 59, 60, 61, 63, 64, 65, 66, 67, 68, 69, 70, 72, 73, 74, 75, 76, 77, 78, 79, 81, 82, 83, 84, 85, 86, 87, 88, 90, 91, 92, 93, 94, 95, 96, 97, 99, 100, 101, 102, 103, 104, 105, 106, 108, 109, 110, 111, 112, 113, 114, 115, 117, 118, 119, 120, 121, 122, 123, 124, 126, 127, 128, 129, 130, 131, 132, 133, 135, 136, 137, 138, 139, 140, 141, 142, 144, 146, 147, 148, 149, 150, 151, 152, 154, 156, 157, 158, 159, 160, 161, 162, 164, 166, 167, 168, 169, 170, 171, 172, 174, 176, 177, 178, 179, 180, 181, 182, 184, 186, 187, 188, 189, 190, 191, 192, 194, 196, 197, 198, 199, 200, 201, 202, 204, 206, 207, 208, 209, 210, 211, 212, 214, 216, 217, 218, 219, 220, 221, 222, 224, 226, 227, 228, 229, 230, 231, 232, 234, 236, 237, 238, 239, 240, 241, 242, 244, 246, 247, 248, 249, 250, 251, 252, 254, 256, 257, 258, 259, 260, 261, 262, 264, 266, 267, 268, 269, 270, 271, 272, 274, 276, 277, 278, 279, 280, 281, 282, 284, 286, 287, 288, 289, 290, 291, 292, 294, 296, 297, 298, 299, 300, 301, 302, 304, 306, 307, 308, 309, 310, 311, 312, 314, 316, 317, 318, 319, 320, 321, 322, 324, 326, 327, 328, 329, 330, 331, 332, 334, 336, 337, 338, 339, 340, 341, 342, 344, 346, 347, 348, 349, 350, 351, 352, 354, 356, 357, 358, 359, 360, 361, 362, 364, 366, 367, 368, 369, 370, 371, 372, 374, 376, 377, 378, 379, 380, 381, 382, 384, 386, 387, 388, 389, 390, 391, 392, 394, 396, 397, 398, 399, 400, 401, 402, 404, 406, 407, 408, 409, 410, 411, 412, 414, 416, 417, 418, 419, 420, 421, 422, 424, 426, 427, 428, 429, 430, 431, 432, 434, 436, 437, 438, 439, 440, 441, 442, 444, 446, 447, 448, 449, 450, 451, 452, 454, 456, 457, 458, 459, 460, 461, 462, 464, 466, 467, 468, 469, 470, 471, 472, 474, 476, 477, 478, 479, 480, 481, 482, 484, 486, 487, 488, 489, 490, 491, 492, 494, 496, 497, 498, 499, 500, 501, 502, 504, 506, 507, 508, 509, 510, 511, 512, 514, 516, 517, 518, 519, 520, 521, 522, 524, 526, 527, 528, 529, 530, 531, 532, 534, 536, 537, 538, 539, 540, 541, 542, 544, 546, 547, 548, 549, 550, 551, 552, 554, 556, 557, 558, 559, 560, 561, 562, 564, 566, 567, 568, 569, 570, 571, 572, 574, 576, 577, 578, 579, 580, 581, 582, 584, 586, 587, 588, 589, 590, 591, 592, 594, 596, 597, 598, 599, 600, 601, 602, 604, 606, 607, 608, 609, 610, 611, 612, 614, 616, 617, 618, 619, 620, 621, 622, 624, 626, 627, 628, 629, 630, 631, 632, 634, 636, 637, 638, 639, 640, 641, 642, 644, 646, 647, 648, 649, 650, 651, 652, 654, 656, 657, 658, 659, 660, 661, 662, 664, 666, 667, 668, 669, 670, 671, 672, 674, 676, 677, 678, 679, 680, 681, 682, 684, 686, 687, 688, 689, 690, 691, 692, 694, 696, 697, 698, 699, 700, 701, 702, 704, 706, 707, 708, 709, 710, 711, 712, 714, 716, 717, 718, 719, 720, 721, 722, 724, 726, 727, 728, 729, 730, 731, 732, 734, 736, 737, 738, 739, 740, 741, 742, 744, 746, 747, 748, 749, 750, 751, 752, 754, 756, 757, 758, 759, 760, 761, 762, 764, 766, 767, 768, 769, 770, 771, 772, 774, 776, 777, 778, 779, 780, 781, 782, 784, 786, 787, 788, 789, 790, 791, 792, 794, 796, 797, 798, 799, 800, 801, 802, 804, 806, 807, 808, 809, 810, 811, 812, 814, 816, 817, 818, 819, 820, 821, 822, 824, 826, 827, 828, 829, 830, 831, 832, 834, 836, 837, 838, 839, 840, 841, 842, 844, 846, 847, 848, 849, 850, 851, 852, 854, 856, 857, 858, 859, 860, 861, 862, 864, 866, 867, 868, 869, 870, 871, 872, 874, 876, 877, 878, 879, 880, 881, 882, 884, 886, 887, 888, 889, 890, 891, 892, 894, 896, 897, 898, 899, 900, 901, 902, 904, 906, 907, 908, 909, 910, 911, 912, 914, 916, 917, 918, 919, 920, 921, 922, 924, 926, 927, 928, 929, 930, 931, 932, 934, 936, 937, 938, 939, 940, 941, 942, 944, 946, 947, 948, 949, 950, 951, 952, 954, 956, 957, 958, 959, 960, 961, 962, 964, 966, 967, 968, 969, 970, 971, 972, 974, 976, 977, 978, 979, 980, 981, 982, 984, 986, 987, 988, 989, 990, 991, 992, 994, 996, 997, 998, 999, 1000, 1001, 1002, 1004, 1006, 1007, 1008, 1009, 1010, 1011, 1012, 1014, 1016, 1017, 1018, 1019, 1020, 1021, 1022, 1024, 1026, 1027, 1028, 1029, 1030, 1031, 1032, 1034, 1036, 1037, 1038, 1039, 1040, 1041, 1042, 1044, 1047, 1048, 1049, 1050, 1051, 1052, 1053, 1055, 1058, 1059, 1060, 1061, 1062, 1063, 1064, 1066, 1069, 1070, 1071, 1072, 1073, 1074, 1075, 1077, 1080, 1081, 1082, 1083, 1084, 1085, 1086, 1088, 1091, 1092, 1093, 1094, 1095, 1096, 1097, 1099, 1102, 1103, 1104, 1105, 1106, 1107, 1108, 1110, 1113, 1114, 1115, 1116, 1117, 1118, 1119, 1121, 1124, 1125, 1126, 1127, 1128, 1129, 1130, 1132, 1135, 1136, 1137, 1138, 1139, 1140, 1141, 1143, 1146, 1147, 1148, 1149, 1150, 1151, 1152, 1154, 1157, 1158, 1159, 1160, 1161, 1162, 1163, 1165, 1168, 1169, 1170, 1171, 1172, 1173, 1174, 1176, 1179, 1180, 1181, 1182, 1183, 1184, 1185, 1187, 1190, 1191, 1192, 1193, 1194, 1195, 1196, 1198, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1209, 1212, 1213, 1214, 1215, 1216, 1217, 1218, 1220, 1223, 1224, 1225, 1226, 1227, 1228, 1229, 1231, 1234, 1235, 1236, 1237, 1238, 1239, 1240, 1242, 1245, 1246, 1247, 1248, 1249, 1250, 1251, 1253, 1256, 1257, 1258, 1259, 1260, 1261, 1262, 1264, 1267, 1268, 1269, 1270, 1271, 1272, 1273, 1275, 1278, 1279, 1280, 1281, 1282, 1283, 1284, 1286, 1289, 1290, 1291, 1292, 1293, 1294, 1295, 1297, 1300, 1301, 1302, 1303, 1304, 1305, 1306, 1308, 1311, 1312, 1313, 1314, 1315, 1316, 1317, 1319, 1322, 1323, 1324, 1325, 1326, 1327, 1328, 1330, 1333, 1334, 1335, 1336, 1337, 1338, 1339, 1341, 1344, 1345, 1346, 1347, 1348, 1349, 1350, 1352, 1355, 1356, 1357, 1358, 1359, 1360, 1361, 1363, 1366, 1367, 1368, 1369, 1370, 1371, 1372, 1374, 1377, 1378, 1379, 1380, 1381, 1382, 1383, 1385, 1388, 1389, 1390, 1391, 1392, 1393, 1394, 1396, 1399, 1400, 1401, 1402, 1403, 1404, 1405, 1407, 1410, 1411, 1412, 1413, 1414, 1415, 1416, 1418, 1421, 1422, 1423, 1424, 1425, 1426, 1427, 1429, 1432, 1433, 1434, 1435, 1436, 1437, 1438, 1440, 1443, 1444, 1445, 1446, 1447, 1448, 1449, 1451, 1454, 1455, 1456, 1457, 1458, 1459, 1460, 1462, 1465, 1466, 1467, 1468, 1469, 1470, 1471, 1473, 1476, 1477, 1478, 1479, 1480, 1481, 1482, 1484, 1487, 1488, 1489, 1490, 1491, 1492, 1493, 1495, 1498, 1499, 1500, 1501, 1502, 1503, 1504, 1506, 1509, 1510, 1511, 1512, 1513, 1514, 1515, 1517, 1520, 1521, 1522, 1523, 1524, 1525, 1526, 1528, 1531, 1532, 1533, 1534, 1535, 1536, 1537, 1539, 1542, 1543, 1544, 1545, 1546, 1547, 1548, 1550, 1553, 1554, 1555, 1556, 1557, 1558, 1559, 1561, 1564, 1565, 1566, 1567, 1568, 1569, 1570, 1572, 1575, 1576, 1577, 1578, 1579, 1580, 1581, 1583, 1586, 1587, 1588, 1589, 1590, 1591, 1592, 1594, 1597, 1598, 1599, 1600, 1601, 1602, 1603, 1605, 1608, 1609, 1610, 1611, 1612, 1613, 1614, 1616, 1619, 1620, 1621, 1622, 1623, 1624, 1625, 1627, 1630, 1631, 1632, 1633, 1634, 1635, 1636, 1638, 1641, 1642, 1643, 1644, 1645, 1646, 1647, 1649, 1652, 1653, 1654, 1655, 1656, 1657, 1658, 1660, 1663, 1664, 1665, 1666, 1667, 1668, 1669, 1671, 1674, 1675, 1676, 1677, 1678, 1679, 1680, 1682, 1685, 1686, 1687, 1688, 1689, 1690, 1691, 1693, 1696, 1697, 1698, 1699, 1700, 1701, 1702, 1704, 1707, 1708, 1709, 1710, 1711, 1712, 1713, 1715, 1718, 1719, 1720, 1721, 1722, 1723, 1724, 1726, 1729, 1730, 1731, 1732, 1733, 1734, 1735, 1737, 1740, 1741, 1742, 1743, 1744, 1745, 1746, 1748, 1751, 1752, 1753, 1754, 1755, 1756, 1757, 1759, 1762, 1763, 1764, 1765, 1766, 1767, 1768, 1770, 1773, 1774, 1775, 1776, 1777, 1778, 1779, 1781, 1784, 1785, 1786, 1787, 1788, 1789, 1790, 1792, 1795, 1796, 1797, 1798, 1799, 1800, 1801, 1803, 1806, 1807, 1808, 1809, 1810, 1811, 1812, 1814, 1817, 1818, 1819, 1820, 1821, 1822, 1823, 1825, 1828, 1829, 1830, 1831, 1832, 1833, 1834, 1836, 1839, 1840, 1841, 1842, 1843, 1844, 1845, 1847, 1850], "span_over": "user", "n_tokens": 3689, "head": "broad"}, {"p_ids": [1905, 25, 198, 27, 22446, 3364, 420, 846, 731, 198, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 24, 6473, 264, 13618, 9403, 1484, 13, 198, 510, 22446, 29, 198], "blocks": [{"kind": "span", "ends": [40], "label_ranges": [[22, 40]], "words_in_block": true, "ids": [271, 6065, 913, 25, 9111, 198, 6196, 25, 1156, 198, 14162, 25, 198, 22365, 42824, 6088, 264, 9403, 30, 198, 22933, 25, 198, 14057, 29, 198, 4754, 4532, 3147, 64, 9403, 1484, 2129, 767, 3147, 31320, 8934, 198, 510, 1448, 29, 271, 1138, 25, 198, 27, 22446, 3364, 420, 846, 731, 198, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 24, 6473, 264, 13618, 9403, 1484, 13, 198, 510, 22446, 29, 198, 271, 8679, 1396, 9111, 314, 279, 2100, 1414, 421, 22651, 799, 314, 279, 8928, 13, 198, 6205, 581, 25], "query": 849, "w_pos": [52, 53, 54, 55, 56, 58, 61, 62, 63, 64, 65, 66, 67, 69, 72, 73, 74, 75, 76, 77, 78, 80, 83, 84, 85, 86, 87, 88, 89, 91, 94, 95, 96, 97, 98, 99, 100, 102, 105, 106, 107, 108, 109, 110, 111, 113, 116, 117, 118, 119, 120, 121, 122, 124, 127, 128, 129, 130, 131, 132, 133, 135, 138, 139, 140, 141, 142, 143, 144, 146, 149, 150, 151, 152, 153, 154, 155, 157, 160, 161, 162, 163, 164, 165, 166, 168, 171, 172, 173, 174, 175, 176, 177, 179, 182, 183, 184, 185, 186, 187, 188, 190, 193, 194, 195, 196, 197, 198, 199, 201, 204, 205, 206, 207, 208, 209, 210, 212, 215, 216, 217, 218, 219, 220, 221, 223, 226, 227, 228, 229, 230, 231, 232, 234, 237, 238, 239, 240, 241, 242, 243, 245, 248, 249, 250, 251, 252, 253, 254, 256, 259, 260, 261, 262, 263, 264, 265, 267, 270, 271, 272, 273, 274, 275, 276, 278, 281, 282, 283, 284, 285, 286, 287, 289, 292, 293, 294, 295, 296, 297, 298, 300, 303, 304, 305, 306, 307, 308, 309, 311, 314, 315, 316, 317, 318, 319, 320, 322, 325, 326, 327, 328, 329, 330, 331, 333, 336, 337, 338, 339, 340, 341, 342, 344, 347, 348, 349, 350, 351, 352, 353, 355, 358, 359, 360, 361, 362, 363, 364, 366, 369, 370, 371, 372, 373, 374, 375, 377, 380, 381, 382, 383, 384, 385, 386, 388, 391, 392, 393, 394, 395, 396, 397, 399, 402, 403, 404, 405, 406, 407, 408, 410, 413, 414, 415, 416, 417, 418, 419, 421, 424, 425, 426, 427, 428, 429, 430, 432, 435, 436, 437, 438, 439, 440, 441, 443, 446, 447, 448, 449, 450, 451, 452, 454, 457, 458, 459, 460, 461, 462, 463, 465, 468, 469, 470, 471, 472, 473, 474, 476, 479, 480, 481, 482, 483, 484, 485, 487, 490, 491, 492, 493, 494, 495, 496, 498, 501, 502, 503, 504, 505, 506, 507, 509, 512, 513, 514, 515, 516, 517, 518, 520, 523, 524, 525, 526, 527, 528, 529, 531, 534, 535, 536, 537, 538, 539, 540, 542, 545, 546, 547, 548, 549, 550, 551, 553, 556, 557, 558, 559, 560, 561, 562, 564, 567, 568, 569, 570, 571, 572, 573, 575, 578, 579, 580, 581, 582, 583, 584, 586, 589, 590, 591, 592, 593, 594, 595, 597, 600, 601, 602, 603, 604, 605, 606, 608, 611, 612, 613, 614, 615, 616, 617, 619, 622, 623, 624, 625, 626, 627, 628, 630, 633, 634, 635, 636, 637, 638, 639, 641, 644, 645, 646, 647, 648, 649, 650, 652, 655, 656, 657, 658, 659, 660, 661, 663, 666, 667, 668, 669, 670, 671, 672, 674, 677, 678, 679, 680, 681, 682, 683, 685, 688, 689, 690, 691, 692, 693, 694, 696, 699, 700, 701, 702, 703, 704, 705, 707, 710, 711, 712, 713, 714, 715, 716, 718, 721, 722, 723, 724, 725, 726, 727, 729, 732, 733, 734, 735, 736, 737, 738, 740, 743, 744, 745, 746, 747, 748, 749, 751, 754, 755, 756, 757, 758, 759, 760, 762, 765, 766, 767, 768, 769, 770, 771, 773, 776, 777, 778, 779, 780, 781, 782, 784, 787, 788, 789, 790, 791, 792, 793, 795, 798, 799, 800, 801, 802, 803, 804, 806, 809, 810, 811, 812, 813, 814, 815, 817, 820, 821, 822, 823, 824, 825]}], "trunc": {"shrink_cut": 0, "protected_cut": 0}, "seg_lens": {"user": 774}, "layout": "c", "names": ["delivery"], "w_offs": [[6916, 6923], [6924, 6932], [6933, 6938], [6938, 6939], [6940, 6948], [6949, 6952], [6953, 6960], [6961, 6962], [6963, 6970], [6971, 6979], [6980, 6985], [6985, 6986], [6987, 6995], [6996, 6999], [7000, 7007], [7008, 7009], [7010, 7017], [7018, 7026], [7027, 7032], [7032, 7033], [7034, 7042], [7043, 7046], [7047, 7054], [7055, 7056], [7057, 7064], [7065, 7073], [7074, 7079], [7079, 7080], [7081, 7089], [7090, 7093], [7094, 7101], [7102, 7103], [7104, 7111], [7112, 7120], [7121, 7126], [7126, 7127], [7128, 7136], [7137, 7140], [7141, 7148], [7149, 7150], [7151, 7158], [7159, 7167], [7168, 7173], [7173, 7174], [7175, 7183], [7184, 7187], [7188, 7195], [7196, 7197], [7198, 7205], [7206, 7214], [7215, 7220], [7220, 7221], [7222, 7230], [7231, 7234], [7235, 7242], [7243, 7244], [7245, 7252], [7253, 7261], [7262, 7267], [7267, 7268], [7269, 7277], [7278, 7281], [7282, 7289], [7290, 7291], [7292, 7299], [7300, 7308], [7309, 7314], [7314, 7315], [7316, 7324], [7325, 7328], [7329, 7336], [7337, 7338], [7339, 7346], [7347, 7355], [7356, 7361], [7361, 7362], [7363, 7371], [7372, 7375], [7376, 7383], [7384, 7385], [7386, 7393], [7394, 7402], [7403, 7408], [7408, 7409], [7410, 7418], [7419, 7422], [7423, 7430], [7431, 7432], [7433, 7440], [7441, 7449], [7450, 7455], [7455, 7456], [7457, 7465], [7466, 7469], [7470, 7477], [7478, 7479], [7480, 7487], [7488, 7496], [7497, 7502], [7502, 7503], [7504, 7512], [7513, 7516], [7517, 7524], [7525, 7526], [7527, 7534], [7535, 7543], [7544, 7549], [7549, 7550], [7551, 7559], [7560, 7563], [7564, 7571], [7572, 7573], [7574, 7581], [7582, 7590], [7591, 7596], [7596, 7597], [7598, 7606], [7607, 7610], [7611, 7618], [7619, 7620], [7621, 7628], [7629, 7637], [7638, 7643], [7643, 7644], [7645, 7653], [7654, 7657], [7658, 7665], [7666, 7667], [7668, 7675], [7676, 7684], [7685, 7690], [7690, 7691], [7692, 7700], [7701, 7704], [7705, 7712], [7713, 7714], [7715, 7722], [7723, 7731], [7732, 7737], [7737, 7738], [7739, 7747], [7748, 7751], [7752, 7759], [7760, 7761], [7762, 7769], [7770, 7778], [7779, 7784], [7784, 7785], [7786, 7794], [7795, 7798], [7799, 7806], [7807, 7808], [7809, 7816], [7817, 7825], [7826, 7831], [7831, 7832], [7833, 7841], [7842, 7845], [7846, 7853], [7854, 7855], [7856, 7863], [7864, 7872], [7873, 7878], [7878, 7879], [7880, 7888], [7889, 7892], [7893, 7900], [7901, 7902], [7903, 7910], [7911, 7919], [7920, 7925], [7925, 7926], [7927, 7935], [7936, 7939], [7940, 7947], [7948, 7949], [7950, 7957], [7958, 7966], [7967, 7972], [7972, 7973], [7974, 7982], [7983, 7986], [7987, 7994], [7995, 7996], [7997, 8004], [8005, 8013], [8014, 8019], [8019, 8020], [8021, 8029], [8030, 8033], [8034, 8041], [8042, 8043], [8044, 8051], [8052, 8060], [8061, 8066], [8066, 8067], [8068, 8076], [8077, 8080], [8081, 8088], [8089, 8090], [8091, 8098], [8099, 8107], [8108, 8113], [8113, 8114], [8115, 8123], [8124, 8127], [8128, 8135], [8136, 8137], [8138, 8145], [8146, 8154], [8155, 8160], [8160, 8161], [8162, 8170], [8171, 8174], [8175, 8182], [8183, 8184], [8185, 8192], [8193, 8201], [8202, 8207], [8207, 8208], [8209, 8217], [8218, 8221], [8222, 8229], [8230, 8231], [8232, 8239], [8240, 8248], [8249, 8254], [8254, 8255], [8256, 8264], [8265, 8268], [8269, 8276], [8277, 8278], [8279, 8286], [8287, 8295], [8296, 8301], [8301, 8302], [8303, 8311], [8312, 8315], [8316, 8323], [8324, 8325], [8326, 8333], [8334, 8342], [8343, 8348], [8348, 8349], [8350, 8358], [8359, 8362], [8363, 8370], [8371, 8372], [8373, 8380], [8381, 8389], [8390, 8395], [8395, 8396], [8397, 8405], [8406, 8409], [8410, 8417], [8418, 8419], [8420, 8427], [8428, 8436], [8437, 8442], [8442, 8443], [8444, 8452], [8453, 8456], [8457, 8464], [8465, 8466], [8467, 8474], [8475, 8483], [8484, 8489], [8489, 8490], [8491, 8499], [8500, 8503], [8504, 8511], [8512, 8513], [8514, 8521], [8522, 8530], [8531, 8536], [8536, 8537], [8538, 8546], [8547, 8550], [8551, 8558], [8559, 8560], [8561, 8568], [8569, 8577], [8578, 8583], [8583, 8584], [8585, 8593], [8594, 8597], [8598, 8605], [8606, 8607], [8608, 8615], [8616, 8624], [8625, 8630], [8630, 8631], [8632, 8640], [8641, 8644], [8645, 8652], [8653, 8654], [8655, 8662], [8663, 8671], [8672, 8677], [8677, 8678], [8679, 8687], [8688, 8691], [8692, 8699], [8700, 8701], [8702, 8709], [8710, 8718], [8719, 8724], [8724, 8725], [8726, 8734], [8735, 8738], [8739, 8746], [8747, 8748], [8749, 8756], [8757, 8765], [8766, 8771], [8771, 8772], [8773, 8781], [8782, 8785], [8786, 8793], [8794, 8795], [8796, 8803], [8804, 8812], [8813, 8818], [8818, 8819], [8820, 8828], [8829, 8832], [8833, 8840], [8841, 8842], [8843, 8850], [8851, 8859], [8860, 8865], [8865, 8866], [8867, 8875], [8876, 8879], [8880, 8887], [8888, 8889], [8890, 8897], [8898, 8906], [8907, 8912], [8912, 8913], [8914, 8922], [8923, 8926], [8927, 8934], [8935, 8936], [8937, 8944], [8945, 8953], [8954, 8959], [8959, 8960], [8961, 8969], [8970, 8973], [8974, 8981], [8982, 8983], [8984, 8991], [8992, 9000], [9001, 9006], [9006, 9007], [9008, 9016], [9017, 9020], [9021, 9028], [9029, 9030], [9031, 9038], [9039, 9047], [9048, 9053], [9053, 9054], [9055, 9063], [9064, 9067], [9068, 9075], [9076, 9077], [9078, 9085], [9086, 9094], [9095, 9100], [9100, 9101], [9102, 9110], [9111, 9114], [9115, 9122], [9123, 9124], [9125, 9132], [9133, 9141], [9142, 9147], [9147, 9148], [9149, 9157], [9158, 9161], [9162, 9169], [9170, 9171], [9172, 9179], [9180, 9188], [9189, 9194], [9194, 9195], [9196, 9204], [9205, 9208], [9209, 9216], [9217, 9218], [9219, 9226], [9227, 9235], [9236, 9241], [9241, 9242], [9243, 9251], [9252, 9255], [9256, 9263], [9264, 9265], [9266, 9273], [9274, 9282], [9283, 9288], [9288, 9289], [9290, 9298], [9299, 9302], [9303, 9310], [9311, 9312], [9313, 9320], [9321, 9329], [9330, 9335], [9335, 9336], [9337, 9345], [9346, 9349], [9350, 9357], [9358, 9359], [9360, 9367], [9368, 9376], [9377, 9382], [9382, 9383], [9384, 9392], [9393, 9396], [9397, 9404], [9405, 9406], [9407, 9414], [9415, 9423], [9424, 9429], [9429, 9430], [9431, 9439], [9440, 9443], [9444, 9451], [9452, 9453], [9454, 9461], [9462, 9470], [9471, 9476], [9476, 9477], [9478, 9486], [9487, 9490], [9491, 9498], [9499, 9500], [9501, 9508], [9509, 9517], [9518, 9523], [9523, 9524], [9525, 9533], [9534, 9537], [9538, 9545], [9546, 9547], [9548, 9555], [9556, 9564], [9565, 9570], [9570, 9571], [9572, 9580], [9581, 9584], [9585, 9592], [9593, 9594], [9595, 9602], [9603, 9611], [9612, 9617], [9617, 9618], [9619, 9627], [9628, 9631], [9632, 9639], [9640, 9641], [9642, 9649], [9650, 9658], [9659, 9664], [9664, 9665], [9666, 9674], [9675, 9678], [9679, 9686], [9687, 9688], [9689, 9696], [9697, 9705], [9706, 9711], [9711, 9712], [9713, 9721], [9722, 9725], [9726, 9733], [9734, 9735], [9736, 9743], [9744, 9752], [9753, 9758], [9758, 9759], [9760, 9768], [9769, 9772], [9773, 9780], [9781, 9782], [9783, 9790], [9791, 9799], [9800, 9805], [9805, 9806], [9807, 9815], [9816, 9819], [9820, 9827], [9828, 9829], [9830, 9837], [9838, 9846], [9847, 9852], [9852, 9853], [9854, 9862], [9863, 9866], [9867, 9874], [9875, 9876], [9877, 9884], [9885, 9893], [9894, 9899], [9899, 9900], [9901, 9909], [9910, 9913], [9914, 9921], [9922, 9923], [9924, 9931], [9932, 9940], [9941, 9946], [9946, 9947], [9948, 9956], [9957, 9960], [9961, 9968], [9969, 9970], [9971, 9978], [9979, 9987], [9988, 9993], [9993, 9994], [9995, 10003], [10004, 10007], [10008, 10015], [10016, 10017], [10018, 10025], [10026, 10034], [10035, 10040], [10040, 10041], [10042, 10050], [10051, 10054], [10055, 10062], [10063, 10064], [10065, 10072], [10073, 10081], [10082, 10087], [10087, 10088], [10089, 10097], [10098, 10101], [10102, 10109], [10110, 10111], [10112, 10119], [10120, 10128], [10129, 10134], [10134, 10135], [10136, 10144], [10145, 10148], [10149, 10156], [10157, 10158], [10159, 10166], [10167, 10175], [10176, 10181], [10181, 10182], [10183, 10191], [10192, 10195], [10196, 10203], [10204, 10205], [10206, 10213], [10214, 10222], [10223, 10228], [10228, 10229]], "w_pos": [52, 53, 54, 55, 56, 58, 61, 62, 63, 64, 65, 66, 67, 69, 72, 73, 74, 75, 76, 77, 78, 80, 83, 84, 85, 86, 87, 88, 89, 91, 94, 95, 96, 97, 98, 99, 100, 102, 105, 106, 107, 108, 109, 110, 111, 113, 116, 117, 118, 119, 120, 121, 122, 124, 127, 128, 129, 130, 131, 132, 133, 135, 138, 139, 140, 141, 142, 143, 144, 146, 149, 150, 151, 152, 153, 154, 155, 157, 160, 161, 162, 163, 164, 165, 166, 168, 171, 172, 173, 174, 175, 176, 177, 179, 182, 183, 184, 185, 186, 187, 188, 190, 193, 194, 195, 196, 197, 198, 199, 201, 204, 205, 206, 207, 208, 209, 210, 212, 215, 216, 217, 218, 219, 220, 221, 223, 226, 227, 228, 229, 230, 231, 232, 234, 237, 238, 239, 240, 241, 242, 243, 245, 248, 249, 250, 251, 252, 253, 254, 256, 259, 260, 261, 262, 263, 264, 265, 267, 270, 271, 272, 273, 274, 275, 276, 278, 281, 282, 283, 284, 285, 286, 287, 289, 292, 293, 294, 295, 296, 297, 298, 300, 303, 304, 305, 306, 307, 308, 309, 311, 314, 315, 316, 317, 318, 319, 320, 322, 325, 326, 327, 328, 329, 330, 331, 333, 336, 337, 338, 339, 340, 341, 342, 344, 347, 348, 349, 350, 351, 352, 353, 355, 358, 359, 360, 361, 362, 363, 364, 366, 369, 370, 371, 372, 373, 374, 375, 377, 380, 381, 382, 383, 384, 385, 386, 388, 391, 392, 393, 394, 395, 396, 397, 399, 402, 403, 404, 405, 406, 407, 408, 410, 413, 414, 415, 416, 417, 418, 419, 421, 424, 425, 426, 427, 428, 429, 430, 432, 435, 436, 437, 438, 439, 440, 441, 443, 446, 447, 448, 449, 450, 451, 452, 454, 457, 458, 459, 460, 461, 462, 463, 465, 468, 469, 470, 471, 472, 473, 474, 476, 479, 480, 481, 482, 483, 484, 485, 487, 490, 491, 492, 493, 494, 495, 496, 498, 501, 502, 503, 504, 505, 506, 507, 509, 512, 513, 514, 515, 516, 517, 518, 520, 523, 524, 525, 526, 527, 528, 529, 531, 534, 535, 536, 537, 538, 539, 540, 542, 545, 546, 547, 548, 549, 550, 551, 553, 556, 557, 558, 559, 560, 561, 562, 564, 567, 568, 569, 570, 571, 572, 573, 575, 578, 579, 580, 581, 582, 583, 584, 586, 589, 590, 591, 592, 593, 594, 595, 597, 600, 601, 602, 603, 604, 605, 606, 608, 611, 612, 613, 614, 615, 616, 617, 619, 622, 623, 624, 625, 626, 627, 628, 630, 633, 634, 635, 636, 637, 638, 639, 641, 644, 645, 646, 647, 648, 649, 650, 652, 655, 656, 657, 658, 659, 660, 661, 663, 666, 667, 668, 669, 670, 671, 672, 674, 677, 678, 679, 680, 681, 682, 683, 685, 688, 689, 690, 691, 692, 693, 694, 696, 699, 700, 701, 702, 703, 704, 705, 707, 710, 711, 712, 713, 714, 715, 716, 718, 721, 722, 723, 724, 725, 726, 727, 729, 732, 733, 734, 735, 736, 737, 738, 740, 743, 744, 745, 746, 747, 748, 749, 751, 754, 755, 756, 757, 758, 759, 760, 762, 765, 766, 767, 768, 769, 770, 771, 773, 776, 777, 778, 779, 780, 781, 782, 784, 787, 788, 789, 790, 791, 792, 793, 795, 798, 799, 800, 801, 802, 803, 804, 806, 809, 810, 811, 812, 813, 814, 815, 817, 820, 821, 822, 823, 824, 825], "span_over": "user", "n_tokens": 1639, "head": "broad"}]} diff --git a/tests/vela/native_bundle.rs b/tests/vela/native_bundle.rs new file mode 100644 index 00000000..21ad21af --- /dev/null +++ b/tests/vela/native_bundle.rs @@ -0,0 +1,207 @@ +//! Bundle-gate tests: a synthetic bundle built in a unique temp +//! directory (no weights, no network) drives the manifest gate — pins, +//! inventory counts, per-file hashes and the manifest SHA-256 anchor. + +use omni_vela_native::bundle::{self, Bundle}; +use serde_json::json; + +fn temp_dir(name: &str) -> std::path::PathBuf { + let dir = + std::env::temp_dir().join(format!("vela-native-bundle-{name}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).expect("tempdir"); + dir +} + +fn sha256(bytes: &[u8]) -> String { + use sha2::{Digest, Sha256}; + let digest = Sha256::new_with_prefix(bytes).finalize(); + digest.iter().map(|byte| format!("{byte:02x}")).collect() +} + +/// The SHA-256 of the manifest as it currently sits on disk — the +/// anchor a loader must pin from outside the bundle directory. +fn manifest_sha(dir: &std::path::Path) -> String { + sha256(&std::fs::read(dir.join("vela_native_manifest.json")).expect("manifest")) +} + +/// Rewrite the manifest through one JSON mutation and re-serialize it +/// (the same shape an exporter or a tamperer would produce). +fn edit_manifest(dir: &std::path::Path, edit: impl FnOnce(&mut serde_json::Value)) { + let manifest_path = dir.join("vela_native_manifest.json"); + let mut manifest: serde_json::Value = + serde_json::from_slice(&std::fs::read(&manifest_path).expect("read")).expect("json"); + edit(&mut manifest); + std::fs::write(&manifest_path, serde_json::to_vec(&manifest).expect("json")).expect("write"); +} + +/// The minimal manifest of a correctly exported member; `files` are the +/// bundle files the gate hashes. +fn write_bundle(dir: &std::path::Path) { + let files: [(&str, &[u8]); 6] = [ + ("backbone.safetensors", b"backbone tensors"), + ("heads.safetensors", b"head tensors"), + ("config.json", b"{}"), + ("calibration.json", b"{}"), + ("tokenizer.json", b"{}"), + ("tokenizer_config.json", b"{}"), + ]; + let mut hashes = serde_json::Map::new(); + for (name, content) in files { + std::fs::write(dir.join(name), content).expect("write"); + hashes.insert(name.to_string(), json!(sha256(content))); + } + let manifest = json!({ + "format": bundle::FORMAT, + "model_id": bundle::MODEL_ID, + "model_name": "vllm-sr/Vela-2.0-0.8B", + "max_length": 16384, + "source": {"repo": "vllm-sr/Vela-2.0-0.8B", "revision": bundle::SOURCE_REVISION}, + "backbone": {"tensors": 320, "bf16": 204, "fp32": 116}, + "heads": {"tensors": 37}, + "files_sha256": hashes, + }); + std::fs::write( + dir.join("vela_native_manifest.json"), + serde_json::to_vec(&manifest).expect("manifest"), + ) + .expect("write"); +} + +#[test] +fn a_matching_bundle_loads_and_exposes_the_member() { + let dir = temp_dir("ok"); + write_bundle(&dir); + let loaded = Bundle::load(&dir, &manifest_sha(&dir)).expect("loads"); + assert_eq!(loaded.model_name(), "vllm-sr/Vela-2.0-0.8B"); + assert_eq!(loaded.max_length(), 16384); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn a_tampered_file_fails_its_hash() { + let dir = temp_dir("tampered"); + write_bundle(&dir); + std::fs::write(dir.join("calibration.json"), b"{\"tampered\": true}").expect("write"); + let error = Bundle::load(&dir, &manifest_sha(&dir)).unwrap_err(); + assert!(error.to_string().contains("calibration.json"), "{error}"); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn a_rewritten_manifest_fails_the_anchor() { + // Re-hashing a swapped file and rewriting files_sha256 used to pass + // the gate: the manifest now has to match an anchor fixed outside + // the bundle directory, so the rewrite itself is refused. + let dir = temp_dir("rewritten"); + write_bundle(&dir); + let anchor = manifest_sha(&dir); + edit_manifest(&dir, |manifest| { + manifest["files_sha256"]["calibration.json"] = + json!(sha256(b"{\"swapped\": true}").as_str()); + }); + std::fs::write(dir.join("calibration.json"), b"{\"swapped\": true}").expect("write"); + let error = Bundle::load(&dir, &anchor).unwrap_err(); + assert!( + error.to_string().contains("pinned SHA-256 anchor"), + "{error}" + ); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn a_foreign_revision_is_refused() { + let dir = temp_dir("foreign"); + write_bundle(&dir); + edit_manifest(&dir, |manifest| { + manifest["source"]["revision"] = json!("d692394127f6f87fc581d8634c9558fe4776efce"); + }); + let error = Bundle::load(&dir, &manifest_sha(&dir)).unwrap_err(); + assert!(error.to_string().contains("pinned"), "{error}"); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn an_unexpected_inventory_is_refused() { + let dir = temp_dir("inventory"); + write_bundle(&dir); + edit_manifest(&dir, |manifest| { + manifest["backbone"]["bf16"] = json!(320); + manifest["backbone"]["fp32"] = json!(0); + }); + let error = Bundle::load(&dir, &manifest_sha(&dir)).unwrap_err(); + assert!(error.to_string().contains("inventory"), "{error}"); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn an_extra_unlisted_file_fails_the_gate() { + let dir = temp_dir("extra"); + write_bundle(&dir); + std::fs::write(dir.join("extra.bin"), b"not in the manifest").expect("write"); + let error = Bundle::load(&dir, &manifest_sha(&dir)).unwrap_err(); + assert!(error.to_string().contains("extra.bin"), "{error}"); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn an_absolute_files_key_is_refused() { + let dir = temp_dir("absolute-key"); + write_bundle(&dir); + edit_manifest(&dir, |manifest| { + manifest["files_sha256"]["/etc/passwd"] = json!(sha256(b"")); + }); + let error = Bundle::load(&dir, &manifest_sha(&dir)).unwrap_err(); + assert!(error.to_string().contains("plain relative name"), "{error}"); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn a_parent_escaping_files_key_is_refused() { + let dir = temp_dir("parent-key"); + write_bundle(&dir); + edit_manifest(&dir, |manifest| { + manifest["files_sha256"]["../outside.bin"] = json!(sha256(b"")); + }); + let error = Bundle::load(&dir, &manifest_sha(&dir)).unwrap_err(); + assert!(error.to_string().contains("plain relative name"), "{error}"); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn a_symlinked_files_key_is_refused() { + let dir = temp_dir("symlink-key"); + write_bundle(&dir); + let outside = temp_dir("symlink-outside"); + std::fs::write(outside.join("secret.bin"), b"outside the bundle").expect("write"); + std::os::unix::fs::symlink(outside.join("secret.bin"), dir.join("linked.safetensors")) + .expect("symlink"); + edit_manifest(&dir, |manifest| { + manifest["files_sha256"]["linked.safetensors"] = json!(sha256(b"outside the bundle")); + }); + let error = Bundle::load(&dir, &manifest_sha(&dir)).unwrap_err(); + assert!(error.to_string().contains("symlink"), "{error}"); + let _ = std::fs::remove_dir_all(&dir); + let _ = std::fs::remove_dir_all(&outside); +} + +#[test] +fn an_empty_files_map_fails_the_gate() { + let dir = temp_dir("empty-files"); + write_bundle(&dir); + edit_manifest(&dir, |manifest| { + manifest["files_sha256"] = json!({}); + }); + let error = Bundle::load(&dir, &manifest_sha(&dir)).unwrap_err(); + assert!(error.to_string().contains("files_sha256"), "{error}"); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn a_missing_manifest_names_the_export_step() { + let dir = temp_dir("empty"); + std::fs::create_dir_all(&dir).expect("tempdir"); + let error = Bundle::load(&dir, &sha256(b"any anchor")).unwrap_err(); + assert!(error.to_string().contains("export_native.py"), "{error}"); + let _ = std::fs::remove_dir_all(&dir); +} diff --git a/tests/vela/native_contract.rs b/tests/vela/native_contract.rs new file mode 100644 index 00000000..ebe73454 --- /dev/null +++ b/tests/vela/native_contract.rs @@ -0,0 +1,126 @@ +//! Wire-contract tests for the native Vela 2.0 worker, mirroring the +//! reference worker's protocol tests: the envelope triples must be +//! byte-equal to the pinned runtime's rejections. + +use omni_vela_native::contract::{self, RequestError}; +use serde_json::json; + +fn body(raw: &str) -> serde_json::Map { + contract::parse_body(raw.as_bytes()).expect("valid body") +} + +fn detail(loc: &[&str], msg: &str, type_: &str) -> serde_json::Value { + let mut path = vec![json!("body")]; + path.extend(loc.iter().map(|part| json!(part))); + json!({"detail": [{"loc": path, "msg": msg, "type": type_}]}) +} + +const SERVED: &str = "vllm-sr/Vela-2.0-0.8B"; + +fn raw_body() -> String { + format!( + r#"{{"model": "{SERVED}", "state": "My card was charged twice.", "questions": {{"billing": {{"type": "noul", "instructions": "any?"}}}}}}"# + ) +} + +#[test] +fn the_envelope_passes_through_unchanged() { + let parsed = body(&raw_body()); + contract::admit_request(&parsed, SERVED).expect("admitted"); + assert_eq!(parsed.len(), 3); +} + +#[test] +fn a_missing_field_reports_upstreams_missing_triple() { + for drop in ["state", "model", "questions"] { + let mut parsed = body(&raw_body()); + parsed.shift_remove(drop); + let error = contract::admit_request(&parsed, SERVED).unwrap_err(); + assert_eq!( + error, + RequestError::envelope(&[drop], "Field required", "missing") + ); + assert_eq!(error.body, detail(&[drop], "Field required", "missing")); + } +} + +#[test] +fn an_extra_field_reports_upstreams_extra_forbidden() { + let mut parsed = body(&raw_body()); + parsed.insert("images".into(), json!([])); + let error = contract::admit_request(&parsed, SERVED).unwrap_err(); + assert_eq!(error.status, 422); + assert_eq!( + error.body, + detail( + &["images"], + "Extra inputs are not permitted", + "extra_forbidden" + ) + ); +} + +#[test] +fn a_blank_model_is_rejected_before_the_name_check() { + let mut parsed = body(&raw_body()); + parsed.insert("model".into(), json!(" ")); + let error = contract::admit_request(&parsed, SERVED).unwrap_err(); + assert_eq!( + error.body, + detail( + &["model"], + "model must not be empty or whitespace", + "value_error" + ) + ); +} + +#[test] +fn another_members_name_is_rejected_with_the_model_triple() { + let mut parsed = body(&raw_body()); + parsed.insert("model".into(), json!("vllm-sr/Vela-2.0-0.3B")); + let error = contract::admit_request(&parsed, SERVED).unwrap_err(); + assert_eq!( + error.body, + detail( + &["model"], + "Model is not served by this runtime", + "value_error.model" + ) + ); +} + +#[test] +fn duplicate_keys_are_rejected_with_the_reference_message() { + let error = + contract::parse_body(br#"{"model": "x", "model": "y", "state": "s", "questions": {}}"#) + .unwrap_err(); + assert_eq!(error.status, 400); + assert_eq!(error.body, json!({"detail": "duplicate JSON key 'model'"})); +} + +#[test] +fn invalid_json_is_a_generic_400() { + let error = contract::parse_body(b"{not json").unwrap_err(); + assert_eq!(error.status, 400); + assert_eq!( + error.body, + json!({"detail": "request body must contain valid JSON and UTF-8 text"}) + ); +} + +#[test] +fn a_non_object_body_is_rejected() { + let error = contract::parse_body(b"[1, 2]").unwrap_err(); + assert_eq!(error.status, 400); + assert_eq!( + error.body, + json!({"detail": "request must be a JSON object"}) + ); +} + +#[test] +fn compile_admits_and_returns_the_body() { + let parsed = contract::compile(raw_body().as_bytes(), SERVED).expect("admitted"); + assert!(parsed.contains_key("questions")); +} diff --git a/tests/vela/native_contract_unit.rs b/tests/vela/native_contract_unit.rs new file mode 100644 index 00000000..e2a60834 --- /dev/null +++ b/tests/vela/native_contract_unit.rs @@ -0,0 +1,14 @@ +//! Private-item tests for the native crate's wire contract, loaded by +//! the `#[path]` wiring in `src/models/vela/native/src/contract.rs`. + +use super::*; + +#[test] +fn duplicate_keys_mirror_the_reference_message() { + let error = normalize_parse_error("request body is not valid JSON: duplicate key \"state\""); + assert_eq!(error.status, 400); + assert_eq!( + error.body, + serde_json::json!({"detail": "duplicate JSON key 'state'"}) + ); +} diff --git a/tests/vela/native_heads.rs b/tests/vela/native_heads.rs new file mode 100644 index 00000000..f8b0a854 --- /dev/null +++ b/tests/vela/native_heads.rs @@ -0,0 +1,423 @@ +//! Head and scoring parity against the recorded head-input +//! fixtures: the Rust heads must reproduce the reference's logits from +//! the same hidden states, and the scoring mirror must turn those +//! logits into the same predict-level answers. Needs the exported +//! bundle (heads + calibration) via `VELA_NATIVE_BUNDLE`; hidden states +//! load from the per-case safetensors fixtures. + +use std::collections::HashMap; +use std::path::PathBuf; + +use omni_vela_native::heads::Heads; +use omni_vela_native::scoring::{Calibration, SpanOut, aggregate_windows, row_answers}; +use serde_json::Value; + +mod native_parity; +use native_parity::max_logit_drift; + +const CASES: [&str; 5] = [ + "example-request", + "example-request-set-threshold", + "example-request-cjk", + "example-request-score", + "long-span-windowed", +]; + +/// One row's collected span calls: (head name, logits). +type RowSpanCalls = Vec<(String, Vec>)>; + +fn fixtures_dir() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../../../tests/vela/fixtures") +} + +fn load_json(name: &str) -> Value { + serde_json::from_str(&std::fs::read_to_string(fixtures_dir().join(name)).expect("fixture")) + .expect("fixture json") +} + +fn f32_matrix(raw: &[u8], name: &str) -> Vec> { + let file = safetensors::SafeTensors::deserialize(raw).expect("safetensors"); + let view = file.tensor(name).expect("tensor"); + assert_eq!(view.dtype(), safetensors::Dtype::F32); + let shape = view.shape(); + let data: Vec = view + .data() + .as_chunks::<4>() + .0 + .iter() + .map(|c| f32::from_le_bytes(*c)) + .collect(); + match shape.len() { + 1 => vec![data], + 2 => data.chunks(shape[1]).map(<[f32]>::to_vec).collect(), + 3 => data + .chunks(shape[1] * shape[2]) + .map(|block| block.chunks(shape[2]).map(<[f32]>::to_vec).collect()) + .next() + .unwrap_or_default(), + _ => panic!("unexpected shape {shape:?} for {name}"), + } +} + +fn f32_stack(raw: &[u8], name: &str) -> Vec>> { + let file = safetensors::SafeTensors::deserialize(raw).expect("safetensors"); + let view = file.tensor(name).expect("tensor"); + let shape = view.shape(); + assert_eq!(shape.len(), 3, "{name} must be 3D"); + let data: Vec = view + .data() + .as_chunks::<4>() + .0 + .iter() + .map(|c| f32::from_le_bytes(*c)) + .collect(); + data.chunks(shape[1] * shape[2]) + .map(|block| block.chunks(shape[2]).map(<[f32]>::to_vec).collect()) + .collect() +} + +fn bundle_dir() -> PathBuf { + PathBuf::from( + std::env::var("VELA_NATIVE_BUNDLE").expect("set VELA_NATIVE_BUNDLE to an exported bundle"), + ) +} + +fn manifest_anchor() -> String { + std::env::var("VELA_NATIVE_MANIFEST_SHA256") + .expect("set VELA_NATIVE_MANIFEST_SHA256 to the manifest digest the exporter prints") +} + +/// Largest absolute difference between two nested JSON numbers; a +/// structural mismatch — an object key missing on either side, arrays +/// of unequal length, or a differing non-numeric value — scores the +/// full 1.0. +fn max_drift(a: &Value, b: &Value) -> f64 { + match (a, b) { + (Value::Number(x), Value::Number(y)) => { + (x.as_f64().unwrap_or(0.0) - y.as_f64().unwrap_or(0.0)).abs() + } + (Value::Array(x), Value::Array(y)) if x.len() == y.len() => x + .iter() + .zip(y) + .map(|(x, y)| max_drift(x, y)) + .fold(0.0, f64::max), + (Value::Object(x), Value::Object(y)) => x + .iter() + .map(|(k, v)| y.get(k).map(|w| max_drift(v, w)).unwrap_or(1.0)) + .chain(y.keys().filter(|k| !x.contains_key(*k)).map(|_| 1.0)) + .fold(0.0, f64::max), + (_, _) if a == b => 0.0, + _ => 1.0, + } +} + +#[test] +#[ignore = "requires VELA_NATIVE_BUNDLE at an exported bundle; CPU only"] +fn the_heads_reproduce_the_reference_logits_and_the_scoring_the_answers() { + let bundle = bundle_dir(); + let anchor = manifest_anchor(); + let heads = Heads::load(&bundle, &anchor).expect("heads load"); + let cal = Calibration::load(&bundle, &anchor).expect("calibration loads"); + // The env gates may only tighten the documented ceilings: an + // unparseable value is an error, and a wider one clamps back down. + let logits_gate = std::env::var("VELA_HEADS_LOGIT_GATE") + .map(|v| { + v.parse::() + .expect("VELA_HEADS_LOGIT_GATE must be a number") + }) + .unwrap_or(4e-5) + .min(4e-5); + let answers_gate = std::env::var("VELA_HEADS_ANSWER_GATE") + .map(|v| { + v.parse::() + .expect("VELA_HEADS_ANSWER_GATE must be a number") + }) + .unwrap_or(1e-6) + .min(1e-6); + println!("gates: logit {logits_gate:.3e} answer {answers_gate:.3e}"); + + for case in CASES { + let fixture = load_json(&format!("head-input-{case}.json")); + let plan = load_json(&format!("native-plan-{case}.json")); + let raw = std::fs::read(fixtures_dir().join(format!("head-input-{case}.safetensors"))) + .expect("safetensors fixture"); + let request = fixture["request"].as_object().expect("request"); + + // Re-derive the per-row thresholds (the request's own overrides). + let mut thresholds: HashMap = HashMap::new(); + for (qid, question) in request["questions"].as_object().expect("questions") { + if let Some(t) = question.get("threshold").and_then(Value::as_f64) { + thresholds.insert(qid.clone(), t); + } + } + + // Drive every recorded head call through the Rust heads. + let mut q_calls_by_row: HashMap> = HashMap::new(); + let mut span_calls_by_row: HashMap = HashMap::new(); + let mut worst_logit = 0.0f64; + for call in fixture["head_calls"].as_array().expect("head calls") { + let row = call["row"].as_u64().expect("row") as usize; + match call["kind"].as_str().expect("kind") { + "q" => { + let cands = f32_stack(&raw, call["cand"].as_str().expect("cand")); + let queries = f32_matrix(&raw, call["query"].as_str().expect("query")); + let want = call["logits"].as_array().expect("logits"); + assert_eq!(cands.len(), want.len(), "{case}: block count"); + assert_eq!(queries.len(), cands.len(), "{case}: query count"); + let mut got = Vec::new(); + for (cand, query) in cands.iter().zip(&queries) { + got.push(heads.forward_candidate(cand, query)); + } + for (g, w) in got.iter().zip(want) { + let w: Vec = w + .as_array() + .unwrap() + .iter() + .map(|v| v.as_f64().unwrap()) + .collect(); + worst_logit = worst_logit.max(max_logit_drift(g, &w)); + } + q_calls_by_row.entry(row).or_default().push(call.clone()); + } + "span" => { + let which = call["head"].as_str().expect("head"); + let words = f32_matrix(&raw, call["words"].as_str().expect("words")); + let labels = f32_matrix(&raw, call["labels"].as_str().expect("labels")); + let head = if which == "broad" { + &heads.broad + } else { + &heads.router + }; + let got = head.forward(&words, &labels); + let want = call["logits"].as_array().expect("logits"); + assert_eq!(got.len(), want.len(), "{case}: span words"); + for (g, w) in got.iter().zip(want) { + let w: Vec = w + .as_array() + .unwrap() + .iter() + .map(|v| v.as_f64().unwrap()) + .collect(); + worst_logit = worst_logit.max(max_logit_drift(g, &w)); + } + span_calls_by_row + .entry(row) + .or_default() + .push((which.to_string(), got)); + } + _ => panic!("unknown head call kind"), + } + } + assert!( + worst_logit <= logits_gate, + "{case}: head logits drift {worst_logit} > {logits_gate}" + ); + + // Assemble predict answers row by row and compare. + let mut answers: Vec = Vec::new(); + for (row_index, row) in plan["rows"].as_array().expect("rows").iter().enumerate() { + let blocks: Vec = row["blocks"] + .as_array() + .expect("blocks") + .iter() + .filter(|b| b["kind"] == "q") + .cloned() + .collect(); + let mut qblocks = Vec::new(); + if let Some(calls) = q_calls_by_row.get(&row_index) { + // One stacked call carries all of the row's question + // blocks, in block order. + let call = calls.first().expect("q call"); + let cands = f32_stack(&raw, call["cand"].as_str().expect("cand")); + let queries = f32_matrix(&raw, call["query"].as_str().expect("query")); + for (block, (cand, query)) in blocks.iter().zip(cands.iter().zip(&queries)) { + let names: Vec = block["names"] + .as_array() + .unwrap() + .iter() + .map(|n| n.as_str().unwrap().to_string()) + .collect(); + let has_abs = block["has_abs"].as_bool().unwrap_or(false); + let mut logits = heads.forward_candidate(cand, query); + if block["qtype"] == "set" { + for value in logits.iter_mut() { + *value += heads.set_bias; + } + } + logits.truncate(names.len() + usize::from(has_abs)); + qblocks.push(omni_vela_native::scoring::QBlock { + qid: block["qid"].as_str().unwrap().to_string(), + qtype: block["qtype"].as_str().unwrap().to_string(), + names, + has_abs, + logits, + }); + } + } + // The span part of this row: original word offsets, the span + // block's names, and the part's kept token count. + let mut span_out = None; + let span_blocks: Vec<&Value> = row["blocks"] + .as_array() + .unwrap() + .iter() + .filter(|b| b["kind"] == "span") + .collect(); + let window_rows = plan["window_rows"].as_array().cloned().unwrap_or_default(); + let row_span_calls = span_calls_by_row + .get(&row_index) + .cloned() + .unwrap_or_default(); + let has_span = !span_blocks.is_empty() && !row_span_calls.is_empty(); + if has_span { + let names: Vec = span_block_names(row); + let w_offs: Vec<(usize, usize)> = row["w_offs"] + .as_array() + .map(|a| { + a.iter() + .map(|o| { + ( + o[0].as_u64().unwrap_or(0) as usize, + o[1].as_u64().unwrap_or(0) as usize, + ) + }) + .collect() + }) + .unwrap_or_default(); + let over = row["span_over"].as_str().unwrap_or("user").to_string(); + let n_tokens = row["seg_lens"][&over].as_u64().unwrap_or(0) as usize; + // Windowed rows: the full render is skipped (the first + // call) and the windows aggregate in order; else the + // single call is the answer's logits. + let (which, logits) = if window_rows.is_empty() { + row_span_calls.last().expect("span call").clone() + } else { + let which = row_span_calls[0].0.clone(); + let window_logits: Vec>> = + row_span_calls[1..].iter().map(|(_, l)| l.clone()).collect(); + let window_offs: Vec> = window_rows + .iter() + .map(|w| { + w["w_offs"] + .as_array() + .map(|a| { + a.iter() + .map(|o| { + ( + o[0].as_u64().unwrap_or(0) as usize, + o[1].as_u64().unwrap_or(0) as usize, + ) + }) + .collect() + }) + .unwrap_or_default() + }) + .collect(); + ( + which, + aggregate_windows(&names, &w_offs, &window_logits, &window_offs), + ) + }; + + span_out = Some(SpanOut { + qid: span_qid(&plan, row_index), // row 0 -> spans[0], row i -> spans[i] + over, + names, + offs: w_offs, + logits, + head: which.clone(), + head_reason: row["head_reason"].as_str().map(str::to_string), + label_alias: row.get("label_alias").filter(|v| !v.is_null()).cloned(), + n_tokens, + }); + } + // The span text: the targeted part's rendered text. + let span_text = row["span_over"].as_str().and_then(|role| { + plan["predict_request"]["parts"] + .as_array() + .unwrap() + .iter() + .find(|p| p["type"] == *role) + .map(|p| p["text"].as_str().unwrap().to_string()) + }); + let mut row_answers = row_answers( + &cal, + &qblocks, + span_out.as_ref(), + &thresholds, + span_text.as_deref(), + ); + // The span answer's id is filled by the caller above. + answers.append(&mut row_answers); + } + + // The reference orders answers by the predict request's + // question order, not by the row walk. + let order: Vec = plan["plan"] + .as_array() + .unwrap() + .iter() + .map(|p| p["qid"].as_str().unwrap().to_string()) + .collect(); + let mut answers = answers; + answers.sort_by_key(|a| { + order + .iter() + .position(|qid| qid == a["id"].as_str().unwrap_or("")) + .unwrap_or(usize::MAX) + }); + + let want = fixture["predict"]["answers"].as_array().expect("answers"); + let drift = max_drift(&Value::Array(answers.clone()), &Value::Array(want.clone())); + assert!( + drift <= answers_gate, + "{case}: answer drift {drift} > {answers_gate}\ngot: {}\nwant: {}", + serde_json::to_string_pretty(&answers).unwrap(), + serde_json::to_string_pretty(&want).unwrap(), + ); + // Structure: same answer count, same ids and discrete picks. + assert_eq!(answers.len(), want.len(), "{case}: answer count"); + for (got, expected) in answers.iter().zip(want) { + assert_eq!(got["id"], expected["id"], "{case}: answer order"); + if got["type"] != "span" { + assert_eq!(got["answer"], expected["answer"], "{case}: discrete pick"); + } else { + assert_eq!( + got["threshold_rule"], expected["threshold_rule"], + "{case}: rule" + ); + let got_spans = got["spans"].as_array().cloned().unwrap_or_default(); + let want_spans = expected["spans"].as_array().cloned().unwrap_or_default(); + assert_eq!(got_spans.len(), want_spans.len(), "{case}: span count"); + for (g, w) in got_spans.iter().zip(&want_spans) { + assert_eq!(g["label"], w["label"], "{case}: span label"); + assert_eq!(g["text"], w["text"], "{case}: span text"); + assert_eq!(g["start"], w["start"], "{case}: span start"); + assert_eq!(g["end"], w["end"], "{case}: span end"); + } + } + } + println!("{case}: logit drift {worst_logit:.3e}, answer drift {drift:.3e}"); + } +} + +fn span_block_names(row: &Value) -> Vec { + row["names"] + .as_array() + .map(|a| a.iter().map(|n| n.as_str().unwrap().to_string()).collect()) + .unwrap_or_default() +} + +fn span_qid(plan: &Value, row_index: usize) -> String { + // Row 0 carries the first span question; row i carries spans[i]. + let spans: Vec<&Value> = plan["plan"] + .as_array() + .unwrap() + .iter() + .filter(|p| p["type"] == "span") + .collect(); + let picked = spans.get(row_index).or_else(|| spans.first()); + picked + .map(|p| p["qid"].as_str().unwrap().to_string()) + .unwrap_or_else(|| format!("row{row_index}")) +} diff --git a/tests/vela/native_parity.rs b/tests/vela/native_parity.rs new file mode 100644 index 00000000..b47ca776 --- /dev/null +++ b/tests/vela/native_parity.rs @@ -0,0 +1,36 @@ +//! Logit comparisons shared by the CPU-head and GPU-executor replays. + +pub fn max_logit_drift(got: &[f32], want: &[f64]) -> f64 { + assert_eq!(got.len(), want.len(), "logit count"); + assert!( + got.iter().all(|v| v.is_finite()), + "non-finite replay logits" + ); + assert!( + want.iter().all(|v| v.is_finite()), + "non-finite reference logits" + ); + got.iter() + .zip(want) + .map(|(a, b)| (a - *b as f32).abs() as f64) + .fold(0.0, f64::max) +} + +#[test] +fn finite_logits_keep_their_measured_drift() { + assert_eq!(max_logit_drift(&[1.0, 2.5], &[1.0, 2.0]), 0.5); +} + +#[test] +fn non_finite_logits_are_rejected() { + for invalid in [f32::NAN, f32::INFINITY, f32::NEG_INFINITY] { + assert!(std::panic::catch_unwind(|| max_logit_drift(&[invalid], &[0.0])).is_err()); + assert!(std::panic::catch_unwind(|| max_logit_drift(&[0.0], &[invalid as f64])).is_err()); + } +} + +#[test] +fn missing_or_extra_logits_are_rejected() { + assert!(std::panic::catch_unwind(|| max_logit_drift(&[], &[0.0])).is_err()); + assert!(std::panic::catch_unwind(|| max_logit_drift(&[0.0], &[])).is_err()); +} diff --git a/tests/vela/native_processing.rs b/tests/vela/native_processing.rs new file mode 100644 index 00000000..cca92cd9 --- /dev/null +++ b/tests/vela/native_processing.rs @@ -0,0 +1,149 @@ +//! Processing tests against the frozen token-plan fixtures. +//! +//! The mapping tests (state → parts, SystemOne → predict, plan) run +//! everywhere with no tokenizer and no weights. The full render parity +//! — rows and window rows byte-for-byte against the fixtures — needs +//! the exported bundle's tokenizer and is `#[ignore]`d, opt-in with +//! `-- --ignored`: +//! +//! VELA_NATIVE_BUNDLE=weights/vela-2.0-0.8b-native \ +//! VELA_NATIVE_MANIFEST_SHA256= \ +//! cargo test -p omni-vela-native --test vela_native_processing -- --ignored --nocapture + +use std::path::PathBuf; + +use omni_vela_native::questions::system_one_to_predict; +use omni_vela_native::state::system_one_state; +use serde_json::Value; + +const CASES: [&str; 5] = [ + "example-request", + "example-request-set-threshold", + "example-request-cjk", + "example-request-score", + "long-span-windowed", +]; + +fn fixture(case: &str) -> Value { + let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../../../tests/vela/fixtures"); + serde_json::from_str( + &std::fs::read_to_string(root.join(format!("native-plan-{case}.json"))) + .expect("fixture present; record it with recipe/vela/export_token_plan.py"), + ) + .expect("fixture JSON") +} + +// ------------------------------------------------------- mapping (no tokens) + +fn mapped(case: &str) -> Value { + let fix = fixture(case); + let request = fix["request"].as_object().expect("request"); + let state = system_one_state(&request["state"]).expect("state maps"); + let questions = request["questions"].as_object().expect("questions"); + let mapped = system_one_to_predict(state, questions).expect("maps"); + serde_json::json!({ + "predict_request": serde_json::to_value(&mapped.request).unwrap(), + "plan": serde_json::to_value(&mapped.plan).unwrap(), + }) +} + +#[test] +fn the_predict_mapping_matches_every_fixture() { + for case in CASES { + let got = mapped(case); + let fix = fixture(case); + assert_eq!( + got["predict_request"], fix["predict_request"], + "predict_request differs for {case}" + ); + assert_eq!(got["plan"], fix["plan"], "plan differs for {case}"); + } +} + +#[test] +fn an_invalid_question_reports_the_pinned_triples() { + let state = system_one_state(&serde_json::json!("s")).expect("state"); + let questions = serde_json::json!({ + "q": {"type": "choice", "instructions": "ok?", "criteria": {"a": "x"}} + }); + let error = system_one_to_predict(state, questions.as_object().unwrap()).unwrap_err(); + assert_eq!( + error.body, + serde_json::json!({"detail": [{"loc": ["body", "questions", "q", "criteria"], + "msg": "choice criteria must be an object with 2 to 255 entries", "type": "value_error"}]}) + ); + + let state = system_one_state(&serde_json::json!("s")).expect("state"); + let questions = serde_json::json!({ + "q": {"type": "span", "instructions": "which?", "criteria": {"a": "x"}, + "head": "other", "over": "request"} + }); + let error = system_one_to_predict(state, questions.as_object().unwrap()).unwrap_err(); + assert_eq!( + error.body["detail"][0]["loc"], + serde_json::json!(["body", "questions", "q", "head"]) + ); + assert_eq!(error.body["detail"][0]["type"], "literal_error"); +} + +#[test] +fn a_set_label_collision_is_rejected_after_the_walk() { + let state = system_one_state(&serde_json::json!("s")).expect("state"); + let questions = serde_json::json!({ + "issues": {"type": "set", "instructions": "which?", "criteria": {"billing": "b"}}, + "issues.billing": {"type": "noul", "instructions": "which?"} + }); + let error = system_one_to_predict(state, questions.as_object().unwrap()).unwrap_err(); + assert_eq!( + error.body["detail"][0]["msg"], + "collides with the Noul view of a set label" + ); +} + +// ------------------------------------------------- full render (opt-in) + +fn bundle_dir() -> PathBuf { + let path = PathBuf::from( + std::env::var("VELA_NATIVE_BUNDLE") + .expect("set VELA_NATIVE_BUNDLE to the export directory"), + ); + assert!( + path.is_dir(), + "VELA_NATIVE_BUNDLE is not a directory: {}", + path.display() + ); + path +} + +fn manifest_anchor() -> String { + std::env::var("VELA_NATIVE_MANIFEST_SHA256") + .expect("set VELA_NATIVE_MANIFEST_SHA256 to the manifest digest the exporter prints") +} + +#[test] +#[ignore = "requires VELA_NATIVE_BUNDLE at an exported bundle; CPU only"] +fn the_rendered_rows_match_every_fixture() { + let bundle = bundle_dir(); + let processor = omni_vela_native::processing::Processor::load(&bundle, &manifest_anchor()) + .expect("processor loads the exported bundle"); + for case in CASES { + let fix = fixture(case); + let request = fix["request"].as_object().expect("request"); + let planned = match processor.plan(request) { + Ok(planned) => planned, + Err(error) => panic!( + "plan failed for {case}: {}", + serde_json::to_string(&error.body).unwrap() + ), + }; + let got = serde_json::json!({ + "rows": serde_json::to_value(&planned.rows).unwrap(), + "window_rows": serde_json::to_value(&planned.window_rows).unwrap(), + }); + assert_eq!(got["rows"], fix["rows"], "rows differ for {case}"); + assert_eq!( + got["window_rows"], fix["window_rows"], + "window rows differ for {case}" + ); + } +} diff --git a/tests/vela/native_units.rs b/tests/vela/native_units.rs new file mode 100644 index 00000000..9bcb8498 --- /dev/null +++ b/tests/vela/native_units.rs @@ -0,0 +1,139 @@ +//! Unit tests for the native crate's state mapping and word splitter, +//! extracted from the modules per the repository's test-placement +//! rule (the private contract test lives in native_contract_unit.rs, +//! loaded by the `#[path]` wiring in contract.rs). + +use omni_vela_native::scoring::decode_spans; +use omni_vela_native::state::system_one_state; +use omni_vela_native::words::{split_words, unit_ids, word_first_tokens}; +use serde_json::json; + +// ------------------------------------------------------------- state + +#[test] +fn a_plain_string_is_one_user_part() { + let parts = system_one_state(&json!("hello")).expect("parts"); + assert_eq!(parts.parts.len(), 1); + assert_eq!(parts.parts[0].r#type, "user"); + assert_eq!(parts.fields["request"].end, 5); +} + +#[test] +fn typed_keys_group_into_ordered_parts_with_field_ranges() { + let parts = system_one_state(&json!({ + "answer": "A", "request": "R", "source": "S", "extra": "E", + })) + .expect("parts"); + let roles: Vec<&str> = parts.parts.iter().map(|p| p.r#type.as_str()).collect(); + assert_eq!(roles, ["user", "context", "answer"]); + let context = &parts.parts[1]; + assert_eq!(context.text, "source:\nS\n\nextra:\nE"); + assert_eq!(parts.fields["source"].start, "source:\n".chars().count()); + assert_eq!( + parts.fields["extra"].start, + "source:\nS\n\nextra:\n".chars().count() + ); +} + +#[test] +fn an_empty_role_part_is_rejected() { + let error = system_one_state(&json!({"request": " "})).unwrap_err(); + assert_eq!( + error.body["detail"][0]["msg"], + serde_json::json!("the user part of the state is empty") + ); +} + +#[test] +fn a_string_holding_a_json_object_is_read_as_that_object() { + let parts = system_one_state(&json!("{\"request\": \"text\"}")).expect("parts"); + assert_eq!(parts.parts.len(), 1); + assert_eq!(parts.fields["request"].end, 4); +} + +// ------------------------------------------------------------- words + +#[test] +fn words_split_like_the_reference() { + let text = "Hi, I'm Tom Baker (tom.baker@example.com)."; + let words: Vec = split_words(text) + .iter() + .map(|(a, b)| text.chars().skip(*a).take(b - a).collect()) + .collect(); + assert_eq!( + words, + vec![ + "Hi", + ",", + "I", + "'", + "m", + "Tom", + "Baker", + "(", + "tom.baker@example.com", + ")", + "." + ] + ); +} + +#[test] +fn cjk_characters_are_one_unit_each() { + let text = "追踪 parcel 仓库"; + let words: Vec = split_words(text) + .iter() + .map(|(a, b)| text.chars().skip(*a).take(b - a).collect()) + .collect(); + assert_eq!(words, vec!["追", "踪", "parcel", "仓", "库"]); + // Mixed content: each CJK char its own unit, the ASCII word one + // unit, the space outside every unit. + let chars: Vec = "追踪 parcel".chars().collect(); + assert_eq!(unit_ids(&chars), vec![0, 1, -1, 2, 2, 2, 2, 2, 2]); +} + +#[test] +fn first_tokens_skip_empty_tokens() { + let text = "alpha beta"; + let offsets = vec![(0, 0), (0, 5), (6, 10)]; + let (word_offsets, first) = word_first_tokens(text, &offsets); + assert_eq!(word_offsets, vec![(0, 5), (6, 10)]); + assert_eq!(first, vec![1, 2]); +} + +#[test] +fn a_rejected_word_ends_the_current_span() { + for rejected in [0.1, 0.5] { + let got = decode_spans( + &[vec![0.9], vec![rejected], vec![0.9]], + &[(0, 5), (6, 9), (10, 13)], + &["PERSON".into()], + "Alice met Bob", + 0.5, + ); + assert_eq!(got.len(), 2, "rejected probability {rejected}"); + assert_eq!(got[0]["start"], 0); + assert_eq!(got[0]["end"], 5); + assert_eq!(got[1]["start"], 10); + assert_eq!(got[1]["end"], 13); + for span in got { + assert_eq!(span["label"], "PERSON"); + assert_eq!(span["probability"], f64::from(0.9f32)); + } + } +} + +#[test] +fn empty_offsets_do_not_break_consecutive_labeled_words() { + let got = decode_spans( + &[vec![0.9], vec![0.1], vec![0.9]], + &[(0, 5), (5, 5), (6, 9)], + &["PERSON".into()], + "Alice Bob", + 0.5, + ); + assert_eq!(got.len(), 1); + assert_eq!(got[0]["start"], 0); + assert_eq!(got[0]["end"], 9); + assert_eq!(got[0]["label"], "PERSON"); +} diff --git a/tests/vela/test_export_fixtures.py b/tests/vela/test_export_fixtures.py new file mode 100644 index 00000000..e971497d --- /dev/null +++ b/tests/vela/test_export_fixtures.py @@ -0,0 +1,65 @@ +"""Fixture exporters must leave their verified package reusable (no weights needed).""" + +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path +from types import ModuleType, SimpleNamespace + +import pytest + +from models.vela import hybrid + +from .test_hybrid import build_package, small_files + + +@pytest.mark.parametrize("exporter_name", ["export_token_plan", "export_head_inputs"]) +@pytest.mark.parametrize("preload_runtime", [False, True]) +def test_exporter_keeps_package_verifiable(tmp_path, monkeypatch, exporter_name, preload_runtime): + package = tmp_path / "package" + package.mkdir() + files = small_files() + files["vela2_inference.py"] = b""" +from types import SimpleNamespace +class Vela2: + @staticmethod + def from_pretrained(*args, **kwargs): + import configuration_vela2 + return SimpleNamespace(run=SimpleNamespace(model=None)) +""" + files["configuration_vela2.py"] = b"loaded = True\n" + files["modeling_vela2.py"] = b"loaded = True\n" + pins = build_package(package, files) + monkeypatch.setattr(hybrid, "PINNED_FILES_SHA256", pins) + # The test covers imports and initialization only; no numerical library + # is needed by the empty case list. + monkeypatch.setitem(sys.modules, "numpy", ModuleType("numpy")) + monkeypatch.setitem(sys.modules, "safetensors", ModuleType("safetensors")) + monkeypatch.setitem(sys.modules, "safetensors.numpy", SimpleNamespace(save_file=None)) + monkeypatch.setattr(sys, "path", sys.path.copy()) + monkeypatch.setattr(sys, "dont_write_bytecode", False) + for name in ("vela2_inference", "configuration_vela2", "modeling_vela2"): + monkeypatch.setitem(sys.modules, name, None) + monkeypatch.delitem(sys.modules, name) + if preload_runtime: + # An already imported entry point still imports package modules lazily. + spec = importlib.util.spec_from_file_location("vela2_inference", package / "vela2_inference.py") + runtime = importlib.util.module_from_spec(spec) + exec(files["vela2_inference.py"], runtime.__dict__) + monkeypatch.setitem(sys.modules, "vela2_inference", runtime) + + path = Path(__file__).resolve().parents[2] / "recipe" / "vela" / f"{exporter_name}.py" + spec = importlib.util.spec_from_file_location(exporter_name, path) + exporter = importlib.util.module_from_spec(spec) + spec.loader.exec_module(exporter) + monkeypatch.setattr(exporter, "cases", lambda: []) + if exporter_name == "export_head_inputs": + monkeypatch.setattr(exporter, "Recorder", lambda *_args: None) + + args = ["--model", str(package), "--out", str(tmp_path / "fixtures")] + for _ in range(2): + assert exporter.main(args) == 0 + assert not sys.dont_write_bytecode + hybrid.verify_package(package) + assert not list(package.rglob("__pycache__"))