diff --git a/Cargo.lock b/Cargo.lock index ff9de569..7d5c98b7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -199,9 +199,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.4.7" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54413ede23c2daf518f35156dfde027feb2374004d63bd497f983c8db9c0e313" +checksum = "86e055fa6e3e40ad6187ba25b6855eeb635d5271f8e0f0e1e9a5826b8d75bea0" dependencies = [ "find-msvc-tools", "shlex", @@ -271,9 +271,9 @@ dependencies = [ [[package]] name = "crc32fast" -version = "1.5.2" +version = "1.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78" +checksum = "0ba106e844045ddc02c20768ac4b0b7cb45f2cc2ecbd833ac9f5dacb183c2055" dependencies = [ "cfg-if", ] @@ -438,20 +438,20 @@ checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] name = "either" -version = "1.18.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" +checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" [[package]] name = "equivalent" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" [[package]] name = "errno" @@ -497,9 +497,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.13" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef25905e51abafe4dcea6c15fec58c57b601cdbd0ee53d22ea1d3016c587d39b" +checksum = "1c66915cc5391097de7a2b0b972e8505aa33c0df401fd3a6811dbdb32881ba48" [[package]] name = "flate2" @@ -563,7 +563,7 @@ checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -729,13 +729,12 @@ dependencies = [ [[package]] name = "hyper" -version = "1.11.1" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27b501faa50e7a26c3d3560ca625132f4078a17771f4810baf70475ae48cbe43" +checksum = "2c3e324da4c95177d6291d4c8730197c0d1822f8a9766814a4a44fa5ab797c9c" dependencies = [ "atomic-waker", "bytes", - "futures-channel", "futures-core", "http", "http-body", @@ -790,9 +789,9 @@ dependencies = [ [[package]] name = "icu_collections" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" +checksum = "fe090a42a0d3140fed8c63263ed1fecc02422cb2f4bfe55cd8de2de1b8e105dd" dependencies = [ "displaydoc", "potential_utf", @@ -804,9 +803,9 @@ dependencies = [ [[package]] name = "icu_locale_core" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" +checksum = "25d29b57c17c6528d0dc4c2141cfa2d08200465acb537a3909c8b9b8a4a984c0" dependencies = [ "displaydoc", "litemap", @@ -817,9 +816,9 @@ dependencies = [ [[package]] name = "icu_normalizer" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" +checksum = "404a3b8c486f205c87fe7f23ab6894d84a4ad76d6e4671ae1981ba7def87c599" dependencies = [ "icu_collections", "icu_normalizer_data", @@ -831,15 +830,15 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" +checksum = "20f0088bfb7441f7a37fa0c9e20cb1e9cba1cac6b9ce8cd2a6dc44294f8e8cbc" [[package]] name = "icu_properties" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" +checksum = "2ea83d991c56158f7cf898bf86436c1dfe7c07ee82fe88951f2695c9f37a379f" dependencies = [ "displaydoc", "icu_collections", @@ -852,15 +851,15 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" +checksum = "b98f29af82d02b163fe50161388350e743bb3107cc8e8c2f7314cb505faddfcf" [[package]] name = "icu_provider" -version = "2.3.1" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" +checksum = "413099315636c588ee8f011190b329b971a09a8fae9d157e01dcae8379468514" dependencies = [ "displaydoc", "icu_locale_core", @@ -968,9 +967,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.189" +version = "0.2.191" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" [[package]] name = "libloading" @@ -1042,9 +1041,9 @@ checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" [[package]] name = "memmap2" -version = "0.9.11" +version = "0.9.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0" +checksum = "6b0ca1155a9ecc25aa12e8313a119f32edf0255fea2f44839c4ac0a12b2ac157" dependencies = [ "libc", ] @@ -1083,9 +1082,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.2.3" +version = "1.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b18443e9c262bfe8fa82f51666e2642c53393f7e5c27b3e1aeab922cff5b9d8" +checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee" dependencies = [ "libc", "wasi", @@ -1245,7 +1244,7 @@ dependencies = [ "omni-runtime", "safetensors 0.8.0", "serde_json", - "sha2 0.11.0", + "sha2 0.11.1", "tempfile", "tokenizers 0.22.2", "tokio", @@ -1341,6 +1340,19 @@ dependencies = [ "tokio", ] +[[package]] +name = "omni-vela-native" +version = "0.1.0" +dependencies = [ + "anyhow", + "omni-qwen3-5-native", + "onig", + "serde", + "serde_json", + "sha2 0.11.1", + "tokenizers 0.22.2", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -1473,9 +1485,9 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.18" +version = "0.11.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9746dbde176634f4f2f1faf2404e30a31b2bc1e9cafb5329c95d8177a18c9fc" +checksum = "0e750cca55fe4f0439a15d0bb529da9651e79993e8e72c61a899a36d462befbe" dependencies = [ "bytes", "getrandom 0.4.3", @@ -1495,9 +1507,9 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.15" +version = "0.5.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" +checksum = "af66907df18639dcf4db56ca65490cabc4b27a97dbadd96f2926cca73298f016" dependencies = [ "cfg_aliases", "libc", @@ -1720,9 +1732,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.45" +version = "0.23.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" dependencies = [ "once_cell", "ring", @@ -1744,9 +1756,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.15" +version = "0.103.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" +checksum = "4443b1b6d39d7c663139942cc4558262562d2c84123bb6282ef177009ca8b281" dependencies = [ "ring", "rustls-pki-types", @@ -1815,15 +1827,16 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] name = "serde_json" -version = "1.0.151" +version = "1.0.154" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" dependencies = [ + "foldhash", "indexmap", "itoa", "memchr", @@ -1868,9 +1881,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.11.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +checksum = "47d7069beb7d6ac7b9acd1039986e73443f24234f41074da099d6f994ac9ad19" dependencies = [ "cfg-if", "cpufeatures 0.3.1", @@ -1907,9 +1920,9 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "smallvec" -version = "1.16.2" +version = "1.16.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9395f0f0eee849a9b707b2f06bb92a6a422090e2123bb2ef8e87a0e61892a8e" +checksum = "d0d5d4b1981b8bbbb7aed204cae5726d683d5d6ac7e74ec54122ea28f315e7c8" [[package]] name = "socket2" @@ -1970,9 +1983,9 @@ dependencies = [ [[package]] name = "syn" -version = "3.0.6" +version = "3.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" dependencies = [ "proc-macro2", "quote", @@ -1996,7 +2009,7 @@ checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -2029,14 +2042,14 @@ checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] name = "tinystr" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" +checksum = "abc840c238154d58f9e86b2792261edfde565a17696e1cda1a449de27979801a" dependencies = [ "displaydoc", "zerovec", @@ -2116,9 +2129,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.53.1" +version = "1.54.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +checksum = "b52fd3255258d26fac7264ab5a76b53c13367136dc011d3d6ad4f8ed070396a3" dependencies = [ "bytes", "libc", @@ -2138,14 +2151,14 @@ checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] name = "tokio-rustls" -version = "0.26.5" +version = "0.26.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0c85f2c3ef0b1cd58b36682f4b17aaa995f0e5db534d85692b4903abce21f67" +checksum = "c9cc2678c2cdd569ef8215e2afd7954ada2ae20b4fdd2c5fe6139a3b02d105db" dependencies = [ "rustls", "tokio", @@ -2153,9 +2166,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.19" +version = "0.7.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" +checksum = "e464cf451ba96ebfc6f9b6542f17ee8b8956e33f1e40d9690624e59d7a7f8a4b" dependencies = [ "bytes", "futures-core", @@ -2244,9 +2257,9 @@ checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" [[package]] name = "unicode-ident" -version = "1.0.26" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" [[package]] name = "unicode-normalization-alignments" @@ -2301,9 +2314,9 @@ checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] name = "want" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +checksum = "ec4cdd0dd910afe868b7ef477227d8d538b46b3075031afee8a9f2acb0a2ed0b" dependencies = [ "try-lock", ] @@ -2366,7 +2379,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", "wasm-bindgen-shared", ] @@ -2414,9 +2427,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "1.0.9" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" +checksum = "f5f280bbb2b287bc8e7de7e3852945a0ff50d1bbcb5ed42b6c7a568c546df171" dependencies = [ "rustls-pki-types", ] @@ -2523,9 +2536,9 @@ checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" [[package]] name = "yoke" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +checksum = "417ef3fcf6c300b7dbd25c85531a746931c02ad536f3cf4786ed1e88223340b3" dependencies = [ "stable_deref_trait", "yoke-derive", @@ -2534,30 +2547,30 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33811428bee40dbceb6d545e95754741d17a6aef9a4849f0fd62e2ba4f412a78" +checksum = "ec8ebde2db3681e8c9980cc27822030e68752690ddfa9473e739aeb4dbde6d71" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", "synstructure", ] [[package]] name = "zerocopy" -version = "0.8.59" +version = "0.8.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6df92bf3d9227be3d53173901ddbffac2babc27ae50f397776ffd6dc33f800cb" +checksum = "e5fe1f8f1b06191a00962174c61aa5005e0bb391a6d80d07e24d115c01a92ed8" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.59" +version = "0.8.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac4f328cf2f05d084e496c3e9c3f33ed0a183656a16e1fcec4d464d8373aec82" +checksum = "863ad3ac83293fb4d740aedbfdc9240dd8d1a50c1099acd76ce80ce7c7230c7f" dependencies = [ "proc-macro2", "quote", @@ -2581,15 +2594,15 @@ checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", "synstructure", ] [[package]] name = "zeroize" -version = "1.9.0" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" +checksum = "e13084392c5e4bc371903e2935a5eaeed24905a7511356b883835e18a78f6879" [[package]] name = "zerotrie" @@ -2604,9 +2617,9 @@ dependencies = [ [[package]] name = "zerovec" -version = "0.11.8" +version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" +checksum = "f58a2799cfe9a1670c7ea8edbd52ef8edb93cb37a7f83bf31ba9e35d9dae6a40" dependencies = [ "yoke", "zerofrom", @@ -2615,13 +2628,13 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.6" +version = "0.11.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" +checksum = "c3743ddbf47738d929cd1ae2e199d40d0162979dfe31aef8ae646d2bf00bf97a" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index d4d00a3d..13ec2aa8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,3 +1,3 @@ [workspace] -members = ["src/frontend", "src/runtime", "src/models/clm", "src/models/cua_s1/native", "src/models/qwen3_5/native", "src/models/open_jev/native", "src/models/jev_vl/native", "src/models/omnijev/native", "src/models/laya", "src/backends/cuda", "src/models/decider/native", "src/models/jemm/native", "src/models/startlux/native"] +members = ["src/frontend", "src/runtime", "src/models/clm", "src/models/cua_s1/native", "src/models/qwen3_5/native", "src/models/open_jev/native", "src/models/jev_vl/native", "src/models/omnijev/native", "src/models/laya", "src/backends/cuda", "src/models/decider/native", "src/models/jemm/native", "src/models/vela/native", "src/models/startlux/native"] resolver = "3" diff --git a/README.md b/README.md index 6e35c3ed..ccebd2dc 100644 --- a/README.md +++ b/README.md @@ -167,9 +167,10 @@ Python screenshot worker, Valen-Preview-0923 has a Python reference worker for text and one-image state, CLM has a stub-encoder contract recipe, and OmniJev-4B v1.1 runs on a native Rust/CUDA worker. D3 (Decision 3.0) has a Python reference worker that runs the checkpoint's own -pinned runtime for text, image and video state, and Vela 2.0 0.3B has one -that serves all five SystemOne question kinds (`choice`/`noul`/`score` plus -the `span`/`set` extensions) on the checkpoint's own pinned runtime. +pinned runtime for text, image and video state, and the Vela 2.0 members (0.3B and the 0.8B +hybrid) have one that serves all five +SystemOne question kinds (`choice`/`noul`/`score` plus the `span`/`set` +extensions) on each checkpoint's own pinned runtime. pplx-decider-v1.1-27b has a Python reference worker for the pinned Perplexity checkpoint. Public231 concurrency-1 latency for heads `1ba07d1`, `f968e576`, `ffae182`, `4b99b73`, and `fcf5e2b` is recorded in the recipe: @@ -191,6 +192,7 @@ checkpoint. Public231 concurrency-1 latency for heads `1ba07d1`, `f968e576`, | OmniJev-4B v1.1 | [Native Rust/CUDA worker](recipe/omnijev/native.md), eager: one image, Choice, Noul and Score; [reference comparison on sm_89](recipe/omnijev/native.md#comparison-with-the-reference) | | D3 (Decision 3.0) | [Python reference worker](recipe/d3/reference.md) on the checkpoint's pinned runtime; `choice`/`noul`/`score` over text, image and video state; author-run on A100 for text and media serving (see the integration and media PRs) | | Vela 2.0 0.3B | [Python reference worker](recipe/vela/reference.md) on the checkpoint's pinned runtime; all five kinds (`choice`/`noul`/`score`/`span`/`set`) for text state; CPU FP32 validated (author-run, evidence in the integration PR) | +| Vela 2.0 0.8B hybrid | [Python reference worker](recipe/vela/reference.md) on the checkpoint's pinned runtime; same surface plus `span_heads`; [native library](recipe/vela/native.md) (bundle export, wire contract, token-level processing); CPU FP32 parity recorded in the integration PR | | Clef-flash (9B) and Clef (27B) | [Python reference worker](recipe/clef/reference.md) for text `choice`/`noul`/`score`; flash is the bring-up variant. Image, video and native CUDA are not served. L20X-labeled lab Public231 C1 primary is tip `594528a9`: p50 164.969 ms, p95 216.755 ms, `/health` `fla+causal_conv1d`, versus `9030065` run1 p50 247.518 ms, p95 413.025 ms (Δp50 −82.549 ms, Δp95 −196.270 ms) | | pplx-decider-v1.1-27b | [Python reference worker](recipe/pplx_decider/README.md) on the pinned checkpoint; `choice`/`noul`/`score` text plus up to four images. Lab L20X-labeled GPUs are the H200 target. Kept Public231 primary is `fcf5e2b`: 231/231, p50 87.782 ms, p95 291.183 ms, grouped-value FlashInfer with q and k packed before the L2 view (p50 −1.30%, p95 −1.56% versus `4b99b73` 88.942 / 295.797). `4b99b73` was p50 −4.20%, p95 −0.22% versus `ffae182` 92.844 / 296.458. Uncached all-sequence FlashInfer `3c72f02` is not the primary (p50 88.808 ms, p95 300.751 ms). Short-sequence FlashInfer `a50b9aa` missed its gate (p50 95.446 ms, p95 300.871 ms). cuDNN-first SDPA `0dcaa0ac` missed its gate (p50 138.501 ms, p95 332.859 ms). Fused gate-up `8b4ee326` missed its gate (p50 102.437 ms, p95 304.970 ms). Always-on dynamic compile `e36b32ed` missed the both-percentiles gate (p50 97.751 ms, p95 263.124 ms; p95 improved). The 512-token split `7e3b1be` missed its gate (p50 100.037 ms, p95 257.084 ms). Isolated-median compile `34ba5de` missed its gate (p50 97.084 ms, p95 256.679 ms; the 136-token probe was eager). The FlashInfer state-workspace ring `17a7b120` missed its gate (p50 89.165 ms, p95 295.723 ms). Always-on FLA residual RMSNorm `3afb472` missed its gate (p50 93.935 ms, p95 274.902 ms). Length-gated FLA residual RMSNorm `705ae089` missed its gate (p50 94.604 ms, p95 274.835 ms). `use_cp=auto` `a6f4d9a` missed its gate (p50 112.294 ms, p95 299.419 ms). Grouped-value `3ecdf5a` crashed in warmup (0/231, no p50 or p95). FLA fused gated RMSNorm `801ef7b` missed its gate (p50 89.172 ms, p95 274.790 ms). FLA `swiglu_fwd` `2bbbe4e` missed its gate (owner closeout p50 97.613 ms, p95 292.795 ms) and is not installed | diff --git a/docs/supported-models.md b/docs/supported-models.md index c5dbc4c2..ef8323b5 100644 --- a/docs/supported-models.md +++ b/docs/supported-models.md @@ -29,6 +29,7 @@ Models that are being added are also tracked in issues labeled [new model](https | OmniJev-4B v1.1 | [Native Rust/CUDA worker](../recipe/omnijev/native.md) on the shared Qwen3.5 vision and language path, eager | Contract, preparation, pixels, heads and finishing checked against the reference; no CPU inference | Compared with the reference on compute capability 8.9 for [140 answers](../recipe/omnijev/native.md#comparison-with-the-reference) | Not supported | Compute capability 8.0 or newer, CUDA toolkit to build, the export from `export.py` | | D3 (Decision 3.0) | [Python reference worker](../recipe/d3/reference.md) on the checkpoint's pinned runtime — text, image and video state over `choice`/`noul`/`score`, with the deliberate transport differences documented in the recipe; the native worker is in progress ([#136](https://github.com/ThinkFlowLab/system1-omni/issues/136)) | Unverified | Author-run on NVIDIA A100 (sm_80): serving smoke and reference parity for text and media (exact decisions, max delta 0.0); evidence in the integration and media PRs. Owner: @MrDongsls. | Unverified | Python 3.12, the [pinned runtime requirements](../recipe/d3/requirements.txt), the [pinned package](https://huggingface.co/vllm-sr/d3) (~52 GB); [media budgets and the 64 MiB total request cap](../recipe/d3/reference.md) | | Vela 2.0 0.3B | [Python reference worker](../recipe/vela/reference.md) running the checkpoint's pinned runtime | **CPU FP32 validated** (author-run, x86 WSL2, 24 cores): startup with all-kind warmup, the four example requests plus five synthetic rejections, and exact-match reference parity against the pinned upstream server on all nine (four 200s byte-equal incl. key order, agreed 422/413); byte-identical frontend forwarding; evidence in the integration PR. Owner: @MrDongsls. | Unverified (bf16-autocast is an untested deployment option) | Not supported | Python 3.12, the [pinned requirements](../recipe/vela/requirements.txt), the [pinned package](https://huggingface.co/vllm-sr/Vela-2.0-0.3B) (1.24 GB FP32 weights; ~4.4 GB with ONNX/assets) | +| Vela 2.0 0.8B hybrid | [Python reference worker](../recipe/vela/reference.md) running the checkpoint's pinned runtime (same wire layer as 0.3B); [native library](../recipe/vela/native.md) — bundle export, wire contract, token-level processing | Same surface as 0.3B plus the `span_heads` field (router/broad dispatch); **CPU FP32 parity** (author-run; evidence in the integration PR) | Unverified (bf16-autocast is the evaluated upstream setting but untested here) | Not supported | Python 3.12, the [pinned requirements](../recipe/vela/requirements.txt), the [pinned package](https://huggingface.co/vllm-sr/Vela-2.0-0.8B) (2.02 GB shard + 4.5 MB broad head; ~3 GB FP32 parameters) | | Clef-flash (9B) | [Python reference worker](../recipe/clef/reference.md); text `choice`/`noul`/`score`; non-thinking prompt; `thinking.enabled: false` admitted and omitted from the prompt; 16384-token refusal | Contract tests only; no inference | Operator-reported Public231 C1 on the L20X-labeled lab host. Baseline `9030065` run1 p50 247.518 ms, p95 413.025 ms, 231/231. Tip `594528a9` remeasure, warmup 5, concurrency 1, stock Public231 with `thinking.enabled` false, metric `successful_latency`: 231/231, p50 164.969 ms, p95 216.755 ms, `/health` `fla+causal_conv1d`. Versus run1: Δp50 −82.549 ms (−33.35%), Δp95 −196.270 ms (−47.52%). Not re-run here | Not supported | Python 3.12, [card-tested torch 2.11 and transformers 5.10.2](../recipe/clef/requirements.txt), pinned `Cloudflare/clef-flash` snapshot (~19.1 GB) | | Clef (27B) | Same worker and text contract as Clef-flash, selected with `--variant clef` | Contract tests only; no inference | Not run | Not supported | Same stack; pinned `Cloudflare/clef` snapshot (~55.0 GB). Weights and joint-head config differ from flash | | pplx-decider-v1.1-27b | [Python reference worker](../recipe/pplx_decider/README.md) on the checkpoint's own runtime. Text and up to four images; `choice`/`noul`/`score`. Native CUDA executor is not this worker. Lab L20X-labeled GPUs are the H200 target. The causal-conv1d build keeps sm_90 and sm_89. Capability 8.9 is ready on FLA chunk. Capability 9.0 uses all-sequence FlashInfer. The kept primary rotates 48 `cu_seqlens` buffers per device and length, passes `use_cp` false, skips the q/k repeat, and packs q and k before the L2-norm view. `801ef7b` fused the 48 linear gated RMSNorms and missed the median (p50 89.172 ms, p95 274.790 ms), so that kernel stays off. `2bbbe4e` replaced `silu(gate) * up` with FLA `swiglu_fwd` and the owner closeout rejected it (p50 97.613 ms, p95 292.795 ms), so that kernel stays off. Gate and up GEMMs stay separate. Residual RMSNorm and the gated output norm stay eager. An unknown `PPLX_GDN_PREFILL` fails closed on every device, including capability 8.9. `3ecdf5a` crashed in warmup (0/231). Always-on fusion `3afb472` and the 512-token gate `705ae089` missed the median. The 48-slot FlashInfer state ring `17a7b120` missed the gate. `PPLX_GDN_PREFILL=fla` reproduces the prior `f968e576` chunk. `PPLX_GDN_PREFILL=short` is the rejected ≤256 dispatch. SDPA stays on the checkpoint order. The fused gate-up GEMM is refused. The forward stays eager. `PPLX_COMPILE=dynamic` and `PPLX_COMPILE=dynamic-tail` are rejected opt-ins | Contract checks only; no inference | Reported load on 1× lab L20X (sm_89, ~140 GiB) at head `1ba07d1`: BF16 resident ~50333 MiB, no OOM. Historical primary: 231/231, p50 98.777 ms, p95 341.202 ms. Prior primary is the `f968e576` remeasure on the lab L20X-labeled GPU, `compute_capability=9.0`, `fla-chunk`, `flashinfer` null: 231/231, p50 94.077 ms, p95 300.200 ms (p50 −4.76%, p95 −12.02%). Prior primary `ffae182`: 231/231, p50 92.844 ms, p95 296.458 ms (p50 −1.31%, p95 −1.25%). Prior primary `4b99b73`: 231/231, p50 88.942 ms, p95 295.797 ms (p50 −4.20%, p95 −0.22%), 48-slot `cu_seqlens`, quality `choice_acc=0.9065` (139), `noul_acc=0.8784` (74), `score_mae=0.2035` (18). Kept primary is `fcf5e2b`: 231/231, p50 87.782 ms, p95 291.183 ms (p50 −1.30%, p95 −1.56%), grouped-value FlashInfer with q and k packed, same quality digits. Uncached all-sequence FlashInfer `3c72f02`: p50 88.808 ms, p95 300.751 ms, gate failed. Short-sequence FlashInfer `a50b9aa`: p50 95.446 ms, p95 300.871 ms, gate failed. cuDNN-first SDPA `0dcaa0ac`: p50 138.501 ms, p95 332.859 ms, gate failed. Fused gate-up `8b4ee326`: p50 102.437 ms, p95 304.970 ms, gate failed. Always-on dynamic compile `e36b32ed`: p50 97.751 ms, p95 263.124 ms, gate failed (p95 improved). 512-token compile `7e3b1be`: p50 100.037 ms, p95 257.084 ms, gate failed. Isolated-median compile `34ba5de`: p50 97.084 ms, p95 256.679 ms, gate failed (median probe was eager). FlashInfer state-workspace ring `17a7b120`: p50 89.165 ms, p95 295.723 ms, gate failed. Always-on FLA residual RMSNorm `3afb472`: p50 93.935 ms, p95 274.902 ms, gate failed. Length-gated FLA residual RMSNorm `705ae089`: p50 94.604 ms, p95 274.835 ms, gate failed. `use_cp=auto` `a6f4d9a`: p50 112.294 ms, p95 299.419 ms, gate failed. Grouped-value `3ecdf5a` crashed in warmup (0/231, no p50 or p95). The fused linear gated-norm candidate has no sample. Does not enter the H800 board | Not supported | Python 3.12, pinned checkpoint revision `14c9d90137962d300f3de95633f7b6535e80c5ce`, checkpoint `requirements.txt`, `causal-conv1d` 1.7.0, `flash-linear-attention` 0.5.2, optional `flashinfer-python` 0.6.16.post3. Shard sizes in the release manifest sum with the readout to about 48.6 GiB before activations | @@ -44,9 +45,10 @@ StartLux serves Instant only; AutoThink is not implemented, and the weights are D3's reference worker serves the same three kinds for text, image and video state (inline base64 media under the pinned runtime's per-item budgets); its media serving evidence is author-run on the A100 above. -Vela 2.0 0.3B's reference worker serves all five kinds (`choice`, `noul`, -`score`, plus the `span`/`set` extensions with their Noul views) for text -state on CPU FP32; the hybrid members (0.8B/4B/9B) are separate slices. +Vela 2.0's reference workers (0.3B and the 0.8B hybrid) serve all five +kinds (`choice`, `noul`, `score`, plus the `span`/`set` extensions with +their Noul views, and `span_heads` on the hybrid) for text state on CPU +FP32; the larger hybrid members (4B/9B) are separate slices. Clef-flash and Clef serve the same three kinds for text only. The worker refuses image and video inputs. The flash CUDA cell records two operator-reported Public231 concurrency-1 results on the L20X-labeled lab diff --git a/mkdocs.yml b/mkdocs.yml index 7e3be0ad..b4bc3858 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -84,7 +84,8 @@ nav: - CLM stub-encoder contract: recipe/clm/README.md - LFM2.5-350M choice worker: recipe/lfm2/README.md - D3 reference worker: recipe/d3/reference.md - - Vela 2.0 0.3B reference worker: recipe/vela/reference.md + - Vela 2.0 reference workers: recipe/vela/reference.md + - Vela 2.0 0.8B native library: recipe/vela/native.md - Clef reference worker: recipe/clef/reference.md - pplx-decider reference worker: recipe/pplx_decider/README.md - Contributing: CONTRIBUTING.md diff --git a/recipe/README.md b/recipe/README.md index 041c091d..c7a5c294 100644 --- a/recipe/README.md +++ b/recipe/README.md @@ -48,10 +48,14 @@ For a first real decision, follow the [complete CPU walkthrough](../docs/getting verify its runtime manifest, and serve `choice`/`noul`/`score` decisions over text, image and video state through the Python worker and the Rust frontend (text serving and parity validated on A100; media budgets and caps in the recipe). -- [Vela 2.0 0.3B reference worker](vela/reference.md): download the pinned +- [Vela 2.0 reference workers (0.3B and the 0.8B hybrid)](vela/reference.md): download the pinned vllm-sr/Vela-2.0-0.3B package, pass its full-hash package gate, and serve all five SystemOne question kinds (`choice`/`noul`/`score` plus the `span`/`set` - extensions) through the Python worker and the Rust frontend on CPU FP32. + extensions) through the Python worker and the Rust frontend on CPU FP32; + or vllm-sr/Vela-2.0-0.8B at revision `326b01d8` behind its own 10-file gate. +- [Vela 2.0 0.8B native library](vela/native.md): export the pinned-bundle + manifest gate, wire contract and token-level processing for the 0.8B hybrid, + checked against frozen fixtures; serving stays with the reference Python worker. - [Clef reference worker](clef/reference.md): load pinned Clef-flash (9B) or Clef (27B) and serve text `choice`/`noul`/`score`. Image, video and native CUDA are not served. Operator-reported flash Public231 concurrency-1 diff --git a/recipe/vela/compare_reference.py b/recipe/vela/compare_reference.py index fde4cd15..3588e06a 100644 --- a/recipe/vela/compare_reference.py +++ b/recipe/vela/compare_reference.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Compare the Vela 2.0 0.3B worker pipeline against the pinned upstream server. +"""Compare the Vela 2.0 worker pipeline against the pinned upstream server. Runs each fixture request through two paths that share one loaded pinned engine, so the comparison isolates this repository's admission and @@ -13,8 +13,8 @@ finishing). Both paths execute the same verified runtime, and the worker delegates -response construction to the same pinned code — so the criterion is the -charter's reference parity: exact match, including JSON key order, with +response construction to the same pinned code — so the criterion is +reference parity: exact match, including JSON key order, with rejections agreeing on status and body (``{"detail": [{"loc", "msg", "type"}]}``). ``--export`` additionally freezes the oracle fixture — request, SystemOne response and the predict()-level answers — for the @@ -24,11 +24,14 @@ The default fixture set is the recipe's four example requests plus five synthetic rejections: an invalid span question (whole-request -422), a schema that alone exceeds the 8,192-token window (413), and +422), a schema that alone exceeds the member's input window (413), and the three envelope 422s (missing field, extra field, wrong model name) that freeze the worker's preamble admission against the pinned -runtime's. CPU FP32 is the oracle setting; GPU bf16-autocast is a -deployment option, not this tool's default. +runtime's. The example files' ``model`` field is rewritten to the +served member's name, and the hybrid member's oracle fixtures export +under an ``oracle-hybrid-`` prefix, so both members share one tool. +CPU FP32 is the oracle setting; GPU bf16-autocast is a deployment +option, not this tool's default. """ from __future__ import annotations @@ -44,7 +47,7 @@ def parse_args(argv=None): parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) - parser.add_argument("--model", required=True, help="pinned Vela 2.0 0.3B package directory") + parser.add_argument("--model", required=True, help="pinned Vela 2.0 package directory") parser.add_argument("--device", default="cpu", help="torch device (default: the CPU FP32 oracle)") parser.add_argument( "--fixture", @@ -62,8 +65,8 @@ def parse_args(argv=None): return parser.parse_args(argv) -def over_limit_body(): - """A schema that alone cannot fit the 8,192-token window. +def over_limit_body(model=MODEL_NAME): + """A schema that alone cannot fit the member's input window. 255 choice options with ~300-character descriptions: far past ``max_length`` before any state is rendered, which is exactly the @@ -79,7 +82,7 @@ def over_limit_body(): for index in range(255) } return { - "model": MODEL_NAME, + "model": model, "state": "short", "questions": { "big": {"type": "choice", "instructions": "Pick one.", "criteria": criteria} @@ -87,11 +90,11 @@ def over_limit_body(): } -def invalid_span_body(): +def invalid_span_body(model=MODEL_NAME): """A span question whose criteria is a list, not a label mapping.""" return { - "model": MODEL_NAME, + "model": model, "state": "short", "questions": { "bad": { @@ -109,50 +112,63 @@ def missing_model_body(): return {"state": "short", "questions": {"q": {"type": "noul", "instructions": "any?"}}} -def extra_field_body(): +def extra_field_body(model=MODEL_NAME): """An envelope with a field the preamble does not permit ('extra_forbidden').""" return { - "model": MODEL_NAME, + "model": model, "state": "short", "questions": {"q": {"type": "noul", "instructions": "any?"}}, "images": [], } -def wrong_model_body(): +def wrong_model_body(served=MODEL_NAME): """Another member's name on this runtime ('value_error.model' 422).""" + other = "vllm-sr/Vela-2.0-0.3B" if served.endswith("0.8B") else "vllm-sr/Vela-2.0-0.8B" return { - "model": "vllm-sr/Vela-2.0-0.8B", + "model": other, "state": "short", "questions": {"q": {"type": "noul", "instructions": "any?"}}, } -def default_cases(): +def default_cases(served=MODEL_NAME): return [ ("example-request", HERE / "example-request.json"), ("example-request-set-threshold", HERE / "example-request-set-threshold.json"), ("example-request-cjk", HERE / "example-request-cjk.json"), ("example-request-score", HERE / "example-request-score.json"), - ("invalid-span-422", invalid_span_body()), - ("over-limit-413", over_limit_body()), + ("invalid-span-422", invalid_span_body(served)), + ("over-limit-413", over_limit_body(served)), ("missing-model-422", missing_model_body()), - ("extra-field-422", extra_field_body()), - ("wrong-model-422", wrong_model_body()), + ("extra-field-422", extra_field_body(served)), + ("wrong-model-422", wrong_model_body(served)), ] -def load_cases(paths): +def load_cases(paths, served=MODEL_NAME): if not paths: - cases = default_cases() + cases = default_cases(served) else: cases = [(Path(path).name, path) for path in paths] - return [ - (name, body if isinstance(body, dict) else json.loads(Path(body).read_text(encoding="utf-8"))) - for name, body in cases - ] + loaded = [] + for name, source in cases: + body = source if isinstance(source, dict) else json.loads(Path(source).read_text(encoding="utf-8")) + # The example files are member-agnostic apart from their model field; + # serving the other member rewrites it so the case exercises the 200 + # path it was written for, on both compared paths alike. Synthetic + # bodies (including the deliberate wrong-model one) are never touched. + if ( + not isinstance(source, dict) + and isinstance(body, dict) + and body.get("model") != served + and body.get("model") in (MODEL_NAME, "vllm-sr/Vela-2.0-0.8B") + ): + body = {**body, "model": served} + loaded.append((name, body)) + return loaded def reference_response(client, body): @@ -245,10 +261,10 @@ def main(argv=None): sys.path.insert(0, str(HERE.parents[1] / "src")) from fastapi.testclient import TestClient - from frontend.vela import decide - from models.vela.engine import Vela2Executor + from frontend.vela import decide, detect_member - executor = Vela2Executor(args.model, device=args.device) + executor_class, _default_name = detect_member(args.model) + executor = executor_class(args.model, device=args.device) # Both runtime imports stay inside a no-bytecode window: the package # gate refuses existing caches, and vela2_serve is a fresh import that # would otherwise write the __pycache__ the next worker start rejects. @@ -262,11 +278,15 @@ def main(argv=None): app = vela2_serve.create_app(executor.engine, model_name=executor.model_name, api_key=None) client = TestClient(app) + served = executor.model_name + # The hybrid member's oracle fixtures get their own prefix so both + # members' frozen baselines can live in one directory. + prefix = "oracle-hybrid-" if getattr(executor, "is_hybrid", False) else "oracle-" exported = 0 failed = 0 try: - for name, body in load_cases(args.fixture): + for name, body in load_cases(args.fixture, served): raw = json.dumps(body).encode("utf-8") worker = worker_response(raw, executor, decide) reference = reference_response(client, body) @@ -278,7 +298,7 @@ def main(argv=None): req, _plan = vela2_inference.system_one_to_predict(body["state"], body["questions"]) target = Path(args.export) if target.is_dir(): - target = target / f"oracle-{name}.json" + target = target / f"{prefix}{name}.json" export_oracle(name, body, worker[1], executor.engine.predict(req), target) exported += 1 print() diff --git a/recipe/vela/export_native.py b/recipe/vela/export_native.py new file mode 100644 index 00000000..96e52ac6 --- /dev/null +++ b/recipe/vela/export_native.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 +"""Export the pinned Vela 2.0 0.8B package into a native bundle. + +The bundle re-keys the hybrid +backbone for the shared qwen3_5 native loader — ``backbone.X`` tensors +become ``model.X`` (the loader's prefix probe accepts ``model.``), with +**per-tensor dtypes preserved verbatim** (Linear/conv1d in BF16, +embeddings/norms/A_log/dt_bias in FP32 — the reference's own mixed +layout, no re-save rounding) — and copies the Vela-owned F32 heads and +the side-loaded broad span head as separate files, plus calibration and +tokenizer. ``config.json`` is the package's ``backbone_config`` written +at the root, which is exactly the shape the family ``Config::load`` +reads. The manifest pins every source file by SHA-256 (re-using the +worker's own gate table) and records the exported tensor inventory. + + PYTHONPATH=src .venv/bin/python recipe/vela/export_native.py \ + --package weights/vela-2.0-0.8b --out weights/vela-2.0-0.8b-native +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import shutil +import sys +from pathlib import Path + +BACKBONE_PREFIX = "backbone." +HEAD_PREFIXES = ("head.", "span2.", "set_bias") +FORMAT = "vela2-native-bundle/1" + + +def parse_args(argv=None): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--package", required=True, help="pinned Vela 2.0 0.8B package directory") + parser.add_argument("--out", required=True, help="output bundle directory (created)") + parser.add_argument( + "--bf16-out", + default=None, + help="also write a uniform-BF16 backbone directory for the shared qwen3_5 " + "loader (model.safetensors + config.json)", + ) + return parser.parse_args(argv) + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(8 * 1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def main(argv=None) -> int: + args = parse_args(argv) + sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "src")) + import torch + from safetensors.torch import load_file, save_file + + from models.vela.hybrid import ( + HYBRID_REPO, + HYBRID_REVISION, + PINNED_FILES_SHA256, + verify_package, + ) + + package = Path(args.package) + out = Path(args.out) + verify_package(package) # the worker's own gate: provenance before export + config = json.loads((package / "config.json").read_text(encoding="utf-8")) + backbone_config = config["backbone_config"] + + # The shard map names every backbone and head tensor; broad_head is a + # separate side-loaded file. + index = json.loads((package / "model.safetensors.index.json").read_text(encoding="utf-8")) + weight_map: dict[str, str] = index["weight_map"] + shard_files = sorted(set(weight_map.values())) + tensors = {} + for name in shard_files: + tensors.update(load_file(str(package / name))) + broad = load_file(str(package / "broad_head.safetensors")) + + backbone, heads = {}, {} + dtypes: dict[str, dict[str, str]] = {"backbone": {}, "heads": {}} + for name, tensor in sorted(weight_map.items()): + value = tensors[name] + dtype = str(value.dtype).removeprefix("torch.") + if name.startswith(BACKBONE_PREFIX): + rekeyed = "model." + name[len(BACKBONE_PREFIX):] + backbone[rekeyed] = value + dtypes["backbone"][rekeyed] = f"{dtype}:{list(value.shape)}" + elif name.startswith(HEAD_PREFIXES): + heads[name] = value + dtypes["heads"][name] = f"{dtype}:{list(value.shape)}" + else: + raise SystemExit(f"unexpected tensor key {name!r}; update the export for the new prefix") + for name, value in sorted(broad.items()): + heads[name] = value + dtypes["heads"][name] = f"{str(value.dtype).removeprefix('torch.')}:{list(value.shape)}" + + out.mkdir(parents=True, exist_ok=True) + save_file(backbone, str(out / "backbone.safetensors")) + save_file(heads, str(out / "heads.safetensors")) + # config.json for the family loader: backbone_config at the root. + (out / "config.json").write_text(json.dumps(backbone_config, indent=1) + "\n", encoding="utf-8") + for name in ("calibration.json", "tokenizer.json", "tokenizer_config.json"): + shutil.copyfile(package / name, out / name) + + bf16 = sum(1 for d in dtypes["backbone"].values() if d.startswith("bfloat16")) + fp32 = len(dtypes["backbone"]) - bf16 + manifest = { + "format": FORMAT, + "model_id": "vllm-sr/Vela-2.0-0.8B", + "model_name": config["model_name"], + "max_length": config["max_length"], + "description": config.get("description", ""), + "source": { + "repo": HYBRID_REPO, + "revision": HYBRID_REVISION, + # The worker gate's own table, re-verified above. + "pinned_files_sha256": dict(PINNED_FILES_SHA256), + "shard_files": shard_files, + }, + "backbone": { + "tensors": len(backbone), + "bf16": bf16, + "fp32": fp32, + "note": "per-tensor dtypes preserved from the package; backbone.X re-keyed to model.X", + "inventory": dtypes["backbone"], + }, + "heads": { + "tensors": len(heads), + "inventory": dtypes["heads"], + "note": "Vela-owned F32 heads (CandidateHead, set_bias, span2 router, span_broad) in one file", + }, + "files_sha256": {}, + } + for name in [ + "backbone.safetensors", "heads.safetensors", "config.json", + "calibration.json", "tokenizer.json", "tokenizer_config.json", + ]: + manifest["files_sha256"][name] = sha256(out / name) + (out / "vela_native_manifest.json").write_text( + json.dumps(manifest, indent=1, sort_keys=True) + "\n", encoding="utf-8" + ) + + if args.bf16_out: + # Uniform-BF16 re-save for the family crate's loader: every + # backbone tensor rounds to bfloat16 once, at export (the + # reference's own GPU setting runs the backbone under bf16 + # autocast too). + bf16_dir = Path(args.bf16_out) + bf16_dir.mkdir(parents=True, exist_ok=True) + uniform = {name: value.to(torch.bfloat16) for name, value in backbone.items()} + save_file(uniform, str(bf16_dir / "model.safetensors")) + (bf16_dir / "config.json").write_text(json.dumps(backbone_config, indent=1) + "\n", encoding="utf-8") + print(f"bf16 backbone: {len(uniform)} tensors -> {bf16_dir}") + + print(f"export ok: {out}") + print(f" backbone: {len(backbone)} tensors ({bf16} bf16, {fp32} fp32), re-keyed {BACKBONE_PREFIX} -> model.") + print(f" heads: {len(heads)} tensors (F32, incl. span_broad)") + print(f" config: backbone_config at root, model_type={backbone_config.get('model_type')}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/recipe/vela/export_token_plan.py b/recipe/vela/export_token_plan.py new file mode 100644 index 00000000..39bfb2ea --- /dev/null +++ b/recipe/vela/export_token_plan.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +"""Freeze the token-level row plan of the pinned Vela 2.0 0.8B runtime. + +This tool records, for a set of request +bodies, exactly what the pinned runtime feeds the backbone — the +SystemOne → predict mapping, the per-row decomposition (parts pass + +question blocks; one span question per row), and the rendered records +(``p_ids``, block token ids with option/label end positions, query +positions, span word positions, truncation and segment lengths). The +native Rust processor must reproduce these byte-for-byte; the frozen +files under ``tests/vela/fixtures/native-plan-*.json`` are that parity +target. Model outputs are not recorded here — the oracle fixtures from +``compare_reference.py`` own the output side. + + PYTHONPATH=src .venv/bin/python recipe/vela/export_token_plan.py \ + --model weights/vela-2.0-0.8b --out tests/vela/fixtures/ +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +import numpy as np + +HERE = Path(__file__).resolve().parent +MODEL_NAME = "vllm-sr/Vela-2.0-0.8B" + + +def parse_args(argv=None): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--model", required=True, help="pinned Vela 2.0 0.8B package directory") + parser.add_argument("--out", default=None, help="fixture output directory") + return parser.parse_args(argv) + + +def long_span_body(): + """A span target beyond HYBRID_RMAX (2,048 tokens) to exercise the + windowed layout: the full-render span block is skipped and one row + per 1,800-token window (stride 1,536, cut at word starts) carries + the span question.""" + + filler = " ".join(f"Sentence {i} reports a routine delivery event." for i in range(220)) + return { + "model": MODEL_NAME, + "state": {"request": filler}, + "questions": { + "claims": { + "type": "span", + "instructions": "Which spans mention a delivery?", + "criteria": {"delivery": "a delivery event"}, + "over": "request", + } + }, + } + + +def cases(): + named = [ + ("example-request", HERE / "example-request.json"), + ("example-request-set-threshold", HERE / "example-request-set-threshold.json"), + ("example-request-cjk", HERE / "example-request-cjk.json"), + ("example-request-score", HERE / "example-request-score.json"), + ] + loaded = [ + ( + name, + {**json.loads(path.read_text(encoding="utf-8")), "model": MODEL_NAME}, + ) + for name, path in named + ] + loaded.append(("long-span-windowed", long_span_body())) + return loaded + + +def plain(value): + """numpy-free JSON for the rendered records.""" + + if isinstance(value, dict): + return {key: plain(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [plain(item) for item in value] + if isinstance(value, np.ndarray): + return plain(value.tolist()) + if isinstance(value, (np.integer,)): + return int(value) + if isinstance(value, (np.floating,)): + return float(value) + if isinstance(value, (np.bool_,)): + return bool(value) + return value + + +def rows_of(engine, body): + """Mirror ``Vela2._predict``'s row decomposition and render each row.""" + + from vela2_inference import system_one_to_predict + + req, plan = system_one_to_predict(body["state"], body["questions"]) + parts, qs, _thr = engine._to_generic(req) + spans = [q for q in qs if q["type"] == "span"] + rest = [q for q in qs if q["type"] != "span"] + rows = [dict(parts=parts, questions=rest + spans[:1])] + [ + dict(parts=parts, questions=[s]) for s in spans[1:] + ] + rendered, windows = [], [] + for row in rows: + if not row["questions"]: + continue + prepped = engine._prep(row) + rec = plain(engine._render(prepped)) + rec["head"] = prepped["head"] + rec["head_reason"] = prepped["head_reason"] + rendered.append(rec) + for wrow in engine._window_rows(prepped) or []: + wrec = plain(engine._render(wrow)) + wrec["head"] = prepped["head"] + windows.append(wrec) + return plain(req), plain(plan), rendered, windows + + +def main(argv=None) -> int: + args = parse_args(argv) + sys.path.insert(0, str(Path(args.model).resolve())) + from vela2_inference import Vela2 + + engine = Vela2.from_pretrained(args.model, backend="torch", device="cpu") + out = Path(args.out) if args.out else HERE.parents[1] / "tests" / "vela" / "fixtures" + out.mkdir(parents=True, exist_ok=True) + for name, body in cases(): + req, plan, rows, windows = rows_of(engine, body) + fixture = { + "case": name, + "request": body, + "predict_request": req, + "plan": plan, + "rows": rows, + "window_rows": windows, + } + target = out / f"native-plan-{name}.json" + target.write_text(json.dumps(fixture, ensure_ascii=False) + "\n", encoding="utf-8") + tokens = sum(r["n_tokens"] for r in rows) + sum(r["n_tokens"] for r in windows) + print( + f"{name}: {len(rows)} row(s) + {len(windows)} window row(s), " + f"{tokens} tokens -> {target.name}" + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/recipe/vela/native.md b/recipe/vela/native.md new file mode 100644 index 00000000..23ca355e --- /dev/null +++ b/recipe/vela/native.md @@ -0,0 +1,74 @@ +# Vela 2.0 0.8B native library + +The native implementation for +[vllm-sr/Vela-2.0-0.8B](https://huggingface.co/vllm-sr/Vela-2.0-0.8B): +the exported-bundle gate, the wire contract, and the token-level +processing — all mirrored from the pinned runtime and checked against +frozen fixtures. This is a library; serving runs through the +[reference recipe](reference.md)'s Python worker. + +Run the commands from the repository root with the same `.venv` and +pinned package as the reference recipe. The export needs about 6 GB of +free disk next to the package; the checks need no GPU. + +## Export the native bundle + +~~~sh +PYTHONPATH=src .venv/bin/python recipe/vela/export_native.py \ + --package weights/vela-2.0-0.8b --out weights/vela-2.0-0.8b-native +~~~ + +The export re-runs the reference worker's package gate first +(provenance), then writes `weights/vela-2.0-0.8b-native/`: the backbone +re-keyed `backbone.` → `model.` with **per-tensor dtypes preserved** +(204 BF16 Linear/conv1d, 116 F32 embed/norms/A_log/dt_bias — the +reference's own mixed layout), the 37 F32 heads in `heads.safetensors` +(including the side-loaded broad span head), `backbone_config` at the +config root (the shared qwen3_5 `Config::load` shape), calibration and +tokenizer, and `vela_native_manifest.json` pinning the source +revision, the gate's file table, the tensor inventory and every file's +SHA-256. + +## Freeze the token-plan fixtures (opt-in, needs the package) + +~~~sh +PYTHONPATH=src .venv/bin/python recipe/vela/export_token_plan.py \ + --model weights/vela-2.0-0.8b --out tests/vela/fixtures/ +~~~ + +The token plans record what the pinned runtime feeds the backbone (the +predict mapping, the per-row decomposition, the rendered `p_ids`, +blocks and readout positions, truncation, a windowed long-span case). +The example-request token plan totals 1,041 tokens, matching the +oracle fixture's `usage.input_tokens`. + +## Checks + +~~~sh +cargo test -p omni-vela-native +cargo fmt --all --check +cargo clippy -p omni-vela-native --all-targets -- -D warnings +VELA_NATIVE_BUNDLE="$PWD/weights/vela-2.0-0.8b-native" \ + cargo test -p omni-vela-native --test vela_native_processing -- --ignored --nocapture +~~~ + +27 tests run everywhere with no weights and no GPU: the wire contract +against the pinned runtime's 422/400 bodies, the bundle manifest gate, +and the predict-mapping/plan parity on every frozen fixture. The +`#[ignore]`d render check (run it with `-- --ignored`, as above) +replays the five token plans through the Rust processing +field-for-field. For a field-level diff of one case: + +~~~sh +VELA_NATIVE_BUNDLE="$PWD/weights/vela-2.0-0.8b-native" \ + cargo run -p omni-vela-native --example parity -- example-request +~~~ + +## Scope and limits + +- Implemented: the bundle export + manifest gate, the wire admission, + and the token-level processing (state mapping, predict mapping with + the validation triples, word units, span-head dispatch, the DEV2.0 + rendering, the truncation budget, window rows). +- Not implemented: the F32 heads, scoring and span decoding, backbone + execution, and an HTTP worker. No performance claims are made. diff --git a/recipe/vela/reference.md b/recipe/vela/reference.md index 98d30f9c..559a7afb 100644 --- a/recipe/vela/reference.md +++ b/recipe/vela/reference.md @@ -1,9 +1,13 @@ -# Vela 2.0 0.3B reference worker +# Vela 2.0 reference workers (0.3B, 0.8B hybrid) -This recipe serves Vela 2.0 0.3B through the Python reference worker and -the existing Rust frontend. The model-side implementation is under -[`src/models/vela/`](../../src/models/vela/), and the worker is -[`src/frontend/vela.py`](../../src/frontend/vela.py). +This recipe serves the Vela 2.0 members through the Python reference +worker and the existing Rust frontend: the 0.3B encoder and the 0.8B +hybrid, one pinned member per worker process. The model-side +implementation is under +[`src/models/vela/`](../../src/models/vela/) (0.3B in `engine.py`, the +0.8B hybrid in `hybrid.py`), and the worker is +[`src/frontend/vela.py`](../../src/frontend/vela.py), which picks the +executor from the package's declared `model_type`. Run the commands from the repository root with Python 3.12. The default and evidence-backed setting is CPU FP32 — the reference oracle; a CUDA @@ -14,33 +18,34 @@ description, not here. ## Scope -The slice is: +Scope: -- package: [`vllm-sr/Vela-2.0-0.3B`](https://huggingface.co/vllm-sr/Vela-2.0-0.3B) - at the pinned revision below; -- the Python reference worker behind the Rust frontend, executing the +- packages: [`vllm-sr/Vela-2.0-0.3B`](https://huggingface.co/vllm-sr/Vela-2.0-0.3B) + and [`vllm-sr/Vela-2.0-0.8B`](https://huggingface.co/vllm-sr/Vela-2.0-0.8B) + at the pinned revisions below; +- the Python reference worker behind the Rust frontend, executing each checkpoint's own pinned runtime (`vela2_inference.py` and siblings) after a full-hash package gate; - text/JSON state and all five typed kinds — `choice`, `noul`, `score`, `span`, `set` — in one request, with the span/set response extensions - (`spans`/`sets`/`thresholds` and their Noul views) served - forward-untouched; -- FP32 parameters, one bidirectional forward per assembled sequence, + (`spans`/`sets`/`thresholds`, `span_heads` on the hybrid, and their + Noul views) served forward-untouched; +- FP32 parameters, one bidirectional forward per assembled sequence + (0.3B) or one parts pass with per-question continuation (hybrid), calibrated readouts; zero generated tokens. -Out of scope for this slice: the hybrid members (0.8B/4B/9B — their -reference workers are a separate slice), the ONNX backend, any native -Rust/CUDA executor (the frozen RFC's ModernBERT-profile route), and -`noul_calibration` (a hybrid-only feature that does not exist on 0.3B). +Out of scope: the larger hybrid members (4B/9B), the ONNX backend of +the 0.3B, any native Rust/CUDA executor, and `noul_calibration` (off +by default; documented as a reference-server feature only). ## Architecture and ownership ~~~text Rust frontend - -> Python worker + -> Python worker (executor chosen by config.json model_type) -> admit: JSON transport + upstream envelope preamble (protocol) - -> verify: config identity pin, full SHA-256 of the 7-file table, - exact weights size, then import it (engine) + -> verify: config identity pin, full SHA-256 of the pinned table, + exact weights size, then import it (engine / hybrid) -> execute: the checkpoint's own vela2_inference system_one_request -> finish: response-order/key-set checks + formula mirrors (postprocess) ~~~ @@ -53,28 +58,37 @@ The ownership boundary follows triples; deeper validation (state parts, question semantics, the 413) is the pinned runtime's, verbatim; - `preprocess.py` maps the runtime's `SystemOneError` to 422/413; -- `engine.py` owns the pinned runtime and the package gate; -- `postprocess.py` owns the pure formulas and response finishing; -- `frontend.vela` owns HTTP lifecycle, the single-flight 503 lock, the - `VELA2_API_KEY` 401 gate and warmup-before-readiness; +- `engine.py` owns the 0.3B pinned runtime and its package gate; + `hybrid.py` mirrors it for the 0.8B (a 10-file pin table including + the shard index and the side-loaded `broad_head.safetensors`); +- `postprocess.py` owns the pure formulas and response finishing, + member-aware for the hybrid's `span_heads` field; +- `frontend.vela` owns HTTP lifecycle, member detection, the + single-flight 503 lock, the `VELA2_API_KEY` 401 gate and + warmup-before-readiness; - The Rust frontend forwards `/v1/systemone` and `/health` unchanged. ## Pinned artifacts -| Artifact | Value | -| --- | --- | -| Package | [vllm-sr/Vela-2.0-0.3B](https://huggingface.co/vllm-sr/Vela-2.0-0.3B) | -| Pinned revision | `d6f03aa9baca4f9017fa8f8e9c2764d3bed3f409` | -| Package gate | config structural identity + a 7-file SHA-256 table pinned per revision in `engine.py`, full hash before import, no unlisted importables, `__pycache__` or directory symlinks in the package root, exact 1,236,473,252-byte weights check | -| License | Apache-2.0; tokenizer material under Gemma Terms of Use | +| Artifact | 0.3B | 0.8B hybrid | +| --- | --- | --- | +| Package | [vllm-sr/Vela-2.0-0.3B](https://huggingface.co/vllm-sr/Vela-2.0-0.3B) | [vllm-sr/Vela-2.0-0.8B](https://huggingface.co/vllm-sr/Vela-2.0-0.8B) | +| Pinned revision | `d6f03aa9baca4f9017fa8f8e9c2764d3bed3f409` | `326b01d81f61fff7fd4ae570e54b87a49119e490` | +| Package gate | config structural identity + a 7-file SHA-256 table in `engine.py`; no unlisted importables, `__pycache__` or directory symlinks in the package root | config structural identity (incl. `model_name`) + a 10-file SHA-256 table in `hybrid.py`, cross-checked against the package's own SHA256SUMS | +| Weights check | exact 1,236,473,252 bytes | exact 2,022,189,488 bytes + hashed `broad_head.safetensors` | +| Served name | `vllm-sr/Vela-2.0-0.3B` | `vllm-sr/Vela-2.0-0.8B` (from `config.json`) | +| Input window | 8,192 tokens | 16,384 tokens | +| License | Apache-2.0; tokenizer material under Gemma Terms of Use | Apache-2.0; Qwen3.5-0.8B lineage notices in the package | -The gate's file table is checked in (not read from the package, which -ships no manifest), so a later snapshot — self-consistent or not — -cannot pass as the pinned revision. Re-check before changing the pin: +Each gate's file table is checked in (never read from the package), so +a later snapshot — self-consistent or not — cannot pass as the pinned +revision. Re-check before changing a pin: ~~~sh curl -sS "https://huggingface.co/api/models/vllm-sr/Vela-2.0-0.3B/revision/d6f03aa9baca4f9017fa8f8e9c2764d3bed3f409" | jq -r '.sha' sha256sum weights/vela-2.0-0.3b/{vela2_inference.py,modeling_vela2.py,configuration_vela2.py,vela2_serve.py,config.json,calibration.json,tokenizer.json} +curl -sS "https://huggingface.co/api/models/vllm-sr/Vela-2.0-0.8B/revision/326b01d81f61fff7fd4ae570e54b87a49119e490" | jq -r '.sha' +sha256sum weights/vela-2.0-0.8b/{vela2_inference.py,modeling_vela2.py,configuration_vela2.py,vela2_serve.py,config.json,calibration.json,tokenizer.json,tokenizer_config.json,model.safetensors.index.json,broad_head.safetensors} ~~~ ## Install and prepare the artifacts @@ -88,15 +102,21 @@ python3.12 -m venv .venv ~~~sh export VELA_PKG="$PWD/weights/vela-2.0-0.3b" export VELA_REV=d6f03aa9baca4f9017fa8f8e9c2764d3bed3f409 +export VELA_HYBRID_PKG="$PWD/weights/vela-2.0-0.8b" +export VELA_HYBRID_REV=326b01d81f61fff7fd4ae570e54b87a49119e490 mkdir -p weights .venv/bin/hf download vllm-sr/Vela-2.0-0.3B --revision "$VELA_REV" --local-dir "$VELA_PKG" +.venv/bin/hf download vllm-sr/Vela-2.0-0.8B --revision "$VELA_HYBRID_REV" --local-dir "$VELA_HYBRID_PKG" ~~~ -The full package is about 4.4 GB (the 1.24 GB FP32 safetensors, a 34 MB +The 0.3B package is about 4.4 GB (the 1.24 GB FP32 safetensors, a 34 MB tokenizer, the ONNX exports and doc assets); serving needs roughly 2 GB of RAM for the FP32 model plus working memory for the batch rows. -The worker refuses to start when the config is not the pinned member's, -when any of the 7 gated files fails its hash, when the package root +The 0.8B package is about 2.1 GB (a 2.02 GB shard plus the 4.5 MB +broad-head file); FP32 parameters take about 3 GB of RAM on CPU (about +3 GB of GPU memory on CUDA, where the backbone runs under bf16 +autocast). A worker refuses to start when the config is not its pinned +member's, when any gated file fails its hash, when the package root holds an importable entry the table does not name, an existing `__pycache__` or a directory symlink, or when the weights file is not the pinned size — that @@ -106,12 +126,15 @@ gate runs before any package code is imported. ~~~sh PYTHONPATH=src .venv/bin/python -m frontend.vela \ - --model "$VELA_PKG" --device cpu --port 8000 + --model "$VELA_PKG" --device cpu --port 8000 # 0.3B +PYTHONPATH=src .venv/bin/python -m frontend.vela \ + --model "$VELA_HYBRID_PKG" --device cpu --port 8001 # 0.8B hybrid ~~~ -The worker verifies the package, loads the FP32 model, runs a real -warmup request covering every question kind, and only then binds the -port: +The worker detects the member from the package's `model_type`, verifies +the package, loads the model, runs a real warmup request covering every +question kind (the hybrid's warmup also drives both span heads), and +only then binds the port: ~~~sh curl -sS -i http://127.0.0.1:8000/health # {"status": "ok"} (upstream verbatim) @@ -199,12 +222,13 @@ the [model contract](../../src/models/vela/README.md). In short: transport errors follow the shared worker discipline; every invalid field rejects the whole request with the upstream 422 `{"detail": [{"loc", "msg", "type"}]}`; 413 `too_long` fires only when -questions plus schema alone exceed 8,192 tokens; and over-limit state -text is **truncated as the reference truncates it** (shrink part first, -floor 64 then 1, then protected parts; windowed re-coverage of cut -labelled parts) — a deliberate, loudly documented divergence from the -repository's no-silent-truncation convention, with the guardrail risk -recorded in the model contract. +questions plus schema alone exceed the member's input window (8,192 on +0.3B, 16,384 on the hybrid); and over-limit state text is **truncated +as the reference truncates it** (shrink part first, floor 64 then 1, +then protected parts; windowed re-coverage of cut labelled parts) — a +deliberate, loudly documented divergence from the repository's +no-silent-truncation convention, with the guardrail risk recorded in +the model contract. ## Repository-side validation @@ -229,26 +253,32 @@ invalid-span 422, an over-limit 413, and the missing-field / extra-field / wrong-model envelope 422s) — through two paths that share one loaded engine: the pinned `vela2_serve.py` app via FastAPI's test client, and the production -`frontend.vela.decide` pipeline. The criterion is the charter's -reference parity: **exact match** including JSON key order for 200s, +`frontend.vela.decide` pipeline. The criterion is reference parity: +**exact match** including JSON key order for 200s, and agreement on status and body for rejections (needs `pip install fastapi uvicorn httpx` on top of the requirements file). `--export tests/vela/fixtures/oracle-.json` freezes each passing 200 case — request, SystemOne response and predict()-level answers — as -the frozen comparison baseline for the frozen RFC's native encoder -slices, and writes it only after a passing comparison. +the frozen comparison baseline, and writes it only after a passing +comparison. ~~~sh PYTHONPATH=src .venv/bin/python recipe/vela/compare_reference.py \ --model "$VELA_PKG" --device cpu \ --export tests/vela/fixtures/ +PYTHONPATH=src .venv/bin/python recipe/vela/compare_reference.py \ + --model "$VELA_HYBRID_PKG" --device cpu \ + --export tests/vela/fixtures/ ~~~ Run it by hand and record the verbatim output, host, package revision and dependency versions in the pull request description. CPU FP32 is the oracle setting; keep flash-linear-attention out of the fixture path -(it does not exist on this encoder member anyway) and do not substitute -GPU bf16-autocast numbers for the baseline. +(it changes the hybrid's numbers and its parity against the fallback +path is not documented upstream) and do not substitute GPU +bf16-autocast numbers for the baseline. The tool rewrites the example +files' `model` field to the served member and exports the hybrid's +oracle fixtures under an `oracle-hybrid-` prefix. ## Troubleshooting and limits @@ -257,15 +287,15 @@ GPU bf16-autocast numbers for the baseline. `__pycache__` bytecode caches, directory symlinks and wrong weights sizes at startup, before importing anything. - The 8 MiB body cap is slice-scoped (upstream sets no total-body - limit); a valid request bounded by the 8,192-token window is far - below it. -- `/health` and `/ready` are upstream's minimal bodies on this member, - not the richer shape of some other workers in this repository. + limit); a valid request bounded by either member's input window is + far below it. +- `/health` and `/ready` are upstream's minimal bodies on both + members, not the richer shape of some other workers in this + repository. - Long states are truncated (see Error behavior); if a consumer must never miss tail text, check `usage.input_tokens` against the expected budget or enforce limits upstream of the worker. - GPU bf16-autocast serving (`--device cuda`) is untested in this - slice's evidence; treat it as a deployment option to validate - separately. + evidence; treat it as a deployment option to validate separately. - GPU evidence for the supported-models table is recorded in the integration pull request, not in this recipe. diff --git a/recipe/vela/test_compare_reference.py b/recipe/vela/test_compare_reference.py index d97aaa45..f76fb69d 100644 --- a/recipe/vela/test_compare_reference.py +++ b/recipe/vela/test_compare_reference.py @@ -86,7 +86,7 @@ def check(name: str, condition: bool, detail: str = "") -> None: lines, ok = tool.compare_case("fixture", (200, response()), (422, rejection)) check("worker answering a rejected request fails", not ok, "\n".join(lines)) - # 3. The default case set carries the charter's fixture list shape. + # 3. The default case set carries the fixture list's shape. cases = tool.default_cases() names = [name for name, _ in cases] check( diff --git a/src/frontend/vela.py b/src/frontend/vela.py index c863a018..78f46631 100644 --- a/src/frontend/vela.py +++ b/src/frontend/vela.py @@ -1,10 +1,13 @@ -"""Python reference worker for Vela 2.0 0.3B. +"""Python reference worker for the Vela 2.0 members (0.3B, 0.8B hybrid). The Rust frontend remains the public transport layer. This worker owns only HTTP lifecycle and request orchestration; wire admission lives in models.vela.protocol, the pinned-runtime error mapping in -models.vela.preprocess, the pinned runtime and its package gate in -models.vela.engine, and response finishing in models.vela.postprocess. +models.vela.preprocess, the pinned runtimes and their package gates in +models.vela.engine (0.3B) and models.vela.hybrid (0.8B), and response +finishing in models.vela.postprocess. The executor is chosen by the +package's declared ``model_type``; one worker process serves one pinned +member. The error surface has two layers: transport errors follow the shared Python-worker discipline, and every request rejection the pinned runtime @@ -13,7 +16,8 @@ (or 413 ``too_long``), passed through verbatim. There are no per-question errors. Over-limit state text is truncated exactly as the pinned reference truncates it (see the model contract); only questions plus -schema that cannot fit the 8,192-token window are refused. +schema that cannot fit the member's input window (8,192 or 16,384 +tokens) are refused. """ from __future__ import annotations @@ -26,17 +30,21 @@ import socket import threading from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path from typing import Any from models.vela.engine import Vela2Executor +from models.vela.hybrid import EXPECTED_MODEL_NAME as HYBRID_MODEL_NAME, Vela2HybridExecutor from models.vela.postprocess import finish_response from models.vela.preprocess import map_runtime_error, to_request_error from models.vela.protocol import MODEL_NAME, MAX_BODY, RequestError, parse_body, admit_request LOG = logging.getLogger(__name__) -# Upstream vela2_serve.py /v1/models entry, kept verbatim so clients see -# the reference server's description and release date. +# Upstream vela2_serve.py /v1/models entry for the 0.3B member, kept +# verbatim so clients see the reference server's description and release +# date; the hybrid members source their entry from config.json, exactly +# as their vela2_serve.py does. MODELS_ENTRY = { "name": None, # the served model name, filled per request "release_date": "2026-09-29", @@ -48,7 +56,57 @@ UNAUTHORIZED_BODY = {"detail": "Missing or invalid API key"} -def decide(raw: bytes, executor: Vela2Executor) -> dict[str, Any]: +def detect_member(package: str | Path) -> tuple[type, str]: + """Pick the pinned executor for a package's declared member. + + Returns the executor class and the member's default served name; the + structural gate inside the executor is the real authority — this + only routes the right gate to the package. + """ + + config_path = Path(package) / "config.json" + if not config_path.is_file(): + raise ValueError(f"the package is missing config.json: {package}") + try: + config = json.loads(config_path.read_text(encoding="utf-8")) + except (OSError, ValueError) as exc: + raise ValueError(f"config.json in {package} is not readable JSON") from exc + if not isinstance(config, dict): + raise ValueError(f"config.json in {package} is not a JSON object") + model_type = config.get("model_type") + if model_type == "vela2-unified": + return Vela2Executor, MODEL_NAME + if model_type == "vela2-decoder": + return Vela2HybridExecutor, HYBRID_MODEL_NAME + raise ValueError( + f"config.json declares model_type {model_type!r}; this worker serves the " + "vela2-unified (0.3B) and vela2-decoder (0.8B) members of Vela 2.0" + ) + + +def models_entry(executor) -> dict[str, Any]: + """The /v1/models entry, verbatim per member's reference server.""" + + if getattr(executor, "is_hybrid", False): + # Upstream hybrid vela2_serve.py: name/description/release_date + # straight from config.json. + config = getattr(executor, "engine", None) + config = getattr(config, "config", None) if config is not None else None + config = config or {} + return { + "name": executor.model_name, + "release_date": config.get("release_date", ""), + "description": config.get( + "description", + "Vela 2.0: SystemOne Choice / Noul / Score with span and set extensions", + ), + } + entry = dict(MODELS_ENTRY) + entry["name"] = executor.model_name + return entry + + +def decide(raw: bytes, executor) -> dict[str, Any]: """Admit, execute, and finish one complete request.""" body = admit_request(parse_body(raw), executor.served) @@ -62,13 +120,13 @@ def decide(raw: bytes, executor: Vela2Executor) -> dict[str, Any]: # layer cannot foresee (state rendering, question semantics, the # window budget); its rejections are client errors, not failures. raise map_runtime_error(exc) from exc - return finish_response(questions, response) + return finish_response(questions, response, hybrid=getattr(executor, "is_hybrid", False)) class WorkerServer(ThreadingHTTPServer): daemon_threads = False - def __init__(self, address, executor: Vela2Executor, api_key: str | None = None): + def __init__(self, address, executor: Vela2Executor | Vela2HybridExecutor, api_key: str | None = None): self.executor = executor self.api_key = api_key self.inference_lock = threading.Lock() @@ -139,9 +197,7 @@ def do_GET(self): elif path == "/v1/models": if self._unauthorized(): return - entry = dict(MODELS_ENTRY) - entry["name"] = self.server.executor.model_name - self.send_json(200, {"models": [entry]}) + self.send_json(200, {"models": [models_entry(self.server.executor)]}) else: self.send_json(404, {"detail": "unknown route"}) @@ -190,12 +246,16 @@ def do_POST(self): def parse_args(argv=None): - parser = argparse.ArgumentParser(description="Python reference worker for Vela 2.0 0.3B") - parser.add_argument("--model", required=True, help="pinned Vela 2.0 0.3B package directory") - parser.add_argument("--device", default="cpu", help="torch device (reference contract: cpu)") + parser = argparse.ArgumentParser(description="Python reference worker for Vela 2.0 members") + parser.add_argument("--model", required=True, + help="pinned Vela 2.0 package directory (0.3B or the 0.8B hybrid)") + parser.add_argument("--device", default="cpu", + help="torch device (reference contract: cpu; cuda runs the hybrid " + "backbone under bf16 autocast with FP32 heads)") parser.add_argument("--batch-size", type=int, default=None) - parser.add_argument("--name", default=os.environ.get("VELA2_MODEL_NAME", MODEL_NAME), - help="model name accepted in the request's model field") + parser.add_argument("--name", default=os.environ.get("VELA2_MODEL_NAME"), + help="model name accepted in the request's model field " + "(default: the pinned member's served name)") parser.add_argument("--host", default="127.0.0.1") parser.add_argument("--port", type=int, default=8000) return parser.parse_args(argv) @@ -204,10 +264,11 @@ def parse_args(argv=None): def main(argv=None): args = parse_args(argv) logging.basicConfig(level=logging.INFO) - executor = Vela2Executor( + executor_class, default_name = detect_member(args.model) + executor = executor_class( args.model, device=args.device, - model_name=args.name, + model_name=args.name or default_name, batch_size=args.batch_size, ) server = None diff --git a/src/models/vela/README.md b/src/models/vela/README.md index 8c6c65f6..9d2c8ab9 100644 --- a/src/models/vela/README.md +++ b/src/models/vela/README.md @@ -1,48 +1,64 @@ -# Vela 2.0 0.3B reference contract +# Vela 2.0 reference contracts (0.3B, 0.8B hybrid) -This directory contains the System1-Omni integration slice for +This directory contains the System1-Omni integration slices for [Vela 2.0 0.3B](https://huggingface.co/vllm-sr/Vela-2.0-0.3B) -(`vela2-unified`). It is a reference-serving boundary only: a Python -worker that runs the checkpoint's own pinned runtime for text state and -all five typed question kinds (`choice`, `noul`, `score`, `span`, `set`) -on `/v1/systemone`. Native execution is not part of this slice; per the -frozen RFC this worker is the Phase-1 entry that freezes the -text/token_ids input, the position-level final-hidden readout contract, -the error/length boundaries and the parity baseline for later -ModernBERT-profile work. +(`vela2-unified`) and +[Vela 2.0 0.8B](https://huggingface.co/vllm-sr/Vela-2.0-0.8B) +(`vela2-decoder`, the first hybrid member): a Python worker that runs +the checkpoint's own pinned runtime for text state and all five typed +question kinds (`choice`, `noul`, `score`, `span`, `set`) on +`/v1/systemone`, one pinned member per worker process. The 0.3B +worker's contract freezes the text/token_ids input, the +position-level final-hidden readout contract, the error/length +boundaries and the parity baseline. The shared Vela 2.0 wire module is deliberately shaped for the family: the envelope admission (`protocol`), the pinned-runtime error mapping (`preprocess`) and response finishing (`postprocess`) are member-neutral; -this slice ships the 0.3B pinned executor (`engine`). The hybrid members -(0.8B/4B/9B) reuse the same wire layer with their own executors. +this directory ships the 0.3B pinned executor (`engine`) and the 0.8B +hybrid executor (`hybrid`). The worker picks the executor from the +package's declared `model_type`. The 4B/9B members are not +implemented. ## Pinned artifacts -| Artifact | Value | -| --- | --- | -| Package | [vllm-sr/Vela-2.0-0.3B](https://huggingface.co/vllm-sr/Vela-2.0-0.3B) | -| Pinned revision | `d6f03aa9baca4f9017fa8f8e9c2764d3bed3f409` | -| Identity | `vela2-unified` / `Vela2Model`, 309,114,369 parameters, all FP32 (1,236,473,252-byte safetensors) | -| Runtime | the checkpoint's own flat files: `vela2_inference.py`, `modeling_vela2.py`, `configuration_vela2.py`, `vela2_serve.py` | -| Calibration | the package's own `calibration.json` (never cross-copied between members) | -| License | Apache-2.0; the inherited tokenizer material carries Gemma Terms of Use | -| Runtime dependencies | [recipe/vela/requirements.txt](../../recipe/vela/requirements.txt) (torch, transformers, safetensors, tokenizers, numpy) | - -The package ships no manifest, so the executor's startup gate checks the -config's structural identity (model type, architecture, `max_length` -8,192, the 8 marker ids), then hashes every small file the runtime reads -or executes — the four runtime modules, `config.json`, -`calibration.json` and `tokenizer.json` — against the table checked in -for the pinned revision, before anything is imported, and refuses -importable entries the table does not name plus any existing -`__pycache__` bytecode cache or directory symlink (`model.safetensors` is -checked by exact size; the runtime's strict safetensors load rejects -truncated or structurally mismatched weights, but not same-size -bit-level corruption). +| Artifact | 0.3B | 0.8B hybrid | +| --- | --- | --- | +| Package | [vllm-sr/Vela-2.0-0.3B](https://huggingface.co/vllm-sr/Vela-2.0-0.3B) | [vllm-sr/Vela-2.0-0.8B](https://huggingface.co/vllm-sr/Vela-2.0-0.8B) | +| Pinned revision | `d6f03aa9baca4f9017fa8f8e9c2764d3bed3f409` | `326b01d81f61fff7fd4ae570e54b87a49119e490` | +| Identity | `vela2-unified` / `Vela2Model`, 309,114,369 parameters, all FP32 (1,236,473,252-byte safetensors) | `vela2-decoder` / `Vela2DecoderModel`, 754,565,954 parameters (2,022,189,488-byte shard + hashed 4,480,172-byte `broad_head.safetensors`) | +| Input window | 8,192 tokens | 16,384 tokens | +| Runtime | the checkpoint's own flat files: `vela2_inference.py`, `modeling_vela2.py`, `configuration_vela2.py`, `vela2_serve.py` | the same four modules, decoder-specific | +| Calibration | the package's own `calibration.json` (never cross-copied between members — shipped thresholds differ) | same rule; the side-loaded broad head needs its `broad_head` block | +| License | Apache-2.0; the inherited tokenizer material carries Gemma Terms of Use | Apache-2.0; Qwen3.5-0.8B lineage notices ship in the package | +| Runtime dependencies | [recipe/vela/requirements.txt](../../recipe/vela/requirements.txt) (torch, transformers, safetensors, tokenizers, numpy) | same | + +The 0.3B package ships no manifest, so its gate checks the config's +structural identity (model type, architecture, `max_length` 8,192, the +8 marker ids), then hashes every small file the runtime reads or +executes — the four runtime modules, `config.json`, `calibration.json` +and `tokenizer.json` — against the table checked in for the pinned +revision, before anything is imported, and refuses importable entries +the table does not name plus any existing `__pycache__` bytecode cache +or directory symlink (`model.safetensors` is checked by exact size; the +runtime's strict safetensors load rejects truncated or structurally +mismatched weights, but not same-size bit-level corruption). The 0.8B +gate mirrors this with a 10-file table — the same four runtime modules, +`config.json` (identity includes the served `model_name`, so a 4B or 9B +package fails with its own name), `calibration.json`, `tokenizer.json`, +`tokenizer_config.json`, `model.safetensors.index.json` (the shard map) +and the hashed `broad_head.safetensors` — with the 2 GB shard checked +by exact size. A later self-consistent snapshot rewrites its own files and fails the checked-in pin. +On either member the runtime loads parameters in FP32; on `cuda` the +hybrid runs its backbone under bf16 autocast with FP32 heads (the +evaluated setting), and on `cpu` it runs FP32 throughout. The hybrid's +served name comes from `config.json` (`vllm-sr/Vela-2.0-0.8B`); the +runtime's family-generic `DEFAULT_MODEL_NAME` is never used as a +default. + ## Current serving scope - The request envelope is upstream's three-field form, exactly: @@ -103,8 +119,14 @@ The response envelope is the pinned runtime's, passed through verbatim answers echo the value without a rule field). Serving-time temperatures divide the logits per question type; the learned `tau`/`tau_span` live inside the model. - The hybrid-only fields (`span_heads`, `noul_calibration`) do not - exist on this member. + The hybrid-only `noul_calibration` block is default-off on every + member (env `VELA2_NOUL_CALIBRATION` or the reference server's flag + enable it); the worker serves the default path only. On the 0.3B the + `span_heads` field does not exist; on the hybrid it is emitted + together with `spans`, keyed by the span question ids + (`{"router"|"broad"}` per question, after the runtime's six-rule + dispatch: the `"head"` override, trained ids, exact router label + sets, PII subsets, hallucination aliases, else broad). - `usage.input_tokens` is the sum over every rendered sequence — span window rows and extra span-question rows included, nothing deduplicated; `usage.output_tokens` is always 0. @@ -128,7 +150,8 @@ per-question errors on this path: upstream server would answer 500); no such path is reachable on the pinned runtime's request path. 3. HTTP 413 (`too_long`) only when the questions plus schema alone - exceed the 8,192-token window. + exceed the member's input window (8,192 on 0.3B, 16,384 on the + hybrid). 4. HTTP 401 (`{"detail": "Missing or invalid API key"}`) only when `VELA2_API_KEY` is set and the request does not carry exactly `Authorization: Bearer ` (applies to `/v1/systemone` and @@ -166,14 +189,17 @@ every question kind), so answering at all means ready. - `preprocess.py` owns the mapping of the pinned runtime's `SystemOneError` to HTTP (422, or 413 for `too_long`) and of its other ValueErrors to client errors. -- `engine.py` owns the pinned runtime, the pre-import package gate, the - served name, device state and the all-kind warmup. +- `engine.py` owns the 0.3B pinned runtime, the pre-import package gate, the + served name, device state and the all-kind warmup; `hybrid.py` mirrors + it for the 0.8B (its warmup also drives both span heads). - `postprocess.py` owns the pure confidence/score formulas (mirrored for the CPU tests) and response finishing: answer-order and key-set - reconstruction including set views, usage accounting, and refusal of - hybrid-only fields on this member. -- `src/frontend/vela.py` owns HTTP lifecycle, the single-flight lock, - the API-key gate and orchestration. + reconstruction including set views, usage accounting, the + member-aware `span_heads` check, and refusal of hybrid-only fields + on the 0.3B. +- `src/frontend/vela.py` owns HTTP lifecycle, member detection from the + package's `model_type`, the per-member `/v1/models` entry, the + single-flight lock, the API-key gate and orchestration. - The Rust frontend owns transport and forwarding, not model state. The server-side error surface and envelope semantics are recorded as @@ -184,16 +210,55 @@ the shared worker-contract issue, and nothing is posted upstream. - CPU contract tests, no weights, no torch: `PYTHONPATH=src python -m pytest tests/vela -q` (envelope triples, - transport discipline, package gate, formula mirrors, response - finishing, the HTTP surface with a contract fake). + transport discipline, both package gates, formula mirrors, response + finishing including the hybrid `span_heads` surface, member + detection, the HTTP surface with a contract fake). - Reference parity is exact match on recorded fixtures under the pinned versions: `recipe/vela/compare_reference.py` drives the recipe fixtures plus five synthetic rejections (invalid-span 422, over-limit 413, and the missing-field / extra-field / wrong-model envelope 422s) through both the pinned `vela2_serve.py` app and the - production worker pipeline over one shared engine, and freezes the - oracle fixture for the native encoder slices. CPU FP32 (no + production worker pipeline over one shared engine, for either member, + and freezes the oracle fixture for the native slices (`oracle-` for + 0.3B, `oracle-hybrid-` for 0.8B). CPU FP32 (no flash-linear-attention) is the oracle setting; GPU bf16-autocast is - a deployment option outside this slice's evidence. + a deployment option outside these slices' evidence. - The serving procedure, hardware requirements and the recorded evidence pointer are [recipe/vela/reference.md](../../recipe/vela/reference.md). + +## Native path (0.8B) + +The native implementation lives in [`native/`](native/) as the +`omni-vela-native` crate, a library around the exported bundle: + +- [`recipe/vela/export_native.py`](../../recipe/vela/export_native.py) + exports the pinned package into a native bundle: the 320 backbone + tensors re-keyed `backbone.` → `model.` with **per-tensor dtypes + preserved** (204 BF16 Linear/conv1d + 116 F32 embed/norms/A_log/ + dt_bias — the reference's own layout), the 37 F32 head tensors + (CandidateHead, `set_bias`, span2 router, span_broad) in one file, + `backbone_config` at the config root, calibration and tokenizer + copied, and a manifest pinning the source revision, the worker + gate's own file table, the tensor inventory and every bundle file's + SHA-256. +- `native/src/bundle.rs` re-verifies that manifest (revision, + inventory counts, per-file hashes) before anything loads it. +- `native/src/contract.rs` is the wire admission, mirroring + `protocol.py` and the pinned runtime's preamble with byte-equal + 422/400 bodies. +- `native/src/{state,questions,words,render,processing}.rs` are the + token-level processing: the state → parts mapping, the SystemOne → + predict mapping with the pinned runtime's validation triples (Noul + folded into a two-option choice; choice renders `[ABS]`, the + reference's abstain default), the GLiNER2/CJK word splitter, the + six-rule span-head dispatch with the hallucination-alias label + replacement, the DEV2.0 prompt rendering (parts pass, question + blocks with option end and query positions, span layout a/c with + word positions, the truncation budget, the 16,384-window 413) and + the long-target window rows. All five frozen token plans + (`tests/vela/fixtures/native-plan-*.json`) match field-for-field. + +The F32 heads, scoring, backbone execution and an HTTP worker are not +implemented. Checks: `cargo test -p omni-vela-native` (26 tests, no +weights, no GPU; render parity opt-in with `VELA_NATIVE_BUNDLE`; see +[recipe/vela/native.md](../../recipe/vela/native.md)). diff --git a/src/models/vela/engine.py b/src/models/vela/engine.py index 60e27d1c..ac2f316a 100644 --- a/src/models/vela/engine.py +++ b/src/models/vela/engine.py @@ -126,7 +126,7 @@ def verify_config(root: Path) -> dict[str, Any]: return config -def verify_files(root: Path, table: dict[str, str] | None = None) -> list[str]: +def verify_files(root: Path, table: dict[str, str] | None = None, revision: str = REFERENCE_REVISION) -> list[str]: """Hash every pinned file in full and return their names. Must run before any package code is imported: the table names the @@ -138,7 +138,9 @@ def verify_files(root: Path, table: dict[str, str] | None = None) -> list[str]: ``__init__`` in every module form, or a namespace package's nested modules — would shadow one of the pinned runtime's dependencies — and any ``__pycache__`` or directory symlink, whose contents the - walk cannot verify against the pin. + walk cannot verify against the pin. ``revision`` names the pin the + table belongs to in hash-failure messages (the hybrid member passes + its own). """ table = dict(table or PINNED_FILES_SHA256) @@ -151,7 +153,7 @@ def verify_files(root: Path, table: dict[str, str] | None = None) -> list[str]: problems.append(f"missing pinned file {name}") continue if _sha256_safe(target) != digest: - problems.append(f"{name}: SHA-256 differs from the pin for {REFERENCE_REVISION}") + problems.append(f"{name}: SHA-256 differs from the pin for {revision}") for entry in sorted(Path(root).iterdir()): if entry.name in table: continue @@ -239,6 +241,10 @@ def verify_package(root: Path, table: dict[str, str] | None = None) -> None: class Vela2Executor: """Load and drive one immutable pinned Vela 2.0 0.3B runtime instance.""" + # Response finishing uses this to refuse the hybrid-only ``span_heads`` + # field (hybrid.py's executor sets it True). + is_hybrid = False + def __init__( self, package: str | Path, diff --git a/src/models/vela/hybrid.py b/src/models/vela/hybrid.py new file mode 100644 index 00000000..e02fa6fb --- /dev/null +++ b/src/models/vela/hybrid.py @@ -0,0 +1,241 @@ +"""Pinned executor for Vela 2.0 0.8B (``vela2-decoder``), the first hybrid member. + +Mirrors :mod:`models.vela.engine` for the compact hybrid member. The +``vela2-decoder`` package ships the same flat runtime modules as the +0.3B (with decoder-specific contents), so the gate reuses the same +helpers and the same one-package-per-process discipline; what differs +is the pin table, the structural identity of ``config.json``, and the +weights layout — one 2 GB shard checked by exact size plus the +side-loaded second span head. + +The runtime loads parameters in FP32 either way; on ``cuda`` it runs +the backbone under bf16 autocast with FP32 heads (the evaluated +setting), on ``cpu`` it runs FP32 throughout. The served model name +comes from ``config.json`` ``model_name`` (upstream +``DEFAULT_MODEL_NAME`` is the family-generic ``vllm-sr/Vela-2.0`` and +must not be used as a default here). +""" + +from __future__ import annotations + +import json +import sys +import threading +from pathlib import Path +from typing import Any + +from .engine import ( + put_first_on_sys_path, + verify_files, + verify_no_foreign_runtime, +) + +HYBRID_REPO = "vllm-sr/Vela-2.0-0.8B" +HYBRID_REVISION = "326b01d81f61fff7fd4ae570e54b87a49119e490" +# Structural identity of the pinned config.json. model_name is unique +# per member, so a 4B or 9B package fails with its own name. +EXPECTED_MODEL_TYPE = "vela2-decoder" +EXPECTED_ARCHITECTURE = "Vela2DecoderModel" +EXPECTED_MAX_LENGTH = 16384 +EXPECTED_MODEL_NAME = "vllm-sr/Vela-2.0-0.8B" +WEIGHTS_NAME = "model-00001-of-00001.safetensors" +EXPECTED_WEIGHTS_BYTES = 2022189488 +# The files this gate hashes, transcribed from the pinned revision and +# cross-checked against the package's own SHA256SUMS; repinning means +# updating this table together with HYBRID_REVISION. Every file the +# runtime reads or executes is here: the four runtime modules, +# ``vela2_serve.py`` (never imported, part of the pinned surface), +# ``config.json``, ``calibration.json``, ``tokenizer.json`` (the loaded +# tokenizer), ``tokenizer_config.json`` (listed by the runtime's own +# download manifest), ``model.safetensors.index.json`` (the shard map +# the loader reads) and ``broad_head.safetensors`` (the side-loaded +# second span head; 4.5 MB hashes in milliseconds). The 2 GB shard is +# checked by exact size only, as on 0.3B. +PINNED_FILES_SHA256 = { + "vela2_serve.py": "68e1ef278561f452aa0ca6afd6aa288636cecdb0b37fd7f7146a3c9bf17aa787", + "vela2_inference.py": "11de1322788942153a22db164a3fb26abcaa4c5bb4b057cd572e277457c17dc2", + "modeling_vela2.py": "b7ecd8dcacfb24dbe8249c7b96829a737f4e2b493fc2157497f1b62d9e02727d", + "configuration_vela2.py": "c9a41e434a9d2cfeb3ab6581c00679d241b3468c04d160a811d6fe2c85cd071f", + "config.json": "0637ca193b9159b6ece25837913956477164d3392d9b5dd1e6b74b95a484004a", + "calibration.json": "d96e447962d0dc1d58fa8422dddbae8b99960e21fd75a10ef9b096f700a211f5", + "tokenizer.json": "06b9509352d2af50381ab2247e083b80d32d5c0aba91c272ca9ff729b6a0e523", + "tokenizer_config.json": "bee8eba30f0eb4af73c0fe2cd06d0f89b657d7819941c438157ec42f7c80ea87", + "model.safetensors.index.json": "b53d1c79219677289bdb1541a1373dfe53119884f248b23c94a27aef7338919d", + "broad_head.safetensors": "687d7a435d6de846796274e417b6333f83df3085a6e7c3bca4c01cb4d75a30f0", +} +# A warmup that touches every question kind's readout path on both span +# heads: the CandidateHead option rows (choice/set share it, noul and +# score ride the same candidate readout), the span decoder and the +# threshold echo, one router-head span (the unsupported label is a +# trained alias) and one broad-head span (an open label with no trained +# match). Model text only; the result is discarded. +WARMUP_REQUEST = { + "state": { + "request": "Hello, I would like to check the status of my order.", + "answer": "Your order was delivered yesterday.", + }, + "model": None, # filled with the served name at warmup time + "questions": { + "kind": { + "type": "choice", + "instructions": "What is this message about?", + "criteria": {"order": "an order's status", "other": "anything else"}, + }, + "claims": { + "type": "set", + "instructions": "Which topics does the message mention?", + "criteria": {"delivery": "delivery of an order", "billing": "payments or charges"}, + }, + "tone": { + "type": "score", + "instructions": "How formal is the message?", + "criteria": ["casual", "neutral", "formal"], + }, + "polite": { + "type": "noul", + "instructions": "Is the message polite?", + }, + "unsupported": { + "type": "span", + "instructions": "Which spans of the answer are not supported by the request?", + "criteria": {"unsupported": "a claim not supported by the request"}, + }, + "products": { + "type": "span", + "instructions": "Which products or services are mentioned?", + "criteria": {"product": "a product or a service"}, + "over": "request", + }, + }, +} + + +def verify_config(root: Path) -> dict[str, Any]: + """Read config.json and check the pinned member's structural identity.""" + + path = Path(root) / "config.json" + if not path.is_file(): + raise ValueError(f"the package is missing config.json; this worker only serves {HYBRID_REPO}") + config = json.loads(path.read_text(encoding="utf-8")) + if config.get("model_type") != EXPECTED_MODEL_TYPE: + raise ValueError( + f"config.json declares model_type {config.get('model_type')!r}; this worker only " + f"serves the {EXPECTED_MODEL_TYPE} member {HYBRID_REPO} at {HYBRID_REVISION}" + ) + if config.get("architectures") != [EXPECTED_ARCHITECTURE]: + raise ValueError(f"config.json architectures must be [{EXPECTED_ARCHITECTURE!r}]") + if config.get("max_length") != EXPECTED_MAX_LENGTH: + raise ValueError(f"config.json max_length must be {EXPECTED_MAX_LENGTH}") + if config.get("model_name") != EXPECTED_MODEL_NAME: + raise ValueError( + f"config.json model_name must be {EXPECTED_MODEL_NAME!r}; another Vela 2.0 member " + "needs its own pinned executor" + ) + return config + + +def verify_weights(root: Path) -> None: + """Check the pinned shard is present at its exact pinned size.""" + + target = Path(root) / WEIGHTS_NAME + if not target.is_file(): + raise ValueError(f"the package is missing {WEIGHTS_NAME}") + size = target.stat().st_size + if size != EXPECTED_WEIGHTS_BYTES: + raise ValueError( + f"{WEIGHTS_NAME} is {size} bytes; the pinned revision's file is {EXPECTED_WEIGHTS_BYTES}" + ) + + +def verify_package(root: Path, table: dict[str, str] | None = None) -> None: + """The full pre-import gate for one hybrid package directory.""" + + verify_config(root) + verify_files(root, table or PINNED_FILES_SHA256, HYBRID_REVISION) + verify_weights(root) + + +class Vela2HybridExecutor: + """Load and drive one immutable pinned Vela 2.0 0.8B runtime instance.""" + + is_hybrid = True + + def __init__( + self, + package: str | Path, + *, + device: str = "cpu", + model_name: str | None = None, + batch_size: int | None = None, + token_budget: int | None = None, + ) -> None: + root = Path(package) + if not root.is_dir(): + raise ValueError(f"the Vela 2.0 0.8B package directory does not exist: {root}") + verify_package(root) + + # Same discipline as the 0.3B executor: the just-verified package + # root goes first on sys.path so the flat runtime modules resolve + # to the checkpoint's own copies, and a runtime module already + # imported from another directory is refused — one worker process + # serves one package. + verify_no_foreign_runtime(root) + put_first_on_sys_path(root) + # Import with bytecode writing off: the gate refuses existing + # bytecode caches, so a __pycache__ this import wrote would fail + # every later gate run of the same package. + previous_dont_write_bytecode = sys.dont_write_bytecode + sys.dont_write_bytecode = True + try: + import vela2_inference + finally: + sys.dont_write_bytecode = previous_dont_write_bytecode + + self.SystemOneError = vela2_inference.SystemOneError + self._system_one_request = vela2_inference.system_one_request + self.engine = vela2_inference.Vela2.from_pretrained( + root, backend="torch", device=device, + **({"batch_size": batch_size} if batch_size is not None else {}), + **({"token_budget": token_budget} if token_budget is not None else {}), + ) + # The decoder members declare their served name in config.json; + # the runtime's DEFAULT_MODEL_NAME is family-generic. + self.model_name = model_name or self.engine.model_name + self.max_length = int(self.engine.max_len) + self._lock = threading.RLock() + self._closed = False + + @property + def served(self) -> set[str]: + """The model names this executor answers to (upstream: one).""" + + return {self.model_name} + + def system_one(self, body: dict[str, Any]) -> dict[str, Any]: + """Run one admitted request body; the response is upstream's, verbatim. + + Hybrid responses carry one extra top-level field, ``span_heads``, + alongside ``spans``; response finishing checks it in + :func:`models.vela.postprocess.finish_response`. + """ + + with self._lock: + if self._closed: + raise RuntimeError("Vela 2.0 hybrid executor is closed") + return self._system_one_request(self.engine, body, served=self.served) + + def warmup(self) -> None: + """Run one real request of every question kind on both span heads.""" + + request = dict(WARMUP_REQUEST) + request["model"] = self.model_name + response = self.system_one(request) + if not isinstance(response, dict) or "answers" not in response: + raise RuntimeError("warmup did not produce a SystemOne response") + + def close(self) -> None: + with self._lock: + if self._closed: + return + self._closed = True + self.engine = None diff --git a/src/models/vela/native/Cargo.toml b/src/models/vela/native/Cargo.toml new file mode 100644 index 00000000..a69e15b6 --- /dev/null +++ b/src/models/vela/native/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "omni-vela-native" +version = "0.1.0" +edition = "2024" +publish = false +description = "Native library for vllm-sr/Vela-2.0-0.8B: bundle gate, wire contract and token-level processing" + +[dependencies] +anyhow = "1.0.100" +onig = { version = "6", default-features = false } +omni-qwen3-5-native = { path = "../../qwen3_5/native" } +serde = { version = "1", features = ["derive"] } +tokenizers = { version = "=0.22.2", default-features = false, features = ["onig"] } +serde_json = { version = "1.0.149", features = ["float_roundtrip", "preserve_order"] } +sha2 = "0.11" + +[lib] +path = "src/lib.rs" + +[[test]] +name = "vela_native_contract" +path = "../../../../tests/vela/native_contract.rs" + +[[test]] +name = "vela_native_bundle" +path = "../../../../tests/vela/native_bundle.rs" + +[[test]] +name = "vela_native_processing" +path = "../../../../tests/vela/native_processing.rs" + +[[test]] +name = "vela_native_units" +path = "../../../../tests/vela/native_units.rs" diff --git a/src/models/vela/native/examples/parity.rs b/src/models/vela/native/examples/parity.rs new file mode 100644 index 00000000..071d0f52 --- /dev/null +++ b/src/models/vela/native/examples/parity.rs @@ -0,0 +1,87 @@ +//! Debug helper: compare the rendered rows against one fixture and +//! print the first differing field. Run with: +//! +//! VELA_NATIVE_BUNDLE=/abs/path cargo run -p omni-vela-native --example parity -- example-request + +use std::path::PathBuf; + +use serde_json::Value; + +fn main() { + let case = std::env::args().nth(1).unwrap_or("example-request".into()); + let bundle = PathBuf::from( + std::env::var("VELA_NATIVE_BUNDLE").expect("VELA_NATIVE_BUNDLE points at the export"), + ); + let fixture: Value = serde_json::from_str( + &std::fs::read_to_string( + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../../../../tests/vela/fixtures") + .join(format!("native-plan-{case}.json")), + ) + .expect("fixture"), + ) + .expect("fixture json"); + + let processor = omni_vela_native::processing::Processor::load(&bundle).expect("load"); + let request = fixture["request"].as_object().expect("request"); + match processor.plan(request) { + Ok(planned) => { + let got = serde_json::json!({ + "rows": serde_json::to_value(&planned.rows).unwrap(), + "window_rows": serde_json::to_value(&planned.window_rows).unwrap(), + }); + compare( + "rows", + got["rows"].as_array().unwrap(), + fixture["rows"].as_array().unwrap(), + ); + if !fixture["window_rows"].is_null() { + compare( + "window_rows", + got["window_rows"].as_array().unwrap(), + fixture["window_rows"].as_array().unwrap(), + ); + } + println!("parity: no field differences found"); + } + Err(error) => println!( + "plan error: {}", + serde_json::to_string(&error.body).unwrap() + ), + } +} + +fn compare(name: &str, got: &[Value], want: &[Value]) { + println!("{name}: {} vs {} row(s)", got.len(), want.len()); + for (index, (a, b)) in got.iter().zip(want).enumerate() { + let (Some(a), Some(b)) = (a.as_object(), b.as_object()) else { + continue; + }; + for key in b.keys() { + if a.get(key) != b.get(key) { + println!("row {index} field {key}: differs"); + println!( + " want: {}", + serde_json::to_string(&b[key]) + .unwrap() + .chars() + .take(300) + .collect::() + ); + println!( + " got: {}", + serde_json::to_string(&a[key]) + .unwrap() + .chars() + .take(300) + .collect::() + ); + } + } + for key in a.keys() { + if !b.contains_key(key) { + println!("row {index} field {key}: extra in ours"); + } + } + } +} diff --git a/src/models/vela/native/src/bundle.rs b/src/models/vela/native/src/bundle.rs new file mode 100644 index 00000000..b31afc96 --- /dev/null +++ b/src/models/vela/native/src/bundle.rs @@ -0,0 +1,142 @@ +//! The exported native bundle's manifest gate. +//! +//! [`recipe/vela/export_native.py`](../../../recipe/vela/export_native.py) +//! produces the bundle from the pinned package; this module loads its +//! manifest, pins the source revision and the member's tensor +//! inventory, and re-hashes every bundle file before the (future) +//! executor touches it — the same provenance-before-import discipline +//! as the reference worker's package gate. + +use std::collections::HashSet; +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result, ensure}; +use serde_json::Value; + +pub const FORMAT: &str = "vela2-native-bundle/1"; +pub const MODEL_ID: &str = "vllm-sr/Vela-2.0-0.8B"; +pub const SOURCE_REVISION: &str = "326b01d81f61fff7fd4ae570e54b87a49119e490"; +/// This member's exported inventory, pinned from the pinned package: +/// 320 backbone tensors (204 BF16 Linear/conv1d + 116 F32) and 37 F32 +/// head tensors (CandidateHead, set_bias, span2 router, span_broad). +pub const BACKBONE_TENSORS: (usize, usize, usize) = (320, 204, 116); +pub const HEAD_TENSORS: usize = 37; + +#[derive(Debug)] +pub struct Bundle { + pub dir: PathBuf, + pub manifest: Value, +} + +impl Bundle { + /// Load and fully verify one exported bundle directory. + pub fn load(dir: &Path) -> Result { + let manifest: Value = serde_json::from_slice( + &std::fs::read(dir.join("vela_native_manifest.json")) + .context("export the pinned package first; see recipe/vela/export_native.py")?, + ) + .context("vela_native_manifest.json must be JSON")?; + ensure!( + manifest["format"] == FORMAT + && manifest["model_id"] == MODEL_ID + && manifest["source"]["revision"] == SOURCE_REVISION, + "expected a bundle exported from the pinned {MODEL_ID} package" + ); + let backbone = &manifest["backbone"]; + ensure!( + backbone["tensors"].as_u64() == Some(BACKBONE_TENSORS.0 as u64) + && backbone["bf16"].as_u64() == Some(BACKBONE_TENSORS.1 as u64) + && backbone["fp32"].as_u64() == Some(BACKBONE_TENSORS.2 as u64), + "unexpected backbone inventory; re-export from the pinned package" + ); + ensure!( + manifest["heads"]["tensors"].as_u64() == Some(HEAD_TENSORS as u64), + "unexpected head inventory; re-export from the pinned package" + ); + let files = manifest["files_sha256"] + .as_object() + .context("files_sha256 must be an object")?; + ensure!( + !files.is_empty(), + "files_sha256 pins no files; the gate must cover the whole bundle" + ); + for (name, digest) in files { + let actual = hash_file(&dir.join(name))?; + ensure!( + actual == digest.as_str().unwrap_or(""), + "{name} does not match the manifest SHA-256" + ); + } + // Fail closed on the directory, as the reference gate walks the + // package root: the export writes a flat bundle, so the + // directory must hold exactly the pinned files plus this + // manifest — nothing unlisted, at any depth. + let pinned: HashSet<&str> = files + .keys() + .map(String::as_str) + .chain(std::iter::once("vela_native_manifest.json")) + .collect(); + let mut unpinned: Vec = std::fs::read_dir(dir) + .with_context(|| format!("cannot list the bundle directory {}", dir.display()))? + .map(|entry| { + entry + .expect("a readable bundle entry") + .file_name() + .to_string_lossy() + .into_owned() + }) + .filter(|name| !pinned.contains(name.as_str())) + .collect(); + unpinned.sort(); + ensure!( + unpinned.is_empty(), + "unpinned file(s) in the bundle directory: {unpinned:?}" + ); + ensure!( + manifest["max_length"].as_u64() == Some(16_384) + && manifest["model_name"].as_str() == Some(MODEL_ID), + "unexpected member identity in the manifest" + ); + Ok(Self { + dir: dir.to_owned(), + manifest, + }) + } + + /// The served model name (upstream: config.json `model_name`). + pub fn model_name(&self) -> &str { + self.manifest["model_name"].as_str().unwrap_or(MODEL_ID) + } + + /// The member's input window. + pub fn max_length(&self) -> usize { + self.manifest["max_length"].as_u64().unwrap_or(16_384) as usize + } +} + +fn hash_file(path: &Path) -> Result { + use anyhow::bail; + use sha2::{Digest, Sha256}; + use std::io::Read; + + let mut hasher = Sha256::new(); + let file = match std::fs::File::open(path) { + Ok(file) => file, + Err(error) => bail!("cannot read {}: {error}", path.display()), + }; + let mut reader = std::io::BufReader::new(file); + let mut chunk = vec![0u8; 8 * 1024 * 1024]; + loop { + let read = reader.read(&mut chunk)?; + if read == 0 { + break; + } + hasher.update(&chunk[..read]); + } + Ok(hex(&hasher.finalize())) +} + +/// Lower-hex of a digest; sha2 0.11's output has no `LowerHex` impl. +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|byte| format!("{byte:02x}")).collect() +} diff --git a/src/models/vela/native/src/contract.rs b/src/models/vela/native/src/contract.rs new file mode 100644 index 00000000..a2304730 --- /dev/null +++ b/src/models/vela/native/src/contract.rs @@ -0,0 +1,139 @@ +//! Vela 2.0 wire admission for the native worker, mirroring +//! `models.vela.protocol` and the pinned runtime's request preamble so +//! clients cannot tell this layer from the reference worker: the +//! three-field envelope with upstream's exact `{"detail": [{"loc", +//! "msg", "type"}]}` triples, and the whole-request error model. +//! +//! Deeper validation — state rendering, question semantics, the 413 — +//! lives in the processing modules and mirrors the pinned runtime +//! against the `native-plan-*` fixtures. One known residual +//! difference: a non-finite JSON number answers the generic 400 +//! message here, where the Python worker names the value; duplicate +//! keys are mirrored verbatim. + +use serde_json::{Map, Value, json}; + +/// Slice-scoped transport cap, as on the reference worker. +pub const MAX_BODY: usize = 8 * 1024 * 1024; +/// The envelope fields, in upstream's missing-field check order. +pub const REQUEST_FIELDS: [&str; 3] = ["state", "model", "questions"]; + +/// A request rejection: an HTTP status and the JSON body a client sees. +#[derive(Debug, PartialEq)] +pub struct RequestError { + pub status: u16, + pub body: Value, +} + +impl RequestError { + fn detail(status: u16, message: &str) -> Self { + Self { + status, + body: json!({"detail": message}), + } + } + + /// One upstream `SystemOneError` as a whole-request 422 triple. + pub fn envelope(loc: &[&str], msg: &str, type_: &str) -> Self { + let mut path = vec![Value::from("body")]; + path.extend(loc.iter().map(|part| Value::from(*part))); + Self { + status: 422, + body: json!({"detail": [{"loc": path, "msg": msg, "type": type_}]}), + } + } +} + +/// Decode a bounded JSON body and reject ambiguous JSON values. +pub fn parse_body(raw: &[u8]) -> Result, RequestError> { + if raw.len() > MAX_BODY { + return Err(RequestError::detail(413, "request body exceeds 8 MiB")); + } + let message = + omni_qwen3_5_native::json::parse(raw).map_err(|error| normalize_parse_error(&error))?; + Ok(message) +} + +/// Mirror the reference worker's 400 messages: duplicate keys and +/// non-object bodies verbatim, everything else the shared generic +/// message. +pub(crate) fn normalize_parse_error(error: &str) -> RequestError { + if let Some(rest) = error.split("duplicate key ").nth(1) { + // serde appends its position suffix after the quoted key. + if let Some(key) = rest + .strip_prefix('"') + .and_then(|tail| tail.split('"').next()) + { + return RequestError::detail(400, &format!("duplicate JSON key '{key}'")); + } + } + // The shared parser's exact non-object error, answered with the + // reference worker's wording. + if error == "request body must be a JSON object" { + return RequestError::detail(400, "request must be a JSON object"); + } + RequestError::detail(400, "request body must contain valid JSON and UTF-8 text") +} + +/// Admit the envelope exactly as upstream's request preamble does, for +/// one served model name: the three required fields present, no extra +/// field, `model` non-blank and equal to the served name. +pub fn admit_request(body: &Map, served: &str) -> Result<(), RequestError> { + for field in REQUEST_FIELDS { + if !body.contains_key(field) { + return Err(RequestError::envelope( + &[field], + "Field required", + "missing", + )); + } + } + let mut extra: Vec<&String> = body + .keys() + .filter(|key| !REQUEST_FIELDS.contains(&key.as_str())) + .collect(); + extra.sort(); + if let Some(first) = extra.first() { + return Err(RequestError::envelope( + &[first], + "Extra inputs are not permitted", + "extra_forbidden", + )); + } + let model = body["model"].as_str().unwrap_or(""); + if model.trim().is_empty() { + return Err(RequestError::envelope( + &["model"], + "model must not be empty or whitespace", + "value_error", + )); + } + if model != served { + return Err(RequestError::envelope( + &["model"], + "Model is not served by this runtime", + "value_error.model", + )); + } + Ok(()) +} + +/// Admit one complete raw request body: parse, then envelope-check. +pub fn compile(raw: &[u8], served: &str) -> Result, RequestError> { + let body = parse_body(raw)?; + admit_request(&body, served)?; + Ok(body) +} + +/// A pinned-runtime `ValueError` rejection as the reference worker maps +/// it (`map_runtime_error`): a whole-request 422 naming the runtime. +pub fn runtime_rejection(message: &str) -> RequestError { + RequestError::detail( + 422, + &format!("request rejected by the pinned runtime: {message}"), + ) +} + +#[cfg(test)] +#[path = "../../../../../tests/vela/native_contract_unit.rs"] +mod tests; diff --git a/src/models/vela/native/src/lib.rs b/src/models/vela/native/src/lib.rs new file mode 100644 index 00000000..1d849284 --- /dev/null +++ b/src/models/vela/native/src/lib.rs @@ -0,0 +1,13 @@ +//! Native library for vllm-sr/Vela-2.0-0.8B: the exported-bundle gate, +//! the wire contract, and the token-level processing (state → parts, +//! the SystemOne → predict mapping with the pinned runtime's validation +//! triples, word units, the DEV2.0 prompt rendering — mirrored against +//! the frozen `native-plan-*` fixtures). + +pub mod bundle; +pub mod contract; +pub mod processing; +pub mod questions; +pub mod render; +pub mod state; +pub mod words; diff --git a/src/models/vela/native/src/processing.rs b/src/models/vela/native/src/processing.rs new file mode 100644 index 00000000..12370856 --- /dev/null +++ b/src/models/vela/native/src/processing.rs @@ -0,0 +1,274 @@ +//! The processor: one admitted request body → predict mapping, plan, +//! and rendered rows, mirroring the reference path +//! (`system_one_to_predict` → `_to_generic` → `_predict`'s row split → +//! `_prep`/`_render`, plus the long-target window rows). Everything the +//! frozen `native-plan-*` fixtures record. + +use std::collections::HashSet; +use std::path::Path; + +use anyhow::{Context, Result}; +use serde_json::{Map, Value}; + +use crate::contract::RequestError; +use crate::questions::{PredictQuestion, system_one_to_predict}; +use crate::render::{ + Encoder, GenericQuestion, HALU_LABEL, PreparedPart, RMAX, RenderContext, RenderedRow, + window_part, +}; +use crate::state::system_one_state; + +pub struct Processor { + encoder: Encoder, + max_length: usize, + router_label_sets: Vec>, + has_broad: bool, + halu_desc: String, +} + +pub struct Planned { + pub predict_request: Value, + pub plan: Value, + pub rows: Vec, + pub window_rows: Vec, +} + +impl Processor { + /// Load from an exported native bundle (tokenizer + calibration; + /// the manifest gate runs first). + pub fn load(bundle: &Path) -> Result { + let manifest = crate::bundle::Bundle::load(bundle)?; + let calibration: Value = serde_json::from_slice( + &std::fs::read(bundle.join("calibration.json")).context("calibration.json")?, + ) + .context("calibration.json must be JSON")?; + let broad = calibration.get("broad_head").filter(|b| !b.is_null()); + let router_label_sets: Vec> = broad + .and_then(|b| b.get("router_label_sets")) + .and_then(Value::as_array) + .filter(|sets| !sets.is_empty()) + .map(|sets| { + sets.iter() + .filter_map(Value::as_array) + .map(|labels| { + labels + .iter() + .filter_map(Value::as_str) + .map(str::to_owned) + .collect() + }) + .collect() + }) + .unwrap_or_else(|| { + vec![ + calibration["pii_types"].as_array().map(|types| { + types + .iter() + .filter_map(Value::as_str) + .map(str::to_owned) + .collect() + }), + calibration["halu_schema"]["labels"] + .as_object() + .map(|labels| labels.keys().cloned().collect()), + ] + .into_iter() + .flatten() + .collect() + }); + let has_broad = broad.is_some(); // the bundle gate pinned the broad tensors + let halu_desc = calibration["halu_schema"]["labels"][HALU_LABEL] + .as_str() + .unwrap_or("a claim not supported by the context") + .to_owned(); + Ok(Self { + encoder: Encoder::load(&bundle.join("tokenizer.json"))?, + max_length: manifest.max_length(), + router_label_sets, + has_broad, + halu_desc, + }) + } + + /// The full planning path for one admitted request body. + pub fn plan(&self, body: &Map) -> Result { + let state = system_one_state(&body["state"])?; + let questions = body["questions"] + .as_object() + .ok_or_else(crate::questions::empty_questions_error)?; + let mapped = system_one_to_predict(state, questions)?; + let predict_request = serde_json::to_value(&mapped.request).expect("serializable"); + let plan = serde_json::to_value(&mapped.plan).expect("serializable"); + let state_parts = mapped.request.parts.clone(); + + // `_to_generic`: predict questions → generic render questions + // (Noul is already folded; choice always carries [ABS]). + let generic: Vec = + mapped.request.questions.iter().map(to_generic).collect(); + // `_predict`'s row split: non-span questions plus the first span + // in one row, every further span its own row. + let spans: Vec<&GenericQuestion> = generic.iter().filter(|q| q.kind == "span").collect(); + let rest: Vec = generic + .iter() + .filter(|q| q.kind != "span") + .cloned() + .collect(); + let mut rows: Vec> = Vec::new(); + if !rest.is_empty() || !spans.is_empty() { + let mut first = rest.clone(); + if let Some(span) = spans.first() { + first.push((*span).clone()); + } + rows.push(first); + for span in spans.iter().skip(1) { + rows.push(vec![(*span).clone()]); + } + } + let context = RenderContext { + encoder: &self.encoder, + max_length: self.max_length, + router_label_sets: &self.router_label_sets, + has_broad: self.has_broad, + halu_desc: self.halu_desc.clone(), + }; + let span_roles: HashSet = + spans.iter().flat_map(|q| q.over.iter().cloned()).collect(); + let mut rendered = Vec::new(); + let mut window_rows = Vec::new(); + let mut parts_by_role = std::collections::HashMap::new(); + for part in self.prepared_parts(&state_parts, &span_roles) { + parts_by_role.insert(part.role.clone(), part); + } + for row in rows { + if row.is_empty() { + continue; + } + let (adjusted, dispatch) = context.prepare_questions(&row)?; + let parts: Vec = state_parts + .iter() + .map(|part| { + parts_by_role + .get(&part.r#type) + .cloned() + .expect("prepared part") + }) + .collect(); + let record = context.render(&parts, &adjusted, &dispatch)?; + // `_window_rows`: a span target beyond RMAX becomes one row + // per window (the full-render span block stays on the row). + let span = row.iter().find(|q| q.kind == "span"); + if let Some(span) = span { + let index = parts.iter().position(|p| p.role == span.over[0]); + if let Some(index) = index { + let part = &parts[index]; + let has_words = part.words.is_some(); + if part.ids.len() > RMAX && has_words { + let (_, first_tokens) = part.words.as_ref().expect("words"); + for (a, b) in crate::render::span_windows(first_tokens, part.ids.len()) { + let mut windowed_parts = parts.clone(); + windowed_parts[index] = window_part(part, a, b); + let (w_adjusted, w_dispatch) = context.prepare_questions(&row)?; + let mut record = + context.render(&windowed_parts, &w_adjusted, &w_dispatch)?; + // The runtime's window rows carry the head only. + record.window = true; + window_rows.push(record); + } + } + } + } + rendered.push(record); + } + Ok(Planned { + predict_request, + plan, + rows: rendered, + window_rows, + }) + } + + /// The prepared parts for one request: tokens everywhere, offsets + /// and word units on span-target parts. + fn prepared_parts( + &self, + state_parts: &[crate::state::Part], + span_roles: &HashSet, + ) -> Vec { + state_parts + .iter() + .map(|part| { + let (ids, offsets) = self.encoder.ids_offsets(&part.text); + if span_roles.contains(&part.r#type) { + let words = crate::words::word_first_tokens(&part.text, &offsets); + PreparedPart { + role: part.r#type.clone(), + ids, + offsets: Some(offsets), + words: Some(words), + } + } else { + PreparedPart { + role: part.r#type.clone(), + ids, + offsets: None, + words: None, + } + } + }) + .collect() + } +} + +/// `_to_generic` for the SystemOne path: options/labels carry over in +/// order (score levels have empty descriptions), the target part is the +/// resolved `over`, choice always carries `[ABS]` (the reference's +/// `abstain` default is true and the mapper never sets it), and span +/// keeps its threshold and head override. +fn to_generic(question: &PredictQuestion) -> GenericQuestion { + let over: Vec = match &question.target_part { + Value::String(role) => vec![role.clone()], + Value::Array(roles) => roles + .iter() + .filter_map(Value::as_str) + .map(str::to_owned) + .collect(), + _ => Vec::new(), + }; + let (names, descriptions) = match (&question.labels, &question.options) { + (Some(labels), _) => ( + labels.iter().map(|n| n.name.clone()).collect(), + labels.iter().map(|n| n.description.clone()).collect(), + ), + (None, Some(Value::Array(items))) => { + // Objects carry name/description; score levels are strings. + let mut names = Vec::new(); + let mut descriptions = Vec::new(); + for item in items { + match item { + Value::String(name) => { + names.push(name.clone()); + descriptions.push(String::new()); + } + Value::Object(named) => { + names.push(named["name"].as_str().unwrap_or_default().to_owned()); + descriptions + .push(named["description"].as_str().unwrap_or_default().to_owned()); + } + _ => {} + } + } + (names, descriptions) + } + _ => (Vec::new(), Vec::new()), + }; + GenericQuestion { + id: question.id.clone(), + kind: question.kind.clone(), + over, + text: question.text.clone(), + names, + descriptions, + threshold: question.threshold, + head_override: question.head.clone(), + } +} diff --git a/src/models/vela/native/src/questions.rs b/src/models/vela/native/src/questions.rs new file mode 100644 index 00000000..4c095032 --- /dev/null +++ b/src/models/vela/native/src/questions.rs @@ -0,0 +1,441 @@ +//! The SystemOne (state, questions) → predict-request mapping, mirroring +//! the pinned runtime's `system_one_to_predict` exactly: the validation +//! walk and its whole-request error triples, the Noul folding into a +//! two-option choice, and the per-question plan used to build responses. + +use serde_json::{Map, Value}; + +use crate::contract::RequestError; +use crate::state::{Part, StateParts, content_text}; + +/// The `questions` container rejection shared by the mapping and the +/// processor's body walk. +pub fn empty_questions_error() -> RequestError { + qerror( + &["questions"], + "questions must be a non-empty object of named questions", + ) +} + +pub const SYSTEM_ONE_TYPES: [&str; 5] = ["noul", "choice", "score", "span", "set"]; +pub const NOUL_DEFAULT_NO: &str = "No. The statement or question is not satisfied."; +pub const NOUL_DEFAULT_YES: &str = "Yes. The statement or question is satisfied."; + +/// One option / label of a predict question (name, description). +#[derive(Debug, Clone, serde::Serialize, PartialEq)] +pub struct Named { + pub name: String, + pub description: String, +} + +/// The predict-level question, exactly as it serializes into the frozen +/// fixtures: Noul questions arrive here already folded into a choice. +#[derive(Debug, Clone, serde::Serialize)] +pub struct PredictQuestion { + pub id: String, + #[serde(rename = "type")] + pub kind: String, + pub text: String, + pub target_part: Value, + #[serde(skip_serializing_if = "Option::is_none")] + pub labels: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub options: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub threshold: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub head: Option, +} + +#[derive(Debug, Clone, serde::Serialize)] +pub struct PredictRequest { + pub parts: Vec, + pub questions: Vec, +} + +/// The response-side plan entry for one original question. +#[derive(Debug, Clone, serde::Serialize)] +pub struct PlanEntry { + pub qid: String, + #[serde(rename = "type")] + pub kind: String, + pub criteria: Value, + pub range: Option<(usize, usize)>, + pub part: Value, +} + +#[derive(Debug)] +pub struct Mapped { + pub request: PredictRequest, + pub plan: Vec, +} + +fn qerror(loc: &[&str], msg: &str) -> RequestError { + RequestError::envelope(loc, msg, "value_error") +} + +fn nonblank(value: &Value) -> bool { + match value { + Value::String(text) => !text.trim().is_empty(), + _ => true, + } +} + +/// `over` (state key | part name | list) → (target part(s), field range). +fn resolve_over( + over: &Value, + parts: &StateParts, + qloc: &[&str], +) -> Result<(Value, Option<(usize, usize)>), RequestError> { + let roles: Vec<&str> = parts.parts.iter().map(|p| p.r#type.as_str()).collect(); + let items: Vec<&Value> = match over { + Value::Array(list) => list.iter().collect(), + single => vec![single], + }; + let mut out: Vec<&str> = Vec::new(); + let mut range = None; + for item in items { + let Some(name) = item.as_str() else { + return Err(qerror( + &[qloc[0], qloc[1], "over"], + "over must be a state key, a part name or a list of them", + )); + }; + if let Some(field) = parts.fields.get(name) { + if !over.is_array() { + range = Some((field.start, field.end)); + } + if !out.contains(&field.role) { + out.push(field.role); + } + } else if roles.contains(&name) { + if !out.contains(&name) { + out.push(name); + } + } else { + let role = crate::state::key_role(name); + if roles.contains(&role) && !out.contains(&role) { + out.push(role); + } else { + return Err(qerror( + &[qloc[0], qloc[1], "over"], + &format!( + "'{name}' is not a field of the state (fields: [{fields}], parts: [{roles}])", + fields = parts + .fields + .keys() + .map(|k| format!("'{k}'")) + .collect::>() + .join(", "), + roles = roles.join(", "), + ), + )); + } + } + } + if out.is_empty() { + return Err(qerror( + &[qloc[0], qloc[1], "over"], + "over must not be empty", + )); + } + // Sequence order, so the pooled range covers exactly these parts. + let ordered: Vec<&str> = roles + .iter() + .filter(|role| out.contains(role)) + .copied() + .collect(); + let target = match ordered.len() { + 1 => Value::String(ordered[0].into()), + _ => Value::Array( + ordered + .into_iter() + .map(|r| Value::String(r.into())) + .collect(), + ), + }; + Ok((target, range)) +} + +/// The full mapping. `state` and `questions` are the admitted request's +/// fields; validation runs in the pinned runtime's order so error +/// precedence matches. +pub fn system_one_to_predict( + state_parts: StateParts, + questions: &Map, +) -> Result { + if questions.is_empty() { + return Err(qerror( + &["questions"], + "questions must be a non-empty object of named questions", + )); + } + let roles: Vec<&str> = state_parts + .parts + .iter() + .map(|p| p.r#type.as_str()) + .collect(); + let mut predict = Vec::new(); + let mut plan = Vec::new(); + for (qid, question) in questions { + if qid.trim().is_empty() { + return Err(qerror( + &["questions"], + "question IDs must not be empty or whitespace", + )); + } + let Some(question) = question.as_object() else { + return Err(qerror( + &["questions", qid], + "a question must be an object with a type", + )); + }; + let loc = ["questions", qid]; + let kind = question.get("type").and_then(Value::as_str).unwrap_or(""); + if !SYSTEM_ONE_TYPES.contains(&kind) { + return Err(RequestError::envelope( + &[loc[0], loc[1], "type"], + "type must be one of ['noul', 'choice', 'score', 'span', 'set']", + "literal_error", + )); + } + let mut allowed: Vec<&str> = vec!["type", "instructions", "criteria", "over"]; + if kind == "span" || kind == "set" { + allowed.push("threshold"); + } + if kind == "span" { + allowed.push("head"); + } + let extra: Vec<&String> = question + .keys() + .filter(|key| !allowed.contains(&key.as_str())) + .collect(); + if let Some(first) = extra.first() { + return Err(RequestError::envelope( + &[loc[0], loc[1], first], + "Extra inputs are not permitted", + "extra_forbidden", + )); + } + let instructions = question.get("instructions").unwrap_or(&Value::Null); + if !nonblank(instructions) { + return Err(qerror( + &[loc[0], loc[1], "instructions"], + "instructions must not be empty or whitespace", + )); + } + let text = match instructions { + Value::Null => String::new(), + other => content_text(other), + }; + let criteria = question.get("criteria").unwrap_or(&Value::Null); + // Target part: the resolved `over`, the span default, or all parts. + let (target_part, range) = match question.get("over") { + Some(Value::Null) | None => { + if kind == "span" { + let role = if roles.contains(&"answer") { + "answer" + } else if roles.contains(&"user") { + "user" + } else { + roles[0] + }; + (Value::String(role.into()), None) + } else if roles.len() == 1 { + (Value::String(roles[0].into()), None) + } else { + ( + Value::Array(roles.iter().map(|r| Value::String((*r).into())).collect()), + None, + ) + } + } + Some(over) => resolve_over(over, &state_parts, &loc)?, + }; + let mut predict_question = PredictQuestion { + id: qid.clone(), + kind: kind.to_string(), + text: text.clone(), + target_part: target_part.clone(), + labels: None, + options: None, + threshold: None, + head: None, + }; + let mut plan_criteria = criteria.clone(); + match kind { + "noul" => { + if !criteria.is_null() + && (!criteria.is_object() + || criteria + .as_object() + .is_some_and(|map| map.keys().any(|k| k != "true" && k != "false"))) + { + return Err(qerror( + &[loc[0], loc[1], "criteria"], + "noul criteria is an object with optional true / false", + )); + } + let map = criteria.as_object().cloned().unwrap_or_default(); + for key in ["true", "false"] { + if let Some(value) = map.get(key) + && !nonblank(value) + { + return Err(qerror( + &[loc[0], loc[1], "criteria", key], + "Noul criterion must not be empty or whitespace", + )); + } + } + let no = match map.get("false") { + Some(Value::Null) | None => NOUL_DEFAULT_NO.to_string(), + Some(value) => content_text(value), + }; + let yes = match map.get("true") { + Some(Value::Null) | None => NOUL_DEFAULT_YES.to_string(), + Some(value) => content_text(value), + }; + predict_question.kind = "choice".into(); + predict_question.options = Some(serde_json::json!([ + {"name": "no", "description": no}, + {"name": "yes", "description": yes}, + ])); + } + "choice" | "set" | "span" => { + let (lo, hi) = if kind == "choice" { (2, 255) } else { (1, 255) }; + let Some(map) = criteria.as_object() else { + return Err(qerror( + &[loc[0], loc[1], "criteria"], + &format!("{kind} criteria must be an object with {lo} to {hi} entries"), + )); + }; + if !(lo..=hi).contains(&map.len()) { + return Err(qerror( + &[loc[0], loc[1], "criteria"], + &format!("{kind} criteria must be an object with {lo} to {hi} entries"), + )); + } + let mut named = Vec::new(); + for (name, value) in map { + if name.trim().is_empty() || !nonblank(value) { + return Err(qerror( + &[loc[0], loc[1], "criteria", name], + "option names and descriptions must not be empty or whitespace", + )); + } + let description = match value { + Value::Null => String::new(), + other => content_text(other), + }; + named.push(Named { + name: name.clone(), + description, + }); + } + if kind == "span" { + predict_question.labels = Some(named); + } else { + predict_question.options = Some(serde_json::to_value(&named).unwrap()); + } + } + _ => { + // score + let Some(levels) = criteria.as_array() else { + return Err(qerror( + &[loc[0], loc[1], "criteria"], + "score criteria must be a list of 2 to 10 levels", + )); + }; + if !(2..=10).contains(&levels.len()) { + return Err(qerror( + &[loc[0], loc[1], "criteria"], + "score criteria must be a list of 2 to 10 levels", + )); + } + let mut names = Vec::new(); + for (index, value) in levels.iter().enumerate() { + if value.is_null() || !nonblank(value) { + return Err(qerror( + &[loc[0], loc[1], "criteria", &index.to_string()], + "Score criterion must not be empty or whitespace", + )); + } + names.push(content_text(value)); + } + if names.iter().collect::>().len() != names.len() { + return Err(qerror( + &[loc[0], loc[1], "criteria"], + "score levels must be distinct", + )); + } + predict_question.options = Some(serde_json::to_value(&names).unwrap()); + } + } + if let Some(threshold) = question.get("threshold") + && !threshold.is_null() + { + match threshold.as_f64().filter(|_| threshold.is_number()) { + Some(value) if (0.0..=1.0).contains(&value) => { + predict_question.threshold = Some(value); + } + _ => { + return Err(qerror( + &[loc[0], loc[1], "threshold"], + "threshold must be a number in [0, 1]", + )); + } + } + } + if kind == "span" + && let Some(head) = question.get("head") + && !head.is_null() + { + match head.as_str() { + Some(value @ ("router" | "broad")) => { + predict_question.head = Some(value.to_string()); + } + _ => { + return Err(RequestError::envelope( + &[loc[0], loc[1], "head"], + "head must be one of ['router', 'broad']", + "literal_error", + )); + } + } + } + // Python's plan keeps `crit`; noul normalized null -> {} earlier. + if kind == "noul" && plan_criteria.is_null() { + plan_criteria = serde_json::json!({}); + } + predict.push(predict_question); + plan.push(PlanEntry { + qid: qid.clone(), + kind: kind.to_string(), + criteria: plan_criteria, + range, + part: target_part, + }); + } + // The Noul views of set labels must not overwrite a question. + for entry in &plan { + if entry.kind == "set" + && let Value::Object(map) = &entry.criteria + { + for label in map.keys() { + if questions.contains_key(&format!("{}.{}", entry.qid, label)) { + return Err(qerror( + &["questions", &format!("{}.{}", entry.qid, label)], + "collides with the Noul view of a set label", + )); + } + } + } + } + Ok(Mapped { + request: PredictRequest { + parts: state_parts.parts, + questions: predict, + }, + plan, + }) +} diff --git a/src/models/vela/native/src/render.rs b/src/models/vela/native/src/render.rs new file mode 100644 index 00000000..ed3d11fd --- /dev/null +++ b/src/models/vela/native/src/render.rs @@ -0,0 +1,794 @@ +//! Prompt rendering, mirroring the pinned runtime's `_prep` + `_render` +//! (the DEV2.0 decoder layout): the parts pass, per-question blocks +//! with option end and query positions, the span block in layout a/c +//! with word positions, the truncation budget, and the long-target +//! windows. Output structs serialize exactly like the frozen +//! `native-plan-*` fixtures. + +use std::cell::RefCell; +use std::collections::{HashMap, HashSet}; +use std::path::Path; + +use std::collections::BTreeMap; +use tokenizers::Tokenizer; + +use crate::contract::RequestError; +use crate::state::canonical; + +pub const SUFFIX_CHOICE_SCORE: &str = + "\n\nSelect the single option best supported by the context and instructions.\nDecision:"; +pub const SUFFIX_SET: &str = + "\n\nSelect every option supported by the context and instructions.\nDecision:"; +pub const SUFFIX_SPAN: &str = + "\n\nMark every span of the target text that carries one of the labels.\nSpans:"; +pub const ABS_KEY: &str = "[ABS]"; +pub const ABS_DESC: &str = "None of the shown options applies."; +pub const PARTS_HEAD: &str = "Context:\n"; +pub const SEG_CLOSE: &str = "\n\n"; +pub const TEXT_HEAD: &str = "\n\nText:\n"; +pub const HYBRID_RMAX: usize = 2048; +pub const WIN: usize = 1800; +pub const STRIDE: usize = 1536; +pub const RMAX: usize = 2048; + +pub const ROUTER_SPAN_IDS: [&str; 3] = ["pii", "halu", "toxic"]; +pub const HALU_LABEL: &str = "unsupported"; +pub const HALU_ALIASES: [&str; 9] = [ + "unsupported", + "unsupportedclaim", + "hallucinated", + "hallucination", + "hallucinatedspan", + "notsupported", + "unsupportedspan", + "fabricated", + "fabrication", +]; +pub const PII_TYPES: [&str; 17] = [ + "AGE", + "CREDIT_CARD", + "DATE_TIME", + "DOMAIN_NAME", + "EMAIL_ADDRESS", + "GPE", + "IBAN_CODE", + "IP_ADDRESS", + "NRP", + "ORGANIZATION", + "PERSON", + "PHONE_NUMBER", + "STREET_ADDRESS", + "TITLE", + "US_DRIVER_LICENSE", + "US_SSN", + "ZIP_CODE", +]; + +fn seg_open(role: &str) -> String { + format!("\n") +} + +fn option_block(key: &str, desc: &str, tag: &str) -> String { + format!( + "\n<{tag}>\n{}\n", + canonical(&serde_json::json!({"key": key, "description": desc})) + ) +} + +fn q_head(kind: &str, over: &[String], text: &str, span: bool) -> String { + let target = over.join(", "); + format!( + "\n\nTask type: {kind}\nTarget: {target}\nQuestion:\n{text}\n{}", + if span { "Labels:" } else { "Options:" } + ) +} + +// ------------------------------------------------------------------ tokenizer + +/// The pinned tokenizer, with the reference's memoized plain-`ids` view +/// and char-offset encodings (the tokenizers crate reports byte +/// offsets; the reference works in Unicode code points). +/// Ids plus char offsets, the encoder cache entry. +type Encoded = (Vec, Vec<(usize, usize)>); + +pub struct Encoder { + tokenizer: Tokenizer, + cache: RefCell>, +} + +impl Encoder { + pub fn load(path: &Path) -> anyhow::Result { + let tokenizer = Tokenizer::from_file(path).map_err(anyhow::Error::msg)?; + Ok(Self { + tokenizer, + cache: RefCell::new(HashMap::new()), + }) + } + + fn encode(&self, text: &str) -> (Vec, Vec<(usize, usize)>) { + if let Some(hit) = self.cache.borrow().get(text) { + return hit.clone(); + } + let encoding = match self.tokenizer.encode(text, false) { + Ok(encoding) => encoding, + Err(error) => { + // The pinned tokenizer should never fail on valid UTF-8 + // input; surface the error instead of panicking so a + // wrapping worker can map it. + unreachable!("the pinned tokenizer failed on valid input: {error}") + } + }; + let byte_offsets = encoding.get_offsets(); + // Byte → code-point offsets. + let char_starts: Vec = text + .char_indices() + .map(|(byte, _)| byte) + .chain(std::iter::once(text.len())) + .collect(); + let to_char = |byte: usize| char_starts.partition_point(|&start| start < byte); + let offsets: Vec<(usize, usize)> = byte_offsets + .iter() + .map(|(start, end)| (to_char(*start), to_char(*end))) + .collect(); + let entry = (encoding.get_ids().to_vec(), offsets); + self.cache + .borrow_mut() + .insert(text.to_owned(), entry.clone()); + entry + } + + /// Memoized token ids of a whole string (the reference's `_ids`). + pub fn ids(&self, text: &str) -> Vec { + self.encode(text).0 + } + + /// Ids and char offsets (the reference's `_ids_offs`). + pub fn ids_offsets(&self, text: &str) -> (Vec, Vec<(usize, usize)>) { + self.encode(text) + } +} + +// ------------------------------------------------------------- span dispatch + +/// `[\s_\-]+` removed, lower-cased — the reference's `_norm_label`. +pub fn norm_label(name: &str) -> String { + let mut out = String::with_capacity(name.len()); + for ch in name.chars() { + if !ch.is_whitespace() && ch != '_' && ch != '-' { + out.extend(ch.to_lowercase()); + } + } + out +} + +/// The user's label when the span question has exactly one label that +/// is a hallucination alias, else none. +pub fn halu_alias(names: &[String]) -> Option { + if names.len() == 1 && HALU_ALIASES.contains(&norm_label(&names[0]).as_str()) { + Some(names[0].clone()) + } else { + None + } +} + +pub struct SpanDispatch { + pub head: &'static str, + pub reason: &'static str, +} + +/// The reference's six-rule dispatch, in order: `"head"` override, +/// trained ids, an exact router label set, a non-empty subset of the 17 +/// PII types, one hallucination alias, else broad (router when the +/// broad head is not loaded). +pub fn span_head_for( + id: &str, + labels: &HashSet, + head_override: Option<&str>, + router_label_sets: &[HashSet], + broad_available: bool, +) -> Result { + if let Some(head) = head_override { + if !matches!(head, "router" | "broad") { + return Err(format!( + "head must be one of ['router', 'broad'], got '{head:?}'" + )); + } + if head == "broad" && !broad_available { + return Err("the broad span head is not loaded (needs broad_head.safetensors)".into()); + } + return Ok(SpanDispatch { + head: if head == "broad" { "broad" } else { "router" }, + reason: "override", + }); + } + if ROUTER_SPAN_IDS.contains(&id) { + return Ok(SpanDispatch { + head: "router", + reason: "id", + }); + } + if router_label_sets.iter().any(|set| set == labels) { + return Ok(SpanDispatch { + head: "router", + reason: "label_set", + }); + } + let pii: HashSet<&str> = PII_TYPES.into_iter().collect(); + if !labels.is_empty() && labels.iter().all(|label| pii.contains(label.as_str())) { + return Ok(SpanDispatch { + head: "router", + reason: "pii_subset", + }); + } + if halu_alias(&labels.iter().cloned().collect::>()).is_some() { + return Ok(SpanDispatch { + head: "router", + reason: "halu_alias", + }); + } + if !broad_available { + return Ok(SpanDispatch { + head: "router", + reason: "broad_unavailable", + }); + } + Ok(SpanDispatch { + head: "broad", + reason: "default", + }) +} + +// ------------------------------------------------------------------ rendering + +/// One generic-contract question at render time (Noul is already folded +/// into a two-option choice; choice always carries `[ABS]` on the +/// SystemOne path — the reference's `_to_generic` defaults abstain to +/// true and the mapper never sets it). +#[derive(Clone)] +pub struct GenericQuestion { + pub id: String, + pub kind: String, + pub over: Vec, + pub text: String, + pub names: Vec, + pub descriptions: Vec, + pub threshold: Option, + pub head_override: Option, +} + +/// One prepared part: tokens, and for span-target parts also char +/// offsets and word units. +/// Word char offsets and each word's first token index. +pub type WordUnits = (Vec<(usize, usize)>, Vec); + +#[derive(Clone)] +pub struct PreparedPart { + pub role: String, + pub ids: Vec, + pub offsets: Option>, + pub words: Option, +} + +#[derive(Debug, Clone, serde::Serialize, PartialEq)] +pub struct TruncRecord { + pub shrink_cut: usize, + pub protected_cut: usize, +} + +#[derive(Debug, Clone, serde::Serialize, PartialEq)] +#[serde(tag = "kind")] +pub enum Block { + #[serde(rename = "q")] + Question { + ids: Vec, + ends: Vec, + query: usize, + qid: String, + qtype: String, + names: Vec, + has_abs: bool, + }, + #[serde(rename = "span")] + Span { + ends: Vec, + label_ranges: Vec<(usize, usize)>, + words_in_block: bool, + ids: Vec, + query: usize, + w_pos: Vec, + }, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct RenderedRow { + pub p_ids: Vec, + pub blocks: Vec, + pub trunc: TruncRecord, + pub seg_lens: BTreeMap, + pub layout: Option, + pub names: Option>, + pub w_offs: Option>, + pub w_pos: Option>, + pub span_over: Option, + pub n_tokens: usize, + pub head: String, + pub head_reason: Option, + /// The runtime's window rows carry the head only — no head_reason + /// key at all (predict_raw assigns head alone). + pub window: bool, +} + +/// Serialize exactly like the frozen fixtures: main rows always carry +/// `head_reason` (null when the row has no span); window rows omit it. +impl serde::Serialize for RenderedRow { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serialize_row(self, serializer) + } +} + +fn serialize_row(row: &RenderedRow, serializer: S) -> Result +where + S: serde::Serializer, +{ + use serde::ser::SerializeMap; + let mut map = serializer.serialize_map(None)?; + macro_rules! put { + ($key:literal, $value:expr) => { + map.serialize_entry($key, &$value)? + }; + } + put!("p_ids", row.p_ids); + put!("blocks", row.blocks); + put!("trunc", row.trunc); + put!("seg_lens", row.seg_lens); + put!("layout", row.layout); + if let Some(names) = &row.names { + put!("names", names); + } + if let Some(w_offs) = &row.w_offs { + put!("w_offs", w_offs); + } + if let Some(w_pos) = &row.w_pos { + put!("w_pos", w_pos); + } + if let Some(span_over) = &row.span_over { + put!("span_over", span_over); + } + put!("n_tokens", row.n_tokens); + put!("head", row.head); + if !row.window { + put!("head_reason", row.head_reason); + } + map.end() +} + +/// Everything the renderer needs beyond tokens. +pub struct RenderContext<'a> { + pub encoder: &'a Encoder, + pub max_length: usize, + pub router_label_sets: &'a [HashSet], + pub has_broad: bool, + pub halu_desc: String, +} + +/// The whole-request 413: questions plus schema alone do not fit. +pub fn too_long(max_length: usize) -> RequestError { + RequestError { + status: 413, + body: serde_json::json!({"detail": [{"loc": ["body", "questions"], + "msg": format!("the questions alone do not fit the {max_length}-token window"), + "type": "too_long"}]}), + } +} + +/// `schema_lib_r4.assemble`, eval path: the truncation order on the +/// decoder's token costs. Returns kept segment lengths and the record. +fn assemble_budget( + q_lens: &[usize], + el_lens: &[usize], + segs: &mut [(String, usize)], + max_len: usize, + protected: &HashSet, + shrink: Option<&str>, +) -> Result<(BTreeMap, TruncRecord), ()> { + let fixed = 2 + 1 + segs.len(); + fn over( + q_lens: &[usize], + el_lens: &[usize], + segs: &[(String, usize)], + fixed: usize, + max_len: usize, + ) -> i64 { + fixed as i64 + + q_lens.iter().sum::() as i64 + + el_lens.iter().sum::() as i64 + + segs.iter().map(|(_, n)| *n).sum::() as i64 + - max_len as i64 + } + let mut trunc = TruncRecord { + shrink_cut: 0, + protected_cut: 0, + }; + let shrink_index = shrink.and_then(|role| segs.iter().position(|(r, _)| r == role)); + for floor in [64, 1] { + if over(q_lens, el_lens, segs, fixed, max_len) > 0 + && let Some(index) = shrink_index + { + let full = segs[index].1; + let excess = over(q_lens, el_lens, segs, fixed, max_len).max(0) as usize; + let keep = full.min(floor.max(full.saturating_sub(excess))); + trunc.shrink_cut += full - keep; + segs[index].1 = keep; + } + } + if over(q_lens, el_lens, segs, fixed, max_len) > 0 { + for index in 0..segs.len() { + if over(q_lens, el_lens, segs, fixed, max_len) > 0 && protected.contains(&segs[index].0) + { + let full = segs[index].1; + let excess = over(q_lens, el_lens, segs, fixed, max_len).max(0) as usize; + let keep = 1.max(full.saturating_sub(excess)); + trunc.protected_cut += full - keep; + segs[index].1 = keep; + } + } + } + if over(q_lens, el_lens, segs, fixed, max_len) > 0 { + return Err(()); + } + let seg_lens: BTreeMap = + segs.iter().map(|(role, n)| (role.clone(), *n)).collect(); + Ok((seg_lens, trunc)) +} + +/// Token windows [a, b) over a part of n tokens, starting and cut at +/// word starts (`schema_lib_d3_20261001.windows`). +pub fn span_windows(first_tokens: &[usize], n: usize) -> Vec<(usize, usize)> { + let mut starts: Vec = first_tokens.to_vec(); + starts.sort_unstable(); + starts.dedup(); + if starts.is_empty() { + return vec![(0, n)]; + } + let mut out = Vec::new(); + let mut a = starts[0]; + loop { + let limit = a + WIN; + let next: Vec = starts + .iter() + .copied() + .filter(|&s| s > a && s < limit) + .collect(); + let b = if limit >= n { + n + } else if !next.is_empty() { + *starts + .iter() + .filter(|&&s| s < limit && s > a) + .max() + .unwrap() + } else { + n.min(limit) + }; + out.push((a, b)); + if b >= n { + break; + } + let candidate: Vec = starts + .iter() + .copied() + .filter(|&s| s >= a + STRIDE && s < b) + .collect(); + a = candidate.first().copied().unwrap_or(b); + } + out +} + +/// A windowed copy of a prepared part: tokens [a, b) and the words that +/// start inside them. +pub fn window_part(part: &PreparedPart, a: usize, b: usize) -> PreparedPart { + let (word_offsets, first_tokens) = part.words.as_ref().expect("windowed parts carry words"); + let selected: Vec = first_tokens + .iter() + .enumerate() + .filter(|&(_, &t)| t >= a && t < b) + .map(|(index, _)| index) + .collect(); + let kept_offsets: Vec<(usize, usize)> = selected.iter().map(|&i| word_offsets[i]).collect(); + PreparedPart { + role: part.role.clone(), + ids: part.ids[a..b].to_vec(), + offsets: part.offsets.as_ref().map(|offs| offs[a..b].to_vec()), + words: Some(( + kept_offsets, + selected.iter().map(|&i| first_tokens[i] - a).collect(), + )), + } +} + +impl RenderContext<'_> { + /// `_prep`'s question side: the span-head dispatch and the + /// hallucination-alias label replacement (parts arrive prepared — + /// word units need the original text and are the caller's job). + pub fn prepare_questions( + &self, + questions: &[GenericQuestion], + ) -> Result<(Vec, SpanDispatch), RequestError> { + let span = questions.iter().find(|q| q.kind == "span"); + let dispatch = match span { + Some(question) => { + let labels: HashSet = question.names.iter().cloned().collect(); + span_head_for( + &question.id, + &labels, + question.head_override.as_deref(), + self.router_label_sets, + self.has_broad, + ) + .map_err(|message| crate::contract::runtime_rejection(&message))? + } + None => SpanDispatch { + head: "router", + reason: "", + }, + }; + // The trained `unsupported` label renders in place of a single + // hallucination-alias label (the answer gets the user's name). + let mut adjusted = questions.to_vec(); + if let Some(question) = adjusted.iter_mut().find(|q| q.kind == "span") + && dispatch.head == "router" + && let Some(alias) = halu_alias(&question.names) + && alias != HALU_LABEL + { + question.names = vec![HALU_LABEL.to_string()]; + question.descriptions = vec![self.halu_desc.clone()]; + } + Ok((adjusted, dispatch)) + } + + /// `_render` on one prepared row: the parts pass and every block, + /// with the budget applied first. + pub fn render( + &self, + parts: &[PreparedPart], + questions: &[GenericQuestion], + dispatch: &SpanDispatch, + ) -> Result { + let ids_of = |text: &str| self.encoder.ids(text); + let roles: Vec = parts.iter().map(|p| p.role.clone()).collect(); + // Question blocks and their schema costs. + struct RenderedQ { + qid: String, + qtype: String, + names: Vec, + otoks: Vec>, + has_abs: bool, + head: Vec, + suffix: Vec, + abstain_tokens: Vec, + len: usize, + } + let mut rendered_qs: Vec = Vec::new(); + let mut el: Vec<(String, Vec)> = Vec::new(); // span labels + let mut span_question: Option<&GenericQuestion> = None; + for question in questions { + if question.kind == "span" { + span_question = Some(question); + for (name, desc) in question.names.iter().zip(&question.descriptions) { + let tokens = ids_of(&option_block(name, desc, "label")); + el.push((name.clone(), tokens.clone())); + } + continue; + } + let suffix = match question.kind.as_str() { + "set" => SUFFIX_SET, + _ => SUFFIX_CHOICE_SCORE, // choice and score share the wording + }; + let has_abs = question.kind == "choice"; // SystemOne: abstain defaults true + let head = ids_of(&q_head( + &question.kind, + &question.over, + &question.text, + false, + )); + let suffix = ids_of(suffix); + let abstain_tokens = if has_abs { + ids_of(&option_block(ABS_KEY, ABS_DESC, "option")) + } else { + Vec::new() + }; + let mut otoks = Vec::with_capacity(question.names.len()); + let mut len = + 1 + head.len() + suffix.len() + abstain_tokens.len() + usize::from(has_abs); + for (name, desc) in question.names.iter().zip(&question.descriptions) { + let tokens = ids_of(&option_block(name, desc, "option")); + len += 1 + tokens.len(); + otoks.push(tokens); + } + rendered_qs.push(RenderedQ { + qid: question.id.clone(), + qtype: question.kind.clone(), + names: question.names.clone(), + otoks, + has_abs, + head, + suffix, + abstain_tokens, + len, + }); + } + let span_over: Option = span_question.map(|question| question.over[0].clone()); + let mut overs: HashSet = HashSet::new(); + for question in questions { + overs.extend(question.over.iter().cloned()); + } + if let Some(over) = &span_over { + overs.insert(over.clone()); + } + let part_of = |role: &str| parts.iter().find(|p| p.role == role); + let shrink = roles.iter().find(|role| !overs.contains(*role)).cloned(); + let layout = match &span_over { + Some(over) if part_of(over).is_some_and(|p| p.ids.len() <= HYBRID_RMAX) => "c", + Some(_) => "a", + None => "", + }; + // The schema overhead the budget does not see. + let mut budget = ids_of(PARTS_HEAD).len(); + for role in &roles { + budget += ids_of(&seg_open(role)).len() + ids_of(SEG_CLOSE).len(); + } + if let Some(over) = &span_over { + let question = span_question.expect("span question"); + let head = ids_of(&q_head("span", &question.over, &question.text, true)); + let suffix = ids_of(SUFFIX_SPAN); + budget += head.len() + suffix.len(); + if layout == "c" { + let span_len = part_of(over).expect("span part").ids.len(); + budget += ids_of(TEXT_HEAD).len() + + ids_of(&seg_open(over)).len() + + ids_of(SEG_CLOSE).len(); + let headroom = self.max_length.saturating_sub(budget); + budget += span_len.min(headroom / 2); + } + } + let q_lens: Vec = rendered_qs.iter().map(|q| q.len).collect(); + let el_lens: Vec = el.iter().map(|(_, tokens)| 1 + tokens.len()).collect(); + let mut segs: Vec<(String, usize)> = parts + .iter() + .map(|p| (p.role.clone(), p.ids.len())) + .collect(); + let (seg_lens, trunc) = assemble_budget( + &q_lens, + &el_lens, + &mut segs, + 64.max(self.max_length - budget), + &overs, + shrink.as_deref(), + ) + .map_err(|()| too_long(self.max_length))?; + // The parts pass, in part order (not the sorted map). + let mut p_ids = ids_of(PARTS_HEAD); + let mut start: HashMap = HashMap::new(); + for part in parts { + let keep = seg_lens[&part.role]; + let open = ids_of(&seg_open(&part.role)); + start.insert(part.role.clone(), p_ids.len() + open.len()); + p_ids.extend(open); + p_ids.extend(part.ids.iter().take(keep)); + p_ids.extend(ids_of(SEG_CLOSE)); + } + // Question blocks. + let mut blocks: Vec = Vec::new(); + for q in &rendered_qs { + let mut ids = q.head.clone(); + let mut ends = Vec::with_capacity(q.names.len() + usize::from(q.has_abs)); + for tokens in &q.otoks { + ids.extend(tokens); + ends.push(ids.len() - 1); + } + if q.has_abs { + ids.extend(q.abstain_tokens.clone()); + ends.push(ids.len() - 1); + } + ids.extend(q.suffix.clone()); + let query = ids.len() - 1; + blocks.push(Block::Question { + ids, + ends, + query, + qid: q.qid.clone(), + qtype: q.qtype.clone(), + names: q.names.clone(), + has_abs: q.has_abs, + }); + } + // The span block. + let mut row_names = None; + let mut row_w_offs = None; + let mut row_w_pos = None; + if let Some(over) = &span_over + && !el.is_empty() + { + let question = span_question.expect("span question"); + let mut ids = ids_of(&q_head("span", &question.over, &question.text, true)); + let mut ends = Vec::new(); + let mut starts = Vec::new(); + for (_, tokens) in &el { + starts.push(ids.len()); + ids.extend(tokens); + ends.push(ids.len() - 1); + } + let part = part_of(over).expect("span part"); + let n = seg_lens[over]; + let (word_offsets, first_tokens) = part.words.as_ref().expect("span words"); + // Word indices whose first token survives the kept prefix; + // indices, not token values — two words can share one. + let keep: Vec = first_tokens + .iter() + .enumerate() + .filter(|&(_, &t)| t < n) + .map(|(index, _)| index) + .collect(); + let words_in_block = layout == "c"; + if words_in_block { + ids.extend(ids_of(TEXT_HEAD)); + ids.extend(ids_of(&seg_open(over))); + let c0 = ids.len(); + ids.extend(part.ids.iter().take(n)); + ids.extend(ids_of(SEG_CLOSE)); + row_w_pos = Some(keep.iter().map(|&i| c0 + first_tokens[i]).collect()); + } else { + let base = start[over]; + row_w_pos = Some(keep.iter().map(|&i| base + first_tokens[i]).collect()); + } + ids.extend(ids_of(SUFFIX_SPAN)); + let query = ids.len() - 1; + row_names = Some(el.iter().map(|(name, _)| name.clone()).collect()); + row_w_offs = Some(keep.iter().map(|&i| word_offsets[i]).collect()); + let label_ranges: Vec<(usize, usize)> = + starts.into_iter().zip(ends.iter().copied()).collect(); + blocks.push(Block::Span { + ends, + label_ranges, + words_in_block, + ids, + query, + w_pos: row_w_pos.clone().expect("w_pos"), + }); + } + let n_tokens = p_ids.len() + blocks.iter().map(block_len).sum::(); + Ok(RenderedRow { + p_ids, + blocks, + trunc, + seg_lens, + layout: if span_over.is_some() { + Some(layout.to_string()) + } else { + None + }, + names: row_names, + w_offs: row_w_offs, + w_pos: row_w_pos, + span_over, + n_tokens, + head: dispatch.head.to_string(), + head_reason: if dispatch.reason.is_empty() { + None + } else { + Some(dispatch.reason.to_string()) + }, + window: false, + }) + } +} + +fn block_len(block: &Block) -> usize { + match block { + Block::Question { ids, .. } => ids.len(), + Block::Span { ids, .. } => ids.len(), + } +} diff --git a/src/models/vela/native/src/state.rs b/src/models/vela/native/src/state.rs new file mode 100644 index 00000000..d33e42bb --- /dev/null +++ b/src/models/vela/native/src/state.rs @@ -0,0 +1,235 @@ +//! DEV2.0 canonical JSON and the state → typed-parts mapping, mirroring +//! the pinned runtime's `canonical_json`, `content_text` and +//! `system_one_state` exactly (the frozen `native-plan-*` fixtures are +//! the parity target). + +use serde_json::Value; +use std::collections::BTreeMap; + +use crate::contract::RequestError; + +/// `json.dumps(ensure_ascii=False, sort_keys=True, separators=(",", ":"))` +/// — key-sorted compact JSON, non-ASCII literal. +pub fn canonical(value: &Value) -> String { + match value { + Value::Object(map) => { + let mut keys: Vec<&String> = map.keys().collect(); + keys.sort(); + let parts: Vec = keys + .iter() + .map(|key| { + format!( + "{}:{}", + serde_json::to_string(key).unwrap(), + canonical(&map[*key]) + ) + }) + .collect(); + format!("{{{}}}", parts.join(",")) + } + Value::Array(items) => { + let parts: Vec = items.iter().map(canonical).collect(); + format!("[{}]", parts.join(",")) + } + other => serde_json::to_string(other).unwrap(), + } +} + +/// JSON content → model text: strings as they are, anything else as +/// canonical JSON. +pub fn content_text(value: &Value) -> String { + match value { + Value::String(text) => text.clone(), + other => canonical(other), + } +} + +/// The state-key → part-role table (unknown keys land in `context`). +pub fn key_role(key: &str) -> &'static str { + match key.to_ascii_lowercase().as_str() { + "request" | "user" | "prompt" => "user", + "answer" | "response" => "answer", + _ => "context", + } +} + +/// One rendered state part, in user / context / answer order. +#[derive(Debug, Clone, serde::Serialize, PartialEq)] +pub struct Part { + pub r#type: String, + pub text: String, +} + +/// A named state field's span inside its part's text (Unicode code +/// points, as on the reference). +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct FieldRange { + pub role: &'static str, + pub start: usize, + pub end: usize, +} + +#[derive(Debug)] +pub struct StateParts { + pub parts: Vec, + pub fields: BTreeMap, +} + +fn state_error(msg: &str) -> RequestError { + RequestError::envelope(&["state"], msg, "value_error") +} + +/// The SystemOne state → (parts, fields) mapping, mirroring +/// `system_one_state`: a plain string is one user part; a JSON array is +/// canonicalized into one; an object groups its keys by role (request / +/// user / prompt → user; source / context / document / tools / +/// previous_answer → context; answer / response → answer; unknown keys +/// → context) in user / context / answer order — one known-key entry +/// per role renders bare, several (or an unknown key) render as +/// `"key:\n"` blocks joined by blank lines. +pub fn system_one_state(state: &Value) -> Result { + // A string holding a JSON object is read as that object. + let state = match state { + Value::String(text) => { + let stripped = text.trim_start(); + if stripped.starts_with('{') { + if let Value::Object(object) = + serde_json::from_str::(stripped).unwrap_or(Value::Null) + { + Value::Object(object) + } else { + state.clone() + } + } else { + state.clone() + } + } + other => other.clone(), + }; + match &state { + Value::String(text) => { + if text.trim().is_empty() { + return Err(state_error("state must not be empty or whitespace")); + } + let mut fields = BTreeMap::new(); + fields.insert( + "request".into(), + FieldRange { + role: "user", + start: 0, + end: text.chars().count(), + }, + ); + Ok(StateParts { + parts: vec![Part { + r#type: "user".into(), + text: text.clone(), + }], + fields, + }) + } + Value::Array(_) => { + let text = canonical(&state); + let mut fields = BTreeMap::new(); + fields.insert( + "request".into(), + FieldRange { + role: "user", + start: 0, + end: text.chars().count(), + }, + ); + Ok(StateParts { + parts: vec![Part { + r#type: "user".into(), + text, + }], + fields, + }) + } + Value::Object(object) if !object.is_empty() => { + // Group per role, keeping the object's insertion order inside + // each role and remembering which entries had a known key. + let mut by_role: [Vec<(&String, &Value, bool)>; 3] = + [Vec::new(), Vec::new(), Vec::new()]; + for (key, value) in object { + let lower = key.to_ascii_lowercase(); + let known = matches!( + lower.as_str(), + "request" + | "user" + | "prompt" + | "source" + | "context" + | "document" + | "tools" + | "previous_answer" + | "answer" + | "response" + ); + let slot = match key_role(&lower) { + "user" => 0, + "answer" => 2, + _ => 1, + }; + by_role[slot].push((key, value, known)); + } + let mut parts = Vec::new(); + let mut fields = BTreeMap::new(); + for (role, items) in [ + ("user", &by_role[0]), + ("context", &by_role[1]), + ("answer", &by_role[2]), + ] { + if items.is_empty() { + continue; + } + let mut text = String::new(); + let single = items.len() == 1 && items[0].2; + for (index, (key, value, _)) in items.iter().enumerate() { + let rendered = content_text(value); + if single { + fields.insert( + (*key).clone(), + FieldRange { + role, + start: 0, + end: rendered.chars().count(), + }, + ); + text = rendered; + } else { + if index > 0 { + text.push_str("\n\n"); + } + let head = format!("{key}:\n"); + let start = text.chars().count() + head.chars().count(); + fields.insert( + (*key).clone(), + FieldRange { + role, + start, + end: start + rendered.chars().count(), + }, + ); + text.push_str(&head); + text.push_str(&rendered); + } + } + if text.trim().is_empty() { + return Err(state_error(&format!( + "the {role} part of the state is empty" + ))); + } + parts.push(Part { + r#type: role.into(), + text, + }); + } + Ok(StateParts { parts, fields }) + } + _ => Err(state_error( + "state must be a non-empty string, JSON object or JSON array", + )), + } +} diff --git a/src/models/vela/native/src/words.rs b/src/models/vela/native/src/words.rs new file mode 100644 index 00000000..3b947fd3 --- /dev/null +++ b/src/models/vela/native/src/words.rs @@ -0,0 +1,129 @@ +//! Word units and first-token mapping, mirroring the pinned runtime's +//! `schema_lib_r4` `_WORD` splitter (GLiNER2 whitespace splitter + CJK +//! one unit per character + the r4 URL/email/handle rule) and +//! `word_first_tokens`. Offsets are Unicode code points everywhere, as +//! on the reference. + +use onig::Regex; + +/// The CJK ranges of the reference `_WORD` pattern, verbatim. +const CJK: &str = "⺀-⿿　-〿぀-ヿ㄀-ㇿ㐀-䶿一-鿿ꀀ-꓏가-힯豈-﫿︰-﹏＀-￯"; +/// Trailing punctuation the URL rule strips. +const TRAIL: &str = ".,;:!?)\\]}>\"'\u{201d}\u{2019}\u{bb}"; + +fn word_pattern() -> Regex { + let pattern = concat!( + "(?:https?://|www\\.)[^\\s{cjk}]+?(?=[{trail}]*(?![^\\s{cjk}]))", + "|[a-z0-9._%+-]+@[a-z0-9.-]+\\.[a-z]{2,}", + "|@[a-z0-9_]+", + "|[{cjk}]", + "|[^\\W{cjk}]+(?:[-_][^\\W{cjk}]+)*", + "|\\S", + ); + let pattern = pattern.replace("{cjk}", CJK).replace("{trail}", TRAIL); + Regex::with_options( + &pattern, + onig::RegexOptions::REGEX_OPTION_IGNORECASE, + onig::Syntax::ruby(), + ) + .expect("the reference word pattern must compile") +} + +/// Start-of-char byte offsets plus the end sentinel; index it with +/// `partition_point(|&start| start < byte)` to map a byte offset to its +/// code-point offset. +fn char_starts(text: &str) -> Vec { + text.char_indices() + .map(|(byte, _)| byte) + .chain(std::iter::once(text.len())) + .collect() +} + +/// The (start, end) word ranges of `text`, in code points. +pub fn split_words(text: &str) -> Vec<(usize, usize)> { + let regex = word_pattern(); + // Byte offsets → code-point offsets. + let char_of_byte = char_starts(text); + let to_char = |byte: usize| -> usize { char_of_byte.partition_point(|&start| start < byte) }; + regex + .find_iter(text) + .map(|(start, end)| (to_char(start), to_char(end))) + .collect() +} + +/// (word char offsets (W, 2), each word's first overlapping sub-word +/// token index), mirroring `word_first_tokens`: the first token whose +/// cumulative end passes the word start, skipping empty tokens. +pub fn word_first_tokens( + text: &str, + offsets: &[(usize, usize)], +) -> (Vec<(usize, usize)>, Vec) { + let words = split_words(text); + if words.is_empty() || offsets.is_empty() { + return (Vec::new(), Vec::new()); + } + let valid: Vec = offsets.iter().map(|(s, e)| e > s).collect(); + let mut cummax_end: Vec = Vec::with_capacity(offsets.len()); + let mut running: i64 = -1; + for (index, (_start, end)) in offsets.iter().enumerate() { + if valid[index] { + running = running.max(*end as i64); + } + cummax_end.push(running); + } + let mut word_offsets = Vec::new(); + let mut first_tokens = Vec::new(); + for (a, b) in words { + let mut t = cummax_end.partition_point(|&end| end <= a as i64); + while t < offsets.len() && !valid[t] { + t += 1; + } + if t < offsets.len() && offsets[t].0 < b { + word_offsets.push((a, b)); + first_tokens.push(t); + } + } + (word_offsets, first_tokens) +} + +/// The reference's CJK class as a range test (the same ranges the word +/// splitter carries). +pub fn is_cjk_char(ch: char) -> bool { + // The reference class has one contiguous run from U+8C48 to + // U+FAFF and ends the Yi block at U+A4CF (not U+A48F). + matches!(ch as u32, + 0x2E80..=0x2FFF | 0x3000..=0x303F | 0x3040..=0x30FF | 0x3100..=0x31FF + | 0x3400..=0x4DBF | 0x4E00..=0x9FFF | 0xA000..=0xA4CF | 0xAC00..=0xD7AF + | 0x8C48..=0xFAFF | 0xFE30..=0xFE4F | 0xFF00..=0xFFEF) +} + +/// The reference's `_UNIT` split (decode-time units): URL bodies, runs +/// of word characters (excluding CJK, `_` excluded by `[^\W_]`) plus +/// `@._-+`, or any non-space char. Returns each char's unit id, -1 +/// outside units — the decode's `u` array. +pub fn unit_ids(chars: &[char]) -> Vec { + let mut ids = vec![-1i64; chars.len()]; + let text: String = chars.iter().collect(); + let pattern = concat!( + "(?:https?://|www\\.)[^\\s\\]\\)\\}>「」『』\"'“”{cjk}]+?(?=[.,;:!?]*(?![^\\s\\]\\)\\}>「」『』\"'“”{cjk}]))", + "|(?:(?![{cjk}])[^\\W_]|[@._\\-+])+", + "|\\S", + ) + .replace("{cjk}", CJK); + let regex = Regex::with_options( + &pattern, + onig::RegexOptions::REGEX_OPTION_IGNORECASE, + onig::Syntax::ruby(), + ) + .expect("the reference unit pattern must compile"); + // find_iter yields byte offsets; ids indexes characters. + let char_of_byte = char_starts(&text); + let to_char = |byte: usize| -> usize { char_of_byte.partition_point(|&start| start < byte) }; + for (unit, (start, end)) in regex.find_iter(&text).enumerate() { + let (start, end) = (to_char(start), to_char(end)); + for id in ids.iter_mut().take(end).skip(start) { + *id = unit as i64; + } + } + ids +} diff --git a/src/models/vela/postprocess.py b/src/models/vela/postprocess.py index 141b6938..907acb9b 100644 --- a/src/models/vela/postprocess.py +++ b/src/models/vela/postprocess.py @@ -91,26 +91,35 @@ def _question_ids_of(questions: Mapping[str, Any], *kinds: str) -> list[str]: ] -def finish_response(questions: Mapping[str, Any], response: dict[str, Any]) -> dict[str, Any]: +def finish_response( + questions: Mapping[str, Any], + response: dict[str, Any], + *, + hybrid: bool = False, +) -> dict[str, Any]: """Check the runtime response against the admitted questions and pass it through. The pinned runtime owns every value in this response; these checks only refuse a response that could not be reconstructed for the - request that produced it (wrong key set or order, a hybrid-only + request that produced it (wrong key set or order, a member-invented field, broken usage accounting, an answer of unexpected shape). + ``hybrid`` selects the member: the hybrid executor's responses carry + ``span_heads`` alongside ``spans`` (upstream emits both together, + keyed by the span question ids); the 0.3B member's never do. """ if not isinstance(response, dict): raise RuntimeError("the pinned runtime returned a non-object response") keys = set(response) foreign = keys & HYBRID_ONLY_FIELDS - if foreign: + if foreign and not hybrid: raise RuntimeError( f"the pinned runtime returned hybrid-only field(s) {sorted(foreign)}; " "this worker serves the 0.3B member" ) missing = [field for field in REQUIRED_FIELDS if field not in response] - if missing or not keys <= TOP_LEVEL_FIELDS: + allowed = TOP_LEVEL_FIELDS | HYBRID_ONLY_FIELDS if hybrid else TOP_LEVEL_FIELDS + if missing or not keys <= allowed: raise RuntimeError("the pinned runtime returned an unexpected response envelope") answers = response["answers"] expected = expected_answer_order(questions) @@ -126,6 +135,14 @@ def finish_response(questions: Mapping[str, Any], response: dict[str, Any]) -> d raise RuntimeError(f"{field} do not match the request's {field} questions") elif field in response: raise RuntimeError(f"the pinned runtime returned {field} without {field} questions") + if hybrid: + span_ids = _question_ids_of(questions, "span") + if span_ids: + heads = response.get("span_heads") + if not isinstance(heads, dict) or list(heads) != span_ids: + raise RuntimeError("span_heads do not match the request's span questions") + elif "span_heads" in response: + raise RuntimeError("the pinned runtime returned span_heads without span questions") usage = response["usage"] if ( not isinstance(usage, dict) diff --git a/src/models/vela/protocol.py b/src/models/vela/protocol.py index de5dd2db..d9341206 100644 --- a/src/models/vela/protocol.py +++ b/src/models/vela/protocol.py @@ -24,10 +24,10 @@ # hybrid members serve their config.json "model_name" instead. MODEL_NAME = "vllm-sr/Vela-2.0-0.3B" # Slice-scoped transport cap. Upstream vela2_serve.py sets no total-body -# limit (its only size boundary is the 8,192-token input window); a text -# request bounded by that window is orders of magnitude smaller, so this -# cap never binds a valid request. A deliberate difference recorded with -# the integration. +# limit (its only size boundary is the member's token input window); a +# text request bounded by that window is orders of magnitude smaller, so +# this cap never binds a valid request. A deliberate difference recorded +# with the integration. MAX_BODY = 8 * 1024 * 1024 # The envelope fields, in upstream's missing-field check order. REQUEST_FIELDS = ("state", "model", "questions") diff --git a/tests/vela/fixtures/native-plan-example-request-cjk.json b/tests/vela/fixtures/native-plan-example-request-cjk.json new file mode 100644 index 00000000..01973a26 --- /dev/null +++ b/tests/vela/fixtures/native-plan-example-request-cjk.json @@ -0,0 +1 @@ +{"case": "example-request-cjk", "request": {"model": "vllm-sr/Vela-2.0-0.8B", "state": {"request": "客户张伟（zhang.wei@example.cn）于2026年10月1日在上海市浦东新区世纪大道100号办理了业务，联系电话13800138000。", "source": "客户的业务申请需要在三个工作日内审核完成。", "answer": "客户的业务已经即时完成了。"}, "questions": {"pii": {"type": "span", "instructions": "Which spans are personal information?", "criteria": {"PERSON": "a person's name", "EMAIL_ADDRESS": "an e-mail address", "PHONE_NUMBER": "a telephone number", "STREET_ADDRESS": "a street address or postal address"}, "over": "request"}, "unsupported": {"type": "span", "instructions": "Which spans of the answer are not supported by the context?", "criteria": {"unsupported": "a claim not supported by the context"}}, "billing": {"type": "noul", "instructions": "这个请求是否与账单有关？"}}}, "predict_request": {"parts": [{"type": "user", "text": "客户张伟（zhang.wei@example.cn）于2026年10月1日在上海市浦东新区世纪大道100号办理了业务，联系电话13800138000。"}, {"type": "context", "text": "客户的业务申请需要在三个工作日内审核完成。"}, {"type": "answer", "text": "客户的业务已经即时完成了。"}], "questions": [{"id": "pii", "type": "span", "text": "Which spans are personal information?", "target_part": "user", "labels": [{"name": "PERSON", "description": "a person's name"}, {"name": "EMAIL_ADDRESS", "description": "an e-mail address"}, {"name": "PHONE_NUMBER", "description": "a telephone number"}, {"name": "STREET_ADDRESS", "description": "a street address or postal address"}]}, {"id": "unsupported", "type": "span", "text": "Which spans of the answer are not supported by the context?", "target_part": "answer", "labels": [{"name": "unsupported", "description": "a claim not supported by the context"}]}, {"id": "billing", "type": "choice", "text": "这个请求是否与账单有关？", "target_part": ["user", "context", "answer"], "options": [{"name": "no", "description": "No. The statement or question is not satisfied."}, {"name": "yes", "description": "Yes. The statement or question is satisfied."}]}]}, "plan": [{"qid": "pii", "type": "span", "criteria": {"PERSON": "a person's name", "EMAIL_ADDRESS": "an e-mail address", "PHONE_NUMBER": "a telephone number", "STREET_ADDRESS": "a street address or postal address"}, "range": [0, 75], "part": "user"}, {"qid": "unsupported", "type": "span", "criteria": {"unsupported": "a claim not supported by the context"}, "range": null, "part": "answer"}, {"qid": "billing", "type": "noul", "criteria": {}, "range": null, "part": ["user", "context", "answer"]}], "rows": [{"p_ids": [1905, 25, 198, 27, 22446, 3364, 420, 846, 731, 198, 96989, 138667, 9616, 89, 20032, 13, 26234, 34314, 21377, 7313, 95928, 17, 15, 17, 21, 95859, 16, 15, 96212, 16, 95971, 95772, 108914, 121458, 99524, 108611, 16, 15, 15, 96389, 134395, 97192, 3709, 108672, 16, 18, 23, 15, 15, 16, 18, 23, 15, 15, 15, 1710, 198, 510, 22446, 29, 198, 27, 22446, 3364, 420, 2078, 731, 198, 109480, 97192, 97379, 111341, 98868, 96069, 99671, 100616, 97235, 1710, 198, 510, 22446, 29, 198, 27, 22446, 3364, 420, 8944, 731, 198, 109480, 97192, 96790, 111267, 104154, 1710, 198, 510, 22446, 29, 198], "blocks": [{"kind": "q", "ids": [271, 6065, 913, 25, 5576, 198, 6196, 25, 1156, 11, 2193, 11, 4087, 198, 14162, 25, 198, 96488, 101729, 147343, 119526, 97495, 10992, 198, 3670, 25, 198, 13886, 29, 198, 4754, 4532, 3147, 2665, 13, 561, 4952, 466, 3296, 369, 524, 18933, 46292, 767, 3147, 2083, 8934, 198, 510, 1979, 29, 198, 13886, 29, 198, 4754, 4532, 3147, 9175, 13, 561, 4952, 466, 3296, 369, 18933, 46292, 767, 3147, 9405, 8934, 198, 510, 1979, 29, 198, 13886, 29, 198, 4754, 4532, 3147, 3928, 314, 279, 6625, 2519, 16298, 46292, 767, 3147, 58, 55553, 60, 8934, 198, 510, 1979, 29, 271, 3270, 279, 3074, 2904, 1786, 7021, 539, 279, 2193, 321, 10895, 13, 198, 72298, 25], "ends": [50, 74, 98], "query": 114, "qid": "billing", "qtype": "choice", "names": ["no", "yes"], "has_abs": true}, {"kind": "span", "ends": [41, 62, 82, 106], "label_ranges": [[22, 41], [42, 62], [63, 82], [83, 106]], "words_in_block": true, "ids": [271, 6065, 913, 25, 9111, 198, 6196, 25, 1156, 198, 14162, 25, 198, 22365, 42824, 513, 4204, 1928, 30, 198, 22933, 25, 198, 14057, 29, 198, 4754, 4532, 3147, 64, 1637, 579, 803, 2129, 767, 3147, 89974, 8934, 198, 510, 1448, 29, 198, 14057, 29, 198, 4754, 4532, 3147, 276, 378, 11139, 2534, 2129, 767, 3147, 34714, 19522, 8934, 198, 510, 1448, 29, 198, 14057, 29, 198, 4754, 4532, 3147, 64, 19919, 1324, 2129, 767, 3147, 72446, 18777, 8934, 198, 510, 1448, 29, 198, 14057, 29, 198, 4754, 4532, 3147, 64, 8329, 2534, 466, 38436, 2534, 2129, 767, 3147, 759, 54046, 19522, 8934, 198, 510, 1448, 29, 271, 1138, 25, 198, 27, 22446, 3364, 420, 846, 731, 198, 96989, 138667, 9616, 89, 20032, 13, 26234, 34314, 21377, 7313, 95928, 17, 15, 17, 21, 95859, 16, 15, 96212, 16, 95971, 95772, 108914, 121458, 99524, 108611, 16, 15, 15, 96389, 134395, 97192, 3709, 108672, 16, 18, 23, 15, 15, 16, 18, 23, 15, 15, 15, 1710, 198, 510, 22446, 29, 198, 271, 8679, 1396, 9111, 314, 279, 2100, 1414, 421, 22651, 799, 314, 279, 8928, 13, 198, 6205, 581, 25], "query": 187, "w_pos": [118, 118, 119, 119, 120, 121, 127, 128, 129, 133, 134, 136, 137, 138, 139, 140, 140, 140, 141, 141, 141, 141, 142, 142, 143, 143, 144, 147, 148, 148, 148, 149, 149, 150, 151, 151, 151, 151, 152, 163]}], "trunc": {"shrink_cut": 0, "protected_cut": 0}, "seg_lens": {"user": 46, "context": 10, "answer": 6}, "layout": "c", "names": ["PERSON", "EMAIL_ADDRESS", "PHONE_NUMBER", "STREET_ADDRESS"], "w_offs": [[0, 1], [1, 2], [2, 3], [3, 4], [4, 5], [5, 25], [25, 26], [26, 27], [27, 31], [31, 32], [32, 34], [34, 35], [35, 36], [36, 37], [37, 38], [38, 39], [39, 40], [40, 41], [41, 42], [42, 43], [43, 44], [44, 45], [45, 46], [46, 47], [47, 48], [48, 49], [49, 52], [52, 53], [53, 54], [54, 55], [55, 56], [56, 57], [57, 58], [58, 59], [59, 60], [60, 61], [61, 62], [62, 63], [63, 74], [74, 75]], "w_pos": [118, 118, 119, 119, 120, 121, 127, 128, 129, 133, 134, 136, 137, 138, 139, 140, 140, 140, 141, 141, 141, 141, 142, 142, 143, 143, 144, 147, 148, 148, 148, 149, 149, 150, 151, 151, 151, 151, 152, 163], "span_over": "user", "n_tokens": 404, "head": "router", "head_reason": "id"}, {"p_ids": [1905, 25, 198, 27, 22446, 3364, 420, 846, 731, 198, 96989, 138667, 9616, 89, 20032, 13, 26234, 34314, 21377, 7313, 95928, 17, 15, 17, 21, 95859, 16, 15, 96212, 16, 95971, 95772, 108914, 121458, 99524, 108611, 16, 15, 15, 96389, 134395, 97192, 3709, 108672, 16, 18, 23, 15, 15, 16, 18, 23, 15, 15, 15, 1710, 198, 510, 22446, 29, 198, 27, 22446, 3364, 420, 2078, 731, 198, 109480, 97192, 97379, 111341, 98868, 96069, 99671, 100616, 97235, 1710, 198, 510, 22446, 29, 198, 27, 22446, 3364, 420, 8944, 731, 198, 109480, 97192, 96790, 111267, 104154, 1710, 198, 510, 22446, 29, 198], "blocks": [{"kind": "span", "ends": [50], "label_ranges": [[28, 50]], "words_in_block": true, "ids": [271, 6065, 913, 25, 9111, 198, 6196, 25, 4087, 198, 14162, 25, 198, 22365, 42824, 314, 279, 4087, 513, 524, 7021, 539, 279, 2193, 30, 198, 22933, 25, 198, 14057, 29, 198, 4754, 4532, 3147, 64, 3591, 524, 7021, 539, 279, 2193, 2129, 767, 3147, 78349, 8934, 198, 510, 1448, 29, 271, 1138, 25, 198, 27, 22446, 3364, 420, 8944, 731, 198, 109480, 97192, 96790, 111267, 104154, 1710, 198, 510, 22446, 29, 198, 271, 8679, 1396, 9111, 314, 279, 2100, 1414, 421, 22651, 799, 314, 279, 8928, 13, 198, 6205, 581, 25], "query": 91, "w_pos": [62, 62, 62, 63, 63, 64, 64, 65, 65, 66, 66, 66, 67]}], "trunc": {"shrink_cut": 0, "protected_cut": 0}, "seg_lens": {"user": 46, "context": 10, "answer": 6}, "layout": "c", "names": ["unsupported"], "w_offs": [[0, 1], [1, 2], [2, 3], [3, 4], [4, 5], [5, 6], [6, 7], [7, 8], [8, 9], [9, 10], [10, 11], [11, 12], [12, 13]], "w_pos": [62, 62, 62, 63, 63, 64, 64, 65, 65, 66, 66, 66, 67], "span_over": "answer", "n_tokens": 193, "head": "router", "head_reason": "label_set"}], "window_rows": []} diff --git a/tests/vela/fixtures/native-plan-example-request-score.json b/tests/vela/fixtures/native-plan-example-request-score.json new file mode 100644 index 00000000..fc67c0f8 --- /dev/null +++ b/tests/vela/fixtures/native-plan-example-request-score.json @@ -0,0 +1 @@ +{"case": "example-request-score", "request": {"model": "vllm-sr/Vela-2.0-0.8B", "state": "Hello, I would like to check the status of my order. It has been five days and I have heard nothing back.", "questions": {"tone": {"type": "score", "instructions": "How formal is the message?", "criteria": ["casual", "neutral", "formal"]}, "urgency": {"type": "score", "instructions": "How urgently does the sender need a reply?", "criteria": ["no rush", "soon", "today", "immediately"]}}}, "predict_request": {"parts": [{"type": "user", "text": "Hello, I would like to check the status of my order. It has been five days and I have heard nothing back."}], "questions": [{"id": "tone", "type": "score", "text": "How formal is the message?", "target_part": "user", "options": ["casual", "neutral", "formal"]}, {"id": "urgency", "type": "score", "text": "How urgently does the sender need a reply?", "target_part": "user", "options": ["no rush", "soon", "today", "immediately"]}]}, "plan": [{"qid": "tone", "type": "score", "criteria": ["casual", "neutral", "formal"], "range": null, "part": "user"}, {"qid": "urgency", "type": "score", "criteria": ["no rush", "soon", "today", "immediately"], "range": null, "part": "user"}], "rows": [{"p_ids": [1905, 25, 198, 27, 22446, 3364, 420, 846, 731, 198, 9419, 11, 353, 1000, 1040, 310, 1716, 279, 2552, 314, 821, 1906, 13, 1049, 682, 978, 4097, 2756, 321, 353, 599, 6408, 4161, 1142, 13, 198, 510, 22446, 29, 198], "blocks": [{"kind": "q", "ids": [271, 6065, 913, 25, 5284, 198, 6196, 25, 1156, 198, 14162, 25, 198, 4199, 15440, 369, 279, 1876, 30, 198, 3670, 25, 198, 13886, 29, 198, 4754, 4532, 58719, 767, 3147, 23543, 899, 8934, 198, 510, 1979, 29, 198, 13886, 29, 198, 4754, 4532, 58719, 767, 3147, 57555, 8934, 198, 510, 1979, 29, 198, 13886, 29, 198, 4754, 4532, 58719, 767, 3147, 610, 278, 8934, 198, 510, 1979, 29, 271, 3270, 279, 3074, 2904, 1786, 7021, 539, 279, 2193, 321, 10895, 13, 198, 72298, 25], "ends": [37, 52, 68], "query": 84, "qid": "tone", "qtype": "score", "names": ["casual", "neutral", "formal"], "has_abs": false}, {"kind": "q", "ids": [271, 6065, 913, 25, 5284, 198, 6196, 25, 1156, 198, 14162, 25, 198, 4199, 74000, 1503, 279, 4495, 1144, 264, 9559, 30, 198, 3670, 25, 198, 13886, 29, 198, 4754, 4532, 58719, 767, 3147, 2083, 12601, 8934, 198, 510, 1979, 29, 198, 13886, 29, 198, 4754, 4532, 58719, 767, 3147, 63826, 8934, 198, 510, 1979, 29, 198, 13886, 29, 198, 4754, 4532, 58719, 767, 3147, 29129, 8934, 198, 510, 1979, 29, 198, 13886, 29, 198, 4754, 4532, 58719, 767, 3147, 316, 41763, 8934, 198, 510, 1979, 29, 271, 3270, 279, 3074, 2904, 1786, 7021, 539, 279, 2193, 321, 10895, 13, 198, 72298, 25], "ends": [40, 55, 70, 86], "query": 102, "qid": "urgency", "qtype": "score", "names": ["no rush", "soon", "today", "immediately"], "has_abs": false}], "trunc": {"shrink_cut": 0, "protected_cut": 0}, "seg_lens": {"user": 25}, "layout": null, "n_tokens": 228, "head": "router", "head_reason": null}], "window_rows": []} diff --git a/tests/vela/fixtures/native-plan-example-request-set-threshold.json b/tests/vela/fixtures/native-plan-example-request-set-threshold.json new file mode 100644 index 00000000..d0ea9b8f --- /dev/null +++ b/tests/vela/fixtures/native-plan-example-request-set-threshold.json @@ -0,0 +1 @@ +{"case": "example-request-set-threshold", "request": {"model": "vllm-sr/Vela-2.0-0.8B", "state": "My card was charged twice and the parcel never arrived.", "questions": {"issues": {"type": "set", "instructions": "Which issues does the customer report?", "criteria": {"billing": "payments, charges, refunds or invoices", "shipping": "delivery of an order or a parcel", "login": "signing in, passwords or account access"}, "threshold": 0.15}}}, "predict_request": {"parts": [{"type": "user", "text": "My card was charged twice and the parcel never arrived."}], "questions": [{"id": "issues", "type": "set", "text": "Which issues does the customer report?", "target_part": "user", "options": [{"name": "billing", "description": "payments, charges, refunds or invoices"}, {"name": "shipping", "description": "delivery of an order or a parcel"}, {"name": "login", "description": "signing in, passwords or account access"}], "threshold": 0.15}]}, "plan": [{"qid": "issues", "type": "set", "criteria": {"billing": "payments, charges, refunds or invoices", "shipping": "delivery of an order or a parcel", "login": "signing in, passwords or account access"}, "range": null, "part": "user"}], "rows": [{"p_ids": [1905, 25, 198, 27, 22446, 3364, 420, 846, 731, 198, 4888, 3575, 557, 11102, 10598, 321, 279, 28350, 2496, 11136, 13, 198, 510, 22446, 29, 198], "blocks": [{"kind": "q", "ids": [271, 6065, 913, 25, 716, 198, 6196, 25, 1156, 198, 14162, 25, 198, 22365, 4562, 1503, 279, 5813, 1830, 30, 198, 3670, 25, 198, 13886, 29, 198, 4754, 4532, 3147, 60917, 11, 9892, 11, 70057, 466, 61893, 2129, 767, 3147, 37355, 8934, 198, 510, 1979, 29, 198, 13886, 29, 198, 4754, 4532, 3147, 31320, 314, 449, 1906, 466, 264, 28350, 2129, 767, 3147, 25660, 8934, 198, 510, 1979, 29, 198, 13886, 29, 198, 4754, 4532, 3147, 7506, 286, 303, 11, 32599, 466, 2605, 2528, 2129, 767, 3147, 3547, 8934, 198, 510, 1979, 29, 271, 3270, 1396, 2904, 7021, 539, 279, 2193, 321, 10895, 13, 198, 72298, 25], "ends": [45, 68, 92], "query": 106, "qid": "issues", "qtype": "set", "names": ["billing", "shipping", "login"], "has_abs": false}], "trunc": {"shrink_cut": 0, "protected_cut": 0}, "seg_lens": {"user": 11}, "layout": null, "n_tokens": 133, "head": "router", "head_reason": null}], "window_rows": []} diff --git a/tests/vela/fixtures/native-plan-example-request.json b/tests/vela/fixtures/native-plan-example-request.json new file mode 100644 index 00000000..09bab119 --- /dev/null +++ b/tests/vela/fixtures/native-plan-example-request.json @@ -0,0 +1 @@ +{"case": "example-request", "request": {"model": "vllm-sr/Vela-2.0-0.8B", "state": {"request": "Hi, I'm Tom Baker (tom.baker@example.com). What is the maximum daily dose of paracetamol for an adult?", "source": "For adults, the maximum dose of paracetamol is 4 grams in 24 hours, taken as 500 mg to 1 g every 4 to 6 hours.", "answer": "Adults can take up to 6 grams of paracetamol in 24 hours, in doses of 500 mg to 1 g every 4 to 6 hours."}, "questions": {"pii": {"type": "span", "instructions": "Which spans are personal information?", "criteria": {"AGE": "a person's age", "CREDIT_CARD": "a payment card number", "DATE_TIME": "a date, time or date of birth", "DOMAIN_NAME": "an internet domain name or website address", "EMAIL_ADDRESS": "an e-mail address", "GPE": "a country, city, state or other geopolitical place", "IBAN_CODE": "an international bank account number (IBAN)", "IP_ADDRESS": "an IPv4 or IPv6 address", "NRP": "a nationality, religious or political group", "ORGANIZATION": "the name of a company, institution or other organisation", "PERSON": "a person's name", "PHONE_NUMBER": "a telephone number", "STREET_ADDRESS": "a street address or postal address", "TITLE": "a personal or professional title such as Dr. or Mrs.", "US_DRIVER_LICENSE": "a US driver's licence number", "US_SSN": "a US social security number", "ZIP_CODE": "a postal or ZIP code"}, "over": "request"}, "halu": {"type": "span", "instructions": "Which spans of the answer are not supported by the context?", "criteria": {"unsupported": "a claim not supported by the context"}}, "domain": {"type": "choice", "instructions": "Which subject area is this request about?", "over": "request", "criteria": {"health": "medicine, clinical practice, nutrition, ageing or sexual health", "math": "arithmetic, algebra, geometry, statistics or other mathematics", "other": "a subject that fits none of the listed areas"}}}}, "predict_request": {"parts": [{"type": "user", "text": "Hi, I'm Tom Baker (tom.baker@example.com). What is the maximum daily dose of paracetamol for an adult?"}, {"type": "context", "text": "For adults, the maximum dose of paracetamol is 4 grams in 24 hours, taken as 500 mg to 1 g every 4 to 6 hours."}, {"type": "answer", "text": "Adults can take up to 6 grams of paracetamol in 24 hours, in doses of 500 mg to 1 g every 4 to 6 hours."}], "questions": [{"id": "pii", "type": "span", "text": "Which spans are personal information?", "target_part": "user", "labels": [{"name": "AGE", "description": "a person's age"}, {"name": "CREDIT_CARD", "description": "a payment card number"}, {"name": "DATE_TIME", "description": "a date, time or date of birth"}, {"name": "DOMAIN_NAME", "description": "an internet domain name or website address"}, {"name": "EMAIL_ADDRESS", "description": "an e-mail address"}, {"name": "GPE", "description": "a country, city, state or other geopolitical place"}, {"name": "IBAN_CODE", "description": "an international bank account number (IBAN)"}, {"name": "IP_ADDRESS", "description": "an IPv4 or IPv6 address"}, {"name": "NRP", "description": "a nationality, religious or political group"}, {"name": "ORGANIZATION", "description": "the name of a company, institution or other organisation"}, {"name": "PERSON", "description": "a person's name"}, {"name": "PHONE_NUMBER", "description": "a telephone number"}, {"name": "STREET_ADDRESS", "description": "a street address or postal address"}, {"name": "TITLE", "description": "a personal or professional title such as Dr. or Mrs."}, {"name": "US_DRIVER_LICENSE", "description": "a US driver's licence number"}, {"name": "US_SSN", "description": "a US social security number"}, {"name": "ZIP_CODE", "description": "a postal or ZIP code"}]}, {"id": "halu", "type": "span", "text": "Which spans of the answer are not supported by the context?", "target_part": "answer", "labels": [{"name": "unsupported", "description": "a claim not supported by the context"}]}, {"id": "domain", "type": "choice", "text": "Which subject area is this request about?", "target_part": "user", "options": [{"name": "health", "description": "medicine, clinical practice, nutrition, ageing or sexual health"}, {"name": "math", "description": "arithmetic, algebra, geometry, statistics or other mathematics"}, {"name": "other", "description": "a subject that fits none of the listed areas"}]}]}, "plan": [{"qid": "pii", "type": "span", "criteria": {"AGE": "a person's age", "CREDIT_CARD": "a payment card number", "DATE_TIME": "a date, time or date of birth", "DOMAIN_NAME": "an internet domain name or website address", "EMAIL_ADDRESS": "an e-mail address", "GPE": "a country, city, state or other geopolitical place", "IBAN_CODE": "an international bank account number (IBAN)", "IP_ADDRESS": "an IPv4 or IPv6 address", "NRP": "a nationality, religious or political group", "ORGANIZATION": "the name of a company, institution or other organisation", "PERSON": "a person's name", "PHONE_NUMBER": "a telephone number", "STREET_ADDRESS": "a street address or postal address", "TITLE": "a personal or professional title such as Dr. or Mrs.", "US_DRIVER_LICENSE": "a US driver's licence number", "US_SSN": "a US social security number", "ZIP_CODE": "a postal or ZIP code"}, "range": [0, 102], "part": "user"}, {"qid": "halu", "type": "span", "criteria": {"unsupported": "a claim not supported by the context"}, "range": null, "part": "answer"}, {"qid": "domain", "type": "choice", "criteria": {"health": "medicine, clinical practice, nutrition, ageing or sexual health", "math": "arithmetic, algebra, geometry, statistics or other mathematics", "other": "a subject that fits none of the listed areas"}, "range": [0, 102], "part": "user"}], "rows": [{"p_ids": [1905, 25, 198, 27, 22446, 3364, 420, 846, 731, 198, 12675, 11, 353, 2688, 8105, 27466, 318, 36164, 918, 4263, 34314, 877, 553, 3437, 369, 279, 6966, 7071, 18537, 314, 1298, 64955, 235579, 364, 449, 6474, 30, 198, 510, 22446, 29, 198, 27, 22446, 3364, 420, 2078, 731, 198, 2381, 12236, 11, 279, 6966, 18537, 314, 1298, 64955, 235579, 369, 220, 19, 32216, 303, 220, 17, 19, 3979, 11, 4284, 430, 220, 20, 15, 15, 13348, 310, 220, 16, 338, 1396, 220, 19, 310, 220, 21, 3979, 13, 198, 510, 22446, 29, 198, 27, 22446, 3364, 420, 8944, 731, 198, 50281, 82, 628, 1831, 685, 310, 220, 21, 32216, 314, 1298, 64955, 235579, 303, 220, 17, 19, 3979, 11, 303, 32902, 314, 220, 20, 15, 15, 13348, 310, 220, 16, 338, 1396, 220, 19, 310, 220, 21, 3979, 13, 198, 510, 22446, 29, 198], "blocks": [{"kind": "q", "ids": [271, 6065, 913, 25, 5576, 198, 6196, 25, 1156, 198, 14162, 25, 198, 22365, 3704, 2982, 369, 411, 1622, 883, 30, 198, 3670, 25, 198, 13886, 29, 198, 4754, 4532, 3147, 79600, 11, 14075, 6380, 11, 24780, 11, 76242, 466, 7017, 2730, 2129, 767, 3147, 11776, 8934, 198, 510, 1979, 29, 198, 13886, 29, 198, 4754, 4532, 3147, 277, 25092, 11, 45309, 11, 16547, 11, 12760, 466, 975, 36352, 2129, 767, 3147, 10065, 8934, 198, 510, 1979, 29, 198, 13886, 29, 198, 4754, 4532, 3147, 64, 3704, 421, 17759, 6643, 314, 279, 9711, 5493, 2129, 767, 3147, 1520, 8934, 198, 510, 1979, 29, 198, 13886, 29, 198, 4754, 4532, 3147, 3928, 314, 279, 6625, 2519, 16298, 46292, 767, 3147, 58, 55553, 60, 8934, 198, 510, 1979, 29, 271, 3270, 279, 3074, 2904, 1786, 7021, 539, 279, 2193, 321, 10895, 13, 198, 72298, 25], "ends": [50, 77, 102, 126], "query": 142, "qid": "domain", "qtype": "choice", "names": ["health", "math", "other"], "has_abs": true}, {"kind": "span", "ends": [41, 63, 88, 112, 133, 160, 186, 210, 234, 262, 282, 302, 326, 353, 377, 400, 422], "label_ranges": [[22, 41], [42, 63], [64, 88], [89, 112], [113, 133], [134, 160], [161, 186], [187, 210], [211, 234], [235, 262], [263, 282], [283, 302], [303, 326], [327, 353], [354, 377], [378, 400], [401, 422]], "words_in_block": true, "ids": [271, 6065, 913, 25, 9111, 198, 6196, 25, 1156, 198, 14162, 25, 198, 22365, 42824, 513, 4204, 1928, 30, 198, 22933, 25, 198, 14057, 29, 198, 4754, 4532, 3147, 64, 1637, 579, 4092, 2129, 767, 3147, 3466, 8934, 198, 510, 1448, 29, 198, 14057, 29, 198, 4754, 4532, 3147, 64, 7903, 3575, 1324, 2129, 767, 3147, 34, 54569, 44012, 8934, 198, 510, 1448, 29, 198, 14057, 29, 198, 4754, 4532, 3147, 64, 2321, 11, 854, 466, 2321, 314, 6968, 2129, 767, 3147, 6876, 9754, 8934, 198, 510, 1448, 29, 198, 14057, 29, 198, 4754, 4532, 3147, 276, 7361, 7697, 803, 466, 3779, 2534, 2129, 767, 3147, 55214, 4556, 8934, 198, 510, 1448, 29, 198, 14057, 29, 198, 4754, 4532, 3147, 276, 378, 11139, 2534, 2129, 767, 3147, 34714, 19522, 8934, 198, 510, 1448, 29, 198, 14057, 29, 198, 4754, 4532, 3147, 64, 3046, 11, 3177, 11, 1528, 466, 975, 83899, 1925, 2129, 767, 3147, 38, 1679, 8934, 198, 510, 1448, 29, 198, 14057, 29, 198, 4754, 4532, 3147, 276, 6286, 5883, 2605, 1324, 318, 3151, 1058, 77039, 767, 3147, 3151, 1058, 9724, 8934, 198, 510, 1448, 29, 198, 14057, 29, 198, 4754, 4532, 3147, 276, 30521, 19, 466, 30521, 21, 2534, 2129, 767, 3147, 3191, 19522, 8934, 198, 510, 1448, 29, 198, 14057, 29, 198, 4754, 4532, 3147, 64, 56284, 11, 10072, 466, 4788, 1809, 2129, 767, 3147, 45, 20993, 8934, 198, 510, 1448, 29, 198, 14057, 29, 198, 4754, 4532, 3147, 1719, 803, 314, 264, 2724, 11, 14467, 466, 975, 20767, 2129, 767, 3147, 43433, 1058, 47383, 8934, 198, 510, 1448, 29, 198, 14057, 29, 198, 4754, 4532, 3147, 64, 1637, 579, 803, 2129, 767, 3147, 89974, 8934, 198, 510, 1448, 29, 198, 14057, 29, 198, 4754, 4532, 3147, 64, 19919, 1324, 2129, 767, 3147, 72446, 18777, 8934, 198, 510, 1448, 29, 198, 14057, 29, 198, 4754, 4532, 3147, 64, 8329, 2534, 466, 38436, 2534, 2129, 767, 3147, 759, 54046, 19522, 8934, 198, 510, 1448, 29, 198, 14057, 29, 198, 4754, 4532, 3147, 64, 4204, 466, 6376, 2192, 1680, 430, 2832, 13, 466, 17093, 46292, 767, 3147, 48639, 8934, 198, 510, 1448, 29, 198, 14057, 29, 198, 4754, 4532, 3147, 64, 2201, 5404, 579, 28995, 1324, 2129, 767, 3147, 1966, 42585, 55630, 8934, 198, 510, 1448, 29, 198, 14057, 29, 198, 4754, 4532, 3147, 64, 2201, 3471, 4610, 1324, 2129, 767, 3147, 1966, 1063, 18393, 8934, 198, 510, 1448, 29, 198, 14057, 29, 198, 4754, 4532, 3147, 64, 38436, 466, 54778, 1970, 2129, 767, 3147, 59506, 9724, 8934, 198, 510, 1448, 29, 271, 1138, 25, 198, 27, 22446, 3364, 420, 846, 731, 198, 12675, 11, 353, 2688, 8105, 27466, 318, 36164, 918, 4263, 34314, 877, 553, 3437, 369, 279, 6966, 7071, 18537, 314, 1298, 64955, 235579, 364, 449, 6474, 30, 198, 510, 22446, 29, 198, 271, 8679, 1396, 9111, 314, 279, 2100, 1414, 421, 22651, 799, 314, 279, 8928, 13, 198, 6205, 581, 25], "query": 484, "w_pos": [434, 435, 436, 437, 437, 438, 439, 440, 441, 446, 446, 447, 448, 449, 450, 451, 452, 453, 454, 457, 458, 459, 460]}], "trunc": {"shrink_cut": 0, "protected_cut": 0}, "seg_lens": {"user": 27, "context": 39, "answer": 39}, "layout": "c", "names": ["AGE", "CREDIT_CARD", "DATE_TIME", "DOMAIN_NAME", "EMAIL_ADDRESS", "GPE", "IBAN_CODE", "IP_ADDRESS", "NRP", "ORGANIZATION", "PERSON", "PHONE_NUMBER", "STREET_ADDRESS", "TITLE", "US_DRIVER_LICENSE", "US_SSN", "ZIP_CODE"], "w_offs": [[0, 2], [2, 3], [4, 5], [5, 6], [6, 7], [8, 11], [12, 17], [18, 19], [19, 40], [40, 41], [41, 42], [43, 47], [48, 50], [51, 54], [55, 62], [63, 68], [69, 73], [74, 76], [77, 88], [89, 92], [93, 95], [96, 101], [101, 102]], "w_pos": [434, 435, 436, 437, 437, 438, 439, 440, 441, 446, 446, 447, 448, 449, 450, 451, 452, 453, 454, 457, 458, 459, 460], "span_over": "user", "n_tokens": 772, "head": "router", "head_reason": "id"}, {"p_ids": [1905, 25, 198, 27, 22446, 3364, 420, 846, 731, 198, 12675, 11, 353, 2688, 8105, 27466, 318, 36164, 918, 4263, 34314, 877, 553, 3437, 369, 279, 6966, 7071, 18537, 314, 1298, 64955, 235579, 364, 449, 6474, 30, 198, 510, 22446, 29, 198, 27, 22446, 3364, 420, 2078, 731, 198, 2381, 12236, 11, 279, 6966, 18537, 314, 1298, 64955, 235579, 369, 220, 19, 32216, 303, 220, 17, 19, 3979, 11, 4284, 430, 220, 20, 15, 15, 13348, 310, 220, 16, 338, 1396, 220, 19, 310, 220, 21, 3979, 13, 198, 510, 22446, 29, 198, 27, 22446, 3364, 420, 8944, 731, 198, 50281, 82, 628, 1831, 685, 310, 220, 21, 32216, 314, 1298, 64955, 235579, 303, 220, 17, 19, 3979, 11, 303, 32902, 314, 220, 20, 15, 15, 13348, 310, 220, 16, 338, 1396, 220, 19, 310, 220, 21, 3979, 13, 198, 510, 22446, 29, 198], "blocks": [{"kind": "span", "ends": [50], "label_ranges": [[28, 50]], "words_in_block": true, "ids": [271, 6065, 913, 25, 9111, 198, 6196, 25, 4087, 198, 14162, 25, 198, 22365, 42824, 314, 279, 4087, 513, 524, 7021, 539, 279, 2193, 30, 198, 22933, 25, 198, 14057, 29, 198, 4754, 4532, 3147, 64, 3591, 524, 7021, 539, 279, 2193, 2129, 767, 3147, 78349, 8934, 198, 510, 1448, 29, 271, 1138, 25, 198, 27, 22446, 3364, 420, 8944, 731, 198, 50281, 82, 628, 1831, 685, 310, 220, 21, 32216, 314, 1298, 64955, 235579, 303, 220, 17, 19, 3979, 11, 303, 32902, 314, 220, 20, 15, 15, 13348, 310, 220, 16, 338, 1396, 220, 19, 310, 220, 21, 3979, 13, 198, 510, 22446, 29, 198, 271, 8679, 1396, 9111, 314, 279, 2100, 1414, 421, 22651, 799, 314, 279, 8928, 13, 198, 6205, 581, 25], "query": 124, "w_pos": [62, 64, 65, 66, 67, 69, 70, 71, 72, 75, 77, 79, 80, 81, 82, 83, 85, 88, 89, 91, 92, 93, 95, 96, 98, 99, 100]}], "trunc": {"shrink_cut": 0, "protected_cut": 0}, "seg_lens": {"user": 27, "context": 39, "answer": 39}, "layout": "c", "names": ["unsupported"], "w_offs": [[0, 6], [7, 10], [11, 15], [16, 18], [19, 21], [22, 23], [24, 29], [30, 32], [33, 44], [45, 47], [48, 50], [51, 56], [56, 57], [58, 60], [61, 66], [67, 69], [70, 73], [74, 76], [77, 79], [80, 81], [82, 83], [84, 89], [90, 91], [92, 94], [95, 96], [97, 102], [102, 103]], "w_pos": [62, 64, 65, 66, 67, 69, 70, 71, 72, 75, 77, 79, 80, 81, 82, 83, 85, 88, 89, 91, 92, 93, 95, 96, 98, 99, 100], "span_over": "answer", "n_tokens": 269, "head": "router", "head_reason": "id"}], "window_rows": []} diff --git a/tests/vela/fixtures/native-plan-long-span-windowed.json b/tests/vela/fixtures/native-plan-long-span-windowed.json new file mode 100644 index 00000000..0210fa02 --- /dev/null +++ b/tests/vela/fixtures/native-plan-long-span-windowed.json @@ -0,0 +1 @@ +{"case": "long-span-windowed", "request": {"model": "vllm-sr/Vela-2.0-0.8B", "state": {"request": "Sentence 0 reports a routine delivery event. Sentence 1 reports a routine delivery event. Sentence 2 reports a routine delivery event. Sentence 3 reports a routine delivery event. Sentence 4 reports a routine delivery event. Sentence 5 reports a routine delivery event. Sentence 6 reports a routine delivery event. Sentence 7 reports a routine delivery event. Sentence 8 reports a routine delivery event. Sentence 9 reports a routine delivery event. Sentence 10 reports a routine delivery event. Sentence 11 reports a routine delivery event. Sentence 12 reports a routine delivery event. Sentence 13 reports a routine delivery event. Sentence 14 reports a routine delivery event. Sentence 15 reports a routine delivery event. Sentence 16 reports a routine delivery event. Sentence 17 reports a routine delivery event. Sentence 18 reports a routine delivery event. Sentence 19 reports a routine delivery event. Sentence 20 reports a routine delivery event. Sentence 21 reports a routine delivery event. Sentence 22 reports a routine delivery event. Sentence 23 reports a routine delivery event. Sentence 24 reports a routine delivery event. Sentence 25 reports a routine delivery event. Sentence 26 reports a routine delivery event. Sentence 27 reports a routine delivery event. Sentence 28 reports a routine delivery event. Sentence 29 reports a routine delivery event. Sentence 30 reports a routine delivery event. Sentence 31 reports a routine delivery event. Sentence 32 reports a routine delivery event. Sentence 33 reports a routine delivery event. Sentence 34 reports a routine delivery event. Sentence 35 reports a routine delivery event. Sentence 36 reports a routine delivery event. Sentence 37 reports a routine delivery event. Sentence 38 reports a routine delivery event. Sentence 39 reports a routine delivery event. Sentence 40 reports a routine delivery event. Sentence 41 reports a routine delivery event. Sentence 42 reports a routine delivery event. Sentence 43 reports a routine delivery event. Sentence 44 reports a routine delivery event. Sentence 45 reports a routine delivery event. Sentence 46 reports a routine delivery event. Sentence 47 reports a routine delivery event. Sentence 48 reports a routine delivery event. Sentence 49 reports a routine delivery event. Sentence 50 reports a routine delivery event. Sentence 51 reports a routine delivery event. Sentence 52 reports a routine delivery event. Sentence 53 reports a routine delivery event. Sentence 54 reports a routine delivery event. Sentence 55 reports a routine delivery event. Sentence 56 reports a routine delivery event. Sentence 57 reports a routine delivery event. Sentence 58 reports a routine delivery event. Sentence 59 reports a routine delivery event. Sentence 60 reports a routine delivery event. Sentence 61 reports a routine delivery event. Sentence 62 reports a routine delivery event. Sentence 63 reports a routine delivery event. Sentence 64 reports a routine delivery event. Sentence 65 reports a routine delivery event. Sentence 66 reports a routine delivery event. Sentence 67 reports a routine delivery event. Sentence 68 reports a routine delivery event. Sentence 69 reports a routine delivery event. Sentence 70 reports a routine delivery event. Sentence 71 reports a routine delivery event. Sentence 72 reports a routine delivery event. Sentence 73 reports a routine delivery event. Sentence 74 reports a routine delivery event. Sentence 75 reports a routine delivery event. Sentence 76 reports a routine delivery event. Sentence 77 reports a routine delivery event. Sentence 78 reports a routine delivery event. Sentence 79 reports a routine delivery event. Sentence 80 reports a routine delivery event. Sentence 81 reports a routine delivery event. Sentence 82 reports a routine delivery event. Sentence 83 reports a routine delivery event. Sentence 84 reports a routine delivery event. Sentence 85 reports a routine delivery event. Sentence 86 reports a routine delivery event. Sentence 87 reports a routine delivery event. Sentence 88 reports a routine delivery event. Sentence 89 reports a routine delivery event. Sentence 90 reports a routine delivery event. Sentence 91 reports a routine delivery event. Sentence 92 reports a routine delivery event. Sentence 93 reports a routine delivery event. Sentence 94 reports a routine delivery event. Sentence 95 reports a routine delivery event. Sentence 96 reports a routine delivery event. Sentence 97 reports a routine delivery event. Sentence 98 reports a routine delivery event. Sentence 99 reports a routine delivery event. Sentence 100 reports a routine delivery event. Sentence 101 reports a routine delivery event. Sentence 102 reports a routine delivery event. Sentence 103 reports a routine delivery event. Sentence 104 reports a routine delivery event. Sentence 105 reports a routine delivery event. Sentence 106 reports a routine delivery event. Sentence 107 reports a routine delivery event. Sentence 108 reports a routine delivery event. Sentence 109 reports a routine delivery event. Sentence 110 reports a routine delivery event. Sentence 111 reports a routine delivery event. Sentence 112 reports a routine delivery event. Sentence 113 reports a routine delivery event. Sentence 114 reports a routine delivery event. Sentence 115 reports a routine delivery event. Sentence 116 reports a routine delivery event. Sentence 117 reports a routine delivery event. Sentence 118 reports a routine delivery event. Sentence 119 reports a routine delivery event. Sentence 120 reports a routine delivery event. Sentence 121 reports a routine delivery event. Sentence 122 reports a routine delivery event. Sentence 123 reports a routine delivery event. Sentence 124 reports a routine delivery event. Sentence 125 reports a routine delivery event. Sentence 126 reports a routine delivery event. Sentence 127 reports a routine delivery event. Sentence 128 reports a routine delivery event. Sentence 129 reports a routine delivery event. Sentence 130 reports a routine delivery event. Sentence 131 reports a routine delivery event. Sentence 132 reports a routine delivery event. Sentence 133 reports a routine delivery event. Sentence 134 reports a routine delivery event. Sentence 135 reports a routine delivery event. Sentence 136 reports a routine delivery event. Sentence 137 reports a routine delivery event. Sentence 138 reports a routine delivery event. Sentence 139 reports a routine delivery event. Sentence 140 reports a routine delivery event. Sentence 141 reports a routine delivery event. Sentence 142 reports a routine delivery event. Sentence 143 reports a routine delivery event. Sentence 144 reports a routine delivery event. Sentence 145 reports a routine delivery event. Sentence 146 reports a routine delivery event. Sentence 147 reports a routine delivery event. Sentence 148 reports a routine delivery event. Sentence 149 reports a routine delivery event. Sentence 150 reports a routine delivery event. Sentence 151 reports a routine delivery event. Sentence 152 reports a routine delivery event. Sentence 153 reports a routine delivery event. Sentence 154 reports a routine delivery event. Sentence 155 reports a routine delivery event. Sentence 156 reports a routine delivery event. Sentence 157 reports a routine delivery event. Sentence 158 reports a routine delivery event. Sentence 159 reports a routine delivery event. Sentence 160 reports a routine delivery event. Sentence 161 reports a routine delivery event. Sentence 162 reports a routine delivery event. Sentence 163 reports a routine delivery event. Sentence 164 reports a routine delivery event. Sentence 165 reports a routine delivery event. Sentence 166 reports a routine delivery event. Sentence 167 reports a routine delivery event. Sentence 168 reports a routine delivery event. Sentence 169 reports a routine delivery event. Sentence 170 reports a routine delivery event. Sentence 171 reports a routine delivery event. Sentence 172 reports a routine delivery event. Sentence 173 reports a routine delivery event. Sentence 174 reports a routine delivery event. Sentence 175 reports a routine delivery event. Sentence 176 reports a routine delivery event. Sentence 177 reports a routine delivery event. Sentence 178 reports a routine delivery event. Sentence 179 reports a routine delivery event. Sentence 180 reports a routine delivery event. Sentence 181 reports a routine delivery event. Sentence 182 reports a routine delivery event. Sentence 183 reports a routine delivery event. Sentence 184 reports a routine delivery event. Sentence 185 reports a routine delivery event. Sentence 186 reports a routine delivery event. Sentence 187 reports a routine delivery event. Sentence 188 reports a routine delivery event. Sentence 189 reports a routine delivery event. Sentence 190 reports a routine delivery event. Sentence 191 reports a routine delivery event. Sentence 192 reports a routine delivery event. Sentence 193 reports a routine delivery event. Sentence 194 reports a routine delivery event. Sentence 195 reports a routine delivery event. Sentence 196 reports a routine delivery event. Sentence 197 reports a routine delivery event. Sentence 198 reports a routine delivery event. Sentence 199 reports a routine delivery event. Sentence 200 reports a routine delivery event. Sentence 201 reports a routine delivery event. Sentence 202 reports a routine delivery event. Sentence 203 reports a routine delivery event. Sentence 204 reports a routine delivery event. Sentence 205 reports a routine delivery event. Sentence 206 reports a routine delivery event. Sentence 207 reports a routine delivery event. Sentence 208 reports a routine delivery event. Sentence 209 reports a routine delivery event. Sentence 210 reports a routine delivery event. Sentence 211 reports a routine delivery event. Sentence 212 reports a routine delivery event. Sentence 213 reports a routine delivery event. Sentence 214 reports a routine delivery event. Sentence 215 reports a routine delivery event. Sentence 216 reports a routine delivery event. Sentence 217 reports a routine delivery event. Sentence 218 reports a routine delivery event. Sentence 219 reports a routine delivery event."}, "questions": {"claims": {"type": "span", "instructions": "Which spans mention a delivery?", "criteria": {"delivery": "a delivery event"}, "over": "request"}}}, "predict_request": {"parts": [{"type": "user", "text": "Sentence 0 reports a routine delivery event. Sentence 1 reports a routine delivery event. Sentence 2 reports a routine delivery event. Sentence 3 reports a routine delivery event. Sentence 4 reports a routine delivery event. Sentence 5 reports a routine delivery event. Sentence 6 reports a routine delivery event. Sentence 7 reports a routine delivery event. Sentence 8 reports a routine delivery event. Sentence 9 reports a routine delivery event. Sentence 10 reports a routine delivery event. Sentence 11 reports a routine delivery event. Sentence 12 reports a routine delivery event. Sentence 13 reports a routine delivery event. Sentence 14 reports a routine delivery event. Sentence 15 reports a routine delivery event. Sentence 16 reports a routine delivery event. Sentence 17 reports a routine delivery event. Sentence 18 reports a routine delivery event. Sentence 19 reports a routine delivery event. Sentence 20 reports a routine delivery event. Sentence 21 reports a routine delivery event. Sentence 22 reports a routine delivery event. Sentence 23 reports a routine delivery event. Sentence 24 reports a routine delivery event. Sentence 25 reports a routine delivery event. Sentence 26 reports a routine delivery event. Sentence 27 reports a routine delivery event. Sentence 28 reports a routine delivery event. Sentence 29 reports a routine delivery event. Sentence 30 reports a routine delivery event. Sentence 31 reports a routine delivery event. Sentence 32 reports a routine delivery event. Sentence 33 reports a routine delivery event. Sentence 34 reports a routine delivery event. Sentence 35 reports a routine delivery event. Sentence 36 reports a routine delivery event. Sentence 37 reports a routine delivery event. Sentence 38 reports a routine delivery event. Sentence 39 reports a routine delivery event. Sentence 40 reports a routine delivery event. Sentence 41 reports a routine delivery event. Sentence 42 reports a routine delivery event. Sentence 43 reports a routine delivery event. Sentence 44 reports a routine delivery event. Sentence 45 reports a routine delivery event. Sentence 46 reports a routine delivery event. Sentence 47 reports a routine delivery event. Sentence 48 reports a routine delivery event. Sentence 49 reports a routine delivery event. Sentence 50 reports a routine delivery event. Sentence 51 reports a routine delivery event. Sentence 52 reports a routine delivery event. Sentence 53 reports a routine delivery event. Sentence 54 reports a routine delivery event. Sentence 55 reports a routine delivery event. Sentence 56 reports a routine delivery event. Sentence 57 reports a routine delivery event. Sentence 58 reports a routine delivery event. Sentence 59 reports a routine delivery event. Sentence 60 reports a routine delivery event. Sentence 61 reports a routine delivery event. Sentence 62 reports a routine delivery event. Sentence 63 reports a routine delivery event. Sentence 64 reports a routine delivery event. Sentence 65 reports a routine delivery event. Sentence 66 reports a routine delivery event. Sentence 67 reports a routine delivery event. Sentence 68 reports a routine delivery event. Sentence 69 reports a routine delivery event. Sentence 70 reports a routine delivery event. Sentence 71 reports a routine delivery event. Sentence 72 reports a routine delivery event. Sentence 73 reports a routine delivery event. Sentence 74 reports a routine delivery event. Sentence 75 reports a routine delivery event. Sentence 76 reports a routine delivery event. Sentence 77 reports a routine delivery event. Sentence 78 reports a routine delivery event. Sentence 79 reports a routine delivery event. Sentence 80 reports a routine delivery event. Sentence 81 reports a routine delivery event. Sentence 82 reports a routine delivery event. Sentence 83 reports a routine delivery event. Sentence 84 reports a routine delivery event. Sentence 85 reports a routine delivery event. Sentence 86 reports a routine delivery event. Sentence 87 reports a routine delivery event. Sentence 88 reports a routine delivery event. Sentence 89 reports a routine delivery event. Sentence 90 reports a routine delivery event. Sentence 91 reports a routine delivery event. Sentence 92 reports a routine delivery event. Sentence 93 reports a routine delivery event. Sentence 94 reports a routine delivery event. Sentence 95 reports a routine delivery event. Sentence 96 reports a routine delivery event. Sentence 97 reports a routine delivery event. Sentence 98 reports a routine delivery event. Sentence 99 reports a routine delivery event. Sentence 100 reports a routine delivery event. Sentence 101 reports a routine delivery event. Sentence 102 reports a routine delivery event. Sentence 103 reports a routine delivery event. Sentence 104 reports a routine delivery event. Sentence 105 reports a routine delivery event. Sentence 106 reports a routine delivery event. Sentence 107 reports a routine delivery event. Sentence 108 reports a routine delivery event. Sentence 109 reports a routine delivery event. Sentence 110 reports a routine delivery event. Sentence 111 reports a routine delivery event. Sentence 112 reports a routine delivery event. Sentence 113 reports a routine delivery event. Sentence 114 reports a routine delivery event. Sentence 115 reports a routine delivery event. Sentence 116 reports a routine delivery event. Sentence 117 reports a routine delivery event. Sentence 118 reports a routine delivery event. Sentence 119 reports a routine delivery event. Sentence 120 reports a routine delivery event. Sentence 121 reports a routine delivery event. Sentence 122 reports a routine delivery event. Sentence 123 reports a routine delivery event. Sentence 124 reports a routine delivery event. Sentence 125 reports a routine delivery event. Sentence 126 reports a routine delivery event. Sentence 127 reports a routine delivery event. Sentence 128 reports a routine delivery event. Sentence 129 reports a routine delivery event. Sentence 130 reports a routine delivery event. Sentence 131 reports a routine delivery event. Sentence 132 reports a routine delivery event. Sentence 133 reports a routine delivery event. Sentence 134 reports a routine delivery event. Sentence 135 reports a routine delivery event. Sentence 136 reports a routine delivery event. Sentence 137 reports a routine delivery event. Sentence 138 reports a routine delivery event. Sentence 139 reports a routine delivery event. Sentence 140 reports a routine delivery event. Sentence 141 reports a routine delivery event. Sentence 142 reports a routine delivery event. Sentence 143 reports a routine delivery event. Sentence 144 reports a routine delivery event. Sentence 145 reports a routine delivery event. Sentence 146 reports a routine delivery event. Sentence 147 reports a routine delivery event. Sentence 148 reports a routine delivery event. Sentence 149 reports a routine delivery event. Sentence 150 reports a routine delivery event. Sentence 151 reports a routine delivery event. Sentence 152 reports a routine delivery event. Sentence 153 reports a routine delivery event. Sentence 154 reports a routine delivery event. Sentence 155 reports a routine delivery event. Sentence 156 reports a routine delivery event. Sentence 157 reports a routine delivery event. Sentence 158 reports a routine delivery event. Sentence 159 reports a routine delivery event. Sentence 160 reports a routine delivery event. Sentence 161 reports a routine delivery event. Sentence 162 reports a routine delivery event. Sentence 163 reports a routine delivery event. Sentence 164 reports a routine delivery event. Sentence 165 reports a routine delivery event. Sentence 166 reports a routine delivery event. Sentence 167 reports a routine delivery event. Sentence 168 reports a routine delivery event. Sentence 169 reports a routine delivery event. Sentence 170 reports a routine delivery event. Sentence 171 reports a routine delivery event. Sentence 172 reports a routine delivery event. Sentence 173 reports a routine delivery event. Sentence 174 reports a routine delivery event. Sentence 175 reports a routine delivery event. Sentence 176 reports a routine delivery event. Sentence 177 reports a routine delivery event. Sentence 178 reports a routine delivery event. Sentence 179 reports a routine delivery event. Sentence 180 reports a routine delivery event. Sentence 181 reports a routine delivery event. Sentence 182 reports a routine delivery event. Sentence 183 reports a routine delivery event. Sentence 184 reports a routine delivery event. Sentence 185 reports a routine delivery event. Sentence 186 reports a routine delivery event. Sentence 187 reports a routine delivery event. Sentence 188 reports a routine delivery event. Sentence 189 reports a routine delivery event. Sentence 190 reports a routine delivery event. Sentence 191 reports a routine delivery event. Sentence 192 reports a routine delivery event. Sentence 193 reports a routine delivery event. Sentence 194 reports a routine delivery event. Sentence 195 reports a routine delivery event. Sentence 196 reports a routine delivery event. Sentence 197 reports a routine delivery event. Sentence 198 reports a routine delivery event. Sentence 199 reports a routine delivery event. Sentence 200 reports a routine delivery event. Sentence 201 reports a routine delivery event. Sentence 202 reports a routine delivery event. Sentence 203 reports a routine delivery event. Sentence 204 reports a routine delivery event. Sentence 205 reports a routine delivery event. Sentence 206 reports a routine delivery event. Sentence 207 reports a routine delivery event. Sentence 208 reports a routine delivery event. Sentence 209 reports a routine delivery event. Sentence 210 reports a routine delivery event. Sentence 211 reports a routine delivery event. Sentence 212 reports a routine delivery event. Sentence 213 reports a routine delivery event. Sentence 214 reports a routine delivery event. Sentence 215 reports a routine delivery event. Sentence 216 reports a routine delivery event. Sentence 217 reports a routine delivery event. Sentence 218 reports a routine delivery event. Sentence 219 reports a routine delivery event."}], "questions": [{"id": "claims", "type": "span", "text": "Which spans mention a delivery?", "target_part": "user", "labels": [{"name": "delivery", "description": "a delivery event"}]}]}, "plan": [{"qid": "claims", "type": "span", "criteria": {"delivery": "a delivery event"}, "range": [0, 10229], "part": "user"}], "rows": [{"p_ids": [1905, 25, 198, 27, 22446, 3364, 420, 846, 731, 198, 81646, 220, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 24, 6473, 264, 13618, 9403, 1484, 13, 198, 510, 22446, 29, 198], "blocks": [{"kind": "span", "ends": [40], "label_ranges": [[22, 40]], "words_in_block": false, "ids": [271, 6065, 913, 25, 9111, 198, 6196, 25, 1156, 198, 14162, 25, 198, 22365, 42824, 6088, 264, 9403, 30, 198, 22933, 25, 198, 14057, 29, 198, 4754, 4532, 3147, 64, 9403, 1484, 2129, 767, 3147, 31320, 8934, 198, 510, 1448, 29, 271, 8679, 1396, 9111, 314, 279, 2100, 1414, 421, 22651, 799, 314, 279, 8928, 13, 198, 6205, 581, 25], "query": 59, "w_pos": [10, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 23, 24, 25, 26, 27, 28, 30, 31, 32, 33, 34, 35, 36, 37, 39, 40, 41, 42, 43, 44, 45, 46, 48, 49, 50, 51, 52, 53, 54, 55, 57, 58, 59, 60, 61, 62, 63, 64, 66, 67, 68, 69, 70, 71, 72, 73, 75, 76, 77, 78, 79, 80, 81, 82, 84, 85, 86, 87, 88, 89, 90, 91, 93, 94, 95, 96, 97, 98, 99, 100, 102, 104, 105, 106, 107, 108, 109, 110, 112, 114, 115, 116, 117, 118, 119, 120, 122, 124, 125, 126, 127, 128, 129, 130, 132, 134, 135, 136, 137, 138, 139, 140, 142, 144, 145, 146, 147, 148, 149, 150, 152, 154, 155, 156, 157, 158, 159, 160, 162, 164, 165, 166, 167, 168, 169, 170, 172, 174, 175, 176, 177, 178, 179, 180, 182, 184, 185, 186, 187, 188, 189, 190, 192, 194, 195, 196, 197, 198, 199, 200, 202, 204, 205, 206, 207, 208, 209, 210, 212, 214, 215, 216, 217, 218, 219, 220, 222, 224, 225, 226, 227, 228, 229, 230, 232, 234, 235, 236, 237, 238, 239, 240, 242, 244, 245, 246, 247, 248, 249, 250, 252, 254, 255, 256, 257, 258, 259, 260, 262, 264, 265, 266, 267, 268, 269, 270, 272, 274, 275, 276, 277, 278, 279, 280, 282, 284, 285, 286, 287, 288, 289, 290, 292, 294, 295, 296, 297, 298, 299, 300, 302, 304, 305, 306, 307, 308, 309, 310, 312, 314, 315, 316, 317, 318, 319, 320, 322, 324, 325, 326, 327, 328, 329, 330, 332, 334, 335, 336, 337, 338, 339, 340, 342, 344, 345, 346, 347, 348, 349, 350, 352, 354, 355, 356, 357, 358, 359, 360, 362, 364, 365, 366, 367, 368, 369, 370, 372, 374, 375, 376, 377, 378, 379, 380, 382, 384, 385, 386, 387, 388, 389, 390, 392, 394, 395, 396, 397, 398, 399, 400, 402, 404, 405, 406, 407, 408, 409, 410, 412, 414, 415, 416, 417, 418, 419, 420, 422, 424, 425, 426, 427, 428, 429, 430, 432, 434, 435, 436, 437, 438, 439, 440, 442, 444, 445, 446, 447, 448, 449, 450, 452, 454, 455, 456, 457, 458, 459, 460, 462, 464, 465, 466, 467, 468, 469, 470, 472, 474, 475, 476, 477, 478, 479, 480, 482, 484, 485, 486, 487, 488, 489, 490, 492, 494, 495, 496, 497, 498, 499, 500, 502, 504, 505, 506, 507, 508, 509, 510, 512, 514, 515, 516, 517, 518, 519, 520, 522, 524, 525, 526, 527, 528, 529, 530, 532, 534, 535, 536, 537, 538, 539, 540, 542, 544, 545, 546, 547, 548, 549, 550, 552, 554, 555, 556, 557, 558, 559, 560, 562, 564, 565, 566, 567, 568, 569, 570, 572, 574, 575, 576, 577, 578, 579, 580, 582, 584, 585, 586, 587, 588, 589, 590, 592, 594, 595, 596, 597, 598, 599, 600, 602, 604, 605, 606, 607, 608, 609, 610, 612, 614, 615, 616, 617, 618, 619, 620, 622, 624, 625, 626, 627, 628, 629, 630, 632, 634, 635, 636, 637, 638, 639, 640, 642, 644, 645, 646, 647, 648, 649, 650, 652, 654, 655, 656, 657, 658, 659, 660, 662, 664, 665, 666, 667, 668, 669, 670, 672, 674, 675, 676, 677, 678, 679, 680, 682, 684, 685, 686, 687, 688, 689, 690, 692, 694, 695, 696, 697, 698, 699, 700, 702, 704, 705, 706, 707, 708, 709, 710, 712, 714, 715, 716, 717, 718, 719, 720, 722, 724, 725, 726, 727, 728, 729, 730, 732, 734, 735, 736, 737, 738, 739, 740, 742, 744, 745, 746, 747, 748, 749, 750, 752, 754, 755, 756, 757, 758, 759, 760, 762, 764, 765, 766, 767, 768, 769, 770, 772, 774, 775, 776, 777, 778, 779, 780, 782, 784, 785, 786, 787, 788, 789, 790, 792, 794, 795, 796, 797, 798, 799, 800, 802, 804, 805, 806, 807, 808, 809, 810, 812, 814, 815, 816, 817, 818, 819, 820, 822, 824, 825, 826, 827, 828, 829, 830, 832, 834, 835, 836, 837, 838, 839, 840, 842, 844, 845, 846, 847, 848, 849, 850, 852, 854, 855, 856, 857, 858, 859, 860, 862, 864, 865, 866, 867, 868, 869, 870, 872, 874, 875, 876, 877, 878, 879, 880, 882, 884, 885, 886, 887, 888, 889, 890, 892, 894, 895, 896, 897, 898, 899, 900, 902, 904, 905, 906, 907, 908, 909, 910, 912, 914, 915, 916, 917, 918, 919, 920, 922, 924, 925, 926, 927, 928, 929, 930, 932, 934, 935, 936, 937, 938, 939, 940, 942, 944, 945, 946, 947, 948, 949, 950, 952, 954, 955, 956, 957, 958, 959, 960, 962, 964, 965, 966, 967, 968, 969, 970, 972, 974, 975, 976, 977, 978, 979, 980, 982, 984, 985, 986, 987, 988, 989, 990, 992, 994, 995, 996, 997, 998, 999, 1000, 1002, 1005, 1006, 1007, 1008, 1009, 1010, 1011, 1013, 1016, 1017, 1018, 1019, 1020, 1021, 1022, 1024, 1027, 1028, 1029, 1030, 1031, 1032, 1033, 1035, 1038, 1039, 1040, 1041, 1042, 1043, 1044, 1046, 1049, 1050, 1051, 1052, 1053, 1054, 1055, 1057, 1060, 1061, 1062, 1063, 1064, 1065, 1066, 1068, 1071, 1072, 1073, 1074, 1075, 1076, 1077, 1079, 1082, 1083, 1084, 1085, 1086, 1087, 1088, 1090, 1093, 1094, 1095, 1096, 1097, 1098, 1099, 1101, 1104, 1105, 1106, 1107, 1108, 1109, 1110, 1112, 1115, 1116, 1117, 1118, 1119, 1120, 1121, 1123, 1126, 1127, 1128, 1129, 1130, 1131, 1132, 1134, 1137, 1138, 1139, 1140, 1141, 1142, 1143, 1145, 1148, 1149, 1150, 1151, 1152, 1153, 1154, 1156, 1159, 1160, 1161, 1162, 1163, 1164, 1165, 1167, 1170, 1171, 1172, 1173, 1174, 1175, 1176, 1178, 1181, 1182, 1183, 1184, 1185, 1186, 1187, 1189, 1192, 1193, 1194, 1195, 1196, 1197, 1198, 1200, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1211, 1214, 1215, 1216, 1217, 1218, 1219, 1220, 1222, 1225, 1226, 1227, 1228, 1229, 1230, 1231, 1233, 1236, 1237, 1238, 1239, 1240, 1241, 1242, 1244, 1247, 1248, 1249, 1250, 1251, 1252, 1253, 1255, 1258, 1259, 1260, 1261, 1262, 1263, 1264, 1266, 1269, 1270, 1271, 1272, 1273, 1274, 1275, 1277, 1280, 1281, 1282, 1283, 1284, 1285, 1286, 1288, 1291, 1292, 1293, 1294, 1295, 1296, 1297, 1299, 1302, 1303, 1304, 1305, 1306, 1307, 1308, 1310, 1313, 1314, 1315, 1316, 1317, 1318, 1319, 1321, 1324, 1325, 1326, 1327, 1328, 1329, 1330, 1332, 1335, 1336, 1337, 1338, 1339, 1340, 1341, 1343, 1346, 1347, 1348, 1349, 1350, 1351, 1352, 1354, 1357, 1358, 1359, 1360, 1361, 1362, 1363, 1365, 1368, 1369, 1370, 1371, 1372, 1373, 1374, 1376, 1379, 1380, 1381, 1382, 1383, 1384, 1385, 1387, 1390, 1391, 1392, 1393, 1394, 1395, 1396, 1398, 1401, 1402, 1403, 1404, 1405, 1406, 1407, 1409, 1412, 1413, 1414, 1415, 1416, 1417, 1418, 1420, 1423, 1424, 1425, 1426, 1427, 1428, 1429, 1431, 1434, 1435, 1436, 1437, 1438, 1439, 1440, 1442, 1445, 1446, 1447, 1448, 1449, 1450, 1451, 1453, 1456, 1457, 1458, 1459, 1460, 1461, 1462, 1464, 1467, 1468, 1469, 1470, 1471, 1472, 1473, 1475, 1478, 1479, 1480, 1481, 1482, 1483, 1484, 1486, 1489, 1490, 1491, 1492, 1493, 1494, 1495, 1497, 1500, 1501, 1502, 1503, 1504, 1505, 1506, 1508, 1511, 1512, 1513, 1514, 1515, 1516, 1517, 1519, 1522, 1523, 1524, 1525, 1526, 1527, 1528, 1530, 1533, 1534, 1535, 1536, 1537, 1538, 1539, 1541, 1544, 1545, 1546, 1547, 1548, 1549, 1550, 1552, 1555, 1556, 1557, 1558, 1559, 1560, 1561, 1563, 1566, 1567, 1568, 1569, 1570, 1571, 1572, 1574, 1577, 1578, 1579, 1580, 1581, 1582, 1583, 1585, 1588, 1589, 1590, 1591, 1592, 1593, 1594, 1596, 1599, 1600, 1601, 1602, 1603, 1604, 1605, 1607, 1610, 1611, 1612, 1613, 1614, 1615, 1616, 1618, 1621, 1622, 1623, 1624, 1625, 1626, 1627, 1629, 1632, 1633, 1634, 1635, 1636, 1637, 1638, 1640, 1643, 1644, 1645, 1646, 1647, 1648, 1649, 1651, 1654, 1655, 1656, 1657, 1658, 1659, 1660, 1662, 1665, 1666, 1667, 1668, 1669, 1670, 1671, 1673, 1676, 1677, 1678, 1679, 1680, 1681, 1682, 1684, 1687, 1688, 1689, 1690, 1691, 1692, 1693, 1695, 1698, 1699, 1700, 1701, 1702, 1703, 1704, 1706, 1709, 1710, 1711, 1712, 1713, 1714, 1715, 1717, 1720, 1721, 1722, 1723, 1724, 1725, 1726, 1728, 1731, 1732, 1733, 1734, 1735, 1736, 1737, 1739, 1742, 1743, 1744, 1745, 1746, 1747, 1748, 1750, 1753, 1754, 1755, 1756, 1757, 1758, 1759, 1761, 1764, 1765, 1766, 1767, 1768, 1769, 1770, 1772, 1775, 1776, 1777, 1778, 1779, 1780, 1781, 1783, 1786, 1787, 1788, 1789, 1790, 1791, 1792, 1794, 1797, 1798, 1799, 1800, 1801, 1802, 1803, 1805, 1808, 1809, 1810, 1811, 1812, 1813, 1814, 1816, 1819, 1820, 1821, 1822, 1823, 1824, 1825, 1827, 1830, 1831, 1832, 1833, 1834, 1835, 1836, 1838, 1841, 1842, 1843, 1844, 1845, 1846, 1847, 1849, 1852, 1853, 1854, 1855, 1856, 1857, 1858, 1860, 1863, 1864, 1865, 1866, 1867, 1868, 1869, 1871, 1874, 1875, 1876, 1877, 1878, 1879, 1880, 1882, 1885, 1886, 1887, 1888, 1889, 1890, 1891, 1893, 1896, 1897, 1898, 1899, 1900, 1901, 1902, 1904, 1907, 1908, 1909, 1910, 1911, 1912, 1913, 1915, 1918, 1919, 1920, 1921, 1922, 1923, 1924, 1926, 1929, 1930, 1931, 1932, 1933, 1934, 1935, 1937, 1940, 1941, 1942, 1943, 1944, 1945, 1946, 1948, 1951, 1952, 1953, 1954, 1955, 1956, 1957, 1959, 1962, 1963, 1964, 1965, 1966, 1967, 1968, 1970, 1973, 1974, 1975, 1976, 1977, 1978, 1979, 1981, 1984, 1985, 1986, 1987, 1988, 1989, 1990, 1992, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2003, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2014, 2017, 2018, 2019, 2020, 2021, 2022, 2023, 2025, 2028, 2029, 2030, 2031, 2032, 2033, 2034, 2036, 2039, 2040, 2041, 2042, 2043, 2044, 2045, 2047, 2050, 2051, 2052, 2053, 2054, 2055, 2056, 2058, 2061, 2062, 2063, 2064, 2065, 2066, 2067, 2069, 2072, 2073, 2074, 2075, 2076, 2077, 2078, 2080, 2083, 2084, 2085, 2086, 2087, 2088, 2089, 2091, 2094, 2095, 2096, 2097, 2098, 2099, 2100, 2102, 2105, 2106, 2107, 2108, 2109, 2110, 2111, 2113, 2116, 2117, 2118, 2119, 2120, 2121, 2122, 2124, 2127, 2128, 2129, 2130, 2131, 2132, 2133, 2135, 2138, 2139, 2140, 2141, 2142, 2143, 2144, 2146, 2149, 2150, 2151, 2152, 2153, 2154, 2155, 2157, 2160, 2161, 2162, 2163, 2164, 2165, 2166, 2168, 2171, 2172, 2173, 2174, 2175, 2176, 2177, 2179, 2182, 2183, 2184, 2185, 2186, 2187, 2188, 2190, 2193, 2194, 2195, 2196, 2197, 2198, 2199, 2201, 2204, 2205, 2206, 2207, 2208, 2209, 2210, 2212, 2215, 2216, 2217, 2218, 2219, 2220, 2221, 2223, 2226, 2227, 2228, 2229, 2230, 2231, 2232, 2234, 2237, 2238, 2239, 2240, 2241, 2242, 2243, 2245, 2248, 2249, 2250, 2251, 2252, 2253, 2254, 2256, 2259, 2260, 2261, 2262, 2263, 2264, 2265, 2267, 2270, 2271, 2272, 2273, 2274, 2275, 2276, 2278, 2281, 2282, 2283, 2284, 2285, 2286, 2287, 2289, 2292, 2293, 2294, 2295, 2296, 2297, 2298, 2300, 2303, 2304, 2305, 2306, 2307, 2308, 2309, 2311, 2314, 2315, 2316, 2317, 2318, 2319]}], "trunc": {"shrink_cut": 0, "protected_cut": 0}, "seg_lens": {"user": 2310}, "layout": "a", "names": ["delivery"], "w_offs": [[0, 8], [9, 10], [11, 18], [19, 20], [21, 28], [29, 37], [38, 43], [43, 44], [45, 53], [54, 55], [56, 63], [64, 65], [66, 73], [74, 82], [83, 88], [88, 89], [90, 98], [99, 100], [101, 108], [109, 110], [111, 118], [119, 127], [128, 133], [133, 134], [135, 143], [144, 145], [146, 153], [154, 155], [156, 163], [164, 172], [173, 178], [178, 179], [180, 188], [189, 190], [191, 198], [199, 200], [201, 208], [209, 217], [218, 223], [223, 224], [225, 233], [234, 235], [236, 243], [244, 245], [246, 253], [254, 262], [263, 268], [268, 269], [270, 278], [279, 280], [281, 288], [289, 290], [291, 298], [299, 307], [308, 313], [313, 314], [315, 323], [324, 325], [326, 333], [334, 335], [336, 343], [344, 352], [353, 358], [358, 359], [360, 368], [369, 370], [371, 378], [379, 380], [381, 388], [389, 397], [398, 403], [403, 404], [405, 413], [414, 415], [416, 423], [424, 425], [426, 433], [434, 442], [443, 448], [448, 449], [450, 458], [459, 461], [462, 469], [470, 471], [472, 479], [480, 488], [489, 494], [494, 495], [496, 504], [505, 507], [508, 515], [516, 517], [518, 525], [526, 534], [535, 540], [540, 541], [542, 550], [551, 553], [554, 561], [562, 563], [564, 571], [572, 580], [581, 586], [586, 587], [588, 596], [597, 599], [600, 607], [608, 609], [610, 617], [618, 626], [627, 632], [632, 633], [634, 642], [643, 645], [646, 653], [654, 655], [656, 663], [664, 672], [673, 678], [678, 679], [680, 688], [689, 691], [692, 699], [700, 701], [702, 709], [710, 718], [719, 724], [724, 725], [726, 734], [735, 737], [738, 745], [746, 747], [748, 755], [756, 764], [765, 770], [770, 771], [772, 780], [781, 783], [784, 791], [792, 793], [794, 801], [802, 810], [811, 816], [816, 817], [818, 826], [827, 829], [830, 837], [838, 839], [840, 847], [848, 856], [857, 862], [862, 863], [864, 872], [873, 875], [876, 883], [884, 885], [886, 893], [894, 902], [903, 908], [908, 909], [910, 918], [919, 921], [922, 929], [930, 931], [932, 939], [940, 948], [949, 954], [954, 955], [956, 964], [965, 967], [968, 975], [976, 977], [978, 985], [986, 994], [995, 1000], [1000, 1001], [1002, 1010], [1011, 1013], [1014, 1021], [1022, 1023], [1024, 1031], [1032, 1040], [1041, 1046], [1046, 1047], [1048, 1056], [1057, 1059], [1060, 1067], [1068, 1069], [1070, 1077], [1078, 1086], [1087, 1092], [1092, 1093], [1094, 1102], [1103, 1105], [1106, 1113], [1114, 1115], [1116, 1123], [1124, 1132], [1133, 1138], [1138, 1139], [1140, 1148], [1149, 1151], [1152, 1159], [1160, 1161], [1162, 1169], [1170, 1178], [1179, 1184], [1184, 1185], [1186, 1194], [1195, 1197], [1198, 1205], [1206, 1207], [1208, 1215], [1216, 1224], [1225, 1230], [1230, 1231], [1232, 1240], [1241, 1243], [1244, 1251], [1252, 1253], [1254, 1261], [1262, 1270], [1271, 1276], [1276, 1277], [1278, 1286], [1287, 1289], [1290, 1297], [1298, 1299], [1300, 1307], [1308, 1316], [1317, 1322], [1322, 1323], [1324, 1332], [1333, 1335], [1336, 1343], [1344, 1345], [1346, 1353], [1354, 1362], [1363, 1368], [1368, 1369], [1370, 1378], [1379, 1381], [1382, 1389], [1390, 1391], [1392, 1399], [1400, 1408], [1409, 1414], [1414, 1415], [1416, 1424], [1425, 1427], [1428, 1435], [1436, 1437], [1438, 1445], [1446, 1454], [1455, 1460], [1460, 1461], [1462, 1470], [1471, 1473], [1474, 1481], [1482, 1483], [1484, 1491], [1492, 1500], [1501, 1506], [1506, 1507], [1508, 1516], [1517, 1519], [1520, 1527], [1528, 1529], [1530, 1537], [1538, 1546], [1547, 1552], [1552, 1553], [1554, 1562], [1563, 1565], [1566, 1573], [1574, 1575], [1576, 1583], [1584, 1592], [1593, 1598], [1598, 1599], [1600, 1608], [1609, 1611], [1612, 1619], [1620, 1621], [1622, 1629], [1630, 1638], [1639, 1644], [1644, 1645], [1646, 1654], [1655, 1657], [1658, 1665], [1666, 1667], [1668, 1675], [1676, 1684], [1685, 1690], [1690, 1691], [1692, 1700], [1701, 1703], [1704, 1711], [1712, 1713], [1714, 1721], [1722, 1730], [1731, 1736], [1736, 1737], [1738, 1746], [1747, 1749], [1750, 1757], [1758, 1759], [1760, 1767], [1768, 1776], [1777, 1782], [1782, 1783], [1784, 1792], [1793, 1795], [1796, 1803], [1804, 1805], [1806, 1813], [1814, 1822], [1823, 1828], [1828, 1829], [1830, 1838], [1839, 1841], [1842, 1849], [1850, 1851], [1852, 1859], [1860, 1868], [1869, 1874], [1874, 1875], [1876, 1884], [1885, 1887], [1888, 1895], [1896, 1897], [1898, 1905], [1906, 1914], [1915, 1920], [1920, 1921], [1922, 1930], [1931, 1933], [1934, 1941], [1942, 1943], [1944, 1951], [1952, 1960], [1961, 1966], [1966, 1967], [1968, 1976], [1977, 1979], [1980, 1987], [1988, 1989], [1990, 1997], [1998, 2006], [2007, 2012], [2012, 2013], [2014, 2022], [2023, 2025], [2026, 2033], [2034, 2035], [2036, 2043], [2044, 2052], [2053, 2058], [2058, 2059], [2060, 2068], [2069, 2071], [2072, 2079], [2080, 2081], [2082, 2089], [2090, 2098], [2099, 2104], [2104, 2105], [2106, 2114], [2115, 2117], [2118, 2125], [2126, 2127], [2128, 2135], [2136, 2144], [2145, 2150], [2150, 2151], [2152, 2160], [2161, 2163], [2164, 2171], [2172, 2173], [2174, 2181], [2182, 2190], [2191, 2196], [2196, 2197], [2198, 2206], [2207, 2209], [2210, 2217], [2218, 2219], [2220, 2227], [2228, 2236], [2237, 2242], [2242, 2243], [2244, 2252], [2253, 2255], [2256, 2263], [2264, 2265], [2266, 2273], [2274, 2282], [2283, 2288], [2288, 2289], [2290, 2298], [2299, 2301], [2302, 2309], [2310, 2311], [2312, 2319], [2320, 2328], [2329, 2334], [2334, 2335], [2336, 2344], [2345, 2347], [2348, 2355], [2356, 2357], [2358, 2365], [2366, 2374], [2375, 2380], [2380, 2381], [2382, 2390], [2391, 2393], [2394, 2401], [2402, 2403], [2404, 2411], [2412, 2420], [2421, 2426], [2426, 2427], [2428, 2436], [2437, 2439], [2440, 2447], [2448, 2449], [2450, 2457], [2458, 2466], [2467, 2472], [2472, 2473], [2474, 2482], [2483, 2485], [2486, 2493], [2494, 2495], [2496, 2503], [2504, 2512], [2513, 2518], [2518, 2519], [2520, 2528], [2529, 2531], [2532, 2539], [2540, 2541], [2542, 2549], [2550, 2558], [2559, 2564], [2564, 2565], [2566, 2574], [2575, 2577], [2578, 2585], [2586, 2587], [2588, 2595], [2596, 2604], [2605, 2610], [2610, 2611], [2612, 2620], [2621, 2623], [2624, 2631], [2632, 2633], [2634, 2641], [2642, 2650], [2651, 2656], [2656, 2657], [2658, 2666], [2667, 2669], [2670, 2677], [2678, 2679], [2680, 2687], [2688, 2696], [2697, 2702], [2702, 2703], [2704, 2712], [2713, 2715], [2716, 2723], [2724, 2725], [2726, 2733], [2734, 2742], [2743, 2748], [2748, 2749], [2750, 2758], [2759, 2761], [2762, 2769], [2770, 2771], [2772, 2779], [2780, 2788], [2789, 2794], [2794, 2795], [2796, 2804], [2805, 2807], [2808, 2815], [2816, 2817], [2818, 2825], [2826, 2834], [2835, 2840], [2840, 2841], [2842, 2850], [2851, 2853], [2854, 2861], [2862, 2863], [2864, 2871], [2872, 2880], [2881, 2886], [2886, 2887], [2888, 2896], [2897, 2899], [2900, 2907], [2908, 2909], [2910, 2917], [2918, 2926], [2927, 2932], [2932, 2933], [2934, 2942], [2943, 2945], [2946, 2953], [2954, 2955], [2956, 2963], [2964, 2972], [2973, 2978], [2978, 2979], [2980, 2988], [2989, 2991], [2992, 2999], [3000, 3001], [3002, 3009], [3010, 3018], [3019, 3024], [3024, 3025], [3026, 3034], [3035, 3037], [3038, 3045], [3046, 3047], [3048, 3055], [3056, 3064], [3065, 3070], [3070, 3071], [3072, 3080], [3081, 3083], [3084, 3091], [3092, 3093], [3094, 3101], [3102, 3110], [3111, 3116], [3116, 3117], [3118, 3126], [3127, 3129], [3130, 3137], [3138, 3139], [3140, 3147], [3148, 3156], [3157, 3162], [3162, 3163], [3164, 3172], [3173, 3175], [3176, 3183], [3184, 3185], [3186, 3193], [3194, 3202], [3203, 3208], [3208, 3209], [3210, 3218], [3219, 3221], [3222, 3229], [3230, 3231], [3232, 3239], [3240, 3248], [3249, 3254], [3254, 3255], [3256, 3264], [3265, 3267], [3268, 3275], [3276, 3277], [3278, 3285], [3286, 3294], [3295, 3300], [3300, 3301], [3302, 3310], [3311, 3313], [3314, 3321], [3322, 3323], [3324, 3331], [3332, 3340], [3341, 3346], [3346, 3347], [3348, 3356], [3357, 3359], [3360, 3367], [3368, 3369], [3370, 3377], [3378, 3386], [3387, 3392], [3392, 3393], [3394, 3402], [3403, 3405], [3406, 3413], [3414, 3415], [3416, 3423], [3424, 3432], [3433, 3438], [3438, 3439], [3440, 3448], [3449, 3451], [3452, 3459], [3460, 3461], [3462, 3469], [3470, 3478], [3479, 3484], [3484, 3485], [3486, 3494], [3495, 3497], [3498, 3505], [3506, 3507], [3508, 3515], [3516, 3524], [3525, 3530], [3530, 3531], [3532, 3540], [3541, 3543], [3544, 3551], [3552, 3553], [3554, 3561], [3562, 3570], [3571, 3576], [3576, 3577], [3578, 3586], [3587, 3589], [3590, 3597], [3598, 3599], [3600, 3607], [3608, 3616], [3617, 3622], [3622, 3623], [3624, 3632], [3633, 3635], [3636, 3643], [3644, 3645], [3646, 3653], [3654, 3662], [3663, 3668], [3668, 3669], [3670, 3678], [3679, 3681], [3682, 3689], [3690, 3691], [3692, 3699], [3700, 3708], [3709, 3714], [3714, 3715], [3716, 3724], [3725, 3727], [3728, 3735], [3736, 3737], [3738, 3745], [3746, 3754], [3755, 3760], [3760, 3761], [3762, 3770], [3771, 3773], [3774, 3781], [3782, 3783], [3784, 3791], [3792, 3800], [3801, 3806], [3806, 3807], [3808, 3816], [3817, 3819], [3820, 3827], [3828, 3829], [3830, 3837], [3838, 3846], [3847, 3852], [3852, 3853], [3854, 3862], [3863, 3865], [3866, 3873], [3874, 3875], [3876, 3883], [3884, 3892], [3893, 3898], [3898, 3899], [3900, 3908], [3909, 3911], [3912, 3919], [3920, 3921], [3922, 3929], [3930, 3938], [3939, 3944], [3944, 3945], [3946, 3954], [3955, 3957], [3958, 3965], [3966, 3967], [3968, 3975], [3976, 3984], [3985, 3990], [3990, 3991], [3992, 4000], [4001, 4003], [4004, 4011], [4012, 4013], [4014, 4021], [4022, 4030], [4031, 4036], [4036, 4037], [4038, 4046], [4047, 4049], [4050, 4057], [4058, 4059], [4060, 4067], [4068, 4076], [4077, 4082], [4082, 4083], [4084, 4092], [4093, 4095], [4096, 4103], [4104, 4105], [4106, 4113], [4114, 4122], [4123, 4128], [4128, 4129], [4130, 4138], [4139, 4141], [4142, 4149], [4150, 4151], [4152, 4159], [4160, 4168], [4169, 4174], [4174, 4175], [4176, 4184], [4185, 4187], [4188, 4195], [4196, 4197], [4198, 4205], [4206, 4214], [4215, 4220], [4220, 4221], [4222, 4230], [4231, 4233], [4234, 4241], [4242, 4243], [4244, 4251], [4252, 4260], [4261, 4266], [4266, 4267], [4268, 4276], [4277, 4279], [4280, 4287], [4288, 4289], [4290, 4297], [4298, 4306], [4307, 4312], [4312, 4313], [4314, 4322], [4323, 4325], [4326, 4333], [4334, 4335], [4336, 4343], [4344, 4352], [4353, 4358], [4358, 4359], [4360, 4368], [4369, 4371], [4372, 4379], [4380, 4381], [4382, 4389], [4390, 4398], [4399, 4404], [4404, 4405], [4406, 4414], [4415, 4417], [4418, 4425], [4426, 4427], [4428, 4435], [4436, 4444], [4445, 4450], [4450, 4451], [4452, 4460], [4461, 4463], [4464, 4471], [4472, 4473], [4474, 4481], [4482, 4490], [4491, 4496], [4496, 4497], [4498, 4506], [4507, 4509], [4510, 4517], [4518, 4519], [4520, 4527], [4528, 4536], [4537, 4542], [4542, 4543], [4544, 4552], [4553, 4555], [4556, 4563], [4564, 4565], [4566, 4573], [4574, 4582], [4583, 4588], [4588, 4589], [4590, 4598], [4599, 4602], [4603, 4610], [4611, 4612], [4613, 4620], [4621, 4629], [4630, 4635], [4635, 4636], [4637, 4645], [4646, 4649], [4650, 4657], [4658, 4659], [4660, 4667], [4668, 4676], [4677, 4682], [4682, 4683], [4684, 4692], [4693, 4696], [4697, 4704], [4705, 4706], [4707, 4714], [4715, 4723], [4724, 4729], [4729, 4730], [4731, 4739], [4740, 4743], [4744, 4751], [4752, 4753], [4754, 4761], [4762, 4770], [4771, 4776], [4776, 4777], [4778, 4786], [4787, 4790], [4791, 4798], [4799, 4800], [4801, 4808], [4809, 4817], [4818, 4823], [4823, 4824], [4825, 4833], [4834, 4837], [4838, 4845], [4846, 4847], [4848, 4855], [4856, 4864], [4865, 4870], [4870, 4871], [4872, 4880], [4881, 4884], [4885, 4892], [4893, 4894], [4895, 4902], [4903, 4911], [4912, 4917], [4917, 4918], [4919, 4927], [4928, 4931], [4932, 4939], [4940, 4941], [4942, 4949], [4950, 4958], [4959, 4964], [4964, 4965], [4966, 4974], [4975, 4978], [4979, 4986], [4987, 4988], [4989, 4996], [4997, 5005], [5006, 5011], [5011, 5012], [5013, 5021], [5022, 5025], [5026, 5033], [5034, 5035], [5036, 5043], [5044, 5052], [5053, 5058], [5058, 5059], [5060, 5068], [5069, 5072], [5073, 5080], [5081, 5082], [5083, 5090], [5091, 5099], [5100, 5105], [5105, 5106], [5107, 5115], [5116, 5119], [5120, 5127], [5128, 5129], [5130, 5137], [5138, 5146], [5147, 5152], [5152, 5153], [5154, 5162], [5163, 5166], [5167, 5174], [5175, 5176], [5177, 5184], [5185, 5193], [5194, 5199], [5199, 5200], [5201, 5209], [5210, 5213], [5214, 5221], [5222, 5223], [5224, 5231], [5232, 5240], [5241, 5246], [5246, 5247], [5248, 5256], [5257, 5260], [5261, 5268], [5269, 5270], [5271, 5278], [5279, 5287], [5288, 5293], [5293, 5294], [5295, 5303], [5304, 5307], [5308, 5315], [5316, 5317], [5318, 5325], [5326, 5334], [5335, 5340], [5340, 5341], [5342, 5350], [5351, 5354], [5355, 5362], [5363, 5364], [5365, 5372], [5373, 5381], [5382, 5387], [5387, 5388], [5389, 5397], [5398, 5401], [5402, 5409], [5410, 5411], [5412, 5419], [5420, 5428], [5429, 5434], [5434, 5435], [5436, 5444], [5445, 5448], [5449, 5456], [5457, 5458], [5459, 5466], [5467, 5475], [5476, 5481], [5481, 5482], [5483, 5491], [5492, 5495], [5496, 5503], [5504, 5505], [5506, 5513], [5514, 5522], [5523, 5528], [5528, 5529], [5530, 5538], [5539, 5542], [5543, 5550], [5551, 5552], [5553, 5560], [5561, 5569], [5570, 5575], [5575, 5576], [5577, 5585], [5586, 5589], [5590, 5597], [5598, 5599], [5600, 5607], [5608, 5616], [5617, 5622], [5622, 5623], [5624, 5632], [5633, 5636], [5637, 5644], [5645, 5646], [5647, 5654], [5655, 5663], [5664, 5669], [5669, 5670], [5671, 5679], [5680, 5683], [5684, 5691], [5692, 5693], [5694, 5701], [5702, 5710], [5711, 5716], [5716, 5717], [5718, 5726], [5727, 5730], [5731, 5738], [5739, 5740], [5741, 5748], [5749, 5757], [5758, 5763], [5763, 5764], [5765, 5773], [5774, 5777], [5778, 5785], [5786, 5787], [5788, 5795], [5796, 5804], [5805, 5810], [5810, 5811], [5812, 5820], [5821, 5824], [5825, 5832], [5833, 5834], [5835, 5842], [5843, 5851], [5852, 5857], [5857, 5858], [5859, 5867], [5868, 5871], [5872, 5879], [5880, 5881], [5882, 5889], [5890, 5898], [5899, 5904], [5904, 5905], [5906, 5914], [5915, 5918], [5919, 5926], [5927, 5928], [5929, 5936], [5937, 5945], [5946, 5951], [5951, 5952], [5953, 5961], [5962, 5965], [5966, 5973], [5974, 5975], [5976, 5983], [5984, 5992], [5993, 5998], [5998, 5999], [6000, 6008], [6009, 6012], [6013, 6020], [6021, 6022], [6023, 6030], [6031, 6039], [6040, 6045], [6045, 6046], [6047, 6055], [6056, 6059], [6060, 6067], [6068, 6069], [6070, 6077], [6078, 6086], [6087, 6092], [6092, 6093], [6094, 6102], [6103, 6106], [6107, 6114], [6115, 6116], [6117, 6124], [6125, 6133], [6134, 6139], [6139, 6140], [6141, 6149], [6150, 6153], [6154, 6161], [6162, 6163], [6164, 6171], [6172, 6180], [6181, 6186], [6186, 6187], [6188, 6196], [6197, 6200], [6201, 6208], [6209, 6210], [6211, 6218], [6219, 6227], [6228, 6233], [6233, 6234], [6235, 6243], [6244, 6247], [6248, 6255], [6256, 6257], [6258, 6265], [6266, 6274], [6275, 6280], [6280, 6281], [6282, 6290], [6291, 6294], [6295, 6302], [6303, 6304], [6305, 6312], [6313, 6321], [6322, 6327], [6327, 6328], [6329, 6337], [6338, 6341], [6342, 6349], [6350, 6351], [6352, 6359], [6360, 6368], [6369, 6374], [6374, 6375], [6376, 6384], [6385, 6388], [6389, 6396], [6397, 6398], [6399, 6406], [6407, 6415], [6416, 6421], [6421, 6422], [6423, 6431], [6432, 6435], [6436, 6443], [6444, 6445], [6446, 6453], [6454, 6462], [6463, 6468], [6468, 6469], [6470, 6478], [6479, 6482], [6483, 6490], [6491, 6492], [6493, 6500], [6501, 6509], [6510, 6515], [6515, 6516], [6517, 6525], [6526, 6529], [6530, 6537], [6538, 6539], [6540, 6547], [6548, 6556], [6557, 6562], [6562, 6563], [6564, 6572], [6573, 6576], [6577, 6584], [6585, 6586], [6587, 6594], [6595, 6603], [6604, 6609], [6609, 6610], [6611, 6619], [6620, 6623], [6624, 6631], [6632, 6633], [6634, 6641], [6642, 6650], [6651, 6656], [6656, 6657], [6658, 6666], [6667, 6670], [6671, 6678], [6679, 6680], [6681, 6688], [6689, 6697], [6698, 6703], [6703, 6704], [6705, 6713], [6714, 6717], [6718, 6725], [6726, 6727], [6728, 6735], [6736, 6744], [6745, 6750], [6750, 6751], [6752, 6760], [6761, 6764], [6765, 6772], [6773, 6774], [6775, 6782], [6783, 6791], [6792, 6797], [6797, 6798], [6799, 6807], [6808, 6811], [6812, 6819], [6820, 6821], [6822, 6829], [6830, 6838], [6839, 6844], [6844, 6845], [6846, 6854], [6855, 6858], [6859, 6866], [6867, 6868], [6869, 6876], [6877, 6885], [6886, 6891], [6891, 6892], [6893, 6901], [6902, 6905], [6906, 6913], [6914, 6915], [6916, 6923], [6924, 6932], [6933, 6938], [6938, 6939], [6940, 6948], [6949, 6952], [6953, 6960], [6961, 6962], [6963, 6970], [6971, 6979], [6980, 6985], [6985, 6986], [6987, 6995], [6996, 6999], [7000, 7007], [7008, 7009], [7010, 7017], [7018, 7026], [7027, 7032], [7032, 7033], [7034, 7042], [7043, 7046], [7047, 7054], [7055, 7056], [7057, 7064], [7065, 7073], [7074, 7079], [7079, 7080], [7081, 7089], [7090, 7093], [7094, 7101], [7102, 7103], [7104, 7111], [7112, 7120], [7121, 7126], [7126, 7127], [7128, 7136], [7137, 7140], [7141, 7148], [7149, 7150], [7151, 7158], [7159, 7167], [7168, 7173], [7173, 7174], [7175, 7183], [7184, 7187], [7188, 7195], [7196, 7197], [7198, 7205], [7206, 7214], [7215, 7220], [7220, 7221], [7222, 7230], [7231, 7234], [7235, 7242], [7243, 7244], [7245, 7252], [7253, 7261], [7262, 7267], [7267, 7268], [7269, 7277], [7278, 7281], [7282, 7289], [7290, 7291], [7292, 7299], [7300, 7308], [7309, 7314], [7314, 7315], [7316, 7324], [7325, 7328], [7329, 7336], [7337, 7338], [7339, 7346], [7347, 7355], [7356, 7361], [7361, 7362], [7363, 7371], [7372, 7375], [7376, 7383], [7384, 7385], [7386, 7393], [7394, 7402], [7403, 7408], [7408, 7409], [7410, 7418], [7419, 7422], [7423, 7430], [7431, 7432], [7433, 7440], [7441, 7449], [7450, 7455], [7455, 7456], [7457, 7465], [7466, 7469], [7470, 7477], [7478, 7479], [7480, 7487], [7488, 7496], [7497, 7502], [7502, 7503], [7504, 7512], [7513, 7516], [7517, 7524], [7525, 7526], [7527, 7534], [7535, 7543], [7544, 7549], [7549, 7550], [7551, 7559], [7560, 7563], [7564, 7571], [7572, 7573], [7574, 7581], [7582, 7590], [7591, 7596], [7596, 7597], [7598, 7606], [7607, 7610], [7611, 7618], [7619, 7620], [7621, 7628], [7629, 7637], [7638, 7643], [7643, 7644], [7645, 7653], [7654, 7657], [7658, 7665], [7666, 7667], [7668, 7675], [7676, 7684], [7685, 7690], [7690, 7691], [7692, 7700], [7701, 7704], [7705, 7712], [7713, 7714], [7715, 7722], [7723, 7731], [7732, 7737], [7737, 7738], [7739, 7747], [7748, 7751], [7752, 7759], [7760, 7761], [7762, 7769], [7770, 7778], [7779, 7784], [7784, 7785], [7786, 7794], [7795, 7798], [7799, 7806], [7807, 7808], [7809, 7816], [7817, 7825], [7826, 7831], [7831, 7832], [7833, 7841], [7842, 7845], [7846, 7853], [7854, 7855], [7856, 7863], [7864, 7872], [7873, 7878], [7878, 7879], [7880, 7888], [7889, 7892], [7893, 7900], [7901, 7902], [7903, 7910], [7911, 7919], [7920, 7925], [7925, 7926], [7927, 7935], [7936, 7939], [7940, 7947], [7948, 7949], [7950, 7957], [7958, 7966], [7967, 7972], [7972, 7973], [7974, 7982], [7983, 7986], [7987, 7994], [7995, 7996], [7997, 8004], [8005, 8013], [8014, 8019], [8019, 8020], [8021, 8029], [8030, 8033], [8034, 8041], [8042, 8043], [8044, 8051], [8052, 8060], [8061, 8066], [8066, 8067], [8068, 8076], [8077, 8080], [8081, 8088], [8089, 8090], [8091, 8098], [8099, 8107], [8108, 8113], [8113, 8114], [8115, 8123], [8124, 8127], [8128, 8135], [8136, 8137], [8138, 8145], [8146, 8154], [8155, 8160], [8160, 8161], [8162, 8170], [8171, 8174], [8175, 8182], [8183, 8184], [8185, 8192], [8193, 8201], [8202, 8207], [8207, 8208], [8209, 8217], [8218, 8221], [8222, 8229], [8230, 8231], [8232, 8239], [8240, 8248], [8249, 8254], [8254, 8255], [8256, 8264], [8265, 8268], [8269, 8276], [8277, 8278], [8279, 8286], [8287, 8295], [8296, 8301], [8301, 8302], [8303, 8311], [8312, 8315], [8316, 8323], [8324, 8325], [8326, 8333], [8334, 8342], [8343, 8348], [8348, 8349], [8350, 8358], [8359, 8362], [8363, 8370], [8371, 8372], [8373, 8380], [8381, 8389], [8390, 8395], [8395, 8396], [8397, 8405], [8406, 8409], [8410, 8417], [8418, 8419], [8420, 8427], [8428, 8436], [8437, 8442], [8442, 8443], [8444, 8452], [8453, 8456], [8457, 8464], [8465, 8466], [8467, 8474], [8475, 8483], [8484, 8489], [8489, 8490], [8491, 8499], [8500, 8503], [8504, 8511], [8512, 8513], [8514, 8521], [8522, 8530], [8531, 8536], [8536, 8537], [8538, 8546], [8547, 8550], [8551, 8558], [8559, 8560], [8561, 8568], [8569, 8577], [8578, 8583], [8583, 8584], [8585, 8593], [8594, 8597], [8598, 8605], [8606, 8607], [8608, 8615], [8616, 8624], [8625, 8630], [8630, 8631], [8632, 8640], [8641, 8644], [8645, 8652], [8653, 8654], [8655, 8662], [8663, 8671], [8672, 8677], [8677, 8678], [8679, 8687], [8688, 8691], [8692, 8699], [8700, 8701], [8702, 8709], [8710, 8718], [8719, 8724], [8724, 8725], [8726, 8734], [8735, 8738], [8739, 8746], [8747, 8748], [8749, 8756], [8757, 8765], [8766, 8771], [8771, 8772], [8773, 8781], [8782, 8785], [8786, 8793], [8794, 8795], [8796, 8803], [8804, 8812], [8813, 8818], [8818, 8819], [8820, 8828], [8829, 8832], [8833, 8840], [8841, 8842], [8843, 8850], [8851, 8859], [8860, 8865], [8865, 8866], [8867, 8875], [8876, 8879], [8880, 8887], [8888, 8889], [8890, 8897], [8898, 8906], [8907, 8912], [8912, 8913], [8914, 8922], [8923, 8926], [8927, 8934], [8935, 8936], [8937, 8944], [8945, 8953], [8954, 8959], [8959, 8960], [8961, 8969], [8970, 8973], [8974, 8981], [8982, 8983], [8984, 8991], [8992, 9000], [9001, 9006], [9006, 9007], [9008, 9016], [9017, 9020], [9021, 9028], [9029, 9030], [9031, 9038], [9039, 9047], [9048, 9053], [9053, 9054], [9055, 9063], [9064, 9067], [9068, 9075], [9076, 9077], [9078, 9085], [9086, 9094], [9095, 9100], [9100, 9101], [9102, 9110], [9111, 9114], [9115, 9122], [9123, 9124], [9125, 9132], [9133, 9141], [9142, 9147], [9147, 9148], [9149, 9157], [9158, 9161], [9162, 9169], [9170, 9171], [9172, 9179], [9180, 9188], [9189, 9194], [9194, 9195], [9196, 9204], [9205, 9208], [9209, 9216], [9217, 9218], [9219, 9226], [9227, 9235], [9236, 9241], [9241, 9242], [9243, 9251], [9252, 9255], [9256, 9263], [9264, 9265], [9266, 9273], [9274, 9282], [9283, 9288], [9288, 9289], [9290, 9298], [9299, 9302], [9303, 9310], [9311, 9312], [9313, 9320], [9321, 9329], [9330, 9335], [9335, 9336], [9337, 9345], [9346, 9349], [9350, 9357], [9358, 9359], [9360, 9367], [9368, 9376], [9377, 9382], [9382, 9383], [9384, 9392], [9393, 9396], [9397, 9404], [9405, 9406], [9407, 9414], [9415, 9423], [9424, 9429], [9429, 9430], [9431, 9439], [9440, 9443], [9444, 9451], [9452, 9453], [9454, 9461], [9462, 9470], [9471, 9476], [9476, 9477], [9478, 9486], [9487, 9490], [9491, 9498], [9499, 9500], [9501, 9508], [9509, 9517], [9518, 9523], [9523, 9524], [9525, 9533], [9534, 9537], [9538, 9545], [9546, 9547], [9548, 9555], [9556, 9564], [9565, 9570], [9570, 9571], [9572, 9580], [9581, 9584], [9585, 9592], [9593, 9594], [9595, 9602], [9603, 9611], [9612, 9617], [9617, 9618], [9619, 9627], [9628, 9631], [9632, 9639], [9640, 9641], [9642, 9649], [9650, 9658], [9659, 9664], [9664, 9665], [9666, 9674], [9675, 9678], [9679, 9686], [9687, 9688], [9689, 9696], [9697, 9705], [9706, 9711], [9711, 9712], [9713, 9721], [9722, 9725], [9726, 9733], [9734, 9735], [9736, 9743], [9744, 9752], [9753, 9758], [9758, 9759], [9760, 9768], [9769, 9772], [9773, 9780], [9781, 9782], [9783, 9790], [9791, 9799], [9800, 9805], [9805, 9806], [9807, 9815], [9816, 9819], [9820, 9827], [9828, 9829], [9830, 9837], [9838, 9846], [9847, 9852], [9852, 9853], [9854, 9862], [9863, 9866], [9867, 9874], [9875, 9876], [9877, 9884], [9885, 9893], [9894, 9899], [9899, 9900], [9901, 9909], [9910, 9913], [9914, 9921], [9922, 9923], [9924, 9931], [9932, 9940], [9941, 9946], [9946, 9947], [9948, 9956], [9957, 9960], [9961, 9968], [9969, 9970], [9971, 9978], [9979, 9987], [9988, 9993], [9993, 9994], [9995, 10003], [10004, 10007], [10008, 10015], [10016, 10017], [10018, 10025], [10026, 10034], [10035, 10040], [10040, 10041], [10042, 10050], [10051, 10054], [10055, 10062], [10063, 10064], [10065, 10072], [10073, 10081], [10082, 10087], [10087, 10088], [10089, 10097], [10098, 10101], [10102, 10109], [10110, 10111], [10112, 10119], [10120, 10128], [10129, 10134], [10134, 10135], [10136, 10144], [10145, 10148], [10149, 10156], [10157, 10158], [10159, 10166], [10167, 10175], [10176, 10181], [10181, 10182], [10183, 10191], [10192, 10195], [10196, 10203], [10204, 10205], [10206, 10213], [10214, 10222], [10223, 10228], [10228, 10229]], "w_pos": [10, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 23, 24, 25, 26, 27, 28, 30, 31, 32, 33, 34, 35, 36, 37, 39, 40, 41, 42, 43, 44, 45, 46, 48, 49, 50, 51, 52, 53, 54, 55, 57, 58, 59, 60, 61, 62, 63, 64, 66, 67, 68, 69, 70, 71, 72, 73, 75, 76, 77, 78, 79, 80, 81, 82, 84, 85, 86, 87, 88, 89, 90, 91, 93, 94, 95, 96, 97, 98, 99, 100, 102, 104, 105, 106, 107, 108, 109, 110, 112, 114, 115, 116, 117, 118, 119, 120, 122, 124, 125, 126, 127, 128, 129, 130, 132, 134, 135, 136, 137, 138, 139, 140, 142, 144, 145, 146, 147, 148, 149, 150, 152, 154, 155, 156, 157, 158, 159, 160, 162, 164, 165, 166, 167, 168, 169, 170, 172, 174, 175, 176, 177, 178, 179, 180, 182, 184, 185, 186, 187, 188, 189, 190, 192, 194, 195, 196, 197, 198, 199, 200, 202, 204, 205, 206, 207, 208, 209, 210, 212, 214, 215, 216, 217, 218, 219, 220, 222, 224, 225, 226, 227, 228, 229, 230, 232, 234, 235, 236, 237, 238, 239, 240, 242, 244, 245, 246, 247, 248, 249, 250, 252, 254, 255, 256, 257, 258, 259, 260, 262, 264, 265, 266, 267, 268, 269, 270, 272, 274, 275, 276, 277, 278, 279, 280, 282, 284, 285, 286, 287, 288, 289, 290, 292, 294, 295, 296, 297, 298, 299, 300, 302, 304, 305, 306, 307, 308, 309, 310, 312, 314, 315, 316, 317, 318, 319, 320, 322, 324, 325, 326, 327, 328, 329, 330, 332, 334, 335, 336, 337, 338, 339, 340, 342, 344, 345, 346, 347, 348, 349, 350, 352, 354, 355, 356, 357, 358, 359, 360, 362, 364, 365, 366, 367, 368, 369, 370, 372, 374, 375, 376, 377, 378, 379, 380, 382, 384, 385, 386, 387, 388, 389, 390, 392, 394, 395, 396, 397, 398, 399, 400, 402, 404, 405, 406, 407, 408, 409, 410, 412, 414, 415, 416, 417, 418, 419, 420, 422, 424, 425, 426, 427, 428, 429, 430, 432, 434, 435, 436, 437, 438, 439, 440, 442, 444, 445, 446, 447, 448, 449, 450, 452, 454, 455, 456, 457, 458, 459, 460, 462, 464, 465, 466, 467, 468, 469, 470, 472, 474, 475, 476, 477, 478, 479, 480, 482, 484, 485, 486, 487, 488, 489, 490, 492, 494, 495, 496, 497, 498, 499, 500, 502, 504, 505, 506, 507, 508, 509, 510, 512, 514, 515, 516, 517, 518, 519, 520, 522, 524, 525, 526, 527, 528, 529, 530, 532, 534, 535, 536, 537, 538, 539, 540, 542, 544, 545, 546, 547, 548, 549, 550, 552, 554, 555, 556, 557, 558, 559, 560, 562, 564, 565, 566, 567, 568, 569, 570, 572, 574, 575, 576, 577, 578, 579, 580, 582, 584, 585, 586, 587, 588, 589, 590, 592, 594, 595, 596, 597, 598, 599, 600, 602, 604, 605, 606, 607, 608, 609, 610, 612, 614, 615, 616, 617, 618, 619, 620, 622, 624, 625, 626, 627, 628, 629, 630, 632, 634, 635, 636, 637, 638, 639, 640, 642, 644, 645, 646, 647, 648, 649, 650, 652, 654, 655, 656, 657, 658, 659, 660, 662, 664, 665, 666, 667, 668, 669, 670, 672, 674, 675, 676, 677, 678, 679, 680, 682, 684, 685, 686, 687, 688, 689, 690, 692, 694, 695, 696, 697, 698, 699, 700, 702, 704, 705, 706, 707, 708, 709, 710, 712, 714, 715, 716, 717, 718, 719, 720, 722, 724, 725, 726, 727, 728, 729, 730, 732, 734, 735, 736, 737, 738, 739, 740, 742, 744, 745, 746, 747, 748, 749, 750, 752, 754, 755, 756, 757, 758, 759, 760, 762, 764, 765, 766, 767, 768, 769, 770, 772, 774, 775, 776, 777, 778, 779, 780, 782, 784, 785, 786, 787, 788, 789, 790, 792, 794, 795, 796, 797, 798, 799, 800, 802, 804, 805, 806, 807, 808, 809, 810, 812, 814, 815, 816, 817, 818, 819, 820, 822, 824, 825, 826, 827, 828, 829, 830, 832, 834, 835, 836, 837, 838, 839, 840, 842, 844, 845, 846, 847, 848, 849, 850, 852, 854, 855, 856, 857, 858, 859, 860, 862, 864, 865, 866, 867, 868, 869, 870, 872, 874, 875, 876, 877, 878, 879, 880, 882, 884, 885, 886, 887, 888, 889, 890, 892, 894, 895, 896, 897, 898, 899, 900, 902, 904, 905, 906, 907, 908, 909, 910, 912, 914, 915, 916, 917, 918, 919, 920, 922, 924, 925, 926, 927, 928, 929, 930, 932, 934, 935, 936, 937, 938, 939, 940, 942, 944, 945, 946, 947, 948, 949, 950, 952, 954, 955, 956, 957, 958, 959, 960, 962, 964, 965, 966, 967, 968, 969, 970, 972, 974, 975, 976, 977, 978, 979, 980, 982, 984, 985, 986, 987, 988, 989, 990, 992, 994, 995, 996, 997, 998, 999, 1000, 1002, 1005, 1006, 1007, 1008, 1009, 1010, 1011, 1013, 1016, 1017, 1018, 1019, 1020, 1021, 1022, 1024, 1027, 1028, 1029, 1030, 1031, 1032, 1033, 1035, 1038, 1039, 1040, 1041, 1042, 1043, 1044, 1046, 1049, 1050, 1051, 1052, 1053, 1054, 1055, 1057, 1060, 1061, 1062, 1063, 1064, 1065, 1066, 1068, 1071, 1072, 1073, 1074, 1075, 1076, 1077, 1079, 1082, 1083, 1084, 1085, 1086, 1087, 1088, 1090, 1093, 1094, 1095, 1096, 1097, 1098, 1099, 1101, 1104, 1105, 1106, 1107, 1108, 1109, 1110, 1112, 1115, 1116, 1117, 1118, 1119, 1120, 1121, 1123, 1126, 1127, 1128, 1129, 1130, 1131, 1132, 1134, 1137, 1138, 1139, 1140, 1141, 1142, 1143, 1145, 1148, 1149, 1150, 1151, 1152, 1153, 1154, 1156, 1159, 1160, 1161, 1162, 1163, 1164, 1165, 1167, 1170, 1171, 1172, 1173, 1174, 1175, 1176, 1178, 1181, 1182, 1183, 1184, 1185, 1186, 1187, 1189, 1192, 1193, 1194, 1195, 1196, 1197, 1198, 1200, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1211, 1214, 1215, 1216, 1217, 1218, 1219, 1220, 1222, 1225, 1226, 1227, 1228, 1229, 1230, 1231, 1233, 1236, 1237, 1238, 1239, 1240, 1241, 1242, 1244, 1247, 1248, 1249, 1250, 1251, 1252, 1253, 1255, 1258, 1259, 1260, 1261, 1262, 1263, 1264, 1266, 1269, 1270, 1271, 1272, 1273, 1274, 1275, 1277, 1280, 1281, 1282, 1283, 1284, 1285, 1286, 1288, 1291, 1292, 1293, 1294, 1295, 1296, 1297, 1299, 1302, 1303, 1304, 1305, 1306, 1307, 1308, 1310, 1313, 1314, 1315, 1316, 1317, 1318, 1319, 1321, 1324, 1325, 1326, 1327, 1328, 1329, 1330, 1332, 1335, 1336, 1337, 1338, 1339, 1340, 1341, 1343, 1346, 1347, 1348, 1349, 1350, 1351, 1352, 1354, 1357, 1358, 1359, 1360, 1361, 1362, 1363, 1365, 1368, 1369, 1370, 1371, 1372, 1373, 1374, 1376, 1379, 1380, 1381, 1382, 1383, 1384, 1385, 1387, 1390, 1391, 1392, 1393, 1394, 1395, 1396, 1398, 1401, 1402, 1403, 1404, 1405, 1406, 1407, 1409, 1412, 1413, 1414, 1415, 1416, 1417, 1418, 1420, 1423, 1424, 1425, 1426, 1427, 1428, 1429, 1431, 1434, 1435, 1436, 1437, 1438, 1439, 1440, 1442, 1445, 1446, 1447, 1448, 1449, 1450, 1451, 1453, 1456, 1457, 1458, 1459, 1460, 1461, 1462, 1464, 1467, 1468, 1469, 1470, 1471, 1472, 1473, 1475, 1478, 1479, 1480, 1481, 1482, 1483, 1484, 1486, 1489, 1490, 1491, 1492, 1493, 1494, 1495, 1497, 1500, 1501, 1502, 1503, 1504, 1505, 1506, 1508, 1511, 1512, 1513, 1514, 1515, 1516, 1517, 1519, 1522, 1523, 1524, 1525, 1526, 1527, 1528, 1530, 1533, 1534, 1535, 1536, 1537, 1538, 1539, 1541, 1544, 1545, 1546, 1547, 1548, 1549, 1550, 1552, 1555, 1556, 1557, 1558, 1559, 1560, 1561, 1563, 1566, 1567, 1568, 1569, 1570, 1571, 1572, 1574, 1577, 1578, 1579, 1580, 1581, 1582, 1583, 1585, 1588, 1589, 1590, 1591, 1592, 1593, 1594, 1596, 1599, 1600, 1601, 1602, 1603, 1604, 1605, 1607, 1610, 1611, 1612, 1613, 1614, 1615, 1616, 1618, 1621, 1622, 1623, 1624, 1625, 1626, 1627, 1629, 1632, 1633, 1634, 1635, 1636, 1637, 1638, 1640, 1643, 1644, 1645, 1646, 1647, 1648, 1649, 1651, 1654, 1655, 1656, 1657, 1658, 1659, 1660, 1662, 1665, 1666, 1667, 1668, 1669, 1670, 1671, 1673, 1676, 1677, 1678, 1679, 1680, 1681, 1682, 1684, 1687, 1688, 1689, 1690, 1691, 1692, 1693, 1695, 1698, 1699, 1700, 1701, 1702, 1703, 1704, 1706, 1709, 1710, 1711, 1712, 1713, 1714, 1715, 1717, 1720, 1721, 1722, 1723, 1724, 1725, 1726, 1728, 1731, 1732, 1733, 1734, 1735, 1736, 1737, 1739, 1742, 1743, 1744, 1745, 1746, 1747, 1748, 1750, 1753, 1754, 1755, 1756, 1757, 1758, 1759, 1761, 1764, 1765, 1766, 1767, 1768, 1769, 1770, 1772, 1775, 1776, 1777, 1778, 1779, 1780, 1781, 1783, 1786, 1787, 1788, 1789, 1790, 1791, 1792, 1794, 1797, 1798, 1799, 1800, 1801, 1802, 1803, 1805, 1808, 1809, 1810, 1811, 1812, 1813, 1814, 1816, 1819, 1820, 1821, 1822, 1823, 1824, 1825, 1827, 1830, 1831, 1832, 1833, 1834, 1835, 1836, 1838, 1841, 1842, 1843, 1844, 1845, 1846, 1847, 1849, 1852, 1853, 1854, 1855, 1856, 1857, 1858, 1860, 1863, 1864, 1865, 1866, 1867, 1868, 1869, 1871, 1874, 1875, 1876, 1877, 1878, 1879, 1880, 1882, 1885, 1886, 1887, 1888, 1889, 1890, 1891, 1893, 1896, 1897, 1898, 1899, 1900, 1901, 1902, 1904, 1907, 1908, 1909, 1910, 1911, 1912, 1913, 1915, 1918, 1919, 1920, 1921, 1922, 1923, 1924, 1926, 1929, 1930, 1931, 1932, 1933, 1934, 1935, 1937, 1940, 1941, 1942, 1943, 1944, 1945, 1946, 1948, 1951, 1952, 1953, 1954, 1955, 1956, 1957, 1959, 1962, 1963, 1964, 1965, 1966, 1967, 1968, 1970, 1973, 1974, 1975, 1976, 1977, 1978, 1979, 1981, 1984, 1985, 1986, 1987, 1988, 1989, 1990, 1992, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2003, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2014, 2017, 2018, 2019, 2020, 2021, 2022, 2023, 2025, 2028, 2029, 2030, 2031, 2032, 2033, 2034, 2036, 2039, 2040, 2041, 2042, 2043, 2044, 2045, 2047, 2050, 2051, 2052, 2053, 2054, 2055, 2056, 2058, 2061, 2062, 2063, 2064, 2065, 2066, 2067, 2069, 2072, 2073, 2074, 2075, 2076, 2077, 2078, 2080, 2083, 2084, 2085, 2086, 2087, 2088, 2089, 2091, 2094, 2095, 2096, 2097, 2098, 2099, 2100, 2102, 2105, 2106, 2107, 2108, 2109, 2110, 2111, 2113, 2116, 2117, 2118, 2119, 2120, 2121, 2122, 2124, 2127, 2128, 2129, 2130, 2131, 2132, 2133, 2135, 2138, 2139, 2140, 2141, 2142, 2143, 2144, 2146, 2149, 2150, 2151, 2152, 2153, 2154, 2155, 2157, 2160, 2161, 2162, 2163, 2164, 2165, 2166, 2168, 2171, 2172, 2173, 2174, 2175, 2176, 2177, 2179, 2182, 2183, 2184, 2185, 2186, 2187, 2188, 2190, 2193, 2194, 2195, 2196, 2197, 2198, 2199, 2201, 2204, 2205, 2206, 2207, 2208, 2209, 2210, 2212, 2215, 2216, 2217, 2218, 2219, 2220, 2221, 2223, 2226, 2227, 2228, 2229, 2230, 2231, 2232, 2234, 2237, 2238, 2239, 2240, 2241, 2242, 2243, 2245, 2248, 2249, 2250, 2251, 2252, 2253, 2254, 2256, 2259, 2260, 2261, 2262, 2263, 2264, 2265, 2267, 2270, 2271, 2272, 2273, 2274, 2275, 2276, 2278, 2281, 2282, 2283, 2284, 2285, 2286, 2287, 2289, 2292, 2293, 2294, 2295, 2296, 2297, 2298, 2300, 2303, 2304, 2305, 2306, 2307, 2308, 2309, 2311, 2314, 2315, 2316, 2317, 2318, 2319], "span_over": "user", "n_tokens": 2385, "head": "broad", "head_reason": "default"}], "window_rows": [{"p_ids": [1905, 25, 198, 27, 22446, 3364, 420, 846, 731, 198, 81646, 220, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 18, 6473, 198, 510, 22446, 29, 198], "blocks": [{"kind": "span", "ends": [40], "label_ranges": [[22, 40]], "words_in_block": true, "ids": [271, 6065, 913, 25, 9111, 198, 6196, 25, 1156, 198, 14162, 25, 198, 22365, 42824, 6088, 264, 9403, 30, 198, 22933, 25, 198, 14057, 29, 198, 4754, 4532, 3147, 64, 9403, 1484, 2129, 767, 3147, 31320, 8934, 198, 510, 1448, 29, 271, 1138, 25, 198, 27, 22446, 3364, 420, 846, 731, 198, 81646, 220, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 18, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 19, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 20, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 21, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 22, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 23, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 24, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 15, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 16, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 17, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 18, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 19, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 18, 6473, 198, 510, 22446, 29, 198, 271, 8679, 1396, 9111, 314, 279, 2100, 1414, 421, 22651, 799, 314, 279, 8928, 13, 198, 6205, 581, 25], "query": 1874, "w_pos": [52, 54, 55, 56, 57, 58, 59, 60, 61, 63, 64, 65, 66, 67, 68, 69, 70, 72, 73, 74, 75, 76, 77, 78, 79, 81, 82, 83, 84, 85, 86, 87, 88, 90, 91, 92, 93, 94, 95, 96, 97, 99, 100, 101, 102, 103, 104, 105, 106, 108, 109, 110, 111, 112, 113, 114, 115, 117, 118, 119, 120, 121, 122, 123, 124, 126, 127, 128, 129, 130, 131, 132, 133, 135, 136, 137, 138, 139, 140, 141, 142, 144, 146, 147, 148, 149, 150, 151, 152, 154, 156, 157, 158, 159, 160, 161, 162, 164, 166, 167, 168, 169, 170, 171, 172, 174, 176, 177, 178, 179, 180, 181, 182, 184, 186, 187, 188, 189, 190, 191, 192, 194, 196, 197, 198, 199, 200, 201, 202, 204, 206, 207, 208, 209, 210, 211, 212, 214, 216, 217, 218, 219, 220, 221, 222, 224, 226, 227, 228, 229, 230, 231, 232, 234, 236, 237, 238, 239, 240, 241, 242, 244, 246, 247, 248, 249, 250, 251, 252, 254, 256, 257, 258, 259, 260, 261, 262, 264, 266, 267, 268, 269, 270, 271, 272, 274, 276, 277, 278, 279, 280, 281, 282, 284, 286, 287, 288, 289, 290, 291, 292, 294, 296, 297, 298, 299, 300, 301, 302, 304, 306, 307, 308, 309, 310, 311, 312, 314, 316, 317, 318, 319, 320, 321, 322, 324, 326, 327, 328, 329, 330, 331, 332, 334, 336, 337, 338, 339, 340, 341, 342, 344, 346, 347, 348, 349, 350, 351, 352, 354, 356, 357, 358, 359, 360, 361, 362, 364, 366, 367, 368, 369, 370, 371, 372, 374, 376, 377, 378, 379, 380, 381, 382, 384, 386, 387, 388, 389, 390, 391, 392, 394, 396, 397, 398, 399, 400, 401, 402, 404, 406, 407, 408, 409, 410, 411, 412, 414, 416, 417, 418, 419, 420, 421, 422, 424, 426, 427, 428, 429, 430, 431, 432, 434, 436, 437, 438, 439, 440, 441, 442, 444, 446, 447, 448, 449, 450, 451, 452, 454, 456, 457, 458, 459, 460, 461, 462, 464, 466, 467, 468, 469, 470, 471, 472, 474, 476, 477, 478, 479, 480, 481, 482, 484, 486, 487, 488, 489, 490, 491, 492, 494, 496, 497, 498, 499, 500, 501, 502, 504, 506, 507, 508, 509, 510, 511, 512, 514, 516, 517, 518, 519, 520, 521, 522, 524, 526, 527, 528, 529, 530, 531, 532, 534, 536, 537, 538, 539, 540, 541, 542, 544, 546, 547, 548, 549, 550, 551, 552, 554, 556, 557, 558, 559, 560, 561, 562, 564, 566, 567, 568, 569, 570, 571, 572, 574, 576, 577, 578, 579, 580, 581, 582, 584, 586, 587, 588, 589, 590, 591, 592, 594, 596, 597, 598, 599, 600, 601, 602, 604, 606, 607, 608, 609, 610, 611, 612, 614, 616, 617, 618, 619, 620, 621, 622, 624, 626, 627, 628, 629, 630, 631, 632, 634, 636, 637, 638, 639, 640, 641, 642, 644, 646, 647, 648, 649, 650, 651, 652, 654, 656, 657, 658, 659, 660, 661, 662, 664, 666, 667, 668, 669, 670, 671, 672, 674, 676, 677, 678, 679, 680, 681, 682, 684, 686, 687, 688, 689, 690, 691, 692, 694, 696, 697, 698, 699, 700, 701, 702, 704, 706, 707, 708, 709, 710, 711, 712, 714, 716, 717, 718, 719, 720, 721, 722, 724, 726, 727, 728, 729, 730, 731, 732, 734, 736, 737, 738, 739, 740, 741, 742, 744, 746, 747, 748, 749, 750, 751, 752, 754, 756, 757, 758, 759, 760, 761, 762, 764, 766, 767, 768, 769, 770, 771, 772, 774, 776, 777, 778, 779, 780, 781, 782, 784, 786, 787, 788, 789, 790, 791, 792, 794, 796, 797, 798, 799, 800, 801, 802, 804, 806, 807, 808, 809, 810, 811, 812, 814, 816, 817, 818, 819, 820, 821, 822, 824, 826, 827, 828, 829, 830, 831, 832, 834, 836, 837, 838, 839, 840, 841, 842, 844, 846, 847, 848, 849, 850, 851, 852, 854, 856, 857, 858, 859, 860, 861, 862, 864, 866, 867, 868, 869, 870, 871, 872, 874, 876, 877, 878, 879, 880, 881, 882, 884, 886, 887, 888, 889, 890, 891, 892, 894, 896, 897, 898, 899, 900, 901, 902, 904, 906, 907, 908, 909, 910, 911, 912, 914, 916, 917, 918, 919, 920, 921, 922, 924, 926, 927, 928, 929, 930, 931, 932, 934, 936, 937, 938, 939, 940, 941, 942, 944, 946, 947, 948, 949, 950, 951, 952, 954, 956, 957, 958, 959, 960, 961, 962, 964, 966, 967, 968, 969, 970, 971, 972, 974, 976, 977, 978, 979, 980, 981, 982, 984, 986, 987, 988, 989, 990, 991, 992, 994, 996, 997, 998, 999, 1000, 1001, 1002, 1004, 1006, 1007, 1008, 1009, 1010, 1011, 1012, 1014, 1016, 1017, 1018, 1019, 1020, 1021, 1022, 1024, 1026, 1027, 1028, 1029, 1030, 1031, 1032, 1034, 1036, 1037, 1038, 1039, 1040, 1041, 1042, 1044, 1047, 1048, 1049, 1050, 1051, 1052, 1053, 1055, 1058, 1059, 1060, 1061, 1062, 1063, 1064, 1066, 1069, 1070, 1071, 1072, 1073, 1074, 1075, 1077, 1080, 1081, 1082, 1083, 1084, 1085, 1086, 1088, 1091, 1092, 1093, 1094, 1095, 1096, 1097, 1099, 1102, 1103, 1104, 1105, 1106, 1107, 1108, 1110, 1113, 1114, 1115, 1116, 1117, 1118, 1119, 1121, 1124, 1125, 1126, 1127, 1128, 1129, 1130, 1132, 1135, 1136, 1137, 1138, 1139, 1140, 1141, 1143, 1146, 1147, 1148, 1149, 1150, 1151, 1152, 1154, 1157, 1158, 1159, 1160, 1161, 1162, 1163, 1165, 1168, 1169, 1170, 1171, 1172, 1173, 1174, 1176, 1179, 1180, 1181, 1182, 1183, 1184, 1185, 1187, 1190, 1191, 1192, 1193, 1194, 1195, 1196, 1198, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1209, 1212, 1213, 1214, 1215, 1216, 1217, 1218, 1220, 1223, 1224, 1225, 1226, 1227, 1228, 1229, 1231, 1234, 1235, 1236, 1237, 1238, 1239, 1240, 1242, 1245, 1246, 1247, 1248, 1249, 1250, 1251, 1253, 1256, 1257, 1258, 1259, 1260, 1261, 1262, 1264, 1267, 1268, 1269, 1270, 1271, 1272, 1273, 1275, 1278, 1279, 1280, 1281, 1282, 1283, 1284, 1286, 1289, 1290, 1291, 1292, 1293, 1294, 1295, 1297, 1300, 1301, 1302, 1303, 1304, 1305, 1306, 1308, 1311, 1312, 1313, 1314, 1315, 1316, 1317, 1319, 1322, 1323, 1324, 1325, 1326, 1327, 1328, 1330, 1333, 1334, 1335, 1336, 1337, 1338, 1339, 1341, 1344, 1345, 1346, 1347, 1348, 1349, 1350, 1352, 1355, 1356, 1357, 1358, 1359, 1360, 1361, 1363, 1366, 1367, 1368, 1369, 1370, 1371, 1372, 1374, 1377, 1378, 1379, 1380, 1381, 1382, 1383, 1385, 1388, 1389, 1390, 1391, 1392, 1393, 1394, 1396, 1399, 1400, 1401, 1402, 1403, 1404, 1405, 1407, 1410, 1411, 1412, 1413, 1414, 1415, 1416, 1418, 1421, 1422, 1423, 1424, 1425, 1426, 1427, 1429, 1432, 1433, 1434, 1435, 1436, 1437, 1438, 1440, 1443, 1444, 1445, 1446, 1447, 1448, 1449, 1451, 1454, 1455, 1456, 1457, 1458, 1459, 1460, 1462, 1465, 1466, 1467, 1468, 1469, 1470, 1471, 1473, 1476, 1477, 1478, 1479, 1480, 1481, 1482, 1484, 1487, 1488, 1489, 1490, 1491, 1492, 1493, 1495, 1498, 1499, 1500, 1501, 1502, 1503, 1504, 1506, 1509, 1510, 1511, 1512, 1513, 1514, 1515, 1517, 1520, 1521, 1522, 1523, 1524, 1525, 1526, 1528, 1531, 1532, 1533, 1534, 1535, 1536, 1537, 1539, 1542, 1543, 1544, 1545, 1546, 1547, 1548, 1550, 1553, 1554, 1555, 1556, 1557, 1558, 1559, 1561, 1564, 1565, 1566, 1567, 1568, 1569, 1570, 1572, 1575, 1576, 1577, 1578, 1579, 1580, 1581, 1583, 1586, 1587, 1588, 1589, 1590, 1591, 1592, 1594, 1597, 1598, 1599, 1600, 1601, 1602, 1603, 1605, 1608, 1609, 1610, 1611, 1612, 1613, 1614, 1616, 1619, 1620, 1621, 1622, 1623, 1624, 1625, 1627, 1630, 1631, 1632, 1633, 1634, 1635, 1636, 1638, 1641, 1642, 1643, 1644, 1645, 1646, 1647, 1649, 1652, 1653, 1654, 1655, 1656, 1657, 1658, 1660, 1663, 1664, 1665, 1666, 1667, 1668, 1669, 1671, 1674, 1675, 1676, 1677, 1678, 1679, 1680, 1682, 1685, 1686, 1687, 1688, 1689, 1690, 1691, 1693, 1696, 1697, 1698, 1699, 1700, 1701, 1702, 1704, 1707, 1708, 1709, 1710, 1711, 1712, 1713, 1715, 1718, 1719, 1720, 1721, 1722, 1723, 1724, 1726, 1729, 1730, 1731, 1732, 1733, 1734, 1735, 1737, 1740, 1741, 1742, 1743, 1744, 1745, 1746, 1748, 1751, 1752, 1753, 1754, 1755, 1756, 1757, 1759, 1762, 1763, 1764, 1765, 1766, 1767, 1768, 1770, 1773, 1774, 1775, 1776, 1777, 1778, 1779, 1781, 1784, 1785, 1786, 1787, 1788, 1789, 1790, 1792, 1795, 1796, 1797, 1798, 1799, 1800, 1801, 1803, 1806, 1807, 1808, 1809, 1810, 1811, 1812, 1814, 1817, 1818, 1819, 1820, 1821, 1822, 1823, 1825, 1828, 1829, 1830, 1831, 1832, 1833, 1834, 1836, 1839, 1840, 1841, 1842, 1843, 1844, 1845, 1847, 1850]}], "trunc": {"shrink_cut": 0, "protected_cut": 0}, "seg_lens": {"user": 1799}, "layout": "c", "names": ["delivery"], "w_offs": [[0, 8], [9, 10], [11, 18], [19, 20], [21, 28], [29, 37], [38, 43], [43, 44], [45, 53], [54, 55], [56, 63], [64, 65], [66, 73], [74, 82], [83, 88], [88, 89], [90, 98], [99, 100], [101, 108], [109, 110], [111, 118], [119, 127], [128, 133], [133, 134], [135, 143], [144, 145], [146, 153], [154, 155], [156, 163], [164, 172], [173, 178], [178, 179], [180, 188], [189, 190], [191, 198], [199, 200], [201, 208], [209, 217], [218, 223], [223, 224], [225, 233], [234, 235], [236, 243], [244, 245], [246, 253], [254, 262], [263, 268], [268, 269], [270, 278], [279, 280], [281, 288], [289, 290], [291, 298], [299, 307], [308, 313], [313, 314], [315, 323], [324, 325], [326, 333], [334, 335], [336, 343], [344, 352], [353, 358], [358, 359], [360, 368], [369, 370], [371, 378], [379, 380], [381, 388], [389, 397], [398, 403], [403, 404], [405, 413], [414, 415], [416, 423], [424, 425], [426, 433], [434, 442], [443, 448], [448, 449], [450, 458], [459, 461], [462, 469], [470, 471], [472, 479], [480, 488], [489, 494], [494, 495], [496, 504], [505, 507], [508, 515], [516, 517], [518, 525], [526, 534], [535, 540], [540, 541], [542, 550], [551, 553], [554, 561], [562, 563], [564, 571], [572, 580], [581, 586], [586, 587], [588, 596], [597, 599], [600, 607], [608, 609], [610, 617], [618, 626], [627, 632], [632, 633], [634, 642], [643, 645], [646, 653], [654, 655], [656, 663], [664, 672], [673, 678], [678, 679], [680, 688], [689, 691], [692, 699], [700, 701], [702, 709], [710, 718], [719, 724], [724, 725], [726, 734], [735, 737], [738, 745], [746, 747], [748, 755], [756, 764], [765, 770], [770, 771], [772, 780], [781, 783], [784, 791], [792, 793], [794, 801], [802, 810], [811, 816], [816, 817], [818, 826], [827, 829], [830, 837], [838, 839], [840, 847], [848, 856], [857, 862], [862, 863], [864, 872], [873, 875], [876, 883], [884, 885], [886, 893], [894, 902], [903, 908], [908, 909], [910, 918], [919, 921], [922, 929], [930, 931], [932, 939], [940, 948], [949, 954], [954, 955], [956, 964], [965, 967], [968, 975], [976, 977], [978, 985], [986, 994], [995, 1000], [1000, 1001], [1002, 1010], [1011, 1013], [1014, 1021], [1022, 1023], [1024, 1031], [1032, 1040], [1041, 1046], [1046, 1047], [1048, 1056], [1057, 1059], [1060, 1067], [1068, 1069], [1070, 1077], [1078, 1086], [1087, 1092], [1092, 1093], [1094, 1102], [1103, 1105], [1106, 1113], [1114, 1115], [1116, 1123], [1124, 1132], [1133, 1138], [1138, 1139], [1140, 1148], [1149, 1151], [1152, 1159], [1160, 1161], [1162, 1169], [1170, 1178], [1179, 1184], [1184, 1185], [1186, 1194], [1195, 1197], [1198, 1205], [1206, 1207], [1208, 1215], [1216, 1224], [1225, 1230], [1230, 1231], [1232, 1240], [1241, 1243], [1244, 1251], [1252, 1253], [1254, 1261], [1262, 1270], [1271, 1276], [1276, 1277], [1278, 1286], [1287, 1289], [1290, 1297], [1298, 1299], [1300, 1307], [1308, 1316], [1317, 1322], [1322, 1323], [1324, 1332], [1333, 1335], [1336, 1343], [1344, 1345], [1346, 1353], [1354, 1362], [1363, 1368], [1368, 1369], [1370, 1378], [1379, 1381], [1382, 1389], [1390, 1391], [1392, 1399], [1400, 1408], [1409, 1414], [1414, 1415], [1416, 1424], [1425, 1427], [1428, 1435], [1436, 1437], [1438, 1445], [1446, 1454], [1455, 1460], [1460, 1461], [1462, 1470], [1471, 1473], [1474, 1481], [1482, 1483], [1484, 1491], [1492, 1500], [1501, 1506], [1506, 1507], [1508, 1516], [1517, 1519], [1520, 1527], [1528, 1529], [1530, 1537], [1538, 1546], [1547, 1552], [1552, 1553], [1554, 1562], [1563, 1565], [1566, 1573], [1574, 1575], [1576, 1583], [1584, 1592], [1593, 1598], [1598, 1599], [1600, 1608], [1609, 1611], [1612, 1619], [1620, 1621], [1622, 1629], [1630, 1638], [1639, 1644], [1644, 1645], [1646, 1654], [1655, 1657], [1658, 1665], [1666, 1667], [1668, 1675], [1676, 1684], [1685, 1690], [1690, 1691], [1692, 1700], [1701, 1703], [1704, 1711], [1712, 1713], [1714, 1721], [1722, 1730], [1731, 1736], [1736, 1737], [1738, 1746], [1747, 1749], [1750, 1757], [1758, 1759], [1760, 1767], [1768, 1776], [1777, 1782], [1782, 1783], [1784, 1792], [1793, 1795], [1796, 1803], [1804, 1805], [1806, 1813], [1814, 1822], [1823, 1828], [1828, 1829], [1830, 1838], [1839, 1841], [1842, 1849], [1850, 1851], [1852, 1859], [1860, 1868], [1869, 1874], [1874, 1875], [1876, 1884], [1885, 1887], [1888, 1895], [1896, 1897], [1898, 1905], [1906, 1914], [1915, 1920], [1920, 1921], [1922, 1930], [1931, 1933], [1934, 1941], [1942, 1943], [1944, 1951], [1952, 1960], [1961, 1966], [1966, 1967], [1968, 1976], [1977, 1979], [1980, 1987], [1988, 1989], [1990, 1997], [1998, 2006], [2007, 2012], [2012, 2013], [2014, 2022], [2023, 2025], [2026, 2033], [2034, 2035], [2036, 2043], [2044, 2052], [2053, 2058], [2058, 2059], [2060, 2068], [2069, 2071], [2072, 2079], [2080, 2081], [2082, 2089], [2090, 2098], [2099, 2104], [2104, 2105], [2106, 2114], [2115, 2117], [2118, 2125], [2126, 2127], [2128, 2135], [2136, 2144], [2145, 2150], [2150, 2151], [2152, 2160], [2161, 2163], [2164, 2171], [2172, 2173], [2174, 2181], [2182, 2190], [2191, 2196], [2196, 2197], [2198, 2206], [2207, 2209], [2210, 2217], [2218, 2219], [2220, 2227], [2228, 2236], [2237, 2242], [2242, 2243], [2244, 2252], [2253, 2255], [2256, 2263], [2264, 2265], [2266, 2273], [2274, 2282], [2283, 2288], [2288, 2289], [2290, 2298], [2299, 2301], [2302, 2309], [2310, 2311], [2312, 2319], [2320, 2328], [2329, 2334], [2334, 2335], [2336, 2344], [2345, 2347], [2348, 2355], [2356, 2357], [2358, 2365], [2366, 2374], [2375, 2380], [2380, 2381], [2382, 2390], [2391, 2393], [2394, 2401], [2402, 2403], [2404, 2411], [2412, 2420], [2421, 2426], [2426, 2427], [2428, 2436], [2437, 2439], [2440, 2447], [2448, 2449], [2450, 2457], [2458, 2466], [2467, 2472], [2472, 2473], [2474, 2482], [2483, 2485], [2486, 2493], [2494, 2495], [2496, 2503], [2504, 2512], [2513, 2518], [2518, 2519], [2520, 2528], [2529, 2531], [2532, 2539], [2540, 2541], [2542, 2549], [2550, 2558], [2559, 2564], [2564, 2565], [2566, 2574], [2575, 2577], [2578, 2585], [2586, 2587], [2588, 2595], [2596, 2604], [2605, 2610], [2610, 2611], [2612, 2620], [2621, 2623], [2624, 2631], [2632, 2633], [2634, 2641], [2642, 2650], [2651, 2656], [2656, 2657], [2658, 2666], [2667, 2669], [2670, 2677], [2678, 2679], [2680, 2687], [2688, 2696], [2697, 2702], [2702, 2703], [2704, 2712], [2713, 2715], [2716, 2723], [2724, 2725], [2726, 2733], [2734, 2742], [2743, 2748], [2748, 2749], [2750, 2758], [2759, 2761], [2762, 2769], [2770, 2771], [2772, 2779], [2780, 2788], [2789, 2794], [2794, 2795], [2796, 2804], [2805, 2807], [2808, 2815], [2816, 2817], [2818, 2825], [2826, 2834], [2835, 2840], [2840, 2841], [2842, 2850], [2851, 2853], [2854, 2861], [2862, 2863], [2864, 2871], [2872, 2880], [2881, 2886], [2886, 2887], [2888, 2896], [2897, 2899], [2900, 2907], [2908, 2909], [2910, 2917], [2918, 2926], [2927, 2932], [2932, 2933], [2934, 2942], [2943, 2945], [2946, 2953], [2954, 2955], [2956, 2963], [2964, 2972], [2973, 2978], [2978, 2979], [2980, 2988], [2989, 2991], [2992, 2999], [3000, 3001], [3002, 3009], [3010, 3018], [3019, 3024], [3024, 3025], [3026, 3034], [3035, 3037], [3038, 3045], [3046, 3047], [3048, 3055], [3056, 3064], [3065, 3070], [3070, 3071], [3072, 3080], [3081, 3083], [3084, 3091], [3092, 3093], [3094, 3101], [3102, 3110], [3111, 3116], [3116, 3117], [3118, 3126], [3127, 3129], [3130, 3137], [3138, 3139], [3140, 3147], [3148, 3156], [3157, 3162], [3162, 3163], [3164, 3172], [3173, 3175], [3176, 3183], [3184, 3185], [3186, 3193], [3194, 3202], [3203, 3208], [3208, 3209], [3210, 3218], [3219, 3221], [3222, 3229], [3230, 3231], [3232, 3239], [3240, 3248], [3249, 3254], [3254, 3255], [3256, 3264], [3265, 3267], [3268, 3275], [3276, 3277], [3278, 3285], [3286, 3294], [3295, 3300], [3300, 3301], [3302, 3310], [3311, 3313], [3314, 3321], [3322, 3323], [3324, 3331], [3332, 3340], [3341, 3346], [3346, 3347], [3348, 3356], [3357, 3359], [3360, 3367], [3368, 3369], [3370, 3377], [3378, 3386], [3387, 3392], [3392, 3393], [3394, 3402], [3403, 3405], [3406, 3413], [3414, 3415], [3416, 3423], [3424, 3432], [3433, 3438], [3438, 3439], [3440, 3448], [3449, 3451], [3452, 3459], [3460, 3461], [3462, 3469], [3470, 3478], [3479, 3484], [3484, 3485], [3486, 3494], [3495, 3497], [3498, 3505], [3506, 3507], [3508, 3515], [3516, 3524], [3525, 3530], [3530, 3531], [3532, 3540], [3541, 3543], [3544, 3551], [3552, 3553], [3554, 3561], [3562, 3570], [3571, 3576], [3576, 3577], [3578, 3586], [3587, 3589], [3590, 3597], [3598, 3599], [3600, 3607], [3608, 3616], [3617, 3622], [3622, 3623], [3624, 3632], [3633, 3635], [3636, 3643], [3644, 3645], [3646, 3653], [3654, 3662], [3663, 3668], [3668, 3669], [3670, 3678], [3679, 3681], [3682, 3689], [3690, 3691], [3692, 3699], [3700, 3708], [3709, 3714], [3714, 3715], [3716, 3724], [3725, 3727], [3728, 3735], [3736, 3737], [3738, 3745], [3746, 3754], [3755, 3760], [3760, 3761], [3762, 3770], [3771, 3773], [3774, 3781], [3782, 3783], [3784, 3791], [3792, 3800], [3801, 3806], [3806, 3807], [3808, 3816], [3817, 3819], [3820, 3827], [3828, 3829], [3830, 3837], [3838, 3846], [3847, 3852], [3852, 3853], [3854, 3862], [3863, 3865], [3866, 3873], [3874, 3875], [3876, 3883], [3884, 3892], [3893, 3898], [3898, 3899], [3900, 3908], [3909, 3911], [3912, 3919], [3920, 3921], [3922, 3929], [3930, 3938], [3939, 3944], [3944, 3945], [3946, 3954], [3955, 3957], [3958, 3965], [3966, 3967], [3968, 3975], [3976, 3984], [3985, 3990], [3990, 3991], [3992, 4000], [4001, 4003], [4004, 4011], [4012, 4013], [4014, 4021], [4022, 4030], [4031, 4036], [4036, 4037], [4038, 4046], [4047, 4049], [4050, 4057], [4058, 4059], [4060, 4067], [4068, 4076], [4077, 4082], [4082, 4083], [4084, 4092], [4093, 4095], [4096, 4103], [4104, 4105], [4106, 4113], [4114, 4122], [4123, 4128], [4128, 4129], [4130, 4138], [4139, 4141], [4142, 4149], [4150, 4151], [4152, 4159], [4160, 4168], [4169, 4174], [4174, 4175], [4176, 4184], [4185, 4187], [4188, 4195], [4196, 4197], [4198, 4205], [4206, 4214], [4215, 4220], [4220, 4221], [4222, 4230], [4231, 4233], [4234, 4241], [4242, 4243], [4244, 4251], [4252, 4260], [4261, 4266], [4266, 4267], [4268, 4276], [4277, 4279], [4280, 4287], [4288, 4289], [4290, 4297], [4298, 4306], [4307, 4312], [4312, 4313], [4314, 4322], [4323, 4325], [4326, 4333], [4334, 4335], [4336, 4343], [4344, 4352], [4353, 4358], [4358, 4359], [4360, 4368], [4369, 4371], [4372, 4379], [4380, 4381], [4382, 4389], [4390, 4398], [4399, 4404], [4404, 4405], [4406, 4414], [4415, 4417], [4418, 4425], [4426, 4427], [4428, 4435], [4436, 4444], [4445, 4450], [4450, 4451], [4452, 4460], [4461, 4463], [4464, 4471], [4472, 4473], [4474, 4481], [4482, 4490], [4491, 4496], [4496, 4497], [4498, 4506], [4507, 4509], [4510, 4517], [4518, 4519], [4520, 4527], [4528, 4536], [4537, 4542], [4542, 4543], [4544, 4552], [4553, 4555], [4556, 4563], [4564, 4565], [4566, 4573], [4574, 4582], [4583, 4588], [4588, 4589], [4590, 4598], [4599, 4602], [4603, 4610], [4611, 4612], [4613, 4620], [4621, 4629], [4630, 4635], [4635, 4636], [4637, 4645], [4646, 4649], [4650, 4657], [4658, 4659], [4660, 4667], [4668, 4676], [4677, 4682], [4682, 4683], [4684, 4692], [4693, 4696], [4697, 4704], [4705, 4706], [4707, 4714], [4715, 4723], [4724, 4729], [4729, 4730], [4731, 4739], [4740, 4743], [4744, 4751], [4752, 4753], [4754, 4761], [4762, 4770], [4771, 4776], [4776, 4777], [4778, 4786], [4787, 4790], [4791, 4798], [4799, 4800], [4801, 4808], [4809, 4817], [4818, 4823], [4823, 4824], [4825, 4833], [4834, 4837], [4838, 4845], [4846, 4847], [4848, 4855], [4856, 4864], [4865, 4870], [4870, 4871], [4872, 4880], [4881, 4884], [4885, 4892], [4893, 4894], [4895, 4902], [4903, 4911], [4912, 4917], [4917, 4918], [4919, 4927], [4928, 4931], [4932, 4939], [4940, 4941], [4942, 4949], [4950, 4958], [4959, 4964], [4964, 4965], [4966, 4974], [4975, 4978], [4979, 4986], [4987, 4988], [4989, 4996], [4997, 5005], [5006, 5011], [5011, 5012], [5013, 5021], [5022, 5025], [5026, 5033], [5034, 5035], [5036, 5043], [5044, 5052], [5053, 5058], [5058, 5059], [5060, 5068], [5069, 5072], [5073, 5080], [5081, 5082], [5083, 5090], [5091, 5099], [5100, 5105], [5105, 5106], [5107, 5115], [5116, 5119], [5120, 5127], [5128, 5129], [5130, 5137], [5138, 5146], [5147, 5152], [5152, 5153], [5154, 5162], [5163, 5166], [5167, 5174], [5175, 5176], [5177, 5184], [5185, 5193], [5194, 5199], [5199, 5200], [5201, 5209], [5210, 5213], [5214, 5221], [5222, 5223], [5224, 5231], [5232, 5240], [5241, 5246], [5246, 5247], [5248, 5256], [5257, 5260], [5261, 5268], [5269, 5270], [5271, 5278], [5279, 5287], [5288, 5293], [5293, 5294], [5295, 5303], [5304, 5307], [5308, 5315], [5316, 5317], [5318, 5325], [5326, 5334], [5335, 5340], [5340, 5341], [5342, 5350], [5351, 5354], [5355, 5362], [5363, 5364], [5365, 5372], [5373, 5381], [5382, 5387], [5387, 5388], [5389, 5397], [5398, 5401], [5402, 5409], [5410, 5411], [5412, 5419], [5420, 5428], [5429, 5434], [5434, 5435], [5436, 5444], [5445, 5448], [5449, 5456], [5457, 5458], [5459, 5466], [5467, 5475], [5476, 5481], [5481, 5482], [5483, 5491], [5492, 5495], [5496, 5503], [5504, 5505], [5506, 5513], [5514, 5522], [5523, 5528], [5528, 5529], [5530, 5538], [5539, 5542], [5543, 5550], [5551, 5552], [5553, 5560], [5561, 5569], [5570, 5575], [5575, 5576], [5577, 5585], [5586, 5589], [5590, 5597], [5598, 5599], [5600, 5607], [5608, 5616], [5617, 5622], [5622, 5623], [5624, 5632], [5633, 5636], [5637, 5644], [5645, 5646], [5647, 5654], [5655, 5663], [5664, 5669], [5669, 5670], [5671, 5679], [5680, 5683], [5684, 5691], [5692, 5693], [5694, 5701], [5702, 5710], [5711, 5716], [5716, 5717], [5718, 5726], [5727, 5730], [5731, 5738], [5739, 5740], [5741, 5748], [5749, 5757], [5758, 5763], [5763, 5764], [5765, 5773], [5774, 5777], [5778, 5785], [5786, 5787], [5788, 5795], [5796, 5804], [5805, 5810], [5810, 5811], [5812, 5820], [5821, 5824], [5825, 5832], [5833, 5834], [5835, 5842], [5843, 5851], [5852, 5857], [5857, 5858], [5859, 5867], [5868, 5871], [5872, 5879], [5880, 5881], [5882, 5889], [5890, 5898], [5899, 5904], [5904, 5905], [5906, 5914], [5915, 5918], [5919, 5926], [5927, 5928], [5929, 5936], [5937, 5945], [5946, 5951], [5951, 5952], [5953, 5961], [5962, 5965], [5966, 5973], [5974, 5975], [5976, 5983], [5984, 5992], [5993, 5998], [5998, 5999], [6000, 6008], [6009, 6012], [6013, 6020], [6021, 6022], [6023, 6030], [6031, 6039], [6040, 6045], [6045, 6046], [6047, 6055], [6056, 6059], [6060, 6067], [6068, 6069], [6070, 6077], [6078, 6086], [6087, 6092], [6092, 6093], [6094, 6102], [6103, 6106], [6107, 6114], [6115, 6116], [6117, 6124], [6125, 6133], [6134, 6139], [6139, 6140], [6141, 6149], [6150, 6153], [6154, 6161], [6162, 6163], [6164, 6171], [6172, 6180], [6181, 6186], [6186, 6187], [6188, 6196], [6197, 6200], [6201, 6208], [6209, 6210], [6211, 6218], [6219, 6227], [6228, 6233], [6233, 6234], [6235, 6243], [6244, 6247], [6248, 6255], [6256, 6257], [6258, 6265], [6266, 6274], [6275, 6280], [6280, 6281], [6282, 6290], [6291, 6294], [6295, 6302], [6303, 6304], [6305, 6312], [6313, 6321], [6322, 6327], [6327, 6328], [6329, 6337], [6338, 6341], [6342, 6349], [6350, 6351], [6352, 6359], [6360, 6368], [6369, 6374], [6374, 6375], [6376, 6384], [6385, 6388], [6389, 6396], [6397, 6398], [6399, 6406], [6407, 6415], [6416, 6421], [6421, 6422], [6423, 6431], [6432, 6435], [6436, 6443], [6444, 6445], [6446, 6453], [6454, 6462], [6463, 6468], [6468, 6469], [6470, 6478], [6479, 6482], [6483, 6490], [6491, 6492], [6493, 6500], [6501, 6509], [6510, 6515], [6515, 6516], [6517, 6525], [6526, 6529], [6530, 6537], [6538, 6539], [6540, 6547], [6548, 6556], [6557, 6562], [6562, 6563], [6564, 6572], [6573, 6576], [6577, 6584], [6585, 6586], [6587, 6594], [6595, 6603], [6604, 6609], [6609, 6610], [6611, 6619], [6620, 6623], [6624, 6631], [6632, 6633], [6634, 6641], [6642, 6650], [6651, 6656], [6656, 6657], [6658, 6666], [6667, 6670], [6671, 6678], [6679, 6680], [6681, 6688], [6689, 6697], [6698, 6703], [6703, 6704], [6705, 6713], [6714, 6717], [6718, 6725], [6726, 6727], [6728, 6735], [6736, 6744], [6745, 6750], [6750, 6751], [6752, 6760], [6761, 6764], [6765, 6772], [6773, 6774], [6775, 6782], [6783, 6791], [6792, 6797], [6797, 6798], [6799, 6807], [6808, 6811], [6812, 6819], [6820, 6821], [6822, 6829], [6830, 6838], [6839, 6844], [6844, 6845], [6846, 6854], [6855, 6858], [6859, 6866], [6867, 6868], [6869, 6876], [6877, 6885], [6886, 6891], [6891, 6892], [6893, 6901], [6902, 6905], [6906, 6913], [6914, 6915], [6916, 6923], [6924, 6932], [6933, 6938], [6938, 6939], [6940, 6948], [6949, 6952], [6953, 6960], [6961, 6962], [6963, 6970], [6971, 6979], [6980, 6985], [6985, 6986], [6987, 6995], [6996, 6999], [7000, 7007], [7008, 7009], [7010, 7017], [7018, 7026], [7027, 7032], [7032, 7033], [7034, 7042], [7043, 7046], [7047, 7054], [7055, 7056], [7057, 7064], [7065, 7073], [7074, 7079], [7079, 7080], [7081, 7089], [7090, 7093], [7094, 7101], [7102, 7103], [7104, 7111], [7112, 7120], [7121, 7126], [7126, 7127], [7128, 7136], [7137, 7140], [7141, 7148], [7149, 7150], [7151, 7158], [7159, 7167], [7168, 7173], [7173, 7174], [7175, 7183], [7184, 7187], [7188, 7195], [7196, 7197], [7198, 7205], [7206, 7214], [7215, 7220], [7220, 7221], [7222, 7230], [7231, 7234], [7235, 7242], [7243, 7244], [7245, 7252], [7253, 7261], [7262, 7267], [7267, 7268], [7269, 7277], [7278, 7281], [7282, 7289], [7290, 7291], [7292, 7299], [7300, 7308], [7309, 7314], [7314, 7315], [7316, 7324], [7325, 7328], [7329, 7336], [7337, 7338], [7339, 7346], [7347, 7355], [7356, 7361], [7361, 7362], [7363, 7371], [7372, 7375], [7376, 7383], [7384, 7385], [7386, 7393], [7394, 7402], [7403, 7408], [7408, 7409], [7410, 7418], [7419, 7422], [7423, 7430], [7431, 7432], [7433, 7440], [7441, 7449], [7450, 7455], [7455, 7456], [7457, 7465], [7466, 7469], [7470, 7477], [7478, 7479], [7480, 7487], [7488, 7496], [7497, 7502], [7502, 7503], [7504, 7512], [7513, 7516], [7517, 7524], [7525, 7526], [7527, 7534], [7535, 7543], [7544, 7549], [7549, 7550], [7551, 7559], [7560, 7563], [7564, 7571], [7572, 7573], [7574, 7581], [7582, 7590], [7591, 7596], [7596, 7597], [7598, 7606], [7607, 7610], [7611, 7618], [7619, 7620], [7621, 7628], [7629, 7637], [7638, 7643], [7643, 7644], [7645, 7653], [7654, 7657], [7658, 7665], [7666, 7667], [7668, 7675], [7676, 7684], [7685, 7690], [7690, 7691], [7692, 7700], [7701, 7704], [7705, 7712], [7713, 7714], [7715, 7722], [7723, 7731], [7732, 7737], [7737, 7738], [7739, 7747], [7748, 7751], [7752, 7759], [7760, 7761], [7762, 7769], [7770, 7778], [7779, 7784], [7784, 7785], [7786, 7794], [7795, 7798], [7799, 7806], [7807, 7808], [7809, 7816], [7817, 7825], [7826, 7831], [7831, 7832], [7833, 7841], [7842, 7845], [7846, 7853], [7854, 7855], [7856, 7863], [7864, 7872], [7873, 7878], [7878, 7879], [7880, 7888], [7889, 7892], [7893, 7900], [7901, 7902], [7903, 7910], [7911, 7919], [7920, 7925], [7925, 7926], [7927, 7935], [7936, 7939], [7940, 7947], [7948, 7949], [7950, 7957], [7958, 7966], [7967, 7972], [7972, 7973], [7974, 7982], [7983, 7986], [7987, 7994], [7995, 7996], [7997, 8004], [8005, 8013], [8014, 8019], [8019, 8020], [8021, 8029], [8030, 8033], [8034, 8041]], "w_pos": [52, 54, 55, 56, 57, 58, 59, 60, 61, 63, 64, 65, 66, 67, 68, 69, 70, 72, 73, 74, 75, 76, 77, 78, 79, 81, 82, 83, 84, 85, 86, 87, 88, 90, 91, 92, 93, 94, 95, 96, 97, 99, 100, 101, 102, 103, 104, 105, 106, 108, 109, 110, 111, 112, 113, 114, 115, 117, 118, 119, 120, 121, 122, 123, 124, 126, 127, 128, 129, 130, 131, 132, 133, 135, 136, 137, 138, 139, 140, 141, 142, 144, 146, 147, 148, 149, 150, 151, 152, 154, 156, 157, 158, 159, 160, 161, 162, 164, 166, 167, 168, 169, 170, 171, 172, 174, 176, 177, 178, 179, 180, 181, 182, 184, 186, 187, 188, 189, 190, 191, 192, 194, 196, 197, 198, 199, 200, 201, 202, 204, 206, 207, 208, 209, 210, 211, 212, 214, 216, 217, 218, 219, 220, 221, 222, 224, 226, 227, 228, 229, 230, 231, 232, 234, 236, 237, 238, 239, 240, 241, 242, 244, 246, 247, 248, 249, 250, 251, 252, 254, 256, 257, 258, 259, 260, 261, 262, 264, 266, 267, 268, 269, 270, 271, 272, 274, 276, 277, 278, 279, 280, 281, 282, 284, 286, 287, 288, 289, 290, 291, 292, 294, 296, 297, 298, 299, 300, 301, 302, 304, 306, 307, 308, 309, 310, 311, 312, 314, 316, 317, 318, 319, 320, 321, 322, 324, 326, 327, 328, 329, 330, 331, 332, 334, 336, 337, 338, 339, 340, 341, 342, 344, 346, 347, 348, 349, 350, 351, 352, 354, 356, 357, 358, 359, 360, 361, 362, 364, 366, 367, 368, 369, 370, 371, 372, 374, 376, 377, 378, 379, 380, 381, 382, 384, 386, 387, 388, 389, 390, 391, 392, 394, 396, 397, 398, 399, 400, 401, 402, 404, 406, 407, 408, 409, 410, 411, 412, 414, 416, 417, 418, 419, 420, 421, 422, 424, 426, 427, 428, 429, 430, 431, 432, 434, 436, 437, 438, 439, 440, 441, 442, 444, 446, 447, 448, 449, 450, 451, 452, 454, 456, 457, 458, 459, 460, 461, 462, 464, 466, 467, 468, 469, 470, 471, 472, 474, 476, 477, 478, 479, 480, 481, 482, 484, 486, 487, 488, 489, 490, 491, 492, 494, 496, 497, 498, 499, 500, 501, 502, 504, 506, 507, 508, 509, 510, 511, 512, 514, 516, 517, 518, 519, 520, 521, 522, 524, 526, 527, 528, 529, 530, 531, 532, 534, 536, 537, 538, 539, 540, 541, 542, 544, 546, 547, 548, 549, 550, 551, 552, 554, 556, 557, 558, 559, 560, 561, 562, 564, 566, 567, 568, 569, 570, 571, 572, 574, 576, 577, 578, 579, 580, 581, 582, 584, 586, 587, 588, 589, 590, 591, 592, 594, 596, 597, 598, 599, 600, 601, 602, 604, 606, 607, 608, 609, 610, 611, 612, 614, 616, 617, 618, 619, 620, 621, 622, 624, 626, 627, 628, 629, 630, 631, 632, 634, 636, 637, 638, 639, 640, 641, 642, 644, 646, 647, 648, 649, 650, 651, 652, 654, 656, 657, 658, 659, 660, 661, 662, 664, 666, 667, 668, 669, 670, 671, 672, 674, 676, 677, 678, 679, 680, 681, 682, 684, 686, 687, 688, 689, 690, 691, 692, 694, 696, 697, 698, 699, 700, 701, 702, 704, 706, 707, 708, 709, 710, 711, 712, 714, 716, 717, 718, 719, 720, 721, 722, 724, 726, 727, 728, 729, 730, 731, 732, 734, 736, 737, 738, 739, 740, 741, 742, 744, 746, 747, 748, 749, 750, 751, 752, 754, 756, 757, 758, 759, 760, 761, 762, 764, 766, 767, 768, 769, 770, 771, 772, 774, 776, 777, 778, 779, 780, 781, 782, 784, 786, 787, 788, 789, 790, 791, 792, 794, 796, 797, 798, 799, 800, 801, 802, 804, 806, 807, 808, 809, 810, 811, 812, 814, 816, 817, 818, 819, 820, 821, 822, 824, 826, 827, 828, 829, 830, 831, 832, 834, 836, 837, 838, 839, 840, 841, 842, 844, 846, 847, 848, 849, 850, 851, 852, 854, 856, 857, 858, 859, 860, 861, 862, 864, 866, 867, 868, 869, 870, 871, 872, 874, 876, 877, 878, 879, 880, 881, 882, 884, 886, 887, 888, 889, 890, 891, 892, 894, 896, 897, 898, 899, 900, 901, 902, 904, 906, 907, 908, 909, 910, 911, 912, 914, 916, 917, 918, 919, 920, 921, 922, 924, 926, 927, 928, 929, 930, 931, 932, 934, 936, 937, 938, 939, 940, 941, 942, 944, 946, 947, 948, 949, 950, 951, 952, 954, 956, 957, 958, 959, 960, 961, 962, 964, 966, 967, 968, 969, 970, 971, 972, 974, 976, 977, 978, 979, 980, 981, 982, 984, 986, 987, 988, 989, 990, 991, 992, 994, 996, 997, 998, 999, 1000, 1001, 1002, 1004, 1006, 1007, 1008, 1009, 1010, 1011, 1012, 1014, 1016, 1017, 1018, 1019, 1020, 1021, 1022, 1024, 1026, 1027, 1028, 1029, 1030, 1031, 1032, 1034, 1036, 1037, 1038, 1039, 1040, 1041, 1042, 1044, 1047, 1048, 1049, 1050, 1051, 1052, 1053, 1055, 1058, 1059, 1060, 1061, 1062, 1063, 1064, 1066, 1069, 1070, 1071, 1072, 1073, 1074, 1075, 1077, 1080, 1081, 1082, 1083, 1084, 1085, 1086, 1088, 1091, 1092, 1093, 1094, 1095, 1096, 1097, 1099, 1102, 1103, 1104, 1105, 1106, 1107, 1108, 1110, 1113, 1114, 1115, 1116, 1117, 1118, 1119, 1121, 1124, 1125, 1126, 1127, 1128, 1129, 1130, 1132, 1135, 1136, 1137, 1138, 1139, 1140, 1141, 1143, 1146, 1147, 1148, 1149, 1150, 1151, 1152, 1154, 1157, 1158, 1159, 1160, 1161, 1162, 1163, 1165, 1168, 1169, 1170, 1171, 1172, 1173, 1174, 1176, 1179, 1180, 1181, 1182, 1183, 1184, 1185, 1187, 1190, 1191, 1192, 1193, 1194, 1195, 1196, 1198, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1209, 1212, 1213, 1214, 1215, 1216, 1217, 1218, 1220, 1223, 1224, 1225, 1226, 1227, 1228, 1229, 1231, 1234, 1235, 1236, 1237, 1238, 1239, 1240, 1242, 1245, 1246, 1247, 1248, 1249, 1250, 1251, 1253, 1256, 1257, 1258, 1259, 1260, 1261, 1262, 1264, 1267, 1268, 1269, 1270, 1271, 1272, 1273, 1275, 1278, 1279, 1280, 1281, 1282, 1283, 1284, 1286, 1289, 1290, 1291, 1292, 1293, 1294, 1295, 1297, 1300, 1301, 1302, 1303, 1304, 1305, 1306, 1308, 1311, 1312, 1313, 1314, 1315, 1316, 1317, 1319, 1322, 1323, 1324, 1325, 1326, 1327, 1328, 1330, 1333, 1334, 1335, 1336, 1337, 1338, 1339, 1341, 1344, 1345, 1346, 1347, 1348, 1349, 1350, 1352, 1355, 1356, 1357, 1358, 1359, 1360, 1361, 1363, 1366, 1367, 1368, 1369, 1370, 1371, 1372, 1374, 1377, 1378, 1379, 1380, 1381, 1382, 1383, 1385, 1388, 1389, 1390, 1391, 1392, 1393, 1394, 1396, 1399, 1400, 1401, 1402, 1403, 1404, 1405, 1407, 1410, 1411, 1412, 1413, 1414, 1415, 1416, 1418, 1421, 1422, 1423, 1424, 1425, 1426, 1427, 1429, 1432, 1433, 1434, 1435, 1436, 1437, 1438, 1440, 1443, 1444, 1445, 1446, 1447, 1448, 1449, 1451, 1454, 1455, 1456, 1457, 1458, 1459, 1460, 1462, 1465, 1466, 1467, 1468, 1469, 1470, 1471, 1473, 1476, 1477, 1478, 1479, 1480, 1481, 1482, 1484, 1487, 1488, 1489, 1490, 1491, 1492, 1493, 1495, 1498, 1499, 1500, 1501, 1502, 1503, 1504, 1506, 1509, 1510, 1511, 1512, 1513, 1514, 1515, 1517, 1520, 1521, 1522, 1523, 1524, 1525, 1526, 1528, 1531, 1532, 1533, 1534, 1535, 1536, 1537, 1539, 1542, 1543, 1544, 1545, 1546, 1547, 1548, 1550, 1553, 1554, 1555, 1556, 1557, 1558, 1559, 1561, 1564, 1565, 1566, 1567, 1568, 1569, 1570, 1572, 1575, 1576, 1577, 1578, 1579, 1580, 1581, 1583, 1586, 1587, 1588, 1589, 1590, 1591, 1592, 1594, 1597, 1598, 1599, 1600, 1601, 1602, 1603, 1605, 1608, 1609, 1610, 1611, 1612, 1613, 1614, 1616, 1619, 1620, 1621, 1622, 1623, 1624, 1625, 1627, 1630, 1631, 1632, 1633, 1634, 1635, 1636, 1638, 1641, 1642, 1643, 1644, 1645, 1646, 1647, 1649, 1652, 1653, 1654, 1655, 1656, 1657, 1658, 1660, 1663, 1664, 1665, 1666, 1667, 1668, 1669, 1671, 1674, 1675, 1676, 1677, 1678, 1679, 1680, 1682, 1685, 1686, 1687, 1688, 1689, 1690, 1691, 1693, 1696, 1697, 1698, 1699, 1700, 1701, 1702, 1704, 1707, 1708, 1709, 1710, 1711, 1712, 1713, 1715, 1718, 1719, 1720, 1721, 1722, 1723, 1724, 1726, 1729, 1730, 1731, 1732, 1733, 1734, 1735, 1737, 1740, 1741, 1742, 1743, 1744, 1745, 1746, 1748, 1751, 1752, 1753, 1754, 1755, 1756, 1757, 1759, 1762, 1763, 1764, 1765, 1766, 1767, 1768, 1770, 1773, 1774, 1775, 1776, 1777, 1778, 1779, 1781, 1784, 1785, 1786, 1787, 1788, 1789, 1790, 1792, 1795, 1796, 1797, 1798, 1799, 1800, 1801, 1803, 1806, 1807, 1808, 1809, 1810, 1811, 1812, 1814, 1817, 1818, 1819, 1820, 1821, 1822, 1823, 1825, 1828, 1829, 1830, 1831, 1832, 1833, 1834, 1836, 1839, 1840, 1841, 1842, 1843, 1844, 1845, 1847, 1850], "span_over": "user", "n_tokens": 3689, "head": "broad"}, {"p_ids": [1905, 25, 198, 27, 22446, 3364, 420, 846, 731, 198, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 24, 6473, 264, 13618, 9403, 1484, 13, 198, 510, 22446, 29, 198], "blocks": [{"kind": "span", "ends": [40], "label_ranges": [[22, 40]], "words_in_block": true, "ids": [271, 6065, 913, 25, 9111, 198, 6196, 25, 1156, 198, 14162, 25, 198, 22365, 42824, 6088, 264, 9403, 30, 198, 22933, 25, 198, 14057, 29, 198, 4754, 4532, 3147, 64, 9403, 1484, 2129, 767, 3147, 31320, 8934, 198, 510, 1448, 29, 271, 1138, 25, 198, 27, 22446, 3364, 420, 846, 731, 198, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 20, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 21, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 22, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 23, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 16, 24, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 15, 24, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 15, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 16, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 17, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 18, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 19, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 20, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 21, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 22, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 23, 6473, 264, 13618, 9403, 1484, 13, 76802, 220, 17, 16, 24, 6473, 264, 13618, 9403, 1484, 13, 198, 510, 22446, 29, 198, 271, 8679, 1396, 9111, 314, 279, 2100, 1414, 421, 22651, 799, 314, 279, 8928, 13, 198, 6205, 581, 25], "query": 849, "w_pos": [52, 53, 54, 55, 56, 58, 61, 62, 63, 64, 65, 66, 67, 69, 72, 73, 74, 75, 76, 77, 78, 80, 83, 84, 85, 86, 87, 88, 89, 91, 94, 95, 96, 97, 98, 99, 100, 102, 105, 106, 107, 108, 109, 110, 111, 113, 116, 117, 118, 119, 120, 121, 122, 124, 127, 128, 129, 130, 131, 132, 133, 135, 138, 139, 140, 141, 142, 143, 144, 146, 149, 150, 151, 152, 153, 154, 155, 157, 160, 161, 162, 163, 164, 165, 166, 168, 171, 172, 173, 174, 175, 176, 177, 179, 182, 183, 184, 185, 186, 187, 188, 190, 193, 194, 195, 196, 197, 198, 199, 201, 204, 205, 206, 207, 208, 209, 210, 212, 215, 216, 217, 218, 219, 220, 221, 223, 226, 227, 228, 229, 230, 231, 232, 234, 237, 238, 239, 240, 241, 242, 243, 245, 248, 249, 250, 251, 252, 253, 254, 256, 259, 260, 261, 262, 263, 264, 265, 267, 270, 271, 272, 273, 274, 275, 276, 278, 281, 282, 283, 284, 285, 286, 287, 289, 292, 293, 294, 295, 296, 297, 298, 300, 303, 304, 305, 306, 307, 308, 309, 311, 314, 315, 316, 317, 318, 319, 320, 322, 325, 326, 327, 328, 329, 330, 331, 333, 336, 337, 338, 339, 340, 341, 342, 344, 347, 348, 349, 350, 351, 352, 353, 355, 358, 359, 360, 361, 362, 363, 364, 366, 369, 370, 371, 372, 373, 374, 375, 377, 380, 381, 382, 383, 384, 385, 386, 388, 391, 392, 393, 394, 395, 396, 397, 399, 402, 403, 404, 405, 406, 407, 408, 410, 413, 414, 415, 416, 417, 418, 419, 421, 424, 425, 426, 427, 428, 429, 430, 432, 435, 436, 437, 438, 439, 440, 441, 443, 446, 447, 448, 449, 450, 451, 452, 454, 457, 458, 459, 460, 461, 462, 463, 465, 468, 469, 470, 471, 472, 473, 474, 476, 479, 480, 481, 482, 483, 484, 485, 487, 490, 491, 492, 493, 494, 495, 496, 498, 501, 502, 503, 504, 505, 506, 507, 509, 512, 513, 514, 515, 516, 517, 518, 520, 523, 524, 525, 526, 527, 528, 529, 531, 534, 535, 536, 537, 538, 539, 540, 542, 545, 546, 547, 548, 549, 550, 551, 553, 556, 557, 558, 559, 560, 561, 562, 564, 567, 568, 569, 570, 571, 572, 573, 575, 578, 579, 580, 581, 582, 583, 584, 586, 589, 590, 591, 592, 593, 594, 595, 597, 600, 601, 602, 603, 604, 605, 606, 608, 611, 612, 613, 614, 615, 616, 617, 619, 622, 623, 624, 625, 626, 627, 628, 630, 633, 634, 635, 636, 637, 638, 639, 641, 644, 645, 646, 647, 648, 649, 650, 652, 655, 656, 657, 658, 659, 660, 661, 663, 666, 667, 668, 669, 670, 671, 672, 674, 677, 678, 679, 680, 681, 682, 683, 685, 688, 689, 690, 691, 692, 693, 694, 696, 699, 700, 701, 702, 703, 704, 705, 707, 710, 711, 712, 713, 714, 715, 716, 718, 721, 722, 723, 724, 725, 726, 727, 729, 732, 733, 734, 735, 736, 737, 738, 740, 743, 744, 745, 746, 747, 748, 749, 751, 754, 755, 756, 757, 758, 759, 760, 762, 765, 766, 767, 768, 769, 770, 771, 773, 776, 777, 778, 779, 780, 781, 782, 784, 787, 788, 789, 790, 791, 792, 793, 795, 798, 799, 800, 801, 802, 803, 804, 806, 809, 810, 811, 812, 813, 814, 815, 817, 820, 821, 822, 823, 824, 825]}], "trunc": {"shrink_cut": 0, "protected_cut": 0}, "seg_lens": {"user": 774}, "layout": "c", "names": ["delivery"], "w_offs": [[6916, 6923], [6924, 6932], [6933, 6938], [6938, 6939], [6940, 6948], [6949, 6952], [6953, 6960], [6961, 6962], [6963, 6970], [6971, 6979], [6980, 6985], [6985, 6986], [6987, 6995], [6996, 6999], [7000, 7007], [7008, 7009], [7010, 7017], [7018, 7026], [7027, 7032], [7032, 7033], [7034, 7042], [7043, 7046], [7047, 7054], [7055, 7056], [7057, 7064], [7065, 7073], [7074, 7079], [7079, 7080], [7081, 7089], [7090, 7093], [7094, 7101], [7102, 7103], [7104, 7111], [7112, 7120], [7121, 7126], [7126, 7127], [7128, 7136], [7137, 7140], [7141, 7148], [7149, 7150], [7151, 7158], [7159, 7167], [7168, 7173], [7173, 7174], [7175, 7183], [7184, 7187], [7188, 7195], [7196, 7197], [7198, 7205], [7206, 7214], [7215, 7220], [7220, 7221], [7222, 7230], [7231, 7234], [7235, 7242], [7243, 7244], [7245, 7252], [7253, 7261], [7262, 7267], [7267, 7268], [7269, 7277], [7278, 7281], [7282, 7289], [7290, 7291], [7292, 7299], [7300, 7308], [7309, 7314], [7314, 7315], [7316, 7324], [7325, 7328], [7329, 7336], [7337, 7338], [7339, 7346], [7347, 7355], [7356, 7361], [7361, 7362], [7363, 7371], [7372, 7375], [7376, 7383], [7384, 7385], [7386, 7393], [7394, 7402], [7403, 7408], [7408, 7409], [7410, 7418], [7419, 7422], [7423, 7430], [7431, 7432], [7433, 7440], [7441, 7449], [7450, 7455], [7455, 7456], [7457, 7465], [7466, 7469], [7470, 7477], [7478, 7479], [7480, 7487], [7488, 7496], [7497, 7502], [7502, 7503], [7504, 7512], [7513, 7516], [7517, 7524], [7525, 7526], [7527, 7534], [7535, 7543], [7544, 7549], [7549, 7550], [7551, 7559], [7560, 7563], [7564, 7571], [7572, 7573], [7574, 7581], [7582, 7590], [7591, 7596], [7596, 7597], [7598, 7606], [7607, 7610], [7611, 7618], [7619, 7620], [7621, 7628], [7629, 7637], [7638, 7643], [7643, 7644], [7645, 7653], [7654, 7657], [7658, 7665], [7666, 7667], [7668, 7675], [7676, 7684], [7685, 7690], [7690, 7691], [7692, 7700], [7701, 7704], [7705, 7712], [7713, 7714], [7715, 7722], [7723, 7731], [7732, 7737], [7737, 7738], [7739, 7747], [7748, 7751], [7752, 7759], [7760, 7761], [7762, 7769], [7770, 7778], [7779, 7784], [7784, 7785], [7786, 7794], [7795, 7798], [7799, 7806], [7807, 7808], [7809, 7816], [7817, 7825], [7826, 7831], [7831, 7832], [7833, 7841], [7842, 7845], [7846, 7853], [7854, 7855], [7856, 7863], [7864, 7872], [7873, 7878], [7878, 7879], [7880, 7888], [7889, 7892], [7893, 7900], [7901, 7902], [7903, 7910], [7911, 7919], [7920, 7925], [7925, 7926], [7927, 7935], [7936, 7939], [7940, 7947], [7948, 7949], [7950, 7957], [7958, 7966], [7967, 7972], [7972, 7973], [7974, 7982], [7983, 7986], [7987, 7994], [7995, 7996], [7997, 8004], [8005, 8013], [8014, 8019], [8019, 8020], [8021, 8029], [8030, 8033], [8034, 8041], [8042, 8043], [8044, 8051], [8052, 8060], [8061, 8066], [8066, 8067], [8068, 8076], [8077, 8080], [8081, 8088], [8089, 8090], [8091, 8098], [8099, 8107], [8108, 8113], [8113, 8114], [8115, 8123], [8124, 8127], [8128, 8135], [8136, 8137], [8138, 8145], [8146, 8154], [8155, 8160], [8160, 8161], [8162, 8170], [8171, 8174], [8175, 8182], [8183, 8184], [8185, 8192], [8193, 8201], [8202, 8207], [8207, 8208], [8209, 8217], [8218, 8221], [8222, 8229], [8230, 8231], [8232, 8239], [8240, 8248], [8249, 8254], [8254, 8255], [8256, 8264], [8265, 8268], [8269, 8276], [8277, 8278], [8279, 8286], [8287, 8295], [8296, 8301], [8301, 8302], [8303, 8311], [8312, 8315], [8316, 8323], [8324, 8325], [8326, 8333], [8334, 8342], [8343, 8348], [8348, 8349], [8350, 8358], [8359, 8362], [8363, 8370], [8371, 8372], [8373, 8380], [8381, 8389], [8390, 8395], [8395, 8396], [8397, 8405], [8406, 8409], [8410, 8417], [8418, 8419], [8420, 8427], [8428, 8436], [8437, 8442], [8442, 8443], [8444, 8452], [8453, 8456], [8457, 8464], [8465, 8466], [8467, 8474], [8475, 8483], [8484, 8489], [8489, 8490], [8491, 8499], [8500, 8503], [8504, 8511], [8512, 8513], [8514, 8521], [8522, 8530], [8531, 8536], [8536, 8537], [8538, 8546], [8547, 8550], [8551, 8558], [8559, 8560], [8561, 8568], [8569, 8577], [8578, 8583], [8583, 8584], [8585, 8593], [8594, 8597], [8598, 8605], [8606, 8607], [8608, 8615], [8616, 8624], [8625, 8630], [8630, 8631], [8632, 8640], [8641, 8644], [8645, 8652], [8653, 8654], [8655, 8662], [8663, 8671], [8672, 8677], [8677, 8678], [8679, 8687], [8688, 8691], [8692, 8699], [8700, 8701], [8702, 8709], [8710, 8718], [8719, 8724], [8724, 8725], [8726, 8734], [8735, 8738], [8739, 8746], [8747, 8748], [8749, 8756], [8757, 8765], [8766, 8771], [8771, 8772], [8773, 8781], [8782, 8785], [8786, 8793], [8794, 8795], [8796, 8803], [8804, 8812], [8813, 8818], [8818, 8819], [8820, 8828], [8829, 8832], [8833, 8840], [8841, 8842], [8843, 8850], [8851, 8859], [8860, 8865], [8865, 8866], [8867, 8875], [8876, 8879], [8880, 8887], [8888, 8889], [8890, 8897], [8898, 8906], [8907, 8912], [8912, 8913], [8914, 8922], [8923, 8926], [8927, 8934], [8935, 8936], [8937, 8944], [8945, 8953], [8954, 8959], [8959, 8960], [8961, 8969], [8970, 8973], [8974, 8981], [8982, 8983], [8984, 8991], [8992, 9000], [9001, 9006], [9006, 9007], [9008, 9016], [9017, 9020], [9021, 9028], [9029, 9030], [9031, 9038], [9039, 9047], [9048, 9053], [9053, 9054], [9055, 9063], [9064, 9067], [9068, 9075], [9076, 9077], [9078, 9085], [9086, 9094], [9095, 9100], [9100, 9101], [9102, 9110], [9111, 9114], [9115, 9122], [9123, 9124], [9125, 9132], [9133, 9141], [9142, 9147], [9147, 9148], [9149, 9157], [9158, 9161], [9162, 9169], [9170, 9171], [9172, 9179], [9180, 9188], [9189, 9194], [9194, 9195], [9196, 9204], [9205, 9208], [9209, 9216], [9217, 9218], [9219, 9226], [9227, 9235], [9236, 9241], [9241, 9242], [9243, 9251], [9252, 9255], [9256, 9263], [9264, 9265], [9266, 9273], [9274, 9282], [9283, 9288], [9288, 9289], [9290, 9298], [9299, 9302], [9303, 9310], [9311, 9312], [9313, 9320], [9321, 9329], [9330, 9335], [9335, 9336], [9337, 9345], [9346, 9349], [9350, 9357], [9358, 9359], [9360, 9367], [9368, 9376], [9377, 9382], [9382, 9383], [9384, 9392], [9393, 9396], [9397, 9404], [9405, 9406], [9407, 9414], [9415, 9423], [9424, 9429], [9429, 9430], [9431, 9439], [9440, 9443], [9444, 9451], [9452, 9453], [9454, 9461], [9462, 9470], [9471, 9476], [9476, 9477], [9478, 9486], [9487, 9490], [9491, 9498], [9499, 9500], [9501, 9508], [9509, 9517], [9518, 9523], [9523, 9524], [9525, 9533], [9534, 9537], [9538, 9545], [9546, 9547], [9548, 9555], [9556, 9564], [9565, 9570], [9570, 9571], [9572, 9580], [9581, 9584], [9585, 9592], [9593, 9594], [9595, 9602], [9603, 9611], [9612, 9617], [9617, 9618], [9619, 9627], [9628, 9631], [9632, 9639], [9640, 9641], [9642, 9649], [9650, 9658], [9659, 9664], [9664, 9665], [9666, 9674], [9675, 9678], [9679, 9686], [9687, 9688], [9689, 9696], [9697, 9705], [9706, 9711], [9711, 9712], [9713, 9721], [9722, 9725], [9726, 9733], [9734, 9735], [9736, 9743], [9744, 9752], [9753, 9758], [9758, 9759], [9760, 9768], [9769, 9772], [9773, 9780], [9781, 9782], [9783, 9790], [9791, 9799], [9800, 9805], [9805, 9806], [9807, 9815], [9816, 9819], [9820, 9827], [9828, 9829], [9830, 9837], [9838, 9846], [9847, 9852], [9852, 9853], [9854, 9862], [9863, 9866], [9867, 9874], [9875, 9876], [9877, 9884], [9885, 9893], [9894, 9899], [9899, 9900], [9901, 9909], [9910, 9913], [9914, 9921], [9922, 9923], [9924, 9931], [9932, 9940], [9941, 9946], [9946, 9947], [9948, 9956], [9957, 9960], [9961, 9968], [9969, 9970], [9971, 9978], [9979, 9987], [9988, 9993], [9993, 9994], [9995, 10003], [10004, 10007], [10008, 10015], [10016, 10017], [10018, 10025], [10026, 10034], [10035, 10040], [10040, 10041], [10042, 10050], [10051, 10054], [10055, 10062], [10063, 10064], [10065, 10072], [10073, 10081], [10082, 10087], [10087, 10088], [10089, 10097], [10098, 10101], [10102, 10109], [10110, 10111], [10112, 10119], [10120, 10128], [10129, 10134], [10134, 10135], [10136, 10144], [10145, 10148], [10149, 10156], [10157, 10158], [10159, 10166], [10167, 10175], [10176, 10181], [10181, 10182], [10183, 10191], [10192, 10195], [10196, 10203], [10204, 10205], [10206, 10213], [10214, 10222], [10223, 10228], [10228, 10229]], "w_pos": [52, 53, 54, 55, 56, 58, 61, 62, 63, 64, 65, 66, 67, 69, 72, 73, 74, 75, 76, 77, 78, 80, 83, 84, 85, 86, 87, 88, 89, 91, 94, 95, 96, 97, 98, 99, 100, 102, 105, 106, 107, 108, 109, 110, 111, 113, 116, 117, 118, 119, 120, 121, 122, 124, 127, 128, 129, 130, 131, 132, 133, 135, 138, 139, 140, 141, 142, 143, 144, 146, 149, 150, 151, 152, 153, 154, 155, 157, 160, 161, 162, 163, 164, 165, 166, 168, 171, 172, 173, 174, 175, 176, 177, 179, 182, 183, 184, 185, 186, 187, 188, 190, 193, 194, 195, 196, 197, 198, 199, 201, 204, 205, 206, 207, 208, 209, 210, 212, 215, 216, 217, 218, 219, 220, 221, 223, 226, 227, 228, 229, 230, 231, 232, 234, 237, 238, 239, 240, 241, 242, 243, 245, 248, 249, 250, 251, 252, 253, 254, 256, 259, 260, 261, 262, 263, 264, 265, 267, 270, 271, 272, 273, 274, 275, 276, 278, 281, 282, 283, 284, 285, 286, 287, 289, 292, 293, 294, 295, 296, 297, 298, 300, 303, 304, 305, 306, 307, 308, 309, 311, 314, 315, 316, 317, 318, 319, 320, 322, 325, 326, 327, 328, 329, 330, 331, 333, 336, 337, 338, 339, 340, 341, 342, 344, 347, 348, 349, 350, 351, 352, 353, 355, 358, 359, 360, 361, 362, 363, 364, 366, 369, 370, 371, 372, 373, 374, 375, 377, 380, 381, 382, 383, 384, 385, 386, 388, 391, 392, 393, 394, 395, 396, 397, 399, 402, 403, 404, 405, 406, 407, 408, 410, 413, 414, 415, 416, 417, 418, 419, 421, 424, 425, 426, 427, 428, 429, 430, 432, 435, 436, 437, 438, 439, 440, 441, 443, 446, 447, 448, 449, 450, 451, 452, 454, 457, 458, 459, 460, 461, 462, 463, 465, 468, 469, 470, 471, 472, 473, 474, 476, 479, 480, 481, 482, 483, 484, 485, 487, 490, 491, 492, 493, 494, 495, 496, 498, 501, 502, 503, 504, 505, 506, 507, 509, 512, 513, 514, 515, 516, 517, 518, 520, 523, 524, 525, 526, 527, 528, 529, 531, 534, 535, 536, 537, 538, 539, 540, 542, 545, 546, 547, 548, 549, 550, 551, 553, 556, 557, 558, 559, 560, 561, 562, 564, 567, 568, 569, 570, 571, 572, 573, 575, 578, 579, 580, 581, 582, 583, 584, 586, 589, 590, 591, 592, 593, 594, 595, 597, 600, 601, 602, 603, 604, 605, 606, 608, 611, 612, 613, 614, 615, 616, 617, 619, 622, 623, 624, 625, 626, 627, 628, 630, 633, 634, 635, 636, 637, 638, 639, 641, 644, 645, 646, 647, 648, 649, 650, 652, 655, 656, 657, 658, 659, 660, 661, 663, 666, 667, 668, 669, 670, 671, 672, 674, 677, 678, 679, 680, 681, 682, 683, 685, 688, 689, 690, 691, 692, 693, 694, 696, 699, 700, 701, 702, 703, 704, 705, 707, 710, 711, 712, 713, 714, 715, 716, 718, 721, 722, 723, 724, 725, 726, 727, 729, 732, 733, 734, 735, 736, 737, 738, 740, 743, 744, 745, 746, 747, 748, 749, 751, 754, 755, 756, 757, 758, 759, 760, 762, 765, 766, 767, 768, 769, 770, 771, 773, 776, 777, 778, 779, 780, 781, 782, 784, 787, 788, 789, 790, 791, 792, 793, 795, 798, 799, 800, 801, 802, 803, 804, 806, 809, 810, 811, 812, 813, 814, 815, 817, 820, 821, 822, 823, 824, 825], "span_over": "user", "n_tokens": 1639, "head": "broad"}]} diff --git a/tests/vela/fixtures/oracle-hybrid-example-request-cjk.json b/tests/vela/fixtures/oracle-hybrid-example-request-cjk.json new file mode 100644 index 00000000..ddf6c71c --- /dev/null +++ b/tests/vela/fixtures/oracle-hybrid-example-request-cjk.json @@ -0,0 +1,155 @@ +{ + "case": "example-request-cjk", + "request": { + "model": "vllm-sr/Vela-2.0-0.8B", + "state": { + "request": "客户张伟（zhang.wei@example.cn）于2026年10月1日在上海市浦东新区世纪大道100号办理了业务，联系电话13800138000。", + "source": "客户的业务申请需要在三个工作日内审核完成。", + "answer": "客户的业务已经即时完成了。" + }, + "questions": { + "pii": { + "type": "span", + "instructions": "Which spans are personal information?", + "criteria": { + "PERSON": "a person's name", + "EMAIL_ADDRESS": "an e-mail address", + "PHONE_NUMBER": "a telephone number", + "STREET_ADDRESS": "a street address or postal address" + }, + "over": "request" + }, + "unsupported": { + "type": "span", + "instructions": "Which spans of the answer are not supported by the context?", + "criteria": { + "unsupported": "a claim not supported by the context" + } + }, + "billing": { + "type": "noul", + "instructions": "这个请求是否与账单有关？" + } + } + }, + "system_one": { + "model": "vllm-sr/Vela-2.0-0.8B", + "answers": { + "pii": { + "type": "noul", + "noul": 0.8401349388137276 + }, + "unsupported": { + "type": "noul", + "noul": 0.28838747148371713 + }, + "billing": { + "type": "noul", + "noul": 0.1674766566538726 + } + }, + "usage": { + "input_tokens": 597, + "output_tokens": 0 + }, + "spans": { + "pii": [ + { + "label": "PHONE_NUMBER", + "start": 32, + "end": 34, + "text": "10", + "probability": 0.4658691883087158 + }, + { + "label": "STREET_ADDRESS", + "start": 38, + "end": 52, + "text": "上海市浦东新区世纪大道100", + "probability": 0.4821831285953522 + }, + { + "label": "PHONE_NUMBER", + "start": 63, + "end": 74, + "text": "13800138000", + "probability": 0.4298103451728821 + } + ], + "unsupported": [] + }, + "span_heads": { + "pii": "router", + "unsupported": "router" + }, + "thresholds": { + "pii": 0.2914, + "unsupported": 0.5 + } + }, + "predict": { + "answers": [ + { + "id": "pii", + "type": "span", + "target_part": "user", + "spans": [ + { + "start": 32, + "end": 34, + "label": "PHONE_NUMBER", + "probability": 0.4658691883087158, + "text": "10" + }, + { + "start": 38, + "end": 52, + "label": "STREET_ADDRESS", + "probability": 0.4821831285953522, + "text": "上海市浦东新区世纪大道100" + }, + { + "start": 63, + "end": 74, + "label": "PHONE_NUMBER", + "probability": 0.4298103451728821, + "text": "13800138000" + } + ], + "threshold": 0.2914, + "threshold_rule": "pii:length", + "head": "router", + "head_reason": "id" + }, + { + "id": "unsupported", + "type": "span", + "target_part": "answer", + "spans": [], + "threshold": 0.5, + "threshold_rule": "span:*", + "head": "router", + "head_reason": "label_set" + }, + { + "id": "billing", + "type": "choice", + "answer": "no", + "probabilities": { + "no": 0.8325233433461274, + "yes": 0.1674766566538726 + }, + "abstain_probability": 0.0009211774164155532 + } + ], + "meta": { + "tokens": { + "user": 46, + "context": 10, + "answer": 6 + }, + "windows": 0, + "sequences": 2 + } + } +} diff --git a/tests/vela/fixtures/oracle-hybrid-example-request-score.json b/tests/vela/fixtures/oracle-hybrid-example-request-score.json new file mode 100644 index 00000000..9491e486 --- /dev/null +++ b/tests/vela/fixtures/oracle-hybrid-example-request-score.json @@ -0,0 +1,105 @@ +{ + "case": "example-request-score", + "request": { + "model": "vllm-sr/Vela-2.0-0.8B", + "state": "Hello, I would like to check the status of my order. It has been five days and I have heard nothing back.", + "questions": { + "tone": { + "type": "score", + "instructions": "How formal is the message?", + "criteria": [ + "casual", + "neutral", + "formal" + ] + }, + "urgency": { + "type": "score", + "instructions": "How urgently does the sender need a reply?", + "criteria": [ + "no rush", + "soon", + "today", + "immediately" + ] + } + } + }, + "system_one": { + "model": "vllm-sr/Vela-2.0-0.8B", + "answers": { + "tone": { + "type": "score", + "score": 1.1498890868381673, + "confidence": 0.3046350733874953, + "legend": { + "0": "casual", + "1": "neutral", + "2": "formal" + }, + "probabilities": { + "0": 0.16807713462834117, + "1": 0.5139566439051507, + "2": 0.3179662214665082 + } + }, + "urgency": { + "type": "score", + "score": 0.9787050638239541, + "confidence": 0.19233689920934172, + "legend": { + "0": "no rush", + "1": "soon", + "2": "today", + "3": "immediately" + }, + "probabilities": { + "0": 0.44128118285871365, + "1": 0.21311503083546, + "2": 0.2712213259289846, + "3": 0.07438246037684161 + } + } + }, + "usage": { + "input_tokens": 228, + "output_tokens": 0 + } + }, + "predict": { + "answers": [ + { + "id": "tone", + "type": "score", + "answer": "neutral", + "probabilities": { + "casual": 0.16807713462834117, + "neutral": 0.5139566439051507, + "formal": 0.3179662214665082 + }, + "expected_index": 1.1498890868381673, + "score": 0.5749445434190836 + }, + { + "id": "urgency", + "type": "score", + "answer": "no rush", + "probabilities": { + "no rush": 0.44128118285871365, + "soon": 0.21311503083546, + "today": 0.2712213259289846, + "immediately": 0.07438246037684161 + }, + "expected_index": 0.9787050638239541, + "score": 0.3262350212746513 + } + ], + "meta": { + "tokens": { + "user": 25 + }, + "windows": 0, + "sequences": 1 + } + } +} diff --git a/tests/vela/fixtures/oracle-hybrid-example-request-set-threshold.json b/tests/vela/fixtures/oracle-hybrid-example-request-set-threshold.json new file mode 100644 index 00000000..bab3d048 --- /dev/null +++ b/tests/vela/fixtures/oracle-hybrid-example-request-set-threshold.json @@ -0,0 +1,81 @@ +{ + "case": "example-request-set-threshold", + "request": { + "model": "vllm-sr/Vela-2.0-0.8B", + "state": "My card was charged twice and the parcel never arrived.", + "questions": { + "issues": { + "type": "set", + "instructions": "Which issues does the customer report?", + "criteria": { + "billing": "payments, charges, refunds or invoices", + "shipping": "delivery of an order or a parcel", + "login": "signing in, passwords or account access" + }, + "threshold": 0.15 + } + } + }, + "system_one": { + "model": "vllm-sr/Vela-2.0-0.8B", + "answers": { + "issues.billing": { + "type": "noul", + "noul": 0.6211446881608428 + }, + "issues.shipping": { + "type": "noul", + "noul": 0.8205402483597802 + }, + "issues.login": { + "type": "noul", + "noul": 0.00012988178108716601 + } + }, + "usage": { + "input_tokens": 133, + "output_tokens": 0 + }, + "sets": { + "issues": { + "selected": [ + "billing", + "shipping" + ], + "probabilities": { + "billing": 0.6211446881608428, + "shipping": 0.8205402483597802, + "login": 0.00012988178108716601 + } + } + }, + "thresholds": { + "issues": 0.15 + } + }, + "predict": { + "answers": [ + { + "id": "issues", + "type": "set", + "answer": [ + "billing", + "shipping" + ], + "probabilities": { + "billing": 0.6211446881608428, + "shipping": 0.8205402483597802, + "login": 0.00012988178108716601 + }, + "threshold": 0.15 + } + ], + "meta": { + "tokens": { + "user": 11 + }, + "windows": 0, + "sequences": 1 + } + } +} diff --git a/tests/vela/fixtures/oracle-hybrid-example-request.json b/tests/vela/fixtures/oracle-hybrid-example-request.json new file mode 100644 index 00000000..801ef7aa --- /dev/null +++ b/tests/vela/fixtures/oracle-hybrid-example-request.json @@ -0,0 +1,183 @@ +{ + "case": "example-request", + "request": { + "model": "vllm-sr/Vela-2.0-0.8B", + "state": { + "request": "Hi, I'm Tom Baker (tom.baker@example.com). What is the maximum daily dose of paracetamol for an adult?", + "source": "For adults, the maximum dose of paracetamol is 4 grams in 24 hours, taken as 500 mg to 1 g every 4 to 6 hours.", + "answer": "Adults can take up to 6 grams of paracetamol in 24 hours, in doses of 500 mg to 1 g every 4 to 6 hours." + }, + "questions": { + "pii": { + "type": "span", + "instructions": "Which spans are personal information?", + "criteria": { + "AGE": "a person's age", + "CREDIT_CARD": "a payment card number", + "DATE_TIME": "a date, time or date of birth", + "DOMAIN_NAME": "an internet domain name or website address", + "EMAIL_ADDRESS": "an e-mail address", + "GPE": "a country, city, state or other geopolitical place", + "IBAN_CODE": "an international bank account number (IBAN)", + "IP_ADDRESS": "an IPv4 or IPv6 address", + "NRP": "a nationality, religious or political group", + "ORGANIZATION": "the name of a company, institution or other organisation", + "PERSON": "a person's name", + "PHONE_NUMBER": "a telephone number", + "STREET_ADDRESS": "a street address or postal address", + "TITLE": "a personal or professional title such as Dr. or Mrs.", + "US_DRIVER_LICENSE": "a US driver's licence number", + "US_SSN": "a US social security number", + "ZIP_CODE": "a postal or ZIP code" + }, + "over": "request" + }, + "halu": { + "type": "span", + "instructions": "Which spans of the answer are not supported by the context?", + "criteria": { + "unsupported": "a claim not supported by the context" + } + }, + "domain": { + "type": "choice", + "instructions": "Which subject area is this request about?", + "over": "request", + "criteria": { + "health": "medicine, clinical practice, nutrition, ageing or sexual health", + "math": "arithmetic, algebra, geometry, statistics or other mathematics", + "other": "a subject that fits none of the listed areas" + } + } + } + }, + "system_one": { + "model": "vllm-sr/Vela-2.0-0.8B", + "answers": { + "pii": { + "type": "noul", + "noul": 0.9965421673633966 + }, + "halu": { + "type": "noul", + "noul": 0.9972382713052719 + }, + "domain": { + "type": "choice", + "choice": "health", + "confidence": 0.9288764270674694, + "probabilities": { + "health": 0.9621185645193945, + "math": 0.004639298028680393, + "other": 0.03324213745192508 + } + } + }, + "usage": { + "input_tokens": 1041, + "output_tokens": 0 + }, + "spans": { + "pii": [ + { + "label": "PERSON", + "start": 8, + "end": 17, + "text": "Tom Baker", + "probability": 0.983367919921875 + }, + { + "label": "EMAIL_ADDRESS", + "start": 19, + "end": 40, + "text": "tom.baker@example.com", + "probability": 0.9851056337356567 + } + ], + "halu": [ + { + "label": "unsupported", + "start": 19, + "end": 29, + "text": "to 6 grams", + "probability": 0.7782317996025085 + } + ] + }, + "span_heads": { + "pii": "router", + "halu": "router" + }, + "thresholds": { + "pii": 0.2914, + "halu": 0.35 + } + }, + "predict": { + "answers": [ + { + "id": "pii", + "type": "span", + "target_part": "user", + "spans": [ + { + "start": 8, + "end": 17, + "label": "PERSON", + "probability": 0.983367919921875, + "text": "Tom Baker" + }, + { + "start": 19, + "end": 40, + "label": "EMAIL_ADDRESS", + "probability": 0.9851056337356567, + "text": "tom.baker@example.com" + } + ], + "threshold": 0.2914, + "threshold_rule": "pii:length", + "head": "router", + "head_reason": "id" + }, + { + "id": "halu", + "type": "span", + "target_part": "answer", + "spans": [ + { + "start": 19, + "end": 29, + "label": "unsupported", + "probability": 0.7782317996025085, + "text": "to 6 grams" + } + ], + "threshold": 0.35, + "threshold_rule": "halu", + "head": "router", + "head_reason": "id" + }, + { + "id": "domain", + "type": "choice", + "answer": "health", + "probabilities": { + "health": 0.9621185645193945, + "math": 0.004639298028680393, + "other": 0.03324213745192508 + }, + "abstain_probability": 0.04287965117790319 + } + ], + "meta": { + "tokens": { + "user": 27, + "context": 39, + "answer": 39 + }, + "windows": 0, + "sequences": 2 + } + } +} diff --git a/tests/vela/native_bundle.rs b/tests/vela/native_bundle.rs new file mode 100644 index 00000000..dd33fba8 --- /dev/null +++ b/tests/vela/native_bundle.rs @@ -0,0 +1,135 @@ +//! Bundle-gate tests: a synthetic bundle built in a unique temp +//! directory (no weights, no network) drives the manifest gate — pins, +//! inventory counts and per-file hashes. + +use omni_vela_native::bundle::{self, Bundle}; +use serde_json::json; + +fn temp_dir(name: &str) -> std::path::PathBuf { + let dir = + std::env::temp_dir().join(format!("vela-native-bundle-{name}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).expect("tempdir"); + dir +} + +fn sha256(bytes: &[u8]) -> String { + use sha2::{Digest, Sha256}; + let digest = Sha256::new_with_prefix(bytes).finalize(); + digest.iter().map(|byte| format!("{byte:02x}")).collect() +} + +/// The minimal manifest of a correctly exported member; `files` are the +/// bundle files the gate hashes. +fn write_bundle(dir: &std::path::Path) { + let files: [(&str, &[u8]); 6] = [ + ("backbone.safetensors", b"backbone tensors"), + ("heads.safetensors", b"head tensors"), + ("config.json", b"{}"), + ("calibration.json", b"{}"), + ("tokenizer.json", b"{}"), + ("tokenizer_config.json", b"{}"), + ]; + let mut hashes = serde_json::Map::new(); + for (name, content) in files { + std::fs::write(dir.join(name), content).expect("write"); + hashes.insert(name.to_string(), json!(sha256(content))); + } + let manifest = json!({ + "format": bundle::FORMAT, + "model_id": bundle::MODEL_ID, + "model_name": "vllm-sr/Vela-2.0-0.8B", + "max_length": 16384, + "source": {"repo": "vllm-sr/Vela-2.0-0.8B", "revision": bundle::SOURCE_REVISION}, + "backbone": {"tensors": 320, "bf16": 204, "fp32": 116}, + "heads": {"tensors": 37}, + "files_sha256": hashes, + }); + std::fs::write( + dir.join("vela_native_manifest.json"), + serde_json::to_vec(&manifest).expect("manifest"), + ) + .expect("write"); +} + +#[test] +fn a_matching_bundle_loads_and_exposes_the_member() { + let dir = temp_dir("ok"); + write_bundle(&dir); + let loaded = Bundle::load(&dir).expect("loads"); + assert_eq!(loaded.model_name(), "vllm-sr/Vela-2.0-0.8B"); + assert_eq!(loaded.max_length(), 16384); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn a_tampered_file_fails_its_hash() { + let dir = temp_dir("tampered"); + write_bundle(&dir); + std::fs::write(dir.join("calibration.json"), b"{\"tampered\": true}").expect("write"); + let error = Bundle::load(&dir).unwrap_err(); + assert!(error.to_string().contains("calibration.json"), "{error}"); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn a_foreign_revision_is_refused() { + let dir = temp_dir("foreign"); + write_bundle(&dir); + let manifest_path = dir.join("vela_native_manifest.json"); + let mut manifest: serde_json::Value = + serde_json::from_slice(&std::fs::read(&manifest_path).expect("read")).expect("json"); + manifest["source"]["revision"] = json!("d692394127f6f87fc581d8634c9558fe4776efce"); + std::fs::write(&manifest_path, serde_json::to_vec(&manifest).expect("json")).expect("write"); + let error = Bundle::load(&dir).unwrap_err(); + assert!(error.to_string().contains("pinned"), "{error}"); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn an_unexpected_inventory_is_refused() { + let dir = temp_dir("inventory"); + write_bundle(&dir); + let manifest_path = dir.join("vela_native_manifest.json"); + let mut manifest: serde_json::Value = + serde_json::from_slice(&std::fs::read(&manifest_path).expect("read")).expect("json"); + manifest["backbone"]["bf16"] = json!(320); + manifest["backbone"]["fp32"] = json!(0); + std::fs::write(&manifest_path, serde_json::to_vec(&manifest).expect("json")).expect("write"); + let error = Bundle::load(&dir).unwrap_err(); + assert!(error.to_string().contains("inventory"), "{error}"); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn an_extra_unlisted_file_fails_the_gate() { + let dir = temp_dir("extra"); + write_bundle(&dir); + std::fs::write(dir.join("extra.bin"), b"not in the manifest").expect("write"); + let error = Bundle::load(&dir).unwrap_err(); + assert!(error.to_string().contains("extra.bin"), "{error}"); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn an_empty_files_map_fails_the_gate() { + let dir = temp_dir("empty-files"); + write_bundle(&dir); + let manifest_path = dir.join("vela_native_manifest.json"); + let mut manifest: serde_json::Value = + serde_json::from_slice(&std::fs::read(&manifest_path).expect("read")).expect("json"); + manifest["files_sha256"] = json!({}); + std::fs::write(&manifest_path, serde_json::to_vec(&manifest).expect("json")).expect("write"); + let error = Bundle::load(&dir).unwrap_err(); + assert!(error.to_string().contains("files_sha256"), "{error}"); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn a_missing_manifest_names_the_export_step() { + let dir = temp_dir("empty"); + std::fs::create_dir_all(&dir).expect("tempdir"); + let error = Bundle::load(&dir).unwrap_err(); + assert!(error.to_string().contains("export_native.py"), "{error}"); + let _ = std::fs::remove_dir_all(&dir); +} diff --git a/tests/vela/native_contract.rs b/tests/vela/native_contract.rs new file mode 100644 index 00000000..ebe73454 --- /dev/null +++ b/tests/vela/native_contract.rs @@ -0,0 +1,126 @@ +//! Wire-contract tests for the native Vela 2.0 worker, mirroring the +//! reference worker's protocol tests: the envelope triples must be +//! byte-equal to the pinned runtime's rejections. + +use omni_vela_native::contract::{self, RequestError}; +use serde_json::json; + +fn body(raw: &str) -> serde_json::Map { + contract::parse_body(raw.as_bytes()).expect("valid body") +} + +fn detail(loc: &[&str], msg: &str, type_: &str) -> serde_json::Value { + let mut path = vec![json!("body")]; + path.extend(loc.iter().map(|part| json!(part))); + json!({"detail": [{"loc": path, "msg": msg, "type": type_}]}) +} + +const SERVED: &str = "vllm-sr/Vela-2.0-0.8B"; + +fn raw_body() -> String { + format!( + r#"{{"model": "{SERVED}", "state": "My card was charged twice.", "questions": {{"billing": {{"type": "noul", "instructions": "any?"}}}}}}"# + ) +} + +#[test] +fn the_envelope_passes_through_unchanged() { + let parsed = body(&raw_body()); + contract::admit_request(&parsed, SERVED).expect("admitted"); + assert_eq!(parsed.len(), 3); +} + +#[test] +fn a_missing_field_reports_upstreams_missing_triple() { + for drop in ["state", "model", "questions"] { + let mut parsed = body(&raw_body()); + parsed.shift_remove(drop); + let error = contract::admit_request(&parsed, SERVED).unwrap_err(); + assert_eq!( + error, + RequestError::envelope(&[drop], "Field required", "missing") + ); + assert_eq!(error.body, detail(&[drop], "Field required", "missing")); + } +} + +#[test] +fn an_extra_field_reports_upstreams_extra_forbidden() { + let mut parsed = body(&raw_body()); + parsed.insert("images".into(), json!([])); + let error = contract::admit_request(&parsed, SERVED).unwrap_err(); + assert_eq!(error.status, 422); + assert_eq!( + error.body, + detail( + &["images"], + "Extra inputs are not permitted", + "extra_forbidden" + ) + ); +} + +#[test] +fn a_blank_model_is_rejected_before_the_name_check() { + let mut parsed = body(&raw_body()); + parsed.insert("model".into(), json!(" ")); + let error = contract::admit_request(&parsed, SERVED).unwrap_err(); + assert_eq!( + error.body, + detail( + &["model"], + "model must not be empty or whitespace", + "value_error" + ) + ); +} + +#[test] +fn another_members_name_is_rejected_with_the_model_triple() { + let mut parsed = body(&raw_body()); + parsed.insert("model".into(), json!("vllm-sr/Vela-2.0-0.3B")); + let error = contract::admit_request(&parsed, SERVED).unwrap_err(); + assert_eq!( + error.body, + detail( + &["model"], + "Model is not served by this runtime", + "value_error.model" + ) + ); +} + +#[test] +fn duplicate_keys_are_rejected_with_the_reference_message() { + let error = + contract::parse_body(br#"{"model": "x", "model": "y", "state": "s", "questions": {}}"#) + .unwrap_err(); + assert_eq!(error.status, 400); + assert_eq!(error.body, json!({"detail": "duplicate JSON key 'model'"})); +} + +#[test] +fn invalid_json_is_a_generic_400() { + let error = contract::parse_body(b"{not json").unwrap_err(); + assert_eq!(error.status, 400); + assert_eq!( + error.body, + json!({"detail": "request body must contain valid JSON and UTF-8 text"}) + ); +} + +#[test] +fn a_non_object_body_is_rejected() { + let error = contract::parse_body(b"[1, 2]").unwrap_err(); + assert_eq!(error.status, 400); + assert_eq!( + error.body, + json!({"detail": "request must be a JSON object"}) + ); +} + +#[test] +fn compile_admits_and_returns_the_body() { + let parsed = contract::compile(raw_body().as_bytes(), SERVED).expect("admitted"); + assert!(parsed.contains_key("questions")); +} diff --git a/tests/vela/native_contract_unit.rs b/tests/vela/native_contract_unit.rs new file mode 100644 index 00000000..e2a60834 --- /dev/null +++ b/tests/vela/native_contract_unit.rs @@ -0,0 +1,14 @@ +//! Private-item tests for the native crate's wire contract, loaded by +//! the `#[path]` wiring in `src/models/vela/native/src/contract.rs`. + +use super::*; + +#[test] +fn duplicate_keys_mirror_the_reference_message() { + let error = normalize_parse_error("request body is not valid JSON: duplicate key \"state\""); + assert_eq!(error.status, 400); + assert_eq!( + error.body, + serde_json::json!({"detail": "duplicate JSON key 'state'"}) + ); +} diff --git a/tests/vela/native_processing.rs b/tests/vela/native_processing.rs new file mode 100644 index 00000000..5857b54a --- /dev/null +++ b/tests/vela/native_processing.rs @@ -0,0 +1,143 @@ +//! Processing tests against the frozen token-plan fixtures. +//! +//! The mapping tests (state → parts, SystemOne → predict, plan) run +//! everywhere with no tokenizer and no weights. The full render parity +//! — rows and window rows byte-for-byte against the fixtures — needs +//! the exported bundle's tokenizer and is `#[ignore]`d, opt-in with +//! `-- --ignored`: +//! +//! VELA_NATIVE_BUNDLE=weights/vela-2.0-0.8b-native \ +//! cargo test -p omni-vela-native --test vela_native_processing -- --ignored --nocapture + +use std::path::PathBuf; + +use omni_vela_native::questions::system_one_to_predict; +use omni_vela_native::state::system_one_state; +use serde_json::Value; + +const CASES: [&str; 5] = [ + "example-request", + "example-request-set-threshold", + "example-request-cjk", + "example-request-score", + "long-span-windowed", +]; + +fn fixture(case: &str) -> Value { + let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../../../tests/vela/fixtures"); + serde_json::from_str( + &std::fs::read_to_string(root.join(format!("native-plan-{case}.json"))) + .expect("fixture present; record it with recipe/vela/export_token_plan.py"), + ) + .expect("fixture JSON") +} + +// ------------------------------------------------------- mapping (no tokens) + +fn mapped(case: &str) -> Value { + let fix = fixture(case); + let request = fix["request"].as_object().expect("request"); + let state = system_one_state(&request["state"]).expect("state maps"); + let questions = request["questions"].as_object().expect("questions"); + let mapped = system_one_to_predict(state, questions).expect("maps"); + serde_json::json!({ + "predict_request": serde_json::to_value(&mapped.request).unwrap(), + "plan": serde_json::to_value(&mapped.plan).unwrap(), + }) +} + +#[test] +fn the_predict_mapping_matches_every_fixture() { + for case in CASES { + let got = mapped(case); + let fix = fixture(case); + assert_eq!( + got["predict_request"], fix["predict_request"], + "predict_request differs for {case}" + ); + assert_eq!(got["plan"], fix["plan"], "plan differs for {case}"); + } +} + +#[test] +fn an_invalid_question_reports_the_pinned_triples() { + let state = system_one_state(&serde_json::json!("s")).expect("state"); + let questions = serde_json::json!({ + "q": {"type": "choice", "instructions": "ok?", "criteria": {"a": "x"}} + }); + let error = system_one_to_predict(state, questions.as_object().unwrap()).unwrap_err(); + assert_eq!( + error.body, + serde_json::json!({"detail": [{"loc": ["body", "questions", "q", "criteria"], + "msg": "choice criteria must be an object with 2 to 255 entries", "type": "value_error"}]}) + ); + + let state = system_one_state(&serde_json::json!("s")).expect("state"); + let questions = serde_json::json!({ + "q": {"type": "span", "instructions": "which?", "criteria": {"a": "x"}, + "head": "other", "over": "request"} + }); + let error = system_one_to_predict(state, questions.as_object().unwrap()).unwrap_err(); + assert_eq!( + error.body["detail"][0]["loc"], + serde_json::json!(["body", "questions", "q", "head"]) + ); + assert_eq!(error.body["detail"][0]["type"], "literal_error"); +} + +#[test] +fn a_set_label_collision_is_rejected_after_the_walk() { + let state = system_one_state(&serde_json::json!("s")).expect("state"); + let questions = serde_json::json!({ + "issues": {"type": "set", "instructions": "which?", "criteria": {"billing": "b"}}, + "issues.billing": {"type": "noul", "instructions": "which?"} + }); + let error = system_one_to_predict(state, questions.as_object().unwrap()).unwrap_err(); + assert_eq!( + error.body["detail"][0]["msg"], + "collides with the Noul view of a set label" + ); +} + +// ------------------------------------------------- full render (opt-in) + +fn bundle_dir() -> PathBuf { + let path = PathBuf::from( + std::env::var("VELA_NATIVE_BUNDLE") + .expect("set VELA_NATIVE_BUNDLE to the export directory"), + ); + assert!( + path.is_dir(), + "VELA_NATIVE_BUNDLE is not a directory: {}", + path.display() + ); + path +} + +#[test] +#[ignore = "requires VELA_NATIVE_BUNDLE at an exported bundle; CPU only"] +fn the_rendered_rows_match_every_fixture() { + let bundle = bundle_dir(); + let processor = omni_vela_native::processing::Processor::load(&bundle) + .expect("processor loads the exported bundle"); + for case in CASES { + let fix = fixture(case); + let request = fix["request"].as_object().expect("request"); + let planned = match processor.plan(request) { + Ok(planned) => planned, + Err(error) => panic!( + "plan failed for {case}: {}", + serde_json::to_string(&error.body).unwrap() + ), + }; + let got = serde_json::json!({ + "rows": serde_json::to_value(&planned.rows).unwrap(), + "window_rows": serde_json::to_value(&planned.window_rows).unwrap(), + }); + assert_eq!(got["rows"], fix["rows"], "rows differ for {case}"); + assert_eq!( + got["window_rows"], fix["window_rows"], + "window rows differ for {case}" + ); + } +} diff --git a/tests/vela/native_units.rs b/tests/vela/native_units.rs new file mode 100644 index 00000000..ce695237 --- /dev/null +++ b/tests/vela/native_units.rs @@ -0,0 +1,101 @@ +//! Unit tests for the native crate's state mapping and word splitter, +//! extracted from the modules per the repository's test-placement +//! rule (the private contract test lives in native_contract_unit.rs, +//! loaded by the `#[path]` wiring in contract.rs). + +use omni_vela_native::state::system_one_state; +use omni_vela_native::words::{split_words, unit_ids, word_first_tokens}; +use serde_json::json; + +// ------------------------------------------------------------- state + +#[test] +fn a_plain_string_is_one_user_part() { + let parts = system_one_state(&json!("hello")).expect("parts"); + assert_eq!(parts.parts.len(), 1); + assert_eq!(parts.parts[0].r#type, "user"); + assert_eq!(parts.fields["request"].end, 5); +} + +#[test] +fn typed_keys_group_into_ordered_parts_with_field_ranges() { + let parts = system_one_state(&json!({ + "answer": "A", "request": "R", "source": "S", "extra": "E", + })) + .expect("parts"); + let roles: Vec<&str> = parts.parts.iter().map(|p| p.r#type.as_str()).collect(); + assert_eq!(roles, ["user", "context", "answer"]); + let context = &parts.parts[1]; + assert_eq!(context.text, "source:\nS\n\nextra:\nE"); + assert_eq!(parts.fields["source"].start, "source:\n".chars().count()); + assert_eq!( + parts.fields["extra"].start, + "source:\nS\n\nextra:\n".chars().count() + ); +} + +#[test] +fn an_empty_role_part_is_rejected() { + let error = system_one_state(&json!({"request": " "})).unwrap_err(); + assert_eq!( + error.body["detail"][0]["msg"], + serde_json::json!("the user part of the state is empty") + ); +} + +#[test] +fn a_string_holding_a_json_object_is_read_as_that_object() { + let parts = system_one_state(&json!("{\"request\": \"text\"}")).expect("parts"); + assert_eq!(parts.parts.len(), 1); + assert_eq!(parts.fields["request"].end, 4); +} + +// ------------------------------------------------------------- words + +#[test] +fn words_split_like_the_reference() { + let text = "Hi, I'm Tom Baker (tom.baker@example.com)."; + let words: Vec = split_words(text) + .iter() + .map(|(a, b)| text.chars().skip(*a).take(b - a).collect()) + .collect(); + assert_eq!( + words, + vec![ + "Hi", + ",", + "I", + "'", + "m", + "Tom", + "Baker", + "(", + "tom.baker@example.com", + ")", + "." + ] + ); +} + +#[test] +fn cjk_characters_are_one_unit_each() { + let text = "追踪 parcel 仓库"; + let words: Vec = split_words(text) + .iter() + .map(|(a, b)| text.chars().skip(*a).take(b - a).collect()) + .collect(); + assert_eq!(words, vec!["追", "踪", "parcel", "仓", "库"]); + // Mixed content: each CJK char its own unit, the ASCII word one + // unit, the space outside every unit. + let chars: Vec = "追踪 parcel".chars().collect(); + assert_eq!(unit_ids(&chars), vec![0, 1, -1, 2, 2, 2, 2, 2, 2]); +} + +#[test] +fn first_tokens_skip_empty_tokens() { + let text = "alpha beta"; + let offsets = vec![(0, 0), (0, 5), (6, 10)]; + let (word_offsets, first) = word_first_tokens(text, &offsets); + assert_eq!(word_offsets, vec![(0, 5), (6, 10)]); + assert_eq!(first, vec![1, 2]); +} diff --git a/tests/vela/test_hybrid.py b/tests/vela/test_hybrid.py new file mode 100644 index 00000000..ee3fb300 --- /dev/null +++ b/tests/vela/test_hybrid.py @@ -0,0 +1,310 @@ +"""Tests for the Vela 2.0 0.8B hybrid member: gate, worker dispatch, response finishing. + +Gate tests mirror test_engine_manifest: temp directories with an +injectable pin table and recomputed hashes — no weights, no torch, no +network. Worker tests drive FakeVela2Hybrid (from test_worker's fake, +plus the hybrid response surface) through decide/detect_member/ +models_entry: no weights, no torch, no GPU. + + PYTHONPATH=src python -m pytest tests/vela -q +""" + +from __future__ import annotations + +import hashlib +import json +import re +from pathlib import Path + +import pytest + +from frontend.vela import decide, detect_member, models_entry +from models.vela.engine import Vela2Executor, verify_files +from models.vela.hybrid import ( + EXPECTED_ARCHITECTURE, + EXPECTED_MAX_LENGTH, + EXPECTED_MODEL_NAME, + EXPECTED_MODEL_TYPE, + EXPECTED_WEIGHTS_BYTES, + HYBRID_REPO, + HYBRID_REVISION, + PINNED_FILES_SHA256, + WEIGHTS_NAME, + Vela2HybridExecutor, + verify_config, + verify_package, + verify_weights, +) +from models.vela.postprocess import finish_response + +from .test_protocol import body +from .test_worker import FakeVela2, five_kind_body + +CONFIG = { + "model_type": EXPECTED_MODEL_TYPE, + "architectures": [EXPECTED_ARCHITECTURE], + "max_length": EXPECTED_MAX_LENGTH, + "model_name": EXPECTED_MODEL_NAME, +} + + +def table_of(files: dict[str, bytes]) -> dict[str, str]: + return {name: hashlib.sha256(content).hexdigest() for name, content in files.items()} + + +def small_files(): + return { + "vela2_serve.py": b"upstream server stub\n", + "vela2_inference.py": b"upstream engine stub\n", + "modeling_vela2.py": b"upstream model stub\n", + "configuration_vela2.py": b"upstream config stub\n", + "calibration.json": b'{"temperature": {}}', + "tokenizer.json": b'{"version": "stub"}', + "tokenizer_config.json": b'{"model_type": "stub"}', + "model.safetensors.index.json": b'{"metadata": {"total_size": 1}}', + "broad_head.safetensors": b"second span head stub\n", + } + + +def build_package(root: Path, files: dict[str, bytes], config: dict | None = None) -> dict[str, str]: + for name, content in files.items(): + (root / name).write_bytes(content) + (root / "config.json").write_text( + json.dumps(config if config is not None else CONFIG), encoding="utf-8" + ) + with (root / WEIGHTS_NAME).open("wb") as stream: + stream.truncate(EXPECTED_WEIGHTS_BYTES) + files = dict(files) + files["config.json"] = (root / "config.json").read_bytes() + return table_of(files) + + +def hybrid_body(): + value = five_kind_body() + value["model"] = EXPECTED_MODEL_NAME + return value + + +class FakeVela2Hybrid(FakeVela2): + """The pinned hybrid runtime's contract, driven without its code.""" + + is_hybrid = True + + def __init__(self, model_name=EXPECTED_MODEL_NAME, too_long=False): + super().__init__(model_name, too_long) + self.max_length = 16384 + + class _Engine: + config = { + "model_name": EXPECTED_MODEL_NAME, + "description": ( + "Vela 2.0 0.8B: SystemOne Choice / Noul / Score " + "with span and set extensions, 16,384-token input" + ), + "release_date": "", + } + + self.engine = _Engine() + + def system_one(self, body): + response = super().system_one(body) + if "spans" in response: + # Upstream emits span_heads together with spans, keyed by the + # span question ids; router stands in for both heads here. + response["span_heads"] = {qid: "router" for qid in response["spans"]} + return response + + +# --------------------------------------------------------------------- gate + + +def test_the_checked_in_hybrid_pin_table_shape_is_locked(): + assert HYBRID_REPO == "vllm-sr/Vela-2.0-0.8B" + assert HYBRID_REVISION == "326b01d81f61fff7fd4ae570e54b87a49119e490" + assert set(PINNED_FILES_SHA256) == { + "vela2_serve.py", "vela2_inference.py", "modeling_vela2.py", + "configuration_vela2.py", "config.json", "calibration.json", "tokenizer.json", + "tokenizer_config.json", "model.safetensors.index.json", "broad_head.safetensors", + } + for digest in PINNED_FILES_SHA256.values(): + assert re.fullmatch(r"[0-9a-f]{64}", digest) + + +def test_verification_accepts_a_package_matching_its_table(tmp_path): + table = build_package(tmp_path, small_files()) + assert verify_files(tmp_path, table) == sorted(table) + verify_config(tmp_path) + verify_weights(tmp_path) + verify_package(tmp_path, table) + + +def test_a_unified_package_is_refused_with_its_own_name(tmp_path): + build_package( + tmp_path, small_files(), + config={**CONFIG, "model_type": "vela2-unified"}, + ) + with pytest.raises(ValueError, match="vela2-unified.*only serves the vela2-decoder member"): + verify_config(tmp_path) + + +@pytest.mark.parametrize( + ("override", "fragment"), + [ + ({"architectures": ["OtherModel"]}, "architectures"), + ({"max_length": 8192}, "max_length"), + ({"model_name": "vllm-sr/Vela-2.0-4B"}, "needs its own pinned executor"), + ], +) +def test_foreign_config_structures_are_refused(tmp_path, override, fragment): + build_package(tmp_path, small_files(), config={**CONFIG, **override}) + with pytest.raises(ValueError, match=fragment): + verify_config(tmp_path) + + +def test_tampered_files_are_rejected(tmp_path): + table = build_package(tmp_path, small_files()) + victim = tmp_path / "broad_head.safetensors" + victim.write_bytes(victim.read_bytes() + b"tampered\n") + # The shared gate must name the hybrid's pin, not the 0.3B's. + with pytest.raises( + ValueError, match=f"broad_head.safetensors: SHA-256 differs from the pin for {HYBRID_REVISION}" + ): + verify_files(tmp_path, table, HYBRID_REVISION) + + +def test_a_missing_broad_head_is_rejected(tmp_path): + table = build_package(tmp_path, small_files()) + (tmp_path / "broad_head.safetensors").unlink() + with pytest.raises(ValueError, match="missing pinned file broad_head.safetensors"): + verify_files(tmp_path, table) + + +def test_a_later_self_consistent_snapshot_is_refused_by_the_checked_in_pin(tmp_path): + build_package(tmp_path, small_files()) + with pytest.raises(ValueError, match="SHA-256 differs from the pin"): + verify_files(tmp_path, PINNED_FILES_SHA256) + + +def test_weights_are_checked_by_exact_pinned_size(tmp_path): + build_package(tmp_path, small_files()) + verify_weights(tmp_path) + + with (tmp_path / WEIGHTS_NAME).open("wb") as stream: + stream.truncate(EXPECTED_WEIGHTS_BYTES - 1) + with pytest.raises(ValueError, match="is 2022189487 bytes"): + verify_weights(tmp_path) + + (tmp_path / WEIGHTS_NAME).unlink() + with pytest.raises(ValueError, match=f"missing {WEIGHTS_NAME}"): + verify_weights(tmp_path) + + +# ------------------------------------------------------------- worker wiring + + +def test_detect_member_routes_each_declared_member(tmp_path): + unified = tmp_path / "unified" + unified.mkdir() + (unified / "config.json").write_text(json.dumps({"model_type": "vela2-unified"})) + decoder = tmp_path / "decoder" + decoder.mkdir() + (decoder / "config.json").write_text(json.dumps({"model_type": "vela2-decoder"})) + + cls, name = detect_member(unified) + assert cls is Vela2Executor and name == "vllm-sr/Vela-2.0-0.3B" + cls, name = detect_member(decoder) + assert cls is Vela2HybridExecutor and name == EXPECTED_MODEL_NAME + + +def test_detect_member_refuses_packages_without_a_usable_config(tmp_path): + empty = tmp_path / "empty" + empty.mkdir() + with pytest.raises(ValueError, match="missing config.json"): + detect_member(empty) + foreign = tmp_path / "foreign" + foreign.mkdir() + (foreign / "config.json").write_text(json.dumps({"model_type": "vela2-unified-xl"})) + with pytest.raises(ValueError, match="vela2-unified-xl"): + detect_member(foreign) + broken = tmp_path / "broken" + broken.mkdir() + (broken / "config.json").write_text("{not json") + with pytest.raises(ValueError, match="not readable JSON"): + detect_member(broken) + array = tmp_path / "array" + array.mkdir() + (array / "config.json").write_text("[]") + with pytest.raises(ValueError, match="not a JSON object"): + detect_member(array) + + +def test_models_entry_is_verbatim_per_member(): + hybrid = models_entry(FakeVela2Hybrid()) + assert hybrid == { + "name": EXPECTED_MODEL_NAME, + "release_date": "", + "description": ( + "Vela 2.0 0.8B: SystemOne Choice / Noul / Score " + "with span and set extensions, 16,384-token input" + ), + } + assert models_entry(FakeVela2())["release_date"] == "2026-09-29" + + +# -------------------------------------------------------- response finishing + + +def test_decide_answers_all_five_kinds_with_span_heads(): + response = decide(json.dumps(hybrid_body()).encode(), FakeVela2Hybrid()) + assert set(response) == {"model", "answers", "usage", "spans", "sets", "thresholds", "span_heads"} + assert list(response["span_heads"]) == ["unsupported"] + assert response["model"] == EXPECTED_MODEL_NAME + + +def test_decide_accepts_a_hybrid_response_without_span_questions(): + value = body(model=EXPECTED_MODEL_NAME) + response = decide(json.dumps(value).encode(), FakeVela2Hybrid()) + assert "span_heads" not in response and "spans" not in response + + +def test_a_hybrid_response_missing_span_heads_is_refused(): + class Headless(FakeVela2Hybrid): + def system_one(self, body): + response = super().system_one(body) + response.pop("span_heads", None) + return response + + with pytest.raises(RuntimeError, match="span_heads do not match"): + decide(json.dumps(hybrid_body()).encode(), Headless()) + + +def test_a_hybrid_response_with_stray_span_heads_is_refused(): + class Stray(FakeVela2Hybrid): + def system_one(self, body): + response = super().system_one(body) + response["span_heads"] = {"ghost": "router"} + return response + + with pytest.raises(RuntimeError, match="span_heads without span questions"): + decide(json.dumps(body(model=EXPECTED_MODEL_NAME)).encode(), Stray()) + + +def test_the_unified_member_still_refuses_span_heads(): + class UnifiedLeak(FakeVela2): + def system_one(self, body): + response = super().system_one(body) + if "spans" in response: + response["span_heads"] = {qid: "router" for qid in response["spans"]} + return response + + with pytest.raises(RuntimeError, match="this worker serves the 0.3B member"): + decide(json.dumps(five_kind_body()).encode(), UnifiedLeak()) + + +def test_finish_response_keeps_the_hybrid_envelope_tight(): + questions = hybrid_body()["questions"] + response = FakeVela2Hybrid().system_one(hybrid_body()) + assert finish_response(questions, response, hybrid=True) is response + del response["answers"] + with pytest.raises(RuntimeError, match="unexpected response envelope"): + finish_response(questions, response, hybrid=True) diff --git a/tests/vela/test_oracle_hybrid_fixtures.py b/tests/vela/test_oracle_hybrid_fixtures.py new file mode 100644 index 00000000..3114a904 --- /dev/null +++ b/tests/vela/test_oracle_hybrid_fixtures.py @@ -0,0 +1,63 @@ +"""Ties the frozen hybrid oracle fixtures to the recipe examples they came from. + +The 0.8B hybrid fixtures recorded by ``recipe/vela/compare_reference.py`` +under the ``oracle-hybrid-`` prefix keep the recipe's example requests +(with the ``model`` field rewritten to the hybrid's served name) and their +recorded ``system_one`` responses, which must pass the production +``finish_response`` gate on the hybrid setting; the 0.3B oracles have no +equivalent consumer yet. No weights, no torch. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from models.vela.hybrid import EXPECTED_MODEL_NAME +from models.vela.postprocess import finish_response + +TESTS = Path(__file__).parent +RECIPE = TESTS.parent.parent / "recipe" / "vela" + +CASES = [ + ("oracle-hybrid-example-request", "example-request.json"), + ("oracle-hybrid-example-request-cjk", "example-request-cjk.json"), + ("oracle-hybrid-example-request-set-threshold", "example-request-set-threshold.json"), + ("oracle-hybrid-example-request-score", "example-request-score.json"), +] + + +def oracle(case: str) -> dict: + return json.loads((TESTS / "fixtures" / f"{case}.json").read_text()) + + +def recipe_body(example: str) -> dict: + body = json.loads((RECIPE / example).read_text()) + # The tool's load_cases rewrite for file cases: the examples are + # member-agnostic apart from the model field. + return {**body, "model": EXPECTED_MODEL_NAME} + + +@pytest.mark.parametrize("case,example", CASES) +def test_the_oracle_request_is_the_recipe_example(case: str, example: str): + recorded = oracle(case) + served = recipe_body(example) + assert recorded["request"] == served + # Same insertion order, so byte-for-byte serialization holds too. + assert list(recorded["request"]) == list(served) + + +@pytest.mark.parametrize("case,example", CASES) +def test_the_recorded_response_passes_the_worker_gate(case: str, example: str): + recorded = oracle(case) + response = recorded["system_one"] + assert response["model"] == EXPECTED_MODEL_NAME + assert finish_response(recorded["request"]["questions"], response, hybrid=True) is response + if any(q.get("type") == "span" for q in recorded["request"]["questions"].values()): + assert set(response["span_heads"]) == { + qid for qid, q in recorded["request"]["questions"].items() if q.get("type") == "span" + } + # The frozen case label still names the recipe file it belongs to. + assert recorded["case"] == example.removesuffix(".json") diff --git a/tests/vela/test_worker_server.py b/tests/vela/test_worker_server.py index ce2d106b..9221a933 100644 --- a/tests/vela/test_worker_server.py +++ b/tests/vela/test_worker_server.py @@ -292,3 +292,51 @@ def test_the_correct_bearer_answers_normally(keyed_server): headers=(("Authorization", "Bearer secret-key"),), ) assert status == 200 and json.loads(raw)["answers"]["billing"] == {"type": "noul", "noul": 0.75} + + +@pytest.fixture() +def hybrid_server(): + from .test_hybrid import FakeVela2Hybrid, hybrid_body + + instance = start(FakeVela2Hybrid()) + yield instance, hybrid_body() + stop(instance) + + +def test_the_hybrid_member_serves_its_own_models_entry(hybrid_server): + server, _ = hybrid_server + status, _, raw = request(server, "GET", "/v1/models") + assert status == 200 + assert json.loads(raw) == { + "models": [ + { + "name": "vllm-sr/Vela-2.0-0.8B", + "release_date": "", + "description": ( + "Vela 2.0 0.8B: SystemOne Choice / Noul / Score " + "with span and set extensions, 16,384-token input" + ), + } + ] + } + + +def test_the_hybrid_member_answers_all_five_kinds_with_span_heads(hybrid_server): + server, hybrid_body = hybrid_server + status, ctype, raw = post(server, json.dumps(hybrid_body).encode("utf-8")) + assert status == 200 and ctype == "application/json" + response = json.loads(raw) + assert response["model"] == "vllm-sr/Vela-2.0-0.8B" + assert set(response) == { + "model", "answers", "usage", "spans", "sets", "thresholds", "span_heads" + } + assert list(response["span_heads"]) == ["unsupported"] + assert response["answers"]["unsupported"] == {"type": "noul", "noul": 0.9} + + +def test_the_hybrid_member_rejects_another_model_name(hybrid_server): + server, _ = hybrid_server + wrong = body(model="vllm-sr/Vela-2.0-0.3B") + status, _, raw = post(server, json.dumps(wrong).encode("utf-8")) + assert status == 422 + assert json.loads(raw)["detail"][0]["type"] == "value_error.model"