From 346a4713fddfc4a2e9044c2529d8861e3bc3f82d Mon Sep 17 00:00:00 2001 From: Szymon Kurcab Date: Fri, 18 Sep 2026 20:28:48 +0200 Subject: [PATCH] fix: validate answers against each provider's actual contract One validator served both providers, and it was wrong in both directions. Too strict for OpenRouter. It required a numeric `confidence` on every choice and score answer. OpenRouter's published schema (DecisionsChoiceAnswer, DecisionsScoreAnswer in openrouter.ai/openapi.json) requires only `type` plus `choice`/`score`; confidence is optional. A response the gateway considers valid raised InvalidResponse. Too lax for Typesafe. docs.typesafe.ai/api documents `probabilities` and `confidence` as required on choice, and those plus `legend` on score. When any of them was missing or the wrong shape the client substituted `{}` and returned a typed answer with an empty distribution, so code thresholding on probabilities read an empty map as a legitimate answer. And too lax for everyone about values. `is_a?(Numeric)` accepts a `noul` of 2, and JSON parses `1e999` into Float::INFINITY, which is Numeric and survives every check downstream. A `choice` was taken as given even when it was not one of the labels the question offered -- the one place a wrong string does real damage, since that is what the application routes on. So: providers declare `required_answer_fields`, the client enforces that per provider, missing-but-optional comes back nil or {} while present-but-malformed always raises, numbers must be finite and in range, and a choice must be one the question offered. InvalidResponse now names the field rather than only the question id. Two existing tests asserted the confidence-is-required behaviour and one asserted the silent fallback to {}; they now assert the contract each provider actually publishes. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 18 ++ README.md | 27 +++ lib/ruby_decision_model/client.rb | 151 +++++++++--- lib/ruby_decision_model/providers/base.rb | 17 ++ lib/ruby_decision_model/providers/typesafe.rb | 13 + test/answers_test.rb | 228 ++++++++++++++++++ test/client_test.rb | 40 +-- 7 files changed, 444 insertions(+), 50 deletions(-) create mode 100644 test/answers_test.rb diff --git a/CHANGELOG.md b/CHANGELOG.md index 846df76..3180340 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,23 @@ # Changelog +## Unreleased + +- Answer validation follows each provider's published contract instead of one + shared guess. Typesafe documents `probabilities`, `confidence`, and `legend` + as required, and a response missing one now raises `InvalidResponse` rather + than being handed back with an empty Hash in its place. OpenRouter's schema + requires only `type` plus `choice`/`score`, and answers without a + `confidence` are no longer rejected there; `confidence` is `nil` instead. + Providers declare this with `required_answer_fields`. +- Numeric answer fields are checked for finiteness and range. `noul`, + `confidence`, and probability values must be finite and within 0..1; + `{"noul": 1e999}` parses to `Infinity` and used to be accepted. +- A `choice` that is not one of the question's own criteria raises + `InvalidResponse` instead of reaching application routing as an unknown + label. +- `InvalidResponse` messages now name the field that was wrong, not just the + question id. + ## 0.1.0 - 2026-09-18 Provider-neutral release. One `Client`, two providers behind it. diff --git a/README.md b/README.md index 79638d6..7e80509 100644 --- a/README.md +++ b/README.md @@ -124,6 +124,33 @@ Answers come back typed: `Answers::Noul` (`noul`, `probabilities`), `response.nouls`, `response.choices`, and `response.scores` return the answers of one type keyed the same way as `response.answers`. +### What counts as a valid answer + +Each provider declares which answer fields its published contract guarantees, +and the client holds it to that rather than to one shared guess. + +| | OpenRouter requires | Typesafe requires | +| --- | --- | --- | +| noul | `noul` | `noul` | +| choice | `choice` | `choice`, `confidence`, `probabilities` | +| score | `score` | `score`, `confidence`, `probabilities`, `legend` | + +A required field that is missing raises `InvalidResponse` naming the question +and the field. A field the provider does not guarantee comes back `nil`, or +`{}` for a map, rather than raising: OpenRouter's schema makes `confidence` +optional, so `answer.confidence` can be `nil` there. A field that is present +but the wrong shape always raises; it is never replaced with an empty value. + +Numbers are checked as well as typed. `noul`, `confidence`, and every value in +a `probabilities` map must be finite and within 0..1 — `1e999` parses to +`Infinity` and would otherwise sail through an `is_a?(Numeric)` check — and a +`choice` must be one of the keys the question's `criteria` offered, so a label +the application has no branch for cannot reach its routing. + +A provider written by hand declares its own contract with +`required_answer_fields`; naming a field the client does not model raises +`ConfigurationError` when the client is built. + Score `probabilities` and `legend` are keyed by the wire's string level keys (`"0"`, `"1"`, ...), not by the criteria labels. Choice `probabilities` sum to approximately 1; treat them as calibrated, not normalized. diff --git a/lib/ruby_decision_model/client.rb b/lib/ruby_decision_model/client.rb index 3457726..2623609 100644 --- a/lib/ruby_decision_model/client.rb +++ b/lib/ruby_decision_model/client.rb @@ -35,6 +35,7 @@ def initialize(provider: nil, api_key: nil, model: nil, base_url: nil, timeout: transport: nil, sleeper: ->(seconds) { sleep(seconds) }, retry: {}, random: -> { rand }, clock: -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }) @provider = resolve_provider(provider, api_key: api_key, base_url: base_url) + validate_answer_contract(@provider) unless @provider.api_key? raise ConfigurationError, "api_key is required for #{@provider.name}: pass api_key: or set #{@provider.env_var}" @@ -88,6 +89,23 @@ def resolve_provider(provider, api_key:, base_url:) end end + # A provider that requires a field the client does not model would be + # declaring a contract nothing enforces. Say so when the client is built + # rather than letting the guarantee quietly do nothing. + def validate_answer_contract(provider) + provider.required_answer_fields.each do |type, names| + known = ANSWER_FIELDS[type] + raise ConfigurationError, "#{provider.name} requires fields for unknown answer type #{type.inspect}" if known.nil? + + unknown = names.map(&:to_s) - known.keys + next if unknown.empty? + + raise ConfigurationError, + "#{provider.name} requires answer fields the client does not model: " \ + "#{unknown.join(', ')} on #{type} answers" + end + end + def default_transport lambda do |url:, headers:, body:| uri = URI.parse(url) @@ -208,9 +226,9 @@ def parse_success(response_body, response_headers, questions) if answer_hash.is_a?(Hash) && answer_hash["type"] == expected_type begin - normalized[id] = normalize_answer(expected_type, answer_hash) - rescue MalformedAnswer - malformed << id + normalized[id] = normalize_answer(expected_type, answer_hash, question) + rescue MalformedAnswer => e + malformed << "#{id} (#{e.message})" end else missing << id @@ -219,7 +237,7 @@ def parse_success(response_body, response_headers, questions) if malformed.any? raise InvalidResponse.new( - "malformed answer fields for: #{malformed.join(', ')}", + "malformed answer fields for: #{malformed.join('; ')}", answers: normalized ) end @@ -255,36 +273,106 @@ def request_id_from(headers) class MalformedAnswer < StandardError; end - def normalize_answer(type, hash) + # Every field an answer can carry, and the shape each one has to have. + # A field absent from this map is ignored; Response#raw still has it. + ANSWER_FIELDS = { + # Neither provider's schema has probabilities on a noul answer -- the + # value is the probability -- but it is read when one turns up rather + # than dropped, and Answers::Noul has carried the field since 0.0.1. + "noul" => { "noul" => :probability, "probabilities" => :distribution }, + "choice" => { "choice" => :label, "confidence" => :probability, "probabilities" => :distribution }, + "score" => { "score" => :number, "confidence" => :probability, + "probabilities" => :distribution, "legend" => :map } + }.freeze + + # Probabilities are floats off a model, so a distribution can land a hair + # over 1.0 without being wrong. + PROBABILITY_TOLERANCE = 1e-6 + + def normalize_answer(type, hash, question) + fields = ANSWER_FIELDS[type] + raise MalformedAnswer, "unsupported answer type #{type.inspect}" if fields.nil? + + required = @provider.required_answer_fields.fetch(type, fields.keys) + values = fields.to_h { |name, shape| [name, read_answer_field(hash, name, shape, required)] } + check_choice_is_offered(values["choice"], question) if type == "choice" + + build_answer(type, values) + end + + def read_answer_field(hash, name, shape, required) + raw = hash[name] + + if raw.nil? + raise MalformedAnswer, "#{name} is missing" if required.include?(name) + + return %i[distribution map].include?(shape) ? {} : nil + end + + case shape + when :probability then unit_interval(name, raw) + when :number then finite_number(name, raw) + when :label then label(name, raw) + when :distribution then distribution(name, raw) + when :map then raw.is_a?(Hash) ? raw : raise(MalformedAnswer, "#{name} is not an object") + end + end + + def finite_number(name, raw) + raise MalformedAnswer, "#{name} is not a number" unless raw.is_a?(Numeric) + # JSON turns 1e999 into Infinity and some encoders emit NaN. Neither is + # an answer, and both survive every is_a?(Numeric) check downstream. + raise MalformedAnswer, "#{name} is not finite (#{raw})" unless raw.finite? + + raw.to_f + end + + def unit_interval(name, raw) + value = finite_number(name, raw) + unless value >= -PROBABILITY_TOLERANCE && value <= 1.0 + PROBABILITY_TOLERANCE + raise MalformedAnswer, "#{name} is outside 0..1 (#{value})" + end + + value.clamp(0.0, 1.0) + end + + def label(name, raw) + raise MalformedAnswer, "#{name} is not a string" unless raw.is_a?(String) + raise MalformedAnswer, "#{name} is empty" if raw.empty? + + raw + end + + def distribution(name, raw) + raise MalformedAnswer, "#{name} is not an object" unless raw.is_a?(Hash) + + raw.to_h { |key, value| [key, unit_interval("#{name}[#{key.inspect}]", value)] } + end + + # A choice the question never offered cannot be routed on, and reading it + # as a label the application knows is exactly the mistake this guards. + def check_choice_is_offered(choice, question) + return if choice.nil? + + criteria = question.is_a?(Hash) ? (question["criteria"] || question[:criteria]) : nil + return unless criteria.is_a?(Hash) + + offered = criteria.keys.map(&:to_s) + return if offered.include?(choice) + + raise MalformedAnswer, "choice #{choice.inspect} is not one of the question's criteria (#{offered.join(', ')})" + end + + def build_answer(type, values) case type when "noul" - noul = hash["noul"] - raise MalformedAnswer unless noul.is_a?(Numeric) - - Answers::Noul.new(noul: noul.to_f, probabilities: hash_or_empty(hash["probabilities"])) + Answers::Noul.new(noul: values["noul"], probabilities: values["probabilities"]) when "choice" - choice = hash["choice"] - confidence = hash["confidence"] - raise MalformedAnswer unless choice.is_a?(String) && confidence.is_a?(Numeric) - - Answers::Choice.new( - choice: choice, - confidence: confidence.to_f, - probabilities: hash_or_empty(hash["probabilities"]) - ) + Answers::Choice.new(choice: values["choice"], confidence: values["confidence"], + probabilities: values["probabilities"]) when "score" - score = hash["score"] - confidence = hash["confidence"] - raise MalformedAnswer unless score.is_a?(Numeric) && confidence.is_a?(Numeric) - - Answers::Score.new( - score: score.to_f, - confidence: confidence.to_f, - probabilities: hash_or_empty(hash["probabilities"]), - legend: hash_or_empty(hash["legend"]) - ) - else - raise MalformedAnswer + Answers::Score.new(score: values["score"], confidence: values["confidence"], + probabilities: values["probabilities"], legend: values["legend"]) end end @@ -294,8 +382,5 @@ def question_type(question) question["type"] || question[:type] end - def hash_or_empty(value) - value.is_a?(Hash) ? value : {} - end end end diff --git a/lib/ruby_decision_model/providers/base.rb b/lib/ruby_decision_model/providers/base.rb index 85f7089..f6bb202 100644 --- a/lib/ruby_decision_model/providers/base.rb +++ b/lib/ruby_decision_model/providers/base.rb @@ -47,6 +47,23 @@ def reports_cost? false end + # The answer fields this provider's published contract declares + # required. The client raises InvalidResponse when one is missing + # instead of quietly substituting nil or an empty Hash, and leaves the + # rest nil when a provider does not send them. + # + # The default is the minimum that makes an answer an answer: the value + # field itself. A provider that guarantees more says so. + MINIMUM_ANSWER_FIELDS = { + "noul" => %w[noul], + "choice" => %w[choice], + "score" => %w[score] + }.freeze + + def required_answer_fields + MINIMUM_ANSWER_FIELDS + end + def base_url (@base_url || default_base_url).to_s.chomp("/") end diff --git a/lib/ruby_decision_model/providers/typesafe.rb b/lib/ruby_decision_model/providers/typesafe.rb index c8a1ee7..c74a3b9 100644 --- a/lib/ruby_decision_model/providers/typesafe.rb +++ b/lib/ruby_decision_model/providers/typesafe.rb @@ -33,6 +33,19 @@ def default_model def aliases ALIASES end + + # https://docs.typesafe.ai/api documents probabilities, confidence, and + # legend as required on the answers that carry them, so a response + # missing one is a broken response rather than a sparse one. + REQUIRED_ANSWER_FIELDS = { + "noul" => %w[noul], + "choice" => %w[choice probabilities confidence], + "score" => %w[score probabilities confidence legend] + }.freeze + + def required_answer_fields + REQUIRED_ANSWER_FIELDS + end end end end diff --git a/test/answers_test.rb b/test/answers_test.rb new file mode 100644 index 0000000..f4f332b --- /dev/null +++ b/test/answers_test.rb @@ -0,0 +1,228 @@ +# frozen_string_literal: true + +require "test_helper" + +# What the client will and will not accept as an answer, per provider. +class AnswersTest < Minitest::Test + def noul_question + { "urgent" => RubyDecisionModel::Questions.noul("Is this urgent?") } + end + + def choice_question + { "dept" => RubyDecisionModel::Questions.choice("Which team?", criteria: { "billing" => nil, "auth" => nil }) } + end + + def score_question + { "sev" => RubyDecisionModel::Questions.score("How bad?", criteria: %w[low medium high]) } + end + + def body(answers) + JSON.generate("model" => "jev-latest", "answers" => answers, "usage" => {}) + end + + def client_for(provider, answers) + RubyDecisionModel::Client.new( + provider: provider, api_key: "k", sleeper: no_sleep, + transport: FakeTransport.new([[200, body(answers)]]) + ) + end + + def ask(provider, questions, answers) + client_for(provider, answers).ask(state: {}, questions: questions) + end + + def refute_accepted(provider, questions, answers, matching) + error = assert_raises(RubyDecisionModel::InvalidResponse) { ask(provider, questions, answers) } + assert_match matching, error.message + error + end + + # --- fields Typesafe documents as required are required --- + + def test_typesafe_choice_without_probabilities_is_rejected + refute_accepted(:typesafe, choice_question, + { "dept" => { "type" => "choice", "choice" => "billing", "confidence" => 0.9 } }, + /probabilities is missing/) + end + + def test_typesafe_choice_without_confidence_is_rejected + refute_accepted(:typesafe, choice_question, + { "dept" => { "type" => "choice", "choice" => "billing", "probabilities" => { "billing" => 1.0 } } }, + /confidence is missing/) + end + + def test_typesafe_score_without_legend_is_rejected + refute_accepted(:typesafe, score_question, + { "sev" => { "type" => "score", "score" => 1.0, "confidence" => 0.5, + "probabilities" => { "0" => 1.0 } } }, + /legend is missing/) + end + + def test_the_error_names_the_question_and_the_field + error = refute_accepted(:typesafe, choice_question, + { "dept" => { "type" => "choice", "choice" => "billing" } }, + /dept/) + assert_match(/confidence is missing/, error.message) + end + + def test_a_typesafe_answer_with_every_documented_field_is_accepted + response = ask(:typesafe, score_question, + { "sev" => { "type" => "score", "score" => 1.6, "confidence" => 0.78, + "probabilities" => { "0" => 0.05, "1" => 0.3, "2" => 0.65 }, + "legend" => { "0" => "low", "1" => "medium", "2" => "high" } } }) + + answer = response["sev"] + assert_in_delta 1.6, answer.score + assert_in_delta 0.78, answer.confidence + assert_equal 3, answer.legend.size + end + + # --- OpenRouter's schema makes the same fields optional --- + + def test_open_router_choice_without_probabilities_or_confidence_is_accepted + answer = ask(:open_router, choice_question, + { "dept" => { "type" => "choice", "choice" => "billing" } })["dept"] + + assert_equal "billing", answer.choice + assert_nil answer.confidence + assert_empty answer.probabilities + end + + def test_open_router_still_requires_the_answer_itself + refute_accepted(:open_router, choice_question, { "dept" => { "type" => "choice" } }, /choice is missing/) + refute_accepted(:open_router, noul_question, { "urgent" => { "type" => "noul" } }, /noul is missing/) + end + + # --- numbers have to be numbers that exist --- + + def test_an_infinite_noul_is_rejected + # JSON.parse turns 1e999 into Float::INFINITY, which passes is_a?(Numeric). + infinite = '{"model":"m","answers":{"urgent":{"type":"noul","noul":1e999}},"usage":{}}' + client = RubyDecisionModel::Client.new(provider: :typesafe, api_key: "k", sleeper: no_sleep, + transport: FakeTransport.new([[200, infinite]])) + + error = nil + capture_io do # JSON warns "Float 1e999 out of range" on the way past + error = assert_raises(RubyDecisionModel::InvalidResponse) { client.ask(state: {}, questions: noul_question) } + end + assert_match(/noul is not finite/, error.message) + end + + def test_a_noul_outside_zero_to_one_is_rejected + refute_accepted(:typesafe, noul_question, { "urgent" => { "type" => "noul", "noul" => 2 } }, + /noul is outside 0\.\.1/) + refute_accepted(:typesafe, noul_question, { "urgent" => { "type" => "noul", "noul" => -0.5 } }, + /noul is outside 0\.\.1/) + end + + def test_a_confidence_outside_zero_to_one_is_rejected + refute_accepted(:open_router, choice_question, + { "dept" => { "type" => "choice", "choice" => "billing", "confidence" => 1.4 } }, + /confidence is outside 0\.\.1/) + end + + def test_float_slop_just_past_one_is_tolerated_and_clamped + answer = ask(:open_router, noul_question, + { "urgent" => { "type" => "noul", "noul" => 1.0000000001 } })["urgent"] + + assert_in_delta 1.0, answer.noul + assert_operator answer.noul, :<=, 1.0 + end + + def test_a_probability_that_is_not_a_number_is_rejected + refute_accepted(:open_router, choice_question, + { "dept" => { "type" => "choice", "choice" => "billing", + "probabilities" => { "billing" => "high" } } }, + /probabilities\["billing"\] is not a number/) + end + + def test_probabilities_that_are_not_an_object_are_rejected_rather_than_erased + refute_accepted(:open_router, choice_question, + { "dept" => { "type" => "choice", "choice" => "billing", "probabilities" => [0.9, 0.1] } }, + /probabilities is not an object/) + end + + def test_a_legend_that_is_not_an_object_is_rejected_rather_than_erased + refute_accepted(:open_router, score_question, + { "sev" => { "type" => "score", "score" => 1.0, "legend" => "low, medium, high" } }, + /legend is not an object/) + end + + # --- a choice has to be one the question offered --- + + def test_a_choice_outside_the_questions_criteria_is_rejected + error = refute_accepted(:open_router, choice_question, + { "dept" => { "type" => "choice", "choice" => "marketing" } }, + /not one of the question's criteria/) + assert_match(/billing, auth/, error.message) + end + + def test_an_empty_choice_is_rejected + refute_accepted(:open_router, choice_question, { "dept" => { "type" => "choice", "choice" => "" } }, + /choice is empty/) + end + + def test_a_choice_is_not_checked_when_the_question_carries_no_criteria_map + # A hand-rolled question that bypassed Questions.choice has nothing to + # check against, so the label is taken as given. + answer = ask(:open_router, { "dept" => { "type" => "choice", "instructions" => "Which team?" } }, + { "dept" => { "type" => "choice", "choice" => "anything" } })["dept"] + + assert_equal "anything", answer.choice + end + + def test_symbol_keyed_criteria_still_match + questions = { "dept" => { type: "choice", instructions: "Which?", criteria: { billing: nil } } } + answer = ask(:open_router, questions, { "dept" => { "type" => "choice", "choice" => "billing" } })["dept"] + + assert_equal "billing", answer.choice + end + + def test_a_noul_probabilities_map_is_read_when_a_provider_sends_one + answer = ask(:open_router, noul_question, + { "urgent" => { "type" => "noul", "noul" => 0.8, + "probabilities" => { "true" => 0.8, "false" => 0.2 } } })["urgent"] + + assert_in_delta 0.8, answer.probabilities["true"] + end + + # --- a provider can declare its own contract --- + + def custom_provider(required) + Class.new(RubyDecisionModel::Providers::Base) do + define_method(:required_answer_fields) { required } + def name = :strict + def env_var = "STRICT_API_KEY" + def default_base_url = "https://strict.example" + def endpoint_path = "/v1/decide" + def default_model = "strict-1" + end + end + + def test_a_custom_provider_declares_which_fields_it_guarantees + provider = custom_provider("choice" => %w[choice probabilities]).new(api_key: "k") + client = RubyDecisionModel::Client.new( + provider: provider, sleeper: no_sleep, + transport: FakeTransport.new([[200, body("dept" => { "type" => "choice", "choice" => "billing" })]]) + ) + + error = assert_raises(RubyDecisionModel::InvalidResponse) { client.ask(state: {}, questions: choice_question) } + assert_match(/probabilities is missing/, error.message) + end + + def test_a_provider_requiring_a_field_the_client_does_not_model_is_rejected + error = assert_raises(RubyDecisionModel::ConfigurationError) do + RubyDecisionModel::Client.new(provider: custom_provider("noul" => %w[noul legend]).new(api_key: "k"), + transport: FakeTransport.new([])) + end + assert_match(/does not model: legend on noul/, error.message) + end + + def test_a_provider_requiring_an_unknown_answer_type_is_rejected + error = assert_raises(RubyDecisionModel::ConfigurationError) do + RubyDecisionModel::Client.new(provider: custom_provider("ranking" => %w[order]).new(api_key: "k"), + transport: FakeTransport.new([])) + end + assert_match(/unknown answer type/, error.message) + end +end diff --git a/test/client_test.rb b/test/client_test.rb index de5e058..47ee564 100644 --- a/test/client_test.rb +++ b/test/client_test.rb @@ -250,36 +250,40 @@ def test_malformed_takes_priority_over_missing_when_both_present assert_includes error.message, "urgent" end - def test_malformed_choice_missing_confidence_raises_invalid_response + # OpenRouter's published schema requires only type + choice/score, so a + # confidence-less answer is valid there and comes back nil. See + # answers_test.rb for the same bodies against Typesafe, which documents + # confidence as required and rejects them. + def test_choice_without_confidence_is_valid_on_open_router body = JSON.generate( "id" => "resp_7", "model" => "typesafe/jev-1.13", "answers" => { "category" => { "type" => "choice", "choice" => "bug" } }, "usage" => {} ) - transport = FakeTransport.new([[200, body]]) - client = build_client(transport) + client = build_client(FakeTransport.new([[200, body]])) - error = assert_raises(RubyDecisionModel::InvalidResponse) do - client.ask(state: {}, questions: { "category" => questions["category"] }) - end - assert_includes error.message, "category" + answer = client.ask(state: {}, questions: { "category" => questions["category"] })["category"] + + assert_equal "bug", answer.choice + assert_nil answer.confidence + assert_empty answer.probabilities end - def test_malformed_score_missing_confidence_raises_invalid_response + def test_score_without_confidence_is_valid_on_open_router body = JSON.generate( "id" => "resp_8", "model" => "typesafe/jev-1.13", "answers" => { "severity" => { "type" => "score", "score" => 1.4 } }, "usage" => {} ) - transport = FakeTransport.new([[200, body]]) - client = build_client(transport) + client = build_client(FakeTransport.new([[200, body]])) - error = assert_raises(RubyDecisionModel::InvalidResponse) do - client.ask(state: {}, questions: { "severity" => questions["severity"] }) - end - assert_includes error.message, "severity" + answer = client.ask(state: {}, questions: { "severity" => questions["severity"] })["severity"] + + assert_in_delta 1.4, answer.score + assert_nil answer.confidence + assert_empty answer.legend end def test_unsupported_question_type_raises_invalid_response @@ -299,7 +303,7 @@ def test_unsupported_question_type_raises_invalid_response assert_includes error.message, "mystery" end - def test_non_hash_probabilities_fall_back_to_empty_hash + def test_non_hash_probabilities_are_rejected_rather_than_erased body = JSON.generate( "id" => "resp_10", "model" => "typesafe/jev-1.13", @@ -309,8 +313,10 @@ def test_non_hash_probabilities_fall_back_to_empty_hash transport = FakeTransport.new([[200, body]]) client = build_client(transport) - response = client.ask(state: {}, questions: { "urgent" => questions["urgent"] }) - assert_equal({}, response["urgent"].probabilities) + error = assert_raises(RubyDecisionModel::InvalidResponse) do + client.ask(state: {}, questions: { "urgent" => questions["urgent"] }) + end + assert_match(/probabilities is not an object/, error.message) end def test_junk_usage_fields_become_nil