From 117c6008a42ab907e80cb40db003853123380b3f Mon Sep 17 00:00:00 2001 From: Szymon Kurcab Date: Fri, 18 Sep 2026 20:49:00 +0200 Subject: [PATCH] fix: stop quietly coercing the usage numbers usage read its fields through Integer(value, exception: false) and Float(value, exception: false), which is a parser, not a check. It accepted things that are not token counts and produced a number anyway: 30.9 became 30, "120" became 120. And everything it could not read at all -- [], true, "120.5", an error object sitting where usage should be -- became nil, which is exactly what a provider that does not report the field produces. There was no way to tell "this provider sends no cost" from "this response was malformed". Requests are billed per input token. A usage number that silently lost its fraction, or silently became nil, is worse than no number, because whatever is adding these up keeps adding them up. A present field now has to be a whole token count -- an Integer, or a Float with nothing after the point, since JSON has one number type and 120 may arrive as 120.0. Anything else raises InvalidResponse naming the field and showing the value. An absent field is still nil, and a provider that does not report cost still reports nil for it without looking at what was sent. The test that asserted junk becomes nil now asserts it is rejected. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 9 +++ README.md | 3 + lib/ruby_decision_model/providers/base.rb | 36 +++++++++- test/client_test.rb | 23 ++++-- test/usage_test.rb | 85 +++++++++++++++++++++++ 5 files changed, 148 insertions(+), 8 deletions(-) create mode 100644 test/usage_test.rb diff --git a/CHANGELOG.md b/CHANGELOG.md index 846df76..e04c7ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## Unreleased + +- Usage fields are validated rather than coerced. `Integer(value, exception: + false)` turned `30.9` into `30`, `"120"` into `120`, and anything it could + not read — `[]`, `true`, an error object where usage should be — into `nil`, + which was indistinguishable from a provider that does not report the field. + A present field that is not a whole token count now raises `InvalidResponse`; + an absent one is still `nil`. Same for `cost`. + ## 0.1.0 - 2026-09-18 Provider-neutral release. One `Client`, two providers behind it. diff --git a/README.md b/README.md index 79638d6..cb7727c 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,9 @@ client = RubyDecisionModel::Client.new(api_key: "sk-or-...") Requests go to `https://openrouter.ai/api/alpha/decisions`. The default model is `typesafe/jev-1.13`. Usage reports `input_tokens`, `output_tokens`, and `cost`. +A usage field the provider omits is `nil`; one that is present but is not a +whole token count raises `InvalidResponse` rather than being rounded off or +turned into `nil`. ### Typesafe native API diff --git a/lib/ruby_decision_model/providers/base.rb b/lib/ruby_decision_model/providers/base.rb index 85f7089..3c789a8 100644 --- a/lib/ruby_decision_model/providers/base.rb +++ b/lib/ruby_decision_model/providers/base.rb @@ -84,12 +84,42 @@ def usage(parsed) raw = parsed.is_a?(Hash) && parsed["usage"].is_a?(Hash) ? parsed["usage"] : {} Response::Usage.new( - input_tokens: Integer(raw["input_tokens"], exception: false), - output_tokens: Integer(raw["output_tokens"], exception: false), - cost: reports_cost? ? Float(raw["cost"], exception: false) : nil + input_tokens: token_count(raw, "input_tokens"), + output_tokens: token_count(raw, "output_tokens"), + cost: reports_cost? ? cost(raw) : nil ) end + # A token count is a whole number of tokens. Integer(..., exception: + # false) accepted a good deal more than that: 30.9 became 30, "120" + # became 120, and anything it could not read at all -- [], true, an + # error object where usage should be -- became nil, indistinguishable + # from a provider that simply does not report the field. Requests are + # billed per input token, so a usage number that quietly became nil or + # lost its fraction is worse than no number. + # + # An absent field is still nil. A present one has to be a count. + def token_count(raw, name) + return nil unless raw.key?(name) + + value = raw[name] + return nil if value.nil? + return value if value.is_a?(Integer) + return Integer(value) if value.is_a?(Float) && value.finite? && (value % 1).zero? + + raise InvalidResponse, "usage.#{name} is not a token count: #{value.inspect}" + end + + def cost(raw) + return nil unless raw.key?("cost") + + value = raw["cost"] + return nil if value.nil? + return value.to_f if value.is_a?(Numeric) && value.finite? + + raise InvalidResponse, "usage.cost is not a number: #{value.inspect}" + end + # Keeps the API key out of logs and error output. def inspect "#<#{self.class.name} name=#{name.inspect} base_url=#{base_url.inspect} api_key=#{api_key? ? '[REDACTED]' : 'nil'}>" diff --git a/test/client_test.rb b/test/client_test.rb index de5e058..88f0818 100644 --- a/test/client_test.rb +++ b/test/client_test.rb @@ -313,7 +313,7 @@ def test_non_hash_probabilities_fall_back_to_empty_hash assert_equal({}, response["urgent"].probabilities) end - def test_junk_usage_fields_become_nil + def test_junk_usage_fields_are_rejected_rather_than_nilled body = JSON.generate( "id" => "resp_4", "model" => "typesafe/jev-1.13", @@ -327,10 +327,23 @@ def test_junk_usage_fields_become_nil transport = FakeTransport.new([[200, body]]) client = build_client(transport) - response = client.ask(state: {}, questions: questions) + error = assert_raises(RubyDecisionModel::InvalidResponse) { client.ask(state: {}, questions: questions) } + assert_match(/usage\.input_tokens is not a token count/, error.message) + end + + def test_an_absent_usage_field_is_still_nil + body = JSON.generate( + "id" => "resp_4b", + "model" => "typesafe/jev-1.13", + "answers" => { "urgent" => { "type" => "noul", "noul" => 0.5 } }, + "usage" => { "input_tokens" => 120 } + ) + client = build_client(FakeTransport.new([[200, body]])) + + usage = client.ask(state: {}, questions: { "urgent" => questions["urgent"] }).usage - assert_nil response.usage.input_tokens - assert_nil response.usage.output_tokens - assert_nil response.usage.cost + assert_equal 120, usage.input_tokens + assert_nil usage.output_tokens + assert_nil usage.cost end end diff --git a/test/usage_test.rb b/test/usage_test.rb new file mode 100644 index 0000000..f6fa1d1 --- /dev/null +++ b/test/usage_test.rb @@ -0,0 +1,85 @@ +# frozen_string_literal: true + +require "test_helper" + +# Requests are billed per input token, so a usage number is either a number +# or an error -- never a quietly coerced one. +class UsageTest < Minitest::Test + def usage_for(raw, provider: :open_router) + RubyDecisionModel::Providers.build(provider, api_key: "k").usage("usage" => raw) + end + + def test_integers_pass_through + usage = usage_for({ "input_tokens" => 120, "output_tokens" => 30, "cost" => 0.0012 }) + + assert_equal 120, usage.input_tokens + assert_equal 30, usage.output_tokens + assert_in_delta 0.0012, usage.cost + end + + def test_a_whole_float_is_accepted_as_a_count + # JSON has one number type, so 120 may arrive as 120.0. + assert_equal 120, usage_for({ "input_tokens" => 120.0 }).input_tokens + end + + def test_a_fractional_token_count_is_rejected_rather_than_truncated + error = assert_raises(RubyDecisionModel::InvalidResponse) { usage_for({ "output_tokens" => 30.9 }) } + assert_match(/usage\.output_tokens is not a token count/, error.message) + end + + def test_a_numeric_string_is_rejected + # "120" used to become 120 and "120.5" used to become nil. + assert_raises(RubyDecisionModel::InvalidResponse) { usage_for({ "input_tokens" => "120" }) } + assert_raises(RubyDecisionModel::InvalidResponse) { usage_for({ "input_tokens" => "120.5" }) } + end + + def test_structural_junk_is_rejected + [[], {}, true, "junk"].each do |junk| + assert_raises(RubyDecisionModel::InvalidResponse, "expected #{junk.inspect} to be rejected") do + usage_for({ "input_tokens" => junk }) + end + end + end + + def test_an_infinite_count_is_rejected + assert_raises(RubyDecisionModel::InvalidResponse) { usage_for({ "input_tokens" => Float::INFINITY }) } + end + + def test_an_absent_field_is_nil + usage = usage_for({ "input_tokens" => 10 }) + + assert_equal 10, usage.input_tokens + assert_nil usage.output_tokens + end + + def test_an_explicitly_null_field_is_nil + assert_nil usage_for({ "input_tokens" => nil }).input_tokens + end + + def test_an_absent_usage_object_is_all_nil + usage = RubyDecisionModel::Providers.build(:open_router, api_key: "k").usage({}) + + assert_nil usage.input_tokens + assert_nil usage.output_tokens + assert_nil usage.cost + end + + # --- cost --- + + def test_cost_is_nil_on_a_provider_that_does_not_report_it + assert_nil usage_for({ "cost" => 0.5 }, provider: :typesafe).cost + end + + def test_a_cost_that_is_not_a_number_is_rejected + error = assert_raises(RubyDecisionModel::InvalidResponse) { usage_for({ "cost" => "junk" }) } + assert_match(/usage\.cost is not a number/, error.message) + end + + def test_an_integer_cost_becomes_a_float + assert_in_delta 1.0, usage_for({ "cost" => 1 }).cost + end + + def test_a_typesafe_provider_ignores_a_junk_cost_it_never_reports + assert_nil usage_for({ "cost" => "junk" }, provider: :typesafe).cost + end +end