diff --git a/CHANGELOG.md b/CHANGELOG.md index 846df76..e04c7ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## Unreleased + +- Usage fields are validated rather than coerced. `Integer(value, exception: + false)` turned `30.9` into `30`, `"120"` into `120`, and anything it could + not read — `[]`, `true`, an error object where usage should be — into `nil`, + which was indistinguishable from a provider that does not report the field. + A present field that is not a whole token count now raises `InvalidResponse`; + an absent one is still `nil`. Same for `cost`. + ## 0.1.0 - 2026-09-18 Provider-neutral release. One `Client`, two providers behind it. diff --git a/README.md b/README.md index 79638d6..cb7727c 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,9 @@ client = RubyDecisionModel::Client.new(api_key: "sk-or-...") Requests go to `https://openrouter.ai/api/alpha/decisions`. The default model is `typesafe/jev-1.13`. Usage reports `input_tokens`, `output_tokens`, and `cost`. +A usage field the provider omits is `nil`; one that is present but is not a +whole token count raises `InvalidResponse` rather than being rounded off or +turned into `nil`. ### Typesafe native API diff --git a/lib/ruby_decision_model/providers/base.rb b/lib/ruby_decision_model/providers/base.rb index 85f7089..3c789a8 100644 --- a/lib/ruby_decision_model/providers/base.rb +++ b/lib/ruby_decision_model/providers/base.rb @@ -84,12 +84,42 @@ def usage(parsed) raw = parsed.is_a?(Hash) && parsed["usage"].is_a?(Hash) ? parsed["usage"] : {} Response::Usage.new( - input_tokens: Integer(raw["input_tokens"], exception: false), - output_tokens: Integer(raw["output_tokens"], exception: false), - cost: reports_cost? ? Float(raw["cost"], exception: false) : nil + input_tokens: token_count(raw, "input_tokens"), + output_tokens: token_count(raw, "output_tokens"), + cost: reports_cost? ? cost(raw) : nil ) end + # A token count is a whole number of tokens. Integer(..., exception: + # false) accepted a good deal more than that: 30.9 became 30, "120" + # became 120, and anything it could not read at all -- [], true, an + # error object where usage should be -- became nil, indistinguishable + # from a provider that simply does not report the field. Requests are + # billed per input token, so a usage number that quietly became nil or + # lost its fraction is worse than no number. + # + # An absent field is still nil. A present one has to be a count. + def token_count(raw, name) + return nil unless raw.key?(name) + + value = raw[name] + return nil if value.nil? + return value if value.is_a?(Integer) + return Integer(value) if value.is_a?(Float) && value.finite? && (value % 1).zero? + + raise InvalidResponse, "usage.#{name} is not a token count: #{value.inspect}" + end + + def cost(raw) + return nil unless raw.key?("cost") + + value = raw["cost"] + return nil if value.nil? + return value.to_f if value.is_a?(Numeric) && value.finite? + + raise InvalidResponse, "usage.cost is not a number: #{value.inspect}" + end + # Keeps the API key out of logs and error output. def inspect "#<#{self.class.name} name=#{name.inspect} base_url=#{base_url.inspect} api_key=#{api_key? ? '[REDACTED]' : 'nil'}>" diff --git a/test/client_test.rb b/test/client_test.rb index de5e058..88f0818 100644 --- a/test/client_test.rb +++ b/test/client_test.rb @@ -313,7 +313,7 @@ def test_non_hash_probabilities_fall_back_to_empty_hash assert_equal({}, response["urgent"].probabilities) end - def test_junk_usage_fields_become_nil + def test_junk_usage_fields_are_rejected_rather_than_nilled body = JSON.generate( "id" => "resp_4", "model" => "typesafe/jev-1.13", @@ -327,10 +327,23 @@ def test_junk_usage_fields_become_nil transport = FakeTransport.new([[200, body]]) client = build_client(transport) - response = client.ask(state: {}, questions: questions) + error = assert_raises(RubyDecisionModel::InvalidResponse) { client.ask(state: {}, questions: questions) } + assert_match(/usage\.input_tokens is not a token count/, error.message) + end + + def test_an_absent_usage_field_is_still_nil + body = JSON.generate( + "id" => "resp_4b", + "model" => "typesafe/jev-1.13", + "answers" => { "urgent" => { "type" => "noul", "noul" => 0.5 } }, + "usage" => { "input_tokens" => 120 } + ) + client = build_client(FakeTransport.new([[200, body]])) + + usage = client.ask(state: {}, questions: { "urgent" => questions["urgent"] }).usage - assert_nil response.usage.input_tokens - assert_nil response.usage.output_tokens - assert_nil response.usage.cost + assert_equal 120, usage.input_tokens + assert_nil usage.output_tokens + assert_nil usage.cost end end diff --git a/test/usage_test.rb b/test/usage_test.rb new file mode 100644 index 0000000..f6fa1d1 --- /dev/null +++ b/test/usage_test.rb @@ -0,0 +1,85 @@ +# frozen_string_literal: true + +require "test_helper" + +# Requests are billed per input token, so a usage number is either a number +# or an error -- never a quietly coerced one. +class UsageTest < Minitest::Test + def usage_for(raw, provider: :open_router) + RubyDecisionModel::Providers.build(provider, api_key: "k").usage("usage" => raw) + end + + def test_integers_pass_through + usage = usage_for({ "input_tokens" => 120, "output_tokens" => 30, "cost" => 0.0012 }) + + assert_equal 120, usage.input_tokens + assert_equal 30, usage.output_tokens + assert_in_delta 0.0012, usage.cost + end + + def test_a_whole_float_is_accepted_as_a_count + # JSON has one number type, so 120 may arrive as 120.0. + assert_equal 120, usage_for({ "input_tokens" => 120.0 }).input_tokens + end + + def test_a_fractional_token_count_is_rejected_rather_than_truncated + error = assert_raises(RubyDecisionModel::InvalidResponse) { usage_for({ "output_tokens" => 30.9 }) } + assert_match(/usage\.output_tokens is not a token count/, error.message) + end + + def test_a_numeric_string_is_rejected + # "120" used to become 120 and "120.5" used to become nil. + assert_raises(RubyDecisionModel::InvalidResponse) { usage_for({ "input_tokens" => "120" }) } + assert_raises(RubyDecisionModel::InvalidResponse) { usage_for({ "input_tokens" => "120.5" }) } + end + + def test_structural_junk_is_rejected + [[], {}, true, "junk"].each do |junk| + assert_raises(RubyDecisionModel::InvalidResponse, "expected #{junk.inspect} to be rejected") do + usage_for({ "input_tokens" => junk }) + end + end + end + + def test_an_infinite_count_is_rejected + assert_raises(RubyDecisionModel::InvalidResponse) { usage_for({ "input_tokens" => Float::INFINITY }) } + end + + def test_an_absent_field_is_nil + usage = usage_for({ "input_tokens" => 10 }) + + assert_equal 10, usage.input_tokens + assert_nil usage.output_tokens + end + + def test_an_explicitly_null_field_is_nil + assert_nil usage_for({ "input_tokens" => nil }).input_tokens + end + + def test_an_absent_usage_object_is_all_nil + usage = RubyDecisionModel::Providers.build(:open_router, api_key: "k").usage({}) + + assert_nil usage.input_tokens + assert_nil usage.output_tokens + assert_nil usage.cost + end + + # --- cost --- + + def test_cost_is_nil_on_a_provider_that_does_not_report_it + assert_nil usage_for({ "cost" => 0.5 }, provider: :typesafe).cost + end + + def test_a_cost_that_is_not_a_number_is_rejected + error = assert_raises(RubyDecisionModel::InvalidResponse) { usage_for({ "cost" => "junk" }) } + assert_match(/usage\.cost is not a number/, error.message) + end + + def test_an_integer_cost_becomes_a_float + assert_in_delta 1.0, usage_for({ "cost" => 1 }).cost + end + + def test_a_typesafe_provider_ignores_a_junk_cost_it_never_reports + assert_nil usage_for({ "cost" => "junk" }, provider: :typesafe).cost + end +end