Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,14 @@
# Changelog

## Unreleased

- Usage fields are validated rather than coerced. `Integer(value, exception:
false)` turned `30.9` into `30`, `"120"` into `120`, and anything it could
not read — `[]`, `true`, an error object where usage should be — into `nil`,
which was indistinguishable from a provider that does not report the field.
A present field that is not a whole token count now raises `InvalidResponse`;
an absent one is still `nil`. Same for `cost`.

## 0.1.0 - 2026-09-18

Provider-neutral release. One `Client`, two providers behind it.
Expand Down
3 changes: 3 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,9 @@ client = RubyDecisionModel::Client.new(api_key: "sk-or-...")

Requests go to `https://openrouter.ai/api/alpha/decisions`. The default model is
`typesafe/jev-1.13`. Usage reports `input_tokens`, `output_tokens`, and `cost`.
A usage field the provider omits is `nil`; one that is present but is not a
whole token count raises `InvalidResponse` rather than being rounded off or
turned into `nil`.

### Typesafe native API

Expand Down
36 changes: 33 additions & 3 deletions lib/ruby_decision_model/providers/base.rb
Original file line number Diff line number Diff line change
Expand Up @@ -84,12 +84,42 @@ def usage(parsed)
raw = parsed.is_a?(Hash) && parsed["usage"].is_a?(Hash) ? parsed["usage"] : {}

Response::Usage.new(
input_tokens: Integer(raw["input_tokens"], exception: false),
output_tokens: Integer(raw["output_tokens"], exception: false),
cost: reports_cost? ? Float(raw["cost"], exception: false) : nil
input_tokens: token_count(raw, "input_tokens"),
output_tokens: token_count(raw, "output_tokens"),
cost: reports_cost? ? cost(raw) : nil
)
end

# A token count is a whole number of tokens. Integer(..., exception:
# false) accepted a good deal more than that: 30.9 became 30, "120"
# became 120, and anything it could not read at all -- [], true, an
# error object where usage should be -- became nil, indistinguishable
# from a provider that simply does not report the field. Requests are
# billed per input token, so a usage number that quietly became nil or
# lost its fraction is worse than no number.
#
# An absent field is still nil. A present one has to be a count.
def token_count(raw, name)
return nil unless raw.key?(name)

value = raw[name]
return nil if value.nil?
return value if value.is_a?(Integer)
return Integer(value) if value.is_a?(Float) && value.finite? && (value % 1).zero?

raise InvalidResponse, "usage.#{name} is not a token count: #{value.inspect}"
end

def cost(raw)
return nil unless raw.key?("cost")

value = raw["cost"]
return nil if value.nil?
return value.to_f if value.is_a?(Numeric) && value.finite?

raise InvalidResponse, "usage.cost is not a number: #{value.inspect}"
end

# Keeps the API key out of logs and error output.
def inspect
"#<#{self.class.name} name=#{name.inspect} base_url=#{base_url.inspect} api_key=#{api_key? ? '[REDACTED]' : 'nil'}>"
Expand Down
23 changes: 18 additions & 5 deletions test/client_test.rb
Original file line number Diff line number Diff line change
Expand Up @@ -313,7 +313,7 @@ def test_non_hash_probabilities_fall_back_to_empty_hash
assert_equal({}, response["urgent"].probabilities)
end

def test_junk_usage_fields_become_nil
def test_junk_usage_fields_are_rejected_rather_than_nilled
body = JSON.generate(
"id" => "resp_4",
"model" => "typesafe/jev-1.13",
Expand All @@ -327,10 +327,23 @@ def test_junk_usage_fields_become_nil
transport = FakeTransport.new([[200, body]])
client = build_client(transport)

response = client.ask(state: {}, questions: questions)
error = assert_raises(RubyDecisionModel::InvalidResponse) { client.ask(state: {}, questions: questions) }
assert_match(/usage\.input_tokens is not a token count/, error.message)
end

def test_an_absent_usage_field_is_still_nil
body = JSON.generate(
"id" => "resp_4b",
"model" => "typesafe/jev-1.13",
"answers" => { "urgent" => { "type" => "noul", "noul" => 0.5 } },
"usage" => { "input_tokens" => 120 }
)
client = build_client(FakeTransport.new([[200, body]]))

usage = client.ask(state: {}, questions: { "urgent" => questions["urgent"] }).usage

assert_nil response.usage.input_tokens
assert_nil response.usage.output_tokens
assert_nil response.usage.cost
assert_equal 120, usage.input_tokens
assert_nil usage.output_tokens
assert_nil usage.cost
end
end
85 changes: 85 additions & 0 deletions test/usage_test.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
# frozen_string_literal: true

require "test_helper"

# Requests are billed per input token, so a usage number is either a number
# or an error -- never a quietly coerced one.
class UsageTest < Minitest::Test
def usage_for(raw, provider: :open_router)
RubyDecisionModel::Providers.build(provider, api_key: "k").usage("usage" => raw)
end

def test_integers_pass_through
usage = usage_for({ "input_tokens" => 120, "output_tokens" => 30, "cost" => 0.0012 })

assert_equal 120, usage.input_tokens
assert_equal 30, usage.output_tokens
assert_in_delta 0.0012, usage.cost
end

def test_a_whole_float_is_accepted_as_a_count
# JSON has one number type, so 120 may arrive as 120.0.
assert_equal 120, usage_for({ "input_tokens" => 120.0 }).input_tokens
end

def test_a_fractional_token_count_is_rejected_rather_than_truncated
error = assert_raises(RubyDecisionModel::InvalidResponse) { usage_for({ "output_tokens" => 30.9 }) }
assert_match(/usage\.output_tokens is not a token count/, error.message)
end

def test_a_numeric_string_is_rejected
# "120" used to become 120 and "120.5" used to become nil.
assert_raises(RubyDecisionModel::InvalidResponse) { usage_for({ "input_tokens" => "120" }) }
assert_raises(RubyDecisionModel::InvalidResponse) { usage_for({ "input_tokens" => "120.5" }) }
end

def test_structural_junk_is_rejected
[[], {}, true, "junk"].each do |junk|
assert_raises(RubyDecisionModel::InvalidResponse, "expected #{junk.inspect} to be rejected") do
usage_for({ "input_tokens" => junk })
end
end
end

def test_an_infinite_count_is_rejected
assert_raises(RubyDecisionModel::InvalidResponse) { usage_for({ "input_tokens" => Float::INFINITY }) }
end

def test_an_absent_field_is_nil
usage = usage_for({ "input_tokens" => 10 })

assert_equal 10, usage.input_tokens
assert_nil usage.output_tokens
end

def test_an_explicitly_null_field_is_nil
assert_nil usage_for({ "input_tokens" => nil }).input_tokens
end

def test_an_absent_usage_object_is_all_nil
usage = RubyDecisionModel::Providers.build(:open_router, api_key: "k").usage({})

assert_nil usage.input_tokens
assert_nil usage.output_tokens
assert_nil usage.cost
end

# --- cost ---

def test_cost_is_nil_on_a_provider_that_does_not_report_it
assert_nil usage_for({ "cost" => 0.5 }, provider: :typesafe).cost
end

def test_a_cost_that_is_not_a_number_is_rejected
error = assert_raises(RubyDecisionModel::InvalidResponse) { usage_for({ "cost" => "junk" }) }
assert_match(/usage\.cost is not a number/, error.message)
end

def test_an_integer_cost_becomes_a_float
assert_in_delta 1.0, usage_for({ "cost" => 1 }).cost
end

def test_a_typesafe_provider_ignores_a_junk_cost_it_never_reports
assert_nil usage_for({ "cost" => "junk" }, provider: :typesafe).cost
end
end