From 3c524fee0c4d4e9dc38e45de39454b1d199b80a3 Mon Sep 17 00:00:00 2001 From: Obie Fernandez Date: Tue, 6 Oct 2026 22:00:41 -0600 Subject: [PATCH 1/4] feat: add OpenAI, Cloudflare, Perplexity, Databricks and System One providers The client keeps one shape for questions and answers while each provider speaks its vendor's wire format. - OpenAI Decisions (gpt-6-luna) translates both ways between System One and OpenAI's named-array format, including refusals and boolean choices for noul criteria. Ambiguous responses (an id answered twice, duplicate probability values) raise InvalidResponse. - Cloudflare Clef and Clef-flash on Workers AI, Perplexity pplx-decider, Databricks ai_decide, and any /v1/systemone server (Ollama, autojev, strands-decider). - Client#ask takes images: as data URLs; Images builds them. - Providers own the request id header, default timeout, vendor error text (key masked, control characters removed, capped), and response normalization. Noul answers always carry a true/false split. - RUBY_DECISION_MODEL_PROVIDER names a provider, including for api_key:. - Write timeouts retry as TimeoutError; unencodable state raises RequestError. 0.1.0 custom providers keep working. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Lz7fQbXMHCS4fYAm6p9n97 --- lib/ruby_decision_model.rb | 1 + lib/ruby_decision_model/client.rb | 134 +++-- lib/ruby_decision_model/errors.rb | 8 +- lib/ruby_decision_model/images.rb | 33 ++ lib/ruby_decision_model/providers.rb | 48 +- lib/ruby_decision_model/providers/base.rb | 165 ++++++- .../providers/cloudflare.rb | 119 +++++ .../providers/databricks.rb | 87 ++++ .../providers/open_router.rb | 10 +- lib/ruby_decision_model/providers/openai.rb | 237 +++++++++ .../providers/perplexity.rb | 73 +++ .../providers/system_one.rb | 90 ++++ lib/ruby_decision_model/providers/typesafe.rb | 1 + lib/ruby_decision_model/retry_policy.rb | 2 +- test/client_test.rb | 81 ++- test/openai_provider_test.rb | 357 ++++++++++++++ test/providers_test.rb | 357 ++++++++++++++ test/retry_test.rb | 11 + test/test_helper.rb | 62 ++- test/vendor_providers_test.rb | 460 ++++++++++++++++++ 20 files changed, 2263 insertions(+), 73 deletions(-) create mode 100644 lib/ruby_decision_model/images.rb create mode 100644 lib/ruby_decision_model/providers/cloudflare.rb create mode 100644 lib/ruby_decision_model/providers/databricks.rb create mode 100644 lib/ruby_decision_model/providers/openai.rb create mode 100644 lib/ruby_decision_model/providers/perplexity.rb create mode 100644 lib/ruby_decision_model/providers/system_one.rb create mode 100644 test/openai_provider_test.rb create mode 100644 test/vendor_providers_test.rb diff --git a/lib/ruby_decision_model.rb b/lib/ruby_decision_model.rb index fb00614..6cb462b 100644 --- a/lib/ruby_decision_model.rb +++ b/lib/ruby_decision_model.rb @@ -6,6 +6,7 @@ require_relative "ruby_decision_model/version" require_relative "ruby_decision_model/errors" require_relative "ruby_decision_model/questions" +require_relative "ruby_decision_model/images" require_relative "ruby_decision_model/answers" require_relative "ruby_decision_model/response" require_relative "ruby_decision_model/providers" diff --git a/lib/ruby_decision_model/client.rb b/lib/ruby_decision_model/client.rb index 3457726..b3668d2 100644 --- a/lib/ruby_decision_model/client.rb +++ b/lib/ruby_decision_model/client.rb @@ -7,41 +7,48 @@ module RubyDecisionModel class Client - REQUEST_ID_HEADER = "x-typesafe-request-id" - # Kept from 0.0.1 for callers that referenced them. They describe the - # OpenRouter provider and the default RetryPolicy; prefer those directly. + # Typesafe and OpenRouter providers and the default RetryPolicy; prefer + # those directly. Each provider now names its own request id header. + REQUEST_ID_HEADER = "x-typesafe-request-id" DEFAULT_BASE_URL = Providers::OpenRouter.new.default_base_url DEFAULT_MODEL = Providers::OpenRouter.new.default_model MAX_ATTEMPTS = RetryPolicy.new.max_retries + 1 RETRYABLE_STATUSES = RetryPolicy::DEFAULT_STATUSES RETRYABLE_EXCEPTIONS = (RetryPolicy::TIMEOUT_EXCEPTIONS + RetryPolicy::CONNECTION_EXCEPTIONS).freeze - attr_reader :provider, :model, :retry_policy, :timeout + # Connecting is quick wherever the answer is slow, so the open timeout + # stays short unless the caller sets timeout: explicitly. + DEFAULT_OPEN_TIMEOUT = 5 - # provider: :open_router, :typesafe, or a Providers::Base instance. When - # nil, api_key: alone selects OpenRouter; otherwise the - # environment decides (TYPESAFE_API_KEY, then OPENROUTER_API_KEY). + attr_reader :provider, :model, :retry_policy, :timeout, :open_timeout + + # provider: a name from Providers.names or a Providers::Base instance. + # When nil, RUBY_DECISION_MODEL_PROVIDER names one if set; + # otherwise api_key: alone selects OpenRouter, and with no + # api_key: the environment decides (see Providers.from_env). # api_key: overrides the provider's env var. # model: nil means the provider default; aliases resolve per provider. # base_url: overrides the provider base URL. + # timeout: read timeout in seconds, and open timeout too when given; + # nil means the provider's read default (5 for Jev-speed APIs, + # 30 where the vendor documents multi-second responses) with a + # 5 second open timeout. # transport: callable(url:, headers:, body:) returning # [status, body_string, headers_hash] (a 2-element return is # still accepted and treated as having no headers). # retry: a RetryPolicy or a Hash of overrides. # random: callable returning a Float in 0...1, used for backoff jitter. # clock: callable returning monotonic seconds, used for total_timeout. - def initialize(provider: nil, api_key: nil, model: nil, base_url: nil, timeout: 5, + def initialize(provider: nil, api_key: nil, model: nil, base_url: nil, timeout: nil, transport: nil, sleeper: ->(seconds) { sleep(seconds) }, retry: {}, random: -> { rand }, clock: -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }) @provider = resolve_provider(provider, api_key: api_key, base_url: base_url) - unless @provider.api_key? - raise ConfigurationError, - "api_key is required for #{@provider.name}: pass api_key: or set #{@provider.env_var}" - end + @provider.validate! @model = @provider.resolve_model(model) - @timeout = timeout + @timeout = timeout || @provider.default_timeout + @open_timeout = timeout || DEFAULT_OPEN_TIMEOUT @transport = transport || default_transport @sleeper = sleeper @retry_policy = RetryPolicy.from(binding.local_variable_get(:retry)) @@ -53,18 +60,46 @@ def base_url @provider.base_url end - def ask(state:, questions:) + # images: data URLs (see Images) for providers that read images. Each + # provider places them where its API expects. + def ask(state:, questions:, images: nil) raise RequestError, "questions must not be empty" if questions.nil? || questions.empty? - body = @provider.request_body(model: @model, state: state, questions: questions) + body = build_body(state, questions, images) status, response_body, response_headers = perform_with_retry( - url: @provider.url, headers: @provider.headers, body: body + url: request_url, headers: @provider.headers, body: body ) handle_response(status, response_body, response_headers, questions) end private + # Providers written against 0.1.0 may override url without the model + # argument added in 0.2.0. + def request_url + takes_model = @provider.method(:url).parameters.any? { |type, _| %i[req opt rest].include?(type) } + takes_model ? @provider.url(@model) : @provider.url + end + + def build_body(state, questions, images) + images = Array(images) + return @provider.request_body(model: @model, state: state, questions: questions) if images.empty? + + raise RequestError, "#{@provider.name} does not accept images" unless @provider.supports_images? + + images.each do |image| + next if image.is_a?(String) && image.start_with?("data:image/") + + raise RequestError, "images must be data URLs (data:image/...); see RubyDecisionModel::Images" + end + + @provider.request_body(model: @model, state: state, questions: questions, images: images) + rescue JSON::GeneratorError, JSON::NestingError => e + # The generator's message can quote the offending value, so it stays + # on #cause rather than in a message that may be logged. + raise RequestError, "state or questions could not be encoded as JSON (#{e.class})" + end + def resolve_provider(provider, api_key:, base_url:) case provider when Providers::Base @@ -81,7 +116,7 @@ def resolve_provider(provider, api_key:, base_url:) "no provider configured: pass provider: or api_key:, or set one of #{Providers.env_vars.join(', ')}" ) else - Providers.build(:open_router, api_key: api_key, base_url: base_url) + Providers.build(Providers.named_in_env || :open_router, api_key: api_key, base_url: base_url) end else raise ConfigurationError, "provider must be a Symbol or a Providers::Base, got #{provider.class}" @@ -93,8 +128,9 @@ def default_transport uri = URI.parse(url) http = Net::HTTP.new(uri.host, uri.port) http.use_ssl = uri.scheme == "https" - http.open_timeout = @timeout + http.open_timeout = @open_timeout http.read_timeout = @timeout + http.write_timeout = @timeout request = Net::HTTP::Post.new(uri.request_uri) headers.each { |k, v| request[k] = v } @@ -162,25 +198,23 @@ def raise_transport_error(exception) raise TransportError.new("transport error: #{exception.message}", cause_error: exception) end + ERROR_CLASSES = { + 401 => [Unauthorized, "unauthorized"], + 413 => [PayloadTooLarge, "payload too large"], + 422 => [UnprocessableEntity, "unprocessable entity"], + 429 => [RateLimited, "rate limited"], + 529 => [Overloaded, "overloaded"] + }.freeze + private_constant :ERROR_CLASSES + def handle_response(status, response_body, response_headers, questions) - case status - when 200..299 - parse_success(response_body, response_headers, questions) - when 401 - raise Unauthorized.new("unauthorized", status: status, body: response_body, headers: response_headers) - when 413 - raise PayloadTooLarge.new("payload too large", status: status, body: response_body, headers: response_headers) - when 422 - raise UnprocessableEntity.new("unprocessable entity", status: status, body: response_body, - headers: response_headers) - when 429 - raise RateLimited.new("rate limited", status: status, body: response_body, headers: response_headers) - when 529 - raise Overloaded.new("overloaded", status: status, body: response_body, headers: response_headers) - else - raise ApiError.new("api error (status #{status})", status: status, body: response_body, - headers: response_headers) - end + return parse_success(response_body, response_headers, questions) if (200..299).cover?(status) + + klass, message = ERROR_CLASSES.fetch(status) { [ApiError, "api error (status #{status})"] } + detail = @provider.error_message(response_body) + message = "#{message}: #{detail}" if detail + + raise klass.new(message, status: status, body: response_body, headers: response_headers) end def parse_success(response_body, response_headers, questions) @@ -194,12 +228,16 @@ def parse_success(response_body, response_headers, questions) raise InvalidResponse, "response body was not a JSON object" unless parsed.is_a?(Hash) - raw_answers = parsed["answers"] + canonical = @provider.normalize_response(parsed, questions: questions) + raise InvalidResponse, "response body was not a JSON object" unless canonical.is_a?(Hash) + + raw_answers = canonical["answers"] raw_answers = {} unless raw_answers.is_a?(Hash) normalized = {} malformed = [] missing = [] + refused = [] questions.each do |raw_id, question| id = raw_id.to_s @@ -213,6 +251,7 @@ def parse_success(response_body, response_headers, questions) malformed << id end else + refused << id if answer_hash.is_a?(Hash) && answer_hash["type"] == "refusal" missing << id end end @@ -225,28 +264,27 @@ def parse_success(response_body, response_headers, questions) end if missing.any? - raise MissingAnswers.new( - "missing or wrong-type answers for: #{missing.join(', ')}", - answers: normalized, - missing: missing - ) + message = "missing or wrong-type answers for: #{missing.join(', ')}" + message += " (refused: #{refused.join(', ')})" if refused.any? + raise MissingAnswers.new(message, answers: normalized, missing: missing, refused: refused) end Response.new( answers: normalized, - usage: @provider.usage(parsed), - model: parsed["model"], - id: parsed["id"], + usage: @provider.usage(canonical), + model: canonical["model"], + id: canonical["id"], raw: parsed, request_id: request_id_from(response_headers) ) end def request_id_from(headers) - return nil unless headers.is_a?(Hash) + header = @provider.request_id_header + return nil unless header && headers.is_a?(Hash) headers.each do |key, value| - next unless key.to_s.casecmp?(REQUEST_ID_HEADER) + next unless key.to_s.casecmp?(header) return value.is_a?(Array) ? value.first : value end @@ -259,7 +297,7 @@ def normalize_answer(type, hash) case type when "noul" noul = hash["noul"] - raise MalformedAnswer unless noul.is_a?(Numeric) + raise MalformedAnswer unless noul.is_a?(Numeric) && noul.between?(0, 1) Answers::Noul.new(noul: noul.to_f, probabilities: hash_or_empty(hash["probabilities"])) when "choice" diff --git a/lib/ruby_decision_model/errors.rb b/lib/ruby_decision_model/errors.rb index d35c51a..c93fdcf 100644 --- a/lib/ruby_decision_model/errors.rb +++ b/lib/ruby_decision_model/errors.rb @@ -48,12 +48,16 @@ def initialize(message, answers: {}) end end + # Raised when any question went unanswered. `missing` lists every such id. + # `refused` lists the ones the provider explicitly declined, a subset of + # `missing`. Answers that did arrive are on `answers`. class MissingAnswers < InvalidResponse - attr_reader :missing + attr_reader :missing, :refused - def initialize(message, answers: {}, missing: []) + def initialize(message, answers: {}, missing: [], refused: []) super(message, answers: answers) @missing = missing + @refused = refused end end end diff --git a/lib/ruby_decision_model/images.rb b/lib/ruby_decision_model/images.rb new file mode 100644 index 0000000..dcea4a6 --- /dev/null +++ b/lib/ruby_decision_model/images.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module RubyDecisionModel + # Builds the base64 data URLs that Client#ask takes as images:. Every + # provider that reads images wants them embedded; none fetch a remote URL. + module Images + CONTENT_TYPES = { + ".png" => "image/png", + ".jpg" => "image/jpeg", + ".jpeg" => "image/jpeg", + ".webp" => "image/webp", + ".gif" => "image/gif" + }.freeze + + module_function + + def data_url(bytes, content_type:) + raise ArgumentError, "content_type must be an image/* type, got #{content_type.inspect}" unless content_type.to_s.start_with?("image/") + + # pack("m0") is strict base64 without the base64 gem, which leaves the + # default gems in Ruby 3.4. + "data:#{content_type};base64,#{[bytes].pack('m0')}" + end + + # Reads a file and infers its type from the extension. + def from_file(path, content_type: nil) + content_type ||= CONTENT_TYPES[File.extname(path.to_s).downcase] + raise ArgumentError, "cannot infer an image type for #{path}; pass content_type:" if content_type.nil? + + data_url(File.binread(path), content_type: content_type) + end + end +end diff --git a/lib/ruby_decision_model/providers.rb b/lib/ruby_decision_model/providers.rb index 05a9eae..eedb69d 100644 --- a/lib/ruby_decision_model/providers.rb +++ b/lib/ruby_decision_model/providers.rb @@ -3,42 +3,76 @@ require_relative "providers/base" require_relative "providers/open_router" require_relative "providers/typesafe" +require_relative "providers/openai" +require_relative "providers/cloudflare" +require_relative "providers/perplexity" +require_relative "providers/databricks" +require_relative "providers/system_one" module RubyDecisionModel module Providers REGISTRY = { open_router: OpenRouter, - typesafe: Typesafe + typesafe: Typesafe, + openai: OpenAI, + cloudflare: Cloudflare, + perplexity: Perplexity, + databricks: Databricks, + system_one: SystemOne }.freeze + # Names a provider from the environment, ahead of any key sniffing. + PROVIDER_ENV_VAR = "RUBY_DECISION_MODEL_PROVIDER" + module_function def names REGISTRY.keys end + # Names are matched without regard to case, and `-` reads as `_`, so + # "OpenAI" and "open-router" from an env file both resolve. def build(name, api_key: nil, base_url: nil) - klass = REGISTRY[name.to_s.to_sym] + klass = REGISTRY[name.to_s.strip.downcase.tr("-", "_").to_sym] raise ConfigurationError, "unknown provider #{name.inspect}; known providers: #{names.join(', ')}" if klass.nil? klass.new(api_key: api_key, base_url: base_url) end # Order in which environment variables are consulted when no provider or - # api_key is given. Typesafe wins when both keys are set. - ENV_PRIORITY = [Typesafe, OpenRouter].freeze + # api_key is given. Typesafe wins when several are set. Only settings + # that exist for decision models take part; general-purpose credentials + # such as OPENAI_API_KEY or CLOUDFLARE_API_TOKEN never pick a provider + # on their own. Name those with RUBY_DECISION_MODEL_PROVIDER. + ENV_PRIORITY = [Typesafe, OpenRouter, SystemOne].freeze + + # The provider name RUBY_DECISION_MODEL_PROVIDER holds, or nil. + def named_in_env + named = ENV.fetch(PROVIDER_ENV_VAR, nil).to_s.strip + named.empty? ? nil : named + end - # Picks a provider from the environment, or nil when no key is set. + # Picks a provider from the environment, or nil when nothing is set. + # RUBY_DECISION_MODEL_PROVIDER wins when present, even if that provider + # turns out to be missing its key, so the error names the right thing. def from_env + named = named_in_env + return build(named) if named + ENV_PRIORITY.each do |klass| provider = klass.new - return provider if provider.api_key? + return provider if provider.configured? end nil end def env_vars - ENV_PRIORITY.map { |klass| klass.new.env_var } + [PROVIDER_ENV_VAR] + ENV_PRIORITY.map { |klass| env_var_for(klass) } + end + + def env_var_for(klass) + klass == SystemOne ? SystemOne::BASE_URL_ENV_VAR : klass.new.env_var end + private_class_method :env_var_for end end diff --git a/lib/ruby_decision_model/providers/base.rb b/lib/ruby_decision_model/providers/base.rb index 85f7089..bcf643a 100644 --- a/lib/ruby_decision_model/providers/base.rb +++ b/lib/ruby_decision_model/providers/base.rb @@ -6,8 +6,14 @@ module RubyDecisionModel module Providers # A provider owns everything that differs between decision-model APIs: # where requests go, how they are authenticated, which model is the - # default, which model names are aliases, and how usage is read back. - # Client keeps the public API and delegates these questions here. + # default, which model names are aliases, how the request is encoded, and + # how the response is read back. Client keeps the public API and + # delegates these questions here. + # + # The defaults speak the System One wire format that Typesafe published + # with Jev: questions keyed by id, answers keyed by id. A provider whose + # API differs overrides request_body and normalize_response, translating + # to and from that shape, so Client only ever sees System One answers. class Base attr_reader :api_key @@ -47,11 +53,39 @@ def reports_cost? false end + # Read timeout in seconds when Client gets timeout: nil. The open + # timeout stays at Client::DEFAULT_OPEN_TIMEOUT. + def default_timeout + 5 + end + + # Whether requests may carry images. Client refuses images: for a + # provider that says no, before anything goes over the wire. + def supports_images? + false + end + + # Response header carrying the provider's request id, or nil. + # Typesafe's, which 0.1.0 read for every provider and System One + # servers that follow Typesafe send too. Vendors with their own header + # override it. + def request_id_header + "x-typesafe-request-id" + end + + # Whether a request without an API key is meaningful. Self-hosted + # servers often run without auth. + def requires_api_key? + true + end + + # Values from env files often carry stray whitespace. def base_url - (@base_url || default_base_url).to_s.chomp("/") + (@base_url || default_base_url).to_s.strip.chomp("/") end - def url + # The model is passed for providers that put it in the path. + def url(_model = nil) "#{base_url}#{endpoint_path}" end @@ -59,6 +93,22 @@ def api_key? !(api_key.nil? || api_key.to_s.strip.empty?) end + # Raises ConfigurationError naming whatever is missing. + def validate! + return unless requires_api_key? && !api_key? + + raise ConfigurationError, "api_key is required for #{name}: pass api_key: or set #{env_var}" + end + + # True when validate! would pass. Used to pick a provider from the + # environment. + def configured? + validate! + true + rescue ConfigurationError + false + end + # Nil or blank means the provider default. Known aliases resolve to the # provider's canonical name. Anything else passes through untouched. def resolve_model(model) @@ -68,16 +118,47 @@ def resolve_model(model) end def headers - { - "Authorization" => "Bearer #{api_key}", + headers = { "Content-Type" => "application/json", "Accept" => "application/json", "User-Agent" => "ruby_decision_model/#{VERSION}" } - end - - def request_body(model:, state:, questions:) - JSON.generate({ "model" => model, "state" => state, "questions" => questions }) + # Keys read from files often end in a newline, which Net::HTTP + # rejects in a header. + headers["Authorization"] = "Bearer #{api_key.to_s.strip}" if api_key? + headers + end + + # Client passes images: only when the caller supplied some, so a + # subclass that does not take images can keep a three-keyword + # signature. + def request_body(model:, state:, questions:, images: nil) + body = { "model" => model, "state" => state, "questions" => questions } + body["images"] = images if images + JSON.generate(body) + end + + # Turns a parsed success body into the System One shape Client reads: + # a Hash with "answers" keyed by question id, plus "id", "model", and + # "usage". `questions` is the Hash the caller asked, for providers that + # need it to map answers back. Noul answers that arrive without a + # probability split get one. + def normalize_response(parsed, questions:) + fill_noul_probabilities(parsed) + end + + # The vendor's reason for a failed request, read from the error body, + # or nil. Appended to the ApiError message. Knows the shapes the + # supported APIs use: + # {"error": {"message": ...}} OpenAI, Perplexity, OpenRouter + # {"errors": [{"message": ...}]} Cloudflare + # {"detail": [{"loc", "msg"}]} Typesafe and other FastAPI servers + # {"message": ...} Databricks REST + def error_message(body) + parsed = JSON.parse(body.to_s) + parsed.is_a?(Hash) ? vendor_text(error_text(parsed)) : nil + rescue JSON::ParserError, EncodingError + nil end def usage(parsed) @@ -102,6 +183,70 @@ def configure(api_key: nil, base_url: nil) @base_url = base_url unless base_url.nil? self end + + private + + ERROR_MESSAGE_LIMIT = 500 + + def error_text(parsed) + error = parsed["error"] + return error if error.is_a?(String) + return error["message"].to_s if error.is_a?(Hash) && error["message"] + + errors = parsed["errors"] + if errors.is_a?(Array) && errors.any? + return errors.map { |item| item.is_a?(Hash) ? item["message"] || item.to_s : item.to_s }.join("; ") + end + + detail = parsed["detail"] + return detail if detail.is_a?(String) + return detail.map { |item| detail_text(item) }.join("; ") if detail.is_a?(Array) && detail.any? + + (parsed["message"] || parsed["error_message"])&.to_s + end + + def detail_text(item) + return item.to_s unless item.is_a?(Hash) + + location = Array(item["loc"]).join(".") + location.empty? ? item["msg"].to_s : "#{location}: #{item['msg']}" + end + + # Vendor-supplied text bound for an exception message, which tends to + # end up in logs: blank becomes nil, the API key is masked in case a + # vendor echoes it, and the length is capped. + def vendor_text(text) + # Net::HTTP hands back binary bodies; a proxy's error page can carry + # bytes that are not UTF-8, which strip and gsub would raise on. + text = text.to_s.dup.force_encoding(Encoding::UTF_8).scrub + # Control characters could forge log lines or drive a terminal when + # the vendor echoes caller input; each run becomes one space. + text = text.gsub(/[[:cntrl:]]+/, " ").strip + return nil if text.empty? + + key = api_key.to_s.strip + text = text.gsub(key, "[REDACTED]") if key.length >= 8 + text.length > ERROR_MESSAGE_LIMIT ? "#{text[0, ERROR_MESSAGE_LIMIT]}..." : text + end + + # Several APIs answer a noul with the probability alone. Fill in the + # two-way split Jev sends so Answers::Noul#probabilities reads the same + # everywhere. + def noul_probabilities(answer) + return answer unless answer.is_a?(Hash) && answer["type"] == "noul" + return answer if answer["probabilities"].is_a?(Hash) && !answer["probabilities"].empty? + + probability = answer["noul"] + return answer unless probability.is_a?(Numeric) + + answer.merge("probabilities" => { "true" => probability.to_f, "false" => 1.0 - probability.to_f }) + end + + def fill_noul_probabilities(parsed) + return parsed unless parsed.is_a?(Hash) && parsed["answers"].is_a?(Hash) + + parsed.merge("answers" => parsed["answers"].transform_values { |answer| noul_probabilities(answer) }) + end end end end diff --git a/lib/ruby_decision_model/providers/cloudflare.rb b/lib/ruby_decision_model/providers/cloudflare.rb new file mode 100644 index 0000000..de384ee --- /dev/null +++ b/lib/ruby_decision_model/providers/cloudflare.rb @@ -0,0 +1,119 @@ +# frozen_string_literal: true + +require_relative "base" + +module RubyDecisionModel + module Providers + # Cloudflare's Clef and Clef-flash on Workers AI. The body is System One + # with Cloudflare's images extension; the model also appears in the + # path, and the account id is required. Responses arrive inside the + # Workers AI envelope ({"result": ..., "success": true}), which is + # unwrapped here. + class Cloudflare < Base + ALIASES = { + "cloudflare/clef" => "clef", + "cloudflare/clef-flash" => "clef-flash", + "@cf/cloudflare/clef" => "clef", + "@cf/cloudflare/clef-flash" => "clef-flash" + }.freeze + + # Cloudflare's own tooling reads CLOUDFLARE_API_TOKEN; the Clef docs + # use CLOUDFLARE_AUTH_TOKEN. Either works, in that order. + FALLBACK_ENV_VAR = "CLOUDFLARE_AUTH_TOKEN" + ACCOUNT_ENV_VAR = "CLOUDFLARE_ACCOUNT_ID" + + # Both end up in the request path, so they are held to characters that + # cannot leave it. + ACCOUNT_ID_PATTERN = /\A[A-Za-z0-9_-]+\z/ + MODEL_PATTERN = /\A[A-Za-z0-9][A-Za-z0-9._-]*\z/ + + attr_reader :account_id + + def initialize(api_key: nil, base_url: nil, account_id: nil) + # A blank CLOUDFLARE_API_TOKEN must not hide CLOUDFLARE_AUTH_TOKEN. + token = [env_var, FALLBACK_ENV_VAR].filter_map { |name| ENV.fetch(name, nil) }.find { |v| !v.strip.empty? } + super(api_key: api_key || token, base_url: base_url) + @account_id = (account_id || ENV.fetch(ACCOUNT_ENV_VAR, nil))&.to_s&.strip + end + + def name + :cloudflare + end + + def env_var + "CLOUDFLARE_API_TOKEN" + end + + def default_base_url + "https://api.cloudflare.com/client/v4" + end + + def endpoint_path + "/accounts/#{account_id}/ai/run/@cf/cloudflare" + end + + # Checked here as well as at build time, since configure can change the + # account id on a provider a client already holds. + def url(model = nil) + model ||= default_model + unless account_id.to_s.match?(ACCOUNT_ID_PATTERN) && model.to_s.match?(MODEL_PATTERN) + raise ConfigurationError, + "cloudflare account_id and model must stay inside the URL path, got #{account_id.inspect} and #{model.inspect}" + end + + "#{base_url}#{endpoint_path}/#{model}" + end + + def default_model + "clef" + end + + def aliases + ALIASES + end + + def supports_images? + true + end + + def request_id_header + "cf-ray" + end + + def validate! + super + if account_id.nil? || account_id.to_s.strip.empty? + raise ConfigurationError, "account_id is required for cloudflare: pass account_id: or set #{ACCOUNT_ENV_VAR}" + end + return if account_id.to_s.match?(ACCOUNT_ID_PATTERN) + + raise ConfigurationError, "cloudflare account_id must be letters, digits, - or _, got #{account_id.inspect}" + end + + def resolve_model(model) + resolved = super + return resolved if resolved.match?(MODEL_PATTERN) + + raise ConfigurationError, "cloudflare model must be a bare Workers AI name such as clef, got #{model.inspect}" + end + + # Unwraps the Workers AI envelope. A body that says success: false + # carries no answers, so its errors become the exception message. + def normalize_response(parsed, questions:) + if parsed["success"] == false + detail = vendor_text(error_text(parsed)) + raise InvalidResponse, ["cloudflare reported failure", detail].compact.join(": ") + end + + body = parsed["result"].is_a?(Hash) ? parsed["result"] : parsed + super(body, questions: questions) + end + + def configure(api_key: nil, base_url: nil, account_id: nil) + super(api_key: api_key, base_url: base_url) + @account_id = account_id.to_s.strip unless account_id.nil? + self + end + end + end +end diff --git a/lib/ruby_decision_model/providers/databricks.rb b/lib/ruby_decision_model/providers/databricks.rb new file mode 100644 index 0000000..6a223d0 --- /dev/null +++ b/lib/ruby_decision_model/providers/databricks.rb @@ -0,0 +1,87 @@ +# frozen_string_literal: true + +require_relative "base" + +module RubyDecisionModel + module Providers + # Databricks ai_decide over REST (beta). Questions are System One, but + # the workspace serves one managed model, so the body has no model and + # Client#model is nil. Answers arrive under "response", and a noul + # carries "probability" where System One says "noul". Usage and ids are + # not reported. + class Databricks < Base + HOST_ENV_VAR = "DATABRICKS_HOST" + + def initialize(api_key: nil, base_url: nil) + super + @base_url ||= ENV.fetch(HOST_ENV_VAR, nil) + end + + def name + :databricks + end + + def env_var + "DATABRICKS_TOKEN" + end + + def default_base_url + nil + end + + # DATABRICKS_HOST is often set without a scheme. + def base_url + host = super + return host if host.empty? || host.match?(%r{\Ahttps?://}) + + "https://#{host}" + end + + # Beta serverless endpoint with no published latency figures. + def default_timeout + 30 + end + + def endpoint_path + "/api/2.0/ai-functions/ai-decide" + end + + def default_model + nil + end + + def resolve_model(model) + return nil if model.nil? || model.to_s.strip.empty? + + raise ConfigurationError, "databricks serves ai_decide's managed model and takes no model: (got #{model.inspect})" + end + + def validate! + super + return unless base_url.empty? + + raise ConfigurationError, "base_url is required for databricks: pass base_url: or set #{HOST_ENV_VAR}" + end + + def request_body(model:, state:, questions:) + JSON.generate("state" => state, "questions" => questions) + end + + def normalize_response(parsed, questions:) + if parsed["response"].nil? && parsed["error_message"] + raise InvalidResponse, "databricks ai_decide error: #{vendor_text(parsed['error_message'])}" + end + + response = parsed["response"].is_a?(Hash) ? parsed["response"] : {} + answers = response["answers"].is_a?(Hash) ? response["answers"] : {} + answers = answers.transform_values do |answer| + next answer unless answer.is_a?(Hash) && answer["type"] == "noul" && !answer.key?("noul") + + answer.merge("noul" => answer["probability"]) + end + + super({ "answers" => answers }, questions: questions) + end + end + end +end diff --git a/lib/ruby_decision_model/providers/open_router.rb b/lib/ruby_decision_model/providers/open_router.rb index 89ac172..16bce93 100644 --- a/lib/ruby_decision_model/providers/open_router.rb +++ b/lib/ruby_decision_model/providers/open_router.rb @@ -5,9 +5,17 @@ module RubyDecisionModel module Providers class OpenRouter < Base + # Short names resolve to the decision models OpenRouter routes, so the + # same model: works here and on each vendor's own provider. ALIASES = { "jev" => "typesafe/jev-1.13", - "jev-latest" => "typesafe/jev-1.13" + "jev-latest" => "typesafe/jev-1.13", + "luna" => "openai/gpt-6-luna-decisions", + "gpt-6-luna" => "openai/gpt-6-luna-decisions", + "clef" => "cloudflare/clef", + "clef-flash" => "cloudflare/clef-flash", + "pplx-decider" => "perplexity/pplx-decider-v1-27b", + "pplx-decider-v1-27b" => "perplexity/pplx-decider-v1-27b" }.freeze def name diff --git a/lib/ruby_decision_model/providers/openai.rb b/lib/ruby_decision_model/providers/openai.rb new file mode 100644 index 0000000..6d792d4 --- /dev/null +++ b/lib/ruby_decision_model/providers/openai.rb @@ -0,0 +1,237 @@ +# frozen_string_literal: true + +require_relative "base" + +module RubyDecisionModel + module Providers + # OpenAI's Decisions API (public beta, gpt-6-luna). Its wire format + # differs from System One in every direction, so this provider + # translates both ways: + # + # - state becomes `input`. Strings pass through; anything else is sent + # as JSON text, since the API has no structured input. Images become + # input_image parts on a single user message. + # - questions become a named array. noul is `predicate`; a noul with + # true/false criteria becomes a boolean `choice` so the descriptions + # reach the model, and its answer is read back as a noul. choice + # criteria become `choices`, score criteria become `levels`. + # - answers come back as an array of predicate/choice/score/refusal + # objects with probability arrays, and are rebuilt keyed by id with + # probability Hashes and a score legend. + class OpenAI < Base + ALIASES = { + "luna" => "gpt-6-luna", + "openai/gpt-6-luna" => "gpt-6-luna", + "openai/gpt-6-luna-decisions" => "gpt-6-luna", + "gpt-6-luna-decisions" => "gpt-6-luna" + }.freeze + + def name + :openai + end + + def env_var + "OPENAI_API_KEY" + end + + def default_base_url + "https://api.openai.com/v1" + end + + def endpoint_path + "/decisions" + end + + def default_model + "gpt-6-luna" + end + + def aliases + ALIASES + end + + # Inputs run to a million tokens, and OpenAI publishes no latency + # figures for this endpoint yet. + def default_timeout + 30 + end + + def supports_images? + true + end + + def request_id_header + "x-request-id" + end + + def request_body(model:, state:, questions:, images: nil) + JSON.generate( + "model" => model, + "input" => input(state, images), + "questions" => questions.map { |id, question| encode_question(id.to_s, question) } + ) + end + + # Answers are matched by name. An unnamed answer falls back to its + # position, since the API answers in question order, but never onto an + # id some other answer names. An id answered twice is ambiguous, so it + # reads as malformed rather than letting one answer (or a refusal) + # silently replace another. + def normalize_response(parsed, questions:) + by_id = questions.to_h { |id, question| [id.to_s, question] } + ids = by_id.keys + entries = Array(parsed["answers"]) + named = entries.filter_map { |answer| answer["name"] if answer.is_a?(Hash) && answer["name"].is_a?(String) } + answers = {} + + entries.each_with_index do |answer, index| + next unless answer.is_a?(Hash) + + id = answer["name"].is_a?(String) ? answer["name"] : ids[index] + next if id.nil? || (!answer["name"].is_a?(String) && named.include?(id)) + + answers[id] = answers.key?(id) ? ambiguous(by_id[id]) : decode_answer(answer, by_id[id]) + end + + { "id" => parsed["id"], "model" => parsed["model"], "answers" => answers, "usage" => parsed["usage"] } + end + + private + + def input(state, images) + text = state_text(state) + return text if images.nil? || images.empty? + + content = [] + content << { "type" => "input_text", "text" => text } unless text.empty? + images.each { |url| content << { "type" => "input_image", "image_url" => url } } + [{ "role" => "user", "content" => content }] + end + + def state_text(state) + case state + when String then state + when nil then "" + else JSON.generate(state) + end + end + + def encode_question(id, question) + question = stringify(question) + instructions = question["instructions"] + raise RequestError, "question #{id} needs instructions for openai" if instructions.nil? + + instructions = JSON.generate(instructions) unless instructions.is_a?(String) + criteria = question["criteria"] + + case question["type"] + when "noul" + return { "name" => id, "type" => "predicate", "instructions" => instructions } unless criteria.is_a?(Hash) + + descriptions = stringify(criteria) + { "name" => id, "type" => "choice", "instructions" => instructions, + "choices" => [true, false].map { |value| described({ "value" => value }, descriptions[value.to_s]) } } + when "choice" + { "name" => id, "type" => "choice", "instructions" => instructions, + "choices" => stringify(criteria).map { |value, text| described({ "value" => value }, text) } } + when "score" + { "name" => id, "type" => "score", "instructions" => instructions, + "levels" => Array(criteria).map { |level| encode_level(level) } } + else + question.merge("name" => id) + end + end + + # Jev score criteria are descriptions: usually strings, sometimes + # objects. A Hash with a label keeps its label and description; + # anything else becomes the label as text. + def encode_level(level) + level = stringify(level) if level.is_a?(Hash) + return described({ "label" => level["label"].to_s }, level["description"]) if level.is_a?(Hash) && level.key?("label") + return { "label" => level } if level.is_a?(String) + + { "label" => JSON.generate(level) } + end + + def described(hash, description) + return hash if description.nil? + + hash.merge("description" => description.is_a?(String) ? description : JSON.generate(description)) + end + + def decode_answer(answer, question) + question = stringify(question) + + case answer["type"] + when "predicate" + { "type" => "noul", "noul" => answer["probability"] } + when "choice" + question["type"] == "noul" ? decode_boolean_choice(answer) : decode_choice(answer) + when "score" + decode_score(answer, Array(question["criteria"])) + when "refusal" + { "type" => "refusal" } + else + answer + end.then { |decoded| noul_probabilities(decoded) } + end + + # Carries the expected type with none of its fields, so Client reports + # the id as malformed. + def ambiguous(question) + { "type" => stringify(question)["type"] } + end + + def decode_boolean_choice(answer) + return { "type" => "noul" } if duplicate_values?(answer) + + entry = probability_entries(answer).find { |item| item["value"] == true } + { "type" => "noul", "noul" => entry && entry["probability"] } + end + + def decode_choice(answer) + return { "type" => "choice" } if duplicate_values?(answer) + + probabilities = probability_entries(answer).to_h { |item| [item["value"].to_s, item["probability"]] } + choice = answer["choice"] + choice = choice.to_s if [true, false].include?(choice) + + { "type" => "choice", "choice" => choice, "confidence" => answer["confidence"], "probabilities" => probabilities } + end + + # The legend maps each level index to the criteria entry the caller + # wrote, as Jev does, falling back to the label OpenAI echoes. + def decode_score(answer, criteria) + return { "type" => "score" } if duplicate_values?(answer) + + entries = probability_entries(answer) + legend = entries.to_h do |item| + index = item["value"] + [index.to_s, index.is_a?(Integer) && index.between?(0, criteria.size - 1) ? criteria[index] : item["label"]] + end + + { "type" => "score", "score" => answer["score"], "confidence" => answer["confidence"], + "probabilities" => entries.to_h { |item| [item["value"].to_s, item["probability"]] }, + "legend" => legend } + end + + def probability_entries(answer) + Array(answer["probabilities"]).select { |item| item.is_a?(Hash) && item.key?("value") } + end + + # Two entries for one value make the split ambiguous. The answer then + # carries no fields and Client reports it as malformed. Values compare + # as the string keys the decoded probabilities use, so true and "true" + # (or 1 and "1") count as the same value rather than one silently + # replacing the other. + def duplicate_values?(answer) + values = probability_entries(answer).map { |item| item["value"].to_s } + values.uniq.size != values.size + end + + def stringify(hash) + hash.is_a?(Hash) ? hash.transform_keys(&:to_s) : {} + end + end + end +end diff --git a/lib/ruby_decision_model/providers/perplexity.rb b/lib/ruby_decision_model/providers/perplexity.rb new file mode 100644 index 0000000..1d9374b --- /dev/null +++ b/lib/ruby_decision_model/providers/perplexity.rb @@ -0,0 +1,73 @@ +# frozen_string_literal: true + +require_relative "base" + +module RubyDecisionModel + module Providers + # Perplexity's Decisions API serving pplx-decider. The body is System One. + # Images ride inside state as OpenAI-style image_url parts, so state + # becomes an array when images are present. + class Perplexity < Base + ALIASES = { + "pplx-decider" => "pplx-decider-v1.1-27b", + "perplexity/pplx-decider-v1-27b" => "pplx-decider-v1-27b" + }.freeze + + def name + :perplexity + end + + def env_var + "PERPLEXITY_API_KEY" + end + + def default_base_url + "https://api.perplexity.ai" + end + + def endpoint_path + "/v1/decisions" + end + + def default_model + "pplx-decider-v1.1-27b" + end + + def aliases + ALIASES + end + + # Perplexity documents 5 to 23 seconds for large inputs and uses 30 in + # its own examples. + def default_timeout + 30 + end + + def supports_images? + true + end + + def request_id_header + "x-request-id" + end + + def request_body(model:, state:, questions:, images: nil) + return super(model: model, state: state, questions: questions) if images.nil? || images.empty? + + parts = images.map { |url| { "type" => "image_url", "image_url" => { "url" => url } } } + super(model: model, state: state_parts(state) + parts, questions: questions) + end + + private + + def state_parts(state) + case state + when nil then [] + when String then state.empty? ? [] : [state] + when Array then state + else [JSON.generate(state)] + end + end + end + end +end diff --git a/lib/ruby_decision_model/providers/system_one.rb b/lib/ruby_decision_model/providers/system_one.rb new file mode 100644 index 0000000..1767666 --- /dev/null +++ b/lib/ruby_decision_model/providers/system_one.rb @@ -0,0 +1,90 @@ +# frozen_string_literal: true + +require "uri" +require_relative "base" + +module RubyDecisionModel + module Providers + # Any server that speaks Typesafe's System One API at /v1/systemone: + # Ollama, the autojev server that ships with pplx-decider's weights, + # strands-decider's local server, and hosted lookalikes. The base URL is + # required and has no /v1 suffix. The API key and model are optional, + # since local servers often run without auth and some pick their own + # model. SYSTEM_ONE_BASE_URL and SYSTEM_ONE_API_KEY follow Pydantic AI. + class SystemOne < Base + BASE_URL_ENV_VAR = "SYSTEM_ONE_BASE_URL" + + def initialize(api_key: nil, base_url: nil) + super + @base_url ||= ENV.fetch(BASE_URL_ENV_VAR, nil) + end + + def name + :system_one + end + + def env_var + "SYSTEM_ONE_API_KEY" + end + + def default_base_url + nil + end + + def endpoint_path + "/v1/systemone" + end + + def default_model + nil + end + + def requires_api_key? + false + end + + # A local server may load the model on the first request. + def default_timeout + 30 + end + + # Sent as the System One images extension; servers without image + # support reject the request. + def supports_images? + true + end + + def validate! + super + return unless base_url.empty? + + raise ConfigurationError, "base_url is required for system_one: pass base_url: or set #{BASE_URL_ENV_VAR}" + end + + # Checked when a request is built rather than in validate!, so a + # client's own base_url: can still replace an unusable + # SYSTEM_ONE_BASE_URL. Ollama hosts are often written as + # localhost:11434, and whether the server speaks http or https is not + # ours to guess; an empty host would quietly mean this machine. + def url(model = nil) + uri = URI.parse(base_url) + unless uri.is_a?(URI::HTTP) && !uri.host.to_s.empty? + raise ConfigurationError, "system_one base_url needs http:// or https:// and a host, got #{base_url.inspect}" + end + + super + rescue URI::InvalidURIError + raise ConfigurationError, "system_one base_url is not a URL: #{base_url.inspect}" + end + + def request_body(model:, state:, questions:, images: nil) + body = {} + body["model"] = model unless model.nil? + body["state"] = state + body["questions"] = questions + body["images"] = images if images + JSON.generate(body) + end + end + end +end diff --git a/lib/ruby_decision_model/providers/typesafe.rb b/lib/ruby_decision_model/providers/typesafe.rb index c8a1ee7..9002d25 100644 --- a/lib/ruby_decision_model/providers/typesafe.rb +++ b/lib/ruby_decision_model/providers/typesafe.rb @@ -7,6 +7,7 @@ module Providers class Typesafe < Base ALIASES = { "typesafe/jev-1.13" => "jev-latest", + "~typesafe/jev-latest" => "jev-latest", "jev" => "jev-latest" }.freeze diff --git a/lib/ruby_decision_model/retry_policy.rb b/lib/ruby_decision_model/retry_policy.rb index 00774c5..6194635 100644 --- a/lib/ruby_decision_model/retry_policy.rb +++ b/lib/ruby_decision_model/retry_policy.rb @@ -21,7 +21,7 @@ class RetryPolicy < Data.define( ) DEFAULT_STATUSES = ([408, 429] + (500..599).to_a).freeze - TIMEOUT_EXCEPTIONS = [Net::OpenTimeout, Net::ReadTimeout].freeze + TIMEOUT_EXCEPTIONS = [Net::OpenTimeout, Net::ReadTimeout, Net::WriteTimeout].freeze CONNECTION_EXCEPTIONS = [ Errno::ECONNRESET, Errno::ECONNREFUSED, diff --git a/test/client_test.rb b/test/client_test.rb index de5e058..c7e32c8 100644 --- a/test/client_test.rb +++ b/test/client_test.rb @@ -125,6 +125,19 @@ def test_500_raises_api_error assert_equal 500, error.status end + # Value: protects=529 maps to Overloaded and carries the vendor error text in the message + # Value: fails_when=ERROR_CLASSES loses 529 or the detail append is skipped for non-4xx statuses + # Value: why_new=no existing test hits 529; only 401, 413, 422, 429, 500 are asserted + # Value: seam=none + def test_529_raises_overloaded_with_vendor_detail_after_retries + body = '{"error":{"message":"model is busy"}}' + transport = FakeTransport.new([[529, body]]) + error = assert_raises(RubyDecisionModel::Overloaded) { build_client(transport).ask(state: {}, questions: questions) } + assert_equal 529, error.status + assert_includes error.message, "overloaded: model is busy" + assert_equal body, error.body + end + # --- retry --- def test_retries_once_on_503_then_succeeds @@ -303,14 +316,78 @@ def test_non_hash_probabilities_fall_back_to_empty_hash body = JSON.generate( "id" => "resp_10", "model" => "typesafe/jev-1.13", - "answers" => { "urgent" => { "type" => "noul", "noul" => 0.5, "probabilities" => %w[not a hash] } }, + "answers" => { "category" => { "type" => "choice", "choice" => "bug", "confidence" => 0.9, + "probabilities" => %w[not a hash] } }, + "usage" => {} + ) + transport = FakeTransport.new([[200, body]]) + client = build_client(transport) + + response = client.ask(state: {}, questions: { "category" => questions["category"] }) + assert_equal({}, response["category"].probabilities) + end + + def test_noul_outside_zero_to_one_is_malformed + [1.7, -0.1, 1.0001].each do |value| + body = JSON.generate("answers" => { "urgent" => { "type" => "noul", "noul" => value } }) + client = build_client(FakeTransport.new([[200, body]])) + + error = assert_raises(RubyDecisionModel::InvalidResponse, value.to_s) do + client.ask(state: {}, questions: { "urgent" => questions["urgent"] }) + end + refute_kind_of RubyDecisionModel::MissingAnswers, error + end + + [0, 1].each do |value| + body = JSON.generate("answers" => { "urgent" => { "type" => "noul", "noul" => value } }) + response = build_client(FakeTransport.new([[200, body]])).ask(state: {}, questions: { "urgent" => questions["urgent"] }) + assert_in_delta value, response["urgent"].noul + end + end + + def test_deeply_nested_state_raises_request_error + state = JSON.parse(("[" * 100) + ("]" * 100)) + transport = FakeTransport.new([[200, success_body]]) + + error = assert_raises(RubyDecisionModel::RequestError) do + build_client(transport).ask(state: { "nested" => state }, questions: questions) + end + assert_kind_of JSON::NestingError, error.cause + assert_empty transport.calls + end + + def test_state_with_invalid_utf8_raises_request_error_before_sending + transport = FakeTransport.new([[200, success_body]]) + + assert_raises(RubyDecisionModel::RequestError) do + build_client(transport).ask(state: { "title" => "bad \xFF".b.force_encoding("UTF-8") }, questions: questions) + end + assert_empty transport.calls + end + + def test_noul_without_probabilities_gets_a_true_false_split + body = JSON.generate( + "model" => "jev-1.13.0", + "answers" => { "urgent" => { "type" => "noul", "noul" => 0.75 } }, "usage" => {} ) transport = FakeTransport.new([[200, body]]) client = build_client(transport) response = client.ask(state: {}, questions: { "urgent" => questions["urgent"] }) - assert_equal({}, response["urgent"].probabilities) + assert_equal({ "true" => 0.75, "false" => 0.25 }, response["urgent"].probabilities) + assert_nil response.id + end + + def test_noul_with_junk_probabilities_gets_a_true_false_split + body = JSON.generate( + "answers" => { "urgent" => { "type" => "noul", "noul" => 0.5, "probabilities" => %w[not a hash] } } + ) + transport = FakeTransport.new([[200, body]]) + client = build_client(transport) + + response = client.ask(state: {}, questions: { "urgent" => questions["urgent"] }) + assert_equal({ "true" => 0.5, "false" => 0.5 }, response["urgent"].probabilities) end def test_junk_usage_fields_become_nil diff --git a/test/openai_provider_test.rb b/test/openai_provider_test.rb new file mode 100644 index 0000000..c04a173 --- /dev/null +++ b/test/openai_provider_test.rb @@ -0,0 +1,357 @@ +# frozen_string_literal: true + +require "test_helper" + +class OpenAIProviderTest < Minitest::Test + Q = RubyDecisionModel::Questions + + def questions + { + "urgent" => Q.noul("Is this urgent?"), + "department" => Q.choice("Which department?", criteria: { "billing" => "Charges and refunds", "other" => nil }), + "severity" => Q.score("How severe?", criteria: ["Cosmetic", "Workaround", "Blocked"]) + } + end + + # Shapes follow the OpenAI Decisions guide and API reference. + def success_body + JSON.generate( + "model" => "gpt-6-luna", + "answers" => [ + { "type" => "predicate", "name" => "urgent", "probability" => 0.92 }, + { "type" => "choice", "name" => "department", "choice" => "billing", "confidence" => 0.93, + "probabilities" => [{ "value" => "billing", "probability" => 0.95 }, + { "value" => "other", "probability" => 0.05 }] }, + { "type" => "score", "name" => "severity", "score" => 1.1, "confidence" => 0.55, + "probabilities" => [{ "value" => 0, "label" => "Cosmetic", "probability" => 0.1 }, + { "value" => 1, "label" => "Workaround", "probability" => 0.7 }, + { "value" => 2, "label" => "Blocked", "probability" => 0.2 }] } + ], + "usage" => { "input_tokens" => 40, "input_tokens_details" => { "cached_tokens" => 0 }, + "output_tokens" => 0, "total_tokens" => 40 } + ) + end + + def client(transport, **options) + RubyDecisionModel::Client.new(provider: :openai, api_key: "sk-test", transport: transport, sleeper: no_sleep, + **options) + end + + def test_defaults + provider = RubyDecisionModel::Providers::OpenAI.new(api_key: "k") + assert_equal "https://api.openai.com/v1/decisions", provider.url + assert_equal "gpt-6-luna", provider.resolve_model(nil) + assert_equal "OPENAI_API_KEY", provider.env_var + end + + def test_aliases + provider = RubyDecisionModel::Providers::OpenAI.new(api_key: "k") + assert_equal "gpt-6-luna", provider.resolve_model("luna") + assert_equal "gpt-6-luna", provider.resolve_model("openai/gpt-6-luna-decisions") + assert_equal "gpt-6-luna-2026-10-06", provider.resolve_model("gpt-6-luna-2026-10-06") + end + + def test_request_translates_questions_and_state + transport = FakeTransport.new([[200, success_body, {}]]) + client(transport).ask(state: { title: "Charged twice" }, questions: questions) + + call = transport.calls.first + assert_equal "https://api.openai.com/v1/decisions", call[:url] + assert_equal "Bearer sk-test", call[:headers]["Authorization"] + + body = JSON.parse(call[:body]) + assert_equal %w[model input questions], body.keys + assert_equal "gpt-6-luna", body["model"] + assert_equal '{"title":"Charged twice"}', body["input"] + assert_equal( + [ + { "name" => "urgent", "type" => "predicate", "instructions" => "Is this urgent?" }, + { "name" => "department", "type" => "choice", "instructions" => "Which department?", + "choices" => [{ "value" => "billing", "description" => "Charges and refunds" }, { "value" => "other" }] }, + { "name" => "severity", "type" => "score", "instructions" => "How severe?", + "levels" => [{ "label" => "Cosmetic" }, { "label" => "Workaround" }, { "label" => "Blocked" }] } + ], + body["questions"] + ) + end + + def test_string_state_passes_through_as_input + transport = FakeTransport.new([[200, success_body, {}]]) + client(transport).ask(state: "I was charged twice.", questions: questions) + + assert_equal "I was charged twice.", JSON.parse(transport.calls.first[:body])["input"] + end + + def test_noul_with_criteria_becomes_a_boolean_choice_and_reads_back_as_a_noul + body = JSON.generate( + "model" => "gpt-6-luna", + "answers" => [{ "type" => "choice", "name" => "urgent", "choice" => true, "confidence" => 0.8, + "probabilities" => [{ "value" => true, "probability" => 0.85 }, + { "value" => false, "probability" => 0.15 }] }], + "usage" => { "input_tokens" => 1, "output_tokens" => 0 } + ) + transport = FakeTransport.new([[200, body, {}]]) + question = Q.noul("Is this urgent?", criteria: { true => "Explicitly time-sensitive", "false" => "No urgency" }) + + response = client(transport).ask(state: "x", questions: { "urgent" => question }) + + sent = JSON.parse(transport.calls.first[:body])["questions"].first + assert_equal "choice", sent["type"] + assert_equal [{ "value" => true, "description" => "Explicitly time-sensitive" }, + { "value" => false, "description" => "No urgency" }], sent["choices"] + + assert_instance_of RubyDecisionModel::Answers::Noul, response["urgent"] + assert_in_delta 0.85, response["urgent"].noul + end + + def test_response_is_rebuilt_keyed_by_question_id + transport = FakeTransport.new([[200, success_body, { "x-request-id" => "req_abc" }]]) + response = client(transport).ask(state: "x", questions: questions) + + assert_in_delta 0.92, response["urgent"].noul + assert_equal({ "true" => 0.92, "false" => 0.08 }, response["urgent"].probabilities.transform_values { _1.round(2) }) + + department = response["department"] + assert_equal "billing", department.choice + assert_in_delta 0.93, department.confidence + assert_equal({ "billing" => 0.95, "other" => 0.05 }, department.probabilities) + + severity = response["severity"] + assert_in_delta 1.1, severity.score + assert_equal({ "0" => 0.1, "1" => 0.7, "2" => 0.2 }, severity.probabilities) + assert_equal({ "0" => "Cosmetic", "1" => "Workaround", "2" => "Blocked" }, severity.legend) + + assert_equal "gpt-6-luna", response.model + assert_nil response.id + assert_equal "req_abc", response.request_id + assert_equal 40, response.usage.input_tokens + assert_equal 0, response.usage.output_tokens + assert_nil response.usage.cost + assert_kind_of Array, response.raw["answers"] + end + + def test_symbol_question_ids + transport = FakeTransport.new([[200, success_body, {}]]) + response = client(transport).ask(state: "x", questions: questions.transform_keys(&:to_sym)) + + assert_equal "billing", response["department"].choice + assert_equal "urgent", JSON.parse(transport.calls.first[:body])["questions"].first["name"] + end + + def test_refusal_raises_missing_answers_naming_the_refused_question + body = JSON.generate( + "model" => "gpt-6-luna", + "answers" => [{ "type" => "predicate", "name" => "urgent", "probability" => 0.4 }, + { "type" => "refusal", "name" => "department" }], + "usage" => { "input_tokens" => 1, "output_tokens" => 0 } + ) + transport = FakeTransport.new([[200, body, {}]]) + asked = questions.slice("urgent", "department") + + error = assert_raises(RubyDecisionModel::MissingAnswers) { client(transport).ask(state: "x", questions: asked) } + assert_equal ["department"], error.missing + assert_equal ["department"], error.refused + assert_includes error.message, "refused: department" + assert_in_delta 0.4, error.answers["urgent"].noul + end + + def test_unnamed_answers_match_questions_by_position + body = JSON.generate( + "model" => "gpt-6-luna", + "answers" => [{ "type" => "predicate", "name" => nil, "probability" => 0.3 }], + "usage" => {} + ) + transport = FakeTransport.new([[200, body, {}]]) + + response = client(transport).ask(state: "x", questions: questions.slice("urgent")) + assert_in_delta 0.3, response["urgent"].noul + end + + def test_answers_without_name_keys_map_back_by_position_and_strays_are_ignored + body = JSON.generate( + "model" => "gpt-6-luna", + "answers" => [ + { "type" => "predicate", "probability" => 0.6 }, + { "type" => "choice", "choice" => "billing", "confidence" => 0.9, + "probabilities" => [{ "value" => "billing", "probability" => 0.9 }, { "value" => "other", "probability" => 0.1 }] }, + { "type" => "score", "score" => 1.0, "confidence" => 0.5, + "probabilities" => [{ "value" => 1, "label" => "Workaround", "probability" => 1.0 }] }, + "not an answer", + { "type" => "mystery", "name" => "unasked", "value" => 1 } + ], + "usage" => {} + ) + + response = client(FakeTransport.new([[200, body, {}]])).ask(state: "x", questions: questions) + + assert_in_delta 0.6, response["urgent"].noul + assert_equal "billing", response["department"].choice + assert_equal({ "1" => "Workaround" }, response["severity"].legend) + assert_equal %w[department severity urgent], response.answers.keys.sort + end + + def test_an_unnamed_answer_never_lands_on_an_id_another_answer_names + body = JSON.generate( + "model" => "gpt-6-luna", + "answers" => [{ "type" => "predicate", "name" => "department", "probability" => 0.9 }, + { "type" => "predicate", "probability" => 0.1 }, + { "type" => "predicate", "name" => "urgent", "probability" => 0.5 }], + "usage" => {} + ) + asked = { "urgent" => Q.noul("Urgent?"), "department" => Q.noul("Billing?") } + + response = client(FakeTransport.new([[200, body, {}]])).ask(state: "x", questions: asked) + + assert_in_delta 0.5, response["urgent"].noul + assert_in_delta 0.9, response["department"].noul + end + + def test_an_id_answered_twice_is_malformed_not_overwritten + body = JSON.generate( + "model" => "gpt-6-luna", + "answers" => [{ "type" => "refusal", "name" => "urgent" }, + { "type" => "predicate", "name" => "urgent", "probability" => 0.99 }], + "usage" => {} + ) + + error = assert_raises(RubyDecisionModel::InvalidResponse) do + client(FakeTransport.new([[200, body, {}]])).ask(state: "x", questions: questions.slice("urgent")) + end + refute_kind_of RubyDecisionModel::MissingAnswers, error + assert_includes error.message, "urgent" + end + + def test_duplicate_probability_values_are_malformed + duplicate_true = { "type" => "choice", "name" => "urgent", "choice" => true, "confidence" => 0.5, + "probabilities" => [{ "value" => true, "probability" => 0.1 }, + { "value" => true, "probability" => 0.9 }, + { "value" => false, "probability" => 0.0 }] } + duplicate_choice = { "type" => "choice", "name" => "department", "choice" => "billing", "confidence" => 0.5, + "probabilities" => [{ "value" => "billing", "probability" => 0.2 }, + { "value" => "billing", "probability" => 0.8 }] } + mixed_types = { "type" => "score", "name" => "severity", "score" => 1.0, "confidence" => 0.5, + "probabilities" => [{ "value" => 1, "label" => "Workaround", "probability" => 0.9 }, + { "value" => "1", "label" => "Workaround", "probability" => 0.1 }] } + { + "urgent" => [duplicate_true, Q.noul("Urgent?", criteria: { "true" => "yes", "false" => "no" })], + "department" => [duplicate_choice, questions["department"]], + "severity" => [mixed_types, questions["severity"]] + }.each do |id, (answer, question)| + body = JSON.generate("model" => "gpt-6-luna", "answers" => [answer], "usage" => {}) + error = assert_raises(RubyDecisionModel::InvalidResponse, id) do + client(FakeTransport.new([[200, body, {}]])).ask(state: "x", questions: { id => question }) + end + refute_kind_of RubyDecisionModel::MissingAnswers, error, id + end + end + + def test_state_that_cannot_be_encoded_raises_request_error + transport = FakeTransport.new([[200, success_body, {}]]) + + error = assert_raises(RubyDecisionModel::RequestError) do + client(transport).ask(state: { score: Float::NAN }, questions: questions) + end + assert_kind_of JSON::GeneratorError, error.cause + assert_empty transport.calls + end + + def test_images_become_input_image_parts + transport = FakeTransport.new([[200, success_body, {}]]) + image = RubyDecisionModel::Images.data_url("\x89PNG".b, content_type: "image/png") + + client(transport).ask(state: "Inspect the photo.", questions: questions, images: [image]) + + input = JSON.parse(transport.calls.first[:body])["input"] + assert_equal( + [{ "role" => "user", "content" => [{ "type" => "input_text", "text" => "Inspect the photo." }, + { "type" => "input_image", "image_url" => image }] }], + input + ) + end + + def test_images_without_state_send_only_image_parts + transport = FakeTransport.new([[200, success_body, {}]]) + image = RubyDecisionModel::Images.data_url("GIF89a".b, content_type: "image/gif") + + client(transport).ask(state: nil, questions: questions, images: [image]) + + content = JSON.parse(transport.calls.first[:body])["input"].first["content"] + assert_equal ["input_image"], content.map { _1["type"] } + end + + def test_question_without_instructions_raises_before_sending + transport = FakeTransport.new([[200, success_body, {}]]) + + error = assert_raises(RubyDecisionModel::RequestError) do + client(transport).ask(state: "x", questions: { "urgent" => { "type" => "noul" } }) + end + assert_includes error.message, "urgent" + assert_empty transport.calls + end + + # Value: protects=boolean choice values and out-of-range score indexes decode to usable strings/labels + # Value: fails_when=decode_choice stops stringifying true/false or decode_score indexes criteria out of range + # Value: why_new=existing tests only cover string choices and in-range score indexes + # Value: seam=none + def test_boolean_choice_values_are_stringified_and_out_of_range_score_index_uses_the_echoed_label + body = JSON.generate( + "model" => "gpt-6-luna", + "answers" => [ + { "type" => "choice", "name" => "flag", "choice" => true, "confidence" => 0.7, + "probabilities" => [{ "value" => true, "probability" => 0.7 }, { "value" => false, "probability" => 0.3 }] }, + { "type" => "score", "name" => "severity", "score" => 4.0, "confidence" => 0.5, + "probabilities" => [{ "value" => 4, "label" => "Catastrophic", "probability" => 1.0 }] } + ] + ) + qs = { "flag" => Q.choice("Flag?", criteria: { "true" => nil, "false" => nil }), + "severity" => Q.score("How severe?", criteria: %w[Low High]) } + response = client(FakeTransport.new([[200, body, {}]])).ask(state: "x", questions: qs) + + assert_equal "true", response["flag"].choice + assert_equal({ "true" => 0.7, "false" => 0.3 }, response["flag"].probabilities) + assert_equal({ "4" => "Catastrophic" }, response["severity"].legend) + end + + # Value: protects=non-string instructions are sent as JSON text so the API accepts them + # Value: fails_when=encode_question forwards a Hash instructions value unchanged + # Value: why_new=every existing test uses string instructions + # Value: seam=none + def test_non_string_instructions_are_sent_as_json_text + transport = FakeTransport.new([[200, success_body, {}]]) + qs = { "urgent" => { type: "noul", instructions: { "ask" => "Is this urgent?" } } } + response = client(transport).ask(state: "x", questions: qs) + + assert_in_delta 0.92, response["urgent"].noul + sent = JSON.parse(transport.calls.first[:body])["questions"].first + assert_equal "predicate", sent["type"] + assert_equal '{"ask":"Is this urgent?"}', sent["instructions"] + end + + def test_malformed_predicate_or_boolean_choice_raises_invalid_response + { + "predicate without probability" => { "type" => "predicate", "name" => "urgent" }, + "boolean choice without a true entry" => { "type" => "choice", "name" => "urgent", "choice" => false, + "confidence" => 1.0, + "probabilities" => [{ "value" => false, "probability" => 1.0 }] } + }.each do |label, answer| + body = JSON.generate("model" => "gpt-6-luna", "answers" => [answer], "usage" => {}) + question = label.start_with?("boolean") ? Q.noul("Urgent?", criteria: { "true" => "yes", "false" => "no" }) : Q.noul("Urgent?") + + error = assert_raises(RubyDecisionModel::InvalidResponse, label) do + client(FakeTransport.new([[200, body, {}]])).ask(state: "x", questions: { "urgent" => question }) + end + refute_kind_of RubyDecisionModel::MissingAnswers, error, label + assert_includes error.message, "urgent", label + end + end + + def test_score_levels_with_label_and_description + transport = FakeTransport.new([[200, success_body, {}]]) + levels = [{ label: "Cosmetic", description: "Appearance only" }, { "label" => "Blocked" }] + + client(transport).ask(state: "x", questions: { "severity" => Q.score("How severe?", criteria: levels) }) + + sent = JSON.parse(transport.calls.first[:body])["questions"].first["levels"] + assert_equal [{ "label" => "Cosmetic", "description" => "Appearance only" }, { "label" => "Blocked" }], sent + end +end diff --git a/test/providers_test.rb b/test/providers_test.rb index 1ab71b2..14ee9d4 100644 --- a/test/providers_test.rb +++ b/test/providers_test.rb @@ -159,6 +159,24 @@ def test_typesafe_aliases assert_equal "jev-1.12", provider.resolve_model("jev-1.12") end + def test_open_router_aliases_for_other_vendors_models + provider = RubyDecisionModel::Providers::OpenRouter.new(api_key: "k") + assert_equal "openai/gpt-6-luna-decisions", provider.resolve_model("luna") + assert_equal "openai/gpt-6-luna-decisions", provider.resolve_model("gpt-6-luna") + assert_equal "cloudflare/clef", provider.resolve_model("clef") + assert_equal "cloudflare/clef-flash", provider.resolve_model("clef-flash") + assert_equal "perplexity/pplx-decider-v1-27b", provider.resolve_model("pplx-decider") + assert_equal "liquid/d1", provider.resolve_model("liquid/d1") + end + + def test_short_names_work_on_the_vendors_own_provider + assert_equal "gpt-6-luna", RubyDecisionModel::Providers::OpenAI.new(api_key: "k").resolve_model("luna") + cloudflare = RubyDecisionModel::Providers::Cloudflare.new(api_key: "k", account_id: "a") + assert_equal "clef-flash", cloudflare.resolve_model("clef-flash") + assert_equal "clef", cloudflare.resolve_model("cloudflare/clef") + assert_equal "jev-latest", RubyDecisionModel::Providers::Typesafe.new(api_key: "k").resolve_model("~typesafe/jev-latest") + end + def test_client_model_is_resolved_after_aliasing client = RubyDecisionModel::Client.new(provider: :typesafe, api_key: "k", model: "jev", transport: FakeTransport.new([])) @@ -245,6 +263,345 @@ def test_response_model_is_passed_through_as_returned assert_equal "jev-1.13", response.model end + # --- environment selection beyond the first two providers --- + + def test_provider_env_var_names_a_provider + without_provider_env do + with_env("RUBY_DECISION_MODEL_PROVIDER" => "openai", "OPENAI_API_KEY" => "sk-env", "TYPESAFE_API_KEY" => "ts") do + client = RubyDecisionModel::Client.new(transport: FakeTransport.new([])) + assert_equal :openai, client.provider.name + assert_equal "sk-env", client.provider.api_key + assert_equal "gpt-6-luna", client.model + end + end + end + + def test_provider_env_var_without_that_providers_key_names_the_key + without_provider_env do + with_env("RUBY_DECISION_MODEL_PROVIDER" => "perplexity", "TYPESAFE_API_KEY" => "ts") do + error = assert_raises(RubyDecisionModel::ConfigurationError) do + RubyDecisionModel::Client.new(transport: FakeTransport.new([])) + end + assert_includes error.message, "PERPLEXITY_API_KEY" + end + end + end + + def test_provider_env_var_with_unknown_name_raises + without_provider_env do + with_env("RUBY_DECISION_MODEL_PROVIDER" => "nope") do + assert_raises(RubyDecisionModel::ConfigurationError) do + RubyDecisionModel::Client.new(transport: FakeTransport.new([])) + end + end + end + end + + def test_general_purpose_keys_do_not_pick_a_provider + without_provider_env do + with_env("OPENAI_API_KEY" => "sk", "PERPLEXITY_API_KEY" => "pplx", "CLOUDFLARE_API_TOKEN" => "cf", + "CLOUDFLARE_ACCOUNT_ID" => "acc", "DATABRICKS_HOST" => "h", "DATABRICKS_TOKEN" => "t") do + error = assert_raises(RubyDecisionModel::ConfigurationError) do + RubyDecisionModel::Client.new(transport: FakeTransport.new([])) + end + assert_includes error.message, "RUBY_DECISION_MODEL_PROVIDER" + assert_includes error.message, "SYSTEM_ONE_BASE_URL" + end + end + end + + def test_open_router_beats_a_system_one_base_url + without_provider_env do + with_env("OPENROUTER_API_KEY" => "or", "SYSTEM_ONE_BASE_URL" => "http://localhost:11434") do + assert_equal :open_router, RubyDecisionModel::Client.new(transport: FakeTransport.new([])).provider.name + end + end + end + + def test_every_registered_provider_builds + RubyDecisionModel::Providers.names.each do |name| + provider = RubyDecisionModel::Providers.build(name, api_key: "k", base_url: "https://example.test") + assert_equal name, provider.name + end + end + + # --- images --- + + def test_images_are_refused_by_providers_that_do_not_read_them + transport = FakeTransport.new([[200, success_body, {}]]) + client = RubyDecisionModel::Client.new(provider: :typesafe, api_key: "k", transport: transport, sleeper: no_sleep) + image = RubyDecisionModel::Images.data_url("x", content_type: "image/png") + + error = assert_raises(RubyDecisionModel::RequestError) { client.ask(state: {}, questions: questions, images: [image]) } + assert_includes error.message, "typesafe" + assert_empty transport.calls + end + + def test_images_must_be_data_urls + transport = FakeTransport.new([[200, success_body, {}]]) + client = RubyDecisionModel::Client.new(provider: :openai, api_key: "k", transport: transport, sleeper: no_sleep) + + assert_raises(RubyDecisionModel::RequestError) do + client.ask(state: {}, questions: questions, images: ["https://example.test/cat.png"]) + end + assert_empty transport.calls + end + + def test_empty_images_are_ignored + transport = FakeTransport.new([[200, success_body, {}]]) + client = RubyDecisionModel::Client.new(provider: :typesafe, api_key: "k", transport: transport, sleeper: no_sleep) + + client.ask(state: {}, questions: questions, images: []) + refute JSON.parse(transport.calls.first[:body]).key?("images") + end + + def test_image_data_url_and_from_file + assert_equal "data:image/png;base64,aGk=", RubyDecisionModel::Images.data_url("hi", content_type: "image/png") + assert_raises(ArgumentError) { RubyDecisionModel::Images.data_url("hi", content_type: "text/plain") } + + Dir.mktmpdir do |dir| + path = File.join(dir, "photo.JPG") + File.binwrite(path, "\xFF\xD8".b) + assert_equal "data:image/jpeg;base64,/9g=", RubyDecisionModel::Images.from_file(path) + + other = File.join(dir, "photo.bmp") + File.binwrite(other, "BM") + assert_raises(ArgumentError) { RubyDecisionModel::Images.from_file(other) } + assert_equal "data:image/bmp;base64,Qk0=", RubyDecisionModel::Images.from_file(other, content_type: "image/bmp") + end + end + + # --- timeouts --- + + def test_timeout_defaults_come_from_the_provider + build = lambda do |name, **options| + RubyDecisionModel::Client.new(provider: name, api_key: "k", transport: FakeTransport.new([]), **options) + end + + assert_equal 5, build.call(:open_router).timeout + assert_equal 5, build.call(:typesafe).timeout + assert_equal 30, build.call(:perplexity).timeout + assert_equal 30, build.call(:openai).timeout + assert_equal 30, build.call(:system_one, base_url: "http://localhost:11434").timeout + assert_equal 12, build.call(:perplexity, timeout: 12).timeout + end + + # --- vendor error messages --- + + def error_for(body, status: 400, provider: :open_router) + transport = FakeTransport.new([[status, body, {}]]) + client = RubyDecisionModel::Client.new(provider: provider, api_key: "k", transport: transport, sleeper: no_sleep, + retry: { max_retries: 0 }) + assert_raises(RubyDecisionModel::ApiError) { client.ask(state: {}, questions: questions) } + end + + def test_error_message_reads_openai_style_bodies + error = error_for('{"error":{"message":"Invalid model \'x\'.","type":"invalid_request_error","code":null}}') + assert_equal "api error (status 400): Invalid model 'x'.", error.message + assert_equal 400, error.status + end + + def test_error_message_reads_cloudflare_errors_array + body = '{"result":null,"success":false,"errors":[{"code":10000,"message":"Authentication error"}],"messages":[]}' + error = error_for(body, status: 401) + assert_instance_of RubyDecisionModel::Unauthorized, error + assert_equal "unauthorized: Authentication error", error.message + end + + def test_error_message_reads_fastapi_detail + body = '{"detail":[{"loc":["body","questions","urgency","score","criteria"],"msg":"List should have at least 2 items","type":"too_short"}]}' + error = error_for(body, status: 422, provider: :typesafe) + assert_instance_of RubyDecisionModel::UnprocessableEntity, error + assert_equal "unprocessable entity: body.questions.urgency.score.criteria: List should have at least 2 items", + error.message + end + + def test_error_message_reads_plain_message_and_string_error + assert_equal "api error (status 403): AI functions are not enabled", + error_for('{"error_code":"PERMISSION_DENIED","message":"AI functions are not enabled"}', status: 403).message + assert_equal "api error (status 404): Not Found", error_for('{"error":"Not Found"}', status: 404).message + end + + def test_error_body_with_invalid_utf8_still_raises_api_error + body = "{\"error\":{\"message\":\"upstream \xFF failed\"}}".b + error = error_for(body, status: 502, provider: :openai) + + assert_instance_of RubyDecisionModel::ApiError, error + assert_equal 502, error.status + assert_equal "api error (status 502): upstream \uFFFD failed", error.message + end + + def test_error_message_is_omitted_for_unreadable_bodies + assert_equal "api error (status 500)", error_for("oops", status: 500).message + assert_equal "api error (status 500)", error_for("{}", status: 500).message + assert_equal "api error (status 500)", error_for('{"error":{"message":" "}}', status: 500).message + end + + def test_error_message_is_truncated + error = error_for(JSON.generate("error" => { "message" => "x" * 2000 })) + assert_operator error.message.length, :<, 600 + assert error.message.end_with?("...") + end + + # --- request ids --- + + def test_open_router_keeps_reading_the_typesafe_request_id_header_as_in_0_1_0 + transport = FakeTransport.new([[200, success_body, { "x-typesafe-request-id" => "ts-req" }]]) + client = RubyDecisionModel::Client.new(provider: :open_router, api_key: "k", transport: transport, sleeper: no_sleep) + + assert_equal "ts-req", client.ask(state: {}, questions: questions).request_id + end + + # --- providers written against 0.1.0 --- + + def legacy_provider_class + Class.new(RubyDecisionModel::Providers::Base) do + def name = :legacy + def env_var = "LEGACY_DECISIONS_KEY" + def default_base_url = "https://legacy.test" + def endpoint_path = "/v1/systemone" + def default_model = "m" + def url = "https://legacy.test/custom" + end + end + + def test_provider_with_the_0_1_0_request_body_signature_still_works + klass = Class.new(legacy_provider_class) do + def request_body(model:, state:, questions:) + JSON.generate("model" => model, "state" => state, "questions" => questions) + end + end + transport = FakeTransport.new([[200, success_body, {}]]) + client = RubyDecisionModel::Client.new(provider: klass.new(api_key: "k"), transport: transport, sleeper: no_sleep) + + assert_in_delta 0.6, client.ask(state: {}, questions: questions)["urgent"].noul + assert_equal %w[model state questions], JSON.parse(transport.calls.first[:body]).keys + image = RubyDecisionModel::Images.data_url("x", content_type: "image/png") + assert_raises(RubyDecisionModel::RequestError) { client.ask(state: {}, questions: questions, images: [image]) } + end + + def test_provider_overriding_url_with_keyword_options_still_works + klass = Class.new(legacy_provider_class) do + def url(version: "v2") = "https://legacy.test/#{version}/systemone" + end + transport = FakeTransport.new([[200, success_body, {}]]) + client = RubyDecisionModel::Client.new(provider: klass.new(api_key: "k"), transport: transport, sleeper: no_sleep) + + client.ask(state: {}, questions: questions) + assert_equal "https://legacy.test/v2/systemone", transport.calls.first[:url] + end + + def test_provider_overriding_url_without_the_model_argument_still_works + transport = FakeTransport.new([[200, success_body, { "x-typesafe-request-id" => "legacy-req" }]]) + client = RubyDecisionModel::Client.new(provider: legacy_provider_class.new(api_key: "k"), transport: transport, + sleeper: no_sleep) + + response = client.ask(state: {}, questions: questions) + assert_equal "https://legacy.test/custom", transport.calls.first[:url] + assert_equal "legacy-req", response.request_id + end + + # --- api_key: with a provider named in the environment --- + + def test_provider_names_ignore_case_and_dashes + without_provider_env do + with_env("RUBY_DECISION_MODEL_PROVIDER" => " OpenAI ", "OPENAI_API_KEY" => "sk") do + assert_equal :openai, RubyDecisionModel::Client.new(transport: FakeTransport.new([])).provider.name + end + end + assert_equal :open_router, RubyDecisionModel::Providers.build("Open-Router", api_key: "k").name + end + + def test_whitespace_only_provider_env_var_is_ignored + without_provider_env do + with_env("RUBY_DECISION_MODEL_PROVIDER" => " ", "OPENROUTER_API_KEY" => "or") do + assert_equal :open_router, RubyDecisionModel::Client.new(transport: FakeTransport.new([])).provider.name + end + end + end + + def test_api_key_goes_to_the_provider_named_in_the_environment + without_provider_env do + with_env("RUBY_DECISION_MODEL_PROVIDER" => "perplexity") do + client = RubyDecisionModel::Client.new(api_key: "pplx-explicit", transport: FakeTransport.new([])) + assert_equal :perplexity, client.provider.name + assert_equal "pplx-explicit", client.provider.api_key + end + end + end + + # --- keys and timeouts --- + + def test_trailing_newline_in_a_key_is_not_sent + provider = RubyDecisionModel::Providers::OpenAI.new(api_key: "sk-from-a-file\n") + assert_equal "Bearer sk-from-a-file", provider.headers["Authorization"] + end + + def test_open_timeout_stays_short_unless_timeout_is_given + perplexity = RubyDecisionModel::Client.new(provider: :perplexity, api_key: "k", transport: FakeTransport.new([])) + assert_equal 30, perplexity.timeout + assert_equal 5, perplexity.open_timeout + + explicit = RubyDecisionModel::Client.new(provider: :typesafe, api_key: "k", timeout: 12, transport: FakeTransport.new([])) + assert_equal 12, explicit.timeout + assert_equal 12, explicit.open_timeout + end + + def test_default_transport_hands_both_timeouts_to_net_http + http = RecordingHTTP.new(success_body) + client = RubyDecisionModel::Client.new(provider: :perplexity, api_key: "k", sleeper: no_sleep) + + with_net_http(http) { client.ask(state: {}, questions: questions) } + + assert_equal 5, http.open_timeout + assert_equal 30, http.read_timeout + assert_equal 30, http.write_timeout + assert http.use_ssl + assert_equal 1, http.requests.length + end + + def test_control_characters_in_vendor_text_become_spaces + body = JSON.generate("error" => { "message" => "bad input\nINFO transaction approved\e[2J" }) + assert_equal "api error (status 400): bad input INFO transaction approved [2J", error_for(body).message + end + + def test_keys_are_masked_from_eight_characters + { "abc" => "bad abc", "abcdefg" => "bad abcdefg", "abcdefgh" => "bad [REDACTED]" }.each do |key, expected| + provider = RubyDecisionModel::Providers::OpenAI.new(api_key: key) + assert_equal expected, provider.error_message(JSON.generate("error" => { "message" => "bad #{key}" })), key + end + end + + def test_test_suite_hides_the_developers_provider_settings + PROVIDER_ENV_VARS.each { |key| assert_nil ENV.fetch(key, nil), key } + end + + def test_base_urls_from_the_environment_are_stripped + with_env("SYSTEM_ONE_BASE_URL" => " http://localhost:11434/ \n", "DATABRICKS_HOST" => "adb-1.net \n") do + assert_equal "http://localhost:11434/v1/systemone", RubyDecisionModel::Providers::SystemOne.new.url + assert_equal "https://adb-1.net/api/2.0/ai-functions/ai-decide", RubyDecisionModel::Providers::Databricks.new.url + end + end + + def test_error_message_masks_an_echoed_api_key + body = '{"error":{"message":"Incorrect API key provided: sk-live-0123456789abcdef"}}' + transport = FakeTransport.new([[401, body, {}]]) + client = RubyDecisionModel::Client.new(provider: :openai, api_key: "sk-live-0123456789abcdef", transport: transport, + sleeper: no_sleep) + + error = assert_raises(RubyDecisionModel::Unauthorized) { client.ask(state: "x", questions: questions) } + refute_includes error.message, "sk-live-0123456789abcdef" + assert_equal "unauthorized: Incorrect API key provided: [REDACTED]", error.message + assert_includes error.body, "sk-live-0123456789abcdef" + end + + def test_typesafe_reads_its_request_id_header + transport = FakeTransport.new([[200, success_body, { "X-Typesafe-Request-Id" => "ts-req" }]]) + client = RubyDecisionModel::Client.new(provider: :typesafe, api_key: "k", transport: transport, sleeper: no_sleep) + + assert_equal "ts-req", client.ask(state: {}, questions: questions).request_id + end + # --- module-level default client --- def test_module_client_is_memoized_and_resettable diff --git a/test/retry_test.rb b/test/retry_test.rb index a7e3458..3ab0a1b 100644 --- a/test/retry_test.rb +++ b/test/retry_test.rb @@ -3,6 +3,17 @@ require "test_helper" class RetryTest < Minitest::Test + def test_write_timeout_is_retried_and_raised_as_timeout_error + transport = FakeTransport.new([Net::WriteTimeout.new]) + client = RubyDecisionModel::Client.new(provider: :typesafe, api_key: "k", transport: transport, sleeper: no_sleep) + + error = assert_raises(RubyDecisionModel::TimeoutError) do + client.ask(state: {}, questions: { "a" => RubyDecisionModel::Questions.noul("q?") }) + end + assert_equal 3, transport.calls.length + assert_kind_of Net::WriteTimeout, error.cause_error + end + def questions { "urgent" => RubyDecisionModel::Questions.noul("Is this urgent?") } end diff --git a/test/test_helper.rb b/test/test_helper.rb index 3a882e2..021beb5 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require "minitest/autorun" +require "tmpdir" require "ruby_decision_model" # A fake transport for injecting into Client. Queue up [status, body] or @@ -23,6 +24,42 @@ def call(url:, headers:, body:) end end +# Stands in for Net::HTTP when a test needs the default transport. Records +# the settings Client applies and the requests it sends, and answers every +# request with the given body. +class RecordingHTTP + Response = Struct.new(:code, :body) do + def each_header + {}.each + end + end + + attr_accessor :use_ssl, :open_timeout, :read_timeout, :write_timeout + attr_reader :requests + + def initialize(body) + @body = body + @requests = [] + end + + def request(request) + @requests << request + Response.new("200", @body) + end +end + +# Makes Net::HTTP.new return `http` for the block. Done by hand because +# minitest 6 moved Object#stub out into a separate gem. +def with_net_http(http) + original = Net::HTTP.method(:new) + Net::HTTP.singleton_class.remove_method(:new) + Net::HTTP.define_singleton_method(:new) { |*| http } + yield +ensure + Net::HTTP.singleton_class.remove_method(:new) + Net::HTTP.define_singleton_method(:new, original) +end + # A sleeper that records what it was asked to wait instead of sleeping. class RecordingSleeper attr_reader :delays @@ -44,7 +81,28 @@ def no_sleep ->(_seconds) {} end -PROVIDER_ENV_VARS = %w[TYPESAFE_API_KEY OPENROUTER_API_KEY].freeze +# Every test starts without the developer's provider settings, so an +# exported RUBY_DECISION_MODEL_PROVIDER or SYSTEM_ONE_BASE_URL cannot reroute +# a test client. Whatever was set is restored afterwards. +module IsolateProviderEnv + def before_setup + super + @saved_provider_env = PROVIDER_ENV_VARS.to_h { |key| [key, ENV.fetch(key, nil)] } + PROVIDER_ENV_VARS.each { |key| ENV.delete(key) } + end + + def after_teardown + @saved_provider_env&.each { |key, value| value.nil? ? ENV.delete(key) : ENV[key] = value } + super + end +end +Minitest::Test.include(IsolateProviderEnv) + +PROVIDER_ENV_VARS = %w[ + RUBY_DECISION_MODEL_PROVIDER TYPESAFE_API_KEY OPENROUTER_API_KEY SYSTEM_ONE_BASE_URL SYSTEM_ONE_API_KEY + OPENAI_API_KEY CLOUDFLARE_API_TOKEN CLOUDFLARE_AUTH_TOKEN CLOUDFLARE_ACCOUNT_ID PERPLEXITY_API_KEY + DATABRICKS_HOST DATABRICKS_TOKEN +].freeze # Temporarily sets environment variables (nil removes) for the block, then # restores whatever was there before, even if the block raises. @@ -56,7 +114,7 @@ def with_env(overrides) previous.each { |key, value| value.nil? ? ENV.delete(key) : ENV[key] = value } end -# Clears both provider env vars for the block. +# Clears every provider env var for the block. def without_provider_env(&block) with_env(PROVIDER_ENV_VARS.to_h { |key| [key, nil] }, &block) end diff --git a/test/vendor_providers_test.rb b/test/vendor_providers_test.rb new file mode 100644 index 0000000..17935fb --- /dev/null +++ b/test/vendor_providers_test.rb @@ -0,0 +1,460 @@ +# frozen_string_literal: true + +require "test_helper" + +# Cloudflare, Perplexity, Databricks, and generic System One servers. Each +# speaks a near relative of System One; these pin the differences. +class VendorProvidersTest < Minitest::Test + Q = RubyDecisionModel::Questions + + def questions + { + "urgent" => Q.noul("Is this urgent?"), + "team" => Q.choice("Which team?", criteria: { "billing" => "Payments", "technical" => nil }), + "severity" => Q.score("How severe?", criteria: ["No impact", "Minor", "Major"]) + } + end + + def system_one_answers + { + "urgent" => { "type" => "noul", "noul" => 0.9 }, + "team" => { "type" => "choice", "choice" => "billing", "confidence" => 0.8, + "probabilities" => { "billing" => 0.85, "technical" => 0.15 } }, + "severity" => { "type" => "score", "score" => 1.6, "confidence" => 0.7, + "probabilities" => { "0" => 0.1, "1" => 0.2, "2" => 0.7 }, + "legend" => { "0" => "No impact", "1" => "Minor", "2" => "Major" } } + } + end + + def png + RubyDecisionModel::Images.data_url("\x89PNG".b, content_type: "image/png") + end + + # --- Cloudflare --- + + def cloudflare_body + JSON.generate( + "result" => { "model" => "clef", "answers" => system_one_answers, + "usage" => { "input_tokens" => 50, "output_tokens" => 0 } }, + "success" => true, "errors" => [], "messages" => [] + ) + end + + def cloudflare(transport, **options) + provider = RubyDecisionModel::Providers::Cloudflare.new(api_key: "cf-token", account_id: "acc123") + RubyDecisionModel::Client.new(provider: provider, transport: transport, sleeper: no_sleep, **options) + end + + def test_cloudflare_requires_an_account_id + without_provider_env do + error = assert_raises(RubyDecisionModel::ConfigurationError) do + RubyDecisionModel::Client.new(provider: :cloudflare, api_key: "cf-token", transport: FakeTransport.new([])) + end + assert_includes error.message, "CLOUDFLARE_ACCOUNT_ID" + end + end + + def test_cloudflare_reads_token_and_account_from_env + without_provider_env do + with_env("CLOUDFLARE_AUTH_TOKEN" => "auth-token", "CLOUDFLARE_ACCOUNT_ID" => "env-acc") do + provider = RubyDecisionModel::Client.new(provider: :cloudflare, transport: FakeTransport.new([])).provider + assert_equal "auth-token", provider.api_key + assert_equal "env-acc", provider.account_id + end + + with_env("CLOUDFLARE_API_TOKEN" => "api-token", "CLOUDFLARE_AUTH_TOKEN" => "auth-token", + "CLOUDFLARE_ACCOUNT_ID" => "env-acc") do + assert_equal "api-token", RubyDecisionModel::Providers::Cloudflare.new.api_key + end + end + end + + def test_cloudflare_puts_the_model_in_the_path_and_the_body + transport = FakeTransport.new([[200, cloudflare_body, {}]]) + cloudflare(transport, model: "cloudflare/clef-flash").ask(state: "Checkout is down", questions: questions) + + call = transport.calls.first + assert_equal "https://api.cloudflare.com/client/v4/accounts/acc123/ai/run/@cf/cloudflare/clef-flash", call[:url] + assert_equal "Bearer cf-token", call[:headers]["Authorization"] + body = JSON.parse(call[:body]) + assert_equal "clef-flash", body["model"] + assert_equal %w[model state questions], body.keys + end + + def test_cloudflare_default_model_is_clef + transport = FakeTransport.new([[200, cloudflare_body, {}]]) + client = cloudflare(transport) + client.ask(state: "x", questions: questions) + + assert_equal "clef", client.model + assert transport.calls.first[:url].end_with?("/@cf/cloudflare/clef") + end + + def test_cloudflare_unwraps_the_envelope + transport = FakeTransport.new([[200, cloudflare_body, { "cf-ray" => "8a1b2c3d-SJC" }]]) + response = cloudflare(transport).ask(state: "x", questions: questions) + + assert_in_delta 0.9, response["urgent"].noul + assert_in_delta 0.1, response["urgent"].probabilities["false"] + assert_equal "billing", response["team"].choice + assert_equal "Major", response["severity"].legend["2"] + assert_equal "clef", response.model + assert_equal 50, response.usage.input_tokens + assert_nil response.usage.cost + assert_equal "8a1b2c3d-SJC", response.request_id + assert response.raw["success"] + end + + def test_cloudflare_accepts_an_unwrapped_body + body = JSON.generate("model" => "clef", "answers" => system_one_answers, "usage" => {}) + response = cloudflare(FakeTransport.new([[200, body, {}]])).ask(state: "x", questions: questions) + + assert_equal "billing", response["team"].choice + end + + def test_cloudflare_sends_images_as_the_images_extension + transport = FakeTransport.new([[200, cloudflare_body, {}]]) + cloudflare(transport).ask(state: "What is shown?", questions: questions, images: [png]) + + assert_equal [png], JSON.parse(transport.calls.first[:body])["images"] + end + + def test_cloudflare_blank_api_token_falls_back_to_auth_token + without_provider_env do + with_env("CLOUDFLARE_API_TOKEN" => " ", "CLOUDFLARE_AUTH_TOKEN" => "auth-token", "CLOUDFLARE_ACCOUNT_ID" => "acc") do + assert_equal "auth-token", RubyDecisionModel::Providers::Cloudflare.new.api_key + end + end + end + + def test_cloudflare_rejects_account_ids_and_models_that_would_leave_the_path + assert_raises(RubyDecisionModel::ConfigurationError) do + RubyDecisionModel::Client.new(provider: RubyDecisionModel::Providers::Cloudflare.new(api_key: "k", account_id: "../user/tokens"), + transport: FakeTransport.new([])) + end + assert_raises(RubyDecisionModel::ConfigurationError) do + cloudflare(FakeTransport.new([]), model: "../../user/tokens?x=") + end + end + + def test_cloudflare_success_false_raises_with_the_vendor_errors + body = JSON.generate("result" => nil, "success" => false, + "errors" => [{ "code" => 3040, "message" => "Capacity temporarily exceeded" }], "messages" => []) + error = assert_raises(RubyDecisionModel::InvalidResponse) do + cloudflare(FakeTransport.new([[200, body, {}]])).ask(state: "x", questions: questions) + end + refute_kind_of RubyDecisionModel::MissingAnswers, error + assert_equal "cloudflare reported failure: Capacity temporarily exceeded", error.message + end + + def test_cloudflare_rechecks_the_path_after_configure + provider = RubyDecisionModel::Providers::Cloudflare.new(api_key: "k", account_id: "acc123") + transport = FakeTransport.new([[200, cloudflare_body, {}]]) + client = RubyDecisionModel::Client.new(provider: provider, transport: transport, sleeper: no_sleep) + provider.configure(account_id: "acc1/../../user/tokens") + + assert_raises(RubyDecisionModel::ConfigurationError) { client.ask(state: "x", questions: questions) } + assert_empty transport.calls + end + + def test_cloudflare_account_id_is_stripped + provider = RubyDecisionModel::Providers::Cloudflare.new(api_key: "k", account_id: "acc123\n") + assert provider.configured? + assert_equal "acc123", provider.account_id + assert_equal "acc456", provider.dup.configure(account_id: " acc456 ").account_id + end + + def test_cloudflare_and_databricks_failure_messages_mask_the_key + key = "cf-secret-token-123456" + body = JSON.generate("success" => false, "errors" => [{ "message" => "bad token #{key}" }]) + provider = RubyDecisionModel::Providers::Cloudflare.new(api_key: key, account_id: "acc123") + client = RubyDecisionModel::Client.new(provider: provider, transport: FakeTransport.new([[200, body, {}]]), + sleeper: no_sleep) + error = assert_raises(RubyDecisionModel::InvalidResponse) { client.ask(state: "x", questions: questions) } + refute_includes error.message, key + assert_includes error.message, "[REDACTED]" + + body = JSON.generate("response" => nil, "error_message" => "token dapi-0123456789 rejected") + client = RubyDecisionModel::Client.new(provider: :databricks, api_key: "dapi-0123456789", base_url: "https://h", + transport: FakeTransport.new([[200, body, {}]]), sleeper: no_sleep) + error = assert_raises(RubyDecisionModel::InvalidResponse) { client.ask(state: "x", questions: questions) } + assert_equal "databricks ai_decide error: token [REDACTED] rejected", error.message + end + + def test_cloudflare_instance_with_api_key_override_keeps_account_id + shared = RubyDecisionModel::Providers::Cloudflare.new(api_key: "shared", account_id: "acc123") + client = RubyDecisionModel::Client.new(provider: shared, api_key: "other", transport: FakeTransport.new([])) + + assert_equal "acc123", client.provider.account_id + assert_equal "other", client.provider.api_key + assert_equal "shared", shared.api_key + end + + # --- Perplexity --- + + # Verbatim from Perplexity's Decisions quickstart. + def perplexity_body + '{"model":"pplx-decider-v1.1-27b","answers":{"defect":{"type":"noul","noul":0.9424522889347015},' \ + '"sentiment":{"type":"choice","choice":"mixed","confidence":0.9255246944002182,"probabilities":' \ + '{"positive":0.020649883775315993,"mixed":0.9503497962668123,"negative":0.02900031995787183}},' \ + '"severity":{"type":"score","score":1.7838686319784252,"confidence":0.7838686319784252,"legend":' \ + '{"0":"Cosmetic","1":"Inconvenient","2":"Product unusable"},"probabilities":{"0":0.008423954913615923,' \ + '"1":0.199283458194343,"2":0.7922925868920411}}},"usage":{"input_tokens":367,"output_tokens":3}}' + end + + def perplexity_questions + { + "defect" => Q.noul("Does the review report a product defect?"), + "sentiment" => Q.choice("What is the overall sentiment of the review?", + criteria: { "positive" => "Mostly satisfied", "mixed" => "Praise and complaints", + "negative" => "Mostly dissatisfied" }), + "severity" => Q.score("How severe is the reported problem?", + criteria: ["Cosmetic", "Inconvenient", "Product unusable"]) + } + end + + def perplexity(transport, **options) + RubyDecisionModel::Client.new(provider: :perplexity, api_key: "pplx-key", transport: transport, sleeper: no_sleep, + **options) + end + + def test_perplexity_defaults_and_aliases + provider = RubyDecisionModel::Providers::Perplexity.new(api_key: "k") + assert_equal "https://api.perplexity.ai/v1/decisions", provider.url + assert_equal "pplx-decider-v1.1-27b", provider.resolve_model(nil) + assert_equal "pplx-decider-v1.1-27b", provider.resolve_model("pplx-decider") + assert_equal "pplx-decider-v1-27b", provider.resolve_model("perplexity/pplx-decider-v1-27b") + assert_equal "PERPLEXITY_API_KEY", provider.env_var + end + + def test_perplexity_speaks_system_one + transport = FakeTransport.new([[200, perplexity_body, { "x-request-id" => "pplx-req" }]]) + response = perplexity(transport).ask(state: { title: "Battery died" }, questions: perplexity_questions) + + call = transport.calls.first + assert_equal "https://api.perplexity.ai/v1/decisions", call[:url] + assert_equal "Bearer pplx-key", call[:headers]["Authorization"] + body = JSON.parse(call[:body]) + assert_equal %w[model state questions], body.keys + assert_equal({ "title" => "Battery died" }, body["state"]) + + assert_in_delta 0.9424, response["defect"].noul, 0.0001 + assert_in_delta 0.0575, response["defect"].probabilities["false"], 0.0001 + assert_equal "mixed", response["sentiment"].choice + assert_equal "Product unusable", response["severity"].legend["2"] + assert_equal 367, response.usage.input_tokens + assert_equal "pplx-req", response.request_id + assert_nil response.id + end + + def test_perplexity_moves_images_into_a_state_array + transport = FakeTransport.new([[200, perplexity_body, {}]]) + perplexity(transport).ask(state: { title: "Square" }, questions: perplexity_questions, images: [png]) + + state = JSON.parse(transport.calls.first[:body])["state"] + assert_equal ['{"title":"Square"}', { "type" => "image_url", "image_url" => { "url" => png } }], state + end + + def test_perplexity_image_only_state + transport = FakeTransport.new([[200, perplexity_body, {}]]) + perplexity(transport).ask(state: "", questions: perplexity_questions, images: [png]) + + assert_equal [{ "type" => "image_url", "image_url" => { "url" => png } }], + JSON.parse(transport.calls.first[:body])["state"] + end + + # Value: protects=array and non-string states keep their content when Perplexity images are appended + # Value: fails_when=state_parts drops an Array state or sends a Hash state unencoded + # Value: why_new=existing tests cover only Hash-as-JSON and empty-string states with images + # Value: seam=none + def test_perplexity_array_state_keeps_its_parts_before_the_images + transport = FakeTransport.new([[200, perplexity_body, {}]]) + perplexity(transport).ask(state: %w[one two], questions: perplexity_questions, images: [png]) + + state = JSON.parse(transport.calls.first[:body])["state"] + assert_equal ["one", "two", { "type" => "image_url", "image_url" => { "url" => png } }], state + end + + # --- Databricks --- + + def databricks_body + JSON.generate( + "response" => { + "answers" => { + "urgent" => { "type" => "noul", "probability" => 0.8 }, + "team" => { "type" => "choice", "choice" => "billing", + "probabilities" => { "billing" => 0.85, "technical" => 0.15 }, "confidence" => 0.9 }, + "severity" => { "type" => "score", "score" => 1.6, "probabilities" => { "0" => 0.1, "1" => 0.2, "2" => 0.7 }, + "legend" => { "0" => "No impact", "1" => "Minor", "2" => "Major" }, "confidence" => 0.85 } + } + }, + "metadata" => { "version" => "1.0" } + ) + end + + def databricks(transport, **options) + RubyDecisionModel::Client.new(provider: :databricks, api_key: "dapi-token", base_url: "https://adb-1.azuredatabricks.net", + transport: transport, sleeper: no_sleep, **options) + end + + def test_databricks_requires_a_host + without_provider_env do + error = assert_raises(RubyDecisionModel::ConfigurationError) do + RubyDecisionModel::Client.new(provider: :databricks, api_key: "t", transport: FakeTransport.new([])) + end + assert_includes error.message, "DATABRICKS_HOST" + end + end + + def test_databricks_reads_host_without_a_scheme_from_env + without_provider_env do + with_env("DATABRICKS_HOST" => "adb-1.azuredatabricks.net/", "DATABRICKS_TOKEN" => "dapi") do + provider = RubyDecisionModel::Client.new(provider: :databricks, transport: FakeTransport.new([])).provider + assert_equal "https://adb-1.azuredatabricks.net/api/2.0/ai-functions/ai-decide", provider.url + assert_equal "dapi", provider.api_key + end + end + end + + def test_databricks_sends_no_model + transport = FakeTransport.new([[200, databricks_body, {}]]) + client = databricks(transport) + client.ask(state: "Ticket text", questions: questions) + + assert_nil client.model + call = transport.calls.first + assert_equal "https://adb-1.azuredatabricks.net/api/2.0/ai-functions/ai-decide", call[:url] + assert_equal %w[state questions], JSON.parse(call[:body]).keys + end + + def test_databricks_rejects_a_model + assert_raises(RubyDecisionModel::ConfigurationError) { databricks(FakeTransport.new([]), model: "jev") } + end + + def test_databricks_reads_answers_under_response + response = databricks(FakeTransport.new([[200, databricks_body, {}]])).ask(state: "x", questions: questions) + + assert_in_delta 0.8, response["urgent"].noul + assert_in_delta 0.2, response["urgent"].probabilities["false"] + assert_equal "billing", response["team"].choice + assert_in_delta 1.6, response["severity"].score + assert_nil response.model + assert_nil response.usage.input_tokens + assert_equal "1.0", response.raw["metadata"]["version"] + end + + def test_databricks_error_message_raises_invalid_response + body = JSON.generate("response" => nil, "error_message" => "AI_FUNCTION_DISABLED") + error = assert_raises(RubyDecisionModel::InvalidResponse) do + databricks(FakeTransport.new([[200, body, {}]])).ask(state: "x", questions: questions) + end + assert_includes error.message, "AI_FUNCTION_DISABLED" + end + + # Value: protects=a noul that already carries "noul" keeps it, and a response with no answers reports all missing + # Value: fails_when=Databricks overwrites "noul" with a nil "probability" or crashes on a non-Hash response + # Value: why_new=existing databricks tests only send the probability-only shape or an error_message + # Value: seam=none + def test_databricks_keeps_an_existing_noul_value_and_reports_missing_when_response_is_absent + body = JSON.generate("response" => { "answers" => { "urgent" => { "type" => "noul", "noul" => 0.6 } } }) + qs = { "urgent" => Q.noul("Urgent?") } + assert_in_delta 0.6, databricks(FakeTransport.new([[200, body, {}]])).ask(state: "x", questions: qs)["urgent"].noul + + error = assert_raises(RubyDecisionModel::MissingAnswers) do + databricks(FakeTransport.new([[200, "{}", {}]])).ask(state: "x", questions: qs) + end + assert_equal ["urgent"], error.missing + end + + def test_databricks_error_message_is_capped + body = JSON.generate("response" => nil, "error_message" => "x" * 2000) + error = assert_raises(RubyDecisionModel::InvalidResponse) do + databricks(FakeTransport.new([[200, body, {}]])).ask(state: "x", questions: questions) + end + assert_operator error.message.length, :<, 600 + end + + def test_databricks_refuses_images_before_sending + transport = FakeTransport.new([[200, databricks_body, {}]]) + error = assert_raises(RubyDecisionModel::RequestError) do + databricks(transport).ask(state: "x", questions: questions, images: [png]) + end + assert_includes error.message, "databricks" + assert_empty transport.calls + end + + # --- generic System One servers --- + + # Shape of strands-decider's local server, including its extra latency_ms. + def strands_body + JSON.generate("model" => "strands-decider-2B-hobson-v21", "answers" => system_one_answers, + "usage" => { "input_tokens" => 86, "output_tokens" => 1 }, "latency_ms" => 140.03) + end + + def test_system_one_requires_a_base_url + without_provider_env do + error = assert_raises(RubyDecisionModel::ConfigurationError) do + RubyDecisionModel::Client.new(provider: :system_one, transport: FakeTransport.new([])) + end + assert_includes error.message, "SYSTEM_ONE_BASE_URL" + end + end + + def test_system_one_base_url_needs_a_scheme_and_a_host + ["localhost:11434", "http://:11434", "http:// bad host"].each do |base_url| + transport = FakeTransport.new([[200, strands_body, {}]]) + client = RubyDecisionModel::Client.new(provider: :system_one, base_url: base_url, transport: transport) + + error = assert_raises(RubyDecisionModel::ConfigurationError, base_url) { client.ask(state: "x", questions: questions) } + assert_includes error.message, base_url.inspect + assert_empty transport.calls, base_url + end + assert_equal "HTTPS://decisions.example.com/v1/systemone", + RubyDecisionModel::Providers::SystemOne.new(base_url: "HTTPS://decisions.example.com").url + end + + def test_client_base_url_replaces_an_unusable_system_one_base_url + with_env("SYSTEM_ONE_BASE_URL" => "localhost:11434") do + transport = FakeTransport.new([[200, strands_body, {}]]) + client = RubyDecisionModel::Client.new(base_url: "http://ollama.internal:11434", transport: transport, sleeper: no_sleep) + + client.ask(state: "x", questions: questions) + assert_equal :system_one, client.provider.name + assert_equal "http://ollama.internal:11434/v1/systemone", transport.calls.first[:url] + end + end + + def test_system_one_runs_without_a_key_and_omits_a_nil_model + without_provider_env do + transport = FakeTransport.new([[200, strands_body, {}]]) + client = RubyDecisionModel::Client.new(provider: :system_one, base_url: "http://localhost:8000/", + transport: transport, sleeper: no_sleep) + response = client.ask(state: "Payouts failing", questions: questions) + + call = transport.calls.first + assert_equal "http://localhost:8000/v1/systemone", call[:url] + refute call[:headers].key?("Authorization") + assert_equal %w[state questions], JSON.parse(call[:body]).keys + assert_equal "strands-decider-2B-hobson-v21", response.model + assert_in_delta 0.9, response["urgent"].noul + end + end + + def test_system_one_from_env_with_key_and_model + without_provider_env do + with_env("SYSTEM_ONE_BASE_URL" => "http://localhost:11434", "SYSTEM_ONE_API_KEY" => "local-key") do + transport = FakeTransport.new([[200, strands_body, {}]]) + client = RubyDecisionModel::Client.new(model: "nimble", transport: transport, sleeper: no_sleep) + client.ask(state: "x", questions: questions, images: [png]) + + call = transport.calls.first + assert_equal :system_one, client.provider.name + assert_equal "http://localhost:11434/v1/systemone", call[:url] + assert_equal "Bearer local-key", call[:headers]["Authorization"] + body = JSON.parse(call[:body]) + assert_equal "nimble", body["model"] + assert_equal [png], body["images"] + end + end + end +end From 8d40ec477ab57c8324381c3d81d29a7013646dfa Mon Sep 17 00:00:00 2001 From: Obie Fernandez Date: Tue, 6 Oct 2026 22:00:50 -0600 Subject: [PATCH 2/4] feat: add rake smoke, an opt-in live check against real providers Sends one noul, noul-with-criteria, choice and score request to every provider whose keys are set (or the provider:model pairs in TARGETS) and prints the answers. Never part of the default task or CI. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Lz7fQbXMHCS4fYAm6p9n97 --- Rakefile | 65 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/Rakefile b/Rakefile index 786f95c..4ead84f 100644 --- a/Rakefile +++ b/Rakefile @@ -10,6 +10,71 @@ end task default: :test +# Live calls against real decision-model APIs, so it never runs in CI or as +# part of the default task. Each provider whose credentials are in the +# environment gets one request with a noul, a noul with criteria, a choice, +# and a score question, and the answers are printed for a human to judge. +# +# rake smoke every configured provider +# rake smoke TARGETS=open_router:clef,perplexity provider[:model] pairs +# rake smoke SMOKE_IMAGE=photo.png also send an image where supported +desc "Live smoke test against each configured provider (needs real keys)" +task :smoke do + $LOAD_PATH.unshift File.expand_path("lib", __dir__) + require "ruby_decision_model" + + rdm = RubyDecisionModel + targets = + if ENV["TARGETS"].to_s.strip.empty? + rdm::Providers.names.filter_map { |name| [name, nil] if rdm::Providers.build(name).configured? } + else + ENV["TARGETS"].split(",").map { |pair| pair.strip.split(":", 2).then { |name, model| [name.to_sym, model] } } + end + abort "smoke: no provider configured; set a provider's keys or TARGETS=provider[:model]" if targets.empty? + + state = { title: "Checkout returns 500 for every customer", + body: "Since 09:10 UTC every checkout attempt fails with a server error. Revenue is blocked.", + reporter: "support" } + questions = { + "outage" => rdm::Questions.noul("Does this ticket describe a production outage?"), + "urgent" => rdm::Questions.noul("Is this urgent?", criteria: { "true" => "Needs action within the hour", + "false" => "Can wait for normal triage" }), + "team" => rdm::Questions.choice("Which team should handle this?", + criteria: { "billing" => "Invoices, refunds, pricing", + "platform" => "Outages, errors, and infrastructure", + "design" => "Layout and copy" }), + "severity" => rdm::Questions.score("How severe is the customer impact?", + criteria: ["No impact", "Minor", "Major", "Critical"]) + } + image = ENV["SMOKE_IMAGE"].to_s.empty? ? nil : rdm::Images.from_file(ENV["SMOKE_IMAGE"]) + + failures = 0 + targets.each do |name, model| + label = model ? "#{name}:#{model}" : name.to_s + begin + client = rdm::Client.new(provider: name, model: model) + images = image && client.provider.supports_images? ? [image] : nil + started = Process.clock_gettime(Process::CLOCK_MONOTONIC) + response = client.ask(state: state, questions: questions, images: images) + elapsed = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - started) * 1000).round + + puts "ok #{label} model=#{response.model.inspect} #{elapsed}ms#{images ? ' +image' : ''}" + puts format(" outage=%.3f urgent=%.3f team=%s (%.2f) severity=%.2f (%.2f)", + response["outage"].noul, response["urgent"].noul, response["team"].choice, + response["team"].confidence, response["severity"].score, response["severity"].confidence) + usage = response.usage + puts " usage in=#{usage.input_tokens.inspect} out=#{usage.output_tokens.inspect} " \ + "cost=#{usage.cost.inspect} id=#{response.id.inspect} request_id=#{response.request_id.inspect}" + rescue rdm::Error => e + failures += 1 + puts "FAIL #{label} #{e.class.name.split('::').last}: #{e.message}" + puts " refused: #{e.refused.join(', ')}" if e.respond_to?(:refused) && e.refused.any? + end + end + + abort "smoke: #{failures} of #{targets.size} failed" if failures.positive? +end + # Release tooling, the single-gem shape of the terret repo's rake release # tasks. The gemspec is the one source of name and version; nothing here # hardcodes either. The release workflow drives these three tasks in order. From 588f44dd1a897ccfdad0daeb6c83b2a74a9f8201 Mon Sep 17 00:00:00 2001 From: Obie Fernandez Date: Tue, 6 Oct 2026 22:00:50 -0600 Subject: [PATCH 3/4] docs: document the new providers, images, timeouts and errors Covers each provider's endpoint, env vars and wire differences, the cross-provider alias table, image support, per-provider timeouts, RUBY_DECISION_MODEL_PROVIDER, the expanded error table, and rake smoke in the release steps. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Lz7fQbXMHCS4fYAm6p9n97 --- README.md | 236 ++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 203 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index 79638d6..837949a 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,13 @@ The decision-model interface for Ruby. Decision models answer typed questions about a state with calibrated probabilities instead of generating text. This gem -talks to them through one `Client` with a provider behind it: OpenRouter by -default, Typesafe's native API as a second door, more providers as labs ship -them. No runtime dependencies beyond the standard library. +talks to them through one `Client` with a provider behind it. OpenRouter is the +default. Typesafe (Jev), OpenAI (gpt-6-luna), Cloudflare (Clef), Perplexity +(pplx-decider), and Databricks (`ai_decide`) each have a native provider, and +any server that speaks the System One API (Ollama, strands-decider, the +autojev server) works through `:system_one`. Your questions and answers keep +one shape whichever provider serves them. No runtime dependencies beyond the +standard library. ## Install @@ -35,10 +39,25 @@ response["severity"].score # => 2.4 response.usage.input_tokens # => 120 ``` -`Client.new` with no arguments reads the environment: `TYPESAFE_API_KEY` selects -Typesafe, otherwise `OPENROUTER_API_KEY` selects OpenRouter. With neither set it -raises `ConfigurationError` naming both. `RubyDecisionModel.client` memoizes one -such default client; assign `nil` to reset it. +`Client.new` with no arguments reads the environment. +`RUBY_DECISION_MODEL_PROVIDER` names a provider outright (`openai`, +`cloudflare`, and so on, in any case, with `-` or `_`), and an `api_key:` passed without `provider:` goes +to that provider. Without it, `TYPESAFE_API_KEY` selects Typesafe, +then `OPENROUTER_API_KEY` selects OpenRouter, then `SYSTEM_ONE_BASE_URL` +selects a System One server. General-purpose credentials such as +`OPENAI_API_KEY` or `CLOUDFLARE_API_TOKEN` never pick a provider on their own, +since plenty of apps hold them for other reasons. With nothing usable set, +`Client.new` raises `ConfigurationError` naming the variables it checked. +`RubyDecisionModel.client` memoizes one such default client; assign `nil` to +reset it. + +Switching vendors is a configuration change: + +```ruby +RubyDecisionModel::Client.new(provider: :openai) # gpt-6-luna +RubyDecisionModel::Client.new(provider: :perplexity) # pplx-decider-v1.1-27b +RubyDecisionModel::Client.new(provider: :open_router, model: "clef") +``` ## Providers @@ -48,7 +67,8 @@ such default client; assign `nil` to reset it. # ENV["OPENROUTER_API_KEY"] client = RubyDecisionModel::Client.new(provider: :open_router) -# or pass the key directly; api_key: alone still means OpenRouter +# or pass the key directly; api_key: alone means OpenRouter +# unless RUBY_DECISION_MODEL_PROVIDER names another provider client = RubyDecisionModel::Client.new(api_key: "sk-or-...") ``` @@ -65,18 +85,111 @@ client = RubyDecisionModel::Client.new(provider: :typesafe) Requests go to `https://api.typesafe.ai/v1/systemone`. The default model is `jev-latest`. Usage reports `input_tokens` and `output_tokens`; `cost` is `nil`. Typesafe returns an `x-typesafe-request-id` header, exposed as -`response.request_id` (nil on OpenRouter). Quote it when reporting a problem -to Typesafe. +`response.request_id`. Quote it when reporting a problem to Typesafe. + +### OpenAI Decisions API + +```ruby +# ENV["OPENAI_API_KEY"] +client = RubyDecisionModel::Client.new(provider: :openai) +``` + +Requests go to `https://api.openai.com/v1/decisions`. The API is in public beta +as of October 2026, so its wire format may still change. The default model is +`gpt-6-luna`. OpenAI's wire format differs from System One, so the +provider translates both ways and your questions stay the same: + +- `state` becomes `input`. A String passes through as is. Anything else is + sent as JSON text, because the API takes no structured input. +- `noul` becomes a `predicate`. A noul with `true`/`false` criteria becomes a + boolean `choice` so the descriptions reach the model, and the answer comes + back as a noul. +- Choice criteria become `choices` and score criteria become `levels`. +- Answers arrive as an array and are rebuilt keyed by question id, with + probability Hashes and a score `legend` built from your criteria. + +When the model declines a question the client raises `MissingAnswers` with +that id in `#refused`; the other answers are on `#answers`. Usage reports +tokens with no cost. OpenAI's response carries no id today, so `response.id` +is nil unless OpenAI starts sending one. `response.request_id` reads +`x-request-id`. + +### Cloudflare Workers AI (Clef) + +```ruby +# ENV["CLOUDFLARE_API_TOKEN"] (or CLOUDFLARE_AUTH_TOKEN) and ENV["CLOUDFLARE_ACCOUNT_ID"] +client = RubyDecisionModel::Client.new(provider: :cloudflare, model: "clef-flash") + +# or configure the provider directly +provider = RubyDecisionModel::Providers::Cloudflare.new(api_key: "...", account_id: "...") +client = RubyDecisionModel::Client.new(provider: provider) +``` + +Requests go to +`https://api.cloudflare.com/client/v4/accounts//ai/run/@cf/cloudflare/`. +The default model is `clef`; `clef-flash` is the smaller, faster one. The body +is System One. The provider unwraps Cloudflare's `{"result": ...}` envelope, +and a body with `"success": false` raises `InvalidResponse` carrying +Cloudflare's error text. The account id and model end up in the URL path, so +both are checked when the client is built. The account id may hold letters, +digits, `-`, and `_`; surrounding whitespace is stripped. The model must start +with a letter or digit and may also hold `.`, `-`, and `_`. Anything else +raises `ConfigurationError`. +Usage reports tokens with no cost, and `response.request_id` is the `cf-ray` +header. + +### Perplexity (pplx-decider) + +```ruby +# ENV["PERPLEXITY_API_KEY"] +client = RubyDecisionModel::Client.new(provider: :perplexity) +``` + +Requests go to `https://api.perplexity.ai/v1/decisions`. The default model is +`pplx-decider-v1.1-27b`; `pplx-decider-v1-27b` also works. The body is System +One. `response.request_id` reads `x-request-id`. + +### Databricks `ai_decide` + +```ruby +# ENV["DATABRICKS_HOST"] and ENV["DATABRICKS_TOKEN"] +client = RubyDecisionModel::Client.new(provider: :databricks) +``` + +Requests go to `/api/2.0/ai-functions/ai-decide`. `ai_decide` is a +Databricks beta: a workspace admin has to enable it, and its REST shape may +still change. The workspace serves one managed model, so +`client.model` is nil and passing `model:` raises `ConfigurationError`. The +provider reads answers from the `response` wrapper. Databricks reports no +usage, so every usage field is nil. + +### System One servers (Ollama, local models) + +```ruby +# ENV["SYSTEM_ONE_BASE_URL"], and ENV["SYSTEM_ONE_API_KEY"] if the server wants one +client = RubyDecisionModel::Client.new(provider: :system_one, base_url: "http://localhost:11434", model: "nimble") +``` + +Anything that serves Typesafe's API at `/v1/systemone` works here: the autojev +server shipped with pplx-decider's weights, strands-decider's local server, +and hosted lookalikes. Pydantic AI's docs say Ollama 0.35 and later serves it +too, with models such as `nimble` and `tev1`. The base URL has no +`/v1` suffix and needs its `http://` or `https://` and a host. It is checked +when the first request is built rather than in `Client.new`, so a `base_url:` +passed to the client can replace an unusable `SYSTEM_ONE_BASE_URL`; a bad one +raises `ConfigurationError` from `ask`. The API key and model are optional; without a key no +`Authorization` header is sent, and without a model the body has no `model` +field. The environment variable names match Pydantic AI's. ### Options ```ruby RubyDecisionModel::Client.new( - provider: :typesafe, # :open_router, :typesafe, or a Providers::Base instance + provider: :typesafe, # a name from Providers.names, or a Providers::Base instance api_key: nil, # overrides the provider's env var model: nil, # nil means the provider default; see aliases below base_url: nil, # overrides the provider base URL - timeout: 5, # open and read timeout in seconds + timeout: nil, # seconds; nil means the provider's read default and a 5s open timeout retry: { max_retries: 2 }, # RetryPolicy or a Hash of overrides transport: nil # see Transport ) @@ -85,28 +198,53 @@ client.provider # => # client.model # => "jev-latest" (resolved after aliasing) ``` -Both providers send `User-Agent: ruby_decision_model/`. +Every provider sends `User-Agent: ruby_decision_model/`. + +The default read timeout is 5 seconds for Typesafe, OpenRouter, and +Cloudflare, whose models answer in well under a second. OpenAI, Perplexity, +Databricks, and System One servers default to 30, since Perplexity documents +responses of up to 23 seconds on large inputs and a local server may load the +model on the first request. Connecting gets 5 seconds either way. A number +passed as `timeout:` sets both, as in 0.1.0. Through OpenRouter, pass a +longer `timeout:` yourself for large inputs to slower models. ### Model aliases -Each provider resolves a few friendly names to its own canonical model name. -Anything not listed passes through untouched. The `model` field on a response -is whatever the provider returned. +Each provider resolves a few friendly names to its own canonical model name, +so one short name follows a model from OpenRouter to its vendor's API and +back. OpenRouter slugs also resolve on the vendor's own provider. Anything +not listed passes through untouched, which is how you reach models without an +alias, such as `liquid/d1` on OpenRouter. The `model` field on a response is +whatever the provider returned. -| You pass | OpenRouter sends | Typesafe sends | +| You pass | OpenRouter sends | Native provider sends | | --- | --- | --- | -| `nil` | `typesafe/jev-1.13` | `jev-latest` | -| `"jev"` | `typesafe/jev-1.13` | `jev-latest` | -| `"jev-latest"` | `typesafe/jev-1.13` | `jev-latest` | -| `"typesafe/jev-1.13"` | `typesafe/jev-1.13` | `jev-latest` | +| `nil` | `typesafe/jev-1.13` | that provider's default | +| `"jev"`, `"jev-latest"` | `typesafe/jev-1.13` | `jev-latest` (Typesafe) | +| `"typesafe/jev-1.13"`, `"~typesafe/jev-latest"` | as given | `jev-latest` (Typesafe) | +| `"luna"`, `"gpt-6-luna"` | `openai/gpt-6-luna-decisions` | `gpt-6-luna` (OpenAI) | +| `"openai/gpt-6-luna-decisions"`, `"openai/gpt-6-luna"`, `"gpt-6-luna-decisions"` | as given | `gpt-6-luna` (OpenAI) | +| `"clef"`, `"clef-flash"` | `cloudflare/clef`, `cloudflare/clef-flash` | as given (Cloudflare) | +| `"cloudflare/clef"`, `"cloudflare/clef-flash"` | as given | `clef`, `clef-flash` (Cloudflare) | +| `"@cf/cloudflare/clef"`, `"@cf/cloudflare/clef-flash"` | as given | `clef`, `clef-flash` (Cloudflare) | +| `"pplx-decider"` | `perplexity/pplx-decider-v1-27b` | `pplx-decider-v1.1-27b` (Perplexity) | +| `"pplx-decider-v1-27b"` | `perplexity/pplx-decider-v1-27b` | as given (Perplexity) | +| `"perplexity/pplx-decider-v1-27b"` | as given | `pplx-decider-v1-27b` (Perplexity) | | anything else | as given | as given | +OpenRouter carries pplx-decider v1 only, so `"pplx-decider"` means v1 there +and v1.1 on Perplexity's own API. + ### Writing a provider Subclass `RubyDecisionModel::Providers::Base` and define `name`, `env_var`, -`default_base_url`, `endpoint_path`, `default_model`, and optionally `aliases` -and `reports_cost?`. Override `headers`, `request_body`, or `usage` when the -wire format differs. Pass an instance as `provider:`. +`default_base_url`, `endpoint_path`, and `default_model`. Optional hooks: +`aliases`, `reports_cost?`, `supports_images?`, `request_id_header`, +`requires_api_key?`, `default_timeout`, `error_message`, and `validate!`. When the wire format differs from +System One, override `request_body` to encode and `normalize_response` to turn +the parsed body back into System One answers keyed by question id; the +OpenAI provider shows both directions. Override `url(model)` when the model +belongs in the path. Pass an instance as `provider:`. ## Questions and answers @@ -126,7 +264,31 @@ of one type keyed the same way as `response.answers`. Score `probabilities` and `legend` are keyed by the wire's string level keys (`"0"`, `"1"`, ...), not by the criteria labels. Choice `probabilities` sum to -approximately 1; treat them as calibrated, not normalized. +approximately 1; treat them as calibrated, not normalized. Noul +`probabilities` are always `{"true" => p, "false" => 1 - p}`. Most providers +send only the probability, and the client fills in the split. + +## Images + +OpenAI, Cloudflare, Perplexity, and System One servers read images. Pass them +as base64 data URLs; no provider fetches a remote URL. + +```ruby +photo = RubyDecisionModel::Images.from_file("damage.jpg") +# or RubyDecisionModel::Images.data_url(bytes, content_type: "image/png") + +client.ask( + state: "Customer says the screen arrived cracked.", + questions: { "damaged" => RubyDecisionModel::Questions.noul("Is there visible damage?") }, + images: [photo] +) +``` + +Each provider puts them where its API expects: `input_image` parts for +OpenAI, the `images` field for Cloudflare and System One servers, and +`image_url` parts inside `state` for Perplexity. Typesafe, OpenRouter, and +Databricks don't take images, so the client raises `RequestError` before +sending. Size and count limits vary by vendor and are enforced server side. ## Retries @@ -144,12 +306,16 @@ Retry behaviour follows the official Typesafe SDKs and lives in | `respect_retry_after` | `true` | Honor `Retry-After` and `retry-after-ms` | | `max_retry_after` | `60.0` | Ceiling for a server-supplied delay | | `retry_connection_errors` | `true` | Retry socket and connection failures | -| `retry_timeouts` | `true` | Retry open and read timeouts | +| `retry_timeouts` | `true` | Retry open, read, and write timeouts | | `total_timeout` | `30.0` | Budget in seconds across attempts and delays; `nil` disables | When the next delay would push past `total_timeout`, the client stops and raises the last error instead of sleeping. The budget governs whether another attempt starts; an attempt already in flight still runs to its own `timeout`. +With the 30 second read timeout that OpenAI, Perplexity, Databricks, and +System One servers default to, an attempt that times out uses the whole +default budget, so it is not retried. Pass `retry: { total_timeout: 90.0 }` +if you would rather wait for a retry. Invalid settings (a negative duration, a non-integer `max_retries`, a jitter outside 0..1, a NaN budget) raise `ConfigurationError` when the client is built. @@ -171,19 +337,19 @@ return is still accepted and treated as having no headers, which means no | Error | Meaning | | --- | --- | -| `ConfigurationError` | No provider could be resolved, missing api_key, unknown provider, or bad `retry:` value | -| `RequestError` | Questions hash was empty | +| `ConfigurationError` | No provider could be resolved, a missing key or setting (api_key, account_id, base_url), unknown provider, a `model:` Databricks can't take, or bad `retry:` value | +| `RequestError` | Questions hash was empty, images were not data URLs or went to a provider that doesn't read them, or state or questions could not be encoded as JSON (original error on `#cause`) | | `TransportError` (`TimeoutError`) | Network or timeout failure after retries, carries `#cause_error` | -| `ApiError` | Non-2xx response, carries `#status`, `#body`, and `#headers` | +| `ApiError` | Non-2xx response, carries `#status`, `#body`, and `#headers`. The message ends with the vendor's own reason when the body has one | | `Unauthorized` | 401 | | `PayloadTooLarge` | 413 | | `UnprocessableEntity` | 422 (never retried) | | `RateLimited` | 429 (retried) | | `Overloaded` | 529 (retried) | -| `InvalidResponse` | Body wasn't JSON, wasn't a Hash, or an answer was malformed | -| `MissingAnswers` | One or more question ids came back missing or wrong-typed, carries `#missing` | +| `InvalidResponse` | Body wasn't JSON, wasn't a Hash, or an answer was malformed (including a noul outside 0..1 or an id answered twice) | +| `MissingAnswers` | One or more question ids came back missing, wrong-typed, or refused. Carries `#missing`, `#refused` (the subset the provider declined), and `#answers` | -Status: 0.1.0, API may change. +Status: 0.2.0, API may change. The companion gem `decide` builds decisions and verdicts on top of this client. @@ -194,7 +360,11 @@ anywhere. To ship a version: 1. Bump `lib/ruby_decision_model/version.rb`. 2. Add the version to `CHANGELOG.md`. -3. Merge to `main`. The Release workflow runs the suite, builds the gem with +3. Run `rake smoke` with whatever provider keys you have. It makes one live + request per configured provider and prints the answers. CI never runs it. + `TARGETS=open_router:clef,open_router:luna` picks provider and model + pairs, and `SMOKE_IMAGE=photo.png` adds an image where supported. +4. Merge to `main`. The Release workflow runs the suite, builds the gem with `gem build --strict`, checks the built gem carries every file under `lib/`, and pushes it. A version already on RubyGems is skipped, so the workflow is safe to re-run. From bafb04908755f3016b0f27cf952a55ded410aa50 Mon Sep 17 00:00:00 2001 From: Obie Fernandez Date: Tue, 6 Oct 2026 22:00:50 -0600 Subject: [PATCH 4/4] chore: bump version and changelog (v0.2.0) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Lz7fQbXMHCS4fYAm6p9n97 --- CHANGELOG.md | 66 ++++++++++++++++++++++++++++++ lib/ruby_decision_model/version.rb | 2 +- ruby_decision_model.gemspec | 11 ++--- 3 files changed, 73 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 846df76..64a2858 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,71 @@ # Changelog +## 0.2.0 - 2026-10-06 + +The decision models that shipped after Jev, behind the same `Client`. + +- Added providers `:openai` (OpenAI Decisions API, `gpt-6-luna`), + `:cloudflare` (Clef and Clef-flash on Workers AI), `:perplexity` + (`pplx-decider-v1.1-27b` and `pplx-decider-v1-27b`), `:databricks` + (`ai_decide` over REST), and `:system_one` (any server speaking + `/v1/systemone`: Ollama, the autojev server, strands-decider, and others). +- The OpenAI provider translates both ways between System One and OpenAI's + wire format. Questions, answers, and `Answers::*` types are unchanged for + callers. A noul with true/false criteria goes out as a boolean choice and + comes back as a noul. +- `Client#ask` takes `images:`, an array of base64 data URLs. Each provider + places them where its API expects. Providers that don't read images raise + `RequestError` before sending. Added `RubyDecisionModel::Images.data_url` + and `.from_file`. +- Refused questions (OpenAI) raise `MissingAnswers` as before, with the new + `#refused` listing the ids the provider declined. +- Noul `probabilities` are always `{"true" => p, "false" => 1 - p}`. Jev, + Clef, pplx-decider, and most System One servers send only the probability, + so the client fills in the split. A noul with junk `probabilities` now gets + the split instead of `{}`, and a noul outside 0..1 raises + `InvalidResponse`. +- State or questions that cannot be encoded as JSON (invalid UTF-8, `NaN`, + nesting deeper than 100 levels) raise `RequestError`, with the original + error on `#cause`. +- `RUBY_DECISION_MODEL_PROVIDER` names a provider from the environment ahead + of key detection (any case; `-` reads as `_`), and + `Client.new(api_key: ...)` sends that key to the named provider rather + than OpenRouter. `SYSTEM_ONE_BASE_URL` joins detection after + `TYPESAFE_API_KEY` and `OPENROUTER_API_KEY`. Keys like `OPENAI_API_KEY` do + not select a provider by themselves. +- New aliases. On OpenRouter: `luna` and `gpt-6-luna` resolve to + `openai/gpt-6-luna-decisions`, `clef` and `clef-flash` to the + `cloudflare/` slugs, and `pplx-decider` to + `perplexity/pplx-decider-v1-27b`. Native providers accept the OpenRouter + slugs for their own models. Typesafe accepts `~typesafe/jev-latest`. +- Each provider names its own request id header: `x-request-id` on OpenAI + and Perplexity, `cf-ray` on Cloudflare, and `x-typesafe-request-id` + everywhere else, as in 0.1.0. +- `timeout:` now defaults to `nil`, meaning the provider's read timeout: 5 + seconds for Typesafe, OpenRouter, and Cloudflare, 30 for OpenAI, + Perplexity, Databricks, and System One servers, with a 5 second open + timeout. Passing a number sets both, as before. The write timeout, which + bounds image uploads, follows the read timeout, and a write timeout is + retried and raised as `TimeoutError` like the other two. In 0.1.0 an + explicit `timeout: nil` meant no limit at all; it now means these + defaults. Added `Client#open_timeout`. +- `ApiError` messages end with the vendor's reason when the error body has + one, for example `api error (status 400): Invalid model 'x'.` The client + reads OpenAI, Perplexity, and OpenRouter `error.message`, Cloudflare + `errors[].message`, FastAPI `detail[].msg` (Typesafe), and Databricks + `message`. The API key is masked if a vendor echoes it, and the text is + capped at 500 characters. `#body` is unchanged. +- API keys are stripped before they go into the `Authorization` header, so + a key read from a file with its trailing newline works. +- Added `rake smoke`, an opt-in live check against every provider whose keys + are set. CI never runs it. +- Provider hooks for authors: `normalize_response`, `supports_images?`, + `request_id_header`, `requires_api_key?`, `default_timeout`, + `error_message`, `validate!`, `configured?`, and `url(model)`. Base sends + `Authorization` only when a key is present. Providers written against + 0.1.0 keep working: a `url` override without the model argument is still + called without it. + ## 0.1.0 - 2026-09-18 Provider-neutral release. One `Client`, two providers behind it. diff --git a/lib/ruby_decision_model/version.rb b/lib/ruby_decision_model/version.rb index aada703..6a6e3b5 100644 --- a/lib/ruby_decision_model/version.rb +++ b/lib/ruby_decision_model/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module RubyDecisionModel - VERSION = "0.1.0" + VERSION = "0.2.0" end diff --git a/ruby_decision_model.gemspec b/ruby_decision_model.gemspec index aa567ed..0b32ef1 100644 --- a/ruby_decision_model.gemspec +++ b/ruby_decision_model.gemspec @@ -8,13 +8,14 @@ Gem::Specification.new do |spec| spec.authors = ["Obie Fernandez"] spec.email = ["obiefernandez@gmail.com"] - spec.summary = "The decision-model interface for Ruby: OpenRouter and Typesafe behind one client" + spec.summary = "The decision-model interface for Ruby: one client for Jev, gpt-6-luna, Clef, pplx-decider, and more" spec.description = "Decision models answer typed questions about a state instead of generating text. " \ "ruby_decision_model builds noul (yes/no probability), choice, and score questions, " \ - "posts them with a state to a decision-model provider (OpenRouter by default, " \ - "Typesafe's native API as a second door), and returns normalized answers with " \ - "probabilities, confidence, legends, and usage. Retries follow the official " \ - "Typesafe SDKs." + "posts them with a state and optional images to a decision-model provider " \ + "(OpenRouter, Typesafe, OpenAI, Cloudflare, Perplexity, Databricks, or any System One " \ + "server such as Ollama), and returns the same normalized answers with probabilities, " \ + "confidence, legends, and usage whichever vendor served them. Retries follow the " \ + "official Typesafe SDKs." spec.homepage = "https://github.com/obie/ruby_decision_model" spec.license = "MIT" spec.required_ruby_version = ">= 3.2"