diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6eadd8e..1b254dd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,11 +9,13 @@ repos: - id: check-json exclude: ^.devcontainer/devcontainer.json - id: pretty-format-json - exclude: ^.devcontainer/devcontainer.json + exclude: ^.devcontainer/devcontainer.json|^tests/test_provider_golden/cases/provider_sample_.*\.json$ args: [--autofix, --no-sort-keys] - id: end-of-file-fixer - exclude: ^tests/test_provider_golden/cases/.*\.sse$ + # Recorded bodies keep their upstream bytes: no added newline, no re-encoding. + exclude: ^tests/test_provider_golden/cases/.*\.sse$|^tests/test_provider_golden/cases/provider_sample_.*\.json$ - id: trailing-whitespace + exclude: ^tests/test_provider_golden/cases/provider_sample_.*\.json$ - repo: https://github.com/astral-sh/ruff-pre-commit rev: "v0.14.10" diff --git a/tests/test_provider_golden.py b/tests/test_provider_golden.py index 841617b..d0082d9 100644 --- a/tests/test_provider_golden.py +++ b/tests/test_provider_golden.py @@ -571,3 +571,170 @@ async def test_chat_stream_selects_choice_zero(service: FakeService) -> None: assert response.tool_calls == [] assert response.finish_reason == "stop" assert response.token_usage == republic.TokenUsage(input_tokens=79, output_tokens=42) + + +async def test_zai_reports_split_reasoning_and_usage(service: FakeService) -> None: + load_case(service, "provider_sample_zai_glm_reasoning") + model = republic.get_model("zai:glm-4.7-flash", api_key="test-key", http_client=service.client()) + + response, _ = await respond( + model, [republic.system("You are a helpful assistant"), republic.user("Say hi")], max_tokens=4000 + ) + + assert service.requests[0].url == "https://api.z.ai/api/paas/v4/chat/completions" + assert service.requests[0].headers["authorization"] == "Bearer test-key" + body = service.body() + assert body["model"] == "glm-4.7-flash" + # Z.ai's GLM chat limits output with `max_tokens`; `max_completion_tokens` is not part of its API. + assert body["max_tokens"] == 4000 + assert "max_completion_tokens" not in body + # `reasoning_content` is the model's thinking, kept out of the answer text. + assert response.reasoning.startswith("1. **Analyze the Request:**") + assert response.text == "Olá! (Oh-lah)" + assert response.finish_reason == "stop" + assert response.token_usage == republic.TokenUsage( + input_tokens=15, output_tokens=249, reasoning_tokens=239, cached_tokens=3 + ) + + +async def test_zai_reasoning_content_returns_in_the_next_turn(service: FakeService) -> None: + load_case(service, "provider_sample_zai_glm_tool") + model = republic.get_model("zai:glm-4.7-flash", api_key="test-key", http_client=service.client()) + prompt = [republic.system("You are a helpful assistant"), republic.user("What's the weather in Florence?")] + + first, _ = await respond(model, prompt, max_tokens=4000, tools=[WEATHER], tool_choice="auto") + + assert first.finish_reason == "tool_calls" + assert len(first.tool_calls) == 1 + call = first.tool_calls[0] + assert call.name == "weather" + assert call.id == "call_-7807249954186984837" + assert call.args == {"location": "Florence,Italy"} + assert first.reasoning + + second, _ = await respond( + model, [*prompt, first.message, republic.tool(call, "40 C")], max_tokens=4000, tools=[WEATHER] + ) + + assert second.finish_reason == "stop" + assert "40" in second.text + assert_tool_exchange(service, "chat", [call], {"weather": "40 C"}) + assert service.body()["messages"][2]["reasoning_content"] == first.reasoning + + +async def test_grok_reports_reasoning_tokens_without_replaying_them(service: FakeService) -> None: + load_case(service, "provider_sample_grok_tool") + model = republic.get_model("grok:grok-4-fast", api_key="test-key", api_format="chat", http_client=service.client()) + prompt = [republic.system("You are a helpful assistant"), republic.user("What's the weather in Florence?")] + + first, _ = await respond(model, prompt, max_tokens=4000, tools=[WEATHER], tool_choice="auto") + + assert service.requests[0].url == "https://api.x.ai/v1/chat/completions" + assert first.finish_reason == "tool_calls" + assert len(first.tool_calls) == 1 + call = first.tool_calls[0] + assert call.name == "weather" + assert call.id == "call_98286056" + assert call.args == {"location": "Florence, Italy"} + assert first.token_usage == republic.TokenUsage( + input_tokens=361, output_tokens=26, reasoning_tokens=93, cached_tokens=153 + ) + + second, _ = await respond( + model, [*prompt, first.message, republic.tool(call, "40 C")], max_tokens=4000, tools=[WEATHER] + ) + + assert second.finish_reason == "stop" + assert "40" in second.text + assert_tool_exchange(service, "chat", [call], {"weather": "40 C"}) + assert "reasoning_content" not in service.body()["messages"][2] + + +async def test_openrouter_kimi_returns_recorded_call_identity(service: FakeService) -> None: + load_case(service, "provider_sample_openrouter_kimi_tool") + model = republic.get_model( + "openrouter:moonshotai/kimi-k2.5", api_key="test-key", api_format="chat", http_client=service.client() + ) + prompt = [republic.system("You are a helpful assistant"), republic.user("What's the weather in Florence?")] + + first, _ = await respond(model, prompt, max_tokens=4000, tools=[WEATHER], tool_choice="auto") + + assert service.requests[0].url == "https://openrouter.ai/api/v1/chat/completions" + assert first.finish_reason == "tool_calls" + assert len(first.tool_calls) == 1 + call = first.tool_calls[0] + assert call.name == "weather" + # The gateway keeps the upstream call id, which is not shaped like OpenAI's. + assert call.id == "functions.weather:0" + assert call.args == {"location": "Florence"} + assert first.reasoning.startswith(" The user is asking for the weather in Florence") + assert first.text == "" + + second, _ = await respond( + model, [*prompt, first.message, republic.tool(call, "40 C")], max_tokens=4000, tools=[WEATHER] + ) + + assert second.finish_reason == "stop" + assert "40" in second.text + assert_tool_exchange(service, "chat", [call], {"weather": "40 C"}) + + +async def test_openrouter_streams_reasoning_and_final_usage(service: FakeService) -> None: + load_case(service, "provider_sample_openrouter_grok_streaming") + model = republic.get_model( + "openrouter:x-ai/grok-4.1-fast", api_key="test-key", api_format="chat", http_client=service.client() + ) + + response, _ = await respond( + model, + [republic.system("You are a helpful assistant"), republic.user("Say hi")], + streaming=True, + max_tokens=4000, + ) + + assert response.finish_reason == "stop" + assert response.text == "Oi!" + # The recording carries OpenRouter's `: OPENROUTER PROCESSING` heartbeats and a null reasoning + # field on the closing chunks; only the `reasoning` text becomes reasoning. + assert response.reasoning.startswith("First, the core policies") + assert response.token_usage == republic.TokenUsage( + input_tokens=165, output_tokens=167, reasoning_tokens=165, cached_tokens=164 + ) + + +async def test_azure_uses_api_key_header_and_deployment_name(service: FakeService) -> None: + azure_api_base = "https://fantasy-playground-resource.openai.azure.com/openai/v1" + load_case(service, "provider_sample_azure_tool") + model = republic.get_model( + "azure-openai:gpt-5-mini", + api_key="test-key", + api_base=azure_api_base, + api_format="chat", + http_client=service.client(), + ) + prompt = [republic.system("You are a helpful assistant"), republic.user("What's the weather in Florence?")] + + first, _ = await respond(model, prompt, max_tokens=4000, tools=[WEATHER], tool_choice="auto") + + assert service.requests[0].url == f"{azure_api_base}/chat/completions" + # This API-key case uses `api-key`; Azure AD bearer auth is a separate supported auth mode. + assert service.requests[0].headers["api-key"] == "test-key" + assert "authorization" not in service.requests[0].headers + body = service.body() + assert body["model"] == "gpt-5-mini" + assert body["max_completion_tokens"] == 4000 + assert first.finish_reason == "tool_calls" + assert len(first.tool_calls) == 1 + call = first.tool_calls[0] + assert call.name == "weather" + assert call.id == "call_mbWCiHb6eF0qjzWrYgK52fWS" + assert call.args == {"location": "Florence, Italy"} + + second, _ = await respond( + model, [*prompt, first.message, republic.tool(call, "40 C")], max_tokens=4000, tools=[WEATHER] + ) + + assert second.finish_reason == "stop" + assert "40" in second.text + assert service.requests[1].headers["api-key"] == "test-key" + assert_tool_exchange(service, "chat", [call], {"weather": "40 C"}) diff --git a/tests/test_provider_golden/PROVIDER-SAMPLES.md b/tests/test_provider_golden/PROVIDER-SAMPLES.md new file mode 100644 index 0000000..166e469 --- /dev/null +++ b/tests/test_provider_golden/PROVIDER-SAMPLES.md @@ -0,0 +1,169 @@ +# Provider samples: provenance + +Inputs for `tests/test_provider_golden.py`. Every case under +`cases/provider_sample_*` here is a response body recorded by a real client against the real +service: six cases holding ten byte-verified bodies. Nothing here is a live recording made by us: +no credentials were used and no model was called, and re-recording is out of scope for a test-run +environment. + +Schema-constructed samples are not recordings and are deliberately absent from this file. + +Each entry records the local SHA-256, the immutable upstream location, the license, the exact +transformations applied, and how the sample is classified. Expected values in the tests come from +the recorded payload's fields and the contract evidence identified below, never observed Republic +output. Republic source identifies routing and local integration conventions; +source inspection alone does not establish a provider-wide wire requirement. + +## Case index + +| Case | Provider family | Kind | Bodies | +| --- | --- | --- | --- | +| `provider_sample_zai_glm_reasoning` | Z.ai (ZAI) | recorded | `0.json` | +| `provider_sample_zai_glm_tool` | Z.ai (ZAI) | recorded | `0.json`, `1.json` | +| `provider_sample_grok_tool` | Grok (xAI direct) | recorded | `0.json`, `1.json` | +| `provider_sample_openrouter_kimi_tool` | OpenRouter (Kimi upstream) | recorded | `0.json`, `1.json` | +| `provider_sample_openrouter_grok_streaming` | OpenRouter (xAI upstream) | recorded | `0.sse` | +| `provider_sample_azure_tool` | Azure OpenAI | recorded | `0.json`, `1.json` | + +Bodies are numbered in the order Republic sends them. A `tool*` case holds the first response (the +model's tool call) and the second (the answer after the caller's tool output), so the recorded +request bodies also show what a real client sent back. + +## Recorded samples: Fantasy VCR cassettes + +Source repository: [`charmbracelet/fantasy`](https://github.com/charmbracelet/fantasy) at commit +`b0fadc9ae2e7da81a5d0262563d3ced9b594bd75`, license Apache-2.0 +([`LICENSE`](https://github.com/charmbracelet/fantasy/blob/b0fadc9ae2e7da81a5d0262563d3ced9b594bd75/LICENSE), +SHA-256 `0d542e0c8804e39aa7f37eb00da5a762149dc682d7829451287e11b938e94594`). A copy is kept beside +this file as `LICENSE.fantasy`. + +These cassettes are `go-vcr` recordings of the provider's own test suite: each interaction stores +a real outgoing request and the real response body, and the suite replays them through an HTTP +transport. The bodies below are the `response.body` field verbatim, so they are genuine +provider-authored API responses, not hand-written fixtures. + +We fetch from `raw.githubusercontent.com` at the pinned commit and compare SHA-256 against the same +path in the commit tarball, so the bytes are verified against two independent transports. + +### `provider_sample_zai_glm_reasoning/0.json` + +- Case: `TestOpenAICompatibleCommon/zai-glm-4-7-flash/simple.yaml` +- Path: `providertests/testdata/TestOpenAICompatibleCommon/zai-glm-4-7-flash/simple.yaml` +- Upstream file SHA-256: `f435f5086e66e034a1629a0d028026fbc1aad6ea17a8b77b0777fd937b86315d` +- Local SHA-256: `ef3ef5b8a4d45d15c6f213693de9a31daa0d368f2b83297490605d5fdc73bbb6` +- Classification: recorded, upstream real response +- Transformation: the YAML `response.body` string is written out as bytes. No reformatting, no field + edits, no re-encoding. +- Why it is necessary: GLM chat answers with `reasoning_content` beside `content`, reports + `completion_tokens_details.reasoning_tokens` and `prompt_tokens_details.cached_tokens`. + This is the only family here whose recorded + non-streaming answer keeps thinking and answer text apart in one message. +- Limitations: the recording is one short greeting; it exercises usage and reasoning decoding, not + a long answer. Its request uses `/api/coding/paas/v4/chat/completions`; it does not establish + Republic's default `/api/paas/v4` base. The URL assertion checks Republic's configured routing, + not a route verified by this cassette. The cassette uses `max_tokens`, so it does not show what + Z.ai does when + `max_completion_tokens` is sent — the test asserts Republic's own mapping instead. + +### `provider_sample_zai_glm_tool/0.json`, `1.json` + +- Case: `TestOpenAICompatibleCommon/zai-glm-4-7-flash/tool.yaml` +- Upstream file SHA-256: `dc98aee75f2f046941fb6c271d7e4a5ee3900a8cd2363d0b589375482867e5f0` +- Local SHA-256: `8d417387f2e02926bee4edddd39b9edae3c60471e6a62f4a8531386d0b235fa0` (`0.json`), + `087bcb0050ff0c982fc6eb97ecff5e23febd56354c1dabe9f5a910a47ed2149e` (`1.json`) +- Classification: recorded, upstream real responses +- Transformation: byte-identical extraction of both `response.body` values. +- Why it is necessary: the round trip is the point. The first response carries a Z.ai tool call + (`call_-7807249954186984837`) and thinking; the recorded second request shows the real client + resending that thinking as `reasoning_content`. Republic's `MaxTokensReasoningContentChat` exists + for exactly this, and only a recorded pair can show it. +- Limitations: one tool, one turn. The recorded arguments `{"location":"Florence,Italy"}` are + treated as an opaque string by the test, which compares them to the decoded call rather than + re-encoding them. + +### `provider_sample_grok_tool/0.json`, `1.json` + +- Case: `TestOpenAICompatibleCommon/xai-grok-4-fast/tool.yaml` +- Upstream file SHA-256: `2ace0fca86c730b56e7efca378bf919d418fbf1407bf242bdea4c243997b5f28` +- Local SHA-256: `6c674d6b5a3f86e88e18338e67b279c7cf75439fe66751c7002d972746d349cc` (`0.json`), + `ad6987149618b8f1ec90b2cd953670234e98d47453ca77884a0b6d370eeac718` (`1.json`) +- Classification: recorded, upstream real responses +- Transformation: byte-identical extraction of both `response.body` values. +- Why it is necessary: this is xAI's own `api.x.ai` endpoint, not a gateway. It shows Grok's + non-OpenAI usage envelope (`reasoning_tokens`, `cached_tokens`, `cost_in_usd_ticks`) and that no + reasoning field is replayed on the next turn — the counterpart to the Z.ai case. +- Limitations: `grok-4-fast-reasoning` reports reasoning tokens but no reasoning text in this + recording, so the test asserts token accounting and replay shape, not reasoning decoding. + Republic's `Grok` provider uses the xAI OAuth login by default; the test passes + `api_key="test-key"` with `api_format="chat"`, because a bearer header is what the recorded + request used. The OAuth flow itself is covered by `tests/test_grok.py`. + +### `provider_sample_openrouter_kimi_tool/0.json`, `1.json` + +- Case: `TestOpenRouterCommon/kimi/tool.yaml` +- Upstream file SHA-256: `bfc1bca915d600116d6aabf58f121dd30e7e11b713f8cd088a9743444d0293d4` +- Local SHA-256: `4fcf68ece5e374717a4c89c2c8f43142c31ebaca91c5a1792c53b74fb5cb215f` (`0.json`), + `f6e81e87249dab8a9c0da78196de3d195781c7bbab0c4843ceb644d08bd33b5c` (`1.json`) +- Classification: recorded, upstream real responses +- Transformation: byte-identical extraction of both `response.body` values. The recorded bodies + begin with several blank/whitespace lines from the gateway; these are kept as recorded. +- Why it is necessary: the tool-call id `functions.weather:0` is a gateway-shaped id, unlike + OpenAI's `call_…`. It proves Republic neither rewrites nor rejects foreign ids, and that the id + survives into the outgoing `tool_call_id`. Kimi's thinking arrives in `reasoning`, the field + Republic reads for gateways. +- Limitations: Kimi is reached through OpenRouter here; the direct Moonshot provider differs only in + base URL and dialect hooks, which the shared corpus already covers. The `provider` field of the + recording is whichever upstream OpenRouter routed to, so it is not a stable Moonshot fact. + +### `provider_sample_openrouter_grok_streaming/0.sse` + +- Case: `TestOpenRouterCommon/grok/simple_streaming.yaml` +- Upstream file SHA-256: `4f22a8f7c52af1223f70b2f99ef7ee090576c731a35025ceef9d669d59c0d047` +- Local SHA-256: `3f9e271722ffa27d7a65d58f4e402680b0dfb15449a702defd037567f743343b` +- Classification: recorded, upstream real SSE stream +- Transformation: the YAML block-scalar `response.body` is written out as bytes unchanged, + including the three `: OPENROUTER PROCESSING` comment lines and the trailing `data: [DONE]`. +- Why it is necessary: OpenRouter streams `reasoning` incrementally with no `reasoning_content`, + sends usage in a final chunk after the finish reason, and emits SSE comments as keepalives. It is + the streaming counterpart to the Kimi case and the only sample here with gateway keepalives. +- Limitations: xAI is the upstream, so it does not distinguish OpenRouter from xAI beyond the + gateway envelope. The recorded request used `"usage":{"include":true}` (an OpenRouter body + extension); Republic sends `stream_options.include_usage`. The response recording alone does + not validate that request option. + +### `provider_sample_azure_tool/0.json`, `1.json` + +- Case: `TestAzureCommon/azure-gpt-5-mini/tool.yaml` +- Upstream file SHA-256: `4848812cfd263a7ea3a32d0cd1ab3c8dda0f8137566a85ca2a4ff5a9b29789de` +- Local SHA-256: `793f9d33718111aecacaf0ac23441a448c9b52cd89fbd899b51b518c2d2522ae` (`0.json`), + `7c6d0e0708e0e4d8885fea90d3041e224d2553d3ab0786ed1e594c5373a71251` (`1.json`) +- Classification: recorded, upstream real responses +- Transformation: byte-identical extraction of both `response.body` values. The recorded host + `fantasy-playground-resource.openai.azure.com` is kept inside the body only insofar as it appears + there; the test's `api_base` uses the same resource name so the URL is not invented. +- Why it is necessary: Azure's v1 API answers with extra `content_filter_results` and + `prompt_filter_results` blocks, names the model `gpt-5-mini-2025-08-07`, and authenticates with + `api-key`. It is the only sample that exercises the Azure auth header and deployment naming. +- Limitations: the recording is chat, not the Responses API. It contains no `api-key` header of its + own — the VCR matches only method/URL/body — so the header assertion comes from Republic's + `AzureOpenAI._api_key_auth`, not from the recording. Independent evidence is the pinned OpenAI + SDK `src/openai/lib/azure.py` at `59738a289ee82c7b86bf1e10230cceb6daa9716d`, SHA-256 + `002f5e8b264e0b28534855f1e0245e5d2b8591a1e696bc872de4e9a98508322a`. It supports + both API-key and Azure AD bearer modes; only API-key mode is sampled here. + +## Families not sampled here + +The following supported families have no case in the recorded corpus. In short: `deepseek`, +`moonshot` (direct), `ollama` +and `together` reuse the shared chat mechanics already covered by `test_compatible_providers.py` +and `test_provider_golden.py`, and adding a fifth near-identical body would not test a new +distinction. `minimax`'s `reasoning_split` request field and default Messages routing are covered by +existing tests; no pinned recording of its `reasoning_details` reply was found in the sources above. + +## Reproducing + +```console +uv run pytest tests/test_provider_golden.py -q +``` + +No network access, credentials, or model calls are needed; `FakeService` answers from these files. diff --git a/tests/test_provider_golden/README.md b/tests/test_provider_golden/README.md index dfbdd3f..3f8f360 100644 --- a/tests/test_provider_golden/README.md +++ b/tests/test_provider_golden/README.md @@ -146,3 +146,12 @@ event accounting uses the common golden stream check. The existing The input carries three interleaved choices; only choice 0 reaches the single-response API. The case uses the common golden runner and streaming consistency assertions; there is no second copy of the input in the format tests. + + +## Additional provider recordings + +The `provider_sample_*` cases exercise Z.ai reasoning/usage and reasoning replay, xAI usage, +OpenRouter tool identity and streaming reasoning, and Azure authentication/routing through the +same golden runner and numbered response format. `PROVIDER-SAMPLES.md` pins their Fantasy cassette +sources, transformations and response hashes. Recorded provenance describes the inputs; it does not +make Republic's own request conventions evidence of the recorded service's requirements. diff --git a/tests/test_provider_golden/cases/provider_sample_azure_tool/0.json b/tests/test_provider_golden/cases/provider_sample_azure_tool/0.json new file mode 100644 index 0000000..d8ca940 --- /dev/null +++ b/tests/test_provider_golden/cases/provider_sample_azure_tool/0.json @@ -0,0 +1 @@ +{"choices":[{"content_filter_results":{},"finish_reason":"tool_calls","index":0,"logprobs":null,"message":{"annotations":[],"content":null,"refusal":null,"role":"assistant","tool_calls":[{"function":{"arguments":"{\"location\":\"Florence, Italy\"}","name":"weather"},"id":"call_mbWCiHb6eF0qjzWrYgK52fWS","type":"function"}]}}],"created":1782321616,"id":"chatcmpl-DuLYmgf20hGu0QAyj7XLR7WSNqeW0","model":"gpt-5-mini-2025-08-07","object":"chat.completion","prompt_filter_results":[{"prompt_index":0,"content_filter_results":{"hate":{"filtered":false,"severity":"safe"},"jailbreak":{"detected":false,"filtered":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}}}],"service_tier":"default","system_fingerprint":null,"usage":{"completion_tokens":26,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"latency_checkpoint":{"engine_tbt_ms":11,"engine_ttft_ms":67,"engine_ttlt_ms":1042,"pre_inference_ms":162,"service_tbt_ms":11,"service_ttft_ms":615,"service_ttlt_ms":1584,"total_duration_ms":1432,"user_visible_ttft_ms":453},"prompt_tokens":145,"prompt_tokens_details":{"audio_tokens":0,"cached_tokens":0},"total_tokens":171}} diff --git a/tests/test_provider_golden/cases/provider_sample_azure_tool/1.json b/tests/test_provider_golden/cases/provider_sample_azure_tool/1.json new file mode 100644 index 0000000..efd7fb1 --- /dev/null +++ b/tests/test_provider_golden/cases/provider_sample_azure_tool/1.json @@ -0,0 +1 @@ +{"choices":[{"content_filter_results":{"hate":{"filtered":false,"severity":"safe"},"protected_material_code":{"detected":false,"filtered":false},"protected_material_text":{"detected":false,"filtered":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"finish_reason":"stop","index":0,"logprobs":null,"message":{"annotations":[],"content":"Current temperature in Florence, Italy: 40 °C (104 °F).\n\nThat’s quite hot — recommendations: stay hydrated, avoid strenuous outdoor activity during midday, seek shade or air-conditioned places, wear light clothing and sunscreen, and check for local heat advisories. Would you like an hourly forecast, the next few days, or details like humidity and wind?","refusal":null,"role":"assistant"}}],"created":1782321618,"id":"chatcmpl-DuLYopOLgCaC41qFg4mR61lgZ1JgH","model":"gpt-5-mini-2025-08-07","object":"chat.completion","prompt_filter_results":[{"prompt_index":0,"content_filter_results":{}}],"service_tier":"default","system_fingerprint":null,"usage":{"completion_tokens":339,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":256,"rejected_prediction_tokens":0},"latency_checkpoint":{"engine_tbt_ms":11,"engine_ttft_ms":39,"engine_ttlt_ms":4111,"pre_inference_ms":232,"service_tbt_ms":11,"service_ttft_ms":675,"service_ttlt_ms":4729,"total_duration_ms":4505,"user_visible_ttft_ms":442},"prompt_tokens":176,"prompt_tokens_details":{"audio_tokens":0,"cached_tokens":0},"total_tokens":515}} diff --git a/tests/test_provider_golden/cases/provider_sample_grok_tool/0.json b/tests/test_provider_golden/cases/provider_sample_grok_tool/0.json new file mode 100644 index 0000000..f7fea95 --- /dev/null +++ b/tests/test_provider_golden/cases/provider_sample_grok_tool/0.json @@ -0,0 +1 @@ +{"id":"546bcc82-b37d-ca5d-aee1-0f2d503ee0b8","object":"chat.completion","created":1773345550,"model":"grok-4-fast-reasoning","choices":[{"index":0,"message":{"role":"assistant","content":"","tool_calls":[{"id":"call_98286056","function":{"name":"weather","arguments":"{\"location\":\"Florence, Italy\"}"},"type":"function"}],"refusal":null},"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":361,"completion_tokens":26,"total_tokens":480,"prompt_tokens_details":{"text_tokens":361,"audio_tokens":0,"image_tokens":0,"cached_tokens":153},"completion_tokens_details":{"reasoning_tokens":93,"audio_tokens":0,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0},"num_sources_used":0,"cost_in_usd_ticks":1087500},"system_fingerprint":"fp_3d7ed3b779"} \ No newline at end of file diff --git a/tests/test_provider_golden/cases/provider_sample_grok_tool/1.json b/tests/test_provider_golden/cases/provider_sample_grok_tool/1.json new file mode 100644 index 0000000..8ea77c3 --- /dev/null +++ b/tests/test_provider_golden/cases/provider_sample_grok_tool/1.json @@ -0,0 +1 @@ +{"id":"533294f1-af60-2023-e993-8ab87543ed84","object":"chat.completion","created":1773345552,"model":"grok-4-fast-reasoning","choices":[{"index":0,"message":{"role":"assistant","content":"The current weather in Florence, Italy, is 40°C. It feels quite hot—stay hydrated! If you need more details like forecast or conditions, let me know.","refusal":null},"finish_reason":"stop"}],"usage":{"prompt_tokens":493,"completion_tokens":35,"total_tokens":582,"prompt_tokens_details":{"text_tokens":493,"audio_tokens":0,"image_tokens":0,"cached_tokens":481},"completion_tokens_details":{"reasoning_tokens":54,"audio_tokens":0,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0},"num_sources_used":0,"cost_in_usd_ticks":709500},"system_fingerprint":"fp_3d7ed3b779"} \ No newline at end of file diff --git a/tests/test_provider_golden/cases/provider_sample_openrouter_grok_streaming/0.sse b/tests/test_provider_golden/cases/provider_sample_openrouter_grok_streaming/0.sse new file mode 100644 index 0000000..3430db7 --- /dev/null +++ b/tests/test_provider_golden/cases/provider_sample_openrouter_grok_streaming/0.sse @@ -0,0 +1,100 @@ +: OPENROUTER PROCESSING + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":"First","reasoning_details":[{"type":"reasoning.summary","summary":"First","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":",","reasoning_details":[{"type":"reasoning.summary","summary":",","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":" the","reasoning_details":[{"type":"reasoning.summary","summary":" the","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":" core","reasoning_details":[{"type":"reasoning.summary","summary":" core","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":" policies","reasoning_details":[{"type":"reasoning.summary","summary":" policies","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":":","reasoning_details":[{"type":"reasoning.summary","summary":":","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":" No","reasoning_details":[{"type":"reasoning.summary","summary":" No","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":" assistance","reasoning_details":[{"type":"reasoning.summary","summary":" assistance","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":" with","reasoning_details":[{"type":"reasoning.summary","summary":" with","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":" criminal","reasoning_details":[{"type":"reasoning.summary","summary":" criminal","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":" activity","reasoning_details":[{"type":"reasoning.summary","summary":" activity","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":",","reasoning_details":[{"type":"reasoning.summary","summary":",","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":" no","reasoning_details":[{"type":"reasoning.summary","summary":" no","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":" overly","reasoning_details":[{"type":"reasoning.summary","summary":" overly","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":" realistic","reasoning_details":[{"type":"reasoning.summary","summary":" realistic","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":" help","reasoning_details":[{"type":"reasoning.summary","summary":" help","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":" with","reasoning_details":[{"type":"reasoning.summary","summary":" with","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":" crimes","reasoning_details":[{"type":"reasoning.summary","summary":" crimes","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":" even","reasoning_details":[{"type":"reasoning.summary","summary":" even","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":" in","reasoning_details":[{"type":"reasoning.summary","summary":" in","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":" hypot","reasoning_details":[{"type":"reasoning.summary","summary":" hypot","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":"het","reasoning_details":[{"type":"reasoning.summary","summary":"het","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":"icals","reasoning_details":[{"type":"reasoning.summary","summary":"icals","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":",","reasoning_details":[{"type":"reasoning.summary","summary":",","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":" short","reasoning_details":[{"type":"reasoning.summary","summary":" short","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":" responses","reasoning_details":[{"type":"reasoning.summary","summary":" responses","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":" to","reasoning_details":[{"type":"reasoning.summary","summary":" to","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":" jail","reasoning_details":[{"type":"reasoning.summary","summary":" jail","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":"break","reasoning_details":[{"type":"reasoning.summary","summary":"break","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":" attempts","reasoning_details":[{"type":"reasoning.summary","summary":" attempts","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":".","reasoning_details":[{"type":"reasoning.summary","summary":".","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":" This","reasoning_details":[{"type":"reasoning.summary","summary":" This","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":" query","reasoning_details":[{"type":"reasoning.summary","summary":" query","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":" is","reasoning_details":[{"type":"reasoning.summary","summary":" is","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":" harmless","reasoning_details":[{"type":"reasoning.summary","summary":" harmless","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":":","reasoning_details":[{"type":"reasoning.summary","summary":":","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":" \"","reasoning_details":[{"type":"reasoning.summary","summary":" \"","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":"Say","reasoning_details":[{"type":"reasoning.summary","summary":"Say","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":" hi","reasoning_details":[{"type":"reasoning.summary","summary":" hi","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":" in","reasoning_details":[{"type":"reasoning.summary","summary":" in","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":" Po...","reasoning_details":[{"type":"reasoning.summary","summary":" Po...","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +: OPENROUTER PROCESSING + +: OPENROUTER PROCESSING + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":null,"reasoning_details":[{"type":"reasoning.encrypted","data":"B/XIFKX05qXzekUEb12SSn4BIjmJ51LDdv30TMRAsHya7nZDMRR6fgLO2bRGyvmcxJoBh+zTWaizcAgiPaPWCDhgCNDv9GzUBhAgYnAbtNISeOGG2SSVoTIMSOcH77dXVLUVAlF0fbYbrebbJcJl9mDhvP09ddekxG9flwdVBDK/4uZ7yomvH8dS1lG34NeWpzbivkqQRDvPWOBBFNQzW3gE01nRHZArutRBieQyKDuMQPwXsorJuA1glLg5YrRfAljVpuq8NJIqzwfzgx1iwJtxULMnKDJNnJkEJPcANc+HarCdRyPWgfdylm/UMGQV12uTJJ6d3fJM5Hq8mV5E1R0oVbRqVPOGC+U60v49tLKG/VSxpFvRajMXTotmMCWpfyKb68NJq2yxT7IFjBoDxx7+UVHyCXVO5VKTfeac3lIWeorbzXrtNQH9x1bHaNCIrH/Y3kFiSQ2GjOX5tlFYWip8U0dbLlQplTKnEajVep3GF+HESe9KdmnIsB2wrtKrz54wXUiPj8wuo1uKWPdBgiI1EFUBOXro4V+GGvjQiKZlH02l7S7GEmgW0b40fMf5evJzqjXjY2OwR+RKkXnDEtqqRSoeA146l7mz/QZd44ZHuzlRwZq/FpRUhq2hbBEQTysy73XIN2yMB/H8o6tevCo1UOo95Nxj6GAS/dGzUglexXYOUGRkkaSlBTr4n4WHpzN0H4gryrFQCsx3EhZIbR/W0SmNUC7KUhTvNHBOng90+0j1+hIMtYMip+fjxrwS1yu7UBZe6bFKqHaSUSXuL+VoS3vgrIOVThBNNWW7teu+2/LTkWr9WwKqv+GR0Cq1dZARGEglGmr96CEzTBTOYgRGBTkZk6bp3HpS10X8UkKMzIBHo+ttrzwft15DZOILgT4wZsYDvR5M3kKQBm2QRBYgkC8gIWTWtelgJ8Xa/A65/2ck8+Gu5VxwC3e/HsMytwuwQFChSl5OqUCmMiLGJgUR3zg","format":"xai-responses-v1","id":"rs_b69ec351-62d7-3ea7-bb06-d9699f84799e","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"Oi","role":"assistant"},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"!","role":"assistant"},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","service_tier":"default","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":null},"finish_reason":"stop","native_finish_reason":"completed"}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","service_tier":"default","choices":[{"index":0,"delta":{"content":"","role":"assistant"},"finish_reason":"stop","native_finish_reason":"completed"}],"usage":{"prompt_tokens":165,"completion_tokens":167,"total_tokens":332,"cost":0.0000919,"is_byok":false,"prompt_tokens_details":{"cached_tokens":164,"cache_write_tokens":0,"audio_tokens":0,"video_tokens":0},"cost_details":{"upstream_inference_cost":0.0000919,"upstream_inference_prompt_cost":0.0000084,"upstream_inference_completions_cost":0.0000835},"completion_tokens_details":{"reasoning_tokens":165,"image_tokens":0,"audio_tokens":0}}} + +data: [DONE] + diff --git a/tests/test_provider_golden/cases/provider_sample_openrouter_kimi_tool/0.json b/tests/test_provider_golden/cases/provider_sample_openrouter_kimi_tool/0.json new file mode 100644 index 0000000..9fa8dd3 --- /dev/null +++ b/tests/test_provider_golden/cases/provider_sample_openrouter_kimi_tool/0.json @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + +{"id":"gen-1777319891-AbXaksK95wX8kHwAXxXX","object":"chat.completion","created":1777319891,"model":"moonshotai/kimi-k2.5-0127","provider":"Chutes","system_fingerprint":null,"choices":[{"index":0,"logprobs":null,"finish_reason":"tool_calls","native_finish_reason":"tool_calls","message":{"role":"assistant","content":null,"refusal":null,"reasoning":" The user is asking for the weather in Florence, Italy. I have a weather function available that takes a location parameter (which should be the city). I should call the weather function with \"Florence\" as the location.","tool_calls":[{"type":"function","index":0,"id":"functions.weather:0","function":{"name":"weather","arguments":"{\"location\": \"Florence\"}"}}],"reasoning_details":[{"type":"reasoning.text","text":" The user is asking for the weather in Florence, Italy. I have a weather function available that takes a location parameter (which should be the city). I should call the weather function with \"Florence\" as the location.","format":"unknown","index":0}]}}],"usage":{"prompt_tokens":69,"completion_tokens":66,"total_tokens":135,"cost":0.00016236,"is_byok":false,"prompt_tokens_details":{"cached_tokens":0,"cache_write_tokens":0,"audio_tokens":0,"video_tokens":0},"cost_details":{"upstream_inference_cost":0.00016236,"upstream_inference_prompt_cost":0.00003036,"upstream_inference_completions_cost":0.000132},"completion_tokens_details":{"reasoning_tokens":47,"image_tokens":0,"audio_tokens":0}}} \ No newline at end of file diff --git a/tests/test_provider_golden/cases/provider_sample_openrouter_kimi_tool/1.json b/tests/test_provider_golden/cases/provider_sample_openrouter_kimi_tool/1.json new file mode 100644 index 0000000..b9c8c44 --- /dev/null +++ b/tests/test_provider_golden/cases/provider_sample_openrouter_kimi_tool/1.json @@ -0,0 +1,39 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +{"id":"gen-1777319895-NVPGjsD98YPuLRBFo9BH","object":"chat.completion","created":1777319895,"model":"moonshotai/kimi-k2.5-0127","provider":"Chutes","system_fingerprint":null,"choices":[{"index":0,"logprobs":null,"finish_reason":"stop","native_finish_reason":"stop","message":{"role":"assistant","content":" The weather in Florence, Italy is currently **40°C** (104°F). That's quite hot!","refusal":null,"reasoning":" The weather in Florence, Italy is currently 40°C. That's quite hot - about 104°F. This seems like it might be unusually warm weather for Florence, possibly during a heat wave or in the peak of summer.","reasoning_details":[{"type":"reasoning.text","text":" The weather in Florence, Italy is currently 40°C. That's quite hot - about 104°F. This seems like it might be unusually warm weather for Florence, possibly during a heat wave or in the peak of summer.","format":"unknown","index":0}]}}],"usage":{"prompt_tokens":108,"completion_tokens":67,"total_tokens":175,"cost":0.00018152,"is_byok":false,"prompt_tokens_details":{"cached_tokens":0,"cache_write_tokens":0,"audio_tokens":0,"video_tokens":0},"cost_details":{"upstream_inference_cost":0.00018152,"upstream_inference_prompt_cost":0.00004752,"upstream_inference_completions_cost":0.000134},"completion_tokens_details":{"reasoning_tokens":46,"image_tokens":0,"audio_tokens":0}}} \ No newline at end of file diff --git a/tests/test_provider_golden/cases/provider_sample_zai_glm_reasoning/0.json b/tests/test_provider_golden/cases/provider_sample_zai_glm_reasoning/0.json new file mode 100644 index 0000000..47cd164 --- /dev/null +++ b/tests/test_provider_golden/cases/provider_sample_zai_glm_reasoning/0.json @@ -0,0 +1 @@ +{"choices":[{"finish_reason":"stop","index":0,"message":{"content":"Olá! (Oh-lah)","reasoning_content":"1. **Analyze the Request:** The user wants me to say \"hi\" in Portuguese.\n\n2. **Identify the Language:** Portuguese.\n\n3. **Identify the Translation for \"Hi\":**\n * Standard/Common greeting: \"Olá\"\n * Informal/Casual greeting: \"Oi\"\n * Pronunciation guides (optional but helpful): \"oh-lah\", \"oy\"\n\n4. **Formulate the Response:**\n * Option 1: Just \"Olá\" (too brief).\n * Option 2: \"Olá!\" (better).\n * Option 3: \"Olá! Oi!\" (covers formal and informal).\n * Option 4: Include pronunciation for better UX. \"Olá! (oh-lah)\"\n\n5. **Select the Best Response:** \"Olá!\" is the standard, universally understood greeting. I will also provide the phonetic pronunciation to be helpful.\n\n * *Draft:* \"Olá! (oh-lah)\"\n\n6. **Final Polish:** Keep it simple and friendly.\n\n * Output: Olá!","role":"assistant"}}],"created":1773349367,"id":"20260313050242595b2cb031a74737","model":"glm-4.7-flash","object":"chat.completion","request_id":"20260313050242595b2cb031a74737","usage":{"completion_tokens":249,"completion_tokens_details":{"reasoning_tokens":239},"prompt_tokens":15,"prompt_tokens_details":{"cached_tokens":3},"total_tokens":264}} \ No newline at end of file diff --git a/tests/test_provider_golden/cases/provider_sample_zai_glm_tool/0.json b/tests/test_provider_golden/cases/provider_sample_zai_glm_tool/0.json new file mode 100644 index 0000000..dfaff5e --- /dev/null +++ b/tests/test_provider_golden/cases/provider_sample_zai_glm_tool/0.json @@ -0,0 +1 @@ +{"choices":[{"finish_reason":"tool_calls","index":0,"message":{"content":"I'll check the weather in Florence, Italy for you.","reasoning_content":"The user is asking for the weather in Florence, Italy. I have a weather function available that can get weather information for a location. The function requires a \"location\" parameter, and the user has provided \"Florence,Italy\" as the location. I should use this function to get the weather information.","role":"assistant","tool_calls":[{"function":{"arguments":"{\"location\":\"Florence,Italy\"}","name":"weather"},"id":"call_-7807249954186984837","index":0,"type":"function"}]}}],"created":1773349384,"id":"202603130503035652f6eb725d4b48","model":"glm-4.7-flash","object":"chat.completion","request_id":"202603130503035652f6eb725d4b48","usage":{"completion_tokens":88,"completion_tokens_details":{"reasoning_tokens":62},"prompt_tokens":173,"prompt_tokens_details":{"cached_tokens":43},"total_tokens":261}} \ No newline at end of file diff --git a/tests/test_provider_golden/cases/provider_sample_zai_glm_tool/1.json b/tests/test_provider_golden/cases/provider_sample_zai_glm_tool/1.json new file mode 100644 index 0000000..8cdf17e --- /dev/null +++ b/tests/test_provider_golden/cases/provider_sample_zai_glm_tool/1.json @@ -0,0 +1 @@ +{"choices":[{"finish_reason":"stop","index":0,"message":{"content":"The weather in Florence, Italy is currently 40°C (104°F). That's quite hot! Make sure to stay hydrated and seek shade if you're out and about.","reasoning_content":"The weather function returned \"40 C\" which indicates the temperature in Florence, Italy is 40 degrees Celsius. This seems quite hot, which would be consistent with Florence being in Italy and possibly experiencing hot weather. I should provide this information to the user in a clear and helpful way.","role":"assistant"}}],"created":1773349386,"id":"20260313050305e26cddfc278c448a","model":"glm-4.7-flash","object":"chat.completion","request_id":"20260313050305e26cddfc278c448a","usage":{"completion_tokens":94,"completion_tokens_details":{"reasoning_tokens":57},"prompt_tokens":267,"prompt_tokens_details":{"cached_tokens":173},"total_tokens":361}} \ No newline at end of file