From 43e52471e995bc5d7d09e71b58f8955a0b6acdf5 Mon Sep 17 00:00:00 2001 From: Chojan Shang Date: Sat, 14 Nov 2026 00:06:04 +0800 Subject: [PATCH 1/4] fix(chat): isolate choice zero in streamed responses --- src/republic/formats/chat.py | 16 +- tests/test_provider_golden.py | 12 ++ tests/test_provider_golden/LICENSE.openai | 201 ++++++++++++++++++ tests/test_provider_golden/README.md | 11 + .../openai_chat_multi_choice_streaming/0.sse | 100 +++++++++ 5 files changed, 339 insertions(+), 1 deletion(-) create mode 100644 tests/test_provider_golden/LICENSE.openai create mode 100644 tests/test_provider_golden/cases/openai_chat_multi_choice_streaming/0.sse diff --git a/src/republic/formats/chat.py b/src/republic/formats/chat.py index c002e66..41b7554 100644 --- a/src/republic/formats/chat.py +++ b/src/republic/formats/chat.py @@ -164,7 +164,9 @@ def feed(self, event: str, data: str) -> Iterable[Delta]: return chunk = json.loads(data) _raise_for_error(chunk) - for choice in chunk.get("choices") or (): + # The API exposes one response, so only the choice indexed 0 contributes; other + # choices' text, reasoning, calls and finish reason are dropped rather than merged. + if (choice := _indexed_choice(chunk)) is not None: delta = choice.get("delta") or {} if reasoning := self._api_format.reasoning_text(delta): yield ReasoningDelta(reasoning) @@ -184,6 +186,7 @@ def feed(self, event: str, data: str) -> Iterable[Delta]: arguments=function.get("arguments") or "", ) yield _info(chunk, choice) + # Usage is request-level, not per choice, so it survives whatever the choices say. if usage := chunk.get("usage"): yield _usage(usage) @@ -260,6 +263,17 @@ def _citation_deltas(message: Mapping[str, Any]) -> Iterable[Delta]: yield CitationAdded(Citation(citation["url"], title=citation.get("title"))) +def _indexed_choice(chunk: Mapping[str, Any]) -> Mapping[str, Any] | None: + """The choice with index 0 in a stream chunk, whatever its position in the array. + + Single-response dialects omit ``index``, so a missing index counts as 0. + """ + for choice in chunk.get("choices") or (): + if choice.get("index", 0) == 0: + return choice + return None + + def _info(data: Mapping[str, Any], choice: Mapping[str, Any]) -> ResponseInfo: finish_reason = choice.get("finish_reason") return ResponseInfo( diff --git a/tests/test_provider_golden.py b/tests/test_provider_golden.py index 417e7c7..841617b 100644 --- a/tests/test_provider_golden.py +++ b/tests/test_provider_golden.py @@ -559,3 +559,15 @@ async def test_anthropic_cumulative_usage_includes_thinking(service: FakeService assert response.token_usage == republic.TokenUsage( input_tokens=59, output_tokens=8, reasoning_tokens=3, cached_tokens=7, cache_write_tokens=12 ) + + +async def test_chat_stream_selects_choice_zero(service: FakeService) -> None: + load_case(service, "openai_chat_multi_choice_streaming") + + response, _ = await respond(make_model(service, "chat"), [republic.user("Weather?")], streaming=True) + + assert response.text == '{"city":"San Francisco","temperature":65,"units":"f"}' + assert response.reasoning == "" + assert response.tool_calls == [] + assert response.finish_reason == "stop" + assert response.token_usage == republic.TokenUsage(input_tokens=79, output_tokens=42) diff --git a/tests/test_provider_golden/LICENSE.openai b/tests/test_provider_golden/LICENSE.openai new file mode 100644 index 0000000..cbb5bb2 --- /dev/null +++ b/tests/test_provider_golden/LICENSE.openai @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2026 OpenAI + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/tests/test_provider_golden/README.md b/tests/test_provider_golden/README.md index 61e2495..dfbdd3f 100644 --- a/tests/test_provider_golden/README.md +++ b/tests/test_provider_golden/README.md @@ -135,3 +135,14 @@ This SDK input is not claimed as a production recording. Cumulative usage includ Republic counts cached reads/writes in input tokens and thinking in output tokens. Public usage event accounting uses the common golden stream check. The existing `goose_messages_cache_streaming` case covers preservation of omitted counts. + + +## Chat multi-choice regression + +`cases/openai_chat_multi_choice_streaming/0.sse` is the byte-identical SDK fixture used by +[openai-python@59738a2](https://github.com/openai/openai-python/blob/59738a289ee82c7b86bf1e10230cceb6daa9716d/tests/lib/chat/test_completions_streaming.py), +`test_parse_pydantic_model_multiple_choices` (Apache-2.0; `LICENSE.openai`). Its SHA-256 is +`a491adda08c3d4fde95f5b2ee3f60f7f745f1a56d82e62f58031cc2add502380`. +The input carries three interleaved choices; only choice 0 reaches the single-response API. +The case uses the common golden runner and streaming consistency assertions; there is no +second copy of the input in the format tests. diff --git a/tests/test_provider_golden/cases/openai_chat_multi_choice_streaming/0.sse b/tests/test_provider_golden/cases/openai_chat_multi_choice_streaming/0.sse new file mode 100644 index 0000000..160e65d --- /dev/null +++ b/tests/test_provider_golden/cases/openai_chat_multi_choice_streaming/0.sse @@ -0,0 +1,100 @@ +data: {"id":"chatcmpl-ABfw2KKFuVXmEJgVwYfBvejMAdWtq","object":"chat.completion.chunk","created":1727346170,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_b40fb1c6fb","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-ABfw2KKFuVXmEJgVwYfBvejMAdWtq","object":"chat.completion.chunk","created":1727346170,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_b40fb1c6fb","choices":[{"index":0,"delta":{"content":"{\""},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-ABfw2KKFuVXmEJgVwYfBvejMAdWtq","object":"chat.completion.chunk","created":1727346170,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_b40fb1c6fb","choices":[{"index":1,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-ABfw2KKFuVXmEJgVwYfBvejMAdWtq","object":"chat.completion.chunk","created":1727346170,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_b40fb1c6fb","choices":[{"index":1,"delta":{"content":"{\""},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-ABfw2KKFuVXmEJgVwYfBvejMAdWtq","object":"chat.completion.chunk","created":1727346170,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_b40fb1c6fb","choices":[{"index":2,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-ABfw2KKFuVXmEJgVwYfBvejMAdWtq","object":"chat.completion.chunk","created":1727346170,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_b40fb1c6fb","choices":[{"index":2,"delta":{"content":"{\""},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-ABfw2KKFuVXmEJgVwYfBvejMAdWtq","object":"chat.completion.chunk","created":1727346170,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_b40fb1c6fb","choices":[{"index":0,"delta":{"content":"city"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-ABfw2KKFuVXmEJgVwYfBvejMAdWtq","object":"chat.completion.chunk","created":1727346170,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_b40fb1c6fb","choices":[{"index":1,"delta":{"content":"city"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-ABfw2KKFuVXmEJgVwYfBvejMAdWtq","object":"chat.completion.chunk","created":1727346170,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_b40fb1c6fb","choices":[{"index":2,"delta":{"content":"city"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-ABfw2KKFuVXmEJgVwYfBvejMAdWtq","object":"chat.completion.chunk","created":1727346170,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_b40fb1c6fb","choices":[{"index":0,"delta":{"content":"\":\""},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-ABfw2KKFuVXmEJgVwYfBvejMAdWtq","object":"chat.completion.chunk","created":1727346170,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_b40fb1c6fb","choices":[{"index":1,"delta":{"content":"\":\""},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-ABfw2KKFuVXmEJgVwYfBvejMAdWtq","object":"chat.completion.chunk","created":1727346170,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_b40fb1c6fb","choices":[{"index":2,"delta":{"content":"\":\""},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-ABfw2KKFuVXmEJgVwYfBvejMAdWtq","object":"chat.completion.chunk","created":1727346170,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_b40fb1c6fb","choices":[{"index":0,"delta":{"content":"San"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-ABfw2KKFuVXmEJgVwYfBvejMAdWtq","object":"chat.completion.chunk","created":1727346170,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_b40fb1c6fb","choices":[{"index":1,"delta":{"content":"San"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-ABfw2KKFuVXmEJgVwYfBvejMAdWtq","object":"chat.completion.chunk","created":1727346170,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_b40fb1c6fb","choices":[{"index":2,"delta":{"content":"San"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-ABfw2KKFuVXmEJgVwYfBvejMAdWtq","object":"chat.completion.chunk","created":1727346170,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_b40fb1c6fb","choices":[{"index":0,"delta":{"content":" Francisco"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-ABfw2KKFuVXmEJgVwYfBvejMAdWtq","object":"chat.completion.chunk","created":1727346170,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_b40fb1c6fb","choices":[{"index":1,"delta":{"content":" Francisco"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-ABfw2KKFuVXmEJgVwYfBvejMAdWtq","object":"chat.completion.chunk","created":1727346170,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_b40fb1c6fb","choices":[{"index":2,"delta":{"content":" Francisco"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-ABfw2KKFuVXmEJgVwYfBvejMAdWtq","object":"chat.completion.chunk","created":1727346170,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_b40fb1c6fb","choices":[{"index":0,"delta":{"content":"\",\""},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-ABfw2KKFuVXmEJgVwYfBvejMAdWtq","object":"chat.completion.chunk","created":1727346170,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_b40fb1c6fb","choices":[{"index":1,"delta":{"content":"\",\""},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-ABfw2KKFuVXmEJgVwYfBvejMAdWtq","object":"chat.completion.chunk","created":1727346170,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_b40fb1c6fb","choices":[{"index":2,"delta":{"content":"\",\""},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-ABfw2KKFuVXmEJgVwYfBvejMAdWtq","object":"chat.completion.chunk","created":1727346170,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_b40fb1c6fb","choices":[{"index":0,"delta":{"content":"temperature"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-ABfw2KKFuVXmEJgVwYfBvejMAdWtq","object":"chat.completion.chunk","created":1727346170,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_b40fb1c6fb","choices":[{"index":0,"delta":{"content":"\":"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-ABfw2KKFuVXmEJgVwYfBvejMAdWtq","object":"chat.completion.chunk","created":1727346170,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_b40fb1c6fb","choices":[{"index":1,"delta":{"content":"temperature"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-ABfw2KKFuVXmEJgVwYfBvejMAdWtq","object":"chat.completion.chunk","created":1727346170,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_b40fb1c6fb","choices":[{"index":1,"delta":{"content":"\":"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-ABfw2KKFuVXmEJgVwYfBvejMAdWtq","object":"chat.completion.chunk","created":1727346170,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_b40fb1c6fb","choices":[{"index":2,"delta":{"content":"temperature"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-ABfw2KKFuVXmEJgVwYfBvejMAdWtq","object":"chat.completion.chunk","created":1727346170,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_b40fb1c6fb","choices":[{"index":2,"delta":{"content":"\":"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-ABfw2KKFuVXmEJgVwYfBvejMAdWtq","object":"chat.completion.chunk","created":1727346170,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_b40fb1c6fb","choices":[{"index":0,"delta":{"content":"65"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-ABfw2KKFuVXmEJgVwYfBvejMAdWtq","object":"chat.completion.chunk","created":1727346170,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_b40fb1c6fb","choices":[{"index":0,"delta":{"content":",\""},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-ABfw2KKFuVXmEJgVwYfBvejMAdWtq","object":"chat.completion.chunk","created":1727346170,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_b40fb1c6fb","choices":[{"index":1,"delta":{"content":"61"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-ABfw2KKFuVXmEJgVwYfBvejMAdWtq","object":"chat.completion.chunk","created":1727346170,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_b40fb1c6fb","choices":[{"index":1,"delta":{"content":",\""},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-ABfw2KKFuVXmEJgVwYfBvejMAdWtq","object":"chat.completion.chunk","created":1727346170,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_b40fb1c6fb","choices":[{"index":2,"delta":{"content":"59"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-ABfw2KKFuVXmEJgVwYfBvejMAdWtq","object":"chat.completion.chunk","created":1727346170,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_b40fb1c6fb","choices":[{"index":2,"delta":{"content":",\""},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-ABfw2KKFuVXmEJgVwYfBvejMAdWtq","object":"chat.completion.chunk","created":1727346170,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_b40fb1c6fb","choices":[{"index":0,"delta":{"content":"units"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-ABfw2KKFuVXmEJgVwYfBvejMAdWtq","object":"chat.completion.chunk","created":1727346170,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_b40fb1c6fb","choices":[{"index":1,"delta":{"content":"units"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-ABfw2KKFuVXmEJgVwYfBvejMAdWtq","object":"chat.completion.chunk","created":1727346170,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_b40fb1c6fb","choices":[{"index":2,"delta":{"content":"units"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-ABfw2KKFuVXmEJgVwYfBvejMAdWtq","object":"chat.completion.chunk","created":1727346170,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_b40fb1c6fb","choices":[{"index":0,"delta":{"content":"\":\""},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-ABfw2KKFuVXmEJgVwYfBvejMAdWtq","object":"chat.completion.chunk","created":1727346170,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_b40fb1c6fb","choices":[{"index":1,"delta":{"content":"\":\""},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-ABfw2KKFuVXmEJgVwYfBvejMAdWtq","object":"chat.completion.chunk","created":1727346170,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_b40fb1c6fb","choices":[{"index":2,"delta":{"content":"\":\""},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-ABfw2KKFuVXmEJgVwYfBvejMAdWtq","object":"chat.completion.chunk","created":1727346170,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_b40fb1c6fb","choices":[{"index":0,"delta":{"content":"f"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-ABfw2KKFuVXmEJgVwYfBvejMAdWtq","object":"chat.completion.chunk","created":1727346170,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_b40fb1c6fb","choices":[{"index":1,"delta":{"content":"f"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-ABfw2KKFuVXmEJgVwYfBvejMAdWtq","object":"chat.completion.chunk","created":1727346170,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_b40fb1c6fb","choices":[{"index":2,"delta":{"content":"f"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-ABfw2KKFuVXmEJgVwYfBvejMAdWtq","object":"chat.completion.chunk","created":1727346170,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_b40fb1c6fb","choices":[{"index":0,"delta":{"content":"\"}"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-ABfw2KKFuVXmEJgVwYfBvejMAdWtq","object":"chat.completion.chunk","created":1727346170,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_b40fb1c6fb","choices":[{"index":1,"delta":{"content":"\"}"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-ABfw2KKFuVXmEJgVwYfBvejMAdWtq","object":"chat.completion.chunk","created":1727346170,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_b40fb1c6fb","choices":[{"index":2,"delta":{"content":"\"}"},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-ABfw2KKFuVXmEJgVwYfBvejMAdWtq","object":"chat.completion.chunk","created":1727346170,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_b40fb1c6fb","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}]} + +data: {"id":"chatcmpl-ABfw2KKFuVXmEJgVwYfBvejMAdWtq","object":"chat.completion.chunk","created":1727346170,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_b40fb1c6fb","choices":[{"index":1,"delta":{},"logprobs":null,"finish_reason":"stop"}]} + +data: {"id":"chatcmpl-ABfw2KKFuVXmEJgVwYfBvejMAdWtq","object":"chat.completion.chunk","created":1727346170,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_b40fb1c6fb","choices":[{"index":2,"delta":{},"logprobs":null,"finish_reason":"stop"}]} + +data: {"id":"chatcmpl-ABfw2KKFuVXmEJgVwYfBvejMAdWtq","object":"chat.completion.chunk","created":1727346170,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_b40fb1c6fb","choices":[],"usage":{"prompt_tokens":79,"completion_tokens":42,"total_tokens":121,"completion_tokens_details":{"reasoning_tokens":0}}} + +data: [DONE] + From c57626f033e2caf6e2154d28092d5764eaa8dd69 Mon Sep 17 00:00:00 2001 From: Chojan Shang Date: Sat, 14 Nov 2026 00:06:15 +0800 Subject: [PATCH 2/4] test: extend the shared provider golden suite --- .pre-commit-config.yaml | 6 +- tests/test_provider_golden.py | 167 +++++++++++++++++ .../test_provider_golden/PROVIDER-SAMPLES.md | 169 ++++++++++++++++++ tests/test_provider_golden/README.md | 9 + .../cases/provider_sample_azure_tool/0.json | 1 + .../cases/provider_sample_azure_tool/1.json | 1 + .../cases/provider_sample_grok_tool/0.json | 1 + .../cases/provider_sample_grok_tool/1.json | 1 + .../0.sse | 100 +++++++++++ .../0.json | 15 ++ .../1.json | 39 ++++ .../provider_sample_zai_glm_reasoning/0.json | 1 + .../cases/provider_sample_zai_glm_tool/0.json | 1 + .../cases/provider_sample_zai_glm_tool/1.json | 1 + 14 files changed, 510 insertions(+), 2 deletions(-) create mode 100644 tests/test_provider_golden/PROVIDER-SAMPLES.md create mode 100644 tests/test_provider_golden/cases/provider_sample_azure_tool/0.json create mode 100644 tests/test_provider_golden/cases/provider_sample_azure_tool/1.json create mode 100644 tests/test_provider_golden/cases/provider_sample_grok_tool/0.json create mode 100644 tests/test_provider_golden/cases/provider_sample_grok_tool/1.json create mode 100644 tests/test_provider_golden/cases/provider_sample_openrouter_grok_streaming/0.sse create mode 100644 tests/test_provider_golden/cases/provider_sample_openrouter_kimi_tool/0.json create mode 100644 tests/test_provider_golden/cases/provider_sample_openrouter_kimi_tool/1.json create mode 100644 tests/test_provider_golden/cases/provider_sample_zai_glm_reasoning/0.json create mode 100644 tests/test_provider_golden/cases/provider_sample_zai_glm_tool/0.json create mode 100644 tests/test_provider_golden/cases/provider_sample_zai_glm_tool/1.json diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6eadd8e..1b254dd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,11 +9,13 @@ repos: - id: check-json exclude: ^.devcontainer/devcontainer.json - id: pretty-format-json - exclude: ^.devcontainer/devcontainer.json + exclude: ^.devcontainer/devcontainer.json|^tests/test_provider_golden/cases/provider_sample_.*\.json$ args: [--autofix, --no-sort-keys] - id: end-of-file-fixer - exclude: ^tests/test_provider_golden/cases/.*\.sse$ + # Recorded bodies keep their upstream bytes: no added newline, no re-encoding. + exclude: ^tests/test_provider_golden/cases/.*\.sse$|^tests/test_provider_golden/cases/provider_sample_.*\.json$ - id: trailing-whitespace + exclude: ^tests/test_provider_golden/cases/provider_sample_.*\.json$ - repo: https://github.com/astral-sh/ruff-pre-commit rev: "v0.14.10" diff --git a/tests/test_provider_golden.py b/tests/test_provider_golden.py index 841617b..d0082d9 100644 --- a/tests/test_provider_golden.py +++ b/tests/test_provider_golden.py @@ -571,3 +571,170 @@ async def test_chat_stream_selects_choice_zero(service: FakeService) -> None: assert response.tool_calls == [] assert response.finish_reason == "stop" assert response.token_usage == republic.TokenUsage(input_tokens=79, output_tokens=42) + + +async def test_zai_reports_split_reasoning_and_usage(service: FakeService) -> None: + load_case(service, "provider_sample_zai_glm_reasoning") + model = republic.get_model("zai:glm-4.7-flash", api_key="test-key", http_client=service.client()) + + response, _ = await respond( + model, [republic.system("You are a helpful assistant"), republic.user("Say hi")], max_tokens=4000 + ) + + assert service.requests[0].url == "https://api.z.ai/api/paas/v4/chat/completions" + assert service.requests[0].headers["authorization"] == "Bearer test-key" + body = service.body() + assert body["model"] == "glm-4.7-flash" + # Z.ai's GLM chat limits output with `max_tokens`; `max_completion_tokens` is not part of its API. + assert body["max_tokens"] == 4000 + assert "max_completion_tokens" not in body + # `reasoning_content` is the model's thinking, kept out of the answer text. + assert response.reasoning.startswith("1. **Analyze the Request:**") + assert response.text == "Olá! (Oh-lah)" + assert response.finish_reason == "stop" + assert response.token_usage == republic.TokenUsage( + input_tokens=15, output_tokens=249, reasoning_tokens=239, cached_tokens=3 + ) + + +async def test_zai_reasoning_content_returns_in_the_next_turn(service: FakeService) -> None: + load_case(service, "provider_sample_zai_glm_tool") + model = republic.get_model("zai:glm-4.7-flash", api_key="test-key", http_client=service.client()) + prompt = [republic.system("You are a helpful assistant"), republic.user("What's the weather in Florence?")] + + first, _ = await respond(model, prompt, max_tokens=4000, tools=[WEATHER], tool_choice="auto") + + assert first.finish_reason == "tool_calls" + assert len(first.tool_calls) == 1 + call = first.tool_calls[0] + assert call.name == "weather" + assert call.id == "call_-7807249954186984837" + assert call.args == {"location": "Florence,Italy"} + assert first.reasoning + + second, _ = await respond( + model, [*prompt, first.message, republic.tool(call, "40 C")], max_tokens=4000, tools=[WEATHER] + ) + + assert second.finish_reason == "stop" + assert "40" in second.text + assert_tool_exchange(service, "chat", [call], {"weather": "40 C"}) + assert service.body()["messages"][2]["reasoning_content"] == first.reasoning + + +async def test_grok_reports_reasoning_tokens_without_replaying_them(service: FakeService) -> None: + load_case(service, "provider_sample_grok_tool") + model = republic.get_model("grok:grok-4-fast", api_key="test-key", api_format="chat", http_client=service.client()) + prompt = [republic.system("You are a helpful assistant"), republic.user("What's the weather in Florence?")] + + first, _ = await respond(model, prompt, max_tokens=4000, tools=[WEATHER], tool_choice="auto") + + assert service.requests[0].url == "https://api.x.ai/v1/chat/completions" + assert first.finish_reason == "tool_calls" + assert len(first.tool_calls) == 1 + call = first.tool_calls[0] + assert call.name == "weather" + assert call.id == "call_98286056" + assert call.args == {"location": "Florence, Italy"} + assert first.token_usage == republic.TokenUsage( + input_tokens=361, output_tokens=26, reasoning_tokens=93, cached_tokens=153 + ) + + second, _ = await respond( + model, [*prompt, first.message, republic.tool(call, "40 C")], max_tokens=4000, tools=[WEATHER] + ) + + assert second.finish_reason == "stop" + assert "40" in second.text + assert_tool_exchange(service, "chat", [call], {"weather": "40 C"}) + assert "reasoning_content" not in service.body()["messages"][2] + + +async def test_openrouter_kimi_returns_recorded_call_identity(service: FakeService) -> None: + load_case(service, "provider_sample_openrouter_kimi_tool") + model = republic.get_model( + "openrouter:moonshotai/kimi-k2.5", api_key="test-key", api_format="chat", http_client=service.client() + ) + prompt = [republic.system("You are a helpful assistant"), republic.user("What's the weather in Florence?")] + + first, _ = await respond(model, prompt, max_tokens=4000, tools=[WEATHER], tool_choice="auto") + + assert service.requests[0].url == "https://openrouter.ai/api/v1/chat/completions" + assert first.finish_reason == "tool_calls" + assert len(first.tool_calls) == 1 + call = first.tool_calls[0] + assert call.name == "weather" + # The gateway keeps the upstream call id, which is not shaped like OpenAI's. + assert call.id == "functions.weather:0" + assert call.args == {"location": "Florence"} + assert first.reasoning.startswith(" The user is asking for the weather in Florence") + assert first.text == "" + + second, _ = await respond( + model, [*prompt, first.message, republic.tool(call, "40 C")], max_tokens=4000, tools=[WEATHER] + ) + + assert second.finish_reason == "stop" + assert "40" in second.text + assert_tool_exchange(service, "chat", [call], {"weather": "40 C"}) + + +async def test_openrouter_streams_reasoning_and_final_usage(service: FakeService) -> None: + load_case(service, "provider_sample_openrouter_grok_streaming") + model = republic.get_model( + "openrouter:x-ai/grok-4.1-fast", api_key="test-key", api_format="chat", http_client=service.client() + ) + + response, _ = await respond( + model, + [republic.system("You are a helpful assistant"), republic.user("Say hi")], + streaming=True, + max_tokens=4000, + ) + + assert response.finish_reason == "stop" + assert response.text == "Oi!" + # The recording carries OpenRouter's `: OPENROUTER PROCESSING` heartbeats and a null reasoning + # field on the closing chunks; only the `reasoning` text becomes reasoning. + assert response.reasoning.startswith("First, the core policies") + assert response.token_usage == republic.TokenUsage( + input_tokens=165, output_tokens=167, reasoning_tokens=165, cached_tokens=164 + ) + + +async def test_azure_uses_api_key_header_and_deployment_name(service: FakeService) -> None: + azure_api_base = "https://fantasy-playground-resource.openai.azure.com/openai/v1" + load_case(service, "provider_sample_azure_tool") + model = republic.get_model( + "azure-openai:gpt-5-mini", + api_key="test-key", + api_base=azure_api_base, + api_format="chat", + http_client=service.client(), + ) + prompt = [republic.system("You are a helpful assistant"), republic.user("What's the weather in Florence?")] + + first, _ = await respond(model, prompt, max_tokens=4000, tools=[WEATHER], tool_choice="auto") + + assert service.requests[0].url == f"{azure_api_base}/chat/completions" + # This API-key case uses `api-key`; Azure AD bearer auth is a separate supported auth mode. + assert service.requests[0].headers["api-key"] == "test-key" + assert "authorization" not in service.requests[0].headers + body = service.body() + assert body["model"] == "gpt-5-mini" + assert body["max_completion_tokens"] == 4000 + assert first.finish_reason == "tool_calls" + assert len(first.tool_calls) == 1 + call = first.tool_calls[0] + assert call.name == "weather" + assert call.id == "call_mbWCiHb6eF0qjzWrYgK52fWS" + assert call.args == {"location": "Florence, Italy"} + + second, _ = await respond( + model, [*prompt, first.message, republic.tool(call, "40 C")], max_tokens=4000, tools=[WEATHER] + ) + + assert second.finish_reason == "stop" + assert "40" in second.text + assert service.requests[1].headers["api-key"] == "test-key" + assert_tool_exchange(service, "chat", [call], {"weather": "40 C"}) diff --git a/tests/test_provider_golden/PROVIDER-SAMPLES.md b/tests/test_provider_golden/PROVIDER-SAMPLES.md new file mode 100644 index 0000000..166e469 --- /dev/null +++ b/tests/test_provider_golden/PROVIDER-SAMPLES.md @@ -0,0 +1,169 @@ +# Provider samples: provenance + +Inputs for `tests/test_provider_golden.py`. Every case under +`cases/provider_sample_*` here is a response body recorded by a real client against the real +service: six cases holding ten byte-verified bodies. Nothing here is a live recording made by us: +no credentials were used and no model was called, and re-recording is out of scope for a test-run +environment. + +Schema-constructed samples are not recordings and are deliberately absent from this file. + +Each entry records the local SHA-256, the immutable upstream location, the license, the exact +transformations applied, and how the sample is classified. Expected values in the tests come from +the recorded payload's fields and the contract evidence identified below, never observed Republic +output. Republic source identifies routing and local integration conventions; +source inspection alone does not establish a provider-wide wire requirement. + +## Case index + +| Case | Provider family | Kind | Bodies | +| --- | --- | --- | --- | +| `provider_sample_zai_glm_reasoning` | Z.ai (ZAI) | recorded | `0.json` | +| `provider_sample_zai_glm_tool` | Z.ai (ZAI) | recorded | `0.json`, `1.json` | +| `provider_sample_grok_tool` | Grok (xAI direct) | recorded | `0.json`, `1.json` | +| `provider_sample_openrouter_kimi_tool` | OpenRouter (Kimi upstream) | recorded | `0.json`, `1.json` | +| `provider_sample_openrouter_grok_streaming` | OpenRouter (xAI upstream) | recorded | `0.sse` | +| `provider_sample_azure_tool` | Azure OpenAI | recorded | `0.json`, `1.json` | + +Bodies are numbered in the order Republic sends them. A `tool*` case holds the first response (the +model's tool call) and the second (the answer after the caller's tool output), so the recorded +request bodies also show what a real client sent back. + +## Recorded samples: Fantasy VCR cassettes + +Source repository: [`charmbracelet/fantasy`](https://github.com/charmbracelet/fantasy) at commit +`b0fadc9ae2e7da81a5d0262563d3ced9b594bd75`, license Apache-2.0 +([`LICENSE`](https://github.com/charmbracelet/fantasy/blob/b0fadc9ae2e7da81a5d0262563d3ced9b594bd75/LICENSE), +SHA-256 `0d542e0c8804e39aa7f37eb00da5a762149dc682d7829451287e11b938e94594`). A copy is kept beside +this file as `LICENSE.fantasy`. + +These cassettes are `go-vcr` recordings of the provider's own test suite: each interaction stores +a real outgoing request and the real response body, and the suite replays them through an HTTP +transport. The bodies below are the `response.body` field verbatim, so they are genuine +provider-authored API responses, not hand-written fixtures. + +We fetch from `raw.githubusercontent.com` at the pinned commit and compare SHA-256 against the same +path in the commit tarball, so the bytes are verified against two independent transports. + +### `provider_sample_zai_glm_reasoning/0.json` + +- Case: `TestOpenAICompatibleCommon/zai-glm-4-7-flash/simple.yaml` +- Path: `providertests/testdata/TestOpenAICompatibleCommon/zai-glm-4-7-flash/simple.yaml` +- Upstream file SHA-256: `f435f5086e66e034a1629a0d028026fbc1aad6ea17a8b77b0777fd937b86315d` +- Local SHA-256: `ef3ef5b8a4d45d15c6f213693de9a31daa0d368f2b83297490605d5fdc73bbb6` +- Classification: recorded, upstream real response +- Transformation: the YAML `response.body` string is written out as bytes. No reformatting, no field + edits, no re-encoding. +- Why it is necessary: GLM chat answers with `reasoning_content` beside `content`, reports + `completion_tokens_details.reasoning_tokens` and `prompt_tokens_details.cached_tokens`. + This is the only family here whose recorded + non-streaming answer keeps thinking and answer text apart in one message. +- Limitations: the recording is one short greeting; it exercises usage and reasoning decoding, not + a long answer. Its request uses `/api/coding/paas/v4/chat/completions`; it does not establish + Republic's default `/api/paas/v4` base. The URL assertion checks Republic's configured routing, + not a route verified by this cassette. The cassette uses `max_tokens`, so it does not show what + Z.ai does when + `max_completion_tokens` is sent — the test asserts Republic's own mapping instead. + +### `provider_sample_zai_glm_tool/0.json`, `1.json` + +- Case: `TestOpenAICompatibleCommon/zai-glm-4-7-flash/tool.yaml` +- Upstream file SHA-256: `dc98aee75f2f046941fb6c271d7e4a5ee3900a8cd2363d0b589375482867e5f0` +- Local SHA-256: `8d417387f2e02926bee4edddd39b9edae3c60471e6a62f4a8531386d0b235fa0` (`0.json`), + `087bcb0050ff0c982fc6eb97ecff5e23febd56354c1dabe9f5a910a47ed2149e` (`1.json`) +- Classification: recorded, upstream real responses +- Transformation: byte-identical extraction of both `response.body` values. +- Why it is necessary: the round trip is the point. The first response carries a Z.ai tool call + (`call_-7807249954186984837`) and thinking; the recorded second request shows the real client + resending that thinking as `reasoning_content`. Republic's `MaxTokensReasoningContentChat` exists + for exactly this, and only a recorded pair can show it. +- Limitations: one tool, one turn. The recorded arguments `{"location":"Florence,Italy"}` are + treated as an opaque string by the test, which compares them to the decoded call rather than + re-encoding them. + +### `provider_sample_grok_tool/0.json`, `1.json` + +- Case: `TestOpenAICompatibleCommon/xai-grok-4-fast/tool.yaml` +- Upstream file SHA-256: `2ace0fca86c730b56e7efca378bf919d418fbf1407bf242bdea4c243997b5f28` +- Local SHA-256: `6c674d6b5a3f86e88e18338e67b279c7cf75439fe66751c7002d972746d349cc` (`0.json`), + `ad6987149618b8f1ec90b2cd953670234e98d47453ca77884a0b6d370eeac718` (`1.json`) +- Classification: recorded, upstream real responses +- Transformation: byte-identical extraction of both `response.body` values. +- Why it is necessary: this is xAI's own `api.x.ai` endpoint, not a gateway. It shows Grok's + non-OpenAI usage envelope (`reasoning_tokens`, `cached_tokens`, `cost_in_usd_ticks`) and that no + reasoning field is replayed on the next turn — the counterpart to the Z.ai case. +- Limitations: `grok-4-fast-reasoning` reports reasoning tokens but no reasoning text in this + recording, so the test asserts token accounting and replay shape, not reasoning decoding. + Republic's `Grok` provider uses the xAI OAuth login by default; the test passes + `api_key="test-key"` with `api_format="chat"`, because a bearer header is what the recorded + request used. The OAuth flow itself is covered by `tests/test_grok.py`. + +### `provider_sample_openrouter_kimi_tool/0.json`, `1.json` + +- Case: `TestOpenRouterCommon/kimi/tool.yaml` +- Upstream file SHA-256: `bfc1bca915d600116d6aabf58f121dd30e7e11b713f8cd088a9743444d0293d4` +- Local SHA-256: `4fcf68ece5e374717a4c89c2c8f43142c31ebaca91c5a1792c53b74fb5cb215f` (`0.json`), + `f6e81e87249dab8a9c0da78196de3d195781c7bbab0c4843ceb644d08bd33b5c` (`1.json`) +- Classification: recorded, upstream real responses +- Transformation: byte-identical extraction of both `response.body` values. The recorded bodies + begin with several blank/whitespace lines from the gateway; these are kept as recorded. +- Why it is necessary: the tool-call id `functions.weather:0` is a gateway-shaped id, unlike + OpenAI's `call_…`. It proves Republic neither rewrites nor rejects foreign ids, and that the id + survives into the outgoing `tool_call_id`. Kimi's thinking arrives in `reasoning`, the field + Republic reads for gateways. +- Limitations: Kimi is reached through OpenRouter here; the direct Moonshot provider differs only in + base URL and dialect hooks, which the shared corpus already covers. The `provider` field of the + recording is whichever upstream OpenRouter routed to, so it is not a stable Moonshot fact. + +### `provider_sample_openrouter_grok_streaming/0.sse` + +- Case: `TestOpenRouterCommon/grok/simple_streaming.yaml` +- Upstream file SHA-256: `4f22a8f7c52af1223f70b2f99ef7ee090576c731a35025ceef9d669d59c0d047` +- Local SHA-256: `3f9e271722ffa27d7a65d58f4e402680b0dfb15449a702defd037567f743343b` +- Classification: recorded, upstream real SSE stream +- Transformation: the YAML block-scalar `response.body` is written out as bytes unchanged, + including the three `: OPENROUTER PROCESSING` comment lines and the trailing `data: [DONE]`. +- Why it is necessary: OpenRouter streams `reasoning` incrementally with no `reasoning_content`, + sends usage in a final chunk after the finish reason, and emits SSE comments as keepalives. It is + the streaming counterpart to the Kimi case and the only sample here with gateway keepalives. +- Limitations: xAI is the upstream, so it does not distinguish OpenRouter from xAI beyond the + gateway envelope. The recorded request used `"usage":{"include":true}` (an OpenRouter body + extension); Republic sends `stream_options.include_usage`. The response recording alone does + not validate that request option. + +### `provider_sample_azure_tool/0.json`, `1.json` + +- Case: `TestAzureCommon/azure-gpt-5-mini/tool.yaml` +- Upstream file SHA-256: `4848812cfd263a7ea3a32d0cd1ab3c8dda0f8137566a85ca2a4ff5a9b29789de` +- Local SHA-256: `793f9d33718111aecacaf0ac23441a448c9b52cd89fbd899b51b518c2d2522ae` (`0.json`), + `7c6d0e0708e0e4d8885fea90d3041e224d2553d3ab0786ed1e594c5373a71251` (`1.json`) +- Classification: recorded, upstream real responses +- Transformation: byte-identical extraction of both `response.body` values. The recorded host + `fantasy-playground-resource.openai.azure.com` is kept inside the body only insofar as it appears + there; the test's `api_base` uses the same resource name so the URL is not invented. +- Why it is necessary: Azure's v1 API answers with extra `content_filter_results` and + `prompt_filter_results` blocks, names the model `gpt-5-mini-2025-08-07`, and authenticates with + `api-key`. It is the only sample that exercises the Azure auth header and deployment naming. +- Limitations: the recording is chat, not the Responses API. It contains no `api-key` header of its + own — the VCR matches only method/URL/body — so the header assertion comes from Republic's + `AzureOpenAI._api_key_auth`, not from the recording. Independent evidence is the pinned OpenAI + SDK `src/openai/lib/azure.py` at `59738a289ee82c7b86bf1e10230cceb6daa9716d`, SHA-256 + `002f5e8b264e0b28534855f1e0245e5d2b8591a1e696bc872de4e9a98508322a`. It supports + both API-key and Azure AD bearer modes; only API-key mode is sampled here. + +## Families not sampled here + +The following supported families have no case in the recorded corpus. In short: `deepseek`, +`moonshot` (direct), `ollama` +and `together` reuse the shared chat mechanics already covered by `test_compatible_providers.py` +and `test_provider_golden.py`, and adding a fifth near-identical body would not test a new +distinction. `minimax`'s `reasoning_split` request field and default Messages routing are covered by +existing tests; no pinned recording of its `reasoning_details` reply was found in the sources above. + +## Reproducing + +```console +uv run pytest tests/test_provider_golden.py -q +``` + +No network access, credentials, or model calls are needed; `FakeService` answers from these files. diff --git a/tests/test_provider_golden/README.md b/tests/test_provider_golden/README.md index dfbdd3f..3f8f360 100644 --- a/tests/test_provider_golden/README.md +++ b/tests/test_provider_golden/README.md @@ -146,3 +146,12 @@ event accounting uses the common golden stream check. The existing The input carries three interleaved choices; only choice 0 reaches the single-response API. The case uses the common golden runner and streaming consistency assertions; there is no second copy of the input in the format tests. + + +## Additional provider recordings + +The `provider_sample_*` cases exercise Z.ai reasoning/usage and reasoning replay, xAI usage, +OpenRouter tool identity and streaming reasoning, and Azure authentication/routing through the +same golden runner and numbered response format. `PROVIDER-SAMPLES.md` pins their Fantasy cassette +sources, transformations and response hashes. Recorded provenance describes the inputs; it does not +make Republic's own request conventions evidence of the recorded service's requirements. diff --git a/tests/test_provider_golden/cases/provider_sample_azure_tool/0.json b/tests/test_provider_golden/cases/provider_sample_azure_tool/0.json new file mode 100644 index 0000000..d8ca940 --- /dev/null +++ b/tests/test_provider_golden/cases/provider_sample_azure_tool/0.json @@ -0,0 +1 @@ +{"choices":[{"content_filter_results":{},"finish_reason":"tool_calls","index":0,"logprobs":null,"message":{"annotations":[],"content":null,"refusal":null,"role":"assistant","tool_calls":[{"function":{"arguments":"{\"location\":\"Florence, Italy\"}","name":"weather"},"id":"call_mbWCiHb6eF0qjzWrYgK52fWS","type":"function"}]}}],"created":1782321616,"id":"chatcmpl-DuLYmgf20hGu0QAyj7XLR7WSNqeW0","model":"gpt-5-mini-2025-08-07","object":"chat.completion","prompt_filter_results":[{"prompt_index":0,"content_filter_results":{"hate":{"filtered":false,"severity":"safe"},"jailbreak":{"detected":false,"filtered":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}}}],"service_tier":"default","system_fingerprint":null,"usage":{"completion_tokens":26,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"latency_checkpoint":{"engine_tbt_ms":11,"engine_ttft_ms":67,"engine_ttlt_ms":1042,"pre_inference_ms":162,"service_tbt_ms":11,"service_ttft_ms":615,"service_ttlt_ms":1584,"total_duration_ms":1432,"user_visible_ttft_ms":453},"prompt_tokens":145,"prompt_tokens_details":{"audio_tokens":0,"cached_tokens":0},"total_tokens":171}} diff --git a/tests/test_provider_golden/cases/provider_sample_azure_tool/1.json b/tests/test_provider_golden/cases/provider_sample_azure_tool/1.json new file mode 100644 index 0000000..efd7fb1 --- /dev/null +++ b/tests/test_provider_golden/cases/provider_sample_azure_tool/1.json @@ -0,0 +1 @@ +{"choices":[{"content_filter_results":{"hate":{"filtered":false,"severity":"safe"},"protected_material_code":{"detected":false,"filtered":false},"protected_material_text":{"detected":false,"filtered":false},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"finish_reason":"stop","index":0,"logprobs":null,"message":{"annotations":[],"content":"Current temperature in Florence, Italy: 40 °C (104 °F).\n\nThat’s quite hot — recommendations: stay hydrated, avoid strenuous outdoor activity during midday, seek shade or air-conditioned places, wear light clothing and sunscreen, and check for local heat advisories. Would you like an hourly forecast, the next few days, or details like humidity and wind?","refusal":null,"role":"assistant"}}],"created":1782321618,"id":"chatcmpl-DuLYopOLgCaC41qFg4mR61lgZ1JgH","model":"gpt-5-mini-2025-08-07","object":"chat.completion","prompt_filter_results":[{"prompt_index":0,"content_filter_results":{}}],"service_tier":"default","system_fingerprint":null,"usage":{"completion_tokens":339,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":256,"rejected_prediction_tokens":0},"latency_checkpoint":{"engine_tbt_ms":11,"engine_ttft_ms":39,"engine_ttlt_ms":4111,"pre_inference_ms":232,"service_tbt_ms":11,"service_ttft_ms":675,"service_ttlt_ms":4729,"total_duration_ms":4505,"user_visible_ttft_ms":442},"prompt_tokens":176,"prompt_tokens_details":{"audio_tokens":0,"cached_tokens":0},"total_tokens":515}} diff --git a/tests/test_provider_golden/cases/provider_sample_grok_tool/0.json b/tests/test_provider_golden/cases/provider_sample_grok_tool/0.json new file mode 100644 index 0000000..f7fea95 --- /dev/null +++ b/tests/test_provider_golden/cases/provider_sample_grok_tool/0.json @@ -0,0 +1 @@ +{"id":"546bcc82-b37d-ca5d-aee1-0f2d503ee0b8","object":"chat.completion","created":1773345550,"model":"grok-4-fast-reasoning","choices":[{"index":0,"message":{"role":"assistant","content":"","tool_calls":[{"id":"call_98286056","function":{"name":"weather","arguments":"{\"location\":\"Florence, Italy\"}"},"type":"function"}],"refusal":null},"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":361,"completion_tokens":26,"total_tokens":480,"prompt_tokens_details":{"text_tokens":361,"audio_tokens":0,"image_tokens":0,"cached_tokens":153},"completion_tokens_details":{"reasoning_tokens":93,"audio_tokens":0,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0},"num_sources_used":0,"cost_in_usd_ticks":1087500},"system_fingerprint":"fp_3d7ed3b779"} \ No newline at end of file diff --git a/tests/test_provider_golden/cases/provider_sample_grok_tool/1.json b/tests/test_provider_golden/cases/provider_sample_grok_tool/1.json new file mode 100644 index 0000000..8ea77c3 --- /dev/null +++ b/tests/test_provider_golden/cases/provider_sample_grok_tool/1.json @@ -0,0 +1 @@ +{"id":"533294f1-af60-2023-e993-8ab87543ed84","object":"chat.completion","created":1773345552,"model":"grok-4-fast-reasoning","choices":[{"index":0,"message":{"role":"assistant","content":"The current weather in Florence, Italy, is 40°C. It feels quite hot—stay hydrated! If you need more details like forecast or conditions, let me know.","refusal":null},"finish_reason":"stop"}],"usage":{"prompt_tokens":493,"completion_tokens":35,"total_tokens":582,"prompt_tokens_details":{"text_tokens":493,"audio_tokens":0,"image_tokens":0,"cached_tokens":481},"completion_tokens_details":{"reasoning_tokens":54,"audio_tokens":0,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0},"num_sources_used":0,"cost_in_usd_ticks":709500},"system_fingerprint":"fp_3d7ed3b779"} \ No newline at end of file diff --git a/tests/test_provider_golden/cases/provider_sample_openrouter_grok_streaming/0.sse b/tests/test_provider_golden/cases/provider_sample_openrouter_grok_streaming/0.sse new file mode 100644 index 0000000..3430db7 --- /dev/null +++ b/tests/test_provider_golden/cases/provider_sample_openrouter_grok_streaming/0.sse @@ -0,0 +1,100 @@ +: OPENROUTER PROCESSING + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":"First","reasoning_details":[{"type":"reasoning.summary","summary":"First","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":",","reasoning_details":[{"type":"reasoning.summary","summary":",","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":" the","reasoning_details":[{"type":"reasoning.summary","summary":" the","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":" core","reasoning_details":[{"type":"reasoning.summary","summary":" core","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":" policies","reasoning_details":[{"type":"reasoning.summary","summary":" policies","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":":","reasoning_details":[{"type":"reasoning.summary","summary":":","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":" No","reasoning_details":[{"type":"reasoning.summary","summary":" No","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":" assistance","reasoning_details":[{"type":"reasoning.summary","summary":" assistance","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":" with","reasoning_details":[{"type":"reasoning.summary","summary":" with","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":" criminal","reasoning_details":[{"type":"reasoning.summary","summary":" criminal","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":" activity","reasoning_details":[{"type":"reasoning.summary","summary":" activity","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":",","reasoning_details":[{"type":"reasoning.summary","summary":",","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":" no","reasoning_details":[{"type":"reasoning.summary","summary":" no","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":" overly","reasoning_details":[{"type":"reasoning.summary","summary":" overly","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":" realistic","reasoning_details":[{"type":"reasoning.summary","summary":" realistic","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":" help","reasoning_details":[{"type":"reasoning.summary","summary":" help","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":" with","reasoning_details":[{"type":"reasoning.summary","summary":" with","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":" crimes","reasoning_details":[{"type":"reasoning.summary","summary":" crimes","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":" even","reasoning_details":[{"type":"reasoning.summary","summary":" even","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":" in","reasoning_details":[{"type":"reasoning.summary","summary":" in","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":" hypot","reasoning_details":[{"type":"reasoning.summary","summary":" hypot","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":"het","reasoning_details":[{"type":"reasoning.summary","summary":"het","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":"icals","reasoning_details":[{"type":"reasoning.summary","summary":"icals","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":",","reasoning_details":[{"type":"reasoning.summary","summary":",","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":" short","reasoning_details":[{"type":"reasoning.summary","summary":" short","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":" responses","reasoning_details":[{"type":"reasoning.summary","summary":" responses","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":" to","reasoning_details":[{"type":"reasoning.summary","summary":" to","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":" jail","reasoning_details":[{"type":"reasoning.summary","summary":" jail","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":"break","reasoning_details":[{"type":"reasoning.summary","summary":"break","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":" attempts","reasoning_details":[{"type":"reasoning.summary","summary":" attempts","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":".","reasoning_details":[{"type":"reasoning.summary","summary":".","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":" This","reasoning_details":[{"type":"reasoning.summary","summary":" This","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":" query","reasoning_details":[{"type":"reasoning.summary","summary":" query","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":" is","reasoning_details":[{"type":"reasoning.summary","summary":" is","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":" harmless","reasoning_details":[{"type":"reasoning.summary","summary":" harmless","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":":","reasoning_details":[{"type":"reasoning.summary","summary":":","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":" \"","reasoning_details":[{"type":"reasoning.summary","summary":" \"","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":"Say","reasoning_details":[{"type":"reasoning.summary","summary":"Say","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":" hi","reasoning_details":[{"type":"reasoning.summary","summary":" hi","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":" in","reasoning_details":[{"type":"reasoning.summary","summary":" in","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":" Po...","reasoning_details":[{"type":"reasoning.summary","summary":" Po...","format":"xai-responses-v1","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +: OPENROUTER PROCESSING + +: OPENROUTER PROCESSING + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":null,"reasoning_details":[{"type":"reasoning.encrypted","data":"B/XIFKX05qXzekUEb12SSn4BIjmJ51LDdv30TMRAsHya7nZDMRR6fgLO2bRGyvmcxJoBh+zTWaizcAgiPaPWCDhgCNDv9GzUBhAgYnAbtNISeOGG2SSVoTIMSOcH77dXVLUVAlF0fbYbrebbJcJl9mDhvP09ddekxG9flwdVBDK/4uZ7yomvH8dS1lG34NeWpzbivkqQRDvPWOBBFNQzW3gE01nRHZArutRBieQyKDuMQPwXsorJuA1glLg5YrRfAljVpuq8NJIqzwfzgx1iwJtxULMnKDJNnJkEJPcANc+HarCdRyPWgfdylm/UMGQV12uTJJ6d3fJM5Hq8mV5E1R0oVbRqVPOGC+U60v49tLKG/VSxpFvRajMXTotmMCWpfyKb68NJq2yxT7IFjBoDxx7+UVHyCXVO5VKTfeac3lIWeorbzXrtNQH9x1bHaNCIrH/Y3kFiSQ2GjOX5tlFYWip8U0dbLlQplTKnEajVep3GF+HESe9KdmnIsB2wrtKrz54wXUiPj8wuo1uKWPdBgiI1EFUBOXro4V+GGvjQiKZlH02l7S7GEmgW0b40fMf5evJzqjXjY2OwR+RKkXnDEtqqRSoeA146l7mz/QZd44ZHuzlRwZq/FpRUhq2hbBEQTysy73XIN2yMB/H8o6tevCo1UOo95Nxj6GAS/dGzUglexXYOUGRkkaSlBTr4n4WHpzN0H4gryrFQCsx3EhZIbR/W0SmNUC7KUhTvNHBOng90+0j1+hIMtYMip+fjxrwS1yu7UBZe6bFKqHaSUSXuL+VoS3vgrIOVThBNNWW7teu+2/LTkWr9WwKqv+GR0Cq1dZARGEglGmr96CEzTBTOYgRGBTkZk6bp3HpS10X8UkKMzIBHo+ttrzwft15DZOILgT4wZsYDvR5M3kKQBm2QRBYgkC8gIWTWtelgJ8Xa/A65/2ck8+Gu5VxwC3e/HsMytwuwQFChSl5OqUCmMiLGJgUR3zg","format":"xai-responses-v1","id":"rs_b69ec351-62d7-3ea7-bb06-d9699f84799e","index":0}]},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"Oi","role":"assistant"},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","choices":[{"index":0,"delta":{"content":"!","role":"assistant"},"finish_reason":null,"native_finish_reason":null}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","service_tier":"default","choices":[{"index":0,"delta":{"content":"","role":"assistant","reasoning":null},"finish_reason":"stop","native_finish_reason":"completed"}]} + +data: {"id":"gen-1777319837-oWuxtkDuFDqmpKXSp8it","object":"chat.completion.chunk","created":1777319837,"model":"x-ai/grok-4.1-fast","provider":"xAI","service_tier":"default","choices":[{"index":0,"delta":{"content":"","role":"assistant"},"finish_reason":"stop","native_finish_reason":"completed"}],"usage":{"prompt_tokens":165,"completion_tokens":167,"total_tokens":332,"cost":0.0000919,"is_byok":false,"prompt_tokens_details":{"cached_tokens":164,"cache_write_tokens":0,"audio_tokens":0,"video_tokens":0},"cost_details":{"upstream_inference_cost":0.0000919,"upstream_inference_prompt_cost":0.0000084,"upstream_inference_completions_cost":0.0000835},"completion_tokens_details":{"reasoning_tokens":165,"image_tokens":0,"audio_tokens":0}}} + +data: [DONE] + diff --git a/tests/test_provider_golden/cases/provider_sample_openrouter_kimi_tool/0.json b/tests/test_provider_golden/cases/provider_sample_openrouter_kimi_tool/0.json new file mode 100644 index 0000000..9fa8dd3 --- /dev/null +++ b/tests/test_provider_golden/cases/provider_sample_openrouter_kimi_tool/0.json @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + +{"id":"gen-1777319891-AbXaksK95wX8kHwAXxXX","object":"chat.completion","created":1777319891,"model":"moonshotai/kimi-k2.5-0127","provider":"Chutes","system_fingerprint":null,"choices":[{"index":0,"logprobs":null,"finish_reason":"tool_calls","native_finish_reason":"tool_calls","message":{"role":"assistant","content":null,"refusal":null,"reasoning":" The user is asking for the weather in Florence, Italy. I have a weather function available that takes a location parameter (which should be the city). I should call the weather function with \"Florence\" as the location.","tool_calls":[{"type":"function","index":0,"id":"functions.weather:0","function":{"name":"weather","arguments":"{\"location\": \"Florence\"}"}}],"reasoning_details":[{"type":"reasoning.text","text":" The user is asking for the weather in Florence, Italy. I have a weather function available that takes a location parameter (which should be the city). I should call the weather function with \"Florence\" as the location.","format":"unknown","index":0}]}}],"usage":{"prompt_tokens":69,"completion_tokens":66,"total_tokens":135,"cost":0.00016236,"is_byok":false,"prompt_tokens_details":{"cached_tokens":0,"cache_write_tokens":0,"audio_tokens":0,"video_tokens":0},"cost_details":{"upstream_inference_cost":0.00016236,"upstream_inference_prompt_cost":0.00003036,"upstream_inference_completions_cost":0.000132},"completion_tokens_details":{"reasoning_tokens":47,"image_tokens":0,"audio_tokens":0}}} \ No newline at end of file diff --git a/tests/test_provider_golden/cases/provider_sample_openrouter_kimi_tool/1.json b/tests/test_provider_golden/cases/provider_sample_openrouter_kimi_tool/1.json new file mode 100644 index 0000000..b9c8c44 --- /dev/null +++ b/tests/test_provider_golden/cases/provider_sample_openrouter_kimi_tool/1.json @@ -0,0 +1,39 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +{"id":"gen-1777319895-NVPGjsD98YPuLRBFo9BH","object":"chat.completion","created":1777319895,"model":"moonshotai/kimi-k2.5-0127","provider":"Chutes","system_fingerprint":null,"choices":[{"index":0,"logprobs":null,"finish_reason":"stop","native_finish_reason":"stop","message":{"role":"assistant","content":" The weather in Florence, Italy is currently **40°C** (104°F). That's quite hot!","refusal":null,"reasoning":" The weather in Florence, Italy is currently 40°C. That's quite hot - about 104°F. This seems like it might be unusually warm weather for Florence, possibly during a heat wave or in the peak of summer.","reasoning_details":[{"type":"reasoning.text","text":" The weather in Florence, Italy is currently 40°C. That's quite hot - about 104°F. This seems like it might be unusually warm weather for Florence, possibly during a heat wave or in the peak of summer.","format":"unknown","index":0}]}}],"usage":{"prompt_tokens":108,"completion_tokens":67,"total_tokens":175,"cost":0.00018152,"is_byok":false,"prompt_tokens_details":{"cached_tokens":0,"cache_write_tokens":0,"audio_tokens":0,"video_tokens":0},"cost_details":{"upstream_inference_cost":0.00018152,"upstream_inference_prompt_cost":0.00004752,"upstream_inference_completions_cost":0.000134},"completion_tokens_details":{"reasoning_tokens":46,"image_tokens":0,"audio_tokens":0}}} \ No newline at end of file diff --git a/tests/test_provider_golden/cases/provider_sample_zai_glm_reasoning/0.json b/tests/test_provider_golden/cases/provider_sample_zai_glm_reasoning/0.json new file mode 100644 index 0000000..47cd164 --- /dev/null +++ b/tests/test_provider_golden/cases/provider_sample_zai_glm_reasoning/0.json @@ -0,0 +1 @@ +{"choices":[{"finish_reason":"stop","index":0,"message":{"content":"Olá! (Oh-lah)","reasoning_content":"1. **Analyze the Request:** The user wants me to say \"hi\" in Portuguese.\n\n2. **Identify the Language:** Portuguese.\n\n3. **Identify the Translation for \"Hi\":**\n * Standard/Common greeting: \"Olá\"\n * Informal/Casual greeting: \"Oi\"\n * Pronunciation guides (optional but helpful): \"oh-lah\", \"oy\"\n\n4. **Formulate the Response:**\n * Option 1: Just \"Olá\" (too brief).\n * Option 2: \"Olá!\" (better).\n * Option 3: \"Olá! Oi!\" (covers formal and informal).\n * Option 4: Include pronunciation for better UX. \"Olá! (oh-lah)\"\n\n5. **Select the Best Response:** \"Olá!\" is the standard, universally understood greeting. I will also provide the phonetic pronunciation to be helpful.\n\n * *Draft:* \"Olá! (oh-lah)\"\n\n6. **Final Polish:** Keep it simple and friendly.\n\n * Output: Olá!","role":"assistant"}}],"created":1773349367,"id":"20260313050242595b2cb031a74737","model":"glm-4.7-flash","object":"chat.completion","request_id":"20260313050242595b2cb031a74737","usage":{"completion_tokens":249,"completion_tokens_details":{"reasoning_tokens":239},"prompt_tokens":15,"prompt_tokens_details":{"cached_tokens":3},"total_tokens":264}} \ No newline at end of file diff --git a/tests/test_provider_golden/cases/provider_sample_zai_glm_tool/0.json b/tests/test_provider_golden/cases/provider_sample_zai_glm_tool/0.json new file mode 100644 index 0000000..dfaff5e --- /dev/null +++ b/tests/test_provider_golden/cases/provider_sample_zai_glm_tool/0.json @@ -0,0 +1 @@ +{"choices":[{"finish_reason":"tool_calls","index":0,"message":{"content":"I'll check the weather in Florence, Italy for you.","reasoning_content":"The user is asking for the weather in Florence, Italy. I have a weather function available that can get weather information for a location. The function requires a \"location\" parameter, and the user has provided \"Florence,Italy\" as the location. I should use this function to get the weather information.","role":"assistant","tool_calls":[{"function":{"arguments":"{\"location\":\"Florence,Italy\"}","name":"weather"},"id":"call_-7807249954186984837","index":0,"type":"function"}]}}],"created":1773349384,"id":"202603130503035652f6eb725d4b48","model":"glm-4.7-flash","object":"chat.completion","request_id":"202603130503035652f6eb725d4b48","usage":{"completion_tokens":88,"completion_tokens_details":{"reasoning_tokens":62},"prompt_tokens":173,"prompt_tokens_details":{"cached_tokens":43},"total_tokens":261}} \ No newline at end of file diff --git a/tests/test_provider_golden/cases/provider_sample_zai_glm_tool/1.json b/tests/test_provider_golden/cases/provider_sample_zai_glm_tool/1.json new file mode 100644 index 0000000..8cdf17e --- /dev/null +++ b/tests/test_provider_golden/cases/provider_sample_zai_glm_tool/1.json @@ -0,0 +1 @@ +{"choices":[{"finish_reason":"stop","index":0,"message":{"content":"The weather in Florence, Italy is currently 40°C (104°F). That's quite hot! Make sure to stay hydrated and seek shade if you're out and about.","reasoning_content":"The weather function returned \"40 C\" which indicates the temperature in Florence, Italy is 40 degrees Celsius. This seems quite hot, which would be consistent with Florence being in Italy and possibly experiencing hot weather. I should provide this information to the user in a clear and helpful way.","role":"assistant"}}],"created":1773349386,"id":"20260313050305e26cddfc278c448a","model":"glm-4.7-flash","object":"chat.completion","request_id":"20260313050305e26cddfc278c448a","usage":{"completion_tokens":94,"completion_tokens_details":{"reasoning_tokens":57},"prompt_tokens":267,"prompt_tokens_details":{"cached_tokens":173},"total_tokens":361}} \ No newline at end of file From 809ecce383dba8ea484c41f8b2d25c96a73afab8 Mon Sep 17 00:00:00 2001 From: Chojan Shang Date: Sat, 14 Nov 2026 00:06:27 +0800 Subject: [PATCH 3/4] fix(responses): preserve output around null items --- src/republic/formats/responses.py | 5 ++++ tests/test_provider_golden.py | 10 +++++++ tests/test_provider_golden/README.md | 12 ++++++++ .../0.sse | 28 +++++++++++++++++++ 4 files changed, 55 insertions(+) create mode 100644 tests/test_provider_golden/cases/responses_null_output_items_streaming/0.sse diff --git a/src/republic/formats/responses.py b/src/republic/formats/responses.py index 8e6a3cd..af1078c 100644 --- a/src/republic/formats/responses.py +++ b/src/republic/formats/responses.py @@ -142,6 +142,11 @@ def feed(self, event: str, data: str) -> Iterable[Delta]: payload = json.loads(data) if payload.get("type") in {"response.completed", "response.incomplete"}: self.completed = True + if ( + payload.get("type") in {"response.output_item.added", "response.output_item.done"} + and payload.get("item") is None + ): + return yield from self._feed_payload(payload) def _feed_payload(self, payload: Mapping[str, Any]) -> Iterable[Delta]: diff --git a/tests/test_provider_golden.py b/tests/test_provider_golden.py index d0082d9..2900ef0 100644 --- a/tests/test_provider_golden.py +++ b/tests/test_provider_golden.py @@ -738,3 +738,13 @@ async def test_azure_uses_api_key_header_and_deployment_name(service: FakeServic assert "40" in second.text assert service.requests[1].headers["api-key"] == "test-key" assert_tool_exchange(service, "chat", [call], {"weather": "40 C"}) + + +async def test_responses_null_items_preserve_text_and_tools(service: FakeService) -> None: + load_case(service, "responses_null_output_items_streaming") + + response, _ = await respond(make_model(service, "responses"), [republic.user("Hi")], streaming=True) + + assert response.text == '{"answer":4}' + assert response.tool_calls == [republic.ToolCall("call_test", "lookup", '{"answer":4}')] + assert response.finish_reason == "tool_calls" diff --git a/tests/test_provider_golden/README.md b/tests/test_provider_golden/README.md index 3f8f360..0fa1676 100644 --- a/tests/test_provider_golden/README.md +++ b/tests/test_provider_golden/README.md @@ -155,3 +155,15 @@ OpenRouter tool identity and streaming reasoning, and Azure authentication/routi same golden runner and numbered response format. `PROVIDER-SAMPLES.md` pins their Fantasy cassette sources, transformations and response hashes. Recorded provenance describes the inputs; it does not make Republic's own request conventions evidence of the recorded service's requirements. + + +## Responses null-item regression + +`cases/responses_null_output_items_streaming/0.sse` adapts the synthetic input in +[openai-python@59738a2](https://github.com/openai/openai-python/blob/59738a289ee82c7b86bf1e10230cceb6daa9716d/tests/lib/responses/test_null_output_items.py) +(Apache-2.0; `LICENSE.openai`). Its SHA-256 is +`609c654ee2aa50d590995e15d2515009e868651c02e65c7321d177834ffb8302`. +The upstream construction is serialized as SSE, retaining event order and sequence numbers. +It is not a production recording and does not establish that null items mean keepalives. +The input is replayed through the OpenAI Responses provider. Valid text and calls following +empty output-item events survive. This case does not establish non-streaming or Copilot behavior. diff --git a/tests/test_provider_golden/cases/responses_null_output_items_streaming/0.sse b/tests/test_provider_golden/cases/responses_null_output_items_streaming/0.sse new file mode 100644 index 0000000..e14903a --- /dev/null +++ b/tests/test_provider_golden/cases/responses_null_output_items_streaming/0.sse @@ -0,0 +1,28 @@ +data: {"type": "response.created", "response": {"id": "resp_test", "output": []}, "sequence_number": 0} + +data: {"type": "response.output_item.added", "output_index": 0, "item": null, "sequence_number": 1} + +data: {"type": "response.output_item.added", "output_index": 0, "item": {"id": "msg_test", "type": "message", "role": "assistant", "status": "in_progress", "content": []}, "sequence_number": 2} + +data: {"type": "response.output_item.added", "output_index": 0, "item": null, "sequence_number": 3} + +data: {"type": "response.output_item.added", "output_index": 1, "item": {"id": "fc_test", "type": "function_call", "call_id": "call_test", "name": "lookup", "arguments": "", "status": "in_progress"}, "sequence_number": 4} + +data: {"type": "response.content_part.added", "output_index": 0, "content_index": 0, "part": {"type": "output_text", "text": "", "annotations": [], "logprobs": []}, "sequence_number": 5} + +data: {"type": "response.output_text.delta", "output_index": 0, "content_index": 0, "item_id": "msg_test", "delta": "{\"answer\":", "logprobs": [], "sequence_number": 6} + +data: {"type": "response.function_call_arguments.delta", "output_index": 1, "item_id": "fc_test", "delta": "{\"answer\":", "sequence_number": 7} + +data: {"type": "response.output_text.delta", "output_index": 0, "content_index": 0, "item_id": "msg_test", "delta": "4}", "logprobs": [], "sequence_number": 8} + +data: {"type": "response.function_call_arguments.delta", "output_index": 1, "item_id": "fc_test", "delta": "4}", "sequence_number": 9} + +data: {"type": "response.output_text.done", "output_index": 0, "content_index": 0, "item_id": "msg_test", "text": "{\"answer\":4}", "logprobs": [], "sequence_number": 10} + +data: {"type": "response.output_item.done", "output_index": 1, "item": {"id": "fc_test", "type": "function_call", "call_id": "call_test", "name": "lookup", "arguments": "{\"answer\":4}", "status": "completed"}, "sequence_number": 11} + +data: {"type": "response.output_item.done", "output_index": 0, "item": {"id": "msg_test", "type": "message", "role": "assistant", "status": "completed", "content": [{"type": "output_text", "text": "{\"answer\":4}", "annotations": [], "logprobs": []}]}, "sequence_number": 12} + +data: {"type": "response.completed", "response": {"id": "resp_test", "status": "completed", "output": null}, "sequence_number": 13} + From d37984b9040fc8f6da8ae1845c59fd10c395ec2f Mon Sep 17 00:00:00 2001 From: Chojan Shang Date: Sat, 14 Nov 2026 00:19:00 +0800 Subject: [PATCH 4/4] fix(responses): retain source-backed lifecycle replacements --- src/republic/formats/responses.py | 85 ++++++++++++++++--- tests/test_provider_golden.py | 11 +++ tests/test_provider_golden/README.md | 15 ++++ .../responses_lifecycle_replacements/0.sse | 18 ++++ 4 files changed, 116 insertions(+), 13 deletions(-) create mode 100644 tests/test_provider_golden/cases/responses_lifecycle_replacements/0.sse diff --git a/src/republic/formats/responses.py b/src/republic/formats/responses.py index af1078c..e5f8ebd 100644 --- a/src/republic/formats/responses.py +++ b/src/republic/formats/responses.py @@ -133,14 +133,30 @@ def _input_items(self, message: Message) -> list[Mapping[str, Any]]: class _ResponsesStreamParser(StreamParser): def __init__(self) -> None: + # Keep first-appearance order while replacing snapshots by their stable identity. + self._items: dict[tuple[str, str | None], Mapping[str, Any] | None] = {} + self._call_keys: dict[str, str] = {} self._streamed_calls: set[str] = set() + self._finished_calls: set[str] = set() + self._streamed_messages: set[str | None] = set() def feed(self, event: str, data: str) -> Iterable[Delta]: if data == "[DONE]": + yield from self._flush_items() self.completed = True return payload = json.loads(data) + if payload.get("type") in { + "response.created", + "response.in_progress", + "response.queued", + "response.completed", + "response.incomplete", + }: + for item in payload["response"].get("output") or (): + self._remember_item(item) if payload.get("type") in {"response.completed", "response.incomplete"}: + yield from self._flush_items() self.completed = True if ( payload.get("type") in {"response.output_item.added", "response.output_item.done"} @@ -152,28 +168,75 @@ def feed(self, event: str, data: str) -> Iterable[Delta]: def _feed_payload(self, payload: Mapping[str, Any]) -> Iterable[Delta]: match payload.get("type"): case "response.output_text.delta": + self._streamed_messages.add(payload.get("item_id")) yield TextDelta(payload["delta"]) case "response.output_text.annotation.added": yield from _citation_deltas([payload["annotation"]]) case "response.refusal.delta": + self._streamed_messages.add(payload.get("item_id")) yield RefusalDelta(payload["delta"]) case "response.reasoning_summary_text.delta" | "response.reasoning_text.delta": yield ReasoningDelta(payload["delta"]) case "response.reasoning_summary_part.added" if payload.get("summary_index", 0) > 0: yield ReasoningDelta("\n\n") - case "response.output_item.added" if payload["item"].get("type") == "function_call": - item = payload["item"] - self._streamed_calls.add(_call_key(item)) - yield ToolCallFragment(_call_key(item), id=item["call_id"], name=item["name"]) + case "response.output_item.added" | "response.output_item.done": + yield from self._item_event(payload) case "response.function_call_arguments.delta": - yield ToolCallFragment(payload["item_id"], arguments=payload["delta"]) - case "response.output_item.done" if _call_key_or_none(payload["item"]) in self._streamed_calls: - yield ToolCallFragment(_call_key(payload["item"]), done=True) - case "response.output_item.done": - yield from _item_deltas(payload["item"], streamed=True) + key = self._call_keys.get(payload["item_id"], payload["item_id"]) + self._streamed_calls.add(key) + yield ToolCallFragment(key, arguments=payload["delta"]) case _: yield from _lifecycle_deltas(payload) + def _item_event(self, payload: Mapping[str, Any]) -> Iterable[Delta]: + item = payload["item"] + added = payload["type"] == "response.output_item.added" + self._remember_item(item, snapshot=not added) + if item.get("type") == "function_call": + key = item["call_id"] + if added: + yield ToolCallFragment(key, id=key, name=item["name"]) + elif key in self._streamed_calls: + yield from self._finish_call(item) + elif not added and item.get("type") != "message": + yield from _item_deltas(item, streamed=True) + + def _remember_item(self, item: Mapping[str, Any], *, snapshot: bool = True) -> None: + match item.get("type"): + case "function_call": + identity = item["call_id"] + self._call_keys[_call_key(item)] = identity + key = ("function_call", identity) + case "message": + key = ("message", item.get("id")) + case _: + return + if snapshot: + self._items[key] = item + else: + # Added items reserve order, but their content may still be incomplete. + self._items.setdefault(key, None) + + def _flush_items(self) -> Iterable[Delta]: + for item in self._items.values(): + if item is None: + continue + if item["type"] == "function_call": + yield from self._finish_call(item) + elif None not in self._streamed_messages and item.get("id") not in self._streamed_messages: + yield from _item_deltas(item, streamed=False) + self._items.clear() + + def _finish_call(self, item: Mapping[str, Any]) -> Iterable[Delta]: + key = item["call_id"] + if key in self._finished_calls: + return + # Ordinary argument deltas and their completion remain live. Snapshot-only calls + # wait for the terminal event so intermediate lifecycle replacements can settle. + arguments = "" if key in self._streamed_calls else item["arguments"] + yield ToolCallFragment(key, id=key, name=item["name"], arguments=arguments, done=True) + self._finished_calls.add(key) + def _lifecycle_deltas(payload: Mapping[str, Any]) -> Iterable[Delta]: match payload.get("type"): @@ -295,10 +358,6 @@ def _call_key(item: Mapping[str, Any]) -> str: return item.get("id") or item["call_id"] -def _call_key_or_none(item: Mapping[str, Any]) -> str | None: - return _call_key(item) if item.get("type") == "function_call" else None - - def _response_deltas(response: Mapping[str, Any]) -> Iterable[Delta]: yield _info(response) if usage := response.get("usage"): diff --git a/tests/test_provider_golden.py b/tests/test_provider_golden.py index 2900ef0..d4ad5e3 100644 --- a/tests/test_provider_golden.py +++ b/tests/test_provider_golden.py @@ -748,3 +748,14 @@ async def test_responses_null_items_preserve_text_and_tools(service: FakeService assert response.text == '{"answer":4}' assert response.tool_calls == [republic.ToolCall("call_test", "lookup", '{"answer":4}')] assert response.finish_reason == "tool_calls" + + +async def test_responses_lifecycle_replacements(service: FakeService) -> None: + load_case(service, "responses_lifecycle_replacements") + + response, _ = await respond(make_model(service, "responses"), [republic.user("Read the stream.")], streaming=True) + + assert response.text == "Updated result" + assert response.id == "resp_fixture" + assert response.finish_reason == "tool_calls" + assert response.tool_calls == [republic.ToolCall("call_fixture", "lookup", '{"answer":5}')] diff --git a/tests/test_provider_golden/README.md b/tests/test_provider_golden/README.md index 0fa1676..46f2dce 100644 --- a/tests/test_provider_golden/README.md +++ b/tests/test_provider_golden/README.md @@ -167,3 +167,18 @@ The upstream construction is serialized as SSE, retaining event order and sequen It is not a production recording and does not establish that null items mean keepalives. The input is replayed through the OpenAI Responses provider. Valid text and calls following empty output-item events survive. This case does not establish non-streaming or Copilot behavior. + + +## Responses lifecycle replacements + +`cases/responses_lifecycle_replacements/0.sse` serializes the events constructed by +[`openai-python@59738a289ee82c7b86bf1e10230cceb6daa9716d`](https://github.com/openai/openai-python/blob/59738a289ee82c7b86bf1e10230cceb6daa9716d/tests/lib/responses/test_lifecycle_replacements.py), +`test_final_output_keeps_completed_lifecycle_replacements`, with `terminal="null"`, +`duplicate=False` and `tool_id="null"` (Apache-2.0; `LICENSE.openai`). Its SHA-256 is +`f782d678c56e05ae3026c835d2d3bde30e6a110d36aa465cf1512da2a88b1c48`. Serialization uses the upstream `json.dumps` expression, including sequence numbers. + +This is an upstream SDK synthetic robustness case, not a production recording. It covers stable +message IDs, sparse item-event indices, reordered/partial lifecycle snapshots, replacement text +and tool arguments, and null terminal output. The final answer and call must contain the latest +replacement once. It does not establish support for changed/absent message IDs or Copilot-specific +combinations. The existing golden runner checks public streamed/final consistency. diff --git a/tests/test_provider_golden/cases/responses_lifecycle_replacements/0.sse b/tests/test_provider_golden/cases/responses_lifecycle_replacements/0.sse new file mode 100644 index 0000000..80b0638 --- /dev/null +++ b/tests/test_provider_golden/cases/responses_lifecycle_replacements/0.sse @@ -0,0 +1,18 @@ +data: {"type": "response.created", "response": {"id": "resp_fixture", "output": []}, "sequence_number": 0} + +data: {"type": "response.output_item.added", "output_index": 3, "item": {"id": "msg_fixture", "type": "message", "role": "assistant", "status": "completed", "content": [{"type": "output_text", "text": "Original result", "annotations": []}]}, "sequence_number": 1} + +data: {"type": "response.output_item.done", "output_index": 3, "item": {"id": "msg_fixture", "type": "message", "role": "assistant", "status": "completed", "content": [{"type": "output_text", "text": "Original result", "annotations": []}]}, "sequence_number": 2} + +data: {"type": "response.output_item.added", "output_index": 7, "item": {"type": "function_call", "call_id": "call_fixture", "name": "lookup", "arguments": "{\"answer\":4}", "status": "completed", "id": null}, "sequence_number": 3} + +data: {"type": "response.output_item.done", "output_index": 7, "item": {"type": "function_call", "call_id": "call_fixture", "name": "lookup", "arguments": "{\"answer\":4}", "status": "completed", "id": null}, "sequence_number": 4} + +data: {"type": "response.in_progress", "response": {"id": "resp_fixture", "output": [{"type": "function_call", "call_id": "call_fixture", "name": "lookup", "arguments": "{\"answer\":5}", "status": "completed", "id": "fc_fixture"}, {"id": "msg_fixture", "type": "message", "role": "assistant", "status": "completed", "content": [{"type": "output_text", "text": "Updated result", "annotations": []}]}]}, "sequence_number": 5} + +data: {"type": "response.queued", "response": {"id": "resp_fixture", "output": [{"type": "function_call", "call_id": "call_fixture", "name": "lookup", "arguments": "{\"answer\":6}", "status": "completed", "id": "fc_fixture"}]}, "sequence_number": 6} + +data: {"type": "response.in_progress", "response": {"id": "resp_fixture", "output": [{"type": "function_call", "call_id": "call_fixture", "name": "lookup", "arguments": "{\"answer\":5}", "status": "completed", "id": "fc_fixture"}]}, "sequence_number": 7} + +data: {"type": "response.completed", "response": {"id": "resp_fixture", "status": "completed", "output": null}, "sequence_number": 8} +