Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,13 @@ repos:
- id: check-json
exclude: ^.devcontainer/devcontainer.json
- id: pretty-format-json
exclude: ^.devcontainer/devcontainer.json
exclude: ^.devcontainer/devcontainer.json|^tests/test_provider_golden/cases/provider_sample_.*\.json$
args: [--autofix, --no-sort-keys]
- id: end-of-file-fixer
exclude: ^tests/test_provider_golden/cases/.*\.sse$
# Recorded bodies keep their upstream bytes: no added newline, no re-encoding.
exclude: ^tests/test_provider_golden/cases/.*\.sse$|^tests/test_provider_golden/cases/provider_sample_.*\.json$
- id: trailing-whitespace
exclude: ^tests/test_provider_golden/cases/provider_sample_.*\.json$

- repo: https://github.com/astral-sh/ruff-pre-commit
rev: "v0.14.10"
Expand Down
16 changes: 15 additions & 1 deletion src/republic/formats/chat.py
Original file line number Diff line number Diff line change
Expand Up @@ -164,7 +164,9 @@ def feed(self, event: str, data: str) -> Iterable[Delta]:
return
chunk = json.loads(data)
_raise_for_error(chunk)
for choice in chunk.get("choices") or ():
# The API exposes one response, so only the choice indexed 0 contributes; other
# choices' text, reasoning, calls and finish reason are dropped rather than merged.
if (choice := _indexed_choice(chunk)) is not None:
delta = choice.get("delta") or {}
if reasoning := self._api_format.reasoning_text(delta):
yield ReasoningDelta(reasoning)
Expand All @@ -184,6 +186,7 @@ def feed(self, event: str, data: str) -> Iterable[Delta]:
arguments=function.get("arguments") or "",
)
yield _info(chunk, choice)
# Usage is request-level, not per choice, so it survives whatever the choices say.
if usage := chunk.get("usage"):
yield _usage(usage)

Expand Down Expand Up @@ -260,6 +263,17 @@ def _citation_deltas(message: Mapping[str, Any]) -> Iterable[Delta]:
yield CitationAdded(Citation(citation["url"], title=citation.get("title")))


def _indexed_choice(chunk: Mapping[str, Any]) -> Mapping[str, Any] | None:
"""The choice with index 0 in a stream chunk, whatever its position in the array.

Single-response dialects omit ``index``, so a missing index counts as 0.
"""
for choice in chunk.get("choices") or ():
if choice.get("index", 0) == 0:
return choice
return None


def _info(data: Mapping[str, Any], choice: Mapping[str, Any]) -> ResponseInfo:
finish_reason = choice.get("finish_reason")
return ResponseInfo(
Expand Down
90 changes: 77 additions & 13 deletions src/republic/formats/responses.py
Original file line number Diff line number Diff line change
Expand Up @@ -133,42 +133,110 @@ def _input_items(self, message: Message) -> list[Mapping[str, Any]]:

class _ResponsesStreamParser(StreamParser):
def __init__(self) -> None:
# Keep first-appearance order while replacing snapshots by their stable identity.
self._items: dict[tuple[str, str | None], Mapping[str, Any] | None] = {}
self._call_keys: dict[str, str] = {}
self._streamed_calls: set[str] = set()
self._finished_calls: set[str] = set()
self._streamed_messages: set[str | None] = set()

def feed(self, event: str, data: str) -> Iterable[Delta]:
if data == "[DONE]":
yield from self._flush_items()
self.completed = True
return
payload = json.loads(data)
if payload.get("type") in {
"response.created",
"response.in_progress",
"response.queued",
"response.completed",
"response.incomplete",
}:
for item in payload["response"].get("output") or ():
self._remember_item(item)
if payload.get("type") in {"response.completed", "response.incomplete"}:
yield from self._flush_items()
self.completed = True
if (
payload.get("type") in {"response.output_item.added", "response.output_item.done"}
and payload.get("item") is None
):
return
yield from self._feed_payload(payload)

def _feed_payload(self, payload: Mapping[str, Any]) -> Iterable[Delta]:
match payload.get("type"):
case "response.output_text.delta":
self._streamed_messages.add(payload.get("item_id"))
yield TextDelta(payload["delta"])
case "response.output_text.annotation.added":
yield from _citation_deltas([payload["annotation"]])
case "response.refusal.delta":
self._streamed_messages.add(payload.get("item_id"))
yield RefusalDelta(payload["delta"])
case "response.reasoning_summary_text.delta" | "response.reasoning_text.delta":
yield ReasoningDelta(payload["delta"])
case "response.reasoning_summary_part.added" if payload.get("summary_index", 0) > 0:
yield ReasoningDelta("\n\n")
case "response.output_item.added" if payload["item"].get("type") == "function_call":
item = payload["item"]
self._streamed_calls.add(_call_key(item))
yield ToolCallFragment(_call_key(item), id=item["call_id"], name=item["name"])
case "response.output_item.added" | "response.output_item.done":
yield from self._item_event(payload)
case "response.function_call_arguments.delta":
yield ToolCallFragment(payload["item_id"], arguments=payload["delta"])
case "response.output_item.done" if _call_key_or_none(payload["item"]) in self._streamed_calls:
yield ToolCallFragment(_call_key(payload["item"]), done=True)
case "response.output_item.done":
yield from _item_deltas(payload["item"], streamed=True)
key = self._call_keys.get(payload["item_id"], payload["item_id"])
self._streamed_calls.add(key)
yield ToolCallFragment(key, arguments=payload["delta"])
case _:
yield from _lifecycle_deltas(payload)

def _item_event(self, payload: Mapping[str, Any]) -> Iterable[Delta]:
item = payload["item"]
added = payload["type"] == "response.output_item.added"
self._remember_item(item, snapshot=not added)
if item.get("type") == "function_call":
key = item["call_id"]
if added:
yield ToolCallFragment(key, id=key, name=item["name"])
elif key in self._streamed_calls:
yield from self._finish_call(item)
elif not added and item.get("type") != "message":
yield from _item_deltas(item, streamed=True)

def _remember_item(self, item: Mapping[str, Any], *, snapshot: bool = True) -> None:
match item.get("type"):
case "function_call":
identity = item["call_id"]
self._call_keys[_call_key(item)] = identity
key = ("function_call", identity)
case "message":
key = ("message", item.get("id"))
case _:
return
if snapshot:
self._items[key] = item
else:
# Added items reserve order, but their content may still be incomplete.
self._items.setdefault(key, None)

def _flush_items(self) -> Iterable[Delta]:
for item in self._items.values():
if item is None:
continue
if item["type"] == "function_call":
yield from self._finish_call(item)
elif None not in self._streamed_messages and item.get("id") not in self._streamed_messages:
Comment thread
PsiACE marked this conversation as resolved.
Resolved
yield from _item_deltas(item, streamed=False)
self._items.clear()

def _finish_call(self, item: Mapping[str, Any]) -> Iterable[Delta]:
key = item["call_id"]
if key in self._finished_calls:
return
# Ordinary argument deltas and their completion remain live. Snapshot-only calls
# wait for the terminal event so intermediate lifecycle replacements can settle.
arguments = "" if key in self._streamed_calls else item["arguments"]
yield ToolCallFragment(key, id=key, name=item["name"], arguments=arguments, done=True)
self._finished_calls.add(key)


def _lifecycle_deltas(payload: Mapping[str, Any]) -> Iterable[Delta]:
match payload.get("type"):
Expand Down Expand Up @@ -290,10 +358,6 @@ def _call_key(item: Mapping[str, Any]) -> str:
return item.get("id") or item["call_id"]


def _call_key_or_none(item: Mapping[str, Any]) -> str | None:
return _call_key(item) if item.get("type") == "function_call" else None


def _response_deltas(response: Mapping[str, Any]) -> Iterable[Delta]:
yield _info(response)
if usage := response.get("usage"):
Expand Down
200 changes: 200 additions & 0 deletions tests/test_provider_golden.py
Original file line number Diff line number Diff line change
Expand Up @@ -559,3 +559,203 @@ async def test_anthropic_cumulative_usage_includes_thinking(service: FakeService
assert response.token_usage == republic.TokenUsage(
input_tokens=59, output_tokens=8, reasoning_tokens=3, cached_tokens=7, cache_write_tokens=12
)


async def test_chat_stream_selects_choice_zero(service: FakeService) -> None:
load_case(service, "openai_chat_multi_choice_streaming")

response, _ = await respond(make_model(service, "chat"), [republic.user("Weather?")], streaming=True)

assert response.text == '{"city":"San Francisco","temperature":65,"units":"f"}'
assert response.reasoning == ""
assert response.tool_calls == []
assert response.finish_reason == "stop"
assert response.token_usage == republic.TokenUsage(input_tokens=79, output_tokens=42)


async def test_zai_reports_split_reasoning_and_usage(service: FakeService) -> None:
load_case(service, "provider_sample_zai_glm_reasoning")
model = republic.get_model("zai:glm-4.7-flash", api_key="test-key", http_client=service.client())

response, _ = await respond(
model, [republic.system("You are a helpful assistant"), republic.user("Say hi")], max_tokens=4000
)

assert service.requests[0].url == "https://api.z.ai/api/paas/v4/chat/completions"
assert service.requests[0].headers["authorization"] == "Bearer test-key"
body = service.body()
assert body["model"] == "glm-4.7-flash"
# Z.ai's GLM chat limits output with `max_tokens`; `max_completion_tokens` is not part of its API.
assert body["max_tokens"] == 4000
assert "max_completion_tokens" not in body
# `reasoning_content` is the model's thinking, kept out of the answer text.
assert response.reasoning.startswith("1. **Analyze the Request:**")
assert response.text == "Olá! (Oh-lah)"
assert response.finish_reason == "stop"
assert response.token_usage == republic.TokenUsage(
input_tokens=15, output_tokens=249, reasoning_tokens=239, cached_tokens=3
)


async def test_zai_reasoning_content_returns_in_the_next_turn(service: FakeService) -> None:
load_case(service, "provider_sample_zai_glm_tool")
model = republic.get_model("zai:glm-4.7-flash", api_key="test-key", http_client=service.client())
prompt = [republic.system("You are a helpful assistant"), republic.user("What's the weather in Florence?")]

first, _ = await respond(model, prompt, max_tokens=4000, tools=[WEATHER], tool_choice="auto")

assert first.finish_reason == "tool_calls"
assert len(first.tool_calls) == 1
call = first.tool_calls[0]
assert call.name == "weather"
assert call.id == "call_-7807249954186984837"
assert call.args == {"location": "Florence,Italy"}
assert first.reasoning

second, _ = await respond(
model, [*prompt, first.message, republic.tool(call, "40 C")], max_tokens=4000, tools=[WEATHER]
)

assert second.finish_reason == "stop"
assert "40" in second.text
assert_tool_exchange(service, "chat", [call], {"weather": "40 C"})
assert service.body()["messages"][2]["reasoning_content"] == first.reasoning


async def test_grok_reports_reasoning_tokens_without_replaying_them(service: FakeService) -> None:
load_case(service, "provider_sample_grok_tool")
model = republic.get_model("grok:grok-4-fast", api_key="test-key", api_format="chat", http_client=service.client())
prompt = [republic.system("You are a helpful assistant"), republic.user("What's the weather in Florence?")]

first, _ = await respond(model, prompt, max_tokens=4000, tools=[WEATHER], tool_choice="auto")

assert service.requests[0].url == "https://api.x.ai/v1/chat/completions"
assert first.finish_reason == "tool_calls"
assert len(first.tool_calls) == 1
call = first.tool_calls[0]
assert call.name == "weather"
assert call.id == "call_98286056"
assert call.args == {"location": "Florence, Italy"}
assert first.token_usage == republic.TokenUsage(
input_tokens=361, output_tokens=26, reasoning_tokens=93, cached_tokens=153
)

second, _ = await respond(
model, [*prompt, first.message, republic.tool(call, "40 C")], max_tokens=4000, tools=[WEATHER]
)

assert second.finish_reason == "stop"
assert "40" in second.text
assert_tool_exchange(service, "chat", [call], {"weather": "40 C"})
assert "reasoning_content" not in service.body()["messages"][2]


async def test_openrouter_kimi_returns_recorded_call_identity(service: FakeService) -> None:
load_case(service, "provider_sample_openrouter_kimi_tool")
model = republic.get_model(
"openrouter:moonshotai/kimi-k2.5", api_key="test-key", api_format="chat", http_client=service.client()
)
prompt = [republic.system("You are a helpful assistant"), republic.user("What's the weather in Florence?")]

first, _ = await respond(model, prompt, max_tokens=4000, tools=[WEATHER], tool_choice="auto")

assert service.requests[0].url == "https://openrouter.ai/api/v1/chat/completions"
assert first.finish_reason == "tool_calls"
assert len(first.tool_calls) == 1
call = first.tool_calls[0]
assert call.name == "weather"
# The gateway keeps the upstream call id, which is not shaped like OpenAI's.
assert call.id == "functions.weather:0"
assert call.args == {"location": "Florence"}
assert first.reasoning.startswith(" The user is asking for the weather in Florence")
assert first.text == ""

second, _ = await respond(
model, [*prompt, first.message, republic.tool(call, "40 C")], max_tokens=4000, tools=[WEATHER]
)

assert second.finish_reason == "stop"
assert "40" in second.text
assert_tool_exchange(service, "chat", [call], {"weather": "40 C"})


async def test_openrouter_streams_reasoning_and_final_usage(service: FakeService) -> None:
load_case(service, "provider_sample_openrouter_grok_streaming")
model = republic.get_model(
"openrouter:x-ai/grok-4.1-fast", api_key="test-key", api_format="chat", http_client=service.client()
)

response, _ = await respond(
model,
[republic.system("You are a helpful assistant"), republic.user("Say hi")],
streaming=True,
max_tokens=4000,
)

assert response.finish_reason == "stop"
assert response.text == "Oi!"
# The recording carries OpenRouter's `: OPENROUTER PROCESSING` heartbeats and a null reasoning
# field on the closing chunks; only the `reasoning` text becomes reasoning.
assert response.reasoning.startswith("First, the core policies")
assert response.token_usage == republic.TokenUsage(
input_tokens=165, output_tokens=167, reasoning_tokens=165, cached_tokens=164
)


async def test_azure_uses_api_key_header_and_deployment_name(service: FakeService) -> None:
azure_api_base = "https://fantasy-playground-resource.openai.azure.com/openai/v1"
load_case(service, "provider_sample_azure_tool")
model = republic.get_model(
"azure-openai:gpt-5-mini",
api_key="test-key",
api_base=azure_api_base,
api_format="chat",
http_client=service.client(),
)
prompt = [republic.system("You are a helpful assistant"), republic.user("What's the weather in Florence?")]

first, _ = await respond(model, prompt, max_tokens=4000, tools=[WEATHER], tool_choice="auto")

assert service.requests[0].url == f"{azure_api_base}/chat/completions"
# This API-key case uses `api-key`; Azure AD bearer auth is a separate supported auth mode.
assert service.requests[0].headers["api-key"] == "test-key"
assert "authorization" not in service.requests[0].headers
body = service.body()
assert body["model"] == "gpt-5-mini"
assert body["max_completion_tokens"] == 4000
assert first.finish_reason == "tool_calls"
assert len(first.tool_calls) == 1
call = first.tool_calls[0]
assert call.name == "weather"
assert call.id == "call_mbWCiHb6eF0qjzWrYgK52fWS"
assert call.args == {"location": "Florence, Italy"}

second, _ = await respond(
model, [*prompt, first.message, republic.tool(call, "40 C")], max_tokens=4000, tools=[WEATHER]
)

assert second.finish_reason == "stop"
assert "40" in second.text
assert service.requests[1].headers["api-key"] == "test-key"
assert_tool_exchange(service, "chat", [call], {"weather": "40 C"})


async def test_responses_null_items_preserve_text_and_tools(service: FakeService) -> None:
load_case(service, "responses_null_output_items_streaming")

response, _ = await respond(make_model(service, "responses"), [republic.user("Hi")], streaming=True)

assert response.text == '{"answer":4}'
assert response.tool_calls == [republic.ToolCall("call_test", "lookup", '{"answer":4}')]
assert response.finish_reason == "tool_calls"


async def test_responses_lifecycle_replacements(service: FakeService) -> None:
load_case(service, "responses_lifecycle_replacements")

response, _ = await respond(make_model(service, "responses"), [republic.user("Read the stream.")], streaming=True)

assert response.text == "Updated result"
assert response.id == "resp_fixture"
assert response.finish_reason == "tool_calls"
assert response.tool_calls == [republic.ToolCall("call_fixture", "lookup", '{"answer":5}')]
Loading
Loading