From 60e9c8cd8ce6fdef05e13c4712b5e6cc5fef9de3 Mon Sep 17 00:00:00 2001 From: Hermes Date: Sat, 31 Oct 2026 10:33:41 +0100 Subject: [PATCH] tools.hermes-voice: post the whole call transcript after the hang-up The per-turn transcript thread came from the plugin's post_llm_call hook, which Hermes skips for interrupted turns: a delegation the page replaced (a correction, more speech) never reached the thread, so a call could show only its hang-up part. The clients now remember where a call starts in the conversation and send all its lines (both sides) with the hang-up; hermes-voice posts them as one message to the Matrix home room through the existing voice-approval webhook route. The plugin's transcript hook and thread code go away. --- machines/coaxmetal/hermes-voice.nix | 6 +- .../src/us/lassul/assist/Conversation.java | 7 ++- .../src/us/lassul/assist/VoiceCall.java | 12 +++- tools/hermes-voice/hermes-plugin/__init__.py | 62 ------------------- tools/hermes-voice/hermes_voice.py | 30 +++++++-- tools/hermes-voice/static/app.js | 22 ++++--- 6 files changed, 56 insertions(+), 83 deletions(-) diff --git a/machines/coaxmetal/hermes-voice.nix b/machines/coaxmetal/hermes-voice.nix index 0af761ee..e4db1102 100644 --- a/machines/coaxmetal/hermes-voice.nix +++ b/machines/coaxmetal/hermes-voice.nix @@ -87,8 +87,7 @@ in # matrix_dm_send and end_call tools, on for every platform like any plugin # toolset (end_call refuses outside voice sessions), /assist, which # answers approvals asked in the Matrix DM through hermes-voice - # (authenticated with API_SERVER_KEY), and the call transcript: one thread - # per voice call in the Matrix home room. + # (authenticated with API_SERVER_KEY). services.hermes-agent.extraPlugins = [ (pkgs.runCommand "hermes-voice" { } '' cp -r ${../../tools/hermes-voice/hermes-plugin} $out @@ -96,7 +95,8 @@ in ]; services.hermes-agent.environment.HERMES_VOICE_URL = "http://localhost:${toString port}"; - # How hermes-voice asks for an approval when no call can hear it: a + # How hermes-voice asks for an approval when no call can hear it, and + # where it posts each call's transcript after the hang-up: a # deliver_only route on the gateway's webhook platform (hermes.nix) posts # the text to the Matrix home room, no agent turn. services.hermes-agent.settings.platforms.webhook.extra.routes.voice-approval = { diff --git a/tools/hermes-voice/android/src/us/lassul/assist/Conversation.java b/tools/hermes-voice/android/src/us/lassul/assist/Conversation.java index 7929b143..99b1e725 100644 --- a/tools/hermes-voice/android/src/us/lassul/assist/Conversation.java +++ b/tools/hermes-voice/android/src/us/lassul/assist/Conversation.java @@ -178,9 +178,14 @@ void add(String speaker, String text) { /** Lines since the last hand-over to Hermes, of the given speakers. */ List undelegated(String... speakers) { + return since(delegated, speakers); + } + + /** Lines from index start on, of the given speakers. */ + List since(int start, String... speakers) { List out = new ArrayList<>(); List wanted = Arrays.asList(speakers); - for (Line line : lines.subList(Math.min(delegated, lines.size()), lines.size())) { + for (Line line : lines.subList(Math.min(start, lines.size()), lines.size())) { if (wanted.contains(line.speaker)) { out.add(line); } diff --git a/tools/hermes-voice/android/src/us/lassul/assist/VoiceCall.java b/tools/hermes-voice/android/src/us/lassul/assist/VoiceCall.java index 597cc46c..9cfb4c5f 100644 --- a/tools/hermes-voice/android/src/us/lassul/assist/VoiceCall.java +++ b/tools/hermes-voice/android/src/us/lassul/assist/VoiceCall.java @@ -80,6 +80,8 @@ interface Listener { private final Api api; private final Conversation conversation; private final Listener listener; + // Where this call starts in the conversation, for its transcript. + private final int firstLine; private final Handler main = new Handler(Looper.getMainLooper()); private final ExecutorService io = Executors.newCachedThreadPool(); @@ -160,6 +162,7 @@ private static final class Approval { this.api = api; this.conversation = conversation; this.listener = listener; + this.firstLine = conversation.lines.size(); } void start() { @@ -712,17 +715,20 @@ private void finish(String reason, boolean failed) { listener.onFinished(reason, failed); } - // Tell the server the call is over, with what the user said since the last delegation: a - // request spoken right before hanging up still reaches Hermes. + // Tell the server the call is over, with what the user said since the last delegation (a + // request spoken right before hanging up still reaches Hermes) and the whole call for its + // transcript. private void hangUp() { List lines = conversation.undelegated("user", "assistant"); + List call = conversation.since(firstLine, "user", "assistant"); conversation.delegated = conversation.lines.size(); conversation.save(); final JSONObject body; try { body = new JSONObject() .put("hermes_session", conversation.hermesSession) - .put("lines", Conversation.toJson(lines)); + .put("lines", Conversation.toJson(lines)) + .put("call", Conversation.toJson(call)); } catch (JSONException error) { return; } diff --git a/tools/hermes-voice/hermes-plugin/__init__.py b/tools/hermes-voice/hermes-plugin/__init__.py index 827a0736..a4fd3d3a 100644 --- a/tools/hermes-voice/hermes-plugin/__init__.py +++ b/tools/hermes-voice/hermes-plugin/__init__.py @@ -17,20 +17,12 @@ allowed users (MATRIX_ALLOWED_USERS), so the answer is the user's own; the agent cannot issue it. The code may be left out while exactly one approval waits. - -Every voice call gets its own thread in the Matrix home room, holding the -call's transcript: after each turn of a voice session (post_llm_call), the -spoken conversation the turn was delegated with (both sides, small talk -included) and Hermes' answer go into that call's thread. The thread root is -posted on the call's first turn; the webhook route cannot thread, since it -does not return the root's event id. """ import asyncio import json import logging import os -import time import urllib.error import urllib.request @@ -135,15 +127,6 @@ def end_call(args, session_id=None, **_kw): return json.dumps({"status": "The call ends once your reply has been spoken."}) -# hermes-voice's delegation text (hermes_voice.transcript_message). -TRANSCRIPT_HEADER = "Spoken conversation since the last request, newest last:" -# Thread root event id per call (its voice-* session id). Only in memory: a -# call still running across a gateway restart continues in a new thread. -THREADS = {} -# Serializes the posts (thread root first), on the gateway loop. -POST_LOCK = None - - def matrix_adapter(): """The gateway runner and its live Matrix adapter (None when missing).""" from gateway.run import _gateway_runner_ref @@ -157,50 +140,6 @@ def matrix_adapter(): return runner, None -async def post_transcript(adapter, room, call_id, text): - global POST_LOCK - if POST_LOCK is None: - POST_LOCK = asyncio.Lock() - async with POST_LOCK: - root = THREADS.get(call_id) - if root is None: - root = await adapter.create_handoff_thread( - room, "📞 Voice call, " + time.strftime("%a %d %b %Y %H:%M") - ) - if root is None: - log.warning("voice transcript: no thread for %s", call_id) - return - THREADS[call_id] = root - result = await adapter.send(room, text, metadata={"thread_id": root}) - if not result.success: - log.warning("voice transcript: posting for %s failed: %s", call_id, result.error) - - -def transcript(session_id=None, user_message=None, assistant_response=None, platform=None, **_kw): - """post_llm_call: put a voice turn into its call's Matrix thread.""" - if getattr(platform, "value", platform) != "api_server": - return - call_id = voice_call_id(session_id) - if call_id is None: - return - spoken = str(user_message or "").removeprefix(TRANSCRIPT_HEADER).strip() - answer = str(assistant_response or "").strip() - text = "\n".join(part for part in (spoken, answer and f"Hermes: {answer}") if part) - room = os.environ.get("MATRIX_HOME_ROOM") - runner, adapter = matrix_adapter() - loop = getattr(runner, "_gateway_loop", None) - if not (text and room and adapter and loop): - return - from agent.async_utils import safe_schedule_threadsafe - - safe_schedule_threadsafe( - post_transcript(adapter, room, call_id, text), - loop, - logger=log, - log_message="voice transcript: scheduling the post failed", - ) - - def assist(raw_args): from agent.secret_scope import get_secret @@ -251,7 +190,6 @@ def register(ctx): handler=end_call, emoji="📴", ) - ctx.register_hook("post_llm_call", transcript) ctx.register_command( "assist", handler=assist, diff --git a/tools/hermes-voice/hermes_voice.py b/tools/hermes-voice/hermes_voice.py index c726513e..88ebb6bf 100644 --- a/tools/hermes-voice/hermes_voice.py +++ b/tools/hermes-voice/hermes_voice.py @@ -14,7 +14,8 @@ corrected or replaced the request); hanging up just ends the call, and the run finishes whatever the user asked for. On hang-up the page sends what the user said since the last delegation, so a request spoken right before hanging up -still reaches Hermes. When a run asks for approval while no page is listening, +still reaches Hermes, and the whole call (both sides), which goes to the +owner's Matrix DM as its transcript. When a run asks for approval while no page is listening, the question goes to the owner's Matrix DM (the gateway's Matrix home room, through its webhook platform), and the hermes-voice plugin's `/assist approve|deny ` command answers it. @@ -266,16 +267,19 @@ def create_live_session(config, sdp, history, voice): return {"session_id": answer["session"]["id"], "sdp": answer["transport"]["sdp"]} -def transcript_message(lines): +def render_lines(lines): + """Transcript lines as `User: …` / `Assistant: …` text, empty ones left out.""" rendered = [] for line in lines: text = str(line.get("text", "")).strip() if text: who = "Assistant" if line.get("speaker") == "assistant" else "User" rendered.append(f"{who}: {text}") - return "Spoken conversation since the last request, newest last:\n" + "\n".join( - rendered - ) + return "\n".join(rendered) + + +def transcript_message(lines): + return "Spoken conversation since the last request, newest last:\n" + render_lines(lines) def iter_sse(response): @@ -759,6 +763,7 @@ def handle_hangup(self, request): runs = [run for run in RUNS.values() if run.hermes_session == hermes_session] for run in runs: run.hang_up() + self.post_call_transcript(hermes_session, request.get("call")) if not any(line.get("speaker") == "user" for line in lines): self.send_json(HTTPStatus.OK, {"run_id": None}) return @@ -775,6 +780,19 @@ def handle_hangup(self, request): log.info("hang-up of %s left a request: run %s", hermes_session, run_id) self.send_json(HTTPStatus.OK, {"run_id": run_id}) + def post_call_transcript(self, hermes_session, lines): + """The whole call, both sides, as one message in the owner's Matrix DM.""" + if not self.config.matrix_webhook or not isinstance(lines, list): + return + spoken = render_lines(line for line in lines if isinstance(line, dict)) + if not spoken: + return + title = "📞 Voice call, ended " + time.strftime("%a %d %b %Y %H:%M") + try: + notify_matrix(self.config, f"{title}\n\n{spoken}") + except (urllib.error.URLError, OSError) as error: + log.warning("call transcript of %s not posted: %s", hermes_session, error) + def handle_approve(self, request): code = request.get("code") choice = request.get("choice") @@ -831,7 +849,7 @@ def main(): parser.add_argument( "--matrix-webhook", help="Hermes webhook route (deliver_only to Matrix, signed with API_SERVER_KEY) that " - "receives approvals no call can answer", + "receives approvals no call can answer and the transcript of every call", ) parser.add_argument("--config", help="JSON file: voice, model, idle_seconds, glossary, instructions") parser.add_argument( diff --git a/tools/hermes-voice/static/app.js b/tools/hermes-voice/static/app.js index c7b4114b..9c2c13d7 100644 --- a/tools/hermes-voice/static/app.js +++ b/tools/hermes-voice/static/app.js @@ -185,6 +185,8 @@ class Call { this.closing = false; this.ending = null; this.eventCounter = 0; + // Where this call starts in the conversation, for its transcript. + this.firstLine = state.lines.length; } async start() { @@ -562,21 +564,25 @@ class Call { } // Tell the server the call is over, with what the user said since the last - // delegation: a request spoken right before hanging up still reaches - // Hermes. keepalive lets the request outlive the page. + // delegation (a request spoken right before hanging up still reaches + // Hermes) and the whole call for its transcript. keepalive lets the + // request outlive the page. hangUp() { - const lines = state.lines - .slice(state.delegated) - .filter( - (line) => line.speaker === "user" || line.speaker === "assistant", - ); + const spoken = (line) => + line.speaker === "user" || line.speaker === "assistant"; + const lines = state.lines.slice(state.delegated).filter(spoken); + const callLines = state.lines.slice(this.firstLine).filter(spoken); state.delegated = state.lines.length; saveState(); api("/hangup", { method: "POST", keepalive: true, headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ hermes_session: state.hermesSession, lines }), + body: JSON.stringify({ + hermes_session: state.hermesSession, + lines, + call: callLines, + }), }).catch(() => {}); }