tools.hermes-voice: post the whole call transcript after the hang-up

machines/coaxmetal/hermes-voice.nix

@@ -87,16 +87,16 @@ in
   # matrix_dm_send and end_call tools, on for every platform like any plugin
   # toolset (end_call refuses outside voice sessions), /assist, which
   # answers approvals asked in the Matrix DM through hermes-voice
-  # (authenticated with API_SERVER_KEY), and the call transcript: one thread
-  # per voice call in the Matrix home room.
+  # (authenticated with API_SERVER_KEY).
   services.hermes-agent.extraPlugins = [
     (pkgs.runCommand "hermes-voice" { } ''
       cp -r ${../../tools/hermes-voice/hermes-plugin} $out
     '')
   ];
   services.hermes-agent.environment.HERMES_VOICE_URL = "http://localhost:${toString port}";
 
-  # How hermes-voice asks for an approval when no call can hear it: a
+  # How hermes-voice asks for an approval when no call can hear it, and
+  # where it posts each call's transcript after the hang-up: a
   # deliver_only route on the gateway's webhook platform (hermes.nix) posts
   # the text to the Matrix home room, no agent turn.
   services.hermes-agent.settings.platforms.webhook.extra.routes.voice-approval = {

tools/hermes-voice/android/src/us/lassul/assist/Conversation.java

@@ -178,9 +178,14 @@ void add(String speaker, String text) {
 
     /** Lines since the last hand-over to Hermes, of the given speakers. */
     List<Line> undelegated(String... speakers) {
+        return since(delegated, speakers);
+    }
+
+    /** Lines from index start on, of the given speakers. */
+    List<Line> since(int start, String... speakers) {
         List<Line> out = new ArrayList<>();
         List<String> wanted = Arrays.asList(speakers);
-        for (Line line : lines.subList(Math.min(delegated, lines.size()), lines.size())) {
+        for (Line line : lines.subList(Math.min(start, lines.size()), lines.size())) {
             if (wanted.contains(line.speaker)) {
                 out.add(line);
             }

tools/hermes-voice/android/src/us/lassul/assist/VoiceCall.java

@@ -80,6 +80,8 @@ interface Listener {
     private final Api api;
     private final Conversation conversation;
     private final Listener listener;
+    // Where this call starts in the conversation, for its transcript.
+    private final int firstLine;
     private final Handler main = new Handler(Looper.getMainLooper());
     private final ExecutorService io = Executors.newCachedThreadPool();
 
@@ -160,6 +162,7 @@ private static final class Approval {
         this.api = api;
         this.conversation = conversation;
         this.listener = listener;
+        this.firstLine = conversation.lines.size();
     }
 
     void start() {
@@ -712,17 +715,20 @@ private void finish(String reason, boolean failed) {
         listener.onFinished(reason, failed);
     }
 
-    // Tell the server the call is over, with what the user said since the last delegation: a
-    // request spoken right before hanging up still reaches Hermes.
+    // Tell the server the call is over, with what the user said since the last delegation (a
+    // request spoken right before hanging up still reaches Hermes) and the whole call for its
+    // transcript.
     private void hangUp() {
         List<Conversation.Line> lines = conversation.undelegated("user", "assistant");
+        List<Conversation.Line> call = conversation.since(firstLine, "user", "assistant");
         conversation.delegated = conversation.lines.size();
         conversation.save();
         final JSONObject body;
         try {
             body = new JSONObject()
                     .put("hermes_session", conversation.hermesSession)
-                    .put("lines", Conversation.toJson(lines));
+                    .put("lines", Conversation.toJson(lines))
+                    .put("call", Conversation.toJson(call));
         } catch (JSONException error) {
             return;
         }

tools/hermes-voice/hermes-plugin/__init__.py

@@ -17,20 +17,12 @@
 allowed users (MATRIX_ALLOWED_USERS), so the answer is the user's own; the
 agent cannot issue it. The code may be left out while exactly one approval
 waits.
-
-Every voice call gets its own thread in the Matrix home room, holding the
-call's transcript: after each turn of a voice session (post_llm_call), the
-spoken conversation the turn was delegated with (both sides, small talk
-included) and Hermes' answer go into that call's thread. The thread root is
-posted on the call's first turn; the webhook route cannot thread, since it
-does not return the root's event id.
 """
 
 import asyncio
 import json
 import logging
 import os
-import time
 import urllib.error
 import urllib.request
 
@@ -135,15 +127,6 @@ def end_call(args, session_id=None, **_kw):
     return json.dumps({"status": "The call ends once your reply has been spoken."})
 
 
-# hermes-voice's delegation text (hermes_voice.transcript_message).
-TRANSCRIPT_HEADER = "Spoken conversation since the last request, newest last:"
-# Thread root event id per call (its voice-* session id). Only in memory: a
-# call still running across a gateway restart continues in a new thread.
-THREADS = {}
-# Serializes the posts (thread root first), on the gateway loop.
-POST_LOCK = None
-
-
 def matrix_adapter():
     """The gateway runner and its live Matrix adapter (None when missing)."""
     from gateway.run import _gateway_runner_ref
@@ -157,50 +140,6 @@ def matrix_adapter():
     return runner, None
 
 
-async def post_transcript(adapter, room, call_id, text):
-    global POST_LOCK
-    if POST_LOCK is None:
-        POST_LOCK = asyncio.Lock()
-    async with POST_LOCK:
-        root = THREADS.get(call_id)
-        if root is None:
-            root = await adapter.create_handoff_thread(
-                room, "📞 Voice call, " + time.strftime("%a %d %b %Y %H:%M")
-            )
-            if root is None:
-                log.warning("voice transcript: no thread for %s", call_id)
-                return
-            THREADS[call_id] = root
-        result = await adapter.send(room, text, metadata={"thread_id": root})
-        if not result.success:
-            log.warning("voice transcript: posting for %s failed: %s", call_id, result.error)
-
-
-def transcript(session_id=None, user_message=None, assistant_response=None, platform=None, **_kw):
-    """post_llm_call: put a voice turn into its call's Matrix thread."""
-    if getattr(platform, "value", platform) != "api_server":
-        return
-    call_id = voice_call_id(session_id)
-    if call_id is None:
-        return
-    spoken = str(user_message or "").removeprefix(TRANSCRIPT_HEADER).strip()
-    answer = str(assistant_response or "").strip()
-    text = "\n".join(part for part in (spoken, answer and f"Hermes: {answer}") if part)
-    room = os.environ.get("MATRIX_HOME_ROOM")
-    runner, adapter = matrix_adapter()
-    loop = getattr(runner, "_gateway_loop", None)
-    if not (text and room and adapter and loop):
-        return
-    from agent.async_utils import safe_schedule_threadsafe
-
-    safe_schedule_threadsafe(
-        post_transcript(adapter, room, call_id, text),
-        loop,
-        logger=log,
-        log_message="voice transcript: scheduling the post failed",
-    )
-
-
 def assist(raw_args):
     from agent.secret_scope import get_secret
 
@@ -251,7 +190,6 @@ def register(ctx):
         handler=end_call,
         emoji="📴",
     )
-    ctx.register_hook("post_llm_call", transcript)
     ctx.register_command(
         "assist",
         handler=assist,

tools/hermes-voice/hermes_voice.py

@@ -14,7 +14,8 @@
 corrected or replaced the request); hanging up just ends the call, and the run
 finishes whatever the user asked for. On hang-up the page sends what the user
 said since the last delegation, so a request spoken right before hanging up
-still reaches Hermes. When a run asks for approval while no page is listening,
+still reaches Hermes, and the whole call (both sides), which goes to the
+owner's Matrix DM as its transcript. When a run asks for approval while no page is listening,
 the question goes to the owner's Matrix DM (the gateway's Matrix home room,
 through its webhook platform), and the hermes-voice plugin's
 `/assist approve|deny <code>` command answers it.
@@ -266,16 +267,19 @@ def create_live_session(config, sdp, history, voice):
     return {"session_id": answer["session"]["id"], "sdp": answer["transport"]["sdp"]}
 
 
-def transcript_message(lines):
+def render_lines(lines):
+    """Transcript lines as `User: …` / `Assistant: …` text, empty ones left out."""
     rendered = []
     for line in lines:
         text = str(line.get("text", "")).strip()
         if text:
             who = "Assistant" if line.get("speaker") == "assistant" else "User"
             rendered.append(f"{who}: {text}")
-    return "Spoken conversation since the last request, newest last:\n" + "\n".join(
-        rendered
-    )
+    return "\n".join(rendered)
+
+
+def transcript_message(lines):
+    return "Spoken conversation since the last request, newest last:\n" + render_lines(lines)
 
 
 def iter_sse(response):
@@ -759,6 +763,7 @@ def handle_hangup(self, request):
             runs = [run for run in RUNS.values() if run.hermes_session == hermes_session]
         for run in runs:
             run.hang_up()
+        self.post_call_transcript(hermes_session, request.get("call"))
         if not any(line.get("speaker") == "user" for line in lines):
             self.send_json(HTTPStatus.OK, {"run_id": None})
             return
@@ -775,6 +780,19 @@ def handle_hangup(self, request):
         log.info("hang-up of %s left a request: run %s", hermes_session, run_id)
         self.send_json(HTTPStatus.OK, {"run_id": run_id})
 
+    def post_call_transcript(self, hermes_session, lines):
+        """The whole call, both sides, as one message in the owner's Matrix DM."""
+        if not self.config.matrix_webhook or not isinstance(lines, list):
+            return
+        spoken = render_lines(line for line in lines if isinstance(line, dict))
+        if not spoken:
+            return
+        title = "📞 Voice call, ended " + time.strftime("%a %d %b %Y %H:%M")
+        try:
+            notify_matrix(self.config, f"{title}\n\n{spoken}")
+        except (urllib.error.URLError, OSError) as error:
+            log.warning("call transcript of %s not posted: %s", hermes_session, error)
+
     def handle_approve(self, request):
         code = request.get("code")
         choice = request.get("choice")
@@ -831,7 +849,7 @@ def main():
     parser.add_argument(
         "--matrix-webhook",
         help="Hermes webhook route (deliver_only to Matrix, signed with API_SERVER_KEY) that "
-        "receives approvals no call can answer",
+        "receives approvals no call can answer and the transcript of every call",
     )
     parser.add_argument("--config", help="JSON file: voice, model, idle_seconds, glossary, instructions")
     parser.add_argument(

tools/hermes-voice/static/app.js

@@ -185,6 +185,8 @@ class Call {
     this.closing = false;
     this.ending = null;
     this.eventCounter = 0;
+    // Where this call starts in the conversation, for its transcript.
+    this.firstLine = state.lines.length;
   }
 
   async start() {
@@ -562,21 +564,25 @@ class Call {
   }
 
   // Tell the server the call is over, with what the user said since the last
-  // delegation: a request spoken right before hanging up still reaches
-  // Hermes. keepalive lets the request outlive the page.
+  // delegation (a request spoken right before hanging up still reaches
+  // Hermes) and the whole call for its transcript. keepalive lets the
+  // request outlive the page.
   hangUp() {
-    const lines = state.lines
-      .slice(state.delegated)
-      .filter(
-        (line) => line.speaker === "user" || line.speaker === "assistant",
-      );
+    const spoken = (line) =>
+      line.speaker === "user" || line.speaker === "assistant";
+    const lines = state.lines.slice(state.delegated).filter(spoken);
+    const callLines = state.lines.slice(this.firstLine).filter(spoken);
     state.delegated = state.lines.length;
     saveState();
     api("/hangup", {
       method: "POST",
       keepalive: true,
       headers: { "Content-Type": "application/json" },
-      body: JSON.stringify({ hermes_session: state.hermesSession, lines }),
+      body: JSON.stringify({
+        hermes_session: state.hermesSession,
+        lines,
+        call: callLines,
+      }),
     }).catch(() => {});
   }