fix(links): decode base64url dialog audio in the whisper links

common/lib/dialog_body.py

@@ -0,0 +1,87 @@
+"""Decoding for inline dialog bodies.
+
+draft-ietf-vcon-vcon-core-04 allows exactly three values for a dialog's
+``encoding``: ``"base64url"``, ``"json"`` or ``"none"``. Telephony adapters
+are moving to ``base64url`` *without* padding for inline audio bodies.
+
+Standard ``base64.b64decode`` chokes on that: with ``validate=True`` it
+raises on the missing padding, and in its default (non-validating) mode it
+silently drops the ``-``/``_`` characters that are only valid in the
+URL-safe alphabet and hands back corrupted audio instead of raising.
+
+:func:`decode_inline_dialog_body` is the one place that should turn a
+dialog's inline ``body`` into raw bytes. It dispatches on ``encoding``
+instead of guessing, so a body encoded in a way we do not understand raises
+rather than silently returning garbage.
+"""
+
+import base64
+import binascii
+from typing import Any, Dict
+
+
+def _restore_base64url_padding(body: str) -> str:
+    """Pad a base64url string out to a multiple of 4 characters.
+
+    Adapters that emit base64url without padding are a valid rendering of
+    the same bytes -- padding is only there to make the length a multiple
+    of 4, and ``-``/``_`` already disambiguate the alphabet, so restoring
+    it here is lossless.
+    """
+    return body + "=" * (-len(body) % 4)
+
+
+def decode_inline_dialog_body(dialog: Dict[str, Any]) -> bytes:
+    """Decode a dialog's inline ``body`` per its ``encoding`` field.
+
+    Args:
+        dialog: A vCon dialog object carrying an inline ``body`` (as opposed
+            to a ``url``-referenced one).
+
+    Returns:
+        The decoded raw bytes.
+
+    Raises:
+        ValueError: ``encoding`` is missing/legacy and the body is not valid
+            base64; ``encoding`` is ``"none"`` (the body is already a plain
+            string, not encoded audio bytes); or ``encoding`` is some other,
+            unrecognized value.
+        KeyError: ``dialog`` has no ``body``.
+    """
+    body = dialog["body"]
+    encoding = dialog.get("encoding")
+    if isinstance(body, str):
+        # Some producers wrap base64 at 76 columns (MIME style). The old
+        # non-validating decode skipped the line breaks; keep accepting them
+        # while still rejecting any other out-of-alphabet character.
+        body = "".join(body.split())
+
+    if encoding == "base64url":
+        if not isinstance(body, str):
+            raise ValueError(f"Cannot decode base64url dialog body of type {type(body).__name__}; expected str")
+        try:
+            padded = _restore_base64url_padding(body)
+            # base64.urlsafe_b64decode() just translates "-"/"_" to "+"/"/"
+            # and calls b64decode() *without* validate=True, so it silently
+            # drops any other out-of-alphabet character instead of raising.
+            # Translate ourselves and validate explicitly to fail loudly on
+            # garbage input rather than return corrupted audio.
+            standard = padded.translate({ord("-"): "+", ord("_"): "/"})
+            return base64.b64decode(standard, validate=True)
+        except (binascii.Error, ValueError) as e:
+            raise ValueError(f"Invalid base64url dialog body: {e}") from e
+
+    if encoding == "base64" or encoding is None:
+        # Legacy: vCons stored before the encoding field was spec-mandated,
+        # or producers that never adopted base64url, use plain base64.
+        if not isinstance(body, str):
+            raise ValueError(f"Cannot decode base64 dialog body of type {type(body).__name__}; expected str")
+        try:
+            return base64.b64decode(body, validate=True)
+        except (binascii.Error, ValueError) as e:
+            raise ValueError(f"Invalid base64 dialog body: {e}") from e
+
+    if encoding == "none":
+        raise ValueError("Dialog body encoding is 'none' (a plain string), not audio bytes")
+
+    raise ValueError(f"Unrecognized dialog body encoding: {encoding!r}")

common/tests/lib/test_dialog_body.py

@@ -0,0 +1,80 @@
+import base64
+
+import pytest
+
+from lib.dialog_body import decode_inline_dialog_body
+
+
+def test_base64url_without_padding_round_trips():
+    raw = b"hello world, this is audio-ish binary data"
+    body = base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=")
+    assert decode_inline_dialog_body({"encoding": "base64url", "body": body}) == raw
+
+
+def test_base64url_with_padding_round_trips():
+    raw = b"hello world, this is audio-ish binary dat"  # len % 3 != 0 -> padded
+    body = base64.urlsafe_b64encode(raw).decode("ascii")
+    assert body.endswith("=")
+    assert decode_inline_dialog_body({"encoding": "base64url", "body": body}) == raw
+
+
+def test_legacy_base64_round_trips():
+    raw = b"legacy audio bytes"
+    body = base64.b64encode(raw).decode("ascii")
+    assert decode_inline_dialog_body({"encoding": "base64", "body": body}) == raw
+
+
+def test_missing_encoding_treated_as_legacy_base64():
+    raw = b"vcon stored before encoding was mandatory"
+    body = base64.b64encode(raw).decode("ascii")
+    assert decode_inline_dialog_body({"body": body}) == raw
+
+
+def test_bytes_that_encode_to_dash_and_underscore_in_base64url_round_trip_exactly():
+    # 0xFB -> "-" and 0xFF -> "_" only in the URL-safe alphabet ("+"/"/" in
+    # standard base64). Plain base64.b64decode in its default, non-validating
+    # mode silently drops "-"/"_" instead of raising, which is exactly the
+    # silent-corruption failure mode this helper exists to prevent.
+    raw = bytes([0xFB, 0xFF, 0x00, 0x10, 0x83, 0xFB, 0xFF])
+    body = base64.urlsafe_b64encode(raw).decode("ascii")
+    assert "-" in body or "_" in body
+    unpadded = body.rstrip("=")
+
+    decoded = decode_inline_dialog_body({"encoding": "base64url", "body": unpadded})
+    assert decoded == raw
+
+    # Prove the corruption this guards against: plain b64decode on the
+    # same (padded) base64url text does NOT raise and does NOT return the
+    # original bytes.
+    corrupted = base64.b64decode(body)
+    assert corrupted != raw
+
+
+def test_none_encoding_raises_instead_of_returning_string_as_audio():
+    with pytest.raises(ValueError):
+        decode_inline_dialog_body({"encoding": "none", "body": "just text"})
+
+
+def test_unrecognized_encoding_raises():
+    with pytest.raises(ValueError):
+        decode_inline_dialog_body({"encoding": "json", "body": "{}"})
+
+
+def test_invalid_base64url_body_raises():
+    with pytest.raises(ValueError):
+        decode_inline_dialog_body({"encoding": "base64url", "body": "not valid base64!!"})
+
+
+def test_invalid_legacy_base64_body_raises():
+    with pytest.raises(ValueError):
+        decode_inline_dialog_body({"body": "not valid base64!!"})
+
+
+def test_line_wrapped_bodies_decode():
+    raw = bytes(range(256)) * 2
+    std = base64.b64encode(raw).decode()
+    wrapped = "\n".join(std[i : i + 76] for i in range(0, len(std), 76))
+    assert decode_inline_dialog_body({"body": wrapped, "encoding": "base64"}) == raw
+    url = base64.urlsafe_b64encode(raw).decode().rstrip("=")
+    wrapped_url = "\r\n".join(url[i : i + 76] for i in range(0, len(url), 76))
+    assert decode_inline_dialog_body({"body": wrapped_url, "encoding": "base64url"}) == raw

conserver/links/groq_whisper/__init__.py

@@ -12,6 +12,7 @@
 """
 
 from lib.redaction import safe_opts
+from lib.dialog_body import decode_inline_dialog_body
 import base64
 import hashlib
 import logging
@@ -139,8 +140,9 @@ def get_file_content(dialog: dict) -> bytes:
         Exception: If file cannot be retrieved or verified
     """
     if "body" in dialog:
-        # body contains the base64 encoded content. Decode and return
-        return base64.b64decode(dialog["body"])
+        # body contains the base64/base64url encoded content. Decode per
+        # the dialog's own encoding and return.
+        return decode_inline_dialog_body(dialog)
 
     elif "url" in dialog:
         # Handle external file

conserver/links/groq_whisper/test_groq_whisper.py

@@ -451,6 +451,17 @@ def test_get_file_content_from_body():
     content = get_file_content(dialog)
     assert content == b"test audio content"
 
+def test_get_file_content_from_base64url_body_without_padding():
+    """base64url dialogs (no padding, per draft-ietf-vcon-vcon-core-04) decode
+    to the original bytes, not the corrupted/raised result plain b64decode gives."""
+    raw = b"test audio content, base64url, no padding"
+    dialog = {
+        "encoding": "base64url",
+        "body": base64.urlsafe_b64encode(raw).decode('utf-8').rstrip('='),
+    }
+    content = get_file_content(dialog)
+    assert content == raw
+
 @patch('links.groq_whisper.requests.get')
 def test_get_file_content_from_url(mock_get):
     """Test extracting file content from URL reference"""

conserver/links/hugging_face_whisper/__init__.py

@@ -11,6 +11,7 @@
 """
 
 from lib.redaction import safe_opts
+from lib.dialog_body import decode_inline_dialog_body
 import base64
 import hashlib
 import logging
@@ -83,8 +84,9 @@ def get_file_content(dialog: dict) -> bytes:
         Exception: If file cannot be retrieved or verified
     """
     if "body" in dialog:
-        # body contains the base64 encoded content. Decode and return
-        return base64.b64decode(dialog["body"])
+        # body contains the base64/base64url encoded content. Decode per
+        # the dialog's own encoding and return.
+        return decode_inline_dialog_body(dialog)
 
     elif "url" in dialog:
         # Handle external file

conserver/links/hugging_face_whisper/tests/test_hugging_face_whisper_dialog_body.py

@@ -0,0 +1,44 @@
+"""get_file_content() decode coverage for CON-1112.
+
+The rest of this link's test suite (test_hugging_face_whisper.py,
+test_hugging_face_whisper_integration.py) is commented out pending an HF API
+key, but get_file_content() itself is pure and needs no network or model
+call, so it is covered here unconditionally.
+"""
+
+import base64
+
+import pytest
+
+from links.hugging_face_whisper import get_file_content
+
+
+def test_get_file_content_from_legacy_base64_body():
+    dialog = {"body": base64.b64encode(b"test audio content").decode("utf-8")}
+    assert get_file_content(dialog) == b"test audio content"
+
+
+def test_get_file_content_from_base64url_body_without_padding():
+    """base64url dialogs (no padding, per draft-ietf-vcon-vcon-core-04) decode
+    to the original bytes, not the corrupted/raised result plain b64decode gives."""
+    raw = b"test audio content, base64url, no padding"
+    dialog = {
+        "encoding": "base64url",
+        "body": base64.urlsafe_b64encode(raw).decode("utf-8").rstrip("="),
+    }
+    assert get_file_content(dialog) == raw
+
+
+def test_get_file_content_from_base64url_body_with_padding():
+    raw = b"test audio content, base64url, padded"
+    dialog = {
+        "encoding": "base64url",
+        "body": base64.urlsafe_b64encode(raw).decode("utf-8"),
+    }
+    assert get_file_content(dialog) == raw
+
+
+def test_get_file_content_none_encoding_raises():
+    dialog = {"encoding": "none", "body": "just text, not audio"}
+    with pytest.raises(ValueError):
+        get_file_content(dialog)

conserver/links/wtf_transcribe/__init__.py

@@ -23,7 +23,6 @@
         url-timeout: 300
 """
 
-import base64
 import json
 import logging
 import requests
@@ -32,6 +31,7 @@
 from lib.vcon_redis import VconRedis
 from lib.logging_utils import init_logger
 from lib.error_tracking import init_error_tracker
+from lib.dialog_body import decode_inline_dialog_body
 
 init_error_tracker()
 logger = init_logger(__name__)
@@ -111,13 +111,13 @@ def is_base64url_dialog(dialog):
     return has_base64url_encoding(dialog) and has_body(dialog)
 
 def base64url_dialog_to_binary(dialog):
-    return base64.urlsafe_b64decode(dialog["body"])
+    return decode_inline_dialog_body(dialog)
 
 def is_base64_dialog(dialog):
     return has_body(dialog)
 
 def base64_dialog_to_binary(dialog):
-    return base64.b64decode(dialog["body"])
+    return decode_inline_dialog_body(dialog)
 
 def dialog_to_binary(dialog, url_timeout=60):
     if is_url_dialog(dialog):

conserver/links/wtf_transcribe/test_wtf_transcribe_unit.py

@@ -92,6 +92,11 @@ def test_dialog_to_binary_handles_url_base64url_base64_and_invalid_inputs():
     base64url_body = base64.urlsafe_b64encode(b"url-safe").decode("ascii")
     assert dialog_to_binary({"encoding": "base64url", "body": base64url_body}) == b"url-safe"
 
+    # Telephony adapters emit base64url WITHOUT padding (draft-ietf-vcon-vcon-core-04);
+    # plain base64.urlsafe_b64decode raises on that, so this must still round-trip.
+    unpadded_base64url_body = base64.urlsafe_b64encode(b"url-safe, unpadded").decode("ascii").rstrip("=")
+    assert dialog_to_binary({"encoding": "base64url", "body": unpadded_base64url_body}) == b"url-safe, unpadded"
+
     base64_body = base64.b64encode(b"plain").decode("ascii")
     assert dialog_to_binary({"body": base64_body}) == b"plain"