From 8375a7d63465197182c28db1e29b6dab44856ee2 Mon Sep 17 00:00:00 2001 From: Thomas Howe Date: Fri, 25 Sep 2026 17:55:04 -0400 Subject: [PATCH 1/3] feat(lib): add decode_inline_dialog_body for base64url dialog bodies draft-ietf-vcon-vcon-core-04 allows base64url, json or none as dialog encoding, and telephony adapters are moving to unpadded base64url for inline audio. Plain base64.b64decode either raises on the missing padding or, in its default non-validating mode, silently drops "-"/"_" and returns corrupted audio instead of raising. decode_inline_dialog_body() dispatches on the dialog's own encoding: base64url (with or without padding) via a validated decode, legacy base64/missing encoding for vCons stored before encoding was mandatory, and a clear ValueError for "none" or anything unrecognized rather than guessing. Co-Authored-By: Claude Opus 5.5 --- common/lib/dialog_body.py | 82 ++++++++++++++++++++++++++++ common/tests/lib/test_dialog_body.py | 70 ++++++++++++++++++++++++ 2 files changed, 152 insertions(+) create mode 100644 common/lib/dialog_body.py create mode 100644 common/tests/lib/test_dialog_body.py diff --git a/common/lib/dialog_body.py b/common/lib/dialog_body.py new file mode 100644 index 0000000..61652dd --- /dev/null +++ b/common/lib/dialog_body.py @@ -0,0 +1,82 @@ +"""Decoding for inline dialog bodies. + +draft-ietf-vcon-vcon-core-04 allows exactly three values for a dialog's +``encoding``: ``"base64url"``, ``"json"`` or ``"none"``. Telephony adapters +are moving to ``base64url`` *without* padding for inline audio bodies. + +Standard ``base64.b64decode`` chokes on that: with ``validate=True`` it +raises on the missing padding, and in its default (non-validating) mode it +silently drops the ``-``/``_`` characters that are only valid in the +URL-safe alphabet and hands back corrupted audio instead of raising. + +:func:`decode_inline_dialog_body` is the one place that should turn a +dialog's inline ``body`` into raw bytes. It dispatches on ``encoding`` +instead of guessing, so a body encoded in a way we do not understand raises +rather than silently returning garbage. +""" + +import base64 +import binascii +from typing import Any, Dict + + +def _restore_base64url_padding(body: str) -> str: + """Pad a base64url string out to a multiple of 4 characters. + + Adapters that emit base64url without padding are a valid rendering of + the same bytes -- padding is only there to make the length a multiple + of 4, and ``-``/``_`` already disambiguate the alphabet, so restoring + it here is lossless. + """ + return body + "=" * (-len(body) % 4) + + +def decode_inline_dialog_body(dialog: Dict[str, Any]) -> bytes: + """Decode a dialog's inline ``body`` per its ``encoding`` field. + + Args: + dialog: A vCon dialog object carrying an inline ``body`` (as opposed + to a ``url``-referenced one). + + Returns: + The decoded raw bytes. + + Raises: + ValueError: ``encoding`` is missing/legacy and the body is not valid + base64; ``encoding`` is ``"none"`` (the body is already a plain + string, not encoded audio bytes); or ``encoding`` is some other, + unrecognized value. + KeyError: ``dialog`` has no ``body``. + """ + body = dialog["body"] + encoding = dialog.get("encoding") + + if encoding == "base64url": + if not isinstance(body, str): + raise ValueError(f"Cannot decode base64url dialog body of type {type(body).__name__}; expected str") + try: + padded = _restore_base64url_padding(body) + # base64.urlsafe_b64decode() just translates "-"/"_" to "+"/"/" + # and calls b64decode() *without* validate=True, so it silently + # drops any other out-of-alphabet character instead of raising. + # Translate ourselves and validate explicitly to fail loudly on + # garbage input rather than return corrupted audio. + standard = padded.translate({ord("-"): "+", ord("_"): "/"}) + return base64.b64decode(standard, validate=True) + except (binascii.Error, ValueError) as e: + raise ValueError(f"Invalid base64url dialog body: {e}") from e + + if encoding == "base64" or encoding is None: + # Legacy: vCons stored before the encoding field was spec-mandated, + # or producers that never adopted base64url, use plain base64. + if not isinstance(body, str): + raise ValueError(f"Cannot decode base64 dialog body of type {type(body).__name__}; expected str") + try: + return base64.b64decode(body, validate=True) + except (binascii.Error, ValueError) as e: + raise ValueError(f"Invalid base64 dialog body: {e}") from e + + if encoding == "none": + raise ValueError("Dialog body encoding is 'none' (a plain string), not audio bytes") + + raise ValueError(f"Unrecognized dialog body encoding: {encoding!r}") diff --git a/common/tests/lib/test_dialog_body.py b/common/tests/lib/test_dialog_body.py new file mode 100644 index 0000000..5de8d6a --- /dev/null +++ b/common/tests/lib/test_dialog_body.py @@ -0,0 +1,70 @@ +import base64 + +import pytest + +from lib.dialog_body import decode_inline_dialog_body + + +def test_base64url_without_padding_round_trips(): + raw = b"hello world, this is audio-ish binary data" + body = base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=") + assert decode_inline_dialog_body({"encoding": "base64url", "body": body}) == raw + + +def test_base64url_with_padding_round_trips(): + raw = b"hello world, this is audio-ish binary dat" # len % 3 != 0 -> padded + body = base64.urlsafe_b64encode(raw).decode("ascii") + assert body.endswith("=") + assert decode_inline_dialog_body({"encoding": "base64url", "body": body}) == raw + + +def test_legacy_base64_round_trips(): + raw = b"legacy audio bytes" + body = base64.b64encode(raw).decode("ascii") + assert decode_inline_dialog_body({"encoding": "base64", "body": body}) == raw + + +def test_missing_encoding_treated_as_legacy_base64(): + raw = b"vcon stored before encoding was mandatory" + body = base64.b64encode(raw).decode("ascii") + assert decode_inline_dialog_body({"body": body}) == raw + + +def test_bytes_that_encode_to_dash_and_underscore_in_base64url_round_trip_exactly(): + # 0xFB -> "-" and 0xFF -> "_" only in the URL-safe alphabet ("+"/"/" in + # standard base64). Plain base64.b64decode in its default, non-validating + # mode silently drops "-"/"_" instead of raising, which is exactly the + # silent-corruption failure mode this helper exists to prevent. + raw = bytes([0xFB, 0xFF, 0x00, 0x10, 0x83, 0xFB, 0xFF]) + body = base64.urlsafe_b64encode(raw).decode("ascii") + assert "-" in body or "_" in body + unpadded = body.rstrip("=") + + decoded = decode_inline_dialog_body({"encoding": "base64url", "body": unpadded}) + assert decoded == raw + + # Prove the corruption this guards against: plain b64decode on the + # same (padded) base64url text does NOT raise and does NOT return the + # original bytes. + corrupted = base64.b64decode(body) + assert corrupted != raw + + +def test_none_encoding_raises_instead_of_returning_string_as_audio(): + with pytest.raises(ValueError): + decode_inline_dialog_body({"encoding": "none", "body": "just text"}) + + +def test_unrecognized_encoding_raises(): + with pytest.raises(ValueError): + decode_inline_dialog_body({"encoding": "json", "body": "{}"}) + + +def test_invalid_base64url_body_raises(): + with pytest.raises(ValueError): + decode_inline_dialog_body({"encoding": "base64url", "body": "not valid base64!!"}) + + +def test_invalid_legacy_base64_body_raises(): + with pytest.raises(ValueError): + decode_inline_dialog_body({"body": "not valid base64!!"}) From 73f3dc9e0fcff6f9da1353bd6dc67cc091b5f52c Mon Sep 17 00:00:00 2001 From: Thomas Howe Date: Fri, 25 Sep 2026 17:55:12 -0400 Subject: [PATCH 2/3] fix(links): decode inline dialog audio via shared base64url-safe helper groq_whisper and hugging_face_whisper decoded inline dialog audio with plain base64.b64decode(dialog["body"]) regardless of the dialog's encoding field, which corrupts or rejects unpadded base64url bodies from adapters moving to that encoding. wtf_transcribe already branched on encoding == "base64url" but used a non-validating, unpadded urlsafe_b64decode that has the same failure mode. Switch all three to common/lib/dialog_body.decode_inline_dialog_body(), which validates and pads as needed. URL-referenced (non-inline) dialogs are unchanged in all three links. Co-Authored-By: Claude Opus 5.5 --- conserver/links/groq_whisper/__init__.py | 6 ++- .../links/groq_whisper/test_groq_whisper.py | 11 +++++ .../links/hugging_face_whisper/__init__.py | 6 ++- .../test_hugging_face_whisper_dialog_body.py | 44 +++++++++++++++++++ conserver/links/wtf_transcribe/__init__.py | 6 +-- .../test_wtf_transcribe_unit.py | 5 +++ 6 files changed, 71 insertions(+), 7 deletions(-) create mode 100644 conserver/links/hugging_face_whisper/tests/test_hugging_face_whisper_dialog_body.py diff --git a/conserver/links/groq_whisper/__init__.py b/conserver/links/groq_whisper/__init__.py index 56b96e2..78f7b90 100644 --- a/conserver/links/groq_whisper/__init__.py +++ b/conserver/links/groq_whisper/__init__.py @@ -12,6 +12,7 @@ """ from lib.redaction import safe_opts +from lib.dialog_body import decode_inline_dialog_body import base64 import hashlib import logging @@ -139,8 +140,9 @@ def get_file_content(dialog: dict) -> bytes: Exception: If file cannot be retrieved or verified """ if "body" in dialog: - # body contains the base64 encoded content. Decode and return - return base64.b64decode(dialog["body"]) + # body contains the base64/base64url encoded content. Decode per + # the dialog's own encoding and return. + return decode_inline_dialog_body(dialog) elif "url" in dialog: # Handle external file diff --git a/conserver/links/groq_whisper/test_groq_whisper.py b/conserver/links/groq_whisper/test_groq_whisper.py index 9f17c94..a0d058c 100644 --- a/conserver/links/groq_whisper/test_groq_whisper.py +++ b/conserver/links/groq_whisper/test_groq_whisper.py @@ -451,6 +451,17 @@ def test_get_file_content_from_body(): content = get_file_content(dialog) assert content == b"test audio content" +def test_get_file_content_from_base64url_body_without_padding(): + """base64url dialogs (no padding, per draft-ietf-vcon-vcon-core-04) decode + to the original bytes, not the corrupted/raised result plain b64decode gives.""" + raw = b"test audio content, base64url, no padding" + dialog = { + "encoding": "base64url", + "body": base64.urlsafe_b64encode(raw).decode('utf-8').rstrip('='), + } + content = get_file_content(dialog) + assert content == raw + @patch('links.groq_whisper.requests.get') def test_get_file_content_from_url(mock_get): """Test extracting file content from URL reference""" diff --git a/conserver/links/hugging_face_whisper/__init__.py b/conserver/links/hugging_face_whisper/__init__.py index 13bb5a1..35cb0bd 100644 --- a/conserver/links/hugging_face_whisper/__init__.py +++ b/conserver/links/hugging_face_whisper/__init__.py @@ -11,6 +11,7 @@ """ from lib.redaction import safe_opts +from lib.dialog_body import decode_inline_dialog_body import base64 import hashlib import logging @@ -83,8 +84,9 @@ def get_file_content(dialog: dict) -> bytes: Exception: If file cannot be retrieved or verified """ if "body" in dialog: - # body contains the base64 encoded content. Decode and return - return base64.b64decode(dialog["body"]) + # body contains the base64/base64url encoded content. Decode per + # the dialog's own encoding and return. + return decode_inline_dialog_body(dialog) elif "url" in dialog: # Handle external file diff --git a/conserver/links/hugging_face_whisper/tests/test_hugging_face_whisper_dialog_body.py b/conserver/links/hugging_face_whisper/tests/test_hugging_face_whisper_dialog_body.py new file mode 100644 index 0000000..5f7a669 --- /dev/null +++ b/conserver/links/hugging_face_whisper/tests/test_hugging_face_whisper_dialog_body.py @@ -0,0 +1,44 @@ +"""get_file_content() decode coverage for CON-1112. + +The rest of this link's test suite (test_hugging_face_whisper.py, +test_hugging_face_whisper_integration.py) is commented out pending an HF API +key, but get_file_content() itself is pure and needs no network or model +call, so it is covered here unconditionally. +""" + +import base64 + +import pytest + +from links.hugging_face_whisper import get_file_content + + +def test_get_file_content_from_legacy_base64_body(): + dialog = {"body": base64.b64encode(b"test audio content").decode("utf-8")} + assert get_file_content(dialog) == b"test audio content" + + +def test_get_file_content_from_base64url_body_without_padding(): + """base64url dialogs (no padding, per draft-ietf-vcon-vcon-core-04) decode + to the original bytes, not the corrupted/raised result plain b64decode gives.""" + raw = b"test audio content, base64url, no padding" + dialog = { + "encoding": "base64url", + "body": base64.urlsafe_b64encode(raw).decode("utf-8").rstrip("="), + } + assert get_file_content(dialog) == raw + + +def test_get_file_content_from_base64url_body_with_padding(): + raw = b"test audio content, base64url, padded" + dialog = { + "encoding": "base64url", + "body": base64.urlsafe_b64encode(raw).decode("utf-8"), + } + assert get_file_content(dialog) == raw + + +def test_get_file_content_none_encoding_raises(): + dialog = {"encoding": "none", "body": "just text, not audio"} + with pytest.raises(ValueError): + get_file_content(dialog) diff --git a/conserver/links/wtf_transcribe/__init__.py b/conserver/links/wtf_transcribe/__init__.py index e00fc89..7323a35 100644 --- a/conserver/links/wtf_transcribe/__init__.py +++ b/conserver/links/wtf_transcribe/__init__.py @@ -23,7 +23,6 @@ url-timeout: 300 """ -import base64 import json import logging import requests @@ -32,6 +31,7 @@ from lib.vcon_redis import VconRedis from lib.logging_utils import init_logger from lib.error_tracking import init_error_tracker +from lib.dialog_body import decode_inline_dialog_body init_error_tracker() logger = init_logger(__name__) @@ -111,13 +111,13 @@ def is_base64url_dialog(dialog): return has_base64url_encoding(dialog) and has_body(dialog) def base64url_dialog_to_binary(dialog): - return base64.urlsafe_b64decode(dialog["body"]) + return decode_inline_dialog_body(dialog) def is_base64_dialog(dialog): return has_body(dialog) def base64_dialog_to_binary(dialog): - return base64.b64decode(dialog["body"]) + return decode_inline_dialog_body(dialog) def dialog_to_binary(dialog, url_timeout=60): if is_url_dialog(dialog): diff --git a/conserver/links/wtf_transcribe/test_wtf_transcribe_unit.py b/conserver/links/wtf_transcribe/test_wtf_transcribe_unit.py index 6432c7a..85e3e02 100644 --- a/conserver/links/wtf_transcribe/test_wtf_transcribe_unit.py +++ b/conserver/links/wtf_transcribe/test_wtf_transcribe_unit.py @@ -92,6 +92,11 @@ def test_dialog_to_binary_handles_url_base64url_base64_and_invalid_inputs(): base64url_body = base64.urlsafe_b64encode(b"url-safe").decode("ascii") assert dialog_to_binary({"encoding": "base64url", "body": base64url_body}) == b"url-safe" + # Telephony adapters emit base64url WITHOUT padding (draft-ietf-vcon-vcon-core-04); + # plain base64.urlsafe_b64decode raises on that, so this must still round-trip. + unpadded_base64url_body = base64.urlsafe_b64encode(b"url-safe, unpadded").decode("ascii").rstrip("=") + assert dialog_to_binary({"encoding": "base64url", "body": unpadded_base64url_body}) == b"url-safe, unpadded" + base64_body = base64.b64encode(b"plain").decode("ascii") assert dialog_to_binary({"body": base64_body}) == b"plain" From 2c5bbd8894721b33a1b86168ce9555dba95cc239 Mon Sep 17 00:00:00 2001 From: Thomas Howe Date: Fri, 25 Sep 2026 17:58:10 -0400 Subject: [PATCH 3/3] fix(lib): accept line-wrapped base64 in decode_inline_dialog_body The previous non-validating decode skipped line breaks, so MIME-style bodies wrapped at 76 columns decoded. Strip whitespace before the validating decode so those keep working; other stray characters still fail. Co-Authored-By: Claude Opus 5.5 --- common/lib/dialog_body.py | 5 +++++ common/tests/lib/test_dialog_body.py | 10 ++++++++++ 2 files changed, 15 insertions(+) diff --git a/common/lib/dialog_body.py b/common/lib/dialog_body.py index 61652dd..5423d8d 100644 --- a/common/lib/dialog_body.py +++ b/common/lib/dialog_body.py @@ -50,6 +50,11 @@ def decode_inline_dialog_body(dialog: Dict[str, Any]) -> bytes: """ body = dialog["body"] encoding = dialog.get("encoding") + if isinstance(body, str): + # Some producers wrap base64 at 76 columns (MIME style). The old + # non-validating decode skipped the line breaks; keep accepting them + # while still rejecting any other out-of-alphabet character. + body = "".join(body.split()) if encoding == "base64url": if not isinstance(body, str): diff --git a/common/tests/lib/test_dialog_body.py b/common/tests/lib/test_dialog_body.py index 5de8d6a..1c2ed5e 100644 --- a/common/tests/lib/test_dialog_body.py +++ b/common/tests/lib/test_dialog_body.py @@ -68,3 +68,13 @@ def test_invalid_base64url_body_raises(): def test_invalid_legacy_base64_body_raises(): with pytest.raises(ValueError): decode_inline_dialog_body({"body": "not valid base64!!"}) + + +def test_line_wrapped_bodies_decode(): + raw = bytes(range(256)) * 2 + std = base64.b64encode(raw).decode() + wrapped = "\n".join(std[i : i + 76] for i in range(0, len(std), 76)) + assert decode_inline_dialog_body({"body": wrapped, "encoding": "base64"}) == raw + url = base64.urlsafe_b64encode(raw).decode().rstrip("=") + wrapped_url = "\r\n".join(url[i : i + 76] for i in range(0, len(url), 76)) + assert decode_inline_dialog_body({"body": wrapped_url, "encoding": "base64url"}) == raw