diff --git a/common/lib/dialog_body.py b/common/lib/dialog_body.py new file mode 100644 index 00000000..5423d8d1 --- /dev/null +++ b/common/lib/dialog_body.py @@ -0,0 +1,87 @@ +"""Decoding for inline dialog bodies. + +draft-ietf-vcon-vcon-core-04 allows exactly three values for a dialog's +``encoding``: ``"base64url"``, ``"json"`` or ``"none"``. Telephony adapters +are moving to ``base64url`` *without* padding for inline audio bodies. + +Standard ``base64.b64decode`` chokes on that: with ``validate=True`` it +raises on the missing padding, and in its default (non-validating) mode it +silently drops the ``-``/``_`` characters that are only valid in the +URL-safe alphabet and hands back corrupted audio instead of raising. + +:func:`decode_inline_dialog_body` is the one place that should turn a +dialog's inline ``body`` into raw bytes. It dispatches on ``encoding`` +instead of guessing, so a body encoded in a way we do not understand raises +rather than silently returning garbage. +""" + +import base64 +import binascii +from typing import Any, Dict + + +def _restore_base64url_padding(body: str) -> str: + """Pad a base64url string out to a multiple of 4 characters. + + Adapters that emit base64url without padding are a valid rendering of + the same bytes -- padding is only there to make the length a multiple + of 4, and ``-``/``_`` already disambiguate the alphabet, so restoring + it here is lossless. + """ + return body + "=" * (-len(body) % 4) + + +def decode_inline_dialog_body(dialog: Dict[str, Any]) -> bytes: + """Decode a dialog's inline ``body`` per its ``encoding`` field. + + Args: + dialog: A vCon dialog object carrying an inline ``body`` (as opposed + to a ``url``-referenced one). + + Returns: + The decoded raw bytes. + + Raises: + ValueError: ``encoding`` is missing/legacy and the body is not valid + base64; ``encoding`` is ``"none"`` (the body is already a plain + string, not encoded audio bytes); or ``encoding`` is some other, + unrecognized value. + KeyError: ``dialog`` has no ``body``. + """ + body = dialog["body"] + encoding = dialog.get("encoding") + if isinstance(body, str): + # Some producers wrap base64 at 76 columns (MIME style). The old + # non-validating decode skipped the line breaks; keep accepting them + # while still rejecting any other out-of-alphabet character. + body = "".join(body.split()) + + if encoding == "base64url": + if not isinstance(body, str): + raise ValueError(f"Cannot decode base64url dialog body of type {type(body).__name__}; expected str") + try: + padded = _restore_base64url_padding(body) + # base64.urlsafe_b64decode() just translates "-"/"_" to "+"/"/" + # and calls b64decode() *without* validate=True, so it silently + # drops any other out-of-alphabet character instead of raising. + # Translate ourselves and validate explicitly to fail loudly on + # garbage input rather than return corrupted audio. + standard = padded.translate({ord("-"): "+", ord("_"): "/"}) + return base64.b64decode(standard, validate=True) + except (binascii.Error, ValueError) as e: + raise ValueError(f"Invalid base64url dialog body: {e}") from e + + if encoding == "base64" or encoding is None: + # Legacy: vCons stored before the encoding field was spec-mandated, + # or producers that never adopted base64url, use plain base64. + if not isinstance(body, str): + raise ValueError(f"Cannot decode base64 dialog body of type {type(body).__name__}; expected str") + try: + return base64.b64decode(body, validate=True) + except (binascii.Error, ValueError) as e: + raise ValueError(f"Invalid base64 dialog body: {e}") from e + + if encoding == "none": + raise ValueError("Dialog body encoding is 'none' (a plain string), not audio bytes") + + raise ValueError(f"Unrecognized dialog body encoding: {encoding!r}") diff --git a/common/tests/lib/test_dialog_body.py b/common/tests/lib/test_dialog_body.py new file mode 100644 index 00000000..1c2ed5ea --- /dev/null +++ b/common/tests/lib/test_dialog_body.py @@ -0,0 +1,80 @@ +import base64 + +import pytest + +from lib.dialog_body import decode_inline_dialog_body + + +def test_base64url_without_padding_round_trips(): + raw = b"hello world, this is audio-ish binary data" + body = base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=") + assert decode_inline_dialog_body({"encoding": "base64url", "body": body}) == raw + + +def test_base64url_with_padding_round_trips(): + raw = b"hello world, this is audio-ish binary dat" # len % 3 != 0 -> padded + body = base64.urlsafe_b64encode(raw).decode("ascii") + assert body.endswith("=") + assert decode_inline_dialog_body({"encoding": "base64url", "body": body}) == raw + + +def test_legacy_base64_round_trips(): + raw = b"legacy audio bytes" + body = base64.b64encode(raw).decode("ascii") + assert decode_inline_dialog_body({"encoding": "base64", "body": body}) == raw + + +def test_missing_encoding_treated_as_legacy_base64(): + raw = b"vcon stored before encoding was mandatory" + body = base64.b64encode(raw).decode("ascii") + assert decode_inline_dialog_body({"body": body}) == raw + + +def test_bytes_that_encode_to_dash_and_underscore_in_base64url_round_trip_exactly(): + # 0xFB -> "-" and 0xFF -> "_" only in the URL-safe alphabet ("+"/"/" in + # standard base64). Plain base64.b64decode in its default, non-validating + # mode silently drops "-"/"_" instead of raising, which is exactly the + # silent-corruption failure mode this helper exists to prevent. + raw = bytes([0xFB, 0xFF, 0x00, 0x10, 0x83, 0xFB, 0xFF]) + body = base64.urlsafe_b64encode(raw).decode("ascii") + assert "-" in body or "_" in body + unpadded = body.rstrip("=") + + decoded = decode_inline_dialog_body({"encoding": "base64url", "body": unpadded}) + assert decoded == raw + + # Prove the corruption this guards against: plain b64decode on the + # same (padded) base64url text does NOT raise and does NOT return the + # original bytes. + corrupted = base64.b64decode(body) + assert corrupted != raw + + +def test_none_encoding_raises_instead_of_returning_string_as_audio(): + with pytest.raises(ValueError): + decode_inline_dialog_body({"encoding": "none", "body": "just text"}) + + +def test_unrecognized_encoding_raises(): + with pytest.raises(ValueError): + decode_inline_dialog_body({"encoding": "json", "body": "{}"}) + + +def test_invalid_base64url_body_raises(): + with pytest.raises(ValueError): + decode_inline_dialog_body({"encoding": "base64url", "body": "not valid base64!!"}) + + +def test_invalid_legacy_base64_body_raises(): + with pytest.raises(ValueError): + decode_inline_dialog_body({"body": "not valid base64!!"}) + + +def test_line_wrapped_bodies_decode(): + raw = bytes(range(256)) * 2 + std = base64.b64encode(raw).decode() + wrapped = "\n".join(std[i : i + 76] for i in range(0, len(std), 76)) + assert decode_inline_dialog_body({"body": wrapped, "encoding": "base64"}) == raw + url = base64.urlsafe_b64encode(raw).decode().rstrip("=") + wrapped_url = "\r\n".join(url[i : i + 76] for i in range(0, len(url), 76)) + assert decode_inline_dialog_body({"body": wrapped_url, "encoding": "base64url"}) == raw diff --git a/conserver/links/groq_whisper/__init__.py b/conserver/links/groq_whisper/__init__.py index 56b96e23..78f7b902 100644 --- a/conserver/links/groq_whisper/__init__.py +++ b/conserver/links/groq_whisper/__init__.py @@ -12,6 +12,7 @@ """ from lib.redaction import safe_opts +from lib.dialog_body import decode_inline_dialog_body import base64 import hashlib import logging @@ -139,8 +140,9 @@ def get_file_content(dialog: dict) -> bytes: Exception: If file cannot be retrieved or verified """ if "body" in dialog: - # body contains the base64 encoded content. Decode and return - return base64.b64decode(dialog["body"]) + # body contains the base64/base64url encoded content. Decode per + # the dialog's own encoding and return. + return decode_inline_dialog_body(dialog) elif "url" in dialog: # Handle external file diff --git a/conserver/links/groq_whisper/test_groq_whisper.py b/conserver/links/groq_whisper/test_groq_whisper.py index 9f17c944..a0d058cf 100644 --- a/conserver/links/groq_whisper/test_groq_whisper.py +++ b/conserver/links/groq_whisper/test_groq_whisper.py @@ -451,6 +451,17 @@ def test_get_file_content_from_body(): content = get_file_content(dialog) assert content == b"test audio content" +def test_get_file_content_from_base64url_body_without_padding(): + """base64url dialogs (no padding, per draft-ietf-vcon-vcon-core-04) decode + to the original bytes, not the corrupted/raised result plain b64decode gives.""" + raw = b"test audio content, base64url, no padding" + dialog = { + "encoding": "base64url", + "body": base64.urlsafe_b64encode(raw).decode('utf-8').rstrip('='), + } + content = get_file_content(dialog) + assert content == raw + @patch('links.groq_whisper.requests.get') def test_get_file_content_from_url(mock_get): """Test extracting file content from URL reference""" diff --git a/conserver/links/hugging_face_whisper/__init__.py b/conserver/links/hugging_face_whisper/__init__.py index 13bb5a11..35cb0bd3 100644 --- a/conserver/links/hugging_face_whisper/__init__.py +++ b/conserver/links/hugging_face_whisper/__init__.py @@ -11,6 +11,7 @@ """ from lib.redaction import safe_opts +from lib.dialog_body import decode_inline_dialog_body import base64 import hashlib import logging @@ -83,8 +84,9 @@ def get_file_content(dialog: dict) -> bytes: Exception: If file cannot be retrieved or verified """ if "body" in dialog: - # body contains the base64 encoded content. Decode and return - return base64.b64decode(dialog["body"]) + # body contains the base64/base64url encoded content. Decode per + # the dialog's own encoding and return. + return decode_inline_dialog_body(dialog) elif "url" in dialog: # Handle external file diff --git a/conserver/links/hugging_face_whisper/tests/test_hugging_face_whisper_dialog_body.py b/conserver/links/hugging_face_whisper/tests/test_hugging_face_whisper_dialog_body.py new file mode 100644 index 00000000..5f7a6691 --- /dev/null +++ b/conserver/links/hugging_face_whisper/tests/test_hugging_face_whisper_dialog_body.py @@ -0,0 +1,44 @@ +"""get_file_content() decode coverage for CON-1112. + +The rest of this link's test suite (test_hugging_face_whisper.py, +test_hugging_face_whisper_integration.py) is commented out pending an HF API +key, but get_file_content() itself is pure and needs no network or model +call, so it is covered here unconditionally. +""" + +import base64 + +import pytest + +from links.hugging_face_whisper import get_file_content + + +def test_get_file_content_from_legacy_base64_body(): + dialog = {"body": base64.b64encode(b"test audio content").decode("utf-8")} + assert get_file_content(dialog) == b"test audio content" + + +def test_get_file_content_from_base64url_body_without_padding(): + """base64url dialogs (no padding, per draft-ietf-vcon-vcon-core-04) decode + to the original bytes, not the corrupted/raised result plain b64decode gives.""" + raw = b"test audio content, base64url, no padding" + dialog = { + "encoding": "base64url", + "body": base64.urlsafe_b64encode(raw).decode("utf-8").rstrip("="), + } + assert get_file_content(dialog) == raw + + +def test_get_file_content_from_base64url_body_with_padding(): + raw = b"test audio content, base64url, padded" + dialog = { + "encoding": "base64url", + "body": base64.urlsafe_b64encode(raw).decode("utf-8"), + } + assert get_file_content(dialog) == raw + + +def test_get_file_content_none_encoding_raises(): + dialog = {"encoding": "none", "body": "just text, not audio"} + with pytest.raises(ValueError): + get_file_content(dialog) diff --git a/conserver/links/wtf_transcribe/__init__.py b/conserver/links/wtf_transcribe/__init__.py index e00fc89c..7323a35e 100644 --- a/conserver/links/wtf_transcribe/__init__.py +++ b/conserver/links/wtf_transcribe/__init__.py @@ -23,7 +23,6 @@ url-timeout: 300 """ -import base64 import json import logging import requests @@ -32,6 +31,7 @@ from lib.vcon_redis import VconRedis from lib.logging_utils import init_logger from lib.error_tracking import init_error_tracker +from lib.dialog_body import decode_inline_dialog_body init_error_tracker() logger = init_logger(__name__) @@ -111,13 +111,13 @@ def is_base64url_dialog(dialog): return has_base64url_encoding(dialog) and has_body(dialog) def base64url_dialog_to_binary(dialog): - return base64.urlsafe_b64decode(dialog["body"]) + return decode_inline_dialog_body(dialog) def is_base64_dialog(dialog): return has_body(dialog) def base64_dialog_to_binary(dialog): - return base64.b64decode(dialog["body"]) + return decode_inline_dialog_body(dialog) def dialog_to_binary(dialog, url_timeout=60): if is_url_dialog(dialog): diff --git a/conserver/links/wtf_transcribe/test_wtf_transcribe_unit.py b/conserver/links/wtf_transcribe/test_wtf_transcribe_unit.py index 6432c7ae..85e3e021 100644 --- a/conserver/links/wtf_transcribe/test_wtf_transcribe_unit.py +++ b/conserver/links/wtf_transcribe/test_wtf_transcribe_unit.py @@ -92,6 +92,11 @@ def test_dialog_to_binary_handles_url_base64url_base64_and_invalid_inputs(): base64url_body = base64.urlsafe_b64encode(b"url-safe").decode("ascii") assert dialog_to_binary({"encoding": "base64url", "body": base64url_body}) == b"url-safe" + # Telephony adapters emit base64url WITHOUT padding (draft-ietf-vcon-vcon-core-04); + # plain base64.urlsafe_b64decode raises on that, so this must still round-trip. + unpadded_base64url_body = base64.urlsafe_b64encode(b"url-safe, unpadded").decode("ascii").rstrip("=") + assert dialog_to_binary({"encoding": "base64url", "body": unpadded_base64url_body}) == b"url-safe, unpadded" + base64_body = base64.b64encode(b"plain").decode("ascii") assert dialog_to_binary({"body": base64_body}) == b"plain"