From 07b91ac6a69e324d26b7814c28172ff8c0c2908d Mon Sep 17 00:00:00 2001 From: Thomas Howe Date: Fri, 25 Sep 2026 18:17:56 -0400 Subject: [PATCH 1/2] fix(vcon): stop stringifying JSON attachment/analysis bodies (CON-1111) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit draft-ietf-vcon-vcon-core-04 §2.3.2: with encoding "json" the body is the JSON value itself (object/array), not a stringified copy. Adapters now emit raw values, so the conserver must stop converting them back into strings on write. - common/vcon.py: add_attachment/add_analysis keep dict/list bodies as values with encoding forced to "json"; JSON validation only applies to str bodies claiming encoding "json" (the legacy -02 stringified shape). - common/lib/vcon_redis.py: VconRedis._enforce_spec_on_write no longer stringifies dict/list bodies. It still fixes a real mismatch (dict/list body with encoding "none"/absent) by stamping encoding to "json" without touching the value. - common/lib/vcon_egress_compat.py: the inverse (_body_to_legacy) now handles both shapes — a native dict/list body under encoding "json" is flipped straight to legacy encoding "none"; a str body is parsed first as before. - common/lib/links/filters.py: is_included's includes substring match now also matches a dict/list body via its JSON-serialized form, since it can no longer assume body is always a string. - conserver/links/post_analysis_to_slack: the customer_frustration includes check and the summary-body-as-Slack-text path now tolerate a raw dict/list body the same way. Every other storage backend (postgres, mongo, s3, elasticsearch, utopia, redis_storage, vcon_mcp, file) persists the whole vCon dict as JSON/BSON/ JSONB rather than treating body specially, so raw bodies already round-trip unchanged; elasticsearch already had an isinstance(body, str) guard before parsing. No migration: existing stored vCons keep string bodies, and every reader (Vcon.decoded_body, egress_compat, filters, storage get()) still accepts both shapes. Co-Authored-By: Claude Opus 5.5 --- common/lib/links/filters.py | 17 +++++-- common/lib/vcon_egress_compat.py | 40 +++++++++------ common/lib/vcon_redis.py | 35 +++++++------ common/vcon.py | 50 +++++++++---------- .../links/post_analysis_to_slack/__init__.py | 19 +++++-- 5 files changed, 100 insertions(+), 61 deletions(-) diff --git a/common/lib/links/filters.py b/common/lib/links/filters.py index 8e6d73e..b01ee1e 100644 --- a/common/lib/links/filters.py +++ b/common/lib/links/filters.py @@ -1,3 +1,4 @@ +import json import random from typing import Literal, Optional, TypedDict from lib.logging_utils import init_logger @@ -68,11 +69,19 @@ def is_included(options: Optional[FilterOptions], _vcon) -> bool: if isinstance(tags, list) and includes in tags: return True continue - # Per spec §2.3.2 ``body`` is always a String regardless of - # encoding, so substring-match directly without any decode. + # Per draft-ietf-vcon-vcon-core-04 §2.3.2, an ``encoding: "json"`` + # body is the JSON value itself (dict/list), not a string — so a + # plain substring match only works directly for str bodies (the + # legacy -02 stringified shape, or encoding "none"/"base64url"). + # For a dict/list body, match against its JSON-serialized form so + # ``includes`` still finds tokens inside it. body = element.get("body") - if isinstance(body, str) and includes in body: - return True + if isinstance(body, str): + if includes in body: + return True + elif isinstance(body, (dict, list)): + if includes in json.dumps(body): + return True except Exception as e: logger.error(f"Error checking inclusion: {e}") return False diff --git a/common/lib/vcon_egress_compat.py b/common/lib/vcon_egress_compat.py index 1cfda66..34b3b79 100644 --- a/common/lib/vcon_egress_compat.py +++ b/common/lib/vcon_egress_compat.py @@ -5,11 +5,13 @@ The conserver normalizes every vCon *up* to the current spec (``vcon: "0.4.0"``) on read and write. Downstream consumers built against an older schema (e.g. ``0.0.1``) break on the canonical shape — notably the ``type`` → ``purpose`` -attachment rename and the write-path serialization of dict/list analysis and -attachment bodies into JSON strings (``encoding: "json"``). This module converts -an *outgoing* payload back to a legacy version — reversing the field renames and -re-inflating those JSON-string bodies to native objects with ``encoding: "none"`` -— so those consumers keep working while a migration is planned. +attachment rename and, per draft-ietf-vcon-vcon-core-04 §2.3.2, dict/list +analysis/attachment bodies carried as native JSON values under +``encoding: "json"`` rather than stringified. This module converts an +*outgoing* payload back to a legacy version — reversing the field renames and +turning those native/stringified JSON bodies into legacy-shape native objects +with ``encoding: "none"`` — so those consumers keep working while a migration +is planned. It never mutates the canonical in-pipeline copy: callers pass ``vcon.to_dict()`` and receive a new, deep-copied, downgraded dict. Enable it per egress point via @@ -68,22 +70,30 @@ def _entry_to_legacy(entry: Dict[str, Any]) -> None: def _body_to_legacy(entry: Dict[str, Any]) -> None: - """Inverse of ``VconRedis._stringify_json_body``. - - The spec write-path serializes dict/list ``body`` values to a JSON string - and sets ``encoding: "json"``. The legacy 0.0.1 shape carries the native - object/array with ``encoding: "none"``, so parse it back. Applied to - analysis and attachment entries only — dialog bodies are not stringified on - write. Left untouched if the body isn't valid JSON. + """Inverse of the -04 write path's ``encoding: "json"`` shape. + + Per draft-ietf-vcon-vcon-core-04 §2.3.2, a spec-current ``encoding: "json"`` + body is the JSON value itself (dict/list) — no parsing needed, just flip + ``encoding`` to the legacy 0.0.1 ``"none"``. A str body under + ``encoding: "json"`` is the older -02 stringified shape (still seen in + storage rows written before this change, or from writers that pass a + pre-serialized string); parse it back to a native value. Applied to + analysis and attachment entries only — dialog bodies are not touched on + write. Left untouched if a str body isn't valid JSON. """ if not isinstance(entry, dict): return - if entry.get("encoding") == "json" and isinstance(entry.get("body"), str): + if entry.get("encoding") != "json": + return + body = entry.get("body") + if isinstance(body, str): try: - entry["body"] = json.loads(entry["body"]) + entry["body"] = json.loads(body) except (ValueError, TypeError): return - entry["encoding"] = "none" + elif not isinstance(body, (dict, list)): + return + entry["encoding"] = "none" def _attachment_to_legacy(att: Dict[str, Any]) -> None: diff --git a/common/lib/vcon_redis.py b/common/lib/vcon_redis.py index 8de076b..997aba2 100644 --- a/common/lib/vcon_redis.py +++ b/common/lib/vcon_redis.py @@ -1,4 +1,3 @@ -import json from datetime import datetime from typing import Any, Optional from config import Configuration @@ -36,20 +35,26 @@ class VconRedis: DEFAULT_TTL = VCON_REDIS_EXPIRY @staticmethod - def _stringify_json_body(entry: Any) -> None: - """Force ``body`` to be a string + correct ``encoding`` per speckit. - - The speckit non-negotiable: analysis/attachment bodies are - strings. JSON content pairs ``body: json.dumps(...)`` with - ``encoding: "json"``. vcon-lib's ``add_analysis`` currently - emits dict/list bodies with ``encoding: "none"``, which we - normalize on the way out so storage is spec-correct. + def _normalize_json_body_encoding(entry: Any) -> None: + """Fix a mismatched ``encoding`` on a dict/list ``body`` per -04. + + Per draft-ietf-vcon-vcon-core-04 §2.3.2, with ``encoding: "json"`` + the ``body`` is the JSON value itself (object/array), not a + stringified copy. A dict/list body should therefore carry + ``encoding: "json"`` as-is; we no longer ``json.dumps`` it away. + The only thing to fix here is a real mismatch: a dict/list body + that arrived with ``encoding: "none"`` (or no encoding at all, + e.g. from vcon-lib's ``add_analysis``) is stamped to ``"json"`` + without touching the value. + + A ``str`` body with ``encoding: "json"`` is left untouched — that's + the legacy -02 stringified shape, which readers (``Vcon.decoded_body``) + still decode. """ if not isinstance(entry, dict): return body = entry.get("body") - if isinstance(body, (dict, list)): - entry["body"] = json.dumps(body) + if isinstance(body, (dict, list)) and entry.get("encoding") != "json": entry["encoding"] = "json" @classmethod @@ -81,11 +86,13 @@ def _enforce_spec_on_write(cls, vcon_dict: dict) -> dict: # speckit: empty ``redacted: {}`` should be omitted. if vcon_dict.get("redacted") == {}: vcon_dict.pop("redacted", None) - # speckit: analysis/attachment bodies are strings. + # draft-ietf-vcon-vcon-core-04 §2.3.2: with encoding "json" the body + # is the JSON value itself; only fix a real dict/list-vs-encoding + # mismatch, never stringify. for entry in vcon_dict.get("analysis", []) or []: - cls._stringify_json_body(entry) + cls._normalize_json_body_encoding(entry) for entry in vcon_dict.get("attachments", []) or []: - cls._stringify_json_body(entry) + cls._normalize_json_body_encoding(entry) return vcon_dict def store_vcon(self, vCon: vcon.Vcon, ttl: Optional[int] = None) -> None: diff --git a/common/vcon.py b/common/vcon.py index dc45565..1d75724 100644 --- a/common/vcon.py +++ b/common/vcon.py @@ -70,19 +70,21 @@ def tags(self): def decoded_body(entry): """Return an attachment/analysis ``body`` as a live Python value. - Per draft-ietf-vcon-vcon-core-02 §2.3.2 ``body`` is *always* a String; - the ``encoding`` tells you how to interpret it: - - - ``json`` → body is a JSON-encoded object/array, parse with ``json.loads``. + Per draft-ietf-vcon-vcon-core-04 §2.3.2, with ``encoding: "json"`` + the ``body`` is the JSON value itself (object/array) — that's the + common case now and is returned as-is. A JSON-decoded value never + needs to be str-decoded, so this only parses when ``encoding`` is + ``"json"`` *and* ``body`` is still a ``str`` — the legacy -02 + stringified shape that older producers or storage rows may still + carry. + + - ``json`` + dict/list body → returned as-is (current spec shape). + - ``json`` + str body → parsed with ``json.loads`` (legacy shape). - ``base64url`` → body is base64url-encoded bytes, returned verbatim (binary decoding is caller-specific). - - ``none`` → body is a freeform string, returned verbatim. - - For backwards compatibility with legacy writers that placed a raw - dict/list under ``body`` with ``encoding: none``, the dict/list is - returned as-is. ``VconRedis._enforce_spec_on_write`` later normalises - that to spec-correct ``encoding: json`` + stringified body, after - which this helper still returns the same Python value on reload. + - ``none`` → body is a freeform string, returned verbatim. A raw + dict/list under ``body`` with ``encoding: none`` (a real writer + mismatch) is likewise returned as-is. Returns ``None`` if ``entry`` is falsy. """ @@ -176,15 +178,14 @@ def add_attachment(self, *, body: Union[dict, list, str], type: str, encoding="n if encoding not in ['json', 'none', 'base64url']: raise Exception("Invalid encoding") - # Per draft-ietf-vcon-vcon-core-02 §2.3.2 ``body`` is always a String. - # If a caller hands us a dict/list as a convenience, JSON-encode it - # immediately so any reader that touches the attachment between now - # and storage sees the spec-correct shape. + # Per draft-ietf-vcon-vcon-core-04 §2.3.2, with encoding "json" the + # body is the JSON value itself (object/array), not a stringified + # copy. A dict/list body is stored as-is with encoding forced to + # "json"; a str body is only validated as JSON when it claims to be + # encoded that way (the legacy -02 stringified shape). if isinstance(body, (dict, list)): - body = json.dumps(body) encoding = "json" - - if encoding == "json": + elif encoding == "json": try: json.loads(body) except Exception as e: @@ -209,15 +210,14 @@ def add_analysis(self, *, type: str, dialog: Union[list, int], vendor: str, body if encoding not in ['json', 'none', 'base64url']: raise Exception("Invalid encoding") - # Per draft-ietf-vcon-vcon-core-02 §2.3.2 ``body`` is always a String. - # If a caller hands us a dict/list as a convenience, JSON-encode it - # immediately so any reader that touches the analysis between now - # and storage sees the spec-correct shape. + # Per draft-ietf-vcon-vcon-core-04 §2.3.2, with encoding "json" the + # body is the JSON value itself (object/array), not a stringified + # copy. A dict/list body is stored as-is with encoding forced to + # "json"; a str body is only validated as JSON when it claims to be + # encoded that way (the legacy -02 stringified shape). if isinstance(body, (dict, list)): - body = json.dumps(body) encoding = "json" - - if encoding == "json": + elif encoding == "json": try: json.loads(body) except Exception as e: diff --git a/conserver/links/post_analysis_to_slack/__init__.py b/conserver/links/post_analysis_to_slack/__init__.py index 57dcd75..5c9387f 100644 --- a/conserver/links/post_analysis_to_slack/__init__.py +++ b/conserver/links/post_analysis_to_slack/__init__.py @@ -1,3 +1,5 @@ +import json + from lib.vcon_redis import VconRedis from lib.logging_utils import init_logger from lib.metrics import increment_counter @@ -122,9 +124,14 @@ def run(vcon_id, link_name, opts=default_options): # we need to skip first one an only post the second one to slack if a["type"] != opts["only_if"]["analysis_type"]: continue - # Per draft-ietf-vcon-vcon-core-02 §2.3.2 body is always a String — - # substring-match directly without decoding. - if opts["only_if"]["includes"] not in a["body"]: + # Per draft-ietf-vcon-vcon-core-04 §2.3.2, an ``encoding: "json"`` + # body is the JSON value itself (dict/list) rather than a string, so + # substring-match its JSON-serialized form in that case; a str body + # (the common case for this analysis type, and the legacy -02 + # stringified shape) is matched directly. + body = a["body"] + haystack = body if isinstance(body, str) else json.dumps(body) + if opts["only_if"]["includes"] not in haystack: continue if a.get("was_posted_to_slack"): continue @@ -140,7 +147,13 @@ def run(vcon_id, link_name, opts=default_options): team_name = get_team(vcon) dealer_name = get_dealer(vcon) summary = get_summary(vcon, a["dialog"]) + # The summary body is posted as Slack message text, which must be a + # string. It's normally already one (encoding "none"/legacy -02 + # stringified "json"); a raw dict/list body (spec-current -04 + # "json" encoding) is serialized for display. abstract = summary["body"] + if not isinstance(abstract, str): + abstract = json.dumps(abstract) if team_name and team_name != "strolid": channel_name = f"team-{team_name}-alerts" From aa21bc959c61e8af9bd3e45ceded9d68d60694df Mon Sep 17 00:00:00 2001 From: Thomas Howe Date: Fri, 25 Sep 2026 18:18:07 -0400 Subject: [PATCH 2/2] test(vcon): cover raw JSON body behavior across storage and links (CON-1111) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Updates existing tests that asserted the old stringify-on-write behavior, and adds coverage for the new draft-ietf-vcon-vcon-core-04 §2.3.2 shape: - common/tests/test_vcon_redis_spec_enforce.py, common/tests/test_vcon_egress_compat.py, common/tests/test_encoding.py, common/tests/lib/filters/test_is_included.py, tests/core/test_vcon.py, common/tests/test_api_branches.py: updated to expect dict/list bodies to stay native (not JSON-stringified), plus new cases for the encoding-mismatch fix, the legacy stringified shape staying untouched, and filters/is_included matching a raw dict/list body. - common/tests/test_api.py: new end-to-end test posting a dict lawful_basis body and a list tags body through POST /vcon and reading them back via GET /vcon/{uuid} as a dict and a list. - tests/storage/test_s3.py, tests/storage/test_postgres_body_roundtrip.py (new), tests/storage/test_mongo_body_roundtrip.py (new), tests/storage/test_elasticsearch_body_roundtrip.py (new), tests/storage/test_vcon_mcp_body_roundtrip.py (new), common/storage/file/test_file_storage.py, common/storage/redis_storage/test_redis_storage.py (new): each storage backend round-trips a raw dict/list body unchanged, with external services mocked. Full suite: 750 passed, 17 skipped (was 731 passed, 17 skipped on origin/main baseline; +19 net from new/updated tests). Co-Authored-By: Claude Opus 5.5 --- common/storage/file/test_file_storage.py | 28 +++++ .../redis_storage/test_redis_storage.py | 43 +++++++ common/tests/lib/filters/test_is_included.py | 53 ++++++++ common/tests/test_api.py | 43 +++++++ common/tests/test_api_branches.py | 11 +- common/tests/test_encoding.py | 10 +- common/tests/test_vcon_egress_compat.py | 21 ++++ common/tests/test_vcon_redis_spec_enforce.py | 56 +++++++-- tests/core/test_vcon.py | 53 ++++++-- .../test_elasticsearch_body_roundtrip.py | 70 +++++++++++ tests/storage/test_mongo_body_roundtrip.py | 90 ++++++++++++++ tests/storage/test_postgres_body_roundtrip.py | 115 ++++++++++++++++++ tests/storage/test_s3.py | 35 ++++++ tests/storage/test_vcon_mcp_body_roundtrip.py | 62 ++++++++++ 14 files changed, 663 insertions(+), 27 deletions(-) create mode 100644 common/storage/redis_storage/test_redis_storage.py create mode 100644 tests/storage/test_elasticsearch_body_roundtrip.py create mode 100644 tests/storage/test_mongo_body_roundtrip.py create mode 100644 tests/storage/test_postgres_body_roundtrip.py create mode 100644 tests/storage/test_vcon_mcp_body_roundtrip.py diff --git a/common/storage/file/test_file_storage.py b/common/storage/file/test_file_storage.py index 01f31ac..5027c19 100644 --- a/common/storage/file/test_file_storage.py +++ b/common/storage/file/test_file_storage.py @@ -255,6 +255,34 @@ def test_get_prefers_uncompressed_when_both_exist(self, temp_storage_dir, sample assert result["marker"] == "uncompressed" +class TestRawJsonBodyRoundtrip: + """CON-1111: an encoding='json' dict/list body (draft-ietf-vcon-vcon- + core-04 §2.3.2) must survive save()+get() unchanged. File storage writes + the whole vcon.dumps() JSON text, so this guards against anything + upstream re-stringifying it.""" + + def test_save_then_get_preserves_raw_dict_and_list_body(self, temp_storage_dir): + vcon = Vcon.build_new() + vcon.add_attachment(body={"lawful_basis": "consent"}, type="lawful_basis") + vcon.add_attachment(body=["source:test"], type="tags") + + opts = {"path": temp_storage_dir, "compression": False, "organize_by_date": False} + + with patch("storage.file.VconRedis") as MockVconRedis: + mock_redis = MagicMock() + mock_redis.get_vcon.return_value = vcon + MockVconRedis.return_value = mock_redis + save(vcon.uuid, opts) + + result = get(vcon.uuid, opts) + lawful_basis = next(a for a in result["attachments"] if a["type"] == "lawful_basis") + tags = next(a for a in result["attachments"] if a["type"] == "tags") + assert lawful_basis["body"] == {"lawful_basis": "consent"} + assert lawful_basis["encoding"] == "json" + assert tags["body"] == ["source:test"] + assert tags["encoding"] == "json" + + class TestDelete: """Tests for delete function.""" diff --git a/common/storage/redis_storage/test_redis_storage.py b/common/storage/redis_storage/test_redis_storage.py new file mode 100644 index 0000000..ae80c45 --- /dev/null +++ b/common/storage/redis_storage/test_redis_storage.py @@ -0,0 +1,43 @@ +"""CON-1111: redis_storage round-trips a raw JSON body. + +``storage.redis_storage.save`` writes ``vcon.dumps()`` verbatim as a plain +Redis string; ``get`` ``json.loads`` it back. Neither touches ``body`` +fields, so a raw dict/list body (draft-ietf-vcon-vcon-core-04 §2.3.2, +encoding "json") must ride along unchanged. +""" + +from unittest.mock import MagicMock, patch + +from vcon import Vcon +from storage.redis_storage import save, get + + +def test_save_then_get_preserves_raw_dict_and_list_body(): + vcon = Vcon.build_new() + vcon.add_attachment(body={"lawful_basis": "consent"}, type="lawful_basis") + vcon.add_attachment(body=["source:test"], type="tags") + + opts = {"prefix": "vcon_storage", "expires": 3600} + store = {} + + with patch("storage.redis_storage.VconRedis") as MockVconRedis, \ + patch("storage.redis_storage.redis") as mock_redis: + mock_vcon_redis = MagicMock() + mock_vcon_redis.get_vcon.return_value = vcon + MockVconRedis.return_value = mock_vcon_redis + + def _set(key, value, ex=None): + store[key] = value + + mock_redis.set.side_effect = _set + mock_redis.get.side_effect = lambda key: store.get(key) + + save(vcon.uuid, opts) + result = get(vcon.uuid, opts) + + lawful_basis = next(a for a in result["attachments"] if a["type"] == "lawful_basis") + tags = next(a for a in result["attachments"] if a["type"] == "tags") + assert lawful_basis["body"] == {"lawful_basis": "consent"} + assert lawful_basis["encoding"] == "json" + assert tags["body"] == ["source:test"] + assert tags["encoding"] == "json" diff --git a/common/tests/lib/filters/test_is_included.py b/common/tests/lib/filters/test_is_included.py index f3cf885..d5208e4 100644 --- a/common/tests/lib/filters/test_is_included.py +++ b/common/tests/lib/filters/test_is_included.py @@ -72,3 +72,56 @@ async def test_is_included(): }, _vcon, ) + + +@pytest.mark.asyncio +async def test_is_included_matches_raw_dict_body(): + # draft-ietf-vcon-vcon-core-04 §2.3.2: with encoding "json" the body is + # the JSON value itself (a dict here), not a string. The substring match + # must still find tokens inside it via its JSON-serialized form. + _vcon = vcon.Vcon.build_new() + _vcon.add_analysis( + type="customer_frustration", + body={"verdict": "NEEDS REVIEW", "score": 0.9}, + dialog=0, + vendor="FooBar Inc.", + ) + assert is_included( + { + "only_if": { + "section": "analysis", + "type": "customer_frustration", + "includes": "NEEDS REVIEW", + } + }, + _vcon, + ) + assert not is_included( + { + "only_if": { + "section": "analysis", + "type": "customer_frustration", + "includes": "ALL CLEAR", + } + }, + _vcon, + ) + + +@pytest.mark.asyncio +async def test_is_included_matches_raw_list_attachment_body(): + _vcon = vcon.Vcon.build_new() + _vcon.add_attachment( + body=["category:12", "category:99"], + type="tags2", + ) + assert is_included( + { + "only_if": { + "section": "attachments", + "type": "tags2", + "includes": "category:99", + } + }, + _vcon, + ) diff --git a/common/tests/test_api.py b/common/tests/test_api.py index 7442318..24cc197 100644 --- a/common/tests/test_api.py +++ b/common/tests/test_api.py @@ -99,6 +99,49 @@ def test_create_vcon_with_extra_attribute(): assert response.json()["meta"] == {"foo": "bar"} +@pytest.mark.anyio +def test_post_and_get_vcon_round_trips_raw_json_bodies(): + """CON-1111: draft-ietf-vcon-vcon-core-04 §2.3.2 — with encoding "json" + the body is the JSON value itself (object/array), not a stringified + copy. A dict lawful_basis attachment body and a list tags attachment + body must round-trip through POST /vcon + GET /vcon/{uuid} as a dict + and a list, not JSON strings. + """ + test_vcon = generate_mock_vcon() + test_vcon["attachments"] = [ + { + "purpose": "lawful_basis", + "body": { + "lawful_basis": "consent", + "purpose_grants": [{"purpose": "analysis", "granted": True}], + }, + "encoding": "json", + }, + { + "purpose": "tags", + "body": ["source:test", "direction:out"], + "encoding": "json", + }, + ] + post_vcon(test_vcon) + + with TestClient(api.app, headers={CONSERVER_HEADER_NAME: CONSERVER_API_TOKEN}) as client: + response = client.get("/vcon/{}".format(test_vcon["uuid"])) + assert response.status_code == 200 + body = response.json() + + lawful_basis = next(a for a in body["attachments"] if a["purpose"] == "lawful_basis") + tags = next(a for a in body["attachments"] if a["purpose"] == "tags") + + assert isinstance(lawful_basis["body"], dict) + assert lawful_basis["body"]["lawful_basis"] == "consent" + assert lawful_basis["encoding"] == "json" + + assert isinstance(tags["body"], list) + assert tags["body"] == ["source:test", "direction:out"] + assert tags["encoding"] == "json" + + _INVALID_FIXTURES_DIR = os.path.join(os.path.dirname(__file__), "invalid_fixtures") diff --git a/common/tests/test_api_branches.py b/common/tests/test_api_branches.py index c7252d5..891eda0 100644 --- a/common/tests/test_api_branches.py +++ b/common/tests/test_api_branches.py @@ -75,7 +75,12 @@ async def test_sync_vcon_from_storage_canonicalizes_legacy_payload(redis_async): """A storage backend may hold a legacy / egress-converted 0.0.1 payload (see egress_format_version). The fallback must canonicalize it before caching to Redis or returning to the client, so downstream links/clients - never see legacy field names or native (non-stringified) bodies.""" + never see legacy field names or a body/encoding mismatch. + + Per draft-ietf-vcon-vcon-core-04 §2.3.2, canonicalizing a dict/list body + no longer stringifies it — encoding "json" now means the body IS that + dict/list, so the fallback only needs to fix the mismatched "none" + encoding on the way in, keeping the value native.""" vcon_uuid = uuid4() legacy = { "uuid": str(vcon_uuid), @@ -94,9 +99,9 @@ async def test_sync_vcon_from_storage_canonicalizes_legacy_payload(redis_async): # Returned payload is canonical. att = result["attachments"][0] assert att.get("purpose") == "tags" and "type" not in att - assert isinstance(att["body"], str) and att["encoding"] == "json" + assert att["body"] == ["a", "b"] and att["encoding"] == "json" an = result["analysis"][0] - assert isinstance(an["body"], str) and an["encoding"] == "json" + assert an["body"] == {"k": "v"} and an["encoding"] == "json" # The copy cached back into Redis is canonical too (not the raw legacy dict). cached = json.loads(redis_async.set.await_args.args[1]) diff --git a/common/tests/test_encoding.py b/common/tests/test_encoding.py index 6161978..66c536d 100644 --- a/common/tests/test_encoding.py +++ b/common/tests/test_encoding.py @@ -10,9 +10,9 @@ def test_encoding(): print("test_vcon:", _vcon) test_vcon = vcon.Vcon(_vcon) - # Spec (draft-ietf-vcon-vcon-core-02 §2.3.2): body is always a String. A - # dict/list passed in for convenience is JSON-encoded at the boundary and - # the encoding is forced to "json". + # Spec (draft-ietf-vcon-vcon-core-04 §2.3.2): with encoding "json" body is + # the JSON value itself. A dict/list passed in is kept as-is and the + # encoding is forced to "json" (no stringification). test_vcon.add_attachment( type="test_encoding_dict_coerced", body={"key": "value"}, @@ -20,7 +20,7 @@ def test_encoding(): ) assert test_vcon.find_attachment_by_purpose("test_encoding_dict_coerced") == { "type": "test_encoding_dict_coerced", - "body": json.dumps({"key": "value"}), + "body": {"key": "value"}, "encoding": "json", } @@ -31,7 +31,7 @@ def test_encoding(): ) assert test_vcon.find_attachment_by_purpose("test_encoding_list_coerced") == { "type": "test_encoding_list_coerced", - "body": json.dumps(["key", "value"]), + "body": ["key", "value"], "encoding": "json", } diff --git a/common/tests/test_vcon_egress_compat.py b/common/tests/test_vcon_egress_compat.py index dcebd74..e41a5da 100644 --- a/common/tests/test_vcon_egress_compat.py +++ b/common/tests/test_vcon_egress_compat.py @@ -114,6 +114,27 @@ def test_json_string_body_inflated_to_native(): assert out["analysis"][0]["encoding"] == "none" +def test_raw_dict_and_list_json_body_flipped_to_legacy_encoding(): + """The -04 write path (spec-current): encoding 'json' body is already a + native dict/list. No parsing needed — just flip encoding to 'none' to + match the legacy 0.0.1 shape, leaving the value itself untouched.""" + canonical = _canonical_vcon() + canonical["attachments"] = [ + {"purpose": "tags", "body": ["source:crexendo", "direction:out"], "encoding": "json"}, + {"purpose": "tenant", "body": {"id": 385}, "encoding": "json"}, + ] + canonical["analysis"] = [ + {"type": "transcript", "dialog": 0, "vendor": "x", "body": {"transcript": "hi"}, "encoding": "json"}, + ] + out = to_legacy(canonical, "0.0.1") + assert out["attachments"][0]["body"] == ["source:crexendo", "direction:out"] + assert out["attachments"][0]["encoding"] == "none" + assert out["attachments"][1]["body"] == {"id": 385} + assert out["attachments"][1]["encoding"] == "none" + assert out["analysis"][0]["body"] == {"transcript": "hi"} + assert out["analysis"][0]["encoding"] == "none" + + def test_plain_string_body_untouched(): """A non-JSON body (encoding != 'json') is left exactly as-is.""" canonical = _canonical_vcon() diff --git a/common/tests/test_vcon_redis_spec_enforce.py b/common/tests/test_vcon_redis_spec_enforce.py index 74630d9..14bf6b7 100644 --- a/common/tests/test_vcon_redis_spec_enforce.py +++ b/common/tests/test_vcon_redis_spec_enforce.py @@ -51,7 +51,10 @@ def test_returns_same_dict(): assert VconRedis._enforce_spec_on_write(d) is d -def test_stringifies_dict_analysis_body(): +def test_keeps_dict_analysis_body_raw_and_fixes_encoding(): + # draft-ietf-vcon-vcon-core-04 §2.3.2: encoding "json" bodies are the + # JSON value itself. A dict body that arrived with encoding "none" is a + # real mismatch — fix the encoding, but never stringify the value. d = { "uuid": "u", "analysis": [ @@ -65,12 +68,11 @@ def test_stringifies_dict_analysis_body(): } VconRedis._enforce_spec_on_write(d) a = d["analysis"][0] - assert isinstance(a["body"], str) - assert a["body"] == '{"k": "v"}' + assert a["body"] == {"k": "v"} assert a["encoding"] == "json" -def test_stringifies_list_analysis_body(): +def test_keeps_list_analysis_body_raw_and_fixes_encoding(): d = { "uuid": "u", "analysis": [ @@ -84,7 +86,27 @@ def test_stringifies_list_analysis_body(): } VconRedis._enforce_spec_on_write(d) a = d["analysis"][0] - assert isinstance(a["body"], str) + assert a["body"] == [{"entry_id": "abc"}] + assert a["encoding"] == "json" + + +def test_keeps_dict_body_raw_when_encoding_already_json(): + # Adapters now emit raw values directly with encoding already "json" — + # no mismatch to fix, value must be untouched. + d = { + "uuid": "u", + "analysis": [ + { + "type": "summary", + "vendor": "openai", + "body": {"k": "v"}, + "encoding": "json", + } + ], + } + VconRedis._enforce_spec_on_write(d) + a = d["analysis"][0] + assert a["body"] == {"k": "v"} assert a["encoding"] == "json" @@ -106,7 +128,27 @@ def test_leaves_string_body_alone(): assert a["encoding"] == "none" -def test_stringifies_dict_attachment_body(): +def test_leaves_legacy_stringified_json_body_alone(): + # A str body already carrying encoding "json" is the legacy -02 shape. + # It must not be re-parsed or otherwise mutated on write. + d = { + "uuid": "u", + "analysis": [ + { + "type": "summary", + "vendor": "openai", + "body": '{"k": "v"}', + "encoding": "json", + } + ], + } + VconRedis._enforce_spec_on_write(d) + a = d["analysis"][0] + assert a["body"] == '{"k": "v"}' + assert a["encoding"] == "json" + + +def test_keeps_dict_attachment_body_raw_and_fixes_encoding(): d = { "uuid": "u", "attachments": [ @@ -115,7 +157,7 @@ def test_stringifies_dict_attachment_body(): } VconRedis._enforce_spec_on_write(d) att = d["attachments"][0] - assert isinstance(att["body"], str) + assert att["body"] == {"tag": "x"} assert att["encoding"] == "json" diff --git a/tests/core/test_vcon.py b/tests/core/test_vcon.py index 8d36aa5..96b78b7 100644 --- a/tests/core/test_vcon.py +++ b/tests/core/test_vcon.py @@ -44,10 +44,10 @@ def test_add_attachment(): vcon = Vcon.build_new() vcon.add_attachment(body={"key": "value"}, type="test_type") attachment = vcon.find_attachment_by_purpose("test_type") - # Per spec body is always String — a dict input is JSON-encoded at the - # boundary and ``encoding`` is forced to ``json``. The original Python - # value round-trips via Vcon.decoded_body. - assert attachment["body"] == json.dumps({"key": "value"}) + # Per draft-ietf-vcon-vcon-core-04 §2.3.2, with encoding "json" body is + # the JSON value itself — a dict input is kept as-is and ``encoding`` is + # forced to ``json``, no stringification. + assert attachment["body"] == {"key": "value"} assert attachment["encoding"] == "json" assert Vcon.decoded_body(attachment) == {"key": "value"} @@ -59,17 +59,44 @@ def test_add_attachment_keeps_freeform_string_body_unchanged(): assert attachment == {"type": "note", "body": "just text", "encoding": "none"} +def test_add_attachment_keeps_list_body_raw(): + vcon = Vcon.build_new() + vcon.add_attachment(body=["category:1", "category:2"], type="tags") + attachment = vcon.find_attachment_by_purpose("tags") + assert attachment["body"] == ["category:1", "category:2"] + assert attachment["encoding"] == "json" + + +def test_add_attachment_only_validates_json_for_str_body(): + # A dict/list body is never JSON-validated (it's already a Python + # value); only a str body claiming encoding="json" is checked. + vcon = Vcon.build_new() + with pytest.raises(Exception): + vcon.add_attachment(body="not-valid-json", type="bad", encoding="json") + # A dict body never raises — it's stored as-is, no JSON validation. + vcon.add_attachment(body={"k": "v"}, type="ok") + assert vcon.find_attachment_by_purpose("ok")["body"] == {"k": "v"} + + def test_add_analysis(): vcon = Vcon.build_new() vcon.add_analysis(type="test_type", dialog=[1, 2], vendor="test_vendor", body={"key": "value"}) analysis = vcon.find_analysis_by_type("test_type") - assert analysis["body"] == json.dumps({"key": "value"}) + assert analysis["body"] == {"key": "value"} assert analysis["encoding"] == "json" assert analysis["dialog"] == [1, 2] assert analysis["vendor"] == "test_vendor" assert Vcon.decoded_body(analysis) == {"key": "value"} +def test_add_analysis_keeps_list_body_raw(): + vcon = Vcon.build_new() + vcon.add_analysis(type="scitt_receipt", dialog=0, vendor="scittles", body=[{"entry_id": "abc"}]) + analysis = vcon.find_analysis_by_type("scitt_receipt") + assert analysis["body"] == [{"entry_id": "abc"}] + assert analysis["encoding"] == "json" + + def test_add_party(): vcon = Vcon.build_new() vcon.add_party({"id": "party1"}) @@ -109,7 +136,7 @@ def test_find_attachment_by_type(): warnings.simplefilter("ignore", DeprecationWarning) assert vcon.find_attachment_by_type("test_type") == { "type": "test_type", - "body": json.dumps({"key": "value"}), + "body": {"key": "value"}, "encoding": "json", } assert vcon.find_attachment_by_type("nonexistent_type") is None @@ -210,7 +237,7 @@ def test_find_analysis_by_type(): "type": "test_type", "dialog": [1, 2], "vendor": "test_vendor", - "body": json.dumps({"key": "value"}), + "body": {"key": "value"}, "encoding": "json", } assert vcon.find_analysis_by_type("nonexistent_type") is None @@ -264,11 +291,13 @@ def test_error_handling(): # --------------------------------------------------------------------------- -# Body decoding regression coverage. The Redis store path -# (VconRedis._enforce_spec_on_write) stringifies dict/list bodies to JSON and -# rewrites encoding to "json" per draft-ietf-vcon-vcon-core-02. Read-side -# callers must round-trip through Vcon.decoded_body so they don't see a string -# where they used to see a dict/list. +# Body decoding regression coverage. Per draft-ietf-vcon-vcon-core-04 §2.3.2, +# an ``encoding: "json"`` body is the JSON value itself (dict/list); the +# Redis store path (VconRedis._enforce_spec_on_write) only fixes a mismatched +# encoding, never stringifies. Some rows may still carry the legacy -02 +# stringified shape (a str body under encoding "json"). Read-side callers +# must round-trip through Vcon.decoded_body so they see the same Python value +# regardless of which shape is on disk. # --------------------------------------------------------------------------- diff --git a/tests/storage/test_elasticsearch_body_roundtrip.py b/tests/storage/test_elasticsearch_body_roundtrip.py new file mode 100644 index 0000000..bcf8225 --- /dev/null +++ b/tests/storage/test_elasticsearch_body_roundtrip.py @@ -0,0 +1,70 @@ +"""CON-1111: elasticsearch storage indexes a raw JSON body correctly. + +Per draft-ietf-vcon-vcon-core-04 §2.3.2, an ``encoding: "json"`` body is the +JSON value itself. ``storage.elasticsearch.save`` already special-cases +``encoding == "json" and isinstance(body, str)`` before parsing — a raw +dict/list body falls through untouched, which is already correct. This +locks that behavior in. + +Follows tests/storage/test_egress_compat_integration.py's sys.modules +pattern for importing the module without opening a real Redis connection. +""" + +import sys +from unittest.mock import MagicMock, patch + +_ORIG = {name: sys.modules.get(name) for name in ("lib.logging_utils", "lib.vcon_redis")} +sys.modules["lib.logging_utils"] = MagicMock(init_logger=MagicMock(return_value=MagicMock())) +sys.modules["lib.vcon_redis"] = MagicMock() + +from storage import elasticsearch as es_storage # noqa: E402 + +for _name, _mod in _ORIG.items(): + if _mod is not None: + sys.modules[_name] = _mod + else: + sys.modules.pop(_name, None) + + +def _mock_vcon(canonical): + mock = MagicMock() + mock.to_dict.return_value = canonical + mock.find_attachment_by_purpose.side_effect = lambda p: next( + (a for a in canonical["attachments"] if a.get("purpose") == p), None + ) + return mock + + +def test_elasticsearch_indexes_raw_dict_and_list_bodies_unchanged(): + canonical = { + "uuid": "test-uuid", + "vcon": "0.4.0", + "dialog": [{"start": "2026-06-05T00:00:00+00:00"}], + "parties": [], + "attachments": [ + {"purpose": "tenant", "body": {"id": 385}, "encoding": "json"}, + {"purpose": "tags", "body": ["source:test"], "encoding": "json"}, + ], + "analysis": [ + {"type": "summary", "body": {"text": "hi"}, "encoding": "json"}, + ], + } + opts = {"url": "http://es:9200", "username": "u", "password": "p"} + + with patch.object(es_storage, "VconRedis") as redis_cls, \ + patch.object(es_storage, "elasticsearch") as es_lib: + redis_cls.return_value.get_vcon.return_value = _mock_vcon(canonical) + es = es_lib.Elasticsearch.return_value + + es_storage.save("test-uuid", opts) + + indexed = {c.kwargs["index"]: c.kwargs["document"] for c in es.index.call_args_list} + tenant_doc = next(d for idx, d in indexed.items() if idx.startswith("vcon_attachments_tenant")) + tags_doc = next(d for idx, d in indexed.items() if idx.startswith("vcon_attachments_tags")) + summary_doc = next(d for idx, d in indexed.items() if idx == "vcon_analysis_summary") + + assert tenant_doc["body"] == {"id": 385} + assert tags_doc["body"] == ["source:test"] + assert summary_doc["body"] == {"text": "hi"} + # common_attributes.tenant_id resolution also exercises the raw-dict path. + assert indexed[next(idx for idx in indexed if idx.startswith("vcon_attachments_tenant"))]["tenant_id"] == 385 diff --git a/tests/storage/test_mongo_body_roundtrip.py b/tests/storage/test_mongo_body_roundtrip.py new file mode 100644 index 0000000..6d039f6 --- /dev/null +++ b/tests/storage/test_mongo_body_roundtrip.py @@ -0,0 +1,90 @@ +"""CON-1111: mongo storage round-trips a raw JSON body. + +Per draft-ietf-vcon-vcon-core-04 §2.3.2, an ``encoding: "json"`` body is the +JSON value itself. The mongo storage module stores the whole vCon dict as a +BSON document (``prepare_vcon_for_mongo``/``collection.update_one``) — it +never treats ``body`` specially, so a raw dict/list body should ride along +unchanged through both ``save`` and ``fetch``/``get``. + +Follows tests/storage/test_s3.py's sys.modules pattern: lib.vcon_redis / +lib.logging_utils are mocked before importing the module under test so +importing it doesn't open a real Redis connection, then restored. +""" + +import sys +from unittest.mock import MagicMock, patch + +_orig_lib_logging_utils = sys.modules.get("lib.logging_utils") +_orig_lib_vcon_redis = sys.modules.get("lib.vcon_redis") + +sys.modules["lib.logging_utils"] = MagicMock(init_logger=MagicMock(return_value=MagicMock())) +sys.modules["lib.vcon_redis"] = MagicMock() + +from storage.mongo import save, get # noqa: E402 + +if _orig_lib_logging_utils is not None: + sys.modules["lib.logging_utils"] = _orig_lib_logging_utils +else: + sys.modules.pop("lib.logging_utils", None) + +if _orig_lib_vcon_redis is not None: + sys.modules["lib.vcon_redis"] = _orig_lib_vcon_redis +else: + sys.modules.pop("lib.vcon_redis", None) + + +def _canonical_with_raw_body(): + return { + "vcon": "0.4.0", + "uuid": "test-uuid", + "created_at": "2026-06-05T00:00:00.000Z", + "dialog": [], + "attachments": [ + {"purpose": "lawful_basis", "body": {"lawful_basis": "consent"}, "encoding": "json"}, + {"purpose": "tags", "body": ["source:test"], "encoding": "json"}, + ], + "analysis": [], + } + + +def _mock_vcon(canonical): + mock = MagicMock() + mock.to_dict.return_value = canonical + mock.uuid = canonical["uuid"] + return mock + + +def test_mongo_save_preserves_raw_dict_and_list_body(): + opts = {"url": "mongodb://localhost:27017/", "database": "conserver", "collection": "vcons"} + canonical = _canonical_with_raw_body() + + with patch("storage.mongo.VconRedis") as redis_cls, \ + patch("storage.mongo.pymongo.MongoClient") as mongo_client_cls: + redis_cls.return_value.get_vcon.return_value = _mock_vcon(canonical) + collection = MagicMock() + mongo_client_cls.return_value.__getitem__.return_value.__getitem__.return_value = collection + collection.update_one.return_value = MagicMock(modified_count=1, upserted_id=None) + + save("test-uuid", opts) + + args, _ = collection.update_one.call_args + stored = args[1]["$set"] + lawful_basis = next(a for a in stored["attachments"] if a["purpose"] == "lawful_basis") + tags = next(a for a in stored["attachments"] if a["purpose"] == "tags") + assert lawful_basis["body"] == {"lawful_basis": "consent"} + assert tags["body"] == ["source:test"] + + +def test_mongo_get_returns_raw_body_unchanged(): + opts = {"url": "mongodb://localhost:27017/", "database": "conserver", "collection": "vcons"} + stored_doc = {**_canonical_with_raw_body(), "_id": "test-uuid"} + + with patch("storage.mongo.pymongo.MongoClient") as mongo_client_cls: + collection = MagicMock() + mongo_client_cls.return_value.__getitem__.return_value.__getitem__.return_value = collection + collection.find_one.return_value = stored_doc + + result = get("test-uuid", opts) + + assert result["attachments"][0]["body"] == {"lawful_basis": "consent"} + assert result["attachments"][1]["body"] == ["source:test"] diff --git a/tests/storage/test_postgres_body_roundtrip.py b/tests/storage/test_postgres_body_roundtrip.py new file mode 100644 index 0000000..0af728c --- /dev/null +++ b/tests/storage/test_postgres_body_roundtrip.py @@ -0,0 +1,115 @@ +"""CON-1111: postgres storage round-trips a raw JSON body (CON-1111). + +Per draft-ietf-vcon-vcon-core-04 §2.3.2, an ``encoding: "json"`` body is the +JSON value itself. The postgres storage module stores the whole vCon dict +under a ``BinaryJSONField`` (JSONB) — it never treats ``body`` specially, so +a raw dict/list body should just ride along unchanged. This mocks out the +peewee model/DB entirely and asserts the ``vcon_json`` payload handed to it +still carries the raw body (dict and legacy stringified str alike). + +Follows tests/storage/test_s3.py's sys.modules pattern: lib.vcon_redis / +lib.logging_utils are mocked before importing the module under test so +importing it doesn't open a real Redis connection, then restored. +""" + +import sys +from unittest.mock import MagicMock, patch + +_orig_lib_logging_utils = sys.modules.get("lib.logging_utils") +_orig_lib_vcon_redis = sys.modules.get("lib.vcon_redis") + +sys.modules["lib.logging_utils"] = MagicMock(init_logger=MagicMock(return_value=MagicMock())) +sys.modules["lib.vcon_redis"] = MagicMock() + +from storage.postgres import save # noqa: E402 + +if _orig_lib_logging_utils is not None: + sys.modules["lib.logging_utils"] = _orig_lib_logging_utils +else: + sys.modules.pop("lib.logging_utils", None) + +if _orig_lib_vcon_redis is not None: + sys.modules["lib.vcon_redis"] = _orig_lib_vcon_redis +else: + sys.modules.pop("lib.vcon_redis", None) + + +def _mock_vcon(canonical): + mock = MagicMock() + mock.to_dict.return_value = canonical + mock.uuid = canonical["uuid"] + mock.vcon = canonical["vcon"] + mock.created_at = canonical["created_at"] + mock.subject = canonical.get("subject") + return mock + + +def _canonical_with_raw_body(): + return { + "vcon": "0.4.0", + "uuid": "test-uuid", + "created_at": "2026-06-05T00:00:00+00:00", + "subject": None, + "attachments": [ + {"purpose": "lawful_basis", "body": {"lawful_basis": "consent"}, "encoding": "json"}, + {"purpose": "tags", "body": ["source:test"], "encoding": "json"}, + ], + "analysis": [], + } + + +def test_postgres_save_preserves_raw_dict_and_list_body(): + opts = { + "database": "vcon_db", "user": "u", "password": "p", + "host": "localhost", "port": 5432, "table_name": "vcons", + } + canonical = _canonical_with_raw_body() + + with patch("storage.postgres.VconRedis") as redis_cls, \ + patch("storage.postgres.get_db_connection") as get_db, \ + patch("storage.postgres.create_vcons_model") as create_model: + redis_cls.return_value.get_vcon.return_value = _mock_vcon(canonical) + get_db.return_value = MagicMock() + + model_cls = MagicMock() + insert_query = MagicMock() + model_cls.insert.return_value = insert_query + insert_query.on_conflict.return_value.execute.return_value = None + create_model.return_value = model_cls + + save("test-uuid", opts) + + # The dict passed to insert(**vcon_data) is what would be stored + # as the JSONB vcon_json column. + _, kwargs = model_cls.insert.call_args + stored = kwargs["vcon_json"] + lawful_basis = next(a for a in stored["attachments"] if a["purpose"] == "lawful_basis") + tags = next(a for a in stored["attachments"] if a["purpose"] == "tags") + assert lawful_basis["body"] == {"lawful_basis": "consent"} + assert lawful_basis["encoding"] == "json" + assert tags["body"] == ["source:test"] + assert tags["encoding"] == "json" + + +def test_postgres_get_returns_stored_json_field_unchanged(): + from storage.postgres import get + + opts = { + "database": "vcon_db", "user": "u", "password": "p", + "host": "localhost", "port": 5432, "table_name": "vcons", + } + stored_payload = _canonical_with_raw_body() + + with patch("storage.postgres.get_db_connection") as get_db, \ + patch("storage.postgres.create_vcons_model") as create_model: + get_db.return_value = MagicMock() + model_cls = MagicMock() + row = MagicMock() + row.vcon_json = stored_payload + model_cls.get.return_value = row + create_model.return_value = model_cls + + result = get("test-uuid", opts) + + assert result["attachments"][0]["body"] == {"lawful_basis": "consent"} + assert result["attachments"][1]["body"] == ["source:test"] diff --git a/tests/storage/test_s3.py b/tests/storage/test_s3.py index 2e7f324..71c96f1 100644 --- a/tests/storage/test_s3.py +++ b/tests/storage/test_s3.py @@ -376,6 +376,41 @@ def test_save_with_region(self, mock_vcon, base_opts): region_name="us-east-2", ) + def test_save_preserves_raw_dict_and_list_body(self, base_opts): + """CON-1111: an encoding='json' dict/list body (draft-ietf-vcon-vcon- + core-04 §2.3.2) must survive the json.dumps/put_object round trip + unchanged — s3 stores the whole vcon dict, so this is really a + regression guard against something upstream re-stringifying it.""" + canonical = { + "uuid": "test-uuid", + "vcon": "0.4.0", + "attachments": [ + {"purpose": "lawful_basis", "body": {"lawful_basis": "consent"}, "encoding": "json"}, + {"purpose": "tags", "body": ["source:test"], "encoding": "json"}, + ], + } + mock = MagicMock() + mock.to_dict.return_value = canonical + mock.created_at = "2025-12-10T15:30:00Z" + + with patch("storage.s3.VconRedis") as mock_redis_class, \ + patch("storage.s3.boto3.client") as mock_boto_client: + mock_redis = MagicMock() + mock_redis.get_vcon.return_value = mock + mock_redis_class.return_value = mock_redis + + mock_s3 = MagicMock() + mock_boto_client.return_value = mock_s3 + + save("test-uuid", base_opts) + + body_call = next(c for c in mock_s3.put_object.call_args_list if c.kwargs["Key"].endswith(".vcon")) + stored = json.loads(body_call.kwargs["Body"]) + lawful_basis = next(a for a in stored["attachments"] if a["purpose"] == "lawful_basis") + tags = next(a for a in stored["attachments"] if a["purpose"] == "tags") + assert lawful_basis["body"] == {"lawful_basis": "consent"} + assert tags["body"] == ["source:test"] + def test_save_raises_exception_on_error(self, mock_vcon, base_opts): """Test that save raises exception on S3 error.""" with patch("storage.s3.VconRedis") as mock_redis_class, \ diff --git a/tests/storage/test_vcon_mcp_body_roundtrip.py b/tests/storage/test_vcon_mcp_body_roundtrip.py new file mode 100644 index 0000000..95cb8c9 --- /dev/null +++ b/tests/storage/test_vcon_mcp_body_roundtrip.py @@ -0,0 +1,62 @@ +"""CON-1111: vcon-mcp REST storage round-trips a raw JSON body. + +``storage.vcon_mcp.save`` POSTs ``vcon.to_dict()`` as the request JSON body +verbatim — it never touches ``body`` fields, so a raw dict/list body must +ride along unchanged both on save (outgoing JSON) and get (the response is +handed back as-is). +""" + +from unittest.mock import MagicMock, patch + +from storage.vcon_mcp import save, get + + +def _canonical_with_raw_body(): + return { + "uuid": "test-uuid", + "vcon": "0.4.0", + "attachments": [ + {"purpose": "lawful_basis", "body": {"lawful_basis": "consent"}, "encoding": "json"}, + {"purpose": "tags", "body": ["source:test"], "encoding": "json"}, + ], + } + + +def test_vcon_mcp_save_posts_raw_body_unchanged(): + canonical = _canonical_with_raw_body() + mock_vcon = MagicMock() + mock_vcon.to_dict.return_value = canonical + + with patch("storage.vcon_mcp.VconRedis") as redis_cls, \ + patch("storage.vcon_mcp._session") as session_factory: + redis_cls.return_value.get_vcon.return_value = mock_vcon + session = MagicMock() + session.__enter__.return_value = session + session.__exit__.return_value = False + session.post.return_value = MagicMock(status_code=201, raise_for_status=lambda: None) + session_factory.return_value = session + + save("test-uuid", {"base_url": "http://mcp.local/api/v1"}) + + posted = session.post.call_args.kwargs["json"] + assert posted["attachments"][0]["body"] == {"lawful_basis": "consent"} + assert posted["attachments"][1]["body"] == ["source:test"] + + +def test_vcon_mcp_get_returns_raw_body_unchanged(): + canonical = _canonical_with_raw_body() + + with patch("storage.vcon_mcp._session") as session_factory: + session = MagicMock() + session.__enter__.return_value = session + session.__exit__.return_value = False + response = MagicMock(status_code=200) + response.raise_for_status = lambda: None + response.json.return_value = {"vcon": canonical} + session.get.return_value = response + session_factory.return_value = session + + result = get("test-uuid", {"base_url": "http://mcp.local/api/v1"}) + + assert result["attachments"][0]["body"] == {"lawful_basis": "consent"} + assert result["attachments"][1]["body"] == ["source:test"]