fix(vcon): keep draft-04 JSON bodies as values on write

common/lib/links/filters.py

@@ -1,3 +1,4 @@
+import json
 import random
 from typing import Literal, Optional, TypedDict
 from lib.logging_utils import init_logger
@@ -68,11 +69,19 @@ def is_included(options: Optional[FilterOptions], _vcon) -> bool:
                 if isinstance(tags, list) and includes in tags:
                     return True
                 continue
-            # Per spec §2.3.2 ``body`` is always a String regardless of
-            # encoding, so substring-match directly without any decode.
+            # Per draft-ietf-vcon-vcon-core-04 §2.3.2, an ``encoding: "json"``
+            # body is the JSON value itself (dict/list), not a string — so a
+            # plain substring match only works directly for str bodies (the
+            # legacy -02 stringified shape, or encoding "none"/"base64url").
+            # For a dict/list body, match against its JSON-serialized form so
+            # ``includes`` still finds tokens inside it.
             body = element.get("body")
-            if isinstance(body, str) and includes in body:
-                return True
+            if isinstance(body, str):
+                if includes in body:
+                    return True
+            elif isinstance(body, (dict, list)):
+                if includes in json.dumps(body):
+                    return True
     except Exception as e:
         logger.error(f"Error checking inclusion: {e}")
     return False

common/lib/vcon_egress_compat.py

@@ -5,11 +5,13 @@
 The conserver normalizes every vCon *up* to the current spec (``vcon: "0.4.0"``)
 on read and write. Downstream consumers built against an older schema (e.g.
 ``0.0.1``) break on the canonical shape — notably the ``type`` → ``purpose``
-attachment rename and the write-path serialization of dict/list analysis and
-attachment bodies into JSON strings (``encoding: "json"``). This module converts
-an *outgoing* payload back to a legacy version — reversing the field renames and
-re-inflating those JSON-string bodies to native objects with ``encoding: "none"``
-— so those consumers keep working while a migration is planned.
+attachment rename and, per draft-ietf-vcon-vcon-core-04 §2.3.2, dict/list
+analysis/attachment bodies carried as native JSON values under
+``encoding: "json"`` rather than stringified. This module converts an
+*outgoing* payload back to a legacy version — reversing the field renames and
+turning those native/stringified JSON bodies into legacy-shape native objects
+with ``encoding: "none"`` — so those consumers keep working while a migration
+is planned.
 
 It never mutates the canonical in-pipeline copy: callers pass ``vcon.to_dict()``
 and receive a new, deep-copied, downgraded dict. Enable it per egress point via
@@ -68,22 +70,30 @@ def _entry_to_legacy(entry: Dict[str, Any]) -> None:
 
 
 def _body_to_legacy(entry: Dict[str, Any]) -> None:
-    """Inverse of ``VconRedis._stringify_json_body``.
-
-    The spec write-path serializes dict/list ``body`` values to a JSON string
-    and sets ``encoding: "json"``. The legacy 0.0.1 shape carries the native
-    object/array with ``encoding: "none"``, so parse it back. Applied to
-    analysis and attachment entries only — dialog bodies are not stringified on
-    write. Left untouched if the body isn't valid JSON.
+    """Inverse of the -04 write path's ``encoding: "json"`` shape.
+
+    Per draft-ietf-vcon-vcon-core-04 §2.3.2, a spec-current ``encoding: "json"``
+    body is the JSON value itself (dict/list) — no parsing needed, just flip
+    ``encoding`` to the legacy 0.0.1 ``"none"``. A str body under
+    ``encoding: "json"`` is the older -02 stringified shape (still seen in
+    storage rows written before this change, or from writers that pass a
+    pre-serialized string); parse it back to a native value. Applied to
+    analysis and attachment entries only — dialog bodies are not touched on
+    write. Left untouched if a str body isn't valid JSON.
     """
     if not isinstance(entry, dict):
         return
-    if entry.get("encoding") == "json" and isinstance(entry.get("body"), str):
+    if entry.get("encoding") != "json":
+        return
+    body = entry.get("body")
+    if isinstance(body, str):
         try:
-            entry["body"] = json.loads(entry["body"])
+            entry["body"] = json.loads(body)
         except (ValueError, TypeError):
             return
-        entry["encoding"] = "none"
+    elif not isinstance(body, (dict, list)):
+        return
+    entry["encoding"] = "none"
 
 
 def _attachment_to_legacy(att: Dict[str, Any]) -> None:

common/lib/vcon_redis.py

@@ -1,4 +1,3 @@
-import json
 from datetime import datetime
 from typing import Any, Optional
 from config import Configuration
@@ -36,20 +35,26 @@ class VconRedis:
     DEFAULT_TTL = VCON_REDIS_EXPIRY
 
     @staticmethod
-    def _stringify_json_body(entry: Any) -> None:
-        """Force ``body`` to be a string + correct ``encoding`` per speckit.
-
-        The speckit non-negotiable: analysis/attachment bodies are
-        strings. JSON content pairs ``body: json.dumps(...)`` with
-        ``encoding: "json"``. vcon-lib's ``add_analysis`` currently
-        emits dict/list bodies with ``encoding: "none"``, which we
-        normalize on the way out so storage is spec-correct.
+    def _normalize_json_body_encoding(entry: Any) -> None:
+        """Fix a mismatched ``encoding`` on a dict/list ``body`` per -04.
+
+        Per draft-ietf-vcon-vcon-core-04 §2.3.2, with ``encoding: "json"``
+        the ``body`` is the JSON value itself (object/array), not a
+        stringified copy. A dict/list body should therefore carry
+        ``encoding: "json"`` as-is; we no longer ``json.dumps`` it away.
+        The only thing to fix here is a real mismatch: a dict/list body
+        that arrived with ``encoding: "none"`` (or no encoding at all,
+        e.g. from vcon-lib's ``add_analysis``) is stamped to ``"json"``
+        without touching the value.
+
+        A ``str`` body with ``encoding: "json"`` is left untouched — that's
+        the legacy -02 stringified shape, which readers (``Vcon.decoded_body``)
+        still decode.
         """
         if not isinstance(entry, dict):
             return
         body = entry.get("body")
-        if isinstance(body, (dict, list)):
-            entry["body"] = json.dumps(body)
+        if isinstance(body, (dict, list)) and entry.get("encoding") != "json":
             entry["encoding"] = "json"
 
     @classmethod
@@ -81,11 +86,13 @@ def _enforce_spec_on_write(cls, vcon_dict: dict) -> dict:
         # speckit: empty ``redacted: {}`` should be omitted.
         if vcon_dict.get("redacted") == {}:
             vcon_dict.pop("redacted", None)
-        # speckit: analysis/attachment bodies are strings.
+        # draft-ietf-vcon-vcon-core-04 §2.3.2: with encoding "json" the body
+        # is the JSON value itself; only fix a real dict/list-vs-encoding
+        # mismatch, never stringify.
         for entry in vcon_dict.get("analysis", []) or []:
-            cls._stringify_json_body(entry)
+            cls._normalize_json_body_encoding(entry)
         for entry in vcon_dict.get("attachments", []) or []:
-            cls._stringify_json_body(entry)
+            cls._normalize_json_body_encoding(entry)
         return vcon_dict
 
     def store_vcon(self, vCon: vcon.Vcon, ttl: Optional[int] = None) -> None:

common/storage/file/test_file_storage.py

@@ -255,6 +255,34 @@ def test_get_prefers_uncompressed_when_both_exist(self, temp_storage_dir, sample
         assert result["marker"] == "uncompressed"
 
 
+class TestRawJsonBodyRoundtrip:
+    """CON-1111: an encoding='json' dict/list body (draft-ietf-vcon-vcon-
+    core-04 §2.3.2) must survive save()+get() unchanged. File storage writes
+    the whole vcon.dumps() JSON text, so this guards against anything
+    upstream re-stringifying it."""
+
+    def test_save_then_get_preserves_raw_dict_and_list_body(self, temp_storage_dir):
+        vcon = Vcon.build_new()
+        vcon.add_attachment(body={"lawful_basis": "consent"}, type="lawful_basis")
+        vcon.add_attachment(body=["source:test"], type="tags")
+
+        opts = {"path": temp_storage_dir, "compression": False, "organize_by_date": False}
+
+        with patch("storage.file.VconRedis") as MockVconRedis:
+            mock_redis = MagicMock()
+            mock_redis.get_vcon.return_value = vcon
+            MockVconRedis.return_value = mock_redis
+            save(vcon.uuid, opts)
+
+        result = get(vcon.uuid, opts)
+        lawful_basis = next(a for a in result["attachments"] if a["type"] == "lawful_basis")
+        tags = next(a for a in result["attachments"] if a["type"] == "tags")
+        assert lawful_basis["body"] == {"lawful_basis": "consent"}
+        assert lawful_basis["encoding"] == "json"
+        assert tags["body"] == ["source:test"]
+        assert tags["encoding"] == "json"
+
+
 class TestDelete:
     """Tests for delete function."""
 

common/storage/redis_storage/test_redis_storage.py

@@ -0,0 +1,43 @@
+"""CON-1111: redis_storage round-trips a raw JSON body.
+
+``storage.redis_storage.save`` writes ``vcon.dumps()`` verbatim as a plain
+Redis string; ``get`` ``json.loads`` it back. Neither touches ``body``
+fields, so a raw dict/list body (draft-ietf-vcon-vcon-core-04 §2.3.2,
+encoding "json") must ride along unchanged.
+"""
+
+from unittest.mock import MagicMock, patch
+
+from vcon import Vcon
+from storage.redis_storage import save, get
+
+
+def test_save_then_get_preserves_raw_dict_and_list_body():
+    vcon = Vcon.build_new()
+    vcon.add_attachment(body={"lawful_basis": "consent"}, type="lawful_basis")
+    vcon.add_attachment(body=["source:test"], type="tags")
+
+    opts = {"prefix": "vcon_storage", "expires": 3600}
+    store = {}
+
+    with patch("storage.redis_storage.VconRedis") as MockVconRedis, \
+         patch("storage.redis_storage.redis") as mock_redis:
+        mock_vcon_redis = MagicMock()
+        mock_vcon_redis.get_vcon.return_value = vcon
+        MockVconRedis.return_value = mock_vcon_redis
+
+        def _set(key, value, ex=None):
+            store[key] = value
+
+        mock_redis.set.side_effect = _set
+        mock_redis.get.side_effect = lambda key: store.get(key)
+
+        save(vcon.uuid, opts)
+        result = get(vcon.uuid, opts)
+
+    lawful_basis = next(a for a in result["attachments"] if a["type"] == "lawful_basis")
+    tags = next(a for a in result["attachments"] if a["type"] == "tags")
+    assert lawful_basis["body"] == {"lawful_basis": "consent"}
+    assert lawful_basis["encoding"] == "json"
+    assert tags["body"] == ["source:test"]
+    assert tags["encoding"] == "json"

common/tests/lib/filters/test_is_included.py

@@ -72,3 +72,56 @@ async def test_is_included():
         },
         _vcon,
     )
+
+
+@pytest.mark.asyncio
+async def test_is_included_matches_raw_dict_body():
+    # draft-ietf-vcon-vcon-core-04 §2.3.2: with encoding "json" the body is
+    # the JSON value itself (a dict here), not a string. The substring match
+    # must still find tokens inside it via its JSON-serialized form.
+    _vcon = vcon.Vcon.build_new()
+    _vcon.add_analysis(
+        type="customer_frustration",
+        body={"verdict": "NEEDS REVIEW", "score": 0.9},
+        dialog=0,
+        vendor="FooBar Inc.",
+    )
+    assert is_included(
+        {
+            "only_if": {
+                "section": "analysis",
+                "type": "customer_frustration",
+                "includes": "NEEDS REVIEW",
+            }
+        },
+        _vcon,
+    )
+    assert not is_included(
+        {
+            "only_if": {
+                "section": "analysis",
+                "type": "customer_frustration",
+                "includes": "ALL CLEAR",
+            }
+        },
+        _vcon,
+    )
+
+
+@pytest.mark.asyncio
+async def test_is_included_matches_raw_list_attachment_body():
+    _vcon = vcon.Vcon.build_new()
+    _vcon.add_attachment(
+        body=["category:12", "category:99"],
+        type="tags2",
+    )
+    assert is_included(
+        {
+            "only_if": {
+                "section": "attachments",
+                "type": "tags2",
+                "includes": "category:99",
+            }
+        },
+        _vcon,
+    )

common/tests/test_api.py

@@ -99,6 +99,49 @@ def test_create_vcon_with_extra_attribute():
         assert response.json()["meta"] == {"foo": "bar"}
 
 
+@pytest.mark.anyio
+def test_post_and_get_vcon_round_trips_raw_json_bodies():
+    """CON-1111: draft-ietf-vcon-vcon-core-04 §2.3.2 — with encoding "json"
+    the body is the JSON value itself (object/array), not a stringified
+    copy. A dict lawful_basis attachment body and a list tags attachment
+    body must round-trip through POST /vcon + GET /vcon/{uuid} as a dict
+    and a list, not JSON strings.
+    """
+    test_vcon = generate_mock_vcon()
+    test_vcon["attachments"] = [
+        {
+            "purpose": "lawful_basis",
+            "body": {
+                "lawful_basis": "consent",
+                "purpose_grants": [{"purpose": "analysis", "granted": True}],
+            },
+            "encoding": "json",
+        },
+        {
+            "purpose": "tags",
+            "body": ["source:test", "direction:out"],
+            "encoding": "json",
+        },
+    ]
+    post_vcon(test_vcon)
+
+    with TestClient(api.app, headers={CONSERVER_HEADER_NAME: CONSERVER_API_TOKEN}) as client:
+        response = client.get("/vcon/{}".format(test_vcon["uuid"]))
+        assert response.status_code == 200
+        body = response.json()
+
+    lawful_basis = next(a for a in body["attachments"] if a["purpose"] == "lawful_basis")
+    tags = next(a for a in body["attachments"] if a["purpose"] == "tags")
+
+    assert isinstance(lawful_basis["body"], dict)
+    assert lawful_basis["body"]["lawful_basis"] == "consent"
+    assert lawful_basis["encoding"] == "json"
+
+    assert isinstance(tags["body"], list)
+    assert tags["body"] == ["source:test", "direction:out"]
+    assert tags["encoding"] == "json"
+
+
 _INVALID_FIXTURES_DIR = os.path.join(os.path.dirname(__file__), "invalid_fixtures")
 
 

common/tests/test_api_branches.py

@@ -75,7 +75,12 @@ async def test_sync_vcon_from_storage_canonicalizes_legacy_payload(redis_async):
     """A storage backend may hold a legacy / egress-converted 0.0.1 payload
     (see egress_format_version). The fallback must canonicalize it before
     caching to Redis or returning to the client, so downstream links/clients
-    never see legacy field names or native (non-stringified) bodies."""
+    never see legacy field names or a body/encoding mismatch.
+
+    Per draft-ietf-vcon-vcon-core-04 §2.3.2, canonicalizing a dict/list body
+    no longer stringifies it — encoding "json" now means the body IS that
+    dict/list, so the fallback only needs to fix the mismatched "none"
+    encoding on the way in, keeping the value native."""
     vcon_uuid = uuid4()
     legacy = {
         "uuid": str(vcon_uuid),
@@ -94,9 +99,9 @@ async def test_sync_vcon_from_storage_canonicalizes_legacy_payload(redis_async):
     # Returned payload is canonical.
     att = result["attachments"][0]
     assert att.get("purpose") == "tags" and "type" not in att
-    assert isinstance(att["body"], str) and att["encoding"] == "json"
+    assert att["body"] == ["a", "b"] and att["encoding"] == "json"
     an = result["analysis"][0]
-    assert isinstance(an["body"], str) and an["encoding"] == "json"
+    assert an["body"] == {"k": "v"} and an["encoding"] == "json"
 
     # The copy cached back into Redis is canonical too (not the raw legacy dict).
     cached = json.loads(redis_async.set.await_args.args[1])

common/tests/test_encoding.py

@@ -10,17 +10,17 @@ def test_encoding():
     print("test_vcon:", _vcon)
     test_vcon = vcon.Vcon(_vcon)
 
-    # Spec (draft-ietf-vcon-vcon-core-02 §2.3.2): body is always a String. A
-    # dict/list passed in for convenience is JSON-encoded at the boundary and
-    # the encoding is forced to "json".
+    # Spec (draft-ietf-vcon-vcon-core-04 §2.3.2): with encoding "json" body is
+    # the JSON value itself. A dict/list passed in is kept as-is and the
+    # encoding is forced to "json" (no stringification).
     test_vcon.add_attachment(
         type="test_encoding_dict_coerced",
         body={"key": "value"},
         encoding="json",
     )
     assert test_vcon.find_attachment_by_purpose("test_encoding_dict_coerced") == {
         "type": "test_encoding_dict_coerced",
-        "body": json.dumps({"key": "value"}),
+        "body": {"key": "value"},
         "encoding": "json",
     }
 
@@ -31,7 +31,7 @@ def test_encoding():
     )
     assert test_vcon.find_attachment_by_purpose("test_encoding_list_coerced") == {
         "type": "test_encoding_list_coerced",
-        "body": json.dumps(["key", "value"]),
+        "body": ["key", "value"],
         "encoding": "json",
     }