Skip to content

chore: feature integration branch for the ENVITED-X pipeline #3

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
jdsika wants to merge 2 commits into
base: main
Choose a base branch
from
Draft

chore: feature integration branch for the ENVITED-X pipeline #3

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
844ac19
fix: JSON-LD collection syntax can silently duplicate shared RDF lists
jdsika Sep 9, 2026
7dd4a55
fix: preserve explicit JSON-LD list descriptions efficiently
edmondchuc Oct 26, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
115 changes: 88 additions & 27 deletions rdflib/plugins/serializers/jsonld.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
from __future__ import annotations

import warnings
from collections import Counter
from typing import IO, Any, Optional

from rdflib.graph import DATASET_DEFAULT_GRAPH_ID, Graph, _ObjectType
Expand Down Expand Up @@ -155,6 +156,11 @@ def __init__(self, context: Context, use_native_types: bool, use_rdf_type: bool)
self.context = context
self.use_native_types = context.active or use_native_types
self.use_rdf_type = use_rdf_type
# Statements per blank-node object, across every graph being converted.
self._bnode_usages: Optional[Counter[Identifier]] = None
self._pending_nodes: list[BNode] = []
# Rejected prefixes are reusable only during a single graph conversion.
self._rejected_lists: Optional[set[Identifier]] = None

def convert(self, graph: Graph):
# TODO: bug in rdflib dataset parsing (nquads et al):
Expand Down Expand Up @@ -187,6 +193,10 @@ def convert(self, graph: Graph):
graphs = [graph]
default_graph_id = graph.identifier

self._bnode_usages = Counter(
o for g in graphs for o in g.objects() if isinstance(o, BNode)
)

context = self.context

objs: list[Any] = []
Expand Down Expand Up @@ -225,15 +235,25 @@ def convert(self, graph: Graph):

def from_graph(self, graph: Graph):
nodemap: dict[Any, Any] = {}

for s in set(graph.subjects()):
## only iri:s and unreferenced (rest will be promoted to top if needed)
if isinstance(s, URIRef) or (
isinstance(s, BNode) and not any(graph.subjects(None, s))
):
self.process_subject(graph, s, nodemap)

return list(nodemap.values())
self._pending_nodes = []
self._rejected_lists = set()

try:
for s in set(graph.subjects()):
## only iri:s and unreferenced (rest will be promoted to top if needed)
if isinstance(s, URIRef) or (
isinstance(s, BNode) and not any(graph.subjects(None, s))
):
self.process_subject(graph, s, nodemap)

# Explicit blank-node chains can be arbitrarily long. Process their
# descriptions iteratively instead of recursing through each reference.
while self._pending_nodes:
self.process_subject(graph, self._pending_nodes.pop(), nodemap)

return list(nodemap.values())
finally:
self._rejected_lists = None

def process_subject(self, graph: Graph, s: IdentifiedNode, nodemap):
if isinstance(s, URIRef):
Expand Down Expand Up @@ -270,12 +290,16 @@ def add_to_node(
):
context = self.context

collection = None
if isinstance(o, Literal):
datatype = str(o.datatype) if o.datatype else None
language = o.language
term = context.find_term(str(p), datatype, language=language)
else:
containers = [LIST, None] if graph.value(o, RDF.first) else [None]
# Only offer an @list term for a chain that can be written as @list.
if graph.value(o, RDF.first):
collection = self.to_collection(graph, o)
containers = [LIST, None] if collection is not None else [None]
for container in containers:
for coercion in (ID, VOCAB, UNDEF):
# type error: Argument 2 to "find_term" of "Context" has incompatible type "object"; expected "Union[str, Defined, None]"
Expand Down Expand Up @@ -305,12 +329,18 @@ def add_to_node(
):
node = str(o)

if LIST in term.container:
if LIST in term.container and collection is not None:
node = [
self.type_coerce(v, term.type)
or self.to_raw_value(graph, s, v, nodemap)
for v in self.to_collection(graph, o)
for v in collection
]
if term.type == ID:
self._pending_nodes.extend(
v
for v in collection
if isinstance(v, BNode) and any(graph.predicate_objects(v))
)
elif LANG in term.container and language:
value = s_node.setdefault(p_key, {})
values = value.get(language)
Expand Down Expand Up @@ -342,6 +372,10 @@ def add_to_node(

if node is None:
node = self.to_raw_value(graph, s, o, nodemap)
elif isinstance(o, BNode) and term and LIST not in term.container:
# Coercion emits only the reference; preserve its description too.
if any(graph.predicate_objects(o)):
self._pending_nodes.append(o)

value = s_node.get(p_key)
if value:
Expand Down Expand Up @@ -381,15 +415,7 @@ def to_raw_value(
]
return {context.list_key: coll}
elif isinstance(o, BNode):
embed = (
False # TODO: self.context.active or using startnode and only one ref
)
onode = self.process_subject(graph, o, nodemap)
if onode:
if embed and not any(s2 for s2 in graph.subjects(None, o) if s2 != s):
return onode
else:
nodemap[onode[context.id_key]] = onode
self._pending_nodes.append(o)
return {context.id_key: o.n3()}
elif isinstance(o, URIRef):
# TODO: embed if o != startnode (else reverse)
Expand All @@ -416,27 +442,62 @@ def to_raw_value(
else:
return v

def _usages(self, graph: Graph, node: Identifier) -> int:
"""Count the statements whose object is `node`.

Inside `convert` this covers every graph being converted; called on
its own, `graph` is taken to be the whole input.
"""
if self._bnode_usages is None:
return len(list(graph.subject_predicates(node)))
return self._bnode_usages[node]

def to_collection(self, graph: Graph, l_: Identifier):
"""Return the members of the `rdf:List` headed by `l_`, or None.

None means the chain must not be written as `@list`. Besides a
malformed or cyclic chain, that includes a chain any of whose cells,
the head included, is the object of more than one statement in any of
the graphs being converted. `@list` inlines the cells at a single
reference and writes them nowhere else, so another reference would be
left pointing at a node the output never defines. This is the
"referenced once" condition of the Serialize RDF as JSON-LD Algorithm
(JSON-LD 1.1 Processing Algorithms and API, section 8.4).
"""
rejected = self._rejected_lists
if rejected is None:
rejected = set()
if l_ in rejected:
return None
if l_ != RDF.nil and not graph.value(l_, RDF.first):
return None
list_nodes: list[Optional[_ObjectType]] = []
chain = set([l_])
chain: set[Identifier] = set()
while l_:
if l_ == RDF.nil:
return list_nodes
if isinstance(l_, URIRef):
return None
if (
l_ in chain
or l_ in rejected
or isinstance(l_, URIRef)
or self._usages(graph, l_) != 1
):
break
chain.add(l_)
first, rest = None, None
for p, o in graph.predicate_objects(l_):
if not first and p == RDF.first:
first = o
elif not rest and p == RDF.rest:
rest = o
elif p != RDF.type or o != RDF.List:
rejected.update(chain)
return None
list_nodes.append(first)
# type error: Incompatible types in assignment (expression has type "Optional[Node]", variable has type "Identifier")
l_ = rest # type: ignore[assignment]
if l_ in chain:
return None
chain.add(l_)

# Every visited prefix reaches the same invalid suffix. Remember only
# these rejections, so explicit cells do not repeatedly scan that suffix.
rejected.update(chain)
return None
Loading