From b98b49611905fb7a854201a855dcf6c0d9267149 Mon Sep 17 00:00:00 2001 From: Carlo van Driesten Date: Tue, 8 Sep 2026 18:09:43 +0200 Subject: [PATCH 1/5] feat(rdf): add opt-in diff-stable blank-node labels RDFC-1.0 canonicalization already makes RDF output deterministic: isomorphic graphs always serialize identically. It does not make output diffable. Blank nodes are numbered `c14nN` in a single global order, so inserting one class can renumber every blank node after it and rewrite most of the file. A one-line semantic change lands as a whole-file diff, which makes generated OWL/SHACL hard to review and noisy to keep under version control. Add a `diff_stable` argument to `canonicalize_rdf_graph()` and a `--diff-stable/--no-diff-stable` flag to the four RDF generators. When enabled, blank-node labels are derived from each node's own neighbourhood via Weisfeiler-Lehman refinement, so an edit relabels only the blank nodes it actually touches. Measured churn on a real schema (add one class, count changed lines): generator default --diff-stable owlgen 2091 17 shexgen 796 50 shaclgen 291 13 rdfgen 115 25 Output stays deterministic and isomorphic either way; only the choice of label changes. Off by default, because enabling it relabels existing output. The refinement itself lives in `diffable-rdf`, whose only dependencies (rdflib, pyoxigraph) are already linkml-runtime dependencies at higher versions, so this adds no new transitive dependencies. --- .../linkml/src/linkml/generators/owlgen.py | 28 +++- .../linkml/src/linkml/generators/rdfgen.py | 28 +++- .../linkml/src/linkml/generators/shaclgen.py | 28 +++- .../linkml/src/linkml/generators/shexgen.py | 28 +++- packages/linkml_runtime/pyproject.toml | 1 + .../linkml_runtime/utils/rdf_canonicalize.py | 23 ++- .../test_utils/test_rdf_canonicalize.py | 135 ++++++++++++++++++ 7 files changed, 266 insertions(+), 5 deletions(-) diff --git a/packages/linkml/src/linkml/generators/owlgen.py b/packages/linkml/src/linkml/generators/owlgen.py index 50be266df0..7751eab34d 100644 --- a/packages/linkml/src/linkml/generators/owlgen.py +++ b/packages/linkml/src/linkml/generators/owlgen.py @@ -122,6 +122,22 @@ class OwlSchemaGenerator(Generator): """Suffix to add to the schema name to create the ontology URI, e.g. .owl.ttl""" # ObjectVars + diff_stable: bool = False + """Label blank nodes so that unrelated edits leave them untouched. + + Output is already deterministic: RDFC-1.0 guarantees that isomorphic + graphs serialize identically. It does not guarantee that *similar* + graphs serialize *similarly* — blank nodes are numbered ``c14nN`` in a + global order, so adding one class can renumber every blank node after + it and rewrite most of the file. + + When ``True``, blank-node labels are instead derived from each node's + own neighbourhood via Weisfeiler-Lehman refinement, so an edit relabels + only the blank nodes it actually touches. The output stays + deterministic and isomorphic either way; only the choice of label + changes. Off by default because enabling it relabels existing output. + """ + metadata_profile: MetadataProfile | None = None """Deprecated - use metadata_profiles.""" @@ -353,7 +369,7 @@ def serialize(self, **kwargs: Any) -> str: """ self.as_graph() fmt = "turtle" if self.format in ["owl", "ttl"] else self.format - return canonicalize_rdf_graph(self.graph, output_format=fmt) + return canonicalize_rdf_graph(self.graph, output_format=fmt, diff_stable=self.diff_stable) def add_metadata(self, e: Definition | PermissibleValue, uri: URIRef) -> None: """ @@ -1844,6 +1860,16 @@ def slot_owl_type(self, slot: SlotDefinition) -> URIRef: "specified language tag. Element-level in_language overrides this." ), ) +@click.option( + "--diff-stable/--no-diff-stable", + default=False, + show_default=True, + help=( + "Derive blank-node labels from each node's own neighbourhood so that " + "unrelated edits leave them unchanged. Output is deterministic either " + "way; this makes successive versions of a file diff cleanly." + ), +) @click.version_option(__version__, "-V", "--version") def cli(yamlfile: str, metadata_profile: str, **kwargs: Any) -> None: """Generate an OWL representation of a LinkML model diff --git a/packages/linkml/src/linkml/generators/rdfgen.py b/packages/linkml/src/linkml/generators/rdfgen.py index 2da1701787..052fcff12e 100644 --- a/packages/linkml/src/linkml/generators/rdfgen.py +++ b/packages/linkml/src/linkml/generators/rdfgen.py @@ -78,6 +78,22 @@ class RDFGenerator(Generator): uses_schemaloader = True # ObjectVars + diff_stable: bool = False + """Label blank nodes so that unrelated edits leave them untouched. + + Output is already deterministic: RDFC-1.0 guarantees that isomorphic + graphs serialize identically. It does not guarantee that *similar* + graphs serialize *similarly* — blank nodes are numbered ``c14nN`` in a + global order, so adding one class can renumber every blank node after + it and rewrite most of the file. + + When ``True``, blank-node labels are instead derived from each node's + own neighbourhood via Weisfeiler-Lehman refinement, so an edit relabels + only the blank nodes it actually touches. The output stays + deterministic and isomorphic either way; only the choice of label + changes. Off by default because enabling it relabels existing output. + """ + emit_metadata: bool = False context: list[str] = None original_schema: SchemaDefinition = None @@ -89,7 +105,7 @@ def __post_init__(self): def _data(self, g: Graph) -> str: fmt = "turtle" if self.format == "ttl" else self.format - return canonicalize_rdf_graph(g, output_format=fmt) + return canonicalize_rdf_graph(g, output_format=fmt, diff_stable=self.diff_stable) def end_schema(self, output: str | None = None, context: str = None, **_) -> str: gen = JSONLDGenerator( @@ -137,6 +153,16 @@ def end_schema(self, output: str | None = None, context: str = None, **_) -> str multiple=True, help="JSONLD context file (default: vendored meta.context.jsonld)", ) +@click.option( + "--diff-stable/--no-diff-stable", + default=False, + show_default=True, + help=( + "Derive blank-node labels from each node's own neighbourhood so that " + "unrelated edits leave them unchanged. Output is deterministic either " + "way; this makes successive versions of a file diff cleanly." + ), +) @click.version_option(__version__, "-V", "--version") def cli(yamlfile, **kwargs): """Generate an RDF representation of a LinkML model""" diff --git a/packages/linkml/src/linkml/generators/shaclgen.py b/packages/linkml/src/linkml/generators/shaclgen.py index 4731b9f0b8..5d42fea272 100644 --- a/packages/linkml/src/linkml/generators/shaclgen.py +++ b/packages/linkml/src/linkml/generators/shaclgen.py @@ -142,6 +142,22 @@ class ShaclGenerator(Generator): ignores any per-slot ``in_language``. """ + diff_stable: bool = False + """Label blank nodes so that unrelated edits leave them untouched. + + Output is already deterministic: RDFC-1.0 guarantees that isomorphic + graphs serialize identically. It does not guarantee that *similar* + graphs serialize *similarly* — blank nodes are numbered ``c14nN`` in a + global order, so adding one class can renumber every blank node after + it and rewrite most of the file. + + When ``True``, blank-node labels are instead derived from each node's + own neighbourhood via Weisfeiler-Lehman refinement, so an edit relabels + only the blank nodes it actually touches. The output stays + deterministic and isomorphic either way; only the choice of label + changes. Off by default because enabling it relabels existing output. + """ + emit_rules: bool = True """Emit ``sh:sparql`` constraints from LinkML ``rules:`` blocks. @@ -196,7 +212,7 @@ def generate_header(self) -> str: def serialize(self, **args) -> str: g = self.as_graph() fmt = "turtle" if self.format in ["owl", "ttl"] else self.format - return canonicalize_rdf_graph(g, output_format=fmt) + return canonicalize_rdf_graph(g, output_format=fmt, diff_stable=self.diff_stable) def as_graph(self) -> Graph: sv = self.schemaview @@ -929,6 +945,16 @@ def add_simple_data_type(func: Callable, r: ElementName) -> None: "sh:NodeShape. Use --no-emit-rules to suppress rule generation." ), ) +@click.option( + "--diff-stable/--no-diff-stable", + default=False, + show_default=True, + help=( + "Derive blank-node labels from each node's own neighbourhood so that " + "unrelated edits leave them unchanged. Output is deterministic either " + "way; this makes successive versions of a file diff cleanly." + ), +) @click.version_option(__version__, "-V", "--version") def cli(yamlfile, **args): """Generate SHACL turtle from a LinkML model""" diff --git a/packages/linkml/src/linkml/generators/shexgen.py b/packages/linkml/src/linkml/generators/shexgen.py index 40a93ffbc9..7c43b13bc0 100644 --- a/packages/linkml/src/linkml/generators/shexgen.py +++ b/packages/linkml/src/linkml/generators/shexgen.py @@ -40,6 +40,22 @@ class ShExGenerator(Generator): uses_schemaloader = True # ObjectVars + diff_stable: bool = False + """Label blank nodes so that unrelated edits leave them untouched. + + Output is already deterministic: RDFC-1.0 guarantees that isomorphic + graphs serialize identically. It does not guarantee that *similar* + graphs serialize *similarly* — blank nodes are numbered ``c14nN`` in a + global order, so adding one class can renumber every blank node after + it and rewrite most of the file. + + When ``True``, blank-node labels are instead derived from each node's + own neighbourhood via Weisfeiler-Lehman refinement, so an edit relabels + only the blank nodes it actually touches. The output stays + deterministic and isomorphic either way; only the choice of label + changes. Off by default because enabling it relabels existing output. + """ + shex: Schema = field(default_factory=lambda: Schema()) # ShEx Schema being generated shapes: list = field(default_factory=lambda: []) shape: Shape | None = None # Current shape being defined @@ -177,7 +193,7 @@ def end_schema(self, output: str | None = None, **_) -> str: g = Graph() g.parse(data=shex, format="json-ld", version="1.1") g.bind("owl", OWL) - shex = canonicalize_rdf_graph(g, output_format="turtle") + shex = canonicalize_rdf_graph(g, output_format="turtle", diff_stable=self.diff_stable) elif self.format == "shex": g = Graph() self.namespaces.load_graph(g) @@ -258,6 +274,16 @@ def _get_subproperty_values(self, slot: SlotDefinition) -> list: help="If --expand-subproperty-of (default), slots with subproperty_of will generate NodeConstraint " "values containing all slot descendants. Use --no-expand-subproperty-of to disable this behavior.", ) +@click.option( + "--diff-stable/--no-diff-stable", + default=False, + show_default=True, + help=( + "Derive blank-node labels from each node's own neighbourhood so that " + "unrelated edits leave them unchanged. Output is deterministic either " + "way; this makes successive versions of a file diff cleanly." + ), +) @click.version_option(__version__, "-V", "--version") def cli(yamlfile, **args): """Generate a ShEx Schema for a LinkML model""" diff --git a/packages/linkml_runtime/pyproject.toml b/packages/linkml_runtime/pyproject.toml index 1ae9da3083..b879dec7ee 100644 --- a/packages/linkml_runtime/pyproject.toml +++ b/packages/linkml_runtime/pyproject.toml @@ -48,6 +48,7 @@ dependencies = [ "prefixmaps >=0.1.4", "curies>=0.14.6", "pyoxigraph>=0.5.11", + "diffable-rdf>=0.2.0", "pydantic>=2.13.5,<3.0.0", "isodate >=0.7.2, <1.0.0; python_version < '3.11'", ] diff --git a/packages/linkml_runtime/src/linkml_runtime/utils/rdf_canonicalize.py b/packages/linkml_runtime/src/linkml_runtime/utils/rdf_canonicalize.py index 31c5580348..e0fadae34a 100644 --- a/packages/linkml_runtime/src/linkml_runtime/utils/rdf_canonicalize.py +++ b/packages/linkml_runtime/src/linkml_runtime/utils/rdf_canonicalize.py @@ -42,6 +42,7 @@ import pyoxigraph as ox import rdflib +from diffable_rdf import wl_relabel_quads from rdflib.compare import to_canonical_graph @@ -287,6 +288,7 @@ def _is_safe_prefix_iri(iri: str) -> bool: def canonicalize_rdf_graph( graph: rdflib.Graph, output_format: str = "turtle", + diff_stable: bool = False, ) -> str: """Serialize an rdflib Graph deterministically using RDFC-1.0 canonicalization. @@ -300,6 +302,12 @@ def canonicalize_rdf_graph( :param graph: The rdflib Graph to serialize. :param output_format: Target serialization format (e.g. ``"turtle"``, ``"nt"``). + :param diff_stable: Derive blank-node labels from each node's own + neighbourhood instead of RDFC-1.0's global ``c14nN`` numbering, so that + editing one part of a schema does not renumber unrelated blank nodes. + Output is deterministic and isomorphic either way; only the choice of + label changes. Off by default because enabling it relabels existing + output. :return: Deterministic string serialization of the graph. """ ox_format = _FORMAT_MAP.get(output_format.lower()) @@ -339,13 +347,26 @@ def canonicalize_rdf_graph( # 3. Canonicalize blank node labels with RDFC-1.0. dataset.canonicalize(ox.CanonicalizationAlgorithm.RDFC_1_0) + quads = list(dataset) + + # 3b. Optionally re-label blank nodes with locality-sensitive hashes. + # RDFC-1.0 guarantees that identical graphs get identical labels, but it + # does not guarantee that *similar* graphs get similar labels: the labels + # are assigned by a global ordering, so inserting one blank node can + # renumber every label after it and turn a one-line semantic change into a + # whole-file diff. Weisfeiler-Lehman labels are derived only from each + # node's local neighbourhood, so unrelated regions keep their labels. + # Output stays deterministic and isomorphic either way; this only changes + # which label each blank node receives. + if diff_stable: + quads = wl_relabel_quads(quads) + # 4. Sort triples for deterministic ordering. # RDFC-1.0 stabilizes blank-node labels but pyoxigraph's Dataset # iteration order is not sorted and varies across processes (verified # empirically against pyoxigraph 0.5.8). The explicit string-key sort # is load-bearing for byte-identical output across runs; see # tests/linkml_runtime/test_utils/test_rdf_canonicalize.py::test_sort_is_load_bearing. - quads = list(dataset) sorted_triples = sorted( (ox.Triple(q.subject, q.predicate, q.object) for q in quads), key=lambda t: (str(t.subject), str(t.predicate), str(t.object)), diff --git a/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py b/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py index 63dbf095a1..0ca79c3c90 100644 --- a/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py +++ b/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py @@ -507,3 +507,138 @@ def run(seed: str) -> str: assert out_a == out_b, ( "Fallback output differs across PYTHONHASHSEED values; blank-node canonicalization may be missing" ) + + +def _shapes_graph(count: int, extra: bool = False) -> Graph: + """A graph shaped like generator output: one blank node per named subject.""" + g = Graph() + g.bind("ex", "http://example.com/") + names = [f"{i:02d}" for i in range(count)] + (["AAAinserted"] if extra else []) + for name in names: + subject = URIRef(f"http://example.com/Shape{name}") + prop = BNode() + g.add((subject, URIRef("http://example.com/property"), prop)) + g.add((prop, URIRef("http://example.com/path"), URIRef(f"http://example.com/p{name}"))) + return g + + +def _changed_line_count(before: str, after: str) -> int: + import difflib + + diff = difflib.unified_diff(before.splitlines(), after.splitlines(), n=0, lineterm="") + return sum(1 for line in diff if line[:1] in "+-" and not line.startswith(("+++", "---"))) + + +def test_diff_stable_is_opt_in(): + """The default must keep producing exactly the output it produced before.""" + graph = _make_graph_with_bnodes() + assert canonicalize_rdf_graph(graph) == canonicalize_rdf_graph(graph, diff_stable=False) + + +def test_diff_stable_preserves_semantics(): + """Relabelling blank nodes must not change what the graph means.""" + graph = _make_graph_with_bnodes() + + plain = rdflib.Graph() + plain.parse(data=canonicalize_rdf_graph(graph), format="turtle") + stable = rdflib.Graph() + stable.parse(data=canonicalize_rdf_graph(graph, diff_stable=True), format="turtle") + + assert rdflib.compare.isomorphic(plain, stable) + + +def test_diff_stable_is_deterministic(): + """Diff stability must not cost determinism, which is the stronger property.""" + graph = _make_graph_with_bnodes() + outputs = {canonicalize_rdf_graph(graph, diff_stable=True) for _ in range(5)} + assert len(outputs) == 1 + + +def test_diff_stable_confines_an_insertion_to_the_lines_it_touches(): + """Inserting one subject must not relabel the blank nodes of the others. + + RDFC-1.0 numbers blank nodes ``c14nN`` in a global order, so a subject + sorting before the others shifts every subsequent label and rewrites + most of the file. This is the entire reason the option exists, so the + assertion is on the *ratio*, not on an absolute line count that would + be brittle across rdflib versions. + """ + before, after = _shapes_graph(20), _shapes_graph(20, extra=True) + + baseline = _changed_line_count(canonicalize_rdf_graph(before), canonicalize_rdf_graph(after)) + stable = _changed_line_count( + canonicalize_rdf_graph(before, diff_stable=True), + canonicalize_rdf_graph(after, diff_stable=True), + ) + + assert stable < baseline / 4, f"expected diff-stable output to churn far less; got {stable} vs baseline {baseline}" + + +_DIFF_STABLE_SCHEMA = """\ +id: https://example.org/diffstable +name: diffstable +prefixes: + linkml: https://w3id.org/linkml/ + ex: https://example.org/diffstable/ +default_prefix: ex +default_range: string +imports: + - linkml:types +classes: + Person: + slots: [name, knows] + Organization: + slots: [name] +slots: + name: + range: string + knows: + range: Person + multivalued: true +""" + + +def _generator_cases(): + """The four generators that serialize RDF, with the args that make them do so.""" + from linkml.generators.owlgen import OwlSchemaGenerator + from linkml.generators.rdfgen import RDFGenerator + from linkml.generators.shaclgen import ShaclGenerator + from linkml.generators.shexgen import ShExGenerator + + return [ + pytest.param(OwlSchemaGenerator, {}, id="owlgen"), + # rdfgen and shexgen resolve JSON-LD contexts (linkml types, shex.jsonld); + # the `network` marker serves those from local stubs. See tests/conftest.py. + pytest.param(RDFGenerator, {}, id="rdfgen", marks=pytest.mark.network), + pytest.param(ShaclGenerator, {}, id="shaclgen"), + # ShExGenerator only emits RDF in this format; its default is ShExC text, + # where blank-node labelling does not apply. + pytest.param(ShExGenerator, {"format": "rdf"}, id="shexgen", marks=pytest.mark.network), + ] + + +@pytest.mark.parametrize(("generator", "kwargs"), _generator_cases()) +def test_diff_stable_reaches_every_rdf_generator(tmp_path, generator, kwargs): + """Every RDF generator must actually apply the option, not merely accept it. + + Asserting only that the attribute exists would pass even if a generator + forgot to pass it down to :func:`canonicalize_rdf_graph`. Instead this + checks the observable consequence: RDFC-1.0 names blank nodes ``c14nN``, + while diff-stable labels are neighbourhood hashes, so a generator that + honours the flag emits no ``c14nN`` label at all. + """ + assert generator.diff_stable is False, f"{generator.__name__} must default to off" + + schema = tmp_path / "schema.yaml" + schema.write_text(_DIFF_STABLE_SCHEMA, encoding="utf-8", newline="\n") + + plain = generator(str(schema), **kwargs).serialize() + stable = generator(str(schema), diff_stable=True, **kwargs).serialize() + + # Guards the test itself: if the fixture stopped producing blank nodes the + # assertion below would hold vacuously. + assert re.search(r"c14n\d+", plain), f"{generator.__name__} output has no blank nodes to relabel" + assert not re.search(r"c14n\d+", stable), ( + f"{generator.__name__} still emits RDFC-1.0 blank-node labels with diff_stable=True; " + "the flag is probably not threaded into canonicalize_rdf_graph()" + ) From a55ba23124182ef94593ad26fdc33f00b67135a4 Mon Sep 17 00:00:00 2001 From: Carlo van Driesten Date: Fri, 11 Sep 2026 10:52:51 +0200 Subject: [PATCH 2/5] fix(rdf): bump diffable-rdf to 0.3.0 and stop diff_stable silently no-opping Bump the floor to diffable-rdf 0.3.0 and add the missing uv.lock entry: the dependency was declared in pyproject.toml but never locked, so "uv lock --check" and the "uv sync --frozen" anti-malware gate would both have failed CI. 0.3.0 also fixes two defects in the Weisfeiler-Lehman labelling this feature relies on. Disconnected blank-node components now converge independently, so an edit in one region no longer relabels an unrelated one. And the suffix used to tell structurally indistinguishable nodes apart was assigned in c14nN *text* order, so c14n10 sorted between c14n1 and c14n2 -- adding a tenth tied blank node relabelled eight of the nine already there, the exact opposite of what this labelling is for. Separately, diff_stable=True was silently ignored whenever pyoxigraph refused the graph and canonicalize_rdf_graph degraded to rdflib. Weisfeiler-Lehman refinement consumes canonical pyoxigraph quads, and that path exists precisely because there are none, so the argument could not be honoured -- but the caller was never told. "shaclgen --include-annotations --diff-stable" reaches it, via the literal predicate an annotation tag without a ':' produces, and returned output byte-identical to --no-diff-stable. It now warns, with a regression test asserting the warning and the byte-identical output that makes silence misleading. --- packages/linkml_runtime/pyproject.toml | 2 +- .../linkml_runtime/utils/rdf_canonicalize.py | 20 ++++++++++++++- .../test_utils/test_rdf_canonicalize.py | 25 +++++++++++++++++++ uv.lock | 15 +++++++++++ 4 files changed, 60 insertions(+), 2 deletions(-) diff --git a/packages/linkml_runtime/pyproject.toml b/packages/linkml_runtime/pyproject.toml index b879dec7ee..ede7112692 100644 --- a/packages/linkml_runtime/pyproject.toml +++ b/packages/linkml_runtime/pyproject.toml @@ -48,7 +48,7 @@ dependencies = [ "prefixmaps >=0.1.4", "curies>=0.14.6", "pyoxigraph>=0.5.11", - "diffable-rdf>=0.2.0", + "diffable-rdf>=0.3.0", "pydantic>=2.13.5,<3.0.0", "isodate >=0.7.2, <1.0.0; python_version < '3.11'", ] diff --git a/packages/linkml_runtime/src/linkml_runtime/utils/rdf_canonicalize.py b/packages/linkml_runtime/src/linkml_runtime/utils/rdf_canonicalize.py index e0fadae34a..8e28488d55 100644 --- a/packages/linkml_runtime/src/linkml_runtime/utils/rdf_canonicalize.py +++ b/packages/linkml_runtime/src/linkml_runtime/utils/rdf_canonicalize.py @@ -307,7 +307,8 @@ def canonicalize_rdf_graph( editing one part of a schema does not renumber unrelated blank nodes. Output is deterministic and isomorphic either way; only the choice of label changes. Off by default because enabling it relabels existing - output. + output. Has no effect on the rdflib fallback path (non-standard RDF), + which warns rather than silently ignoring the request. :return: Deterministic string serialization of the graph. """ ox_format = _FORMAT_MAP.get(output_format.lower()) @@ -338,6 +339,23 @@ def canonicalize_rdf_graph( RDFCanonicalizationWarning, stacklevel=2, ) + if diff_stable: + # Weisfeiler-Lehman refinement consumes canonical pyoxigraph quads, + # and this path exists precisely because pyoxigraph refused the + # graph, so there are none to refine. The fallback is deterministic + # but not diff-stable: say so rather than returning output that + # silently ignores the argument. ``shaclgen --include-annotations`` + # reaches this path, because an annotation tag without a ``:`` + # becomes a literal predicate. + warnings.warn( + "diff_stable=True was requested but this graph took the rdflib fallback, " + "which cannot apply Weisfeiler-Lehman blank-node labels. Output is " + "deterministic but NOT diff-stable: an unrelated edit may still renumber " + "blank nodes. Make the offending terms standard RDF (absolute IRIs, IRI " + "predicates) to get diff-stable labels.", + RDFCanonicalizationWarning, + stacklevel=2, + ) return _deterministic_fallback_serialize(graph, output_format) dataset = ox.Dataset() diff --git a/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py b/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py index 0ca79c3c90..1bc07583af 100644 --- a/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py +++ b/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py @@ -642,3 +642,28 @@ def test_diff_stable_reaches_every_rdf_generator(tmp_path, generator, kwargs): f"{generator.__name__} still emits RDFC-1.0 blank-node labels with diff_stable=True; " "the flag is probably not threaded into canonicalize_rdf_graph()" ) + + +def test_diff_stable_warns_instead_of_silently_no_opping_on_the_fallback(): + """A request the fallback cannot honour must be reported, not ignored. + + ``wl_relabel_quads`` consumes canonical pyoxigraph quads, and the rdflib + fallback exists precisely because pyoxigraph refused the graph. Returning + the same bytes for ``diff_stable=True`` and ``diff_stable=False`` without + saying so lets a caller believe the output is diff-stable when it is not. + """ + graph = _make_graph_with_bnodes() + # A relative IRI is non-standard RDF, so pyoxigraph rejects the graph and + # canonicalize_rdf_graph degrades to rdflib -- the same path that + # ``shaclgen --include-annotations`` takes via its literal predicates. + graph.add((URIRef("testing"), URIRef("http://example.com/p"), Literal("v"))) + + with pytest.warns(RDFCanonicalizationWarning, match="NOT diff-stable"): + stable = canonicalize_rdf_graph(graph, diff_stable=True) + + with pytest.warns(RDFCanonicalizationWarning): + plain = canonicalize_rdf_graph(graph, diff_stable=False) + + # The warning is the contract: the bytes really are identical, which is + # exactly why staying silent would be misleading. + assert stable == plain diff --git a/uv.lock b/uv.lock index d8cddaa9e1..e862cfc897 100644 --- a/uv.lock +++ b/uv.lock @@ -1085,6 +1085,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/84/d0/205d54408c08b13550c733c4b85429e7ead111c7f0014309637425520a9a/deprecated-1.3.1-py2.py3-none-any.whl", hash = "sha256:597bfef186b6f60181535a29fbe44865ce137a5079f295b479886c82729d5f3f", size = 11298, upload-time = "2025-10-30T08:19:00.758Z" }, ] +[[package]] +name = "diffable-rdf" +version = "0.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyoxigraph" }, + { name = "rdflib" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fe/f3/45771c536aea850f2884206349e35910bbabcaebeec890c5c0cc08b040f9/diffable_rdf-0.3.0.tar.gz", hash = "sha256:6fcf84d9323cd35fc75f423bd73836469cfad198ebb7e6ab86a8743724cd000d", size = 829395, upload-time = "2026-09-11T08:00:26.268Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/27/22/ef80dba96cd558f59d2ab225602bde318a925d437866379303e1f41a4e96/diffable_rdf-0.3.0-py3-none-any.whl", hash = "sha256:8756137423f9f28beacb5cd99ba8d66317328a0c936b1dbb4527831c7d9c6784", size = 43578, upload-time = "2026-09-11T08:00:24.542Z" }, +] + [[package]] name = "distlib" version = "0.4.0" @@ -2618,6 +2631,7 @@ dependencies = [ { name = "click" }, { name = "curies" }, { name = "deprecated" }, + { name = "diffable-rdf" }, { name = "hbreader" }, { name = "isodate", marker = "python_full_version < '3.11'" }, { name = "json-flattener" }, @@ -2650,6 +2664,7 @@ requires-dist = [ { name = "coverage", marker = "extra == 'dev'" }, { name = "curies", specifier = ">=0.14.6" }, { name = "deprecated" }, + { name = "diffable-rdf", specifier = ">=0.3.0" }, { name = "hbreader" }, { name = "isodate", marker = "python_full_version < '3.11'", specifier = ">=0.7.2,<1.0.0" }, { name = "json-flattener", specifier = ">=0.1.9" }, From a4d52530c7f367e274ae86ad70d76dfa64afffa5 Mon Sep 17 00:00:00 2001 From: Carlo van Driesten Date: Fri, 11 Sep 2026 13:45:48 +0200 Subject: [PATCH 3/5] build(deps): require diffable-rdf 0.4.0 0.4.0 carries graph.base through the library's rdflib fallback, verifying that every absolute IRI of the source survives a re-read rather than dropping the directive outright, and adds a diff_stable parameter to canonicalize_rdf_graph. The lock entry is written by hand because the workspace sets exclude-newer = "7 days", which filters any release younger than that from resolution; 0.3.0 was pinned the same way for the same reason, and both become resolvable normally on 2026-09-18. uv lock --check and uv sync --all-groups both accept the entry. https://github.com/ASCS-eV/diffable-rdf/releases/tag/v0.4.0 --- packages/linkml_runtime/pyproject.toml | 2 +- uv.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/linkml_runtime/pyproject.toml b/packages/linkml_runtime/pyproject.toml index ede7112692..c940eeac56 100644 --- a/packages/linkml_runtime/pyproject.toml +++ b/packages/linkml_runtime/pyproject.toml @@ -48,7 +48,7 @@ dependencies = [ "prefixmaps >=0.1.4", "curies>=0.14.6", "pyoxigraph>=0.5.11", - "diffable-rdf>=0.3.0", + "diffable-rdf>=0.4.0", "pydantic>=2.13.5,<3.0.0", "isodate >=0.7.2, <1.0.0; python_version < '3.11'", ] diff --git a/uv.lock b/uv.lock index e862cfc897..3c867304c8 100644 --- a/uv.lock +++ b/uv.lock @@ -1087,15 +1087,15 @@ wheels = [ [[package]] name = "diffable-rdf" -version = "0.3.0" +version = "0.4.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pyoxigraph" }, { name = "rdflib" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/fe/f3/45771c536aea850f2884206349e35910bbabcaebeec890c5c0cc08b040f9/diffable_rdf-0.3.0.tar.gz", hash = "sha256:6fcf84d9323cd35fc75f423bd73836469cfad198ebb7e6ab86a8743724cd000d", size = 829395, upload-time = "2026-09-11T08:00:26.268Z" } +sdist = { url = "https://files.pythonhosted.org/packages/86/df/1d71c0c984eac1cfd25531aabf84528941b01083c875da49d44882d1b5af/diffable_rdf-0.4.0.tar.gz", hash = "sha256:a84afaa10332e6a039b6ddcabf76bced3049e4da467c88ee786d0fa35218111e", size = 838040, upload-time = "2026-09-11T11:43:12.844Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/27/22/ef80dba96cd558f59d2ab225602bde318a925d437866379303e1f41a4e96/diffable_rdf-0.3.0-py3-none-any.whl", hash = "sha256:8756137423f9f28beacb5cd99ba8d66317328a0c936b1dbb4527831c7d9c6784", size = 43578, upload-time = "2026-09-11T08:00:24.542Z" }, + { url = "https://files.pythonhosted.org/packages/7f/49/b62ee864fe6769b759f04e3709fd79711b1cdf49223b1bb0e985e507ead3/diffable_rdf-0.4.0-py3-none-any.whl", hash = "sha256:c59c57429465f786fdbc8d38ddbb687da16748c5752f0c2ac4a77b5527aa9aed", size = 46126, upload-time = "2026-09-11T11:43:11.536Z" }, ] [[package]] @@ -2664,7 +2664,7 @@ requires-dist = [ { name = "coverage", marker = "extra == 'dev'" }, { name = "curies", specifier = ">=0.14.6" }, { name = "deprecated" }, - { name = "diffable-rdf", specifier = ">=0.3.0" }, + { name = "diffable-rdf", specifier = ">=0.4.0" }, { name = "hbreader" }, { name = "isodate", marker = "python_full_version < '3.11'", specifier = ">=0.7.2,<1.0.0" }, { name = "json-flattener", specifier = ">=0.1.9" }, From f7a79fec916c5ac3977cb263d579d88d6f85f988 Mon Sep 17 00:00:00 2001 From: Carlo van Driesten Date: Fri, 11 Sep 2026 11:08:44 +0200 Subject: [PATCH 4/5] test(rdf): record the canonicalizer gaps against the extracted library Asserts each correctness property against both linkml's copy and diffable_rdf, marking whichever implementation does not hold it as a strict xfail, so the file is a ratchet in both directions. Nine gaps run one way, two of them silent data corruption. One ran the other way -- the library dropped @base on the degraded path -- and that was the last property blocking delegation. diffable-rdf 0.4.0 fixed it, so that case now passes on both sides and carries no mark. --- .../test_rdf_canonicalize_defects.py | 353 ++++++++++++++++++ 1 file changed, 353 insertions(+) create mode 100644 tests/linkml_runtime/test_utils/test_rdf_canonicalize_defects.py diff --git a/tests/linkml_runtime/test_utils/test_rdf_canonicalize_defects.py b/tests/linkml_runtime/test_utils/test_rdf_canonicalize_defects.py new file mode 100644 index 0000000000..806be2e3eb --- /dev/null +++ b/tests/linkml_runtime/test_utils/test_rdf_canonicalize_defects.py @@ -0,0 +1,353 @@ +"""Correctness differences between linkml's RDF canonicalizer and the extracted library. + +``linkml_runtime.utils.rdf_canonicalize`` and +``diffable_rdf.canonicalize_rdf_graph`` are the same code: the module was +extracted into a standalone library at the maintainers' request +(linkml/linkml#3295). The two copies have since diverged, so each test here +asserts one correctness property against *both* implementations and marks the +one that does not hold it as a strict xfail. + +Every remaining gap runs one way -- the extracted copy received fixes the local +one did not -- and two of them are silent data corruption, where the output +parses cleanly and says something the input never said. That is worse than a +crash, because a generated artifact gets committed and reviewed on the +assumption that it means what the schema meant. + +One gap used to run the other way: the library dropped ``@base`` on the +degraded path. That was the last property blocking delegation, and +diffable-rdf 0.4.0 fixed it, so the case now passes on both sides and is kept +unmarked. Asserting both directions is what made the fix land where it +belongs. + +The xfails are strict, so this file is a ratchet in both directions: when a fix +lands on either side, its case starts passing and the suite fails until the +mark is removed. +""" + +import os +import subprocess +import sys +import textwrap +import warnings + +import diffable_rdf +import pytest +import rdflib +from rdflib import BNode, Graph, Literal, Namespace, URIRef +from rdflib.namespace import RDF + +from linkml_runtime.utils.rdf_canonicalize import canonicalize_rdf_graph as linkml_canonicalize + +EX = Namespace("http://example.org/") + +_IMPLEMENTATIONS = ( + (linkml_canonicalize, "linkml"), + (diffable_rdf.canonicalize_rdf_graph, "diffable-rdf"), +) + + +def _impls(**known_failures: str): + """Parametrize over both implementations, xfailing the ones named. + + :param known_failures: implementation id (``linkml`` / ``diffable_rdf``) + mapped to why that implementation does not hold the property. + """ + params = [] + for fn, ident in _IMPLEMENTATIONS: + reason = known_failures.get(ident.replace("-", "_")) + marks = [pytest.mark.xfail(strict=True, reason=reason)] if reason else [] + params.append(pytest.param(fn, id=ident, marks=marks)) + return params + + +def _apply(fn, graph, output_format="turtle"): + """Call an implementation, ignoring the warnings both emit on the fallback path.""" + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + return fn(graph, output_format) + + +# -------------------------------------------------------------------------- +# Silent data corruption +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "canonicalize", + _impls(linkml="passes graph.base to the serializer without checking it is safe to relativize against"), +) +def test_base_with_a_fragment_does_not_rewrite_every_iri(canonicalize): + """A base IRI ending in ``#`` must not change the graph's terms. + + Relativizing ``http://ex.org/d#a`` against base ``http://ex.org/d#`` gives + ``<#a>``, which is correct per RFC 3986 -- but rdflib's parser resolves a + fragment reference by concatenation and reads it back as + ``http://ex.org/d##a``. Every term of the graph silently changes, and the + output parses cleanly, so nothing reports it. + """ + graph = Graph(base="http://ex.org/d#") + graph.add((URIRef("http://ex.org/d#a"), EX.p, Literal("v"))) + + round_tripped = Graph() + round_tripped.parse(data=_apply(canonicalize, graph), format="turtle") + + assert {str(s) for s in round_tripped.subjects()} == {"http://ex.org/d#a"} + + +@pytest.mark.parametrize( + "canonicalize", + _impls(linkml="rdflib's collection syntax materializes a shared list tail once per referencing list"), +) +def test_a_shared_rdf_list_tail_is_not_duplicated(canonicalize): + """Two lists sharing a tail must not gain triples on the way out. + + rdflib's Turtle writer renders ``( ... )`` collection syntax per list, so a + tail referenced from two lists is written twice as two separate blank + nodes. The result asserts more than the input did. + """ + graph = Graph() + graph.bind("ex", EX) + tail = BNode() + graph.add((tail, RDF.first, Literal("shared"))) + graph.add((tail, RDF.rest, RDF.nil)) + for name in ("l1", "l2"): + head = BNode() + graph.add((EX[name], EX.list, head)) + graph.add((head, RDF.first, Literal(name))) + graph.add((head, RDF.rest, tail)) + # A relative IRI is non-standard RDF, so pyoxigraph refuses the graph and + # both implementations degrade to rdflib -- the path this defect lives on. + graph.add((URIRef("testing"), EX.p, Literal("forces-fallback"))) + + round_tripped = Graph() + round_tripped.parse(data=_apply(canonicalize, graph), format="turtle") + + assert len(round_tripped) == len(graph) + + +# -------------------------------------------------------------------------- +# Output that no parser will read +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "canonicalize", + _impls(linkml="the rdflib fallback writes N-Triples for a graph pyoxigraph already refused"), +) +def test_nt_output_either_parses_or_refuses(canonicalize): + """N-Triples admits only absolute IRIs, so a relative one must not be written. + + The fallback is taken *because* the graph holds a term pyoxigraph rejected. + Writing it out anyway produces a file that fails on line 1. Refusing with a + message naming the term and a format that can carry the graph is the fix; + returning unreadable text is not. + """ + graph = Graph() + graph.add((URIRef("testing"), EX.p, Literal("v"))) + + try: + serialized = _apply(canonicalize, graph, "nt") + except ValueError: + return # refused, with an explanation -- the correct outcome + + rdflib.Graph().parse(data=serialized, format="nt") + + +@pytest.mark.parametrize( + "canonicalize", + _impls(linkml="sorts fallback output with str.splitlines(), which breaks on U+2028 and five other characters"), +) +def test_a_unicode_line_separator_in_a_literal_survives(canonicalize): + """Sorting N-Triples lines must split on newlines only. + + N-Triples permits U+2028 raw inside a quoted literal, but + ``str.splitlines()`` breaks on it as well as on U+2029, U+0085, U+000B, + U+000C and U+001C-1E. One statement becomes two lines, the halves sort + independently, the separator is rewritten as a newline, and the document + no longer parses. + """ + graph = Graph() + graph.add((EX.s, EX.p, Literal("a\u2028b"))) + graph.add((URIRef("testing"), EX.p, Literal("forces-fallback"))) + + try: + serialized = _apply(canonicalize, graph, "nt") + except ValueError: + return # refused because of the relative IRI, before the sort + + assert "\u2028" in serialized + rdflib.Graph().parse(data=serialized, format="nt") + + +# -------------------------------------------------------------------------- +# Non-determinism, which is the property the module exists to provide +# -------------------------------------------------------------------------- + +_ACROSS_PROCESSES = textwrap.dedent( + """ + import sys, warnings + warnings.simplefilter("ignore") + from rdflib import BNode, Graph, Literal, Namespace, URIRef + from rdflib.namespace import RDF + + module, output_format, force_fallback = sys.argv[1], sys.argv[2], sys.argv[3] == "1" + if module == "linkml": + from linkml_runtime.utils.rdf_canonicalize import canonicalize_rdf_graph + else: + from diffable_rdf import canonicalize_rdf_graph + + EX = Namespace("http://example.org/") + g = Graph() + if force_fallback: + g.add((URIRef("testing"), EX.p, Literal("forces-fallback"))) + for i in range(12): + b = BNode() + g.add((EX["s%02d" % i], EX.child, b)) + g.add((b, EX.k, Literal("v%d" % i))) + g.add((b, RDF.type, EX.T)) + for host in "abcdefgh": + g.add((URIRef("http://%s.example/s" % host), + URIRef("http://%s.example/p" % host), + URIRef("http://%s.example/o" % host))) + sys.stdout.write(canonicalize_rdf_graph(g, output_format)) + """ +) + + +def _outputs_across_processes(module, output_format, force_fallback): + """Serialize the same graph in four processes with different hash seeds.""" + outputs = set() + for seed in ("0", "1", "12345", "999"): + completed = subprocess.run( + [sys.executable, "-c", _ACROSS_PROCESSES, module, output_format, "1" if force_fallback else "0"], + capture_output=True, + text=True, + encoding="utf-8", + env={**os.environ, "PYTHONHASHSEED": seed}, + check=False, + ) + if completed.returncode != 0: + pytest.fail(f"{module}/{output_format} failed:\n{completed.stderr[-2000:]}") + outputs.add(completed.stdout) + return outputs + + +_XML_TRAVERSAL = "degraded RDF/XML is ordered by rdflib's graph traversal, which follows set iteration order" +_NS_NAMES = "auto-generated ns1/ns2 prefix names are allocated in traversal order" +_JSONLD_UNMAPPED = "json-ld is absent from the format map, so it falls through to rdflib with no determinism guarantee" + + +@pytest.mark.parametrize( + ("module", "output_format", "force_fallback"), + [ + pytest.param( + "linkml", "xml", True, id="linkml-xml-fallback", marks=pytest.mark.xfail(strict=True, reason=_XML_TRAVERSAL) + ), + pytest.param("diffable_rdf", "xml", True, id="diffable-rdf-xml-fallback"), + pytest.param( + "linkml", + "turtle", + True, + id="linkml-turtle-fallback", + marks=pytest.mark.xfail(strict=True, reason=_NS_NAMES), + ), + pytest.param("diffable_rdf", "turtle", True, id="diffable-rdf-turtle-fallback"), + pytest.param( + "linkml", + "json-ld", + False, + id="linkml-jsonld", + marks=pytest.mark.xfail(strict=True, reason=_JSONLD_UNMAPPED), + ), + pytest.param("diffable_rdf", "json-ld", False, id="diffable-rdf-jsonld"), + ], +) +def test_output_is_byte_identical_across_processes(module, output_format, force_fallback): + """The same graph must serialize to the same bytes in any process. + + Three separate causes break this in the local copy. Degraded RDF/XML is + ordered by rdflib's own graph traversal, which follows set iteration order. + Auto-generated ``ns1``/``ns2`` prefix names are allocated in traversal order + too, so the *names* move even when the triples do not. And ``json-ld`` is + not in the format map at all, so it falls through to rdflib's serializer, + which carries no determinism guarantee. + """ + assert len(_outputs_across_processes(module, output_format, force_fallback)) == 1 + + +# -------------------------------------------------------------------------- +# Interface +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "canonicalize", + _impls(linkml="returns the serializer's own trailing whitespace, which differs by format"), +) +def test_every_format_ends_with_exactly_one_newline(canonicalize): + """A committed artifact should not depend on which format wrote it. + + Passing through whatever the serializer produced gives no trailing newline + for RDF/XML and one for Turtle, so POSIX text tools disagree about whether + the file has a last line. + """ + counts = {} + for output_format in ("turtle", "nt", "xml", "trig", "n3"): + graph = Graph() + graph.bind("ex", EX) + graph.add((EX.s, EX.p, Literal("v"))) + serialized = _apply(canonicalize, graph, output_format) + counts[output_format] = len(serialized) - len(serialized.rstrip("\n")) + + assert counts == dict.fromkeys(counts, 1) + + +@pytest.mark.parametrize( + "canonicalize", + _impls(linkml="Dataset is a Graph subclass, so the type check accepts it and the named graphs are flattened"), +) +def test_a_dataset_is_refused_rather_than_partly_serialized(canonicalize): + """A Dataset is a Graph subclass, so it is accepted and quietly flattened. + + Every named graph collapses into one document and the graph names are + dropped, which is a different dataset. The caller asked for something this + function cannot express and should be told so. + """ + dataset = rdflib.Dataset() + dataset.graph(URIRef("http://ex/g1")).add((EX.a, EX.p, Literal("in-g1"))) + dataset.graph(URIRef("http://ex/g2")).add((EX.b, EX.p, Literal("in-g2"))) + + with pytest.raises(TypeError): + _apply(canonicalize, dataset) + + +# -------------------------------------------------------------------------- +# A property both implementations now hold +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize("canonicalize", _impls()) +def test_base_survives_the_degraded_path(canonicalize): + """``@base`` must not disappear just because pyoxigraph refused the graph. + + ``rdflib_dumper.dumps(..., prefix_map={"@base": ...})`` is a supported way + to ask for a document whose IRIs are written relative to a base, and the + metamodel routinely produces graphs that fall back (a bare ``status: + testing`` on a ``uriorcurie`` slot serializes to the relative ````, + which pyoxigraph rejects). Losing the directive on exactly that path means + the feature works only for graphs that never needed the fallback. + + This ran the other way until diffable-rdf 0.4.0, which was the last + property blocking delegation. The case is kept because it is the one the + two implementations reach differently: linkml passes ``graph.base`` to the + serializer unconditionally, which is why it also fails + :func:`test_base_with_a_fragment_does_not_rewrite_every_iri` above, while + the library keeps the base only after confirming that re-reading the + result still yields every absolute IRI of the source. + """ + graph = Graph(base="http://example.org/default/") + graph.bind("ex", EX) + graph.add((EX.s, EX.p, Literal("v"))) + graph.add((URIRef("testing"), EX.p, Literal("forces-fallback"))) + + assert "@base ." in _apply(canonicalize, graph) From c8b683a4415304776b3216d353dbfbca306c62fc Mon Sep 17 00:00:00 2001 From: Carlo van Driesten Date: Fri, 11 Sep 2026 14:05:26 +0200 Subject: [PATCH 5/5] refactor(rdf): delegate canonicalization to the diffable-rdf library The implementation was extracted into diffable-rdf at the maintainers' request in linkml/linkml#3295, but linkml kept its own copy and the two drifted. This deletes the copy and calls the library, which is what the extraction was for. Nine correctness fixes come with it, each already asserted in test_rdf_canonicalize_defects.py and each previously a strict xfail on the linkml side: - a base ending in # no longer rewrites every IRI that merely shares its prefix (silent corruption: output parsed, meaning changed) - a shared rdf:List tail is no longer duplicated (9 triples in, 11 out) - N-Triples refuses a relative IRI instead of writing a file its own parser rejects - literals containing U+2028, U+2029, U+0085 and the other separators str.splitlines() treats as line breaks survive the line sort - degraded RDF/XML, degraded Turtle and json-ld are byte-identical across processes - every format ends with exactly one newline - a Dataset is refused rather than silently flattened Behaviour changes for callers: nt output for a graph containing a relative IRI now raises ValueError rather than writing an unparseable file, and json-ld is canonicalized rather than handed to rdflib, so it no longer warns. All four RDF generators produce byte-identical output. The library reports degradation through logging; linkml reports it through warnings so it is visible without logging configuration. _DegradedPathWarnings bridges the two, and the tests pin the properties that makes load-bearing: the warning is attributed to the caller's line, the library's logger is left as it was found, a caller who configured logging still receives the record, and warnings survive an exception. --- .../linkml_runtime/utils/rdf_canonicalize.py | 499 ++++-------------- .../test_utils/test_rdf_canonicalize.py | 190 ++++++- .../test_rdf_canonicalize_defects.py | 129 ++--- 3 files changed, 311 insertions(+), 507 deletions(-) diff --git a/packages/linkml_runtime/src/linkml_runtime/utils/rdf_canonicalize.py b/packages/linkml_runtime/src/linkml_runtime/utils/rdf_canonicalize.py index 8e28488d55..47e8dc2682 100644 --- a/packages/linkml_runtime/src/linkml_runtime/utils/rdf_canonicalize.py +++ b/packages/linkml_runtime/src/linkml_runtime/utils/rdf_canonicalize.py @@ -1,49 +1,41 @@ -"""Deterministic RDF serialization via pyoxigraph RDFC-1.0 canonicalization. - -This module provides a function to canonicalize an rdflib Graph using -pyoxigraph's RDFC-1.0 implementation, producing deterministic output -with stable blank node labels and sorted triples. - -**Known limitations:** - -1. **xsd:string normalization**: pyoxigraph follows RDF 1.1, where plain - string literals and ``"text"^^xsd:string`` are identical. The output - will never contain explicit ``^^xsd:string`` annotations. Code that - re-parses the output with rdflib will see ``Literal("x")`` (datatype - ``None``) rather than ``Literal("x", datatype=XSD.string)``. - -2. **Non-standard RDF**: Graphs with literal predicates (e.g. SHACL - annotation mode) or relative IRIs (e.g. the metamodel's - ``bibo:status ``) are rejected by pyoxigraph. This function - falls back to rdflib's serializer for such graphs, but still - canonicalizes blank-node labels (and sorts line-oriented formats) so - the fallback output remains deterministic across processes. - -3. **Numeric short forms**: pyoxigraph uses Turtle short forms for - ``xsd:integer`` (``42``), ``xsd:boolean`` (``true``), and - ``xsd:decimal`` (``1.23``). rdflib parses these back with the - correct datatype, so this is lossless. - -4. **Base IRI / prefix collision**: When a graph has ``@base`` and a - prefix whose namespace equals the base IRI (e.g. rdflib's auto-bound - ``base:`` prefix), pyoxigraph emits CURIEs like ``base:label`` that - rdflib rejects. We skip such prefixes during serialization. - -5. **Trailing escaped dot in PN_LOCAL**: pyoxigraph emits CURIEs like - ``prefix:local\\.`` for IRIs whose local part ends with ``.``. This - is valid Turtle (PN_LOCAL_ESC), but rdflib's notation3 parser rejects - it because it conflicts with the statement-terminator dot. We - post-process the output to expand such CURIEs to full ```` form. +"""Deterministic RDF serialization, delegated to the ``diffable-rdf`` library. + +This module used to carry the implementation. It was extracted into +`diffable-rdf `_ at the maintainers' +request (linkml/linkml#3295: *"the implementation should live elsewhere ... +independent rdflib sidecar library?"*), and this module is now a thin adapter +over it. :func:`canonicalize_rdf_graph` keeps its signature, so nothing that +imports it needs to change. + +The adapter exists for one reason: the two projects report degraded output +differently. The library logs through the ``logging`` module, which is silent +unless the application configured a handler. linkml deliberately uses +:func:`warnings.warn` so a schema author running ``gen-owl`` sees that the +output took a fallback path without having to opt in to logging first. So the +library's warnings are captured and re-emitted as +:class:`RDFCanonicalizationWarning`. + +What the library does, in short: the graph is transferred to pyoxigraph via +N-Triples, canonicalized with RDFC-1.0, sorted, and serialized back. Graphs +pyoxigraph refuses -- literal predicates from SHACL annotation mode, relative +IRIs such as the metamodel's ``bibo:status `` -- fall back to rdflib +with blank-node labels canonicalized by :func:`rdflib.compare.to_canonical_graph`, +which is content-derived rather than run-local, so the fallback is still +reproducible across processes. The full contract, including the limitations +that used to be listed here (``xsd:string`` normalization, numeric short forms, +base/prefix collisions, ``PN_LOCAL`` escaping), is documented in the library's +``docs/api.md``. + +Delegating also adopts nine correctness fixes the extracted copy received and +this one never did, two of them silent data corruption. Each is pinned by a +test in ``tests/linkml_runtime/test_utils/test_rdf_canonicalize_defects.py``. """ -import io -import re +import logging import warnings -import pyoxigraph as ox import rdflib -from diffable_rdf import wl_relabel_quads -from rdflib.compare import to_canonical_graph +from diffable_rdf import canonicalize_rdf_graph as _canonicalize_rdf_graph class RDFCanonicalizationWarning(UserWarning): @@ -56,233 +48,59 @@ class RDFCanonicalizationWarning(UserWarning): """ -# Mapping from rdflib/LinkML format strings to pyoxigraph RdfFormat objects. -_FORMAT_MAP: dict[str, ox.RdfFormat] = { - "turtle": ox.RdfFormat.TURTLE, - "ttl": ox.RdfFormat.TURTLE, - "nt": ox.RdfFormat.N_TRIPLES, - "ntriples": ox.RdfFormat.N_TRIPLES, - "n-triples": ox.RdfFormat.N_TRIPLES, - "nt11": ox.RdfFormat.N_TRIPLES, - "nquads": ox.RdfFormat.N_QUADS, - "n-quads": ox.RdfFormat.N_QUADS, - "xml": ox.RdfFormat.RDF_XML, - "rdf/xml": ox.RdfFormat.RDF_XML, - "trig": ox.RdfFormat.TRIG, - "n3": ox.RdfFormat.N3, -} - -# Formats that support prefix declarations. -_PREFIX_FORMATS = frozenset({ox.RdfFormat.TURTLE, ox.RdfFormat.TRIG, ox.RdfFormat.N3, ox.RdfFormat.RDF_XML}) - -# Line-oriented formats (one triple/quad per line) whose fallback output must -# be sorted, because rdflib does not emit them in a stable order even after -# blank-node canonicalization. -_LINE_ORIENTED_FORMATS = frozenset({"nt", "ntriples", "n-triples", "nt11", "nquads", "n-quads"}) - - -def _deterministic_fallback_serialize(graph: rdflib.Graph, output_format: str) -> str: - """Serialize a graph that pyoxigraph cannot canonicalize, deterministically. - - pyoxigraph rejects some graphs that rdflib accepts -- notably graphs - containing relative IRIs (e.g. the metamodel's ``bibo:status ``) - or literal predicates (SHACL annotation mode). A plain - ``graph.serialize()`` for such graphs is *not* reproducible across - processes: rdflib assigns blank-node labels non-deterministically, so the - structure and grouping of the output varies run to run. - - To degrade gracefully instead of silently emitting non-deterministic - output, we canonicalize blank-node labels with rdflib's own - isomorphism-based canonicalization (:func:`rdflib.compare.to_canonical_graph`, - which uses a content-derived hash, not run-local ids) and preserve the - original prefix and base bindings that the canonical graph drops. For - line-oriented formats we additionally sort the serialized lines, since - rdflib does not emit N-Triples/N-Quads in a stable order. - - Relative IRIs are preserved verbatim (not resolved against the base): the - goal is deterministic output, and silently rewriting ```` into an - absolute IRI would mask what is really a data problem in the source graph. - - :param graph: The rdflib Graph that pyoxigraph could not parse. - :param output_format: Target serialization format (e.g. ``"turtle"``, ``"nt"``). - :return: Deterministic string serialization of the graph. - """ - canonical = to_canonical_graph(graph) - # to_canonical_graph builds a fresh graph without the source's namespace - # bindings; rebind them so the output does not fall back to rdflib's - # non-deterministic auto-generated ``ns1:``/``ns2:`` prefixes. - for prefix, namespace in graph.namespace_manager.namespaces(): - canonical.namespace_manager.bind(prefix, namespace, replace=True) - canonical.base = graph.base - serialized = canonical.serialize(format=output_format) - if output_format.lower() in _LINE_ORIENTED_FORMATS: - lines = [line for line in serialized.splitlines() if line.strip()] - return "\n".join(sorted(lines)) + "\n" - return serialized - - -# Characters that may appear escaped in a Turtle PN_LOCAL via PN_LOCAL_ESC. -_PN_LOCAL_ESC_UNESCAPE = re.compile(r"\\([_~.\-!$&'()*+,;=/?#@%])") - +_LIBRARY_LOGGER = "diffable_rdf" -_CURIE_TRAILING_DOT_PATTERN = re.compile( - r"(?\"'\[\]]*?\\\.)" - r"(?=\s)" -) +class _DegradedPathWarnings(logging.Handler): + """Collect the library's warning logs and re-emit them as Python warnings. -def _iter_turtle_structural_spans(turtle_text: str): - """Yield ``(start, end, is_structural)`` spans of a Turtle string. + Records are collected during the call and re-emitted in :meth:`__exit__` + rather than from :meth:`emit`. Warning at the moment the record arrives + would put eight ``logging`` frames plus an unknown number of library frames + between :func:`warnings.warn` and the caller, and the library's depth + differs per message, so no single ``stacklevel`` could point at the caller. + Re-emitting from ``__exit__`` makes the distance a constant. - Structural spans are regions outside of string literals, IRI refs, and - line comments — i.e. the only places where a CURIE can syntactically - appear. Non-structural spans (literal contents, IRI bodies, comments) - are yielded as-is so they round-trip unchanged. + The handler is attached to the library's package logger rather than the + root logger. The library logs under module-level children such as + ``diffable_rdf.canonicalize``, which propagate to the package logger, so + attaching there catches every module without naming any of them. - Handles single- and triple-quoted literals with backslash escapes for - both ``"`` and ``'`` delimiters, ``<...>`` IRI refs, and ``#...`` line - comments. + ``propagate`` is left alone: a caller who *did* configure logging still + gets the record through their own handlers, and seeing the message once + per mechanism is a better trade than suppressing a handler they asked for. """ - n = len(turtle_text) - i = 0 - while i < n: - ch = turtle_text[i] - if ch in ('"', "'"): - # String literal — find matching delimiter, respecting triple- - # quote form and backslash escapes. - delim = ch - triple = turtle_text[i : i + 3] == delim * 3 - end_marker = delim * 3 if triple else delim - j = i + len(end_marker) - while j < n: - if turtle_text[j] == "\\" and j + 1 < n: - j += 2 - continue - if turtle_text[j : j + len(end_marker)] == end_marker: - j += len(end_marker) - break - j += 1 - yield i, j, False - i = j - elif ch == "<": - # IRI ref — find closing '>' on the same logical line. - j = turtle_text.find(">", i + 1) - if j == -1: - yield i, n, False - i = n - else: - yield i, j + 1, False - i = j + 1 - elif ch == "#": - # Line comment — to end of line. - j = turtle_text.find("\n", i) - j = n if j == -1 else j - yield i, j, False - i = j - else: - # Structural region — accumulate until the next literal / IRI / - # comment opener. Treat ``\X`` as a PN_LOCAL_ESC escape (two - # chars) so that ``\#`` and ``\.`` inside a CURIE local part - # don't trigger comment / boundary handling. - start = i - while i < n: - c = turtle_text[i] - if c == "\\" and i + 1 < n: - i += 2 - continue - if c in ('"', "'", "<", "#"): - break - i += 1 - yield start, i, True - -def _expand_trailing_dot_curies(turtle_text: str, prefixes: dict[str, str]) -> str: - """Replace CURIEs whose local part ends in ``\\.`` with full ```` form. - - rdflib's notation3 parser rejects PN_LOCAL ending in an escaped dot - even though Turtle permits it (PN_LOCAL_ESC). pyoxigraph emits this - form for IRIs ending in ``.`` (e.g. ``biolink:StrandEnum#.``). We - rewrite each such CURIE to its expanded ```` form so the output - round-trips through rdflib. - - The rewrite is applied only to *structural* spans of the document — - string literals, IRI refs, and comments are left untouched. This is - what prevents the regex from mangling CURIE-shaped substrings that - happen to appear inside a literal value. - """ - if not prefixes: - return turtle_text - - def replace(match: re.Match[str]) -> str: - prefix = match.group(1) - local_escaped = match.group(2) - namespace = prefixes.get(prefix) - if namespace is None: - return match.group(0) - local = _PN_LOCAL_ESC_UNESCAPE.sub(r"\1", local_escaped) - return f"<{namespace}{local}>" - - parts: list[str] = [] - for start, end, is_structural in _iter_turtle_structural_spans(turtle_text): - chunk = turtle_text[start:end] - if is_structural: - chunk = _CURIE_TRAILING_DOT_PATTERN.sub(replace, chunk) - parts.append(chunk) - return "".join(parts) - - -def _iri_terms(triples: list["ox.Triple"]) -> set[str]: - """Return the set of IRI strings appearing anywhere in ``triples``. - - Walks subjects, predicates, non-literal objects, and literal datatypes. - Used to filter the prefix dict down to namespaces that are actually - referenced by the canonicalized graph, so the output isn't padded with - unused ``@prefix`` declarations. - """ - iris: set[str] = set() - for t in triples: - for term in (t.subject, t.predicate, t.object): - if isinstance(term, ox.NamedNode): - iris.add(term.value) - elif isinstance(term, ox.Literal): - dt = term.datatype - if dt is not None: - iris.add(dt.value) - return iris - - -def _filter_prefixes_to_used(prefixes: dict[str, str], used_iris: set[str]) -> dict[str, str]: - """Drop prefix bindings whose namespace is not a prefix of any used IRI. - - A prefix is kept if at least one IRI in ``used_iris`` starts with its - namespace string. Parent-namespace matches are honored (e.g. a prefix - bound to ``http://schema.org/`` is kept when ``http://schema.org/Person`` - appears in the graph). - """ - return {prefix: ns for prefix, ns in prefixes.items() if any(iri.startswith(ns) for iri in used_iris)} - - -def _is_safe_prefix_iri(iri: str) -> bool: - """Check whether a namespace IRI is safe for prefix serialization. - - pyoxigraph rejects IRIs with invalid code-points (e.g. double ``#``), - and rdflib's Turtle parser cannot round-trip CURIEs whose namespace - contains query parameters or fragments in unexpected positions. This - function returns ``False`` for such IRIs so they can be skipped during - prefix collection. - """ - # A namespace IRI should end with '/' or '#'. If '#' appears - # *before* the final character, the IRI contains an embedded - # fragment which produces unusable CURIEs. - if "#" in iri[:-1]: - return False - # Query parameters in namespace IRIs produce CURIEs that rdflib - # cannot parse back. - if "?" in iri: - return False - return True + def __init__(self) -> None: + super().__init__(level=logging.WARNING) + self._messages: list[str] = [] + self._logger = logging.getLogger(_LIBRARY_LOGGER) + self._previous_level = logging.NOTSET + + def emit(self, record: logging.LogRecord) -> None: + """Record one formatted message for re-emission on exit.""" + self._messages.append(record.getMessage()) + + def __enter__(self) -> "_DegradedPathWarnings": + """Attach to the library's logger, raising its level if it is silent.""" + self._previous_level = self._logger.level + if not self._logger.isEnabledFor(logging.WARNING): + self._logger.setLevel(logging.WARNING) + self._logger.addHandler(self) + return self + + def __exit__(self, *exc_info: object) -> None: + """Detach, restore the logger, and re-emit what was collected. + + Runs on the exception path too, so a caller who gets an exception still + learns about the degradation that preceded it. + """ + self._logger.removeHandler(self) + self._logger.setLevel(self._previous_level) + for message in self._messages: + # 3 frames: warnings.warn, this method, and the ``with`` statement + # in canonicalize_rdf_graph -- so the warning lands on its caller. + warnings.warn(message, RDFCanonicalizationWarning, stacklevel=3) def canonicalize_rdf_graph( @@ -292,13 +110,15 @@ def canonicalize_rdf_graph( ) -> str: """Serialize an rdflib Graph deterministically using RDFC-1.0 canonicalization. - The graph is transferred to pyoxigraph via N-Triples, canonicalized - with RDFC-1.0, sorted, and serialized back to the requested format. - Prefix bindings from the rdflib Graph are preserved in the output - for formats that support them (Turtle, TriG, N3, RDF/XML). + The graph is transferred to pyoxigraph via N-Triples, canonicalized with + RDFC-1.0, sorted, and serialized back to the requested format. Prefix + bindings from the rdflib Graph are preserved in the output for formats that + support them (Turtle, TriG, N3, RDF/XML). - Falls back to plain rdflib serialization for unsupported formats or - graphs containing non-standard RDF (e.g. literal predicates). + Falls back to plain rdflib serialization for unsupported formats or graphs + containing non-standard RDF (e.g. literal predicates), warning with + :class:`RDFCanonicalizationWarning` so that the caller knows the output is + less strongly guaranteed than usual. :param graph: The rdflib Graph to serialize. :param output_format: Target serialization format (e.g. ``"turtle"``, ``"nt"``). @@ -310,142 +130,11 @@ def canonicalize_rdf_graph( output. Has no effect on the rdflib fallback path (non-standard RDF), which warns rather than silently ignoring the request. :return: Deterministic string serialization of the graph. + :raises ValueError: If the graph cannot be serialized to ``output_format`` + in a form that parses back -- notably a line-oriented format asked to + write a relative IRI, which N-Triples forbids. """ - ox_format = _FORMAT_MAP.get(output_format.lower()) - if ox_format is None: - warnings.warn( - f"pyoxigraph does not support format {output_format!r}; falling back to " - "rdflib serializer. Output will not be deterministically canonicalized.", - RDFCanonicalizationWarning, - stacklevel=2, - ) - return graph.serialize(format=output_format) - - # 1. Transfer rdflib graph to pyoxigraph via N-Triples. - nt_data = graph.serialize(format="nt") - nt_bytes = nt_data.encode("utf-8") if isinstance(nt_data, str) else nt_data - - # 2. Parse into pyoxigraph and build a Dataset for canonicalization. - # Fall back to rdflib if the graph contains non-standard RDF - # (e.g. literal predicates from annotations) that pyoxigraph rejects. - try: - triples = list(ox.parse(io.BytesIO(nt_bytes), format=ox.RdfFormat.N_TRIPLES)) - except SyntaxError: - warnings.warn( - "Graph contains non-standard RDF (e.g. relative IRIs or literal predicates) " - "that pyoxigraph cannot parse; falling back to rdflib. Output is still " - "deterministic (blank-node labels are canonicalized via rdflib) but is not " - "canonicalized with pyoxigraph RDFC-1.0.", - RDFCanonicalizationWarning, - stacklevel=2, - ) - if diff_stable: - # Weisfeiler-Lehman refinement consumes canonical pyoxigraph quads, - # and this path exists precisely because pyoxigraph refused the - # graph, so there are none to refine. The fallback is deterministic - # but not diff-stable: say so rather than returning output that - # silently ignores the argument. ``shaclgen --include-annotations`` - # reaches this path, because an annotation tag without a ``:`` - # becomes a literal predicate. - warnings.warn( - "diff_stable=True was requested but this graph took the rdflib fallback, " - "which cannot apply Weisfeiler-Lehman blank-node labels. Output is " - "deterministic but NOT diff-stable: an unrelated edit may still renumber " - "blank nodes. Make the offending terms standard RDF (absolute IRIs, IRI " - "predicates) to get diff-stable labels.", - RDFCanonicalizationWarning, - stacklevel=2, - ) - return _deterministic_fallback_serialize(graph, output_format) - - dataset = ox.Dataset() - for triple in triples: - dataset.add(ox.Quad(triple.subject, triple.predicate, triple.object, ox.DefaultGraph())) - - # 3. Canonicalize blank node labels with RDFC-1.0. - dataset.canonicalize(ox.CanonicalizationAlgorithm.RDFC_1_0) - - quads = list(dataset) - - # 3b. Optionally re-label blank nodes with locality-sensitive hashes. - # RDFC-1.0 guarantees that identical graphs get identical labels, but it - # does not guarantee that *similar* graphs get similar labels: the labels - # are assigned by a global ordering, so inserting one blank node can - # renumber every label after it and turn a one-line semantic change into a - # whole-file diff. Weisfeiler-Lehman labels are derived only from each - # node's local neighbourhood, so unrelated regions keep their labels. - # Output stays deterministic and isomorphic either way; this only changes - # which label each blank node receives. - if diff_stable: - quads = wl_relabel_quads(quads) - - # 4. Sort triples for deterministic ordering. - # RDFC-1.0 stabilizes blank-node labels but pyoxigraph's Dataset - # iteration order is not sorted and varies across processes (verified - # empirically against pyoxigraph 0.5.8). The explicit string-key sort - # is load-bearing for byte-identical output across runs; see - # tests/linkml_runtime/test_utils/test_rdf_canonicalize.py::test_sort_is_load_bearing. - sorted_triples = sorted( - (ox.Triple(q.subject, q.predicate, q.object) for q in quads), - key=lambda t: (str(t.subject), str(t.predicate), str(t.object)), - ) - - # 5. Collect prefixes for formats that support them. - base_iri = str(graph.base) if graph.base else None - prefixes: dict[str, str] | None = None - if ox_format in _PREFIX_FORMATS: - prefixes = {} - for prefix, namespace in graph.namespace_manager.namespaces(): - if not prefix: # skip empty prefix (base) - continue - ns_str = str(namespace) - # Skip prefixes whose namespace matches the base IRI to avoid - # pyoxigraph emitting CURIEs like `base:label` that conflict - # with the @base directive. - if base_iri and ns_str == base_iri: - continue - # Skip namespace IRIs that pyoxigraph rejects or that produce - # CURIEs rdflib cannot round-trip. Valid namespace IRIs for - # prefix use should end with '/' or '#' and contain no query - # parameters or fragment-like characters in the middle. - if not _is_safe_prefix_iri(ns_str): - continue - prefixes[str(prefix)] = ns_str - # Drop prefix bindings whose namespace is not referenced by any IRI - # in the graph. This prevents the rdflib NamespaceManager's default - # bindings (~30 well-known vocabularies) from being emitted into - # every output file regardless of whether the schema actually uses - # them. - prefixes = _filter_prefixes_to_used(prefixes, _iri_terms(sorted_triples)) - used_prefixes = prefixes - try: - result_bytes = ox.serialize( - sorted_triples, - format=ox_format, - prefixes=prefixes, - base_iri=base_iri, - ) - except ValueError as e: - # pyoxigraph 0.5.x reports rejected prefix IRIs with a message - # that begins with "Invalid prefix" (verified empirically). Only - # swallow that case and retry without prefixes — any other - # ValueError (e.g. invalid base IRI, or an unrelated future - # serializer bug) must propagate so it surfaces as a stack trace - # rather than silently dropping all prefix declarations. - if not str(e).startswith("Invalid prefix"): - raise - warnings.warn( - f"pyoxigraph rejected one or more prefix IRIs ({e}); serializing without " - "prefix declarations. Output remains canonicalized but is more verbose.", - RDFCanonicalizationWarning, - stacklevel=2, - ) - result_bytes = ox.serialize( - sorted_triples, - format=ox_format, - ) - used_prefixes = None - result = result_bytes.decode("utf-8") - if ox_format in _PREFIX_FORMATS and used_prefixes: - result = _expand_trailing_dot_curies(result, used_prefixes) - return result + # Frames between warnings.warn and this function's caller are counted in + # _DegradedPathWarnings.__exit__, which is where the re-emission happens. + with _DegradedPathWarnings(): + return _canonicalize_rdf_graph(graph, output_format=output_format, diff_stable=diff_stable) diff --git a/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py b/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py index 1bc07583af..5a813729dd 100644 --- a/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py +++ b/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py @@ -1,10 +1,14 @@ """Tests for deterministic RDF serialization via pyoxigraph RDFC-1.0.""" +import json +import logging import os import re import subprocess import sys import textwrap +import warnings +from pathlib import Path import pyoxigraph as ox import pytest @@ -12,7 +16,6 @@ from rdflib import BNode, Graph, Literal, URIRef from rdflib.namespace import RDF -from linkml_runtime.utils import rdf_canonicalize as rdf_canon_mod from linkml_runtime.utils.rdf_canonicalize import ( RDFCanonicalizationWarning, canonicalize_rdf_graph, @@ -304,13 +307,13 @@ def fake_serialize(*args, **kwargs): raise ValueError("Invalid prefix bad IRI 'http://example.com/ has space/', Invalid IRI code point ' '") return real_serialize(*args, **kwargs) - monkeypatch.setattr(rdf_canon_mod.ox, "serialize", fake_serialize) + monkeypatch.setattr(ox, "serialize", fake_serialize) g = Graph() g.bind("ex", "http://example.com/") g.add((URIRef("http://example.com/a"), RDF.type, URIRef("http://example.com/Thing"))) - with pytest.warns(RDFCanonicalizationWarning, match="rejected one or more prefix IRIs"): + with pytest.warns(RDFCanonicalizationWarning, match="rejected the prefix or base IRIs"): ttl = canonicalize_rdf_graph(g, output_format="turtle") # Fallback retry should succeed and the output still round-trips. g2 = Graph() @@ -319,22 +322,74 @@ def fake_serialize(*args, **kwargs): def test_unsupported_format_warns_and_falls_back(): - """A format pyoxigraph doesn't know about falls back to rdflib with a warning.""" + """A format the canonicalizer doesn't know about falls back to rdflib with a warning. + + ``longturtle`` is an rdflib serializer plugin with no pyoxigraph + equivalent, so it can only be produced by delegating to rdflib, and + rdflib orders that output by graph traversal. The warning is what tells + the caller the determinism guarantee does not extend to this format. + """ g = Graph() g.bind("ex", "http://example.com/") g.add((URIRef("http://example.com/a"), RDF.type, URIRef("http://example.com/Thing"))) - with pytest.warns(RDFCanonicalizationWarning, match="does not support format"): - result = canonicalize_rdf_graph(g, output_format="json-ld") + with pytest.warns(RDFCanonicalizationWarning, match="not one of the formats"): + result = canonicalize_rdf_graph(g, output_format="longturtle") # The fallback should still produce valid output in the requested format. assert result +def test_json_ld_is_canonicalized_rather_than_handed_to_rdflib(): + """``json-ld`` gets the determinism guarantee, not a fallback warning. + + rdflib's JSON-LD serializer emits objects in traversal order, so this used + to be an unsupported format that warned and returned whatever rdflib felt + like. It is now produced deterministically, which is why no warning is due. + """ + g = _make_graph_with_bnodes() + with warnings.catch_warnings(): + warnings.simplefilter("error", RDFCanonicalizationWarning) + result = canonicalize_rdf_graph(g, output_format="json-ld") + assert json.loads(result) + + +def test_a_rejected_base_iri_is_recovered_rather_than_fatal(monkeypatch): + """A base pyoxigraph refuses costs the base directive, not the document. + + rdflib accepts a relative or otherwise unusable ``base``, and pyoxigraph + then refuses it at serialization time. Dropping the base and retrying + yields a correct document; failing the whole call over a directive that is + optional in every output format does not. + """ + real_serialize = ox.serialize + calls = {"n": 0} + + def fake_serialize(*args, **kwargs): + calls["n"] += 1 + if calls["n"] == 1: + raise ValueError("Invalid base IRI 'broken', Invalid IRI code point ' '") + return real_serialize(*args, **kwargs) + + monkeypatch.setattr(ox, "serialize", fake_serialize) + + g = Graph() + g.bind("ex", "http://example.com/") + g.add((URIRef("http://example.com/a"), RDF.type, URIRef("http://example.com/Thing"))) + + with pytest.warns(RDFCanonicalizationWarning, match="rejected the prefix or base IRIs"): + ttl = canonicalize_rdf_graph(g, output_format="turtle") + + g2 = Graph() + g2.parse(data=ttl, format="turtle") + assert rdflib.compare.isomorphic(g, g2) + + def test_unrelated_value_error_propagates(monkeypatch): - """Any ``ValueError`` whose message doesn't begin with ``Invalid prefix`` is re-raised. + """A ``ValueError`` that is not about prefixes or the base must be re-raised. - Guards against future regressions where a pyoxigraph serializer bug - raises a different ``ValueError`` and gets silently swallowed by the - invalid-prefix fallback path. + The retry-without-prefixes path exists for two specific pyoxigraph + complaints. Widening it to every ``ValueError`` would let a serializer bug + be papered over by a retry that happens to succeed, and the caller would + get a plausible-looking document with no indication anything went wrong. """ real_serialize = ox.serialize calls = {"count": 0} @@ -343,16 +398,16 @@ def fake_serialize(*args, **kwargs): calls["count"] += 1 # First call is the prefixed serialize — raise an unrelated ValueError. if calls["count"] == 1: - raise ValueError("Invalid base IRI 'broken', Invalid IRI code point ' '") + raise ValueError("BUG: serializer state corrupted") return real_serialize(*args, **kwargs) - monkeypatch.setattr(rdf_canon_mod.ox, "serialize", fake_serialize) + monkeypatch.setattr(ox, "serialize", fake_serialize) g = Graph() g.bind("ex", "http://example.com/") g.add((URIRef("http://example.com/a"), RDF.type, URIRef("http://example.com/Thing"))) - with pytest.raises(ValueError, match="Invalid base IRI"): + with pytest.raises(ValueError, match="BUG: serializer state corrupted"): canonicalize_rdf_graph(g, output_format="turtle") @@ -455,7 +510,7 @@ def test_fallback_preserves_relative_iri(): assert "" in result -@pytest.mark.parametrize("output_format", ["turtle", "nt"]) +@pytest.mark.parametrize("output_format", ["turtle", "xml"]) def test_fallback_is_deterministic_across_processes(output_format): """The rdflib fallback produces byte-identical output across processes. @@ -463,9 +518,15 @@ def test_fallback_is_deterministic_across_processes(output_format): parse to fail, exercising the fallback path. The graph also contains several blank nodes: a plain ``graph.serialize()`` would label them non-deterministically, so this test would fail without the blank-node - canonicalization in ``_deterministic_fallback_serialize``. Two - subprocesses with different ``PYTHONHASHSEED`` values must agree byte for - byte. + canonicalization in the fallback. Two subprocesses with different + ``PYTHONHASHSEED`` values must agree byte for byte. + + ``nt`` is not covered here because a fallback graph is by definition one + pyoxigraph refused, and for this graph the reason is a relative IRI, which + N-Triples forbids outright (N-Triples 1.1 §2.2). There is no deterministic + N-Triples document to produce, so the correct answer is a refusal -- + asserted by ``test_nt_fallback_refuses_rather_than_writing_an_unparseable_file`` + below. """ program = textwrap.dedent( f""" @@ -509,6 +570,24 @@ def run(seed: str) -> str: ) +def test_nt_fallback_refuses_rather_than_writing_an_unparseable_file(): + """N-Triples output for a graph with a relative IRI must raise, not lie. + + rdflib's N-Triples serializer reuses Turtle's term rendering and does not + enforce the absolute-IRI rule, so asking it for ``nt`` here yields a file + containing ```` that its own parser then rejects. N-Triples 1.1 + §2.2 permits only absolute IRIs, so no valid document exists for this + graph and a refusal is the only honest answer. Writing the file instead + defers the failure to whoever tries to read it. + """ + g = Graph() + g.bind("ex", "http://example.com/") + g.add((URIRef("http://example.com/s"), URIRef("http://purl.org/ontology/bibo/status"), URIRef("testing"))) + + with pytest.raises(ValueError, match="not an absolute IRI"): + canonicalize_rdf_graph(g, output_format="nt") + + def _shapes_graph(count: int, extra: bool = False) -> Graph: """A graph shaped like generator output: one blank node per named subject.""" g = Graph() @@ -658,7 +737,7 @@ def test_diff_stable_warns_instead_of_silently_no_opping_on_the_fallback(): # ``shaclgen --include-annotations`` takes via its literal predicates. graph.add((URIRef("testing"), URIRef("http://example.com/p"), Literal("v"))) - with pytest.warns(RDFCanonicalizationWarning, match="NOT diff-stable"): + with pytest.warns(RDFCanonicalizationWarning, match="not diff-stable"): stable = canonicalize_rdf_graph(graph, diff_stable=True) with pytest.warns(RDFCanonicalizationWarning): @@ -667,3 +746,78 @@ def test_diff_stable_warns_instead_of_silently_no_opping_on_the_fallback(): # The warning is the contract: the bytes really are identical, which is # exactly why staying silent would be misleading. assert stable == plain + + +# -------------------------------------------------------------------------- +# The bridge from the library's logging to linkml's warnings +# -------------------------------------------------------------------------- + + +def _fallback_graph() -> Graph: + """A graph pyoxigraph refuses, so every call takes a degraded path.""" + g = Graph() + g.bind("ex", "http://example.com/") + g.add((URIRef("http://example.com/s"), URIRef("http://purl.org/ontology/bibo/status"), URIRef("testing"))) + return g + + +def test_degraded_path_warning_points_at_the_caller(): + """The warning must name the line that asked for the serialization. + + ``diffable-rdf`` reports degradation through ``logging``; linkml re-emits + it through ``warnings`` so it is visible without logging configuration. + A re-emitted warning is only actionable if it is attributed to the caller + rather than to the adapter, and the number of frames in between is not + something a reader can eyeball -- hence this test. + """ + graph = _fallback_graph() + + with pytest.warns(RDFCanonicalizationWarning) as caught: + canonicalize_rdf_graph(graph, output_format="turtle") # attribution target + + assert caught[0].filename == __file__ + source = Path(caught[0].filename).read_text(encoding="utf-8").splitlines() + assert "# attribution target" in source[caught[0].lineno - 1] + + +def test_bridging_does_not_leave_the_library_logger_modified(): + """Raising the library's log level to capture records must not be permanent. + + The adapter has to enable ``WARNING`` on the ``diffable_rdf`` logger to see + anything, which is a global mutation. Leaving it raised would silently + change logging behaviour for the rest of the process. + """ + logger = logging.getLogger("diffable_rdf") + level_before = logger.level + handlers_before = list(logger.handlers) + + with pytest.warns(RDFCanonicalizationWarning): + canonicalize_rdf_graph(_fallback_graph(), output_format="turtle") + + assert logger.level == level_before + assert logger.handlers == handlers_before + + +def test_a_caller_who_configured_logging_still_receives_the_record(caplog): + """Re-emitting as a warning must not steal the record from a log handler. + + An application that deliberately configured the ``diffable_rdf`` logger is + asking for these records. The adapter adds a mechanism; it does not get to + remove one. + """ + with caplog.at_level(logging.WARNING, logger="diffable_rdf"), pytest.warns(RDFCanonicalizationWarning): + canonicalize_rdf_graph(_fallback_graph(), output_format="turtle") + + assert any("non-standard RDF" in record.getMessage() for record in caplog.records) + + +def test_warnings_survive_an_exception_from_the_library(): + """Degradation reported before a failure must still reach the caller. + + ``nt`` output for this graph degrades to rdflib and *then* refuses, because + N-Triples has no way to write the relative IRI. Dropping the warning + because the call ended in an exception would hide the first half of the + story, which is the half that explains the second. + """ + with pytest.warns(RDFCanonicalizationWarning, match="non-standard RDF"), pytest.raises(ValueError): + canonicalize_rdf_graph(_fallback_graph(), output_format="nt") diff --git a/tests/linkml_runtime/test_utils/test_rdf_canonicalize_defects.py b/tests/linkml_runtime/test_utils/test_rdf_canonicalize_defects.py index 806be2e3eb..ffca5ae447 100644 --- a/tests/linkml_runtime/test_utils/test_rdf_canonicalize_defects.py +++ b/tests/linkml_runtime/test_utils/test_rdf_canonicalize_defects.py @@ -1,27 +1,24 @@ -"""Correctness differences between linkml's RDF canonicalizer and the extracted library. +"""Correctness properties shared by linkml's RDF canonicalizer and the library behind it. ``linkml_runtime.utils.rdf_canonicalize`` and ``diffable_rdf.canonicalize_rdf_graph`` are the same code: the module was extracted into a standalone library at the maintainers' request -(linkml/linkml#3295). The two copies have since diverged, so each test here -asserts one correctness property against *both* implementations and marks the -one that does not hold it as a strict xfail. - -Every remaining gap runs one way -- the extracted copy received fixes the local -one did not -- and two of them are silent data corruption, where the output -parses cleanly and says something the input never said. That is worse than a -crash, because a generated artifact gets committed and reviewed on the -assumption that it means what the schema meant. - -One gap used to run the other way: the library dropped ``@base`` on the -degraded path. That was the last property blocking delegation, and -diffable-rdf 0.4.0 fixed it, so the case now passes on both sides and is kept -unmarked. Asserting both directions is what made the fix land where it -belongs. - -The xfails are strict, so this file is a ratchet in both directions: when a fix -lands on either side, its case starts passing and the suite fails until the -mark is removed. +(linkml/linkml#3295). The two copies then diverged for a while, and this file +is what made the divergence visible -- each test asserts one correctness +property against *both* implementations, so a fix that landed on one side and +not the other showed up as a failure rather than as nothing at all. + +Nine properties held only in the library, two of them cases of silent data +corruption where the output parsed cleanly and said something the input never +said. One held only in linkml: the library dropped ``@base`` on the degraded +path. Asserting both directions is what got each of them fixed where it +belonged -- the ``@base`` gap in diffable-rdf 0.4.0, the other nine in linkml +by deleting the in-tree copy and calling the library. + +Every case is now unmarked, which is the point: the file is the evidence that +the delegation changed no behaviour it should not have, and it keeps running +against both entry points so that a future divergence fails the suite instead +of going unnoticed. """ import os @@ -46,18 +43,15 @@ ) -def _impls(**known_failures: str): - """Parametrize over both implementations, xfailing the ones named. +def _impls(): + """Parametrize a test over both entry points. - :param known_failures: implementation id (``linkml`` / ``diffable_rdf``) - mapped to why that implementation does not hold the property. + linkml's is a thin adapter over the library's, so the two agree by + construction today. Running both anyway is what turns a future re-fork, or + an adapter that quietly changes behaviour on the way through, into a test + failure. """ - params = [] - for fn, ident in _IMPLEMENTATIONS: - reason = known_failures.get(ident.replace("-", "_")) - marks = [pytest.mark.xfail(strict=True, reason=reason)] if reason else [] - params.append(pytest.param(fn, id=ident, marks=marks)) - return params + return [pytest.param(fn, id=ident) for fn, ident in _IMPLEMENTATIONS] def _apply(fn, graph, output_format="turtle"): @@ -72,10 +66,7 @@ def _apply(fn, graph, output_format="turtle"): # -------------------------------------------------------------------------- -@pytest.mark.parametrize( - "canonicalize", - _impls(linkml="passes graph.base to the serializer without checking it is safe to relativize against"), -) +@pytest.mark.parametrize("canonicalize", _impls()) def test_base_with_a_fragment_does_not_rewrite_every_iri(canonicalize): """A base IRI ending in ``#`` must not change the graph's terms. @@ -94,10 +85,7 @@ def test_base_with_a_fragment_does_not_rewrite_every_iri(canonicalize): assert {str(s) for s in round_tripped.subjects()} == {"http://ex.org/d#a"} -@pytest.mark.parametrize( - "canonicalize", - _impls(linkml="rdflib's collection syntax materializes a shared list tail once per referencing list"), -) +@pytest.mark.parametrize("canonicalize", _impls()) def test_a_shared_rdf_list_tail_is_not_duplicated(canonicalize): """Two lists sharing a tail must not gain triples on the way out. @@ -130,10 +118,7 @@ def test_a_shared_rdf_list_tail_is_not_duplicated(canonicalize): # -------------------------------------------------------------------------- -@pytest.mark.parametrize( - "canonicalize", - _impls(linkml="the rdflib fallback writes N-Triples for a graph pyoxigraph already refused"), -) +@pytest.mark.parametrize("canonicalize", _impls()) def test_nt_output_either_parses_or_refuses(canonicalize): """N-Triples admits only absolute IRIs, so a relative one must not be written. @@ -153,10 +138,7 @@ def test_nt_output_either_parses_or_refuses(canonicalize): rdflib.Graph().parse(data=serialized, format="nt") -@pytest.mark.parametrize( - "canonicalize", - _impls(linkml="sorts fallback output with str.splitlines(), which breaks on U+2028 and five other characters"), -) +@pytest.mark.parametrize("canonicalize", _impls()) def test_a_unicode_line_separator_in_a_literal_survives(canonicalize): """Sorting N-Triples lines must split on newlines only. @@ -240,37 +222,24 @@ def _outputs_across_processes(module, output_format, force_fallback): @pytest.mark.parametrize( ("module", "output_format", "force_fallback"), [ - pytest.param( - "linkml", "xml", True, id="linkml-xml-fallback", marks=pytest.mark.xfail(strict=True, reason=_XML_TRAVERSAL) - ), + pytest.param("linkml", "xml", True, id="linkml-xml-fallback"), pytest.param("diffable_rdf", "xml", True, id="diffable-rdf-xml-fallback"), - pytest.param( - "linkml", - "turtle", - True, - id="linkml-turtle-fallback", - marks=pytest.mark.xfail(strict=True, reason=_NS_NAMES), - ), + pytest.param("linkml", "turtle", True, id="linkml-turtle-fallback"), pytest.param("diffable_rdf", "turtle", True, id="diffable-rdf-turtle-fallback"), - pytest.param( - "linkml", - "json-ld", - False, - id="linkml-jsonld", - marks=pytest.mark.xfail(strict=True, reason=_JSONLD_UNMAPPED), - ), + pytest.param("linkml", "json-ld", False, id="linkml-jsonld"), pytest.param("diffable_rdf", "json-ld", False, id="diffable-rdf-jsonld"), ], ) def test_output_is_byte_identical_across_processes(module, output_format, force_fallback): """The same graph must serialize to the same bytes in any process. - Three separate causes break this in the local copy. Degraded RDF/XML is - ordered by rdflib's own graph traversal, which follows set iteration order. - Auto-generated ``ns1``/``ns2`` prefix names are allocated in traversal order - too, so the *names* move even when the triples do not. And ``json-ld`` is - not in the format map at all, so it falls through to rdflib's serializer, - which carries no determinism guarantee. + Three separate causes used to break this in the in-tree copy, and all three + are fixed by delegating. Degraded RDF/XML was ordered by rdflib's own graph + traversal, which follows set iteration order. Auto-generated ``ns1``/``ns2`` + prefix names were allocated in traversal order too, so the *names* moved + even when the triples did not. And ``json-ld`` was not in the format map at + all, so it fell through to rdflib's serializer, which carries no + determinism guarantee. """ assert len(_outputs_across_processes(module, output_format, force_fallback)) == 1 @@ -280,10 +249,7 @@ def test_output_is_byte_identical_across_processes(module, output_format, force_ # -------------------------------------------------------------------------- -@pytest.mark.parametrize( - "canonicalize", - _impls(linkml="returns the serializer's own trailing whitespace, which differs by format"), -) +@pytest.mark.parametrize("canonicalize", _impls()) def test_every_format_ends_with_exactly_one_newline(canonicalize): """A committed artifact should not depend on which format wrote it. @@ -302,10 +268,7 @@ def test_every_format_ends_with_exactly_one_newline(canonicalize): assert counts == dict.fromkeys(counts, 1) -@pytest.mark.parametrize( - "canonicalize", - _impls(linkml="Dataset is a Graph subclass, so the type check accepts it and the named graphs are flattened"), -) +@pytest.mark.parametrize("canonicalize", _impls()) def test_a_dataset_is_refused_rather_than_partly_serialized(canonicalize): """A Dataset is a Graph subclass, so it is accepted and quietly flattened. @@ -337,13 +300,11 @@ def test_base_survives_the_degraded_path(canonicalize): which pyoxigraph rejects). Losing the directive on exactly that path means the feature works only for graphs that never needed the fallback. - This ran the other way until diffable-rdf 0.4.0, which was the last - property blocking delegation. The case is kept because it is the one the - two implementations reach differently: linkml passes ``graph.base`` to the - serializer unconditionally, which is why it also fails - :func:`test_base_with_a_fragment_does_not_rewrite_every_iri` above, while - the library keeps the base only after confirming that re-reading the - result still yields every absolute IRI of the source. + This ran the other way until diffable-rdf 0.4.0. It was the last property + blocking delegation, because linkml held it and the library did not, so + adopting the library would have been a regression. Fixing it there rather + than keeping the in-tree copy alive is what let the other nine gaps close + at once. """ graph = Graph(base="http://example.org/default/") graph.bind("ex", EX)