From b98b49611905fb7a854201a855dcf6c0d9267149 Mon Sep 17 00:00:00 2001 From: Carlo van Driesten Date: Tue, 8 Sep 2026 18:09:43 +0200 Subject: [PATCH 1/4] feat(rdf): add opt-in diff-stable blank-node labels RDFC-1.0 canonicalization already makes RDF output deterministic: isomorphic graphs always serialize identically. It does not make output diffable. Blank nodes are numbered `c14nN` in a single global order, so inserting one class can renumber every blank node after it and rewrite most of the file. A one-line semantic change lands as a whole-file diff, which makes generated OWL/SHACL hard to review and noisy to keep under version control. Add a `diff_stable` argument to `canonicalize_rdf_graph()` and a `--diff-stable/--no-diff-stable` flag to the four RDF generators. When enabled, blank-node labels are derived from each node's own neighbourhood via Weisfeiler-Lehman refinement, so an edit relabels only the blank nodes it actually touches. Measured churn on a real schema (add one class, count changed lines): generator default --diff-stable owlgen 2091 17 shexgen 796 50 shaclgen 291 13 rdfgen 115 25 Output stays deterministic and isomorphic either way; only the choice of label changes. Off by default, because enabling it relabels existing output. The refinement itself lives in `diffable-rdf`, whose only dependencies (rdflib, pyoxigraph) are already linkml-runtime dependencies at higher versions, so this adds no new transitive dependencies. --- .../linkml/src/linkml/generators/owlgen.py | 28 +++- .../linkml/src/linkml/generators/rdfgen.py | 28 +++- .../linkml/src/linkml/generators/shaclgen.py | 28 +++- .../linkml/src/linkml/generators/shexgen.py | 28 +++- packages/linkml_runtime/pyproject.toml | 1 + .../linkml_runtime/utils/rdf_canonicalize.py | 23 ++- .../test_utils/test_rdf_canonicalize.py | 135 ++++++++++++++++++ 7 files changed, 266 insertions(+), 5 deletions(-) diff --git a/packages/linkml/src/linkml/generators/owlgen.py b/packages/linkml/src/linkml/generators/owlgen.py index 50be266df0..7751eab34d 100644 --- a/packages/linkml/src/linkml/generators/owlgen.py +++ b/packages/linkml/src/linkml/generators/owlgen.py @@ -122,6 +122,22 @@ class OwlSchemaGenerator(Generator): """Suffix to add to the schema name to create the ontology URI, e.g. .owl.ttl""" # ObjectVars + diff_stable: bool = False + """Label blank nodes so that unrelated edits leave them untouched. + + Output is already deterministic: RDFC-1.0 guarantees that isomorphic + graphs serialize identically. It does not guarantee that *similar* + graphs serialize *similarly* — blank nodes are numbered ``c14nN`` in a + global order, so adding one class can renumber every blank node after + it and rewrite most of the file. + + When ``True``, blank-node labels are instead derived from each node's + own neighbourhood via Weisfeiler-Lehman refinement, so an edit relabels + only the blank nodes it actually touches. The output stays + deterministic and isomorphic either way; only the choice of label + changes. Off by default because enabling it relabels existing output. + """ + metadata_profile: MetadataProfile | None = None """Deprecated - use metadata_profiles.""" @@ -353,7 +369,7 @@ def serialize(self, **kwargs: Any) -> str: """ self.as_graph() fmt = "turtle" if self.format in ["owl", "ttl"] else self.format - return canonicalize_rdf_graph(self.graph, output_format=fmt) + return canonicalize_rdf_graph(self.graph, output_format=fmt, diff_stable=self.diff_stable) def add_metadata(self, e: Definition | PermissibleValue, uri: URIRef) -> None: """ @@ -1844,6 +1860,16 @@ def slot_owl_type(self, slot: SlotDefinition) -> URIRef: "specified language tag. Element-level in_language overrides this." ), ) +@click.option( + "--diff-stable/--no-diff-stable", + default=False, + show_default=True, + help=( + "Derive blank-node labels from each node's own neighbourhood so that " + "unrelated edits leave them unchanged. Output is deterministic either " + "way; this makes successive versions of a file diff cleanly." + ), +) @click.version_option(__version__, "-V", "--version") def cli(yamlfile: str, metadata_profile: str, **kwargs: Any) -> None: """Generate an OWL representation of a LinkML model diff --git a/packages/linkml/src/linkml/generators/rdfgen.py b/packages/linkml/src/linkml/generators/rdfgen.py index 2da1701787..052fcff12e 100644 --- a/packages/linkml/src/linkml/generators/rdfgen.py +++ b/packages/linkml/src/linkml/generators/rdfgen.py @@ -78,6 +78,22 @@ class RDFGenerator(Generator): uses_schemaloader = True # ObjectVars + diff_stable: bool = False + """Label blank nodes so that unrelated edits leave them untouched. + + Output is already deterministic: RDFC-1.0 guarantees that isomorphic + graphs serialize identically. It does not guarantee that *similar* + graphs serialize *similarly* — blank nodes are numbered ``c14nN`` in a + global order, so adding one class can renumber every blank node after + it and rewrite most of the file. + + When ``True``, blank-node labels are instead derived from each node's + own neighbourhood via Weisfeiler-Lehman refinement, so an edit relabels + only the blank nodes it actually touches. The output stays + deterministic and isomorphic either way; only the choice of label + changes. Off by default because enabling it relabels existing output. + """ + emit_metadata: bool = False context: list[str] = None original_schema: SchemaDefinition = None @@ -89,7 +105,7 @@ def __post_init__(self): def _data(self, g: Graph) -> str: fmt = "turtle" if self.format == "ttl" else self.format - return canonicalize_rdf_graph(g, output_format=fmt) + return canonicalize_rdf_graph(g, output_format=fmt, diff_stable=self.diff_stable) def end_schema(self, output: str | None = None, context: str = None, **_) -> str: gen = JSONLDGenerator( @@ -137,6 +153,16 @@ def end_schema(self, output: str | None = None, context: str = None, **_) -> str multiple=True, help="JSONLD context file (default: vendored meta.context.jsonld)", ) +@click.option( + "--diff-stable/--no-diff-stable", + default=False, + show_default=True, + help=( + "Derive blank-node labels from each node's own neighbourhood so that " + "unrelated edits leave them unchanged. Output is deterministic either " + "way; this makes successive versions of a file diff cleanly." + ), +) @click.version_option(__version__, "-V", "--version") def cli(yamlfile, **kwargs): """Generate an RDF representation of a LinkML model""" diff --git a/packages/linkml/src/linkml/generators/shaclgen.py b/packages/linkml/src/linkml/generators/shaclgen.py index 4731b9f0b8..5d42fea272 100644 --- a/packages/linkml/src/linkml/generators/shaclgen.py +++ b/packages/linkml/src/linkml/generators/shaclgen.py @@ -142,6 +142,22 @@ class ShaclGenerator(Generator): ignores any per-slot ``in_language``. """ + diff_stable: bool = False + """Label blank nodes so that unrelated edits leave them untouched. + + Output is already deterministic: RDFC-1.0 guarantees that isomorphic + graphs serialize identically. It does not guarantee that *similar* + graphs serialize *similarly* — blank nodes are numbered ``c14nN`` in a + global order, so adding one class can renumber every blank node after + it and rewrite most of the file. + + When ``True``, blank-node labels are instead derived from each node's + own neighbourhood via Weisfeiler-Lehman refinement, so an edit relabels + only the blank nodes it actually touches. The output stays + deterministic and isomorphic either way; only the choice of label + changes. Off by default because enabling it relabels existing output. + """ + emit_rules: bool = True """Emit ``sh:sparql`` constraints from LinkML ``rules:`` blocks. @@ -196,7 +212,7 @@ def generate_header(self) -> str: def serialize(self, **args) -> str: g = self.as_graph() fmt = "turtle" if self.format in ["owl", "ttl"] else self.format - return canonicalize_rdf_graph(g, output_format=fmt) + return canonicalize_rdf_graph(g, output_format=fmt, diff_stable=self.diff_stable) def as_graph(self) -> Graph: sv = self.schemaview @@ -929,6 +945,16 @@ def add_simple_data_type(func: Callable, r: ElementName) -> None: "sh:NodeShape. Use --no-emit-rules to suppress rule generation." ), ) +@click.option( + "--diff-stable/--no-diff-stable", + default=False, + show_default=True, + help=( + "Derive blank-node labels from each node's own neighbourhood so that " + "unrelated edits leave them unchanged. Output is deterministic either " + "way; this makes successive versions of a file diff cleanly." + ), +) @click.version_option(__version__, "-V", "--version") def cli(yamlfile, **args): """Generate SHACL turtle from a LinkML model""" diff --git a/packages/linkml/src/linkml/generators/shexgen.py b/packages/linkml/src/linkml/generators/shexgen.py index 40a93ffbc9..7c43b13bc0 100644 --- a/packages/linkml/src/linkml/generators/shexgen.py +++ b/packages/linkml/src/linkml/generators/shexgen.py @@ -40,6 +40,22 @@ class ShExGenerator(Generator): uses_schemaloader = True # ObjectVars + diff_stable: bool = False + """Label blank nodes so that unrelated edits leave them untouched. + + Output is already deterministic: RDFC-1.0 guarantees that isomorphic + graphs serialize identically. It does not guarantee that *similar* + graphs serialize *similarly* — blank nodes are numbered ``c14nN`` in a + global order, so adding one class can renumber every blank node after + it and rewrite most of the file. + + When ``True``, blank-node labels are instead derived from each node's + own neighbourhood via Weisfeiler-Lehman refinement, so an edit relabels + only the blank nodes it actually touches. The output stays + deterministic and isomorphic either way; only the choice of label + changes. Off by default because enabling it relabels existing output. + """ + shex: Schema = field(default_factory=lambda: Schema()) # ShEx Schema being generated shapes: list = field(default_factory=lambda: []) shape: Shape | None = None # Current shape being defined @@ -177,7 +193,7 @@ def end_schema(self, output: str | None = None, **_) -> str: g = Graph() g.parse(data=shex, format="json-ld", version="1.1") g.bind("owl", OWL) - shex = canonicalize_rdf_graph(g, output_format="turtle") + shex = canonicalize_rdf_graph(g, output_format="turtle", diff_stable=self.diff_stable) elif self.format == "shex": g = Graph() self.namespaces.load_graph(g) @@ -258,6 +274,16 @@ def _get_subproperty_values(self, slot: SlotDefinition) -> list: help="If --expand-subproperty-of (default), slots with subproperty_of will generate NodeConstraint " "values containing all slot descendants. Use --no-expand-subproperty-of to disable this behavior.", ) +@click.option( + "--diff-stable/--no-diff-stable", + default=False, + show_default=True, + help=( + "Derive blank-node labels from each node's own neighbourhood so that " + "unrelated edits leave them unchanged. Output is deterministic either " + "way; this makes successive versions of a file diff cleanly." + ), +) @click.version_option(__version__, "-V", "--version") def cli(yamlfile, **args): """Generate a ShEx Schema for a LinkML model""" diff --git a/packages/linkml_runtime/pyproject.toml b/packages/linkml_runtime/pyproject.toml index 1ae9da3083..b879dec7ee 100644 --- a/packages/linkml_runtime/pyproject.toml +++ b/packages/linkml_runtime/pyproject.toml @@ -48,6 +48,7 @@ dependencies = [ "prefixmaps >=0.1.4", "curies>=0.14.6", "pyoxigraph>=0.5.11", + "diffable-rdf>=0.2.0", "pydantic>=2.13.5,<3.0.0", "isodate >=0.7.2, <1.0.0; python_version < '3.11'", ] diff --git a/packages/linkml_runtime/src/linkml_runtime/utils/rdf_canonicalize.py b/packages/linkml_runtime/src/linkml_runtime/utils/rdf_canonicalize.py index 31c5580348..e0fadae34a 100644 --- a/packages/linkml_runtime/src/linkml_runtime/utils/rdf_canonicalize.py +++ b/packages/linkml_runtime/src/linkml_runtime/utils/rdf_canonicalize.py @@ -42,6 +42,7 @@ import pyoxigraph as ox import rdflib +from diffable_rdf import wl_relabel_quads from rdflib.compare import to_canonical_graph @@ -287,6 +288,7 @@ def _is_safe_prefix_iri(iri: str) -> bool: def canonicalize_rdf_graph( graph: rdflib.Graph, output_format: str = "turtle", + diff_stable: bool = False, ) -> str: """Serialize an rdflib Graph deterministically using RDFC-1.0 canonicalization. @@ -300,6 +302,12 @@ def canonicalize_rdf_graph( :param graph: The rdflib Graph to serialize. :param output_format: Target serialization format (e.g. ``"turtle"``, ``"nt"``). + :param diff_stable: Derive blank-node labels from each node's own + neighbourhood instead of RDFC-1.0's global ``c14nN`` numbering, so that + editing one part of a schema does not renumber unrelated blank nodes. + Output is deterministic and isomorphic either way; only the choice of + label changes. Off by default because enabling it relabels existing + output. :return: Deterministic string serialization of the graph. """ ox_format = _FORMAT_MAP.get(output_format.lower()) @@ -339,13 +347,26 @@ def canonicalize_rdf_graph( # 3. Canonicalize blank node labels with RDFC-1.0. dataset.canonicalize(ox.CanonicalizationAlgorithm.RDFC_1_0) + quads = list(dataset) + + # 3b. Optionally re-label blank nodes with locality-sensitive hashes. + # RDFC-1.0 guarantees that identical graphs get identical labels, but it + # does not guarantee that *similar* graphs get similar labels: the labels + # are assigned by a global ordering, so inserting one blank node can + # renumber every label after it and turn a one-line semantic change into a + # whole-file diff. Weisfeiler-Lehman labels are derived only from each + # node's local neighbourhood, so unrelated regions keep their labels. + # Output stays deterministic and isomorphic either way; this only changes + # which label each blank node receives. + if diff_stable: + quads = wl_relabel_quads(quads) + # 4. Sort triples for deterministic ordering. # RDFC-1.0 stabilizes blank-node labels but pyoxigraph's Dataset # iteration order is not sorted and varies across processes (verified # empirically against pyoxigraph 0.5.8). The explicit string-key sort # is load-bearing for byte-identical output across runs; see # tests/linkml_runtime/test_utils/test_rdf_canonicalize.py::test_sort_is_load_bearing. - quads = list(dataset) sorted_triples = sorted( (ox.Triple(q.subject, q.predicate, q.object) for q in quads), key=lambda t: (str(t.subject), str(t.predicate), str(t.object)), diff --git a/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py b/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py index 63dbf095a1..0ca79c3c90 100644 --- a/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py +++ b/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py @@ -507,3 +507,138 @@ def run(seed: str) -> str: assert out_a == out_b, ( "Fallback output differs across PYTHONHASHSEED values; blank-node canonicalization may be missing" ) + + +def _shapes_graph(count: int, extra: bool = False) -> Graph: + """A graph shaped like generator output: one blank node per named subject.""" + g = Graph() + g.bind("ex", "http://example.com/") + names = [f"{i:02d}" for i in range(count)] + (["AAAinserted"] if extra else []) + for name in names: + subject = URIRef(f"http://example.com/Shape{name}") + prop = BNode() + g.add((subject, URIRef("http://example.com/property"), prop)) + g.add((prop, URIRef("http://example.com/path"), URIRef(f"http://example.com/p{name}"))) + return g + + +def _changed_line_count(before: str, after: str) -> int: + import difflib + + diff = difflib.unified_diff(before.splitlines(), after.splitlines(), n=0, lineterm="") + return sum(1 for line in diff if line[:1] in "+-" and not line.startswith(("+++", "---"))) + + +def test_diff_stable_is_opt_in(): + """The default must keep producing exactly the output it produced before.""" + graph = _make_graph_with_bnodes() + assert canonicalize_rdf_graph(graph) == canonicalize_rdf_graph(graph, diff_stable=False) + + +def test_diff_stable_preserves_semantics(): + """Relabelling blank nodes must not change what the graph means.""" + graph = _make_graph_with_bnodes() + + plain = rdflib.Graph() + plain.parse(data=canonicalize_rdf_graph(graph), format="turtle") + stable = rdflib.Graph() + stable.parse(data=canonicalize_rdf_graph(graph, diff_stable=True), format="turtle") + + assert rdflib.compare.isomorphic(plain, stable) + + +def test_diff_stable_is_deterministic(): + """Diff stability must not cost determinism, which is the stronger property.""" + graph = _make_graph_with_bnodes() + outputs = {canonicalize_rdf_graph(graph, diff_stable=True) for _ in range(5)} + assert len(outputs) == 1 + + +def test_diff_stable_confines_an_insertion_to_the_lines_it_touches(): + """Inserting one subject must not relabel the blank nodes of the others. + + RDFC-1.0 numbers blank nodes ``c14nN`` in a global order, so a subject + sorting before the others shifts every subsequent label and rewrites + most of the file. This is the entire reason the option exists, so the + assertion is on the *ratio*, not on an absolute line count that would + be brittle across rdflib versions. + """ + before, after = _shapes_graph(20), _shapes_graph(20, extra=True) + + baseline = _changed_line_count(canonicalize_rdf_graph(before), canonicalize_rdf_graph(after)) + stable = _changed_line_count( + canonicalize_rdf_graph(before, diff_stable=True), + canonicalize_rdf_graph(after, diff_stable=True), + ) + + assert stable < baseline / 4, f"expected diff-stable output to churn far less; got {stable} vs baseline {baseline}" + + +_DIFF_STABLE_SCHEMA = """\ +id: https://example.org/diffstable +name: diffstable +prefixes: + linkml: https://w3id.org/linkml/ + ex: https://example.org/diffstable/ +default_prefix: ex +default_range: string +imports: + - linkml:types +classes: + Person: + slots: [name, knows] + Organization: + slots: [name] +slots: + name: + range: string + knows: + range: Person + multivalued: true +""" + + +def _generator_cases(): + """The four generators that serialize RDF, with the args that make them do so.""" + from linkml.generators.owlgen import OwlSchemaGenerator + from linkml.generators.rdfgen import RDFGenerator + from linkml.generators.shaclgen import ShaclGenerator + from linkml.generators.shexgen import ShExGenerator + + return [ + pytest.param(OwlSchemaGenerator, {}, id="owlgen"), + # rdfgen and shexgen resolve JSON-LD contexts (linkml types, shex.jsonld); + # the `network` marker serves those from local stubs. See tests/conftest.py. + pytest.param(RDFGenerator, {}, id="rdfgen", marks=pytest.mark.network), + pytest.param(ShaclGenerator, {}, id="shaclgen"), + # ShExGenerator only emits RDF in this format; its default is ShExC text, + # where blank-node labelling does not apply. + pytest.param(ShExGenerator, {"format": "rdf"}, id="shexgen", marks=pytest.mark.network), + ] + + +@pytest.mark.parametrize(("generator", "kwargs"), _generator_cases()) +def test_diff_stable_reaches_every_rdf_generator(tmp_path, generator, kwargs): + """Every RDF generator must actually apply the option, not merely accept it. + + Asserting only that the attribute exists would pass even if a generator + forgot to pass it down to :func:`canonicalize_rdf_graph`. Instead this + checks the observable consequence: RDFC-1.0 names blank nodes ``c14nN``, + while diff-stable labels are neighbourhood hashes, so a generator that + honours the flag emits no ``c14nN`` label at all. + """ + assert generator.diff_stable is False, f"{generator.__name__} must default to off" + + schema = tmp_path / "schema.yaml" + schema.write_text(_DIFF_STABLE_SCHEMA, encoding="utf-8", newline="\n") + + plain = generator(str(schema), **kwargs).serialize() + stable = generator(str(schema), diff_stable=True, **kwargs).serialize() + + # Guards the test itself: if the fixture stopped producing blank nodes the + # assertion below would hold vacuously. + assert re.search(r"c14n\d+", plain), f"{generator.__name__} output has no blank nodes to relabel" + assert not re.search(r"c14n\d+", stable), ( + f"{generator.__name__} still emits RDFC-1.0 blank-node labels with diff_stable=True; " + "the flag is probably not threaded into canonicalize_rdf_graph()" + ) From a55ba23124182ef94593ad26fdc33f00b67135a4 Mon Sep 17 00:00:00 2001 From: Carlo van Driesten Date: Fri, 11 Sep 2026 10:52:51 +0200 Subject: [PATCH 2/4] fix(rdf): bump diffable-rdf to 0.3.0 and stop diff_stable silently no-opping Bump the floor to diffable-rdf 0.3.0 and add the missing uv.lock entry: the dependency was declared in pyproject.toml but never locked, so "uv lock --check" and the "uv sync --frozen" anti-malware gate would both have failed CI. 0.3.0 also fixes two defects in the Weisfeiler-Lehman labelling this feature relies on. Disconnected blank-node components now converge independently, so an edit in one region no longer relabels an unrelated one. And the suffix used to tell structurally indistinguishable nodes apart was assigned in c14nN *text* order, so c14n10 sorted between c14n1 and c14n2 -- adding a tenth tied blank node relabelled eight of the nine already there, the exact opposite of what this labelling is for. Separately, diff_stable=True was silently ignored whenever pyoxigraph refused the graph and canonicalize_rdf_graph degraded to rdflib. Weisfeiler-Lehman refinement consumes canonical pyoxigraph quads, and that path exists precisely because there are none, so the argument could not be honoured -- but the caller was never told. "shaclgen --include-annotations --diff-stable" reaches it, via the literal predicate an annotation tag without a ':' produces, and returned output byte-identical to --no-diff-stable. It now warns, with a regression test asserting the warning and the byte-identical output that makes silence misleading. --- packages/linkml_runtime/pyproject.toml | 2 +- .../linkml_runtime/utils/rdf_canonicalize.py | 20 ++++++++++++++- .../test_utils/test_rdf_canonicalize.py | 25 +++++++++++++++++++ uv.lock | 15 +++++++++++ 4 files changed, 60 insertions(+), 2 deletions(-) diff --git a/packages/linkml_runtime/pyproject.toml b/packages/linkml_runtime/pyproject.toml index b879dec7ee..ede7112692 100644 --- a/packages/linkml_runtime/pyproject.toml +++ b/packages/linkml_runtime/pyproject.toml @@ -48,7 +48,7 @@ dependencies = [ "prefixmaps >=0.1.4", "curies>=0.14.6", "pyoxigraph>=0.5.11", - "diffable-rdf>=0.2.0", + "diffable-rdf>=0.3.0", "pydantic>=2.13.5,<3.0.0", "isodate >=0.7.2, <1.0.0; python_version < '3.11'", ] diff --git a/packages/linkml_runtime/src/linkml_runtime/utils/rdf_canonicalize.py b/packages/linkml_runtime/src/linkml_runtime/utils/rdf_canonicalize.py index e0fadae34a..8e28488d55 100644 --- a/packages/linkml_runtime/src/linkml_runtime/utils/rdf_canonicalize.py +++ b/packages/linkml_runtime/src/linkml_runtime/utils/rdf_canonicalize.py @@ -307,7 +307,8 @@ def canonicalize_rdf_graph( editing one part of a schema does not renumber unrelated blank nodes. Output is deterministic and isomorphic either way; only the choice of label changes. Off by default because enabling it relabels existing - output. + output. Has no effect on the rdflib fallback path (non-standard RDF), + which warns rather than silently ignoring the request. :return: Deterministic string serialization of the graph. """ ox_format = _FORMAT_MAP.get(output_format.lower()) @@ -338,6 +339,23 @@ def canonicalize_rdf_graph( RDFCanonicalizationWarning, stacklevel=2, ) + if diff_stable: + # Weisfeiler-Lehman refinement consumes canonical pyoxigraph quads, + # and this path exists precisely because pyoxigraph refused the + # graph, so there are none to refine. The fallback is deterministic + # but not diff-stable: say so rather than returning output that + # silently ignores the argument. ``shaclgen --include-annotations`` + # reaches this path, because an annotation tag without a ``:`` + # becomes a literal predicate. + warnings.warn( + "diff_stable=True was requested but this graph took the rdflib fallback, " + "which cannot apply Weisfeiler-Lehman blank-node labels. Output is " + "deterministic but NOT diff-stable: an unrelated edit may still renumber " + "blank nodes. Make the offending terms standard RDF (absolute IRIs, IRI " + "predicates) to get diff-stable labels.", + RDFCanonicalizationWarning, + stacklevel=2, + ) return _deterministic_fallback_serialize(graph, output_format) dataset = ox.Dataset() diff --git a/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py b/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py index 0ca79c3c90..1bc07583af 100644 --- a/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py +++ b/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py @@ -642,3 +642,28 @@ def test_diff_stable_reaches_every_rdf_generator(tmp_path, generator, kwargs): f"{generator.__name__} still emits RDFC-1.0 blank-node labels with diff_stable=True; " "the flag is probably not threaded into canonicalize_rdf_graph()" ) + + +def test_diff_stable_warns_instead_of_silently_no_opping_on_the_fallback(): + """A request the fallback cannot honour must be reported, not ignored. + + ``wl_relabel_quads`` consumes canonical pyoxigraph quads, and the rdflib + fallback exists precisely because pyoxigraph refused the graph. Returning + the same bytes for ``diff_stable=True`` and ``diff_stable=False`` without + saying so lets a caller believe the output is diff-stable when it is not. + """ + graph = _make_graph_with_bnodes() + # A relative IRI is non-standard RDF, so pyoxigraph rejects the graph and + # canonicalize_rdf_graph degrades to rdflib -- the same path that + # ``shaclgen --include-annotations`` takes via its literal predicates. + graph.add((URIRef("testing"), URIRef("http://example.com/p"), Literal("v"))) + + with pytest.warns(RDFCanonicalizationWarning, match="NOT diff-stable"): + stable = canonicalize_rdf_graph(graph, diff_stable=True) + + with pytest.warns(RDFCanonicalizationWarning): + plain = canonicalize_rdf_graph(graph, diff_stable=False) + + # The warning is the contract: the bytes really are identical, which is + # exactly why staying silent would be misleading. + assert stable == plain diff --git a/uv.lock b/uv.lock index d8cddaa9e1..e862cfc897 100644 --- a/uv.lock +++ b/uv.lock @@ -1085,6 +1085,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/84/d0/205d54408c08b13550c733c4b85429e7ead111c7f0014309637425520a9a/deprecated-1.3.1-py2.py3-none-any.whl", hash = "sha256:597bfef186b6f60181535a29fbe44865ce137a5079f295b479886c82729d5f3f", size = 11298, upload-time = "2025-10-30T08:19:00.758Z" }, ] +[[package]] +name = "diffable-rdf" +version = "0.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyoxigraph" }, + { name = "rdflib" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fe/f3/45771c536aea850f2884206349e35910bbabcaebeec890c5c0cc08b040f9/diffable_rdf-0.3.0.tar.gz", hash = "sha256:6fcf84d9323cd35fc75f423bd73836469cfad198ebb7e6ab86a8743724cd000d", size = 829395, upload-time = "2026-09-11T08:00:26.268Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/27/22/ef80dba96cd558f59d2ab225602bde318a925d437866379303e1f41a4e96/diffable_rdf-0.3.0-py3-none-any.whl", hash = "sha256:8756137423f9f28beacb5cd99ba8d66317328a0c936b1dbb4527831c7d9c6784", size = 43578, upload-time = "2026-09-11T08:00:24.542Z" }, +] + [[package]] name = "distlib" version = "0.4.0" @@ -2618,6 +2631,7 @@ dependencies = [ { name = "click" }, { name = "curies" }, { name = "deprecated" }, + { name = "diffable-rdf" }, { name = "hbreader" }, { name = "isodate", marker = "python_full_version < '3.11'" }, { name = "json-flattener" }, @@ -2650,6 +2664,7 @@ requires-dist = [ { name = "coverage", marker = "extra == 'dev'" }, { name = "curies", specifier = ">=0.14.6" }, { name = "deprecated" }, + { name = "diffable-rdf", specifier = ">=0.3.0" }, { name = "hbreader" }, { name = "isodate", marker = "python_full_version < '3.11'", specifier = ">=0.7.2,<1.0.0" }, { name = "json-flattener", specifier = ">=0.1.9" }, From a4d52530c7f367e274ae86ad70d76dfa64afffa5 Mon Sep 17 00:00:00 2001 From: Carlo van Driesten Date: Fri, 11 Sep 2026 13:45:48 +0200 Subject: [PATCH 3/4] build(deps): require diffable-rdf 0.4.0 0.4.0 carries graph.base through the library's rdflib fallback, verifying that every absolute IRI of the source survives a re-read rather than dropping the directive outright, and adds a diff_stable parameter to canonicalize_rdf_graph. The lock entry is written by hand because the workspace sets exclude-newer = "7 days", which filters any release younger than that from resolution; 0.3.0 was pinned the same way for the same reason, and both become resolvable normally on 2026-09-18. uv lock --check and uv sync --all-groups both accept the entry. https://github.com/ASCS-eV/diffable-rdf/releases/tag/v0.4.0 --- packages/linkml_runtime/pyproject.toml | 2 +- uv.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/linkml_runtime/pyproject.toml b/packages/linkml_runtime/pyproject.toml index ede7112692..c940eeac56 100644 --- a/packages/linkml_runtime/pyproject.toml +++ b/packages/linkml_runtime/pyproject.toml @@ -48,7 +48,7 @@ dependencies = [ "prefixmaps >=0.1.4", "curies>=0.14.6", "pyoxigraph>=0.5.11", - "diffable-rdf>=0.3.0", + "diffable-rdf>=0.4.0", "pydantic>=2.13.5,<3.0.0", "isodate >=0.7.2, <1.0.0; python_version < '3.11'", ] diff --git a/uv.lock b/uv.lock index e862cfc897..3c867304c8 100644 --- a/uv.lock +++ b/uv.lock @@ -1087,15 +1087,15 @@ wheels = [ [[package]] name = "diffable-rdf" -version = "0.3.0" +version = "0.4.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pyoxigraph" }, { name = "rdflib" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/fe/f3/45771c536aea850f2884206349e35910bbabcaebeec890c5c0cc08b040f9/diffable_rdf-0.3.0.tar.gz", hash = "sha256:6fcf84d9323cd35fc75f423bd73836469cfad198ebb7e6ab86a8743724cd000d", size = 829395, upload-time = "2026-09-11T08:00:26.268Z" } +sdist = { url = "https://files.pythonhosted.org/packages/86/df/1d71c0c984eac1cfd25531aabf84528941b01083c875da49d44882d1b5af/diffable_rdf-0.4.0.tar.gz", hash = "sha256:a84afaa10332e6a039b6ddcabf76bced3049e4da467c88ee786d0fa35218111e", size = 838040, upload-time = "2026-09-11T11:43:12.844Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/27/22/ef80dba96cd558f59d2ab225602bde318a925d437866379303e1f41a4e96/diffable_rdf-0.3.0-py3-none-any.whl", hash = "sha256:8756137423f9f28beacb5cd99ba8d66317328a0c936b1dbb4527831c7d9c6784", size = 43578, upload-time = "2026-09-11T08:00:24.542Z" }, + { url = "https://files.pythonhosted.org/packages/7f/49/b62ee864fe6769b759f04e3709fd79711b1cdf49223b1bb0e985e507ead3/diffable_rdf-0.4.0-py3-none-any.whl", hash = "sha256:c59c57429465f786fdbc8d38ddbb687da16748c5752f0c2ac4a77b5527aa9aed", size = 46126, upload-time = "2026-09-11T11:43:11.536Z" }, ] [[package]] @@ -2664,7 +2664,7 @@ requires-dist = [ { name = "coverage", marker = "extra == 'dev'" }, { name = "curies", specifier = ">=0.14.6" }, { name = "deprecated" }, - { name = "diffable-rdf", specifier = ">=0.3.0" }, + { name = "diffable-rdf", specifier = ">=0.4.0" }, { name = "hbreader" }, { name = "isodate", marker = "python_full_version < '3.11'", specifier = ">=0.7.2,<1.0.0" }, { name = "json-flattener", specifier = ">=0.1.9" }, From f7a79fec916c5ac3977cb263d579d88d6f85f988 Mon Sep 17 00:00:00 2001 From: Carlo van Driesten Date: Fri, 11 Sep 2026 11:08:44 +0200 Subject: [PATCH 4/4] test(rdf): record the canonicalizer gaps against the extracted library Asserts each correctness property against both linkml's copy and diffable_rdf, marking whichever implementation does not hold it as a strict xfail, so the file is a ratchet in both directions. Nine gaps run one way, two of them silent data corruption. One ran the other way -- the library dropped @base on the degraded path -- and that was the last property blocking delegation. diffable-rdf 0.4.0 fixed it, so that case now passes on both sides and carries no mark. --- .../test_rdf_canonicalize_defects.py | 353 ++++++++++++++++++ 1 file changed, 353 insertions(+) create mode 100644 tests/linkml_runtime/test_utils/test_rdf_canonicalize_defects.py diff --git a/tests/linkml_runtime/test_utils/test_rdf_canonicalize_defects.py b/tests/linkml_runtime/test_utils/test_rdf_canonicalize_defects.py new file mode 100644 index 0000000000..806be2e3eb --- /dev/null +++ b/tests/linkml_runtime/test_utils/test_rdf_canonicalize_defects.py @@ -0,0 +1,353 @@ +"""Correctness differences between linkml's RDF canonicalizer and the extracted library. + +``linkml_runtime.utils.rdf_canonicalize`` and +``diffable_rdf.canonicalize_rdf_graph`` are the same code: the module was +extracted into a standalone library at the maintainers' request +(linkml/linkml#3295). The two copies have since diverged, so each test here +asserts one correctness property against *both* implementations and marks the +one that does not hold it as a strict xfail. + +Every remaining gap runs one way -- the extracted copy received fixes the local +one did not -- and two of them are silent data corruption, where the output +parses cleanly and says something the input never said. That is worse than a +crash, because a generated artifact gets committed and reviewed on the +assumption that it means what the schema meant. + +One gap used to run the other way: the library dropped ``@base`` on the +degraded path. That was the last property blocking delegation, and +diffable-rdf 0.4.0 fixed it, so the case now passes on both sides and is kept +unmarked. Asserting both directions is what made the fix land where it +belongs. + +The xfails are strict, so this file is a ratchet in both directions: when a fix +lands on either side, its case starts passing and the suite fails until the +mark is removed. +""" + +import os +import subprocess +import sys +import textwrap +import warnings + +import diffable_rdf +import pytest +import rdflib +from rdflib import BNode, Graph, Literal, Namespace, URIRef +from rdflib.namespace import RDF + +from linkml_runtime.utils.rdf_canonicalize import canonicalize_rdf_graph as linkml_canonicalize + +EX = Namespace("http://example.org/") + +_IMPLEMENTATIONS = ( + (linkml_canonicalize, "linkml"), + (diffable_rdf.canonicalize_rdf_graph, "diffable-rdf"), +) + + +def _impls(**known_failures: str): + """Parametrize over both implementations, xfailing the ones named. + + :param known_failures: implementation id (``linkml`` / ``diffable_rdf``) + mapped to why that implementation does not hold the property. + """ + params = [] + for fn, ident in _IMPLEMENTATIONS: + reason = known_failures.get(ident.replace("-", "_")) + marks = [pytest.mark.xfail(strict=True, reason=reason)] if reason else [] + params.append(pytest.param(fn, id=ident, marks=marks)) + return params + + +def _apply(fn, graph, output_format="turtle"): + """Call an implementation, ignoring the warnings both emit on the fallback path.""" + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + return fn(graph, output_format) + + +# -------------------------------------------------------------------------- +# Silent data corruption +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "canonicalize", + _impls(linkml="passes graph.base to the serializer without checking it is safe to relativize against"), +) +def test_base_with_a_fragment_does_not_rewrite_every_iri(canonicalize): + """A base IRI ending in ``#`` must not change the graph's terms. + + Relativizing ``http://ex.org/d#a`` against base ``http://ex.org/d#`` gives + ``<#a>``, which is correct per RFC 3986 -- but rdflib's parser resolves a + fragment reference by concatenation and reads it back as + ``http://ex.org/d##a``. Every term of the graph silently changes, and the + output parses cleanly, so nothing reports it. + """ + graph = Graph(base="http://ex.org/d#") + graph.add((URIRef("http://ex.org/d#a"), EX.p, Literal("v"))) + + round_tripped = Graph() + round_tripped.parse(data=_apply(canonicalize, graph), format="turtle") + + assert {str(s) for s in round_tripped.subjects()} == {"http://ex.org/d#a"} + + +@pytest.mark.parametrize( + "canonicalize", + _impls(linkml="rdflib's collection syntax materializes a shared list tail once per referencing list"), +) +def test_a_shared_rdf_list_tail_is_not_duplicated(canonicalize): + """Two lists sharing a tail must not gain triples on the way out. + + rdflib's Turtle writer renders ``( ... )`` collection syntax per list, so a + tail referenced from two lists is written twice as two separate blank + nodes. The result asserts more than the input did. + """ + graph = Graph() + graph.bind("ex", EX) + tail = BNode() + graph.add((tail, RDF.first, Literal("shared"))) + graph.add((tail, RDF.rest, RDF.nil)) + for name in ("l1", "l2"): + head = BNode() + graph.add((EX[name], EX.list, head)) + graph.add((head, RDF.first, Literal(name))) + graph.add((head, RDF.rest, tail)) + # A relative IRI is non-standard RDF, so pyoxigraph refuses the graph and + # both implementations degrade to rdflib -- the path this defect lives on. + graph.add((URIRef("testing"), EX.p, Literal("forces-fallback"))) + + round_tripped = Graph() + round_tripped.parse(data=_apply(canonicalize, graph), format="turtle") + + assert len(round_tripped) == len(graph) + + +# -------------------------------------------------------------------------- +# Output that no parser will read +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "canonicalize", + _impls(linkml="the rdflib fallback writes N-Triples for a graph pyoxigraph already refused"), +) +def test_nt_output_either_parses_or_refuses(canonicalize): + """N-Triples admits only absolute IRIs, so a relative one must not be written. + + The fallback is taken *because* the graph holds a term pyoxigraph rejected. + Writing it out anyway produces a file that fails on line 1. Refusing with a + message naming the term and a format that can carry the graph is the fix; + returning unreadable text is not. + """ + graph = Graph() + graph.add((URIRef("testing"), EX.p, Literal("v"))) + + try: + serialized = _apply(canonicalize, graph, "nt") + except ValueError: + return # refused, with an explanation -- the correct outcome + + rdflib.Graph().parse(data=serialized, format="nt") + + +@pytest.mark.parametrize( + "canonicalize", + _impls(linkml="sorts fallback output with str.splitlines(), which breaks on U+2028 and five other characters"), +) +def test_a_unicode_line_separator_in_a_literal_survives(canonicalize): + """Sorting N-Triples lines must split on newlines only. + + N-Triples permits U+2028 raw inside a quoted literal, but + ``str.splitlines()`` breaks on it as well as on U+2029, U+0085, U+000B, + U+000C and U+001C-1E. One statement becomes two lines, the halves sort + independently, the separator is rewritten as a newline, and the document + no longer parses. + """ + graph = Graph() + graph.add((EX.s, EX.p, Literal("a\u2028b"))) + graph.add((URIRef("testing"), EX.p, Literal("forces-fallback"))) + + try: + serialized = _apply(canonicalize, graph, "nt") + except ValueError: + return # refused because of the relative IRI, before the sort + + assert "\u2028" in serialized + rdflib.Graph().parse(data=serialized, format="nt") + + +# -------------------------------------------------------------------------- +# Non-determinism, which is the property the module exists to provide +# -------------------------------------------------------------------------- + +_ACROSS_PROCESSES = textwrap.dedent( + """ + import sys, warnings + warnings.simplefilter("ignore") + from rdflib import BNode, Graph, Literal, Namespace, URIRef + from rdflib.namespace import RDF + + module, output_format, force_fallback = sys.argv[1], sys.argv[2], sys.argv[3] == "1" + if module == "linkml": + from linkml_runtime.utils.rdf_canonicalize import canonicalize_rdf_graph + else: + from diffable_rdf import canonicalize_rdf_graph + + EX = Namespace("http://example.org/") + g = Graph() + if force_fallback: + g.add((URIRef("testing"), EX.p, Literal("forces-fallback"))) + for i in range(12): + b = BNode() + g.add((EX["s%02d" % i], EX.child, b)) + g.add((b, EX.k, Literal("v%d" % i))) + g.add((b, RDF.type, EX.T)) + for host in "abcdefgh": + g.add((URIRef("http://%s.example/s" % host), + URIRef("http://%s.example/p" % host), + URIRef("http://%s.example/o" % host))) + sys.stdout.write(canonicalize_rdf_graph(g, output_format)) + """ +) + + +def _outputs_across_processes(module, output_format, force_fallback): + """Serialize the same graph in four processes with different hash seeds.""" + outputs = set() + for seed in ("0", "1", "12345", "999"): + completed = subprocess.run( + [sys.executable, "-c", _ACROSS_PROCESSES, module, output_format, "1" if force_fallback else "0"], + capture_output=True, + text=True, + encoding="utf-8", + env={**os.environ, "PYTHONHASHSEED": seed}, + check=False, + ) + if completed.returncode != 0: + pytest.fail(f"{module}/{output_format} failed:\n{completed.stderr[-2000:]}") + outputs.add(completed.stdout) + return outputs + + +_XML_TRAVERSAL = "degraded RDF/XML is ordered by rdflib's graph traversal, which follows set iteration order" +_NS_NAMES = "auto-generated ns1/ns2 prefix names are allocated in traversal order" +_JSONLD_UNMAPPED = "json-ld is absent from the format map, so it falls through to rdflib with no determinism guarantee" + + +@pytest.mark.parametrize( + ("module", "output_format", "force_fallback"), + [ + pytest.param( + "linkml", "xml", True, id="linkml-xml-fallback", marks=pytest.mark.xfail(strict=True, reason=_XML_TRAVERSAL) + ), + pytest.param("diffable_rdf", "xml", True, id="diffable-rdf-xml-fallback"), + pytest.param( + "linkml", + "turtle", + True, + id="linkml-turtle-fallback", + marks=pytest.mark.xfail(strict=True, reason=_NS_NAMES), + ), + pytest.param("diffable_rdf", "turtle", True, id="diffable-rdf-turtle-fallback"), + pytest.param( + "linkml", + "json-ld", + False, + id="linkml-jsonld", + marks=pytest.mark.xfail(strict=True, reason=_JSONLD_UNMAPPED), + ), + pytest.param("diffable_rdf", "json-ld", False, id="diffable-rdf-jsonld"), + ], +) +def test_output_is_byte_identical_across_processes(module, output_format, force_fallback): + """The same graph must serialize to the same bytes in any process. + + Three separate causes break this in the local copy. Degraded RDF/XML is + ordered by rdflib's own graph traversal, which follows set iteration order. + Auto-generated ``ns1``/``ns2`` prefix names are allocated in traversal order + too, so the *names* move even when the triples do not. And ``json-ld`` is + not in the format map at all, so it falls through to rdflib's serializer, + which carries no determinism guarantee. + """ + assert len(_outputs_across_processes(module, output_format, force_fallback)) == 1 + + +# -------------------------------------------------------------------------- +# Interface +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "canonicalize", + _impls(linkml="returns the serializer's own trailing whitespace, which differs by format"), +) +def test_every_format_ends_with_exactly_one_newline(canonicalize): + """A committed artifact should not depend on which format wrote it. + + Passing through whatever the serializer produced gives no trailing newline + for RDF/XML and one for Turtle, so POSIX text tools disagree about whether + the file has a last line. + """ + counts = {} + for output_format in ("turtle", "nt", "xml", "trig", "n3"): + graph = Graph() + graph.bind("ex", EX) + graph.add((EX.s, EX.p, Literal("v"))) + serialized = _apply(canonicalize, graph, output_format) + counts[output_format] = len(serialized) - len(serialized.rstrip("\n")) + + assert counts == dict.fromkeys(counts, 1) + + +@pytest.mark.parametrize( + "canonicalize", + _impls(linkml="Dataset is a Graph subclass, so the type check accepts it and the named graphs are flattened"), +) +def test_a_dataset_is_refused_rather_than_partly_serialized(canonicalize): + """A Dataset is a Graph subclass, so it is accepted and quietly flattened. + + Every named graph collapses into one document and the graph names are + dropped, which is a different dataset. The caller asked for something this + function cannot express and should be told so. + """ + dataset = rdflib.Dataset() + dataset.graph(URIRef("http://ex/g1")).add((EX.a, EX.p, Literal("in-g1"))) + dataset.graph(URIRef("http://ex/g2")).add((EX.b, EX.p, Literal("in-g2"))) + + with pytest.raises(TypeError): + _apply(canonicalize, dataset) + + +# -------------------------------------------------------------------------- +# A property both implementations now hold +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize("canonicalize", _impls()) +def test_base_survives_the_degraded_path(canonicalize): + """``@base`` must not disappear just because pyoxigraph refused the graph. + + ``rdflib_dumper.dumps(..., prefix_map={"@base": ...})`` is a supported way + to ask for a document whose IRIs are written relative to a base, and the + metamodel routinely produces graphs that fall back (a bare ``status: + testing`` on a ``uriorcurie`` slot serializes to the relative ````, + which pyoxigraph rejects). Losing the directive on exactly that path means + the feature works only for graphs that never needed the fallback. + + This ran the other way until diffable-rdf 0.4.0, which was the last + property blocking delegation. The case is kept because it is the one the + two implementations reach differently: linkml passes ``graph.base`` to the + serializer unconditionally, which is why it also fails + :func:`test_base_with_a_fragment_does_not_rewrite_every_iri` above, while + the library keeps the base only after confirming that re-reading the + result still yields every absolute IRI of the source. + """ + graph = Graph(base="http://example.org/default/") + graph.bind("ex", EX) + graph.add((EX.s, EX.p, Literal("v"))) + graph.add((URIRef("testing"), EX.p, Literal("forces-fallback"))) + + assert "@base ." in _apply(canonicalize, graph)