diff --git a/docs/generators/owl.rst b/docs/generators/owl.rst index 4b6f076fe8..87b0f402c4 100644 --- a/docs/generators/owl.rst +++ b/docs/generators/owl.rst @@ -311,6 +311,48 @@ Other examples translation of Biolink schema to OWL +Deterministic output +^^^^^^^^^^^^^^^^^^^^ + +``gen-owl`` output is deterministic by default. The graph is canonicalized with +`RDFC-1.0 `_ before serialization, so repeated +runs over the same schema -- and any two isomorphic graphs -- produce +byte-identical Turtle. No flag is needed, and checked-in artifacts do not churn +between runs. + +RDFC-1.0 numbers blank nodes sequentially (``_:c14n0``, ``_:c14n1``, ...) in +canonical order. That is stable for a fixed graph, but inserting a single +statement can shift the numbering of every blank node ordered after it, so an +unrelated one-line schema edit may rewrite large parts of the file. Pass +``--diff-stable`` to derive each label from the node's own neighbourhood +instead, so that only the blank nodes an edit actually touches are renamed: + +.. code:: bash + + gen-owl --diff-stable schema.yaml + +Both modes are deterministic and yield isomorphic graphs; only the choice of +label differs. ``--diff-stable`` is off by default because turning it on +relabels the blank nodes in existing output once. + +The same ``--diff-stable/--no-diff-stable`` option is available on ``gen-rdf``, +``gen-shacl`` and ``gen-shex``. + +Graphs that are not standard RDF -- literal predicates, as produced by +``gen-shacl`` in annotation mode, or relative IRIs such as the metamodel's +``bibo:status `` -- cannot be canonicalized under RDFC-1.0. Those fall +back to plain rdflib serialization, with blank-node labels canonicalized by +``rdflib.compare.to_canonical_graph``. Those labels are content-derived rather +than run-local, so the fallback remains reproducible across processes. It emits +an ``RDFCanonicalizationWarning``, and ``--diff-stable`` has no effect on that +path -- it warns rather than silently ignoring the request. + +Canonicalization itself is implemented by the +`diffable-rdf `_ library; +``linkml_runtime.utils.rdf_canonicalize.canonicalize_rdf_graph`` is a thin +adapter that re-emits the library's log warnings as Python warnings. + + Docs ---- diff --git a/packages/linkml/src/linkml/generators/owlgen.py b/packages/linkml/src/linkml/generators/owlgen.py index 50be266df0..7751eab34d 100644 --- a/packages/linkml/src/linkml/generators/owlgen.py +++ b/packages/linkml/src/linkml/generators/owlgen.py @@ -122,6 +122,22 @@ class OwlSchemaGenerator(Generator): """Suffix to add to the schema name to create the ontology URI, e.g. .owl.ttl""" # ObjectVars + diff_stable: bool = False + """Label blank nodes so that unrelated edits leave them untouched. + + Output is already deterministic: RDFC-1.0 guarantees that isomorphic + graphs serialize identically. It does not guarantee that *similar* + graphs serialize *similarly* — blank nodes are numbered ``c14nN`` in a + global order, so adding one class can renumber every blank node after + it and rewrite most of the file. + + When ``True``, blank-node labels are instead derived from each node's + own neighbourhood via Weisfeiler-Lehman refinement, so an edit relabels + only the blank nodes it actually touches. The output stays + deterministic and isomorphic either way; only the choice of label + changes. Off by default because enabling it relabels existing output. + """ + metadata_profile: MetadataProfile | None = None """Deprecated - use metadata_profiles.""" @@ -353,7 +369,7 @@ def serialize(self, **kwargs: Any) -> str: """ self.as_graph() fmt = "turtle" if self.format in ["owl", "ttl"] else self.format - return canonicalize_rdf_graph(self.graph, output_format=fmt) + return canonicalize_rdf_graph(self.graph, output_format=fmt, diff_stable=self.diff_stable) def add_metadata(self, e: Definition | PermissibleValue, uri: URIRef) -> None: """ @@ -1844,6 +1860,16 @@ def slot_owl_type(self, slot: SlotDefinition) -> URIRef: "specified language tag. Element-level in_language overrides this." ), ) +@click.option( + "--diff-stable/--no-diff-stable", + default=False, + show_default=True, + help=( + "Derive blank-node labels from each node's own neighbourhood so that " + "unrelated edits leave them unchanged. Output is deterministic either " + "way; this makes successive versions of a file diff cleanly." + ), +) @click.version_option(__version__, "-V", "--version") def cli(yamlfile: str, metadata_profile: str, **kwargs: Any) -> None: """Generate an OWL representation of a LinkML model diff --git a/packages/linkml/src/linkml/generators/rdfgen.py b/packages/linkml/src/linkml/generators/rdfgen.py index 2da1701787..052fcff12e 100644 --- a/packages/linkml/src/linkml/generators/rdfgen.py +++ b/packages/linkml/src/linkml/generators/rdfgen.py @@ -78,6 +78,22 @@ class RDFGenerator(Generator): uses_schemaloader = True # ObjectVars + diff_stable: bool = False + """Label blank nodes so that unrelated edits leave them untouched. + + Output is already deterministic: RDFC-1.0 guarantees that isomorphic + graphs serialize identically. It does not guarantee that *similar* + graphs serialize *similarly* — blank nodes are numbered ``c14nN`` in a + global order, so adding one class can renumber every blank node after + it and rewrite most of the file. + + When ``True``, blank-node labels are instead derived from each node's + own neighbourhood via Weisfeiler-Lehman refinement, so an edit relabels + only the blank nodes it actually touches. The output stays + deterministic and isomorphic either way; only the choice of label + changes. Off by default because enabling it relabels existing output. + """ + emit_metadata: bool = False context: list[str] = None original_schema: SchemaDefinition = None @@ -89,7 +105,7 @@ def __post_init__(self): def _data(self, g: Graph) -> str: fmt = "turtle" if self.format == "ttl" else self.format - return canonicalize_rdf_graph(g, output_format=fmt) + return canonicalize_rdf_graph(g, output_format=fmt, diff_stable=self.diff_stable) def end_schema(self, output: str | None = None, context: str = None, **_) -> str: gen = JSONLDGenerator( @@ -137,6 +153,16 @@ def end_schema(self, output: str | None = None, context: str = None, **_) -> str multiple=True, help="JSONLD context file (default: vendored meta.context.jsonld)", ) +@click.option( + "--diff-stable/--no-diff-stable", + default=False, + show_default=True, + help=( + "Derive blank-node labels from each node's own neighbourhood so that " + "unrelated edits leave them unchanged. Output is deterministic either " + "way; this makes successive versions of a file diff cleanly." + ), +) @click.version_option(__version__, "-V", "--version") def cli(yamlfile, **kwargs): """Generate an RDF representation of a LinkML model""" diff --git a/packages/linkml/src/linkml/generators/shaclgen.py b/packages/linkml/src/linkml/generators/shaclgen.py index 4731b9f0b8..5d42fea272 100644 --- a/packages/linkml/src/linkml/generators/shaclgen.py +++ b/packages/linkml/src/linkml/generators/shaclgen.py @@ -142,6 +142,22 @@ class ShaclGenerator(Generator): ignores any per-slot ``in_language``. """ + diff_stable: bool = False + """Label blank nodes so that unrelated edits leave them untouched. + + Output is already deterministic: RDFC-1.0 guarantees that isomorphic + graphs serialize identically. It does not guarantee that *similar* + graphs serialize *similarly* — blank nodes are numbered ``c14nN`` in a + global order, so adding one class can renumber every blank node after + it and rewrite most of the file. + + When ``True``, blank-node labels are instead derived from each node's + own neighbourhood via Weisfeiler-Lehman refinement, so an edit relabels + only the blank nodes it actually touches. The output stays + deterministic and isomorphic either way; only the choice of label + changes. Off by default because enabling it relabels existing output. + """ + emit_rules: bool = True """Emit ``sh:sparql`` constraints from LinkML ``rules:`` blocks. @@ -196,7 +212,7 @@ def generate_header(self) -> str: def serialize(self, **args) -> str: g = self.as_graph() fmt = "turtle" if self.format in ["owl", "ttl"] else self.format - return canonicalize_rdf_graph(g, output_format=fmt) + return canonicalize_rdf_graph(g, output_format=fmt, diff_stable=self.diff_stable) def as_graph(self) -> Graph: sv = self.schemaview @@ -929,6 +945,16 @@ def add_simple_data_type(func: Callable, r: ElementName) -> None: "sh:NodeShape. Use --no-emit-rules to suppress rule generation." ), ) +@click.option( + "--diff-stable/--no-diff-stable", + default=False, + show_default=True, + help=( + "Derive blank-node labels from each node's own neighbourhood so that " + "unrelated edits leave them unchanged. Output is deterministic either " + "way; this makes successive versions of a file diff cleanly." + ), +) @click.version_option(__version__, "-V", "--version") def cli(yamlfile, **args): """Generate SHACL turtle from a LinkML model""" diff --git a/packages/linkml/src/linkml/generators/shexgen.py b/packages/linkml/src/linkml/generators/shexgen.py index 40a93ffbc9..7c43b13bc0 100644 --- a/packages/linkml/src/linkml/generators/shexgen.py +++ b/packages/linkml/src/linkml/generators/shexgen.py @@ -40,6 +40,22 @@ class ShExGenerator(Generator): uses_schemaloader = True # ObjectVars + diff_stable: bool = False + """Label blank nodes so that unrelated edits leave them untouched. + + Output is already deterministic: RDFC-1.0 guarantees that isomorphic + graphs serialize identically. It does not guarantee that *similar* + graphs serialize *similarly* — blank nodes are numbered ``c14nN`` in a + global order, so adding one class can renumber every blank node after + it and rewrite most of the file. + + When ``True``, blank-node labels are instead derived from each node's + own neighbourhood via Weisfeiler-Lehman refinement, so an edit relabels + only the blank nodes it actually touches. The output stays + deterministic and isomorphic either way; only the choice of label + changes. Off by default because enabling it relabels existing output. + """ + shex: Schema = field(default_factory=lambda: Schema()) # ShEx Schema being generated shapes: list = field(default_factory=lambda: []) shape: Shape | None = None # Current shape being defined @@ -177,7 +193,7 @@ def end_schema(self, output: str | None = None, **_) -> str: g = Graph() g.parse(data=shex, format="json-ld", version="1.1") g.bind("owl", OWL) - shex = canonicalize_rdf_graph(g, output_format="turtle") + shex = canonicalize_rdf_graph(g, output_format="turtle", diff_stable=self.diff_stable) elif self.format == "shex": g = Graph() self.namespaces.load_graph(g) @@ -258,6 +274,16 @@ def _get_subproperty_values(self, slot: SlotDefinition) -> list: help="If --expand-subproperty-of (default), slots with subproperty_of will generate NodeConstraint " "values containing all slot descendants. Use --no-expand-subproperty-of to disable this behavior.", ) +@click.option( + "--diff-stable/--no-diff-stable", + default=False, + show_default=True, + help=( + "Derive blank-node labels from each node's own neighbourhood so that " + "unrelated edits leave them unchanged. Output is deterministic either " + "way; this makes successive versions of a file diff cleanly." + ), +) @click.version_option(__version__, "-V", "--version") def cli(yamlfile, **args): """Generate a ShEx Schema for a LinkML model""" diff --git a/packages/linkml_runtime/pyproject.toml b/packages/linkml_runtime/pyproject.toml index 1ae9da3083..c940eeac56 100644 --- a/packages/linkml_runtime/pyproject.toml +++ b/packages/linkml_runtime/pyproject.toml @@ -48,6 +48,7 @@ dependencies = [ "prefixmaps >=0.1.4", "curies>=0.14.6", "pyoxigraph>=0.5.11", + "diffable-rdf>=0.4.0", "pydantic>=2.13.5,<3.0.0", "isodate >=0.7.2, <1.0.0; python_version < '3.11'", ] diff --git a/packages/linkml_runtime/src/linkml_runtime/utils/rdf_canonicalize.py b/packages/linkml_runtime/src/linkml_runtime/utils/rdf_canonicalize.py index 31c5580348..47e8dc2682 100644 --- a/packages/linkml_runtime/src/linkml_runtime/utils/rdf_canonicalize.py +++ b/packages/linkml_runtime/src/linkml_runtime/utils/rdf_canonicalize.py @@ -1,48 +1,41 @@ -"""Deterministic RDF serialization via pyoxigraph RDFC-1.0 canonicalization. - -This module provides a function to canonicalize an rdflib Graph using -pyoxigraph's RDFC-1.0 implementation, producing deterministic output -with stable blank node labels and sorted triples. - -**Known limitations:** - -1. **xsd:string normalization**: pyoxigraph follows RDF 1.1, where plain - string literals and ``"text"^^xsd:string`` are identical. The output - will never contain explicit ``^^xsd:string`` annotations. Code that - re-parses the output with rdflib will see ``Literal("x")`` (datatype - ``None``) rather than ``Literal("x", datatype=XSD.string)``. - -2. **Non-standard RDF**: Graphs with literal predicates (e.g. SHACL - annotation mode) or relative IRIs (e.g. the metamodel's - ``bibo:status ``) are rejected by pyoxigraph. This function - falls back to rdflib's serializer for such graphs, but still - canonicalizes blank-node labels (and sorts line-oriented formats) so - the fallback output remains deterministic across processes. - -3. **Numeric short forms**: pyoxigraph uses Turtle short forms for - ``xsd:integer`` (``42``), ``xsd:boolean`` (``true``), and - ``xsd:decimal`` (``1.23``). rdflib parses these back with the - correct datatype, so this is lossless. - -4. **Base IRI / prefix collision**: When a graph has ``@base`` and a - prefix whose namespace equals the base IRI (e.g. rdflib's auto-bound - ``base:`` prefix), pyoxigraph emits CURIEs like ``base:label`` that - rdflib rejects. We skip such prefixes during serialization. - -5. **Trailing escaped dot in PN_LOCAL**: pyoxigraph emits CURIEs like - ``prefix:local\\.`` for IRIs whose local part ends with ``.``. This - is valid Turtle (PN_LOCAL_ESC), but rdflib's notation3 parser rejects - it because it conflicts with the statement-terminator dot. We - post-process the output to expand such CURIEs to full ```` form. +"""Deterministic RDF serialization, delegated to the ``diffable-rdf`` library. + +This module used to carry the implementation. It was extracted into +`diffable-rdf `_ at the maintainers' +request (linkml/linkml#3295: *"the implementation should live elsewhere ... +independent rdflib sidecar library?"*), and this module is now a thin adapter +over it. :func:`canonicalize_rdf_graph` keeps its signature, so nothing that +imports it needs to change. + +The adapter exists for one reason: the two projects report degraded output +differently. The library logs through the ``logging`` module, which is silent +unless the application configured a handler. linkml deliberately uses +:func:`warnings.warn` so a schema author running ``gen-owl`` sees that the +output took a fallback path without having to opt in to logging first. So the +library's warnings are captured and re-emitted as +:class:`RDFCanonicalizationWarning`. + +What the library does, in short: the graph is transferred to pyoxigraph via +N-Triples, canonicalized with RDFC-1.0, sorted, and serialized back. Graphs +pyoxigraph refuses -- literal predicates from SHACL annotation mode, relative +IRIs such as the metamodel's ``bibo:status `` -- fall back to rdflib +with blank-node labels canonicalized by :func:`rdflib.compare.to_canonical_graph`, +which is content-derived rather than run-local, so the fallback is still +reproducible across processes. The full contract, including the limitations +that used to be listed here (``xsd:string`` normalization, numeric short forms, +base/prefix collisions, ``PN_LOCAL`` escaping), is documented in the library's +``docs/api.md``. + +Delegating also adopts nine correctness fixes the extracted copy received and +this one never did, two of them silent data corruption. Each is pinned by a +test in ``tests/linkml_runtime/test_utils/test_rdf_canonicalize_defects.py``. """ -import io -import re +import logging import warnings -import pyoxigraph as ox import rdflib -from rdflib.compare import to_canonical_graph +from diffable_rdf import canonicalize_rdf_graph as _canonicalize_rdf_graph class RDFCanonicalizationWarning(UserWarning): @@ -55,358 +48,93 @@ class RDFCanonicalizationWarning(UserWarning): """ -# Mapping from rdflib/LinkML format strings to pyoxigraph RdfFormat objects. -_FORMAT_MAP: dict[str, ox.RdfFormat] = { - "turtle": ox.RdfFormat.TURTLE, - "ttl": ox.RdfFormat.TURTLE, - "nt": ox.RdfFormat.N_TRIPLES, - "ntriples": ox.RdfFormat.N_TRIPLES, - "n-triples": ox.RdfFormat.N_TRIPLES, - "nt11": ox.RdfFormat.N_TRIPLES, - "nquads": ox.RdfFormat.N_QUADS, - "n-quads": ox.RdfFormat.N_QUADS, - "xml": ox.RdfFormat.RDF_XML, - "rdf/xml": ox.RdfFormat.RDF_XML, - "trig": ox.RdfFormat.TRIG, - "n3": ox.RdfFormat.N3, -} - -# Formats that support prefix declarations. -_PREFIX_FORMATS = frozenset({ox.RdfFormat.TURTLE, ox.RdfFormat.TRIG, ox.RdfFormat.N3, ox.RdfFormat.RDF_XML}) - -# Line-oriented formats (one triple/quad per line) whose fallback output must -# be sorted, because rdflib does not emit them in a stable order even after -# blank-node canonicalization. -_LINE_ORIENTED_FORMATS = frozenset({"nt", "ntriples", "n-triples", "nt11", "nquads", "n-quads"}) +_LIBRARY_LOGGER = "diffable_rdf" -def _deterministic_fallback_serialize(graph: rdflib.Graph, output_format: str) -> str: - """Serialize a graph that pyoxigraph cannot canonicalize, deterministically. +class _DegradedPathWarnings(logging.Handler): + """Collect the library's warning logs and re-emit them as Python warnings. - pyoxigraph rejects some graphs that rdflib accepts -- notably graphs - containing relative IRIs (e.g. the metamodel's ``bibo:status ``) - or literal predicates (SHACL annotation mode). A plain - ``graph.serialize()`` for such graphs is *not* reproducible across - processes: rdflib assigns blank-node labels non-deterministically, so the - structure and grouping of the output varies run to run. + Records are collected during the call and re-emitted in :meth:`__exit__` + rather than from :meth:`emit`. Warning at the moment the record arrives + would put eight ``logging`` frames plus an unknown number of library frames + between :func:`warnings.warn` and the caller, and the library's depth + differs per message, so no single ``stacklevel`` could point at the caller. + Re-emitting from ``__exit__`` makes the distance a constant. - To degrade gracefully instead of silently emitting non-deterministic - output, we canonicalize blank-node labels with rdflib's own - isomorphism-based canonicalization (:func:`rdflib.compare.to_canonical_graph`, - which uses a content-derived hash, not run-local ids) and preserve the - original prefix and base bindings that the canonical graph drops. For - line-oriented formats we additionally sort the serialized lines, since - rdflib does not emit N-Triples/N-Quads in a stable order. + The handler is attached to the library's package logger rather than the + root logger. The library logs under module-level children such as + ``diffable_rdf.canonicalize``, which propagate to the package logger, so + attaching there catches every module without naming any of them. - Relative IRIs are preserved verbatim (not resolved against the base): the - goal is deterministic output, and silently rewriting ```` into an - absolute IRI would mask what is really a data problem in the source graph. - - :param graph: The rdflib Graph that pyoxigraph could not parse. - :param output_format: Target serialization format (e.g. ``"turtle"``, ``"nt"``). - :return: Deterministic string serialization of the graph. + ``propagate`` is left alone: a caller who *did* configure logging still + gets the record through their own handlers, and seeing the message once + per mechanism is a better trade than suppressing a handler they asked for. """ - canonical = to_canonical_graph(graph) - # to_canonical_graph builds a fresh graph without the source's namespace - # bindings; rebind them so the output does not fall back to rdflib's - # non-deterministic auto-generated ``ns1:``/``ns2:`` prefixes. - for prefix, namespace in graph.namespace_manager.namespaces(): - canonical.namespace_manager.bind(prefix, namespace, replace=True) - canonical.base = graph.base - serialized = canonical.serialize(format=output_format) - if output_format.lower() in _LINE_ORIENTED_FORMATS: - lines = [line for line in serialized.splitlines() if line.strip()] - return "\n".join(sorted(lines)) + "\n" - return serialized - -# Characters that may appear escaped in a Turtle PN_LOCAL via PN_LOCAL_ESC. -_PN_LOCAL_ESC_UNESCAPE = re.compile(r"\\([_~.\-!$&'()*+,;=/?#@%])") - - -_CURIE_TRAILING_DOT_PATTERN = re.compile( - r"(?\"'\[\]]*?\\\.)" - r"(?=\s)" -) - - -def _iter_turtle_structural_spans(turtle_text: str): - """Yield ``(start, end, is_structural)`` spans of a Turtle string. - - Structural spans are regions outside of string literals, IRI refs, and - line comments — i.e. the only places where a CURIE can syntactically - appear. Non-structural spans (literal contents, IRI bodies, comments) - are yielded as-is so they round-trip unchanged. - - Handles single- and triple-quoted literals with backslash escapes for - both ``"`` and ``'`` delimiters, ``<...>`` IRI refs, and ``#...`` line - comments. - """ - n = len(turtle_text) - i = 0 - while i < n: - ch = turtle_text[i] - if ch in ('"', "'"): - # String literal — find matching delimiter, respecting triple- - # quote form and backslash escapes. - delim = ch - triple = turtle_text[i : i + 3] == delim * 3 - end_marker = delim * 3 if triple else delim - j = i + len(end_marker) - while j < n: - if turtle_text[j] == "\\" and j + 1 < n: - j += 2 - continue - if turtle_text[j : j + len(end_marker)] == end_marker: - j += len(end_marker) - break - j += 1 - yield i, j, False - i = j - elif ch == "<": - # IRI ref — find closing '>' on the same logical line. - j = turtle_text.find(">", i + 1) - if j == -1: - yield i, n, False - i = n - else: - yield i, j + 1, False - i = j + 1 - elif ch == "#": - # Line comment — to end of line. - j = turtle_text.find("\n", i) - j = n if j == -1 else j - yield i, j, False - i = j - else: - # Structural region — accumulate until the next literal / IRI / - # comment opener. Treat ``\X`` as a PN_LOCAL_ESC escape (two - # chars) so that ``\#`` and ``\.`` inside a CURIE local part - # don't trigger comment / boundary handling. - start = i - while i < n: - c = turtle_text[i] - if c == "\\" and i + 1 < n: - i += 2 - continue - if c in ('"', "'", "<", "#"): - break - i += 1 - yield start, i, True - - -def _expand_trailing_dot_curies(turtle_text: str, prefixes: dict[str, str]) -> str: - """Replace CURIEs whose local part ends in ``\\.`` with full ```` form. - - rdflib's notation3 parser rejects PN_LOCAL ending in an escaped dot - even though Turtle permits it (PN_LOCAL_ESC). pyoxigraph emits this - form for IRIs ending in ``.`` (e.g. ``biolink:StrandEnum#.``). We - rewrite each such CURIE to its expanded ```` form so the output - round-trips through rdflib. - - The rewrite is applied only to *structural* spans of the document — - string literals, IRI refs, and comments are left untouched. This is - what prevents the regex from mangling CURIE-shaped substrings that - happen to appear inside a literal value. - """ - if not prefixes: - return turtle_text - - def replace(match: re.Match[str]) -> str: - prefix = match.group(1) - local_escaped = match.group(2) - namespace = prefixes.get(prefix) - if namespace is None: - return match.group(0) - local = _PN_LOCAL_ESC_UNESCAPE.sub(r"\1", local_escaped) - return f"<{namespace}{local}>" - - parts: list[str] = [] - for start, end, is_structural in _iter_turtle_structural_spans(turtle_text): - chunk = turtle_text[start:end] - if is_structural: - chunk = _CURIE_TRAILING_DOT_PATTERN.sub(replace, chunk) - parts.append(chunk) - return "".join(parts) - - -def _iri_terms(triples: list["ox.Triple"]) -> set[str]: - """Return the set of IRI strings appearing anywhere in ``triples``. - - Walks subjects, predicates, non-literal objects, and literal datatypes. - Used to filter the prefix dict down to namespaces that are actually - referenced by the canonicalized graph, so the output isn't padded with - unused ``@prefix`` declarations. - """ - iris: set[str] = set() - for t in triples: - for term in (t.subject, t.predicate, t.object): - if isinstance(term, ox.NamedNode): - iris.add(term.value) - elif isinstance(term, ox.Literal): - dt = term.datatype - if dt is not None: - iris.add(dt.value) - return iris - - -def _filter_prefixes_to_used(prefixes: dict[str, str], used_iris: set[str]) -> dict[str, str]: - """Drop prefix bindings whose namespace is not a prefix of any used IRI. - - A prefix is kept if at least one IRI in ``used_iris`` starts with its - namespace string. Parent-namespace matches are honored (e.g. a prefix - bound to ``http://schema.org/`` is kept when ``http://schema.org/Person`` - appears in the graph). - """ - return {prefix: ns for prefix, ns in prefixes.items() if any(iri.startswith(ns) for iri in used_iris)} - - -def _is_safe_prefix_iri(iri: str) -> bool: - """Check whether a namespace IRI is safe for prefix serialization. - - pyoxigraph rejects IRIs with invalid code-points (e.g. double ``#``), - and rdflib's Turtle parser cannot round-trip CURIEs whose namespace - contains query parameters or fragments in unexpected positions. This - function returns ``False`` for such IRIs so they can be skipped during - prefix collection. - """ - # A namespace IRI should end with '/' or '#'. If '#' appears - # *before* the final character, the IRI contains an embedded - # fragment which produces unusable CURIEs. - if "#" in iri[:-1]: - return False - # Query parameters in namespace IRIs produce CURIEs that rdflib - # cannot parse back. - if "?" in iri: - return False - return True + def __init__(self) -> None: + super().__init__(level=logging.WARNING) + self._messages: list[str] = [] + self._logger = logging.getLogger(_LIBRARY_LOGGER) + self._previous_level = logging.NOTSET + + def emit(self, record: logging.LogRecord) -> None: + """Record one formatted message for re-emission on exit.""" + self._messages.append(record.getMessage()) + + def __enter__(self) -> "_DegradedPathWarnings": + """Attach to the library's logger, raising its level if it is silent.""" + self._previous_level = self._logger.level + if not self._logger.isEnabledFor(logging.WARNING): + self._logger.setLevel(logging.WARNING) + self._logger.addHandler(self) + return self + + def __exit__(self, *exc_info: object) -> None: + """Detach, restore the logger, and re-emit what was collected. + + Runs on the exception path too, so a caller who gets an exception still + learns about the degradation that preceded it. + """ + self._logger.removeHandler(self) + self._logger.setLevel(self._previous_level) + for message in self._messages: + # 3 frames: warnings.warn, this method, and the ``with`` statement + # in canonicalize_rdf_graph -- so the warning lands on its caller. + warnings.warn(message, RDFCanonicalizationWarning, stacklevel=3) def canonicalize_rdf_graph( graph: rdflib.Graph, output_format: str = "turtle", + diff_stable: bool = False, ) -> str: """Serialize an rdflib Graph deterministically using RDFC-1.0 canonicalization. - The graph is transferred to pyoxigraph via N-Triples, canonicalized - with RDFC-1.0, sorted, and serialized back to the requested format. - Prefix bindings from the rdflib Graph are preserved in the output - for formats that support them (Turtle, TriG, N3, RDF/XML). + The graph is transferred to pyoxigraph via N-Triples, canonicalized with + RDFC-1.0, sorted, and serialized back to the requested format. Prefix + bindings from the rdflib Graph are preserved in the output for formats that + support them (Turtle, TriG, N3, RDF/XML). - Falls back to plain rdflib serialization for unsupported formats or - graphs containing non-standard RDF (e.g. literal predicates). + Falls back to plain rdflib serialization for unsupported formats or graphs + containing non-standard RDF (e.g. literal predicates), warning with + :class:`RDFCanonicalizationWarning` so that the caller knows the output is + less strongly guaranteed than usual. :param graph: The rdflib Graph to serialize. :param output_format: Target serialization format (e.g. ``"turtle"``, ``"nt"``). + :param diff_stable: Derive blank-node labels from each node's own + neighbourhood instead of RDFC-1.0's global ``c14nN`` numbering, so that + editing one part of a schema does not renumber unrelated blank nodes. + Output is deterministic and isomorphic either way; only the choice of + label changes. Off by default because enabling it relabels existing + output. Has no effect on the rdflib fallback path (non-standard RDF), + which warns rather than silently ignoring the request. :return: Deterministic string serialization of the graph. + :raises ValueError: If the graph cannot be serialized to ``output_format`` + in a form that parses back -- notably a line-oriented format asked to + write a relative IRI, which N-Triples forbids. """ - ox_format = _FORMAT_MAP.get(output_format.lower()) - if ox_format is None: - warnings.warn( - f"pyoxigraph does not support format {output_format!r}; falling back to " - "rdflib serializer. Output will not be deterministically canonicalized.", - RDFCanonicalizationWarning, - stacklevel=2, - ) - return graph.serialize(format=output_format) - - # 1. Transfer rdflib graph to pyoxigraph via N-Triples. - nt_data = graph.serialize(format="nt") - nt_bytes = nt_data.encode("utf-8") if isinstance(nt_data, str) else nt_data - - # 2. Parse into pyoxigraph and build a Dataset for canonicalization. - # Fall back to rdflib if the graph contains non-standard RDF - # (e.g. literal predicates from annotations) that pyoxigraph rejects. - try: - triples = list(ox.parse(io.BytesIO(nt_bytes), format=ox.RdfFormat.N_TRIPLES)) - except SyntaxError: - warnings.warn( - "Graph contains non-standard RDF (e.g. relative IRIs or literal predicates) " - "that pyoxigraph cannot parse; falling back to rdflib. Output is still " - "deterministic (blank-node labels are canonicalized via rdflib) but is not " - "canonicalized with pyoxigraph RDFC-1.0.", - RDFCanonicalizationWarning, - stacklevel=2, - ) - return _deterministic_fallback_serialize(graph, output_format) - - dataset = ox.Dataset() - for triple in triples: - dataset.add(ox.Quad(triple.subject, triple.predicate, triple.object, ox.DefaultGraph())) - - # 3. Canonicalize blank node labels with RDFC-1.0. - dataset.canonicalize(ox.CanonicalizationAlgorithm.RDFC_1_0) - - # 4. Sort triples for deterministic ordering. - # RDFC-1.0 stabilizes blank-node labels but pyoxigraph's Dataset - # iteration order is not sorted and varies across processes (verified - # empirically against pyoxigraph 0.5.8). The explicit string-key sort - # is load-bearing for byte-identical output across runs; see - # tests/linkml_runtime/test_utils/test_rdf_canonicalize.py::test_sort_is_load_bearing. - quads = list(dataset) - sorted_triples = sorted( - (ox.Triple(q.subject, q.predicate, q.object) for q in quads), - key=lambda t: (str(t.subject), str(t.predicate), str(t.object)), - ) - - # 5. Collect prefixes for formats that support them. - base_iri = str(graph.base) if graph.base else None - prefixes: dict[str, str] | None = None - if ox_format in _PREFIX_FORMATS: - prefixes = {} - for prefix, namespace in graph.namespace_manager.namespaces(): - if not prefix: # skip empty prefix (base) - continue - ns_str = str(namespace) - # Skip prefixes whose namespace matches the base IRI to avoid - # pyoxigraph emitting CURIEs like `base:label` that conflict - # with the @base directive. - if base_iri and ns_str == base_iri: - continue - # Skip namespace IRIs that pyoxigraph rejects or that produce - # CURIEs rdflib cannot round-trip. Valid namespace IRIs for - # prefix use should end with '/' or '#' and contain no query - # parameters or fragment-like characters in the middle. - if not _is_safe_prefix_iri(ns_str): - continue - prefixes[str(prefix)] = ns_str - # Drop prefix bindings whose namespace is not referenced by any IRI - # in the graph. This prevents the rdflib NamespaceManager's default - # bindings (~30 well-known vocabularies) from being emitted into - # every output file regardless of whether the schema actually uses - # them. - prefixes = _filter_prefixes_to_used(prefixes, _iri_terms(sorted_triples)) - used_prefixes = prefixes - try: - result_bytes = ox.serialize( - sorted_triples, - format=ox_format, - prefixes=prefixes, - base_iri=base_iri, - ) - except ValueError as e: - # pyoxigraph 0.5.x reports rejected prefix IRIs with a message - # that begins with "Invalid prefix" (verified empirically). Only - # swallow that case and retry without prefixes — any other - # ValueError (e.g. invalid base IRI, or an unrelated future - # serializer bug) must propagate so it surfaces as a stack trace - # rather than silently dropping all prefix declarations. - if not str(e).startswith("Invalid prefix"): - raise - warnings.warn( - f"pyoxigraph rejected one or more prefix IRIs ({e}); serializing without " - "prefix declarations. Output remains canonicalized but is more verbose.", - RDFCanonicalizationWarning, - stacklevel=2, - ) - result_bytes = ox.serialize( - sorted_triples, - format=ox_format, - ) - used_prefixes = None - result = result_bytes.decode("utf-8") - if ox_format in _PREFIX_FORMATS and used_prefixes: - result = _expand_trailing_dot_curies(result, used_prefixes) - return result + # Frames between warnings.warn and this function's caller are counted in + # _DegradedPathWarnings.__exit__, which is where the re-emission happens. + with _DegradedPathWarnings(): + return _canonicalize_rdf_graph(graph, output_format=output_format, diff_stable=diff_stable) diff --git a/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py b/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py index 63dbf095a1..5a813729dd 100644 --- a/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py +++ b/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py @@ -1,10 +1,14 @@ """Tests for deterministic RDF serialization via pyoxigraph RDFC-1.0.""" +import json +import logging import os import re import subprocess import sys import textwrap +import warnings +from pathlib import Path import pyoxigraph as ox import pytest @@ -12,7 +16,6 @@ from rdflib import BNode, Graph, Literal, URIRef from rdflib.namespace import RDF -from linkml_runtime.utils import rdf_canonicalize as rdf_canon_mod from linkml_runtime.utils.rdf_canonicalize import ( RDFCanonicalizationWarning, canonicalize_rdf_graph, @@ -304,13 +307,13 @@ def fake_serialize(*args, **kwargs): raise ValueError("Invalid prefix bad IRI 'http://example.com/ has space/', Invalid IRI code point ' '") return real_serialize(*args, **kwargs) - monkeypatch.setattr(rdf_canon_mod.ox, "serialize", fake_serialize) + monkeypatch.setattr(ox, "serialize", fake_serialize) g = Graph() g.bind("ex", "http://example.com/") g.add((URIRef("http://example.com/a"), RDF.type, URIRef("http://example.com/Thing"))) - with pytest.warns(RDFCanonicalizationWarning, match="rejected one or more prefix IRIs"): + with pytest.warns(RDFCanonicalizationWarning, match="rejected the prefix or base IRIs"): ttl = canonicalize_rdf_graph(g, output_format="turtle") # Fallback retry should succeed and the output still round-trips. g2 = Graph() @@ -319,22 +322,74 @@ def fake_serialize(*args, **kwargs): def test_unsupported_format_warns_and_falls_back(): - """A format pyoxigraph doesn't know about falls back to rdflib with a warning.""" + """A format the canonicalizer doesn't know about falls back to rdflib with a warning. + + ``longturtle`` is an rdflib serializer plugin with no pyoxigraph + equivalent, so it can only be produced by delegating to rdflib, and + rdflib orders that output by graph traversal. The warning is what tells + the caller the determinism guarantee does not extend to this format. + """ g = Graph() g.bind("ex", "http://example.com/") g.add((URIRef("http://example.com/a"), RDF.type, URIRef("http://example.com/Thing"))) - with pytest.warns(RDFCanonicalizationWarning, match="does not support format"): - result = canonicalize_rdf_graph(g, output_format="json-ld") + with pytest.warns(RDFCanonicalizationWarning, match="not one of the formats"): + result = canonicalize_rdf_graph(g, output_format="longturtle") # The fallback should still produce valid output in the requested format. assert result +def test_json_ld_is_canonicalized_rather_than_handed_to_rdflib(): + """``json-ld`` gets the determinism guarantee, not a fallback warning. + + rdflib's JSON-LD serializer emits objects in traversal order, so this used + to be an unsupported format that warned and returned whatever rdflib felt + like. It is now produced deterministically, which is why no warning is due. + """ + g = _make_graph_with_bnodes() + with warnings.catch_warnings(): + warnings.simplefilter("error", RDFCanonicalizationWarning) + result = canonicalize_rdf_graph(g, output_format="json-ld") + assert json.loads(result) + + +def test_a_rejected_base_iri_is_recovered_rather_than_fatal(monkeypatch): + """A base pyoxigraph refuses costs the base directive, not the document. + + rdflib accepts a relative or otherwise unusable ``base``, and pyoxigraph + then refuses it at serialization time. Dropping the base and retrying + yields a correct document; failing the whole call over a directive that is + optional in every output format does not. + """ + real_serialize = ox.serialize + calls = {"n": 0} + + def fake_serialize(*args, **kwargs): + calls["n"] += 1 + if calls["n"] == 1: + raise ValueError("Invalid base IRI 'broken', Invalid IRI code point ' '") + return real_serialize(*args, **kwargs) + + monkeypatch.setattr(ox, "serialize", fake_serialize) + + g = Graph() + g.bind("ex", "http://example.com/") + g.add((URIRef("http://example.com/a"), RDF.type, URIRef("http://example.com/Thing"))) + + with pytest.warns(RDFCanonicalizationWarning, match="rejected the prefix or base IRIs"): + ttl = canonicalize_rdf_graph(g, output_format="turtle") + + g2 = Graph() + g2.parse(data=ttl, format="turtle") + assert rdflib.compare.isomorphic(g, g2) + + def test_unrelated_value_error_propagates(monkeypatch): - """Any ``ValueError`` whose message doesn't begin with ``Invalid prefix`` is re-raised. + """A ``ValueError`` that is not about prefixes or the base must be re-raised. - Guards against future regressions where a pyoxigraph serializer bug - raises a different ``ValueError`` and gets silently swallowed by the - invalid-prefix fallback path. + The retry-without-prefixes path exists for two specific pyoxigraph + complaints. Widening it to every ``ValueError`` would let a serializer bug + be papered over by a retry that happens to succeed, and the caller would + get a plausible-looking document with no indication anything went wrong. """ real_serialize = ox.serialize calls = {"count": 0} @@ -343,16 +398,16 @@ def fake_serialize(*args, **kwargs): calls["count"] += 1 # First call is the prefixed serialize — raise an unrelated ValueError. if calls["count"] == 1: - raise ValueError("Invalid base IRI 'broken', Invalid IRI code point ' '") + raise ValueError("BUG: serializer state corrupted") return real_serialize(*args, **kwargs) - monkeypatch.setattr(rdf_canon_mod.ox, "serialize", fake_serialize) + monkeypatch.setattr(ox, "serialize", fake_serialize) g = Graph() g.bind("ex", "http://example.com/") g.add((URIRef("http://example.com/a"), RDF.type, URIRef("http://example.com/Thing"))) - with pytest.raises(ValueError, match="Invalid base IRI"): + with pytest.raises(ValueError, match="BUG: serializer state corrupted"): canonicalize_rdf_graph(g, output_format="turtle") @@ -455,7 +510,7 @@ def test_fallback_preserves_relative_iri(): assert "" in result -@pytest.mark.parametrize("output_format", ["turtle", "nt"]) +@pytest.mark.parametrize("output_format", ["turtle", "xml"]) def test_fallback_is_deterministic_across_processes(output_format): """The rdflib fallback produces byte-identical output across processes. @@ -463,9 +518,15 @@ def test_fallback_is_deterministic_across_processes(output_format): parse to fail, exercising the fallback path. The graph also contains several blank nodes: a plain ``graph.serialize()`` would label them non-deterministically, so this test would fail without the blank-node - canonicalization in ``_deterministic_fallback_serialize``. Two - subprocesses with different ``PYTHONHASHSEED`` values must agree byte for - byte. + canonicalization in the fallback. Two subprocesses with different + ``PYTHONHASHSEED`` values must agree byte for byte. + + ``nt`` is not covered here because a fallback graph is by definition one + pyoxigraph refused, and for this graph the reason is a relative IRI, which + N-Triples forbids outright (N-Triples 1.1 §2.2). There is no deterministic + N-Triples document to produce, so the correct answer is a refusal -- + asserted by ``test_nt_fallback_refuses_rather_than_writing_an_unparseable_file`` + below. """ program = textwrap.dedent( f""" @@ -507,3 +568,256 @@ def run(seed: str) -> str: assert out_a == out_b, ( "Fallback output differs across PYTHONHASHSEED values; blank-node canonicalization may be missing" ) + + +def test_nt_fallback_refuses_rather_than_writing_an_unparseable_file(): + """N-Triples output for a graph with a relative IRI must raise, not lie. + + rdflib's N-Triples serializer reuses Turtle's term rendering and does not + enforce the absolute-IRI rule, so asking it for ``nt`` here yields a file + containing ```` that its own parser then rejects. N-Triples 1.1 + §2.2 permits only absolute IRIs, so no valid document exists for this + graph and a refusal is the only honest answer. Writing the file instead + defers the failure to whoever tries to read it. + """ + g = Graph() + g.bind("ex", "http://example.com/") + g.add((URIRef("http://example.com/s"), URIRef("http://purl.org/ontology/bibo/status"), URIRef("testing"))) + + with pytest.raises(ValueError, match="not an absolute IRI"): + canonicalize_rdf_graph(g, output_format="nt") + + +def _shapes_graph(count: int, extra: bool = False) -> Graph: + """A graph shaped like generator output: one blank node per named subject.""" + g = Graph() + g.bind("ex", "http://example.com/") + names = [f"{i:02d}" for i in range(count)] + (["AAAinserted"] if extra else []) + for name in names: + subject = URIRef(f"http://example.com/Shape{name}") + prop = BNode() + g.add((subject, URIRef("http://example.com/property"), prop)) + g.add((prop, URIRef("http://example.com/path"), URIRef(f"http://example.com/p{name}"))) + return g + + +def _changed_line_count(before: str, after: str) -> int: + import difflib + + diff = difflib.unified_diff(before.splitlines(), after.splitlines(), n=0, lineterm="") + return sum(1 for line in diff if line[:1] in "+-" and not line.startswith(("+++", "---"))) + + +def test_diff_stable_is_opt_in(): + """The default must keep producing exactly the output it produced before.""" + graph = _make_graph_with_bnodes() + assert canonicalize_rdf_graph(graph) == canonicalize_rdf_graph(graph, diff_stable=False) + + +def test_diff_stable_preserves_semantics(): + """Relabelling blank nodes must not change what the graph means.""" + graph = _make_graph_with_bnodes() + + plain = rdflib.Graph() + plain.parse(data=canonicalize_rdf_graph(graph), format="turtle") + stable = rdflib.Graph() + stable.parse(data=canonicalize_rdf_graph(graph, diff_stable=True), format="turtle") + + assert rdflib.compare.isomorphic(plain, stable) + + +def test_diff_stable_is_deterministic(): + """Diff stability must not cost determinism, which is the stronger property.""" + graph = _make_graph_with_bnodes() + outputs = {canonicalize_rdf_graph(graph, diff_stable=True) for _ in range(5)} + assert len(outputs) == 1 + + +def test_diff_stable_confines_an_insertion_to_the_lines_it_touches(): + """Inserting one subject must not relabel the blank nodes of the others. + + RDFC-1.0 numbers blank nodes ``c14nN`` in a global order, so a subject + sorting before the others shifts every subsequent label and rewrites + most of the file. This is the entire reason the option exists, so the + assertion is on the *ratio*, not on an absolute line count that would + be brittle across rdflib versions. + """ + before, after = _shapes_graph(20), _shapes_graph(20, extra=True) + + baseline = _changed_line_count(canonicalize_rdf_graph(before), canonicalize_rdf_graph(after)) + stable = _changed_line_count( + canonicalize_rdf_graph(before, diff_stable=True), + canonicalize_rdf_graph(after, diff_stable=True), + ) + + assert stable < baseline / 4, f"expected diff-stable output to churn far less; got {stable} vs baseline {baseline}" + + +_DIFF_STABLE_SCHEMA = """\ +id: https://example.org/diffstable +name: diffstable +prefixes: + linkml: https://w3id.org/linkml/ + ex: https://example.org/diffstable/ +default_prefix: ex +default_range: string +imports: + - linkml:types +classes: + Person: + slots: [name, knows] + Organization: + slots: [name] +slots: + name: + range: string + knows: + range: Person + multivalued: true +""" + + +def _generator_cases(): + """The four generators that serialize RDF, with the args that make them do so.""" + from linkml.generators.owlgen import OwlSchemaGenerator + from linkml.generators.rdfgen import RDFGenerator + from linkml.generators.shaclgen import ShaclGenerator + from linkml.generators.shexgen import ShExGenerator + + return [ + pytest.param(OwlSchemaGenerator, {}, id="owlgen"), + # rdfgen and shexgen resolve JSON-LD contexts (linkml types, shex.jsonld); + # the `network` marker serves those from local stubs. See tests/conftest.py. + pytest.param(RDFGenerator, {}, id="rdfgen", marks=pytest.mark.network), + pytest.param(ShaclGenerator, {}, id="shaclgen"), + # ShExGenerator only emits RDF in this format; its default is ShExC text, + # where blank-node labelling does not apply. + pytest.param(ShExGenerator, {"format": "rdf"}, id="shexgen", marks=pytest.mark.network), + ] + + +@pytest.mark.parametrize(("generator", "kwargs"), _generator_cases()) +def test_diff_stable_reaches_every_rdf_generator(tmp_path, generator, kwargs): + """Every RDF generator must actually apply the option, not merely accept it. + + Asserting only that the attribute exists would pass even if a generator + forgot to pass it down to :func:`canonicalize_rdf_graph`. Instead this + checks the observable consequence: RDFC-1.0 names blank nodes ``c14nN``, + while diff-stable labels are neighbourhood hashes, so a generator that + honours the flag emits no ``c14nN`` label at all. + """ + assert generator.diff_stable is False, f"{generator.__name__} must default to off" + + schema = tmp_path / "schema.yaml" + schema.write_text(_DIFF_STABLE_SCHEMA, encoding="utf-8", newline="\n") + + plain = generator(str(schema), **kwargs).serialize() + stable = generator(str(schema), diff_stable=True, **kwargs).serialize() + + # Guards the test itself: if the fixture stopped producing blank nodes the + # assertion below would hold vacuously. + assert re.search(r"c14n\d+", plain), f"{generator.__name__} output has no blank nodes to relabel" + assert not re.search(r"c14n\d+", stable), ( + f"{generator.__name__} still emits RDFC-1.0 blank-node labels with diff_stable=True; " + "the flag is probably not threaded into canonicalize_rdf_graph()" + ) + + +def test_diff_stable_warns_instead_of_silently_no_opping_on_the_fallback(): + """A request the fallback cannot honour must be reported, not ignored. + + ``wl_relabel_quads`` consumes canonical pyoxigraph quads, and the rdflib + fallback exists precisely because pyoxigraph refused the graph. Returning + the same bytes for ``diff_stable=True`` and ``diff_stable=False`` without + saying so lets a caller believe the output is diff-stable when it is not. + """ + graph = _make_graph_with_bnodes() + # A relative IRI is non-standard RDF, so pyoxigraph rejects the graph and + # canonicalize_rdf_graph degrades to rdflib -- the same path that + # ``shaclgen --include-annotations`` takes via its literal predicates. + graph.add((URIRef("testing"), URIRef("http://example.com/p"), Literal("v"))) + + with pytest.warns(RDFCanonicalizationWarning, match="not diff-stable"): + stable = canonicalize_rdf_graph(graph, diff_stable=True) + + with pytest.warns(RDFCanonicalizationWarning): + plain = canonicalize_rdf_graph(graph, diff_stable=False) + + # The warning is the contract: the bytes really are identical, which is + # exactly why staying silent would be misleading. + assert stable == plain + + +# -------------------------------------------------------------------------- +# The bridge from the library's logging to linkml's warnings +# -------------------------------------------------------------------------- + + +def _fallback_graph() -> Graph: + """A graph pyoxigraph refuses, so every call takes a degraded path.""" + g = Graph() + g.bind("ex", "http://example.com/") + g.add((URIRef("http://example.com/s"), URIRef("http://purl.org/ontology/bibo/status"), URIRef("testing"))) + return g + + +def test_degraded_path_warning_points_at_the_caller(): + """The warning must name the line that asked for the serialization. + + ``diffable-rdf`` reports degradation through ``logging``; linkml re-emits + it through ``warnings`` so it is visible without logging configuration. + A re-emitted warning is only actionable if it is attributed to the caller + rather than to the adapter, and the number of frames in between is not + something a reader can eyeball -- hence this test. + """ + graph = _fallback_graph() + + with pytest.warns(RDFCanonicalizationWarning) as caught: + canonicalize_rdf_graph(graph, output_format="turtle") # attribution target + + assert caught[0].filename == __file__ + source = Path(caught[0].filename).read_text(encoding="utf-8").splitlines() + assert "# attribution target" in source[caught[0].lineno - 1] + + +def test_bridging_does_not_leave_the_library_logger_modified(): + """Raising the library's log level to capture records must not be permanent. + + The adapter has to enable ``WARNING`` on the ``diffable_rdf`` logger to see + anything, which is a global mutation. Leaving it raised would silently + change logging behaviour for the rest of the process. + """ + logger = logging.getLogger("diffable_rdf") + level_before = logger.level + handlers_before = list(logger.handlers) + + with pytest.warns(RDFCanonicalizationWarning): + canonicalize_rdf_graph(_fallback_graph(), output_format="turtle") + + assert logger.level == level_before + assert logger.handlers == handlers_before + + +def test_a_caller_who_configured_logging_still_receives_the_record(caplog): + """Re-emitting as a warning must not steal the record from a log handler. + + An application that deliberately configured the ``diffable_rdf`` logger is + asking for these records. The adapter adds a mechanism; it does not get to + remove one. + """ + with caplog.at_level(logging.WARNING, logger="diffable_rdf"), pytest.warns(RDFCanonicalizationWarning): + canonicalize_rdf_graph(_fallback_graph(), output_format="turtle") + + assert any("non-standard RDF" in record.getMessage() for record in caplog.records) + + +def test_warnings_survive_an_exception_from_the_library(): + """Degradation reported before a failure must still reach the caller. + + ``nt`` output for this graph degrades to rdflib and *then* refuses, because + N-Triples has no way to write the relative IRI. Dropping the warning + because the call ended in an exception would hide the first half of the + story, which is the half that explains the second. + """ + with pytest.warns(RDFCanonicalizationWarning, match="non-standard RDF"), pytest.raises(ValueError): + canonicalize_rdf_graph(_fallback_graph(), output_format="nt") diff --git a/tests/linkml_runtime/test_utils/test_rdf_canonicalize_defects.py b/tests/linkml_runtime/test_utils/test_rdf_canonicalize_defects.py new file mode 100644 index 0000000000..ffca5ae447 --- /dev/null +++ b/tests/linkml_runtime/test_utils/test_rdf_canonicalize_defects.py @@ -0,0 +1,314 @@ +"""Correctness properties shared by linkml's RDF canonicalizer and the library behind it. + +``linkml_runtime.utils.rdf_canonicalize`` and +``diffable_rdf.canonicalize_rdf_graph`` are the same code: the module was +extracted into a standalone library at the maintainers' request +(linkml/linkml#3295). The two copies then diverged for a while, and this file +is what made the divergence visible -- each test asserts one correctness +property against *both* implementations, so a fix that landed on one side and +not the other showed up as a failure rather than as nothing at all. + +Nine properties held only in the library, two of them cases of silent data +corruption where the output parsed cleanly and said something the input never +said. One held only in linkml: the library dropped ``@base`` on the degraded +path. Asserting both directions is what got each of them fixed where it +belonged -- the ``@base`` gap in diffable-rdf 0.4.0, the other nine in linkml +by deleting the in-tree copy and calling the library. + +Every case is now unmarked, which is the point: the file is the evidence that +the delegation changed no behaviour it should not have, and it keeps running +against both entry points so that a future divergence fails the suite instead +of going unnoticed. +""" + +import os +import subprocess +import sys +import textwrap +import warnings + +import diffable_rdf +import pytest +import rdflib +from rdflib import BNode, Graph, Literal, Namespace, URIRef +from rdflib.namespace import RDF + +from linkml_runtime.utils.rdf_canonicalize import canonicalize_rdf_graph as linkml_canonicalize + +EX = Namespace("http://example.org/") + +_IMPLEMENTATIONS = ( + (linkml_canonicalize, "linkml"), + (diffable_rdf.canonicalize_rdf_graph, "diffable-rdf"), +) + + +def _impls(): + """Parametrize a test over both entry points. + + linkml's is a thin adapter over the library's, so the two agree by + construction today. Running both anyway is what turns a future re-fork, or + an adapter that quietly changes behaviour on the way through, into a test + failure. + """ + return [pytest.param(fn, id=ident) for fn, ident in _IMPLEMENTATIONS] + + +def _apply(fn, graph, output_format="turtle"): + """Call an implementation, ignoring the warnings both emit on the fallback path.""" + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + return fn(graph, output_format) + + +# -------------------------------------------------------------------------- +# Silent data corruption +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize("canonicalize", _impls()) +def test_base_with_a_fragment_does_not_rewrite_every_iri(canonicalize): + """A base IRI ending in ``#`` must not change the graph's terms. + + Relativizing ``http://ex.org/d#a`` against base ``http://ex.org/d#`` gives + ``<#a>``, which is correct per RFC 3986 -- but rdflib's parser resolves a + fragment reference by concatenation and reads it back as + ``http://ex.org/d##a``. Every term of the graph silently changes, and the + output parses cleanly, so nothing reports it. + """ + graph = Graph(base="http://ex.org/d#") + graph.add((URIRef("http://ex.org/d#a"), EX.p, Literal("v"))) + + round_tripped = Graph() + round_tripped.parse(data=_apply(canonicalize, graph), format="turtle") + + assert {str(s) for s in round_tripped.subjects()} == {"http://ex.org/d#a"} + + +@pytest.mark.parametrize("canonicalize", _impls()) +def test_a_shared_rdf_list_tail_is_not_duplicated(canonicalize): + """Two lists sharing a tail must not gain triples on the way out. + + rdflib's Turtle writer renders ``( ... )`` collection syntax per list, so a + tail referenced from two lists is written twice as two separate blank + nodes. The result asserts more than the input did. + """ + graph = Graph() + graph.bind("ex", EX) + tail = BNode() + graph.add((tail, RDF.first, Literal("shared"))) + graph.add((tail, RDF.rest, RDF.nil)) + for name in ("l1", "l2"): + head = BNode() + graph.add((EX[name], EX.list, head)) + graph.add((head, RDF.first, Literal(name))) + graph.add((head, RDF.rest, tail)) + # A relative IRI is non-standard RDF, so pyoxigraph refuses the graph and + # both implementations degrade to rdflib -- the path this defect lives on. + graph.add((URIRef("testing"), EX.p, Literal("forces-fallback"))) + + round_tripped = Graph() + round_tripped.parse(data=_apply(canonicalize, graph), format="turtle") + + assert len(round_tripped) == len(graph) + + +# -------------------------------------------------------------------------- +# Output that no parser will read +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize("canonicalize", _impls()) +def test_nt_output_either_parses_or_refuses(canonicalize): + """N-Triples admits only absolute IRIs, so a relative one must not be written. + + The fallback is taken *because* the graph holds a term pyoxigraph rejected. + Writing it out anyway produces a file that fails on line 1. Refusing with a + message naming the term and a format that can carry the graph is the fix; + returning unreadable text is not. + """ + graph = Graph() + graph.add((URIRef("testing"), EX.p, Literal("v"))) + + try: + serialized = _apply(canonicalize, graph, "nt") + except ValueError: + return # refused, with an explanation -- the correct outcome + + rdflib.Graph().parse(data=serialized, format="nt") + + +@pytest.mark.parametrize("canonicalize", _impls()) +def test_a_unicode_line_separator_in_a_literal_survives(canonicalize): + """Sorting N-Triples lines must split on newlines only. + + N-Triples permits U+2028 raw inside a quoted literal, but + ``str.splitlines()`` breaks on it as well as on U+2029, U+0085, U+000B, + U+000C and U+001C-1E. One statement becomes two lines, the halves sort + independently, the separator is rewritten as a newline, and the document + no longer parses. + """ + graph = Graph() + graph.add((EX.s, EX.p, Literal("a\u2028b"))) + graph.add((URIRef("testing"), EX.p, Literal("forces-fallback"))) + + try: + serialized = _apply(canonicalize, graph, "nt") + except ValueError: + return # refused because of the relative IRI, before the sort + + assert "\u2028" in serialized + rdflib.Graph().parse(data=serialized, format="nt") + + +# -------------------------------------------------------------------------- +# Non-determinism, which is the property the module exists to provide +# -------------------------------------------------------------------------- + +_ACROSS_PROCESSES = textwrap.dedent( + """ + import sys, warnings + warnings.simplefilter("ignore") + from rdflib import BNode, Graph, Literal, Namespace, URIRef + from rdflib.namespace import RDF + + module, output_format, force_fallback = sys.argv[1], sys.argv[2], sys.argv[3] == "1" + if module == "linkml": + from linkml_runtime.utils.rdf_canonicalize import canonicalize_rdf_graph + else: + from diffable_rdf import canonicalize_rdf_graph + + EX = Namespace("http://example.org/") + g = Graph() + if force_fallback: + g.add((URIRef("testing"), EX.p, Literal("forces-fallback"))) + for i in range(12): + b = BNode() + g.add((EX["s%02d" % i], EX.child, b)) + g.add((b, EX.k, Literal("v%d" % i))) + g.add((b, RDF.type, EX.T)) + for host in "abcdefgh": + g.add((URIRef("http://%s.example/s" % host), + URIRef("http://%s.example/p" % host), + URIRef("http://%s.example/o" % host))) + sys.stdout.write(canonicalize_rdf_graph(g, output_format)) + """ +) + + +def _outputs_across_processes(module, output_format, force_fallback): + """Serialize the same graph in four processes with different hash seeds.""" + outputs = set() + for seed in ("0", "1", "12345", "999"): + completed = subprocess.run( + [sys.executable, "-c", _ACROSS_PROCESSES, module, output_format, "1" if force_fallback else "0"], + capture_output=True, + text=True, + encoding="utf-8", + env={**os.environ, "PYTHONHASHSEED": seed}, + check=False, + ) + if completed.returncode != 0: + pytest.fail(f"{module}/{output_format} failed:\n{completed.stderr[-2000:]}") + outputs.add(completed.stdout) + return outputs + + +_XML_TRAVERSAL = "degraded RDF/XML is ordered by rdflib's graph traversal, which follows set iteration order" +_NS_NAMES = "auto-generated ns1/ns2 prefix names are allocated in traversal order" +_JSONLD_UNMAPPED = "json-ld is absent from the format map, so it falls through to rdflib with no determinism guarantee" + + +@pytest.mark.parametrize( + ("module", "output_format", "force_fallback"), + [ + pytest.param("linkml", "xml", True, id="linkml-xml-fallback"), + pytest.param("diffable_rdf", "xml", True, id="diffable-rdf-xml-fallback"), + pytest.param("linkml", "turtle", True, id="linkml-turtle-fallback"), + pytest.param("diffable_rdf", "turtle", True, id="diffable-rdf-turtle-fallback"), + pytest.param("linkml", "json-ld", False, id="linkml-jsonld"), + pytest.param("diffable_rdf", "json-ld", False, id="diffable-rdf-jsonld"), + ], +) +def test_output_is_byte_identical_across_processes(module, output_format, force_fallback): + """The same graph must serialize to the same bytes in any process. + + Three separate causes used to break this in the in-tree copy, and all three + are fixed by delegating. Degraded RDF/XML was ordered by rdflib's own graph + traversal, which follows set iteration order. Auto-generated ``ns1``/``ns2`` + prefix names were allocated in traversal order too, so the *names* moved + even when the triples did not. And ``json-ld`` was not in the format map at + all, so it fell through to rdflib's serializer, which carries no + determinism guarantee. + """ + assert len(_outputs_across_processes(module, output_format, force_fallback)) == 1 + + +# -------------------------------------------------------------------------- +# Interface +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize("canonicalize", _impls()) +def test_every_format_ends_with_exactly_one_newline(canonicalize): + """A committed artifact should not depend on which format wrote it. + + Passing through whatever the serializer produced gives no trailing newline + for RDF/XML and one for Turtle, so POSIX text tools disagree about whether + the file has a last line. + """ + counts = {} + for output_format in ("turtle", "nt", "xml", "trig", "n3"): + graph = Graph() + graph.bind("ex", EX) + graph.add((EX.s, EX.p, Literal("v"))) + serialized = _apply(canonicalize, graph, output_format) + counts[output_format] = len(serialized) - len(serialized.rstrip("\n")) + + assert counts == dict.fromkeys(counts, 1) + + +@pytest.mark.parametrize("canonicalize", _impls()) +def test_a_dataset_is_refused_rather_than_partly_serialized(canonicalize): + """A Dataset is a Graph subclass, so it is accepted and quietly flattened. + + Every named graph collapses into one document and the graph names are + dropped, which is a different dataset. The caller asked for something this + function cannot express and should be told so. + """ + dataset = rdflib.Dataset() + dataset.graph(URIRef("http://ex/g1")).add((EX.a, EX.p, Literal("in-g1"))) + dataset.graph(URIRef("http://ex/g2")).add((EX.b, EX.p, Literal("in-g2"))) + + with pytest.raises(TypeError): + _apply(canonicalize, dataset) + + +# -------------------------------------------------------------------------- +# A property both implementations now hold +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize("canonicalize", _impls()) +def test_base_survives_the_degraded_path(canonicalize): + """``@base`` must not disappear just because pyoxigraph refused the graph. + + ``rdflib_dumper.dumps(..., prefix_map={"@base": ...})`` is a supported way + to ask for a document whose IRIs are written relative to a base, and the + metamodel routinely produces graphs that fall back (a bare ``status: + testing`` on a ``uriorcurie`` slot serializes to the relative ````, + which pyoxigraph rejects). Losing the directive on exactly that path means + the feature works only for graphs that never needed the fallback. + + This ran the other way until diffable-rdf 0.4.0. It was the last property + blocking delegation, because linkml held it and the library did not, so + adopting the library would have been a regression. Fixing it there rather + than keeping the in-tree copy alive is what let the other nine gaps close + at once. + """ + graph = Graph(base="http://example.org/default/") + graph.bind("ex", EX) + graph.add((EX.s, EX.p, Literal("v"))) + graph.add((URIRef("testing"), EX.p, Literal("forces-fallback"))) + + assert "@base ." in _apply(canonicalize, graph) diff --git a/uv.lock b/uv.lock index d8cddaa9e1..3c867304c8 100644 --- a/uv.lock +++ b/uv.lock @@ -1085,6 +1085,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/84/d0/205d54408c08b13550c733c4b85429e7ead111c7f0014309637425520a9a/deprecated-1.3.1-py2.py3-none-any.whl", hash = "sha256:597bfef186b6f60181535a29fbe44865ce137a5079f295b479886c82729d5f3f", size = 11298, upload-time = "2025-10-30T08:19:00.758Z" }, ] +[[package]] +name = "diffable-rdf" +version = "0.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyoxigraph" }, + { name = "rdflib" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/86/df/1d71c0c984eac1cfd25531aabf84528941b01083c875da49d44882d1b5af/diffable_rdf-0.4.0.tar.gz", hash = "sha256:a84afaa10332e6a039b6ddcabf76bced3049e4da467c88ee786d0fa35218111e", size = 838040, upload-time = "2026-09-11T11:43:12.844Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/49/b62ee864fe6769b759f04e3709fd79711b1cdf49223b1bb0e985e507ead3/diffable_rdf-0.4.0-py3-none-any.whl", hash = "sha256:c59c57429465f786fdbc8d38ddbb687da16748c5752f0c2ac4a77b5527aa9aed", size = 46126, upload-time = "2026-09-11T11:43:11.536Z" }, +] + [[package]] name = "distlib" version = "0.4.0" @@ -2618,6 +2631,7 @@ dependencies = [ { name = "click" }, { name = "curies" }, { name = "deprecated" }, + { name = "diffable-rdf" }, { name = "hbreader" }, { name = "isodate", marker = "python_full_version < '3.11'" }, { name = "json-flattener" }, @@ -2650,6 +2664,7 @@ requires-dist = [ { name = "coverage", marker = "extra == 'dev'" }, { name = "curies", specifier = ">=0.14.6" }, { name = "deprecated" }, + { name = "diffable-rdf", specifier = ">=0.4.0" }, { name = "hbreader" }, { name = "isodate", marker = "python_full_version < '3.11'", specifier = ">=0.7.2,<1.0.0" }, { name = "json-flattener", specifier = ">=0.1.9" },