From 84af8180524bad81526ceac32956cdd1bdff6eb4 Mon Sep 17 00:00:00 2001 From: Carlo van Driesten Date: Tue, 8 Sep 2026 18:09:43 +0200 Subject: [PATCH 01/14] feat(rdf): add opt-in diff-stable blank-node labels RDFC-1.0 canonicalization already makes RDF output deterministic: isomorphic graphs always serialize identically. It does not make output diffable. Blank nodes are numbered `c14nN` in a single global order, so inserting one class can renumber every blank node after it and rewrite most of the file. A one-line semantic change lands as a whole-file diff, which makes generated OWL/SHACL hard to review and noisy to keep under version control. Add a `diff_stable` argument to `canonicalize_rdf_graph()` and a `--diff-stable/--no-diff-stable` flag to the four RDF generators. When enabled, blank-node labels are derived from each node's own neighbourhood via Weisfeiler-Lehman refinement, so an edit relabels only the blank nodes it actually touches. Measured churn on a real schema (add one class, count changed lines): generator default --diff-stable owlgen 2091 17 shexgen 796 50 shaclgen 291 13 rdfgen 115 25 Output stays deterministic and isomorphic either way; only the choice of label changes. Off by default, because enabling it relabels existing output. The refinement itself lives in `diffable-rdf`, whose only dependencies (rdflib, pyoxigraph) are already linkml-runtime dependencies at higher versions, so this adds no new transitive dependencies. --- .../linkml/src/linkml/generators/owlgen.py | 28 +++- .../linkml/src/linkml/generators/rdfgen.py | 28 +++- .../linkml/src/linkml/generators/shaclgen.py | 28 +++- .../linkml/src/linkml/generators/shexgen.py | 28 +++- packages/linkml_runtime/pyproject.toml | 1 + .../linkml_runtime/utils/rdf_canonicalize.py | 23 ++- .../test_utils/test_rdf_canonicalize.py | 135 ++++++++++++++++++ 7 files changed, 266 insertions(+), 5 deletions(-) diff --git a/packages/linkml/src/linkml/generators/owlgen.py b/packages/linkml/src/linkml/generators/owlgen.py index 50be266df0..7751eab34d 100644 --- a/packages/linkml/src/linkml/generators/owlgen.py +++ b/packages/linkml/src/linkml/generators/owlgen.py @@ -122,6 +122,22 @@ class OwlSchemaGenerator(Generator): """Suffix to add to the schema name to create the ontology URI, e.g. .owl.ttl""" # ObjectVars + diff_stable: bool = False + """Label blank nodes so that unrelated edits leave them untouched. + + Output is already deterministic: RDFC-1.0 guarantees that isomorphic + graphs serialize identically. It does not guarantee that *similar* + graphs serialize *similarly* — blank nodes are numbered ``c14nN`` in a + global order, so adding one class can renumber every blank node after + it and rewrite most of the file. + + When ``True``, blank-node labels are instead derived from each node's + own neighbourhood via Weisfeiler-Lehman refinement, so an edit relabels + only the blank nodes it actually touches. The output stays + deterministic and isomorphic either way; only the choice of label + changes. Off by default because enabling it relabels existing output. + """ + metadata_profile: MetadataProfile | None = None """Deprecated - use metadata_profiles.""" @@ -353,7 +369,7 @@ def serialize(self, **kwargs: Any) -> str: """ self.as_graph() fmt = "turtle" if self.format in ["owl", "ttl"] else self.format - return canonicalize_rdf_graph(self.graph, output_format=fmt) + return canonicalize_rdf_graph(self.graph, output_format=fmt, diff_stable=self.diff_stable) def add_metadata(self, e: Definition | PermissibleValue, uri: URIRef) -> None: """ @@ -1844,6 +1860,16 @@ def slot_owl_type(self, slot: SlotDefinition) -> URIRef: "specified language tag. Element-level in_language overrides this." ), ) +@click.option( + "--diff-stable/--no-diff-stable", + default=False, + show_default=True, + help=( + "Derive blank-node labels from each node's own neighbourhood so that " + "unrelated edits leave them unchanged. Output is deterministic either " + "way; this makes successive versions of a file diff cleanly." + ), +) @click.version_option(__version__, "-V", "--version") def cli(yamlfile: str, metadata_profile: str, **kwargs: Any) -> None: """Generate an OWL representation of a LinkML model diff --git a/packages/linkml/src/linkml/generators/rdfgen.py b/packages/linkml/src/linkml/generators/rdfgen.py index 2da1701787..052fcff12e 100644 --- a/packages/linkml/src/linkml/generators/rdfgen.py +++ b/packages/linkml/src/linkml/generators/rdfgen.py @@ -78,6 +78,22 @@ class RDFGenerator(Generator): uses_schemaloader = True # ObjectVars + diff_stable: bool = False + """Label blank nodes so that unrelated edits leave them untouched. + + Output is already deterministic: RDFC-1.0 guarantees that isomorphic + graphs serialize identically. It does not guarantee that *similar* + graphs serialize *similarly* — blank nodes are numbered ``c14nN`` in a + global order, so adding one class can renumber every blank node after + it and rewrite most of the file. + + When ``True``, blank-node labels are instead derived from each node's + own neighbourhood via Weisfeiler-Lehman refinement, so an edit relabels + only the blank nodes it actually touches. The output stays + deterministic and isomorphic either way; only the choice of label + changes. Off by default because enabling it relabels existing output. + """ + emit_metadata: bool = False context: list[str] = None original_schema: SchemaDefinition = None @@ -89,7 +105,7 @@ def __post_init__(self): def _data(self, g: Graph) -> str: fmt = "turtle" if self.format == "ttl" else self.format - return canonicalize_rdf_graph(g, output_format=fmt) + return canonicalize_rdf_graph(g, output_format=fmt, diff_stable=self.diff_stable) def end_schema(self, output: str | None = None, context: str = None, **_) -> str: gen = JSONLDGenerator( @@ -137,6 +153,16 @@ def end_schema(self, output: str | None = None, context: str = None, **_) -> str multiple=True, help="JSONLD context file (default: vendored meta.context.jsonld)", ) +@click.option( + "--diff-stable/--no-diff-stable", + default=False, + show_default=True, + help=( + "Derive blank-node labels from each node's own neighbourhood so that " + "unrelated edits leave them unchanged. Output is deterministic either " + "way; this makes successive versions of a file diff cleanly." + ), +) @click.version_option(__version__, "-V", "--version") def cli(yamlfile, **kwargs): """Generate an RDF representation of a LinkML model""" diff --git a/packages/linkml/src/linkml/generators/shaclgen.py b/packages/linkml/src/linkml/generators/shaclgen.py index 4731b9f0b8..5d42fea272 100644 --- a/packages/linkml/src/linkml/generators/shaclgen.py +++ b/packages/linkml/src/linkml/generators/shaclgen.py @@ -142,6 +142,22 @@ class ShaclGenerator(Generator): ignores any per-slot ``in_language``. """ + diff_stable: bool = False + """Label blank nodes so that unrelated edits leave them untouched. + + Output is already deterministic: RDFC-1.0 guarantees that isomorphic + graphs serialize identically. It does not guarantee that *similar* + graphs serialize *similarly* — blank nodes are numbered ``c14nN`` in a + global order, so adding one class can renumber every blank node after + it and rewrite most of the file. + + When ``True``, blank-node labels are instead derived from each node's + own neighbourhood via Weisfeiler-Lehman refinement, so an edit relabels + only the blank nodes it actually touches. The output stays + deterministic and isomorphic either way; only the choice of label + changes. Off by default because enabling it relabels existing output. + """ + emit_rules: bool = True """Emit ``sh:sparql`` constraints from LinkML ``rules:`` blocks. @@ -196,7 +212,7 @@ def generate_header(self) -> str: def serialize(self, **args) -> str: g = self.as_graph() fmt = "turtle" if self.format in ["owl", "ttl"] else self.format - return canonicalize_rdf_graph(g, output_format=fmt) + return canonicalize_rdf_graph(g, output_format=fmt, diff_stable=self.diff_stable) def as_graph(self) -> Graph: sv = self.schemaview @@ -929,6 +945,16 @@ def add_simple_data_type(func: Callable, r: ElementName) -> None: "sh:NodeShape. Use --no-emit-rules to suppress rule generation." ), ) +@click.option( + "--diff-stable/--no-diff-stable", + default=False, + show_default=True, + help=( + "Derive blank-node labels from each node's own neighbourhood so that " + "unrelated edits leave them unchanged. Output is deterministic either " + "way; this makes successive versions of a file diff cleanly." + ), +) @click.version_option(__version__, "-V", "--version") def cli(yamlfile, **args): """Generate SHACL turtle from a LinkML model""" diff --git a/packages/linkml/src/linkml/generators/shexgen.py b/packages/linkml/src/linkml/generators/shexgen.py index 40a93ffbc9..7c43b13bc0 100644 --- a/packages/linkml/src/linkml/generators/shexgen.py +++ b/packages/linkml/src/linkml/generators/shexgen.py @@ -40,6 +40,22 @@ class ShExGenerator(Generator): uses_schemaloader = True # ObjectVars + diff_stable: bool = False + """Label blank nodes so that unrelated edits leave them untouched. + + Output is already deterministic: RDFC-1.0 guarantees that isomorphic + graphs serialize identically. It does not guarantee that *similar* + graphs serialize *similarly* — blank nodes are numbered ``c14nN`` in a + global order, so adding one class can renumber every blank node after + it and rewrite most of the file. + + When ``True``, blank-node labels are instead derived from each node's + own neighbourhood via Weisfeiler-Lehman refinement, so an edit relabels + only the blank nodes it actually touches. The output stays + deterministic and isomorphic either way; only the choice of label + changes. Off by default because enabling it relabels existing output. + """ + shex: Schema = field(default_factory=lambda: Schema()) # ShEx Schema being generated shapes: list = field(default_factory=lambda: []) shape: Shape | None = None # Current shape being defined @@ -177,7 +193,7 @@ def end_schema(self, output: str | None = None, **_) -> str: g = Graph() g.parse(data=shex, format="json-ld", version="1.1") g.bind("owl", OWL) - shex = canonicalize_rdf_graph(g, output_format="turtle") + shex = canonicalize_rdf_graph(g, output_format="turtle", diff_stable=self.diff_stable) elif self.format == "shex": g = Graph() self.namespaces.load_graph(g) @@ -258,6 +274,16 @@ def _get_subproperty_values(self, slot: SlotDefinition) -> list: help="If --expand-subproperty-of (default), slots with subproperty_of will generate NodeConstraint " "values containing all slot descendants. Use --no-expand-subproperty-of to disable this behavior.", ) +@click.option( + "--diff-stable/--no-diff-stable", + default=False, + show_default=True, + help=( + "Derive blank-node labels from each node's own neighbourhood so that " + "unrelated edits leave them unchanged. Output is deterministic either " + "way; this makes successive versions of a file diff cleanly." + ), +) @click.version_option(__version__, "-V", "--version") def cli(yamlfile, **args): """Generate a ShEx Schema for a LinkML model""" diff --git a/packages/linkml_runtime/pyproject.toml b/packages/linkml_runtime/pyproject.toml index 1ae9da3083..b879dec7ee 100644 --- a/packages/linkml_runtime/pyproject.toml +++ b/packages/linkml_runtime/pyproject.toml @@ -48,6 +48,7 @@ dependencies = [ "prefixmaps >=0.1.4", "curies>=0.14.6", "pyoxigraph>=0.5.11", + "diffable-rdf>=0.2.0", "pydantic>=2.13.5,<3.0.0", "isodate >=0.7.2, <1.0.0; python_version < '3.11'", ] diff --git a/packages/linkml_runtime/src/linkml_runtime/utils/rdf_canonicalize.py b/packages/linkml_runtime/src/linkml_runtime/utils/rdf_canonicalize.py index 31c5580348..e0fadae34a 100644 --- a/packages/linkml_runtime/src/linkml_runtime/utils/rdf_canonicalize.py +++ b/packages/linkml_runtime/src/linkml_runtime/utils/rdf_canonicalize.py @@ -42,6 +42,7 @@ import pyoxigraph as ox import rdflib +from diffable_rdf import wl_relabel_quads from rdflib.compare import to_canonical_graph @@ -287,6 +288,7 @@ def _is_safe_prefix_iri(iri: str) -> bool: def canonicalize_rdf_graph( graph: rdflib.Graph, output_format: str = "turtle", + diff_stable: bool = False, ) -> str: """Serialize an rdflib Graph deterministically using RDFC-1.0 canonicalization. @@ -300,6 +302,12 @@ def canonicalize_rdf_graph( :param graph: The rdflib Graph to serialize. :param output_format: Target serialization format (e.g. ``"turtle"``, ``"nt"``). + :param diff_stable: Derive blank-node labels from each node's own + neighbourhood instead of RDFC-1.0's global ``c14nN`` numbering, so that + editing one part of a schema does not renumber unrelated blank nodes. + Output is deterministic and isomorphic either way; only the choice of + label changes. Off by default because enabling it relabels existing + output. :return: Deterministic string serialization of the graph. """ ox_format = _FORMAT_MAP.get(output_format.lower()) @@ -339,13 +347,26 @@ def canonicalize_rdf_graph( # 3. Canonicalize blank node labels with RDFC-1.0. dataset.canonicalize(ox.CanonicalizationAlgorithm.RDFC_1_0) + quads = list(dataset) + + # 3b. Optionally re-label blank nodes with locality-sensitive hashes. + # RDFC-1.0 guarantees that identical graphs get identical labels, but it + # does not guarantee that *similar* graphs get similar labels: the labels + # are assigned by a global ordering, so inserting one blank node can + # renumber every label after it and turn a one-line semantic change into a + # whole-file diff. Weisfeiler-Lehman labels are derived only from each + # node's local neighbourhood, so unrelated regions keep their labels. + # Output stays deterministic and isomorphic either way; this only changes + # which label each blank node receives. + if diff_stable: + quads = wl_relabel_quads(quads) + # 4. Sort triples for deterministic ordering. # RDFC-1.0 stabilizes blank-node labels but pyoxigraph's Dataset # iteration order is not sorted and varies across processes (verified # empirically against pyoxigraph 0.5.8). The explicit string-key sort # is load-bearing for byte-identical output across runs; see # tests/linkml_runtime/test_utils/test_rdf_canonicalize.py::test_sort_is_load_bearing. - quads = list(dataset) sorted_triples = sorted( (ox.Triple(q.subject, q.predicate, q.object) for q in quads), key=lambda t: (str(t.subject), str(t.predicate), str(t.object)), diff --git a/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py b/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py index 63dbf095a1..0ca79c3c90 100644 --- a/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py +++ b/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py @@ -507,3 +507,138 @@ def run(seed: str) -> str: assert out_a == out_b, ( "Fallback output differs across PYTHONHASHSEED values; blank-node canonicalization may be missing" ) + + +def _shapes_graph(count: int, extra: bool = False) -> Graph: + """A graph shaped like generator output: one blank node per named subject.""" + g = Graph() + g.bind("ex", "http://example.com/") + names = [f"{i:02d}" for i in range(count)] + (["AAAinserted"] if extra else []) + for name in names: + subject = URIRef(f"http://example.com/Shape{name}") + prop = BNode() + g.add((subject, URIRef("http://example.com/property"), prop)) + g.add((prop, URIRef("http://example.com/path"), URIRef(f"http://example.com/p{name}"))) + return g + + +def _changed_line_count(before: str, after: str) -> int: + import difflib + + diff = difflib.unified_diff(before.splitlines(), after.splitlines(), n=0, lineterm="") + return sum(1 for line in diff if line[:1] in "+-" and not line.startswith(("+++", "---"))) + + +def test_diff_stable_is_opt_in(): + """The default must keep producing exactly the output it produced before.""" + graph = _make_graph_with_bnodes() + assert canonicalize_rdf_graph(graph) == canonicalize_rdf_graph(graph, diff_stable=False) + + +def test_diff_stable_preserves_semantics(): + """Relabelling blank nodes must not change what the graph means.""" + graph = _make_graph_with_bnodes() + + plain = rdflib.Graph() + plain.parse(data=canonicalize_rdf_graph(graph), format="turtle") + stable = rdflib.Graph() + stable.parse(data=canonicalize_rdf_graph(graph, diff_stable=True), format="turtle") + + assert rdflib.compare.isomorphic(plain, stable) + + +def test_diff_stable_is_deterministic(): + """Diff stability must not cost determinism, which is the stronger property.""" + graph = _make_graph_with_bnodes() + outputs = {canonicalize_rdf_graph(graph, diff_stable=True) for _ in range(5)} + assert len(outputs) == 1 + + +def test_diff_stable_confines_an_insertion_to_the_lines_it_touches(): + """Inserting one subject must not relabel the blank nodes of the others. + + RDFC-1.0 numbers blank nodes ``c14nN`` in a global order, so a subject + sorting before the others shifts every subsequent label and rewrites + most of the file. This is the entire reason the option exists, so the + assertion is on the *ratio*, not on an absolute line count that would + be brittle across rdflib versions. + """ + before, after = _shapes_graph(20), _shapes_graph(20, extra=True) + + baseline = _changed_line_count(canonicalize_rdf_graph(before), canonicalize_rdf_graph(after)) + stable = _changed_line_count( + canonicalize_rdf_graph(before, diff_stable=True), + canonicalize_rdf_graph(after, diff_stable=True), + ) + + assert stable < baseline / 4, f"expected diff-stable output to churn far less; got {stable} vs baseline {baseline}" + + +_DIFF_STABLE_SCHEMA = """\ +id: https://example.org/diffstable +name: diffstable +prefixes: + linkml: https://w3id.org/linkml/ + ex: https://example.org/diffstable/ +default_prefix: ex +default_range: string +imports: + - linkml:types +classes: + Person: + slots: [name, knows] + Organization: + slots: [name] +slots: + name: + range: string + knows: + range: Person + multivalued: true +""" + + +def _generator_cases(): + """The four generators that serialize RDF, with the args that make them do so.""" + from linkml.generators.owlgen import OwlSchemaGenerator + from linkml.generators.rdfgen import RDFGenerator + from linkml.generators.shaclgen import ShaclGenerator + from linkml.generators.shexgen import ShExGenerator + + return [ + pytest.param(OwlSchemaGenerator, {}, id="owlgen"), + # rdfgen and shexgen resolve JSON-LD contexts (linkml types, shex.jsonld); + # the `network` marker serves those from local stubs. See tests/conftest.py. + pytest.param(RDFGenerator, {}, id="rdfgen", marks=pytest.mark.network), + pytest.param(ShaclGenerator, {}, id="shaclgen"), + # ShExGenerator only emits RDF in this format; its default is ShExC text, + # where blank-node labelling does not apply. + pytest.param(ShExGenerator, {"format": "rdf"}, id="shexgen", marks=pytest.mark.network), + ] + + +@pytest.mark.parametrize(("generator", "kwargs"), _generator_cases()) +def test_diff_stable_reaches_every_rdf_generator(tmp_path, generator, kwargs): + """Every RDF generator must actually apply the option, not merely accept it. + + Asserting only that the attribute exists would pass even if a generator + forgot to pass it down to :func:`canonicalize_rdf_graph`. Instead this + checks the observable consequence: RDFC-1.0 names blank nodes ``c14nN``, + while diff-stable labels are neighbourhood hashes, so a generator that + honours the flag emits no ``c14nN`` label at all. + """ + assert generator.diff_stable is False, f"{generator.__name__} must default to off" + + schema = tmp_path / "schema.yaml" + schema.write_text(_DIFF_STABLE_SCHEMA, encoding="utf-8", newline="\n") + + plain = generator(str(schema), **kwargs).serialize() + stable = generator(str(schema), diff_stable=True, **kwargs).serialize() + + # Guards the test itself: if the fixture stopped producing blank nodes the + # assertion below would hold vacuously. + assert re.search(r"c14n\d+", plain), f"{generator.__name__} output has no blank nodes to relabel" + assert not re.search(r"c14n\d+", stable), ( + f"{generator.__name__} still emits RDFC-1.0 blank-node labels with diff_stable=True; " + "the flag is probably not threaded into canonicalize_rdf_graph()" + ) From 8ff0ef95b24bdfcdff25ed3227149d467dec0bd6 Mon Sep 17 00:00:00 2001 From: Carlo van Driesten Date: Fri, 11 Sep 2026 10:52:51 +0200 Subject: [PATCH 02/14] fix(rdf): bump diffable-rdf to 0.3.0 and stop diff_stable silently no-opping Bump the floor to diffable-rdf 0.3.0 and add the missing uv.lock entry: the dependency was declared in pyproject.toml but never locked, so "uv lock --check" and the "uv sync --frozen" anti-malware gate would both have failed CI. 0.3.0 also fixes two defects in the Weisfeiler-Lehman labelling this feature relies on. Disconnected blank-node components now converge independently, so an edit in one region no longer relabels an unrelated one. And the suffix used to tell structurally indistinguishable nodes apart was assigned in c14nN *text* order, so c14n10 sorted between c14n1 and c14n2 -- adding a tenth tied blank node relabelled eight of the nine already there, the exact opposite of what this labelling is for. Separately, diff_stable=True was silently ignored whenever pyoxigraph refused the graph and canonicalize_rdf_graph degraded to rdflib. Weisfeiler-Lehman refinement consumes canonical pyoxigraph quads, and that path exists precisely because there are none, so the argument could not be honoured -- but the caller was never told. "shaclgen --include-annotations --diff-stable" reaches it, via the literal predicate an annotation tag without a ':' produces, and returned output byte-identical to --no-diff-stable. It now warns, with a regression test asserting the warning and the byte-identical output that makes silence misleading. --- packages/linkml_runtime/pyproject.toml | 2 +- .../linkml_runtime/utils/rdf_canonicalize.py | 20 ++++++++++++++- .../test_utils/test_rdf_canonicalize.py | 25 +++++++++++++++++++ uv.lock | 15 +++++++++++ 4 files changed, 60 insertions(+), 2 deletions(-) diff --git a/packages/linkml_runtime/pyproject.toml b/packages/linkml_runtime/pyproject.toml index b879dec7ee..ede7112692 100644 --- a/packages/linkml_runtime/pyproject.toml +++ b/packages/linkml_runtime/pyproject.toml @@ -48,7 +48,7 @@ dependencies = [ "prefixmaps >=0.1.4", "curies>=0.14.6", "pyoxigraph>=0.5.11", - "diffable-rdf>=0.2.0", + "diffable-rdf>=0.3.0", "pydantic>=2.13.5,<3.0.0", "isodate >=0.7.2, <1.0.0; python_version < '3.11'", ] diff --git a/packages/linkml_runtime/src/linkml_runtime/utils/rdf_canonicalize.py b/packages/linkml_runtime/src/linkml_runtime/utils/rdf_canonicalize.py index e0fadae34a..8e28488d55 100644 --- a/packages/linkml_runtime/src/linkml_runtime/utils/rdf_canonicalize.py +++ b/packages/linkml_runtime/src/linkml_runtime/utils/rdf_canonicalize.py @@ -307,7 +307,8 @@ def canonicalize_rdf_graph( editing one part of a schema does not renumber unrelated blank nodes. Output is deterministic and isomorphic either way; only the choice of label changes. Off by default because enabling it relabels existing - output. + output. Has no effect on the rdflib fallback path (non-standard RDF), + which warns rather than silently ignoring the request. :return: Deterministic string serialization of the graph. """ ox_format = _FORMAT_MAP.get(output_format.lower()) @@ -338,6 +339,23 @@ def canonicalize_rdf_graph( RDFCanonicalizationWarning, stacklevel=2, ) + if diff_stable: + # Weisfeiler-Lehman refinement consumes canonical pyoxigraph quads, + # and this path exists precisely because pyoxigraph refused the + # graph, so there are none to refine. The fallback is deterministic + # but not diff-stable: say so rather than returning output that + # silently ignores the argument. ``shaclgen --include-annotations`` + # reaches this path, because an annotation tag without a ``:`` + # becomes a literal predicate. + warnings.warn( + "diff_stable=True was requested but this graph took the rdflib fallback, " + "which cannot apply Weisfeiler-Lehman blank-node labels. Output is " + "deterministic but NOT diff-stable: an unrelated edit may still renumber " + "blank nodes. Make the offending terms standard RDF (absolute IRIs, IRI " + "predicates) to get diff-stable labels.", + RDFCanonicalizationWarning, + stacklevel=2, + ) return _deterministic_fallback_serialize(graph, output_format) dataset = ox.Dataset() diff --git a/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py b/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py index 0ca79c3c90..1bc07583af 100644 --- a/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py +++ b/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py @@ -642,3 +642,28 @@ def test_diff_stable_reaches_every_rdf_generator(tmp_path, generator, kwargs): f"{generator.__name__} still emits RDFC-1.0 blank-node labels with diff_stable=True; " "the flag is probably not threaded into canonicalize_rdf_graph()" ) + + +def test_diff_stable_warns_instead_of_silently_no_opping_on_the_fallback(): + """A request the fallback cannot honour must be reported, not ignored. + + ``wl_relabel_quads`` consumes canonical pyoxigraph quads, and the rdflib + fallback exists precisely because pyoxigraph refused the graph. Returning + the same bytes for ``diff_stable=True`` and ``diff_stable=False`` without + saying so lets a caller believe the output is diff-stable when it is not. + """ + graph = _make_graph_with_bnodes() + # A relative IRI is non-standard RDF, so pyoxigraph rejects the graph and + # canonicalize_rdf_graph degrades to rdflib -- the same path that + # ``shaclgen --include-annotations`` takes via its literal predicates. + graph.add((URIRef("testing"), URIRef("http://example.com/p"), Literal("v"))) + + with pytest.warns(RDFCanonicalizationWarning, match="NOT diff-stable"): + stable = canonicalize_rdf_graph(graph, diff_stable=True) + + with pytest.warns(RDFCanonicalizationWarning): + plain = canonicalize_rdf_graph(graph, diff_stable=False) + + # The warning is the contract: the bytes really are identical, which is + # exactly why staying silent would be misleading. + assert stable == plain diff --git a/uv.lock b/uv.lock index d8cddaa9e1..e862cfc897 100644 --- a/uv.lock +++ b/uv.lock @@ -1085,6 +1085,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/84/d0/205d54408c08b13550c733c4b85429e7ead111c7f0014309637425520a9a/deprecated-1.3.1-py2.py3-none-any.whl", hash = "sha256:597bfef186b6f60181535a29fbe44865ce137a5079f295b479886c82729d5f3f", size = 11298, upload-time = "2025-10-30T08:19:00.758Z" }, ] +[[package]] +name = "diffable-rdf" +version = "0.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyoxigraph" }, + { name = "rdflib" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fe/f3/45771c536aea850f2884206349e35910bbabcaebeec890c5c0cc08b040f9/diffable_rdf-0.3.0.tar.gz", hash = "sha256:6fcf84d9323cd35fc75f423bd73836469cfad198ebb7e6ab86a8743724cd000d", size = 829395, upload-time = "2026-09-11T08:00:26.268Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/27/22/ef80dba96cd558f59d2ab225602bde318a925d437866379303e1f41a4e96/diffable_rdf-0.3.0-py3-none-any.whl", hash = "sha256:8756137423f9f28beacb5cd99ba8d66317328a0c936b1dbb4527831c7d9c6784", size = 43578, upload-time = "2026-09-11T08:00:24.542Z" }, +] + [[package]] name = "distlib" version = "0.4.0" @@ -2618,6 +2631,7 @@ dependencies = [ { name = "click" }, { name = "curies" }, { name = "deprecated" }, + { name = "diffable-rdf" }, { name = "hbreader" }, { name = "isodate", marker = "python_full_version < '3.11'" }, { name = "json-flattener" }, @@ -2650,6 +2664,7 @@ requires-dist = [ { name = "coverage", marker = "extra == 'dev'" }, { name = "curies", specifier = ">=0.14.6" }, { name = "deprecated" }, + { name = "diffable-rdf", specifier = ">=0.3.0" }, { name = "hbreader" }, { name = "isodate", marker = "python_full_version < '3.11'", specifier = ">=0.7.2,<1.0.0" }, { name = "json-flattener", specifier = ">=0.1.9" }, From 544b6e8c3e3c86485520623637719527d74d05f5 Mon Sep 17 00:00:00 2001 From: Carlo van Driesten Date: Fri, 11 Sep 2026 13:45:48 +0200 Subject: [PATCH 03/14] build(deps): require diffable-rdf 0.4.0 0.4.0 carries graph.base through the library's rdflib fallback, verifying that every absolute IRI of the source survives a re-read rather than dropping the directive outright, and adds a diff_stable parameter to canonicalize_rdf_graph. The lock entry is written by hand because the workspace sets exclude-newer = "7 days", which filters any release younger than that from resolution; 0.3.0 was pinned the same way for the same reason, and both become resolvable normally on 2026-09-18. uv lock --check and uv sync --all-groups both accept the entry. https://github.com/ASCS-eV/diffable-rdf/releases/tag/v0.4.0 --- packages/linkml_runtime/pyproject.toml | 2 +- uv.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/linkml_runtime/pyproject.toml b/packages/linkml_runtime/pyproject.toml index ede7112692..c940eeac56 100644 --- a/packages/linkml_runtime/pyproject.toml +++ b/packages/linkml_runtime/pyproject.toml @@ -48,7 +48,7 @@ dependencies = [ "prefixmaps >=0.1.4", "curies>=0.14.6", "pyoxigraph>=0.5.11", - "diffable-rdf>=0.3.0", + "diffable-rdf>=0.4.0", "pydantic>=2.13.5,<3.0.0", "isodate >=0.7.2, <1.0.0; python_version < '3.11'", ] diff --git a/uv.lock b/uv.lock index e862cfc897..3c867304c8 100644 --- a/uv.lock +++ b/uv.lock @@ -1087,15 +1087,15 @@ wheels = [ [[package]] name = "diffable-rdf" -version = "0.3.0" +version = "0.4.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pyoxigraph" }, { name = "rdflib" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/fe/f3/45771c536aea850f2884206349e35910bbabcaebeec890c5c0cc08b040f9/diffable_rdf-0.3.0.tar.gz", hash = "sha256:6fcf84d9323cd35fc75f423bd73836469cfad198ebb7e6ab86a8743724cd000d", size = 829395, upload-time = "2026-09-11T08:00:26.268Z" } +sdist = { url = "https://files.pythonhosted.org/packages/86/df/1d71c0c984eac1cfd25531aabf84528941b01083c875da49d44882d1b5af/diffable_rdf-0.4.0.tar.gz", hash = "sha256:a84afaa10332e6a039b6ddcabf76bced3049e4da467c88ee786d0fa35218111e", size = 838040, upload-time = "2026-09-11T11:43:12.844Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/27/22/ef80dba96cd558f59d2ab225602bde318a925d437866379303e1f41a4e96/diffable_rdf-0.3.0-py3-none-any.whl", hash = "sha256:8756137423f9f28beacb5cd99ba8d66317328a0c936b1dbb4527831c7d9c6784", size = 43578, upload-time = "2026-09-11T08:00:24.542Z" }, + { url = "https://files.pythonhosted.org/packages/7f/49/b62ee864fe6769b759f04e3709fd79711b1cdf49223b1bb0e985e507ead3/diffable_rdf-0.4.0-py3-none-any.whl", hash = "sha256:c59c57429465f786fdbc8d38ddbb687da16748c5752f0c2ac4a77b5527aa9aed", size = 46126, upload-time = "2026-09-11T11:43:11.536Z" }, ] [[package]] @@ -2664,7 +2664,7 @@ requires-dist = [ { name = "coverage", marker = "extra == 'dev'" }, { name = "curies", specifier = ">=0.14.6" }, { name = "deprecated" }, - { name = "diffable-rdf", specifier = ">=0.3.0" }, + { name = "diffable-rdf", specifier = ">=0.4.0" }, { name = "hbreader" }, { name = "isodate", marker = "python_full_version < '3.11'", specifier = ">=0.7.2,<1.0.0" }, { name = "json-flattener", specifier = ">=0.1.9" }, From c783edbabf9a503e22a6bfc5b82d7ac1e6d18ff8 Mon Sep 17 00:00:00 2001 From: Carlo van Driesten Date: Fri, 11 Sep 2026 11:08:44 +0200 Subject: [PATCH 04/14] test(rdf): record the canonicalizer gaps against the extracted library Asserts each correctness property against both linkml's copy and diffable_rdf, marking whichever implementation does not hold it as a strict xfail, so the file is a ratchet in both directions. Nine gaps run one way, two of them silent data corruption. One ran the other way -- the library dropped @base on the degraded path -- and that was the last property blocking delegation. diffable-rdf 0.4.0 fixed it, so that case now passes on both sides and carries no mark. --- .../test_rdf_canonicalize_defects.py | 353 ++++++++++++++++++ 1 file changed, 353 insertions(+) create mode 100644 tests/linkml_runtime/test_utils/test_rdf_canonicalize_defects.py diff --git a/tests/linkml_runtime/test_utils/test_rdf_canonicalize_defects.py b/tests/linkml_runtime/test_utils/test_rdf_canonicalize_defects.py new file mode 100644 index 0000000000..806be2e3eb --- /dev/null +++ b/tests/linkml_runtime/test_utils/test_rdf_canonicalize_defects.py @@ -0,0 +1,353 @@ +"""Correctness differences between linkml's RDF canonicalizer and the extracted library. + +``linkml_runtime.utils.rdf_canonicalize`` and +``diffable_rdf.canonicalize_rdf_graph`` are the same code: the module was +extracted into a standalone library at the maintainers' request +(linkml/linkml#3295). The two copies have since diverged, so each test here +asserts one correctness property against *both* implementations and marks the +one that does not hold it as a strict xfail. + +Every remaining gap runs one way -- the extracted copy received fixes the local +one did not -- and two of them are silent data corruption, where the output +parses cleanly and says something the input never said. That is worse than a +crash, because a generated artifact gets committed and reviewed on the +assumption that it means what the schema meant. + +One gap used to run the other way: the library dropped ``@base`` on the +degraded path. That was the last property blocking delegation, and +diffable-rdf 0.4.0 fixed it, so the case now passes on both sides and is kept +unmarked. Asserting both directions is what made the fix land where it +belongs. + +The xfails are strict, so this file is a ratchet in both directions: when a fix +lands on either side, its case starts passing and the suite fails until the +mark is removed. +""" + +import os +import subprocess +import sys +import textwrap +import warnings + +import diffable_rdf +import pytest +import rdflib +from rdflib import BNode, Graph, Literal, Namespace, URIRef +from rdflib.namespace import RDF + +from linkml_runtime.utils.rdf_canonicalize import canonicalize_rdf_graph as linkml_canonicalize + +EX = Namespace("http://example.org/") + +_IMPLEMENTATIONS = ( + (linkml_canonicalize, "linkml"), + (diffable_rdf.canonicalize_rdf_graph, "diffable-rdf"), +) + + +def _impls(**known_failures: str): + """Parametrize over both implementations, xfailing the ones named. + + :param known_failures: implementation id (``linkml`` / ``diffable_rdf``) + mapped to why that implementation does not hold the property. + """ + params = [] + for fn, ident in _IMPLEMENTATIONS: + reason = known_failures.get(ident.replace("-", "_")) + marks = [pytest.mark.xfail(strict=True, reason=reason)] if reason else [] + params.append(pytest.param(fn, id=ident, marks=marks)) + return params + + +def _apply(fn, graph, output_format="turtle"): + """Call an implementation, ignoring the warnings both emit on the fallback path.""" + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + return fn(graph, output_format) + + +# -------------------------------------------------------------------------- +# Silent data corruption +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "canonicalize", + _impls(linkml="passes graph.base to the serializer without checking it is safe to relativize against"), +) +def test_base_with_a_fragment_does_not_rewrite_every_iri(canonicalize): + """A base IRI ending in ``#`` must not change the graph's terms. + + Relativizing ``http://ex.org/d#a`` against base ``http://ex.org/d#`` gives + ``<#a>``, which is correct per RFC 3986 -- but rdflib's parser resolves a + fragment reference by concatenation and reads it back as + ``http://ex.org/d##a``. Every term of the graph silently changes, and the + output parses cleanly, so nothing reports it. + """ + graph = Graph(base="http://ex.org/d#") + graph.add((URIRef("http://ex.org/d#a"), EX.p, Literal("v"))) + + round_tripped = Graph() + round_tripped.parse(data=_apply(canonicalize, graph), format="turtle") + + assert {str(s) for s in round_tripped.subjects()} == {"http://ex.org/d#a"} + + +@pytest.mark.parametrize( + "canonicalize", + _impls(linkml="rdflib's collection syntax materializes a shared list tail once per referencing list"), +) +def test_a_shared_rdf_list_tail_is_not_duplicated(canonicalize): + """Two lists sharing a tail must not gain triples on the way out. + + rdflib's Turtle writer renders ``( ... )`` collection syntax per list, so a + tail referenced from two lists is written twice as two separate blank + nodes. The result asserts more than the input did. + """ + graph = Graph() + graph.bind("ex", EX) + tail = BNode() + graph.add((tail, RDF.first, Literal("shared"))) + graph.add((tail, RDF.rest, RDF.nil)) + for name in ("l1", "l2"): + head = BNode() + graph.add((EX[name], EX.list, head)) + graph.add((head, RDF.first, Literal(name))) + graph.add((head, RDF.rest, tail)) + # A relative IRI is non-standard RDF, so pyoxigraph refuses the graph and + # both implementations degrade to rdflib -- the path this defect lives on. + graph.add((URIRef("testing"), EX.p, Literal("forces-fallback"))) + + round_tripped = Graph() + round_tripped.parse(data=_apply(canonicalize, graph), format="turtle") + + assert len(round_tripped) == len(graph) + + +# -------------------------------------------------------------------------- +# Output that no parser will read +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "canonicalize", + _impls(linkml="the rdflib fallback writes N-Triples for a graph pyoxigraph already refused"), +) +def test_nt_output_either_parses_or_refuses(canonicalize): + """N-Triples admits only absolute IRIs, so a relative one must not be written. + + The fallback is taken *because* the graph holds a term pyoxigraph rejected. + Writing it out anyway produces a file that fails on line 1. Refusing with a + message naming the term and a format that can carry the graph is the fix; + returning unreadable text is not. + """ + graph = Graph() + graph.add((URIRef("testing"), EX.p, Literal("v"))) + + try: + serialized = _apply(canonicalize, graph, "nt") + except ValueError: + return # refused, with an explanation -- the correct outcome + + rdflib.Graph().parse(data=serialized, format="nt") + + +@pytest.mark.parametrize( + "canonicalize", + _impls(linkml="sorts fallback output with str.splitlines(), which breaks on U+2028 and five other characters"), +) +def test_a_unicode_line_separator_in_a_literal_survives(canonicalize): + """Sorting N-Triples lines must split on newlines only. + + N-Triples permits U+2028 raw inside a quoted literal, but + ``str.splitlines()`` breaks on it as well as on U+2029, U+0085, U+000B, + U+000C and U+001C-1E. One statement becomes two lines, the halves sort + independently, the separator is rewritten as a newline, and the document + no longer parses. + """ + graph = Graph() + graph.add((EX.s, EX.p, Literal("a\u2028b"))) + graph.add((URIRef("testing"), EX.p, Literal("forces-fallback"))) + + try: + serialized = _apply(canonicalize, graph, "nt") + except ValueError: + return # refused because of the relative IRI, before the sort + + assert "\u2028" in serialized + rdflib.Graph().parse(data=serialized, format="nt") + + +# -------------------------------------------------------------------------- +# Non-determinism, which is the property the module exists to provide +# -------------------------------------------------------------------------- + +_ACROSS_PROCESSES = textwrap.dedent( + """ + import sys, warnings + warnings.simplefilter("ignore") + from rdflib import BNode, Graph, Literal, Namespace, URIRef + from rdflib.namespace import RDF + + module, output_format, force_fallback = sys.argv[1], sys.argv[2], sys.argv[3] == "1" + if module == "linkml": + from linkml_runtime.utils.rdf_canonicalize import canonicalize_rdf_graph + else: + from diffable_rdf import canonicalize_rdf_graph + + EX = Namespace("http://example.org/") + g = Graph() + if force_fallback: + g.add((URIRef("testing"), EX.p, Literal("forces-fallback"))) + for i in range(12): + b = BNode() + g.add((EX["s%02d" % i], EX.child, b)) + g.add((b, EX.k, Literal("v%d" % i))) + g.add((b, RDF.type, EX.T)) + for host in "abcdefgh": + g.add((URIRef("http://%s.example/s" % host), + URIRef("http://%s.example/p" % host), + URIRef("http://%s.example/o" % host))) + sys.stdout.write(canonicalize_rdf_graph(g, output_format)) + """ +) + + +def _outputs_across_processes(module, output_format, force_fallback): + """Serialize the same graph in four processes with different hash seeds.""" + outputs = set() + for seed in ("0", "1", "12345", "999"): + completed = subprocess.run( + [sys.executable, "-c", _ACROSS_PROCESSES, module, output_format, "1" if force_fallback else "0"], + capture_output=True, + text=True, + encoding="utf-8", + env={**os.environ, "PYTHONHASHSEED": seed}, + check=False, + ) + if completed.returncode != 0: + pytest.fail(f"{module}/{output_format} failed:\n{completed.stderr[-2000:]}") + outputs.add(completed.stdout) + return outputs + + +_XML_TRAVERSAL = "degraded RDF/XML is ordered by rdflib's graph traversal, which follows set iteration order" +_NS_NAMES = "auto-generated ns1/ns2 prefix names are allocated in traversal order" +_JSONLD_UNMAPPED = "json-ld is absent from the format map, so it falls through to rdflib with no determinism guarantee" + + +@pytest.mark.parametrize( + ("module", "output_format", "force_fallback"), + [ + pytest.param( + "linkml", "xml", True, id="linkml-xml-fallback", marks=pytest.mark.xfail(strict=True, reason=_XML_TRAVERSAL) + ), + pytest.param("diffable_rdf", "xml", True, id="diffable-rdf-xml-fallback"), + pytest.param( + "linkml", + "turtle", + True, + id="linkml-turtle-fallback", + marks=pytest.mark.xfail(strict=True, reason=_NS_NAMES), + ), + pytest.param("diffable_rdf", "turtle", True, id="diffable-rdf-turtle-fallback"), + pytest.param( + "linkml", + "json-ld", + False, + id="linkml-jsonld", + marks=pytest.mark.xfail(strict=True, reason=_JSONLD_UNMAPPED), + ), + pytest.param("diffable_rdf", "json-ld", False, id="diffable-rdf-jsonld"), + ], +) +def test_output_is_byte_identical_across_processes(module, output_format, force_fallback): + """The same graph must serialize to the same bytes in any process. + + Three separate causes break this in the local copy. Degraded RDF/XML is + ordered by rdflib's own graph traversal, which follows set iteration order. + Auto-generated ``ns1``/``ns2`` prefix names are allocated in traversal order + too, so the *names* move even when the triples do not. And ``json-ld`` is + not in the format map at all, so it falls through to rdflib's serializer, + which carries no determinism guarantee. + """ + assert len(_outputs_across_processes(module, output_format, force_fallback)) == 1 + + +# -------------------------------------------------------------------------- +# Interface +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "canonicalize", + _impls(linkml="returns the serializer's own trailing whitespace, which differs by format"), +) +def test_every_format_ends_with_exactly_one_newline(canonicalize): + """A committed artifact should not depend on which format wrote it. + + Passing through whatever the serializer produced gives no trailing newline + for RDF/XML and one for Turtle, so POSIX text tools disagree about whether + the file has a last line. + """ + counts = {} + for output_format in ("turtle", "nt", "xml", "trig", "n3"): + graph = Graph() + graph.bind("ex", EX) + graph.add((EX.s, EX.p, Literal("v"))) + serialized = _apply(canonicalize, graph, output_format) + counts[output_format] = len(serialized) - len(serialized.rstrip("\n")) + + assert counts == dict.fromkeys(counts, 1) + + +@pytest.mark.parametrize( + "canonicalize", + _impls(linkml="Dataset is a Graph subclass, so the type check accepts it and the named graphs are flattened"), +) +def test_a_dataset_is_refused_rather_than_partly_serialized(canonicalize): + """A Dataset is a Graph subclass, so it is accepted and quietly flattened. + + Every named graph collapses into one document and the graph names are + dropped, which is a different dataset. The caller asked for something this + function cannot express and should be told so. + """ + dataset = rdflib.Dataset() + dataset.graph(URIRef("http://ex/g1")).add((EX.a, EX.p, Literal("in-g1"))) + dataset.graph(URIRef("http://ex/g2")).add((EX.b, EX.p, Literal("in-g2"))) + + with pytest.raises(TypeError): + _apply(canonicalize, dataset) + + +# -------------------------------------------------------------------------- +# A property both implementations now hold +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize("canonicalize", _impls()) +def test_base_survives_the_degraded_path(canonicalize): + """``@base`` must not disappear just because pyoxigraph refused the graph. + + ``rdflib_dumper.dumps(..., prefix_map={"@base": ...})`` is a supported way + to ask for a document whose IRIs are written relative to a base, and the + metamodel routinely produces graphs that fall back (a bare ``status: + testing`` on a ``uriorcurie`` slot serializes to the relative ````, + which pyoxigraph rejects). Losing the directive on exactly that path means + the feature works only for graphs that never needed the fallback. + + This ran the other way until diffable-rdf 0.4.0, which was the last + property blocking delegation. The case is kept because it is the one the + two implementations reach differently: linkml passes ``graph.base`` to the + serializer unconditionally, which is why it also fails + :func:`test_base_with_a_fragment_does_not_rewrite_every_iri` above, while + the library keeps the base only after confirming that re-reading the + result still yields every absolute IRI of the source. + """ + graph = Graph(base="http://example.org/default/") + graph.bind("ex", EX) + graph.add((EX.s, EX.p, Literal("v"))) + graph.add((URIRef("testing"), EX.p, Literal("forces-fallback"))) + + assert "@base ." in _apply(canonicalize, graph) From afabdc477addc91a019898ba1cb0014ad622d60c Mon Sep 17 00:00:00 2001 From: Carlo van Driesten Date: Fri, 11 Sep 2026 14:05:26 +0200 Subject: [PATCH 05/14] refactor(rdf): delegate canonicalization to the diffable-rdf library The implementation was extracted into diffable-rdf at the maintainers' request in linkml/linkml#3295, but linkml kept its own copy and the two drifted. This deletes the copy and calls the library, which is what the extraction was for. Nine correctness fixes come with it, each already asserted in test_rdf_canonicalize_defects.py and each previously a strict xfail on the linkml side: - a base ending in # no longer rewrites every IRI that merely shares its prefix (silent corruption: output parsed, meaning changed) - a shared rdf:List tail is no longer duplicated (9 triples in, 11 out) - N-Triples refuses a relative IRI instead of writing a file its own parser rejects - literals containing U+2028, U+2029, U+0085 and the other separators str.splitlines() treats as line breaks survive the line sort - degraded RDF/XML, degraded Turtle and json-ld are byte-identical across processes - every format ends with exactly one newline - a Dataset is refused rather than silently flattened Behaviour changes for callers: nt output for a graph containing a relative IRI now raises ValueError rather than writing an unparseable file, and json-ld is canonicalized rather than handed to rdflib, so it no longer warns. All four RDF generators produce byte-identical output. The library reports degradation through logging; linkml reports it through warnings so it is visible without logging configuration. _DegradedPathWarnings bridges the two, and the tests pin the properties that makes load-bearing: the warning is attributed to the caller's line, the library's logger is left as it was found, a caller who configured logging still receives the record, and warnings survive an exception. --- .../linkml_runtime/utils/rdf_canonicalize.py | 499 ++++-------------- .../test_utils/test_rdf_canonicalize.py | 190 ++++++- .../test_rdf_canonicalize_defects.py | 129 ++--- 3 files changed, 311 insertions(+), 507 deletions(-) diff --git a/packages/linkml_runtime/src/linkml_runtime/utils/rdf_canonicalize.py b/packages/linkml_runtime/src/linkml_runtime/utils/rdf_canonicalize.py index 8e28488d55..47e8dc2682 100644 --- a/packages/linkml_runtime/src/linkml_runtime/utils/rdf_canonicalize.py +++ b/packages/linkml_runtime/src/linkml_runtime/utils/rdf_canonicalize.py @@ -1,49 +1,41 @@ -"""Deterministic RDF serialization via pyoxigraph RDFC-1.0 canonicalization. - -This module provides a function to canonicalize an rdflib Graph using -pyoxigraph's RDFC-1.0 implementation, producing deterministic output -with stable blank node labels and sorted triples. - -**Known limitations:** - -1. **xsd:string normalization**: pyoxigraph follows RDF 1.1, where plain - string literals and ``"text"^^xsd:string`` are identical. The output - will never contain explicit ``^^xsd:string`` annotations. Code that - re-parses the output with rdflib will see ``Literal("x")`` (datatype - ``None``) rather than ``Literal("x", datatype=XSD.string)``. - -2. **Non-standard RDF**: Graphs with literal predicates (e.g. SHACL - annotation mode) or relative IRIs (e.g. the metamodel's - ``bibo:status ``) are rejected by pyoxigraph. This function - falls back to rdflib's serializer for such graphs, but still - canonicalizes blank-node labels (and sorts line-oriented formats) so - the fallback output remains deterministic across processes. - -3. **Numeric short forms**: pyoxigraph uses Turtle short forms for - ``xsd:integer`` (``42``), ``xsd:boolean`` (``true``), and - ``xsd:decimal`` (``1.23``). rdflib parses these back with the - correct datatype, so this is lossless. - -4. **Base IRI / prefix collision**: When a graph has ``@base`` and a - prefix whose namespace equals the base IRI (e.g. rdflib's auto-bound - ``base:`` prefix), pyoxigraph emits CURIEs like ``base:label`` that - rdflib rejects. We skip such prefixes during serialization. - -5. **Trailing escaped dot in PN_LOCAL**: pyoxigraph emits CURIEs like - ``prefix:local\\.`` for IRIs whose local part ends with ``.``. This - is valid Turtle (PN_LOCAL_ESC), but rdflib's notation3 parser rejects - it because it conflicts with the statement-terminator dot. We - post-process the output to expand such CURIEs to full ```` form. +"""Deterministic RDF serialization, delegated to the ``diffable-rdf`` library. + +This module used to carry the implementation. It was extracted into +`diffable-rdf `_ at the maintainers' +request (linkml/linkml#3295: *"the implementation should live elsewhere ... +independent rdflib sidecar library?"*), and this module is now a thin adapter +over it. :func:`canonicalize_rdf_graph` keeps its signature, so nothing that +imports it needs to change. + +The adapter exists for one reason: the two projects report degraded output +differently. The library logs through the ``logging`` module, which is silent +unless the application configured a handler. linkml deliberately uses +:func:`warnings.warn` so a schema author running ``gen-owl`` sees that the +output took a fallback path without having to opt in to logging first. So the +library's warnings are captured and re-emitted as +:class:`RDFCanonicalizationWarning`. + +What the library does, in short: the graph is transferred to pyoxigraph via +N-Triples, canonicalized with RDFC-1.0, sorted, and serialized back. Graphs +pyoxigraph refuses -- literal predicates from SHACL annotation mode, relative +IRIs such as the metamodel's ``bibo:status `` -- fall back to rdflib +with blank-node labels canonicalized by :func:`rdflib.compare.to_canonical_graph`, +which is content-derived rather than run-local, so the fallback is still +reproducible across processes. The full contract, including the limitations +that used to be listed here (``xsd:string`` normalization, numeric short forms, +base/prefix collisions, ``PN_LOCAL`` escaping), is documented in the library's +``docs/api.md``. + +Delegating also adopts nine correctness fixes the extracted copy received and +this one never did, two of them silent data corruption. Each is pinned by a +test in ``tests/linkml_runtime/test_utils/test_rdf_canonicalize_defects.py``. """ -import io -import re +import logging import warnings -import pyoxigraph as ox import rdflib -from diffable_rdf import wl_relabel_quads -from rdflib.compare import to_canonical_graph +from diffable_rdf import canonicalize_rdf_graph as _canonicalize_rdf_graph class RDFCanonicalizationWarning(UserWarning): @@ -56,233 +48,59 @@ class RDFCanonicalizationWarning(UserWarning): """ -# Mapping from rdflib/LinkML format strings to pyoxigraph RdfFormat objects. -_FORMAT_MAP: dict[str, ox.RdfFormat] = { - "turtle": ox.RdfFormat.TURTLE, - "ttl": ox.RdfFormat.TURTLE, - "nt": ox.RdfFormat.N_TRIPLES, - "ntriples": ox.RdfFormat.N_TRIPLES, - "n-triples": ox.RdfFormat.N_TRIPLES, - "nt11": ox.RdfFormat.N_TRIPLES, - "nquads": ox.RdfFormat.N_QUADS, - "n-quads": ox.RdfFormat.N_QUADS, - "xml": ox.RdfFormat.RDF_XML, - "rdf/xml": ox.RdfFormat.RDF_XML, - "trig": ox.RdfFormat.TRIG, - "n3": ox.RdfFormat.N3, -} - -# Formats that support prefix declarations. -_PREFIX_FORMATS = frozenset({ox.RdfFormat.TURTLE, ox.RdfFormat.TRIG, ox.RdfFormat.N3, ox.RdfFormat.RDF_XML}) - -# Line-oriented formats (one triple/quad per line) whose fallback output must -# be sorted, because rdflib does not emit them in a stable order even after -# blank-node canonicalization. -_LINE_ORIENTED_FORMATS = frozenset({"nt", "ntriples", "n-triples", "nt11", "nquads", "n-quads"}) - - -def _deterministic_fallback_serialize(graph: rdflib.Graph, output_format: str) -> str: - """Serialize a graph that pyoxigraph cannot canonicalize, deterministically. - - pyoxigraph rejects some graphs that rdflib accepts -- notably graphs - containing relative IRIs (e.g. the metamodel's ``bibo:status ``) - or literal predicates (SHACL annotation mode). A plain - ``graph.serialize()`` for such graphs is *not* reproducible across - processes: rdflib assigns blank-node labels non-deterministically, so the - structure and grouping of the output varies run to run. - - To degrade gracefully instead of silently emitting non-deterministic - output, we canonicalize blank-node labels with rdflib's own - isomorphism-based canonicalization (:func:`rdflib.compare.to_canonical_graph`, - which uses a content-derived hash, not run-local ids) and preserve the - original prefix and base bindings that the canonical graph drops. For - line-oriented formats we additionally sort the serialized lines, since - rdflib does not emit N-Triples/N-Quads in a stable order. - - Relative IRIs are preserved verbatim (not resolved against the base): the - goal is deterministic output, and silently rewriting ```` into an - absolute IRI would mask what is really a data problem in the source graph. - - :param graph: The rdflib Graph that pyoxigraph could not parse. - :param output_format: Target serialization format (e.g. ``"turtle"``, ``"nt"``). - :return: Deterministic string serialization of the graph. - """ - canonical = to_canonical_graph(graph) - # to_canonical_graph builds a fresh graph without the source's namespace - # bindings; rebind them so the output does not fall back to rdflib's - # non-deterministic auto-generated ``ns1:``/``ns2:`` prefixes. - for prefix, namespace in graph.namespace_manager.namespaces(): - canonical.namespace_manager.bind(prefix, namespace, replace=True) - canonical.base = graph.base - serialized = canonical.serialize(format=output_format) - if output_format.lower() in _LINE_ORIENTED_FORMATS: - lines = [line for line in serialized.splitlines() if line.strip()] - return "\n".join(sorted(lines)) + "\n" - return serialized - - -# Characters that may appear escaped in a Turtle PN_LOCAL via PN_LOCAL_ESC. -_PN_LOCAL_ESC_UNESCAPE = re.compile(r"\\([_~.\-!$&'()*+,;=/?#@%])") - +_LIBRARY_LOGGER = "diffable_rdf" -_CURIE_TRAILING_DOT_PATTERN = re.compile( - r"(?\"'\[\]]*?\\\.)" - r"(?=\s)" -) +class _DegradedPathWarnings(logging.Handler): + """Collect the library's warning logs and re-emit them as Python warnings. -def _iter_turtle_structural_spans(turtle_text: str): - """Yield ``(start, end, is_structural)`` spans of a Turtle string. + Records are collected during the call and re-emitted in :meth:`__exit__` + rather than from :meth:`emit`. Warning at the moment the record arrives + would put eight ``logging`` frames plus an unknown number of library frames + between :func:`warnings.warn` and the caller, and the library's depth + differs per message, so no single ``stacklevel`` could point at the caller. + Re-emitting from ``__exit__`` makes the distance a constant. - Structural spans are regions outside of string literals, IRI refs, and - line comments — i.e. the only places where a CURIE can syntactically - appear. Non-structural spans (literal contents, IRI bodies, comments) - are yielded as-is so they round-trip unchanged. + The handler is attached to the library's package logger rather than the + root logger. The library logs under module-level children such as + ``diffable_rdf.canonicalize``, which propagate to the package logger, so + attaching there catches every module without naming any of them. - Handles single- and triple-quoted literals with backslash escapes for - both ``"`` and ``'`` delimiters, ``<...>`` IRI refs, and ``#...`` line - comments. + ``propagate`` is left alone: a caller who *did* configure logging still + gets the record through their own handlers, and seeing the message once + per mechanism is a better trade than suppressing a handler they asked for. """ - n = len(turtle_text) - i = 0 - while i < n: - ch = turtle_text[i] - if ch in ('"', "'"): - # String literal — find matching delimiter, respecting triple- - # quote form and backslash escapes. - delim = ch - triple = turtle_text[i : i + 3] == delim * 3 - end_marker = delim * 3 if triple else delim - j = i + len(end_marker) - while j < n: - if turtle_text[j] == "\\" and j + 1 < n: - j += 2 - continue - if turtle_text[j : j + len(end_marker)] == end_marker: - j += len(end_marker) - break - j += 1 - yield i, j, False - i = j - elif ch == "<": - # IRI ref — find closing '>' on the same logical line. - j = turtle_text.find(">", i + 1) - if j == -1: - yield i, n, False - i = n - else: - yield i, j + 1, False - i = j + 1 - elif ch == "#": - # Line comment — to end of line. - j = turtle_text.find("\n", i) - j = n if j == -1 else j - yield i, j, False - i = j - else: - # Structural region — accumulate until the next literal / IRI / - # comment opener. Treat ``\X`` as a PN_LOCAL_ESC escape (two - # chars) so that ``\#`` and ``\.`` inside a CURIE local part - # don't trigger comment / boundary handling. - start = i - while i < n: - c = turtle_text[i] - if c == "\\" and i + 1 < n: - i += 2 - continue - if c in ('"', "'", "<", "#"): - break - i += 1 - yield start, i, True - -def _expand_trailing_dot_curies(turtle_text: str, prefixes: dict[str, str]) -> str: - """Replace CURIEs whose local part ends in ``\\.`` with full ```` form. - - rdflib's notation3 parser rejects PN_LOCAL ending in an escaped dot - even though Turtle permits it (PN_LOCAL_ESC). pyoxigraph emits this - form for IRIs ending in ``.`` (e.g. ``biolink:StrandEnum#.``). We - rewrite each such CURIE to its expanded ```` form so the output - round-trips through rdflib. - - The rewrite is applied only to *structural* spans of the document — - string literals, IRI refs, and comments are left untouched. This is - what prevents the regex from mangling CURIE-shaped substrings that - happen to appear inside a literal value. - """ - if not prefixes: - return turtle_text - - def replace(match: re.Match[str]) -> str: - prefix = match.group(1) - local_escaped = match.group(2) - namespace = prefixes.get(prefix) - if namespace is None: - return match.group(0) - local = _PN_LOCAL_ESC_UNESCAPE.sub(r"\1", local_escaped) - return f"<{namespace}{local}>" - - parts: list[str] = [] - for start, end, is_structural in _iter_turtle_structural_spans(turtle_text): - chunk = turtle_text[start:end] - if is_structural: - chunk = _CURIE_TRAILING_DOT_PATTERN.sub(replace, chunk) - parts.append(chunk) - return "".join(parts) - - -def _iri_terms(triples: list["ox.Triple"]) -> set[str]: - """Return the set of IRI strings appearing anywhere in ``triples``. - - Walks subjects, predicates, non-literal objects, and literal datatypes. - Used to filter the prefix dict down to namespaces that are actually - referenced by the canonicalized graph, so the output isn't padded with - unused ``@prefix`` declarations. - """ - iris: set[str] = set() - for t in triples: - for term in (t.subject, t.predicate, t.object): - if isinstance(term, ox.NamedNode): - iris.add(term.value) - elif isinstance(term, ox.Literal): - dt = term.datatype - if dt is not None: - iris.add(dt.value) - return iris - - -def _filter_prefixes_to_used(prefixes: dict[str, str], used_iris: set[str]) -> dict[str, str]: - """Drop prefix bindings whose namespace is not a prefix of any used IRI. - - A prefix is kept if at least one IRI in ``used_iris`` starts with its - namespace string. Parent-namespace matches are honored (e.g. a prefix - bound to ``http://schema.org/`` is kept when ``http://schema.org/Person`` - appears in the graph). - """ - return {prefix: ns for prefix, ns in prefixes.items() if any(iri.startswith(ns) for iri in used_iris)} - - -def _is_safe_prefix_iri(iri: str) -> bool: - """Check whether a namespace IRI is safe for prefix serialization. - - pyoxigraph rejects IRIs with invalid code-points (e.g. double ``#``), - and rdflib's Turtle parser cannot round-trip CURIEs whose namespace - contains query parameters or fragments in unexpected positions. This - function returns ``False`` for such IRIs so they can be skipped during - prefix collection. - """ - # A namespace IRI should end with '/' or '#'. If '#' appears - # *before* the final character, the IRI contains an embedded - # fragment which produces unusable CURIEs. - if "#" in iri[:-1]: - return False - # Query parameters in namespace IRIs produce CURIEs that rdflib - # cannot parse back. - if "?" in iri: - return False - return True + def __init__(self) -> None: + super().__init__(level=logging.WARNING) + self._messages: list[str] = [] + self._logger = logging.getLogger(_LIBRARY_LOGGER) + self._previous_level = logging.NOTSET + + def emit(self, record: logging.LogRecord) -> None: + """Record one formatted message for re-emission on exit.""" + self._messages.append(record.getMessage()) + + def __enter__(self) -> "_DegradedPathWarnings": + """Attach to the library's logger, raising its level if it is silent.""" + self._previous_level = self._logger.level + if not self._logger.isEnabledFor(logging.WARNING): + self._logger.setLevel(logging.WARNING) + self._logger.addHandler(self) + return self + + def __exit__(self, *exc_info: object) -> None: + """Detach, restore the logger, and re-emit what was collected. + + Runs on the exception path too, so a caller who gets an exception still + learns about the degradation that preceded it. + """ + self._logger.removeHandler(self) + self._logger.setLevel(self._previous_level) + for message in self._messages: + # 3 frames: warnings.warn, this method, and the ``with`` statement + # in canonicalize_rdf_graph -- so the warning lands on its caller. + warnings.warn(message, RDFCanonicalizationWarning, stacklevel=3) def canonicalize_rdf_graph( @@ -292,13 +110,15 @@ def canonicalize_rdf_graph( ) -> str: """Serialize an rdflib Graph deterministically using RDFC-1.0 canonicalization. - The graph is transferred to pyoxigraph via N-Triples, canonicalized - with RDFC-1.0, sorted, and serialized back to the requested format. - Prefix bindings from the rdflib Graph are preserved in the output - for formats that support them (Turtle, TriG, N3, RDF/XML). + The graph is transferred to pyoxigraph via N-Triples, canonicalized with + RDFC-1.0, sorted, and serialized back to the requested format. Prefix + bindings from the rdflib Graph are preserved in the output for formats that + support them (Turtle, TriG, N3, RDF/XML). - Falls back to plain rdflib serialization for unsupported formats or - graphs containing non-standard RDF (e.g. literal predicates). + Falls back to plain rdflib serialization for unsupported formats or graphs + containing non-standard RDF (e.g. literal predicates), warning with + :class:`RDFCanonicalizationWarning` so that the caller knows the output is + less strongly guaranteed than usual. :param graph: The rdflib Graph to serialize. :param output_format: Target serialization format (e.g. ``"turtle"``, ``"nt"``). @@ -310,142 +130,11 @@ def canonicalize_rdf_graph( output. Has no effect on the rdflib fallback path (non-standard RDF), which warns rather than silently ignoring the request. :return: Deterministic string serialization of the graph. + :raises ValueError: If the graph cannot be serialized to ``output_format`` + in a form that parses back -- notably a line-oriented format asked to + write a relative IRI, which N-Triples forbids. """ - ox_format = _FORMAT_MAP.get(output_format.lower()) - if ox_format is None: - warnings.warn( - f"pyoxigraph does not support format {output_format!r}; falling back to " - "rdflib serializer. Output will not be deterministically canonicalized.", - RDFCanonicalizationWarning, - stacklevel=2, - ) - return graph.serialize(format=output_format) - - # 1. Transfer rdflib graph to pyoxigraph via N-Triples. - nt_data = graph.serialize(format="nt") - nt_bytes = nt_data.encode("utf-8") if isinstance(nt_data, str) else nt_data - - # 2. Parse into pyoxigraph and build a Dataset for canonicalization. - # Fall back to rdflib if the graph contains non-standard RDF - # (e.g. literal predicates from annotations) that pyoxigraph rejects. - try: - triples = list(ox.parse(io.BytesIO(nt_bytes), format=ox.RdfFormat.N_TRIPLES)) - except SyntaxError: - warnings.warn( - "Graph contains non-standard RDF (e.g. relative IRIs or literal predicates) " - "that pyoxigraph cannot parse; falling back to rdflib. Output is still " - "deterministic (blank-node labels are canonicalized via rdflib) but is not " - "canonicalized with pyoxigraph RDFC-1.0.", - RDFCanonicalizationWarning, - stacklevel=2, - ) - if diff_stable: - # Weisfeiler-Lehman refinement consumes canonical pyoxigraph quads, - # and this path exists precisely because pyoxigraph refused the - # graph, so there are none to refine. The fallback is deterministic - # but not diff-stable: say so rather than returning output that - # silently ignores the argument. ``shaclgen --include-annotations`` - # reaches this path, because an annotation tag without a ``:`` - # becomes a literal predicate. - warnings.warn( - "diff_stable=True was requested but this graph took the rdflib fallback, " - "which cannot apply Weisfeiler-Lehman blank-node labels. Output is " - "deterministic but NOT diff-stable: an unrelated edit may still renumber " - "blank nodes. Make the offending terms standard RDF (absolute IRIs, IRI " - "predicates) to get diff-stable labels.", - RDFCanonicalizationWarning, - stacklevel=2, - ) - return _deterministic_fallback_serialize(graph, output_format) - - dataset = ox.Dataset() - for triple in triples: - dataset.add(ox.Quad(triple.subject, triple.predicate, triple.object, ox.DefaultGraph())) - - # 3. Canonicalize blank node labels with RDFC-1.0. - dataset.canonicalize(ox.CanonicalizationAlgorithm.RDFC_1_0) - - quads = list(dataset) - - # 3b. Optionally re-label blank nodes with locality-sensitive hashes. - # RDFC-1.0 guarantees that identical graphs get identical labels, but it - # does not guarantee that *similar* graphs get similar labels: the labels - # are assigned by a global ordering, so inserting one blank node can - # renumber every label after it and turn a one-line semantic change into a - # whole-file diff. Weisfeiler-Lehman labels are derived only from each - # node's local neighbourhood, so unrelated regions keep their labels. - # Output stays deterministic and isomorphic either way; this only changes - # which label each blank node receives. - if diff_stable: - quads = wl_relabel_quads(quads) - - # 4. Sort triples for deterministic ordering. - # RDFC-1.0 stabilizes blank-node labels but pyoxigraph's Dataset - # iteration order is not sorted and varies across processes (verified - # empirically against pyoxigraph 0.5.8). The explicit string-key sort - # is load-bearing for byte-identical output across runs; see - # tests/linkml_runtime/test_utils/test_rdf_canonicalize.py::test_sort_is_load_bearing. - sorted_triples = sorted( - (ox.Triple(q.subject, q.predicate, q.object) for q in quads), - key=lambda t: (str(t.subject), str(t.predicate), str(t.object)), - ) - - # 5. Collect prefixes for formats that support them. - base_iri = str(graph.base) if graph.base else None - prefixes: dict[str, str] | None = None - if ox_format in _PREFIX_FORMATS: - prefixes = {} - for prefix, namespace in graph.namespace_manager.namespaces(): - if not prefix: # skip empty prefix (base) - continue - ns_str = str(namespace) - # Skip prefixes whose namespace matches the base IRI to avoid - # pyoxigraph emitting CURIEs like `base:label` that conflict - # with the @base directive. - if base_iri and ns_str == base_iri: - continue - # Skip namespace IRIs that pyoxigraph rejects or that produce - # CURIEs rdflib cannot round-trip. Valid namespace IRIs for - # prefix use should end with '/' or '#' and contain no query - # parameters or fragment-like characters in the middle. - if not _is_safe_prefix_iri(ns_str): - continue - prefixes[str(prefix)] = ns_str - # Drop prefix bindings whose namespace is not referenced by any IRI - # in the graph. This prevents the rdflib NamespaceManager's default - # bindings (~30 well-known vocabularies) from being emitted into - # every output file regardless of whether the schema actually uses - # them. - prefixes = _filter_prefixes_to_used(prefixes, _iri_terms(sorted_triples)) - used_prefixes = prefixes - try: - result_bytes = ox.serialize( - sorted_triples, - format=ox_format, - prefixes=prefixes, - base_iri=base_iri, - ) - except ValueError as e: - # pyoxigraph 0.5.x reports rejected prefix IRIs with a message - # that begins with "Invalid prefix" (verified empirically). Only - # swallow that case and retry without prefixes — any other - # ValueError (e.g. invalid base IRI, or an unrelated future - # serializer bug) must propagate so it surfaces as a stack trace - # rather than silently dropping all prefix declarations. - if not str(e).startswith("Invalid prefix"): - raise - warnings.warn( - f"pyoxigraph rejected one or more prefix IRIs ({e}); serializing without " - "prefix declarations. Output remains canonicalized but is more verbose.", - RDFCanonicalizationWarning, - stacklevel=2, - ) - result_bytes = ox.serialize( - sorted_triples, - format=ox_format, - ) - used_prefixes = None - result = result_bytes.decode("utf-8") - if ox_format in _PREFIX_FORMATS and used_prefixes: - result = _expand_trailing_dot_curies(result, used_prefixes) - return result + # Frames between warnings.warn and this function's caller are counted in + # _DegradedPathWarnings.__exit__, which is where the re-emission happens. + with _DegradedPathWarnings(): + return _canonicalize_rdf_graph(graph, output_format=output_format, diff_stable=diff_stable) diff --git a/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py b/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py index 1bc07583af..5a813729dd 100644 --- a/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py +++ b/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py @@ -1,10 +1,14 @@ """Tests for deterministic RDF serialization via pyoxigraph RDFC-1.0.""" +import json +import logging import os import re import subprocess import sys import textwrap +import warnings +from pathlib import Path import pyoxigraph as ox import pytest @@ -12,7 +16,6 @@ from rdflib import BNode, Graph, Literal, URIRef from rdflib.namespace import RDF -from linkml_runtime.utils import rdf_canonicalize as rdf_canon_mod from linkml_runtime.utils.rdf_canonicalize import ( RDFCanonicalizationWarning, canonicalize_rdf_graph, @@ -304,13 +307,13 @@ def fake_serialize(*args, **kwargs): raise ValueError("Invalid prefix bad IRI 'http://example.com/ has space/', Invalid IRI code point ' '") return real_serialize(*args, **kwargs) - monkeypatch.setattr(rdf_canon_mod.ox, "serialize", fake_serialize) + monkeypatch.setattr(ox, "serialize", fake_serialize) g = Graph() g.bind("ex", "http://example.com/") g.add((URIRef("http://example.com/a"), RDF.type, URIRef("http://example.com/Thing"))) - with pytest.warns(RDFCanonicalizationWarning, match="rejected one or more prefix IRIs"): + with pytest.warns(RDFCanonicalizationWarning, match="rejected the prefix or base IRIs"): ttl = canonicalize_rdf_graph(g, output_format="turtle") # Fallback retry should succeed and the output still round-trips. g2 = Graph() @@ -319,22 +322,74 @@ def fake_serialize(*args, **kwargs): def test_unsupported_format_warns_and_falls_back(): - """A format pyoxigraph doesn't know about falls back to rdflib with a warning.""" + """A format the canonicalizer doesn't know about falls back to rdflib with a warning. + + ``longturtle`` is an rdflib serializer plugin with no pyoxigraph + equivalent, so it can only be produced by delegating to rdflib, and + rdflib orders that output by graph traversal. The warning is what tells + the caller the determinism guarantee does not extend to this format. + """ g = Graph() g.bind("ex", "http://example.com/") g.add((URIRef("http://example.com/a"), RDF.type, URIRef("http://example.com/Thing"))) - with pytest.warns(RDFCanonicalizationWarning, match="does not support format"): - result = canonicalize_rdf_graph(g, output_format="json-ld") + with pytest.warns(RDFCanonicalizationWarning, match="not one of the formats"): + result = canonicalize_rdf_graph(g, output_format="longturtle") # The fallback should still produce valid output in the requested format. assert result +def test_json_ld_is_canonicalized_rather_than_handed_to_rdflib(): + """``json-ld`` gets the determinism guarantee, not a fallback warning. + + rdflib's JSON-LD serializer emits objects in traversal order, so this used + to be an unsupported format that warned and returned whatever rdflib felt + like. It is now produced deterministically, which is why no warning is due. + """ + g = _make_graph_with_bnodes() + with warnings.catch_warnings(): + warnings.simplefilter("error", RDFCanonicalizationWarning) + result = canonicalize_rdf_graph(g, output_format="json-ld") + assert json.loads(result) + + +def test_a_rejected_base_iri_is_recovered_rather_than_fatal(monkeypatch): + """A base pyoxigraph refuses costs the base directive, not the document. + + rdflib accepts a relative or otherwise unusable ``base``, and pyoxigraph + then refuses it at serialization time. Dropping the base and retrying + yields a correct document; failing the whole call over a directive that is + optional in every output format does not. + """ + real_serialize = ox.serialize + calls = {"n": 0} + + def fake_serialize(*args, **kwargs): + calls["n"] += 1 + if calls["n"] == 1: + raise ValueError("Invalid base IRI 'broken', Invalid IRI code point ' '") + return real_serialize(*args, **kwargs) + + monkeypatch.setattr(ox, "serialize", fake_serialize) + + g = Graph() + g.bind("ex", "http://example.com/") + g.add((URIRef("http://example.com/a"), RDF.type, URIRef("http://example.com/Thing"))) + + with pytest.warns(RDFCanonicalizationWarning, match="rejected the prefix or base IRIs"): + ttl = canonicalize_rdf_graph(g, output_format="turtle") + + g2 = Graph() + g2.parse(data=ttl, format="turtle") + assert rdflib.compare.isomorphic(g, g2) + + def test_unrelated_value_error_propagates(monkeypatch): - """Any ``ValueError`` whose message doesn't begin with ``Invalid prefix`` is re-raised. + """A ``ValueError`` that is not about prefixes or the base must be re-raised. - Guards against future regressions where a pyoxigraph serializer bug - raises a different ``ValueError`` and gets silently swallowed by the - invalid-prefix fallback path. + The retry-without-prefixes path exists for two specific pyoxigraph + complaints. Widening it to every ``ValueError`` would let a serializer bug + be papered over by a retry that happens to succeed, and the caller would + get a plausible-looking document with no indication anything went wrong. """ real_serialize = ox.serialize calls = {"count": 0} @@ -343,16 +398,16 @@ def fake_serialize(*args, **kwargs): calls["count"] += 1 # First call is the prefixed serialize — raise an unrelated ValueError. if calls["count"] == 1: - raise ValueError("Invalid base IRI 'broken', Invalid IRI code point ' '") + raise ValueError("BUG: serializer state corrupted") return real_serialize(*args, **kwargs) - monkeypatch.setattr(rdf_canon_mod.ox, "serialize", fake_serialize) + monkeypatch.setattr(ox, "serialize", fake_serialize) g = Graph() g.bind("ex", "http://example.com/") g.add((URIRef("http://example.com/a"), RDF.type, URIRef("http://example.com/Thing"))) - with pytest.raises(ValueError, match="Invalid base IRI"): + with pytest.raises(ValueError, match="BUG: serializer state corrupted"): canonicalize_rdf_graph(g, output_format="turtle") @@ -455,7 +510,7 @@ def test_fallback_preserves_relative_iri(): assert "" in result -@pytest.mark.parametrize("output_format", ["turtle", "nt"]) +@pytest.mark.parametrize("output_format", ["turtle", "xml"]) def test_fallback_is_deterministic_across_processes(output_format): """The rdflib fallback produces byte-identical output across processes. @@ -463,9 +518,15 @@ def test_fallback_is_deterministic_across_processes(output_format): parse to fail, exercising the fallback path. The graph also contains several blank nodes: a plain ``graph.serialize()`` would label them non-deterministically, so this test would fail without the blank-node - canonicalization in ``_deterministic_fallback_serialize``. Two - subprocesses with different ``PYTHONHASHSEED`` values must agree byte for - byte. + canonicalization in the fallback. Two subprocesses with different + ``PYTHONHASHSEED`` values must agree byte for byte. + + ``nt`` is not covered here because a fallback graph is by definition one + pyoxigraph refused, and for this graph the reason is a relative IRI, which + N-Triples forbids outright (N-Triples 1.1 §2.2). There is no deterministic + N-Triples document to produce, so the correct answer is a refusal -- + asserted by ``test_nt_fallback_refuses_rather_than_writing_an_unparseable_file`` + below. """ program = textwrap.dedent( f""" @@ -509,6 +570,24 @@ def run(seed: str) -> str: ) +def test_nt_fallback_refuses_rather_than_writing_an_unparseable_file(): + """N-Triples output for a graph with a relative IRI must raise, not lie. + + rdflib's N-Triples serializer reuses Turtle's term rendering and does not + enforce the absolute-IRI rule, so asking it for ``nt`` here yields a file + containing ```` that its own parser then rejects. N-Triples 1.1 + §2.2 permits only absolute IRIs, so no valid document exists for this + graph and a refusal is the only honest answer. Writing the file instead + defers the failure to whoever tries to read it. + """ + g = Graph() + g.bind("ex", "http://example.com/") + g.add((URIRef("http://example.com/s"), URIRef("http://purl.org/ontology/bibo/status"), URIRef("testing"))) + + with pytest.raises(ValueError, match="not an absolute IRI"): + canonicalize_rdf_graph(g, output_format="nt") + + def _shapes_graph(count: int, extra: bool = False) -> Graph: """A graph shaped like generator output: one blank node per named subject.""" g = Graph() @@ -658,7 +737,7 @@ def test_diff_stable_warns_instead_of_silently_no_opping_on_the_fallback(): # ``shaclgen --include-annotations`` takes via its literal predicates. graph.add((URIRef("testing"), URIRef("http://example.com/p"), Literal("v"))) - with pytest.warns(RDFCanonicalizationWarning, match="NOT diff-stable"): + with pytest.warns(RDFCanonicalizationWarning, match="not diff-stable"): stable = canonicalize_rdf_graph(graph, diff_stable=True) with pytest.warns(RDFCanonicalizationWarning): @@ -667,3 +746,78 @@ def test_diff_stable_warns_instead_of_silently_no_opping_on_the_fallback(): # The warning is the contract: the bytes really are identical, which is # exactly why staying silent would be misleading. assert stable == plain + + +# -------------------------------------------------------------------------- +# The bridge from the library's logging to linkml's warnings +# -------------------------------------------------------------------------- + + +def _fallback_graph() -> Graph: + """A graph pyoxigraph refuses, so every call takes a degraded path.""" + g = Graph() + g.bind("ex", "http://example.com/") + g.add((URIRef("http://example.com/s"), URIRef("http://purl.org/ontology/bibo/status"), URIRef("testing"))) + return g + + +def test_degraded_path_warning_points_at_the_caller(): + """The warning must name the line that asked for the serialization. + + ``diffable-rdf`` reports degradation through ``logging``; linkml re-emits + it through ``warnings`` so it is visible without logging configuration. + A re-emitted warning is only actionable if it is attributed to the caller + rather than to the adapter, and the number of frames in between is not + something a reader can eyeball -- hence this test. + """ + graph = _fallback_graph() + + with pytest.warns(RDFCanonicalizationWarning) as caught: + canonicalize_rdf_graph(graph, output_format="turtle") # attribution target + + assert caught[0].filename == __file__ + source = Path(caught[0].filename).read_text(encoding="utf-8").splitlines() + assert "# attribution target" in source[caught[0].lineno - 1] + + +def test_bridging_does_not_leave_the_library_logger_modified(): + """Raising the library's log level to capture records must not be permanent. + + The adapter has to enable ``WARNING`` on the ``diffable_rdf`` logger to see + anything, which is a global mutation. Leaving it raised would silently + change logging behaviour for the rest of the process. + """ + logger = logging.getLogger("diffable_rdf") + level_before = logger.level + handlers_before = list(logger.handlers) + + with pytest.warns(RDFCanonicalizationWarning): + canonicalize_rdf_graph(_fallback_graph(), output_format="turtle") + + assert logger.level == level_before + assert logger.handlers == handlers_before + + +def test_a_caller_who_configured_logging_still_receives_the_record(caplog): + """Re-emitting as a warning must not steal the record from a log handler. + + An application that deliberately configured the ``diffable_rdf`` logger is + asking for these records. The adapter adds a mechanism; it does not get to + remove one. + """ + with caplog.at_level(logging.WARNING, logger="diffable_rdf"), pytest.warns(RDFCanonicalizationWarning): + canonicalize_rdf_graph(_fallback_graph(), output_format="turtle") + + assert any("non-standard RDF" in record.getMessage() for record in caplog.records) + + +def test_warnings_survive_an_exception_from_the_library(): + """Degradation reported before a failure must still reach the caller. + + ``nt`` output for this graph degrades to rdflib and *then* refuses, because + N-Triples has no way to write the relative IRI. Dropping the warning + because the call ended in an exception would hide the first half of the + story, which is the half that explains the second. + """ + with pytest.warns(RDFCanonicalizationWarning, match="non-standard RDF"), pytest.raises(ValueError): + canonicalize_rdf_graph(_fallback_graph(), output_format="nt") diff --git a/tests/linkml_runtime/test_utils/test_rdf_canonicalize_defects.py b/tests/linkml_runtime/test_utils/test_rdf_canonicalize_defects.py index 806be2e3eb..ffca5ae447 100644 --- a/tests/linkml_runtime/test_utils/test_rdf_canonicalize_defects.py +++ b/tests/linkml_runtime/test_utils/test_rdf_canonicalize_defects.py @@ -1,27 +1,24 @@ -"""Correctness differences between linkml's RDF canonicalizer and the extracted library. +"""Correctness properties shared by linkml's RDF canonicalizer and the library behind it. ``linkml_runtime.utils.rdf_canonicalize`` and ``diffable_rdf.canonicalize_rdf_graph`` are the same code: the module was extracted into a standalone library at the maintainers' request -(linkml/linkml#3295). The two copies have since diverged, so each test here -asserts one correctness property against *both* implementations and marks the -one that does not hold it as a strict xfail. - -Every remaining gap runs one way -- the extracted copy received fixes the local -one did not -- and two of them are silent data corruption, where the output -parses cleanly and says something the input never said. That is worse than a -crash, because a generated artifact gets committed and reviewed on the -assumption that it means what the schema meant. - -One gap used to run the other way: the library dropped ``@base`` on the -degraded path. That was the last property blocking delegation, and -diffable-rdf 0.4.0 fixed it, so the case now passes on both sides and is kept -unmarked. Asserting both directions is what made the fix land where it -belongs. - -The xfails are strict, so this file is a ratchet in both directions: when a fix -lands on either side, its case starts passing and the suite fails until the -mark is removed. +(linkml/linkml#3295). The two copies then diverged for a while, and this file +is what made the divergence visible -- each test asserts one correctness +property against *both* implementations, so a fix that landed on one side and +not the other showed up as a failure rather than as nothing at all. + +Nine properties held only in the library, two of them cases of silent data +corruption where the output parsed cleanly and said something the input never +said. One held only in linkml: the library dropped ``@base`` on the degraded +path. Asserting both directions is what got each of them fixed where it +belonged -- the ``@base`` gap in diffable-rdf 0.4.0, the other nine in linkml +by deleting the in-tree copy and calling the library. + +Every case is now unmarked, which is the point: the file is the evidence that +the delegation changed no behaviour it should not have, and it keeps running +against both entry points so that a future divergence fails the suite instead +of going unnoticed. """ import os @@ -46,18 +43,15 @@ ) -def _impls(**known_failures: str): - """Parametrize over both implementations, xfailing the ones named. +def _impls(): + """Parametrize a test over both entry points. - :param known_failures: implementation id (``linkml`` / ``diffable_rdf``) - mapped to why that implementation does not hold the property. + linkml's is a thin adapter over the library's, so the two agree by + construction today. Running both anyway is what turns a future re-fork, or + an adapter that quietly changes behaviour on the way through, into a test + failure. """ - params = [] - for fn, ident in _IMPLEMENTATIONS: - reason = known_failures.get(ident.replace("-", "_")) - marks = [pytest.mark.xfail(strict=True, reason=reason)] if reason else [] - params.append(pytest.param(fn, id=ident, marks=marks)) - return params + return [pytest.param(fn, id=ident) for fn, ident in _IMPLEMENTATIONS] def _apply(fn, graph, output_format="turtle"): @@ -72,10 +66,7 @@ def _apply(fn, graph, output_format="turtle"): # -------------------------------------------------------------------------- -@pytest.mark.parametrize( - "canonicalize", - _impls(linkml="passes graph.base to the serializer without checking it is safe to relativize against"), -) +@pytest.mark.parametrize("canonicalize", _impls()) def test_base_with_a_fragment_does_not_rewrite_every_iri(canonicalize): """A base IRI ending in ``#`` must not change the graph's terms. @@ -94,10 +85,7 @@ def test_base_with_a_fragment_does_not_rewrite_every_iri(canonicalize): assert {str(s) for s in round_tripped.subjects()} == {"http://ex.org/d#a"} -@pytest.mark.parametrize( - "canonicalize", - _impls(linkml="rdflib's collection syntax materializes a shared list tail once per referencing list"), -) +@pytest.mark.parametrize("canonicalize", _impls()) def test_a_shared_rdf_list_tail_is_not_duplicated(canonicalize): """Two lists sharing a tail must not gain triples on the way out. @@ -130,10 +118,7 @@ def test_a_shared_rdf_list_tail_is_not_duplicated(canonicalize): # -------------------------------------------------------------------------- -@pytest.mark.parametrize( - "canonicalize", - _impls(linkml="the rdflib fallback writes N-Triples for a graph pyoxigraph already refused"), -) +@pytest.mark.parametrize("canonicalize", _impls()) def test_nt_output_either_parses_or_refuses(canonicalize): """N-Triples admits only absolute IRIs, so a relative one must not be written. @@ -153,10 +138,7 @@ def test_nt_output_either_parses_or_refuses(canonicalize): rdflib.Graph().parse(data=serialized, format="nt") -@pytest.mark.parametrize( - "canonicalize", - _impls(linkml="sorts fallback output with str.splitlines(), which breaks on U+2028 and five other characters"), -) +@pytest.mark.parametrize("canonicalize", _impls()) def test_a_unicode_line_separator_in_a_literal_survives(canonicalize): """Sorting N-Triples lines must split on newlines only. @@ -240,37 +222,24 @@ def _outputs_across_processes(module, output_format, force_fallback): @pytest.mark.parametrize( ("module", "output_format", "force_fallback"), [ - pytest.param( - "linkml", "xml", True, id="linkml-xml-fallback", marks=pytest.mark.xfail(strict=True, reason=_XML_TRAVERSAL) - ), + pytest.param("linkml", "xml", True, id="linkml-xml-fallback"), pytest.param("diffable_rdf", "xml", True, id="diffable-rdf-xml-fallback"), - pytest.param( - "linkml", - "turtle", - True, - id="linkml-turtle-fallback", - marks=pytest.mark.xfail(strict=True, reason=_NS_NAMES), - ), + pytest.param("linkml", "turtle", True, id="linkml-turtle-fallback"), pytest.param("diffable_rdf", "turtle", True, id="diffable-rdf-turtle-fallback"), - pytest.param( - "linkml", - "json-ld", - False, - id="linkml-jsonld", - marks=pytest.mark.xfail(strict=True, reason=_JSONLD_UNMAPPED), - ), + pytest.param("linkml", "json-ld", False, id="linkml-jsonld"), pytest.param("diffable_rdf", "json-ld", False, id="diffable-rdf-jsonld"), ], ) def test_output_is_byte_identical_across_processes(module, output_format, force_fallback): """The same graph must serialize to the same bytes in any process. - Three separate causes break this in the local copy. Degraded RDF/XML is - ordered by rdflib's own graph traversal, which follows set iteration order. - Auto-generated ``ns1``/``ns2`` prefix names are allocated in traversal order - too, so the *names* move even when the triples do not. And ``json-ld`` is - not in the format map at all, so it falls through to rdflib's serializer, - which carries no determinism guarantee. + Three separate causes used to break this in the in-tree copy, and all three + are fixed by delegating. Degraded RDF/XML was ordered by rdflib's own graph + traversal, which follows set iteration order. Auto-generated ``ns1``/``ns2`` + prefix names were allocated in traversal order too, so the *names* moved + even when the triples did not. And ``json-ld`` was not in the format map at + all, so it fell through to rdflib's serializer, which carries no + determinism guarantee. """ assert len(_outputs_across_processes(module, output_format, force_fallback)) == 1 @@ -280,10 +249,7 @@ def test_output_is_byte_identical_across_processes(module, output_format, force_ # -------------------------------------------------------------------------- -@pytest.mark.parametrize( - "canonicalize", - _impls(linkml="returns the serializer's own trailing whitespace, which differs by format"), -) +@pytest.mark.parametrize("canonicalize", _impls()) def test_every_format_ends_with_exactly_one_newline(canonicalize): """A committed artifact should not depend on which format wrote it. @@ -302,10 +268,7 @@ def test_every_format_ends_with_exactly_one_newline(canonicalize): assert counts == dict.fromkeys(counts, 1) -@pytest.mark.parametrize( - "canonicalize", - _impls(linkml="Dataset is a Graph subclass, so the type check accepts it and the named graphs are flattened"), -) +@pytest.mark.parametrize("canonicalize", _impls()) def test_a_dataset_is_refused_rather_than_partly_serialized(canonicalize): """A Dataset is a Graph subclass, so it is accepted and quietly flattened. @@ -337,13 +300,11 @@ def test_base_survives_the_degraded_path(canonicalize): which pyoxigraph rejects). Losing the directive on exactly that path means the feature works only for graphs that never needed the fallback. - This ran the other way until diffable-rdf 0.4.0, which was the last - property blocking delegation. The case is kept because it is the one the - two implementations reach differently: linkml passes ``graph.base`` to the - serializer unconditionally, which is why it also fails - :func:`test_base_with_a_fragment_does_not_rewrite_every_iri` above, while - the library keeps the base only after confirming that re-reading the - result still yields every absolute IRI of the source. + This ran the other way until diffable-rdf 0.4.0. It was the last property + blocking delegation, because linkml held it and the library did not, so + adopting the library would have been a regression. Fixing it there rather + than keeping the in-tree copy alive is what let the other nine gaps close + at once. """ graph = Graph(base="http://example.org/default/") graph.bind("ex", EX) From 1f1917a3b91210341c7dda88e33aa82e482f2257 Mon Sep 17 00:00:00 2001 From: Rayene Messaoud Date: Fri, 11 Sep 2026 14:45:34 +0200 Subject: [PATCH 06/14] feat(gen-shacl): translate presence-implies-value rules to SHACL-SPARQL The rules-to-SHACL-SPARQL converter added in #3451 recognised a single named pattern. This adds the presence-implies-value pattern: a precondition asserting `value_presence: PRESENT` on one slot, and a postcondition constraining another slot with `equals_string` or `equals_string_in`. It reads as "if the guard slot is present, the target slot must be present and hold one of the allowed values", and generalises the existing boolean guard to arbitrary enum values. The boolean guard is now gated on the target slot's range actually being `boolean`. Without that gate a slot of range `string` carrying `equals_string: "true"` was translated as a boolean comparison, which does not match the string `"true"` in the data and so flagged conforming instances as violations. String-ranged slots now fall through to the presence-implies-value pattern and compare as strings. Pattern matching is exact: each converter requires its conditions to set precisely the operators it translates. A rule whose conditions carry anything further -- extra scalar operators, or expression-level any_of / all_of / none_of / exactly_one_of -- is skipped rather than partially translated, since dropping a term would either widen the precondition (false positives) or weaken the postcondition (false negatives). Slot resolution goes through induced slots so that `slot_usage` overrides, `slot_uri` overrides and alias-form keys resolve to the same IRI that `sh:path` emits. Co-authored-by: jdsika --- .../linkml/src/linkml/generators/shaclgen.py | 319 +++++- tests/linkml/test_generators/test_shaclgen.py | 925 ++++++++++++++++++ 2 files changed, 1204 insertions(+), 40 deletions(-) diff --git a/packages/linkml/src/linkml/generators/shaclgen.py b/packages/linkml/src/linkml/generators/shaclgen.py index 5d42fea272..0645f1e6ad 100644 --- a/packages/linkml/src/linkml/generators/shaclgen.py +++ b/packages/linkml/src/linkml/generators/shaclgen.py @@ -163,11 +163,15 @@ class ShaclGenerator(Generator): When ``True`` (default), recognised rule patterns are translated into SHACL-SPARQL constraints (``sh:SPARQLConstraint``) on the corresponding - ``sh:NodeShape``. Currently two patterns are recognised: + ``sh:NodeShape``. Currently three patterns are recognised: * *Boolean guard* — a precondition with ``value_presence: PRESENT`` on a value slot and a postcondition with ``equals_string: "true"`` on a boolean flag slot. + * *Presence implies value* — a precondition with ``value_presence: PRESENT`` + on a value slot and a postcondition with ``equals_string`` or + ``equals_string_in`` on a (typically enum-valued) target slot. This + generalises the boolean guard to arbitrary required values. * *Exclusive value* — a precondition with ``equals_string`` on a slot and a postcondition with ``maximum_cardinality`` on the *same* slot. @@ -445,12 +449,22 @@ def _add_rules(self, g: Graph, shape_uri: URIRef, cls: ClassDefinition) -> None: ``value_presence: PRESENT`` on a value slot and a *postcondition* with ``equals_string: "true"`` on a boolean flag slot. + * **Presence implies value** — a *precondition* with + ``value_presence: PRESENT`` on a value slot and a *postcondition* + with ``equals_string`` or ``equals_string_in`` on a target slot. + Enforces that when the value slot is present, the target slot must + be present and hold one of the allowed values (generalises the + boolean guard to enum-valued targets). + * **Exclusive value** — a *precondition* with ``equals_string`` on a slot and a *postcondition* with ``maximum_cardinality`` on the *same* slot. Enforces that when a specific value is present in a multivalued slot, the total number of values must not exceed the given cardinality (typically 1 for mutual exclusion). + Operator combinations outside these named patterns are not translated; + the rule is skipped rather than partially represented. + See `W3C SHACL §5 `_. """ if not cls.rules: @@ -478,11 +492,11 @@ def _add_rules(self, g: Graph, shape_uri: URIRef, cls: ClassDefinition) -> None: cls.name, ) - if getattr(rule, "elseconditions", None): + if getattr(rule, "elseconditions", None) is not None: logger.warning( "Rule in class %r has elseconditions; " - "only the forward (if/then) branch is emitted as sh:sparql. " - "The else branch cannot be represented in SHACL-SPARQL.", + "SHACL-SPARQL generation emits the forward (if/then) direction only. " + "The else branch is not enforced.", cls.name, ) @@ -505,22 +519,137 @@ def _add_rules(self, g: Graph, shape_uri: URIRef, cls: ClassDefinition) -> None: g.add((constraint, SH.select, Literal(sparql_query))) + # Fields on a slot condition / class expression that carry no constraint + # semantics: they never change which instances satisfy the condition, so + # they are ignored by the operator accounting below. Anything set on a + # condition that is neither here nor explicitly translated by a converter + # makes the rule untranslatable — the converters must SKIP such a rule + # rather than emit a query that silently drops a conjunct (which would + # widen the trigger or narrow the check: a mis-translation, not a skip). + _NON_OPERATOR_FIELDS = frozenset( + { + "name", + "description", + "title", + "deprecated", + "todos", + "notes", + "comments", + "examples", + "in_subset", + "from_schema", + "imported_from", + "source", + "in_language", + "see_also", + "deprecated_element_has_exact_replacement", + "deprecated_element_has_possible_replacement", + "aliases", + "structured_aliases", + "local_names", + "mappings", + "exact_mappings", + "close_mappings", + "related_mappings", + "narrow_mappings", + "broad_mappings", + "created_by", + "contributors", + "created_on", + "last_updated_on", + "modified_by", + "status", + "rank", + "categories", + "keywords", + "extensions", + "annotations", + "alt_descriptions", + "id_prefixes", + "id_prefixes_are_closed", + "definition_uri", + "conforms_to", + "implements", + "instantiates", + } + ) + + @classmethod + def _set_operator_fields(cls, cond) -> set[str]: + """Return the names of the constraint-bearing fields actually set on a + rule condition or class expression. + + A field counts as *set* when it is not ``None`` and not an empty + collection (SchemaView materialises unset multivalued fields as empty + lists / dicts). Scalars are never judged by truthiness, so legitimate + falsy constraints such as ``minimum_value: 0`` or + ``equals_string: ""`` still count as set. Metadata fields + (:data:`_NON_OPERATOR_FIELDS`) are excluded. + + The converters compare this set against the exact operator set they + translate and skip the rule on any mismatch, so an unrecognised (or + future-metamodel) operator can never be silently dropped. + """ + fields: set[str] = set() + for name, value in vars(cond).items(): + if name.startswith("_") or name in cls._NON_OPERATOR_FIELDS: + continue + if value is None: + continue + if isinstance(value, list | dict) and not value: + continue + if isinstance(value, JsonObj) and not as_dict(value): + continue + fields.add(name) + return fields + + def _rule_slot(self, sv, slot_name: str, cls: ClassDefinition): + """Resolve a rule condition's slot key to the slot it names, or ``None`` + when no such slot exists. + + Resolution order mirrors ``sh:path`` in the main slot loop: the induced + (class-specific) slot when the key names one of the class's slots, then + the underscored alias form (a rule key ``my_slot`` for a slot named + ``my slot`` — SchemaView normalises names the same way elsewhere), then + the base slot. Callers treat ``None`` as *unknown slot* and skip the + rule rather than fabricating a predicate no shape uses. + """ + class_slot_names = sv.class_slots(cls.name) + if slot_name in class_slot_names: + return sv.induced_slot(slot_name, cls.name) + canonical = next((s for s in class_slot_names if underscore(s) == underscore(slot_name)), None) + if canonical is not None: + return sv.induced_slot(canonical, cls.name) + return sv.get_slot(slot_name) + def _rule_to_sparql(self, sv, cls: ClassDefinition, rule) -> str | None: """Convert a ``ClassRule`` to a SPARQL SELECT query string. Returns ``None`` when the rule does not match any supported pattern. + Each pattern requires its conditions to set **exactly** the operators + it translates; a rule whose pre/postconditions carry anything more + (extra scalar operators, expression-level ``any_of``/``all_of``/ + ``none_of``/``exactly_one_of``, ...) is skipped rather than partially + translated. """ pre = getattr(rule, "preconditions", None) post = getattr(rule, "postconditions", None) if not pre or not post: return None - pre_slots = getattr(pre, "slot_conditions", None) or {} - post_slots = getattr(post, "slot_conditions", None) or {} + # Expression-level exactness: only a plain conjunction of slot + # conditions is translatable. An any_of/all_of/none_of/exactly_one_of + # branch cannot be honoured by any converter below; dropping it would + # widen the precondition (false positives) or weaken the postcondition + # (false negatives), so the whole rule is skipped. + if self._set_operator_fields(pre) != {"slot_conditions"}: + return None + if self._set_operator_fields(post) != {"slot_conditions"}: + return None + + pre_slots = pre.slot_conditions or {} + post_slots = post.slot_conditions or {} - # Pattern: boolean guard - # preconditions: exactly one slot with value_presence PRESENT - # postconditions: exactly one slot with equals_string "true" if len(pre_slots) == 1 and len(post_slots) == 1: pre_slot_name = next(iter(pre_slots)) post_slot_name = next(iter(post_slots)) @@ -528,31 +657,83 @@ def _rule_to_sparql(self, sv, cls: ClassDefinition, rule) -> str | None: pre_cond = pre_slots[pre_slot_name] post_cond = post_slots[post_slot_name] - # Note: PresenceEnum.PRESENT is a PermissibleValue, but parsed schemas - # return PresenceEnum instances — wrapping ensures type-compatible comparison. - is_value_present = getattr(pre_cond, "value_presence", None) == PresenceEnum(PresenceEnum.PRESENT) - is_flag_true = getattr(post_cond, "equals_string", None) == "true" + pre_ops = self._set_operator_fields(pre_cond) + post_ops = self._set_operator_fields(post_cond) - if is_value_present and is_flag_true: + is_value_present = pre_ops == {"value_presence"} and pre_cond.value_presence == PresenceEnum( + PresenceEnum.PRESENT + ) + + # Pattern: boolean guard + # preconditions: exactly one slot with (only) value_presence PRESENT + # postconditions: exactly one boolean-range slot with (only) + # equals_string "true". The range gate matters: on a non-boolean + # slot the string "true" must be compared as a string, which is + # the presence-implies-value pattern below — without the gate the + # boolean comparison mistranslates and flags conforming data. + if ( + is_value_present + and post_ops == {"equals_string"} + and post_cond.equals_string == "true" + and getattr(self._rule_slot(sv, post_slot_name, cls), "range", None) == "boolean" + ): return self._build_boolean_guard_sparql(sv, cls, post_slot_name, pre_slot_name) + # Pattern: presence implies value (enum guard) + # preconditions: value slot with (only) value_presence PRESENT + # postconditions: target slot with (only) equals_string or (only) + # equals_string_in. + # Semantics: "If the value slot is present, the target slot must be + # present and hold one of the allowed values." Generalises the + # boolean guard (equals_string "true") to arbitrary enum values. + if is_value_present and post_ops in ({"equals_string"}, {"equals_string_in"}): + if post_ops == {"equals_string_in"}: + allowed = list(post_cond.equals_string_in) + else: + allowed = [post_cond.equals_string] + return self._build_presence_implies_value_sparql(sv, cls, pre_slot_name, post_slot_name, allowed) + # Pattern: exclusive value - # preconditions: slot X has equals_string (a specific enum value) - # postconditions: same slot X has maximum_cardinality N + # preconditions: slot X with (only) equals_string (a specific enum value) + # postconditions: same slot X with (only) maximum_cardinality N # Semantics: "If value V is present in slot X, then X has at most N values." - pre_equals = getattr(pre_cond, "equals_string", None) - post_max_card = getattr(post_cond, "maximum_cardinality", None) - - if pre_equals is not None and post_max_card is not None and pre_slot_name == post_slot_name: - return self._build_exclusive_value_sparql(sv, cls, pre_slot_name, pre_equals, int(post_max_card)) + if pre_ops == {"equals_string"} and post_ops == {"maximum_cardinality"} and pre_slot_name == post_slot_name: + return self._build_exclusive_value_sparql( + sv, cls, pre_slot_name, pre_cond.equals_string, int(post_cond.maximum_cardinality) + ) + # Fallback: a small compositional builder for operator combinations not return None - def _build_boolean_guard_sparql(self, sv, cls: ClassDefinition, flag_slot_name: str, value_slot_name: str) -> str: + @staticmethod + def _sparql_string_literal(value: str) -> str: + """Render *value* as a double-quoted SPARQL string literal, escaping the + characters the grammar forbids raw. + + ``equals_string`` / permissible-value names are schema-controlled but + may legitimately contain a double quote, backslash, or newline; without + escaping these would break the ``sh:select`` query (or allow SPARQL + injection). See `SPARQL 1.1 §19.7 escape sequences + `_. + """ + escaped = ( + str(value) + .replace("\\", "\\\\") + .replace('"', '\\"') + .replace("\n", "\\n") + .replace("\r", "\\r") + .replace("\t", "\\t") + ) + return f'"{escaped}"' + + def _build_boolean_guard_sparql( + self, sv, cls: ClassDefinition, flag_slot_name: str, value_slot_name: str + ) -> str | None: """Build a SPARQL SELECT query for the boolean-guard pattern. The query detects violations where the value property is present - but the boolean flag is absent or not ``true``. + but the boolean flag is absent or not ``true``. Returns ``None`` + (rule skipped) when either slot name resolves to no slot. Conforms to `SHACL §5.3.1 `_: @@ -560,6 +741,8 @@ def _build_boolean_guard_sparql(self, sv, cls: ClassDefinition, flag_slot_name: """ flag_uri = self._slot_uri(sv, flag_slot_name, cls) value_uri = self._slot_uri(sv, value_slot_name, cls) + if flag_uri is None or value_uri is None: + return None return ( f"SELECT $this WHERE {{\n" @@ -572,6 +755,44 @@ def _build_boolean_guard_sparql(self, sv, cls: ClassDefinition, flag_slot_name: f"}}" ) + def _build_presence_implies_value_sparql( + self, + sv, + cls: ClassDefinition, + value_slot_name: str, + target_slot_name: str, + allowed_values: list[str], + ) -> str | None: + """Build a SPARQL SELECT query for the presence-implies-value pattern. + + Detects violations where the *value slot* is present but the *target + slot* is absent or holds a value outside the allowed set. This + generalises the boolean-guard pattern to enum-valued targets: it + supports a single required value (``equals_string``) or a set of + acceptable values (``equals_string_in``). + + Each allowed value is resolved via the target slot's enum ``meaning`` + to a full IRI; values without a ``meaning`` (or non-enum targets) fall + back to a plain string literal. + + Conforms to `SHACL §5.3.1 + `_: + ``$this`` is pre-bound to each focus node. + """ + value_uri = self._slot_uri(sv, value_slot_name, cls) + target_uri = self._slot_uri(sv, target_slot_name, cls) + if value_uri is None or target_uri is None: + return None + refs = ", ".join(self._resolve_enum_value_ref(sv, target_slot_name, v, cls) for v in allowed_values) + + return ( + f"SELECT $this WHERE {{\n" + f" $this <{value_uri}> ?value .\n" + f" OPTIONAL {{ $this <{target_uri}> ?target . }}\n" + f" FILTER ( !BOUND(?target) || ?target NOT IN ({refs}) )\n" + f"}}" + ) + def _build_exclusive_value_sparql( self, sv, @@ -599,7 +820,9 @@ def _build_exclusive_value_sparql( ``$this`` is pre-bound to each focus node. """ slot_uri = self._slot_uri(sv, slot_name, cls) - value_ref = self._resolve_enum_value_ref(sv, slot_name, value_name) + if slot_uri is None: + return None + value_ref = self._resolve_enum_value_ref(sv, slot_name, value_name, cls) if max_card == 1: return ( @@ -622,15 +845,20 @@ def _build_exclusive_value_sparql( f"}}" ) - def _resolve_enum_value_ref(self, sv, slot_name: str, value_name: str) -> str: + def _resolve_enum_value_ref(self, sv, slot_name: str, value_name: str, cls: ClassDefinition | None = None) -> str: """Resolve an enum value name to a SPARQL term (IRI or literal). Looks up the slot's range as an enum, finds the permissible value matching *value_name*, and returns its ``meaning`` as a full IRI - wrapped in angle brackets. Falls back to a quoted literal if the - slot is not an enum or the value lacks a ``meaning``. + wrapped in angle brackets. Falls back to an escaped quoted literal if + the slot is not an enum or the value lacks a ``meaning``. + + When *cls* is given and the slot is declared on it, the slot is resolved + in the class's induced context, so a range narrowed via ``slot_usage`` + (a class-specific enum) selects the correct permissible values instead + of the base slot's enum. """ - slot = sv.get_slot(slot_name) + slot = self._rule_slot(sv, slot_name, cls) if cls is not None else sv.get_slot(slot_name) if slot: range_name = slot.range if range_name and range_name in sv.all_enums(): @@ -639,17 +867,27 @@ def _resolve_enum_value_ref(self, sv, slot_name: str, value_name: str) -> str: if pv and pv.meaning: iri = sv.expand_curie(pv.meaning) return f"<{iri}>" - return f'"{value_name}"' - - def _slot_uri(self, sv, slot_name: str, cls: ClassDefinition) -> str: - """Resolve a slot name to a full IRI string for use in SPARQL queries. - - Mirrors the resolution logic used for ``sh:path`` in the main slot loop: - prefer ``sv.get_uri()`` for slots registered in the schema map, fall - back to ``default_prefix:underscored_name``. + return self._sparql_string_literal(value_name) + + def _slot_uri(self, sv, slot_name: str, cls: ClassDefinition) -> str | None: + """Resolve a slot name to a full IRI string for use in SPARQL queries, + or ``None`` when the name resolves to no slot at all (callers then skip + the rule). + + Mirrors the resolution logic used for ``sh:path`` in the main slot loop, + including the **induced** (class-specific) slot: a ``slot_usage`` + override of ``slot_uri`` must yield the same IRI as ``sh:path``. + Otherwise the SPARQL body would query a property the data never uses and + the constraint would silently never fire (a false negative). A slot + that resolves but is not registered in the schema's element map falls + back to ``default_prefix:underscored_name``, again matching ``sh:path``; + an *unknown* name must NOT take that fallback — it would fabricate a + predicate no shape uses and emit a vacuous constraint. """ - slot = sv.get_slot(slot_name) - if slot and slot_name in sv.element_by_schema_map(): + slot = self._rule_slot(sv, slot_name, cls) + if slot is None: + return None + if slot.name in sv.element_by_schema_map(): return sv.get_uri(slot, expand=True) pfx = sv.schema.default_prefix return sv.expand_curie(f"{pfx}:{underscore(slot_name)}") @@ -940,8 +1178,9 @@ def add_simple_data_type(func: Callable, r: ElementName) -> None: show_default=True, help=( "Emit sh:sparql constraints from LinkML rules: blocks. " - "When enabled (default), recognised rule patterns (e.g. boolean-guard) " - "are translated into SHACL-SPARQL constraints on the corresponding " + "When enabled (default), recognised rule patterns (boolean-guard, " + "presence-implies-value, exclusive-value) are translated into " + "SHACL-SPARQL constraints on the corresponding " "sh:NodeShape. Use --no-emit-rules to suppress rule generation." ), ) diff --git a/tests/linkml/test_generators/test_shaclgen.py b/tests/linkml/test_generators/test_shaclgen.py index 8604f712de..2fb7594212 100644 --- a/tests/linkml/test_generators/test_shaclgen.py +++ b/tests/linkml/test_generators/test_shaclgen.py @@ -2784,3 +2784,928 @@ def test_exclusive_value_coexists_with_boolean_guard(): has_boolean = any("BOUND" in q for q in queries) assert has_exclusive, "Expected one exclusive-value SPARQL constraint" assert has_boolean, "Expected one boolean-guard SPARQL constraint" + + +# =========================================================================== +# Presence-implies-value pattern tests (enum guard) +# =========================================================================== +# +# The "presence implies value" pattern generalises the boolean guard to +# enum-valued targets. It translates a LinkML rule where: +# - preconditions: a value slot has value_presence: PRESENT +# - postconditions: a target slot has equals_string (single required value) +# or equals_string_in (a set of acceptable values) +# +# Semantics: "If the value slot is present, the target slot must be present +# and hold one of the allowed values." The motivating use case is the aiSim +# environment model, e.g. "if texture_sky_color is set, sky_model must be +# TextureSky" and "if overcast_sky_illuminance is set, sky_model must be an +# overcast model". +# +# References: +# - W3C SHACL §5 +# - W3C SHACL §5.3.1 +# =========================================================================== + +_PRESENCE_IMPLIES_VALUE_SCHEMA_YAML = """ +id: https://example.org/presence-implies-value +name: presence_implies_value_rules +prefixes: + linkml: https://w3id.org/linkml/ + ex: https://example.org/presence-implies-value/ +imports: + - linkml:types +default_prefix: ex +default_range: string + +enums: + SkyModelEnum: + permissible_values: + ClearSky: + meaning: ex:ClearSky + OvercastSky: + meaning: ex:OvercastSky + MeasuredOvercastSky: + meaning: ex:MeasuredOvercastSky + TextureSky: + meaning: ex:TextureSky + + ModeEnum: + permissible_values: + Auto: + description: Automatic mode (no meaning IRI). + Manual: + description: Manual mode (no meaning IRI). + +slots: + sky_model: + range: SkyModelEnum + slot_uri: ex:sky_model + texture_sky_color: + range: string + slot_uri: ex:texture_sky_color + overcast_sky_illuminance: + range: float + slot_uri: ex:overcast_sky_illuminance + mode: + range: ModeEnum + slot_uri: ex:mode + manual_value: + range: decimal + slot_uri: ex:manual_value + +classes: + Weather: + class_uri: ex:Weather + slots: + - sky_model + - texture_sky_color + - overcast_sky_illuminance + rules: + - description: If texture_sky_color is provided, sky_model must be TextureSky. + preconditions: + slot_conditions: + texture_sky_color: + value_presence: PRESENT + postconditions: + slot_conditions: + sky_model: + equals_string: "TextureSky" + - description: If overcast_sky_illuminance is provided, sky_model must be an overcast model. + preconditions: + slot_conditions: + overcast_sky_illuminance: + value_presence: PRESENT + postconditions: + slot_conditions: + sky_model: + equals_string_in: + - OvercastSky + - MeasuredOvercastSky + + Device: + class_uri: ex:Device + slots: + - mode + - manual_value + rules: + - description: If manual_value is provided, mode must be Manual (literal fallback). + preconditions: + slot_conditions: + manual_value: + value_presence: PRESENT + postconditions: + slot_conditions: + mode: + equals_string: "Manual" +""" + +EX_PIV = rdflib.Namespace("https://example.org/presence-implies-value/") + + +def test_presence_implies_value_generates_sparql(): + """Presence-implies-value rules produce sh:sparql constraints on the NodeShape.""" + g = _parse_shacl(_PRESENCE_IMPLIES_VALUE_SCHEMA_YAML) + + shape = EX_PIV.Weather + sparql_nodes = list(g.objects(shape, SH.sparql)) + assert len(sparql_nodes) == 2, f"Expected 2 sh:sparql constraints, got {len(sparql_nodes)}" + + for node in sparql_nodes: + assert (node, RDF.type, SH.SPARQLConstraint) in g + selects = list(g.objects(node, SH.select)) + assert len(selects) == 1, "Each constraint must have exactly one sh:select" + query = str(selects[0]) + assert "$this" in query, "SPARQL must use $this pre-bound variable" + assert "NOT IN" in query, "presence-implies-value SPARQL must use NOT IN membership test" + assert "FILTER" in query, "SPARQL must have a FILTER clause" + + +def test_presence_implies_value_single_uses_enum_iri(): + """A single equals_string target resolves to the enum meaning IRI.""" + g = _parse_shacl(_PRESENCE_IMPLIES_VALUE_SCHEMA_YAML) + + shape = EX_PIV.Weather + sparql_nodes = list(g.objects(shape, SH.sparql)) + queries = [str(list(g.objects(n, SH.select))[0]) for n in sparql_nodes] + + texture_query = [q for q in queries if "texture_sky_color" in q] + assert len(texture_query) == 1, "Expected exactly one texture_sky_color rule" + query = texture_query[0] + + # value slot and target slot URIs both present + assert str(EX_PIV.texture_sky_color) in query + assert str(EX_PIV.sky_model) in query + # target value resolves to the TextureSky meaning IRI in angle brackets + assert f"<{EX_PIV.TextureSky}>" in query, f"Expected TextureSky IRI, got:\n{query}" + + +def test_presence_implies_value_set_uses_all_iris(): + """equals_string_in resolves every allowed value to its enum meaning IRI.""" + g = _parse_shacl(_PRESENCE_IMPLIES_VALUE_SCHEMA_YAML) + + shape = EX_PIV.Weather + sparql_nodes = list(g.objects(shape, SH.sparql)) + queries = [str(list(g.objects(n, SH.select))[0]) for n in sparql_nodes] + + overcast_query = [q for q in queries if "overcast_sky_illuminance" in q] + assert len(overcast_query) == 1, "Expected exactly one overcast rule" + query = overcast_query[0] + + assert f"<{EX_PIV.OvercastSky}>" in query, f"Expected OvercastSky IRI, got:\n{query}" + assert f"<{EX_PIV.MeasuredOvercastSky}>" in query, f"Expected MeasuredOvercastSky IRI, got:\n{query}" + + +def test_presence_implies_value_no_meaning_falls_back_to_literal(): + """When the target enum value lacks a meaning IRI, it is compared as a literal.""" + g = _parse_shacl(_PRESENCE_IMPLIES_VALUE_SCHEMA_YAML) + + shape = EX_PIV.Device + sparql_nodes = list(g.objects(shape, SH.sparql)) + assert len(sparql_nodes) == 1 + + query = str(list(g.objects(sparql_nodes[0], SH.select))[0]) + assert '"Manual"' in query, f"No-meaning enum should use literal '\"Manual\"', got:\n{query}" + assert f"<{EX_PIV}Manual>" not in query, "Should not emit as IRI when meaning is absent" + + +def test_presence_implies_value_message_from_description(): + """Rule description is emitted as sh:message on the SPARQLConstraint.""" + g = _parse_shacl(_PRESENCE_IMPLIES_VALUE_SCHEMA_YAML) + + shape = EX_PIV.Weather + sparql_nodes = list(g.objects(shape, SH.sparql)) + messages = [str(m) for node in sparql_nodes for m in g.objects(node, SH.message)] + + assert any("sky_model must be TextureSky" in m for m in messages), ( + f"Expected message about TextureSky, got: {messages}" + ) + + +def test_presence_implies_value_sparql_syntax_valid(): + """Generated SPARQL for presence-implies-value rules must be syntactically valid.""" + from rdflib.plugins.sparql import prepareQuery + + g = _parse_shacl(_PRESENCE_IMPLIES_VALUE_SCHEMA_YAML) + + for shape in (EX_PIV.Weather, EX_PIV.Device): + sparql_nodes = list(g.objects(shape, SH.sparql)) + for node in sparql_nodes: + query_text = str(list(g.objects(node, SH.select))[0]) + prepareQuery(query_text) + + +def test_presence_implies_value_pyshacl_end_to_end(): + """End-to-end: pyshacl passes conforming instances and flags violations.""" + import pyshacl + + shacl_ttl = ShaclGenerator(_PRESENCE_IMPLIES_VALUE_SCHEMA_YAML, mergeimports=False, emit_rules=True).serialize() + + # Conforming: guarded slots paired with an allowed sky_model; and an + # unguarded instance (no texture/overcast) is unaffected by the rules. + conforming_data = """ + @prefix ex: . + @prefix xsd: . + + ex:wTexture a ex:Weather ; + ex:texture_sky_color "0,0,0" ; + ex:sky_model ex:TextureSky . + + ex:wOvercast a ex:Weather ; + ex:overcast_sky_illuminance "5000.0"^^xsd:float ; + ex:sky_model ex:OvercastSky . + + ex:wMeasured a ex:Weather ; + ex:overcast_sky_illuminance "4200.0"^^xsd:float ; + ex:sky_model ex:MeasuredOvercastSky . + + ex:wClear a ex:Weather ; + ex:sky_model ex:ClearSky . + """ + + conforms, _, results_text = pyshacl.validate( + data_graph=conforming_data, + shacl_graph=shacl_ttl, + data_graph_format="turtle", + shacl_graph_format="turtle", + advanced=True, + ) + assert conforms, f"Conforming instances should pass SHACL validation:\n{results_text}" + + # Violating: texture_sky_color present but sky_model is ClearSky (not TextureSky). + violating_wrong_value = """ + @prefix ex: . + + ex:wBad a ex:Weather ; + ex:texture_sky_color "0,0,0" ; + ex:sky_model ex:ClearSky . + """ + conforms, _, results_text = pyshacl.validate( + data_graph=violating_wrong_value, + shacl_graph=shacl_ttl, + data_graph_format="turtle", + shacl_graph_format="turtle", + advanced=True, + ) + assert not conforms, f"Wrong-value instance should fail SHACL validation:\n{results_text}" + + # Violating: overcast_sky_illuminance present but sky_model is TextureSky + # (not in the allowed overcast set). + violating_not_in_set = """ + @prefix ex: . + @prefix xsd: . + + ex:wBad2 a ex:Weather ; + ex:overcast_sky_illuminance "5000.0"^^xsd:float ; + ex:sky_model ex:TextureSky . + """ + conforms, _, results_text = pyshacl.validate( + data_graph=violating_not_in_set, + shacl_graph=shacl_ttl, + data_graph_format="turtle", + shacl_graph_format="turtle", + advanced=True, + ) + assert not conforms, f"Not-in-set instance should fail SHACL validation:\n{results_text}" + + # Violating: texture_sky_color present but sky_model entirely absent. + violating_missing_target = """ + @prefix ex: . + + ex:wBad3 a ex:Weather ; + ex:texture_sky_color "0,0,0" . + """ + conforms, _, results_text = pyshacl.validate( + data_graph=violating_missing_target, + shacl_graph=shacl_ttl, + data_graph_format="turtle", + shacl_graph_format="turtle", + advanced=True, + ) + assert not conforms, f"Missing-target instance should fail SHACL validation:\n{results_text}" + + +# =========================================================================== +# Compositional fallback: conditional-required pattern (M1) +# =========================================================================== +# +# Rule shape: +# - preconditions: slot X has equals_string V +# - postconditions: slot Y has required: true +# +# Semantics: "If X = V, then Y must be present." Emitted as an +# sh:SPARQLConstraint whose SELECT matches focus nodes where the precondition +# holds but the required slot is absent (FILTER NOT EXISTS). +# =========================================================================== + + +# =========================================================================== +# Compositional fallback: conditional-absent pattern (M2) +# =========================================================================== +# +# Rule shape: +# - preconditions: slot X has equals_string V +# - postconditions: slot Y has value_presence: ABSENT +# +# Semantics: "If X = V, then Y must NOT be present" (inapplicable slot). +# Emitted as an sh:SPARQLConstraint whose SELECT matches focus nodes where the +# precondition holds and the forbidden slot is present. +# =========================================================================== + + +# =========================================================================== +# Compositional fallback: numeric threshold precondition (M3) +# =========================================================================== +# +# Rule shape: +# - preconditions: slot X has maximum_value N (or minimum_value) +# - postconditions: slot Y has required: true +# +# Semantics: "If X <= N, then Y must be present." The threshold becomes a +# SPARQL FILTER; combined here with the M1 required violation. +# =========================================================================== + + +# =========================================================================== +# Compositional fallback: nested range_expression precondition (M4) +# =========================================================================== +# +# Rule shape: +# - preconditions: slot X (inlined child) has range_expression on an inner +# slot (e.g. sun_position.elevation <= 0) +# - postconditions: slot Y has required: true +# +# Semantics: "If the child's inner value satisfies the condition, then Y must +# be present." The SPARQL binds the child node with one extra hop. +# =========================================================================== + + +# =========================================================================== +# Compositional fallback: has_member list-membership postcondition (M5) +# =========================================================================== +# +# Rule shape: +# - preconditions: any supported precondition (here value_presence PRESENT) +# - postconditions: multivalued slot has_member with a nested +# range_expression constraining the member's inner slots +# +# Semantics: "If the precondition holds, the list must contain a member +# matching the inner conditions." Violation = no such member (FILTER NOT +# EXISTS over the members). Inner enum values resolve against the member +# class (LightControlGroup), which disambiguates the reused `type` slot. +# =========================================================================== + + +# =========================================================================== +# Rule-converter robustness regressions (review hardening) +# +# These guard three defects found while reviewing the rule converters: +# 1. A single precondition combining minimum_value + maximum_value dropped +# all but the first bound (silent under-constraint / false positives). +# 2. A slot_usage `slot_uri` (or enum `range`) override made the SPARQL body +# query the *base* IRI while `sh:path` used the *induced* IRI, so the +# constraint silently never fired (false negative). +# 3. An `equals_string` value containing a quote/backslash produced invalid, +# unparsable SPARQL (broken artifact / injection). +# =========================================================================== + + +_ENUM_NARROWING_SCHEMA_YAML = """ +id: https://example.org/enum-narrowing +name: enum_narrowing_rules +prefixes: + linkml: https://w3id.org/linkml/ + ex: https://example.org/enum-narrowing/ +imports: + - linkml:types +default_prefix: ex +default_range: string + +enums: + BaseMode: + permissible_values: + Active: + meaning: ex:GLOBAL_Active + SceneMode: + permissible_values: + Active: + meaning: ex:LOCAL_Active + +slots: + activator: + range: string + slot_uri: ex:activator + mode: + range: BaseMode + slot_uri: ex:mode + +classes: + Scene: + class_uri: ex:Scene + slots: + - activator + - mode + slot_usage: + mode: + range: SceneMode + rules: + - description: If an activator is present the mode must be Active. + preconditions: + slot_conditions: + activator: + value_presence: PRESENT + postconditions: + slot_conditions: + mode: + equals_string: Active +""" + +EX_EN = rdflib.Namespace("https://example.org/enum-narrowing/") + + +def test_rule_enum_range_narrowed_by_slot_usage(): + """A slot_usage range override to a class-specific enum must resolve the + value's meaning against the induced (narrowed) enum, not the base range.""" + g = _parse_shacl(_ENUM_NARROWING_SCHEMA_YAML) + + nodes = list(g.objects(EX_EN.Scene, SH.sparql)) + assert len(nodes) == 1 + query = str(list(g.objects(nodes[0], SH.select))[0]) + assert str(EX_EN.LOCAL_Active) in query, f"must resolve the narrowed enum meaning, got:\n{query}" + assert "GLOBAL_Active" not in query, f"must not resolve the base enum meaning, got:\n{query}" + + +_ESCAPING_SCHEMA_YAML = """ +id: https://example.org/escaping +name: escaping_rules +prefixes: + linkml: https://w3id.org/linkml/ + ex: https://example.org/escaping/ +imports: + - linkml:types +default_prefix: ex +default_range: string + +slots: + trigger: + range: string + slot_uri: ex:trigger + label: + range: string + slot_uri: ex:label + +classes: + Item: + class_uri: ex:Item + slots: + - trigger + - label + rules: + - description: If a trigger is present the label must equal the quoted marker. + preconditions: + slot_conditions: + trigger: + value_presence: PRESENT + postconditions: + slot_conditions: + label: + equals_string: 'a"b\\\\c' +""" + +EX_ESC = rdflib.Namespace("https://example.org/escaping/") + + +def test_rule_equals_string_special_chars_escaped(): + """An equals_string value with a quote and backslash must be escaped so the + generated SPARQL stays syntactically valid (no injection / broken query).""" + from rdflib.plugins.sparql import prepareQuery + + g = _parse_shacl(_ESCAPING_SCHEMA_YAML) + nodes = list(g.objects(EX_ESC.Item, SH.sparql)) + assert len(nodes) == 1 + query = str(list(g.objects(nodes[0], SH.select))[0]) + + # Would raise ParseException on the unescaped `... = "a"b\c"` form. + prepareQuery(query) + assert '\\"' in query, f"double quote must be escaped, got:\n{query}" + assert "\\\\" in query, f"backslash must be escaped, got:\n{query}" + + +# =========================================================================== +# Audit-fix regression tests: operator exactness, nested-slot resolution, +# numeric bound gating, elseconditions warning +# =========================================================================== + +_PIV_EXTRA_PRE_SCHEMA_YAML = """ +id: https://example.org/piv-extra-pre +name: piv_extra_pre +prefixes: + linkml: https://w3id.org/linkml/ + ex: https://example.org/piv-extra-pre/ +imports: + - linkml:types +default_prefix: ex +default_range: string +slots: + temp: + range: integer + slot_uri: ex:temp + mode: + range: string + slot_uri: ex:mode +classes: + Device: + class_uri: ex:Device + slots: [temp, mode] + rules: + - description: Above 100 the mode must be High (extra precondition operator). + preconditions: + slot_conditions: + temp: + value_presence: PRESENT + minimum_value: 100 + postconditions: + slot_conditions: + mode: + equals_string: "High" +""" + + +def test_rule_extra_precondition_operator_skipped(): + """A precondition combining PRESENT with a threshold must not dispatch to + presence-implies-value: dropping the threshold widens the trigger.""" + g = _parse_shacl(_PIV_EXTRA_PRE_SCHEMA_YAML) + shape = URIRef("https://example.org/piv-extra-pre/Device") + assert list(g.objects(shape, SH.sparql)) == [], "rule with an untranslated conjunct must be skipped" + + +def test_rule_extra_precondition_operator_pyshacl_end_to_end(): + """A device below the threshold satisfies the rule vacuously and must conform.""" + import pyshacl + + shacl_ttl = ShaclGenerator(_PIV_EXTRA_PRE_SCHEMA_YAML, mergeimports=False, emit_rules=True).serialize() + data = """ + @prefix ex: . + + ex:cool a ex:Device ; ex:temp 50 ; ex:mode "Low" . + """ + conforms, _, txt = pyshacl.validate( + data_graph=data, + shacl_graph=shacl_ttl, + data_graph_format="turtle", + shacl_graph_format="turtle", + advanced=True, + ) + assert conforms, f"Below-threshold device must not be flagged:\n{txt}" + + +_POST_BOTH_EQUALS_SCHEMA_YAML = """ +id: https://example.org/post-both-equals +name: post_both_equals +prefixes: + linkml: https://w3id.org/linkml/ + ex: https://example.org/post-both-equals/ +imports: + - linkml:types +default_prefix: ex +default_range: string +slots: + guard: + slot_uri: ex:guard + target: + slot_uri: ex:target +classes: + Thing: + class_uri: ex:Thing + slots: [guard, target] + rules: + - preconditions: + slot_conditions: + guard: + value_presence: PRESENT + postconditions: + slot_conditions: + target: + equals_string: "a" + equals_string_in: ["b", "c"] +""" + + +def test_rule_post_with_both_equals_forms_skipped(): + """equals_string and equals_string_in set together is ambiguous — skip, + do not let one form silently win.""" + g = _parse_shacl(_POST_BOTH_EQUALS_SCHEMA_YAML) + shape = URIRef("https://example.org/post-both-equals/Thing") + assert list(g.objects(shape, SH.sparql)) == [] + + +_MIXED_SCALAR_SCHEMA_YAML = """ +id: https://example.org/mixed-scalar +name: mixed_scalar +prefixes: + linkml: https://w3id.org/linkml/ + ex: https://example.org/mixed-scalar/ +imports: + - linkml:types +default_prefix: ex +default_range: string +slots: + code: + slot_uri: ex:code + note: + slot_uri: ex:note +classes: + Obs: + class_uri: ex:Obs + slots: [code, note] + rules: + - preconditions: + slot_conditions: + code: + equals_string: fog + pattern: "^f" + postconditions: + slot_conditions: + note: + required: true +""" + + +def test_rule_recognized_plus_unrecognized_operator_skipped(): + """A condition mixing a supported operator (equals_string) with an + unsupported one (pattern) must skip — translating only the supported part + widens the trigger.""" + g = _parse_shacl(_MIXED_SCALAR_SCHEMA_YAML) + shape = URIRef("https://example.org/mixed-scalar/Obs") + assert list(g.objects(shape, SH.sparql)) == [] + + +_EXPR_ANY_OF_SCHEMA_YAML = """ +id: https://example.org/expr-any-of +name: expr_any_of +prefixes: + linkml: https://w3id.org/linkml/ + ex: https://example.org/expr-any-of/ +imports: + - linkml:types +default_prefix: ex +default_range: string +slots: + code: + slot_uri: ex:code + other: + slot_uri: ex:other + note: + slot_uri: ex:note +classes: + Obs: + class_uri: ex:Obs + slots: [code, other, note] + rules: + - preconditions: + slot_conditions: + code: + equals_string: fog + any_of: + - slot_conditions: + other: + equals_string: x + - slot_conditions: + other: + equals_string: y + postconditions: + slot_conditions: + note: + required: true +""" + + +def test_rule_expression_level_any_of_skipped(): + """Expression-level any_of on the preconditions cannot be honoured by any + converter; dropping the branch widens the trigger, so the rule is skipped.""" + g = _parse_shacl(_EXPR_ANY_OF_SCHEMA_YAML) + shape = URIRef("https://example.org/expr-any-of/Obs") + assert list(g.objects(shape, SH.sparql)) == [] + + +def test_rule_expression_level_any_of_pyshacl_end_to_end(): + """An instance whose any_of branch is unmet satisfies the rule vacuously + and must conform.""" + import pyshacl + + shacl_ttl = ShaclGenerator(_EXPR_ANY_OF_SCHEMA_YAML, mergeimports=False, emit_rules=True).serialize() + data = """ + @prefix ex: . + + ex:o a ex:Obs ; ex:code "fog" ; ex:other "z" . + """ + conforms, _, txt = pyshacl.validate( + data_graph=data, + shacl_graph=shacl_ttl, + data_graph_format="turtle", + shacl_graph_format="turtle", + advanced=True, + ) + assert conforms, f"Instance with unmet any_of branch must not be flagged:\n{txt}" + + +_POST_MIXED_SCHEMA_YAML = """ +id: https://example.org/post-mixed +name: post_mixed +prefixes: + linkml: https://w3id.org/linkml/ + ex: https://example.org/post-mixed/ +imports: + - linkml:types +default_prefix: ex +default_range: string +slots: + guard: + slot_uri: ex:guard + target: + slot_uri: ex:target +classes: + Thing: + class_uri: ex:Thing + slots: [guard, target] + rules: + - preconditions: + slot_conditions: + guard: + equals_string: on + postconditions: + slot_conditions: + target: + required: true + pattern: "^x" +""" + + +def test_rule_post_mixed_operators_skipped(): + """A postcondition combining required with an untranslated operator must + skip — checking only required weakens the postcondition.""" + g = _parse_shacl(_POST_MIXED_SCHEMA_YAML) + shape = URIRef("https://example.org/post-mixed/Thing") + assert list(g.objects(shape, SH.sparql)) == [] + + +_ABSENT_COMBINED_SCHEMA_YAML = """ +id: https://example.org/absent-combined +name: absent_combined +prefixes: + linkml: https://w3id.org/linkml/ + ex: https://example.org/absent-combined/ +imports: + - linkml:types +default_prefix: ex +default_range: string +slots: + count: + range: integer + slot_uri: ex:count + note: + slot_uri: ex:note +classes: + Obs: + class_uri: ex:Obs + slots: [count, note] + rules: + - preconditions: + slot_conditions: + count: + value_presence: ABSENT + minimum_value: 5 + postconditions: + slot_conditions: + note: + required: true +""" + + +def test_rule_absent_combined_with_bound_skipped(): + """value_presence ABSENT combined with another operator must skip: the + triple-binding translation would invert the declared trigger.""" + g = _parse_shacl(_ABSENT_COMBINED_SCHEMA_YAML) + shape = URIRef("https://example.org/absent-combined/Obs") + assert list(g.objects(shape, SH.sparql)) == [] + + +_STRING_TRUE_SCHEMA_YAML = """ +id: https://example.org/string-true +name: string_true +prefixes: + linkml: https://w3id.org/linkml/ + ex: https://example.org/string-true/ +imports: + - linkml:types +default_prefix: ex +default_range: string +slots: + opt: + slot_uri: ex:opt + status: + range: string + slot_uri: ex:status +classes: + Conf: + class_uri: ex:Conf + slots: [opt, status] + rules: + - description: If opt is present, status must be the string "true". + preconditions: + slot_conditions: + opt: + value_presence: PRESENT + postconditions: + slot_conditions: + status: + equals_string: "true" +""" + + +def test_rule_equals_true_on_string_slot_uses_piv(): + """equals_string "true" on a NON-boolean slot must dispatch to + presence-implies-value (string comparison), not the boolean guard.""" + g = _parse_shacl(_STRING_TRUE_SCHEMA_YAML) + shape = URIRef("https://example.org/string-true/Conf") + sparql_nodes = list(g.objects(shape, SH.sparql)) + assert len(sparql_nodes) == 1 + query = str(list(g.objects(sparql_nodes[0], SH.select))[0]) + assert "NOT IN" in query, f"string-range 'true' must be a string comparison, got:\n{query}" + assert '"true"' in query, "the comparison term must be the string literal" + + +def test_rule_equals_true_on_string_slot_pyshacl_end_to_end(): + """status "true" (string) satisfies the rule; the boolean-guard hijack used + to flag it.""" + import pyshacl + + shacl_ttl = ShaclGenerator(_STRING_TRUE_SCHEMA_YAML, mergeimports=False, emit_rules=True).serialize() + conforming = """ + @prefix ex: . + + ex:ok a ex:Conf ; ex:opt "x" ; ex:status "true" . + """ + conforms, _, txt = pyshacl.validate( + data_graph=conforming, + shacl_graph=shacl_ttl, + data_graph_format="turtle", + shacl_graph_format="turtle", + advanced=True, + ) + assert conforms, f"status 'true' satisfies the rule and must conform:\n{txt}" + + violating = """ + @prefix ex: . + + ex:bad a ex:Conf ; ex:opt "x" ; ex:status "other" . + """ + conforms, _, txt = pyshacl.validate( + data_graph=violating, + shacl_graph=shacl_ttl, + data_graph_format="turtle", + shacl_graph_format="turtle", + advanced=True, + ) + assert not conforms, f"status 'other' violates the rule:\n{txt}" + + +_UNKNOWN_KEY_SCHEMA_YAML = """ +id: https://example.org/unknown-key +name: unknown_key +prefixes: + linkml: https://w3id.org/linkml/ + ex: https://example.org/unknown-key/ +imports: + - linkml:types +default_prefix: ex +default_range: string +slots: + code: + slot_uri: ex:code + note: + slot_uri: ex:note +classes: + Obs: + class_uri: ex:Obs + slots: [code, note] + rules: + - description: A rule keyed on a nonexistent slot must be skipped. + preconditions: + slot_conditions: + no_such_slot: + equals_string: trigger + postconditions: + slot_conditions: + note: + required: true +""" + + +def test_rule_unknown_slot_key_skipped(): + """A rule whose condition keys a slot that does not exist must be skipped: + fabricating a default-prefix predicate would emit a constraint that can + never fire (or, for has_member, always fires).""" + g = _parse_shacl(_UNKNOWN_KEY_SCHEMA_YAML) + shape = URIRef("https://example.org/unknown-key/Obs") + assert list(g.objects(shape, SH.sparql)) == [] From 4fb8efac4c0c3bd0ccac884db0a4481963fa9d03 Mon Sep 17 00:00:00 2001 From: Rayene Messaoud Date: Fri, 11 Sep 2026 14:47:30 +0200 Subject: [PATCH 07/14] feat(gen-shacl): add a compositional fallback for rule-to-SPARQL conversion The named-pattern converters only recognise whole-rule shapes, so a rule one operator away from a known pattern produced no constraint at all. This adds a compositional fallback, tried only after every named pattern has declined, that builds the query from the operators present rather than from a fixed template. Preconditions become a conjunction of graph patterns and FILTERs; the single postcondition becomes its negation. Together they select focus nodes satisfying every precondition while violating the postcondition, which is exactly the SHACL-SPARQL violation contract of SHACL 5.3.1, with $this pre-bound to the focus node. Operator support is declared per operator, and the builder returns None -- skipping the rule -- as soon as it meets one it does not handle, so an unsupported combination is never partially translated. This covers conditional-required and conditional-absent postconditions, numeric threshold preconditions, nested-object preconditions and has_member list membership, in any combination the operators allow. Numeric bounds are validated before interpolation. minimum_value and maximum_value have metamodel range Anything, so YAML strings, dates and .nan / .inf reach the generator unchanged; interpolating them raw either produced unparsable SPARQL that poisons the whole shapes graph at validation time, or -- for a date such as 2020-01-01 -- parsed as an arithmetic expression that silently never fires. Only int and finite float are rendered; anything else skips the rule. String literals are escaped rather than interpolated, and a slot carrying both minimum_value and maximum_value now yields both bounds instead of only the first. Nested and member slots resolve against the range class of their container, so an inner slot is no longer shadowed by a same-named slot on the outer class, and a range narrowed through slot_usage resolves its enum permissible values from the narrowed range. Co-authored-by: jdsika --- .../linkml/src/linkml/generators/shaclgen.py | 261 +++- tests/linkml/test_generators/test_shaclgen.py | 1257 ++++++++++++++++- 2 files changed, 1504 insertions(+), 14 deletions(-) diff --git a/packages/linkml/src/linkml/generators/shaclgen.py b/packages/linkml/src/linkml/generators/shaclgen.py index 0645f1e6ad..590f2a43fa 100644 --- a/packages/linkml/src/linkml/generators/shaclgen.py +++ b/packages/linkml/src/linkml/generators/shaclgen.py @@ -1,4 +1,5 @@ import logging +import math import os import string from collections.abc import Callable @@ -462,8 +463,11 @@ def _add_rules(self, g: Graph, shape_uri: URIRef, cls: ClassDefinition) -> None: multivalued slot, the total number of values must not exceed the given cardinality (typically 1 for mutual exclusion). - Operator combinations outside these named patterns are not translated; - the rule is skipped rather than partially represented. + Operator combinations outside these named patterns are handled by a + small compositional fallback (:meth:`_compose_rule_sparql`) covering + conditional-required / conditional-absent postconditions, numeric + threshold and nested-object preconditions, and ``has_member`` list + membership. See `W3C SHACL §5 `_. """ @@ -703,8 +707,217 @@ def _rule_to_sparql(self, sv, cls: ClassDefinition, rule) -> str | None: ) # Fallback: a small compositional builder for operator combinations not + # covered by the three named patterns above (conditional-required, + # threshold preconditions, list membership, ...). Tried only after the + # named patterns, so their output is unchanged. + composed = self._compose_rule_sparql(sv, cls, rule) + if composed is not None: + return composed + return None + def _compose_rule_sparql(self, sv, cls: ClassDefinition, rule) -> str | None: + """Compose a SHACL-SPARQL violation query for rule shapes not covered + by the three named patterns. + + Translates a conjunction of *precondition* slot conditions and a single + *postcondition* slot condition into one ``SELECT $this`` query that + selects focus nodes which satisfy every precondition but violate the + postcondition. Supported operators grow incrementally in + :meth:`_precondition_patterns` and :meth:`_postcondition_violation`; + the method returns ``None`` (rule skipped, never mis-translated) as soon + as any operator is unsupported. + + A rule's ``postconditions`` are a conjunction, so violating a single + slot condition is sufficient; the single-postcondition case covers the + modeled cross-parameter rules. + + Conforms to `SHACL §5.3.1 + `_: ``$this`` + is pre-bound to each focus node. + """ + pre = getattr(rule, "preconditions", None) + post = getattr(rule, "postconditions", None) + if not pre or not post: + return None + + pre_slots = getattr(pre, "slot_conditions", None) or {} + post_slots = getattr(post, "slot_conditions", None) or {} + if not pre_slots or len(post_slots) != 1: + return None + + pre_lines = self._precondition_patterns(sv, cls, pre_slots) + if pre_lines is None: + return None + + post_slot_name, post_cond = next(iter(post_slots.items())) + violation = self._postcondition_violation(sv, cls, post_slot_name, post_cond) + if violation is None: + return None + + body = "\n".join(f" {line}" for line in (pre_lines + violation)) + return f"SELECT $this WHERE {{\n{body}\n}}" + + def _scalar_filters(self, var: str, cond, resolve: Callable[[str], str]) -> list[str] | None: + """Return the SPARQL ``FILTER`` lines for the scalar operators on *cond*. + + Unlike a first-match dispatch, **every** recognised operator contributes + a line, so a condition combining operators — e.g. a bounded range + ``{minimum_value: X, maximum_value: Y}`` — emits *both* bounds instead of + silently keeping only the first and under-constraining the query. + ``value_presence: PRESENT`` contributes no filter (the caller's triple + binding already enforces presence). + + *resolve* maps an ``equals_string`` value to a SPARQL term (an enum + ``meaning`` IRI or an escaped string literal). + + Returns ``None`` when *cond* sets no recognised scalar operator, sets + any operator *outside* the recognised set (per + :meth:`_set_operator_fields` — partial translation would drop a + conjunct), sets ``value_presence`` to anything but ``PRESENT``, or + carries a non-numeric threshold bound. In every such case the caller + skips the rule it cannot faithfully translate rather than emitting an + under-constrained (or vacuous) query. + """ + op_fields = self._set_operator_fields(cond) + if not op_fields or not op_fields <= {"value_presence", "equals_string", "minimum_value", "maximum_value"}: + return None # unsupported operator present (or none at all): skip + if "value_presence" in op_fields and cond.value_presence != PresenceEnum(PresenceEnum.PRESENT): + # ABSENT (or a future presence value) cannot be expressed as a + # triple binding + filter; translating the other operators anyway + # would invert the declared trigger. + return None + + filters: list[str] = [] + if "equals_string" in op_fields: + filters.append(f"FILTER ( {var} = {resolve(cond.equals_string)} )") + if "minimum_value" in op_fields: + minimum = self._sparql_number(cond.minimum_value) + if minimum is None: + return None + filters.append(f"FILTER ( {var} >= {minimum} )") + if "maximum_value" in op_fields: + maximum = self._sparql_number(cond.maximum_value) + if maximum is None: + return None + filters.append(f"FILTER ( {var} <= {maximum} )") + return filters + + def _precondition_patterns(self, sv, cls: ClassDefinition, pre_slots) -> list[str] | None: + """Translate a conjunction of precondition slot conditions into SPARQL + graph patterns (plus ``FILTER`` lines) that bind focus nodes satisfying + every condition. + + Returns ``None`` if any condition sets no recognised operator. + + Supported operators, which **combine** on a single condition (so a + bounded range ``{minimum_value: X, maximum_value: Y}`` emits both + bounds): ``value_presence: PRESENT``, ``equals_string``, and the numeric + thresholds ``minimum_value`` / ``maximum_value`` (inclusive, per the + LinkML metamodel). A ``range_expression`` with inner ``slot_conditions`` + reaches one hop into an inlined child object. + """ + lines: list[str] = [] + for i, (slot_name, cond) in enumerate(pre_slots.items()): + path = self._slot_uri(sv, slot_name, cls) + if path is None: + return None + var = f"?pre{i}" + if self._set_operator_fields(cond) == {"range_expression"}: + # One-hop into an inlined child object: bind the child node and + # apply the inner slot conditions to it. The nested expression + # must itself be a plain conjunction of slot conditions; a + # condition mixing range_expression with scalar operators (or a + # nested any_of/...) is skipped rather than partially + # translated. + range_expr = cond.range_expression + if self._set_operator_fields(range_expr) != {"slot_conditions"}: + return None + node = f"{var}_node" + lines.append(f"$this <{path}> {node} .") + inner = self._member_conditions(sv, cls, slot_name, node, range_expr.slot_conditions) + if inner is None: + return None + lines.extend(inner) + continue + filters = self._scalar_filters( + var, cond, lambda v, sn=slot_name: self._resolve_enum_value_ref(sv, sn, v, cls) + ) + if filters is None: + return None + lines.append(f"$this <{path}> {var} .") + lines.extend(filters) + return lines + + def _member_conditions( + self, sv, cls: ClassDefinition, container_slot_name: str, node_var: str, slot_conditions + ) -> list[str] | None: + """Constrain the object bound to *node_var* — an instance of the range + class of *container_slot_name* — by a set of inner slot conditions. + + Shared by the nested ``range_expression`` precondition (single inlined + child) and the ``has_member`` postcondition (a list member). Inner + slots live on the container slot's **range class**, so both their + property IRIs and their enum values are resolved in that class's + induced context (the container slot itself is induced against *cls*, + honouring a ``slot_usage`` range narrowing). Resolving against the + outer class instead would emit predicates the member nodes never carry + — the ``sh:path`` on the member shape and the SPARQL body would + diverge, making ``FILTER NOT EXISTS`` member checks vacuously true + (false positives) or preconditions never bind (false negatives). + + Like preconditions, combining operators on one condition emits all of + them. Returns ``None`` for unsupported inner operators or when the + container's range is not a class (inner conditions on a non-class + range cannot be resolved faithfully). + """ + container = self._rule_slot(sv, container_slot_name, cls) + range_name = getattr(container, "range", None) + if not range_name or range_name not in sv.all_classes(): + return None + range_cls = sv.get_class(range_name) + + lines: list[str] = [] + for j, (inner_name, icond) in enumerate(slot_conditions.items()): + ipath = self._slot_uri(sv, inner_name, range_cls) + if ipath is None: + return None + ivar = f"{node_var}_{j}" + filters = self._scalar_filters( + ivar, + icond, + lambda v, inm=inner_name: self._resolve_enum_value_ref(sv, inm, v, range_cls), + ) + if filters is None: + return None + lines.append(f"{node_var} <{ipath}> {ivar} .") + lines.extend(filters) + return lines + + @staticmethod + def _sparql_number(value) -> str | None: + """Render a numeric threshold bound as a SPARQL numeric literal, or + ``None`` when the value is not a finite number (callers then skip the + rule). + + The metamodel range of ``minimum_value`` / ``maximum_value`` is + ``Anything``, so YAML strings, dates, booleans, ``.nan`` / ``.inf`` + all pass through SchemaView unchanged. Interpolating them raw is + unsound: ``"abc"`` yields unparsable SPARQL that poisons the whole + shapes graph at validation time, and a date like ``2020-01-01`` parses + as the arithmetic expression ``2020-01-01 = 2018`` and silently never + fires. Only ``int`` / finite ``float`` (including the + ``extended_int`` / ``extended_float`` runtime subclasses) are + rendered; their ``str`` yields a plain numeric token (e.g. ``4000`` or + ``0.0``) that SPARQL compares with numeric promotion against + ``xsd:float`` / ``xsd:decimal`` data values. + """ + if isinstance(value, bool) or not isinstance(value, int | float): + return None + if isinstance(value, float) and not math.isfinite(value): + return None + return str(value) + @staticmethod def _sparql_string_literal(value: str) -> str: """Render *value* as a double-quoted SPARQL string literal, escaping the @@ -726,6 +939,50 @@ def _sparql_string_literal(value: str) -> str: ) return f'"{escaped}"' + def _postcondition_violation(self, sv, cls: ClassDefinition, slot_name: str, cond) -> list[str] | None: + """Translate a single postcondition slot condition into SPARQL that + matches a *violation* of it. + + Returns ``None`` for operators not handled here, or when the condition + sets anything beyond the single operator a branch translates (dropping + a co-set operator would weaken the postcondition — the rule is skipped + instead). + + Supported operators: + + * ``required: true`` — violation = the target slot is absent on a focus + node that satisfies the preconditions. + * ``value_presence: ABSENT`` — violation = the target slot *is* present + (inapplicable-slot / conditional-absent). + * ``has_member`` with a nested ``range_expression`` — violation = *no* + member of the (multivalued) target slot matches the inner conditions + (list-membership; e.g. the light-group list must contain a + ``{group: Vehicle, type: front_fog_light}`` entry). + """ + path = self._slot_uri(sv, slot_name, cls) + if path is None: + return None + op_fields = self._set_operator_fields(cond) + if op_fields == {"required"} and cond.required is True: + return [f"FILTER NOT EXISTS {{ $this <{path}> ?post . }}"] + if op_fields == {"value_presence"} and cond.value_presence == PresenceEnum(PresenceEnum.ABSENT): + return [f"$this <{path}> ?post ."] + if op_fields == {"has_member"}: + has_member = cond.has_member + if self._set_operator_fields(has_member) != {"range_expression"}: + return None + range_expr = has_member.range_expression + if self._set_operator_fields(range_expr) != {"slot_conditions"}: + return None + member_lines = [f"$this <{path}> ?mem ."] + inner = self._member_conditions(sv, cls, slot_name, "?mem", range_expr.slot_conditions) + if inner is None: + return None + member_lines.extend(inner) + block = " ".join(member_lines) + return [f"FILTER NOT EXISTS {{ {block} }}"] + return None + def _build_boolean_guard_sparql( self, sv, cls: ClassDefinition, flag_slot_name: str, value_slot_name: str ) -> str | None: diff --git a/tests/linkml/test_generators/test_shaclgen.py b/tests/linkml/test_generators/test_shaclgen.py index 2fb7594212..af09ddd34c 100644 --- a/tests/linkml/test_generators/test_shaclgen.py +++ b/tests/linkml/test_generators/test_shaclgen.py @@ -3098,6 +3098,135 @@ def test_presence_implies_value_pyshacl_end_to_end(): # holds but the required slot is absent (FILTER NOT EXISTS). # =========================================================================== +_CONDITIONAL_REQUIRED_SCHEMA_YAML = """ +id: https://example.org/conditional-required +name: conditional_required_rules +prefixes: + linkml: https://w3id.org/linkml/ + ex: https://example.org/conditional-required/ +imports: + - linkml:types +default_prefix: ex +default_range: string + +enums: + SkyModelEnum: + permissible_values: + ClearSky: + meaning: ex:ClearSky + OvercastSky: + meaning: ex:OvercastSky + MeasuredOvercastSky: + meaning: ex:MeasuredOvercastSky + +slots: + sky_model: + range: SkyModelEnum + slot_uri: ex:sky_model + overcast_sky_illuminance: + range: float + slot_uri: ex:overcast_sky_illuminance + +classes: + Weather: + class_uri: ex:Weather + slots: + - sky_model + - overcast_sky_illuminance + rules: + - description: The MeasuredOvercastSky model requires the sky illuminance. + preconditions: + slot_conditions: + sky_model: + equals_string: MeasuredOvercastSky + postconditions: + slot_conditions: + overcast_sky_illuminance: + required: true +""" + +EX_CR = rdflib.Namespace("https://example.org/conditional-required/") + + +def test_conditional_required_generates_sparql(): + """equals_string precondition + required postcondition → one sh:sparql constraint.""" + g = _parse_shacl(_CONDITIONAL_REQUIRED_SCHEMA_YAML) + + shape = EX_CR.Weather + sparql_nodes = list(g.objects(shape, SH.sparql)) + assert len(sparql_nodes) == 1, f"Expected 1 sh:sparql constraint, got {len(sparql_nodes)}" + + node = sparql_nodes[0] + assert (node, RDF.type, SH.SPARQLConstraint) in g + query = str(list(g.objects(node, SH.select))[0]) + + assert "$this" in query, "SPARQL must use $this pre-bound variable (SHACL §5.3.1)" + assert "FILTER NOT EXISTS" in query, "required violation must use FILTER NOT EXISTS" + # precondition references the enum meaning IRI and the trigger slot + assert f"<{EX_CR.MeasuredOvercastSky}>" in query, f"precondition must use the enum IRI, got:\n{query}" + assert str(EX_CR.sky_model) in query + assert str(EX_CR.overcast_sky_illuminance) in query + + +def test_conditional_required_message_from_description(): + """Rule description is emitted as sh:message.""" + g = _parse_shacl(_CONDITIONAL_REQUIRED_SCHEMA_YAML) + messages = [str(m) for node in g.objects(EX_CR.Weather, SH.sparql) for m in g.objects(node, SH.message)] + assert any("requires the sky illuminance" in m for m in messages), messages + + +def test_conditional_required_sparql_syntax_valid(): + """Generated SPARQL must be syntactically valid.""" + from rdflib.plugins.sparql import prepareQuery + + g = _parse_shacl(_CONDITIONAL_REQUIRED_SCHEMA_YAML) + for node in g.objects(EX_CR.Weather, SH.sparql): + prepareQuery(str(list(g.objects(node, SH.select))[0])) + + +def test_conditional_required_pyshacl_end_to_end(): + """End-to-end: pyshacl passes conforming instances and flags the violation.""" + import pyshacl + + shacl_ttl = ShaclGenerator(_CONDITIONAL_REQUIRED_SCHEMA_YAML, mergeimports=False, emit_rules=True).serialize() + + # Conforming: MeasuredOvercastSky WITH illuminance; ClearSky needs nothing. + conforming = """ + @prefix ex: . + @prefix xsd: . + + ex:wMeasured a ex:Weather ; + ex:sky_model ex:MeasuredOvercastSky ; + ex:overcast_sky_illuminance "4200.0"^^xsd:float . + + ex:wClear a ex:Weather ; + ex:sky_model ex:ClearSky . + """ + conforms, _, txt = pyshacl.validate( + data_graph=conforming, + shacl_graph=shacl_ttl, + data_graph_format="turtle", + shacl_graph_format="turtle", + advanced=True, + ) + assert conforms, f"Conforming instances should pass:\n{txt}" + + # Violating: MeasuredOvercastSky WITHOUT the required illuminance. + violating = """ + @prefix ex: . + + ex:wBad a ex:Weather ; + ex:sky_model ex:MeasuredOvercastSky . + """ + conforms, _, txt = pyshacl.validate( + data_graph=violating, + shacl_graph=shacl_ttl, + data_graph_format="turtle", + shacl_graph_format="turtle", + advanced=True, + ) + assert not conforms, f"MeasuredOvercastSky without illuminance should fail:\n{txt}" + # =========================================================================== # Compositional fallback: conditional-absent pattern (M2) @@ -3112,6 +3241,124 @@ def test_presence_implies_value_pyshacl_end_to_end(): # precondition holds and the forbidden slot is present. # =========================================================================== +_CONDITIONAL_ABSENT_SCHEMA_YAML = """ +id: https://example.org/conditional-absent +name: conditional_absent_rules +prefixes: + linkml: https://w3id.org/linkml/ + ex: https://example.org/conditional-absent/ +imports: + - linkml:types +default_prefix: ex +default_range: string + +enums: + SkyModelEnum: + permissible_values: + ClearSky: + meaning: ex:ClearSky + OvercastSky: + meaning: ex:OvercastSky + +slots: + sky_model: + range: SkyModelEnum + slot_uri: ex:sky_model + overcast_sky_illuminance: + range: float + slot_uri: ex:overcast_sky_illuminance + +classes: + Weather: + class_uri: ex:Weather + slots: + - sky_model + - overcast_sky_illuminance + rules: + - description: ClearSky makes overcast_sky_illuminance inapplicable. + preconditions: + slot_conditions: + sky_model: + equals_string: ClearSky + postconditions: + slot_conditions: + overcast_sky_illuminance: + value_presence: ABSENT +""" + +EX_CA = rdflib.Namespace("https://example.org/conditional-absent/") + + +def test_conditional_absent_generates_sparql(): + """equals_string precondition + value_presence ABSENT → one sh:sparql constraint.""" + g = _parse_shacl(_CONDITIONAL_ABSENT_SCHEMA_YAML) + + sparql_nodes = list(g.objects(EX_CA.Weather, SH.sparql)) + assert len(sparql_nodes) == 1, f"Expected 1 sh:sparql constraint, got {len(sparql_nodes)}" + + query = str(list(g.objects(sparql_nodes[0], SH.select))[0]) + assert "$this" in query + # violation = precondition holds AND the forbidden slot is present; the + # forbidden-slot triple must NOT be wrapped in NOT EXISTS. + assert "FILTER NOT EXISTS" not in query, f"conditional-absent must not use NOT EXISTS, got:\n{query}" + assert f"<{EX_CA.ClearSky}>" in query + assert str(EX_CA.overcast_sky_illuminance) in query + + +def test_conditional_absent_sparql_syntax_valid(): + """Generated SPARQL must be syntactically valid.""" + from rdflib.plugins.sparql import prepareQuery + + g = _parse_shacl(_CONDITIONAL_ABSENT_SCHEMA_YAML) + for node in g.objects(EX_CA.Weather, SH.sparql): + prepareQuery(str(list(g.objects(node, SH.select))[0])) + + +def test_conditional_absent_pyshacl_end_to_end(): + """End-to-end: pyshacl passes conforming instances and flags the violation.""" + import pyshacl + + shacl_ttl = ShaclGenerator(_CONDITIONAL_ABSENT_SCHEMA_YAML, mergeimports=False, emit_rules=True).serialize() + + # Conforming: ClearSky without illuminance; OvercastSky may set illuminance. + conforming = """ + @prefix ex: . + @prefix xsd: . + + ex:wClear a ex:Weather ; + ex:sky_model ex:ClearSky . + + ex:wOvercast a ex:Weather ; + ex:sky_model ex:OvercastSky ; + ex:overcast_sky_illuminance "5000.0"^^xsd:float . + """ + conforms, _, txt = pyshacl.validate( + data_graph=conforming, + shacl_graph=shacl_ttl, + data_graph_format="turtle", + shacl_graph_format="turtle", + advanced=True, + ) + assert conforms, f"Conforming instances should pass:\n{txt}" + + # Violating: ClearSky WITH the inapplicable illuminance. + violating = """ + @prefix ex: . + @prefix xsd: . + + ex:wBad a ex:Weather ; + ex:sky_model ex:ClearSky ; + ex:overcast_sky_illuminance "5000.0"^^xsd:float . + """ + conforms, _, txt = pyshacl.validate( + data_graph=violating, + shacl_graph=shacl_ttl, + data_graph_format="turtle", + shacl_graph_format="turtle", + advanced=True, + ) + assert not conforms, f"ClearSky with illuminance should fail:\n{txt}" + # =========================================================================== # Compositional fallback: numeric threshold precondition (M3) @@ -3125,6 +3372,113 @@ def test_presence_implies_value_pyshacl_end_to_end(): # SPARQL FILTER; combined here with the M1 required violation. # =========================================================================== +_THRESHOLD_SCHEMA_YAML = """ +id: https://example.org/threshold +name: threshold_rules +prefixes: + linkml: https://w3id.org/linkml/ + ex: https://example.org/threshold/ +imports: + - linkml:types +default_prefix: ex +default_range: string + +slots: + meteorological_optical_range: + range: float + slot_uri: ex:meteorological_optical_range + fog_note: + range: string + slot_uri: ex:fog_note + +classes: + Weather: + class_uri: ex:Weather + slots: + - meteorological_optical_range + - fog_note + rules: + - description: In fog (optical range at or below 4000) a fog note is required. + preconditions: + slot_conditions: + meteorological_optical_range: + maximum_value: 4000 + postconditions: + slot_conditions: + fog_note: + required: true +""" + +EX_THR = rdflib.Namespace("https://example.org/threshold/") + + +def test_threshold_precondition_generates_sparql(): + """maximum_value precondition emits a numeric FILTER on the trigger slot.""" + g = _parse_shacl(_THRESHOLD_SCHEMA_YAML) + + sparql_nodes = list(g.objects(EX_THR.Weather, SH.sparql)) + assert len(sparql_nodes) == 1, f"Expected 1 sh:sparql constraint, got {len(sparql_nodes)}" + + query = str(list(g.objects(sparql_nodes[0], SH.select))[0]) + assert "<= 4000" in query, f"threshold must emit '<= 4000', got:\n{query}" + assert "FILTER NOT EXISTS" in query, "required postcondition violation must use NOT EXISTS" + assert str(EX_THR.meteorological_optical_range) in query + assert str(EX_THR.fog_note) in query + + +def test_threshold_precondition_sparql_syntax_valid(): + """Generated SPARQL must be syntactically valid.""" + from rdflib.plugins.sparql import prepareQuery + + g = _parse_shacl(_THRESHOLD_SCHEMA_YAML) + for node in g.objects(EX_THR.Weather, SH.sparql): + prepareQuery(str(list(g.objects(node, SH.select))[0])) + + +def test_threshold_precondition_pyshacl_end_to_end(): + """End-to-end: below-threshold requires the note; above-threshold does not.""" + import pyshacl + + shacl_ttl = ShaclGenerator(_THRESHOLD_SCHEMA_YAML, mergeimports=False, emit_rules=True).serialize() + + # Conforming: foggy (400) with a note; clear (5000) needs nothing. + conforming = """ + @prefix ex: . + @prefix xsd: . + + ex:wFog a ex:Weather ; + ex:meteorological_optical_range "400.0"^^xsd:float ; + ex:fog_note "reduced visibility" . + + ex:wClear a ex:Weather ; + ex:meteorological_optical_range "5000.0"^^xsd:float . + """ + conforms, _, txt = pyshacl.validate( + data_graph=conforming, + shacl_graph=shacl_ttl, + data_graph_format="turtle", + shacl_graph_format="turtle", + advanced=True, + ) + assert conforms, f"Conforming instances should pass:\n{txt}" + + # Violating: foggy (400) without the required note. + violating = """ + @prefix ex: . + @prefix xsd: . + + ex:wBad a ex:Weather ; + ex:meteorological_optical_range "400.0"^^xsd:float . + """ + conforms, _, txt = pyshacl.validate( + data_graph=violating, + shacl_graph=shacl_ttl, + data_graph_format="turtle", + shacl_graph_format="turtle", + advanced=True, + ) + assert not conforms, f"Fog without the required note should fail:\n{txt}" + # =========================================================================== # Compositional fallback: nested range_expression precondition (M4) @@ -3139,6 +3493,125 @@ def test_presence_implies_value_pyshacl_end_to_end(): # be present." The SPARQL binds the child node with one extra hop. # =========================================================================== +_NESTED_SCHEMA_YAML = """ +id: https://example.org/nested +name: nested_rules +prefixes: + linkml: https://w3id.org/linkml/ + ex: https://example.org/nested/ +imports: + - linkml:types +default_prefix: ex +default_range: string + +slots: + sun_position: + range: SunPosition + inlined: true + slot_uri: ex:sun_position + elevation: + range: float + slot_uri: ex:elevation + headlight_note: + range: string + slot_uri: ex:headlight_note + +classes: + SunPosition: + class_uri: ex:SunPosition + slots: + - elevation + Weather: + class_uri: ex:Weather + slots: + - sun_position + - headlight_note + rules: + - description: When the sun is at or below the horizon a headlight note is required. + preconditions: + slot_conditions: + sun_position: + range_expression: + slot_conditions: + elevation: + maximum_value: 0.0 + postconditions: + slot_conditions: + headlight_note: + required: true +""" + +EX_NEST = rdflib.Namespace("https://example.org/nested/") + + +def test_nested_precondition_generates_sparql(): + """A nested range_expression precondition emits a two-hop graph pattern.""" + g = _parse_shacl(_NESTED_SCHEMA_YAML) + + sparql_nodes = list(g.objects(EX_NEST.Weather, SH.sparql)) + assert len(sparql_nodes) == 1, f"Expected 1 sh:sparql constraint, got {len(sparql_nodes)}" + + query = str(list(g.objects(sparql_nodes[0], SH.select))[0]) + assert str(EX_NEST.sun_position) in query, "must traverse the container slot" + assert str(EX_NEST.elevation) in query, "must traverse the inner slot" + assert "<= 0.0" in query, f"inner threshold must appear, got:\n{query}" + assert "FILTER NOT EXISTS" in query + assert str(EX_NEST.headlight_note) in query + + +def test_nested_precondition_sparql_syntax_valid(): + """Generated SPARQL must be syntactically valid.""" + from rdflib.plugins.sparql import prepareQuery + + g = _parse_shacl(_NESTED_SCHEMA_YAML) + for node in g.objects(EX_NEST.Weather, SH.sparql): + prepareQuery(str(list(g.objects(node, SH.select))[0])) + + +def test_nested_precondition_pyshacl_end_to_end(): + """End-to-end: sun below horizon requires the note; above horizon does not.""" + import pyshacl + + shacl_ttl = ShaclGenerator(_NESTED_SCHEMA_YAML, mergeimports=False, emit_rules=True).serialize() + + # Conforming: night (elevation -90) with a note; day (45) needs nothing. + conforming = """ + @prefix ex: . + @prefix xsd: . + + ex:wNight a ex:Weather ; + ex:sun_position [ a ex:SunPosition ; ex:elevation "-90.0"^^xsd:float ] ; + ex:headlight_note "on" . + + ex:wDay a ex:Weather ; + ex:sun_position [ a ex:SunPosition ; ex:elevation "45.0"^^xsd:float ] . + """ + conforms, _, txt = pyshacl.validate( + data_graph=conforming, + shacl_graph=shacl_ttl, + data_graph_format="turtle", + shacl_graph_format="turtle", + advanced=True, + ) + assert conforms, f"Conforming instances should pass:\n{txt}" + + # Violating: night (elevation -90) without the required note. + violating = """ + @prefix ex: . + @prefix xsd: . + + ex:wBad a ex:Weather ; + ex:sun_position [ a ex:SunPosition ; ex:elevation "-90.0"^^xsd:float ] . + """ + conforms, _, txt = pyshacl.validate( + data_graph=violating, + shacl_graph=shacl_ttl, + data_graph_format="turtle", + shacl_graph_format="turtle", + advanced=True, + ) + assert not conforms, f"Night without a headlight note should fail:\n{txt}" + # =========================================================================== # Compositional fallback: has_member list-membership postcondition (M5) @@ -3155,19 +3628,359 @@ def test_presence_implies_value_pyshacl_end_to_end(): # class (LightControlGroup), which disambiguates the reused `type` slot. # =========================================================================== +_HAS_MEMBER_SCHEMA_YAML = """ +id: https://example.org/has-member +name: has_member_rules +prefixes: + linkml: https://w3id.org/linkml/ + ex: https://example.org/has-member/ +imports: + - linkml:types +default_prefix: ex +default_range: string + +enums: + LightGroupEnum: + permissible_values: + Vehicle: + meaning: ex:Vehicle + StreetLight: + meaning: ex:StreetLight + LightTypeEnum: + permissible_values: + low_beam_headlight: + meaning: ex:low_beam_headlight + front_fog_light: + meaning: ex:front_fog_light + +slots: + fog_declared: + range: string + slot_uri: ex:fog_declared + enabled_light_control_groups: + range: LightControlGroup + multivalued: true + inlined: true + inlined_as_list: true + slot_uri: ex:enabled_light_control_groups + group: + range: LightGroupEnum + slot_uri: ex:group + type: + range: LightTypeEnum + slot_uri: ex:type + +classes: + LightControlGroup: + class_uri: ex:LightControlGroup + slots: + - group + - type + Weather: + class_uri: ex:Weather + slots: + - fog_declared + - enabled_light_control_groups + rules: + - description: When fog is declared, a front fog light group must be enabled. + preconditions: + slot_conditions: + fog_declared: + value_presence: PRESENT + postconditions: + slot_conditions: + enabled_light_control_groups: + has_member: + range_expression: + slot_conditions: + group: + equals_string: Vehicle + type: + equals_string: front_fog_light +""" + +EX_HM = rdflib.Namespace("https://example.org/has-member/") + + +def test_has_member_generates_sparql(): + """has_member postcondition emits a FILTER NOT EXISTS over list members.""" + g = _parse_shacl(_HAS_MEMBER_SCHEMA_YAML) + + sparql_nodes = list(g.objects(EX_HM.Weather, SH.sparql)) + assert len(sparql_nodes) == 1, f"Expected 1 sh:sparql constraint, got {len(sparql_nodes)}" + + query = str(list(g.objects(sparql_nodes[0], SH.select))[0]) + assert "FILTER NOT EXISTS" in query, "list-membership violation must use FILTER NOT EXISTS" + assert str(EX_HM.enabled_light_control_groups) in query + assert str(EX_HM.group) in query and str(EX_HM.type) in query + # inner enum values resolve against the member class (LightControlGroup), + # so the reused `type` slot picks LightTypeEnum, not another enum. + assert f"<{EX_HM.Vehicle}>" in query, f"group value must be the enum IRI, got:\n{query}" + assert f"<{EX_HM.front_fog_light}>" in query, f"type value must be the enum IRI, got:\n{query}" + + +def test_has_member_sparql_syntax_valid(): + """Generated SPARQL must be syntactically valid.""" + from rdflib.plugins.sparql import prepareQuery + + g = _parse_shacl(_HAS_MEMBER_SCHEMA_YAML) + for node in g.objects(EX_HM.Weather, SH.sparql): + prepareQuery(str(list(g.objects(node, SH.select))[0])) + + +def test_has_member_pyshacl_end_to_end(): + """End-to-end: fog requires a front-fog-light member; otherwise it fails.""" + import pyshacl + + shacl_ttl = ShaclGenerator(_HAS_MEMBER_SCHEMA_YAML, mergeimports=False, emit_rules=True).serialize() + + # Conforming: fog declared WITH a front-fog-light group; and no fog at all. + conforming = """ + @prefix ex: . + + ex:wFog a ex:Weather ; + ex:fog_declared "yes" ; + ex:enabled_light_control_groups + [ a ex:LightControlGroup ; ex:group ex:Vehicle ; ex:type ex:front_fog_light ] . + + ex:wNoFog a ex:Weather . + """ + conforms, _, txt = pyshacl.validate( + data_graph=conforming, + shacl_graph=shacl_ttl, + data_graph_format="turtle", + shacl_graph_format="turtle", + advanced=True, + ) + assert conforms, f"Conforming instances should pass:\n{txt}" + + # Violating: fog declared but only a low-beam group (no front fog light). + violating = """ + @prefix ex: . + + ex:wBad a ex:Weather ; + ex:fog_declared "yes" ; + ex:enabled_light_control_groups + [ a ex:LightControlGroup ; ex:group ex:Vehicle ; ex:type ex:low_beam_headlight ] . + """ + conforms, _, txt = pyshacl.validate( + data_graph=violating, + shacl_graph=shacl_ttl, + data_graph_format="turtle", + shacl_graph_format="turtle", + advanced=True, + ) + assert not conforms, f"Fog without a front-fog-light group should fail:\n{txt}" + + +# =========================================================================== +# Rule-converter robustness regressions (review hardening) +# +# These guard three defects found while reviewing the rule converters: +# 1. A single precondition combining minimum_value + maximum_value dropped +# all but the first bound (silent under-constraint / false positives). +# 2. A slot_usage `slot_uri` (or enum `range`) override made the SPARQL body +# query the *base* IRI while `sh:path` used the *induced* IRI, so the +# constraint silently never fired (false negative). +# 3. An `equals_string` value containing a quote/backslash produced invalid, +# unparsable SPARQL (broken artifact / injection). +# =========================================================================== + +_COMBINED_BOUNDS_SCHEMA_YAML = """ +id: https://example.org/combined-bounds +name: combined_bounds_rules +prefixes: + linkml: https://w3id.org/linkml/ + ex: https://example.org/combined-bounds/ +imports: + - linkml:types +default_prefix: ex +default_range: string + +slots: + reading_value: + range: integer + slot_uri: ex:reading_value + reading_note: + range: string + slot_uri: ex:reading_note + +classes: + Reading: + class_uri: ex:Reading + slots: + - reading_value + - reading_note + rules: + - description: A mid-range reading requires an explanatory note. + preconditions: + slot_conditions: + reading_value: + minimum_value: 10 + maximum_value: 20 + postconditions: + slot_conditions: + reading_note: + required: true +""" + +EX_CB = rdflib.Namespace("https://example.org/combined-bounds/") + + +def test_rule_precondition_combines_min_and_max_bounds(): + """A precondition with both minimum_value and maximum_value must emit both + bounds; the pre-fix first-match dispatch kept only the maximum.""" + g = _parse_shacl(_COMBINED_BOUNDS_SCHEMA_YAML) + + nodes = list(g.objects(EX_CB.Reading, SH.sparql)) + assert len(nodes) == 1, f"Expected 1 sh:sparql constraint, got {len(nodes)}" + query = str(list(g.objects(nodes[0], SH.select))[0]) + assert ">= 10" in query, f"lower bound must be emitted, got:\n{query}" + assert "<= 20" in query, f"upper bound must be emitted, got:\n{query}" + + +def test_rule_combined_bounds_pyshacl_end_to_end(): + """End-to-end: only values inside [10, 20] trigger the required note. + + The below-threshold case is the key assertion — without the lower bound it + would be flagged as a violation.""" + import pyshacl + + shacl_ttl = ShaclGenerator(_COMBINED_BOUNDS_SCHEMA_YAML, mergeimports=False, emit_rules=True).serialize() + + conforming = """ + @prefix ex: . + @prefix xsd: . + + ex:mid a ex:Reading ; ex:reading_value 15 ; ex:reading_note "in range" . + ex:low a ex:Reading ; ex:reading_value 5 . + ex:high a ex:Reading ; ex:reading_value 25 . + """ + conforms, _, txt = pyshacl.validate( + data_graph=conforming, + shacl_graph=shacl_ttl, + data_graph_format="turtle", + shacl_graph_format="turtle", + advanced=True, + ) + assert conforms, f"Out-of-range readings must not require a note:\n{txt}" + + violating = """ + @prefix ex: . + @prefix xsd: . + + ex:bad a ex:Reading ; ex:reading_value 15 . + """ + conforms, _, txt = pyshacl.validate( + data_graph=violating, + shacl_graph=shacl_ttl, + data_graph_format="turtle", + shacl_graph_format="turtle", + advanced=True, + ) + assert not conforms, f"A mid-range reading without a note must fail:\n{txt}" + + +_SLOT_URI_OVERRIDE_SCHEMA_YAML = """ +id: https://example.org/slot-uri-override +name: slot_uri_override_rules +prefixes: + linkml: https://w3id.org/linkml/ + ex: https://example.org/slot-uri-override/ +imports: + - linkml:types +default_prefix: ex +default_range: string + +slots: + trigger: + range: string + slot_uri: ex:GLOBAL_trigger + dependent: + range: string + slot_uri: ex:GLOBAL_dependent + +classes: + Scene: + class_uri: ex:Scene + slots: + - trigger + - dependent + slot_usage: + trigger: + slot_uri: ex:LOCAL_trigger + dependent: + slot_uri: ex:LOCAL_dependent + rules: + - description: If the trigger is present the dependent slot is required. + preconditions: + slot_conditions: + trigger: + value_presence: PRESENT + postconditions: + slot_conditions: + dependent: + required: true +""" + +EX_OVR = rdflib.Namespace("https://example.org/slot-uri-override/") + + +def test_rule_slot_uri_override_matches_sh_path(): + """The SPARQL body must use the same induced (class-local) IRIs as sh:path. + + A slot_usage slot_uri override changes sh:path; if the SPARQL keeps the base + IRI the query targets a property the data never uses and never fires.""" + g = _parse_shacl(_SLOT_URI_OVERRIDE_SCHEMA_YAML) + + paths = {str(o) for o in g.objects(None, SH.path)} + assert str(EX_OVR.LOCAL_trigger) in paths + assert str(EX_OVR.LOCAL_dependent) in paths + + nodes = list(g.objects(EX_OVR.Scene, SH.sparql)) + assert len(nodes) == 1 + query = str(list(g.objects(nodes[0], SH.select))[0]) + assert str(EX_OVR.LOCAL_trigger) in query, f"SPARQL must use the induced IRI, got:\n{query}" + assert str(EX_OVR.LOCAL_dependent) in query, f"SPARQL must use the induced IRI, got:\n{query}" + assert "GLOBAL_" not in query, f"SPARQL must not fall back to the base slot_uri, got:\n{query}" + + +def test_rule_slot_uri_override_pyshacl_end_to_end(): + """End-to-end: the constraint actually fires on data that uses the induced + (LOCAL) IRIs. Before the fix the SPARQL queried the base IRIs, so a missing + dependent slot slipped through as conforming.""" + import pyshacl + + shacl_ttl = ShaclGenerator(_SLOT_URI_OVERRIDE_SCHEMA_YAML, mergeimports=False, emit_rules=True).serialize() + + conforming = """ + @prefix ex: . + + ex:ok a ex:Scene ; ex:LOCAL_trigger "t" ; ex:LOCAL_dependent "d" . + ex:noTrigger a ex:Scene ; ex:LOCAL_dependent "d" . + """ + conforms, _, txt = pyshacl.validate( + data_graph=conforming, + shacl_graph=shacl_ttl, + data_graph_format="turtle", + shacl_graph_format="turtle", + advanced=True, + ) + assert conforms, f"Trigger-with-dependent (and no-trigger) must pass:\n{txt}" -# =========================================================================== -# Rule-converter robustness regressions (review hardening) -# -# These guard three defects found while reviewing the rule converters: -# 1. A single precondition combining minimum_value + maximum_value dropped -# all but the first bound (silent under-constraint / false positives). -# 2. A slot_usage `slot_uri` (or enum `range`) override made the SPARQL body -# query the *base* IRI while `sh:path` used the *induced* IRI, so the -# constraint silently never fired (false negative). -# 3. An `equals_string` value containing a quote/backslash produced invalid, -# unparsable SPARQL (broken artifact / injection). -# =========================================================================== + violating = """ + @prefix ex: . + + ex:bad a ex:Scene ; ex:LOCAL_trigger "t" . + """ + conforms, _, txt = pyshacl.validate( + data_graph=violating, + shacl_graph=shacl_ttl, + data_graph_format="turtle", + shacl_graph_format="turtle", + advanced=True, + ) + assert not conforms, f"Trigger present without the required dependent must fail:\n{txt}" _ENUM_NARROWING_SCHEMA_YAML = """ @@ -3670,6 +4483,426 @@ def test_rule_equals_true_on_string_slot_pyshacl_end_to_end(): assert not conforms, f"status 'other' violates the rule:\n{txt}" +_INNER_OVERRIDE_SCHEMA_YAML = """ +id: https://example.org/inner-override +name: inner_override +prefixes: + linkml: https://w3id.org/linkml/ + ex: https://example.org/inner-override/ +imports: + - linkml:types +default_prefix: ex +default_range: string +slots: + sun_position: + range: SunPosition + inlined: true + slot_uri: ex:sun_position + elevation: + range: float + slot_uri: ex:elevation + headlight_note: + slot_uri: ex:headlight_note +classes: + SunPosition: + class_uri: ex:SunPosition + slots: [elevation] + slot_usage: + elevation: + slot_uri: ex:localElevation + Scene: + class_uri: ex:Scene + slots: [sun_position, headlight_note] + rules: + - description: Below the horizon a headlight note is required. + preconditions: + slot_conditions: + sun_position: + range_expression: + slot_conditions: + elevation: + maximum_value: 0 + postconditions: + slot_conditions: + headlight_note: + required: true +""" + + +def test_rule_nested_inner_slot_uri_resolved_on_range_class(): + """The inner slot of a nested precondition lives on the container's range + class; a slot_usage slot_uri override there must be honoured (sh:path / + SPARQL-body parity one hop down).""" + g = _parse_shacl(_INNER_OVERRIDE_SCHEMA_YAML) + shape = URIRef("https://example.org/inner-override/Scene") + sparql_nodes = list(g.objects(shape, SH.sparql)) + assert len(sparql_nodes) == 1 + query = str(list(g.objects(sparql_nodes[0], SH.select))[0]) + assert "https://example.org/inner-override/localElevation" in query, ( + f"inner slot must use the range class's induced slot_uri, got:\n{query}" + ) + assert "https://example.org/inner-override/elevation" not in query, ( + "the base slot_uri must not leak into the member pattern" + ) + + +def test_rule_nested_inner_slot_uri_override_pyshacl_end_to_end(): + """A night scene without the required note must be flagged — with the + base-URI mistranslation the constraint silently never fired.""" + import pyshacl + + shacl_ttl = ShaclGenerator(_INNER_OVERRIDE_SCHEMA_YAML, mergeimports=False, emit_rules=True).serialize() + # The float is typed explicitly so the sh:datatype property constraint is + # satisfied and the outcome discriminates on the rule constraint alone. + violating = """ + @prefix ex: . + @prefix xsd: . + + ex:night a ex:Scene ; ex:sun_position ex:sp . + ex:sp a ex:SunPosition ; ex:localElevation "-5.0"^^xsd:float . + """ + conforms, _, txt = pyshacl.validate( + data_graph=violating, + shacl_graph=shacl_ttl, + data_graph_format="turtle", + shacl_graph_format="turtle", + advanced=True, + ) + assert not conforms, f"Night scene without headlight note must fail:\n{txt}" + + conforming = """ + @prefix ex: . + @prefix xsd: . + + ex:noon a ex:Scene ; ex:sun_position ex:sp2 . + ex:sp2 a ex:SunPosition ; ex:localElevation "45.0"^^xsd:float . + """ + conforms, _, txt = pyshacl.validate( + data_graph=conforming, + shacl_graph=shacl_ttl, + data_graph_format="turtle", + shacl_graph_format="turtle", + advanced=True, + ) + assert conforms, f"Daytime scene needs no headlight note:\n{txt}" + + +_INNER_COLLISION_SCHEMA_YAML = """ +id: https://example.org/inner-collision +name: inner_collision +prefixes: + linkml: https://w3id.org/linkml/ + ex: https://example.org/inner-collision/ +imports: + - linkml:types +default_prefix: ex +default_range: string +slots: + marker_flag: + slot_uri: ex:marker_flag + items: + range: Item + multivalued: true + inlined: true + inlined_as_list: true + slot_uri: ex:items + type: + slot_uri: ex:defaultType +classes: + Item: + class_uri: ex:Item + slots: [type] + slot_usage: + type: + slot_uri: ex:itemType + Box: + class_uri: ex:Box + slots: [marker_flag, items, type] + slot_usage: + type: + slot_uri: ex:boxType + rules: + - description: A flagged box must contain a marker item. + preconditions: + slot_conditions: + marker_flag: + value_presence: PRESENT + postconditions: + slot_conditions: + items: + has_member: + range_expression: + slot_conditions: + type: + equals_string: marker +""" + + +def test_rule_has_member_inner_slot_not_shadowed_by_outer_class(): + """An inner slot name that also exists on the OUTER class with a different + slot_usage URI must still resolve against the member class.""" + g = _parse_shacl(_INNER_COLLISION_SCHEMA_YAML) + shape = URIRef("https://example.org/inner-collision/Box") + sparql_nodes = list(g.objects(shape, SH.sparql)) + assert len(sparql_nodes) == 1 + query = str(list(g.objects(sparql_nodes[0], SH.select))[0]) + assert "https://example.org/inner-collision/itemType" in query, ( + f"member condition must use the member class's slot URI, got:\n{query}" + ) + assert "boxType" not in query, "the outer class's slot_usage URI must not shadow the member's" + + +def test_rule_has_member_inner_slot_collision_pyshacl_end_to_end(): + """A conforming box (marker item present via the member class's predicate) + must conform — the outer-class shadowing made FILTER NOT EXISTS vacuous.""" + import pyshacl + + shacl_ttl = ShaclGenerator(_INNER_COLLISION_SCHEMA_YAML, mergeimports=False, emit_rules=True).serialize() + conforming = """ + @prefix ex: . + + ex:b a ex:Box ; ex:marker_flag "y" ; ex:items ex:i1 . + ex:i1 a ex:Item ; ex:itemType "marker" . + """ + conforms, _, txt = pyshacl.validate( + data_graph=conforming, + shacl_graph=shacl_ttl, + data_graph_format="turtle", + shacl_graph_format="turtle", + advanced=True, + ) + assert conforms, f"Box with a marker item must conform:\n{txt}" + + +_CONTAINER_NARROWED_SCHEMA_YAML = """ +id: https://example.org/container-narrowed +name: container_narrowed +prefixes: + linkml: https://w3id.org/linkml/ + ex: https://example.org/container-narrowed/ +imports: + - linkml:types +default_prefix: ex +default_range: string +enums: + BaseKindEnum: + permissible_values: + special: + meaning: ex:BASE_special + SpecialKindEnum: + permissible_values: + special: + meaning: ex:SPECIAL_special +slots: + part: + range: BasePart + inlined: true + slot_uri: ex:part + kind: + range: BaseKindEnum + slot_uri: ex:kind + label_note: + slot_uri: ex:label_note +classes: + BasePart: + class_uri: ex:BasePart + slots: [kind] + SpecialPart: + class_uri: ex:SpecialPart + is_a: BasePart + slot_usage: + kind: + range: SpecialKindEnum + Assembly: + class_uri: ex:Assembly + slots: [part, label_note] + slot_usage: + part: + range: SpecialPart + rules: + - description: A special part requires a label note. + preconditions: + slot_conditions: + part: + range_expression: + slot_conditions: + kind: + equals_string: special + postconditions: + slot_conditions: + label_note: + required: true +""" + + +def test_rule_container_range_narrowing_resolves_inner_enum(): + """A slot_usage range-narrowing of the CONTAINER slot must resolve inner + enum values against the narrowed range class's enum.""" + g = _parse_shacl(_CONTAINER_NARROWED_SCHEMA_YAML) + shape = URIRef("https://example.org/container-narrowed/Assembly") + sparql_nodes = list(g.objects(shape, SH.sparql)) + assert len(sparql_nodes) == 1 + query = str(list(g.objects(sparql_nodes[0], SH.select))[0]) + assert "SPECIAL_special" in query, f"inner enum must resolve via the narrowed range, got:\n{query}" + assert "BASE_special" not in query, "the base range's enum must not be used" + + +_NON_NUMERIC_BOUNDS_SCHEMA_YAML = """ +id: https://example.org/non-numeric-bounds +name: non_numeric_bounds +prefixes: + linkml: https://w3id.org/linkml/ + ex: https://example.org/non-numeric-bounds/ +imports: + - linkml:types +default_prefix: ex +default_range: string +slots: + v: + range: integer + slot_uri: ex:v + note: + slot_uri: ex:note +classes: + Obs: + class_uri: ex:Obs + slots: [v, note] + rules: + - preconditions: + slot_conditions: + v: + minimum_value: "abc" + postconditions: + slot_conditions: + note: + required: true + - preconditions: + slot_conditions: + v: + minimum_value: 2020-01-01 + postconditions: + slot_conditions: + note: + required: true + - preconditions: + slot_conditions: + v: + maximum_value: .nan + postconditions: + slot_conditions: + note: + required: true + - preconditions: + slot_conditions: + v: + minimum_value: true + postconditions: + slot_conditions: + note: + required: true +""" + + +def test_rule_non_numeric_bounds_skipped(): + """Non-numeric threshold bounds (string, date, NaN, boolean) must skip the + rule: raw interpolation produced unparsable SPARQL (poisoning the whole + shapes graph) or silently-wrong arithmetic (2020-01-01 == 2018).""" + g = _parse_shacl(_NON_NUMERIC_BOUNDS_SCHEMA_YAML) + shape = URIRef("https://example.org/non-numeric-bounds/Obs") + assert list(g.objects(shape, SH.sparql)) == [] + + +def test_rule_non_numeric_bounds_shapes_graph_still_validates(): + """The generated shapes graph must remain usable by pyshacl — one bad bound + used to raise a ParseException for every validation run.""" + import pyshacl + + shacl_ttl = ShaclGenerator(_NON_NUMERIC_BOUNDS_SCHEMA_YAML, mergeimports=False, emit_rules=True).serialize() + data = """ + @prefix ex: . + + ex:o a ex:Obs ; ex:v 1 . + """ + conforms, _, txt = pyshacl.validate( + data_graph=data, + shacl_graph=shacl_ttl, + data_graph_format="turtle", + shacl_graph_format="turtle", + advanced=True, + ) + assert conforms, f"Shapes graph must stay parseable and the data conform:\n{txt}" + + +def test_has_member_zero_members_pyshacl_end_to_end(): + """A node meeting the precondition with ZERO members violates has_member + ('must contain a matching member'); locks the semantics in.""" + import pyshacl + + shacl_ttl = ShaclGenerator(_HAS_MEMBER_SCHEMA_YAML, mergeimports=False, emit_rules=True).serialize() + violating = """ + @prefix ex: . + + ex:wZero a ex:Weather ; ex:fog_declared "fog" . + """ + conforms, _, txt = pyshacl.validate( + data_graph=violating, + shacl_graph=shacl_ttl, + data_graph_format="turtle", + shacl_graph_format="turtle", + advanced=True, + ) + assert not conforms, f"Zero members cannot contain the required member:\n{txt}" + + +_ALIAS_KEY_SCHEMA_YAML = """ +id: https://example.org/alias-key +name: alias_key +prefixes: + linkml: https://w3id.org/linkml/ + ex: https://example.org/alias-key/ +imports: + - linkml:types +default_prefix: ex +default_range: string +slots: + my slot: + slot_uri: ex:customMySlot + note: + slot_uri: ex:note +classes: + Obs: + class_uri: ex:Obs + slots: ["my slot", note] + rules: + - description: Underscored alias key must resolve to the declared slot. + preconditions: + slot_conditions: + my_slot: + equals_string: trigger + postconditions: + slot_conditions: + note: + required: true +""" + + +def test_rule_alias_form_slot_key_resolves_override(): + """A rule key written `my_slot` for a slot named `my slot` must resolve to + that slot's URI (sh:path parity) instead of fabricating a default-prefix + predicate that makes the constraint vacuous.""" + g = _parse_shacl(_ALIAS_KEY_SCHEMA_YAML) + shape = URIRef("https://example.org/alias-key/Obs") + sparql_nodes = list(g.objects(shape, SH.sparql)) + assert len(sparql_nodes) == 1 + query = str(list(g.objects(sparql_nodes[0], SH.select))[0]) + assert "https://example.org/alias-key/customMySlot" in query, ( + f"alias-form key must resolve to the declared slot_uri, got:\n{query}" + ) + assert "https://example.org/alias-key/my_slot" not in query, ( + "the fabricated default-prefix predicate must not be emitted" + ) + + _UNKNOWN_KEY_SCHEMA_YAML = """ id: https://example.org/unknown-key name: unknown_key From 66078029e4bdfaacde347cfd4c2f052591550085 Mon Sep 17 00:00:00 2001 From: Carlo van Driesten Date: Fri, 11 Sep 2026 14:48:35 +0200 Subject: [PATCH 08/14] docs(shacl): document rule-to-SHACL-SPARQL constraint generation The SHACL generator translates LinkML rules into sh:sparql constraints, but the generator documentation did not mention it, so the feature was undiscoverable and its limits undocumented. Describe the recognised named patterns and the compositional fallback, the --emit-rules flag, the skip-never-mis-translate contract and which rule attributes warn, with a worked YAML-to-Turtle example. Note that SPARQL-based constraints need a processor with SHACL-SPARQL support. --- docs/generators/shacl.rst | 77 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/docs/generators/shacl.rst b/docs/generators/shacl.rst index 3e88f0090f..659e55f63c 100644 --- a/docs/generators/shacl.rst +++ b/docs/generators/shacl.rst @@ -84,6 +84,83 @@ Example Output: shacl:targetClass . +Rule constraints (SHACL-SPARQL) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +LinkML `rules `_ express +cross-parameter, conditional validation ("if slot A holds X, slot B must +..."). Plain per-slot SHACL property shapes cannot express these, so the +generator translates recognised rule shapes into +`SHACL-SPARQL constraints `_ +(``sh:sparql`` / ``sh:SPARQLConstraint``) on the class's ``sh:NodeShape``. +Generation is controlled by ``--emit-rules/--no-emit-rules`` (default: on). + +Three named patterns are recognised first: + +* **Boolean guard** — precondition ``value_presence: PRESENT`` on a value + slot, postcondition ``equals_string: "true"`` on a *boolean-range* flag + slot: if the value is present, the flag must be true. +* **Presence implies value** — precondition ``value_presence: PRESENT``, + postcondition ``equals_string`` / ``equals_string_in`` on a target slot: + if the guard is present, the target must hold one of the allowed values. + Enum values resolve to their ``meaning`` IRIs; values without ``meaning`` + compare as string literals. +* **Exclusive value** — precondition ``equals_string`` and postcondition + ``maximum_cardinality`` on the *same* multivalued slot: if the value is + present, the slot has at most N values. + +Combinations outside the named patterns are handled by a compositional +fallback that conjoins the preconditions and negates a single postcondition: +conditional-required (``required: true``), conditional-absent +(``value_presence: ABSENT``), numeric threshold preconditions +(``minimum_value`` / ``maximum_value``), a one-hop nested precondition into +an inlined child object (``range_expression.slot_conditions``), and +``has_member`` list membership. + +The translation contract is *skip, never mis-translate*: a rule whose +conditions set any operator outside the translated set (including +expression-level ``any_of``/``all_of``/``none_of``/``exactly_one_of``), or +whose slot keys resolve to no slot, is skipped and logged at ``DEBUG``. +``deactivated`` rules are skipped; ``bidirectional``, ``open_world``, and +``elseconditions`` warn (the forward direction is emitted). + +Example: + +.. code-block:: yaml + + classes: + Weather: + slots: [sun_altitude, daytime] + rules: + - description: If sun_altitude is present, daytime must be day or twilight. + preconditions: + slot_conditions: + sun_altitude: + value_presence: PRESENT + postconditions: + slot_conditions: + daytime: + equals_string_in: [day, twilight] + +generates (abridged): + +.. code-block:: turtle + + ex:Weather a sh:NodeShape ; + sh:sparql [ a sh:SPARQLConstraint ; + sh:message "If sun_altitude is present, daytime must be day or twilight." ; + sh:select """SELECT $this WHERE { + $this ?value . + OPTIONAL { $this ?target . } + FILTER ( !BOUND(?target) || ?target NOT IN (, ) ) + }""" ] . + +``$this`` is pre-bound to each focus node per +`SHACL §5.3.1 `_. +Note that SPARQL-based constraints require a SHACL processor with +SHACL-SPARQL support (e.g. ``pyshacl`` with ``advanced=True``). + + Command Line ^^^^^^^^^^^^ From 187d37423dbbe4b43846e1c32ba19adb04dbae9c Mon Sep 17 00:00:00 2001 From: Carlo van Driesten Date: Thu, 7 May 2026 13:58:58 +0200 Subject: [PATCH 09/14] fix(shaclgen): emit sh:pattern for pattern constraints inside any_of MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The SHACL generator translated any_of branches by dispatching solely on `any.range` (class, type, enum, or simple datatype). If a branch specified `pattern:` — either alone or combined with a range — the constraint was silently dropped, producing an empty blank node `[ ]` (trivially satisfied) instead of the intended `[ sh:pattern "..." ]`. This is a problem for schemas that use pattern alternatives in `any_of`, such as the SPDX license field where valid values are either members of a fixed enum (SPDX identifiers), IRIs, or custom identifiers matching the LicenseRef- pattern defined in SPDX Specification v2.3 Annex D (ABNF: license-ref = ["DocumentRef-"(idstring)":"]"LicenseRef-"(idstring)). The fix adds a single check after the range dispatch: if any.pattern: g.add((range_list[-1], SH.pattern, Literal(any.pattern))) This correctly handles: - Pattern-only branches (no range): node gets only sh:pattern - Range + pattern branches: node gets both sh:datatype and sh:pattern - Range-only branches (no pattern): unchanged behaviour The test suite now includes a dedicated schema exercising all three cases, with assertions on both the generated RDF triples and pyshacl validation of conforming/non-conforming data. Signed-off-by: Carlo van Driesten --- .../linkml/src/linkml/generators/shaclgen.py | 5 + .../input/shaclgen/any_of_pattern.yaml | 59 +++++++++ tests/linkml/test_generators/test_shaclgen.py | 121 ++++++++++++++++++ 3 files changed, 185 insertions(+) create mode 100644 tests/linkml/test_generators/input/shaclgen/any_of_pattern.yaml diff --git a/packages/linkml/src/linkml/generators/shaclgen.py b/packages/linkml/src/linkml/generators/shaclgen.py index 590f2a43fa..cdb674f4bf 100644 --- a/packages/linkml/src/linkml/generators/shaclgen.py +++ b/packages/linkml/src/linkml/generators/shaclgen.py @@ -380,6 +380,11 @@ def st_node_pv(p, v): add_simple_data_type(st_node_pv, r) range_list.append(st_node) + # Propagate pattern constraint to the branch node. + # A branch may combine range + pattern (e.g. range: string + # with pattern: "^...") or specify pattern alone (no range). + if any.pattern: + g.add((range_list[-1], SH.pattern, Literal(any.pattern))) Collection(g, or_node, range_list) else: prop_pv_literal(SH.hasValue, s.equals_number) diff --git a/tests/linkml/test_generators/input/shaclgen/any_of_pattern.yaml b/tests/linkml/test_generators/input/shaclgen/any_of_pattern.yaml new file mode 100644 index 0000000000..5b247bb2a1 --- /dev/null +++ b/tests/linkml/test_generators/input/shaclgen/any_of_pattern.yaml @@ -0,0 +1,59 @@ +id: https://w3id.org/linkml/examples/any_of_pattern +name: test_any_of_pattern +description: >- + Test schema for pattern constraints inside any_of branches. + Exercises three cases: (1) pattern-only branch (no range), + (2) range + pattern on the same branch, (3) mixed branches + where some have pattern and some do not. +prefixes: + linkml: https://w3id.org/linkml/ + ex: https://w3id.org/linkml/examples/any_of_pattern/ +imports: + - linkml:types +default_range: string +default_prefix: ex + +enums: + LicenseEnum: + permissible_values: + MIT: + Apache-2.0: + GPL-3.0-only: + +classes: + PatternOnlyBranch: + description: >- + A class where one any_of branch specifies only a pattern + (no range). The generated SHACL sh:or should contain a + node with sh:pattern but no sh:datatype or sh:class. + attributes: + license: + any_of: + - range: LicenseEnum + - range: uri + - pattern: "^LicenseRef-[a-zA-Z0-9\\-\\.]+$" + + RangeWithPattern: + description: >- + A class where an any_of branch combines range + pattern. + The generated SHACL sh:or node should have both sh:datatype + and sh:pattern. + attributes: + identifier: + any_of: + - range: string + pattern: "^[A-Z]{2}-[0-9]{4}$" + - range: integer + + MixedBranches: + description: >- + A class with three any_of branches: one with range only, + one with pattern only, one with range + pattern. Ensures + pattern is emitted only on branches that declare it. + attributes: + code: + any_of: + - range: integer + - pattern: "^CUSTOM-.*$" + - range: string + pattern: "^STD-[0-9]+$" diff --git a/tests/linkml/test_generators/test_shaclgen.py b/tests/linkml/test_generators/test_shaclgen.py index af09ddd34c..8fa68d4c2d 100644 --- a/tests/linkml/test_generators/test_shaclgen.py +++ b/tests/linkml/test_generators/test_shaclgen.py @@ -4942,3 +4942,124 @@ def test_rule_unknown_slot_key_skipped(): g = _parse_shacl(_UNKNOWN_KEY_SCHEMA_YAML) shape = URIRef("https://example.org/unknown-key/Obs") assert list(g.objects(shape, SH.sparql)) == [] + + +# --------------------------------------------------------------------------- +# pattern inside any_of branches +# --------------------------------------------------------------------------- + + +def test_any_of_with_pattern(input_path): + """Test that pattern constraints inside any_of branches emit sh:pattern. + + Exercises three cases: + 1. PatternOnlyBranch: any_of with a pattern-only branch (no range) + 2. RangeWithPattern: any_of with range + pattern on the same branch + 3. MixedBranches: combination of range-only, pattern-only, and range+pattern + """ + shacl = ShaclGenerator(input_path("shaclgen/any_of_pattern.yaml"), mergeimports=True).serialize() + g = rdflib.Graph() + g.parse(data=shacl) + + def get_or_branch_nodes(class_uri: str, slot_local: str) -> list[rdflib.BNode]: + """Return the list of BNodes inside sh:or for a given class property.""" + class_ref = URIRef(class_uri) + for prop_node in g.objects(class_ref, SH.property): + paths = list(g.objects(prop_node, SH.path)) + if any(slot_local in str(p) for p in paths): + for or_head in g.objects(prop_node, SH["or"]): + return list(Collection(g, or_head)) + return [] + + prefix = "https://w3id.org/linkml/examples/any_of_pattern/" + + # Case 1: PatternOnlyBranch — license slot has 3 branches: + # [enum sh:in], [sh:nodeKind sh:IRI], [sh:pattern "^LicenseRef-..."] + branches = get_or_branch_nodes(f"{prefix}PatternOnlyBranch", "license") + assert len(branches) == 3, f"Expected 3 branches, got {len(branches)}" + # Find the branch with sh:pattern + pattern_branches = [b for b in branches if list(g.objects(b, SH.pattern))] + assert len(pattern_branches) == 1, f"Expected 1 pattern branch, got {len(pattern_branches)}" + pattern_val = str(list(g.objects(pattern_branches[0], SH.pattern))[0]) + assert pattern_val == "^LicenseRef-[a-zA-Z0-9\\-\\.]+$" + # The pattern-only branch should NOT have sh:datatype or sh:class + assert list(g.objects(pattern_branches[0], SH.datatype)) == [] + assert list(g.objects(pattern_branches[0], SH["class"])) == [] + + # Case 2: RangeWithPattern — identifier slot has 2 branches: + # [sh:datatype xsd:string + sh:pattern "^[A-Z]{2}-[0-9]{4}$"], [sh:datatype xsd:integer] + branches = get_or_branch_nodes(f"{prefix}RangeWithPattern", "identifier") + assert len(branches) == 2, f"Expected 2 branches, got {len(branches)}" + # Find branch with both datatype and pattern + combo_branches = [b for b in branches if list(g.objects(b, SH.datatype)) and list(g.objects(b, SH.pattern))] + assert len(combo_branches) == 1, f"Expected 1 combo branch, got {len(combo_branches)}" + assert str(list(g.objects(combo_branches[0], SH.pattern))[0]) == "^[A-Z]{2}-[0-9]{4}$" + # The other branch (integer) should NOT have sh:pattern + int_branches = [b for b in branches if b not in combo_branches] + assert list(g.objects(int_branches[0], SH.pattern)) == [] + + # Case 3: MixedBranches — code slot has 3 branches: + # [sh:datatype xsd:integer], [sh:pattern "^CUSTOM-.*$"], [sh:datatype xsd:string + sh:pattern "^STD-[0-9]+$"] + branches = get_or_branch_nodes(f"{prefix}MixedBranches", "code") + assert len(branches) == 3, f"Expected 3 branches, got {len(branches)}" + # Exactly 2 branches should have sh:pattern + pattern_branches = [b for b in branches if list(g.objects(b, SH.pattern))] + assert len(pattern_branches) == 2, f"Expected 2 pattern branches, got {len(pattern_branches)}" + # Collect the patterns + patterns = sorted(str(list(g.objects(b, SH.pattern))[0]) for b in pattern_branches) + assert patterns == ["^CUSTOM-.*$", "^STD-[0-9]+$"] + # The integer-only branch should have no pattern + no_pattern = [b for b in branches if not list(g.objects(b, SH.pattern))] + assert len(no_pattern) == 1 + assert list(g.objects(no_pattern[0], SH.datatype)) == [URIRef("http://www.w3.org/2001/XMLSchema#integer")] + + +def test_any_of_with_pattern_pyshacl_end_to_end(input_path): + """End-to-end: pyshacl accepts values matching an ``any_of`` pattern branch and rejects others. + + This is the behavioural regression guard for the fix. Without ``sh:pattern`` on the + branch node, a pattern-only branch serialises as an empty shape ``[ ]``, which every + value node trivially satisfies — so ``sh:or`` would accept *anything* and the + non-conforming assertions below would fail. + """ + import pyshacl + + shacl_ttl = ShaclGenerator(input_path("shaclgen/any_of_pattern.yaml"), mergeimports=True).serialize() + + def conforms(data_ttl: str) -> tuple[bool, str]: + ok, _, text = pyshacl.validate( + data_graph=data_ttl, + shacl_graph=shacl_ttl, + data_graph_format="turtle", + shacl_graph_format="turtle", + ) + return ok, text + + prefixes = """ + @prefix ex: . + @prefix xsd: . + """ + + # Case 1: pattern-only branch. "MIT" satisfies the enum branch, an IRI satisfies the + # uri branch, and "LicenseRef-..." may only satisfy the pattern-only branch. + for value in ('"MIT"', "", '"LicenseRef-My-Custom.1"'): + ok, text = conforms(f"{prefixes}\nex:l1 a ex:PatternOnlyBranch ; ex:license {value} .") + assert ok, f"license {value} should conform:\n{text}" + # No branch matches: not an enum member, not an IRI, and does not match the pattern. + ok, _ = conforms(f'{prefixes}\nex:l2 a ex:PatternOnlyBranch ; ex:license "NotALicenseRef" .') + assert not ok, "A value matching no any_of branch must be rejected" + + # Case 2: range + pattern on the same branch — both must hold for that branch. + ok, text = conforms(f'{prefixes}\nex:i1 a ex:RangeWithPattern ; ex:identifier "AB-1234" .') + assert ok, f"identifier 'AB-1234' should conform:\n{text}" + ok, text = conforms(f'{prefixes}\nex:i2 a ex:RangeWithPattern ; ex:identifier "42"^^xsd:integer .') + assert ok, f"identifier 42 should conform via the integer branch:\n{text}" + ok, _ = conforms(f'{prefixes}\nex:i3 a ex:RangeWithPattern ; ex:identifier "ab-1234" .') + assert not ok, "A string violating the branch pattern must be rejected" + + # Case 3: mixed branches — each branch accepts only its own values. + for value in ('"7"^^xsd:integer', '"CUSTOM-anything"', '"STD-42"'): + ok, text = conforms(f"{prefixes}\nex:c1 a ex:MixedBranches ; ex:code {value} .") + assert ok, f"code {value} should conform:\n{text}" + ok, _ = conforms(f'{prefixes}\nex:c2 a ex:MixedBranches ; ex:code "STD-xyz" .') + assert not ok, "A value matching no any_of branch must be rejected" From 8bd7fa0d47da819b67d66cf1a00df88d4712918b Mon Sep 17 00:00:00 2001 From: Carlo van Driesten Date: Thu, 18 Jun 2026 09:00:15 +0200 Subject: [PATCH 10/14] feat(jsonschemagen): emit propertyNames from inlined-dict key slot constraints For an inlined-as-dict slot whose range class has an identifier/key slot, render the key slot's string-applicable constraints onto JSON Schema propertyNames (draft-06+) instead of dropping them. In the inlined-dict form the mapping key is the identifier value, so the key slot's constraints constrain the keys. JSON object keys are always strings, so only pattern, enum (equals_string_in) and a string const (equals_string) are emitted; numeric minimum/maximum, numeric const (equals_number) and allOf are excluded -- a numeric const would otherwise reject every key. structured_pattern is honored when materialize_patterns is enabled, consistent with value patterns. Backward compatible: emitted only when a string-applicable key constraint applies. Signed-off-by: Carlo van Driesten --- .../src/linkml/generators/jsonschemagen.py | 39 +++++ .../test_generators/test_jsonschemagen.py | 162 ++++++++++++++++++ 2 files changed, 201 insertions(+) diff --git a/packages/linkml/src/linkml/generators/jsonschemagen.py b/packages/linkml/src/linkml/generators/jsonschemagen.py index 91f61dc869..7ea1310f0d 100644 --- a/packages/linkml/src/linkml/generators/jsonschemagen.py +++ b/packages/linkml/src/linkml/generators/jsonschemagen.py @@ -802,6 +802,40 @@ def get_value_constraints_for_slot(self, slot: SlotDefinition | AnonymousSlotExp return constraints + def get_key_constraints_for_slot(self, slot: SlotDefinition | None) -> JsonSchema: + """Constraints applicable to the *keys* of an inlined-as-dict slot. + + In the inlined-dict form the mapping key *is* the value of the range class's + identifier/key slot (https://linkml.io/linkml/schemas/inlining.html) and is not + repeated inside the value object, so constraints declared on that slot -- or + inherited from its type -- are constraints on the object keys. The result is + intended for JSON Schema ``propertyNames``, which composes conjunctively with + ``additionalProperties``. + + JSON object keys are always strings (JSON Schema Core 2019-09, 9.3.2.5), so only + the string-applicable subset of :meth:`get_value_constraints_for_slot` is + returned: ``pattern`` (including a resolved ``structured_pattern`` and a pattern + inherited from the slot's type), a string ``const`` (``equals_string``) and a + string ``enum`` (``equals_string_in``). Numeric constraints -- ``minimum`` and + ``maximum``, and the numeric ``const`` produced by ``equals_number`` -- and the + ``allOf`` produced by ``range_expression`` are excluded: they cannot be satisfied + by a string key, and a numeric ``const`` would reject *every* key. + + :param slot: the identifier or key slot of the range class + :return: a schema for ``propertyNames``; empty when the key is unconstrained + """ + constraints = self.get_value_constraints_for_slot(slot) + + key_constraints = JsonSchema() + for keyword in ("pattern", "const"): + value = constraints.get(keyword) + if isinstance(value, str): + key_constraints[keyword] = value + enum_values = constraints.get("enum") + if isinstance(enum_values, list) and all(isinstance(value, str) for value in enum_values): + key_constraints["enum"] = enum_values + return key_constraints + def get_subschema_for_slot( self, slot: SlotDefinition | AnonymousSlotExpression, @@ -858,6 +892,11 @@ def get_subschema_for_slot( else: typ = ["object", "null"] prop = JsonSchema({"type": typ, "additionalProperties": additionalProps}) + # The dict keys are the range's identifier/key values, so that + # slot's string-applicable constraints constrain the keys. + key_constraints = self.get_key_constraints_for_slot(range_id_slot) + if key_constraints: + prop["propertyNames"] = key_constraints self.top_level_schema.add_lax_def(reference, self.aliased_slot_name(range_id_slot)) else: prop = JsonSchema.array_of(JsonSchema.ref_for(reference), include_null, required=slot.required) diff --git a/tests/linkml/test_generators/test_jsonschemagen.py b/tests/linkml/test_generators/test_jsonschemagen.py index b914042dbe..f0bfe170ef 100644 --- a/tests/linkml/test_generators/test_jsonschemagen.py +++ b/tests/linkml/test_generators/test_jsonschemagen.py @@ -1626,3 +1626,165 @@ def test_generate_array_error_complex_unbounded_shape(array_error_complex_unboun _ = JsonSchemaGenerator( array_error_complex_unbounded, ).generate() + + +def _inlined_dict_schema( + key_slot_yaml: str, + key_decl: str = "identifier: true", + key_range: str = "string", + extra_yaml: str = "", +) -> str: + """Build a schema with an inlined-as-dict slot whose key slot is configured by + ``key_decl`` (``identifier: true`` or ``key: true``), ``key_range`` (the key slot + range), and ``key_slot_yaml`` (extra YAML lines for the key slot). ``extra_yaml`` is + appended at the top level, for declaring extra ``types``/``enums``.""" + return f""" +id: https://example.org/test-key-constraints +name: test-key-constraints +prefixes: + linkml: https://w3id.org/linkml/ +default_range: string +imports: + - linkml:types +{extra_yaml} +classes: + Container: + tree_root: true + attributes: + entries: + range: Entry + multivalued: true + inlined: true + inlined_as_list: false + Entry: + attributes: + key: + {key_decl} + range: {key_range} +{key_slot_yaml} + val: + range: string +""" + + +@pytest.mark.parametrize("key_decl", ["identifier: true", "key: true"]) +def test_inlined_dict_key_pattern_emits_property_names(key_decl): + """A literal ``pattern`` on the inlined-dict key slot (identifier or key) must be + rendered onto ``propertyNames``.""" + schema = _inlined_dict_schema(' pattern: "^[0-9]+$"', key_decl=key_decl) + generated = json.loads(JsonSchemaGenerator(schema).serialize()) + assert generated["properties"]["entries"]["propertyNames"] == {"pattern": "^[0-9]+$"} + + +def test_inlined_dict_key_enum_emits_property_names(): + """``equals_string_in`` on the key slot becomes an ``enum`` constraint on keys.""" + schema = _inlined_dict_schema(" equals_string_in:\n - a\n - b") + generated = json.loads(JsonSchemaGenerator(schema).serialize()) + assert generated["properties"]["entries"]["propertyNames"] == {"enum": ["a", "b"]} + + +def test_inlined_dict_no_key_constraint_emits_no_property_names(): + """No constraint on the key slot -> no ``propertyNames`` (unchanged behavior).""" + schema = _inlined_dict_schema("") + generated = json.loads(JsonSchemaGenerator(schema).serialize()) + assert "propertyNames" not in generated["properties"]["entries"] + + +def test_inlined_dict_key_structured_pattern_emits_property_names(): + """``structured_pattern`` on the key slot is resolved and rendered onto + ``propertyNames`` -- identical to how value patterns are handled.""" + schema = _inlined_dict_schema(" structured_pattern:\n syntax: '[0-9]+'") + generated = json.loads(JsonSchemaGenerator(schema).serialize()) + + key_pattern = generated["$defs"]["Entry"]["properties"]["key"]["pattern"] + assert generated["properties"]["entries"]["propertyNames"] == {"pattern": key_pattern} + jsonschema.validate({"entries": {"12": {"val": "x"}}}, generated) + with pytest.raises(jsonschema.ValidationError): + jsonschema.validate({"entries": {"bad-key": {"val": "x"}}}, generated) + + +def test_inlined_dict_property_names_rejects_nonmatching_keys(): + """Behavioral check: keys matching the pattern validate; non-matching keys fail.""" + schema = _inlined_dict_schema(' pattern: "^[0-9]+$"') + generated = json.loads(JsonSchemaGenerator(schema).serialize()) + + jsonschema.validate({"entries": {"0": {"val": "x"}}}, generated) + with pytest.raises(jsonschema.ValidationError): + jsonschema.validate({"entries": {"bad-key": {"val": "x"}}}, generated) + + +def test_inlined_dict_key_string_const_emits_property_names(): + """A string ``const`` (``equals_string``) on the key slot becomes a key const.""" + schema = _inlined_dict_schema(" equals_string: fixed") + generated = json.loads(JsonSchemaGenerator(schema).serialize()) + assert generated["properties"]["entries"]["propertyNames"] == {"const": "fixed"} + jsonschema.validate({"entries": {"fixed": {"val": "x"}}}, generated) + with pytest.raises(jsonschema.ValidationError): + jsonschema.validate({"entries": {"other": {"val": "x"}}}, generated) + + +def test_inlined_dict_key_numeric_const_is_not_emitted(): + """A numeric ``const`` (``equals_number``) must NOT be emitted onto propertyNames: + keys are always strings, so a numeric const would reject every key. The keys are + left unconstrained instead.""" + schema = _inlined_dict_schema(" equals_number: 5", key_range="integer") + generated = json.loads(JsonSchemaGenerator(schema).serialize()) + assert "propertyNames" not in generated["properties"]["entries"] + # numeric-looking string keys still validate (unconstrained) + jsonschema.validate({"entries": {"5": {"val": "x"}}}, generated) + jsonschema.validate({"entries": {"anything": {"val": "x"}}}, generated) + + +def test_inlined_dict_key_numeric_bounds_are_not_emitted(): + """Numeric ``minimum``/``maximum`` on the key slot are no-ops on string keys and + must not be emitted (they would be misleading clutter).""" + schema = _inlined_dict_schema(" minimum_value: 1\n maximum_value: 10", key_range="integer") + generated = json.loads(JsonSchemaGenerator(schema).serialize()) + assert "propertyNames" not in generated["properties"]["entries"] + + +@pytest.mark.parametrize( + ("key_range", "extra_yaml"), + [ + ("ncname", ""), + ("DigitString", "types:\n DigitString:\n typeof: string\n pattern: '^[0-9]+$'"), + ], + ids=["base-implied-pattern", "user-defined-type-pattern"], +) +def test_inlined_dict_key_type_pattern_emits_property_names(key_range, extra_yaml): + """A pattern inherited from the key slot's *type* constrains the identifier value just + as a slot-level pattern does, so it must reach ``propertyNames`` too. The emitted key + pattern is exactly the one applied to the identifier inside the value object, so keys + and the (optional) in-object identifier are validated identically.""" + schema = _inlined_dict_schema("", key_range=key_range, extra_yaml=extra_yaml) + generated = json.loads(JsonSchemaGenerator(schema).serialize()) + + value_key_pattern = generated["$defs"]["Entry__identifier_optional"]["properties"]["key"]["pattern"] + assert generated["properties"]["entries"]["propertyNames"] == {"pattern": value_key_pattern} + + +def test_inlined_dict_key_enum_range_emits_no_property_names(): + """A key slot whose range is a LinkML *enum* is compiled to a ``$ref`` on the value + side; ``get_value_constraints_for_slot`` reports no string-applicable constraint for + it, so no ``propertyNames`` is emitted and the keys stay unconstrained.""" + schema = _inlined_dict_schema( + "", key_range="Colour", extra_yaml="enums:\n Colour:\n permissible_values:\n red:\n green:" + ) + generated = json.loads(JsonSchemaGenerator(schema).serialize()) + assert "propertyNames" not in generated["properties"]["entries"] + + +def test_inlined_dict_key_constraints_helper_drops_non_string_values(): + """``get_key_constraints_for_slot`` keeps only string-applicable keywords, regardless + of which upstream constraint produced them: a numeric ``const`` or a non-string + ``enum`` would reject every key, so both are dropped.""" + schema = _inlined_dict_schema("") + generator = JsonSchemaGenerator(schema) + generator.generate() + + slot = SlotDefinition("key", pattern="^[0-9]+$") + assert generator.get_key_constraints_for_slot(slot) == {"pattern": "^[0-9]+$"} + + assert generator.get_key_constraints_for_slot(SlotDefinition("key", equals_number=5)) == {} + assert generator.get_key_constraints_for_slot(SlotDefinition("key", minimum_value=1)) == {} + assert generator.get_key_constraints_for_slot(None) == {} From c8f5088bc731361b75a444239b93c921b9d768f6 Mon Sep 17 00:00:00 2001 From: jdsika Date: Sat, 11 Jul 2026 12:42:16 +0200 Subject: [PATCH 11/14] docs(json-schema): document propertyNames emission from inlined-dict key constraints Signed-off-by: jdsika --- docs/generators/json-schema.rst | 66 +++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/docs/generators/json-schema.rst b/docs/generators/json-schema.rst index 2f8f1d8d91..4378335d5e 100644 --- a/docs/generators/json-schema.rst +++ b/docs/generators/json-schema.rst @@ -378,6 +378,72 @@ will generate: LinkML also supports `Structured patterns `_, these are compiled down to patterns during JSON Schema generation. +Dictionary key constraints (propertyNames) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A multivalued, inlined slot whose range class has an identifier slot is +compiled to a JSON object keyed by that identifier (see *Inlining* above). +When the identifier slot carries string-applicable constraints, they are +emitted as a `propertyNames `_ +schema on the container object, so the *keys* of the dictionary are validated, +not just the values: + +.. code-block:: yaml + + slots: + tags: + range: Tag + multivalued: true + inlined: true + uid: + identifier: true + pattern: "^(0|[1-9][0-9]*)$" + +generates on the container: + +.. code-block:: json + + "tags": { + "additionalProperties": {"$ref": "#/$defs/Tag"}, + "propertyNames": {"pattern": "^(0|[1-9][0-9]*)$"}, + "type": "object" + } + +The constraints carried over from the key slot are the ones applicable to JSON +Schema strings, because object keys are always strings (`JSON Schema Core +2019-09, §9.3.2.5 `_): + +* ``pattern`` -- whether written directly on the slot, resolved from a + ``structured_pattern``, or inherited from the slot's ``range`` type (for + example an identifier with ``range: ncname``, or a user-defined type that + declares a ``pattern``); +* ``equals_string_in``, emitted as ``enum``; +* a string ``equals_string``, emitted as ``const``. + +The emitted key pattern is always the same one that applies to the identifier +*inside* the value object, so a key and a redundantly repeated in-object +identifier are now validated identically. + +Numeric constraints -- ``minimum_value``/``maximum_value``, and the numeric +``const`` produced by ``equals_number`` -- are deliberately **not** carried +over: they cannot be satisfied by a string key, and a numeric ``const`` would +reject every key. The ``allOf`` produced by a ``range_expression``, and the +permissible values of an ``enum``-ranged identifier, are likewise out of scope. + +``propertyNames`` composes conjunctively with ``additionalProperties``, so keys +and values are constrained independently. It is emitted only when the key slot +actually carries one of the constraints listed above; an unconstrained key slot +produces exactly the same output as before. + +.. note:: + + Because type-level patterns are included, an identifier slot whose range is + ``ncname`` (or another pattern-bearing type) gains a ``propertyNames`` + entry even if the slot itself declares no constraint. The generated schema + becomes stricter, but only in ways the model already required: data whose + keys satisfy the declared identifier type is unaffected. + + Rules ^^^^^ From e79834822519474972d2700deceaa3183700a798 Mon Sep 17 00:00:00 2001 From: jdsika Date: Thu, 2 Apr 2026 17:21:36 +0200 Subject: [PATCH 12/14] feat(generators): add --normalize-prefixes flag for well-known prefix names Add an opt-in --normalize-prefixes flag to OWL, SHACL, and JSON-LD Context generators that normalises non-standard prefix aliases to well-known names from a static prefix map (derived from rdflib 7.x defaults, cross-checked against prefix.cc consensus). Key design decisions: - Static frozen map (MappingProxyType) instead of runtime Graph().namespaces() lookup eliminates rdflib version dependency - Both http://schema.org/ and https://schema.org/ map to 'schema' - Shared normalize_graph_prefixes() helper used by OWL and SHACL - Two-phase graph normalisation: Phase 1 normalises schema-declared prefixes, Phase 2 cleans up runtime-injected bindings - Collision detection: skip with warning when standard prefix name is already user-declared for a different namespace - Phase 2 guard prevents overwriting HTTPS bindings with HTTP variants The flag defaults to off, preserving existing behaviour. Tests cover OWL, SHACL, and context generators with sdo->schema, dce->dc, http/https edge case, custom prefix preservation, flag-off backward compatibility, cross-generator consistency, prefix collision detection, schema1 regression prevention, Phase 2 HTTPS guard, empty schema edge case, and static map integrity. Signed-off-by: jdsika Signed-off-by: Carlo van Driesten --- packages/linkml/pyproject.toml | 9 +- .../src/linkml/generators/jsonldcontextgen.py | 82 ++- .../linkml/src/linkml/generators/jsonldgen.py | 2 + .../linkml/src/linkml/generators/owlgen.py | 6 +- .../linkml/src/linkml/generators/shaclgen.py | 6 +- packages/linkml/src/linkml/utils/generator.py | 170 +++++- .../test_generators/test_jsonldcontextgen.py | 115 ++++ .../test_normalize_prefixes.py | 545 ++++++++++++++++++ uv.lock | 10 +- 9 files changed, 932 insertions(+), 13 deletions(-) create mode 100644 tests/linkml/test_generators/test_normalize_prefixes.py diff --git a/packages/linkml/pyproject.toml b/packages/linkml/pyproject.toml index a0e778f2a4..09153fbc48 100644 --- a/packages/linkml/pyproject.toml +++ b/packages/linkml/pyproject.toml @@ -50,7 +50,10 @@ dependencies = [ # Specifier syntax: https://peps.python.org/pep-0631/ "openpyxl", "parse", "prefixcommons >= 0.1.7", - "prefixmaps >= 0.2.2", + # TODO(prefixmaps-0.2.8): Replace git pin with "prefixmaps >= 0.2.8" once released, + # then remove [tool.hatch.metadata] allow-direct-references and regenerate uv.lock. + # Tracked in: https://github.com/linkml/prefixmaps/issues/82 + "prefixmaps @ git+https://github.com/linkml/prefixmaps@75435150a1b31760b9780af2b64a265943a9b263", "pydantic>=2.13.5,<3.0.0", "pyjsg >= 0.12.3", "pyshex >= 0.9.0", @@ -207,6 +210,10 @@ vcs = "git" style = "pep440" fallback-version = "0.0.0" +[tool.hatch.metadata] +# TODO(prefixmaps-0.2.8): Remove this section once the git pin is replaced with >= 0.2.8 +allow-direct-references = true + [tool.hatch.version] source = "uv-dynamic-versioning" diff --git a/packages/linkml/src/linkml/generators/jsonldcontextgen.py b/packages/linkml/src/linkml/generators/jsonldcontextgen.py index 7f83b6dddf..2747403706 100644 --- a/packages/linkml/src/linkml/generators/jsonldcontextgen.py +++ b/packages/linkml/src/linkml/generators/jsonldcontextgen.py @@ -15,7 +15,7 @@ from linkml._version import __version__ from linkml.utils.deprecation import deprecated_fields -from linkml.utils.generator import Generator, shared_arguments +from linkml.utils.generator import Generator, shared_arguments, well_known_prefix_map from linkml_runtime.linkml_model.meta import ClassDefinition, EnumDefinition, SlotDefinition from linkml_runtime.linkml_model.types import SHEX from linkml_runtime.utils.formatutils import camelcase, underscore @@ -93,6 +93,9 @@ class ContextGenerator(Generator): frame_root: str | None = None def __post_init__(self) -> None: + # Must be set before super().__post_init__() because the parent triggers + # the visitor pattern (visit_schema), which accesses _prefix_remap. + self._prefix_remap: dict[str, str] = {} super().__post_init__() if self.namespaces is None: raise TypeError("Schema text must be supplied to context generator. Preparsed schema will not work") @@ -130,8 +133,14 @@ def _collect_external_elements(sv: SchemaView) -> tuple[set[str], set[str]]: external_slots.update(schema_def.slots.keys()) return external_classes, external_slots + def add_prefix(self, ncname: str) -> None: + """Add a prefix, applying well-known prefix normalisation when enabled.""" + super().add_prefix(self._prefix_remap.get(ncname, ncname)) + def visit_schema(self, base: str | Namespace | None = None, output: str | None = None, **_): - # Add any explicitly declared prefixes + # Add any explicitly declared prefixes. + # Direct .add() is safe here: the normalisation block below explicitly + # rewrites emit_prefixes entries for any renamed prefixes (Cases 1-3). for prefix in self.schema.prefixes.values(): self.emit_prefixes.add(prefix.prefix_prefix) @@ -139,6 +148,68 @@ def visit_schema(self, base: str | Namespace | None = None, output: str | None = for pfx in self.schema.emit_prefixes: self.add_prefix(pfx) + # Normalise well-known prefix names when --normalize-prefixes is set. + # If the schema declares a non-standard alias for a namespace that has + # a well-known standard name (e.g. ``sdo`` for + # ``https://schema.org/``), replace the alias with the standard name + # so that generated JSON-LD contexts use the conventional prefix. + # + # Three cases are handled: + # 1. Standard prefix is not yet bound → just rebind from old to new. + # 2. Standard prefix is bound to a *different* URI: + # a. User-declared (in schema.prefixes) → collision, skip with warning. + # b. Runtime default (e.g. linkml-runtime's ``schema: http://…``) + # → remove stale binding, then rebind. + # 3. Standard prefix is already bound to the *same* URI (duplicate) + # → just drop the non-standard alias. + # + # A remap dict is stored for ``_build_element_id`` because + # ``prefix_suffix()`` splits CURIEs on ``:`` without looking up the + # namespace dict. + self._prefix_remap.clear() + if self.normalize_prefixes: + wk = well_known_prefix_map() + for old_pfx in list(self.namespaces): + url = str(self.namespaces[old_pfx]) + std_pfx = wk.get(url) + if not std_pfx or std_pfx == old_pfx: + continue + if std_pfx in self.namespaces: + if str(self.namespaces[std_pfx]) != url: + # Case 2: std_pfx is bound to a different URI. + # If the user explicitly declared std_pfx in the schema, + # it is intentional — skip to avoid data loss. + if std_pfx in self.schema.prefixes: + self.logger.warning( + "Prefix collision: cannot rename '%s' to '%s' because '%s' is " + "already declared for <%s>; skipping normalisation for <%s>", + old_pfx, + std_pfx, + std_pfx, + str(self.namespaces[std_pfx]), + url, + ) + continue + # Not user-declared (e.g. linkml-runtime default) — safe to remove + self.emit_prefixes.discard(std_pfx) + del self.namespaces[std_pfx] + else: + # Case 3: standard prefix already bound to same URI + # — just drop the non-standard alias + del self.namespaces[old_pfx] + if old_pfx in self.emit_prefixes: + self.emit_prefixes.discard(old_pfx) + self.emit_prefixes.add(std_pfx) + self._prefix_remap[old_pfx] = std_pfx + continue + # Case 1 (or Case 2 after stale removal): bind standard name + self.namespaces[std_pfx] = self.namespaces[old_pfx] + del self.namespaces[old_pfx] + if old_pfx in self.emit_prefixes: + self.emit_prefixes.discard(old_pfx) + self.emit_prefixes.add(std_pfx) + self._prefix_remap[old_pfx] = std_pfx + # Add the default prefix if self.schema.default_prefix: dflt = self.namespaces.prefix_for(self.schema.default_prefix) @@ -146,6 +217,8 @@ def visit_schema(self, base: str | Namespace | None = None, output: str | None = self.default_ns = dflt if self.default_ns: default_uri = self.namespaces[self.default_ns] + # Direct .add() is safe: default_ns is already resolved from + # the (possibly normalised) namespace bindings above. self.emit_prefixes.add(self.default_ns) else: default_uri = self.schema.default_prefix @@ -509,6 +582,11 @@ def _build_element_id(self, definition: Any, uri: str) -> None: @return: None """ uri_prefix, uri_suffix = self.namespaces.prefix_suffix(uri) + # Apply well-known prefix normalisation (e.g. sdo → schema). + # prefix_suffix() splits CURIEs on ':' without checking the + # namespace dict, so it may return a stale alias. + if uri_prefix and uri_prefix in self._prefix_remap: + uri_prefix = self._prefix_remap[uri_prefix] is_default_namespace = uri_prefix == self.context_body["@vocab"] or uri_prefix == self.namespaces.prefix_for( self.context_body["@vocab"] ) diff --git a/packages/linkml/src/linkml/generators/jsonldgen.py b/packages/linkml/src/linkml/generators/jsonldgen.py index 91759a3653..118d568327 100644 --- a/packages/linkml/src/linkml/generators/jsonldgen.py +++ b/packages/linkml/src/linkml/generators/jsonldgen.py @@ -190,6 +190,8 @@ def end_schema( # through the same ``--importmap`` the caller supplied. context_kwargs.setdefault("importmap", self.importmap) context_kwargs.setdefault("base_dir", self.base_dir) + # Forward prefix normalisation into the inline @context. + context_kwargs.setdefault("normalize_prefixes", self.normalize_prefixes) add_prefixes = ContextGenerator(self.original_schema, **context_kwargs).serialize() add_prefixes_json = loads(add_prefixes) metamodel_ctx = self.metamodel_context or METAMODEL_CONTEXT_URI diff --git a/packages/linkml/src/linkml/generators/owlgen.py b/packages/linkml/src/linkml/generators/owlgen.py index 7751eab34d..04ac32658f 100644 --- a/packages/linkml/src/linkml/generators/owlgen.py +++ b/packages/linkml/src/linkml/generators/owlgen.py @@ -21,7 +21,7 @@ from linkml._version import __version__ from linkml.generators.common.subproperty import is_xsd_anyuri_range from linkml.utils.deprecation import deprecation_warning -from linkml.utils.generator import Generator, shared_arguments +from linkml.utils.generator import Generator, normalize_graph_prefixes, shared_arguments from linkml.utils.language_tags import LanguageTagResolver from linkml_runtime import SchemaView from linkml_runtime.linkml_model.meta import ( @@ -332,6 +332,10 @@ def as_graph(self) -> Graph: self.graph.bind(prefix, self.metamodel.namespaces[prefix]) for pfx in schema.prefixes.values(): self.graph.namespace_manager.bind(pfx.prefix_prefix, URIRef(pfx.prefix_reference)) + if self.normalize_prefixes: + normalize_graph_prefixes( + graph, {str(v.prefix_prefix): str(v.prefix_reference) for v in schema.prefixes.values()} + ) graph.add((base, RDF.type, OWL.Ontology)) # Add main schema elements diff --git a/packages/linkml/src/linkml/generators/shaclgen.py b/packages/linkml/src/linkml/generators/shaclgen.py index cdb674f4bf..686abf68fb 100644 --- a/packages/linkml/src/linkml/generators/shaclgen.py +++ b/packages/linkml/src/linkml/generators/shaclgen.py @@ -15,7 +15,7 @@ from linkml.generators.common.subproperty import get_subproperty_values, is_uri_range from linkml.generators.shacl.shacl_data_type import ShaclDataType from linkml.generators.shacl.shacl_ifabsent_processor import ShaclIfAbsentProcessor -from linkml.utils.generator import Generator, shared_arguments +from linkml.utils.generator import Generator, normalize_graph_prefixes, shared_arguments from linkml.utils.language_tags import LanguageTagResolver from linkml_runtime.linkml_model.meta import ClassDefinition, ElementName, PresenceEnum from linkml_runtime.utils.formatutils import underscore @@ -228,6 +228,10 @@ def as_graph(self) -> Graph: for pfx in self.schema.prefixes.values(): g.bind(str(pfx.prefix_prefix), pfx.prefix_reference) + if self.normalize_prefixes: + normalize_graph_prefixes( + g, {str(v.prefix_prefix): str(v.prefix_reference) for v in self.schema.prefixes.values()} + ) for c in sv.all_classes(imports=not self.exclude_imports).values(): diff --git a/packages/linkml/src/linkml/utils/generator.py b/packages/linkml/src/linkml/utils/generator.py index 0bda5a8151..252ec4e08c 100644 --- a/packages/linkml/src/linkml/utils/generator.py +++ b/packages/linkml/src/linkml/utils/generator.py @@ -20,12 +20,13 @@ import os import re import sys +import types from collections.abc import Callable, Mapping from copy import deepcopy from dataclasses import dataclass, field from functools import lru_cache from pathlib import Path -from typing import IO, Any, ClassVar, TextIO, Union, cast +from typing import IO, TYPE_CHECKING, Any, ClassVar, TextIO, Union, cast import click import yaml @@ -60,6 +61,9 @@ from linkml_runtime.utils.formatutils import camelcase, underscore from linkml_runtime.utils.namespaces import Namespaces +if TYPE_CHECKING: + from rdflib import Graph + logger = logging.getLogger(__name__) @@ -80,6 +84,154 @@ def _resolved_metamodel(mergeimports): return metamodel +def well_known_prefix_map() -> dict[str, str]: + """Return a mapping from namespace URI to standard prefix name. + + Primary source: the ``linked_data`` context from `prefixmaps + `_ — the canonical curated + registry maintained by the LinkML team. This context provides + correct, community-consensus prefix names (e.g. ``sh`` not ``shacl``, + ``schema`` not ``sdo``). + + Secondary source: the ``merged`` context from prefixmaps, which + combines prefix.cc, bioregistry, and other sources for broad coverage. + + A small ``_PREFIX_OVERRIDES`` map corrects the few cases where the + merged context disagrees with rdflib/W3C canonical names. + + Both ``http`` and ``https`` variants of schema.org and wgs84 are + included because the linkml-runtime historically binds the HTTP form + while rdflib (and the W3C) prefer HTTPS. + + .. note:: + Requires ``prefixmaps >= 0.2.7``. For entries added in + linkml/prefixmaps#81 (W3C/OGC standard prefixes), pin to + ``prefixmaps @ git+https://github.com/linkml/prefixmaps@75435150`` + until v0.2.8 is released. + """ + return dict(_cached_well_known_prefix_map()) + + +@lru_cache(maxsize=1) +def _cached_well_known_prefix_map() -> dict[str, str]: + """Internal cached builder for well_known_prefix_map().""" + from prefixmaps import load_context + + # Layer 1: merged context (broad coverage, first-seen-wins for duplicates). + merged = load_context("merged") + ns_to_prefix: dict[str, str] = {} + for rec in merged.prefix_expansions: + if rec.namespace not in ns_to_prefix: + ns_to_prefix[rec.namespace] = rec.prefix + + # Layer 2: linked_data context (curated, correct names) overrides merged. + ld = load_context("linked_data") + for rec in ld.prefix_expansions: + ns_to_prefix[rec.namespace] = rec.prefix + + # Layer 3: overrides for the few cases where merged/linked_data disagrees + # with the rdflib/W3C canonical forms used by the RDF community. + for ns, pfx in _PREFIX_OVERRIDES.items(): + ns_to_prefix[ns] = pfx + + # Ensure both HTTP/HTTPS schema.org variants resolve to 'schema'. + ns_to_prefix.setdefault("https://schema.org/", "schema") + ns_to_prefix["http://schema.org/"] = "schema" + + # Ensure both HTTP/HTTPS wgs84 variants resolve to 'wgs'. + ns_to_prefix.setdefault("https://www.w3.org/2003/01/geo/wgs84_pos#", "wgs") + + return ns_to_prefix + + +# Overrides: corrections where prefixmaps merged context uses non-standard names +# that differ from rdflib 7.x / W3C canonical forms. +_PREFIX_OVERRIDES: types.MappingProxyType[str, str] = types.MappingProxyType( + { + # merged gives 'geosparql', rdflib/W3C uses 'geo' + "http://www.opengis.net/ont/geosparql#": "geo", + # merged gives 'sc', rdflib/W3C uses 'schema' + "https://schema.org/": "schema", + # merged gives 'WGS84', rdflib uses 'wgs' + "https://www.w3.org/2003/01/geo/wgs84_pos#": "wgs", + "http://www.w3.org/2003/01/geo/wgs84_pos#": "wgs", + } +) + + +def normalize_graph_prefixes(graph: "Graph", schema_prefixes: dict[str, str]) -> None: + """Normalise non-standard prefix aliases in an rdflib Graph. + + For each prefix bound in *schema_prefixes* (mapping prefix name → + namespace URI), check whether ``well_known_prefix_map()`` knows a + standard name for that URI. If the standard name differs from the + schema-declared name, rebind the namespace to the standard name. + + This is the **shared implementation** used by OWL, SHACL, and (via a + different code-path) JSON-LD context generators so that all serialisation + formats agree on prefix names when ``--normalize-prefixes`` is active. + + :param graph: rdflib Graph whose namespace bindings should be adjusted. + :param schema_prefixes: mapping of prefix name → namespace URI string, + typically from ``schema.prefixes``. + """ + from rdflib import Namespace + + wk = well_known_prefix_map() + + # Phase 1: normalise schema-declared prefixes. + for old_pfx, ns_uri in schema_prefixes.items(): + ns_str = str(ns_uri) + std_pfx = wk.get(ns_str) + if not std_pfx or std_pfx == old_pfx: + continue + # Collision: the user explicitly declared std_pfx for a different + # namespace — do not clobber their binding. + if std_pfx in schema_prefixes and schema_prefixes[std_pfx] != ns_str: + logger.warning( + "Prefix collision: cannot rename '%s' to '%s' because '%s' is already " + "declared for <%s>; skipping normalisation for <%s>", + old_pfx, + std_pfx, + std_pfx, + schema_prefixes[std_pfx], + ns_str, + ) + continue + # Rebind: remove old prefix, add standard prefix. + # ``replace=True`` forces the new prefix even if the prefix name + # is already bound to a different namespace. + graph.bind(std_pfx, Namespace(ns_str), override=True, replace=True) + + # Phase 2: normalise runtime-injected bindings (e.g. metamodel defaults). + # The linkml-runtime / rdflib may inject well-known namespaces under + # non-standard prefix names. After Phase 1 rebinds schema-declared + # prefixes, orphaned runtime bindings can appear as ``schema1``, ``dc0``, + # etc. Scan the graph's current bindings and fix any that map to a + # well-known namespace under a non-standard name, provided the standard + # name isn't already claimed by the user for a different namespace. + # + # Guard: if Phase 1 already bound std_pfx to a different URI (e.g. + # ``schema`` → ``https://schema.org/``), do not clobber it with the + # HTTP variant (``http://schema.org/``). Build a snapshot of the + # current bindings after Phase 1 to detect this. + current_bindings = {str(p): str(n) for p, n in graph.namespaces()} + for pfx, ns in list(graph.namespaces()): + pfx_str, ns_str = str(pfx), str(ns) + std_pfx = wk.get(ns_str) + if not std_pfx or std_pfx == pfx_str: + continue + # Same collision check as Phase 1: respect user-declared prefixes. + if std_pfx in schema_prefixes and schema_prefixes[std_pfx] != ns_str: + continue + # Guard: if std_pfx is already bound to a different (correct) URI + # by Phase 1, do not overwrite it. This prevents the HTTP variant + # of schema.org from clobbering the HTTPS binding. + if std_pfx in current_bindings and current_bindings[std_pfx] != ns_str: + continue + graph.bind(std_pfx, Namespace(ns_str), override=True, replace=True) + + @dataclass class Generator(metaclass=abc.ABCMeta): """ @@ -187,6 +339,12 @@ class Generator(metaclass=abc.ABCMeta): stacktrace: bool = False """True means print stack trace, false just error message""" + normalize_prefixes: bool = False + """True means normalise non-standard prefix aliases to well-known names + from the ``prefixmaps`` package (linked_data + merged contexts, with + overrides for rdflib/W3C canonical forms). E.g. ``sdo`` → ``schema`` + for ``https://schema.org/``.""" + include: str | Path | SchemaDefinition | None = None """If set, include extra schema outside of the imports mechanism""" @@ -1042,6 +1200,16 @@ def decorator(f: Command) -> Command: callback=stacktrace_callback, ) ) + f.params.append( + Option( + ("--normalize-prefixes/--no-normalize-prefixes",), + default=False, + show_default=True, + help="Normalise non-standard prefix aliases to rdflib's curated default names " + "(e.g. sdo → schema for https://schema.org/). " + "Supported by OWL, SHACL, and JSON-LD Context generators.", + ) + ) return f diff --git a/tests/linkml/test_generators/test_jsonldcontextgen.py b/tests/linkml/test_generators/test_jsonldcontextgen.py index 8aefc013b1..bdbcfa331c 100644 --- a/tests/linkml/test_generators/test_jsonldcontextgen.py +++ b/tests/linkml/test_generators/test_jsonldcontextgen.py @@ -1669,3 +1669,118 @@ def test_kitchen_sink_employment_event_type_falls_back(kitchen_sink_path): slot_def = ctx["employed_at"] if isinstance(slot_def, dict) and "@context" in slot_def: assert "@vocab" not in slot_def.get("@context", {}) + + +def test_normalize_prefixes_renames_nonstandard_alias(tmp_path): + """When --normalize-prefixes is set, non-standard aliases are replaced by rdflib defaults. + + rdflib binds ``dc`` to ``http://purl.org/dc/elements/1.1/`` by default. + A schema that declares ``dce`` for the same URI should have it normalised + to ``dc`` when the flag is enabled. + + See: rdflib default namespace bindings. + """ + schema = tmp_path / "schema.yaml" + schema.write_text( + """\ +id: https://example.org/test +name: test_normalize +default_prefix: ex +prefixes: + ex: https://example.org/ + linkml: https://w3id.org/linkml/ + dce: http://purl.org/dc/elements/1.1/ +imports: + - linkml:types +classes: + Record: + class_uri: ex:Record + attributes: + title: + range: string + slot_uri: dce:title +""", + encoding="utf-8", + ) + + # Flag OFF (default): non-standard alias preserved + ctx_off = json.loads(ContextGenerator(str(schema), normalize_prefixes=False).serialize())["@context"] + assert "dce" in ctx_off, "With flag off, original prefix 'dce' must be preserved" + + # Flag ON: rdflib default name used + ctx_on = json.loads(ContextGenerator(str(schema), normalize_prefixes=True).serialize())["@context"] + assert "dc" in ctx_on, "With flag on, 'dce' should be normalised to 'dc'" + assert "dce" not in ctx_on, "With flag on, original alias 'dce' should be removed" + assert ctx_on["dc"] == "http://purl.org/dc/elements/1.1/" + + +def test_normalize_prefixes_default_is_off(tmp_path): + """The --normalize-prefixes flag defaults to False — no prefix renaming. + + Ensures backward compatibility: existing schemas produce identical output. + """ + schema = tmp_path / "schema.yaml" + schema.write_text( + """\ +id: https://example.org/test +name: test_default +default_prefix: ex +prefixes: + ex: https://example.org/ + linkml: https://w3id.org/linkml/ + sdo: https://schema.org/ +imports: + - linkml:types +classes: + Thing: + class_uri: sdo:Thing + attributes: + name: + range: string + slot_uri: sdo:name +""", + encoding="utf-8", + ) + + ctx = json.loads(ContextGenerator(str(schema)).serialize())["@context"] + # Without the flag, the schema's own prefix name must be preserved + assert "sdo" in ctx, "Default behavior must preserve schema-declared prefix 'sdo'" + + +def test_normalize_prefixes_curie_remapping(tmp_path): + """CURIEs in element @id values use the normalised prefix name. + + When ``sdo`` is normalised to ``schema``, slot URIs like ``sdo:name`` + must appear as ``schema:name`` in the generated context. + """ + schema = tmp_path / "schema.yaml" + schema.write_text( + """\ +id: https://example.org/test +name: test_curie +default_prefix: ex +prefixes: + ex: https://example.org/ + linkml: https://w3id.org/linkml/ + sdo: https://schema.org/ +imports: + - linkml:types +classes: + Person: + class_uri: sdo:Person + attributes: + full_name: + range: string + slot_uri: sdo:name +""", + encoding="utf-8", + ) + + ctx = json.loads(ContextGenerator(str(schema), normalize_prefixes=True).serialize())["@context"] + # The prefix declaration must use the standard name + assert "schema" in ctx, "Normalised prefix 'schema' must appear" + # Element @id must use the normalised prefix + person = ctx.get("Person", {}) + assert person.get("@id", "").startswith("schema:"), ( + f"Person @id should use normalised prefix 'schema:', got {person}" + ) diff --git a/tests/linkml/test_generators/test_normalize_prefixes.py b/tests/linkml/test_generators/test_normalize_prefixes.py new file mode 100644 index 0000000000..0a832a5791 --- /dev/null +++ b/tests/linkml/test_generators/test_normalize_prefixes.py @@ -0,0 +1,545 @@ +"""Tests for the --normalize-prefixes flag across all generators. + +Verifies that non-standard prefix aliases (e.g. ``sdo`` for ``https://schema.org/``) +are normalised to well-known names (e.g. ``schema``) consistently in OWL, SHACL, +and JSON-LD context output. + +References: +- prefix.cc — community consensus RDF prefix registry +- rdflib 7.x curated default namespace bindings +- W3C Turtle §2.4 — prefix declarations are syntactic sugar +""" + +import json +import logging +import re +import textwrap + +import pytest + +# ── Shared test schema ────────────────────────────────────────────── + +SCHEMA_SDO = textwrap.dedent("""\ + id: https://example.org/test + name: test_normalize + default_prefix: ex + prefixes: + ex: https://example.org/ + linkml: https://w3id.org/linkml/ + sdo: https://schema.org/ + imports: + - linkml:types + classes: + Person: + class_uri: sdo:Person + attributes: + full_name: + range: string + slot_uri: sdo:name +""") + +SCHEMA_DCE = textwrap.dedent("""\ + id: https://example.org/test + name: test_normalize_dce + default_prefix: ex + prefixes: + ex: https://example.org/ + linkml: https://w3id.org/linkml/ + dce: http://purl.org/dc/elements/1.1/ + imports: + - linkml:types + classes: + Record: + class_uri: ex:Record + attributes: + title: + range: string + slot_uri: dce:title +""") + +# HTTP variant — linkml-runtime historically binds schema: http://schema.org/ +# while rdflib (and the W3C) prefer https://schema.org/. The normalize flag +# must handle both. +SCHEMA_HTTP_SDO = textwrap.dedent("""\ + id: https://example.org/test + name: test_http_schema + default_prefix: ex + prefixes: + ex: https://example.org/ + linkml: https://w3id.org/linkml/ + sdo: http://schema.org/ + imports: + - linkml:types + classes: + Place: + class_uri: sdo:Place + attributes: + geo: + range: string + slot_uri: sdo:geo +""") + +# Collision scenario: user declares 'foaf' for a custom namespace AND 'myfoaf' +# for http://xmlns.com/foaf/0.1/. Normalisation must NOT clobber the user's 'foaf'. +# Uses 'foaf' instead of 'schema' because 'schema' is declared in linkml:types, +# which causes a SchemaLoader merge conflict before normalisation even runs. +SCHEMA_COLLISION = textwrap.dedent("""\ + id: https://example.org/test + name: test_collision + default_prefix: ex + prefixes: + ex: https://example.org/ + linkml: https://w3id.org/linkml/ + foaf: https://something-else.org/ + myfoaf: http://xmlns.com/foaf/0.1/ + imports: + - linkml:types + classes: + Agent: + class_uri: myfoaf:Agent + attributes: + label: + range: string + slot_uri: myfoaf:name +""") + + +def _write_schema(tmp_path, content: str, name: str = "schema.yaml") -> str: + """Write schema content to a temporary file and return its path as string.""" + p = tmp_path / name + p.write_text(content, encoding="utf-8") + return str(p) + + +def _turtle_prefixes(ttl: str) -> dict[str, str]: + """Extract @prefix declarations from Turtle output → {prefix: namespace}.""" + result = {} + for m in re.finditer(r"@prefix\s+(\w+):\s+<([^>]+)>", ttl): + result[m.group(1)] = m.group(2) + return result + + +# ── OWL Generator Tests ───────────────────────────────────────────── + + +def test_owl_sdo_normalised_to_schema(tmp_path): + """sdo → schema when --normalize-prefixes is active.""" + from linkml.generators.owlgen import OwlSchemaGenerator + + schema_path = _write_schema(tmp_path, SCHEMA_SDO) + ttl = OwlSchemaGenerator(schema_path, normalize_prefixes=True).serialize() + pfx = _turtle_prefixes(ttl) + assert "schema" in pfx, f"Expected 'schema' prefix in OWL output, got: {sorted(pfx)}" + assert pfx["schema"] == "https://schema.org/" + assert "sdo" not in pfx, "Non-standard 'sdo' prefix should be removed" + + +def test_owl_flag_off_preserves_original(tmp_path): + """Without the flag, schema-declared prefix names are preserved.""" + from linkml.generators.owlgen import OwlSchemaGenerator + + schema_path = _write_schema(tmp_path, SCHEMA_SDO) + ttl = OwlSchemaGenerator(schema_path, normalize_prefixes=False).serialize() + pfx = _turtle_prefixes(ttl) + assert "sdo" in pfx, "With flag off, original prefix 'sdo' must be preserved" + + +def test_owl_dce_normalised_to_dc(tmp_path): + """dce → dc for http://purl.org/dc/elements/1.1/ in graph bindings. + + Note: rdflib's Turtle serializer only emits @prefix declarations for + namespaces actually used in triples. Since the OWL generator may not + produce triples using dc:elements URIs for simple attribute schemas, + we verify the graph's namespace bindings directly. + """ + from linkml.generators.owlgen import OwlSchemaGenerator + + schema_path = _write_schema(tmp_path, SCHEMA_DCE) + gen = OwlSchemaGenerator(schema_path, normalize_prefixes=True) + graph = gen.as_graph() + bound = {str(p): str(n) for p, n in graph.namespaces()} + assert "dc" in bound, f"Expected 'dc' in graph bindings, got: {sorted(bound)}" + assert bound["dc"] == "http://purl.org/dc/elements/1.1/" + + +def test_owl_custom_prefix_not_affected(tmp_path): + """Domain-specific prefixes (e.g. 'ex') are not touched by normalisation.""" + from linkml.generators.owlgen import OwlSchemaGenerator + + schema_path = _write_schema(tmp_path, SCHEMA_SDO) + ttl = OwlSchemaGenerator(schema_path, normalize_prefixes=True).serialize() + pfx = _turtle_prefixes(ttl) + assert "ex" in pfx, "Custom prefix 'ex' must survive normalisation" + assert pfx["ex"] == "https://example.org/" + + +def test_owl_http_schema_org_normalised(tmp_path): + """http://schema.org/ (HTTP variant) also normalises to 'schema'. + + The linkml-runtime historically binds ``schema: http://schema.org/`` + while the W3C and rdflib prefer ``https://schema.org/``. Both + variants must be recognised by the static well-known prefix map. + """ + from linkml.generators.owlgen import OwlSchemaGenerator + + schema_path = _write_schema(tmp_path, SCHEMA_HTTP_SDO) + ttl = OwlSchemaGenerator(schema_path, normalize_prefixes=True).serialize() + pfx = _turtle_prefixes(ttl) + assert "schema" in pfx, f"Expected 'schema' prefix for http://schema.org/, got: {sorted(pfx)}" + assert "sdo" not in pfx + + +def test_owl_no_schema1_from_runtime_http_binding(tmp_path): + """Runtime-injected ``schema: http://schema.org/`` must not create ``schema1``. + + The linkml metamodel (types.yaml) declares ``schema: http://schema.org/`` + (HTTP). When a user schema declares ``sdo: https://schema.org/`` (HTTPS), + normalisation must clean up *both* variants so the output never contains + auto-generated suffixed prefixes like ``schema1``. + """ + from linkml.generators.owlgen import OwlSchemaGenerator + + schema_path = _write_schema(tmp_path, SCHEMA_SDO) + ttl = OwlSchemaGenerator(schema_path, normalize_prefixes=True).serialize() + pfx = _turtle_prefixes(ttl) + suffixed = [p for p in pfx if re.match(r"schema\d+", p)] + assert not suffixed, ( + f"Auto-generated suffixed prefix(es) {suffixed} found — runtime http://schema.org/ binding was not cleaned up" + ) + + +# ── SHACL Generator Tests ─────────────────────────────────────────── + + +def test_shacl_sdo_normalised_to_schema(tmp_path): + """sdo → schema when --normalize-prefixes is active.""" + from linkml.generators.shaclgen import ShaclGenerator + + schema_path = _write_schema(tmp_path, SCHEMA_SDO) + ttl = ShaclGenerator(schema_path, normalize_prefixes=True).serialize() + pfx = _turtle_prefixes(ttl) + assert "schema" in pfx, f"Expected 'schema' prefix in SHACL output, got: {sorted(pfx)}" + assert pfx["schema"] == "https://schema.org/" + assert "sdo" not in pfx, "Non-standard 'sdo' prefix should be removed" + + +def test_shacl_flag_off_preserves_original(tmp_path): + """Without the flag, schema-declared prefix names are preserved.""" + from linkml.generators.shaclgen import ShaclGenerator + + schema_path = _write_schema(tmp_path, SCHEMA_SDO) + ttl = ShaclGenerator(schema_path, normalize_prefixes=False).serialize() + pfx = _turtle_prefixes(ttl) + assert "sdo" in pfx, "With flag off, original prefix 'sdo' must be preserved" + + +def test_shacl_dce_normalised_to_dc(tmp_path): + """dce → dc for http://purl.org/dc/elements/1.1/.""" + from linkml.generators.shaclgen import ShaclGenerator + + schema_path = _write_schema(tmp_path, SCHEMA_DCE) + ttl = ShaclGenerator(schema_path, normalize_prefixes=True).serialize() + pfx = _turtle_prefixes(ttl) + assert "dc" in pfx, f"Expected 'dc' prefix in SHACL output, got: {sorted(pfx)}" + assert pfx["dc"] == "http://purl.org/dc/elements/1.1/" + assert "dce" not in pfx, "Non-standard 'dce' prefix should be removed" + + +def test_shacl_custom_prefix_not_affected(tmp_path): + """Domain-specific prefixes (e.g. 'ex') are not touched by normalisation. + + Note: rdflib only emits @prefix for namespaces used in triples. + We verify graph bindings directly. + """ + from linkml.generators.shaclgen import ShaclGenerator + + schema_path = _write_schema(tmp_path, SCHEMA_SDO) + gen = ShaclGenerator(schema_path, normalize_prefixes=True) + graph = gen.as_graph() + bound = {str(p): str(n) for p, n in graph.namespaces()} + assert "ex" in bound, f"Custom prefix 'ex' must survive in graph bindings, got: {sorted(bound)}" + assert bound["ex"] == "https://example.org/" + + +def test_shacl_http_schema_org_normalised(tmp_path): + """http://schema.org/ (HTTP variant) also normalises to 'schema'.""" + from linkml.generators.shaclgen import ShaclGenerator + + schema_path = _write_schema(tmp_path, SCHEMA_HTTP_SDO) + ttl = ShaclGenerator(schema_path, normalize_prefixes=True).serialize() + pfx = _turtle_prefixes(ttl) + assert "schema" in pfx, f"Expected 'schema' prefix for http://schema.org/, got: {sorted(pfx)}" + assert "sdo" not in pfx + + +def test_shacl_no_schema1_from_runtime_http_binding(tmp_path): + """Runtime-injected ``schema: http://schema.org/`` must not create ``schema1``. + + Same scenario as the OWL test: linkml:types imports bring in + ``schema: http://schema.org/`` while the user schema has + ``sdo: https://schema.org/``. Phase 2 of normalisation must + clean up the orphaned HTTP binding. + """ + from linkml.generators.shaclgen import ShaclGenerator + + schema_path = _write_schema(tmp_path, SCHEMA_SDO) + ttl = ShaclGenerator(schema_path, normalize_prefixes=True).serialize() + pfx = _turtle_prefixes(ttl) + suffixed = [p for p in pfx if re.match(r"schema\d+", p)] + assert not suffixed, ( + f"Auto-generated suffixed prefix(es) {suffixed} found — runtime http://schema.org/ binding was not cleaned up" + ) + + +# ── JSON-LD Context Generator Tests ───────────────────────────────── + + +def test_context_http_schema_org_normalised(tmp_path): + """http://schema.org/ (HTTP variant) normalises to 'schema' in JSON-LD context. + + This covers the edge case where linkml-runtime's ``schema: http://schema.org/`` + conflicts with rdflib's ``schema: https://schema.org/``. The stale binding + must be removed and replaced with the correct one. + """ + from linkml.generators.jsonldcontextgen import ContextGenerator + + schema_path = _write_schema(tmp_path, SCHEMA_HTTP_SDO) + ctx = json.loads(ContextGenerator(schema_path, normalize_prefixes=True).serialize())["@context"] + assert "schema" in ctx, "HTTP schema.org should normalise to 'schema'" + assert "sdo" not in ctx, "Non-standard 'sdo' should be removed" + # The namespace URI must match the schema-declared one (http, not https) + schema_val = ctx["schema"] + if isinstance(schema_val, dict): + schema_val = schema_val.get("@id", "") + assert schema_val == "http://schema.org/", f"Namespace URI must be preserved: got {schema_val}" + + +# ── Static Prefix Map Tests ───────────────────────────────────────── + + +def test_well_known_prefix_map_returns_dict(): + from linkml.utils.generator import well_known_prefix_map + + wk = well_known_prefix_map() + assert isinstance(wk, dict) + assert len(wk) >= 29, f"Expected ≥29 entries, got {len(wk)}" + + +def test_well_known_prefix_map_schema_https(): + from linkml.utils.generator import well_known_prefix_map + + wk = well_known_prefix_map() + assert wk["https://schema.org/"] == "schema" + + +def test_well_known_prefix_map_schema_http_variant(): + """Both http and https schema.org must map to 'schema'.""" + from linkml.utils.generator import well_known_prefix_map + + wk = well_known_prefix_map() + assert wk["http://schema.org/"] == "schema" + + +def test_well_known_prefix_map_dc_elements(): + from linkml.utils.generator import well_known_prefix_map + + wk = well_known_prefix_map() + assert wk["http://purl.org/dc/elements/1.1/"] == "dc" + + +def test_well_known_prefix_map_returns_copy(): + """Callers should not be able to mutate the internal map.""" + from linkml.utils.generator import well_known_prefix_map + + wk1 = well_known_prefix_map() + wk1["http://never-in-any-real-prefix-map.test/"] = "test" + wk2 = well_known_prefix_map() + assert "http://never-in-any-real-prefix-map.test/" not in wk2 + + +def test_well_known_prefix_map_fully_resolved_from_prefixmaps(): + """All rdflib defaults must be resolved from prefixmaps (no residual map). + + This is the proof that pinning prefixmaps to the commit containing + linkml/prefixmaps#81 resolves all well-known prefixes without any + hardcoded fallback. If this test fails after a prefixmaps update, + add the missing prefix to the upstream linked_data.curated.yaml. + """ + from rdflib import Graph as RdfGraph + + from linkml.utils.generator import well_known_prefix_map + + wk = well_known_prefix_map() + rdflib_map = {str(ns): str(pfx) for pfx, ns in RdfGraph().namespaces() if str(pfx)} + missing = {ns: pfx for ns, pfx in rdflib_map.items() if ns not in wk} + assert not missing, f"Prefix map missing rdflib defaults (add to prefixmaps upstream): {missing}" + + +# ── Cross-Generator Consistency Tests ──────────────────────────────── + + +def test_all_generators_normalise_sdo_to_schema(tmp_path): + """OWL, SHACL, and JSON-LD context must all use 'schema' for schema.org.""" + from linkml.generators.jsonldcontextgen import ContextGenerator + from linkml.generators.owlgen import OwlSchemaGenerator + from linkml.generators.shaclgen import ShaclGenerator + + schema_path = _write_schema(tmp_path, SCHEMA_SDO) + + owl_ttl = OwlSchemaGenerator(schema_path, normalize_prefixes=True).serialize() + shacl_ttl = ShaclGenerator(schema_path, normalize_prefixes=True).serialize() + ctx = json.loads(ContextGenerator(schema_path, normalize_prefixes=True).serialize())["@context"] + + owl_pfx = _turtle_prefixes(owl_ttl) + shacl_pfx = _turtle_prefixes(shacl_ttl) + + assert "schema" in owl_pfx, "OWL must use 'schema'" + assert "schema" in shacl_pfx, "SHACL must use 'schema'" + assert "schema" in ctx, "JSON-LD context must use 'schema'" + + assert "sdo" not in owl_pfx, "OWL must not have 'sdo'" + assert "sdo" not in shacl_pfx, "SHACL must not have 'sdo'" + assert "sdo" not in ctx, "JSON-LD context must not have 'sdo'" + + +# ── Prefix Collision Tests ──────────────────────────────────────────── + + +@pytest.mark.parametrize( + "generator_cls,generator_module", + [ + ("OwlSchemaGenerator", "linkml.generators.owlgen"), + ("ShaclGenerator", "linkml.generators.shaclgen"), + ], + ids=["owl", "shacl"], +) +def test_graph_generator_collision_skips_rename(tmp_path, caplog, generator_cls, generator_module): + """Graph generators: myfoaf must NOT be renamed to 'foaf' when user claims that name.""" + import importlib + + mod = importlib.import_module(generator_module) + cls = getattr(mod, generator_cls) + + schema_path = _write_schema(tmp_path, SCHEMA_COLLISION) + with caplog.at_level(logging.WARNING): + gen = cls(schema_path, normalize_prefixes=True) + graph = gen.as_graph() + bound = {str(p): str(n) for p, n in graph.namespaces()} + assert "myfoaf" in bound, "Non-standard 'myfoaf' must remain when collision prevents renaming" + assert bound["myfoaf"] == "http://xmlns.com/foaf/0.1/" + assert "collision" in caplog.text.lower(), f"Expected collision warning, got: {caplog.text}" + + +def test_context_collision_preserves_user_prefix(tmp_path, caplog): + """JSON-LD: user's 'foaf: https://something-else.org/' must survive.""" + from linkml.generators.jsonldcontextgen import ContextGenerator + + schema_path = _write_schema(tmp_path, SCHEMA_COLLISION) + with caplog.at_level(logging.WARNING): + ctx = json.loads(ContextGenerator(schema_path, normalize_prefixes=True).serialize())["@context"] + # User's 'foaf' binding preserved + foaf_val = ctx.get("foaf") + if isinstance(foaf_val, dict): + foaf_val = foaf_val.get("@id", "") + assert foaf_val == "https://something-else.org/", f"User's 'foaf' binding must be preserved, got: {foaf_val}" + # myfoaf must remain (not renamed to foaf) + assert "myfoaf" in ctx, "Non-standard 'myfoaf' must remain when collision prevents renaming" + # Warning emitted + assert "collision" in caplog.text.lower(), f"Expected collision warning, got: {caplog.text}" + + +# ── JSONLDGenerator Flag Forwarding Tests ───────────────────────────── + + +def test_jsonld_generator_forwards_normalize_prefixes(tmp_path): + """JSONLDGenerator must pass normalize_prefixes to embedded ContextGenerator. + + Without forwarding, the inline @context in JSON-LD output would keep + non-standard prefix aliases even when --normalize-prefixes is set. + """ + from linkml.generators.jsonldgen import JSONLDGenerator + + schema_path = _write_schema(tmp_path, SCHEMA_SDO) + out = JSONLDGenerator(schema_path, normalize_prefixes=True).serialize() + parsed = json.loads(out) + # The @context may be a list; find the dict entry + ctx = parsed.get("@context", {}) + if isinstance(ctx, list): + for item in ctx: + if isinstance(item, dict): + ctx = item + break + assert "sdo" not in ctx, "normalize_prefixes not forwarded: 'sdo' still in embedded @context" + + +# ── Phase 2 HTTP/HTTPS Overwrite Bug Tests ──────────────────────────── + + +def test_phase2_does_not_overwrite_https_with_http(tmp_path): + """When Phase 1 binds schema → https://schema.org/, Phase 2 must not + overwrite it with http://schema.org/ from the runtime metamodel. + + Reproduction: linkml:types imports bring schema: http://schema.org/ + (HTTP) while the user schema has sdo: https://schema.org/ (HTTPS). + Phase 1 normalises sdo → schema (HTTPS). Phase 2 must not then + rebind schema → http://schema.org/ when it encounters the runtime + HTTP binding. + """ + from linkml.generators.owlgen import OwlSchemaGenerator + + schema_path = _write_schema(tmp_path, SCHEMA_SDO) + gen = OwlSchemaGenerator(schema_path, normalize_prefixes=True) + graph = gen.as_graph() + bound = {str(p): str(n) for p, n in graph.namespaces()} + assert "schema" in bound, f"Expected 'schema' in bindings, got: {sorted(bound)}" + # MUST be HTTPS (from the user's schema), not HTTP (from runtime) + assert bound["schema"] == "https://schema.org/", ( + f"Phase 2 overwrote HTTPS with HTTP: schema bound to {bound['schema']}" + ) + + +def test_normalize_graph_prefixes_phase2_guard(): + """Direct unit test for the Phase 2 guard in normalize_graph_prefixes. + + Simulates the exact scenario: Phase 1 binds schema → https://schema.org/, + then Phase 2 encounters schema1 → http://schema.org/ and must NOT rebind. + """ + from rdflib import Graph, Namespace, URIRef + + from linkml.utils.generator import normalize_graph_prefixes + + g = Graph(bind_namespaces="none") + # Simulate Phase 1 result + g.bind("schema", Namespace("https://schema.org/")) + # Simulate runtime-injected HTTP variant (would appear as schema1) + g.bind("schema1", Namespace("http://schema.org/")) + # Add a triple so the graph isn't empty + g.add((URIRef("https://example.org/s"), URIRef("https://schema.org/name"), URIRef("https://example.org/o"))) + + normalize_graph_prefixes(g, {"sdo": "https://schema.org/"}) + + bound = {str(p): str(n) for p, n in g.namespaces()} + assert bound.get("schema") == "https://schema.org/", f"Phase 2 guard failed: schema bound to {bound.get('schema')}" + + +def test_empty_schema_no_crash(tmp_path): + """A schema with no custom prefixes must not crash normalize_graph_prefixes.""" + from linkml.generators.owlgen import OwlSchemaGenerator + + (tmp_path / "empty.yaml").write_text( + textwrap.dedent("""\ + id: https://example.org/empty + name: empty + default_prefix: ex + prefixes: + linkml: https://w3id.org/linkml/ + ex: https://example.org/ + imports: + - linkml:types + """), + encoding="utf-8", + ) + # Should not raise + gen = OwlSchemaGenerator(str(tmp_path / "empty.yaml"), normalize_prefixes=True) + ttl = gen.serialize() + assert len(ttl) > 0 diff --git a/uv.lock b/uv.lock index 3c867304c8..a9e2c6c6c3 100644 --- a/uv.lock +++ b/uv.lock @@ -2527,7 +2527,7 @@ requires-dist = [ { name = "parse" }, { name = "platformdirs", specifier = ">=4.5.0" }, { name = "prefixcommons", specifier = ">=0.1.7" }, - { name = "prefixmaps", specifier = ">=0.2.2" }, + { name = "prefixmaps", git = "https://github.com/linkml/prefixmaps?rev=75435150a1b31760b9780af2b64a265943a9b263" }, { name = "pydantic", specifier = ">=2.13.5,<3.0.0" }, { name = "pydantic-settings", specifier = ">=2.15.0" }, { name = "pyjsg", specifier = ">=0.12.3" }, @@ -3736,16 +3736,12 @@ wheels = [ [[package]] name = "prefixmaps" -version = "0.2.6" -source = { registry = "https://pypi.org/simple" } +version = "0.2.7.post2.dev0+7543515" +source = { git = "https://github.com/linkml/prefixmaps?rev=75435150a1b31760b9780af2b64a265943a9b263#75435150a1b31760b9780af2b64a265943a9b263" } dependencies = [ { name = "curies" }, { name = "pyyaml" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/4d/cf/f588bcdfd2c841839b9d59ce219a46695da56aa2805faff937bbafb9ee2b/prefixmaps-0.2.6.tar.gz", hash = "sha256:7421e1244eea610217fa1ba96c9aebd64e8162a930dc0626207cd8bf62ecf4b9", size = 709899, upload-time = "2024-10-17T16:30:57.738Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/89/b2/2b2153173f2819e3d7d1949918612981bc6bd895b75ffa392d63d115f327/prefixmaps-0.2.6-py3-none-any.whl", hash = "sha256:f6cef28a7320fc6337cf411be212948ce570333a0ce958940ef684c7fb192a62", size = 754732, upload-time = "2024-10-17T16:30:55.731Z" }, -] [[package]] name = "prettytable" From e4cdaf9adb67c3c135d5e1fdfd20df4933c61e2e Mon Sep 17 00:00:00 2001 From: jdsika Date: Sat, 11 Jul 2026 12:42:42 +0200 Subject: [PATCH 13/14] docs(owl): document --normalize-prefixes Signed-off-by: jdsika --- docs/generators/owl.rst | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docs/generators/owl.rst b/docs/generators/owl.rst index 4b6f076fe8..1a7ab359d9 100644 --- a/docs/generators/owl.rst +++ b/docs/generators/owl.rst @@ -67,6 +67,26 @@ Mapping .. note:: The current default settings for ``metaclasses`` and ``type-objects`` may change in the future +Prefix normalization +^^^^^^^^^^^^^^^^^^^^ + +Schemas sometimes declare non-standard aliases for well-known namespaces +(e.g. ``sh1:`` for the SHACL namespace, or a versioned alias for ``skos:``). +By default these aliases are carried through into the generated artifact. + +Use ``--normalize-prefixes`` to remap declared prefixes whose namespace IRI +matches a well-known vocabulary to that vocabulary's conventional name in the +output (``owl``, ``rdf``, ``rdfs``, ``skos``, ``sh``, ``xsd``, ...): + +.. code:: bash + + gen-owl --normalize-prefixes schema.yaml + +The mapping is a static, version-independent table; namespace IRIs that are +not in the table are left untouched. The option is also available on +``gen-shacl`` and ``gen-jsonld-context``. + + Enums and PermissibleValues ^^^^^^^^^^^^^^^^^^^^^^^^^^^ From b8a388e0cf3a9e43c2e4ce476719b34a2fe35077 Mon Sep 17 00:00:00 2001 From: Carlo van Driesten Date: Fri, 11 Sep 2026 16:34:30 +0200 Subject: [PATCH 14/14] docs(owl): document deterministic serialization and --diff-stable gen-owl canonicalizes its output with RDFC-1.0 before serializing, and the determinism work adds a --diff-stable option on top of it, but neither is mentioned anywhere in the generator documentation. Describe what is guaranteed without passing any flag, why RDFC-1.0's sequential blank-node numbering can still produce noisy diffs across schema edits, and what --diff-stable changes. Also record the behaviour users meet in practice but cannot discover from --help: that the same option exists on gen-rdf, gen-shacl and gen-shex, and that graphs which are not standard RDF -- literal predicates from SHACL annotation mode, relative IRIs such as the metamodel's bibo:status -- take an rdflib fallback that stays reproducible across processes, warns, and deliberately does not honour --diff-stable. --- docs/generators/owl.rst | 42 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/docs/generators/owl.rst b/docs/generators/owl.rst index 1a7ab359d9..a87f7a6c70 100644 --- a/docs/generators/owl.rst +++ b/docs/generators/owl.rst @@ -331,6 +331,48 @@ Other examples translation of Biolink schema to OWL +Deterministic output +^^^^^^^^^^^^^^^^^^^^ + +``gen-owl`` output is deterministic by default. The graph is canonicalized with +`RDFC-1.0 `_ before serialization, so repeated +runs over the same schema -- and any two isomorphic graphs -- produce +byte-identical Turtle. No flag is needed, and checked-in artifacts do not churn +between runs. + +RDFC-1.0 numbers blank nodes sequentially (``_:c14n0``, ``_:c14n1``, ...) in +canonical order. That is stable for a fixed graph, but inserting a single +statement can shift the numbering of every blank node ordered after it, so an +unrelated one-line schema edit may rewrite large parts of the file. Pass +``--diff-stable`` to derive each label from the node's own neighbourhood +instead, so that only the blank nodes an edit actually touches are renamed: + +.. code:: bash + + gen-owl --diff-stable schema.yaml + +Both modes are deterministic and yield isomorphic graphs; only the choice of +label differs. ``--diff-stable`` is off by default because turning it on +relabels the blank nodes in existing output once. + +The same ``--diff-stable/--no-diff-stable`` option is available on ``gen-rdf``, +``gen-shacl`` and ``gen-shex``. + +Graphs that are not standard RDF -- literal predicates, as produced by +``gen-shacl`` in annotation mode, or relative IRIs such as the metamodel's +``bibo:status `` -- cannot be canonicalized under RDFC-1.0. Those fall +back to plain rdflib serialization, with blank-node labels canonicalized by +``rdflib.compare.to_canonical_graph``. Those labels are content-derived rather +than run-local, so the fallback remains reproducible across processes. It emits +an ``RDFCanonicalizationWarning``, and ``--diff-stable`` has no effect on that +path -- it warns rather than silently ignoring the request. + +Canonicalization itself is implemented by the +`diffable-rdf `_ library; +``linkml_runtime.utils.rdf_canonicalize.canonicalize_rdf_graph`` is a thin +adapter that re-emits the library's log warnings as Python warnings. + + Docs ----