POC: add tool qualification verification report model

docs/internals/requirements/index.rst

@@ -41,3 +41,4 @@ Pages
    requirements
    requirement_coverage
    tooling_verification
+   tool_qualification_example

docs/internals/requirements/tool_qualification_example.rst

@@ -0,0 +1,164 @@
+..
+   # *******************************************************************************
+   # Copyright (c) 2026 Contributors to the Eclipse Foundation
+   #
+   # See the NOTICE file(s) distributed with this work for additional
+   # information regarding copyright ownership.
+   #
+   # This program and the accompanying materials are made available under the
+   # terms of the Apache License 2.0 which is available at
+   # https://www.apache.org/licenses/LICENSE-2.0
+   #
+   # SPDX-License-Identifier: Apache-2.0
+   # *******************************************************************************
+
+.. _tool_qualification_example:
+
+Tool Qualification Report Example
+==================================
+
+This page demonstrates the machine-readable model used for a Docs-as-Code tool
+qualification report. The content is based on the published
+`S-CORE Doc-as-Code Tool Verification Report
+<https://eclipse-score.github.io/score/main/score_tools/tools_documentation/doc_as_code.html#pr-review>`_.
+
+The evaluation considers build and CI behavior, pull-request review, and
+derived views. A malfunction is represented as a nested Need below the use case
+where it occurs. Its ``safety_measure`` is intentionally free text. Generated
+testcase Needs can link back to a malfunction through the existing verification
+links.
+
+.. doc_tool:: Docs-as-Code Tool Qualification
+   :id: doc_tool__docs_as_code
+   :status: evaluated
+   :safety_affected: YES
+   :security_affected: YES
+   :tcl: LOW
+   :tool_version: v7.0.1
+   :realizes: wp__tool_verification_report[version==1]
+   :version: 2
+   :post_template: tool_qualification_report
+
+   This report evaluates the Docs-as-Code toolchain as a qualified tool for
+   producing and verifying safety-relevant documentation.
+
+.. tool-usecase:: Build and CI behavior
+   :id: tool_usecase__docs_as_code__build_ci
+   :version: 1
+
+   The repository contents are the source of truth and gated CI evaluates
+   the generated documentation and traceability data.
+
+   .. tool-malfunction:: Document metamodel enforcement
+      :id: tool_malfunction__docs_as_code__metamodel
+      :safety_affected: YES
+      :detection_sufficient: NO
+      :additional_safety_measure_required: YES
+      :safety_measure: Repository contents are the source of truth and every change is reviewed by a committer.
+      :violates: tool_req__docs_doc_types
+      :version: 1
+
+      A silent false negative can result from a permissive regular
+      expression or a defect in a metamodel check.
+
+   .. tool-malfunction:: Safety-critical linking enforcement
+      :id: tool_malfunction__docs_as_code__safety_links
+      :safety_affected: YES
+      :detection_sufficient: NO
+      :additional_safety_measure_required: YES
+      :safety_measure: Pull-request review checks changes to safety-critical links before merge.
+      :violates: tool_req__docs_common_attr_safety_link_check
+      :version: 1
+
+      A silent false negative could allow an unsafe derivation or other
+      invalid safety-critical link to pass the build.
+
+   .. tool-malfunction:: Requirements coverage statistics
+      :id: tool_malfunction__docs_as_code__coverage
+      :safety_affected: YES
+      :detection_sufficient: NO
+      :additional_safety_measure_required: YES
+      :safety_measure: Coverage results are reviewed as part of the tool qualification evidence.
+      :violates: tool_req__docs_test_linkage_metrics
+      :version: 1
+
+      A defect in the statistics calculation can produce an incorrect
+      coverage result while leaving the build green.
+
+   .. tool-malfunction:: Test linkage
+      :id: tool_malfunction__docs_as_code__test_linkage
+      :safety_affected: YES
+      :detection_sufficient: NO
+      :additional_safety_measure_required: YES
+      :safety_measure: Test metadata and requirement links are reviewed when the test suite changes.
+      :violates: tool_req__docs_test_link_testcase
+      :version: 1
+
+      The safety case may incorrectly conclude that a requirement is tested
+      if a testcase link is missing or points to the wrong need.
+
+   .. tool-malfunction:: Test reference check
+      :id: tool_malfunction__docs_as_code__test_refs
+      :safety_affected: YES
+      :detection_sufficient: NO
+      :additional_safety_measure_required: YES
+      :safety_measure: Pull-request review checks that referenced requirements remain current.
+      :violates: tool_req__docs_test_link_testcase
+      :version: 1
+
+      A missing or outdated requirement reference can invalidate the test
+      evidence without causing a CI failure.
+
+   .. tool-malfunction:: Listing assumptions of use
+      :id: tool_malfunction__docs_as_code__aou_listing
+      :safety_affected: YES
+      :detection_sufficient: NO
+      :additional_safety_measure_required: YES
+      :safety_measure: Each change to an assumption of use is reviewed by a committer.
+      :violates: tool_req__docs_req_link_covers_aou
+      :version: 1
+
+      A missing or incorrect assumption of use can lead to an incomplete
+      safety argument.
+
+.. tool-usecase:: Derived views
+   :id: tool_usecase__docs_as_code__derived_views
+   :version: 1
+
+   Derived views are generated from the machine-readable Need model and are
+   not the source of truth.
+
+   .. tool-malfunction:: Architecture visualization
+      :id: tool_malfunction__docs_as_code__arch_view
+      :safety_affected: YES
+      :detection_sufficient: YES
+      :additional_safety_measure_required: NO
+      :safety_measure: The generated architecture view is inspected during pull-request review.
+      :violates: tool_req__docs_arch_views
+      :version: 1
+
+      An incorrect visualization can misrepresent the architecture. The
+      result is detected by inspecting the derived view.
+
+   .. tool-malfunction:: Backlinks
+      :id: tool_malfunction__docs_as_code__backlinks
+      :safety_affected: NO
+      :detection_sufficient: NO
+      :additional_safety_measure_required: YES
+      :safety_measure: Backlinks are covered by the generated documentation checks.
+      :violates: tool_req__docs_verification_report_need
+      :version: 1
+
+      A generated backlink can be missing or point to the wrong Need.
+
+   .. tool-malfunction:: Documentation generation
+      :id: tool_malfunction__docs_as_code__generation
+      :safety_affected: NO
+      :detection_sufficient: YES
+      :additional_safety_measure_required: NO
+      :safety_measure: The generated HTML is inspected as a derived view.
+      :violates: tool_req__docs_doc_types
+      :version: 1
+
+      The generated HTML can be incomplete, outdated, or rendered
+      incorrectly. The issue is detected by inspecting the derived view.

src/extensions/score_metamodel/metamodel.yaml

@@ -258,6 +258,36 @@ needs_types:
       realizes: workproduct
     parts: 2
 
+  # Tool qualification reports keep use cases top-level for readable rendering.
+  # Their malfunctions are nested below them and use parent_needs as their
+  # semantic relationship.
+  tool-usecase:
+    title: Tool Verification Use Case
+    color: "#E1D5E7"
+    prefix: tool_usecase__
+    mandatory_options:
+      id: ^tool_usecase__docs_as_code__[0-9a-z_-]+$
+    parts: 3
+
+  # A malfunction describes a violation of one or more requirements. Safety
+  # measures are deliberately kept as free text on the malfunction because
+  # they are report evidence, not independently traceable model elements.
+  tool-malfunction:
+    title: Tool Verification Malfunction
+    color: "#F8CECC"
+    prefix: tool_malfunction__
+    mandatory_options:
+      id: ^tool_malfunction__docs_as_code__[0-9a-z_-]+$
+      safety_affected: "^(YES|NO)$"
+      detection_sufficient: "^(YES|NO)$"
+      additional_safety_measure_required: "^(YES|NO)$"
+    optional_options:
+      safety_measure: ^.+$
+    mandatory_links:
+      parent_needs: tool-usecase
+      violates: gd_req, stkh_req, feat_req, comp_req, tool_req, aou_req
+    parts: 3
+
   # Requirements
   # req-Id: tool_req__docs_req_types
   stkh_req:
@@ -1060,6 +1090,13 @@ needs_extra_links:
     incoming: realized by
     outgoing: realizes
 
+  # Sphinx-Needs stores nested Need relationships under parent_needs. Use
+  # report-oriented labels so generated links describe the containment rather
+  # than exposing the implementation-oriented word "parent".
+  parent_needs:
+    incoming: contains
+    outgoing: contained by
+
   # Decision Record
   affects:
     incoming: affected by

src/extensions/score_metamodel/tests/rst/architecture/architecture_tests.rst

@@ -15,6 +15,7 @@
 
 .. test_metadata:: Test Architecture Needs
    :id: test_metadata__architecture
+   :fully_verifies_list: tool_malfunction__docs_as_code__arch_view
    :partially_verifies_list: tool_req__docs_arch_types
    :test_type: requirements_based
    :derivation_technique: requirements_based

src/extensions/score_metamodel/tests/rst/graph/test_metamodel_graph.rst

@@ -14,6 +14,7 @@
 
 .. test_metadata::
    :id: test_metadata__metamodel_graph_checks
+   :fully_verifies_list: tool_malfunction__docs_as_code__safety_links
    :partially_verifies_list: tool_req__docs_common_attr_safety_link_check
    :test_type: requirements_based
    :derivation_technique: requirements_based
@@ -78,3 +79,29 @@
    :status: valid
    :derived_from: feat_req__parent0__abcd
    :expect: unknown outgoing link
+
+
+.. doc_tool:: Nested tool qualification report
+   :id: doc_tool__nested_qualification
+   :status: evaluated
+   :safety_affected: YES
+   :security_affected: NO
+   :tcl: LOW
+   :version: 1
+
+.. tool-usecase:: Nested tool use case
+   :id: tool_usecase__blabla__nested
+   :version: 1
+
+   A use case contains its tool malfunctions as nested needs.
+
+   .. tool-malfunction:: Nested tool malfunction
+      :id: tool_malfunction__blabla__nested
+      :safety_affected: YES
+      :detection_sufficient: NO
+      :additional_safety_measure_required: YES
+      :safety_measure: Review the generated result before release.
+      :violates: gd_guidl__req__engineering
+      :version: 1
+
+      The generated result may contain a silent error.

src/extensions/score_metamodel/tests/test_metamodel_load.py

@@ -14,6 +14,7 @@
 from unittest.mock import mock_open, patch
 
 import pytest
+from attribute_plugin import add_test_properties  # type: ignore[import-untyped]
 from score_metamodel import ProhibitedWordCheck, load_metamodel_data
 
 MODEL_DIR = Path(__file__).absolute().parent / "model"
@@ -188,6 +189,11 @@ def test_sphinx_needs_builtin_links_are_accepted(tmp_path: Path):
     }
 
 
+@add_test_properties(
+    fully_verifies=["tool_malfunction__docs_as_code__metamodel"],
+    test_type="requirements-based",
+    derivation_technique="requirements-analysis",
+)
 def test_all_undeclared_links_are_reported_at_once(tmp_path: Path):
     """Every offending link is listed, so one run shows all the work to do."""
     model = _write_model(
@@ -205,6 +211,33 @@ def test_all_undeclared_links_are_reported_at_once(tmp_path: Path):
     assert "ghost_two" in message
 
 
+@add_test_properties(
+    fully_verifies=["tool_malfunction__docs_as_code__metamodel"],
+    test_type="requirements-based",
+    derivation_technique="requirements-analysis",
+)
 def test_shipped_metamodel_declares_every_link_it_uses():
     """The metamodel shipped with this extension must satisfy the check."""
     load_metamodel_data()
+
+
+@add_test_properties(
+    fully_verifies=["tool_malfunction__docs_as_code__metamodel"],
+    test_type="requirements-based",
+    derivation_technique="requirements-analysis",
+)
+def test_tool_qualification_types_model_nested_traceability():
+    """Tool malfunctions are nested under use cases and carry safety text."""
+    result = load_metamodel_data()
+    types = {need_type["directive"]: need_type for need_type in result.needs_types}
+
+    assert types["tool-usecase"]["mandatory_links_str"] == {}
+    assert types["tool-malfunction"]["mandatory_links_str"] == {
+        "parent_needs": "tool-usecase",
+        "violates": "gd_req, stkh_req, feat_req, comp_req, tool_req, aou_req",
+    }
+    assert types["tool-malfunction"]["optional_options"]["safety_measure"] == "^.+$"
+    assert result.needs_links["parent_needs"] == {
+        "incoming": "contains",
+        "outgoing": "contained by",
+    }

src/extensions/score_metrics/tests/test_traceability_metrics.py

@@ -153,7 +153,10 @@ def test_safe_percent_zero(value1: int, value2: int, expected: float) -> None:
 
 
 @add_test_properties(
-    partially_verifies=["tool_req__docs_test_linkage_metrics"],
+    partially_verifies=[
+        "tool_req__docs_test_linkage_metrics",
+        "tool_malfunction__docs_as_code__coverage",
+    ],
     test_type="requirements-based",
     derivation_technique="requirements-analysis",
 )
@@ -231,7 +234,10 @@ def test_calculate_requirement_metrics_non_fully_linked_2() -> None:
 
 
 @add_test_properties(
-    partially_verifies=["tool_req__docs_test_linkage_metrics"],
+    partially_verifies=[
+        "tool_req__docs_test_linkage_metrics",
+        "tool_malfunction__docs_as_code__coverage",
+    ],
     test_type="interface-test",
     derivation_technique="design-analysis",
 )

src/extensions/score_source_code_linker/tests/test_source_code_link_integration.py

@@ -23,6 +23,7 @@
 from sphinx.testing.util import SphinxTestApp
 from sphinx_needs.data import SphinxNeedsData
 
+from score_pytest.attribute_plugin import add_test_properties
 from src.extensions.score_source_code_linker.helpers import get_github_link
 from src.extensions.score_source_code_linker.needlinks import NeedLink
 from src.extensions.score_source_code_linker.repo_source_links import RepoInfo
@@ -512,6 +513,14 @@ def compare_grouped_json_files(file1: Path, golden_file: Path):
         )
 
 
+@add_test_properties(
+    fully_verifies=[
+        "tool_malfunction__docs_as_code__backlinks",
+        "tool_malfunction__docs_as_code__generation",
+    ],
+    test_type="requirements-based",
+    derivation_technique="requirements-analysis",
+)
 def test_source_link_integration_ok(
     sphinx_app_setup: Callable[[], SphinxTestApp],
     example_source_link_text_all_ok: dict[str, list[NeedLink]],

src/extensions/score_source_code_linker/tests/test_testlink.py

@@ -99,6 +99,10 @@ def test_clean_text_removes_ansi_and_html_unescapes():
 
 @add_test_properties(
     partially_verifies=["tool_req__docs_test_link_testcase"],
+    fully_verifies=[
+        "tool_malfunction__docs_as_code__test_linkage",
+        "tool_malfunction__docs_as_code__test_refs",
+    ],
     test_type="requirements-based",
     derivation_technique="requirements-analysis",
 )

src/extensions/score_source_code_linker/tests/test_xml_parser.py

@@ -277,6 +277,10 @@ def test_parse_properties():
 
 @add_test_properties(
     partially_verifies=["tool_req__docs_test_link_testcase"],
+    fully_verifies=[
+        "tool_malfunction__docs_as_code__test_linkage",
+        "tool_malfunction__docs_as_code__test_refs",
+    ],
     test_type="requirements-based",
     derivation_technique="requirements-analysis",
 )