diff --git a/BUILD b/BUILD index 80284fe40..822c26038 100644 --- a/BUILD +++ b/BUILD @@ -17,8 +17,8 @@ package(default_visibility = ["//visibility:public"]) exports_files(["pyproject.toml"]) docs( - data = [ - "@score_process//:needs_json", + external_needs = [ + "@score_process//:needs_json_file", ], scan_code = [ "//scripts_bazel:sources", diff --git a/docs.bzl b/docs.bzl index 30c0cdaa3..65bcc95c9 100644 --- a/docs.bzl +++ b/docs.bzl @@ -127,7 +127,7 @@ def _missing_requirements(deps): fail(msg) fail("This case should be unreachable?!") -def docs(source_dir = "docs", data = [], deps = [], scan_code = [], test_sources = [], known_good = None, metamodel = None): +def docs(source_dir = "docs", data = [], deps = [], external_needs = [], scan_code = [], test_sources = [], known_good = None, metamodel = None): """Creates all targets related to documentation. By using this function, you'll get any and all updates for documentation targets in one place. @@ -136,6 +136,7 @@ def docs(source_dir = "docs", data = [], deps = [], scan_code = [], test_sources source_dir: The source directory containing documentation files. Defaults to "docs". data: Additional data files to include in the documentation build. deps: Additional dependencies for the documentation build. + external_needs: List of external needs targets to include in the documentation build. scan_code: List of code targets to scan for source code links. test_sources: Optional list of repo-relative directory paths which will be used to filter testcases for documentation generation. When empty (default), all testcases found in `bazel-testlogs` will be used. @@ -143,6 +144,7 @@ def docs(source_dir = "docs", data = [], deps = [], scan_code = [], test_sources metamodel: Optional label to a metamodel.yaml file. When set, the extension loads this file instead of the default metamodel shipped with score_metamodel. """ + # HINT: keep documentation sync docs/reference/bazel_macros.rst metamodel_data = [] metamodel_env = {} @@ -164,7 +166,7 @@ def docs(source_dir = "docs", data = [], deps = [], scan_code = [], test_sources sphinx_build_binary( name = "sphinx_build", visibility = ["//visibility:private"], - data = data + metamodel_data, + data = data + external_needs + metamodel_data, deps = deps, ) @@ -199,13 +201,14 @@ def docs(source_dir = "docs", data = [], deps = [], scan_code = [], test_sources data_with_docs_sources = _rewrite_needs_json_to_docs_sources(data) additional_combo_sourcelinks = _rewrite_needs_json_to_sourcelinks(data) _merge_sourcelinks(name = "merged_sourcelinks", sourcelinks = [":sourcelinks_json"] + additional_combo_sourcelinks, known_good = known_good) - docs_data = data + metamodel_data + [":sourcelinks_json"] - combo_data = data_with_docs_sources + metamodel_data + [":merged_sourcelinks"] + docs_data = data + external_needs + metamodel_data + [":sourcelinks_json"] + combo_data = data_with_docs_sources + external_needs + metamodel_data + [":merged_sourcelinks"] docs_env = { "SOURCE_DIRECTORY": source_dir, "PACKAGE_DIR": native.package_name(), "DATA": str(data), + "EXTERNAL_NEEDS_FILES": str(external_needs), "TEST_SOURCES": str(test_sources), "SCORE_SOURCELINKS": "$(location :sourcelinks_json)", } | metamodel_env @@ -213,6 +216,7 @@ def docs(source_dir = "docs", data = [], deps = [], scan_code = [], test_sources "SOURCE_DIRECTORY": source_dir, "PACKAGE_DIR": native.package_name(), "DATA": str(data_with_docs_sources), + "EXTERNAL_NEEDS_FILES": str(external_needs), "TEST_SOURCES": str(test_sources), "SCORE_SOURCELINKS": "$(location :merged_sourcelinks)", } | metamodel_env @@ -309,13 +313,13 @@ def docs(source_dir = "docs", data = [], deps = [], scan_code = [], test_sources "-T", # show more details in case of errors "--jobs", "auto", - "--define=external_needs_source=" + str(data), + "--define=external_needs_source=" + str(data + external_needs), "--define=score_sourcelinks_json=$(location :sourcelinks_json)", "--define=score_source_code_linker_plain_links=1", ], formats = ["needs"], sphinx = ":sphinx_build", - tools = data + [":sourcelinks_json"], + tools = data + external_needs + [":sourcelinks_json"], visibility = ["//visibility:public"], # Persistent workers cause stale symlinks after dependency version # changes, corrupting the Bazel cache. @@ -330,6 +334,16 @@ def docs(source_dir = "docs", data = [], deps = [], scan_code = [], test_sources visibility = ["//visibility:public"], ) + native.genrule( + # In contrast to the "needs_json" target represents *only* the needs.json file, + # not the whole needs build output. + name = "needs_json_file", + srcs = [":needs_json"], + outs = ["needs.json"], + cmd = "cp $(location :needs_json)/needs.json $@", + visibility = ["//visibility:public"], + ) + native.alias( name = "traceability_gate", actual = Label("//scripts_bazel:traceability_gate"), diff --git a/docs/how-to/other_modules.rst b/docs/how-to/other_modules.rst index a64c65568..2fbcf187d 100644 --- a/docs/how-to/other_modules.rst +++ b/docs/how-to/other_modules.rst @@ -40,9 +40,10 @@ A minimal example (add or extend the existing `bazel_deps` stanza): 2) Extend your `docs` rule so Sphinx picks up the other module's inventory ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The documentation build in this project is exposed via a Bazel macro/rule that accepts a `data` parameter. -Add the external module's ``:needs_json`` target to that list -to have their needs elements available for cross-referencing. +The documentation build is exposed via a Bazel macro that accepts an ``external_needs`` parameter +for external ``:needs_json_file`` targets. +Use ``external_needs`` instead of ``data`` when the target produces needs JSON — +``data`` is meant for non-needs runfiles (e.g. custom tool outputs). Example `BUILD` snippet (consumer module): @@ -76,3 +77,21 @@ Which results in: See the `Sphinx-Needs documentation `_ for more details on cross-referencing needs. + +Data vs. external_needs +~~~~~~~~~~~~~~~~~~~~~~~ + +It might seem like ``data`` and ``external_needs`` behave nearly the same. + +.. code-block:: starlark + + docs( + data = ["@score_process//:needs_json"], + external_needs = ["@score_process//:needs_json_file"], # same? + ) + +For a ``:docs`` they behave the same, +but for ``:docs_combo`` they do not. +In a combo build, the ``data`` are treated differently and will include the documentation +instead of just link the needs online. +However, ``external_needs`` will always be hyperlinks. diff --git a/docs/how-to/setup.md b/docs/how-to/setup.md index 644e679ef..2f7bca8c3 100644 --- a/docs/how-to/setup.md +++ b/docs/how-to/setup.md @@ -73,19 +73,7 @@ docs( ) ``` - -#### Configuration Options - -The `docs()` macro accepts the following arguments: - -| Parameter | Description | Required | -|-----------|-------------|----------| -| `source_dir` | Directory of documentation source files (RST, MD) | Yes | -| `data` | List of `needs_json` targets that should be included in the documentation | No | -| `deps` | Additional Bazel Python dependencies | No | -| `scan_code` | Source code targets to scan for traceability tags | No | -| `known_good` | Label to a "known good" JSON file for source links | No | -| `metamodel` | Label to a custom `metamodel.yaml` that replaces the default metamodel | No | +For configuration options see {ref}`docs_bazel-macros`. ### 4. Copy conf.py diff --git a/docs/reference/bazel_macros.rst b/docs/reference/bazel_macros.rst index 09554b505..95b890af6 100644 --- a/docs/reference/bazel_macros.rst +++ b/docs/reference/bazel_macros.rst @@ -64,6 +64,12 @@ Minimal example (root ``BUILD``) Source code targets to scan for traceability tags (``req-Id:`` annotations). Used to generate the source-code-link JSON that maps tags back to source files. +- ``external_needs`` (list of bazel labels) + External ``:needs_json_file`` targets from other modules/repositories + for referencing their needs. + Users must specify ``:needs_json_file`` explicitly (not ``:needs_json``). + Works the same during ``:docs`` and ``:docs_combo``. + - ``metamodel`` (bazel label, optional) Path to a custom ``metamodel.yaml`` file. When set, the ``score_metamodel`` extension loads **this file instead of** the default metamodel. diff --git a/src/extensions/score_metamodel/external_needs.py b/src/extensions/score_metamodel/external_needs.py index 980dfe1c2..44b7380ea 100644 --- a/src/extensions/score_metamodel/external_needs.py +++ b/src/extensions/score_metamodel/external_needs.py @@ -54,7 +54,7 @@ def _parse_bazel_external_need(s: str) -> ExternalNeedsSource | None: repo, path_to_target = repo_and_path.split("//", 1) repo = repo.lstrip("@") # empty for same-repo `//pkg:needs_json` - if target in ("needs_json", "docs_sources"): + if target in ("needs_json", "needs_json_file", "docs_sources"): return ExternalNeedsSource( bazel_module=repo, path_to_target=path_to_target, @@ -159,12 +159,11 @@ def temp(self: NeedsList): def get_external_needs_source(external_needs_source: str) -> list[ExternalNeedsSource]: if external_needs_source: # Path taken for all invocations via `bazel` - external_needs = parse_external_needs_sources_from_DATA(external_needs_source) + return parse_external_needs_sources_from_DATA(external_needs_source) else: # This is the path taken for anything that doesn't # run via `bazel` e.g. esbonio or other direct executions - external_needs = parse_external_needs_sources_from_bazel_query() # pyright: ignore[reportAny] - return external_needs + return parse_external_needs_sources_from_bazel_query() # pyright: ignore[reportAny] def add_external_needs_json(e: ExternalNeedsSource, config: Config): @@ -182,9 +181,9 @@ def add_external_needs_json(e: ExternalNeedsSource, config: Config): needs_json_data = json.loads(Path(json_file).read_text(encoding="utf-8")) # pyright: ignore[reportAny] except FileNotFoundError: logger.error( - f"Could not find external needs JSON file at {json_file}. " - + "Something went terribly wrong. " - + "Try running `bazel clean --async && rm -rf _build`." + "Could not find external needs JSON file at %s from target %s.", + json_file, + e.target, ) # Attempt to continue, exit code will be non-zero after a logged error anyway. return @@ -227,6 +226,7 @@ def add_external_docs_sources(e: ExternalNeedsSource, config: Config): def connect_external_needs(app: Sphinx, config: Config): extend_needs_json_exporter(config, ["project_url"]) + # Local external needs from DATA (e.g. :needs_json or :docs_sources) external_needs = get_external_needs_source(app.config.external_needs_source) # this sets the default value - required for the needs-config-writer @@ -236,9 +236,41 @@ def connect_external_needs(app: Sphinx, config: Config): for e in external_needs: if e.target == "needs_json": add_external_needs_json(e, app.config) + elif e.target == "needs_json_file": + _add_needs_json_file(e, app.config) elif e.target == "docs_sources": add_external_docs_sources(e, app.config) else: raise ValueError( f"Internal Error. Unknown external needs target: {e.target}" ) + + +def _add_needs_json_file(ext_needs: ExternalNeedsSource, config: Config) -> None: + """Resolve a needs_json_file target from runfiles and register it.""" + json_file_raw = ( + Path(_runfiles_module_dir(ext_needs)) / ext_needs.path_to_target / "needs.json" + ) + r = get_runfiles_dir() + json_file = r / json_file_raw + logger.debug(f"External needs_json_file: {json_file}") + try: + needs_json_data = json.loads( + Path(json_file).read_text(encoding="utf-8") # pyright: ignore[reportAny] + ) + except FileNotFoundError: + logger.error( + "Could not find external needs JSON file at %s from target %s.", + json_file, + ext_needs.target, + ) + return + except json.JSONDecodeError as exc: + logger.error(f"Failed to parse external needs JSON file {json_file}: {exc}") + return + config.needs_external_needs.append( + { # pyright: ignore[reportUnknownMemberType] + "base_url": needs_json_data.get("project_url", "") + "/main", + "json_path": json_file, + } + ) diff --git a/src/extensions/score_metamodel/tests/test_external_needs.py b/src/extensions/score_metamodel/tests/test_external_needs.py index a73ff601f..4c834590c 100644 --- a/src/extensions/score_metamodel/tests/test_external_needs.py +++ b/src/extensions/score_metamodel/tests/test_external_needs.py @@ -24,6 +24,7 @@ import score_metamodel.external_needs as ext_needs from score_metamodel.external_needs import ( ExternalNeedsSource, + _add_needs_json_file, add_external_docs_sources, add_external_needs_json, get_external_needs_source, @@ -202,6 +203,41 @@ def test_add_external_needs_json_appends_entry_local( assert Path(entry["json_path"]) == json_path +def test_add_needs_json_file_appends_entry( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """_add_needs_json_file should load from a :needs_json_file target.""" + # Arrange: create the needs.json at the runfiles path + rel_json = Path("ext_mod+/needs.json") + json_path = tmp_path / rel_json + json_path.parent.mkdir(parents=True, exist_ok=True) + json_path.write_text( + json.dumps({"project_url": "https://example.test/json-file"}), encoding="utf-8" + ) + + runfiles_dir = tmp_path + config = Config() + config.needs_external_needs = [] + + monkeypatch.setattr(ext_needs, "get_runfiles_dir", lambda: runfiles_dir) + + # Act + e = ExternalNeedsSource( + bazel_module="ext_mod", + target="needs_json_file", + path_to_target="", + is_local=False, + ) + _add_needs_json_file(e, config) + + # Assert + assert config.needs_external_needs is not None + assert len(config.needs_external_needs) == 1 + entry = config.needs_external_needs[0] + assert entry["base_url"] == "https://example.test/json-file/main" + assert Path(entry["json_path"]) == json_path + + def test_add_external_needs_json_missing_file_keeps_list_empty( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: diff --git a/src/incremental.py b/src/incremental.py index 0519dff63..5b318fe6d 100644 --- a/src/incremental.py +++ b/src/incremental.py @@ -13,6 +13,7 @@ import argparse import hashlib +import json import logging import os import shutil @@ -39,6 +40,17 @@ def get_env(name: str) -> str: return val +def _merged_external_needs() -> str: + """Combine DATA and EXTERNAL_NEEDS_FILES into one JSON label list. + + Both env vars hold JSON lists of Bazel labels; the extension parses the + resulting `external_needs_source` define uniformly. + """ + data = json.loads(get_env("DATA") or "[]") + external = json.loads(os.environ.get("EXTERNAL_NEEDS_FILES", "[]") or "[]") + return json.dumps(data + external) + + def _compute_hash(files: list[Path]) -> str: h = hashlib.sha256() for f in sorted(files, key=str): @@ -122,7 +134,11 @@ def update_module_hash(build_dir: Path, sentinel_files: list[Path]) -> None: "-T", # show details in case of errors in extensions "--jobs", "auto", - f"--define=external_needs_source={get_env('DATA')}", + # Merge DATA (:needs_json / :docs_sources) with EXTERNAL_NEEDS_FILES + # (:needs_json_file) into a single define. The sphinx_docs rule cannot + # receive per-target env vars, so --define is the only channel that + # works for both the py_binary and the needs_json target. + f"--define=external_needs_source={_merged_external_needs()}", f"--define=testcase_source_dirs={os.environ.get('TEST_SOURCES', '[]')}", ] diff --git a/src/tests/docs_bzl/fixtures/external_needs/consumer/BUILD b/src/tests/docs_bzl/fixtures/external_needs/consumer/BUILD index 977e96715..da54ffc32 100644 --- a/src/tests/docs_bzl/fixtures/external_needs/consumer/BUILD +++ b/src/tests/docs_bzl/fixtures/external_needs/consumer/BUILD @@ -18,6 +18,6 @@ load("//:docs.bzl", "docs") # avoiding the workspace-wide bazel-testlogs scan during the html build. docs( source_dir = "docs", - data = ["//src/tests/docs_bzl/fixtures/external_needs/producer:needs_json"], + external_needs = ["//src/tests/docs_bzl/fixtures/external_needs/producer:needs_json_file"], test_sources = ["src/tests/docs_bzl/fixtures/external_needs/consumer"], ) diff --git a/src/tests/docs_bzl/fixtures/external_needs/consumer_combo/BUILD b/src/tests/docs_bzl/fixtures/external_needs/consumer_combo/BUILD new file mode 100644 index 000000000..9b3a3746b --- /dev/null +++ b/src/tests/docs_bzl/fixtures/external_needs/consumer_combo/BUILD @@ -0,0 +1,20 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +load("//:docs.bzl", "docs") + +docs( + source_dir = "docs", + external_needs = ["//src/tests/docs_bzl/fixtures/external_needs/producer:needs_json_file"], + test_sources = ["src/tests/docs_bzl/fixtures/external_needs/consumer_combo"], +) diff --git a/src/tests/docs_bzl/fixtures/external_needs/consumer_combo/docs/conf.py b/src/tests/docs_bzl/fixtures/external_needs/consumer_combo/docs/conf.py new file mode 100644 index 000000000..9e5c5d250 --- /dev/null +++ b/src/tests/docs_bzl/fixtures/external_needs/consumer_combo/docs/conf.py @@ -0,0 +1,28 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +import os + +project = "External Needs Consumer" +project_url = "https://example.invalid/external-needs-consumer" +extensions = ["score_sphinx_bundle"] + +# FIXME: +# sphinx-needs refuses to add an external need whose type is not registered in +# this project's needs_types, so the consumer must declare the producer's +# `test_req` type. score_metamodel's own checks skip external needs, so the +# id/parts rules never run here. Loaded via the score_metamodel_yaml config +# value because docs(metamodel=...) does not reach the build target. +score_metamodel_yaml = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "metamodel.yaml" +) diff --git a/src/tests/docs_bzl/fixtures/external_needs/consumer_combo/docs/index.rst b/src/tests/docs_bzl/fixtures/external_needs/consumer_combo/docs/index.rst new file mode 100644 index 000000000..17f7631a6 --- /dev/null +++ b/src/tests/docs_bzl/fixtures/external_needs/consumer_combo/docs/index.rst @@ -0,0 +1,19 @@ +.. + # ******************************************************************************* + # Copyright (c) 2026 Contributors to the Eclipse Foundation + # + # See the NOTICE file(s) distributed with this work for additional + # information regarding copyright ownership. + # + # This program and the accompanying materials are made available under the + # terms of the Apache License Version 2.0 which is available at + # https://www.apache.org/licenses/LICENSE-2.0 + # + # SPDX-License-Identifier: Apache-2.0 + # ******************************************************************************* + +External Needs Consumer +======================= + +This document links to a need defined in the producer package: +:need:`test_req__producer__demo`. diff --git a/src/tests/docs_bzl/fixtures/external_needs/consumer_combo/docs/metamodel.yaml b/src/tests/docs_bzl/fixtures/external_needs/consumer_combo/docs/metamodel.yaml new file mode 100644 index 000000000..dc4e1651a --- /dev/null +++ b/src/tests/docs_bzl/fixtures/external_needs/consumer_combo/docs/metamodel.yaml @@ -0,0 +1,29 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +needs_types: + test_req: + title: Test Requirement + prefix: test_req__ + parts: 3 + mandatory_options: + id: ^test_req__[0-9a-zA-Z_]*$ + status: ^(draft|valid)$ + optional_options: + # These are all sphinx-needs default fields. Listing them keeps the + # metamodel/default option overlap exactly at the set score_metamodel + # treats as intentional ({id, tags, status, content, template}), which + # avoids a setup-time warning that -W would turn into a build error. + tags: .* + content: .* + template: .* +links: {} diff --git a/src/tests/docs_bzl/test_docs_bzl.py b/src/tests/docs_bzl/test_docs_bzl.py index 701536777..c1ae0c6dd 100644 --- a/src/tests/docs_bzl/test_docs_bzl.py +++ b/src/tests/docs_bzl/test_docs_bzl.py @@ -72,3 +72,20 @@ def test_consumer_link_resolves(): assert "external-needs-producer/main/index.html#test_req__producer__demo" in html, ( html ) + + +def test_consumer_combo_with_external_needs_resolves(): + """ + Verify that :docs_combo correctly resolves external_needs links. + The consumer_combo fixture uses external_needs (not data), so the + :need: link must resolve to a proper hyperlink, not a dead_link. + """ + result = run_fixture("run", "external_needs/consumer_combo", ":docs_combo") + + index = result.build_dir / "index.html" + html = index.read_text(encoding="utf-8") + + assert "external-needs-producer/main/index.html#test_req__producer__demo" in html, ( + html + ) + assert "needs_dead_link" not in html, html