diff --git a/content/examples/tutorials/cfbs.markdown b/content/examples/tutorials/cfbs.markdown index cb6cbca3f2..d0f5655a0c 100644 --- a/content/examples/tutorials/cfbs.markdown +++ b/content/examples/tutorials/cfbs.markdown @@ -4,7 +4,7 @@ title: Using CFEngine Build command line tools - cfbs --- In this tutorial, we'll take a look at how to work with CFEngine Build projects from the command line. -We assume you have already set up a hub, and installed [`cf-remote`](https://pypi.org/project/cf-remote/) and [`cfbs`](https://pypi.org/project/cfbs/) as we do in [step 1 of the getting started guide](/getting-started/01-installation/). +We assume you have already set up a hub, and installed [`cf-remote`](https://pypi.org/project/cf-remote/) and [`cfbs`](https://pypi.org/project/cfbs/) as we do in [step 1 of the getting started guide](/getting-started/01-installing-cfengine/). When working on a CFEngine Build project, the workflow looks like this: diff --git a/content/examples/tutorials/file_comparison.markdown b/content/examples/tutorials/file_comparison.markdown index 56aec2fab2..9535a4d2b5 100644 --- a/content/examples/tutorials/file_comparison.markdown +++ b/content/examples/tutorials/file_comparison.markdown @@ -6,7 +6,7 @@ aliases: - "/examples-tutorials-file_comparison.html" --- -1. Add the [policy contents][File comparison#Full policy] (also can be downloaded from file_compare_test.cf) to a new file, such as /var/cfengine/masterfiles/file_test.cf. +1. Add the [policy contents][File comparison#Full policy] (also can be downloaded from file_compare_test.cf) to a new file, such as /var/cfengine/masterfiles/file_test.cf. 2. Run the following commands as root on the command line: ```console diff --git a/content/getting-started/hugo-commit.png b/content/examples/tutorials/policy-writing/hugo-commit.png similarity index 100% rename from content/getting-started/hugo-commit.png rename to content/examples/tutorials/policy-writing/hugo-commit.png diff --git a/content/examples/tutorials/promise-type-module-development.markdown b/content/examples/tutorials/promise-type-module-development.markdown index 27e1991d53..5ebc79b4b6 100644 --- a/content/examples/tutorials/promise-type-module-development.markdown +++ b/content/examples/tutorials/promise-type-module-development.markdown @@ -137,6 +137,6 @@ There are several places to look for more information or inspiration when writin - [The real git promise type code](https://github.com/cfengine/modules/tree/c3b7329b240cf7ad062a0a64ee8b607af2cb912a/promise-types/git/) - [HTTP promise type module](https://github.com/cfengine/modules/tree/c861789d4b376147d904fccd76963a92e65eaa97/promise-types/http/) -- [CFEngine custom promise type specification](./reference-promise-types-custom.html) +- [CFEngine custom promise type specification](/reference/promise-types/custom/) - [Blog post: How to implement CFEngine Custom Promise types in Python](https://cfengine.com/blog/2020/how-to-implement-cfengine-custom-promise-types-in-python/) - [Blog post: How to implement CFEngine Custom Promise types in Bash](https://cfengine.com/blog/2021/how-to-implement-cfengine-custom-promise-types-in-bash/) diff --git a/content/reference/masterfiles-policy-framework/cfe_internal-CFE_cfengine.markdown b/content/reference/masterfiles-policy-framework/cfe_internal-cfe_cfengine.markdown similarity index 100% rename from content/reference/masterfiles-policy-framework/cfe_internal-CFE_cfengine.markdown rename to content/reference/masterfiles-policy-framework/cfe_internal-cfe_cfengine.markdown diff --git a/content/reference/promise-types/files/_index.markdown b/content/reference/promise-types/files/_index.markdown index 45db130c9c..5ad1d9b2d4 100644 --- a/content/reference/promise-types/files/_index.markdown +++ b/content/reference/promise-types/files/_index.markdown @@ -234,7 +234,7 @@ files: body classes if_ok(x) { promise_repaired => { "$(x)" }; - promise_kept => { "$(x)" }; +promise_kept => { "$(x)" }; } @@ -260,7 +260,7 @@ file_result => "leaf_name"; body classes if_ok(x) { promise_repaired => { "$(x)" }; - promise_kept => { "$(x)" }; +promise_kept => { "$(x)" }; } diff --git a/content/reference/promise-types/files/edit_line/insert_lines.markdown b/content/reference/promise-types/files/edit_line/insert_lines.markdown index 4477595279..5046fb929d 100644 --- a/content/reference/promise-types/files/edit_line/insert_lines.markdown +++ b/content/reference/promise-types/files/edit_line/insert_lines.markdown @@ -350,7 +350,7 @@ body insert_select example **Type:** `body location` -**See also:** [Common body attributes][Promise types#Common body attributes], [`location` bodies in the standard library](reference-masterfiles-policy-framework-lib-files.html#location-bodies), [`start` location body in the standard library](reference-masterfiles-policy-framework-lib-files.html#location-bodies), [`before(srt)` location body in the standard library](reference-masterfiles-policy-framework-lib-files.html#before), [`after(srt)` location body in the standard library](reference-masterfiles-policy-framework-lib-files.html#after) +**See also:** [Common body attributes][Promise types#Common body attributes], [`location` bodies in the standard library](/reference/masterfiles-policy-framework/lib-files#location-bodies), [`start` location body in the standard library](/reference/masterfiles-policy-framework/lib-files#location-bodies), [`before(srt)` location body in the standard library](/reference/masterfiles-policy-framework/lib-files#before), [`after(srt)` location body in the standard library](/reference/masterfiles-policy-framework/lib-files#after) #### before_after diff --git a/generator/_references.md b/generator/_references.md index e8d5928cbe..925b039803 100644 --- a/generator/_references.md +++ b/generator/_references.md @@ -11,66 +11,57 @@ [github documentation]: https://github.com/cfengine/documentation "Documentation repository on GitHub" [github design-center]: https://github.com/cfengine/design-center "Design-Center repository on GitHub" [evaluate cfengine]: https://cfengine.com/evaluate-enterprise "Evaluate CFEngine" -[cfengine getting started]: https://cfengine.com/enterprise-getting-started "Get started in 10 minutes" [LMDB]: http://symas.com/mdb/ "Symas Lightning Memory-Mapped Database" -[vim_cf3]: https://github.com/neilhwatson/vim_cf3 "CFEngine 3 vim integration" [community package repositories]: http://cfengine.com/cfengine-linux-distros "Community package repositories" [community download page]: https://cfengine.com/community/download/ "Community package download page" [enterprise software download page]: https://cfengine.com/product/free-download/ "Enterprise download page" -[boolean]: reference-language-concepts-promises.html#promise-attributes -[clist]: reference-language-concepts-promises.html#promise-attributes -[anchored]: reference-language-concepts-pattern-matching-and-referencing.html#anchored-vs-unanchored-regular-expressions -[unanchored]: reference-language-concepts-pattern-matching-and-referencing.html#anchored-vs-unanchored-regular-expressions -[datatypes]: reference-language-concepts-variables.html -[utility functions]: tags-utility-functions.html -[data functions]: tags-data-functions.html -[system functions]: tags-system-functions.html -[io functions]: tags-io-functions.html -[comm functions]: tags-communication-functions.html -[files functions]: tags-files-functions.html -[body classes]: reference-promise-types.html#classes -[body common]: reference-components.html#common-control -[body file]: reference-language-concepts-namespaces.html -[body hub control port]: reference-components-cf-hub.html#port -[sys.policy_entry_dirname]: reference-special-variables-sys.html#sys-policy_entry_dirname -[sys.policy_entry_filename]: reference-special-variables-sys.html#sys-policy_entry_filename +[boolean]: /reference/language-concepts/promises#promise-attributes +[clist]: /reference/language-concepts/promises#promise-attributes +[anchored]: /reference/language-concepts/pattern-matching-and-referencing#anchored-vs-unanchored-regular-expressions +[unanchored]: /reference/language-concepts/pattern-matching-and-referencing#anchored-vs-unanchored-regular-expressions +[datatypes]: /reference/language-concepts/variables +[body classes]: /reference/promise-types#classes +[body common]: /reference/components#common-control +[body file]: /reference/language-concepts/namespaces +[body hub control port]: /reference/components/cf-hub#port +[sys.policy_entry_dirname]: /reference/special-variables/sys#syspolicy_entry_dirname +[sys.policy_entry_filename]: /reference/special-variables/sys#syspolicy_entry_filename [supported versions]: https://cfengine.com/extended-support/ -[Inventory API]: api-enterprise-api-ref-inventory.html -[sys.uqhost]: reference-special-variables-sys.html#sys-uqhost -[sys.policy_hub]: reference-special-variables-sys.html#sys-policy_hub -[seed_cp]: reference-masterfiles-policy-framework-lib-files.html#seed_cp -[Masterfiles Policy Framework]: reference-masterfiles-policy-framework.html -[Append to inputs used by main policy]: reference-masterfiles-policy-framework.html#append-to-inputs-used-by-main-policy -[mpf_extra_autorun_inputs]: reference-masterfiles-policy-framework.html#additional-automatically-loaded-inputs -[Append to inputs used by update policy]: reference-masterfiles-policy-framework.html#append-to-inputs-used-by-update-policy -[Classes and decisions]: reference-language-concepts-classes.html -[language-concepts-classes-hard]: reference-language-concepts-classes.html#hard-classes.html "Language concepts -> Classes and decisions: Hard classes" -[lib/files.cf]: reference-masterfiles-policy-framework-lib-files -[lib/packages.cf]: reference-masterfiles-policy-framework-lib-packages -[stdlib-mog]: reference-masterfiles-policy-framework-lib-files.html#mog +[Inventory API]: /api/enterprise-api-ref/inventory +[sys.uqhost]: /reference/special-variables/sys#sysuqhost +[sys.policy_hub]: /reference/special-variables/sys#syspolicy_hub +[seed_cp]: /reference/masterfiles-policy-framework/lib-files#seed_cp +[Masterfiles Policy Framework]: /reference/masterfiles-policy-framework +[Append to inputs used by main policy]: /reference/masterfiles-policy-framework#append-to-inputs-used-by-main-policy +[Append to inputs used by update policy]: /reference/masterfiles-policy-framework#add-additional-policy-files-for-update-inputs +[Classes and decisions]: /reference/language-concepts/classes +[language-concepts-classes-hard]: /reference/language-concepts/classes#hard-classes "Language concepts -> Classes and decisions: Hard classes" +[lib/files.cf]: /reference/masterfiles-policy-framework/lib-files +[lib/packages.cf]: /reference/masterfiles-policy-framework/lib-packages +[stdlib-mog]: /reference/masterfiles-policy-framework/lib-files#mog [jq-project]: https://stedolan.github.io/jq/ "jq is a lightweight and flexible command-line JSON processor. Try online at jqplay.org!" -[Using Vagrant]: getting-started-installation-general-installation-installation-enterprise-vagrant.html "The CFEngine Vagrant environment provides an easy way to test and explore CFEngine Enterprise." -[type]: reference-functions-type.html -[promise-type-measurements]: reference-promise-types-measurements.html -[promise-type-custom]: /reference/promise-types/custom.html -[promise-type-custom-protocol]: reference-promise-types-custom.html#protocol -[component-cf-monitord]: reference-components-cf-monitord.html -[cf-hub]: reference-components-cf-hub.html -[cf-hub#hub_schedule]: reference-components-cf-hub.html#hub_schedule -[cf-hub#exclude_hosts]: reference-components-cf-hub.html#exclude_hosts -[cf-hub#control-promises]: reference-components-cf-hub.html#control-promises -[mpf-configure-measurement-collection]: reference-masterfiles-policy-framework.html#configure-enterprise-measurement-monitoring-collection.html -[mpf-configure-component-restart]: reference-masterfiles-policy-framework.html#configure-mpf-to-automatically-restart-components-on-relevant-data-change.html -[mpf-classification-bundles]: reference-masterfiles-policy-framework.html#classification-bundles-before-autorun.html -[mpf-services-autorun]: reference-masterfiles-policy-framework-services-autorun.html -[package-modules-the-api]: reference-language-concepts-modules-package-module-api.html#the-api -[Functions#collecting functions]: /reference/functions/#collecting-functions -[guest_environments]: /reference/promise-types/guest_environments.html -[defaults]: /reference/promise-types/defaults.html +[Using Vagrant]: /getting-started/01-installing-cfengine/general-installation/installation-enterprise-vagrant "The CFEngine Vagrant environment provides an easy way to test and explore CFEngine Enterprise." +[type]: /reference/functions/type +[promise-type-measurements]: /reference/promise-types/measurements +[promise-type-custom]: /reference/promise-types/custom +[promise-type-custom-protocol]: /reference/promise-types/custom#protocol +[component-cf-monitord]: /reference/components/cf-monitord +[cf-hub]: /reference/components/cf-hub +[cf-hub#hub_schedule]: /reference/components/cf-hub#hub_schedule +[cf-hub#exclude_hosts]: /reference/components/cf-hub#exclude_hosts +[cf-hub#control-promises]: /reference/components/cf-hub#control-promises +[mpf-configure-measurement-collection]: /reference/masterfiles-policy-framework#configure-enterprise-measurementmonitoring-collection +[mpf-configure-component-restart]: /reference/masterfiles-policy-framework#configure-mpf-to-automatically-restart-components-on-relevant-data-change +[mpf-classification-bundles]: /reference/masterfiles-policy-framework#classification-bundles-before-autorun +[mpf-services-autorun]: /reference/masterfiles-policy-framework/services-autorun +[package-modules-the-api]: /reference/language-concepts/modules/package-module-api#the-api +[Functions#collecting functions]: /reference/functions#collecting-functions +[guest_environments]: /reference/promise-types/guest_environments +[defaults]: /reference/promise-types/defaults [getgrgid()]: https://linux.die.net/man/3/getgrgid [getgrnam()]: https://linux.die.net/man/3/getgrnam [getpwuid()]: https://linux.die.net/man/3/getpwuid [getpwnam()]: https://linux.die.net/man/3/getpwnam [select()]: https://linux.die.net/man/3/select [search_up]: /reference/functions/findfiles_up -[High availability]: /examples/tutorials/high-availability/ "CFEngine High availability overview" +[High availability]: /examples/tutorials/high-availability "CFEngine High availability overview" diff --git a/generator/_scripts/cfdoc_link_checker.py b/generator/_scripts/cfdoc_link_checker.py new file mode 100644 index 0000000000..60b622fdfe --- /dev/null +++ b/generator/_scripts/cfdoc_link_checker.py @@ -0,0 +1,126 @@ +import html.parser +import os +import re +import sys +import urllib.parse + +# Tags/attributes that carry a URL we should be able to resolve to a file. +LINK_ATTRS = { + "a": "href", + "img": "src", + "link": "href", + "script": "src", +} + +# Schemes that never point at a file in the built site. +SKIPPED_SCHEMES = ("mailto:", "tel:", "javascript:", "data:") + +# The version switcher (lts_versions_list.html / versions_list.html) links to +# sibling docs builds for other branches/versions, e.g. "../../docs/3.27/", +# "../../docs/lts/" or "../../docs/archive/index.html". Each version/archive +# is built and deployed separately, so those paths never exist in this +# build's own _site and aren't broken links. +VERSION_LINK_RE = re.compile( + r"(?:^|/)docs/(?:master|lts|archive|\d+(?:\.\d+){1,2})(?:/|$)" +) + + +class _LinkExtractor(html.parser.HTMLParser): + """Collects (url, line) for every href/src found in one HTML page.""" + + def __init__(self): + super().__init__(convert_charrefs=True) + self.links = [] + + def handle_starttag(self, tag, attrs): + attr_name = LINK_ATTRS.get(tag) + if attr_name is None: + return + url = dict(attrs).get(attr_name) + if url: + line, _col = self.getpos() + self.links.append((url, line)) + + +def _is_checkable(url): + """Only internal links/assets are worth resolving against the filesystem.""" + url = url.strip() + if not url or url.startswith("#"): + return False + if url.startswith(SKIPPED_SCHEMES): + return False + if VERSION_LINK_RE.search(url): + return False + parsed = urllib.parse.urlsplit(url) + if parsed.scheme or parsed.netloc: + return False # external, e.g. http://, https://, //host/... + return True + + +def _resolve(site_root, html_file, url): + """Map an internal href/src to the file it should point at on disk. + + Returns None for links that don't reference a separate file (e.g. a bare + "?query" link on the current page). + """ + path = url.split("#", 1)[0].split("?", 1)[0] + if not path: + return None + + if path.startswith("/"): + target = os.path.join(site_root, path.lstrip("/")) + else: + target = os.path.normpath(os.path.join(os.path.dirname(html_file), path)) + + # Hugo renders pretty URLs as a directory containing index.html; a link + # is valid whether or not it has the trailing slash the directory implies. + if os.path.isdir(target): + target = os.path.join(target, "index.html") + + return target + + +def _find_html_files(site_root): + for dirpath, _dirnames, filenames in os.walk(site_root): + for filename in filenames: + if filename.endswith(".html"): + yield os.path.join(dirpath, filename) + + +def run(config): + """Checks every internal link/image/script src in the built site and + exits non-zero, listing what's broken and where, if any target is + missing. + + Runs against the generated HTML in CFE_DIR (after the Hugo build) rather + than the markdown source, so it validates Hugo's actual routing (pretty + URLs, aliases, page bundles) instead of a reimplementation of it. + """ + site_root = os.path.join(config["project_directory"], config["CFE_DIR"]) + if not os.path.isdir(site_root): + sys.stderr.write("ERROR: built site not found at %s\n" % site_root) + sys.exit(1) + + broken = [] + for html_file in _find_html_files(site_root): + parser = _LinkExtractor() + with open(html_file, "r", encoding="utf-8", errors="replace") as f: + parser.feed(f.read()) + + for url, line in parser.links: + if not _is_checkable(url): + continue + target = _resolve(site_root, html_file, url) + if target is None: + continue + if not os.path.exists(target): + broken.append((os.path.relpath(html_file, site_root), line, url)) + + if broken: + sys.stderr.write( + "ERROR: %d broken internal link(s)/asset reference(s) found in the built site:\n" + % len(broken) + ) + for page, line, url in broken: + sys.stderr.write(" %s:%d: %s\n" % (page, line, url)) + sys.exit(1) diff --git a/generator/_scripts/cfdoc_postprocess.py b/generator/_scripts/cfdoc_postprocess.py index a2bfe995b6..fe054528a4 100755 --- a/generator/_scripts/cfdoc_postprocess.py +++ b/generator/_scripts/cfdoc_postprocess.py @@ -24,6 +24,7 @@ import cfdoc_environment as environment import cfdoc_sourcelinks as sourcelinks +import cfdoc_link_checker as link_checker import sys @@ -35,4 +36,9 @@ print(sys.exc_info()) exit(1) +# Runs outside the try/except above: it exits directly (like +# cfdoc_references_resolver.run) with a clean broken-link report, rather than +# letting the generic exception handler above swallow that message. +link_checker.run(config) + exit(0)