From 7015725db93723f0d616adf30c34e4e4af478a3b Mon Sep 17 00:00:00 2001 From: Craig Comstock Date: Tue, 14 Apr 2026 17:26:57 -0500 Subject: [PATCH 1/2] fix: refactor modules_presence bundle to filter out sub-directory vendored without regard to trailing slashes findfiles() behavior was unstable from version 3.21.8 through 3.27.0 and so we needed to refactor this policy to not care whether findfiles() would return directories with trailing slashes or not. The goal of the policy is to find any files in modules/packages which are not in modules/packages/vendored and use those instead of the vendored files. Ticket: CFE-4623 Changelog: none --- cfe_internal/update/update_policy.cf | 34 ++++++++++++++++++---------- 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/cfe_internal/update/update_policy.cf b/cfe_internal/update/update_policy.cf index 0a09b0659f..7f8754daf3 100644 --- a/cfe_internal/update/update_policy.cf +++ b/cfe_internal/update/update_policy.cf @@ -158,6 +158,10 @@ bundle agent cfe_internal_update_policy_cpv # scanned for update. { vars: + # TODO: Remove this once sys.keydir is always available (3.26+ only) + "keydir" -> { "CFE-2822" } + string => ifelse(isvariable("sys.keydir"), "$(sys.keydir)", "$(sys.workdir)/ppkeys"); + "inputs_dir" string => translatepath("$(sys.inputdir)"), comment => "Directory containing CFEngine policies", handle => "cfe_internal_update_policy_vars_inputs_dir"; @@ -185,7 +189,7 @@ bundle agent cfe_internal_update_policy_cpv comment => "Path to a policy file", handle => "cfe_internal_update_vars_file_check"; - "ppkeys_file" string => translatepath("$(sys.workdir)/ppkeys/localhost.pub"), + "ppkeys_file" string => translatepath("$(keydir)/localhost.pub"), comment => "Path to public key file", handle => "cfe_internal_update_policy_vars_ppkeys_file"; @@ -780,7 +784,7 @@ body delete u_tidy bundle agent modules_presence # @brief Render vendored and user provided modules from $(sys.inputdir) to $(sys.workdir) # -# @description This bundle manages the contents of $(sys.workdir)/modules by +# @description This bundle manages the contents of moduledir (/var/cfengine/modules) by # first dealing with package module scripts. # Preference is given to user provided package module scripts in # modules/packages directory. If a module there matches a mustache @@ -790,18 +794,24 @@ bundle agent modules_presence # e.g. modules/packages/apt_get takes precedence over modules/packages/apt_get.mustache # # Any other files in the modules directory will be promised to -# be updated in $(sys.workdir)/modules, including any sub-directories. +# be updated in moduledir (/var/cfengine/modules), including any sub-directories. { + vars: + # TODO: Remove this once sys.moduledir is always available (3.26+ only) + "moduledir" string => ifelse(isvariable("sys.moduledir"), "$(sys.moduledir)", "$(sys.workdir)/modules"); "_vendored_dir" string => "$(this.promise_dirname)$(const.dirsep)..$(const.dirsep)..$(const.dirsep)modules$(const.dirsep)packages$(const.dirsep)vendored$(const.dirsep)"; + "_vendored_dir_filter" string => "$(this.promise_dirname)$(const.dirsep)..$(const.dirsep)..$(const.dirsep)modules$(const.dirsep)packages$(const.dirsep)vendored"; "_override_dir" string => "$(this.promise_dirname)$(const.dirsep)..$(const.dirsep)..$(const.dirsep)modules$(const.dirsep)packages$(const.dirsep)"; "_custom_template_dir" string => "$(this.promise_dirname)$(const.dirsep)..$(const.dirsep)..$(const.dirsep)modules$(const.dirsep)mustache$(const.dirsep)"; "_vendored_paths" slist => findfiles("$(_vendored_dir)*.mustache"); "_custom_template_paths" slist => findfiles("$(_custom_template_dir)*.mustache"), if => isdir( "$(_custom_template_dir)" ); - "_package_paths" slist => filter("$(_override_dir)vendored", _package_paths_tmp, "false", "true", 999); + "_package_paths_tmp" slist => findfiles("${_override_dir}*"), + comment => "We get a temp list of files that we have to filter out the vendored sub directory."; windows:: - "_package_paths_tmp" slist => findfiles("$(_override_dir)*"); + "_package_paths" slist => filter("\Q${_vendored_dir_filter}\E.*", _package_paths_tmp, "true", "true", 999), + comment => "The list will include the vendored sub-directory that we don't want so remove it."; "_vendored_modules" slist => maplist(regex_replace("$(this)", "\Q$(_vendored_dir)\E(.*).mustache", "$1", "g"), @(_vendored_paths)); "_override_modules" slist => maplist(regex_replace("$(this)", "\Q$(_override_dir)\E(.*)", "$1", "g"), @(_package_paths)); # replace single backslashes in a windows path with double-backslashes @@ -809,7 +819,8 @@ bundle agent modules_presence # causing PCRE to try and interpret special escape sequences. "_not_vendored_modules_pathname_regex" string => regex_replace("$(sys.inputdir)$(const.dirsep)modules$(const.dirsep)(?!packages$(const.dirsep)vendored).*","\\\\","\\\\\\\\","g"); !windows:: - "_package_paths_tmp" slist => findfiles("$(_override_dir)*"); + "_package_paths" slist => filter("${_vendored_dir_filter}.*", _package_paths_tmp, "true", "true", 999), + comment => "The list will include the vendored sub-directory that we don't want so remove it."; "_vendored_modules" slist => maplist(regex_replace("$(this)", "$(_vendored_dir)(.*).mustache", "$1", "g"), @(_vendored_paths)); "_override_modules" slist => maplist(regex_replace("$(this)", "$(_override_dir)(.*)", "$1", "g"), @(_package_paths)); "_custom_template_modules" slist => maplist(regex_replace("$(this)", "$(_custom_template_dir)(.*).mustache", "$1", "g"), @(_custom_template_paths)); @@ -824,21 +835,21 @@ bundle agent modules_presence # update (see controls/update_def.cf input_name_patterns var. files: - "$(sys.workdir)/modules/packages/$(_vendored_modules)" + "$(moduledir)/packages/$(_vendored_modules)" create => "true", perms => u_mo("755", "root"), unless => canonify("override_vendored_module_$(_vendored_modules)"), edit_template => "$(_vendored_dir)$(_vendored_modules).mustache", template_method => "mustache"; - "$(sys.workdir)/modules/packages/$(_override_modules)" + "$(moduledir)/packages/$(_override_modules)" copy_from => u_cp_missing_ok("$(_override_dir)$(_override_modules)"), perms => u_mo("755", "root"), if => or ( canonify("override_vendored_module_$(_override_modules)"), canonify("override_module_$(_override_modules)")); - "$(sys.workdir)/modules/$(_custom_template_modules)" -> { "ENT-10793" } + "$(moduledir)/$(_custom_template_modules)" -> { "ENT-10793" } comment => "We want to render mustache templated modules", handle => "cfe_internal_update_policy_files_custom_template_modules", template_method => "mustache", @@ -846,7 +857,7 @@ bundle agent modules_presence perms => u_mo("500", "root"), if => fileexists("$(_custom_template_dir)$(_custom_template_modules).mustache"); - "$(sys.workdir)/modules" + "$(moduledir)" comment => "Copy any non-packages modules", handle => "cfe_internal_update_policy_files_nonpackages_modules", copy_from => u_cp("$(sys.inputdir)$(const.dirsep)modules"), @@ -860,14 +871,13 @@ bundle agent modules_presence reports: DEBUG:: "_override_dir: $(_override_dir)"; - "_package_paths_tmp: $(with)" with => storejson(_package_paths_tmp); + "_package_paths: $(with)" with => storejson(_package_paths); "_not_vendored_modules_pathname_regex: $(_not_vendored_modules_pathname_regex)"; "_vendored_modules: $(_vendored_modules)"; "_override_modules: $(_override_modules)"; "_vendored_dir: $(_vendored_dir)"; "_vendored_paths: $(_vendored_paths)"; "_override_dir: $(_override_dir)"; - "_package_paths: $(_package_paths)"; "override_vendored_module_$(_vendored_modules)" if => "override_vendored_module_$(_vendored_modules)"; "override_module_$(_override_modules)" From d47e268bb7a0df5f47a2db9b0b41d04445c7758d Mon Sep 17 00:00:00 2001 From: Craig Comstock Date: Fri, 10 Apr 2026 15:43:29 -0500 Subject: [PATCH 2/2] ci: refactored bootstrap-policy-run test to build with several released versions and master (or from source if a core PR is mentioned in the description) Ticket: CFE-4623 Changelog: none --- .../workflows/bootstrap_policy_run_check.yml | 20 +++++++++++- ci/bootstrap-policy-run.Dockerfile | 2 -- ci/bootstrap-policy-run.cfremote.Dockerfile | 5 +++ ci/bootstrap-policy-run.sh | 10 +++++- ci/bootstrap-policy-run.source.Dockerfile | 6 ++++ ci/docker-bootstrap-policy-run.sh | 32 +++++++++++++------ 6 files changed, 62 insertions(+), 13 deletions(-) delete mode 100644 ci/bootstrap-policy-run.Dockerfile create mode 100644 ci/bootstrap-policy-run.cfremote.Dockerfile create mode 100644 ci/bootstrap-policy-run.source.Dockerfile diff --git a/.github/workflows/bootstrap_policy_run_check.yml b/.github/workflows/bootstrap_policy_run_check.yml index c356cfd3ff..ec9ea7bc7b 100644 --- a/.github/workflows/bootstrap_policy_run_check.yml +++ b/.github/workflows/bootstrap_policy_run_check.yml @@ -3,11 +3,29 @@ on: jobs: bootstrap_policy_run_check: + strategy: + fail-fast: false + matrix: + cfengine_version: [ "3.21.8", "3.24.3", "3.27.0", "master" ] + runs-on: ubuntu-22.04 steps: - name: Checkout uses: actions/checkout@v4 with: path: masterfiles + - name: Get Togethers + uses: cfengine/together-javascript-action@main + id: together + with: + myToken: ${{ secrets.GITHUB_TOKEN }} + - name: Checkout Core + if: ${{ steps.together.outputs.core != null }} + uses: actions/checkout@v4 + with: + repository: cfengine/core + path: core + ref: ${{steps.together.outputs.core || github.base_ref || github.ref}} + submodules: recursive - name: Install, bootstrap, policy run - run: masterfiles/ci/docker-bootstrap-policy-run.sh + run: masterfiles/ci/docker-bootstrap-policy-run.sh ${{ matrix.cfengine_version }} diff --git a/ci/bootstrap-policy-run.Dockerfile b/ci/bootstrap-policy-run.Dockerfile deleted file mode 100644 index fc91a79f52..0000000000 --- a/ci/bootstrap-policy-run.Dockerfile +++ /dev/null @@ -1,2 +0,0 @@ -FROM alpine -RUN apk update && apk add cfengine make automake autoconf git diff --git a/ci/bootstrap-policy-run.cfremote.Dockerfile b/ci/bootstrap-policy-run.cfremote.Dockerfile new file mode 100644 index 0000000000..500318a04e --- /dev/null +++ b/ci/bootstrap-policy-run.cfremote.Dockerfile @@ -0,0 +1,5 @@ +ARG CFENGINE_VERSION="master" +FROM debian +RUN apt update && apt upgrade -y +RUN apt install -y pipx sudo make automake autoconf git procps python3 +RUN pipx install cf-remote diff --git a/ci/bootstrap-policy-run.sh b/ci/bootstrap-policy-run.sh index b2143e3242..ab10f38b1c 100755 --- a/ci/bootstrap-policy-run.sh +++ b/ci/bootstrap-policy-run.sh @@ -1,8 +1,16 @@ #!/usr/bin/env sh set -ex -./autogen.sh --prefix=/var/lib/cfengine +if [ -f ../core/ci/install.sh ]; then + ../core/ci/install.sh +else + # here we use community so that masterfiles has less errors when bootstrapping as it expects an enterprise hub with the -nova package + PATH=/root/.local/bin:$PATH cf-remote --version "$CFENGINE_VERSION" install --edition community --clients localhost +fi +./autogen.sh make install +export PATH=/var/cfengine/bin:$PATH which cf-agent +ps -efl | grep cf- # debug cf-serverd already running somehow? cf-agent -IB $(hostname -i) | tee bootstrap.log cf-agent -KIf update.cf | tee update.log cf-agent -KI | tee promise.log diff --git a/ci/bootstrap-policy-run.source.Dockerfile b/ci/bootstrap-policy-run.source.Dockerfile new file mode 100644 index 0000000000..1e51578ac2 --- /dev/null +++ b/ci/bootstrap-policy-run.source.Dockerfile @@ -0,0 +1,6 @@ +ARG CFENGINE_VERSION="master" +FROM debian +COPY core /core +RUN apt update && apt upgrade -y +# need python3 for apt_get package module to avoid errors +RUN apt install -y sudo make automake autoconf git python3 procps diff --git a/ci/docker-bootstrap-policy-run.sh b/ci/docker-bootstrap-policy-run.sh index 6db6215712..1ed8b5cd57 100755 --- a/ci/docker-bootstrap-policy-run.sh +++ b/ci/docker-bootstrap-policy-run.sh @@ -5,27 +5,41 @@ set -ex COMPUTED_ROOT="$(readlink -e "$(dirname "$0")/../../")" # NTECH_ROOT should be the same, but if available use it so user can do their own thing. NTECH_ROOT=${NTECH_ROOT:-$COMPUTED_ROOT} +CFENGINE_VERSION=${1:-master} +export CFENGINE_VERSION cd "${NTECH_ROOT}/masterfiles" # cleanup rm -f update.log bootstrap.log promise.log -if docker ps | grep mpf; then - docker stop mpf +image_name=bootstrap-${CFENGINE_VERSION} +if docker ps | grep "$image_name"; then + docker stop "$image_name" fi -if docker ps -a | grep mpf; then - docker ps -a | grep mpf | awk '{print $1}' | xargs docker rm +if docker ps -a | grep "$image_name"; then + docker ps -a | grep "$image_name" | awk '{print $1}' | xargs docker rm fi -if docker images | grep mpf; then - docker rmi mpf +if docker images | grep "$image_name"; then + docker rmi "$image_name" +fi + +if [ -d "${NTECH_ROOT}"/core ]; then + docker build -t "$image_name" --build-arg CFENGINE_VERSION="$CFENGINE_VERSION" -f "${NTECH_ROOT}"/masterfiles/ci/bootstrap-policy-run.source.Dockerfile "${NTECH_ROOT}" +else + docker build -t "$image_name" --build-arg CFENGINE_VERSION="$CFENGINE_VERSION" -f "${NTECH_ROOT}"/masterfiles/ci/bootstrap-policy-run.cfremote.Dockerfile "${NTECH_ROOT}" fi # run the test -docker build -t mpf -f "${NTECH_ROOT}"/masterfiles/ci/bootstrap-policy-run.Dockerfile "${NTECH_ROOT}"/masterfiles -docker run --workdir /mpf --volume "${NTECH_ROOT}"/masterfiles:/mpf --tty mpf sh /mpf/ci/bootstrap-policy-run.sh -if grep error *.log; then +docker run -e CFENGINE_VERSION --workdir /masterfiles --volume "${NTECH_ROOT}"/masterfiles:/masterfiles --tty "$image_name" sh /masterfiles/ci/bootstrap-policy-run.sh + +if grep error ./*.log; then echo "fail" exit 1 else echo "success" fi + +if [ ! -f bootstrap.log ] || [ ! -f promise.log ] || [ ! -f update.log ]; then + echo "No log files. Fail." + exit 23 +fi