CFE-4623: Fix modules_presence to handle findfiles() trailing slash behavior (3.24)

.github/workflows/bootstrap_policy_run_check.yml

@@ -3,11 +3,29 @@ on:
 
 jobs:
   bootstrap_policy_run_check:
+    strategy:
+      fail-fast: false
+      matrix:
+        cfengine_version: [ "3.21.8", "3.24.3", "3.27.0", "master" ]
+
     runs-on: ubuntu-22.04
     steps:
       - name: Checkout
         uses: actions/checkout@v4
         with:
           path: masterfiles
+      - name: Get Togethers
+        uses: cfengine/together-javascript-action@main
+        id: together
+        with:
+          myToken: ${{ secrets.GITHUB_TOKEN }}
+      - name: Checkout Core
+        if: ${{ steps.together.outputs.core != null }}
+        uses: actions/checkout@v4
+        with:
+          repository: cfengine/core
+          path: core
+          ref: ${{steps.together.outputs.core || github.base_ref || github.ref}}
+          submodules: recursive
       - name: Install, bootstrap, policy run
-        run: masterfiles/ci/docker-bootstrap-policy-run.sh
+        run: masterfiles/ci/docker-bootstrap-policy-run.sh ${{ matrix.cfengine_version }}

cfe_internal/update/update_policy.cf

@@ -158,6 +158,10 @@ bundle agent cfe_internal_update_policy_cpv
 #   scanned for update.
 {
   vars:
+      # TODO: Remove this once sys.keydir is always available (3.26+ only)
+      "keydir" -> { "CFE-2822" }
+        string => ifelse(isvariable("sys.keydir"), "$(sys.keydir)", "$(sys.workdir)/ppkeys");
+
       "inputs_dir"         string => translatepath("$(sys.inputdir)"),
       comment => "Directory containing CFEngine policies",
       handle => "cfe_internal_update_policy_vars_inputs_dir";
@@ -185,7 +189,7 @@ bundle agent cfe_internal_update_policy_cpv
       comment => "Path to a policy file",
       handle => "cfe_internal_update_vars_file_check";
 
-      "ppkeys_file"        string => translatepath("$(sys.workdir)/ppkeys/localhost.pub"),
+      "ppkeys_file"        string => translatepath("$(keydir)/localhost.pub"),
       comment => "Path to public key file",
       handle => "cfe_internal_update_policy_vars_ppkeys_file";
 
@@ -780,7 +784,7 @@ body delete u_tidy
 bundle agent modules_presence
 # @brief Render vendored and user provided modules from $(sys.inputdir) to $(sys.workdir)
 #
-# @description This bundle manages the contents of $(sys.workdir)/modules by
+# @description This bundle manages the contents of moduledir (/var/cfengine/modules) by
 #              first dealing with package module scripts.
 #              Preference is given to user provided package module scripts in
 #              modules/packages directory. If a module there matches a mustache
@@ -790,26 +794,33 @@ bundle agent modules_presence
 #              e.g. modules/packages/apt_get takes precedence over modules/packages/apt_get.mustache
 #
 #              Any other files in the modules directory will be promised to
-#              be updated in $(sys.workdir)/modules, including any sub-directories.
+#              be updated in moduledir (/var/cfengine/modules), including any sub-directories.
 {
+
   vars:
+    # TODO: Remove this once sys.moduledir is always available (3.26+ only)
+    "moduledir" string => ifelse(isvariable("sys.moduledir"), "$(sys.moduledir)", "$(sys.workdir)/modules");
     "_vendored_dir" string => "$(this.promise_dirname)$(const.dirsep)..$(const.dirsep)..$(const.dirsep)modules$(const.dirsep)packages$(const.dirsep)vendored$(const.dirsep)";
+    "_vendored_dir_filter" string => "$(this.promise_dirname)$(const.dirsep)..$(const.dirsep)..$(const.dirsep)modules$(const.dirsep)packages$(const.dirsep)vendored";
     "_override_dir" string => "$(this.promise_dirname)$(const.dirsep)..$(const.dirsep)..$(const.dirsep)modules$(const.dirsep)packages$(const.dirsep)";
     "_custom_template_dir" string => "$(this.promise_dirname)$(const.dirsep)..$(const.dirsep)..$(const.dirsep)modules$(const.dirsep)mustache$(const.dirsep)";
     "_vendored_paths" slist => findfiles("$(_vendored_dir)*.mustache");
     "_custom_template_paths" slist => findfiles("$(_custom_template_dir)*.mustache"), if => isdir( "$(_custom_template_dir)" );
-    "_package_paths" slist => filter("$(_override_dir)vendored", _package_paths_tmp, "false", "true", 999);
+    "_package_paths_tmp" slist => findfiles("${_override_dir}*"),
+      comment => "We get a temp list of files that we have to filter out the vendored sub directory.";
 
     windows::
-      "_package_paths_tmp" slist => findfiles("$(_override_dir)*");
+      "_package_paths" slist => filter("\Q${_vendored_dir_filter}\E.*", _package_paths_tmp, "true", "true", 999),
+        comment => "The list will include the vendored sub-directory that we don't want so remove it.";
       "_vendored_modules" slist => maplist(regex_replace("$(this)", "\Q$(_vendored_dir)\E(.*).mustache", "$1", "g"), @(_vendored_paths));
       "_override_modules" slist => maplist(regex_replace("$(this)", "\Q$(_override_dir)\E(.*)", "$1", "g"), @(_package_paths));
       # replace single backslashes in a windows path with double-backslashes
       # to avoid problems with things like `C:\Program Files` and `\promises`
       # causing PCRE to try and interpret special escape sequences.
       "_not_vendored_modules_pathname_regex" string => regex_replace("$(sys.inputdir)$(const.dirsep)modules$(const.dirsep)(?!packages$(const.dirsep)vendored).*","\\\\","\\\\\\\\","g");
     !windows::
-      "_package_paths_tmp" slist => findfiles("$(_override_dir)*");
+      "_package_paths" slist => filter("${_vendored_dir_filter}.*", _package_paths_tmp, "true", "true", 999),
+        comment => "The list will include the vendored sub-directory that we don't want so remove it.";
       "_vendored_modules" slist => maplist(regex_replace("$(this)", "$(_vendored_dir)(.*).mustache", "$1", "g"), @(_vendored_paths));
       "_override_modules" slist => maplist(regex_replace("$(this)", "$(_override_dir)(.*)", "$1", "g"), @(_package_paths));
       "_custom_template_modules" slist => maplist(regex_replace("$(this)", "$(_custom_template_dir)(.*).mustache", "$1", "g"), @(_custom_template_paths));
@@ -824,29 +835,29 @@ bundle agent modules_presence
 # update (see controls/update_def.cf input_name_patterns var.
 
   files:
-    "$(sys.workdir)/modules/packages/$(_vendored_modules)"
+    "$(moduledir)/packages/$(_vendored_modules)"
       create => "true",
       perms => u_mo("755", "root"),
       unless => canonify("override_vendored_module_$(_vendored_modules)"),
       edit_template => "$(_vendored_dir)$(_vendored_modules).mustache",
       template_method => "mustache";
 
-    "$(sys.workdir)/modules/packages/$(_override_modules)"
+    "$(moduledir)/packages/$(_override_modules)"
       copy_from => u_cp_missing_ok("$(_override_dir)$(_override_modules)"),
       perms => u_mo("755", "root"),
       if => or (
         canonify("override_vendored_module_$(_override_modules)"),
         canonify("override_module_$(_override_modules)"));
 
-    "$(sys.workdir)/modules/$(_custom_template_modules)" -> { "ENT-10793" }
+    "$(moduledir)/$(_custom_template_modules)" -> { "ENT-10793" }
         comment => "We want to render mustache templated modules",
         handle => "cfe_internal_update_policy_files_custom_template_modules",
         template_method => "mustache",
         edit_template => "$(_custom_template_dir)$(_custom_template_modules).mustache",
         perms => u_mo("500", "root"),
         if => fileexists("$(_custom_template_dir)$(_custom_template_modules).mustache");
 
-    "$(sys.workdir)/modules"
+    "$(moduledir)"
       comment => "Copy any non-packages modules",
       handle => "cfe_internal_update_policy_files_nonpackages_modules",
       copy_from => u_cp("$(sys.inputdir)$(const.dirsep)modules"),
@@ -860,14 +871,13 @@ bundle agent modules_presence
   reports:
     DEBUG::
       "_override_dir: $(_override_dir)";
-      "_package_paths_tmp: $(with)" with => storejson(_package_paths_tmp);
+      "_package_paths: $(with)" with => storejson(_package_paths);
       "_not_vendored_modules_pathname_regex: $(_not_vendored_modules_pathname_regex)";
       "_vendored_modules: $(_vendored_modules)";
       "_override_modules: $(_override_modules)";
       "_vendored_dir: $(_vendored_dir)";
       "_vendored_paths: $(_vendored_paths)";
       "_override_dir: $(_override_dir)";
-      "_package_paths: $(_package_paths)";
       "override_vendored_module_$(_vendored_modules)"
         if => "override_vendored_module_$(_vendored_modules)";
       "override_module_$(_override_modules)"

ci/bootstrap-policy-run.cfremote.Dockerfile

@@ -0,0 +1,5 @@
+ARG CFENGINE_VERSION="master"
+FROM debian
+RUN apt update && apt upgrade -y
+RUN apt install -y pipx sudo make automake autoconf git procps python3
+RUN pipx install cf-remote

ci/bootstrap-policy-run.sh

@@ -1,8 +1,16 @@
 #!/usr/bin/env sh
 set -ex
-./autogen.sh --prefix=/var/lib/cfengine
+if [ -f ../core/ci/install.sh ]; then
+  ../core/ci/install.sh
+else
+  # here we use community so that masterfiles has less errors when bootstrapping as it expects an enterprise hub with the -nova package
+  PATH=/root/.local/bin:$PATH cf-remote --version "$CFENGINE_VERSION" install --edition community --clients localhost
+fi
+./autogen.sh
 make install
+export PATH=/var/cfengine/bin:$PATH
 which cf-agent
+ps -efl | grep cf- # debug cf-serverd already running somehow?
 cf-agent -IB $(hostname -i) | tee bootstrap.log
 cf-agent -KIf update.cf | tee update.log
 cf-agent -KI | tee promise.log

ci/bootstrap-policy-run.source.Dockerfile

@@ -0,0 +1,6 @@
+ARG CFENGINE_VERSION="master"
+FROM debian
+COPY core /core
+RUN apt update && apt upgrade -y
+# need python3 for apt_get package module to avoid errors
+RUN apt install -y sudo make automake autoconf git python3 procps

ci/docker-bootstrap-policy-run.sh

@@ -5,27 +5,41 @@ set -ex
 COMPUTED_ROOT="$(readlink -e "$(dirname "$0")/../../")"
 # NTECH_ROOT should be the same, but if available use it so user can do their own thing.
 NTECH_ROOT=${NTECH_ROOT:-$COMPUTED_ROOT}
+CFENGINE_VERSION=${1:-master}
+export CFENGINE_VERSION
 
 cd "${NTECH_ROOT}/masterfiles"
 
 # cleanup
 rm -f update.log bootstrap.log promise.log
-if docker ps | grep mpf; then
-  docker stop mpf
+image_name=bootstrap-${CFENGINE_VERSION}
+if docker ps | grep "$image_name"; then
+  docker stop "$image_name"
 fi
-if docker ps -a | grep mpf; then
-  docker ps -a | grep mpf | awk '{print $1}' | xargs docker rm
+if docker ps -a | grep "$image_name"; then
+  docker ps -a | grep "$image_name" | awk '{print $1}' | xargs docker rm
 fi
-if docker images | grep mpf; then
-  docker rmi mpf
+if docker images | grep "$image_name"; then
+  docker rmi "$image_name"
+fi
+
+if [ -d "${NTECH_ROOT}"/core ]; then
+  docker build -t "$image_name" --build-arg CFENGINE_VERSION="$CFENGINE_VERSION" -f "${NTECH_ROOT}"/masterfiles/ci/bootstrap-policy-run.source.Dockerfile  "${NTECH_ROOT}"
+else
+  docker build -t "$image_name" --build-arg CFENGINE_VERSION="$CFENGINE_VERSION" -f "${NTECH_ROOT}"/masterfiles/ci/bootstrap-policy-run.cfremote.Dockerfile  "${NTECH_ROOT}"
 fi
 
 # run the test
-docker build -t mpf -f "${NTECH_ROOT}"/masterfiles/ci/bootstrap-policy-run.Dockerfile  "${NTECH_ROOT}"/masterfiles
-docker run --workdir /mpf --volume "${NTECH_ROOT}"/masterfiles:/mpf --tty mpf sh /mpf/ci/bootstrap-policy-run.sh
-if grep error *.log; then
+docker run -e CFENGINE_VERSION --workdir /masterfiles --volume "${NTECH_ROOT}"/masterfiles:/masterfiles --tty "$image_name" sh /masterfiles/ci/bootstrap-policy-run.sh
+
+if grep error ./*.log; then
   echo "fail"
   exit 1
 else
   echo "success"
 fi
+
+if [ ! -f bootstrap.log ] || [ ! -f promise.log ] || [ ! -f update.log ]; then
+  echo "No log files. Fail."
+  exit 23
+fi