ENT-4074: Added a new input-type: file

JSON.md

@@ -855,3 +855,111 @@ $ cat ./out/masterfiles/def.json
   }
 }
 ```
+
+### Referencing a file example
+
+The `"file"` input type lets a module ask the user for the path to an existing file (a script, playbook, etc.) instead of a value typed in directly - the file must already exist, `cfbs` doesn't generate its contents.
+Two optional attributes are available to the `"file"` input-type: `filetype` restricts which file extension(s) are accepted (a string or list of strings; any extension is accepted if omitted), and `while`, just like for `"list"`, lets the user supply any number of files instead of just one.
+
+```json
+      "input": [
+        {
+          "type": "file",
+          "variable": "scripts",
+          "namespace": "cfbs",
+          "bundle": "run_scripts",
+          "label": "Script",
+          "question": "Which script should be run?",
+          "filetype": ".sh",
+          "while": "Do you want to add another script?"
+        }
+      ]
+```
+
+```
+$ cfbs input run-scripts
+Collecting input for module 'run-scripts'
+Which script should be run? /tmp/notes.txt
+'/tmp/notes.txt' does not have one of the accepted file extensions (.sh), please try again
+Which script should be run? /tmp/deploy.sh
+Do you want to add another script? yes
+Which script should be run? /tmp/rollback.sh
+Do you want to add another script? no
+$ cat ./run-scripts/input.json
+[
+  {
+    "type": "file",
+    "variable": "scripts",
+    "namespace": "cfbs",
+    "bundle": "run_scripts",
+    "label": "Script",
+    "question": "Which script should be run?",
+    "filetype": ".sh",
+    "while": "Do you want to add another script?",
+    "response": ["./run-scripts/deploy.sh", "./run-scripts/rollback.sh"]
+  }
+]
+```
+
+Without `"while"`, `"response"` is a single path instead of a list.
+A file already inside the project is referred to as-is; a file from outside (as above) is copied into the module's own directory, next to `input.json`.
+
+Example referencing multiple files at different locations:
+```
+$ eza -T
+.
+├── cfbs.json
+├── test-input-module
+│   ├── input.json
+│   ├── main.cf
+│   └── testfile2.txt
+└── testfile1.txt
+
+$ cfbs input ./test-input-module/
+
+Input already exists for this module, do you want to overwrite it? [yes/y/NO/n] y
+Collecting input for module './test-input-module/'
+What file should this module use in policy? testfile1.txt
+Do you want to specify more inputs? [yes/y/NO/n] y
+What file should this module use in policy? testfile2.txt
+File 'testfile2.txt' not found, please try again
+What file should this module use in policy? test-input-module/testfile2.txt
+Do you want to specify more inputs? [yes/y/NO/n]
+--snip--
+$ cat test-input-module/input.json
+[
+  {
+    "type": "file",
+    "variable": "variable_name",
+    "namespace": "test_input_module",
+    "bundle": "main",
+    "label": "Variable name",
+    "question": "What file should this module use in policy?",
+    "while": "Do you want to specify more inputs?",
+    "response": ["testfile1.txt", "test-input-module/testfile2.txt"]
+  }
+]
+$ cfbs build
+ --snip--
+Build complete, ready to deploy 🐿
+ -> Directory: out/masterfiles
+ -> Tarball:   out/masterfiles.tgz
+To install on this machine: sudo cfbs install
+To deploy on remote hub(s): cf-remote deploy
+$ cat out/masterfiles/def.json
+{
+  "inputs": ["services/cfbs/modules/test-input-module/main.cf"],
+  "vars": { "control_common_bundlesequence_end": ["test_input_module:main"] },
+  "variables": {
+    "test_input_module:main.variable_name": {
+      "value": [
+        "$(sys.workdir)/masterfiles/services/cfbs/modules/test-input-module/testfile1.txt",
+        "$(sys.workdir)/masterfiles/services/cfbs/modules/test-input-module/testfile2.txt"
+      ],
+      "comment": "Added by 'cfbs input'"
+    }
+  }
+}
+```
+As you can see as long as the files referenced exists within the build project, they will all end up inside the modules directory on the host.
+Referenced files not inside the project directory will first be copied inside and then follows suit.

cfbs/build.py

@@ -243,7 +243,81 @@ def _perform_directory_step(args, source, destination, prefix):
     write_json(defjson, merged)
 
 
-def _perform_input_step(args, name, destination, prefix):
+def _already_shipped_destination(path, build_modules, destination):
+    """If `path` is already inside a local module's directory that has its own
+    "directory" build step shipping it to masterfiles, return the on-host path
+    that step will produce. That step already copies the whole directory
+    during this same build, so the caller doesn't need to (and shouldn't)
+    copy the file itself - just point at where it will end up.
+    """
+    abs_path = os.path.abspath(path)
+    for module in build_modules:
+        module_name = module.get("name", "")
+        if not (module_name.startswith("./") and module_name.endswith("/")):
+            continue
+        module_root = os.path.abspath(module_name)
+        if os.path.commonpath([abs_path, module_root]) != module_root:
+            continue
+        for step in module.get("steps", []):
+            operation, args = split_build_step(step)
+            if operation != "directory" or len(args) != 2:
+                continue
+            src, dst = args
+            if src not in (".", "./"):
+                continue
+            rel = os.path.relpath(abs_path, module_root)
+            dst = "" if dst in (".", "./") else dst
+            dest = os.path.join(destination, dst, rel)
+            return "$(sys.workdir)/masterfiles/" + os.path.relpath(dest, destination)
+    return None
+
+
+def _localize_file_inputs(name, input_data, destination, build_modules):
+    """Copy files referenced by "file" type input responses into the built
+    masterfiles, so they're actually part of what gets deployed instead of
+    only existing in the project directory. Rewrites the responses in place
+    to the resulting on-host path.
+
+    If a response is already shipped by another module's own "directory"
+    build step (e.g. the project author set one up manually), that step's
+    destination is used instead of making a redundant copy.
+    """
+    if not isinstance(input_data, list):
+        return
+
+    module_dir_name = name[2:] if name.startswith("./") else name
+    module_dir_name = os.path.basename(module_dir_name.rstrip("/"))
+
+    def _localize(path):
+        if not path or not os.path.isfile(path):
+            return path
+
+        already_shipped = _already_shipped_destination(path, build_modules, destination)
+        if already_shipped is not None:
+            return already_shipped
+
+        dest = os.path.join(
+            destination,
+            "services",
+            "cfbs",
+            "modules",
+            module_dir_name,
+            os.path.basename(path),
+        )
+        cp(path, dest)
+        return "$(sys.workdir)/masterfiles/" + os.path.relpath(dest, destination)
+
+    for element in input_data:
+        if not isinstance(element, dict) or element.get("type") != "file":
+            continue
+        response = element.get("response")
+        if isinstance(response, list):
+            element["response"] = [_localize(path) for path in response]
+        else:
+            element["response"] = _localize(response)
+
+
+def _perform_input_step(args, name, destination, prefix, build_modules):
     src, dst = args
     if dst in [".", "./"]:
         dst = ""
@@ -264,6 +338,7 @@ def _perform_input_step(args, name, destination, prefix):
         )
         return
     extras, original = read_json(src), read_json(dst)
+    _localize_file_inputs(name, extras, destination, build_modules)
     extras = generate_augment(name, extras)
     log.debug("Generated augment: %s", pretty(extras))
     if not extras:
@@ -424,7 +499,7 @@ def perform_build(config: CFBSConfig, diffs_filename=None) -> int:
             elif operation == "directory":
                 _perform_directory_step(args, source, destination, prefix)
             elif operation == "input":
-                _perform_input_step(args, name, destination, prefix)
+                _perform_input_step(args, name, destination, prefix, config["build"])
             elif operation == "policy_files":
                 _perform_policy_files_step(args, destination, prefix)
             elif operation == "bundles":

cfbs/cfbs_config.py

@@ -574,6 +574,44 @@ def _input_multiline_string(input_data):
             )
             return response
 
+        def _input_file(input_data):
+            _check_keys(["question"], input_data)
+            filetypes = input_data.get("filetype")
+            if filetypes is not None and not isinstance(filetypes, list):
+                filetypes = [filetypes]
+
+            def _one_file():
+                while True:
+                    response = prompt_user(
+                        self.non_interactive,
+                        input_data["question"],
+                        default=input_data.get("default"),
+                    )
+                    if self.non_interactive:
+                        return response
+                    if filetypes and not any(
+                        response.endswith(filetype) for filetype in filetypes
+                    ):
+                        print(
+                            "'%s' does not have one of the accepted file extensions (%s), please try again"
+                            % (response, ", ".join(filetypes))
+                        )
+                        continue
+                    if not os.path.isfile(response):
+                        print("File '%s' not found, please try again" % response)
+                        continue
+                    return response
+
+            if "while" not in input_data:
+                return _one_file()
+
+            result = [_one_file()]
+            while prompt_user_yesno(
+                self.non_interactive, input_data["while"], default="no"
+            ):
+                result.append(_one_file())
+            return result
+
         def _input_elements(subtype):
             result = OrderedDict()
             for element in subtype:
@@ -626,6 +664,8 @@ def _input_list(input_data):
                 definition["response"] = _input_string(definition)
             elif definition["type"] == "string-multiline":
                 definition["response"] = _input_multiline_string(definition)
+            elif definition["type"] == "file":
+                definition["response"] = _input_file(definition)
             elif definition["type"] == "list":
                 definition["response"] = _input_list(definition)
             else:

cfbs/commands.py

@@ -1580,14 +1580,58 @@ def input_command(args, input_from="cfbs input"):
 
         input_data = copy.deepcopy(module["input"])
         config.input_command(module_name, input_data)
+        copied_files = _place_file_input(module_name, input_data)
 
         write_json(input_path, input_data)
         do_commit = True
         files_to_commit.append(input_path)
+        files_to_commit.extend(copied_files)
     config.save()
     return CFBSCommandGitResult(0, do_commit, None, files_to_commit)
 
 
+def _place_file_input(module_name, input_data):
+    """Make sure files given as "file" type input are part of the project.
+
+    A file already inside the project is left where it is and simply
+    referred to. A file from outside the project is copied into the
+    module's directory, next to its input.json, and the response is
+    updated to point at that copy. A "file" input using "while" to collect
+    multiple files has a list of paths as its response, each handled the
+    same way.
+
+    Returns the list of paths that were copied into the project, so they
+    can be committed alongside input.json.
+    """
+    project_root = os.path.abspath(".")
+    module_dir = os.path.join(".", module_name)
+    copied_files = []
+
+    def _place(path):
+        if not path or not os.path.isfile(path):
+            return path
+
+        abs_path = os.path.abspath(path)
+        if os.path.commonpath([abs_path, project_root]) == project_root:
+            return path  # Already part of the project, refer to it as-is
+
+        dest = os.path.join(module_dir, os.path.basename(path))
+        cp(abs_path, dest)
+        copied_files.append(dest)
+        return dest
+
+    for definition in input_data:
+        if definition.get("type") != "file":
+            continue
+        response = definition.get("response")
+        if isinstance(response, list):
+            definition["response"] = [_place(path) for path in response]
+        else:
+            definition["response"] = _place(response)
+
+    return copied_files
+
+
 @cfbs_command("set-input")
 @commit_after_command("Set input for module %s", [FIRST_ARG])
 def set_input_command(name, infile):

cfbs/validate.py

@@ -682,10 +682,15 @@ def _validate_module_input(name, module):
                     % field,
                 )
 
-        if input_element["type"] not in ("string", "list", "string-multiline"):
+        if input_element["type"] not in (
+            "string",
+            "string-multiline",
+            "file",
+            "list",
+        ):
             raise CFBSValidationError(
                 name,
-                'The input "type" must be "string", "string-multiline", or "list", not "%s"'
+                'The input "type" must be "string", "string-multiline", "file" or "list", not "%s"'
                 % input_element["type"],
             )
         if not re.fullmatch(r"[a-z_]+", input_element["variable"]):
@@ -766,6 +771,36 @@ def _validate_module_input(name, module):
                         % part["type"],
                     )
 
+        if input_element["type"] == "file":
+            if "filetype" in input_element:
+                filetype = input_element["filetype"]
+                filetypes = filetype if type(filetype) is list else [filetype]
+                if not filetypes:
+                    raise CFBSValidationError(
+                        name,
+                        'The "filetype" field of a "file" input element must be a non-empty file extension, or a non-empty list of them, not "%s"'
+                        % filetype,
+                    )
+                for part in filetypes:
+                    if (
+                        type(part) is not str
+                        or not part.strip()
+                        or not part.startswith(".")
+                    ):
+                        raise CFBSValidationError(
+                            name,
+                            'The "filetype" field of a "file" input element must consist of file extensions starting with ".", not "%s"'
+                            % part,
+                        )
+            if "while" in input_element and (
+                type(input_element["while"]) is not str
+                or not input_element["while"].strip()
+            ):
+                raise CFBSValidationError(
+                    name,
+                    'The "while" prompt in an input "file" element must be a non-empty / non-whitespace string',
+                )
+
 
 def _compare_dict(a, b, ignore=None):
     assert isinstance(a, dict) and isinstance(b, dict)