diff --git a/JSON.md b/JSON.md index 7c946ca0..7d054ec5 100644 --- a/JSON.md +++ b/JSON.md @@ -855,3 +855,111 @@ $ cat ./out/masterfiles/def.json } } ``` + +### Referencing a file example + +The `"file"` input type lets a module ask the user for the path to an existing file (a script, playbook, etc.) instead of a value typed in directly - the file must already exist, `cfbs` doesn't generate its contents. +Two optional attributes are available to the `"file"` input-type: `filetype` restricts which file extension(s) are accepted (a string or list of strings; any extension is accepted if omitted), and `while`, just like for `"list"`, lets the user supply any number of files instead of just one. + +```json + "input": [ + { + "type": "file", + "variable": "scripts", + "namespace": "cfbs", + "bundle": "run_scripts", + "label": "Script", + "question": "Which script should be run?", + "filetype": ".sh", + "while": "Do you want to add another script?" + } + ] +``` + +``` +$ cfbs input run-scripts +Collecting input for module 'run-scripts' +Which script should be run? /tmp/notes.txt +'/tmp/notes.txt' does not have one of the accepted file extensions (.sh), please try again +Which script should be run? /tmp/deploy.sh +Do you want to add another script? yes +Which script should be run? /tmp/rollback.sh +Do you want to add another script? no +$ cat ./run-scripts/input.json +[ + { + "type": "file", + "variable": "scripts", + "namespace": "cfbs", + "bundle": "run_scripts", + "label": "Script", + "question": "Which script should be run?", + "filetype": ".sh", + "while": "Do you want to add another script?", + "response": ["./run-scripts/deploy.sh", "./run-scripts/rollback.sh"] + } +] +``` + +Without `"while"`, `"response"` is a single path instead of a list. +A file already inside the project is referred to as-is; a file from outside (as above) is copied into the module's own directory, next to `input.json`. + +Example referencing multiple files at different locations: +``` +$ eza -T +. +├── cfbs.json +├── test-input-module +│ ├── input.json +│ ├── main.cf +│ └── testfile2.txt +└── testfile1.txt + +$ cfbs input ./test-input-module/ + +Input already exists for this module, do you want to overwrite it? [yes/y/NO/n] y +Collecting input for module './test-input-module/' +What file should this module use in policy? testfile1.txt +Do you want to specify more inputs? [yes/y/NO/n] y +What file should this module use in policy? testfile2.txt +File 'testfile2.txt' not found, please try again +What file should this module use in policy? test-input-module/testfile2.txt +Do you want to specify more inputs? [yes/y/NO/n] +--snip-- +$ cat test-input-module/input.json +[ + { + "type": "file", + "variable": "variable_name", + "namespace": "test_input_module", + "bundle": "main", + "label": "Variable name", + "question": "What file should this module use in policy?", + "while": "Do you want to specify more inputs?", + "response": ["testfile1.txt", "test-input-module/testfile2.txt"] + } +] +$ cfbs build + --snip-- +Build complete, ready to deploy 🐿 + -> Directory: out/masterfiles + -> Tarball: out/masterfiles.tgz +To install on this machine: sudo cfbs install +To deploy on remote hub(s): cf-remote deploy +$ cat out/masterfiles/def.json +{ + "inputs": ["services/cfbs/modules/test-input-module/main.cf"], + "vars": { "control_common_bundlesequence_end": ["test_input_module:main"] }, + "variables": { + "test_input_module:main.variable_name": { + "value": [ + "$(sys.workdir)/masterfiles/services/cfbs/modules/test-input-module/testfile1.txt", + "$(sys.workdir)/masterfiles/services/cfbs/modules/test-input-module/testfile2.txt" + ], + "comment": "Added by 'cfbs input'" + } + } +} +``` +As you can see as long as the files referenced exists within the build project, they will all end up inside the modules directory on the host. +Referenced files not inside the project directory will first be copied inside and then follows suit. diff --git a/cfbs/build.py b/cfbs/build.py index 079a0fe0..57bb223a 100644 --- a/cfbs/build.py +++ b/cfbs/build.py @@ -243,7 +243,81 @@ def _perform_directory_step(args, source, destination, prefix): write_json(defjson, merged) -def _perform_input_step(args, name, destination, prefix): +def _already_shipped_destination(path, build_modules, destination): + """If `path` is already inside a local module's directory that has its own + "directory" build step shipping it to masterfiles, return the on-host path + that step will produce. That step already copies the whole directory + during this same build, so the caller doesn't need to (and shouldn't) + copy the file itself - just point at where it will end up. + """ + abs_path = os.path.abspath(path) + for module in build_modules: + module_name = module.get("name", "") + if not (module_name.startswith("./") and module_name.endswith("/")): + continue + module_root = os.path.abspath(module_name) + if os.path.commonpath([abs_path, module_root]) != module_root: + continue + for step in module.get("steps", []): + operation, args = split_build_step(step) + if operation != "directory" or len(args) != 2: + continue + src, dst = args + if src not in (".", "./"): + continue + rel = os.path.relpath(abs_path, module_root) + dst = "" if dst in (".", "./") else dst + dest = os.path.join(destination, dst, rel) + return "$(sys.workdir)/masterfiles/" + os.path.relpath(dest, destination) + return None + + +def _localize_file_inputs(name, input_data, destination, build_modules): + """Copy files referenced by "file" type input responses into the built + masterfiles, so they're actually part of what gets deployed instead of + only existing in the project directory. Rewrites the responses in place + to the resulting on-host path. + + If a response is already shipped by another module's own "directory" + build step (e.g. the project author set one up manually), that step's + destination is used instead of making a redundant copy. + """ + if not isinstance(input_data, list): + return + + module_dir_name = name[2:] if name.startswith("./") else name + module_dir_name = os.path.basename(module_dir_name.rstrip("/")) + + def _localize(path): + if not path or not os.path.isfile(path): + return path + + already_shipped = _already_shipped_destination(path, build_modules, destination) + if already_shipped is not None: + return already_shipped + + dest = os.path.join( + destination, + "services", + "cfbs", + "modules", + module_dir_name, + os.path.basename(path), + ) + cp(path, dest) + return "$(sys.workdir)/masterfiles/" + os.path.relpath(dest, destination) + + for element in input_data: + if not isinstance(element, dict) or element.get("type") != "file": + continue + response = element.get("response") + if isinstance(response, list): + element["response"] = [_localize(path) for path in response] + else: + element["response"] = _localize(response) + + +def _perform_input_step(args, name, destination, prefix, build_modules): src, dst = args if dst in [".", "./"]: dst = "" @@ -264,6 +338,7 @@ def _perform_input_step(args, name, destination, prefix): ) return extras, original = read_json(src), read_json(dst) + _localize_file_inputs(name, extras, destination, build_modules) extras = generate_augment(name, extras) log.debug("Generated augment: %s", pretty(extras)) if not extras: @@ -424,7 +499,7 @@ def perform_build(config: CFBSConfig, diffs_filename=None) -> int: elif operation == "directory": _perform_directory_step(args, source, destination, prefix) elif operation == "input": - _perform_input_step(args, name, destination, prefix) + _perform_input_step(args, name, destination, prefix, config["build"]) elif operation == "policy_files": _perform_policy_files_step(args, destination, prefix) elif operation == "bundles": diff --git a/cfbs/cfbs_config.py b/cfbs/cfbs_config.py index 02a0a361..832eebc9 100644 --- a/cfbs/cfbs_config.py +++ b/cfbs/cfbs_config.py @@ -574,6 +574,44 @@ def _input_multiline_string(input_data): ) return response + def _input_file(input_data): + _check_keys(["question"], input_data) + filetypes = input_data.get("filetype") + if filetypes is not None and not isinstance(filetypes, list): + filetypes = [filetypes] + + def _one_file(): + while True: + response = prompt_user( + self.non_interactive, + input_data["question"], + default=input_data.get("default"), + ) + if self.non_interactive: + return response + if filetypes and not any( + response.endswith(filetype) for filetype in filetypes + ): + print( + "'%s' does not have one of the accepted file extensions (%s), please try again" + % (response, ", ".join(filetypes)) + ) + continue + if not os.path.isfile(response): + print("File '%s' not found, please try again" % response) + continue + return response + + if "while" not in input_data: + return _one_file() + + result = [_one_file()] + while prompt_user_yesno( + self.non_interactive, input_data["while"], default="no" + ): + result.append(_one_file()) + return result + def _input_elements(subtype): result = OrderedDict() for element in subtype: @@ -626,6 +664,8 @@ def _input_list(input_data): definition["response"] = _input_string(definition) elif definition["type"] == "string-multiline": definition["response"] = _input_multiline_string(definition) + elif definition["type"] == "file": + definition["response"] = _input_file(definition) elif definition["type"] == "list": definition["response"] = _input_list(definition) else: diff --git a/cfbs/commands.py b/cfbs/commands.py index 262d6415..68c95583 100644 --- a/cfbs/commands.py +++ b/cfbs/commands.py @@ -1580,14 +1580,58 @@ def input_command(args, input_from="cfbs input"): input_data = copy.deepcopy(module["input"]) config.input_command(module_name, input_data) + copied_files = _place_file_input(module_name, input_data) write_json(input_path, input_data) do_commit = True files_to_commit.append(input_path) + files_to_commit.extend(copied_files) config.save() return CFBSCommandGitResult(0, do_commit, None, files_to_commit) +def _place_file_input(module_name, input_data): + """Make sure files given as "file" type input are part of the project. + + A file already inside the project is left where it is and simply + referred to. A file from outside the project is copied into the + module's directory, next to its input.json, and the response is + updated to point at that copy. A "file" input using "while" to collect + multiple files has a list of paths as its response, each handled the + same way. + + Returns the list of paths that were copied into the project, so they + can be committed alongside input.json. + """ + project_root = os.path.abspath(".") + module_dir = os.path.join(".", module_name) + copied_files = [] + + def _place(path): + if not path or not os.path.isfile(path): + return path + + abs_path = os.path.abspath(path) + if os.path.commonpath([abs_path, project_root]) == project_root: + return path # Already part of the project, refer to it as-is + + dest = os.path.join(module_dir, os.path.basename(path)) + cp(abs_path, dest) + copied_files.append(dest) + return dest + + for definition in input_data: + if definition.get("type") != "file": + continue + response = definition.get("response") + if isinstance(response, list): + definition["response"] = [_place(path) for path in response] + else: + definition["response"] = _place(response) + + return copied_files + + @cfbs_command("set-input") @commit_after_command("Set input for module %s", [FIRST_ARG]) def set_input_command(name, infile): diff --git a/cfbs/validate.py b/cfbs/validate.py index 19aa0671..7ff6cc8c 100644 --- a/cfbs/validate.py +++ b/cfbs/validate.py @@ -682,10 +682,15 @@ def _validate_module_input(name, module): % field, ) - if input_element["type"] not in ("string", "list", "string-multiline"): + if input_element["type"] not in ( + "string", + "string-multiline", + "file", + "list", + ): raise CFBSValidationError( name, - 'The input "type" must be "string", "string-multiline", or "list", not "%s"' + 'The input "type" must be "string", "string-multiline", "file" or "list", not "%s"' % input_element["type"], ) if not re.fullmatch(r"[a-z_]+", input_element["variable"]): @@ -766,6 +771,36 @@ def _validate_module_input(name, module): % part["type"], ) + if input_element["type"] == "file": + if "filetype" in input_element: + filetype = input_element["filetype"] + filetypes = filetype if type(filetype) is list else [filetype] + if not filetypes: + raise CFBSValidationError( + name, + 'The "filetype" field of a "file" input element must be a non-empty file extension, or a non-empty list of them, not "%s"' + % filetype, + ) + for part in filetypes: + if ( + type(part) is not str + or not part.strip() + or not part.startswith(".") + ): + raise CFBSValidationError( + name, + 'The "filetype" field of a "file" input element must consist of file extensions starting with ".", not "%s"' + % part, + ) + if "while" in input_element and ( + type(input_element["while"]) is not str + or not input_element["while"].strip() + ): + raise CFBSValidationError( + name, + 'The "while" prompt in an input "file" element must be a non-empty / non-whitespace string', + ) + def _compare_dict(a, b, ignore=None): assert isinstance(a, dict) and isinstance(b, dict) diff --git a/tests/shell/060_input_file.sh b/tests/shell/060_input_file.sh new file mode 100644 index 00000000..7c0b0348 --- /dev/null +++ b/tests/shell/060_input_file.sh @@ -0,0 +1,48 @@ +set -e +set -x +cd tests/ +mkdir -p ./tmp/ +cd ./tmp/ +touch cfbs.json && rm cfbs.json +rm -rf .git +rm -rf copy-a-file +rm -rf run-scripts +cp ../shell/060_input_file/example-cfbs.json cfbs.json +echo "some content" > source.txt +echo "some content" > source.sh + +# Interactively, a "file" input must reject a path that doesn't exist and +# prompt again, then accept a path that does exist: +printf '/does/not/exist.txt\n./source.txt\n' | cfbs input copy-a-file | grep "not found, please try again" +grep '"response": "./source.txt"' copy-a-file/input.json +rm -rf copy-a-file + +# Interactively, a "file" input must reject a path whose extension doesn't +# match one of the module-specified "filetype" extensions, then accept one +# that does: +printf './source.sh\n./source.txt\n' | cfbs input copy-a-file | grep "does not have one of the accepted file extensions (.txt, .log), please try again" +grep '"response": "./source.txt"' copy-a-file/input.json +rm -rf copy-a-file + +# In non-interactive mode, a "file" input must fall back to the "default" +# given in the input definition, without prompting or checking existence: +cfbs --non-interactive input copy-a-file +grep '"type": "file"' copy-a-file/input.json +grep '"response": "./source.txt"' copy-a-file/input.json + +cfbs render-input copy-a-file copy-a-file/input.json actual.output +diff actual.output ../shell/060_input_file/expected-augment.json + +# A "file" input with a "while" prompt must let the user supply multiple +# files. Files from outside the project must be copied into the module's +# directory, next to input.json, and "response" updated to a list of the +# (possibly localized) paths: +echo "echo one" > /tmp/one.sh +echo "echo two" > /tmp/two.sh +printf '/tmp/one.sh\nyes\n/tmp/two.sh\nno\n' | cfbs input run-scripts +grep '"response": \[' run-scripts/input.json +grep '"./run-scripts/one.sh"' run-scripts/input.json +grep '"./run-scripts/two.sh"' run-scripts/input.json +test -f run-scripts/one.sh +test -f run-scripts/two.sh +rm -rf run-scripts /tmp/one.sh /tmp/two.sh diff --git a/tests/shell/060_input_file/example-cfbs.json b/tests/shell/060_input_file/example-cfbs.json new file mode 100644 index 00000000..48e57b87 --- /dev/null +++ b/tests/shell/060_input_file/example-cfbs.json @@ -0,0 +1,42 @@ +{ + "name": "Example", + "type": "policy-set", + "description": "Example description", + "git": false, + "build": [ + { + "name": "copy-a-file", + "description": "Copy a file.", + "steps": ["input ./input.json def.json"], + "input": [ + { + "type": "file", + "variable": "source", + "namespace": "cfbs", + "bundle": "copy_a_file", + "label": "Source file", + "question": "Which file should be copied?", + "filetype": [".txt", ".log"], + "default": "./source.txt" + } + ] + }, + { + "name": "run-scripts", + "description": "Run multiple scripts.", + "steps": ["input ./input.json def.json"], + "input": [ + { + "type": "file", + "variable": "scripts", + "namespace": "cfbs", + "bundle": "run_scripts", + "label": "Script", + "question": "Which script should be run?", + "while": "Do you want to add another script?", + "filetype": ".sh" + } + ] + } + ] +} diff --git a/tests/shell/060_input_file/expected-augment.json b/tests/shell/060_input_file/expected-augment.json new file mode 100644 index 00000000..ff519694 --- /dev/null +++ b/tests/shell/060_input_file/expected-augment.json @@ -0,0 +1,8 @@ +{ + "variables": { + "cfbs:copy_a_file.source": { + "value": "./source.txt", + "comment": "Added by 'cfbs input'" + } + } +} diff --git a/tests/shell/all.sh b/tests/shell/all.sh index 152bc363..d095c9cb 100644 --- a/tests/shell/all.sh +++ b/tests/shell/all.sh @@ -103,6 +103,7 @@ run_test tests/shell/056_render_input_list.sh run_test tests/shell/057_render_input_no_response.sh run_test tests/shell/058_render_input_fail.sh run_test tests/shell/059_input_string_multiline.sh +run_test tests/shell/060_input_file.sh # Summary _suite_end=$(date +%s) diff --git a/tests/test_build.py b/tests/test_build.py new file mode 100644 index 00000000..cb1e7ec3 --- /dev/null +++ b/tests/test_build.py @@ -0,0 +1,163 @@ +import os +import copy + +from cfbs.build import _localize_file_inputs + + +def test_localize_file_inputs_copies_single_file(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + os.makedirs("out/masterfiles") + with open("deploy.sh", "w") as f: + f.write("echo hi\n") + + input_data = [ + { + "type": "file", + "variable": "script", + "response": "deploy.sh", + } + ] + + _localize_file_inputs("run-a-script", input_data, "out/masterfiles", []) + + expected_dest = "out/masterfiles/services/cfbs/modules/run-a-script/deploy.sh" + assert os.path.isfile(expected_dest) + assert ( + input_data[0]["response"] + == "$(sys.workdir)/masterfiles/services/cfbs/modules/run-a-script/deploy.sh" + ) + + +def test_localize_file_inputs_copies_list_of_files(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + os.makedirs("out/masterfiles") + with open("one.sh", "w") as f: + f.write("echo one\n") + with open("two.sh", "w") as f: + f.write("echo two\n") + + input_data = [ + { + "type": "file", + "variable": "scripts", + "response": ["one.sh", "two.sh"], + } + ] + + _localize_file_inputs("run-scripts", input_data, "out/masterfiles", []) + + assert input_data[0]["response"] == [ + "$(sys.workdir)/masterfiles/services/cfbs/modules/run-scripts/one.sh", + "$(sys.workdir)/masterfiles/services/cfbs/modules/run-scripts/two.sh", + ] + assert os.path.isfile("out/masterfiles/services/cfbs/modules/run-scripts/one.sh") + assert os.path.isfile("out/masterfiles/services/cfbs/modules/run-scripts/two.sh") + + +def test_localize_file_inputs_strips_local_module_prefix(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + os.makedirs("out/masterfiles") + with open("deploy.sh", "w") as f: + f.write("echo hi\n") + + input_data = [{"type": "file", "variable": "script", "response": "deploy.sh"}] + + _localize_file_inputs("./run-a-script", input_data, "out/masterfiles", []) + + assert os.path.isfile( + "out/masterfiles/services/cfbs/modules/run-a-script/deploy.sh" + ) + + +def test_localize_file_inputs_ignores_missing_file(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + os.makedirs("out/masterfiles") + + input_data = [ + {"type": "file", "variable": "script", "response": "does-not-exist.sh"} + ] + before = copy.deepcopy(input_data) + + _localize_file_inputs("run-a-script", input_data, "out/masterfiles", []) + + assert input_data == before + + +def test_localize_file_inputs_ignores_non_file_types(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + os.makedirs("out/masterfiles") + + input_data = [ + {"type": "string", "variable": "filename", "response": "/tmp/foo.txt"} + ] + before = copy.deepcopy(input_data) + + _localize_file_inputs("some-module", input_data, "out/masterfiles", []) + + assert input_data == before + + +def test_localize_file_inputs_skips_copy_when_already_shipped(tmp_path, monkeypatch): + """When the referenced file is already inside a module directory that has + its own "directory" build step, that step's destination should be used + instead of also copying the file into services/cfbs/modules//. + """ + monkeypatch.chdir(tmp_path) + os.makedirs("out/masterfiles") + os.makedirs("./run-scripts/input_files") + with open("./run-scripts/input_files/deploy.sh", "w") as f: + f.write("echo hi\n") + + build_modules = [ + { + "name": "./run-scripts/input_files/", + "steps": ["directory ./ services/cfbs/run-scripts/input_files/"], + } + ] + input_data = [ + { + "type": "file", + "variable": "script", + "response": "./run-scripts/input_files/deploy.sh", + } + ] + + _localize_file_inputs("run-scripts", input_data, "out/masterfiles", build_modules) + + assert input_data[0]["response"] == ( + "$(sys.workdir)/masterfiles/services/cfbs/run-scripts/input_files/deploy.sh" + ) + # The file was NOT additionally copied to the bespoke destination: + assert not os.path.isfile( + "out/masterfiles/services/cfbs/modules/run-scripts/deploy.sh" + ) + + +def test_localize_file_inputs_ignores_non_directory_steps(tmp_path, monkeypatch): + """A registered module whose steps don't include a "directory ./ ..." + step (e.g. it only copies a single file) shouldn't be mistaken for + already shipping the referenced file - the bespoke copy should still + happen. + """ + monkeypatch.chdir(tmp_path) + os.makedirs("out/masterfiles") + os.makedirs("./run-scripts") + with open("./run-scripts/deploy.sh", "w") as f: + f.write("echo hi\n") + + build_modules = [ + { + "name": "./run-scripts/", + "steps": ["copy deploy.sh services/cfbs/run-scripts/deploy.sh"], + } + ] + input_data = [ + {"type": "file", "variable": "script", "response": "./run-scripts/deploy.sh"} + ] + + _localize_file_inputs("run-scripts", input_data, "out/masterfiles", build_modules) + + assert input_data[0]["response"] == ( + "$(sys.workdir)/masterfiles/services/cfbs/modules/run-scripts/deploy.sh" + ) + assert os.path.isfile("out/masterfiles/services/cfbs/modules/run-scripts/deploy.sh")