ENT-14418: Added --ask-pass, --password-file and --switch-user-command options

.gitignore

@@ -8,6 +8,8 @@ dist
 log
 __pycache__/
 **/.DS_STORE
+/tests/docker/sudo/id_test
+/tests/docker/sudo/id_test.pub
 
 # coverage
 .coverage.*

README.md

@@ -8,6 +8,7 @@ Commands for provisioning hosts in the cloud (AWS or GCP) are also available.
 
 - cf-remote requires python 3.6 or greater.
 - SSH must be configured in such a way that cf-remote can login without a password.
+- The account cf-remote logs in as must be root or be able to `sudo`. Passwordless sudo is not required, see [Switching user on the remote hosts](#switching-user-on-the-remote-hosts).
 - An sftp server for transferring files on UNIX hosts. e.g. openssh-sftp-server for debian-based distributions.
 
 ## Installation
@@ -189,6 +190,44 @@ If you have more than one key in `~/.ssh` you may need to specify which key `cf-
 $ export CF_REMOTE_SSH_KEY="~/.ssh/id_rsa.pub"
 ```
 
+### Switching user on the remote hosts
+
+Most of what `cf-remote` does needs root, so unless it logs in as root it runs commands through `sudo`.
+If `sudo` asks for a password, use `--ask-pass` (`-K`) and `cf-remote` prompts for it once and uses it for all the hosts in the run:
+
+```
+$ cf-remote --ask-pass install --clients ubuntu@10.0.0.5
+Password for switching user:
+```
+
+The password is written to the standard input of the `ssh` process, so it is never part of a command line and doesn't show up in the process list, in the shell history on the target host, or in the output of `--log-level DEBUG`.
+It is only sent to hosts where switching user actually asks for a password.
+
+Where there is nobody to answer a prompt, such as in a script or a CI job, put the password on the first line of a file and point `--password-file` at it:
+
+```
+$ cf-remote --password-file ~/.cf-remote-password install --clients ubuntu@10.0.0.5
+```
+
+`cf-remote` refuses to read the file if others can read it, the same way `ssh` refuses to use a private key with too generous permissions, so `chmod 600` it first.
+
+Use `--switch-user-command` if `sudo` is not what you want to switch user with:
+
+```
+$ cf-remote --ask-pass --switch-user-command "doas /bin/sh -c" info -H bsd-host
+```
+
+The command to run is appended as a single quoted argument.
+The default is `sudo -n bash -c`, or `sudo -S -p '' bash -c` with `--ask-pass`, since `sudo` only reads the password from standard input when it is given `-S`.
+`-n` in the first is because there is no terminal to prompt on, so a `sudo` that wants a password should say so instead of trying to ask; it is left out of the second because it means never prompt, and `sudo` then refuses the password rather than reading it.
+
+Whichever command is used, it is run with `LC_ALL=C`.
+`cf-remote` recognizes "this needs a password you didn't give me" by what the command said, and `sudo` says it in the caller's language on the distributions that ship its translations, which `ssh` carries over by default.
+`sudo` keeps `LC_ALL`, so the command being run is left in the C locale as well; commands run without switching user are not.
+
+A password can only reach a command that reads it from standard input, which in practice means `sudo -S` and the tools that copy its interface, such as `dzdo -S`.
+`doas` and `su` read from a terminal instead, so they work with `--switch-user-command` where they need no password, but cannot be given one by `cf-remote`.
+
 ### Working on the local host
 
 `cf-remote` can work on the local host when the target host is `localhost`. In this case, it executes commands locally without connecting over SSH.

cf_remote/aramid.py

@@ -114,11 +114,31 @@ def _get_put_method_args(method, host, src, dst):
 )
 
 
+def _popen(args, stdin_input=None):
+    """Start a process, giving it a pipe on standard input if we have input for it
+
+    Uses 'Popen.communicate()' to avoid deadlock (see https://docs.python.org/3/library/subprocess.html#subprocess.Popen.stderr).
+
+    An empty string closes the pipe immediately. Anything waiting for input sees EOF at once.
+    """
+    return subprocess.Popen(
+        args,
+        stdin=(subprocess.PIPE if stdin_input is not None else None),
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        universal_newlines=True,
+    )
+
+
 class _Task:
-    def __init__(self, host, proc, action=None, retries=0):  # TODO: timeout=60
+    def __init__(
+        self, host, proc, action=None, retries=0, stdin_input=None
+    ):  # TODO: timeout=60
         self.host = host
         self.proc = proc
         self.action = action
+        self.stdin_input = stdin_input
+        self._input_given = False
         self._max_retries = retries
         self._retries = retries
         self.stdout = ""
@@ -130,8 +150,13 @@ def __init__(self, host, proc, action=None, retries=0):  # TODO: timeout=60
 
     def communicate(self, timeout=1, ignore_failed=False):
         start = time.time()
+        # 'communicate()' keeps writing what the first call handed it, and
+        # raises if a later one hands it the same input again. Timing out is
+        # normal here, so only the first call gets it.
+        stdin_input = None if self._input_given else self.stdin_input
+        self._input_given = True
         try:
-            out, err = self.proc.communicate(timeout=timeout)
+            out, err = self.proc.communicate(input=stdin_input, timeout=timeout)
         except subprocess.TimeoutExpired:
             log.debug("Connection timed out")
             return False
@@ -143,12 +168,8 @@ def communicate(self, timeout=1, ignore_failed=False):
                 if self._retries > 0:
                     # wait for the rest of timeout (if any) and restart the process
                     time.sleep(max(timeout - (time.time() - start), 0))
-                    self.proc = subprocess.Popen(
-                        self.proc.args,
-                        stdout=subprocess.PIPE,
-                        stderr=subprocess.PIPE,
-                        universal_newlines=True,
-                    )
+                    self.proc = _popen(self.proc.args, self.stdin_input)
+                    self._input_given = False  # a new process needs it again
                     self._retries -= 1
                     return False
                 else:
@@ -305,6 +326,7 @@ def execute(
     ignore_failed=False,
     echo=True,
     echo_cmd=False,
+    stdin_input=None,
 ):  # TODO: parallel=False
     """Execute command on remote hosts (in parallel)
 
@@ -321,6 +343,9 @@ def execute(
     :param bool echo: whether to echo the output (STDOUT first followed by
                       STDERR) of the given commands
     :param bool echo_cmd: whether to echo the commands run on the hosts
+    :param str stdin_input: data to write to the standard input of the commands,
+                            for example a password for switching user. If `None`,
+                            standard input is inherited from `cf-remote` itself.
     :return: results of commands executed on the given hosts
     :rtype: dict(:class:`Host` -> list(:class:`ExecutionResult`))
 
@@ -340,18 +365,16 @@ def execute(
         port_args = []
         if host.port != _DEFAULT_SSH_PORT:
             port_args += ["-p", str(host.port)]
-        proc = subprocess.Popen(
+        proc = _popen(
             ["ssh"]
             + DEFAULT_SSH_ARGS
             + port_args
             + host.extra_ssh_args
             + [host.login]
             + [commands[i]],
-            stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE,
-            universal_newlines=True,
+            stdin_input=stdin_input,
         )
-        task = _Task(host, proc, commands[i], retries=retries)
+        task = _Task(host, proc, commands[i], retries=retries, stdin_input=stdin_input)
         host.tasks.append(task)
         tasks.append(task)
 

cf_remote/args.py

@@ -283,6 +283,26 @@ def add_connect_args(sp: argparse.ArgumentParser) -> None:
     )
 
 
+def add_switch_user_args(ap: argparse.ArgumentParser) -> None:
+    password_source = ap.add_mutually_exclusive_group()
+    password_source.add_argument(
+        "--ask-pass",
+        "-K",
+        help="Prompt for the password to switch user with",
+        action="store_true",
+    )
+    password_source.add_argument(
+        "--password-file",
+        help="Read the password to switch user with from the first line of a file",
+        type=str,
+    )
+    ap.add_argument(
+        "--switch-user-command",
+        help="Command to switch user with, e.g. 'doas /bin/sh -c'",
+        type=str,
+    )
+
+
 @cache
 def get_arg_parser() -> argparse.ArgumentParser:
     ap = argparse.ArgumentParser(
@@ -304,6 +324,7 @@ def get_arg_parser() -> argparse.ArgumentParser:
         type=str,
         const=True,
     )
+    add_switch_user_args(ap)
 
     command_help_hint = (
         "Commands (use %s COMMAND --help to get more info)"

cf_remote/commands.py

@@ -53,7 +53,7 @@
 from cf_remote import cloud_data
 
 
-def info(hosts, users=None, all=False):
+def info(hosts, users=None, all=False, switch_user=None):
     if all:
         hosts = _get_all_hosts()
     elif not hosts:
@@ -69,19 +69,25 @@ def info(hosts, users=None, all=False):
     log.debug("hosts='{}'".format(hosts))
     errors = 0
     for host in hosts:
-        data = get_info(host, users=users)
+        data = get_info(host, users=users, switch_user=switch_user)
         if data:
             print_info(data)
         else:
             errors += 1
     return errors
 
 
-def run(hosts, command, users=None, sudo=False, raw=False):
+def run(hosts, command, users=None, sudo=False, raw=False, switch_user=None):
     assert hosts
     errors = 0
     for host in hosts:
-        lines = run_command(host=host, command=command, users=users, sudo=sudo)
+        lines = run_command(
+            host=host,
+            command=command,
+            users=users,
+            sudo=sudo,
+            switch_user=switch_user,
+        )
         if lines is None:
             log.error("Command: '{}'\nFailed on host: '{}'".format(command, host))
             errors += 1
@@ -107,15 +113,15 @@ def run(hosts, command, users=None, sudo=False, raw=False):
     return errors
 
 
-def sudo(hosts, command, users=None, raw=False):
-    return run(hosts, command, users, sudo=True, raw=raw)
+def sudo(hosts, command, users=None, raw=False, switch_user=None):
+    return run(hosts, command, users, sudo=True, raw=raw, switch_user=switch_user)
 
 
-def scp(hosts, files, users=None):
+def scp(hosts, files, users=None, switch_user=None):
     errors = 0
     for host in hosts:
         for file in files:
-            errors += transfer_file(host, file, users)
+            errors += transfer_file(host, file, users, switch_user=switch_user)
     return errors
 
 
@@ -197,7 +203,8 @@ def install(
     edition=None,
     remote_download=False,
     trust_keys=None,
-    insecure=False
+    insecure=False,
+    switch_user=None
 ):
     assert hubs or clients
     assert not (hubs and clients and package)
@@ -269,6 +276,7 @@ def install(
                     insecure=insecure,
                     demo_salt=salt,
                     demo_sha=sha,
+                    switch_user=switch_user,
                 )
             )
 
@@ -305,6 +313,7 @@ def install(
                 show_info=show_host_info,
                 remote_download=remote_download,
                 trust_keys=trust_keys,
+                switch_user=switch_user,
             )
         )
 
@@ -937,14 +946,14 @@ def show(ansible_inventory):
     return 0
 
 
-def uninstall(hosts, purge=False):
+def uninstall(hosts, purge=False, switch_user=None):
     errors = 0
     for host in hosts:
-        errors += uninstall_host(host, purge=purge)
+        errors += uninstall_host(host, purge=purge, switch_user=switch_user)
     return errors
 
 
-def deploy_tarball(hubs, tarball):
+def deploy_tarball(hubs, tarball, switch_user=None):
     assert os.path.isfile(tarball)
 
     if not tarball.endswith((".tgz", ".tar.gz")):
@@ -955,7 +964,7 @@ def deploy_tarball(hubs, tarball):
 
     errors = 0
     for hub in hubs:
-        errors += deploy_masterfiles(hub, tarball)
+        errors += deploy_masterfiles(hub, tarball, switch_user=switch_user)
     return errors
 
 
@@ -981,7 +990,7 @@ def _get_hubs():
     return _get_all_hosts(role="hub")
 
 
-def deploy(hubs, masterfiles):
+def deploy(hubs, masterfiles, switch_user=None):
     if not hubs:
         hubs = _get_hubs()
         if hubs:
@@ -1022,7 +1031,7 @@ def deploy(hubs, masterfiles):
     masterfiles = masterfiles.rstrip("/")
 
     if os.path.isfile(masterfiles):
-        return deploy_tarball(hubs, masterfiles)
+        return deploy_tarball(hubs, masterfiles, switch_user=switch_user)
 
     if masterfiles.endswith((".tgz", ".tar.gz")):
         if not os.path.exists(masterfiles):
@@ -1070,10 +1079,10 @@ def deploy(hubs, masterfiles):
     above = directory[0 : -len("/masterfiles")]
     os.system("rm -rf %s" % tarball)
     os.system("tar -czf %s -C %s masterfiles" % (tarball, above))
-    return deploy_tarball(hubs, tarball)
+    return deploy_tarball(hubs, tarball, switch_user=switch_user)
 
 
-def agent(hosts, bootstrap=None):
+def agent(hosts, bootstrap=None, switch_user=None):
     if bootstrap and len(bootstrap) > 1:
         raise CFRExitError(
             "Cannot boostrap {} to {}. Cannot bootstrap to more than one host.".format(
@@ -1082,7 +1091,7 @@ def agent(hosts, bootstrap=None):
         )
 
     for host in hosts:
-        data = get_info(host)
+        data = get_info(host, switch_user=switch_user)
 
         if not data["agent"]:
             raise CFRExitError("CFEngine not installed on {}".format(host))
@@ -1094,7 +1103,7 @@ def agent(hosts, bootstrap=None):
 
         command = " ".join(args)
 
-        output = run_command(host, command, sudo=True)
+        output = run_command(host, command, sudo=True, switch_user=switch_user)
         if output:
             print(output)