diff --git a/packages/syft-enclave/docker/Dockerfile b/packages/syft-enclave/docker/Dockerfile index 4a6d4523908..8b1a6a7cd86 100644 --- a/packages/syft-enclave/docker/Dockerfile +++ b/packages/syft-enclave/docker/Dockerfile @@ -27,6 +27,12 @@ ENV PATH="/repo/.venv/bin:$PATH" # its libs on LD_LIBRARY_PATH. ENV LD_LIBRARY_PATH=/usr/local/nvidia/lib64 +# Unprivileged account that sandboxed jobs run as (SYFT_JOB_SANDBOX). Jobs are +# untrusted code; running them as root would let them read the Drive token and +# rewrite the enclave's own source. Nologin: it exists to own processes, not to +# be logged into. +RUN useradd --uid 1500 --no-create-home --shell /usr/sbin/nologin syftjob + WORKDIR /app COPY packages/syft-enclave/docker/attestation_server.py . COPY packages/syft-enclave/docker/entrypoint.sh . @@ -45,7 +51,10 @@ SYFT_BOOTSTRAP,\ SYFT_BOOTSTRAP_WIF_AUDIENCE,\ SYFT_BOOTSTRAP_WIF_SECRET,\ SYFT_BOOTSTRAP_SA_SECRET,\ -SYFT_ENCLAVE_TOKEN_CONTENT" +SYFT_ENCLAVE_TOKEN_CONTENT,\ +SYFT_JOB_SANDBOX,\ +SYFT_JOB_SANDBOX_UID,\ +SYFT_JOB_SANDBOX_GID" LABEL "tee.launch_policy.log_redirect"="always" diff --git a/packages/syft-job/src/syft_job/job_runner.py b/packages/syft-job/src/syft_job/job_runner.py index 471eca5d811..fcf3dacb5ad 100644 --- a/packages/syft-job/src/syft_job/job_runner.py +++ b/packages/syft-job/src/syft_job/job_runner.py @@ -1,6 +1,7 @@ import os import shutil import subprocess +import sys import time from datetime import datetime, timezone from pathlib import Path @@ -31,6 +32,300 @@ def get_job_timeout_seconds() -> int: IS_IN_JOB_ENV_VAR = "SYFT_IS_IN_JOB" +# Sandbox mode for job execution. Job code is untrusted, but this runner also +# runs on data owners' own machines, where sandboxing is neither expected nor +# always possible -- hence "off" by default. The enclave opts in explicitly. +# +# off -- execute the job directly (previous behaviour) +# on -- sandbox when supported, warn and continue when not +# require -- sandbox, or refuse to run the job at all +SANDBOX_ENV_VAR = "SYFT_JOB_SANDBOX" +SANDBOX_UID_ENV_VAR = "SYFT_JOB_SANDBOX_UID" +SANDBOX_GID_ENV_VAR = "SYFT_JOB_SANDBOX_GID" +_SANDBOX_MODES = ("off", "on", "require") + +# Environment handed to a sandboxed job. The unsandboxed path still inherits the +# full environment; under the sandbox we pass only what a job legitimately needs, +# so bootstrap secrets in the runner's environment are not exposed to job code. +_SANDBOX_ENV_ALLOWLIST = ( + "PATH", + "HOME", + "LANG", + "LC_ALL", + "TZ", + "TMPDIR", + "LD_LIBRARY_PATH", # GPU deploys need the NVIDIA driver libs + "CUDA_VISIBLE_DEVICES", + "NVIDIA_VISIBLE_DEVICES", + "UV_SYSTEM_PYTHON", + "UV_CACHE_DIR", +) + + +class SandboxUnavailableError(RuntimeError): + """Sandbox required by configuration but not applicable here.""" + + +def get_sandbox_mode() -> str: + """Job sandbox mode from the environment. Defaults to ``off``.""" + mode = os.environ.get(SANDBOX_ENV_VAR, "off").strip().lower() + if mode not in _SANDBOX_MODES: + raise ValueError( + f"{SANDBOX_ENV_VAR} must be one of {_SANDBOX_MODES}, got {mode!r}" + ) + return mode + + +def _sandbox_ids() -> tuple[int, int]: + from .sandbox import DEFAULT_GID, DEFAULT_UID + + return ( + int(os.environ.get(SANDBOX_UID_ENV_VAR, DEFAULT_UID)), + int(os.environ.get(SANDBOX_GID_ENV_VAR, DEFAULT_GID)), + ) + + +def _wrap_in_sandbox(command: List[str]) -> List[str]: + """Prefix ``command`` with the sandbox wrapper. + + The wrapper is invoked by file path rather than as ``-m syft_job.sandbox`` + so that installing the lockdown does not depend on the ``syft_job`` package + (and its dependencies) importing successfully inside the job environment. + """ + from . import sandbox as _sandbox + + uid, gid = _sandbox_ids() + # "on" is explicitly best-effort; "require" must refuse a partial lockdown + # (e.g. running as a non-root user, where privileges cannot be dropped). + best_effort = ["--best-effort"] if get_sandbox_mode() == "on" else [] + return [ + sys.executable, + os.fspath(Path(_sandbox.__file__).resolve()), + "--uid", + str(uid), + "--gid", + str(gid), + *best_effort, + "--", + *command, + ] + + +def _sandbox_available_or_raise(mode: str) -> bool: + """Whether to sandbox. Raises in ``require`` mode if we cannot.""" + from . import sandbox as _sandbox + + supported, reason = _sandbox.is_supported() + if supported: + return True + if mode == "require": + raise SandboxUnavailableError( + f"{SANDBOX_ENV_VAR}=require but the sandbox cannot be applied: {reason}" + ) + print( + f" WARNING: {SANDBOX_ENV_VAR}=on but sandbox unavailable ({reason}); " + f"running job WITHOUT network isolation" + ) + return False + + +def build_job_command(run_script: Path) -> List[str]: + """Command used to launch a job, wrapped in the sandbox when enabled. + + This sandboxes ``run.sh`` wholesale. It is correct only for submissions + whose script does no dependency installation -- installers need local + sockets, which the sandbox denies. Python jobs go through + :func:`build_two_phase_command` instead. + """ + direct = ["bash", str(run_script)] + mode = get_sandbox_mode() + if mode == "off" or not _sandbox_available_or_raise(mode): + return direct + return _wrap_in_sandbox(direct) + + +# Python version used for the job virtualenv. Mirrors +# ``syft_job.client.RUN_SCRIPT_PYTHON_VERSION``, which generates the equivalent +# run.sh for the unsandboxed path. +JOB_PYTHON_VERSION = "3.12" + + +class PhaseAError(RuntimeError): + """Dependency installation failed before the job could be sandboxed.""" + + +def prepare_sandbox_filesystem( + submission_dir: Path, syftbox_folder: Path, uid: int, gid: int +) -> None: + """Make the job tree writable, and the datasite readable, by the sandbox user. + + Without this a sandboxed job fails immediately: it cannot create its + virtualenv or write ``outputs/``, and it cannot read the datasets it was + approved for. Both are needed for the job to do anything useful, and neither + weakens the lockdown -- the Drive credential and the runner's own code stay + root-owned and out of reach. + + A no-op when not running as root, which is the case in tests and on a data + owner's own machine, where the job already runs as the invoking user. + """ + if os.geteuid() != 0: + return + + # The job owns its own working tree: venv, code, outputs. + os.chown(submission_dir, uid, gid) + for root, dirs, files in os.walk(submission_dir): + for name in dirs + files: + path = os.path.join(root, name) + if not os.path.islink(path): + os.chown(path, uid, gid) + + # The datasite is read-only to the job. Ownership stays with root; we only + # open traversal and read. Every parent must be traversable or the job + # cannot reach the datasets at all -- the folder lives under /root (0700) + # by default, since the container runs as root and that is its home. + for parent in list(syftbox_folder.parents)[:-1]: + try: + os.chmod(parent, os.stat(parent).st_mode | 0o011) + except OSError: + pass + for root, dirs, files in os.walk(syftbox_folder): + for name in [root] + [os.path.join(root, n) for n in dirs + files]: + try: + mode = os.stat(name).st_mode + os.chmod(name, mode | (0o011 if os.path.isdir(name) else 0o004)) + except OSError: + pass + + +def _is_submitter_code(spec: str) -> bool: + """Whether a dependency string would fetch and build submitter-chosen code. + + Local paths and VCS URLs execute their own build scripts on install, so + under sandboxing they are refused rather than run: phase A has the network + and runs unsandboxed, which is precisely the position an attacker wants. + """ + s = spec.strip().lower() + if any(s.startswith(p) for p in ("git+", "hg+", "svn+", "bzr+")): + return True + if s.startswith((".", "/", "file://")): + return True + return " @ " in s or s.startswith("-e ") + + +def install_dependencies( + submission_dir: Path, dependencies: List[str], timeout: int = 900 +) -> Path: + """Phase A: build the job's virtualenv with the network available. + + Runs *before* the sandbox is applied, so it must not execute code the + submitter chose. Two rules enforce that: + + * ``syft-client`` is installed from *this* runner's own install source -- + part of the attested enclave image -- not from whatever the submission + declared. It may be a local path, so it is installed without the + wheels-only restriction. + * every submitter-declared dependency is installed ``--only-binary=:all:``, + because building a source distribution runs its build backend. Declared + dependencies that are local paths or VCS URLs are refused outright. + + Returns the interpreter to use for phase B. + """ + from .install_source import get_syft_client_install_source + + code_dir = submission_dir / "code" + venv_dir = code_dir / ".venv" + venv_python = venv_dir / "bin" / "python" + + rejected = [d for d in dependencies if _is_submitter_code(d)] + declared = [d for d in dependencies if not _is_submitter_code(d)] + if rejected: + print( + f" Sandbox: ignoring {len(rejected)} dependency spec(s) that would build " + f"submitter-supplied code: {', '.join(rejected)}" + ) + + steps: List[List[str]] = [ + ["uv", "venv", "--python", JOB_PYTHON_VERSION, str(venv_dir)], + # Trusted: comes from the enclave image, not the submission. + [ + "uv", + "pip", + "install", + "--python", + str(venv_python), + get_syft_client_install_source(), + ], + ] + if declared: + steps.append( + [ + "uv", + "pip", + "install", + "--python", + str(venv_python), + "--only-binary=:all:", + *declared, + ] + ) + + for step in steps: + result = subprocess.run( + step, cwd=code_dir, capture_output=True, text=True, timeout=timeout + ) + if result.returncode != 0: + raise PhaseAError( + f"{' '.join(step[:3])} failed ({result.returncode}):\n" + f"{result.stderr.strip()[-2000:]}" + ) + + return venv_python + + +def build_two_phase_command( + submission_dir: Path, + metadata: JobSubmissionMetadata, + run_script: Path, + timeout: int = 900, +) -> List[str]: + """Install dependencies unsandboxed, then return a sandboxed run command. + + The submitted ``run.sh`` is deliberately not executed for python jobs when + sandboxing: it interleaves installation and execution in one script, so it + cannot be split, and its contents are chosen by the submitter. The two + phases are rebuilt from the declared ``entrypoint`` and ``dependencies`` + instead. + """ + mode = get_sandbox_mode() + if mode == "off" or not _sandbox_available_or_raise(mode): + return ["bash", str(run_script)] + + # Only python submissions carry the metadata needed to split the phases. + if metadata.type != "python" or not metadata.entrypoint: + return _wrap_in_sandbox(["bash", str(run_script)]) + + python = install_dependencies( + submission_dir, list(metadata.dependencies or []), timeout=timeout + ) + return _wrap_in_sandbox( + ["bash", "-c", f'cd code && exec "$0" "$1"', str(python), metadata.entrypoint] + ) + + +def build_job_env(config_folder: str, email: str) -> dict: + """Environment for a job subprocess.""" + if get_sandbox_mode() == "off": + env = os.environ.copy() + else: + env = { + k: v for k, v in os.environ.items() if k in _SANDBOX_ENV_ALLOWLIST + } + env["SYFTBOX_FOLDER"] = config_folder + env["SYFTBOX_EMAIL"] = email + env[IS_IN_JOB_ENV_VAR] = "true" + env["PYTHONUNBUFFERED"] = "1" + return env + def _kill_process_tree(pid: int, timeout: float = 2.0) -> None: """Kill `pid` and every descendant. Cross-platform via psutil.""" @@ -218,6 +513,25 @@ def _find_jobref_from_name(self, job_name: str, user: str | None = None) -> JobR self.config.current_user_email, job_name, ds_email=user ) + def _build_command( + self, ref: JobRef, submission_dir: Path, run_script: Path, timeout: int + ) -> List[str]: + """Command to launch this job, sandboxed and phase-split when enabled.""" + if get_sandbox_mode() == "off": + return build_job_command(run_script) + metadata = self._get_job_metadata(ref) + if metadata is None: + return build_job_command(run_script) + command = build_two_phase_command( + submission_dir, metadata, run_script, timeout=timeout + ) + # After phase A, so the venv it created is handed over too. + uid, gid = _sandbox_ids() + prepare_sandbox_filesystem( + submission_dir, Path(self.config.syftbox_folder_path_str), uid, gid + ) + return command + def _execute_job_streaming(self, ref: JobRef, timeout: int) -> int: """Execute job with real-time streaming output. @@ -235,11 +549,10 @@ def _execute_job_streaming(self, ref: JobRef, timeout: int) -> int: os.chmod(run_script, 0o755) # Prepare environment variables - env = os.environ.copy() - env["SYFTBOX_FOLDER"] = self.config.syftbox_folder_path_str - env["SYFTBOX_EMAIL"] = self.config.current_user_email - env[IS_IN_JOB_ENV_VAR] = "true" - env["PYTHONUNBUFFERED"] = "1" + env = build_job_env( + self.config.syftbox_folder_path_str, self.config.current_user_email + ) + command = self._build_command(ref, submission_dir, run_script, timeout) # stdout/stderr go to review/ stdout_file = review_dir / "stdout.txt" @@ -252,7 +565,7 @@ def _execute_job_streaming(self, ref: JobRef, timeout: int) -> int: open(stderr_file, "w") as stderr_f, ): process = subprocess.Popen( - ["bash", str(run_script)], + command, cwd=submission_dir, # run.sh executes from inbox/ where code/ lives stdout=subprocess.PIPE, stderr=subprocess.PIPE, @@ -322,14 +635,12 @@ def _execute_job_captured(self, ref: JobRef, timeout: int) -> int: os.chmod(run_script, 0o755) # Prepare environment variables - env = os.environ.copy() - env["SYFTBOX_FOLDER"] = self.config.syftbox_folder_path_str - env["SYFTBOX_EMAIL"] = self.config.current_user_email - env[IS_IN_JOB_ENV_VAR] = "true" - env["PYTHONUNBUFFERED"] = "1" + env = build_job_env( + self.config.syftbox_folder_path_str, self.config.current_user_email + ) process = subprocess.Popen( - ["bash", str(run_script)], + self._build_command(ref, submission_dir, run_script, timeout), cwd=submission_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE, diff --git a/packages/syft-job/src/syft_job/sandbox.py b/packages/syft-job/src/syft_job/sandbox.py new file mode 100644 index 00000000000..a80d313d632 --- /dev/null +++ b/packages/syft-job/src/syft_job/sandbox.py @@ -0,0 +1,310 @@ +"""Kernel-enforced lockdown for untrusted job code. + +Run as ``python -m syft_job.sandbox -- [args...]``. The module drops +privileges, installs a seccomp filter that denies socket creation, and then +``execve``s the command. Both restrictions are one-way: the kernel offers no +operation to remove a seccomp filter or to undo ``PR_SET_NO_NEW_PRIVS``, so the +code being launched cannot lift them. + +Because the filter is enforced by the kernel on syscall entry and survives +``execve``, it binds compiled binaries -- C/C++/CUDA -- exactly as it binds +Python. Blocking socket *creation* rather than ``connect`` is deliberate: a +seccomp filter cannot dereference pointers, so it cannot inspect the address +passed to ``connect``, but ``socket``'s arguments are plain integers. Denying +every address family (not just AF_INET) also prevents the process from +receiving an already-connected socket over a unix socket via SCM_RIGHTS. + +``execvp`` is the last statement, so any failure above it means the command is +never executed -- the sandbox fails closed by construction. + +Linux/x86-64 only. ``is_supported()`` reports availability so callers can +degrade or refuse deliberately rather than silently running unprotected. +""" + +from __future__ import annotations + +import ctypes +import errno +import os +import platform +import sys + +__all__ = [ + "is_supported", + "apply_lockdown", + "main", + "SandboxError", + "REFUSED_EXIT_CODE", + "SENTINEL", +] + +# Exit code used when the sandbox refuses to run the command. Distinct from any +# plausible exit code of the job itself so the caller can tell "we refused" from +# "the job failed". +REFUSED_EXIT_CODE = 93 + +# Printed to stderr on refusal; callers grep for this. +SENTINEL = "SYFT_SANDBOX_REFUSED" + +# Default unprivileged account. Overridable for tests and for images that ship a +# dedicated job user. +DEFAULT_UID = 65534 +DEFAULT_GID = 65534 + +# prctl(2) options. +_PR_SET_NO_NEW_PRIVS = 38 +_PR_SET_SECCOMP = 22 +_SECCOMP_MODE_FILTER = 2 + +# Classic BPF opcodes used by the filter. +_BPF_LD = 0x00 +_BPF_W = 0x00 +_BPF_ABS = 0x20 +_BPF_JMP = 0x05 +_BPF_JEQ = 0x10 +_BPF_K = 0x00 +_BPF_RET = 0x06 + +_AUDIT_ARCH_X86_64 = 0xC000003E + +_SECCOMP_RET_ALLOW = 0x7FFF0000 +_SECCOMP_RET_ERRNO = 0x00050000 + +# Offsets into struct seccomp_data. +_OFF_NR = 0 +_OFF_ARCH = 4 + +# x86-64 syscall numbers denied by the filter. +# +# socket/socketpair are the only ways to obtain a socket descriptor, so denying +# them denies networking outright. io_uring can perform network I/O through its +# submission queue without issuing the corresponding syscalls, which a classic +# seccomp filter cannot observe, so it must be denied too. The ptrace family is +# denied to stop the job inspecting or modifying other processes. +_DENIED_SYSCALLS = ( + 41, # socket + 53, # socketpair + 425, # io_uring_setup + 426, # io_uring_enter + 101, # ptrace + 310, # process_vm_readv + 311, # process_vm_writev +) + + +class SandboxError(RuntimeError): + """The lockdown could not be applied.""" + + +class _SockFilter(ctypes.Structure): + """struct sock_filter -- one classic-BPF instruction.""" + + _fields_ = [ + ("code", ctypes.c_ushort), + ("jt", ctypes.c_ubyte), + ("jf", ctypes.c_ubyte), + ("k", ctypes.c_uint), + ] + + +class _SockFprog(ctypes.Structure): + """struct sock_fprog -- the program handed to the kernel.""" + + _fields_ = [ + ("len", ctypes.c_ushort), + ("filter", ctypes.POINTER(_SockFilter)), + ] + + +def _libc() -> ctypes.CDLL: + libc = ctypes.CDLL("libc.so.6", use_errno=True) + # Without explicit argtypes ctypes passes the struct pointer as a 32-bit int + # on x86-64 and the kernel reads a truncated address. + libc.prctl.restype = ctypes.c_int + libc.prctl.argtypes = [ + ctypes.c_int, + ctypes.c_ulong, + ctypes.c_void_p, + ctypes.c_ulong, + ctypes.c_ulong, + ] + return libc + + +def _build_filter() -> tuple[_SockFprog, ctypes.Array]: + """Assemble the BPF program. + + Returns the program and the instruction array. The caller must keep the + array alive until prctl returns: ``_SockFprog`` holds a raw pointer that the + garbage collector does not treat as a reference. + """ + prog: list[_SockFilter] = [] + + prog.append(_SockFilter(_BPF_LD | _BPF_W | _BPF_ABS, 0, 0, _OFF_ARCH)) + arch_at = len(prog) + prog.append(_SockFilter(_BPF_JMP | _BPF_JEQ | _BPF_K, 0, 0, _AUDIT_ARCH_X86_64)) + + prog.append(_SockFilter(_BPF_LD | _BPF_W | _BPF_ABS, 0, 0, _OFF_NR)) + tests_at: list[int] = [] + for nr in _DENIED_SYSCALLS: + tests_at.append(len(prog)) + prog.append(_SockFilter(_BPF_JMP | _BPF_JEQ | _BPF_K, 0, 0, nr)) + + allow_at = len(prog) + prog.append(_SockFilter(_BPF_RET | _BPF_K, 0, 0, _SECCOMP_RET_ALLOW)) + deny_at = len(prog) + prog.append( + _SockFilter(_BPF_RET | _BPF_K, 0, 0, _SECCOMP_RET_ERRNO | errno.EPERM) + ) + + # Jump targets are offsets relative to the instruction *after* the jump, not + # absolute indices. Getting this wrong routes ordinary syscalls into the + # deny branch, which crashes the process immediately. + prog[arch_at].jt = 0 + prog[arch_at].jf = deny_at - arch_at - 1 + for i in tests_at: + prog[i].jt = deny_at - i - 1 + prog[i].jf = 0 + + if allow_at != deny_at - 1: + raise SandboxError("filter assembly produced an unexpected layout") + + instructions = (_SockFilter * len(prog))(*prog) + return _SockFprog(len(prog), instructions), instructions + + +def is_supported() -> tuple[bool, str]: + """Whether the lockdown can be applied here. + + Returns ``(supported, reason)``; ``reason`` is empty when supported. + """ + if sys.platform != "linux": + return False, f"requires Linux, running on {sys.platform}" + if platform.machine() not in ("x86_64", "AMD64"): + return False, f"filter is x86-64 only, running on {platform.machine()}" + try: + _libc() + except OSError as exc: + return False, f"cannot load libc: {exc}" + return True, "" + + +def apply_lockdown( + uid: int = DEFAULT_UID, gid: int = DEFAULT_GID, strict: bool = True +) -> None: + """Drop privileges and install the seccomp filter, in that order. + + Raises ``SandboxError`` if any step fails. On return the calling process + cannot create sockets, cannot regain privilege, and cannot undo either. + + The ordering is enforced by the kernel, not chosen for style: + + * ``setgid`` must precede ``setuid`` -- after dropping the user id the + process is no longer permitted to change its group, so a reversed order + leaves the job in the root group. + * ``PR_SET_NO_NEW_PRIVS`` must precede ``PR_SET_SECCOMP`` -- installing a + filter otherwise requires CAP_SYS_ADMIN, which the enclave container does + not have. + + Only root can change user id, so when invoked as an ordinary user the drop + is impossible. With ``strict`` (the default) that is an error rather than a + silent half-application: the seccomp filter alone still blocks the network, + but it leaves the job able to read credentials and modify the runner's own + files, which is the more valuable half of the protection. Callers that + genuinely want best-effort behaviour must ask for it explicitly. + """ + supported, reason = is_supported() + if not supported: + raise SandboxError(reason) + + libc = _libc() + + if os.getuid() == 0: + try: + os.setgroups([]) + os.setgid(gid) + os.setuid(uid) + except OSError as exc: + raise SandboxError(f"privilege drop failed: {exc}") from exc + if os.getuid() != uid or os.geteuid() != uid: + raise SandboxError("privilege drop did not take effect") + elif os.getuid() != uid: + message = ( + f"cannot drop to uid {uid}: running as uid {os.getuid()}, not root. " + f"The seccomp filter would still apply, but the job would keep this " + f"user's file access" + ) + if strict: + raise SandboxError(message) + print(f"{SENTINEL}-WARNING: {message}", file=sys.stderr, flush=True) + + if libc.prctl(_PR_SET_NO_NEW_PRIVS, 1, None, 0, 0) != 0: + raise SandboxError( + f"PR_SET_NO_NEW_PRIVS failed: {os.strerror(ctypes.get_errno())}" + ) + + prog, _instructions = _build_filter() + rc = libc.prctl( + _PR_SET_SECCOMP, + _SECCOMP_MODE_FILTER, + ctypes.cast(ctypes.byref(prog), ctypes.c_void_p), + 0, + 0, + ) + if rc != 0: + raise SandboxError( + f"PR_SET_SECCOMP failed: {os.strerror(ctypes.get_errno())}" + ) + + +def _refuse(reason: str) -> "None": + print(f"{SENTINEL}: {reason}", file=sys.stderr, flush=True) + # os._exit avoids running atexit handlers or flushing inherited buffers in + # a process that is midway through dropping privileges. + os._exit(REFUSED_EXIT_CODE) + + +def main(argv: list[str] | None = None) -> None: + """Entry point: apply the lockdown, then become the requested command.""" + args = list(sys.argv[1:] if argv is None else argv) + + uid, gid, strict = DEFAULT_UID, DEFAULT_GID, True + while args and args[0].startswith("--"): + flag = args.pop(0) + if flag == "--": + break + if flag == "--best-effort": + # Apply what is possible here rather than refusing. Only for callers + # that accept a partial lockdown; never use where the guarantee is + # being relied upon. + strict = False + elif flag in ("--uid", "--gid"): + if not args: + _refuse(f"{flag} requires a value") + try: + value = int(args.pop(0)) + except ValueError: + _refuse(f"{flag} requires an integer") + if flag == "--uid": + uid = value + else: + gid = value + else: + _refuse(f"unknown option {flag}") + + if not args: + _refuse("no command given") + + try: + apply_lockdown(uid=uid, gid=gid, strict=strict) + except SandboxError as exc: + _refuse(str(exc)) + + try: + os.execvp(args[0], args) + except OSError as exc: + _refuse(f"exec {args[0]!r} failed: {exc}") + + +if __name__ == "__main__": + main() diff --git a/packages/syft-job/tests/test_sandbox.py b/packages/syft-job/tests/test_sandbox.py new file mode 100644 index 00000000000..c3adf7ad4b5 --- /dev/null +++ b/packages/syft-job/tests/test_sandbox.py @@ -0,0 +1,423 @@ +"""Tests for the job sandbox. + +The behavioural tests shell out to sandbox.py and inspect what the sandboxed +process can and cannot do. They deliberately assert on the *kernel's* behaviour +rather than on our code paths -- the guarantee is only worth what the kernel +actually enforces. +""" + +from __future__ import annotations + +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +from syft_job import job_runner, sandbox + +SANDBOX_PY = str(Path(sandbox.__file__).resolve()) +LINUX_X86 = sys.platform == "linux" and os.uname().machine == "x86_64" +requires_linux_x86 = pytest.mark.skipif( + not LINUX_X86, reason="sandbox is Linux/x86-64 only" +) + + +def _run_sandboxed(snippet: str, extra_args: list[str] | None = None): + """Run a Python snippet under the sandbox, as the current user.""" + args = [ + sys.executable, + SANDBOX_PY, + "--uid", + str(os.getuid()), + "--gid", + str(os.getgid()), + *(extra_args or []), + "--", + sys.executable, + "-c", + snippet, + ] + return subprocess.run(args, capture_output=True, text=True, timeout=60) + + +# -------------------------------------------------------------------------- +# configuration / wiring +# -------------------------------------------------------------------------- + + +def test_sandbox_defaults_to_off(monkeypatch): + """Must default off: this runner also runs on data owners' own machines.""" + monkeypatch.delenv(job_runner.SANDBOX_ENV_VAR, raising=False) + assert job_runner.get_sandbox_mode() == "off" + + +def test_invalid_mode_rejected(monkeypatch): + monkeypatch.setenv(job_runner.SANDBOX_ENV_VAR, "sort-of") + with pytest.raises(ValueError): + job_runner.get_sandbox_mode() + + +def test_command_unwrapped_when_off(monkeypatch): + monkeypatch.setenv(job_runner.SANDBOX_ENV_VAR, "off") + assert job_runner.build_job_command(Path("/tmp/run.sh")) == [ + "bash", + "/tmp/run.sh", + ] + + +@requires_linux_x86 +@pytest.mark.parametrize("mode", ["on", "require"]) +def test_command_wrapped_when_enabled(monkeypatch, mode): + monkeypatch.setenv(job_runner.SANDBOX_ENV_VAR, mode) + cmd = job_runner.build_job_command(Path("/tmp/run.sh")) + assert cmd[0] == sys.executable + assert cmd[1] == SANDBOX_PY + # the job itself is still the tail of the command + assert cmd[-2:] == ["bash", "/tmp/run.sh"] + + +def test_require_raises_when_unsupported(monkeypatch): + monkeypatch.setenv(job_runner.SANDBOX_ENV_VAR, "require") + monkeypatch.setattr(sandbox, "is_supported", lambda: (False, "no kernel support")) + with pytest.raises(job_runner.SandboxUnavailableError): + job_runner.build_job_command(Path("/tmp/run.sh")) + + +def test_on_degrades_when_unsupported(monkeypatch, capsys): + monkeypatch.setenv(job_runner.SANDBOX_ENV_VAR, "on") + monkeypatch.setattr(sandbox, "is_supported", lambda: (False, "no kernel support")) + assert job_runner.build_job_command(Path("/tmp/run.sh")) == [ + "bash", + "/tmp/run.sh", + ] + assert "WITHOUT network isolation" in capsys.readouterr().out + + +def test_env_allowlisted_when_sandboxed(monkeypatch): + monkeypatch.setenv(job_runner.SANDBOX_ENV_VAR, "on") + monkeypatch.setenv("SYFT_BOOTSTRAP_SA_SECRET", "super-secret") + monkeypatch.setenv("SYFT_ENCLAVE_TOKEN_CONTENT", "oauth-token") + env = job_runner.build_job_env("/syftbox", "enclave@example.org") + assert "SYFT_BOOTSTRAP_SA_SECRET" not in env + assert "SYFT_ENCLAVE_TOKEN_CONTENT" not in env + assert env["SYFTBOX_FOLDER"] == "/syftbox" + assert env[job_runner.IS_IN_JOB_ENV_VAR] == "true" + + +def test_env_inherited_when_off(monkeypatch): + monkeypatch.setenv(job_runner.SANDBOX_ENV_VAR, "off") + monkeypatch.setenv("SOME_UNRELATED_VAR", "kept") + env = job_runner.build_job_env("/syftbox", "a@b.c") + assert env["SOME_UNRELATED_VAR"] == "kept" + + +# -------------------------------------------------------------------------- +# what the kernel actually enforces +# -------------------------------------------------------------------------- + + +@requires_linux_x86 +def test_supported_on_this_platform(): + supported, reason = sandbox.is_supported() + assert supported, reason + + +@requires_linux_x86 +@pytest.mark.parametrize( + "family,name", [(2, "AF_INET"), (10, "AF_INET6"), (1, "AF_UNIX")] +) +def test_socket_creation_blocked(family, name): + """Every address family, not just the internet ones. + + AF_UNIX matters: a socket's network namespace is fixed at creation, so a + process that can open a unix socket can be handed an already-connected + network socket over SCM_RIGHTS by a co-resident helper. + """ + proc = _run_sandboxed( + f"import socket\n" + f"try:\n" + f" socket.socket({family}, socket.SOCK_STREAM)\n" + f" print('ALLOWED')\n" + f"except OSError as e:\n" + f" print('BLOCKED')\n" + ) + assert proc.stdout.strip() == "BLOCKED", f"{name} was not blocked: {proc.stdout}" + + +@requires_linux_x86 +def test_restriction_survives_exec(): + """The guarantee is about compiled binaries, so it must outlive execve.""" + inner = ( + "import socket\n" + "try:\n" + " socket.socket(2, 1); print('ALLOWED')\n" + "except OSError: print('BLOCKED')\n" + ) + proc = _run_sandboxed( + "import subprocess, sys\n" + f"r = subprocess.run([sys.executable, '-c', {inner!r}], " + "capture_output=True, text=True)\n" + "print(r.stdout.strip())\n" + ) + assert proc.stdout.strip() == "BLOCKED" + + +@requires_linux_x86 +def test_dns_resolution_blocked(): + proc = _run_sandboxed( + "import socket\n" + "try:\n" + " socket.getaddrinfo('example.com', 80); print('RESOLVED')\n" + "except Exception: print('BLOCKED')\n" + ) + assert proc.stdout.strip() == "BLOCKED" + + +@requires_linux_x86 +def test_ordinary_work_unaffected(): + """A lockdown that breaks normal jobs is not shippable.""" + proc = _run_sandboxed( + "import os, tempfile\n" + "total = sum(i * i for i in range(10000))\n" + "with tempfile.TemporaryDirectory() as d:\n" + " p = os.path.join(d, 'out.txt')\n" + " open(p, 'w').write('result')\n" + " assert open(p).read() == 'result'\n" + "print('OK', total)\n" + ) + assert proc.returncode == 0, proc.stderr + assert proc.stdout.startswith("OK 333283335000") + + +@requires_linux_x86 +def test_no_new_privs_is_set(): + proc = _run_sandboxed( + "print([l for l in open('/proc/self/status') if 'NoNewPrivs' in l][0].strip())" + ) + assert proc.stdout.strip().endswith("1") + + +# -------------------------------------------------------------------------- +# fail-closed +# -------------------------------------------------------------------------- + + +@requires_linux_x86 +def test_refuses_unknown_option_without_running_command(): + proc = subprocess.run( + [sys.executable, SANDBOX_PY, "--nope", "--", "echo", "SHOULD_NOT_RUN"], + capture_output=True, + text=True, + timeout=30, + ) + assert proc.returncode == sandbox.REFUSED_EXIT_CODE + assert sandbox.SENTINEL in proc.stderr + assert "SHOULD_NOT_RUN" not in proc.stdout + + +@requires_linux_x86 +def test_refuses_when_no_command_given(): + proc = subprocess.run( + [sys.executable, SANDBOX_PY, "--"], capture_output=True, text=True, timeout=30 + ) + assert proc.returncode == sandbox.REFUSED_EXIT_CODE + + +@requires_linux_x86 +def test_refuses_when_exec_target_missing(): + proc = _run_sandboxed_missing = subprocess.run( + [ + sys.executable, + SANDBOX_PY, + "--uid", + str(os.getuid()), + "--gid", + str(os.getgid()), + "--", + "/nonexistent/binary", + ], + capture_output=True, + text=True, + timeout=30, + ) + assert proc.returncode == sandbox.REFUSED_EXIT_CODE + assert sandbox.SENTINEL in proc.stderr + + +# -------------------------------------------------------------------------- +# ordering guards -- these encode kernel requirements, not style preferences +# -------------------------------------------------------------------------- + + +@requires_linux_x86 +def test_seccomp_requires_no_new_privs_first(): + """Without CAP_SYS_ADMIN the kernel refuses a filter unless no_new_privs is + already set. If this ever stops being true the ordering in apply_lockdown + is no longer load-bearing and the comment there should be revisited.""" + proc = subprocess.run( + [ + sys.executable, + "-c", + "import ctypes, sys\n" + "sys.path.insert(0, %r)\n" % str(Path(sandbox.__file__).parent.parent) + + "from syft_job.sandbox import _build_filter, _libc, " + "_PR_SET_SECCOMP, _SECCOMP_MODE_FILTER\n" + "libc = _libc()\n" + "prog, _keep = _build_filter()\n" + "rc = libc.prctl(_PR_SET_SECCOMP, _SECCOMP_MODE_FILTER, " + "ctypes.cast(ctypes.byref(prog), ctypes.c_void_p), 0, 0)\n" + "print('rc', rc)\n", + ], + capture_output=True, + text=True, + timeout=30, + ) + # Non-root without CAP_SYS_ADMIN: must fail. If running as root with the + # capability the call may succeed, so only assert the failure when we are + # genuinely unprivileged. + if os.getuid() != 0: + assert proc.stdout.strip() == "rc -1", proc.stdout + proc.stderr + + +def test_setgid_must_precede_setuid_is_documented(): + """Guards against a refactor silently reordering the privilege drop.""" + src = Path(sandbox.__file__).read_text() + drop = src[src.index("os.setgroups([])") : src.index("if os.getuid() != uid")] + assert drop.index("os.setgroups") < drop.index("os.setgid") < drop.index( + "os.setuid" + ), "privilege drop order changed: setgroups -> setgid -> setuid is required" + + +# -------------------------------------------------------------------------- +# integration: the lockdown must actually engage through the real runner +# -------------------------------------------------------------------------- + +NETWORK_PROBE_MAIN_PY = """\ +import os, socket + +os.makedirs("outputs", exist_ok=True) +try: + socket.socket(socket.AF_INET, socket.SOCK_STREAM) + verdict = "NETWORK_ALLOWED" +except OSError: + verdict = "NETWORK_BLOCKED" +with open("outputs/verdict.txt", "w") as f: + f.write(verdict) +print(verdict) +""" + + +def _run_probe_job(tmp_path: Path) -> str: + """Submit and run a job that reports whether it can make a socket.""" + from syft_job.client import JobClient + from syft_job.config import SyftJobConfig + from syft_job.job_runner import SyftJobRunner + + do_email, ds_email = "do@test.org", "ds@test.org" + syftbox = tmp_path / "SyftBox" + syftbox.mkdir() + code_file = tmp_path / "main.py" + code_file.write_text(NETWORK_PROBE_MAIN_PY) + + do_config = SyftJobConfig(syftbox_folder=syftbox, current_user_email=do_email) + ds_config = SyftJobConfig(syftbox_folder=syftbox, current_user_email=ds_email) + JobClient(config=ds_config).submit_python_job( + user=do_email, code_path=str(code_file), job_name="probe.job" + ) + do_client = JobClient(config=do_config) + do_client.jobs[0].approve() + SyftJobRunner(config=do_config).process_approved_jobs( + stream_output=False, timeout=900 + ) + + review = do_config.get_review_job_dir(do_email, ds_email, "probe.job") + verdict_file = review / "outputs" / "verdict.txt" + if not verdict_file.exists(): + stderr = (review / "stderr.txt").read_text() if ( + review / "stderr.txt" + ).exists() else "" + raise AssertionError(f"job produced no verdict; stderr:\n{stderr[-3000:]}") + return verdict_file.read_text().strip() + + +@requires_linux_x86 +@pytest.mark.slow +def test_job_has_network_without_sandbox(tmp_path, monkeypatch): + """Control: confirms the probe would otherwise succeed.""" + monkeypatch.setenv(job_runner.SANDBOX_ENV_VAR, "off") + assert _run_probe_job(tmp_path) == "NETWORK_ALLOWED" + + +@requires_linux_x86 +@pytest.mark.slow +def test_job_network_blocked_with_sandbox(tmp_path, monkeypatch): + """The whole point: a real job, run through the real runner, cannot network.""" + monkeypatch.setenv(job_runner.SANDBOX_ENV_VAR, "require") + monkeypatch.setenv(job_runner.SANDBOX_UID_ENV_VAR, str(os.getuid())) + monkeypatch.setenv(job_runner.SANDBOX_GID_ENV_VAR, str(os.getgid())) + assert _run_probe_job(tmp_path) == "NETWORK_BLOCKED" + + +# -------------------------------------------------------------------------- +# partial lockdown must not pass silently +# -------------------------------------------------------------------------- + + +@requires_linux_x86 +@pytest.mark.skipif(os.getuid() == 0, reason="needs to run as a non-root user") +def test_refuses_when_privileges_cannot_be_dropped(): + """As a non-root user the uid drop is impossible. + + The seccomp half would still apply, but the job would keep this user's file + access -- so `require` must refuse rather than half-protect. + """ + proc = subprocess.run( + [sys.executable, SANDBOX_PY, "--uid", "65534", "--", "echo", "SHOULD_NOT_RUN"], + capture_output=True, + text=True, + timeout=30, + ) + assert proc.returncode == sandbox.REFUSED_EXIT_CODE + assert "cannot drop to uid 65534" in proc.stderr + assert "SHOULD_NOT_RUN" not in proc.stdout + + +@requires_linux_x86 +@pytest.mark.skipif(os.getuid() == 0, reason="needs to run as a non-root user") +def test_best_effort_proceeds_but_warns(): + proc = subprocess.run( + [ + sys.executable, + SANDBOX_PY, + "--uid", + "65534", + "--best-effort", + "--", + sys.executable, + "-c", + "import socket\n" + "try:\n" + " socket.socket(2, 1); print('ALLOWED')\n" + "except OSError: print('BLOCKED')\n", + ], + capture_output=True, + text=True, + timeout=60, + ) + assert proc.returncode == 0 + assert proc.stdout.strip() == "BLOCKED" # filter still applied + assert "WARNING" in proc.stderr # but the partial application is announced + + +@requires_linux_x86 +def test_require_mode_does_not_pass_best_effort(monkeypatch): + monkeypatch.setenv(job_runner.SANDBOX_ENV_VAR, "require") + assert "--best-effort" not in job_runner.build_job_command(Path("/tmp/run.sh")) + + +@requires_linux_x86 +def test_on_mode_passes_best_effort(monkeypatch): + monkeypatch.setenv(job_runner.SANDBOX_ENV_VAR, "on") + assert "--best-effort" in job_runner.build_job_command(Path("/tmp/run.sh"))