From 32a366f7131d23df6ecd368ad1df6cc83baa4619 Mon Sep 17 00:00:00 2001 From: bitsofsteve <11381249+bitsofsteve@users.noreply.github.com> Date: Mon, 3 Aug 2026 05:15:16 +0000 Subject: [PATCH 1/4] Add opt-in kernel sandbox for untrusted job code Job code currently runs as a plain subprocess: root, same container as the supervisor, same network, inheriting the full environment. syft-restrict can constrain Python but not the compiled C++/CUDA that some private code ships as, so there is no mechanism today that prevents such code making network requests. Add syft_job.sandbox, which drops privileges, sets no_new_privs and installs a seccomp filter denying socket creation, then execs the job. Both restrictions are one-way -- the kernel offers no operation to remove a seccomp filter -- so the code being launched cannot lift them, and because the filter is enforced on syscall entry and survives execve it binds compiled binaries, not just Python. Denying every address family rather than only AF_INET is deliberate: a socket's network namespace is fixed at creation, so a process that can open a unix socket can be handed an already-connected one over SCM_RIGHTS by a co-resident helper. io_uring is denied for the same reason -- it performs network I/O without the syscalls a classic filter would see. Wired into both Popen sites behind SYFT_JOB_SANDBOX (off/on/require), defaulting to off: this runner also executes jobs on data owners' own machines, where sandboxing is neither expected nor always possible. When enabled the job also receives an allowlisted environment rather than the runner's full one, which today leaks bootstrap secrets. Not yet usable in production: run.sh as generated performs venv creation and dependency installation before the entrypoint, and uv itself needs local sockets (its async runtime uses a UnixStream for signal handling), so sandboxing the whole script breaks the install. Splitting install from execution is a prerequisite and follows in the next commit. --- packages/syft-job/src/syft_job/job_runner.py | 128 +++++++- packages/syft-job/src/syft_job/sandbox.py | 284 ++++++++++++++++++ packages/syft-job/tests/test_sandbox.py | 290 +++++++++++++++++++ 3 files changed, 690 insertions(+), 12 deletions(-) create mode 100644 packages/syft-job/src/syft_job/sandbox.py create mode 100644 packages/syft-job/tests/test_sandbox.py diff --git a/packages/syft-job/src/syft_job/job_runner.py b/packages/syft-job/src/syft_job/job_runner.py index 471eca5d811..217b203ef98 100644 --- a/packages/syft-job/src/syft_job/job_runner.py +++ b/packages/syft-job/src/syft_job/job_runner.py @@ -1,6 +1,7 @@ import os import shutil import subprocess +import sys import time from datetime import datetime, timezone from pathlib import Path @@ -31,6 +32,112 @@ def get_job_timeout_seconds() -> int: IS_IN_JOB_ENV_VAR = "SYFT_IS_IN_JOB" +# Sandbox mode for job execution. Job code is untrusted, but this runner also +# runs on data owners' own machines, where sandboxing is neither expected nor +# always possible -- hence "off" by default. The enclave opts in explicitly. +# +# off -- execute the job directly (previous behaviour) +# on -- sandbox when supported, warn and continue when not +# require -- sandbox, or refuse to run the job at all +SANDBOX_ENV_VAR = "SYFT_JOB_SANDBOX" +SANDBOX_UID_ENV_VAR = "SYFT_JOB_SANDBOX_UID" +SANDBOX_GID_ENV_VAR = "SYFT_JOB_SANDBOX_GID" +_SANDBOX_MODES = ("off", "on", "require") + +# Environment handed to a sandboxed job. The unsandboxed path still inherits the +# full environment; under the sandbox we pass only what a job legitimately needs, +# so bootstrap secrets in the runner's environment are not exposed to job code. +_SANDBOX_ENV_ALLOWLIST = ( + "PATH", + "HOME", + "LANG", + "LC_ALL", + "TZ", + "TMPDIR", + "LD_LIBRARY_PATH", # GPU deploys need the NVIDIA driver libs + "CUDA_VISIBLE_DEVICES", + "NVIDIA_VISIBLE_DEVICES", + "UV_SYSTEM_PYTHON", + "UV_CACHE_DIR", +) + + +class SandboxUnavailableError(RuntimeError): + """Sandbox required by configuration but not applicable here.""" + + +def get_sandbox_mode() -> str: + """Job sandbox mode from the environment. Defaults to ``off``.""" + mode = os.environ.get(SANDBOX_ENV_VAR, "off").strip().lower() + if mode not in _SANDBOX_MODES: + raise ValueError( + f"{SANDBOX_ENV_VAR} must be one of {_SANDBOX_MODES}, got {mode!r}" + ) + return mode + + +def _sandbox_ids() -> tuple[int, int]: + from .sandbox import DEFAULT_GID, DEFAULT_UID + + return ( + int(os.environ.get(SANDBOX_UID_ENV_VAR, DEFAULT_UID)), + int(os.environ.get(SANDBOX_GID_ENV_VAR, DEFAULT_GID)), + ) + + +def build_job_command(run_script: Path) -> List[str]: + """Command used to launch a job, wrapped in the sandbox when enabled. + + The wrapper is invoked by file path rather than as ``-m syft_job.sandbox`` + so that installing the lockdown does not depend on the ``syft_job`` package + (and its dependencies) importing successfully inside the job environment. + """ + direct = ["bash", str(run_script)] + mode = get_sandbox_mode() + if mode == "off": + return direct + + from . import sandbox as _sandbox + + supported, reason = _sandbox.is_supported() + if not supported: + if mode == "require": + raise SandboxUnavailableError( + f"{SANDBOX_ENV_VAR}=require but the sandbox cannot be applied: {reason}" + ) + print( + f" WARNING: {SANDBOX_ENV_VAR}=on but sandbox unavailable ({reason}); " + f"running job WITHOUT network isolation" + ) + return direct + + uid, gid = _sandbox_ids() + return [ + sys.executable, + os.fspath(Path(_sandbox.__file__).resolve()), + "--uid", + str(uid), + "--gid", + str(gid), + "--", + *direct, + ] + + +def build_job_env(config_folder: str, email: str) -> dict: + """Environment for a job subprocess.""" + if get_sandbox_mode() == "off": + env = os.environ.copy() + else: + env = { + k: v for k, v in os.environ.items() if k in _SANDBOX_ENV_ALLOWLIST + } + env["SYFTBOX_FOLDER"] = config_folder + env["SYFTBOX_EMAIL"] = email + env[IS_IN_JOB_ENV_VAR] = "true" + env["PYTHONUNBUFFERED"] = "1" + return env + def _kill_process_tree(pid: int, timeout: float = 2.0) -> None: """Kill `pid` and every descendant. Cross-platform via psutil.""" @@ -235,11 +342,10 @@ def _execute_job_streaming(self, ref: JobRef, timeout: int) -> int: os.chmod(run_script, 0o755) # Prepare environment variables - env = os.environ.copy() - env["SYFTBOX_FOLDER"] = self.config.syftbox_folder_path_str - env["SYFTBOX_EMAIL"] = self.config.current_user_email - env[IS_IN_JOB_ENV_VAR] = "true" - env["PYTHONUNBUFFERED"] = "1" + env = build_job_env( + self.config.syftbox_folder_path_str, self.config.current_user_email + ) + command = build_job_command(run_script) # stdout/stderr go to review/ stdout_file = review_dir / "stdout.txt" @@ -252,7 +358,7 @@ def _execute_job_streaming(self, ref: JobRef, timeout: int) -> int: open(stderr_file, "w") as stderr_f, ): process = subprocess.Popen( - ["bash", str(run_script)], + command, cwd=submission_dir, # run.sh executes from inbox/ where code/ lives stdout=subprocess.PIPE, stderr=subprocess.PIPE, @@ -322,14 +428,12 @@ def _execute_job_captured(self, ref: JobRef, timeout: int) -> int: os.chmod(run_script, 0o755) # Prepare environment variables - env = os.environ.copy() - env["SYFTBOX_FOLDER"] = self.config.syftbox_folder_path_str - env["SYFTBOX_EMAIL"] = self.config.current_user_email - env[IS_IN_JOB_ENV_VAR] = "true" - env["PYTHONUNBUFFERED"] = "1" + env = build_job_env( + self.config.syftbox_folder_path_str, self.config.current_user_email + ) process = subprocess.Popen( - ["bash", str(run_script)], + build_job_command(run_script), cwd=submission_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE, diff --git a/packages/syft-job/src/syft_job/sandbox.py b/packages/syft-job/src/syft_job/sandbox.py new file mode 100644 index 00000000000..02fa1f7b0fd --- /dev/null +++ b/packages/syft-job/src/syft_job/sandbox.py @@ -0,0 +1,284 @@ +"""Kernel-enforced lockdown for untrusted job code. + +Run as ``python -m syft_job.sandbox -- [args...]``. The module drops +privileges, installs a seccomp filter that denies socket creation, and then +``execve``s the command. Both restrictions are one-way: the kernel offers no +operation to remove a seccomp filter or to undo ``PR_SET_NO_NEW_PRIVS``, so the +code being launched cannot lift them. + +Because the filter is enforced by the kernel on syscall entry and survives +``execve``, it binds compiled binaries -- C/C++/CUDA -- exactly as it binds +Python. Blocking socket *creation* rather than ``connect`` is deliberate: a +seccomp filter cannot dereference pointers, so it cannot inspect the address +passed to ``connect``, but ``socket``'s arguments are plain integers. Denying +every address family (not just AF_INET) also prevents the process from +receiving an already-connected socket over a unix socket via SCM_RIGHTS. + +``execvp`` is the last statement, so any failure above it means the command is +never executed -- the sandbox fails closed by construction. + +Linux/x86-64 only. ``is_supported()`` reports availability so callers can +degrade or refuse deliberately rather than silently running unprotected. +""" + +from __future__ import annotations + +import ctypes +import errno +import os +import platform +import sys + +__all__ = ["is_supported", "apply_lockdown", "main", "SandboxError"] + +# Exit code used when the sandbox refuses to run the command. Distinct from any +# plausible exit code of the job itself so the caller can tell "we refused" from +# "the job failed". +REFUSED_EXIT_CODE = 93 + +# Printed to stderr on refusal; callers grep for this. +SENTINEL = "SYFT_SANDBOX_REFUSED" + +# Default unprivileged account. Overridable for tests and for images that ship a +# dedicated job user. +DEFAULT_UID = 65534 +DEFAULT_GID = 65534 + +# prctl(2) options. +_PR_SET_NO_NEW_PRIVS = 38 +_PR_SET_SECCOMP = 22 +_SECCOMP_MODE_FILTER = 2 + +# Classic BPF opcodes used by the filter. +_BPF_LD = 0x00 +_BPF_W = 0x00 +_BPF_ABS = 0x20 +_BPF_JMP = 0x05 +_BPF_JEQ = 0x10 +_BPF_K = 0x00 +_BPF_RET = 0x06 + +_AUDIT_ARCH_X86_64 = 0xC000003E + +_SECCOMP_RET_ALLOW = 0x7FFF0000 +_SECCOMP_RET_ERRNO = 0x00050000 + +# Offsets into struct seccomp_data. +_OFF_NR = 0 +_OFF_ARCH = 4 + +# x86-64 syscall numbers denied by the filter. +# +# socket/socketpair are the only ways to obtain a socket descriptor, so denying +# them denies networking outright. io_uring can perform network I/O through its +# submission queue without issuing the corresponding syscalls, which a classic +# seccomp filter cannot observe, so it must be denied too. The ptrace family is +# denied to stop the job inspecting or modifying other processes. +_DENIED_SYSCALLS = ( + 41, # socket + 53, # socketpair + 425, # io_uring_setup + 426, # io_uring_enter + 101, # ptrace + 310, # process_vm_readv + 311, # process_vm_writev +) + + +class SandboxError(RuntimeError): + """The lockdown could not be applied.""" + + +class _SockFilter(ctypes.Structure): + """struct sock_filter -- one classic-BPF instruction.""" + + _fields_ = [ + ("code", ctypes.c_ushort), + ("jt", ctypes.c_ubyte), + ("jf", ctypes.c_ubyte), + ("k", ctypes.c_uint), + ] + + +class _SockFprog(ctypes.Structure): + """struct sock_fprog -- the program handed to the kernel.""" + + _fields_ = [ + ("len", ctypes.c_ushort), + ("filter", ctypes.POINTER(_SockFilter)), + ] + + +def _libc() -> ctypes.CDLL: + libc = ctypes.CDLL("libc.so.6", use_errno=True) + # Without explicit argtypes ctypes passes the struct pointer as a 32-bit int + # on x86-64 and the kernel reads a truncated address. + libc.prctl.restype = ctypes.c_int + libc.prctl.argtypes = [ + ctypes.c_int, + ctypes.c_ulong, + ctypes.c_void_p, + ctypes.c_ulong, + ctypes.c_ulong, + ] + return libc + + +def _build_filter() -> tuple[_SockFprog, ctypes.Array]: + """Assemble the BPF program. + + Returns the program and the instruction array. The caller must keep the + array alive until prctl returns: ``_SockFprog`` holds a raw pointer that the + garbage collector does not treat as a reference. + """ + prog: list[_SockFilter] = [] + + prog.append(_SockFilter(_BPF_LD | _BPF_W | _BPF_ABS, 0, 0, _OFF_ARCH)) + arch_at = len(prog) + prog.append(_SockFilter(_BPF_JMP | _BPF_JEQ | _BPF_K, 0, 0, _AUDIT_ARCH_X86_64)) + + prog.append(_SockFilter(_BPF_LD | _BPF_W | _BPF_ABS, 0, 0, _OFF_NR)) + tests_at: list[int] = [] + for nr in _DENIED_SYSCALLS: + tests_at.append(len(prog)) + prog.append(_SockFilter(_BPF_JMP | _BPF_JEQ | _BPF_K, 0, 0, nr)) + + allow_at = len(prog) + prog.append(_SockFilter(_BPF_RET | _BPF_K, 0, 0, _SECCOMP_RET_ALLOW)) + deny_at = len(prog) + prog.append( + _SockFilter(_BPF_RET | _BPF_K, 0, 0, _SECCOMP_RET_ERRNO | errno.EPERM) + ) + + # Jump targets are offsets relative to the instruction *after* the jump, not + # absolute indices. Getting this wrong routes ordinary syscalls into the + # deny branch, which crashes the process immediately. + prog[arch_at].jt = 0 + prog[arch_at].jf = deny_at - arch_at - 1 + for i in tests_at: + prog[i].jt = deny_at - i - 1 + prog[i].jf = 0 + + if allow_at != deny_at - 1: + raise SandboxError("filter assembly produced an unexpected layout") + + instructions = (_SockFilter * len(prog))(*prog) + return _SockFprog(len(prog), instructions), instructions + + +def is_supported() -> tuple[bool, str]: + """Whether the lockdown can be applied here. + + Returns ``(supported, reason)``; ``reason`` is empty when supported. + """ + if sys.platform != "linux": + return False, f"requires Linux, running on {sys.platform}" + if platform.machine() not in ("x86_64", "AMD64"): + return False, f"filter is x86-64 only, running on {platform.machine()}" + try: + _libc() + except OSError as exc: + return False, f"cannot load libc: {exc}" + return True, "" + + +def apply_lockdown(uid: int = DEFAULT_UID, gid: int = DEFAULT_GID) -> None: + """Drop privileges and install the seccomp filter, in that order. + + Raises ``SandboxError`` if any step fails. On return the calling process + cannot create sockets, cannot regain privilege, and cannot undo either. + + The ordering is enforced by the kernel, not chosen for style: + + * ``setgid`` must precede ``setuid`` -- after dropping the user id the + process is no longer permitted to change its group, so a reversed order + leaves the job in the root group. + * ``PR_SET_NO_NEW_PRIVS`` must precede ``PR_SET_SECCOMP`` -- installing a + filter otherwise requires CAP_SYS_ADMIN, which the enclave container does + not have. + """ + supported, reason = is_supported() + if not supported: + raise SandboxError(reason) + + libc = _libc() + + if os.getuid() == 0: + try: + os.setgroups([]) + os.setgid(gid) + os.setuid(uid) + except OSError as exc: + raise SandboxError(f"privilege drop failed: {exc}") from exc + if os.getuid() != uid or os.geteuid() != uid: + raise SandboxError("privilege drop did not take effect") + elif os.getuid() != uid: + # Already unprivileged. Continue -- the filter is still worth applying -- + # but do not pretend we dropped to the requested account. + pass + + if libc.prctl(_PR_SET_NO_NEW_PRIVS, 1, None, 0, 0) != 0: + raise SandboxError( + f"PR_SET_NO_NEW_PRIVS failed: {os.strerror(ctypes.get_errno())}" + ) + + prog, _instructions = _build_filter() + rc = libc.prctl( + _PR_SET_SECCOMP, + _SECCOMP_MODE_FILTER, + ctypes.cast(ctypes.byref(prog), ctypes.c_void_p), + 0, + 0, + ) + if rc != 0: + raise SandboxError( + f"PR_SET_SECCOMP failed: {os.strerror(ctypes.get_errno())}" + ) + + +def _refuse(reason: str) -> "None": + print(f"{SENTINEL}: {reason}", file=sys.stderr, flush=True) + # os._exit avoids running atexit handlers or flushing inherited buffers in + # a process that is midway through dropping privileges. + os._exit(REFUSED_EXIT_CODE) + + +def main(argv: list[str] | None = None) -> None: + """Entry point: apply the lockdown, then become the requested command.""" + args = list(sys.argv[1:] if argv is None else argv) + + uid, gid = DEFAULT_UID, DEFAULT_GID + while args and args[0].startswith("--"): + flag = args.pop(0) + if flag == "--": + break + if flag in ("--uid", "--gid"): + if not args: + _refuse(f"{flag} requires a value") + try: + value = int(args.pop(0)) + except ValueError: + _refuse(f"{flag} requires an integer") + if flag == "--uid": + uid = value + else: + gid = value + else: + _refuse(f"unknown option {flag}") + + if not args: + _refuse("no command given") + + try: + apply_lockdown(uid=uid, gid=gid) + except SandboxError as exc: + _refuse(str(exc)) + + try: + os.execvp(args[0], args) + except OSError as exc: + _refuse(f"exec {args[0]!r} failed: {exc}") + + +if __name__ == "__main__": + main() diff --git a/packages/syft-job/tests/test_sandbox.py b/packages/syft-job/tests/test_sandbox.py new file mode 100644 index 00000000000..32007d3a915 --- /dev/null +++ b/packages/syft-job/tests/test_sandbox.py @@ -0,0 +1,290 @@ +"""Tests for the job sandbox. + +The behavioural tests shell out to sandbox.py and inspect what the sandboxed +process can and cannot do. They deliberately assert on the *kernel's* behaviour +rather than on our code paths -- the guarantee is only worth what the kernel +actually enforces. +""" + +from __future__ import annotations + +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +from syft_job import job_runner, sandbox + +SANDBOX_PY = str(Path(sandbox.__file__).resolve()) +LINUX_X86 = sys.platform == "linux" and os.uname().machine == "x86_64" +requires_linux_x86 = pytest.mark.skipif( + not LINUX_X86, reason="sandbox is Linux/x86-64 only" +) + + +def _run_sandboxed(snippet: str, extra_args: list[str] | None = None): + """Run a Python snippet under the sandbox, as the current user.""" + args = [ + sys.executable, + SANDBOX_PY, + "--uid", + str(os.getuid()), + "--gid", + str(os.getgid()), + *(extra_args or []), + "--", + sys.executable, + "-c", + snippet, + ] + return subprocess.run(args, capture_output=True, text=True, timeout=60) + + +# -------------------------------------------------------------------------- +# configuration / wiring +# -------------------------------------------------------------------------- + + +def test_sandbox_defaults_to_off(monkeypatch): + """Must default off: this runner also runs on data owners' own machines.""" + monkeypatch.delenv(job_runner.SANDBOX_ENV_VAR, raising=False) + assert job_runner.get_sandbox_mode() == "off" + + +def test_invalid_mode_rejected(monkeypatch): + monkeypatch.setenv(job_runner.SANDBOX_ENV_VAR, "sort-of") + with pytest.raises(ValueError): + job_runner.get_sandbox_mode() + + +def test_command_unwrapped_when_off(monkeypatch): + monkeypatch.setenv(job_runner.SANDBOX_ENV_VAR, "off") + assert job_runner.build_job_command(Path("/tmp/run.sh")) == [ + "bash", + "/tmp/run.sh", + ] + + +@requires_linux_x86 +@pytest.mark.parametrize("mode", ["on", "require"]) +def test_command_wrapped_when_enabled(monkeypatch, mode): + monkeypatch.setenv(job_runner.SANDBOX_ENV_VAR, mode) + cmd = job_runner.build_job_command(Path("/tmp/run.sh")) + assert cmd[0] == sys.executable + assert cmd[1] == SANDBOX_PY + # the job itself is still the tail of the command + assert cmd[-2:] == ["bash", "/tmp/run.sh"] + + +def test_require_raises_when_unsupported(monkeypatch): + monkeypatch.setenv(job_runner.SANDBOX_ENV_VAR, "require") + monkeypatch.setattr(sandbox, "is_supported", lambda: (False, "no kernel support")) + with pytest.raises(job_runner.SandboxUnavailableError): + job_runner.build_job_command(Path("/tmp/run.sh")) + + +def test_on_degrades_when_unsupported(monkeypatch, capsys): + monkeypatch.setenv(job_runner.SANDBOX_ENV_VAR, "on") + monkeypatch.setattr(sandbox, "is_supported", lambda: (False, "no kernel support")) + assert job_runner.build_job_command(Path("/tmp/run.sh")) == [ + "bash", + "/tmp/run.sh", + ] + assert "WITHOUT network isolation" in capsys.readouterr().out + + +def test_env_allowlisted_when_sandboxed(monkeypatch): + monkeypatch.setenv(job_runner.SANDBOX_ENV_VAR, "on") + monkeypatch.setenv("SYFT_BOOTSTRAP_SA_SECRET", "super-secret") + monkeypatch.setenv("SYFT_ENCLAVE_TOKEN_CONTENT", "oauth-token") + env = job_runner.build_job_env("/syftbox", "enclave@example.org") + assert "SYFT_BOOTSTRAP_SA_SECRET" not in env + assert "SYFT_ENCLAVE_TOKEN_CONTENT" not in env + assert env["SYFTBOX_FOLDER"] == "/syftbox" + assert env[job_runner.IS_IN_JOB_ENV_VAR] == "true" + + +def test_env_inherited_when_off(monkeypatch): + monkeypatch.setenv(job_runner.SANDBOX_ENV_VAR, "off") + monkeypatch.setenv("SOME_UNRELATED_VAR", "kept") + env = job_runner.build_job_env("/syftbox", "a@b.c") + assert env["SOME_UNRELATED_VAR"] == "kept" + + +# -------------------------------------------------------------------------- +# what the kernel actually enforces +# -------------------------------------------------------------------------- + + +@requires_linux_x86 +def test_supported_on_this_platform(): + supported, reason = sandbox.is_supported() + assert supported, reason + + +@requires_linux_x86 +@pytest.mark.parametrize( + "family,name", [(2, "AF_INET"), (10, "AF_INET6"), (1, "AF_UNIX")] +) +def test_socket_creation_blocked(family, name): + """Every address family, not just the internet ones. + + AF_UNIX matters: a socket's network namespace is fixed at creation, so a + process that can open a unix socket can be handed an already-connected + network socket over SCM_RIGHTS by a co-resident helper. + """ + proc = _run_sandboxed( + f"import socket\n" + f"try:\n" + f" socket.socket({family}, socket.SOCK_STREAM)\n" + f" print('ALLOWED')\n" + f"except OSError as e:\n" + f" print('BLOCKED')\n" + ) + assert proc.stdout.strip() == "BLOCKED", f"{name} was not blocked: {proc.stdout}" + + +@requires_linux_x86 +def test_restriction_survives_exec(): + """The guarantee is about compiled binaries, so it must outlive execve.""" + inner = ( + "import socket\n" + "try:\n" + " socket.socket(2, 1); print('ALLOWED')\n" + "except OSError: print('BLOCKED')\n" + ) + proc = _run_sandboxed( + "import subprocess, sys\n" + f"r = subprocess.run([sys.executable, '-c', {inner!r}], " + "capture_output=True, text=True)\n" + "print(r.stdout.strip())\n" + ) + assert proc.stdout.strip() == "BLOCKED" + + +@requires_linux_x86 +def test_dns_resolution_blocked(): + proc = _run_sandboxed( + "import socket\n" + "try:\n" + " socket.getaddrinfo('example.com', 80); print('RESOLVED')\n" + "except Exception: print('BLOCKED')\n" + ) + assert proc.stdout.strip() == "BLOCKED" + + +@requires_linux_x86 +def test_ordinary_work_unaffected(): + """A lockdown that breaks normal jobs is not shippable.""" + proc = _run_sandboxed( + "import os, tempfile\n" + "total = sum(i * i for i in range(10000))\n" + "with tempfile.TemporaryDirectory() as d:\n" + " p = os.path.join(d, 'out.txt')\n" + " open(p, 'w').write('result')\n" + " assert open(p).read() == 'result'\n" + "print('OK', total)\n" + ) + assert proc.returncode == 0, proc.stderr + assert proc.stdout.startswith("OK 333283335000") + + +@requires_linux_x86 +def test_no_new_privs_is_set(): + proc = _run_sandboxed( + "print([l for l in open('/proc/self/status') if 'NoNewPrivs' in l][0].strip())" + ) + assert proc.stdout.strip().endswith("1") + + +# -------------------------------------------------------------------------- +# fail-closed +# -------------------------------------------------------------------------- + + +@requires_linux_x86 +def test_refuses_unknown_option_without_running_command(): + proc = subprocess.run( + [sys.executable, SANDBOX_PY, "--nope", "--", "echo", "SHOULD_NOT_RUN"], + capture_output=True, + text=True, + timeout=30, + ) + assert proc.returncode == sandbox.REFUSED_EXIT_CODE + assert sandbox.SENTINEL in proc.stderr + assert "SHOULD_NOT_RUN" not in proc.stdout + + +@requires_linux_x86 +def test_refuses_when_no_command_given(): + proc = subprocess.run( + [sys.executable, SANDBOX_PY, "--"], capture_output=True, text=True, timeout=30 + ) + assert proc.returncode == sandbox.REFUSED_EXIT_CODE + + +@requires_linux_x86 +def test_refuses_when_exec_target_missing(): + proc = _run_sandboxed_missing = subprocess.run( + [ + sys.executable, + SANDBOX_PY, + "--uid", + str(os.getuid()), + "--gid", + str(os.getgid()), + "--", + "/nonexistent/binary", + ], + capture_output=True, + text=True, + timeout=30, + ) + assert proc.returncode == sandbox.REFUSED_EXIT_CODE + assert sandbox.SENTINEL in proc.stderr + + +# -------------------------------------------------------------------------- +# ordering guards -- these encode kernel requirements, not style preferences +# -------------------------------------------------------------------------- + + +@requires_linux_x86 +def test_seccomp_requires_no_new_privs_first(): + """Without CAP_SYS_ADMIN the kernel refuses a filter unless no_new_privs is + already set. If this ever stops being true the ordering in apply_lockdown + is no longer load-bearing and the comment there should be revisited.""" + proc = subprocess.run( + [ + sys.executable, + "-c", + "import ctypes, sys\n" + "sys.path.insert(0, %r)\n" % str(Path(sandbox.__file__).parent.parent) + + "from syft_job.sandbox import _build_filter, _libc, " + "_PR_SET_SECCOMP, _SECCOMP_MODE_FILTER\n" + "libc = _libc()\n" + "prog, _keep = _build_filter()\n" + "rc = libc.prctl(_PR_SET_SECCOMP, _SECCOMP_MODE_FILTER, " + "ctypes.cast(ctypes.byref(prog), ctypes.c_void_p), 0, 0)\n" + "print('rc', rc)\n", + ], + capture_output=True, + text=True, + timeout=30, + ) + # Non-root without CAP_SYS_ADMIN: must fail. If running as root with the + # capability the call may succeed, so only assert the failure when we are + # genuinely unprivileged. + if os.getuid() != 0: + assert proc.stdout.strip() == "rc -1", proc.stdout + proc.stderr + + +def test_setgid_must_precede_setuid_is_documented(): + """Guards against a refactor silently reordering the privilege drop.""" + src = Path(sandbox.__file__).read_text() + drop = src[src.index("os.setgroups([])") : src.index("if os.getuid() != uid")] + assert drop.index("os.setgroups") < drop.index("os.setgid") < drop.index( + "os.setuid" + ), "privilege drop order changed: setgroups -> setgid -> setuid is required" From 42b71041b1cf20f28204dc5da5ff959204754678 Mon Sep 17 00:00:00 2001 From: bitsofsteve <11381249+bitsofsteve@users.noreply.github.com> Date: Mon, 3 Aug 2026 05:23:34 +0000 Subject: [PATCH 2/4] Split dependency install from execution when sandboxing The sandbox denies every address family, including AF_UNIX, so it cannot wrap run.sh as generated: that script builds a venv and installs dependencies before reaching the entrypoint, and uv dies without local sockets -- its async runtime opens a UnixStream for signal handling, which fails long before any network call is attempted. Rebuild the two phases from the submission's declared entrypoint and dependencies instead of executing the submitted run.sh. Phase A installs with the network available and unsandboxed; phase B runs only the entrypoint under the lockdown. Because phase A runs unsandboxed and with network, it must not execute code the submitter chose, so: - syft-client is installed from this runner's own install source, which is part of the attested enclave image, rather than from whatever the submission declared. It may legitimately be a local path, so it is exempt from the wheels-only rule. - declared dependencies are installed --only-binary=:all:, since building a source distribution runs its build backend; specs naming local paths or VCS URLs are refused outright rather than built. Bash submissions carry no entrypoint metadata to split on, so they are sandboxed wholesale and will fail if they install anything -- acceptable while the flag is opt-in. Adds an integration test pair that runs a probe job through the real runner and asserts it can open a socket with the sandbox off and cannot with require, so a silently degraded sandbox fails the suite rather than passing quietly. syft-job and syft-enclave suites: 143 passed with the sandbox off, 143 passed with it on. --- packages/syft-job/src/syft_job/job_runner.py | 198 ++++++++++++++++--- packages/syft-job/src/syft_job/sandbox.py | 9 +- packages/syft-job/tests/test_sandbox.py | 70 +++++++ 3 files changed, 254 insertions(+), 23 deletions(-) diff --git a/packages/syft-job/src/syft_job/job_runner.py b/packages/syft-job/src/syft_job/job_runner.py index 217b203ef98..82f1a69c347 100644 --- a/packages/syft-job/src/syft_job/job_runner.py +++ b/packages/syft-job/src/syft_job/job_runner.py @@ -85,32 +85,15 @@ def _sandbox_ids() -> tuple[int, int]: ) -def build_job_command(run_script: Path) -> List[str]: - """Command used to launch a job, wrapped in the sandbox when enabled. +def _wrap_in_sandbox(command: List[str]) -> List[str]: + """Prefix ``command`` with the sandbox wrapper. The wrapper is invoked by file path rather than as ``-m syft_job.sandbox`` so that installing the lockdown does not depend on the ``syft_job`` package (and its dependencies) importing successfully inside the job environment. """ - direct = ["bash", str(run_script)] - mode = get_sandbox_mode() - if mode == "off": - return direct - from . import sandbox as _sandbox - supported, reason = _sandbox.is_supported() - if not supported: - if mode == "require": - raise SandboxUnavailableError( - f"{SANDBOX_ENV_VAR}=require but the sandbox cannot be applied: {reason}" - ) - print( - f" WARNING: {SANDBOX_ENV_VAR}=on but sandbox unavailable ({reason}); " - f"running job WITHOUT network isolation" - ) - return direct - uid, gid = _sandbox_ids() return [ sys.executable, @@ -120,10 +103,168 @@ def build_job_command(run_script: Path) -> List[str]: "--gid", str(gid), "--", - *direct, + *command, ] +def _sandbox_available_or_raise(mode: str) -> bool: + """Whether to sandbox. Raises in ``require`` mode if we cannot.""" + from . import sandbox as _sandbox + + supported, reason = _sandbox.is_supported() + if supported: + return True + if mode == "require": + raise SandboxUnavailableError( + f"{SANDBOX_ENV_VAR}=require but the sandbox cannot be applied: {reason}" + ) + print( + f" WARNING: {SANDBOX_ENV_VAR}=on but sandbox unavailable ({reason}); " + f"running job WITHOUT network isolation" + ) + return False + + +def build_job_command(run_script: Path) -> List[str]: + """Command used to launch a job, wrapped in the sandbox when enabled. + + This sandboxes ``run.sh`` wholesale. It is correct only for submissions + whose script does no dependency installation -- installers need local + sockets, which the sandbox denies. Python jobs go through + :func:`build_two_phase_command` instead. + """ + direct = ["bash", str(run_script)] + mode = get_sandbox_mode() + if mode == "off" or not _sandbox_available_or_raise(mode): + return direct + return _wrap_in_sandbox(direct) + + +# Python version used for the job virtualenv. Mirrors +# ``syft_job.client.RUN_SCRIPT_PYTHON_VERSION``, which generates the equivalent +# run.sh for the unsandboxed path. +JOB_PYTHON_VERSION = "3.12" + + +class PhaseAError(RuntimeError): + """Dependency installation failed before the job could be sandboxed.""" + + +def _is_submitter_code(spec: str) -> bool: + """Whether a dependency string would fetch and build submitter-chosen code. + + Local paths and VCS URLs execute their own build scripts on install, so + under sandboxing they are refused rather than run: phase A has the network + and runs unsandboxed, which is precisely the position an attacker wants. + """ + s = spec.strip().lower() + if any(s.startswith(p) for p in ("git+", "hg+", "svn+", "bzr+")): + return True + if s.startswith((".", "/", "file://")): + return True + return " @ " in s or s.startswith("-e ") + + +def install_dependencies( + submission_dir: Path, dependencies: List[str], timeout: int = 900 +) -> Path: + """Phase A: build the job's virtualenv with the network available. + + Runs *before* the sandbox is applied, so it must not execute code the + submitter chose. Two rules enforce that: + + * ``syft-client`` is installed from *this* runner's own install source -- + part of the attested enclave image -- not from whatever the submission + declared. It may be a local path, so it is installed without the + wheels-only restriction. + * every submitter-declared dependency is installed ``--only-binary=:all:``, + because building a source distribution runs its build backend. Declared + dependencies that are local paths or VCS URLs are refused outright. + + Returns the interpreter to use for phase B. + """ + from .install_source import get_syft_client_install_source + + code_dir = submission_dir / "code" + venv_dir = code_dir / ".venv" + venv_python = venv_dir / "bin" / "python" + + rejected = [d for d in dependencies if _is_submitter_code(d)] + declared = [d for d in dependencies if not _is_submitter_code(d)] + if rejected: + print( + f" Sandbox: ignoring {len(rejected)} dependency spec(s) that would build " + f"submitter-supplied code: {', '.join(rejected)}" + ) + + steps: List[List[str]] = [ + ["uv", "venv", "--python", JOB_PYTHON_VERSION, str(venv_dir)], + # Trusted: comes from the enclave image, not the submission. + [ + "uv", + "pip", + "install", + "--python", + str(venv_python), + get_syft_client_install_source(), + ], + ] + if declared: + steps.append( + [ + "uv", + "pip", + "install", + "--python", + str(venv_python), + "--only-binary=:all:", + *declared, + ] + ) + + for step in steps: + result = subprocess.run( + step, cwd=code_dir, capture_output=True, text=True, timeout=timeout + ) + if result.returncode != 0: + raise PhaseAError( + f"{' '.join(step[:3])} failed ({result.returncode}):\n" + f"{result.stderr.strip()[-2000:]}" + ) + + return venv_python + + +def build_two_phase_command( + submission_dir: Path, + metadata: JobSubmissionMetadata, + run_script: Path, + timeout: int = 900, +) -> List[str]: + """Install dependencies unsandboxed, then return a sandboxed run command. + + The submitted ``run.sh`` is deliberately not executed for python jobs when + sandboxing: it interleaves installation and execution in one script, so it + cannot be split, and its contents are chosen by the submitter. The two + phases are rebuilt from the declared ``entrypoint`` and ``dependencies`` + instead. + """ + mode = get_sandbox_mode() + if mode == "off" or not _sandbox_available_or_raise(mode): + return ["bash", str(run_script)] + + # Only python submissions carry the metadata needed to split the phases. + if metadata.type != "python" or not metadata.entrypoint: + return _wrap_in_sandbox(["bash", str(run_script)]) + + python = install_dependencies( + submission_dir, list(metadata.dependencies or []), timeout=timeout + ) + return _wrap_in_sandbox( + ["bash", "-c", f'cd code && exec "$0" "$1"', str(python), metadata.entrypoint] + ) + + def build_job_env(config_folder: str, email: str) -> dict: """Environment for a job subprocess.""" if get_sandbox_mode() == "off": @@ -325,6 +466,19 @@ def _find_jobref_from_name(self, job_name: str, user: str | None = None) -> JobR self.config.current_user_email, job_name, ds_email=user ) + def _build_command( + self, ref: JobRef, submission_dir: Path, run_script: Path, timeout: int + ) -> List[str]: + """Command to launch this job, sandboxed and phase-split when enabled.""" + if get_sandbox_mode() == "off": + return build_job_command(run_script) + metadata = self._get_job_metadata(ref) + if metadata is None: + return build_job_command(run_script) + return build_two_phase_command( + submission_dir, metadata, run_script, timeout=timeout + ) + def _execute_job_streaming(self, ref: JobRef, timeout: int) -> int: """Execute job with real-time streaming output. @@ -345,7 +499,7 @@ def _execute_job_streaming(self, ref: JobRef, timeout: int) -> int: env = build_job_env( self.config.syftbox_folder_path_str, self.config.current_user_email ) - command = build_job_command(run_script) + command = self._build_command(ref, submission_dir, run_script, timeout) # stdout/stderr go to review/ stdout_file = review_dir / "stdout.txt" @@ -433,7 +587,7 @@ def _execute_job_captured(self, ref: JobRef, timeout: int) -> int: ) process = subprocess.Popen( - build_job_command(run_script), + self._build_command(ref, submission_dir, run_script, timeout), cwd=submission_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE, diff --git a/packages/syft-job/src/syft_job/sandbox.py b/packages/syft-job/src/syft_job/sandbox.py index 02fa1f7b0fd..f5a181cb8b9 100644 --- a/packages/syft-job/src/syft_job/sandbox.py +++ b/packages/syft-job/src/syft_job/sandbox.py @@ -29,7 +29,14 @@ import platform import sys -__all__ = ["is_supported", "apply_lockdown", "main", "SandboxError"] +__all__ = [ + "is_supported", + "apply_lockdown", + "main", + "SandboxError", + "REFUSED_EXIT_CODE", + "SENTINEL", +] # Exit code used when the sandbox refuses to run the command. Distinct from any # plausible exit code of the job itself so the caller can tell "we refused" from diff --git a/packages/syft-job/tests/test_sandbox.py b/packages/syft-job/tests/test_sandbox.py index 32007d3a915..9d79d353e0d 100644 --- a/packages/syft-job/tests/test_sandbox.py +++ b/packages/syft-job/tests/test_sandbox.py @@ -288,3 +288,73 @@ def test_setgid_must_precede_setuid_is_documented(): assert drop.index("os.setgroups") < drop.index("os.setgid") < drop.index( "os.setuid" ), "privilege drop order changed: setgroups -> setgid -> setuid is required" + + +# -------------------------------------------------------------------------- +# integration: the lockdown must actually engage through the real runner +# -------------------------------------------------------------------------- + +NETWORK_PROBE_MAIN_PY = """\ +import os, socket + +os.makedirs("outputs", exist_ok=True) +try: + socket.socket(socket.AF_INET, socket.SOCK_STREAM) + verdict = "NETWORK_ALLOWED" +except OSError: + verdict = "NETWORK_BLOCKED" +with open("outputs/verdict.txt", "w") as f: + f.write(verdict) +print(verdict) +""" + + +def _run_probe_job(tmp_path: Path) -> str: + """Submit and run a job that reports whether it can make a socket.""" + from syft_job.client import JobClient + from syft_job.config import SyftJobConfig + from syft_job.job_runner import SyftJobRunner + + do_email, ds_email = "do@test.org", "ds@test.org" + syftbox = tmp_path / "SyftBox" + syftbox.mkdir() + code_file = tmp_path / "main.py" + code_file.write_text(NETWORK_PROBE_MAIN_PY) + + do_config = SyftJobConfig(syftbox_folder=syftbox, current_user_email=do_email) + ds_config = SyftJobConfig(syftbox_folder=syftbox, current_user_email=ds_email) + JobClient(config=ds_config).submit_python_job( + user=do_email, code_path=str(code_file), job_name="probe.job" + ) + do_client = JobClient(config=do_config) + do_client.jobs[0].approve() + SyftJobRunner(config=do_config).process_approved_jobs( + stream_output=False, timeout=900 + ) + + review = do_config.get_review_job_dir(do_email, ds_email, "probe.job") + verdict_file = review / "outputs" / "verdict.txt" + if not verdict_file.exists(): + stderr = (review / "stderr.txt").read_text() if ( + review / "stderr.txt" + ).exists() else "" + raise AssertionError(f"job produced no verdict; stderr:\n{stderr[-3000:]}") + return verdict_file.read_text().strip() + + +@requires_linux_x86 +@pytest.mark.slow +def test_job_has_network_without_sandbox(tmp_path, monkeypatch): + """Control: confirms the probe would otherwise succeed.""" + monkeypatch.setenv(job_runner.SANDBOX_ENV_VAR, "off") + assert _run_probe_job(tmp_path) == "NETWORK_ALLOWED" + + +@requires_linux_x86 +@pytest.mark.slow +def test_job_network_blocked_with_sandbox(tmp_path, monkeypatch): + """The whole point: a real job, run through the real runner, cannot network.""" + monkeypatch.setenv(job_runner.SANDBOX_ENV_VAR, "require") + monkeypatch.setenv(job_runner.SANDBOX_UID_ENV_VAR, str(os.getuid())) + monkeypatch.setenv(job_runner.SANDBOX_GID_ENV_VAR, str(os.getgid())) + assert _run_probe_job(tmp_path) == "NETWORK_BLOCKED" From 79e650260b780c259a4670db2c0a607f1255ffc1 Mon Sep 17 00:00:00 2001 From: bitsofsteve <11381249+bitsofsteve@users.noreply.github.com> Date: Mon, 3 Aug 2026 05:29:08 +0000 Subject: [PATCH 3/4] Refuse a partial lockdown instead of applying it silently Only root can change user id, so apply_lockdown skipped the privilege drop when invoked as an ordinary user and installed just the seccomp filter. That is the less valuable half: the network is blocked, but the job keeps the invoking user's file access, so it can still read the Drive credential and modify the runner's own code. Worse, it happened silently, and under SYFT_JOB_SANDBOX= require -- where an operator has asked for the full guarantee. Make it an error by default. Callers wanting best-effort behaviour must pass --best-effort explicitly, which the runner does for "on" (documented as best-effort) and pointedly does not for "require". Existing tests missed this because they pass --uid $(id -u), so the requested uid already matched and no drop was needed. Added tests that request a different uid as a non-root user. Validated as root in the published enclave image, which is the configuration that actually ships: baseline uid=0 caps=a80425fb nnp=0 network ALLOWED token READABLE code WRITABLE sandboxed uid=1500 caps=0 nnp=1 network BLOCKED token BLOCKED code read-only syft-job and syft-enclave suites: 147 passed with the sandbox off and on. --- packages/syft-job/src/syft_job/job_runner.py | 4 ++ packages/syft-job/src/syft_job/sandbox.py | 33 +++++++--- packages/syft-job/tests/test_sandbox.py | 63 ++++++++++++++++++++ 3 files changed, 93 insertions(+), 7 deletions(-) diff --git a/packages/syft-job/src/syft_job/job_runner.py b/packages/syft-job/src/syft_job/job_runner.py index 82f1a69c347..a37fe8ba321 100644 --- a/packages/syft-job/src/syft_job/job_runner.py +++ b/packages/syft-job/src/syft_job/job_runner.py @@ -95,6 +95,9 @@ def _wrap_in_sandbox(command: List[str]) -> List[str]: from . import sandbox as _sandbox uid, gid = _sandbox_ids() + # "on" is explicitly best-effort; "require" must refuse a partial lockdown + # (e.g. running as a non-root user, where privileges cannot be dropped). + best_effort = ["--best-effort"] if get_sandbox_mode() == "on" else [] return [ sys.executable, os.fspath(Path(_sandbox.__file__).resolve()), @@ -102,6 +105,7 @@ def _wrap_in_sandbox(command: List[str]) -> List[str]: str(uid), "--gid", str(gid), + *best_effort, "--", *command, ] diff --git a/packages/syft-job/src/syft_job/sandbox.py b/packages/syft-job/src/syft_job/sandbox.py index f5a181cb8b9..a80d313d632 100644 --- a/packages/syft-job/src/syft_job/sandbox.py +++ b/packages/syft-job/src/syft_job/sandbox.py @@ -189,7 +189,9 @@ def is_supported() -> tuple[bool, str]: return True, "" -def apply_lockdown(uid: int = DEFAULT_UID, gid: int = DEFAULT_GID) -> None: +def apply_lockdown( + uid: int = DEFAULT_UID, gid: int = DEFAULT_GID, strict: bool = True +) -> None: """Drop privileges and install the seccomp filter, in that order. Raises ``SandboxError`` if any step fails. On return the calling process @@ -203,6 +205,13 @@ def apply_lockdown(uid: int = DEFAULT_UID, gid: int = DEFAULT_GID) -> None: * ``PR_SET_NO_NEW_PRIVS`` must precede ``PR_SET_SECCOMP`` -- installing a filter otherwise requires CAP_SYS_ADMIN, which the enclave container does not have. + + Only root can change user id, so when invoked as an ordinary user the drop + is impossible. With ``strict`` (the default) that is an error rather than a + silent half-application: the seccomp filter alone still blocks the network, + but it leaves the job able to read credentials and modify the runner's own + files, which is the more valuable half of the protection. Callers that + genuinely want best-effort behaviour must ask for it explicitly. """ supported, reason = is_supported() if not supported: @@ -220,9 +229,14 @@ def apply_lockdown(uid: int = DEFAULT_UID, gid: int = DEFAULT_GID) -> None: if os.getuid() != uid or os.geteuid() != uid: raise SandboxError("privilege drop did not take effect") elif os.getuid() != uid: - # Already unprivileged. Continue -- the filter is still worth applying -- - # but do not pretend we dropped to the requested account. - pass + message = ( + f"cannot drop to uid {uid}: running as uid {os.getuid()}, not root. " + f"The seccomp filter would still apply, but the job would keep this " + f"user's file access" + ) + if strict: + raise SandboxError(message) + print(f"{SENTINEL}-WARNING: {message}", file=sys.stderr, flush=True) if libc.prctl(_PR_SET_NO_NEW_PRIVS, 1, None, 0, 0) != 0: raise SandboxError( @@ -254,12 +268,17 @@ def main(argv: list[str] | None = None) -> None: """Entry point: apply the lockdown, then become the requested command.""" args = list(sys.argv[1:] if argv is None else argv) - uid, gid = DEFAULT_UID, DEFAULT_GID + uid, gid, strict = DEFAULT_UID, DEFAULT_GID, True while args and args[0].startswith("--"): flag = args.pop(0) if flag == "--": break - if flag in ("--uid", "--gid"): + if flag == "--best-effort": + # Apply what is possible here rather than refusing. Only for callers + # that accept a partial lockdown; never use where the guarantee is + # being relied upon. + strict = False + elif flag in ("--uid", "--gid"): if not args: _refuse(f"{flag} requires a value") try: @@ -277,7 +296,7 @@ def main(argv: list[str] | None = None) -> None: _refuse("no command given") try: - apply_lockdown(uid=uid, gid=gid) + apply_lockdown(uid=uid, gid=gid, strict=strict) except SandboxError as exc: _refuse(str(exc)) diff --git a/packages/syft-job/tests/test_sandbox.py b/packages/syft-job/tests/test_sandbox.py index 9d79d353e0d..c3adf7ad4b5 100644 --- a/packages/syft-job/tests/test_sandbox.py +++ b/packages/syft-job/tests/test_sandbox.py @@ -358,3 +358,66 @@ def test_job_network_blocked_with_sandbox(tmp_path, monkeypatch): monkeypatch.setenv(job_runner.SANDBOX_UID_ENV_VAR, str(os.getuid())) monkeypatch.setenv(job_runner.SANDBOX_GID_ENV_VAR, str(os.getgid())) assert _run_probe_job(tmp_path) == "NETWORK_BLOCKED" + + +# -------------------------------------------------------------------------- +# partial lockdown must not pass silently +# -------------------------------------------------------------------------- + + +@requires_linux_x86 +@pytest.mark.skipif(os.getuid() == 0, reason="needs to run as a non-root user") +def test_refuses_when_privileges_cannot_be_dropped(): + """As a non-root user the uid drop is impossible. + + The seccomp half would still apply, but the job would keep this user's file + access -- so `require` must refuse rather than half-protect. + """ + proc = subprocess.run( + [sys.executable, SANDBOX_PY, "--uid", "65534", "--", "echo", "SHOULD_NOT_RUN"], + capture_output=True, + text=True, + timeout=30, + ) + assert proc.returncode == sandbox.REFUSED_EXIT_CODE + assert "cannot drop to uid 65534" in proc.stderr + assert "SHOULD_NOT_RUN" not in proc.stdout + + +@requires_linux_x86 +@pytest.mark.skipif(os.getuid() == 0, reason="needs to run as a non-root user") +def test_best_effort_proceeds_but_warns(): + proc = subprocess.run( + [ + sys.executable, + SANDBOX_PY, + "--uid", + "65534", + "--best-effort", + "--", + sys.executable, + "-c", + "import socket\n" + "try:\n" + " socket.socket(2, 1); print('ALLOWED')\n" + "except OSError: print('BLOCKED')\n", + ], + capture_output=True, + text=True, + timeout=60, + ) + assert proc.returncode == 0 + assert proc.stdout.strip() == "BLOCKED" # filter still applied + assert "WARNING" in proc.stderr # but the partial application is announced + + +@requires_linux_x86 +def test_require_mode_does_not_pass_best_effort(monkeypatch): + monkeypatch.setenv(job_runner.SANDBOX_ENV_VAR, "require") + assert "--best-effort" not in job_runner.build_job_command(Path("/tmp/run.sh")) + + +@requires_linux_x86 +def test_on_mode_passes_best_effort(monkeypatch): + monkeypatch.setenv(job_runner.SANDBOX_ENV_VAR, "on") + assert "--best-effort" in job_runner.build_job_command(Path("/tmp/run.sh")) From d38557d5ee89ddf338a09e52794cb4229de01f54 Mon Sep 17 00:00:00 2001 From: bitsofsteve <11381249+bitsofsteve@users.noreply.github.com> Date: Mon, 3 Aug 2026 06:43:38 +0000 Subject: [PATCH 4/4] Give sandboxed jobs the filesystem access they need to run A sandboxed job could not do useful work: it runs as an unprivileged account, so it could neither create its virtualenv nor write outputs/ in a root-owned job tree, nor read the datasets it was approved for -- the datasite lives under /root, which is 0700 because the container runs as root and that is its home. Hand the job's own working tree to the sandbox account, and open traversal and read on the datasite without transferring ownership. Neither weakens the lockdown: the Drive token and the runner's own source stay root-owned and unreachable. It is a no-op when not running as root, which covers tests and data owners' own machines. Also add the syftjob account (uid 1500) to the enclave image, and allow SYFT_JOB_SANDBOX{,_UID,_GID} through the Confidential Space env-override policy, without which the sandbox cannot be enabled on a deployed enclave at all. Verified as root inside an image built from this branch, driving the real submit -> approve -> run -> distribute flow with SYFT_JOB_SANDBOX=require: JOB REPORTED: uid=65534 user=nobody caps=0000000000000000 network=BLOCKED and the trust split fired as intended, refusing the submitted local-path dependency spec in favour of the runner's own install source. 147 passed with the sandbox off and on. --- packages/syft-enclave/docker/Dockerfile | 11 ++++- packages/syft-job/src/syft_job/job_runner.py | 51 +++++++++++++++++++- 2 files changed, 60 insertions(+), 2 deletions(-) diff --git a/packages/syft-enclave/docker/Dockerfile b/packages/syft-enclave/docker/Dockerfile index 4a6d4523908..8b1a6a7cd86 100644 --- a/packages/syft-enclave/docker/Dockerfile +++ b/packages/syft-enclave/docker/Dockerfile @@ -27,6 +27,12 @@ ENV PATH="/repo/.venv/bin:$PATH" # its libs on LD_LIBRARY_PATH. ENV LD_LIBRARY_PATH=/usr/local/nvidia/lib64 +# Unprivileged account that sandboxed jobs run as (SYFT_JOB_SANDBOX). Jobs are +# untrusted code; running them as root would let them read the Drive token and +# rewrite the enclave's own source. Nologin: it exists to own processes, not to +# be logged into. +RUN useradd --uid 1500 --no-create-home --shell /usr/sbin/nologin syftjob + WORKDIR /app COPY packages/syft-enclave/docker/attestation_server.py . COPY packages/syft-enclave/docker/entrypoint.sh . @@ -45,7 +51,10 @@ SYFT_BOOTSTRAP,\ SYFT_BOOTSTRAP_WIF_AUDIENCE,\ SYFT_BOOTSTRAP_WIF_SECRET,\ SYFT_BOOTSTRAP_SA_SECRET,\ -SYFT_ENCLAVE_TOKEN_CONTENT" +SYFT_ENCLAVE_TOKEN_CONTENT,\ +SYFT_JOB_SANDBOX,\ +SYFT_JOB_SANDBOX_UID,\ +SYFT_JOB_SANDBOX_GID" LABEL "tee.launch_policy.log_redirect"="always" diff --git a/packages/syft-job/src/syft_job/job_runner.py b/packages/syft-job/src/syft_job/job_runner.py index a37fe8ba321..fcf3dacb5ad 100644 --- a/packages/syft-job/src/syft_job/job_runner.py +++ b/packages/syft-job/src/syft_job/job_runner.py @@ -154,6 +154,49 @@ class PhaseAError(RuntimeError): """Dependency installation failed before the job could be sandboxed.""" +def prepare_sandbox_filesystem( + submission_dir: Path, syftbox_folder: Path, uid: int, gid: int +) -> None: + """Make the job tree writable, and the datasite readable, by the sandbox user. + + Without this a sandboxed job fails immediately: it cannot create its + virtualenv or write ``outputs/``, and it cannot read the datasets it was + approved for. Both are needed for the job to do anything useful, and neither + weakens the lockdown -- the Drive credential and the runner's own code stay + root-owned and out of reach. + + A no-op when not running as root, which is the case in tests and on a data + owner's own machine, where the job already runs as the invoking user. + """ + if os.geteuid() != 0: + return + + # The job owns its own working tree: venv, code, outputs. + os.chown(submission_dir, uid, gid) + for root, dirs, files in os.walk(submission_dir): + for name in dirs + files: + path = os.path.join(root, name) + if not os.path.islink(path): + os.chown(path, uid, gid) + + # The datasite is read-only to the job. Ownership stays with root; we only + # open traversal and read. Every parent must be traversable or the job + # cannot reach the datasets at all -- the folder lives under /root (0700) + # by default, since the container runs as root and that is its home. + for parent in list(syftbox_folder.parents)[:-1]: + try: + os.chmod(parent, os.stat(parent).st_mode | 0o011) + except OSError: + pass + for root, dirs, files in os.walk(syftbox_folder): + for name in [root] + [os.path.join(root, n) for n in dirs + files]: + try: + mode = os.stat(name).st_mode + os.chmod(name, mode | (0o011 if os.path.isdir(name) else 0o004)) + except OSError: + pass + + def _is_submitter_code(spec: str) -> bool: """Whether a dependency string would fetch and build submitter-chosen code. @@ -479,9 +522,15 @@ def _build_command( metadata = self._get_job_metadata(ref) if metadata is None: return build_job_command(run_script) - return build_two_phase_command( + command = build_two_phase_command( submission_dir, metadata, run_script, timeout=timeout ) + # After phase A, so the venv it created is handed over too. + uid, gid = _sandbox_ids() + prepare_sandbox_filesystem( + submission_dir, Path(self.config.syftbox_folder_path_str), uid, gid + ) + return command def _execute_job_streaming(self, ref: JobRef, timeout: int) -> int: """Execute job with real-time streaming output.