[ML] Add Sandbox2 security integration for PyTorch inference

.buildkite/pipelines/build_linux.json.py

@@ -154,6 +154,25 @@ def main(args):
                     ],
                 })
             else:
+                # Regression canary for the enforced Sandbox2 coverage pinned in
+                # run_tests.sh: reports which user-namespace stage this agent
+                # denies, if any. Kept because that pin fails the build when the
+                # agent loses namespace support, and this step is what says which
+                # stage broke - the alternative is another round of guessing.
+                # Needs no build artifacts, hence no depends_on: it answers in
+                # ~1 minute rather than behind the 45 minute compile. soft_fail
+                # because it is instrumentation, not a gate.
+                pipeline_steps.append({
+                    "label": f"Diagnose user namespaces for linux-{arch} :linux:",
+                    "timeout_in_minutes": "15",
+                    "agents": test_agents[arch],
+                    "commands": [
+                      ".buildkite/scripts/steps/diagnose_userns.sh"
+                    ],
+                    "key": f"diagnose_userns_linux-{arch}",
+                    "soft_fail": True,
+                })
+
                 # aarch64: split into build and test steps
                 build_key = f"build_test_linux-{arch}-{build_type}"
 
@@ -218,8 +237,12 @@ def main(args):
                     ],
                 })
 
-    # Add debug build/test steps for PR builds to detect compilation errors with optimization disabled
-    if os.environ.get("BUILDKITE_PIPELINE_SLUG", "ml-cpp-pr-builds") != "ml-cpp-debug-build" and \
+    # Add debug build/test steps for PR builds to detect compilation errors with
+    # optimization disabled. These are x86_64 steps, so they only belong in the
+    # pipeline when x86_64 was actually requested - an aarch64-only build must
+    # not drag an x86_64 pair along with it.
+    if args.build_x86_64 and \
+            os.environ.get("BUILDKITE_PIPELINE_SLUG", "ml-cpp-pr-builds") != "ml-cpp-debug-build" and \
             os.environ.get("BUILDKITE_PULL_REQUEST", "false") != "false" and \
             not should_skip_version_bump_pr_ci():
         debug_build_key = "build_test_linux-x86_64-RelWithDebInfo-debug"

.buildkite/scripts/steps/build.sh

@@ -73,10 +73,31 @@ if [[ "$HARDWARE_ARCH" = aarch64 && -z "${CPP_CROSS_COMPILE:-}" && "$(uname)" =
     ' | tar xf -
 
     docker run --rm --workdir=/ml-cpp $TEMP_TAG bash -c '
+        set -e
+        # Complete GCC133/Boost toolchain runtime for the *host* enforced
+        # sandbox re-run. Do not enumerate individual SONAMEs — copying the
+        # whole gcc133 lib{,64} tree is the point (libstdc++, libgcc_s,
+        # libgomp, all Boost SONAMEs, and anything else the toolchain ships).
+        # Flatten into one sysroot so a single LD_LIBRARY_PATH entry beats
+        # the AlmaLinux 8 agent /lib64.
+        mkdir -p cmake-build-docker/lib/sysroot
+        if [ -d /usr/local/gcc133/lib64 ]; then
+            cp -a /usr/local/gcc133/lib64/. cmake-build-docker/lib/sysroot/
+        fi
+        if [ -d /usr/local/gcc133/lib ]; then
+            cp -a /usr/local/gcc133/lib/. cmake-build-docker/lib/sysroot/
+        fi
+        if [ ! -d /usr/local/gcc133/lib64 ] && [ ! -d /usr/local/gcc133/lib ]; then
+            echo "error: /usr/local/gcc133/lib{,64} missing from build image" >&2
+            exit 1
+        fi
         {
             find cmake-build-docker/test -name "ml_test_*" -type f -executable 2>/dev/null
-            find cmake-build-docker/lib -name "*.so" 2>/dev/null
-            find build/distribution -name "*.so" -not -path "*.debug*" 2>/dev/null
+            find cmake-build-docker/lib \( -name "*.so" -o -name "*.so.*" \) 2>/dev/null
+            find build/distribution \( -name "*.so" -o -name "*.so.*" \) -not -path "*.debug*" 2>/dev/null
+            # Sandbox2 spawn tests need the installed pytorch_inference binary.
+            find build/distribution -type f -path "*/bin/pytorch_inference" 2>/dev/null
+            find cmake-build-docker/lib/sysroot 2>/dev/null
         } | sort -u > /tmp/bundle-files.txt
         echo "Files in bundle: $(wc -l < /tmp/bundle-files.txt)" >&2
         tar czf - -T /tmp/bundle-files.txt
@@ -109,6 +130,8 @@ elif [[ "$(uname)" = "Linux" ]]; then
         find ${BUILD_DIR}/lib -name "*.so" -o -name "*.dylib" 2>/dev/null
         if [ -d "build/distribution" ]; then
             find build/distribution -type f \( -name "*.so" -o -name "*.dylib" \) -not -path "*.dSYM*" 2>/dev/null
+            # Sandbox2 spawn tests need the installed pytorch_inference binary.
+            find build/distribution -type f -path "*/bin/pytorch_inference" 2>/dev/null
         fi
     } | sort -u > /tmp/test-bundle-files.txt
 
@@ -151,6 +174,8 @@ else
         find ${BUILD_DIR}/lib -name "*.so" -o -name "*.dylib" 2>/dev/null
         if [ -d "build/distribution" ]; then
             find build/distribution -type f \( -name "*.so" -o -name "*.dylib" \) -not -path "*.dSYM*" 2>/dev/null
+            # Sandbox2 spawn tests need the installed pytorch_inference binary.
+            find build/distribution -type f -path "*/bin/pytorch_inference" 2>/dev/null
         fi
     } | sort -u > /tmp/test-bundle-files.txt
 

.buildkite/scripts/steps/diagnose_userns.sh

@@ -0,0 +1,72 @@
+#!/bin/bash
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License
+# 2.0 and the following additional limitation. Functionality enabled by the
+# files subject to the Elastic License 2.0 may only be used in production when
+# invoked by an Elasticsearch process with a license key installed that permits
+# use of machine learning features. You may not use this file except in
+# compliance with the Elastic License 2.0 and the foregoing additional
+# limitation.
+
+# Reports which runner on this agent, if any, can create the user namespace
+# Sandbox2 needs - and for the ones that cannot, the exact stage the kernel
+# denies.
+#
+# Sandbox2 requires all of: unshare(CLONE_NEWUSER), a uid/gid mapping,
+# unshare(CLONE_NEWNS), a private "/", and a fresh mount of procfs. Any one can
+# be denied independently and the failures are indistinguishable from outside,
+# which is why userns_probe.sh reports them stage by stage in each runner.
+#
+# Runs the same probe on the host and under several container configurations,
+# because the blocker differs between them: Docker's default seccomp profile
+# denies unshare(CLONE_NEWUSER) outright, while a mount covering /proc blocks
+# only the final stage. The point is to find one runner where every stage passes,
+# which is where ML_SANDBOX2_REQUIRE=enforced can then be set.
+#
+# Needs no build artifacts, so it runs with no depends_on and answers in about a
+# minute instead of waiting behind a 45 minute compile. Never fails the step:
+# this is instrumentation, not a gate.
+
+set -uo pipefail
+
+MY_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+PROBE="${MY_DIR}/userns_probe.sh"
+BASE_IMAGE="${SANDBOX2_DIAGNOSE_IMAGE:-docker.elastic.co/ml-dev/ml-linux-aarch64-native-build:17}"
+
+echo "--- Sandbox2 userns diagnosis: host"
+bash "$PROBE" host
+
+if ! command -v docker >/dev/null 2>&1; then
+    echo "warning: docker not on PATH; skipping container probes"
+    echo "--- Sandbox2 userns diagnosis: done"
+    exit 0
+fi
+
+if ! docker pull "$BASE_IMAGE" >/dev/null 2>&1; then
+    echo "warning: could not pull ${BASE_IMAGE}; skipping container probes"
+    echo "--- Sandbox2 userns diagnosis: done"
+    exit 0
+fi
+
+# label@@docker flags. Ordered least to most privileged: the cheapest
+# configuration that clears every stage is the one to adopt, since each
+# escalation widens what a compromised test process could reach.
+RUNNERS=(
+    "docker@@"
+    "docker+seccomp-unconfined@@--security-opt seccomp=unconfined"
+    "docker+seccomp+systempaths@@--security-opt seccomp=unconfined --security-opt systempaths=unconfined"
+    "docker-privileged@@--privileged"
+    "docker-privileged+systempaths@@--privileged --security-opt systempaths=unconfined"
+)
+
+for runner in "${RUNNERS[@]}"; do
+    label="${runner%%@@*}"
+    flags="${runner#*@@}"
+    echo "--- Sandbox2 userns diagnosis: ${label}"
+    # shellcheck disable=SC2086 - flags must word-split into separate arguments.
+    docker run --rm -i $flags "$BASE_IMAGE" bash -s "$label" < "$PROBE" ||
+        echo "[${label}] container probe could not run (exit $?)"
+done
+
+echo "--- Sandbox2 userns diagnosis: done"
+exit 0

.buildkite/scripts/steps/run_tests.sh

@@ -81,6 +81,71 @@ if [[ "$HARDWARE_ARCH" = aarch64 && -z "${CPP_CROSS_COMPILE:-}" && "$(uname)" =
     # Seccomp tests run inside the Docker container which shares the host
     # kernel, so the kernel's seccomp filters are exercised without needing
     # a separate outside-Docker run.
+    #
+    # The container's sandbox tests are not load-bearing for coverage: both modes
+    # are pinned on the host below, so whichever mode the container happens to
+    # select is a bonus. Coverage therefore does not depend on Docker's default
+    # seccomp profile continuing to deny unshare(CLONE_NEWUSER).
+
+    # Both coverage modes run on the host, each pinned, so neither can quietly
+    # stop executing. Measured by diagnose_userns.sh on this agent
+    # (core-almalinux-8-aarch64, kernel 4.18):
+    #
+    #   host                             all stages OK
+    #   docker (default)                 denied at unshare(CLONE_NEWUSER)
+    #   docker + seccomp=unconfined      denied at mount(proc), masked /proc paths
+    #   docker + seccomp + systempaths   all stages OK
+    #   docker --privileged              all stages OK
+    #
+    # The host needs no privilege escalation for the enforced half. Runs against
+    # the bundled gcc133 sysroot so the binary does not resolve against
+    # AlmaLinux 8 /lib64.
+    if [[ $TEST_OUTCOME -eq 0 ]]; then
+        REPO_ROOT_ABS="$(pwd)"
+        SYSROOT="$(pwd)/${BUILD_DIR}/lib/sysroot"
+        LIB_DIRS=$(find "$(pwd)/${BUILD_DIR}/lib" "$(pwd)/build/distribution" \
+            \( -name "*.so" -o -name "*.so.*" \) \
+            -exec dirname {} \; 2>/dev/null | sort -u | tr '\n' ':')
+        SANDBOX_TEST_DIR="${REPO_ROOT_ABS}/${BUILD_DIR}/test/lib/sandbox/unittest"
+        # CPP_SRC_HOME must be set: CResourceLocator::cppRootDir() otherwise
+        # falls back to "../../.." on the assumption that the cwd is a source
+        # unittest directory, whereas this runs from the build tree - so the
+        # spawn test looked for pytorch_inference under cmake-build-docker/ and
+        # did not find it. set_env.sh exports it inside the container; nothing
+        # does on the host.
+        export CPP_SRC_HOME="${REPO_ROOT_ABS}"
+        export LD_LIBRARY_PATH="${SYSROOT}:${LIB_DIRS}"
+
+        echo "--- Re-running sandbox unit tests on host (enforced)"
+        # Hardcoded, not overridable: this re-run is the enforced-coverage gate.
+        # An env-var default (ML_SANDBOX2_HOST_REQUIRE:-enforced) would let a
+        # pipeline downgrade it to fail_closed and pass required CI without ever
+        # exercising a real sandbox. If a future agent genuinely cannot run
+        # enforced on the host, diagnose_userns.sh reports which stage it denies.
+        (cd "${SANDBOX_TEST_DIR}" && \
+            ML_SANDBOX2_REQUIRE=enforced \
+            ./ml_test_sandbox) || TEST_OUTCOME=$?
+
+        # Fail-closed coverage - spawn refusal plus the kill-switch hint - is the
+        # security property that keeps an unsandboxed pytorch_inference from ever
+        # starting, so it must not depend on an external default either.
+        #
+        # user.max_user_namespaces is per-user-namespace, so setting it to 0
+        # inside a namespace we own denies every further user namespace in that
+        # subtree. That covers clone(CLONE_NEWUSER), which is what Sandbox2 uses,
+        # as well as unshare(CLONE_NEWUSER), which is what the test's probe uses;
+        # both then fail with ENOSPC. A seccomp profile denying only unshare(2)
+        # would desynchronise the two and make the probe report a capability
+        # Sandbox2 does not have. Verified on kernels 4.18 (this agent) and 7.0.
+        if [[ $TEST_OUTCOME -eq 0 ]]; then
+            echo "--- Re-running sandbox unit tests on host (fail_closed)"
+            unshare --user --map-root-user sh -c "
+                echo 0 > /proc/sys/user/max_user_namespaces || exit 1
+                cd '${SANDBOX_TEST_DIR}' || exit 1
+                ML_SANDBOX2_REQUIRE=fail_closed exec ./ml_test_sandbox
+            " || TEST_OUTCOME=$?
+        fi
+    fi
 
 else
     # --- Linux x86_64 / macOS: run tests directly ---
@@ -98,6 +163,12 @@ else
         export DYLD_LIBRARY_PATH="${LIB_DIRS}${DYLD_LIBRARY_PATH:+:$DYLD_LIBRARY_PATH}"
     fi
 
+    # Linux x86_64 PR agents are Buildkite k8s pods: user namespaces often work
+    # but Sandbox2's mount("proc", "/proc", "proc", ...) returns EPERM, so the
+    # sandbox tests self-select fail-closed coverage here. Nothing is pinned -
+    # these pods' namespace support is a property of the k8s runtime, not of this
+    # repository. macOS has no CSandboxedProcessSpawnerTest_Linux.cc at all.
+
     echo "--- Running tests"
     cmake \
         -DSOURCE_DIR="$(pwd)" \