diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7e074990..006941de 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -154,6 +154,18 @@ jobs: make -j BUILD=build NVCC="$NVCC" make check BUILD=build NVCC="$NVCC" sha256sum build/packs/*.fatbin + python tools/check_exports.py --include include --library build/libsircl.so + make check-asan BUILD=build NVCC="$NVCC" + # The CMake build of the same sources and its tests (the same suites as make check). + - name: Build libsircl with CMake and run ctest + working-directory: spark_transport/libsircl + env: + SIRCL_PACKAGE: ${{ github.workspace }}/spark_transport/sircl + run: | + cmake -S . -B build-cmake -DLIBSIRCL_NVCC="$LIBSIRCL_NVCC" \ + -DLIBSIRCL_NVCC_EXTRA_FLAGS=-DCCCL_DISABLE_CTK_COMPATIBILITY_CHECK + cmake --build build-cmake -j + ctest --test-dir build-cmake --output-on-failure lil-bridge: name: pinned LIL bridge diff --git a/docs/architecture/libsircl.md b/docs/architecture/libsircl.md index 7501d188..68bc6613 100644 --- a/docs/architecture/libsircl.md +++ b/docs/architecture/libsircl.md @@ -225,6 +225,13 @@ refuses a library without the mode. The layer records `fail_stop` in the lock when the library's bytes name `LIBSIRCL_FAIL_STOP`; the stock-image preflight reads the same mark from the host library. +The lane sets no wait variable, so every communicator runs in the startup +wait regime with libsircl's default limit of 600 s (`SIRCL_STARTUP_WAIT_S`) +for its whole life: nothing switches it to the serving regime +(`sirclSetWaitRegime`). A rank whose peer died learns of it only when one of +its waits reaches that limit, 600 s after the wait began; fail-stop then ends +the process. + ### Container settings Each rank's container gets the profile's settings with these changes @@ -237,7 +244,7 @@ Each rank's container gets the profile's settings with these changes | `LIBSIRCL_FAIL_STOP` | `1` | | `LIBSIRCL_NCCL_API_VERSION` | `22705`, libsircl's default: it passes vLLM's and torch's feature gates without opening newer ones | | `LIBSIRCL_TRANSPORT` | `verbs` | -| `LIBSIRCL_POSITION`, `SIRCL_PEER_ROUTES`, `LIBSIRCL_CHAIN_ORDER`, `LIBSIRCL_FORWARD_WINDOWS`, `SIRCL_FORWARD_CHUNK_BYTES`, `LIBSIRCL_RING_WINDOW`, `LIBSIRCL_P2P_WINDOWS`, `SIRCL_P2P_CHUNK_BYTES` | the rank's routing settings, recorded per rank in the deployment lock | +| `LIBSIRCL_POSITION`, `SIRCL_PEER_ROUTES`, `LIBSIRCL_CHAIN_ORDER`, `LIBSIRCL_RELAYED_LANES`, `LIBSIRCL_FORWARD_WINDOWS`, `SIRCL_FORWARD_CHUNK_BYTES`, `LIBSIRCL_RING_WINDOW`, `LIBSIRCL_P2P_WINDOWS`, `SIRCL_P2P_CHUNK_BYTES` | the rank's routing settings, recorded per rank in the deployment lock | | `SIRCL_BOOTSTRAP_ADDR` | the rank's `VLLM_HOST_IP` | | `SIRCL_GID_INDEX` | the profile's `NCCL_IB_GID_INDEX`, else 3 | | `LIBSIRCL_RECEIPT` | `/run/sparkring/sircl/receipts/libsircl`: one receipt per communicator in the deployment's receipt directory on each Spark | diff --git a/integrations/vllm/libsircl/sparkring_libsircl.py b/integrations/vllm/libsircl/sparkring_libsircl.py index 04d24a46..dde22e36 100644 --- a/integrations/vllm/libsircl/sparkring_libsircl.py +++ b/integrations/vllm/libsircl/sparkring_libsircl.py @@ -16,8 +16,11 @@ function PyNccl's ``NCCLLibrary`` binds. PyNccl disables itself, and vLLM falls back to torch's own NCCL, when that load or binding fails; this check makes such a fallback fail the process instead; -3. sets ``VLLM_NCCL_SO_PATH`` to the file; processes that vLLM starts later - inherit the setting. +3. sets ``VLLM_NCCL_SO_PATH`` to the file, and requires vLLM's own reading + of it (``vllm.envs.VLLM_NCCL_SO_PATH``) to name the same file, so a vLLM + whose environment values were read before the plugin ran fails instead of + loading the earlier library; processes that vLLM starts later inherit the + setting. Any other state fails the process, so a deployment that asked for libsircl never runs on another NCCL. The plugin changes nothing else: torch's @@ -26,6 +29,7 @@ """ import ctypes import hashlib +import importlib.util import json import os from pathlib import Path @@ -36,6 +40,7 @@ NAME = "libsircl" ENTRY_POINT = ("vllm.general_plugins", NAME, "sparkring_libsircl:register") _checked = {} +_identity = {} # How PyNccl loads its library (vllm/distributed/device_communicators/pynccl_wrapper.py). LOADER = ctypes.CDLL @@ -53,22 +58,43 @@ def selected(environ=None): library = Path(path) if not library.is_absolute() or library.is_symlink() or not library.is_file(): raise LibsirclSelectionError(f"{LIBRARY}={path} is not an absolute path to a regular file") - stat = library.stat() - key = (path, digest, stat.st_ino, stat.st_size, stat.st_mtime_ns) - if key not in _checked: - with library.open("rb") as stream: + if os.path.realpath(path) != path: + raise LibsirclSelectionError(f"{LIBRARY}={path} is not its own real path ({os.path.realpath(path)}): a " + "directory of it is a symbolic link, or the path is not normalized") + # The file hashed is the open file, and its device and inode identify it for identical(). + with library.open("rb") as stream: + stat = os.fstat(stream.fileno()) + key = (path, digest, stat.st_dev, stat.st_ino, stat.st_size, stat.st_mtime_ns) + if key not in _checked: found = hashlib.file_digest(stream, "sha256").hexdigest() - if found != digest: - raise LibsirclSelectionError(f"{path} has SHA-256 {found}, not {DIGEST}={digest}") - _checked[key] = True + if found != digest: + raise LibsirclSelectionError(f"{path} has SHA-256 {found}, not {DIGEST}={digest}") + _checked[key] = (stat.st_dev, stat.st_ino) + _identity[path] = _checked[key] return path +def identical(path): + """Require ``path`` to still name the file selected() hashed (same device and inode).""" + stat = os.stat(path) + if (stat.st_dev, stat.st_ino) != _identity.get(path): + raise LibsirclSelectionError(f"{path} was replaced after its SHA-256 was checked") + + def pynccl_functions(): - """The functions vLLM's PyNccl binds (``NCCLLibrary.exported_functions``), or None outside vLLM.""" + """The functions vLLM's PyNccl binds (``NCCLLibrary.exported_functions``), or None outside vLLM. Inside a + process where vLLM is importable but its PyNccl wrapper is not, the functions cannot be checked: + LibsirclSelectionError, not a check that passes with nothing to check.""" try: from vllm.distributed.device_communicators.pynccl_wrapper import NCCLLibrary - except ImportError: + except ImportError as error: + try: + importable = importlib.util.find_spec("vllm") is not None + except (ImportError, ValueError): + importable = "vllm" in __import__("sys").modules + if importable: + raise LibsirclSelectionError(f"vLLM is importable but its PyNccl wrapper is not ({error}), so the " + "functions PyNccl binds cannot be checked") from None return None return [function.name for function in NCCLLibrary.exported_functions] @@ -91,8 +117,26 @@ def check_binding(path, *, load=None, functions=None): return info +def resolved_target(): + """vLLM's own value of ``VLLM_NCCL_SO_PATH`` (``vllm.envs``), or None outside vLLM.""" + try: + import vllm.envs as envs + except ImportError: + return None + try: + return getattr(envs, TARGET) + except AttributeError: + raise LibsirclSelectionError(f"vllm.envs has no {TARGET}, so this vLLM may not load the library it " + "names") from None + + def register(): """Point vLLM's PyNccl at the checked libsircl library; vLLM may call this more than once.""" path = selected() check_binding(path) + identical(path) os.environ[TARGET] = path + found = resolved_target() + if found is not None and found != path: + raise LibsirclSelectionError(f"vLLM resolves {TARGET} to {found!r}, not {path}: it read its environment " + "before the plugin set the variable, so PyNccl would load that library") diff --git a/integrations/vllm/libsircl/test_sparkring_libsircl.py b/integrations/vllm/libsircl/test_sparkring_libsircl.py index 1d3855f2..275523b2 100644 --- a/integrations/vllm/libsircl/test_sparkring_libsircl.py +++ b/integrations/vllm/libsircl/test_sparkring_libsircl.py @@ -2,6 +2,8 @@ import hashlib import importlib.util import os +import sys +import types from pathlib import Path import pytest @@ -82,12 +84,52 @@ def load(path): plugin.check_binding("/opt/sparkring/libsircl/lib/libsircl.so.0.6.0", load=load, functions=lambda: functions) -def test_outside_vllm_only_the_identity_is_checked(): - assert plugin.pynccl_functions() is None or isinstance(plugin.pynccl_functions(), list) - info = plugin.check_binding("/lib", load=lambda path: Library(functions=()), functions=lambda: None) +WRAPPER = "vllm.distributed.device_communicators.pynccl_wrapper" + + +def test_outside_vllm_only_the_identity_is_checked(monkeypatch): + monkeypatch.delitem(sys.modules, "vllm", raising=False) + monkeypatch.setitem(sys.modules, WRAPPER, None) + monkeypatch.setattr(plugin.importlib.util, "find_spec", lambda name: None) + assert plugin.pynccl_functions() is None + info = plugin.check_binding("/lib", load=lambda path: Library(functions=()), functions=plugin.pynccl_functions) assert info["library"] == "libsircl" +def test_an_importable_vllm_whose_pynccl_wrapper_fails_to_import_is_refused(monkeypatch): + # The functions PyNccl binds cannot be listed, so the binding check must not pass with nothing to check. + monkeypatch.setitem(sys.modules, "vllm", types.ModuleType("vllm")) + monkeypatch.setitem(sys.modules, WRAPPER, None) + with pytest.raises(plugin.LibsirclSelectionError, match="PyNccl wrapper is not"): + plugin.pynccl_functions() + with pytest.raises(plugin.LibsirclSelectionError, match="PyNccl wrapper is not"): + plugin.check_binding("/lib", load=lambda path: Library(), functions=plugin.pynccl_functions) + + +def test_a_library_replaced_after_its_check_is_refused(tmp_path): + path, digest = library(tmp_path) + assert plugin.selected({plugin.LIBRARY: str(path), plugin.DIGEST: digest}) == str(path) + plugin.identical(str(path)) + other = tmp_path / "other.so" + other.write_bytes(b"ELF something else") + os.replace(other, path) + with pytest.raises(plugin.LibsirclSelectionError, match="replaced after its SHA-256 was checked"): + plugin.identical(str(path)) + + +def test_a_path_through_a_linked_directory_is_not_the_library(tmp_path): + real = tmp_path / "real" + real.mkdir() + path, digest = library(real) + linked = tmp_path / "linked" + try: + linked.symlink_to(real, target_is_directory=True) + except OSError: + pytest.skip("this host cannot create symbolic links") + with pytest.raises(plugin.LibsirclSelectionError, match="not its own real path"): + plugin.selected({plugin.LIBRARY: str(linked / path.name), plugin.DIGEST: digest}) + + def test_a_symbolic_link_is_not_the_library(tmp_path): path, digest = library(tmp_path) link = tmp_path / "link.so" @@ -104,3 +146,47 @@ def test_the_entry_point_names_the_register_function(): assert (group, name) == ("vllm.general_plugins", "libsircl") module, _, function = value.partition(":") assert module == SOURCE.stem and callable(getattr(plugin, function)) + + +class FrozenEnvs(types.ModuleType): + """A ``vllm.envs`` whose values were read before the plugin ran.""" + + def __init__(self, value): + super().__init__("vllm.envs") + self.VLLM_NCCL_SO_PATH = value + + +class LiveEnvs(types.ModuleType): + """A ``vllm.envs`` that reads the environment at each access, as vLLM does without its cache.""" + + def __init__(self): + super().__init__("vllm.envs") + + def __getattr__(self, name): + if name == plugin.TARGET: + return os.environ.get(name) + raise AttributeError(name) + + +@pytest.mark.parametrize("envs, message", [ + (LiveEnvs(), None), + (FrozenEnvs("/opt/sparkring/toolchain/nccl/lib/libnccl.so.2"), "vLLM resolves VLLM_NCCL_SO_PATH to"), + (types.ModuleType("vllm.envs"), "vllm.envs has no VLLM_NCCL_SO_PATH"), +]) +def test_register_confirms_vllm_resolves_the_library(tmp_path, monkeypatch, envs, message): + path, digest = library(tmp_path) + monkeypatch.setenv(plugin.LIBRARY, str(path)) + monkeypatch.setenv(plugin.DIGEST, digest) + monkeypatch.setenv(plugin.TARGET, "/opt/sparkring/toolchain/nccl/lib/libnccl.so.2") + monkeypatch.setattr(plugin, "LOADER", lambda name: Library()) + monkeypatch.setattr(plugin, "pynccl_functions", lambda: ["ncclAllReduce"]) + package = types.ModuleType("vllm") + package.envs = envs + monkeypatch.setitem(sys.modules, "vllm", package) + monkeypatch.setitem(sys.modules, "vllm.envs", envs) + if message is None: + plugin.register() + assert plugin.resolved_target() == str(path) + else: + with pytest.raises(plugin.LibsirclSelectionError, match=message): + plugin.register() diff --git a/runtime/common/libsircl.py b/runtime/common/libsircl.py index e0a7be96..2e475e70 100644 --- a/runtime/common/libsircl.py +++ b/runtime/common/libsircl.py @@ -89,8 +89,9 @@ # The kernel packs every libsircl build carries (runtime/images/libsircl_layer.py builds each from its CUDA # source); a layer receipt that records another set describes another libsircl build. KERNEL_PACKS = ("sircl_kernels", "sircl_fold", "sircl_links", "sircl_p2p") -ROUTE_VARIABLES = ("LIBSIRCL_POSITION", "SIRCL_PEER_ROUTES", "LIBSIRCL_CHAIN_ORDER", "LIBSIRCL_FORWARD_WINDOWS", - "SIRCL_FORWARD_CHUNK_BYTES", "LIBSIRCL_RING_WINDOW", "LIBSIRCL_P2P_WINDOWS", "SIRCL_P2P_CHUNK_BYTES") +ROUTE_VARIABLES = ("LIBSIRCL_POSITION", "SIRCL_PEER_ROUTES", "LIBSIRCL_CHAIN_ORDER", "LIBSIRCL_RELAYED_LANES", + "LIBSIRCL_FORWARD_WINDOWS", "SIRCL_FORWARD_CHUNK_BYTES", "LIBSIRCL_RING_WINDOW", "LIBSIRCL_P2P_WINDOWS", + "SIRCL_P2P_CHUNK_BYTES") LIBRARY_VARIABLE = "SPARKRING_LIBSIRCL_LIBRARY" DIGEST_VARIABLE = "SPARKRING_LIBSIRCL_SHA256" # Every communicator's receipt, .rank..c.json, in the deployment's receipt directory. diff --git a/runtime/common/test_libsircl.py b/runtime/common/test_libsircl.py index 8acc4fcb..4a433b04 100644 --- a/runtime/common/test_libsircl.py +++ b/runtime/common/test_libsircl.py @@ -8,6 +8,7 @@ import json import os from pathlib import Path +import re import subprocess import sys @@ -251,6 +252,25 @@ def test_two_pipeline_stages_and_expert_parallelism_on_grouped_all_gathers_pass( "allgather_reducescatter"]) == [] +def test_the_lane_runs_in_the_startup_wait_regime_at_the_librarys_limit(): + """The lane sets no wait variable, so every communicator runs in the startup regime at libsircl's default + limit; the architecture document's fail-stop section states that regime and limit, so the document, the + adapter and the library change together.""" + _, section = deployment(TP2, "pair", 2, [0, 1]) + environment = libsircl.library_environment(section["libsircl"]) + assert not {"LIBSIRCL_WAIT_REGIME", "SIRCL_STARTUP_WAIT_S", "SIRCL_SERVING_WAIT_S", + "LIBSIRCL_SETUP_TIMEOUT_MS"} & set(environment) + root = Path(__file__).resolve().parents[2] + engine = (root / "spark_transport/libsircl/src/engine.c").read_text(encoding="utf-8") + limit = re.search(r"#define DEFAULT_STARTUP_WAIT_S (\d+)\.0", engine).group(1) + text = (root / "docs/architecture/libsircl.md").read_text(encoding="utf-8") + fail_stop = text[text.index("### Fail-stop"):text.index("### Container settings")] + words = " ".join(fail_stop.split()) + assert "startup wait regime" in words and f"limit of {limit} s" in words + plugin = (root / "integrations/vllm/libsircl/sparkring_libsircl.py").read_text(encoding="utf-8") + assert "sirclSetWaitRegime" not in plugin + + def test_a_library_without_the_fail_stop_mode_is_refused(): value = image() value["libsircl"]["fail_stop"] = False diff --git a/spark_transport/libsircl/CMakeLists.txt b/spark_transport/libsircl/CMakeLists.txt index 77628c0a..4d75bf84 100644 --- a/spark_transport/libsircl/CMakeLists.txt +++ b/spark_transport/libsircl/CMakeLists.txt @@ -1,4 +1,8 @@ cmake_minimum_required(VERSION 3.20) +# Optimized with debug information unless a build type is given, as the Makefile's -O2 -g. +if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) + set(CMAKE_BUILD_TYPE RelWithDebInfo CACHE STRING "CMake build type" FORCE) +endif() file(STRINGS ${CMAKE_CURRENT_SOURCE_DIR}/VERSION LIBSIRCL_VERSION LIMIT_COUNT 1) project(libsircl VERSION ${LIBSIRCL_VERSION} LANGUAGES C) if(NOT UNIX OR APPLE) @@ -14,14 +18,28 @@ find_program(LIBSIRCL_NVCC NAMES nvcc HINTS ENV CUDA_HOME PATH_SUFFIXES bin PATH if(NOT LIBSIRCL_NVCC) message(FATAL_ERROR "libsircl's kernel packs need nvcc (CUDA 13.3 or later): set LIBSIRCL_NVCC") endif() +set(LIBSIRCL_NVCC_EXTRA_FLAGS "" CACHE STRING "flags added to nvcc's (for example -DCCCL_DISABLE_CTK_COMPATIBILITY_CHECK)") set(LIBSIRCL_NVCCFLAGS -O3 -std=c++17 -gencode arch=compute_120,code=sm_120 -gencode arch=compute_121,code=sm_121) +# The architectures each pack holds, for the kernel-entry check: the number of -gencode flags. +set(LIBSIRCL_ARCHITECTURES 0) +foreach(flag ${LIBSIRCL_NVCCFLAGS}) + if(flag STREQUAL "-gencode") + math(EXPR LIBSIRCL_ARCHITECTURES "${LIBSIRCL_ARCHITECTURES} + 1") + endif() +endforeach() +separate_arguments(LIBSIRCL_NVCC_EXTRA UNIX_COMMAND "${LIBSIRCL_NVCC_EXTRA_FLAGS}") +# The architectures the packs hold (the code= values), which a pack-load error names. +string(REGEX MATCHALL "code=[a-z0-9_]+" LIBSIRCL_PACK_CODES "${LIBSIRCL_NVCCFLAGS}") +string(REPLACE "code=" "" LIBSIRCL_PACK_CODES "${LIBSIRCL_PACK_CODES}") +string(REPLACE ";" " " LIBSIRCL_PACK_ARCHITECTURES "${LIBSIRCL_PACK_CODES}") file(GLOB LIBSIRCL_KERNEL_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/kernels/*.cuh) foreach(pack kernels fold links p2p) set(FATBIN ${CMAKE_CURRENT_BINARY_DIR}/packs/sircl_${pack}.fatbin) add_custom_command( OUTPUT ${FATBIN} COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_CURRENT_BINARY_DIR}/packs - COMMAND ${LIBSIRCL_NVCC} ${LIBSIRCL_NVCCFLAGS} -fatbin -o ${FATBIN} ${CMAKE_CURRENT_SOURCE_DIR}/kernels/sircl_${pack}.cu + COMMAND ${LIBSIRCL_NVCC} ${LIBSIRCL_NVCC_EXTRA} ${LIBSIRCL_NVCCFLAGS} -fatbin -o ${FATBIN} + ${CMAKE_CURRENT_SOURCE_DIR}/kernels/sircl_${pack}.cu DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/kernels/sircl_${pack}.cu ${LIBSIRCL_KERNEL_HEADERS}) add_custom_command( OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${pack}_fatbin.c @@ -30,32 +48,23 @@ foreach(pack kernels fold links p2p) DEPENDS ${FATBIN} ${CMAKE_CURRENT_SOURCE_DIR}/tools/embed_fatbin.py) endforeach() -# SIRCL's point-to-point native library is vendored byte-identical; configuration refuses a copy whose -# SHA-256 differs from the one recorded beside it. -file(SHA256 ${CMAKE_CURRENT_SOURCE_DIR}/src/transport/sircl_p2p_proxy.c P2P_PROXY_SHA256) -file(STRINGS ${CMAKE_CURRENT_SOURCE_DIR}/src/transport/sircl_p2p_proxy.c.sha256 P2P_PROXY_RECORDED LIMIT_COUNT 1) -if(NOT P2P_PROXY_SHA256 STREQUAL P2P_PROXY_RECORDED) - message(FATAL_ERROR "src/transport/sircl_p2p_proxy.c differs from the vendored SIRCL source recorded in its .sha256") -endif() -# SIRCL's native proxy, vendored byte-identical the same way. -file(SHA256 ${CMAKE_CURRENT_SOURCE_DIR}/src/transport/sircl_roce_proxy.c ROCE_PROXY_SHA256) -file(STRINGS ${CMAKE_CURRENT_SOURCE_DIR}/src/transport/sircl_roce_proxy.c.sha256 ROCE_PROXY_RECORDED LIMIT_COUNT 1) -if(NOT ROCE_PROXY_SHA256 STREQUAL ROCE_PROXY_RECORDED) - message(FATAL_ERROR "src/transport/sircl_roce_proxy.c differs from the vendored SIRCL source recorded in its .sha256") -endif() -file(STRINGS ${CMAKE_CURRENT_SOURCE_DIR}/src/transport/sircl_roce_proxy.c ROCE_LOCAL_FEATURES_LINES REGEX "roce_local_features") -if(NOT ROCE_LOCAL_FEATURES_LINES) - message(FATAL_ERROR "src/transport/sircl_roce_proxy.c predates SIRCL change LF (no roce_local_features)") -endif() # The engine's setup check of the library's local feature word (p2p_local_features bit 0: p2p_destroy counts # failed verbs calls; SIRCL change LF), on by default as in the Makefile (P2P_FEATURES). option(LIBSIRCL_P2P_FEATURES "check the point-to-point library's local feature word at setup" ON) -if(LIBSIRCL_P2P_FEATURES) - file(STRINGS ${CMAKE_CURRENT_SOURCE_DIR}/src/transport/sircl_p2p_proxy.c P2P_LOCAL_FEATURES_LINES REGEX "p2p_local_features") - if(NOT P2P_LOCAL_FEATURES_LINES) - message(FATAL_ERROR "LIBSIRCL_P2P_FEATURES: src/transport/sircl_p2p_proxy.c predates SIRCL change LF (no p2p_local_features)") - endif() -endif() +# SIRCL's native proxy and point-to-point library are vendored byte-identical (LF, SHA-256 recorded beside +# each, with change LF's feature words): checked at configuration and again before every build of the library. +include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/check_vendored.cmake) +add_custom_target(vendored_sources_guard + COMMAND ${CMAKE_COMMAND} -DSOURCE_DIR=${CMAKE_CURRENT_SOURCE_DIR} + -DP2P_FEATURES=${LIBSIRCL_P2P_FEATURES} -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/check_vendored.cmake + COMMENT "checking the vendored SIRCL native sources") + +# Hardening of every object and of the library, as in the Makefile: stack protector, fortified libc calls, +# full RELRO with immediate binding, and a non-executable stack. +# _FORTIFY_SOURCE needs an optimized build, so a Debug build (-O0) goes without it. +set(LIBSIRCL_HARDENING_COMPILE -fstack-protector-strong $<$>:-U_FORTIFY_SOURCE> + $<$>:-D_FORTIFY_SOURCE=2>) +set(LIBSIRCL_HARDENING_LINK -Wl,-z,relro,-z,now,-z,noexecstack) # SIRCL's native proxy and point-to-point library twice: over the real libibverbs (loaded at run time) and # over the shared-memory verbs stand-in (emulation), each with its own include path. @@ -64,7 +73,8 @@ add_library(transport_emu OBJECT src/transport/proxy_emu.c src/transport/p2p_emu target_include_directories(transport_emu BEFORE PRIVATE src/transport/fake_verbs src/transport) foreach(target transport_hw transport_emu) set_target_properties(${target} PROPERTIES POSITION_INDEPENDENT_CODE ON C_STANDARD 11) - target_compile_options(${target} PRIVATE -Wall -Wextra -Werror -Wno-format-truncation) + target_compile_options(${target} PRIVATE -Wall -Wextra -Werror -Wno-format-truncation ${LIBSIRCL_HARDENING_COMPILE}) + add_dependencies(${target} vendored_sources_guard) endforeach() add_library(sircl SHARED src/api.c src/bootstrap.c src/stubs.c src/aliases.c src/device_api.c src/fabric.c src/engine.c @@ -73,18 +83,21 @@ add_library(sircl SHARED src/api.c src/bootstrap.c src/stubs.c src/aliases.c src ${CMAKE_CURRENT_BINARY_DIR}/p2p_fatbin.c $ $) target_compile_features(sircl PRIVATE c_std_11) target_compile_definitions(sircl PUBLIC LIBSIRCL_CPU_ONLY NCCL_OS_LINUX - PRIVATE LIBSIRCL_VERSION="${PROJECT_VERSION}") + PRIVATE LIBSIRCL_VERSION="${PROJECT_VERSION}" + SCCL_PACK_ARCHITECTURES="${LIBSIRCL_PACK_ARCHITECTURES}") if(LIBSIRCL_P2P_FEATURES) target_compile_definitions(sircl PRIVATE SCCL_P2P_LOCAL_FEATURES) endif() target_include_directories(sircl PUBLIC include PRIVATE src) -target_compile_options(sircl PRIVATE -Wall -Wextra -Werror -Wno-format-truncation) +target_compile_options(sircl PRIVATE -Wall -Wextra -Werror -Wno-format-truncation ${LIBSIRCL_HARDENING_COMPILE}) set_source_files_properties(${CMAKE_CURRENT_BINARY_DIR}/kernels_fatbin.c ${CMAKE_CURRENT_BINARY_DIR}/fold_fatbin.c ${CMAKE_CURRENT_BINARY_DIR}/links_fatbin.c ${CMAKE_CURRENT_BINARY_DIR}/p2p_fatbin.c PROPERTIES COMPILE_OPTIONS "-O0;-w") target_link_libraries(sircl PRIVATE Threads::Threads ${CMAKE_DL_LIBS}) set_target_properties(sircl PROPERTIES NO_SONAME TRUE SUFFIX ".so.${PROJECT_VERSION}") -target_link_options(sircl PRIVATE -Wl,-soname,libnccl.so.2 -Wl,-Bsymbolic-functions -Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/exports.map) +target_link_options(sircl PRIVATE -Wl,-soname,libnccl.so.2 -Wl,-Bsymbolic-functions -Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/exports.map + ${LIBSIRCL_HARDENING_LINK}) +add_dependencies(sircl vendored_sources_guard) # The kernel-entry check (tests/check_entries.c): the pack loader runs on the embedded fatbins through a # stand-in CUDA driver (tests/fake_cuda.c) that reads their cubins offline; the library links after it @@ -102,7 +115,7 @@ target_compile_definitions(check_entries PRIVATE LIBSIRCL_CPU_ONLY NCCL_OS_LINUX target_link_libraries(check_entries PRIVATE Threads::Threads ${CMAKE_DL_LIBS}) add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/entries.ok COMMAND ${CMAKE_COMMAND} -E env LD_LIBRARY_PATH=${CMAKE_CURRENT_BINARY_DIR}/fake-cuda - $ 2 + $ ${LIBSIRCL_ARCHITECTURES} COMMAND ${CMAKE_COMMAND} -E touch ${CMAKE_CURRENT_BINARY_DIR}/entries.ok DEPENDS check_entries fake_cuda) add_custom_target(kernel_entries DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/entries.ok) @@ -113,10 +126,22 @@ add_executable(test_fabric tests/test_fabric.c src/fabric.c) target_include_directories(test_fabric PRIVATE include tests) target_compile_options(test_fabric PRIVATE -Wall -Wextra -Werror) add_test(NAME fabric COMMAND test_fabric) -foreach(name abi api lifecycle engine_cpu) +add_executable(test_link_plan tests/test_link_plan.c) +target_include_directories(test_link_plan PRIVATE src) +target_compile_options(test_link_plan PRIVATE -Wall -Wextra -Werror) +add_test(NAME link_plan COMMAND test_link_plan) +add_executable(test_kernelpack_cache tests/test_kernelpack_cache.c src/kernelpack.c src/cuda_api.c + ${CMAKE_CURRENT_BINARY_DIR}/kernels_fatbin.c ${CMAKE_CURRENT_BINARY_DIR}/fold_fatbin.c + ${CMAKE_CURRENT_BINARY_DIR}/links_fatbin.c ${CMAKE_CURRENT_BINARY_DIR}/p2p_fatbin.c) +target_include_directories(test_kernelpack_cache PRIVATE include src) +target_compile_definitions(test_kernelpack_cache PRIVATE LIBSIRCL_CPU_ONLY NCCL_OS_LINUX) +target_link_libraries(test_kernelpack_cache PRIVATE Threads::Threads ${CMAKE_DL_LIBS}) +add_test(NAME kernelpack_cache COMMAND ${CMAKE_COMMAND} -E env + LD_LIBRARY_PATH=${CMAKE_CURRENT_BINARY_DIR}/fake-cuda $) +foreach(name abi api lifecycle engine_cpu build_flags) add_test(NAME ${name} COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tests/test_${name}.py --library $) endforeach() -foreach(name bootstrap shm_verbs vendored_sources p2p_native site_routes mpi_shim check_nccl_tests) +foreach(name bootstrap shm_verbs vendored_sources p2p_native site_routes mpi_shim check_nccl_tests source_snapshot docs_consistency teardown_race) add_test(NAME ${name} COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tests/test_${name}.py) endforeach() add_test(NAME kernel_entries COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tests/test_kernel_entries.py diff --git a/spark_transport/libsircl/KERNEL_ROUTE.md b/spark_transport/libsircl/KERNEL_ROUTE.md index 16818fcf..cb41e66b 100644 --- a/spark_transport/libsircl/KERNEL_ROUTE.md +++ b/spark_transport/libsircl/KERNEL_ROUTE.md @@ -46,7 +46,9 @@ point-to-point library; they have not run in one group with SIRCL's DSL point-to `SIRCL_LINK_UNROLL`). The ring all-reduce's relay in `sircl_ring_reduce__u` stores each pack of this rank's finished piece to link 3's own slot and to the output from one load and one sum; the two-pass entries, which the library launches by default, store the slot, then copy it to the output - (`LIBSIRCL_RING_REDUCE_PASSES=1` selects the one-pass entries). Both write the same bytes and speak the same protocol, + (`LIBSIRCL_RING_REDUCE_PASSES=1` with `LIBSIRCL_RING_REDUCE_ONE_PASS_RESEARCH=1` selects the one-pass entries, and every + rank must agree). Both speak the same protocol and are designed to write the same bytes; one run of the one-pass + entries on a GB10 returned differing bits on one rank and is not isolated (`STATUS.md`), so they are research-only, and the ring all-gather's own role stores each pack to the slot and the output in one pass as well. The pair exchange entries run the ring all-gather's items and op word on a ring of two, carrying one block each way (or one way) with three roles: sending, receiving (or discarding), and copying the own block in @@ -85,13 +87,13 @@ The source holds no compiled pack. |---|---|---|---| | nvcc 13.3.73, `-O3 -std=c++17`, sm_120 + sm_121 SASS, WSL2 x86_64 | Build time, registers, spills; rebuilds from two source paths | 6.1 s for 154 cubins; 34-64 registers, 0 spill bytes; fatbin 2,714,984 bytes, SHA-256 `c2e6e5a1...be4eaa25`, identical on every rebuild | The pack is reproducible and path independent: one nvcc and one source give one pack, whatever the build directory. | | Same compiler and flags, fold pack | Build time, registers, spills; rebuilds from two source paths | 4.0 s for 120 cubins; 46-64 registers, 0 spill bytes, 0-byte stack frames; fatbin 1,388,776 bytes, SHA-256 `66da585d...9f11eee9`, identical on every rebuild | The fold pack is reproducible like the transport pack. | -| Same compiler and flags, link pack | Build time, registers, spills; rebuilds from two source paths | 55 s for 336 cubins; 56-128 registers, 0 spill bytes, 0-byte stack frames (the pair reduce exchange at unroll 4: 96 registers); fatbin 16,734,040 bytes, SHA-256 `2fb2fb0b...f9b340b7`, identical from both paths | The link pack is reproducible like the others. With a 1,024-thread launch bound (64 registers) the unroll-4 chain reduce-scatter spilled 20-44 bytes and unrolls 6-8 more; the 512-thread bound removes every spill. | +| Same compiler and flags, link pack | Build time, registers, spills; rebuilds from two source paths | 55 s for 336 cubins; 56-128 registers, 0 spill bytes, 0-byte stack frames (the pair reduce exchange at unroll 4: 96 registers); fatbin 16,734,040 bytes, SHA-256 `2fb2fb0b...f9b340b7` (an earlier revision of the link pack's source; `STATUS.md` names the packs the current sources build), identical from both paths | The link pack is reproducible like the others. With a 1,024-thread launch bound (64 registers) the unroll-4 chain reduce-scatter spilled 20-44 bytes and unrolls 6-8 more; the 512-thread bound removes every spill. | | SIRCL's DSL (nvidia-cutlass-dsl 4.5.0.dev0, NVVM 12.9) with `CUTE_DSL_KEEP_PTX`, sm_120a | Arithmetic of the dumped one-shot PTX | `add.f32` (round to nearest, no flush to zero), conversions by SIRCL's own inline PTX | The port uses `add.rn.f32` and the identical conversion PTX, so each reduction is the same IEEE computation. | | SIRCL's emulation harness unmodified (`EmulatedGroup`: ranks as threads of one process, RTX 5090, SIRCL's in-process verbs stand-in and native progress thread); per rank either SIRCL's DSL launchers or the pack launched from C with the session's own arena, counters and stream (`tests/emulation/mixed_group.py`) | 181 checks per run: all-reduce one-shot and two-shot, auto and forced, 16 B to 256 KiB, bf16, fp16, fp32; `all_reduce_large` with padded tails; reduce-scatter (scatter ops) with chunks of 16 B to 256 KiB; all-gather along dimension 0 and the last dimension, direct, padded and tiled; all-to-all; one CUDA graph per rank holding all of them, replayed for two seeds; all-DSL, mixed (even ranks C++, odd ranks DSL) and all-C++ back to back on the same sessions; a C++ rank alone under a 0.5 s serving limit | 905 of 905 passed: `path:0-1` (1 lane), `ring:3` (1 lane), `path:0-3` (2 lanes; one-block and every-block polling) and `ring:8` (2 lanes). Every output equals the host reference and the DSL output of the same inputs bit for bit, eager and in graph replay; the lone rank poisoned its session after 0.51 s and named peer 1, lane 0 and the sequence | The C++ kernels are wire-compatible with SIRCL's and reproduce the SIRCL Python session's outputs bit for bit. | | Same harness, DSL launchers | Compile time of `prepare()` | 104 s (pair) to 512 s (`ring:8`, eight ranks in one process) | The DSL compiles per (dtype, world, rank, lanes, polling) at run time; the pack compiles nothing at run time. | | DSL specializations for 2-8 ranks | Count of one-shot and two-shot cubins per architecture (rank, world, lanes and polling are compile-time constants there) | 35 (world, rank) pairs x 3 dtypes x 2 lanes x 2 polling modes x 2 algorithms = 840, before all-gather, scatter and link kernels | Cubin extraction multiplies files and ties each to one DSL version's undocumented parameter packing. | -| RTX 5090, WSL2, driver 595.79; pack kernel on a poisoned session so each launch returns at once (`tests/emulation/launch_bench.c`) | Host time per `cuLaunchKernel` from C, 20,000 launches in batches of 100 | one-shot 4.25 us mean (3.48 us best batch), two-shot 4.19 us (3.67 us) | A C launch costs what `research/nccl-compat/REPORT.md` section 6.4 measured for an empty kernel with the same parameter count on this machine (3.9 us), against 49.7 us through the DSL executor called from Python. | -| `nvidia-cutlass-dsl` licence as installed (section 1.1) | Distribution grant | Python files of the package in source form; no grant names compiled output; developer tools are for internal use unless identified as distributable | Distributing DSL-generated cubins has no stated permission; the CUDA C++ pack carries only SIRCL's Apache-2.0 terms and the b12x RoCEnante attribution (NOTICE). | +| RTX 5090, WSL2, driver 595.79; pack kernel on a poisoned session so each launch returns at once (`tests/emulation/launch_bench.c`) | Host time per `cuLaunchKernel` from C, 20,000 launches in batches of 100 | one-shot 4.25 us mean (3.48 us best batch), two-shot 4.19 us (3.67 us) | A C launch costs what an earlier measurement on this machine, outside this repository's evidence, found for an empty kernel with the same parameter count (3.9 us), against 49.7 us through the DSL executor called from Python. | +| `nvidia-cutlass-dsl` licence as installed | Distribution grant | Python files of the package in source form; no grant names compiled output; developer tools are for internal use unless identified as distributable | Distributing DSL-generated cubins has no stated permission; the CUDA C++ pack carries only SIRCL's Apache-2.0 terms and the b12x RoCEnante attribution (NOTICE). | ## Consequences @@ -99,7 +101,7 @@ The source holds no compiled pack. - Bit-exactness of every further kernel is proven the same way: the mixed-group check runs the ported kernel against SIRCL's DSL kernel in one group, on the same sessions, eager and captured. - Kernel changes inside the SIRCL package (an abort word in the timed waits, new collectives) are - ported here after they land there; both packs' hashes join every communicator's setup agreement, so + ported here after they land there; all four packs' hashes join every communicator's setup agreement, so ranks with different packs refuse to form a session. - Every kernel of SIRCL's session that a collective of the NCCL API reaches is ported; kernels SIRCL adds later (an abort word in the timed waits, new collectives) are ported after they land there. diff --git a/spark_transport/libsircl/LICENSES/rdma-core-verbs.txt b/spark_transport/libsircl/LICENSES/rdma-core-verbs.txt index 626aea21..08b8715e 100644 --- a/spark_transport/libsircl/LICENSES/rdma-core-verbs.txt +++ b/spark_transport/libsircl/LICENSES/rdma-core-verbs.txt @@ -1,7 +1,8 @@ The RDMA transport of libsircl is compiled against from rdma-core and contains code from that header's inline functions (among them ibv_post_send and ibv_poll_cq), used under the OpenIB.org -BSD license option. The notices below are copied from the header of rdma-core 50.0 on the build -workstation; a release built in another image copies that image's header. +BSD license option. The notices below are copied from the header of rdma-core 50.0. Every libsircl +image layer ships this file as it is in the source tree (runtime/images/libsircl_layer.py copies the +notices listed in runtime/common/libsircl.py), whichever rdma-core header its build compiled against. Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. Copyright (c) 2004, 2011-2012 Intel Corporation. All rights reserved. diff --git a/spark_transport/libsircl/Makefile b/spark_transport/libsircl/Makefile index 966942d5..94984f32 100644 --- a/spark_transport/libsircl/Makefile +++ b/spark_transport/libsircl/Makefile @@ -5,10 +5,17 @@ NVCC ?= nvcc BUILD ?= build # The library's version, from the one version source (VERSION). VERSION := $(shell cat VERSION) -CPPFLAGS += -Iinclude -Isrc -DLIBSIRCL_CPU_ONLY -DNCCL_OS_LINUX -DLIBSIRCL_VERSION='"$(VERSION)"' +# The flags the build needs are appended with `override`, so that CFLAGS or CPPFLAGS given on the command line +# (make CFLAGS=-O1) or in the environment replace only the optional part (the default -O2 -g) and keep them. +override CPPFLAGS += -Iinclude -Isrc -DLIBSIRCL_CPU_ONLY -DNCCL_OS_LINUX -DLIBSIRCL_VERSION='"$(VERSION)"' CFLAGS ?= -O2 -g # Diagnostic messages are bounded copies that truncate by design. -CFLAGS += -std=c11 -fPIC -Wall -Wextra -Werror -Wno-format-truncation -pthread -MMD -MP +override CFLAGS += -std=c11 -fPIC -Wall -Wextra -Werror -Wno-format-truncation -pthread -MMD -MP +# Hardening, whatever the compiler's own defaults: stack protector and fortified libc calls (which need an +# optimized build, -O1 or above), full RELRO with immediate binding and a non-executable stack for the library. +override CFLAGS += -fstack-protector-strong +override CPPFLAGS += -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2 +HARDENING_LDFLAGS = -Wl,-z,relro,-z,now,-z,noexecstack # The library: API layer, bootstrap, fabric arithmetic, session engine, CUDA # driver loader and the embedded kernel packs, plus SIRCL's native proxy and @@ -52,16 +59,20 @@ PACKS = $(FATBIN) $(FOLD_FATBIN) $(LINKS_FATBIN) $(P2P_FATBIN) P2P_PROXY = src/transport/sircl_p2p_proxy.c P2P_FEATURES ?= 1 ifeq ($(P2P_FEATURES),1) -CPPFLAGS += -DSCCL_P2P_LOCAL_FEATURES +override CPPFLAGS += -DSCCL_P2P_LOCAL_FEATURES endif NVCCFLAGS = -O3 -std=c++17 -gencode arch=compute_120,code=sm_120 -gencode arch=compute_121,code=sm_121 +# The architectures the packs hold (the code= values), which a pack-load error names. +comma := , +PACK_ARCHITECTURES = $(patsubst code=%,%,$(filter code=%,$(subst $(comma), ,$(NVCCFLAGS)))) +$(BUILD)/kernelpack.o: override CPPFLAGS += -DSCCL_PACK_ARCHITECTURES='"$(PACK_ARCHITECTURES)"' .PHONY: all check clean generate kernels emulation-tools mpi-shim all: $(BUILD)/libsircl.so $(BUILD) $(BUILD)/transport: mkdir -p $@ -$(BUILD)/%.o: src/%.c include/nccl.h src/internal.h | $(BUILD) +$(BUILD)/%.o: src/%.c include/nccl.h src/internal.h Makefile | $(BUILD) $(CC) $(CPPFLAGS) $(CFLAGS) -c $< -o $@ # SIRCL's native proxy is vendored byte-identical as well (src/transport/sircl_roce_proxy.c, SHA-256 in its # .sha256 file); the build refuses a copy of other contents. The engine checks its local feature word @@ -132,8 +143,10 @@ $(BUILD)/kernels_fatbin.o $(BUILD)/fold_fatbin.o $(BUILD)/links_fatbin.o $(BUILD # and ctypes.CDLL("libnccl.so.2") resolve. libsircl.so is a development link to it. $(BUILD)/libsircl.so: $(BUILD)/libsircl.so.$(VERSION) ln -sf libsircl.so.$(VERSION) $@ -$(BUILD)/libsircl.so.$(VERSION): $(OBJECTS) exports.map $(BUILD)/entries.ok - $(CC) -shared -pthread $(LDFLAGS) -Wl,-soname,libnccl.so.2 -Wl,-Bsymbolic-functions -Wl,--version-script=exports.map -o $@ $(OBJECTS) -ldl +# The Makefile is a prerequisite of the objects and the library, so a change of their flags rebuilds them. +$(filter $(BUILD)/transport/%.o,$(OBJECTS)): Makefile +$(BUILD)/libsircl.so.$(VERSION): $(OBJECTS) exports.map $(BUILD)/entries.ok Makefile + $(CC) -shared -pthread $(LDFLAGS) $(HARDENING_LDFLAGS) -Wl,-soname,libnccl.so.2 -Wl,-Bsymbolic-functions -Wl,--version-script=exports.map -o $@ $(OBJECTS) -ldl # The kernel-entry check (tests/check_entries.c): the library's pack loader runs on the embedded fatbins # through a stand-in CUDA driver (tests/fake_cuda.c) that reads their cubins offline, so every entry the @@ -166,20 +179,43 @@ $(BUILD)/mpi-shim/lib/libmpi.so: tools/mpi-shim/mpi_shim.c tools/mpi-shim/includ $(BUILD)/test_fabric: tests/test_fabric.c $(BUILD)/fabric.o $(CC) $(CPPFLAGS) $(CFLAGS) $^ -o $@ -check: all $(BUILD)/test_fabric - $(BUILD)/test_fabric - $(PYTHON) tests/test_abi.py --library $(abspath $(BUILD)/libsircl.so) - $(PYTHON) tests/test_api.py --library $(abspath $(BUILD)/libsircl.so) - $(PYTHON) tests/test_bootstrap.py - $(PYTHON) tests/test_lifecycle.py --library $(abspath $(BUILD)/libsircl.so) - $(PYTHON) tests/test_engine_cpu.py --library $(abspath $(BUILD)/libsircl.so) - $(PYTHON) tests/test_shm_verbs.py - $(PYTHON) tests/test_vendored_sources.py - $(PYTHON) tests/test_p2p_native.py - $(PYTHON) tests/test_site_routes.py - $(PYTHON) tests/test_mpi_shim.py - $(PYTHON) tests/test_check_nccl_tests.py - $(PYTHON) tests/test_kernel_entries.py --build $(BUILD) +$(BUILD)/test_link_plan: tests/test_link_plan.c src/link_plan.h | $(BUILD) + $(CC) $(CPPFLAGS) $(CFLAGS) $< -o $@ +$(BUILD)/test_kernelpack_cache: tests/test_kernelpack_cache.c $(BUILD)/kernelpack.o $(BUILD)/cuda_api.o \ + $(BUILD)/kernels_fatbin.o $(BUILD)/fold_fatbin.o $(BUILD)/links_fatbin.o \ + $(BUILD)/p2p_fatbin.o + $(CC) $(CPPFLAGS) $(CFLAGS) $^ -o $@ -ldl +check: all $(BUILD)/test_fabric $(BUILD)/test_link_plan $(BUILD)/test_kernelpack_cache + @status=0; \ + $(BUILD)/test_fabric || status=1; \ + $(BUILD)/test_link_plan || status=1; \ + LD_LIBRARY_PATH=$(abspath $(BUILD)/fake-cuda) $(BUILD)/test_kernelpack_cache || status=1; \ + $(PYTHON) tests/test_abi.py --library $(abspath $(BUILD)/libsircl.so) || status=1; \ + $(PYTHON) tests/test_api.py --library $(abspath $(BUILD)/libsircl.so) || status=1; \ + $(PYTHON) tests/test_bootstrap.py || status=1; \ + $(PYTHON) tests/test_lifecycle.py --library $(abspath $(BUILD)/libsircl.so) || status=1; \ + $(PYTHON) tests/test_engine_cpu.py --library $(abspath $(BUILD)/libsircl.so) || status=1; \ + $(PYTHON) tests/test_shm_verbs.py || status=1; \ + $(PYTHON) tests/test_vendored_sources.py || status=1; \ + $(PYTHON) tests/test_p2p_native.py || status=1; \ + $(PYTHON) tests/test_site_routes.py || status=1; \ + $(PYTHON) tests/test_mpi_shim.py || status=1; \ + $(PYTHON) tests/test_check_nccl_tests.py || status=1; \ + $(PYTHON) tests/test_kernel_entries.py --build $(BUILD) || status=1; $(PYTHON) tests/test_build_flags.py --library $(abspath $(BUILD)/libsircl.so) || status=1; $(PYTHON) tests/test_source_snapshot.py || status=1; \ + $(PYTHON) tests/test_docs_consistency.py || status=1; \ + $(PYTHON) tests/test_teardown_race.py || status=1; \ + exit $$status + +# The check suites against a host library built with AddressSanitizer and UndefinedBehaviorSanitizer in +# $(BUILD)-asan. The sanitizer flags reach the sub-make through the environment, so the CFLAGS line above +# still appends the required flags. The interpreters that load the library preload the ASan runtime, which +# must come first in the process; leak detection is off because the interpreter's own allocations would +# dominate it. +.PHONY: check-asan +check-asan: + ASAN_OPTIONS=detect_leaks=0 CFLAGS="-O1 -g -fsanitize=address,undefined -fno-omit-frame-pointer" LDFLAGS="-fsanitize=address,undefined" $(MAKE) BUILD=$(BUILD)-asan all + LIBSIRCL_TEST_ASAN=1 LD_PRELOAD="$$($(CC) -print-file-name=libasan.so)" ASAN_OPTIONS=detect_leaks=0 UBSAN_OPTIONS=halt_on_error=1 CFLAGS="-O1 -g -fsanitize=address,undefined -fno-omit-frame-pointer" LDFLAGS="-fsanitize=address,undefined" $(MAKE) BUILD=$(BUILD)-asan check + generate: $(PYTHON) tools/generate_api.py clean: diff --git a/spark_transport/libsircl/NOTICE b/spark_transport/libsircl/NOTICE index 3c00b48a..b94ffa4a 100644 --- a/spark_transport/libsircl/NOTICE +++ b/spark_transport/libsircl/NOTICE @@ -23,8 +23,10 @@ SIRCL components kernels, device intrinsics and progress-thread protocol derive from RoCEnante, the b12x one-shot RoCE collectives by Local Inference Lab and its contributors (https://github.com/local-inference-lab, b12x commit 236ddff04a5f5064084b46464800b70b34ec7d8c), licensed under the Apache License, - Version 2.0. vendor/SIRCL-NOTICE reproduces SIRCL's NOTICE; SOURCE_SNAPSHOT.json records the - SIRCL files this library follows. + Version 2.0. vendor/SIRCL-NOTICE reproduces SIRCL's NOTICE. SOURCE_SNAPSHOT.json records the SHA-256 + of every file of the SIRCL implementation tree that STATUS.md's evidence was taken against, a dated + capture of a development workspace; the library is built against this repository's + spark_transport/sircl and the vendored copies under src/transport. libibverbs header code The RDMA transport is compiled against from rdma-core and contains code from diff --git a/spark_transport/libsircl/README.md b/spark_transport/libsircl/README.md index cc550703..6816407f 100644 --- a/spark_transport/libsircl/README.md +++ b/spark_transport/libsircl/README.md @@ -1,13 +1,17 @@ # libsircl +Status: **research-only**. No serving or training claim rests on libsircl; `STATUS.md` states what is +implemented and the evidence for it. + libsircl is SIRCL's NCCL-compatible C API: a Linux shared library that implements the public C interface of NVIDIA NCCL 2.32 (the 73 functions of the NCCL 2.32.3 header and their `pnccl` profiling twins) on SIRCL, SparkRing's Switchless Inference RDMA Collective Layer for clusters of NVIDIA DGX Spark systems. A program written against NCCL's C API, such as PyTorch, vLLM, SGLang or nccl-tests, loads libsircl in place of NCCL's `libnccl.so.2` without modification. -Compatibility is at the C API and ABI. libsircl implements a subset of the API (`STATUS.md` lists -each function) and does not exchange data with NVIDIA NCCL processes. libsircl is an independent +Compatibility is at the C API and ABI. libsircl implements 36 of the header's 73 functions +(`tests/api_manifest.json` marks each one implemented or not; the other 37 return `ncclInvalidUsage`) and +does not exchange data with NVIDIA NCCL processes. libsircl is an independent implementation: it contains no NCCL implementation source, it is not NVIDIA NCCL, and it is not sponsored or endorsed by NVIDIA. `STATUS.md` states what is implemented and the evidence; `RUNBOOK.md` gives the build, emulation and hardware commands. @@ -39,7 +43,15 @@ gives the build, emulation and hardware commands. `ncclCommSplit`) writes one stderr line, `libsircl: communicator creation failed (): `, the text `ncclGetLastError` returns, whatever `NCCL_DEBUG` says: frameworks commonly report only `ncclGetErrorString`. With `NCCL_DEBUG` set to `WARN`, `INFO` or `TRACE`, every failed call writes - `libsircl: : `. + `libsircl: : `. A creation that fails after the handle was allocated (the bootstrap or the + session setup failed) still returns the handle, whose `ncclCommGetAsyncError` and `ncclGetLastError` give + the result and reason; the failed communicator holds no bootstrap connection or session, but the caller + must still destroy or abort the handle to free it. +- Result codes have NVIDIA NCCL's values and meaning. `ncclGetErrorString` returns short texts ("invalid + usage", "remote error", ...) without NVIDIA NCCL's hints to rerun with `NCCL_DEBUG`; the reason of a + failed call is in `ncclGetLastError`. `ncclCommDestroy(NULL)` and `ncclCommAbort(NULL)` return + `ncclSuccess`, as in NVIDIA NCCL 2.32.3; `ncclCommFinalize(NULL)` returns `ncclInvalidArgument`, where + NVIDIA NCCL 2.32.3 returns `ncclSuccess`. - `include/nccl.h` is NVIDIA's 2.32.3 header with marked changes (`NCCL_VERSION_CODE` 23203), used only to build libsircl. - The name `libsircl` without a suffix means this library; SIRCL's own test libraries are named @@ -58,7 +70,16 @@ A communicator of W ranks (1 to 8) owns one SIRCL ring session: - **Bootstrap**: a 128-byte unique id carrying the root's address and port, a nonce and an optional site hash; TCP rendezvous over loopback or the wired LAN (`SIRCL_BOOTSTRAP_ADDR`, `SIRCL_BOOTSTRAP_IFNAME` - or `NCCL_SOCKET_IFNAME`); all-gather rounds through the root for setup. + or `NCCL_SOCKET_IFNAME`); all-gather rounds through the root for setup, splits and teardown. The root + runs in the process that created the id and lives until every rank's connection to it closes (each rank's + communicator is destroyed or aborted) or, before every rank joined, until its join window ends + (`SIRCL_BOOTSTRAP_TIMEOUT_MS`, default 10 s, counted from `ncclGetUniqueId`, not from the first rank's + join; a rank that arrives later is told the root rejected the id or disconnected, and the message names + the window). A connection that has not sent its join request within 1 s of being accepted is dropped. + Each all-gather round waits at most `LIBSIRCL_SETUP_TIMEOUT_MS` (1 ms to 1 h, default 600 s); another + value is refused, naming the variable. A process holds at most 64 roots at once: `ncclGetUniqueId` + beyond that returns `ncclInvalidUsage`, naming the limit, until a communicator whose id it created is + destroyed. - **Session engine** (`src/engine.c`): settings agreed across ranks, the pinned arena, device counters, SIRCL's native progress thread (`src/transport/sircl_roce_proxy.c`, a byte-identical copy of SIRCL's `oneshot/_roce_proxy.c` with change LF, its SHA-256 checked by the build and its local feature word, @@ -152,12 +173,12 @@ without it, such a capture is refused with `ncclInvalidUsage` on the rank that n eager call of that size has run there. Point-to-point channel messages stage through a stream-ordered allocation on their channel's stream where there is one, else through their channel's own staging buffer. -Teardown: `ncclCommDestroy` and `ncclCommFinalize` of a ready communicator of two or more ranks wait -for every rank. This rank's enqueued work completes; a first bootstrap round proves that every rank's work +Teardown: `ncclCommDestroy` and `ncclCommFinalize` of a ready or revoked communicator of two or more ranks +wait for every rank. This rank's enqueued work completes; a first bootstrap round proves that every rank's work completed, so every item and flag any kernel waits for has landed (a kernel completes once it has its inbound items, while its progress thread may still owe a forward or a last item to a peer); the progress thread then stops, and a second round proves that every rank's progress thread stopped before any queue -pair, registration or arena is freed. Each round is bounded by the session's wait limit; a rank that does +pair, registration or arena is freed. Each round is bounded by the session's wait limit plus 5 s; a rank that does not arrive in time makes the call return `ncclRemoteError` after the local teardown. The first round also carries each rank's health: a rank whose own work failed (a flag wait that timed out, a failed progress thread, an earlier asynchronous error) returns that error, and every other rank returns `ncclRemoteError` @@ -165,11 +186,21 @@ naming it. The close is terminal: from its start the communicator refuses new wo its error, and a second `ncclCommFinalize` or the destroy that follows returns the same result. When the transport's queue pairs or registrations cannot be released, destroy keeps the communicator's memory allocated and returns `ncclSystemError`. Ranks destroy their communicators in the same order, as they issue -collectives. Finalize runs the rounds once and destroy then only frees; `ncclCommAbort` does not wait. +collectives. Finalize runs the rounds once and destroy then only frees; `ncclCommAbort` does not wait. When an abort finds a +kernel of the communicator still running, its memory stays allocated (quarantined) until a later communicator +creation or destroy in the process finds that kernel finished and frees it; every receipt's `quarantined_bytes` +says how much memory is held so. Finalize is +synchronous for every communicator: a non-blocking one (`blocking` 0 in its config) also runs both rounds +before `ncclCommFinalize` returns, which never returns `ncclInProgress`; a caller that must not block runs +finalize on a thread of its own. Errors of single calls: a collective or point-to-point call refused with `ncclInvalidArgument` did nothing and leaves the calling thread's group as it was (the group's other calls still run at `ncclGroupEnd`); other -errors inside a group become the group's result. A point-to-point call queued in a group holds no reference +errors of a collective, point-to-point or communicator-creation call inside a group become the group's +result. A failed call that enqueues nothing (a query, a registration, `ncclGetUniqueId`, a lifecycle call, an +unsupported function) returns its error and leaves the group as it was. Inside a group the point-to-point +calls on one communicator must use one stream and number at most 1,024; the call that breaks either rule is +refused with `ncclInvalidUsage`, which becomes the group's result. A point-to-point call queued in a group holds no reference on its communicator, so `ncclCommAbort` from another thread returns at once, and that group's `ncclGroupEnd` then fails its calls with `ncclInvalidUsage`. `ncclCommSplit` with a NULL config (or a config whose `blocking` is undefined) takes the parent's blocking mode. @@ -184,6 +215,33 @@ gets an error, never a plain sum); a reduction op that is not built in returns ` the receipt. Nothing is forwarded to another NCCL. `tests/api_manifest.json` marks every function of the header implemented or not. +## Threading, streams and graphs + +These are caller contracts; libsircl does not detect every violation. + +- One communicator is not used from several threads at once without ordering between them. The guards + for concurrent use (bootstrap rounds that run one after another, a second close that waits for the + first) make misuse fail safe, not correct. +- A stream passed to a collective or point-to-point call stays valid until the next call on that + communicator or its teardown: libsircl orders later calls after it and waits for the communicator's + work at finalize and destroy. +- CUDA graph replays of one communicator do not overlap its eager calls or each other; they share the + communicator's sequence and scratch. +- `ncclGetLastError(NULL)` returns the calling thread's last error text (NVIDIA NCCL's is per process); + `ncclGetLastError(comm)` returns the communicator's. +- `ncclConfig_t`: only `blocking` is honored. Every other field is accepted and ignored; under + `NCCL_DEBUG` (`WARN`, `INFO` or `TRACE`) the first config of the process that sets one writes one line + naming it. The `nccl*Config` collectives are unsupported. +- Groups: collectives inside a group run at the call; point-to-point calls are queued and run at the + outermost `ncclGroupEnd`, at most 1,024 per communicator, all on one stream; a group's pending + initializations start at `ncclGroupEnd`. A thread that ends inside its group fails its pending + initializations. +- `SIRCL_SITE_HASH` (64 hexadecimal digits) goes into every unique id this process creates, and a rank + refuses to join an id whose hash differs from its own, so processes of different sites never form a + communicator. +- An unsupported function returns `ncclInvalidUsage`, writes `libsircl: unsupported by this + build` to stderr once per process, and zeroes its handle and value outputs. + ## Build and test ```sh @@ -225,21 +283,22 @@ LF), and refuses a vendored copy that does not define the word; `P2P_FEATURES=0` | `LIBSIRCL_REDUCE_LINK_BLOCKS`, `LIBSIRCL_GATHER_LINK_BLOCKS`, `LIBSIRCL_SCATTER_LINK_BLOCKS` (or SIRCL's `SIRCL_REDUCE_LINK_BLOCKS`, `SIRCL_GATHER_LINK_BLOCKS`, `SIRCL_SCATTER_LINK_BLOCKS`) | blocks per link role (1-64) of every chain and ring link op of that collective (pair exchanges count as all-gathers); unset, `SIRCL_LINK_BLOCKS` when set, else the pair plan's on a pair, else 4. The chain all-reduce's blocks are `SIRCL_CHAIN_BLOCKS`. Ops of one kernel type may use different blocks: a launch's last block is the one whose arrival completes its own grid | | `LIBSIRCL_LINK_DUMP` | a diagnostic: a file prefix; one JSON line in `.rank..c.links.json` when a wait of a communicator first times out or its native progress thread fails, and at destroy: the time (CLOCK_REALTIME nanoseconds, the clock of the native event trace), the link area's words (credit, sent, ready, consumed and inbound flag words of every link), the native counters, the device link counters, the native event trace since the previous dump, whether the progress thread failed and its message, and on the emulation transport the stand-in's report (every queue pair's posted, executed and completed writes; every failed write with the resolution check that failed); after destroy, a line with the times the progress thread's stop was requested and completed and the transport was destroyed | | `LIBSIRCL_LINK_BLOCKS_CYCLE` | a test hook: `[,...]` gives successive link ops these blocks per role in turn, to exercise launches of one kernel type with different grids | -| `LIBSIRCL_RING_REDUCE_PASSES` | the ring all-reduce's relay: `2` (default) stores each pack of its finished piece to link 3's slot, then copies the slot to the output; `1` stores both in one pass (the same bytes and protocol, one read of the slot fewer; research-only, see `STATUS.md`); per rank | +| `LIBSIRCL_RING_REDUCE_PASSES` | the ring all-reduce's relay: `2` (default) stores each pack of its finished piece to link 3's slot, then copies the slot to the output; `1` stores both in one pass (the same protocol, one read of the slot fewer; research-only, see `STATUS.md`, and refused unless `LIBSIRCL_RING_REDUCE_ONE_PASS_RESEARCH=1`); agreed at setup, so every rank sets the same value | | `LIBSIRCL_LINK_TILE_BYTES` | bytes of every chunk one link reduce-scatter op carries (default 16 MiB); a call's chunks split into column tiles, which leave every element's arithmetic unchanged | | `LIBSIRCL_RING_WINDOW` | the ring plan: set on every rank when the ring that closes the chain can run, to the bytes this rank's ring lanes keep unacknowledged through relays (0 for cables); the ring schedules need it. One value per process serves every communicator it joins: a communicator whose ring next is reached through relays (a forward window toward it) while this gives 0, such as a pipeline pair of positions i and i + 4 of a ring of eight, keeps the smallest forward window toward that rank in flight on its ring lanes (the receipt's `ring_window_bytes`) | +| `LIBSIRCL_RELAYED_LANES` | the lanes that cross relays: `=[/],...` per lane, flag 1 for a relayed lane and 0 for a cable; a lane counts as relayed when this marks it or `LIBSIRCL_FORWARD_WINDOWS` gives it a window, so a relayed lane without a forward window keeps the cycle plan off and gets no unwindowed point-to-point channel. `tools/site_routes.py` prints it on every rank of a layout with relays | | `LIBSIRCL_FORWARD_WINDOWS`, `SIRCL_FORWARD_CHUNK_BYTES` | forward windows of relayed lanes: `=[/],...` per lane, 0 for a direct lane, and the chunk a windowed stripe posts in (default 32,768) | | `LIBSIRCL_P2P_CHANNELS` | `off` (default) or `on`, the same on every rank: a communicator of three or more ranks created with `on` has point-to-point channels and carries `ncclSend` and `ncclRecv` between its ranks on them (above); without them such a call is refused | | `SIRCL_P2P_SLOTS`, `SIRCL_P2P_SLOT_BYTES`, `SIRCL_P2P_BLOCKS`, `SIRCL_P2P_THREADS`, `SIRCL_P2P_UNROLL`, `SIRCL_P2P_CHUNK_BYTES` | the channels' geometry, SIRCL's names and defaults, agreed at setup: 8 slots (a power of two, 2-32) of 512 KiB (a multiple of 4096) per channel end, 4 blocks per launch (each taking every fourth item), 512 threads per block (a multiple of 32 from 64 to 512; SIRCL's kernels allow 1024), 4 packs per thread per pass, and a windowed stripe posted in chunks of 32,768 bytes | -| `LIBSIRCL_P2P_WINDOWS` | windows of this process's channel lanes through relays: `=[/],...` per lane, 0 or a multiple of 16 holding 1 to 120 chunks; a lane counts as relayed when `LIBSIRCL_FORWARD_WINDOWS` gives it a window, and a pair with a relayed lane in either direction without one has no channel. `tools/site_routes.py` prints them from SIRCL's point-to-point budget: `--p2p-reserve session` (the default, SIRCL's rule) leaves the collective session's forward windows, and with `--ring-schedules` (the site runs the ring schedules) its ring windows, their room in every relay queue first; `--session-share F` (default 1) sizes the session against F of every queue's share, so the channels get the rest; `none` sizes the channels as the only relayed traffic. On ring:8 the session's forward windows at the default share leave no relayed pair a channel; windows are whole 32 KiB chunks, so any share below 1 (0.95 to 0.05 checked) takes at least one chunk from every relayed session lane (at 0.95: 64-128 KiB become 32-96 KiB) and gives all 40 relayed ordered pairs a channel, as does `--max-window 32768` (every session lane one chunk) | +| `LIBSIRCL_P2P_WINDOWS` | windows of this process's channel lanes through relays: `=[/],...` per lane, 0 or a multiple of 16 holding 1 to 120 chunks; a lane counts as relayed when `LIBSIRCL_RELAYED_LANES` marks it or `LIBSIRCL_FORWARD_WINDOWS` gives it a window, and a pair with a relayed lane in either direction without one has no channel. `tools/site_routes.py` prints them from SIRCL's point-to-point budget: `--p2p-reserve session` (the default, SIRCL's rule) leaves the collective session's forward windows, and with `--ring-schedules` (the site runs the ring schedules) its ring windows, their room in every relay queue first; `--session-share F` (default 1) sizes the session against F of every queue's share, so the channels get the rest; `none` sizes the channels as the only relayed traffic. On ring:8 the session's forward windows at the default share leave no relayed pair a channel; windows are whole 32 KiB chunks, so any share below 1 (0.95 to 0.05 checked) takes at least one chunk from every relayed session lane (at 0.95: 64-128 KiB become 32-96 KiB) and gives all 40 relayed ordered pairs a channel, as does `--max-window 32768` (every session lane one chunk) | | `SIRCL_P2P_PROGRESS_CPU` | read by the channels' progress thread, as in SIRCL: a CPU list that pins it; unset, it runs where `LIBSIRCL_CPU_POLICY` places the transport's progress thread | | `LIBSIRCL_STREAM_ORDERED_ALLOC` | `auto` (default): staging uses stream-ordered allocation (`cuMemAllocAsync`) where the driver and device have it; `off` declines it (the communicator's own staging buffers; captured calls that need more are refused), the same on every rank, for comparisons and tests | -| `SIRCL_STARTUP_WAIT_S`, `SIRCL_SERVING_WAIT_S`, `LIBSIRCL_WAIT_REGIME` | flag-wait limits and the regime a communicator starts in (`startup`, default, or `serving`); `sirclSetWaitRegime` switches it | -| `LIBSIRCL_FAIL_STOP` | `0` (default), `1` or `abort`, per process: with `1` or `abort` the process ends at the first asynchronous error (a flag wait that timed out, a failed progress thread, a failed point-to-point channel) of any communicator of two or more ranks. The error is found by a watcher thread that checks every such communicator every `LIBSIRCL_FAIL_STOP_POLL_MS`, or by the first library call that would report it to the caller (`ncclCommGetAsyncError`, a collective or point-to-point enqueue, `ncclCommFinalize`, `ncclCommDestroy`, `ncclCommAbort`), whichever comes first. The library writes a line to stderr (`libsircl: LIBSIRCL_FAIL_STOP: ending the process at (Unix time; CLOCK_MONOTONIC s) on an asynchronous error of communicator (rank of , found by ): `) and the communicator's receipt, then ends the process: `1` at once with exit status 70 (no atexit handlers, no core dump), `abort` by `abort()` (SIGABRT, after the system's core-dump handling). A caller that only checks the codes of enqueue calls keeps a failed collective's output at most for the wait limit plus one poll, and a caller that reads the error and tears the communicator down never returns from that call; a read of the output within one poll after its stream wait returns, without a library call, is not prevented. Receipts read the error without ending the process | +| `SIRCL_STARTUP_WAIT_S`, `SIRCL_SERVING_WAIT_S`, `LIBSIRCL_WAIT_REGIME` | flag-wait limits and the regime a communicator starts in (`startup`, default, or `serving`; every rank of a communicator must start in the same regime, or setup fails naming the variable); `sirclSetWaitRegime` switches it | +| `LIBSIRCL_FAIL_STOP` | `0` (default), `1` or `abort`, per process and the same on every rank of a communicator (setup fails naming the variable otherwise): with `1` or `abort` the process ends at the first asynchronous error (a flag wait that timed out, a failed progress thread, a failed point-to-point channel) of any communicator of two or more ranks. The error is found by a watcher thread that checks every such communicator every `LIBSIRCL_FAIL_STOP_POLL_MS`, or by the first library call that would report it to the caller (`ncclCommGetAsyncError`, a collective or point-to-point enqueue, `ncclCommFinalize`, `ncclCommDestroy`, `ncclCommAbort`), whichever comes first. The library writes a line to stderr (`libsircl: LIBSIRCL_FAIL_STOP: ending the process at (Unix time; CLOCK_MONOTONIC s) on an asynchronous error of communicator (rank of , found by ): `) and the communicator's receipt, then ends the process: `1` at once with exit status 70 (no atexit handlers, no core dump), `abort` by `abort()` (SIGABRT, after the system's core-dump handling). A caller that only checks the codes of enqueue calls keeps a failed collective's output at most for the wait limit plus one poll, and a caller that reads the error and tears the communicator down never returns from that call; a read of the output within one poll after its stream wait returns, without a library call, is not prevented. Receipts read the error without ending the process | | `LIBSIRCL_FAIL_STOP_POLL_MS` | the fail-stop watcher's period, 1 to 3,600,000 ms (default 5); the tests set it long to show that the library calls alone end the process | | `SIRCL_POST_ORDER`, `SIRCL_PROGRESS_CPU`, `SIRCL_FORWARD_PROOF` | read by the native progress thread, as in SIRCL; `SIRCL_PROGRESS_CPU` pins it to a CPU list | | `LIBSIRCL_CPU_POLICY` | where the progress thread runs without `SIRCL_PROGRESS_CPU`: `performance` (default: on the fastest CPU class the creating thread may use, the Cortex-X925 cores of a GB10, found by `/proc/cpuinfo` part numbers or sysfs `cpu_capacity`) or `none` (where the scheduler puts it). The library never changes the affinity of the application's own threads; receipts name the progress thread's CPUs (`progress_cpus`) | -| `SIRCL_BOOTSTRAP_ADDR`, `SIRCL_BOOTSTRAP_IFNAME`, `NCCL_SOCKET_IFNAME` | the root's LAN address; a rank contacts a non-loopback root only when one is set | +| `SIRCL_BOOTSTRAP_ADDR`, `SIRCL_BOOTSTRAP_IFNAME`, `NCCL_SOCKET_IFNAME` | the root's LAN address; a rank contacts a non-loopback root only when one is set. An interface list (at most 255 characters) takes NCCL's syntax: `prefix` and `=name` select, `^prefix` and `^=name` exclude; a list of exclusions only never selects a loopback interface | | `SIRCL_BOOTSTRAP_TIMEOUT_MS`, `LIBSIRCL_SETUP_TIMEOUT_MS`, `LIBSIRCL_LANE_CHECK_MS` | rendezvous, setup-exchange and lane-check deadlines | | `LIBSIRCL_RECEIPT`, `LIBSIRCL_RECEIPT_INTERVAL_S` | prefix of the receipt files, one per communicator (`.rank..c.json`, n the communicator's number in the process), written when the communicator forms, refreshed after calls at most every `LIBSIRCL_RECEIPT_INTERVAL_S` seconds (default 10; 0: only at creation and destroy) and at destroy, each replaced whole; `sirclGetReceipt` returns the same JSON in process | | `LIBSIRCL_EMU_LANES`, `SIRCL_EMU_FABRIC`, `SIRCL_EMU_SEED`, `SIRCL_EMU_LATENCY_NS` | emulation transport | @@ -314,7 +373,7 @@ and their terms, and every source or binary copy carries `LICENSE`, `NOTICE`, `v unmodified in `vendor/`, adapted with marked changes in `include/nccl.h`) and the NCCL user guide; no NCCL implementation source is used. - The SIRCL implementation this library follows is SIRCL's package in this repository - (`spark_transport/sircl`, `sparkring_sircl` 0.3.1); its tests run against that package + (`spark_transport/sircl`, `sparkring_sircl` 0.3.2); its tests run against that package (`SIRCL_PACKAGE`). `SOURCE_SNAPSHOT.json` records the SHA-256 of each of the 200 files of the SIRCL tree that the evidence in `STATUS.md` was taken against (captured 2026-10-08 06:00 UTC). A change the library needs inside SIRCL is a change to `spark_transport/sircl`. diff --git a/spark_transport/libsircl/RUNBOOK.md b/spark_transport/libsircl/RUNBOOK.md index fb597393..5df82aac 100644 --- a/spark_transport/libsircl/RUNBOOK.md +++ b/spark_transport/libsircl/RUNBOOK.md @@ -1,9 +1,11 @@ # libsircl runbook Commands to build libsircl, check it in GPU emulation on the workstation, run the pair milestone on a -cabled pair of NVIDIA DGX Spark systems, and run relayed groups (a path of four, the cycle of eight). Status: sections -3.1 to 3.3 ran on the pair at positions 0-1 (`STATUS.md`, hardware evidence); sections 3.4, 3.5 and 4 -have not run. Only the operator who holds the ring measurement lock runs sections 3 and 4. +cabled pair of NVIDIA DGX Spark systems, and run relayed groups (a path of four, the cycle of eight). Status +(research-only): sections 3.1 to 3.5 ran on the cabled pairs at positions 0-1 and 6-7, and section 4's +bit-exact check, nccl-tests and timing sweeps on the path of four at positions 4-7 and the cycle of eight +(`STATUS.md`, hardware evidence). Sections 3 and 4 change the Sparks: only the ring's operator, who +schedules the ring's time, runs them. Safety classes, as in SIRCL's ring runbook: @@ -144,7 +146,8 @@ NCCL_TESTS_BUILD=/opt/nccl-tests/build bash tools/emulation_suite.sh /tmp/libsir ## 3. Pair milestone on Sparks at positions 0 and 1 (MUTATES HOST; operator only) -Conditions: the serving image (the image whose ID starts `aba309e4610c`, or its successor), host +Conditions: the serving image (the image whose ID starts `aba309e4610c`; another image gives other library +and pack bytes, which the run's records then name), host networking, GPU 0, the RDMA devices of the cabled pair, containers started the way SIRCL's ring harness starts them. `LAN0` is the wired-LAN address of the Spark at position 0 and `IFACE` the wired-LAN interface name (the site file's `lan_interface`). The route maps are SIRCL's for `path:0-1` with two @@ -237,7 +240,7 @@ python3 tests/emulation/perf_rank.py --library $LIB --world 2 --rank --id-se ``` Exit: every size checked is correct; the eager 8192-byte rows give the milestone's eager latency -measurement (criterion: at or below 20 us) and the graph rows its graph measurement, pending 3.4. +measurement (criterion: at or below 20 us) and the graph rows its graph measurement. ### 3.4 nccl-tests, unmodified, through `LD_PRELOAD` @@ -473,4 +476,6 @@ env [schedule variables] python3 tests/emulation/perf_rank.py --library --id-server $LAN0:29724 --out /tmp/sircl-ccl/perf--rank.json ``` -The library's default large-message schedule stays `pieces` until these measurements choose another. +Without schedule variables the library takes the pair plan on a cabled pair and the cycle plan on a ring that +closes over cables (`README.md`, "Pair default" and "Cycle default"), else `pieces`; these measurements +compare the schedules. diff --git a/spark_transport/libsircl/STATUS.md b/spark_transport/libsircl/STATUS.md index fe7eefe7..cd6ef0e6 100644 --- a/spark_transport/libsircl/STATUS.md +++ b/spark_transport/libsircl/STATUS.md @@ -1,7 +1,9 @@ # libsircl status -Status of the library as a whole: **implemented; verified in GPU emulation; on a cabled pair of NVIDIA -DGX Spark systems, bit-identical to the SIRCL Python session's outputs and timed.** The NCCL-compatible +Status of the library as a whole: **research-only. Implemented; verified in GPU emulation; on a cabled +pair of NVIDIA DGX Spark systems, its float16, bfloat16 and float32 sum all-reduces, all-gathers and +reduce-scatters bit-identical to the SIRCL Python session's outputs (every other datatype and op checked +against the fold model and the rank-order reference), and timed.** The NCCL-compatible API layer, bootstrap, session engine, the four kernel packs, both transports and the point-to-point channels are implemented. The pair at ring positions 0-1 ran `RUNBOOK.md` sections 3.1 to 3.5 on snapshot fb63329a, nccl-tests v2.21.1 included, @@ -24,9 +26,9 @@ hash). Identifiers and logs. A snapshot named by eight hexadecimal digits (`fb63329a`, `0f94c72b`, `a3477af2`, `e31abc5c` and others) is a copy of the library's files that its development workspace wrote, named by the SHA-256 of its file list; this directory's first commit in the repository holds snapshot -`e31abc5c`'s files. Paths under `verification/`, `../sircl-ccl-snapshots/` and `ccl-pair/` name run -logs, snapshot builds and the ring operator's gate outputs that the machines which made them keep; they -are not in this repository. +`e31abc5c`'s files. Paths under `verification/`, `../sircl-ccl-snapshots/` and `ccl-pair/`, and the ring operator's gate +script `gate_multi_0f94c72b.sh`, name run logs, snapshot builds, gate outputs and the script that the +machines which made them keep; they are not in this repository. ## Sources @@ -36,8 +38,8 @@ and the outputs of SIRCL's ring harness runs excluded) and the 39 files that cha the capture before it; at 03:00 CDT (08:00 UTC) every source file of that tree still matched the capture and no source file had been added. SIRCL's own CPU tests passed on it (503 passed, 16 skipped; `verification/sircl-cpu-tests.log`). This repository's SIRCL package is `spark_transport/sircl` -(`sparkring_sircl` 0.3.1). The library -carries byte-identical copies of two native libraries of SIRCL 0.3.1 (with change LF), in the LF-ending +(`sparkring_sircl` 0.3.2). The library +carries byte-identical copies of two native libraries of SIRCL 0.3.2 (with change LF), in the LF-ending bytes SIRCL's release builds its natives from, each SHA-256 recorded beside it and checked by the build (which also refuses carriage returns; `tests/test_vendored_sources.py`): the native proxy `oneshot/_roce_proxy.c` (`src/transport/sircl_roce_proxy.c`, `09db8715...`, the source of the serving @@ -54,14 +56,14 @@ The interface follows NVIDIA's public `nccl.h.in` v2.32.3-1 and the user guide o | Component | Status | Evidence | |---|---|---| -| Symbol surface: 73 `ncclX` host functions and 73 `pnccl` twins of the 2.32.3 header, and the 16 host entry points of NCCL's device API (`nccl_device/core.h` and its barrier and LL all-to-all headers, NCCL 2.28 and later) with their twins, which programs built against current headers import (`src/device_api.c`: `ncclCommQueryProperties` and the team queries implemented, the device communicator, device pointers and requirement builders refused); SONAME `libnccl.so.2`, version 22705; only `nccl*`, `pnccl*` and the three extension functions `sirclGetInfo`, `sirclSetWaitRegime` and `sirclGetReceipt` exported, each by exact name; `tests/api_manifest.json` marks each function of the header implemented or not; `tools/check_exports.py` lists any function a header set declares or a program imports that the library lacks | implemented | ABI suite (5 tests); engine suite's device-API test; generator check; `tools/check_exports.py`: 0 missing against the NCCL 2.29.7 headers and the nccl-tests v2.21.1 binaries built on the workstation; PyTorch and vLLM bind to it (`verification/framework-loading.json`) | -| Kernel-entry check: before the library links, its pack loader resolves every entry it names in the three embedded packs through a stand-in CUDA driver that reads every architecture's cubin offline (`tests/check_entries.c`, `tests/fake_cuda.c`; make and CMake) | implemented | kernel-entry suite (3 tests: every entry present; hiding one entry of each pack fails and names it; refused without the stand-in driver); a loader that names the two-pass ring all-reduce entries, built with link pack `5935b067...` (which has none), fails the build at `sircl_ring_reduce_two_pass_f32_u1` | +| Symbol surface: 73 `ncclX` host functions and 73 `pnccl` twins of the 2.32.3 header, and the 16 host entry points of NCCL's device API (`nccl_device/core.h` and its barrier and LL all-to-all headers, NCCL 2.28 and later) with their twins, which programs built against current headers import (`src/device_api.c`: `ncclCommQueryProperties` and the team queries implemented, the device communicator, device pointers and requirement builders refused); SONAME `libnccl.so.2`, version 22705; only `nccl*`, `pnccl*` and the three extension functions `sirclGetInfo`, `sirclSetWaitRegime` and `sirclGetReceipt` exported, each by exact name; `tests/api_manifest.json` marks each function of the header implemented or not; `tools/check_exports.py` lists any function a header set declares or a program imports that the library lacks | implemented | ABI suite; engine suite's device-API test; generator check; `tools/check_exports.py`: 0 missing against the NCCL 2.29.7 headers and the nccl-tests v2.21.1 binaries built on the workstation; PyTorch and vLLM bind to it (`verification/framework-loading.json`) | +| Kernel-entry check: before the library links, its pack loader resolves every entry it names in the four embedded packs through a stand-in CUDA driver that reads every architecture's cubin offline (`tests/check_entries.c`, `tests/fake_cuda.c`; make and CMake) | implemented | kernel-entry suite (3 tests: every entry present; hiding one entry of each pack fails and names it; refused without the stand-in driver); a loader that names the two-pass ring all-reduce entries, built with link pack `5935b067...` (which has none), fails the build at `sircl_ring_reduce_two_pass_f32_u1` | | Loading: the library links only libc and the dynamic loader, has no constructor that touches CUDA, and opens `libcuda.so.1` only when a communicator is created | implemented | CPU test in an isolated interpreter, with a negative control that initializes CUDA in a constructor; `tools/probe_cuda_on_load.py` (`verification/cuda-on-load.json`) | -| Bootstrap: unique id with root address, port, nonce and site hash; loopback, or a LAN address from `SIRCL_BOOTSTRAP_ADDR`, `SIRCL_BOOTSTRAP_IFNAME` or `NCCL_SOCKET_IFNAME`; all-gather rounds through the root; a rank leaving a complete group fails every later round at once | implemented | bootstrap suite (21 tests); between the two Sparks of the pair (hardware evidence) | -| Communicator lifecycle: blocking and non-blocking init, groups (a collective or point-to-point call refused with `ncclInvalidArgument` leaves its group as it was; a queued point-to-point call holds no reference on its communicator, so `ncclCommAbort` from another thread returns at once and that group's `ncclGroupEnd` fails the calls with `ncclInvalidUsage`), finalize and destroy (wait for every rank; see the teardown row), abort, revoke, a communicator's error and its text published together (one lock), `ncclRedOpCreatePreMulSum` (not implemented) returning `ncclNumOps`, which every collective refuses, async errors from the command ring and the progress thread; `ncclCommSplit` (members ordered by key, then parent rank; `NCCL_SPLIT_NOCOLOR` gets NULL; children keep the parent's route-map position; a NULL config, or a config whose `blocking` is undefined, takes the parent's blocking mode) | implemented | lifecycle (14) and API (17) suites, also under AddressSanitizer and UndefinedBehaviorSanitizer; GPU emulation (below): `library_rank.py`'s checks of an invalid-peer send inside a group (the group's valid send and receive complete), of `ncclCommAbort` with a send queued in another thread's open group (returns within 0.1 s; that group's end returns `ncclInvalidUsage`) and of `ncclRedOpCreatePreMulSum`'s op in an all-reduce of unequal inputs (refused with `ncclInvalidArgument`, the buffer unchanged). Abort cannot stop a kernel waiting for a dead peer before its wait limit (request AW) | +| Bootstrap: unique id with root address, port, nonce and site hash; loopback, or a LAN address from `SIRCL_BOOTSTRAP_ADDR`, `SIRCL_BOOTSTRAP_IFNAME` or `NCCL_SOCKET_IFNAME`; all-gather rounds through the root; a rank leaving a complete group fails every later round at once | implemented | bootstrap suite; between the two Sparks of the pair (hardware evidence) | +| Communicator lifecycle: blocking and non-blocking init, groups (a collective or point-to-point call refused with `ncclInvalidArgument` leaves its group as it was; a queued point-to-point call holds no reference on its communicator, so `ncclCommAbort` from another thread returns at once and that group's `ncclGroupEnd` fails the calls with `ncclInvalidUsage`), finalize and destroy (wait for every rank; see the teardown row), abort, revoke, a communicator's error and its text published together (one lock), `ncclRedOpCreatePreMulSum` (not implemented) returning `ncclNumOps`, which every collective refuses, async errors from the command ring and the progress thread; `ncclCommSplit` (members ordered by key, then parent rank; `NCCL_SPLIT_NOCOLOR` gets NULL; children keep the parent's route-map position; a NULL config, or a config whose `blocking` is undefined, takes the parent's blocking mode) | implemented | lifecycle and API suites, also under AddressSanitizer and UndefinedBehaviorSanitizer (`make check-asan`); GPU emulation (below): `library_rank.py`'s checks of an invalid-peer send inside a group (the group's valid send and receive complete), of `ncclCommAbort` with a send queued in another thread's open group (returns within 0.1 s; that group's end returns `ncclInvalidUsage`) and of `ncclRedOpCreatePreMulSum`'s op in an all-reduce of unequal inputs (refused with `ncclInvalidArgument`, the buffer unchanged). Abort cannot stop a kernel waiting for a dead peer before its wait limit (request AW) | | Communicator creation on a thread with no current CUDA context: the current CUDA device's primary context (the application's CUDA runtime's `cudaGetDevice`, else the only visible device), retained once per process and device and made current on the creating thread; refused with `ncclInvalidUsage`, naming the reason, when the device is unknown (several devices, no runtime) | implemented | Conditions: CPU, `tests/test_engine_cpu.py` on the stand-in driver (`tests/fake_cuda.c`): one device, the runtime's device 1 of two, and two devices without a runtime (refused); GPU on the workstation (RTX 5090, torch 2.12.0+cu130), `tests/emulation/init_without_context.py`: vLLM's PyNccl sequence (device selected with `torch.accelerator.set_device_index`, no allocation; `ncclCommInitRank` inside `torch.accelerator.device_index`; then a one-element float32 all-reduce of a tensor torch allocates afterwards). Measurement: the library of source tree `dbf36074` returned `ncclInvalidUsage` ("no current CUDA context") there, as the installer's `qwen38-flash-next-tp2 --transport libsircl` deployment on a cabled pair of Sparks did at PyNccl's first `ncclCommInitRank` on both ranks (image `1a8c10354eb0`, vLLM v0.1.dev21553+gab86b7073); with this change the call succeeds, torch's first allocation lands in the same primary context and the all-reduce returns 1.0. On the fabric (research run, 2026-10-10, a cabled pair of DGX Sparks at positions 0 and 1 of a ring of four; the installer's two containers of that deployment rerun unchanged except for a research library selected through `SPARKRING_LIBSIRCL_LIBRARY` and `SPARKRING_LIBSIRCL_SHA256`, each built in a container of image `1a8c10354eb0`, so with its nvcc 13.4.92 and kernel packs): source tree `dbf36074` with the creation diagnostic printed `libsircl: invalid usage: no current CUDA context: ...` on both ranks and vLLM stopped at PyNccl's first `ncclCommInitRank`; this tree's library (`cddcaa0e...`) let vLLM load, capture its CUDA graphs and serve: two known-answer chat requests at temperature 0 returned `42` (17 + 25) and `Paris`, and each rank's receipt of the tensor-parallel communicator recorded 7,783 all-reduces (3,420 captured in graphs, 5.43 GB), 133 all-gathers and one pair exchange on the verbs transport and the pair plan, healthy, no error, fail-stop on | | The reason of a failed communicator creation on stderr (`libsircl: communicator creation failed (): `, whatever `NCCL_DEBUG` says); with `NCCL_DEBUG=WARN`, `INFO` or `TRACE`, the reason of every failed call | implemented | CPU: `tests/test_engine_cpu.py` (a refused creation's line; `ncclGetVersion(NULL)` quiet without `NCCL_DEBUG`, named with `NCCL_DEBUG=WARN`) | -| Session engine: settings and both packs' hashes agreed across ranks; route maps keyed by process position (`LIBSIRCL_POSITION`, exchanged at setup), so one map serves every communicator a process joins; pinned arena (cuMemHostAlloc on hardware, shared memory registered with CUDA in emulation), device counters, SIRCL's native proxy, queue-pair connection, lane check, progress thread per communicator | implemented | GPU emulation (below); setup-failure cases; verbs transport on the pair (hardware evidence) | +| Session engine: settings and the four kernel packs' hashes agreed across ranks; route maps keyed by process position (`LIBSIRCL_POSITION`, exchanged at setup), so one map serves every communicator a process joins; pinned arena (cuMemHostAlloc on hardware, shared memory registered with CUDA in emulation), device counters, SIRCL's native proxy, queue-pair connection, lane check, progress thread per communicator | implemented | GPU emulation (below); setup-failure cases; verbs transport on the pair (hardware evidence) | | Transport kernel pack: one-shot and two-shot all-reduce, all-gather, reduce-scatter and all-to-all scatter ops in CUDA C++, sm_120 and sm_121, compiled by nvcc when the library is built; its hash, the fold pack's and the link pack's join the setup agreement | implemented | `KERNEL_ROUTE.md`: 905 of 905 mixed-group checks against SIRCL's DSL kernels; on the pair (hardware evidence) | | Fold kernel pack: rank-ordered local reduction of gathered rows, 12 datatypes x 5 built-in ops, sm_120 and sm_121, compiled when the library is built | implemented | library emulation and the pair: every datatype and op equal to the host model | | `ncclAllReduce`, every datatype, `ncclSum`/`ncclProd`/`ncclMax`/`ncclMin`/`ncclAvg`: float16, bfloat16 and float32 sums in the transport kernels (one-shot to `SIRCL_ONESHOT_MAX_BYTES`, two-shot above, pieces of `SIRCL_LARGE_PIECE_BYTES`, zero-padded tails below 16 bytes, unaligned buffers through scratch, in place); every other datatype and op as an all-gather of slot-sized tiles and the fold | implemented | library emulation, PyTorch emulation, the pair | @@ -75,41 +77,42 @@ The interface follows NVIDIA's public `nccl.h.in` v2.32.3-1 and the user guide o | One-way pair exchanges post the empty direction's items as flags only: the exchange kernel sets bit 24 of the link op word when its input is 0, and the native proxy then posts those items without payload; the peer discards them as before | implemented | Conditions: snapshot ca09d28c (SIRCL's landed proxy with this change, link pack `dc9dd167...`) against snapshot 7d1c6f19 and NVIDIA NCCL 2.32.3 on ConnectX-7 between two Sparks (pair 4-5), nccl-tests v2.21.1, bfloat16, root 0, 512 KiB to 256 MiB, out of place, two runs of this tree and of NVIDIA. Measurement (us at 256 MiB and 4 MiB): broadcast 11,008 and 11,000 / 177 and 173 against 7d1c6f19's 11,976 / 216 and NVIDIA's 11,033 and 11,029 / 208 and 190; reduce 10,993 and 10,993 / 177 and 180 against 12,447 and NVIDIA's 11,025 and 10,990 / 205 and 220; gather 5,505 and 5,495 / 90 and 90 against NVIDIA's 5,531 and 5,543 / 109 and 115; scatter 5,483 and 5,494 / 88 and 86 against NVIDIA's 5,545 and 5,548 / 99 and 104; every job exact; NIC bytes per direction over the five rows 133.1 GB against 232.7 GB without the change (NVIDIA 133.2 GB), as in research snapshot bcff23bc. The all-to-all at API level 22705 (grouped point-to-point) does not use this path: 7,229 / 157 us against NVIDIA's 6,228 and 6,215 / 166 and 176. GPU emulation on the workstation: every one-way exchange of the two-rank library run bit-exact, the same link items with fewer link bytes on both ranks; `library_rank.py`'s flags-only check on a fresh split child, three rounds of ten one-way exchanges of 1.1 to 2.8 MiB (10 to 13 pieces each against 8 link slots, so every exchange wraps the slots at a shifting place) whose empty direction alternates between the ranks (broadcasts, gathers, scatters and ncclReduce sums of roots 0 and 1, sends each way), each round followed by a two-way all-to-all and all-gather: every output exact and 33 pair exchanges for 33 calls under the pair plan on one and two lanes, and on a relayed pair with forward windows and a ring window of 393,216 bytes, which carried them in 1,151 window chunks. Conclusion: the four rooted one-way collectives match NVIDIA within 1% at 256 MiB and are faster from 4 MiB | | The all-to-all of two ranks as grouped point-to-point calls (torch's pattern; nccl-tests' below NCCL API level 22800): a send to and a receive from the peer and a send to and a receive from the rank itself of one size, the receive blocks adjacent in rank order and neither send block overlapping them, carried as one pair exchange whose kernel copies the own block in parallel with the transfer (otherwise a copy ahead of the exchange) | implemented | GPU emulation on the workstation: `library_rank.py` on two ranks, torch's pattern of 1 MiB blocks under the pair plan exact with one pair exchange and no separate local copy, and exact under the pieces schedule (no pair plan, the copy ahead); the emulation suite's nccl-tests lines (11, `alltoall_perf` included) exit 0 with `#wrong` 0. On ConnectX-7 between two Sparks (pair 4-5; snapshot b75d4b26 = this change on ca09d28c, against ca09d28c and NVIDIA NCCL 2.32.3; nccl-tests v2.21.1 `alltoall_perf` at API level 22705, bfloat16, 512 KiB to 256 MiB, out of place, two runs each; us at 256 / 16 / 4 MiB): 6,147 and 6,139 / 428 and 423 / 141 and 142 against ca09d28c's 7,217 and 7,266 / 493 and 493 / 159 and 159 and NVIDIA's 6,216 and 6,220 / 455 and 447 / 163 and 177; every job exact; broadcast, reduce, gather and scatter unchanged within the runs' spread. Conclusion: the all-to-all through torch's grouped point-to-point path is 1% faster than NVIDIA at 256 MiB, 5-7% at 16 MiB and 13-20% at 4 MiB | | Shared-memory verbs stand-in (`src/transport/shm_verbs.c`): the verbs subset of SIRCL's proxy across processes, with seeded interleaving across queue pairs; segments named by process id and process start time, so containers that share the host's `/dev/shm` with reused process ids never share a name, and a name that exists already is skipped (never removed); a report of every queue pair and failed write for link dumps; the emulation runners remove the segments of their ended rank processes | research-only (test infrastructure) | CPU suite across processes (5 tests, one with stale segments under the names the ranks try first: the group runs exact and leaves them untouched; with the skip disabled the same test fails with `shm_open: File exists`); every library emulation run | -| MPI shim for nccl-tests (`tools/mpi-shim`: every MPI call and constant of nccl-tests v2.21.1, including `MPI_Allgatherv`, `MPI_Reduce`, `MPI_Error_string`, splits of any color and `MPI_UNDEFINED`, splits of splits, collectives among a communicator's members over a full TCP mesh; a job tag, `SIRCL_MPI_JOB`, in every hello, so processes of different jobs refuse each other; a watchdog that ends the process when a peer's process ends outside `MPI_Finalize`; with `SIRCL_MPI_DISTINCT_HOSTS=1`, a host name per rank, so nccl-tests on one host in emulation counts one rank per host and every rank uses device 0), nccl-tests driver and evaluator (`tools/nccl_tests_pair.sh`: one shim job and a time limit per test line, the ring harness's CPU placement, a job manifest `jobs.tsv` per rank and exit status 1 when any line failed; `tools/check_nccl_tests.py`: the manifests of every rank, the same jobs on every rank and, given the lines file, every expected job, exit 0 per job, every job's sweep complete, `#wrong` 0 with an all-to-all's in-place `N/A` counted as not covered, receipts whose all-reduce ops (transport, fold, chain, ring) cover their calls), all-reduce timing sweep (`tests/emulation/perf_rank.py`) | research-only (test infrastructure) | MPI shim CPU suite (6 tests, 1 to 4 processes: a peer that exits after `MPI_Init` ends the other within 15 s, ranks of different jobs never join, each rank's host name under `SIRCL_MPI_DISTINCT_HOSTS`); the emulation suite's nccl-tests section on the workstation (two processes, one GPU, 11 lines, every line exit 0 on both ranks and `Out of bounds values : 0 OK`); nccl-tests v2.21.1 (commit `afd59ab`) compiles and links against the shim on the workstation with every test binary, `comm_ops_perf` included (nvcc 13.3, NCCL 2.29.7 headers, sm_121; its GIN device-API tests need NCCL 2.30.7 headers and were not built, and their MPI calls compile and run as C++ against the shim on two processes); evaluator on synthetic outputs (`tests/test_check_nccl_tests.py`, 6 cases: a complete pair, a missing manifest, a partial log with a failed exit, a job missing from the lines file, ranks with different jobs, an in-place `N/A` outside all-to-all); timing sweep in emulation (below) | +| MPI shim for nccl-tests (`tools/mpi-shim`: every MPI call and constant of nccl-tests v2.21.1, including `MPI_Allgatherv`, `MPI_Reduce`, `MPI_Error_string`, splits of any color and `MPI_UNDEFINED`, splits of splits, collectives among a communicator's members over a full TCP mesh; a job tag, `SIRCL_MPI_JOB`, in every hello, so processes of different jobs refuse each other; a watchdog that ends the process when a peer's process ends outside `MPI_Finalize`; with `SIRCL_MPI_DISTINCT_HOSTS=1`, a host name per rank, so nccl-tests on one host in emulation counts one rank per host and every rank uses device 0), nccl-tests driver and evaluator (`tools/nccl_tests_pair.sh`: one shim job and a time limit per test line, the ring harness's CPU placement, a job manifest `jobs.tsv` per rank and exit status 1 when any line failed; `tools/check_nccl_tests.py`: the manifests of every rank, the same jobs on every rank and, given the lines file, every expected job, exit 0 per job, every job's sweep complete, `#wrong` 0 with an all-to-all's in-place `N/A` counted as not covered, receipts whose all-reduce ops (transport, fold, chain, ring) cover their calls), all-reduce timing sweep (`tests/emulation/perf_rank.py`) | research-only (test infrastructure) | MPI shim CPU suite (6 tests, 1 to 4 processes: a peer that exits after `MPI_Init` ends the other within 15 s, ranks of different jobs never join, each rank's host name under `SIRCL_MPI_DISTINCT_HOSTS`); the emulation suite's nccl-tests section on the workstation (two processes, one GPU, 11 lines, every line exit 0 on both ranks and `Out of bounds values : 0 OK`); nccl-tests v2.21.1 (commit `afd59ab`) compiles and links against the shim on the workstation with every test binary, `comm_ops_perf` included (nvcc 13.3, NCCL 2.29.7 headers, sm_121; its GIN device-API tests need NCCL 2.30.7 headers and were not built, and their MPI calls compile and run as C++ against the shim on two processes); evaluator on synthetic outputs (`tests/test_check_nccl_tests.py`, among them a complete pair, a missing manifest, a partial log with a failed exit, a job missing from the lines file, ranks with different jobs, an in-place `N/A` outside all-to-all); timing sweep in emulation (below) | | Link kernel pack (`kernels/sircl_links.cu`, `dc9dd167...`): SIRCL's chain all-reduce and link collectives in CUDA C++ (chain all-gather and reduce-scatter over links 0 and 1; ring all-gather, reduce-scatter and all-reduce over links 2 and 3 with both staggers), chain position and rank order as launch parameters, unroll 1-8, at most 512 threads per block (up to 128 registers per thread, no spills), sm_120 and sm_121, compiled when the library is built; the ring all-reduce's relay in the two-pass form (default: the slot, then the output) and in one pass (`LIBSIRCL_RING_REDUCE_PASSES=1`); the pair exchange entries; a launch's last block is the one whose arrival completes its own grid, so link ops of one kernel type may differ in blocks per role | implemented; the one-pass relay: research-only (on a GB10, one run under forward windows on four ranks returned differing bits on one rank of a split communicator; not yet isolated) | mixed groups against SIRCL's DSL kernels on `path:0-1`, `path:0-3` and `ring:8` of the two-pass pack `5935b067...`: chain and ring schedules and SIRCL's own link checks, every check passed (below); the one-pass pack's mixed groups are queued | | Point-to-point kernel pack (`kernels/sircl_p2p.cu`, `0f39a3b9...`): the send and receive kernels of SIRCL's point-to-point channels (`p2p/_kernels.py`) in CUDA C++, one entry per direction and unroll 1-8, threads, lanes, slots, slot bytes and the native layer's block offsets as launch parameters (`sccl_p2p_args`), at most 512 threads per block (44 to 96 registers, no spills), sm_120 and sm_121, compiled when the library is built; its hash joins the setup agreement | implemented | GPU emulation against SIRCL's native point-to-point library (below, "Point-to-point channels"); not run in one group with SIRCL's DSL point-to-point kernels | -| The cycle plan (README.md, "Cycle default"): a communicator of three or more ranks without `SIRCL_LARGE_SCHEDULE`, given a ring plan, whose ring closes over cables only (every rank's relayed positions exchanged with the ranks' positions, the decision compared with the settings), runs all-reduces, all-gathers and reduce-scatters from 8 MiB as one ring op each and the pieces below; the receipt's `cycle_plan` | implemented | GPU emulation (`tools/emulation_suite.sh`): four and eight ranks with a ring plan and no relays take the plan and every check of `library_rank.py` passes against the pieces digests below 8 MiB and the ring digests from it; four ranks whose ring closes through relays, and four with `SIRCL_LARGE_SCHEDULE=pieces`, do not take it; with every rank's own planner settings the channel test's ring:8 layouts take it and path:0-3 does not. The measurement behind it: the ring schedules overtake the pieces between 4 and 8 MiB on the cycle of eight Sparks (snapshot a3477af2, hardware evidence). The plan itself has not run on Sparks | +| The cycle plan (README.md, "Cycle default"): a communicator of three or more ranks without `SIRCL_LARGE_SCHEDULE`, given a ring plan, whose ring closes over cables only (every rank's relayed positions exchanged with the ranks' positions, the decision compared with the settings), runs all-reduces, all-gathers and reduce-scatters from 8 MiB as one ring op each and the pieces below; the receipt's `cycle_plan` | implemented | GPU emulation (`tools/emulation_suite.sh`): four and eight ranks with a ring plan and no relays take the plan and every check of `library_rank.py` passes against the pieces digests below 8 MiB and the ring digests from it; four ranks whose ring closes through relays, and four with `SIRCL_LARGE_SCHEDULE=pieces`, do not take it; with every rank's own planner settings the channel test's ring:8 layouts take it and path:0-3 does not. The measurement behind it: the ring schedules overtake the pieces between 4 and 8 MiB on the cycle of eight Sparks (snapshot a3477af2, hardware evidence). On Sparks, the image gate of image `1a8c10354eb0`'s library on the cycle of eight recorded every eight-rank communicator without a schedule setting taking the plan (96 communicators; [record](../../performance/records/transport/libsircl-ring8-image-1a8c10354eb0-20261009.md)) | | Schedules of large messages and relayed groups, SIRCL's plan: `SIRCL_LARGE_SCHEDULE` (all-reduce: one ring op for the largest prefix of W equal chunks, or one chain op for the 16-byte-aligned body), `SIRCL_GATHER_SCHEDULE` (one chain or ring all-gather for 16-byte-multiple shards), `SIRCL_SCATTER_SCHEDULE` (chain or ring reduce-scatter of 16-byte-multiple chunks, in column tiles of `LIBSIRCL_LINK_TILE_BYTES`); each `pieces`, `chain`, `auto` (from SIRCL's per-collective chain minimums) or `ring` (from its ring minimums, else as `auto`); unset, `pieces` on groups of three or more and the pair plan on two ranks (a cabled pair, one block per link role: ring all-reduces from 2 MiB in 256 KiB pieces, ring all-gathers from 1 MiB shards with pieces by shard size, ring reduce-scatters from 4 MiB of input, and pair exchanges for all-to-alls, point-to-point exchanges of equal or one-way sizes, broadcasts, scatters, gathers and ncclReduce sums from 1 MiB; a relayed pair: the chain from 8 MiB; README.md, "Pair default"); blocks per link role per collective and per call (`LIBSIRCL_*_LINK_BLOCKS`, `SIRCL_LINK_BLOCKS`, the plan), agreed at setup; decided from agreed settings and sizes only, with unaligned and in-place buffers staged, so every rank runs the same ops; chain order from `LIBSIRCL_CHAIN_ORDER` or the ranks by position | implemented | CPU suites; GPU emulation (below); the pair default on the cabled pair at positions 6-7 (snapshot 0f94c72b, hardware evidence): every nccl-tests row of `RUNBOOK.md` sections 3.4 and 3.5 correct. Its sizes follow SIRCL's ring harness on two cabled pairs (section 3.6); libsircl's large-message times under the pair default have not been compared on Sparks with its other schedules, the ring harness or NVIDIA NCCL | | Op selection from shared state only: a call's schedule, op kinds and count, pieces, blocks per role and link op words follow from its shape, datatype, root and the settings agreed at setup; a rank's buffer alignment, in-place or overlapping buffers, a discarded output (off the root) and its staging state choose only local copies through scratch or staging around the same ops. The staging buffer grows outside capture with every earlier buffer kept until destroy (no limit on the number of sizes); under CUDA graph capture a call that needs more staging than the communicator holds takes a graph allocation on the capturing stream (`cuMemAllocAsync`), freed on that stream when the call returns, so the graph owns it (README.md, "Op selection and staging") | implemented | Conditions: GPU emulation on the workstation; `library_rank.py`'s rank-local staging check on a fresh split child: the collectives of every selection point (one-shot, pieces and padded tiles; chain, ring and link ops; pair exchanges of roots 0 and 1; `ncclAlltoAll`; on two ranks torch's all-to-all pattern and one-way sends each way), with every rank's buffers aligned, then rank 1's unaligned, then rank 1's in place, then captured cold into one graph on a second fresh child with rank 1's unaligned and replayed twice; and the staging growth check, 19 all-gathers of doubling shards from 16 B to 4 MiB with rank 1 unaligned. Measurement (the runs under "Op selection under rank-local staging, and alternating flags-only directions" below): in every run each rank's receipt counts of launched ops (pair exchanges and their bytes, link ops by kind and by blocks per role, transport, chain and point-to-point ops) and of carried transfers (native link ops and items, one-shot ops and phases) equal its aligned run's, and the launched counts are the same on every rank; every output exact; the cold capture is taken on every rank, each of rank 1's staged calls as a graph allocation (16 on the pair default, 20 under the ring schedule on two ranks), and both replays are exact; rank 1 keeps 19 staging buffers under the chain and ring schedules. The same checks against snapshot ba5a337b's library, with a 15 s wait limit: under the pair default, rank 1 refused its cold capture of the staged calls with `ncclInvalidUsage` while rank 0 captured them, rank 0's replays returned wrong outputs at the wait limit and the child's destroy failed; under the ring schedule, rank 1 also refused the 18th and 19th all-gathers of the growth check (`ncclInternalError`, too many staging buffer sizes) while rank 0 launched them, and rank 0's session was poisoned at the wait limit. Conclusion: in these runs a rank's alignment, in-place buffers, role and staging state change no rank's ops | | Progress thread placement: `SIRCL_PROGRESS_CPU` pins it; otherwise `LIBSIRCL_CPU_POLICY=performance` (default) starts it on the fastest CPU class the creating thread may use (GB10: the Cortex-X925 cores), `none` leaves it to the scheduler; the application's threads keep their affinity; receipts name the thread's CPUs (`progress_cpus`) | implemented | build and CPU suites (the workstation's CPUs are of one class); on GB10s (Sparks 6-7, snapshot 0f94c72b, hardware evidence) the default policy started the thread on CPUs 5-9 and 15-19 and `SIRCL_PROGRESS_CPU=19` pinned it to CPU 19, as each receipt's `progress_cpus` records | | The chain and link areas in the arena after the control line, the link counters, piece counters and decision words, SIRCL's link settings (`SIRCL_LINK_*`, staggers) agreed at setup; the ring plan from `LIBSIRCL_RING_WINDOW`; forward windows of relayed lanes from `LIBSIRCL_FORWARD_WINDOWS`; `tools/site_routes.py` prints every rank's route map, chain order, forward windows and ring plan from SIRCL's route planner | implemented | CPU suites; setup-failure cases; `tools/site_routes.py` reproduces the pair's route maps and gives `path:0-3` a ring window of 393,216 bytes on position 3 | -| Teardown: `ncclCommDestroy` and `ncclCommFinalize` of a communicator of two or more ranks wait for this rank's work, then a bootstrap round of every rank that also carries each rank's health, then stop the progress thread, then a second round, before any queue pair, registration or arena is freed; each round bounded by the wait limit; the close is terminal (new work refused, its result kept); a failed release of the transport keeps the memory allocated; `ncclCommAbort` does not wait | implemented | Conditions: GPU emulation on the workstation and on GB10s (snapshot 7d1c6f19, built in the serving image); `tests/emulation/teardown_race.py`: a reversed `ncclCommSplit` child, one ring all-reduce, `ncclCommDestroy` at once, repeated, one rank's writes delayed 5 ms so its predecessor finishes first; with `--expect close-error`, a rank 4 s late under a 2 s wait limit, or the stand-in's deregistrations failed (`SIRCL_EMU_FAIL_DEREG=1`). Measurement: four ranks and two ranks at 8 MiB per rank, 20 rounds each, every round exact with no async error and the parent healthy (GB10: four ranks, 5 runs of 20 rounds, 0 failed); without the rounds (snapshot 6fe2882f) the same check failed 7 of 10 rounds on four ranks on the workstation and 98 of 100 on a GB10 (each failure poisons the parent): a rank's kernel had completed while its progress thread still held an outbound item gated by a peer's credit, destroy stopped that thread and freed its queue pairs, the peer waited out the wait limit, and the peer's late credit write failed (transport retry counter exceeded; the destination queue pair was gone). In close-error mode (workstation and GB10) every round's first finalize or destroy returned the error, a second finalize the same, and a collective after the close was refused. The full emulation suite on a GB10 (snapshot 7d1c6f19) passed every section, the teardown check on four ranks and on two at 8 MiB included. On the fabric (ConnectX-7, four Sparks on a path and a cabled pair, 8 MiB per rank, no delayed rank): 100 rounds each without the rounds and 60 with them (snapshot 7d1c6f19), all exact and healthy, and 10 runs of the relayed-group bit-exact check under the ring schedules (RUNBOOK 4.1) on the path, 0 failed (Sparks 0-3 and 0-1, `ccl-pair/done-hw-teardown-7d1-20261008T194629Z`); the same 60 rounds per layout and 10 checks at snapshot ba5a337b on Sparks 4-7 and 4-5, 0 failed (`ccl-pair/hw-teardown-ba5-47-20261008T225027Z`); the stranded item did not occur at natural timing there; on the cabled pair at positions 6-7 (snapshot 0f94c72b, the rounds of this tree, 2026-10-09): 6 runs of 20 rounds at 8 MiB, 3 with fail-stop off and 3 on, 0 failed, the parent exact and healthy. Result: round 1 proves that every item and flag a kernel waits for has landed; round 2 that no rank posts toward queue pairs or regions about to be freed. Writes no kernel waits for (credits) may still be in flight; each rank destroys its queue pairs before its registrations and memory, so they land in registered memory or are dropped. Limits: the rounds are bounded, but this rank's own wait for its work, the progress thread's join and the verbs destroys are not; in emulation the stand-in resolves a write under its registry lock and applies it after, so a write resolved before a deregistration still lands in the old segment, and emulation does not test a NIC's revocation of a deregistered region. Callers: torch 2.10's `destroy_process_group` shuts its groups down in one order on every rank (finalize, then destroy); vLLM 0.19.1's pynccl `destroy` calls `ncclCommDestroy`; a caller that tears down with `ncclCommAbort` runs no rounds | +| Teardown: `ncclCommDestroy` and `ncclCommFinalize` of a communicator of two or more ranks (also after `ncclCommRevoke`) wait for this rank's work, then a bootstrap round of every rank that also carries each rank's health, then stop the progress thread, then a second round, before any queue pair, registration or arena is freed; each round bounded by the wait limit plus 5 s; the close is terminal (new work refused, its result kept); a failed release of the transport keeps the memory allocated; `ncclCommAbort` does not wait | implemented | Conditions: GPU emulation on the workstation and on GB10s (snapshot 7d1c6f19, built in the serving image); `tests/emulation/teardown_race.py`: a reversed `ncclCommSplit` child, one ring all-reduce, `ncclCommDestroy` at once, repeated, one rank's writes delayed 5 ms so its predecessor finishes first; with `--expect close-error`, a rank 4 s late under a 2 s wait limit, or the stand-in's deregistrations failed (`SIRCL_EMU_FAIL_DEREG=1`). Measurement: four ranks and two ranks at 8 MiB per rank, 20 rounds each, every round exact with no async error and the parent healthy (GB10: four ranks, 5 runs of 20 rounds, 0 failed); without the rounds (snapshot 6fe2882f) the same check failed 7 of 10 rounds on four ranks on the workstation and 98 of 100 on a GB10 (each failure poisons the parent): a rank's kernel had completed while its progress thread still held an outbound item gated by a peer's credit, destroy stopped that thread and freed its queue pairs, the peer waited out the wait limit, and the peer's late credit write failed (transport retry counter exceeded; the destination queue pair was gone). In close-error mode (workstation and GB10) every round's first finalize or destroy returned the error, a second finalize the same, and a collective after the close was refused. The full emulation suite on a GB10 (snapshot 7d1c6f19) passed every section, the teardown check on four ranks and on two at 8 MiB included. On the fabric (ConnectX-7, four Sparks on a path and a cabled pair, 8 MiB per rank, no delayed rank): 100 rounds each without the rounds and 60 with them (snapshot 7d1c6f19), all exact and healthy, and 10 runs of the relayed-group bit-exact check under the ring schedules (RUNBOOK 4.1) on the path, 0 failed (Sparks 0-3 and 0-1, `ccl-pair/done-hw-teardown-7d1-20261008T194629Z`); the same 60 rounds per layout and 10 checks at snapshot ba5a337b on Sparks 4-7 and 4-5, 0 failed (`ccl-pair/hw-teardown-ba5-47-20261008T225027Z`); the stranded item did not occur at natural timing there; on the cabled pair at positions 6-7 (snapshot 0f94c72b, the rounds of this tree, 2026-10-09): 6 runs of 20 rounds at 8 MiB, 3 with fail-stop off and 3 on, 0 failed, the parent exact and healthy. Result: round 1 proves that every item and flag a kernel waits for has landed; round 2 that no rank posts toward queue pairs or regions about to be freed. Writes no kernel waits for (credits) may still be in flight; each rank destroys its queue pairs before its registrations and memory, so they land in registered memory or are dropped. Limits: the rounds are bounded, but this rank's own wait for its work, the progress thread's join and the verbs destroys are not; in emulation the stand-in resolves a write under its registry lock and applies it after, so a write resolved before a deregistration still lands in the old segment, and emulation does not test a NIC's revocation of a deregistered region. Callers: torch 2.10's `destroy_process_group` shuts its groups down in one order on every rank (finalize, then destroy); vLLM 0.19.1's pynccl `destroy` calls `ncclCommDestroy`; a caller that tears down with `ncclCommAbort` runs no rounds | | Destroying or finalizing communicators in a different order on different ranks | unsupported | each teardown round waits for every rank of its communicator, so crossed orders wait out the wait limit on each round (20 s serving, 600 s startup) and return `ncclRemoteError`; they do not wait longer | -| Process exit without `ncclCommDestroy`, `ncclCommFinalize` or `ncclCommAbort` while a peer's collective on the communicator still runs | unsupported | at library unload only the native threads stop; a peer may wait out its wait limit | +| Process exit without `ncclCommDestroy`, `ncclCommFinalize` or `ncclCommAbort` while a peer's collective on the communicator still runs | unsupported | at library unload only the native threads stop; a peer may wait out its wait limit. Calls still running on other threads are cancelled, and the unload waits up to 1 s for them to leave before it frees their communicators; a communicator still in use after that stays allocated (CPU: `tests/test_lifecycle.py`, exit and `dlclose` with a split in progress, also under `make check-asan`) | | `ncclCommDestroy` while replays of captured CUDA graphs of the communicator's collectives still run | unsupported (caller contract) | teardown waits for the engine's last eager launch (a launch under capture records no stream), not for graph replays; a communicator used only under capture has nothing to wait for | | CUDA graph capture of a call that needs more staging than the communicator holds, on a driver or device without stream-ordered allocation (a driver before CUDA 11.2, or a device without memory pools; the GB10 and the workstation's RTX 5090 have them) | unsupported | the call is refused with `ncclInvalidUsage` on the rank that needs the staging while its peers capture theirs; it captures after one eager call of that size has run on that rank. Whether every rank has stream-ordered allocation is agreed at setup | | Fail-stop (`LIBSIRCL_FAIL_STOP=1` or `abort`, per process): at the first asynchronous error (a flag wait that timed out, a failed progress thread, a failed point-to-point channel) of a watched communicator of two or more ranks, found by the watcher thread (every `LIBSIRCL_FAIL_STOP_POLL_MS`, default 5) or by the first library call that would report it (`ncclCommGetAsyncError`, an enqueue's check, `ncclCommFinalize`, `ncclCommDestroy`, `ncclCommAbort`), the library writes the error, the time and what found it to stderr and the communicator's receipt, then ends the process, with `1` at once (exit status 70, no atexit handlers, no core dump), with `abort` by `abort()`; destroy and abort check once more, then take the communicator off the watch before releasing it, and library unload stops the watcher first | implemented | Conditions: the workstation, `tests/emulation/fail_stop.py`, two ranks, SIRCL_STARTUP_WAIT_S 2, rank 1 starting its second all-reduce 12 s late, rank 0 enqueuing its own at once, waiting for its stream, keeping the output and calling the library no more. Measurement (library `638cca1b...`, every time CLOCK_MONOTONIC): 15 runs of the exit and control cases and one of all three: with `1`, the fail-stop line 2.11 to 2.13 s and the process's end 2.43 to 2.53 s after the enqueue, exit status 70; with `abort`, the line 2.11 s after the enqueue and SIGABRT 20.8 s after it (WSL's crash capture ran first); without fail-stop, rank 0's stream wait returned 2.10 to 2.13 s after the enqueue with a wrong output, every call it made returned 0, and the process ran on past 7 s. With `LIBSIRCL_FAIL_STOP=1` the pair default's 533 library checks passed and no process ended; on a build of the same fail-stop code, so did the ring schedules' 989 checks on four ranks and 10 teardown rounds on four ranks. The wait limit itself (research build of this tree whose timed-out flag wait records its `%globaltimer` start and end and the limit it read): 34 timed-out 2 s waits lasted 2.00002 to 2.00095 s of `%globaltimer` with the limit read as 2,000,000 us; a 300 s kernel and five 10 s kernels that sampled `%globaltimer` every 1,024 polls showed no step beyond 11 ms (host-stamping jitter) and a rate of 0.943 to 0.948 of the workstation's CLOCK_MONOTONIC; the workstation's CLOCK_REALTIME stepped back by 1.2 to 1.7 s about every 30 s (WSL2's time synchronization), so two of 28 waits timed by the wall clock looked 0.40 s long while the kernel measured 2.0003 s; earlier runs of this check that used the wall clock showed the same. On the fabric (cabled pair at positions 6-7, snapshot 0f94c72b, 2026-10-09; `teardown_race.py`, rank 1 starting its child all-reduce 8 s late, SIRCL_STARTUP_WAIT_S 2): with `1`, rank 0 exited with status 70 3.9 s after it started, setup included, its fail-stop line naming the timed-out flag wait for the late rank; without fail-stop it exited 1 after 10.4 s with the round reported failed. Library calls (this tree, library `9776657f...`, GPU emulation, 2026-10-09, `verification/emulation/suite-s1/`): `fail_stop.py`'s cases in which rank 0 calls `ncclCommGetAsyncError`, `ncclCommDestroy` or `ncclCommAbort` right after its stream wait, with `LIBSIRCL_FAIL_STOP_POLL_MS=600000` so the watcher does not poll, end rank 0 with status 70 inside the call (2.31 to 2.38 s after the enqueue, the line 2.00 s after it, "found by a library call" or "found by ncclCommDestroy or ncclCommAbort"), the call never returning; against the library before this path the same cases had `ncclCommGetAsyncError` and `ncclCommAbort` return the error to rank 0, which ran on and exited 0. `teardown_race.py --expect fail-stop`, four ranks, rank 3's child all-reduce 8 s late, 2 s wait limit: every rank exited 70 in 6 of 6 runs; in 9 of the 15 non-late exits of 5 runs a library call found the error before the watcher (the gate's path of four lost one rank that way before this path). Conclusion: with fail-stop, a caller that checks only enqueue codes keeps a failed collective's output at most for the wait limit plus one poll; a read within that poll after its stream wait returns is not prevented | | `ncclCommSuspend` and `ncclCommResume` | unsupported | refused with `ncclInvalidUsage` (logged once); a communicator's arena and device memory stay allocated while it exists | -| Point-to-point between two ranks of a communicator of three or more ranks on SIRCL's point-to-point channels (`LIBSIRCL_P2P_CHANNELS=on`, off by default; README.md, "Point-to-point channels"): SIRCL's native point-to-point library vendored byte-identical (`src/transport/sircl_p2p_proxy.c`, `c8ccfcb9...`, SIRCL 0.3.1's copy with change LF, checked by the build, its symbols prefixed per transport and not exported), created inside the communicator's setup exchanges (each rank's lanes and `LIBSIRCL_P2P_WINDOWS` give a channel table every rank derives alike; connection records validated together; lane check; windows of relayed lanes), first-in first-out matching per ordered pair, one stream per channel direction joined to the caller's stream, sends queued before receives, staging of messages not 16-byte aligned or not whole packs (stream-ordered allocation, or each channel's own buffer), refusal on both ranks of a pair without a channel, asynchronous errors and fail-stop on every rank after a kernel's timeout or size mismatch (abort notices), the channels' progress thread stopped in the teardown rounds and their verbs objects counted at destroy, receipts (`channels`) | implemented | GPU emulation on the workstation (below, "Point-to-point channels"): four and eight ranks, ring:8 under the route planner's settings, torch initialized eagerly, nccl-tests' sendrecv, hypercube and alltoallv on four processes; CPU: the native library across processes over the verbs stand-in. Not run on Sparks | +| Point-to-point between two ranks of a communicator of three or more ranks on SIRCL's point-to-point channels (`LIBSIRCL_P2P_CHANNELS=on`, off by default; README.md, "Point-to-point channels"): SIRCL's native point-to-point library vendored byte-identical (`src/transport/sircl_p2p_proxy.c`, `c8ccfcb9...`, SIRCL 0.3.2's copy with change LF, checked by the build, its symbols prefixed per transport and not exported), created inside the communicator's setup exchanges (each rank's lanes and `LIBSIRCL_P2P_WINDOWS` give a channel table every rank derives alike; connection records validated together; lane check; windows of relayed lanes), first-in first-out matching per ordered pair, one stream per channel direction joined to the caller's stream, sends queued before receives, staging of messages not 16-byte aligned or not whole packs (stream-ordered allocation, or each channel's own buffer), refusal on both ranks of a pair without a channel, asynchronous errors and fail-stop on every rank after a kernel's timeout or size mismatch (abort notices), the channels' progress thread stopped in the teardown rounds and their verbs objects counted at destroy, receipts (`channels`) | implemented | GPU emulation on the workstation (below, "Point-to-point channels"): four and eight ranks, ring:8 under the route planner's settings, torch initialized eagerly, nccl-tests' sendrecv, hypercube and alltoallv on four processes; CPU: the native library across processes over the verbs stand-in. Sparks: the path of four and the cycle of eight with channels on, snapshot a3477af2 (hardware evidence below) | | Point-to-point channel items under CUDA graph capture | unsupported | refused with `ncclInvalidUsage`: a replay would reuse the item numbers fixed at enqueue | | Point-to-point channels created at the first `ncclSend` or `ncclRecv` | unsupported | connecting the channels is collective over every rank, and a first send involves two; the channels are created with the communicator when `LIBSIRCL_P2P_CHANNELS=on`, and a communicator without them holds none of their memory, queue pairs or thread | | Point-to-point windows of the route planner (`tools/site_routes.py`, SIRCL's `p2p/budget.py`): `--p2p-reserve session` reserves the collective session's forward windows first, and its ring windows only with `--ring-schedules` (libsircl's ring lanes carry traffic only under the ring schedules); `--session-share F` (default 1) sizes the session's forward and ring windows against F of every relay queue's share and gives the channels the rest; a relayed pair without a window on every lane in both directions has no channel, named in the JSON output | implemented | CPU (`tests/test_site_routes.py`, 8 tests, with `SIRCL_PACKAGE` naming the SIRCL reference tree's `spark_transport/sircl`): path:4-7 gives all 6 relayed ordered pairs a window without `--ring-schedules` and leaves 3 without one with it (rank 3's 393,216-byte ring window reserved); ring:8 leaves all 40 without one at the default plan and gives all 40 one with `--max-window 32768` and with `--session-share` 0.95, 0.5 and 0.25 (at 0.95 every relayed session lane one 32 KiB chunk smaller, the channels' windows 32 KiB); a share of 1 prints the default's output; shares outside (0, 1] are refused; in every case no relay queue holds more than its 393,216-byte share (session plus channels), and `--p2p-reserve none` on ring:8, as the check's negative control, overfills one; the ring:8 files of `tests/data` equal the tool's output | -| The setup check of the native proxy's local feature word (`roce_local_features`, SIRCL change LF): communicator setup fails on every rank, naming both bits, unless bit 0 (`roce_destroy` counts failed verbs calls) and bit 1 (flags-only own items, which the pair exchange without input uses) are set; the build refuses a vendored copy that differs from its recorded SHA-256 or does not define the word | implemented | CPU: the native probe reads bits 0 and 1 from the vendored copy (`tests/test_shm_verbs.py`); the library carries the check (`tests/test_engine_cpu.py`); a build against a modified copy, a copy with CRLF endings, or the copy before LF with its own hash, stops with the reason. GPU emulation: every communicator of the suite passed the check. Not run on Sparks | -| The setup check of the point-to-point library's local feature word (`p2p_local_features` bit 0: `p2p_destroy` counts failed verbs calls; SIRCL change LF): channel setup fails with a named reason when the bit is absent; compiled by default (`P2P_FEATURES=1`, CMake `LIBSIRCL_P2P_FEATURES`), and the build refuses a vendored copy without the word | implemented | CPU: the native probe reads bit 0 from the vendored copy (`tests/test_p2p_native.py`); the default build carries the check (`tests/test_engine_cpu.py`); a default build against the copy before LF stops with the reason, and `P2P_FEATURES=0` builds it. GPU emulation: every channel setup of the suite below passed the check. Not run on Sparks | +| The setup check of the native proxy's local feature word (`roce_local_features`, SIRCL change LF): communicator setup fails on every rank, naming both bits, unless bit 0 (`roce_destroy` counts failed verbs calls) and bit 1 (flags-only own items, which the pair exchange without input uses) are set; the build refuses a vendored copy that differs from its recorded SHA-256 or does not define the word | implemented | CPU: the native probe reads bits 0 and 1 from the vendored copy (`tests/test_shm_verbs.py`); the library carries the check (`tests/test_engine_cpu.py`); a build against a modified copy, a copy with CRLF endings, or the copy before LF with its own hash, stops with the reason. GPU emulation: every communicator of the suite passed the check. Sparks: snapshots 0f94c72b and a3477af2 carry the check, and every communicator of their runs passed it (hardware evidence below) | +| The setup check of the point-to-point library's local feature word (`p2p_local_features` bit 0: `p2p_destroy` counts failed verbs calls; SIRCL change LF): channel setup fails with a named reason when the bit is absent; compiled by default (`P2P_FEATURES=1`, CMake `LIBSIRCL_P2P_FEATURES`), and the build refuses a vendored copy without the word | implemented | CPU: the native probe reads bit 0 from the vendored copy (`tests/test_p2p_native.py`); the default build carries the check (`tests/test_engine_cpu.py`); a default build against the copy before LF stops with the reason, and `P2P_FEATURES=0` builds it. GPU emulation: every channel setup of the suite below passed the check. Sparks: snapshot a3477af2's channel runs passed it (hardware evidence below) | | Pipeline parallelism on two-rank communicators: a pipeline group of two ranks (TP 4 x PP 2 on a ring of eight: positions i and i + 4, every lane through three relays), and torch's lazily initialized ProcessGroupNCCL, which gives each stage pair of a larger pipeline group a two-rank communicator of its own (vLLM 0.19.1's V1 pipeline exchange: `isend_tensor_dict` and `irecv_tensor_dict`, one `torch.distributed.isend` or `irecv` per tensor) | implemented | Conditions: GPU emulation on the workstation (library `02573b7c...`), every rank with the settings `tools/site_routes.py --layout ring:8 --lanes 2` emits for its position (`tests/data/site_routes_ring8_l2.json`). Measurement: `tests/emulation/pp_pairs.py` (eight ranks; tensor-parallel splits of positions 0-3 and 4-7, pair splits of positions i and i + 4; six microbatches per pair at its own pace of a tensor dictionary of 2 MiB, 4 MiB, 6 KiB and 1,000,003 B sent as torch's batch_isend_irecv groups them, one tensor back, and every third microbatch both ways in one group): 64 of 64 checks, every received byte exact; each pair ran 20 pair exchanges through its ring lanes' window of 65,536 B (640 to 1,408 window chunks) and its small transfers through the forward windows; without the ring plan 64 of 64, every transfer through the forward windows. `tests/emulation/torch_pp.py`: vLLM's exchange through torch.distributed, initialized lazily: TP 4 x PP 2 under the same ring:8 settings (an all-reduce on the default group, `new_group` tensor-parallel and pipeline groups, four microbatches) 56 of 56 checks exact over four two-rank point-to-point communicators, each pair's ring lanes windowed; a chain of four stages 12 of 12 over three two-rank communicators. Not run on Sparks | | Communicators of some of a layout's positions (split children; two-rank communicators such as torch's per-pair ones) under the route planner's per-process settings, one `LIBSIRCL_CHAIN_ORDER`, `LIBSIRCL_FORWARD_WINDOWS` and `LIBSIRCL_RING_WINDOW` for every communicator a process joins: the chain order's listed members in its order, the other positions skipped; a ring next reached through relays keeps the smallest forward window toward it when the layout's ring plan gives 0 | implemented | Libraries up to and including snapshot db529218 (the serving image's) refuse every such communicator at setup ("LIBSIRCL_CHAIN_ORDER=0,1,2,3,4,5,6,7 must list the positions of the communicator's ranks once"): `torch_pp.py --shape tp4pp2` failed there with 17 problems (the tensor-parallel groups did not form) and `pp_pairs.py` 8 of 8 (the splits); this tree's library forms them (the pipeline row's runs). With the ring lanes' window not derived (a research build of this tree), every pipeline pair ran its 20 pair exchanges through relays unwindowed (ring window 0 B, 0 window chunks; emulation, which has no relay queues, still exact). Unchanged where the ring next is a cable or the plan gives a window: cabled pairs, `path:0-3` (393,216 B on position 3) and ring:8; the pair default, windowed pair, relayed pair, ring schedules on two, four and eight ranks, the four-rank ring with windows and pieces on four and eight ranks passed every check of `library_rank.py` (533, 533, 531, 533, 989, 1,977, 989, 985 and 1,969) | | `ncclMemAlloc` and `ncclMemFree` (device memory of the current CUDA context, ordinary buffers for every collective, eager and in graphs); buffer and window registration (`ncclCommRegister`, `ncclCommWindowRegister` and their deregistrations) accepted as hints, the handle and the window being the buffer's address, and `ncclWinGetUserPtr` | implemented | API suite (registration and windows on a CPU communicator); nccl-tests v2.21.1 allocates its buffers with `ncclMemAlloc` | -| `ncclRedOpCreatePreMulSum` and other ops beyond the five built in, the per-call `nccl*Config` collectives, shrink, grow, the scalable initialization, host RMA signals, the device communicator and device pointers | unsupported | refused with `ncclInvalidUsage` (an op number beyond the built-in five: `ncclInvalidArgument`, counted in the receipt) and logged once | +| The 37 functions of the header that `tests/api_manifest.json` marks not implemented: `ncclCommShrink`, `ncclCommGetUniqueId`, `ncclCommGrow`, `ncclCommInitRankScalable`, `ncclResetDebugInit`, `ncclCommSuspend`, `ncclCommResume`, `ncclCommMemStats`, `ncclRedOpCreatePreMulSum`, `ncclRedOpDestroy`, `ncclAllReduceConfig`, `ncclBroadcastConfig`, `ncclReduceConfig`, `ncclAllGatherConfig`, `ncclReduceScatterConfig`, `ncclAlltoAllConfig`, `ncclGatherConfig`, `ncclScatterConfig`, `ncclPutSignal`, `ncclSignal`, `ncclWaitSignal`, `ncclGroupSimulateEnd`, `ncclSetEncryption`, `ncclParamBind`, `ncclParamGetI8`, `ncclParamGetI16`, `ncclParamGetI32`, `ncclParamGetI64`, `ncclParamGetU8`, `ncclParamGetU16`, `ncclParamGetU32`, `ncclParamGetU64`, `ncclParamGetStr`, `ncclParamGet`, `ncclParamGetParameter`, `ncclParamGetAllParameterKeys`, `ncclParamDumpAll`; ops beyond the five built in; the device communicator and device pointers | unsupported | refused with `ncclInvalidUsage` (an op number beyond the built-in five: `ncclInvalidArgument`, counted in the receipt) and logged once | | One progress thread per process; an abort word watched by every kernel wait | unsupported | each needs a change to SIRCL's package (`spark_transport/sircl`) that it does not have: an externally driven progress loop with which one thread serves every session of a process (request PO), and an abort word in the command ring that every timed flag wait watches (request AW) | | All-reduce timing on a cabled pair (`tests/emulation/perf_rank.py`: bf16, fp16, fp32, 8 B to 256 MiB, eager and graph) | qualified | the pair (hardware evidence): fp32 8 KiB 9.73 us eager, 9.61 us in graphs | | nccl-tests v2.21.1, unmodified, through `LD_PRELOAD` on a cabled pair: every collective test and point-to-point | qualified | snapshot fb63329a on Sparks 0-1 (hardware evidence): `#wrong 0` on every line of `RUNBOOK.md` sections 3.4 and 3.5; snapshot 0f94c72b, this tree's library and pair plan, on Sparks 6-7, 2026-10-09 (hardware evidence): `#wrong 0` on every row of sections 3.4 and 3.5 and of `sendrecv_perf` to 256 MiB (in-place `N/A` on all-to-all and sendrecv, which have no in-place result) | -| Frameworks on the ring | unsupported | PyTorch in emulation only | +| Frameworks on the ring | research-only | vLLM's PyNccl on a cabled pair of Sparks (the communicator-creation row above; the installer's libsircl lane, `docs/architecture/libsircl.md`); torch's own process groups in GPU emulation only | ## Evidence -The files behind each result are in `verification/` (CPU, build and loading records), -`verification/emulation/` (GPU emulation logs and per-rank results) and `verification/hardware/` (the -cabled pairs). +The results below cite their run records by paths under `verification/` (CPU, build and loading +records), `verification/emulation/` (GPU emulation logs and per-rank results) and `verification/hardware/` +(the Spark runs); as the scope above says, those records stay on the machines that made them and are not +in this repository. ### Hardware: the cabled pair at ring positions 0-1 @@ -387,7 +390,7 @@ DSL or from the pack (`tests/emulation/mixed_group.py`). Measurement, result and `KERNEL_ROUTE.md`. In short: 905 of 905 checks on `path:0-1`, `ring:3`, `path:0-3` (one-block and every-block polling) and `ring:8`, every output bit-exact against the host reference and the DSL's output, eager and in graph replay, mixed groups included; pack `c2e6e5a1...be4eaa25`. The test shim, -rebuilt to load both packs, repeats the `path:0-1` run with 181 of 181 checks passed +rebuilt to load the packs of that build, repeats the `path:0-1` run with 181 of 181 checks passed (`verification/emulation/mixed-pair-rebuilt-shim.log`). ### Library end to end: one process per rank on one GPU @@ -513,7 +516,7 @@ captured in a CUDA graph and replayed twice; `destroy_process_group`; every tens reference bit for bit. Result: every check passed with 2 ranks (46 checks) and with 4 ranks (79 checks; point-to-point runs with two ranks only, since torch then issues it on the four-rank communicator); torch reports `comm_split_count` 1, so `new_group` split the group's communicator through `ncclCommSplit`; every -receipt reports both packs' hashes, `"forwarded":0` and no refusals +receipt reports the hashes of its library's packs, `"forwarded":0` and no refusals (`verification/emulation/torch-pg-w2.log`, `torch-pg-w4.log`). Conclusion: torch's NCCL backend runs unmodified on libsircl in emulation, including object collectives, integer and averaging reductions, subgroups and, on two ranks, point-to-point. @@ -609,19 +612,16 @@ Conditions: RTX 5090, WSL2; pack kernels launched on a poisoned session so each (`tests/emulation/launch_bench.c`); 20,000 launches in batches of 100. Result: one-shot 4.25 us mean (3.48 us best batch), two-shot 4.19 us (3.67 us) per `cuLaunchKernel`. Conclusion: a C launch of the pack costs about 4 us on the workstation; GB10 measured 2.75 us for a DSL kernel through a prebuilt -argument block (handoff section 4), so the eager 8 KiB target of 20 us on a pair keeps its margin. Not +argument block (a measurement outside this repository's evidence), so the eager 8 KiB target of 20 us on a pair keeps its margin. Not measured on GB10. ### CPU suites, builds and loading Conditions: WSL2, GCC 13.3, Python 3.12, no GPU or RDMA device. Results: -- `make check` with `SIRCL_PACKAGE` naming the SIRCL reference's `spark_transport/sircl`: ABI 5, - API 17, bootstrap 22, lifecycle 14, engine 8, shared-memory verbs 5, vendored native sources 4, - point-to-point native library 5, route planner 8, MPI shim 6, nccl-tests evaluator 12 and kernel entries - 3 tests passed; 10,057 fabric - vector checks passed (`verification/library-tests.log`). Without `SIRCL_PACKAGE` the route planner's 8 - tests are skipped. +- `make check` with `SIRCL_PACKAGE` naming the SIRCL reference's `spark_transport/sircl`: every suite + passed, and the fabric vector checks passed (`verification/library-tests.log`). Without `SIRCL_PACKAGE` + the route planner's tests are skipped. - The CUDA-on-load test passes against the library and fails against a control library that initializes CUDA in a constructor; `tools/probe_cuda_on_load.py` on the workstation finds that the library maps and initializes nothing of CUDA, with and without site packages (`verification/cuda-on-load.json`). @@ -629,9 +629,10 @@ Conditions: WSL2, GCC 13.3, Python 3.12, no GPU or RDMA device. Results: `SIRCL_PACKAGE` set (`verification/cmake-tests.log`). Libraries up to snapshot cddad8cd fail 3 of them there: rdma-core's inline `ibv_reg_mr` keeps its call to `ibv_reg_mr_iova2` unoptimized, which their verbs loader did not define, so the library did not load. -- AddressSanitizer and UndefinedBehaviorSanitizer build: lifecycle 13 and API 14 tests passed, engine 3 - of 4 and ABI 4 of 5; the other two replace the preload list, which the sanitizer runtime refuses, and - pass in the plain build (`verification/sanitizer-cpu.log`). +- AddressSanitizer and UndefinedBehaviorSanitizer: `make check-asan` builds the host library into + `$(BUILD)-asan` with both sanitizers and runs every `make check` suite with the sanitizer runtime preloaded + (the CI job "libsircl C library" runs it); every suite passes, and a use-after-free injected into + `ncclCommDestroy` makes the target fail with AddressSanitizer's report. - The kernel packs built from the sources by nvcc 13.3.73 (the workstation's CUDA pip packages, 2026-10-09) have the SHA-256 values the rows above name: transport `c2e6e5a1...`, fold `66da585d...`, link `dc9dd167...` and point-to-point `0f39a3b9...`. The library and its four packs built in 63 s @@ -677,8 +678,8 @@ Conditions: WSL2, GCC 13.3, Python 3.12, no GPU or RDMA device. Results: `tools/site_routes.py --session-share` below 1 (0.95 to 0.05 checked) takes at least one chunk from every relayed session lane (at 0.95: 64-128 KiB become 32-96 KiB) and gives all 40 relayed ordered pairs a 32 KiB channel window, as `--max-window 32768` does (every session lane one chunk); the session's cost is not - yet measured on Sparks. The channels have run in emulation only, where no lane crosses - a relay. + yet measured on Sparks. In emulation no lane crosses a relay; on Sparks the channels ran on the path of + four and the cycle of eight (snapshot a3477af2, hardware evidence above). - The library orders eager calls across streams, not CUDA graph replays: a graph that holds a communicator's collectives must not run while that communicator's other work runs. - The fold path sends every rank's whole message to every rank (an all-gather or all-to-all, then a diff --git a/spark_transport/libsircl/cmake/check_vendored.cmake b/spark_transport/libsircl/cmake/check_vendored.cmake new file mode 100644 index 00000000..cbbfe5aa --- /dev/null +++ b/spark_transport/libsircl/cmake/check_vendored.cmake @@ -0,0 +1,35 @@ +# The vendored SIRCL native sources (src/transport/sircl_roce_proxy.c and sircl_p2p_proxy.c) must be the +# byte-identical LF copies whose SHA-256 is recorded beside them, with SIRCL change LF's local feature words. +# CMakeLists.txt includes this file at configure time and runs it as a script (cmake -P, with SOURCE_DIR and +# P2P_FEATURES set) before every build of the library, as the Makefile's roce_source.ok and p2p_source.ok +# rules do, so a copy changed after configuration fails the build. +if(NOT SOURCE_DIR) + set(SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) +endif() +if(NOT DEFINED P2P_FEATURES) + set(P2P_FEATURES ${LIBSIRCL_P2P_FEATURES}) +endif() +foreach(name roce p2p) + set(source ${SOURCE_DIR}/src/transport/sircl_${name}_proxy.c) + file(SHA256 ${source} found) + file(STRINGS ${source}.sha256 recorded LIMIT_COUNT 1) + if(NOT found STREQUAL recorded) + message(FATAL_ERROR "src/transport/sircl_${name}_proxy.c differs from the vendored SIRCL source recorded in its .sha256") + endif() + # file(STRINGS) drops carriage returns, so the raw contents are searched. + file(READ ${source} contents) + string(FIND "${contents}" "\r" carriage_return) + if(NOT carriage_return EQUAL -1) + message(FATAL_ERROR "src/transport/sircl_${name}_proxy.c has CRLF line endings; vendor SIRCL's LF bytes") + endif() +endforeach() +file(STRINGS ${SOURCE_DIR}/src/transport/sircl_roce_proxy.c roce_features REGEX "roce_local_features") +if(NOT roce_features) + message(FATAL_ERROR "src/transport/sircl_roce_proxy.c predates SIRCL change LF (no roce_local_features)") +endif() +if(P2P_FEATURES) + file(STRINGS ${SOURCE_DIR}/src/transport/sircl_p2p_proxy.c p2p_features REGEX "p2p_local_features") + if(NOT p2p_features) + message(FATAL_ERROR "LIBSIRCL_P2P_FEATURES: src/transport/sircl_p2p_proxy.c predates SIRCL change LF (no p2p_local_features)") + endif() +endif() diff --git a/spark_transport/libsircl/src/api.c b/spark_transport/libsircl/src/api.c index c7f504c6..72a6a435 100644 --- a/spark_transport/libsircl/src/api.c +++ b/spark_transport/libsircl/src/api.c @@ -17,6 +17,7 @@ #include #include #include +#include enum comm_state { INIT, READY, FINALIZED, REVOKED, FAILED }; struct ncclComm { @@ -31,8 +32,9 @@ struct ncclComm { sccl_engine *engine; char error_message[1024]; pthread_mutex_t error_lock; /* an error published after the status (publish_error) with its text */ - int quiesced; /* the close began (ncclCommFinalize, or ncclCommDestroy) */ + int quiesced; /* the close ran (ncclCommFinalize, or ncclCommDestroy); under close_lock */ int close_result; /* its result, returned again by a repeated ncclCommFinalize or destroy */ + pthread_mutex_t close_lock; /* held across the close, so a concurrent second close waits for its result */ uint64_t serial; /* creation number in this process: tells a reused address apart */ atomic_int state, error, grouped, cancelled; }; @@ -54,6 +56,37 @@ typedef struct { static _Thread_local queued_p2p *p2p_queue; static _Thread_local unsigned p2p_count, p2p_capacity; +/* A thread that used group state gets a thread-exit hook (a pthread key's destructor): it frees the thread's + * point-to-point queue and fails the initializations still pending in the thread's open group, which no + * other thread may start or destroy, so their handles can be destroyed. The library destructor deletes the + * key, so no hook runs into an unloaded library. */ +static pthread_key_t group_key; +static pthread_once_t group_key_once = PTHREAD_ONCE_INIT; +static atomic_int group_key_ready; +static void group_thread_exit(void *unused) { + (void)unused; + for (unsigned i = 0; i < pending_count; ++i) { + struct ncclComm *comm = pending[i]; + snprintf(comm->error_message, sizeof(comm->error_message), + "the thread that initialized this communicator in a group exited before its ncclGroupEnd"); + atomic_store(&comm->state, FAILED); + atomic_store(&comm->error, ncclInvalidUsage); + atomic_store(&comm->grouped, 0); + } + pending_count = 0; + group_depth = 0; + free(p2p_queue); + p2p_queue = NULL; + p2p_count = p2p_capacity = 0; +} +static void group_key_create(void) { + if (!pthread_key_create(&group_key, group_thread_exit)) atomic_store(&group_key_ready, 1); +} +static void group_thread_watch(void) { + pthread_once(&group_key_once, group_key_create); + if (atomic_load(&group_key_ready) && !pthread_getspecific(group_key)) pthread_setspecific(group_key, &group_key); +} + /* Make `result` the communicator's terminal error with `message` as its text, unless it already has one; * 1 when this call set it. The text is written under the error lock, and readers copy it under the same * lock (read_error). */ @@ -75,18 +108,33 @@ static int debug_errors(void) { const char *debug = sccl_env("NCCL_DEBUG"); return debug && (!strcasecmp(debug, "WARN") || !strcasecmp(debug, "INFO") || !strcasecmp(debug, "TRACE")); } +/* A failed call: its result and text for this thread. A call that enqueues nothing into the thread's group + * (a query, a registration, ncclGetUniqueId, a lifecycle call, an unsupported function) fails this way and + * leaves the group as it was. */ static ncclResult_t fail(ncclResult_t result, const char *message) { snprintf(last_error, sizeof(last_error), "%s", message); - if (group_depth && group_error == ncclSuccess) group_error = result; if (debug_errors()) fprintf(stderr, "libsircl: %s: %s\n", ncclGetErrorString(result), message); return result; } +/* A failed call that belongs to the thread's group (a communicator creation, a collective or point-to-point + * call refused other than as an invalid argument, the group calls themselves): its result also becomes the + * group's, unless the group already has one. ncclInProgress never becomes a group's result: ncclGroupEnd + * would record it as the terminal error of the group's pending initializations, which would then read as in + * progress forever. */ +static _Thread_local char group_reason[400]; /* the text of the call that set group_error */ +static ncclResult_t group_fail(ncclResult_t result, const char *message) { + if (group_depth && group_error == ncclSuccess && result != ncclInProgress) { + group_error = result; + snprintf(group_reason, sizeof group_reason, "%s", message); + } + return fail(result, message); +} /* A communicator that cannot be created (ncclCommInitRank*, ncclCommSplit): its reason goes to stderr * whatever NCCL_DEBUG says, once, because the frameworks that create communicators commonly report only * ncclGetErrorString, and the text ncclGetLastError keeps is otherwise lost with the process. */ static ncclResult_t creation_failed(ncclResult_t result, const char *message) { int printed = debug_errors(); - fail(result, message); + group_fail(result, message); if (!printed) fprintf(stderr, "libsircl: communicator creation failed (%s): %s\n", ncclGetErrorString(result), message); return result; @@ -95,7 +143,7 @@ static ncclResult_t creation_failed(ncclResult_t result, const char *message) { * leaves the calling thread's group as it was (the call did nothing, and the work queued before it still * runs at ncclGroupEnd); any other error also becomes the group's result. */ static ncclResult_t refuse(ncclResult_t result, const char *message) { - if (result != ncclInvalidArgument) return fail(result, message); + if (result != ncclInvalidArgument) return group_fail(result, message); snprintf(last_error, sizeof(last_error), "%s", message); return result; } @@ -126,12 +174,14 @@ const char *sirclGetInfo(void) { "\"ops\":[\"sum\",\"prod\",\"max\",\"min\",\"avg\"],\"transport_reductions\":[\"float16/sum\"," "\"bfloat16/sum\",\"float32/sum\"],\"all_gather\":\"every datatype\",\"broadcast\":" "\"every datatype\",\"all_to_all\":\"every datatype\",\"gather\":\"every datatype\",\"scatter\":" - "\"every datatype\",\"send_recv\":\"two-rank communicators, and a rank to itself\",\"split\":true}," + "\"every datatype\",\"send_recv\":\"two-rank communicators, a rank to itself, and point-to-point " + "channels on communicators of three or more ranks (LIBSIRCL_P2P_CHANNELS=on)\",\"split\":true}," "\"schedules\":{\"large_all_reduce\":[\"pieces\",\"chain\",\"auto\",\"ring\"],\"all_gather\":" "[\"pieces\",\"chain\",\"auto\",\"ring\"],\"reduce_scatter\":[\"pieces\",\"chain\",\"auto\",\"ring\"]," - "\"default\":\"pieces\"}," + "\"default\":\"pieces; the pair plan on a cabled pair; the cycle plan on a ring that closes over " + "cables\"}," "\"transports\":[\"verbs\",\"emulation\"],\"kernels\":\"ahead-of-time CUDA C++ " - "(sm_120, sm_121): transport, fold and link packs\",\"bootstrap\":" + "(sm_120, sm_121): transport, fold, link and point-to-point packs\",\"bootstrap\":" "\"tcp (loopback or SIRCL_BOOTSTRAP_ADDR/IFNAME)\",\"world\":\"1-8\"}"; } ncclResult_t ncclGetVersion(int *version) { @@ -182,6 +232,14 @@ const char *ncclGetLastError(ncclComm_t handle) { struct ncclComm *comm = acquire(handle); if (comm) { int error = atomic_load(&comm->error); + if (error == ncclSuccess && comm->engine) { + /* As ncclCommGetAsyncError: an asynchronous failure the engine found and the communicator has not yet + * recorded becomes its terminal error, with its description. */ + char message[512]; + ncclResult_t failure = sccl_engine_async_error(comm->engine, message, sizeof message); + if (failure != ncclSuccess && publish_error(comm, failure, message)) atomic_store(&comm->state, FAILED); + error = atomic_load(&comm->error); + } if (error != ncclSuccess && error != ncclInProgress) read_error(comm, last_error, sizeof(last_error)); release(comm); } @@ -197,8 +255,16 @@ static void *initialize(void *arg) { &comm->cancelled, &comm->engine, comm->error_message, sizeof(comm->error_message)); if (!result) comm->device = sccl_engine_device(comm->engine); } - atomic_store(&comm->state, result == 0 ? READY : FAILED); + if (result && comm->bootstrap) { + /* A failed communicator keeps no bootstrap connection: it would hold a socket, and on the root's process + * a broker thread and one of the 64 root slots, until the caller destroys the handle. */ + sccl_bootstrap_close(comm->bootstrap); + comm->bootstrap = NULL; + } + /* The result first (both stores sequentially consistent): a reader that sees READY or FAILED also sees + * the result that goes with it. */ atomic_store(&comm->error, result); + atomic_store(&comm->state, result == 0 ? READY : FAILED); return NULL; } static ncclResult_t start(struct ncclComm *comm) { @@ -206,11 +272,49 @@ static ncclResult_t start(struct ncclComm *comm) { atomic_store(&comm->state, FAILED); snprintf(comm->error_message, sizeof(comm->error_message), "cannot start bootstrap worker"); atomic_store(&comm->error, ncclSystemError); - return fail(ncclSystemError, "cannot start bootstrap worker"); + return group_fail(ncclSystemError, "cannot start bootstrap worker"); } comm->worker_started = 1; return ncclSuccess; } +/* libsircl honors only `blocking` of ncclConfig_t. Under NCCL_DEBUG (WARN, INFO or TRACE) the first config + * of the process that sets another field (an int not NCCL_CONFIG_UNDEF_INT, a name not NULL) within its + * `size` writes one stderr line naming that field. */ +static void log_ignored_config(const ncclConfig_t *config, size_t size) { + static atomic_int logged; + if (!debug_errors() || atomic_load(&logged)) return; + static const struct { const char *name; size_t offset; int pointer; } fields[] = { +#define INT_FIELD(f) {#f, offsetof(ncclConfig_t, f), 0} +#define NAME_FIELD(f) {#f, offsetof(ncclConfig_t, f), 1} + INT_FIELD(cgaClusterSize), INT_FIELD(minCTAs), INT_FIELD(maxCTAs), NAME_FIELD(netName), + INT_FIELD(splitShare), INT_FIELD(trafficClass), NAME_FIELD(commName), INT_FIELD(collnetEnable), + INT_FIELD(CTAPolicy), INT_FIELD(shrinkShare), INT_FIELD(nvlsCTAs), INT_FIELD(nChannelsPerNetPeer), + INT_FIELD(nvlinkCentricSched), INT_FIELD(graphUsageMode), INT_FIELD(numRmaCtx), INT_FIELD(maxP2pPeers), + INT_FIELD(graphStreamOrdering), INT_FIELD(launchOrderImplicit), INT_FIELD(numRmaSig), + INT_FIELD(rmaEagerInit), INT_FIELD(hostCftMode), INT_FIELD(nvlsHostMode), +#undef INT_FIELD +#undef NAME_FIELD + }; + for (size_t i = 0; i < sizeof fields / sizeof fields[0]; ++i) { + size_t width = fields[i].pointer ? sizeof(const char *) : sizeof(int); + if (fields[i].offset + width > size) continue; + int set; + if (fields[i].pointer) { + const char *name; + memcpy(&name, (const char *)config + fields[i].offset, sizeof name); + set = name != NULL; + } else { + int value; + memcpy(&value, (const char *)config + fields[i].offset, sizeof value); + set = value != NCCL_CONFIG_UNDEF_INT; + } + if (set && !atomic_exchange(&logged, 1)) { + fprintf(stderr, "libsircl: ncclConfig_t.%s is set and ignored: libsircl honors only blocking\n", + fields[i].name); + return; + } + } +} /* The blocking mode of a new communicator: the config's, or `inherited` when the config is NULL or leaves the * field undefined (1 for ncclCommInitRank*, the parent's for ncclCommSplit). */ static ncclResult_t config_blocking(const ncclConfig_t *config, int inherited, int *blocking) { @@ -227,6 +331,7 @@ static ncclResult_t config_blocking(const ncclConfig_t *config, int inherited, i if (magic != NCCL_API_MAGIC || version < 21400) return fail(ncclInvalidArgument, "config magic or version is invalid"); memcpy(blocking, (const char *)config + offsetof(ncclConfig_t, blocking), sizeof(*blocking)); + log_ignored_config(config, size); if (*blocking == NCCL_CONFIG_UNDEF_INT) *blocking = inherited; if (*blocking != 0 && *blocking != 1) return fail(ncclInvalidArgument, "config blocking must be 0, 1, or UNDEF"); @@ -297,6 +402,7 @@ static ncclResult_t init_rank(ncclComm_t *out, int nranks, ncclUniqueId id, int struct ncclComm *comm = calloc(1, sizeof(*comm)); if (!comm) return creation_failed(ncclSystemError, "communicator allocation failed"); pthread_mutex_init(&comm->error_lock, NULL); + pthread_mutex_init(&comm->close_lock, NULL); comm->nranks = nranks; comm->rank = rank; comm->blocking = blocking; comm->bootstrap_only = bootstrap_only; comm->ctx = ctx; comm->position = position; memcpy(comm->id, id.internal, 128); @@ -332,6 +438,24 @@ ncclResult_t ncclCommInitRank(ncclComm_t *out, int nranks, ncclUniqueId id, int return ncclCommInitRankConfig(out, nranks, id, rank, NULL); } +/* The result of a split whose all-gather failed, as the engine's setup exchanges report theirs: a peer that + * did not arrive in time or a lost broker is ncclRemoteError; a cancellation (abort) ncclInvalidUsage; an + * invalid setting ncclInvalidArgument; a local socket or allocation failure ncclSystemError. */ +static ncclResult_t split_exchange_result(int bootstrap_result) { + switch (bootstrap_result) { + case ncclInvalidUsage: return ncclInvalidUsage; + case ncclInvalidArgument: return ncclInvalidArgument; + case ncclSystemError: return ncclSystemError; + default: return ncclRemoteError; + } +} +/* A failed split: its reason is recorded before the parent's reference is dropped, so that a library unload + * waiting for the reference finds the call already past its library work. */ +static ncclResult_t split_failed(struct ncclComm *parent, ncclResult_t result, const char *message) { + creation_failed(result, message); + release(parent); + return result; +} /* ncclCommSplit: two rounds over the parent's bootstrap. Round 1 gathers every rank's (color, key); the * ranks of one color form a communicator ordered by key, then parent rank. Round 2 gathers the unique id * each new rank 0 created; every member then joins its communicator with that id, keeping the parent's @@ -351,11 +475,17 @@ ncclResult_t ncclCommSplit(ncclComm_t handle, int color, int key, ncclComm_t *ne int32_t mine[2] = {color, key}; void *all = NULL; unsigned lengths[64]; - if (sccl_bootstrap_allgather(parent->bootstrap, mine, sizeof mine, &all, lengths, &parent->cancelled)) { + int gathered = sccl_bootstrap_allgather(parent->bootstrap, mine, sizeof mine, &all, lengths, &parent->cancelled); + if (gathered) { char message[700]; snprintf(message, sizeof message, "ncclCommSplit: exchanging colors: %s", sccl_bootstrap_error()); - release(parent); - return creation_failed(ncclSystemError, message); + return split_failed(parent, split_exchange_result(gathered), message); + } + for (int r = 0; r < parent->nranks; ++r) { + if (lengths[r] != sizeof mine) { + free(all); + return split_failed(parent, ncclRemoteError, "ncclCommSplit: invalid color record length"); + } } const int32_t *records = all; int members[64], count = 0, rank = -1; @@ -372,22 +502,49 @@ ncclResult_t ncclCommSplit(ncclComm_t handle, int color, int key, ncclComm_t *ne for (int i = 0; i < count; ++i) if (members[i] == parent->rank) rank = i; free(all); + /* Every rank takes part in round 2, including a leader whose unique id could not be created: it sends a + * failure record instead (no id magic; bytes 8-11 its result, big-endian), so its members fail with the + * same result at once and every rank's bootstrap stays in step. */ ncclUniqueId id; memset(&id, 0, sizeof id); ncclResult_t result = ncclSuccess; - if (rank == 0) result = ncclGetUniqueId(&id); - if (result != ncclSuccess) { - release(parent); - return result; + char reason[512] = {0}; + if (rank == 0) { + result = ncclGetUniqueId(&id); + if (result != ncclSuccess) { + snprintf(reason, sizeof reason, "%s", last_error); + memset(&id, 0, sizeof id); + id.internal[8] = (unsigned char)((unsigned)result >> 24); id.internal[9] = (unsigned char)((unsigned)result >> 16); + id.internal[10] = (unsigned char)((unsigned)result >> 8); id.internal[11] = (unsigned char)result; + } } - if (sccl_bootstrap_allgather(parent->bootstrap, id.internal, sizeof id.internal, &all, lengths, &parent->cancelled)) { + gathered = sccl_bootstrap_allgather(parent->bootstrap, id.internal, sizeof id.internal, &all, lengths, + &parent->cancelled); + if (gathered) { char message[700]; snprintf(message, sizeof message, "ncclCommSplit: exchanging unique ids: %s", sccl_bootstrap_error()); - release(parent); - return creation_failed(ncclSystemError, message); + return split_failed(parent, split_exchange_result(gathered), message); + } + for (int r = 0; r < parent->nranks; ++r) { + if (lengths[r] != sizeof id.internal) { + free(all); + return split_failed(parent, ncclRemoteError, "ncclCommSplit: invalid unique-id record length"); + } } if (rank >= 0) memcpy(id.internal, (const unsigned char *)all + 128 * (size_t)members[0], 128); free(all); + if (rank > 0 && id.internal[0] == 0) { + unsigned code = (unsigned)id.internal[8] << 24 | (unsigned)id.internal[9] << 16 | (unsigned)id.internal[10] << 8 | + id.internal[11]; + result = code > ncclSuccess && code < ncclNumResults && code != ncclInProgress ? (ncclResult_t)code : ncclRemoteError; + snprintf(reason, sizeof reason, "the split leader (parent rank %d) could not create the unique id (%s)", + members[0], ncclGetErrorString(result)); + } + if (result != ncclSuccess) { + char message[700]; + snprintf(message, sizeof message, "ncclCommSplit: %s", reason); + return split_failed(parent, result, message); + } int position = parent->position, blocking = parent->blocking; release(parent); if (rank < 0) return ncclSuccess; @@ -467,6 +624,28 @@ static ncclResult_t point_to_point(const char *what, int send, const void *buff, } sccl_p2p_op op = {send, peer, buff, count * item, stream}; if (group_depth) { + /* The group rules the engine applies at ncclGroupEnd, refused at the call that breaks them: the calls of + * one group on one communicator use one stream, and there are at most MAX_GROUP_P2P of them. */ + enum { MAX_GROUP_P2P = 1024 }; + unsigned same = 0; + const queued_p2p *first = NULL; + for (unsigned i = 0; i < p2p_count; ++i) + if (p2p_queue[i].comm == comm && p2p_queue[i].serial == comm->serial) { + if (!first) first = &p2p_queue[i]; + ++same; + } + if (first && first->op.stream != op.stream) { + release(comm); + snprintf(message, sizeof message, "%s: the point-to-point calls of one group on one communicator must use " + "one stream", what); + return refuse(ncclInvalidUsage, message); + } + if (same >= MAX_GROUP_P2P) { + release(comm); + snprintf(message, sizeof message, "%s: a group carries at most %d point-to-point calls on one communicator", + what, MAX_GROUP_P2P); + return refuse(ncclInvalidUsage, message); + } if (p2p_count == p2p_capacity) { unsigned capacity = p2p_capacity ? 2 * p2p_capacity : 64; queued_p2p *grown = realloc(p2p_queue, sizeof(*grown) * capacity); @@ -501,8 +680,9 @@ ncclResult_t ncclRecv(void *recvbuff, size_t count, ncclDataType_t datatype, int ncclResult_t ncclGroupStart(void) { CHECK_PROCESS(); - if (group_depth == 256) return fail(ncclInvalidUsage, "group nesting exceeds 256"); - if (group_depth++ == 0) { group_error = ncclSuccess; pending_count = 0; } + if (group_depth == 256) return group_fail(ncclInvalidUsage, "group nesting exceeds 256"); + if (group_depth++ == 0) { group_error = ncclSuccess; group_reason[0] = 0; pending_count = 0; } + group_thread_watch(); return ncclSuccess; } ncclResult_t ncclGroupEnd(void) { @@ -510,11 +690,15 @@ ncclResult_t ncclGroupEnd(void) { if (!group_depth) return fail(ncclInvalidUsage, "GroupEnd without GroupStart"); if (--group_depth) return ncclSuccess; ncclResult_t result = group_error; + /* The reason the group failed: the first failing call's text, else the first initialization's. */ + char reason[sizeof group_reason + 64]; + snprintf(reason, sizeof reason, "%s", group_reason); for (unsigned i = 0; i < pending_count; ++i) { if (result != ncclSuccess) { - atomic_store(&pending[i]->state, FAILED); - snprintf(pending[i]->error_message, sizeof(pending[i]->error_message), "group initialization rejected"); + snprintf(pending[i]->error_message, sizeof(pending[i]->error_message), "group initialization rejected: %s", + reason); atomic_store(&pending[i]->error, result); + atomic_store(&pending[i]->state, FAILED); } else { ncclResult_t r = start(pending[i]); if (r != ncclSuccess) result = r; @@ -529,6 +713,13 @@ ncclResult_t ncclGroupEnd(void) { if (comm->blocking && comm->worker_started) { pthread_join(comm->worker, NULL); comm->worker_started = 0; ncclResult_t r = (ncclResult_t)atomic_load(&comm->error); + if (r != ncclSuccess && !atomic_load(&comm->cancelled)) { + /* A group-deferred creation that failed reports its reason as an ungrouped one does. */ + char message[1200]; + snprintf(message, sizeof message, "communicator initialization failed: %s", comm->error_message); + creation_failed(r, message); + if (!reason[0]) snprintf(reason, sizeof reason, "%s", message); + } if ((result == ncclSuccess || result == ncclInProgress) && r != ncclSuccess) result = r; } else if (!comm->blocking && result == ncclSuccess) result = ncclInProgress; atomic_store(&comm->grouped, 0); @@ -536,20 +727,39 @@ ncclResult_t ncclGroupEnd(void) { pending_count = 0; if (result != ncclSuccess && result != ncclInProgress) { flush_p2p(0); - return fail(result, "group initialization failed"); + char text[sizeof reason + 64]; + snprintf(text, sizeof text, "group initialization failed%s%s", reason[0] ? ": " : "", reason); + return fail(result, text); } ncclResult_t sent = flush_p2p(1); return sent != ncclSuccess ? sent : result; } ncclResult_t ncclCommInitAll(ncclComm_t *out, int ndev, const int *devlist) { CHECK_PROCESS(); - if (!out || ndev < 1 || ndev > 8) return fail(ncclInvalidArgument, "InitAll requires 1 to 8 output handles"); + if (!out || ndev < 1 || ndev > 8) + return group_fail(ncclInvalidArgument, "InitAll requires 1 to 8 output handles"); for (int i = 0; i < ndev; ++i) out[i] = NULL; for (int i = 0; i < ndev; ++i) if (devlist && devlist[i] < 0) - return fail(ncclInvalidArgument, "negative device number"); + return group_fail(ncclInvalidArgument, "negative device number"); + const char *only = sccl_env("LIBSIRCL_BOOTSTRAP_ONLY"); + if (!only || strcmp(only, "1")) { + int device = devlist ? devlist[0] : 0; + for (int i = 1; i < ndev; ++i) + if ((devlist ? devlist[i] : i) != device) + return creation_failed(ncclInvalidUsage, "InitAll with distinct devices is unsupported; initialize each rank on its device"); + char why[512]; + sccl_CUcontext ctx = sccl_engine_current_context(); + if (!ctx) ctx = sccl_engine_device_context(why, sizeof why); + if (!ctx) return creation_failed(ncclInvalidUsage, why); + const sccl_cuda *driver = sccl_cuda_get(); + sccl_CUdevice current = -1; + if (!driver || driver->CtxGetDevice(¤t) != SCCL_CUDA_SUCCESS || current != device) + return creation_failed(ncclInvalidUsage, "InitAll requires the requested device's context to be current"); + } ncclUniqueId id; ncclResult_t result = ncclGetUniqueId(&id); if (result != ncclSuccess) return result; + int nested = group_depth != 0; result = ncclGroupStart(); if (result != ncclSuccess) return result; for (int i = 0; i < ndev; ++i) { @@ -558,7 +768,20 @@ ncclResult_t ncclCommInitAll(ncclComm_t *out, int ndev, const int *devlist) { out[i]->device = devlist ? devlist[i] : i; } ncclResult_t end = ncclGroupEnd(); - return result != ncclSuccess ? result : end; + if (result == ncclSuccess) result = end; + /* A failed InitAll leaves no handle: every communicator it created is aborted and its output cleared. + * Inside the caller's own group the handles belong to that group until its ncclGroupEnd, which reports + * their failure. */ + if (result != ncclSuccess && result != ncclInProgress && !nested) { + char reason[sizeof last_error]; + snprintf(reason, sizeof reason, "%s", last_error); + for (int i = 0; i < ndev; ++i) { + if (out[i]) ncclCommAbort(out[i]); + out[i] = NULL; + } + snprintf(last_error, sizeof last_error, "%s", reason); + } + return result; } static ncclResult_t query(ncclComm_t handle, int *out, int field) { CHECK_PROCESS(); @@ -567,8 +790,11 @@ static ncclResult_t query(ncclComm_t handle, int *out, int field) { if (!comm) return fail(ncclInvalidArgument, "unknown communicator handle"); ncclResult_t result = (ncclResult_t)atomic_load(&comm->error); if (result == ncclSuccess) *out = field == 0 ? comm->nranks : field == 1 ? comm->rank : comm->device; + /* A failed communicator's result comes with its text, so ncclGetLastError(NULL) says why. */ + char text[sizeof comm->error_message] = {0}; + if (result != ncclSuccess && result != ncclInProgress) read_error(comm, text, sizeof text); release(comm); - return result; + return text[0] ? fail(result, text) : result; } ncclResult_t ncclCommCount(ncclComm_t comm, int *out) { return query(comm, out, 0); } @@ -825,12 +1051,18 @@ static ncclResult_t transition(ncclComm_t handle, int next) { if (!comm) return fail(ncclInvalidArgument, "unknown communicator handle"); int state = atomic_load(&comm->state); ncclResult_t result = (ncclResult_t)atomic_load(&comm->error); - if (result == ncclSuccess) { - if (state == READY || (state == next)) atomic_store(&comm->state, next); - else result = ncclInvalidUsage; + const char *why = "communicator is not ready for this lifecycle operation"; + if (result == ncclInProgress) { + /* As collective_ready refuses an enqueue: the initialization has not finished. */ + result = ncclInvalidUsage; + why = "communicator initialization is still in progress"; + } else if (result == ncclSuccess) { + /* Compare-exchange: a state another thread stored in between is seen, not overwritten. */ + while ((state == READY || state == next) && !atomic_compare_exchange_weak(&comm->state, &state, next)) {} + if (state != READY && state != next) result = ncclInvalidUsage; } release(comm); - return result == ncclSuccess ? result : fail(result, "communicator is not ready for this lifecycle operation"); + return result == ncclSuccess ? result : fail(result, why); } /* Teardown of a communicator of two or more ranks, so that no rank frees what a peer still writes. This rank's * enqueued work completes; round 1 over the bootstrap carries every rank's status (its wait, then its health: @@ -838,17 +1070,26 @@ static ncclResult_t transition(ncclComm_t handle, int next) { * rank's work completed, so every item and flag any rank's kernels wait for has landed; this rank's progress * thread stops and posts nothing more; round 2 proves every progress thread stopped, so nothing is posted * toward this rank's queue pairs or regions any more and destroy may free them. Each round is bounded by the - * session's wait limit. The close is terminal: from its start the communicator refuses new work, and its + * session's wait limit plus 5 s. The close is terminal: from its start the communicator refuses new work, and its * result (this rank's own error first, else ncclRemoteError naming the first rank that reported one, or a * round that not every rank reached) is kept, becomes the communicator's error when it failed, and is what a * repeated ncclCommFinalize or the destroy returns. */ +static ncclResult_t quiesce_locked(struct ncclComm *comm, const char *call); static ncclResult_t quiesce(struct ncclComm *comm, const char *call) { + pthread_mutex_lock(&comm->close_lock); + ncclResult_t result = quiesce_locked(comm, call); + pthread_mutex_unlock(&comm->close_lock); + return result; +} +static ncclResult_t quiesce_locked(struct ncclComm *comm, const char *call) { if (comm->quiesced) return (ncclResult_t)comm->close_result; if (!comm->engine || comm->bootstrap_only || comm->nranks < 2 || !comm->bootstrap) return ncclSuccess; + /* A revoked communicator runs the rounds too: revoke stops new work, not the kernels and forwards already + * in flight, so freeing without the rounds could pull queue pairs and regions from under a peer's writes. */ int state = atomic_load(&comm->state); - if (state != READY && state != FINALIZED && state != FAILED) return ncclSuccess; + if (state == INIT) return ncclSuccess; comm->quiesced = 1; - if (state == READY) atomic_store(&comm->state, FINALIZED); + if (state == READY) atomic_compare_exchange_strong(&comm->state, &state, FINALIZED); char message[700] = {0}; ncclResult_t result = (ncclResult_t)atomic_load(&comm->error); if (result != ncclSuccess) read_error(comm, message, sizeof message); @@ -858,8 +1099,11 @@ static ncclResult_t quiesce(struct ncclComm *comm, const char *call) { snprintf(message, sizeof message, "%s: waiting for this rank's enqueued work failed", call); } if (result == ncclSuccess) result = sccl_engine_async_error(comm->engine, message, sizeof message); - const atomic_int never = 0; - int limit_ms = sccl_engine_wait_limit_ms(comm->engine), arrived = 0; + /* The wait limit plus TEARDOWN_SLACK_MS: a rank whose own work ended at its wait limit (a timed-out flag + * wait) reaches the round just after its peers began waiting, not before. */ + enum { TEARDOWN_SLACK_MS = 5000 }; + int wait_ms = sccl_engine_wait_limit_ms(comm->engine), arrived = 0; + int limit_ms = wait_ms > INT_MAX - TEARDOWN_SLACK_MS ? INT_MAX : wait_ms + TEARDOWN_SLACK_MS; for (int round = 1; round <= 2; ++round) { if (round == 2) sccl_engine_teardown_stop(comm->engine); if (round == 2 && !arrived) break; @@ -867,7 +1111,7 @@ static ncclResult_t quiesce(struct ncclComm *comm, const char *call) { void *all = NULL; unsigned lengths[64] = {0}; arrived = sccl_bootstrap_allgather_within(comm->bootstrap, &mine, sizeof mine, &all, lengths, limit_ms, - &never) == 0; + &comm->cancelled) == 0; if (arrived && round == 1 && result == ncclSuccess) { for (int r = 0; r < comm->nranks; ++r) { int32_t theirs = 0; @@ -885,7 +1129,7 @@ static ncclResult_t quiesce(struct ncclComm *comm, const char *call) { if (result == ncclSuccess) { result = ncclRemoteError; snprintf(message, sizeof message, "%s: teardown round %d: not every rank arrived within the wait limit " - "(%d ms): %s", call, round, limit_ms, sccl_bootstrap_error()); + "plus 5 s (%d ms): %s", call, round, limit_ms, sccl_bootstrap_error()); } if (round == 1) sccl_engine_teardown_stop(comm->engine); break; @@ -909,17 +1153,27 @@ ncclResult_t ncclCommFinalize(ncclComm_t handle) { if (!comm) return fail(ncclInvalidArgument, "unknown communicator handle"); ncclResult_t flushed = comm->engine && atomic_load(&comm->state) == READY ? sccl_engine_finalize(comm->engine) : ncclSuccess; - if (flushed == ncclSuccess) flushed = quiesce(comm, "ncclCommFinalize"); + /* The failure's own text: this rank's wait, else the close's result as the communicator recorded it (a + * repeated finalize returns the first close's result and text). Copied before fail() writes the thread's + * text, which it must not read from. */ + char reason[sizeof last_error] = "ncclCommFinalize: waiting for this rank's enqueued work failed"; + if (flushed == ncclSuccess) { + flushed = quiesce(comm, "ncclCommFinalize"); + if (flushed != ncclSuccess) read_error(comm, reason, sizeof reason); + } release(comm); - if (flushed != ncclSuccess) return fail(flushed, last_error[0] ? last_error : "ncclCommFinalize failed"); + if (flushed != ncclSuccess) return fail(flushed, reason); return transition(handle, FINALIZED); } ncclResult_t ncclCommRevoke(ncclComm_t comm, int flags) { + CHECK_PROCESS(); if (flags) return fail(ncclInvalidArgument, "revoke flags must be zero"); return transition(comm, REVOKED); } static ncclResult_t dispose(ncclComm_t handle, int abort) { CHECK_PROCESS(); + /* NVIDIA NCCL 2.32.3's ncclCommDestroy and ncclCommAbort return ncclSuccess for a NULL communicator. */ + if (!handle) return ncclSuccess; /* Group-owned pending handles cannot be removed before GroupEnd consumes them. */ for (unsigned i = 0; i < pending_count; ++i) if (pending[i] == handle) return fail(ncclInvalidUsage, "communicator initialization is pending in a group"); @@ -936,9 +1190,10 @@ static ncclResult_t dispose(ncclComm_t handle, int abort) { return fail(ncclInvalidUsage, "communicator initialization is pending in a group"); } *link = comm->next; + if (abort) atomic_store(&comm->cancelled, 1); while (comm->refs) pthread_cond_wait(®istry_idle, ®istry_lock); pthread_mutex_unlock(®istry_lock); - atomic_store(&comm->cancelled, 1); + if (atomic_load(&comm->state) == INIT) atomic_store(&comm->cancelled, 1); if (comm->worker_started) pthread_join(comm->worker, NULL); ncclResult_t result = abort ? ncclSuccess : quiesce(comm, "ncclCommDestroy"); char message[sizeof comm->error_message]; @@ -951,25 +1206,49 @@ static ncclResult_t dispose(ncclComm_t handle, int abort) { } sccl_bootstrap_close(comm->bootstrap); pthread_mutex_destroy(&comm->error_lock); + pthread_mutex_destroy(&comm->close_lock); free(comm); return result == ncclSuccess ? ncclSuccess : fail(result, message); } ncclResult_t ncclCommDestroy(ncclComm_t comm) { return dispose(comm, 0); } ncclResult_t ncclCommAbort(ncclComm_t comm) { return dispose(comm, 1); } -/* A library unload must not leave a worker executing code in an unmapped DSO. - * dlclose concurrent with API calls is outside the supported caller contract. */ +/* A library unload (dlclose, or the destructors exit runs) must not leave a worker executing code in an + * unmapped DSO, nor free a communicator that a call on another thread still uses. Every communicator is + * cancelled first, which ends its bootstrap waits within one poll slice (50 ms); calls still inside the + * library hold references, and the unload waits up to UNLOAD_WAIT_MS for them to leave. A communicator still + * referenced after that stays allocated with its bootstrap open: freeing it would pull memory from under the + * call. dlclose concurrent with API calls remains outside the supported caller contract, since a call that + * has released its reference still returns through library code. */ +enum { UNLOAD_WAIT_MS = 1000 }; +static int referenced(const struct ncclComm *list) { + for (const struct ncclComm *entry = list; entry; entry = entry->next) + if (entry->refs) return 1; + return 0; +} __attribute__((destructor)) static void shutdown_communicators(void) { if (!sccl_bootstrap_process_valid()) return; + if (atomic_exchange(&group_key_ready, 0)) pthread_key_delete(group_key); sccl_engine_fail_stop_shutdown(); pthread_mutex_lock(®istry_lock); struct ncclComm *comm = registry; registry = NULL; for (struct ncclComm *entry = comm; entry; entry = entry->next) atomic_store(&entry->cancelled, 1); + struct timespec deadline; + clock_gettime(CLOCK_REALTIME, &deadline); + deadline.tv_sec += UNLOAD_WAIT_MS / 1000; + deadline.tv_nsec += (long)(UNLOAD_WAIT_MS % 1000) * 1000000L; + if (deadline.tv_nsec >= 1000000000L) { ++deadline.tv_sec; deadline.tv_nsec -= 1000000000L; } + while (referenced(comm)) + if (pthread_cond_timedwait(®istry_idle, ®istry_lock, &deadline) == ETIMEDOUT) break; pthread_mutex_unlock(®istry_lock); while (comm) { struct ncclComm *next = comm->next; + pthread_mutex_lock(®istry_lock); + int busy = comm->refs != 0; + pthread_mutex_unlock(®istry_lock); + if (busy) { comm = next; continue; } if (comm->worker_started) pthread_join(comm->worker, NULL); /* At unload only the native threads stop; CUDA may already be torn down. */ sccl_engine_shutdown(comm->engine); diff --git a/spark_transport/libsircl/src/bootstrap.c b/spark_transport/libsircl/src/bootstrap.c index d15cb7fa..8dc61aac 100644 --- a/spark_transport/libsircl/src/bootstrap.c +++ b/spark_transport/libsircl/src/bootstrap.c @@ -21,8 +21,10 @@ #include enum { OK = 0, SYSTEM = 2, ARGUMENT = 4, USAGE = 5, REMOTE = 6, TIMEOUT = 8 }; +/* HELLO_LIMIT_MS: a connection that has not sent its whole hello this long after the root accepted it is + * dropped, so idle connections cannot hold the root's MAX_PENDING slots until the join window ends. */ enum { ID_SIZE = 128, HELLO_SIZE = 136, MAX_RANKS = 8, MAX_PENDING = 32, - MAX_ROOTS = 64, FRAME_HEADER = 16, MAX_FRAME = 1 << 20 }; + MAX_ROOTS = 64, FRAME_HEADER = 16, MAX_FRAME = 1 << 20, HELLO_LIMIT_MS = 1000 }; enum { VERSION_OFF = 8, IP_OFF = 12, PORT_OFF = 16, NONCE_OFF = 20, HASH_OFF = 36, RESERVED_OFF = 68 }; static const unsigned char id_magic[8] = {'S','C','C','L','B','O','0','1'}; @@ -35,10 +37,15 @@ static const unsigned char frame_magic[4] = {'S','C','F','1'}; static const unsigned char gather_magic[4] = {'S','C','R','1'}; static _Thread_local char last_error[192]; -struct sccl_bootstrap { int fd; int nranks; int rank; uint32_t round; }; +/* `failed`: an all-gather on this handle failed after it began sending, so this rank's round count may differ + * from the broker's; every later all-gather fails at once instead of matching another round's records. + * `lock` is held across each all-gather round, so rounds that threads start on one handle at once run one + * after another instead of interleaving their frames on the socket. */ +struct sccl_bootstrap { int fd; int nranks; int rank; uint32_t round; int failed; pthread_mutex_t lock; }; struct peer { int fd; int rank; + int64_t accepted; /* CLOCK_MONOTONIC ms when the root accepted it */ size_t used; unsigned char hello[HELLO_SIZE]; unsigned char frame[FRAME_HEADER]; @@ -108,16 +115,14 @@ static int timeout_ms(int *out) { /* Deadline of one all-gather round (LIBSIRCL_SETUP_TIMEOUT_MS, 1 ms to 1 h; * default 600 s, the startup wait regime: ranks may lag while they allocate, - * register and connect). */ + * register and connect). -1 for a value that is not an integer in range. */ static int frame_deadline_ms(void) { const char *value = sccl_env("LIBSIRCL_SETUP_TIMEOUT_MS"); char *end; - if (value && *value) { - errno = 0; - long parsed = strtol(value, &end, 10); - if (!errno && !*end && parsed >= 1 && parsed <= 3600000) return (int)parsed; - } - return 600000; + if (!value || !*value) return 600000; + errno = 0; + long parsed = strtol(value, &end, 10); + return !errno && !*end && parsed >= 1 && parsed <= 3600000 ? (int)parsed : -1; } static int hex_digit(unsigned char c) { @@ -153,6 +158,9 @@ static uint32_t get32(const unsigned char *p) { ((uint32_t)p[2] << 8) | (uint32_t)p[3]; } +/* Every descriptor the bootstrap creates is close-on-exec from its creation (SOCK_CLOEXEC, accept4, + * pipe2), so a concurrent fork and exec in another thread cannot inherit it and keep a broker connection + * open after this process's rank left. configure_fd sets the flags again for descriptors of any origin. */ static int configure_fd(int fd) { int flags = fcntl(fd, F_GETFL, 0); if (flags < 0 || fcntl(fd, F_SETFL, flags | O_NONBLOCK) < 0 || @@ -301,9 +309,11 @@ static int finish_round(struct root *r) { at += 4 + by_rank[k]->payload_bytes; } int result = 0; - int64_t deadline = now_ms() + frame_deadline_ms(); + int limit = frame_deadline_ms(); + int64_t deadline = now_ms() + (limit > 0 ? limit : 600000); for (int k = 0; k < r->nranks; ++k) { - if (send_all(by_rank[k]->fd, answer, at, deadline, NULL) != OK) result = -1; + /* Bounded by the round's deadline, and given up at once when the library unloads (`stopped`). */ + if (send_all(by_rank[k]->fd, answer, at, deadline, &r->stopped) != OK) result = -1; free(by_rank[k]->payload); by_rank[k]->payload = NULL; by_rank[k]->payload_bytes = 0; by_rank[k]->payload_used = 0; @@ -332,7 +342,7 @@ static int accept_hello(struct root *r, int index) { static void accept_connections(struct root *r) { /* Cap each batch so connections cannot starve the absolute deadline. */ for (int attempt = 0; attempt < MAX_PENDING; ++attempt) { - int fd = accept(r->listen_fd, NULL, NULL); + int fd = accept4(r->listen_fd, NULL, NULL, SOCK_CLOEXEC | SOCK_NONBLOCK); if (fd < 0) { if (errno == EINTR) continue; return; } if (configure_fd(fd) != 0) { close(fd); continue; } int slot; @@ -342,6 +352,7 @@ static void accept_connections(struct root *r) { r->peers[slot].fd = fd; r->peers[slot].rank = -1; r->peers[slot].used = 0; + r->peers[slot].accepted = now_ms(); } } @@ -365,6 +376,20 @@ static void *root_thread(void *arg) { if (ready && !r->joined) break; int delay = ready ? 100 : remaining_ms(r->deadline); if (!ready && !delay) break; + /* Pending connections without a whole hello within HELLO_LIMIT_MS are dropped; wake for the next. */ + if (!ready) { + int64_t now = now_ms(); + for (int i = 0; i < MAX_PENDING; ++i) { + struct peer *p = &r->peers[i]; + if (p->fd < 0 || p->rank >= 0) continue; + int64_t left = p->accepted + HELLO_LIMIT_MS - now; + if (left <= 0) { + drop_peer(r, i, -1); + continue; + } + if (left < delay) delay = (int)left; + } + } int rc = poll(pfds, (nfds_t)n, delay); if (rc < 0) { if (errno == EINTR) continue; break; } for (int j = 0; j < n; ++j) { @@ -489,22 +514,29 @@ static int lan_configured(void) { return 0; } -/* Interface list match: comma-separated prefixes, "=name" for an exact name, - * "^prefix" entries exclude. */ -static int interface_matches(const char *list, const char *name) { +/* Interface list match, in NCCL_SOCKET_IFNAME's syntax: comma-separated entries; "prefix" selects the + * interfaces whose name starts with it and "=name" exactly that one; "^prefix" and "^=name" exclude them. + * Without any selecting entry every interface not excluded matches except a loopback interface, which only + * a selecting entry chooses (a remote rank cannot reach the root's loopback address). */ +static int interface_matches(const char *list, const char *name, int loopback) { char copy[256]; snprintf(copy, sizeof copy, "%s", list); int included = 0, any_include = 0; for (char *save = NULL, *item = strtok_r(copy, ",", &save); item; item = strtok_r(NULL, ",", &save)) { while (*item == ' ') ++item; - if (*item == '^') { - if (!strncmp(name, item + 1, strlen(item + 1))) return 0; + int exclude = *item == '^'; + if (exclude) ++item; + int exact = *item == '='; + if (exact) ++item; + int match = exact ? !strcmp(name, item) : !strncmp(name, item, strlen(item)); + if (exclude) { + if (match) return 0; continue; } any_include = 1; - if (*item == '=' ? !strcmp(name, item + 1) : !strncmp(name, item, strlen(item))) included = 1; + if (match) included = 1; } - return any_include ? included : 1; + return any_include ? included : !loopback; } /* The IPv4 address a new root listens on and advertises: SIRCL_BOOTSTRAP_ADDR, @@ -518,15 +550,19 @@ static int root_address(struct in_addr *out) { return error_result(ARGUMENT, "SIRCL_BOOTSTRAP_ADDR must be an IPv4 address"); return OK; } - const char *list = sccl_env("SIRCL_BOOTSTRAP_IFNAME"); - if (!list || !*list) list = sccl_env("NCCL_SOCKET_IFNAME"); + const char *name = "SIRCL_BOOTSTRAP_IFNAME", *list = sccl_env(name); + if (!list || !*list) list = sccl_env(name = "NCCL_SOCKET_IFNAME"); if (!list || !*list) return OK; + if (strlen(list) > 255) { + (void)snprintf(last_error, sizeof last_error, "%s is longer than 255 characters", name); + return ARGUMENT; + } struct ifaddrs *all = NULL; if (getifaddrs(&all) != 0) return error_result(SYSTEM, "cannot list network interfaces"); int found = 0; for (struct ifaddrs *a = all; a && !found; a = a->ifa_next) { if (!a->ifa_addr || a->ifa_addr->sa_family != AF_INET || !(a->ifa_flags & IFF_UP)) continue; - if (!interface_matches(list, a->ifa_name)) continue; + if (!interface_matches(list, a->ifa_name, (a->ifa_flags & IFF_LOOPBACK) != 0)) continue; *out = ((struct sockaddr_in *)a->ifa_addr)->sin_addr; found = 1; } @@ -582,7 +618,7 @@ int sccl_bootstrap_id(unsigned char id[ID_SIZE]) { memcpy(r->id + IP_OFF, &bind_address.s_addr, 4); memcpy(r->id + HASH_OFF, hash, 32); if (random_nonce(r->id + NONCE_OFF) != 0) goto failed; - r->listen_fd = socket(AF_INET, SOCK_STREAM, 0); + r->listen_fd = socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, 0); if (r->listen_fd < 0 || configure_fd(r->listen_fd) != 0) goto failed; struct sockaddr_in address; memset(&address, 0, sizeof address); @@ -594,7 +630,7 @@ int sccl_bootstrap_id(unsigned char id[ID_SIZE]) { if (getsockname(r->listen_fd, (struct sockaddr *)&address, &length) < 0) goto failed; memcpy(r->id + PORT_OFF, &address.sin_port, 2); - if (pipe(r->wake) != 0 || configure_fd(r->wake[0]) != 0 || + if (pipe2(r->wake, O_CLOEXEC | O_NONBLOCK) != 0 || configure_fd(r->wake[0]) != 0 || configure_fd(r->wake[1]) != 0) goto failed; int64_t now = now_ms(); if (now < 0) goto failed; @@ -604,7 +640,9 @@ int sccl_bootstrap_id(unsigned char id[ID_SIZE]) { for (struct root *entry = roots; entry; entry = entry->next) ++count; if (count >= MAX_ROOTS) { pthread_mutex_unlock(&roots_lock); - result = error_result(USAGE, "at most 64 active bootstrap IDs are supported"); + /* At most 191 characters: the error text buffer. */ + result = error_result(USAGE, "this process already holds 64 bootstrap roots, the limit: an id keeps its root until " + "its communicator is destroyed on every rank or its join window ends"); goto cleanup; } if (pthread_create(&r->thread, NULL, root_thread, r) != 0) { @@ -662,7 +700,7 @@ int sccl_bootstrap_join_cancel(const unsigned char id[ID_SIZE], int nranks, int int64_t now = now_ms(); if (now < 0) return error_result(SYSTEM, "cannot read monotonic clock"); int64_t deadline = now + timeout; - fd = socket(AF_INET, SOCK_STREAM, 0); + fd = socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, 0); if (fd < 0 || configure_fd(fd) != 0) { result = SYSTEM; goto failed; } struct sockaddr_in address; memset(&address, 0, sizeof address); @@ -698,6 +736,8 @@ int sccl_bootstrap_join_cancel(const unsigned char id[ID_SIZE], int nranks, int handle->nranks = nranks; handle->rank = rank; handle->round = 0; + handle->failed = 0; + pthread_mutex_init(&handle->lock, NULL); *out = handle; return OK; failed: @@ -708,7 +748,8 @@ int sccl_bootstrap_join_cancel(const unsigned char id[ID_SIZE], int nranks, int case TIMEOUT: return error_result(result, "bootstrap timed out waiting for every rank"); case USAGE: return error_result(result, "bootstrap rejected a duplicate rank or mismatched world size"); case ARGUMENT: return error_result(result, "bootstrap rejected invalid rank arguments"); - case REMOTE: return error_result(result, "bootstrap endpoint rejected the ID or disconnected"); + case REMOTE: return error_result(result, "bootstrap root rejected the ID or disconnected (its join window, " + "SIRCL_BOOTSTRAP_TIMEOUT_MS from ncclGetUniqueId, may have ended)"); default: return error_result(SYSTEM, "bootstrap socket or allocation failure"); } } @@ -723,7 +764,11 @@ void sccl_bootstrap_close(sccl_bootstrap *handle) { (void)error_result(USAGE, "fork after library use is unsupported; exec is required in the child"); return; } - if (handle) { close(handle->fd); free(handle); } + if (handle) { + close(handle->fd); + pthread_mutex_destroy(&handle->lock); + free(handle); + } reap_finished(); } @@ -737,9 +782,17 @@ int sccl_bootstrap_fd(const sccl_bootstrap *handle) { int sccl_bootstrap_allgather(sccl_bootstrap *handle, const void *data, uint32_t bytes, void **out, uint32_t *lengths, const atomic_int *cancelled) { - return sccl_bootstrap_allgather_within(handle, data, bytes, out, lengths, frame_deadline_ms(), cancelled); + int limit = frame_deadline_ms(); + if (limit < 0) { + if (out) *out = NULL; + return error_result(ARGUMENT, "LIBSIRCL_SETUP_TIMEOUT_MS must be an integer from 1 to 3600000"); + } + return sccl_bootstrap_allgather_within(handle, data, bytes, out, lengths, limit, cancelled); } +static int allgather_locked(sccl_bootstrap *handle, const void *data, uint32_t bytes, void **out, + uint32_t *lengths, int timeout_ms, const atomic_int *cancelled); + int sccl_bootstrap_allgather_within(sccl_bootstrap *handle, const void *data, uint32_t bytes, void **out, uint32_t *lengths, int timeout_ms, const atomic_int *cancelled) { if (out) *out = NULL; @@ -747,6 +800,17 @@ int sccl_bootstrap_allgather_within(sccl_bootstrap *handle, const void *data, ui return error_result(USAGE, "fork after library use is unsupported; exec is required in the child"); if (!handle || !out || !lengths || (bytes && !data) || bytes > MAX_FRAME || timeout_ms < 1) return error_result(ARGUMENT, "all-gather needs a handle, outputs, at most 1 MiB per rank and a timeout"); + pthread_mutex_lock(&handle->lock); + int result = allgather_locked(handle, data, bytes, out, lengths, timeout_ms, cancelled); + pthread_mutex_unlock(&handle->lock); + return result; +} + +/* One all-gather round with the handle's lock held; its deadline counts from the moment the lock is held. */ +static int allgather_locked(sccl_bootstrap *handle, const void *data, uint32_t bytes, void **out, + uint32_t *lengths, int timeout_ms, const atomic_int *cancelled) { + if (handle->failed) + return error_result(REMOTE, "an earlier all-gather on this bootstrap failed, so its rounds are out of step"); int64_t deadline = now_ms() + timeout_ms; unsigned char header[FRAME_HEADER]; memcpy(header, frame_magic, 4); @@ -762,7 +826,10 @@ int sccl_bootstrap_allgather_within(sccl_bootstrap *handle, const void *data, ui get32(header + 8) != (uint32_t)handle->nranks || total > (uint32_t)handle->nranks * (4u + MAX_FRAME)) { result = REMOTE; goto failed; } unsigned char *body = malloc(total ? total : 1); - if (!body) return error_result(SYSTEM, "cannot allocate the all-gather answer"); + if (!body) { + handle->failed = 1; + return error_result(SYSTEM, "cannot allocate the all-gather answer"); + } result = recv_all(handle->fd, body, total, deadline, cancelled); if (result != OK) { free(body); goto failed; } /* Repack as the contributions back to back, without their length words. */ @@ -780,6 +847,7 @@ int sccl_bootstrap_allgather_within(sccl_bootstrap *handle, const void *data, ui *out = body; return OK; failed: + handle->failed = 1; switch (result) { case TIMEOUT: return error_result(TIMEOUT, "bootstrap all-gather timed out waiting for every rank"); case USAGE: return error_result(USAGE, "bootstrap all-gather cancelled"); diff --git a/spark_transport/libsircl/src/cuda_api.c b/spark_transport/libsircl/src/cuda_api.c index 204fa14d..56d9c5b7 100644 --- a/spark_transport/libsircl/src/cuda_api.c +++ b/spark_transport/libsircl/src/cuda_api.c @@ -45,6 +45,7 @@ static void load(void) { SYM(LaunchKernel, "cuLaunchKernel"); SYM(MemAlloc, "cuMemAlloc_v2"); SYM(MemFree, "cuMemFree_v2"); + SYM(PointerGetAttribute, "cuPointerGetAttribute"); SYM(MemsetD32Async, "cuMemsetD32Async"); SYM(MemsetD8Async, "cuMemsetD8Async"); SYM(MemcpyDtoDAsync, "cuMemcpyDtoDAsync_v2"); @@ -65,6 +66,7 @@ static void load(void) { SYM(EventRecord, "cuEventRecord"); SYM(EventDestroy, "cuEventDestroy_v2"); SYM(EventSynchronize, "cuEventSynchronize"); + SYM(EventQuery, "cuEventQuery"); #undef SYM *(void **)(&table.MemAllocAsync) = dlsym(handle, "cuMemAllocAsync"); *(void **)(&table.MemFreeAsync) = dlsym(handle, "cuMemFreeAsync"); diff --git a/spark_transport/libsircl/src/cuda_api.h b/spark_transport/libsircl/src/cuda_api.h index c7bffb3b..3454ac98 100644 --- a/spark_transport/libsircl/src/cuda_api.h +++ b/spark_transport/libsircl/src/cuda_api.h @@ -55,6 +55,7 @@ typedef struct sccl_cuda { unsigned, sccl_CUstream, void **, void **); sccl_CUresult (*MemAlloc)(sccl_CUdeviceptr *, size_t); sccl_CUresult (*MemFree)(sccl_CUdeviceptr); + sccl_CUresult (*PointerGetAttribute)(void *, int, sccl_CUdeviceptr); sccl_CUresult (*MemsetD32Async)(sccl_CUdeviceptr, unsigned, size_t, sccl_CUstream); sccl_CUresult (*MemsetD8Async)(sccl_CUdeviceptr, unsigned char, size_t, sccl_CUstream); sccl_CUresult (*MemcpyDtoDAsync)(sccl_CUdeviceptr, sccl_CUdeviceptr, size_t, sccl_CUstream); @@ -75,6 +76,7 @@ typedef struct sccl_cuda { sccl_CUresult (*EventRecord)(sccl_CUevent, sccl_CUstream); sccl_CUresult (*EventDestroy)(sccl_CUevent); sccl_CUresult (*EventSynchronize)(sccl_CUevent); + sccl_CUresult (*EventQuery)(sccl_CUevent); /* Optional (NULL when the driver lacks them): stream-ordered allocation, which inside a CUDA graph * capture adds allocation and free nodes to the graph. */ sccl_CUresult (*MemAllocAsync)(sccl_CUdeviceptr *, size_t, sccl_CUstream); diff --git a/spark_transport/libsircl/src/engine.c b/spark_transport/libsircl/src/engine.c index e6800be2..8a77c39f 100644 --- a/spark_transport/libsircl/src/engine.c +++ b/spark_transport/libsircl/src/engine.c @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -18,6 +19,7 @@ #include #include "kernelpack.h" +#include "link_plan.h" #include "transport/shm_verbs.h" /* -- SIRCL's native proxy (src/transport/sircl_roce_proxy.c), compiled once per transport ----- */ @@ -284,6 +286,9 @@ enum { #define P2P_DEFAULT_UNROLL 4u #define P2P_DEFAULT_CHUNK_BYTES 32768u +/* The native layer's ring links post through relays in chunks of 32 KiB and accept a ring window of 1 to + * 60 such chunks (roce_set_links). */ +enum { RING_WINDOW_CHUNK = 32768, RING_WINDOW_MAX_CHUNKS = 60 }; enum { ALG_AUTO = -1 }; /* LIBSIRCL_FAIL_STOP's modes (see fail_stop). */ enum { FAIL_STOP_EXIT = 1, FAIL_STOP_ABORT = 2 }; @@ -300,7 +305,9 @@ typedef struct { uint32_t startup_wait_us, serving_wait_us, proxy_abi, graph_staging; char kernel_pack[72], fold_pack[72], links_pack[72]; /* The chain schedule: SIRCL_LARGE_SCHEDULE, geometry, and the chain order (ranks by chain index). */ - uint32_t large_schedule, chain_slots, chain_chunk, chain_blocks, chain_unroll, chain_reserved; + uint32_t large_schedule, chain_slots, chain_chunk, chain_blocks, chain_unroll; + /* Fail-stop's mode (LIBSIRCL_FAIL_STOP, 0 for a communicator of one rank). */ + uint32_t fail_stop; uint64_t chain_slot_bytes; /* Smallest collective (COLL_*) that auto runs as a chain op, and that a ring schedule runs as a ring op. */ uint64_t chain_mins[COLLECTIVES], ring_mins[COLLECTIVES]; @@ -309,14 +316,17 @@ typedef struct { * collective (COLL_*), staggers, whether the ring that closes the chain runs (LIBSIRCL_RING_WINDOW * set), the reduce-scatter's column tile. */ uint32_t gather_schedule, scatter_schedule, link_slots, link_blocks, link_unroll, ring_on; - uint32_t ring_stagger, ring_gather_stagger, link_chunk[COLLECTIVES], link_reserved; + uint32_t ring_stagger, ring_gather_stagger, link_chunk[COLLECTIVES]; + /* The wait regime at setup (LIBSIRCL_WAIT_REGIME: 0 startup, 1 serving). */ + uint32_t wait_regime; uint64_t link_slot_bytes, link_tile; /* Blocks per role and pieces per collective: chunk_set[c] when a link chunk variable sets the piece, * coll_blocks[c] from LIBSIRCL__LINK_BLOCKS (0: unset), blocks_set when SIRCL_LINK_BLOCKS * sets link_blocks; pair_plan when the pair plan chooses the rest by size. */ uint32_t chunk_set[COLLECTIVES], coll_blocks[COLLECTIVES], blocks_set, pair_plan; - /* The cycle plan (see CYCLE_RING_MIN_BYTES) applies. */ - uint32_t cycle_plan, cycle_reserved; + /* The cycle plan (see CYCLE_RING_MIN_BYTES) applies; the ring all-reduce's relay form + * (LIBSIRCL_RING_REDUCE_PASSES, see sccl_engine.ring_reduce_passes). */ + uint32_t cycle_plan, ring_reduce_passes; /* Point-to-point channels: on (LIBSIRCL_P2P_CHANNELS on a communicator of three or more ranks), the native * library's ABI, SIRCL's channel geometry (SIRCL_P2P_*) and the point-to-point pack. */ uint32_t p2p_on, p2p_abi, p2p_slots, p2p_blocks, p2p_threads, p2p_unroll, p2p_chunk, p2p_reserved; @@ -381,9 +391,10 @@ struct sccl_engine { uint32_t ring_window; /* The CPUs the transport's progress thread may run on (a CPU list, or "unpinned"). */ char progress_cpus[160]; - /* The ring all-reduce's form (LIBSIRCL_RING_REDUCE_PASSES): 2 (default), the relay stores its result to - * link 3's slot, then copies the slot to the output; 1, it stores both in one pass. Rank-local: both forms - * speak the same protocol and write the same bytes. */ + /* The ring all-reduce's form (LIBSIRCL_RING_REDUCE_PASSES, agreed at setup): 2 (default), the relay stores + * its result to link 3's slot, then copies the slot to the output; 1, it stores both in one pass. Both + * speak the same protocol and are designed to write the same bytes, but one run of the one-pass form on a + * GB10 returned differing bits on one rank (STATUS.md), so 1 needs LIBSIRCL_RING_REDUCE_ONE_PASS_RESEARCH=1. */ int ring_reduce_passes; uint64_t link_off; sccl_CUdeviceptr link_counters, piece_counters, link_scratch; @@ -433,7 +444,16 @@ struct sccl_engine { sccl_CUstream last_stream; /* Teardown: the progress thread stopped (sccl_engine_teardown_stop), when its stop was requested and * when it had stopped (CLOCK_REALTIME nanoseconds). */ - int stopped; + int stopped, closing; + int lane_check_ms; /* LIBSIRCL_LANE_CHECK_MS: the lane check's bound */ + int traffic_class; /* SIRCL_TRAFFIC_CLASS or NCCL_IB_TC */ + int kp_held; /* this engine holds a reference on its context's kernel-pack entry (sccl_kp_load) */ + /* Quarantine (sccl_engine_destroy after an abort with a launch still running): the events recorded + * behind the last launch and the channels' work, the bytes held, the next quarantined engine. */ + sccl_CUevent quarantine_done[1 + 2 * MAX_WORLD]; + int quarantine_events; + uint64_t quarantine_bytes; + struct sccl_engine *quarantine_next; uint64_t stop_ns, stopped_ns; int have_last; sccl_CUevent order_event; @@ -574,8 +594,9 @@ static int env_u64(const char *name, uint64_t fallback, uint64_t lo, uint64_t hi } char *end; errno = 0; - unsigned long long value = strtoull(text, &end, 10); - if (errno || *end || value < lo || value > hi) { + /* Digits only: strtoull would also take a sign ("-1" wraps to a huge value) and leading spaces. */ + unsigned long long value = text[0] >= '0' && text[0] <= '9' ? strtoull(text, &end, 10) : (end = (char *)text, 0); + if (errno || *end || end == text || value < lo || value > hi) { put_error(err, len, "%s=%s must be an integer from %" PRIu64 " to %" PRIu64, name, text, lo, hi); return -1; } @@ -749,6 +770,11 @@ static int link_settings(sccl_engine *e, shared_settings *s, int pair_plan, char e->ring_window = 0; if (ring && *ring) { if (env_u64("LIBSIRCL_RING_WINDOW", 0, 0, 0xFFFFFFFFu, &v, err, len)) return -1; + if (v && (v < RING_WINDOW_CHUNK || v / RING_WINDOW_CHUNK > RING_WINDOW_MAX_CHUNKS)) { + put_error(err, len, "LIBSIRCL_RING_WINDOW=%" PRIu64 " must be 0 or hold 1 to %d chunks of %d bytes", v, + RING_WINDOW_MAX_CHUNKS, RING_WINDOW_CHUNK); + return -1; + } e->ring_window = (uint32_t)v; } return 0; @@ -761,12 +787,52 @@ typedef struct { uint64_t relayed; } position_record; -/* The positions LIBSIRCL_FORWARD_WINDOWS gives a nonzero window toward (reached through relays), as bits; - * a malformed entry counts as none here (forward_windows() reports it). */ +/* LIBSIRCL_RELAYED_LANES, "=<0|1>[/<0|1>],...": 1 for every lane toward that position that crosses + * relays, whatever its windows. tools/site_routes.py prints it on every rank of a layout with relays, so a + * relayed lane is known as relayed even when no forward or point-to-point window is printed for it (for + * example with --max-window 0). Fills `flags` (position by lane); -1 with `err` on a malformed value. */ +static int relayed_lanes(uint8_t flags[MAX_POSITIONS][MAX_LANES], char *err, size_t len) { + memset(flags, 0, sizeof(uint8_t[MAX_POSITIONS][MAX_LANES])); + const char *text = sccl_env("LIBSIRCL_RELAYED_LANES"); + if (!text || !*text) return 0; + if (strlen(text) >= 1024) { + put_error(err, len, "LIBSIRCL_RELAYED_LANES is longer than 1023 characters"); + return -1; + } + char copy[1024]; + snprintf(copy, sizeof copy, "%s", text); + for (char *save = NULL, *entry = strtok_r(copy, ",", &save); entry; entry = strtok_r(NULL, ",", &save)) { + char *equals = strchr(entry, '='), *end; + long position = equals ? strtol(entry, &end, 10) : -1; + if (!equals || end != equals || position < 0 || position >= MAX_POSITIONS) { + put_error(err, len, "LIBSIRCL_RELAYED_LANES entry '%s' is not =<0|1>[/<0|1>]", entry); + return -1; + } + int lane = 0; + for (char *save2 = NULL, *value = strtok_r(equals + 1, "/", &save2); value; + value = strtok_r(NULL, "/", &save2), ++lane) { + if (lane >= MAX_LANES || (strcmp(value, "0") && strcmp(value, "1"))) { + put_error(err, len, "LIBSIRCL_RELAYED_LANES entry for position %ld needs 1 or 2 flags of 0 or 1", position); + return -1; + } + flags[position][lane] = value[0] == '1'; + } + } + return 0; +} + +/* The positions reached through relays, as bits: those LIBSIRCL_FORWARD_WINDOWS gives a nonzero window + * toward, and those LIBSIRCL_RELAYED_LANES marks. A malformed entry counts as none here (read_settings and + * forward_windows() report it). */ static uint64_t relayed_positions(void) { - const char *text = sccl_env("LIBSIRCL_FORWARD_WINDOWS"); + uint8_t flags[MAX_POSITIONS][MAX_LANES]; uint64_t mask = 0; - if (!text || !*text) return 0; + if (relayed_lanes(flags, NULL, 0) == 0) + for (int p = 0; p < MAX_POSITIONS; ++p) + for (int l = 0; l < MAX_LANES; ++l) + if (flags[p][l]) mask |= 1ull << p; + const char *text = sccl_env("LIBSIRCL_FORWARD_WINDOWS"); + if (!text || !*text) return mask; char copy[1024]; snprintf(copy, sizeof copy, "%s", text); for (char *save = NULL, *entry = strtok_r(copy, ",", &save); entry; entry = strtok_r(NULL, ",", &save)) { @@ -779,9 +845,15 @@ static uint64_t relayed_positions(void) { return mask; } -/* Whether LIBSIRCL_FORWARD_WINDOWS gives a nonzero window on some lane toward rank `peer` (the peer is - * reached through relays). A malformed value counts as none here; forward_windows() reports it. */ +/* Whether rank `peer` is reached through relays: LIBSIRCL_RELAYED_LANES marks a lane toward it, or + * LIBSIRCL_FORWARD_WINDOWS gives a lane toward it a nonzero window. A malformed value counts as none here; + * read_settings and forward_windows() report it. */ static int relayed_toward(const sccl_engine *e, int peer) { + uint8_t flags[MAX_POSITIONS][MAX_LANES]; + int position = e->positions[peer]; + if (relayed_lanes(flags, NULL, 0) == 0 && position >= 0 && position < MAX_POSITIONS) + for (int l = 0; l < MAX_LANES; ++l) + if (flags[position][l]) return 1; const char *text = sccl_env("LIBSIRCL_FORWARD_WINDOWS"); if (!text || !*text) return 0; char copy[1024]; @@ -841,6 +913,7 @@ static int p2p_settings(sccl_engine *e, shared_settings *s, char *err, size_t le return 0; } +static int traffic_class(int *out, char *err, size_t len); static int read_settings(sccl_engine *e, int emulation, char *err, size_t len) { shared_settings *s = &e->shared; uint64_t v; @@ -913,14 +986,51 @@ static int read_settings(sccl_engine *e, int emulation, char *err, size_t len) { return -1; } e->serving = regime && !strcmp(regime, "serving"); + s->wait_regime = (uint32_t)e->serving; const char *stop = sccl_env("LIBSIRCL_FAIL_STOP"); if (stop && *stop && strcmp(stop, "0") && strcmp(stop, "1") && strcmp(stop, "abort")) { put_error(err, len, "LIBSIRCL_FAIL_STOP=%s: 0, 1 or abort", stop); return -1; } e->fail_stop = !stop || !*stop || !strcmp(stop, "0") ? 0 : !strcmp(stop, "1") ? FAIL_STOP_EXIT : FAIL_STOP_ABORT; + uint8_t relayed[MAX_POSITIONS][MAX_LANES]; + if (relayed_lanes(relayed, err, len)) return -1; + /* Read with the other settings, so that a bad value fails the first verdict on every rank. */ + if (env_u64("LIBSIRCL_LANE_CHECK_MS", 10000, 1, 600000, &v, err, len)) return -1; + e->lane_check_ms = (int)v; + if (traffic_class(&e->traffic_class, err, len)) return -1; + /* The list variables are parsed from 1,024-byte copies: a longer value is refused, not cut. */ + static const char *const lists[] = {"SIRCL_PEER_ROUTES", "LIBSIRCL_FORWARD_WINDOWS", "LIBSIRCL_P2P_WINDOWS", + "LIBSIRCL_RELAYED_LANES", "LIBSIRCL_CHAIN_ORDER", "SIRCL_GID_INDEX", + "SIRCL_PROGRESS_CPU", "LIBSIRCL_LINK_BLOCKS_CYCLE"}; + for (size_t i = 0; i < sizeof lists / sizeof lists[0]; ++i) { + const char *text = sccl_env(lists[i]); + if (text && strlen(text) >= 1024) { + put_error(err, len, "%s is longer than 1023 characters", lists[i]); + return -1; + } + } + const char *interval = sccl_env("LIBSIRCL_RECEIPT_INTERVAL_S"); + double seconds = 10.0; + if (interval && *interval) { + char *end; + seconds = strtod(interval, &end); + if (*end || !(seconds >= 0 && seconds <= 86400)) { + put_error(err, len, "LIBSIRCL_RECEIPT_INTERVAL_S=%s must be a number of seconds from 0 to 86400", interval); + return -1; + } + } + e->receipt_interval_ns = (uint64_t)(seconds * 1e9); if (env_u64("LIBSIRCL_RING_REDUCE_PASSES", 2, 1, 2, &v, err, len)) return -1; + const char *one_pass = sccl_env("LIBSIRCL_RING_REDUCE_ONE_PASS_RESEARCH"); + if (v == 1 && (!one_pass || strcmp(one_pass, "1"))) { + put_error(err, len, "LIBSIRCL_RING_REDUCE_PASSES=1 (the one-pass ring relay) is research-only: one run returned " + "differing bits on one rank and the cause is not isolated; set LIBSIRCL_RING_REDUCE_ONE_PASS_RESEARCH=1 " + "to select it"); + return -1; + } e->ring_reduce_passes = (int)v; + s->ring_reduce_passes = (uint32_t)v; const char *policy = sccl_env("LIBSIRCL_CPU_POLICY"); if (policy && *policy && strcmp(policy, "performance") && strcmp(policy, "none")) { put_error(err, len, "LIBSIRCL_CPU_POLICY=%s: performance or none", policy); @@ -1094,6 +1204,8 @@ static const char *settings_difference(const shared_settings *a, const shared_se SAME(algorithm_plus_one, "SIRCL_ALLREDUCE_ALGORITHM"); SAME(startup_wait_us, "SIRCL_STARTUP_WAIT_S"); SAME(serving_wait_us, "SIRCL_SERVING_WAIT_S"); + SAME(wait_regime, "LIBSIRCL_WAIT_REGIME"); + SAME(fail_stop, "LIBSIRCL_FAIL_STOP"); SAME(proxy_abi, "native proxy ABI"); SAME(graph_staging, "stream-ordered allocation (cuMemAllocAsync and the device's memory pools), which " "stages unaligned buffers under CUDA graph capture"); @@ -1113,6 +1225,7 @@ static const char *settings_difference(const shared_settings *a, const shared_se SAME(blocks_set, "SIRCL_LINK_BLOCKS"); SAME(pair_plan, "the pair plan (SIRCL_LARGE_SCHEDULE, LIBSIRCL_FORWARD_WINDOWS, LIBSIRCL_RING_WINDOW)"); SAME(cycle_plan, "the cycle plan (SIRCL_LARGE_SCHEDULE, LIBSIRCL_FORWARD_WINDOWS, LIBSIRCL_RING_WINDOW)"); + SAME(ring_reduce_passes, "LIBSIRCL_RING_REDUCE_PASSES"); SAME(p2p_on, "LIBSIRCL_P2P_CHANNELS"); SAME(p2p_abi, "point-to-point native ABI"); SAME(p2p_slots, "SIRCL_P2P_SLOTS"); @@ -1236,6 +1349,8 @@ static int start_progress(sccl_engine *e) { * rank at that position (SIRCL's routes.forward_windows for the layout; tools/site_routes.py prints it), * 0 for a direct lane; positions outside the communicator are ignored. SIRCL_FORWARD_CHUNK_BYTES is the * chunk a windowed stripe is posted in (SIRCL's default 32,768). */ +/* The native proxy accepts a forward window of 1 to 60 chunks (roce_set_forward). */ +enum { FORWARD_MAX_CHUNKS = 60 }; static int forward_windows(sccl_engine *e, char *err, size_t len) { memset(e->forward, 0, sizeof e->forward); e->forward_lanes = 0; @@ -1266,8 +1381,19 @@ static int forward_windows(sccl_engine *e, char *err, size_t len) { value = strtok_r(NULL, "/", &save2), ++lane) { char *stop; unsigned long bytes = strtoul(value, &stop, 10); - if (*stop || lane >= MAX_LANES || bytes > 0xFFFFFFFFul) { - put_error(err, len, "LIBSIRCL_FORWARD_WINDOWS entry '%s' needs 1 or 2 byte counts", entry); + if (*stop || !*value || bytes > 0xFFFFFFFFul) { + put_error(err, len, "LIBSIRCL_FORWARD_WINDOWS: '%s' toward position %ld is not a byte count", value, position); + return -1; + } + if (lane >= MAX_LANES) { + put_error(err, len, "LIBSIRCL_FORWARD_WINDOWS: the entry for position %ld has more than %d lanes", position, + MAX_LANES); + return -1; + } + if (bytes && (bytes < chunk || bytes / chunk > FORWARD_MAX_CHUNKS)) { + put_error(err, len, "LIBSIRCL_FORWARD_WINDOWS: the window of %lu bytes toward position %ld must be 0 or " + "hold 1 to %d chunks of SIRCL_FORWARD_CHUNK_BYTES (%" PRIu64 ")", bytes, position, + FORWARD_MAX_CHUNKS, chunk); return -1; } if (peer < 0 || peer == e->rank || lane >= e->lanes) continue; @@ -1278,9 +1404,6 @@ static int forward_windows(sccl_engine *e, char *err, size_t len) { return 0; } -/* The native layer's ring links post through relays in chunks of 32 KiB and accept a ring window of 1 to - * 60 such chunks (roce_set_links). */ -enum { RING_WINDOW_CHUNK = 32768, RING_WINDOW_MAX_CHUNKS = 60 }; /* The window of this communicator's ring lanes toward its ring's next rank. LIBSIRCL_RING_WINDOW is the * process's layout's ring plan, the same for every communicator the process joins; for the layout's own @@ -1289,18 +1412,25 @@ enum { RING_WINDOW_CHUNK = 32768, RING_WINDOW_MAX_CHUNKS = 60 }; * pairs i and i + 4 of a ring of eight) may have a ring next reached through relays, which the forward * windows name: its ring lanes then keep the smallest forward window toward that rank in flight, never * posting through relays unwindowed. A nonzero LIBSIRCL_RING_WINDOW is the planner's own value and is kept. */ -static void ring_lane_window(sccl_engine *e) { - if (e->ring_window || e->ring_next < 0) return; +static int ring_lane_window(sccl_engine *e, char *err, size_t len) { + if (e->ring_window || e->ring_next < 0) return 0; uint32_t window = 0; for (int l = 0; l < e->lanes; ++l) { uint32_t lane = e->forward[e->ring_next][l]; if (lane && (!window || lane < window)) window = lane; } - if (!window) return; + if (!window) return 0; + /* Never above the forward window: a forward window below one ring chunk leaves the ring lanes no window + * within it, and the communicator is refused rather than posting past it. */ + if (window < RING_WINDOW_CHUNK) { + put_error(err, len, "the ring lanes toward rank %d cross relays whose forward window (LIBSIRCL_FORWARD_WINDOWS) " + "is %u bytes, below one %d-byte ring chunk", e->ring_next, window, RING_WINDOW_CHUNK); + return -1; + } window = window / RING_WINDOW_CHUNK * RING_WINDOW_CHUNK; - if (window < RING_WINDOW_CHUNK) window = RING_WINDOW_CHUNK; if (window > RING_WINDOW_CHUNK * RING_WINDOW_MAX_CHUNKS) window = RING_WINDOW_CHUNK * RING_WINDOW_MAX_CHUNKS; e->ring_window = window; + return 0; } /* LIBSIRCL_P2P_WINDOWS: =[/],... the window of each of this process's lanes toward the @@ -1335,8 +1465,13 @@ static int p2p_windows(sccl_engine *e, char *err, size_t len) { value = strtok_r(NULL, "/", &save2), ++lane) { char *stop; unsigned long bytes = strtoul(value, &stop, 10); - if (*stop || lane >= MAX_LANES || bytes > 0xFFFFFFFFul) { - put_error(err, len, "LIBSIRCL_P2P_WINDOWS entry for position %ld needs 1 or 2 byte counts", position); + if (*stop || !*value || bytes > 0xFFFFFFFFul) { + put_error(err, len, "LIBSIRCL_P2P_WINDOWS: '%s' toward position %ld is not a byte count", value, position); + return -1; + } + if (lane >= MAX_LANES) { + put_error(err, len, "LIBSIRCL_P2P_WINDOWS: the entry for position %ld has more than %d lanes", position, + MAX_LANES); return -1; } if (bytes && (bytes % PACK || bytes < chunk || bytes / chunk > P2P_MAX_WINDOW_CHUNKS)) { @@ -1351,11 +1486,17 @@ static int p2p_windows(sccl_engine *e, char *err, size_t len) { } } } + /* A lane is relayed when it has a forward window or LIBSIRCL_RELAYED_LANES marks it; a relayed lane without + * a point-to-point window gives its pair no channel (SIRCL's budget refuses unwindowed relayed lanes). */ + uint8_t relayed[MAX_POSITIONS][MAX_LANES]; + if (relayed_lanes(relayed, err, len)) return -1; e->p2p_out = 0; for (int p = 0; p < e->world; ++p) { - int ok = p != e->rank; + int ok = p != e->rank, position = e->positions[p]; for (int l = 0; ok && l < e->lanes; ++l) - if (e->forward[p][l] && !e->p2p_window[p][l]) ok = 0; + if ((e->forward[p][l] || (position >= 0 && position < MAX_POSITIONS && relayed[position][l])) && + !e->p2p_window[p][l]) + ok = 0; if (ok) e->p2p_out |= 1u << p; } return 0; @@ -1520,13 +1661,18 @@ static int gid_indices(route_table *t, char *err, size_t len) { return 0; } -static int traffic_class(void) { +/* The IB traffic class of the queue pairs: SIRCL_TRAFFIC_CLASS, else NCCL_IB_TC, else 0; a value that is not + * an integer from 0 to 255 is refused, naming the variable. */ +static int traffic_class(int *out, char *err, size_t len) { const char *names[] = {"SIRCL_TRAFFIC_CLASS", "NCCL_IB_TC"}; + *out = 0; for (int i = 0; i < 2; ++i) { const char *text = sccl_env(names[i]); if (text && *text) { - long value = strtol(text, NULL, 10); - return value >= 0 && value <= 255 ? (int)value : 0; + uint64_t value; + if (env_u64(names[i], 0, 0, 255, &value, err, len)) return -1; + *out = (int)value; + return 0; } } return 0; @@ -1739,6 +1885,24 @@ static int p2p_close(sccl_engine *e, int free_memory) { return 0; } +/* Stop the point-to-point channels so that every peer learns of it: poison the control line, so this rank's + * channel kernels return and its progress thread takes its failure path, which writes an abort notice into + * every peer's abort word (SIRCL's post_aborts); wait up to P2P_NOTICE_WAIT_NS for the thread to report the + * failure, then stop it, which joins it after the notices were posted. A thread that has not reported by + * then is stopped without notices; peers then fall back to their own wait limits. Not after the progress + * thread was stopped (teardown_stop), when no thread is left to post. */ +enum { P2P_NOTICE_WAIT_NS = 50000000 }; +static void p2p_fault(sccl_engine *e, const char *why); +static uint64_t monotonic_ns(void); +static void p2p_stop_with_notices(sccl_engine *e, const char *why) { + if (!e->p2p || !e->p2p_ctrl || e->stopped) return; + p2p_fault(e, why); + uint64_t until = monotonic_ns() + P2P_NOTICE_WAIT_NS; + const struct timespec pause = {0, 100000}; + while (!e->p2p_transport->failed(e->p2p) && monotonic_ns() < until) nanosleep(&pause, NULL); + e->p2p_transport->stop(e->p2p); +} + /* After the first setup exchange gave every rank's p2p_out: ranks a and b have a channel when each one's lanes * toward the other can carry items, so every rank derives the same table. A rank without a channel still * opens a context (every rank's connection record is validated together); when no pair of the group has one, @@ -1801,7 +1965,7 @@ static int p2p_open(sccl_engine *e, const route_table *routes, char *err, size_t } char native[512] = {0}; e->p2p = e->p2p_transport->create(e->world, e->rank, names, routes->n_devices, lane_devices, e->lanes, - routes->gid_index, traffic_class(), channels, e->p2p_host, e->p2p_bytes, + routes->gid_index, e->traffic_class, channels, e->p2p_host, e->p2p_bytes, (int)s->p2p_slots, s->p2p_slot_bytes, native, sizeof native); if (!e->p2p) { put_error(err, len, "point-to-point native setup: %s", native); @@ -1910,7 +2074,31 @@ static ncclResult_t session_error(sccl_engine *e, char *message, size_t len); /* The line names the time twice: the wall clock for logs, and CLOCK_MONOTONIC (the clock of the host's own * timeouts and of Python's time.monotonic), which a stepped wall clock (a VM's time synchronization) does * not move; and what found the error. */ +/* fail_stop's backstop: whatever the steps before _exit wait for (a stalled stderr pipe, a receipt or link + * dump on a stalled filesystem, a FIFO without a reader), the process ends FAIL_STOP_BACKSTOP_MS after + * fail_stop began, with the fail-stop status. */ +enum { FAIL_STOP_BACKSTOP_MS = 1000 }; +static void *fail_stop_backstop(void *unused) { + (void)unused; + const struct timespec delay = {FAIL_STOP_BACKSTOP_MS / 1000, (FAIL_STOP_BACKSTOP_MS % 1000) * 1000000L}; + nanosleep(&delay, NULL); + _exit(FAIL_STOP_STATUS); + return NULL; +} + static void fail_stop(sccl_engine *e, const char *found_by, const char *message) { + /* The backstop first, before any step that can block. A failure to start it leaves the steps unbounded, + * as they are without it. */ + pthread_t backstop; + pthread_attr_t attributes; + if (!pthread_attr_init(&attributes)) { + pthread_attr_setdetachstate(&attributes, PTHREAD_CREATE_DETACHED); + (void)pthread_create(&backstop, &attributes, fail_stop_backstop, NULL); + pthread_attr_destroy(&attributes); + } + /* Before the process ends, the channels' progress thread posts its abort notices, so the peers' channels + * stop now instead of at their wait limits (bounded; see p2p_stop_with_notices). */ + p2p_stop_with_notices(e, message); char line[1200]; uint64_t now = realtime_ns(), mono = monotonic_ns(); int n = snprintf(line, sizeof line, @@ -2037,10 +2225,13 @@ void sccl_engine_fail_stop_shutdown(void) { if (started) pthread_join(watcher, NULL); } +static void quarantine_reclaim(void); int sccl_engine_create(sccl_bootstrap *bootstrap, int nranks, int rank, int position, sccl_CUcontext ctx, const atomic_int *cancelled, sccl_engine **out, char *err, size_t err_len) { *out = NULL; cu = sccl_cuda_get(); + /* Engines quarantined by earlier aborts whose kernels have finished since are freed first. */ + quarantine_reclaim(); sccl_engine *e = calloc(1, sizeof *e); if (!e) { put_error(err, err_len, "out of memory"); @@ -2064,6 +2255,10 @@ int sccl_engine_create(sccl_bootstrap *bootstrap, int nranks, int rank, int posi return ncclRemoteError; } for (int r = 0; r < nranks; ++r) { + if (all[r].position < 0 || all[r].position >= MAX_POSITIONS) { + if (!local[0]) put_error(local, sizeof local, "rank %d position %d is outside 0-63", r, all[r].position); + continue; + } e->positions[r] = all[r].position; e->relayed[r] = all[r].relayed; } @@ -2105,6 +2300,7 @@ int sccl_engine_create(sccl_bootstrap *bootstrap, int nranks, int rank, int posi e->device = device; if (read_settings(e, emulation, local, sizeof local)) break; if (nranks == 1) e->fail_stop = 0; + e->shared.fail_stop = (uint32_t)e->fail_stop; if (e->fail_stop && watch_start(local, sizeof local)) break; e->shared.proxy_abi = (uint32_t)e->transport->abi_version(); /* Before the setup verdict, so a rank whose native proxy lacks a feature fails setup with every rank. */ @@ -2159,21 +2355,31 @@ int sccl_engine_create(sccl_bootstrap *bootstrap, int nranks, int rank, int posi put_error(local, sizeof local, "%s", sccl_kp_error()); break; } + e->kp_held = 1; + /* Every entry the engine launches, looked up once; a lookup that fails refuses setup here, naming the + * entry, instead of leaving a NULL function for the first launch. */ + int missing = 0; for (int a = 0; a < SCCL_ALG_COUNT; ++a) - for (int d = 0; d < SCCL_DT_COUNT; ++d) sccl_kp_function(ctx, a, d, nranks, &e->fn[a][d]); + for (int d = 0; d < SCCL_DT_COUNT; ++d) missing |= sccl_kp_function(ctx, a, d, nranks, &e->fn[a][d]); for (int k = SCCL_K_SCATTER; k < SCCL_K_COUNT; ++k) - for (int d = 0; d < SCCL_DT_COUNT; ++d) sccl_kp_function(ctx, k, d, nranks, &e->fn[k][d]); + for (int d = 0; d < SCCL_DT_COUNT; ++d) missing |= sccl_kp_function(ctx, k, d, nranks, &e->fn[k][d]); for (int d = 0; d < SCCL_FOLD_DTYPES; ++d) - for (int o = 0; o < SCCL_FOLD_OPS; ++o) sccl_kp_fold_function(ctx, d, o, &e->fold[d][o]); - for (int d = 0; d < SCCL_DT_COUNT; ++d) sccl_kp_chain_function(ctx, d, (int)e->shared.chain_unroll, &e->chain_fn[d]); + for (int o = 0; o < SCCL_FOLD_OPS; ++o) missing |= sccl_kp_fold_function(ctx, d, o, &e->fold[d][o]); + for (int d = 0; d < SCCL_DT_COUNT; ++d) + missing |= sccl_kp_chain_function(ctx, d, (int)e->shared.chain_unroll, &e->chain_fn[d]); for (int k = 0; k < SCCL_LINK_KINDS; ++k) for (int d = 0; d < SCCL_DT_COUNT; ++d) - sccl_kp_link_function(ctx, k, d, (int)e->shared.link_unroll, &e->link_fn[k][d]); + missing |= sccl_kp_link_function(ctx, k, d, (int)e->shared.link_unroll, &e->link_fn[k][d]); for (int d = 0; e->ring_reduce_passes == 2 && d < SCCL_DT_COUNT; ++d) - sccl_kp_ring_reduce_two_pass_function(ctx, d, (int)e->shared.link_unroll, &e->link_fn[SCCL_RING_REDUCE][d]); - sccl_kp_ring_exchange_function(ctx, (int)e->shared.link_unroll, &e->exchange_fn); + missing |= sccl_kp_ring_reduce_two_pass_function(ctx, d, (int)e->shared.link_unroll, + &e->link_fn[SCCL_RING_REDUCE][d]); + missing |= sccl_kp_ring_exchange_function(ctx, (int)e->shared.link_unroll, &e->exchange_fn); for (int d = 0; d < SCCL_DT_COUNT; ++d) - sccl_kp_ring_exchange_reduce_function(ctx, d, (int)e->shared.link_unroll, &e->exchange_reduce_fn[d]); + missing |= sccl_kp_ring_exchange_reduce_function(ctx, d, (int)e->shared.link_unroll, &e->exchange_reduce_fn[d]); + if (missing) { + put_error(local, sizeof local, "kernel pack lookup: %s", sccl_kp_error()); + break; + } if (allocate(e, local, sizeof local)) break; e->ctrl[CTRL_WAIT_LIMIT_US] = e->serving ? e->shared.serving_wait_us : e->shared.startup_wait_us; const char *names[MAX_DEVICES]; @@ -2183,7 +2389,7 @@ int sccl_engine_create(sccl_bootstrap *bootstrap, int nranks, int rank, int posi for (int l = 0; l < routes.lanes; ++l) lane_devices[p * routes.lanes + l] = routes.lane_device[p][l]; char proxy_error[512] = {0}; e->proxy = e->transport->create(nranks, rank, names, routes.n_devices, lane_devices, routes.lanes, - routes.gid_index, traffic_class(), e->host, e->region_bytes, + routes.gid_index, e->traffic_class, e->host, e->region_bytes, e->shared.slot_bytes, proxy_error, sizeof proxy_error); if (!e->proxy) { put_error(local, sizeof local, "native setup: %s", proxy_error); @@ -2193,8 +2399,9 @@ int sccl_engine_create(sccl_bootstrap *bootstrap, int nranks, int rank, int posi put_error(local, sizeof local, "chain schedule: %s", e->transport->error(e->proxy)); } else if (forward_windows(e, local, sizeof local) == 0) { /* The forward windows first: the ring lanes' window may come from them (ring_lane_window). */ - if (e->link_on) ring_lane_window(e); - if (e->link_on && + if (e->link_on && ring_lane_window(e, local, sizeof local) != 0) { + /* refused: `local` says why */ + } else if (e->link_on && e->transport->set_links(e->proxy, e->chain_prev, e->chain_next, e->chain_index, e->ring_prev, e->ring_next, e->ring_window, (int)e->shared.link_slots, e->shared.link_slot_bytes, e->link_off) != 0) { @@ -2245,7 +2452,9 @@ int sccl_engine_create(sccl_bootstrap *bootstrap, int nranks, int rank, int posi const char *why = records[r].error[0] ? records[r].error : NULL; const char *field = why || any_error ? NULL : settings_difference(&records[r].settings, &records[0].settings); if (!why && !field) continue; - used += (size_t)snprintf(text + used, used < sizeof text ? sizeof text - used : 0, "%srank %d: %s%s", + /* A peer's error field is a fixed 400-byte array from the wire: printed at most 399 bytes, so a + * record without a terminating zero is never read past its end. */ + used += (size_t)snprintf(text + used, used < sizeof text ? sizeof text - used : 0, "%srank %d: %.399s%s", used ? "; " : "", r, why ? why : field, why ? "" : " differs from rank 0"); if (used >= sizeof text) used = sizeof text - 1; } @@ -2313,17 +2522,13 @@ int sccl_engine_create(sccl_bootstrap *bootstrap, int nranks, int rank, int posi } /* Exchange 3: prove every lane with one write, then start the progress thread. */ { - uint64_t check_ms; - if (env_u64("LIBSIRCL_LANE_CHECK_MS", 10000, 1, 600000, &check_ms, err, err_len)) { - status = ncclInvalidArgument; - goto fail; - } + int check_ms = e->lane_check_ms; char why[400] = {0}; - int rc = e->transport->lane_check(e->proxy, (int)check_ms); + int rc = e->transport->lane_check(e->proxy, check_ms); if (rc) snprintf(why, sizeof why, "lane check: %s", e->transport->error(e->proxy)); if (!rc && e->p2p) { uint64_t started = monotonic_ns(); - rc = e->p2p_transport->lane_check(e->p2p, (int)check_ms) || p2p_apply_windows(e); + rc = e->p2p_transport->lane_check(e->p2p, check_ms) || p2p_apply_windows(e); e->p2p_check_ns = monotonic_ns() - started; if (rc) snprintf(why, sizeof why, "point-to-point lane check: %s", e->p2p_transport->error(e->p2p)); } @@ -2333,16 +2538,15 @@ int sccl_engine_create(sccl_bootstrap *bootstrap, int nranks, int rank, int posi rc = start_progress(e); if (rc) snprintf(why, sizeof why, "%s", rc == 1 ? e->transport->error(e->proxy) : e->p2p_transport->error(e->p2p)); } + /* Every local step before the last verdict, so that no rank fails alone after its peers succeeded. */ + if (rc == 0 && cuda_check(cu->EventCreate(&e->order_event, SCCL_CU_EVENT_DISABLE_TIMING), + "creating the order event", why, sizeof why)) + rc = -1; if (verdict(bootstrap, nranks, rank, rc == 0, why, cancelled, err, err_len)) { status = ncclSystemError; goto fail; } } - if (cuda_check(cu->EventCreate(&e->order_event, SCCL_CU_EVENT_DISABLE_TIMING), "creating the order event", err, - err_len)) { - status = ncclUnhandledCudaError; - goto fail; - } cu->CtxPopCurrent(&ctx); start_receipts(e); if (e->fail_stop) watch_add(e); @@ -2353,6 +2557,7 @@ fail:; p2p_close(e, pushed); int unreleased = e->proxy ? e->transport->destroy(e->proxy) : 0; e->proxy = NULL; + if (e->kp_held) sccl_kp_release(e->ctx, pushed); if (pushed) { if (!unreleased) release_memory(e); cu->CtxPopCurrent(&ctx); @@ -2631,6 +2836,11 @@ static ncclResult_t launch_link_entry(sccl_engine *e, sccl_CUfunction fn, unsign sccl_CUdeviceptr src, sccl_CUdeviceptr dst, sccl_CUdeviceptr scratch, uint64_t chunk, uint64_t stride, uint32_t piece, uint32_t blocks, int32_t flags, sccl_CUstream stream, char *err, size_t len) { + if (!sccl_packs_fit(chunk) || !sccl_packs_fit(stride) || !sccl_packs_fit(piece)) { + put_error(err, len, "%s: chunk %" PRIu64 " or stride %" PRIu64 " bytes: the pack count exceeds the kernel's int32 " + "argument", what, chunk, stride); + return ncclInvalidArgument; + } sccl_link_args a; memset(&a, 0, sizeof a); a.input = src; @@ -2802,7 +3012,8 @@ static int gather_link_kind(const sccl_engine *e, uint64_t shard) { * -1; chunks a multiple of 16 bytes. */ static int scatter_link_kind(const sccl_engine *e, uint64_t chunk) { uint64_t input = chunk * (uint64_t)e->world; - if (!e->link_on || chunk == 0 || chunk % PACK) return -1; + /* The chunk is also the ops' stride, a pack count of the kernels' int32 arguments. */ + if (!e->link_on || chunk == 0 || chunk % PACK || !sccl_packs_fit(chunk)) return -1; uint32_t schedule = effective_schedule(e, e->shared.scatter_schedule, input, COLL_SCATTER); if (schedule == SCHEDULE_RING) return SCCL_RING_SCATTER; if (schedule == SCHEDULE_PIECES) return -1; @@ -2893,6 +3104,10 @@ static ncclResult_t reduce_link(sccl_engine *e, int ring, int dtype, sccl_CUdevi /* One chain op over `nbytes` (a positive multiple of 16) from src to dst, both 16-byte aligned. */ static ncclResult_t launch_chain(sccl_engine *e, int dtype, sccl_CUdeviceptr src, sccl_CUdeviceptr dst, uint64_t nbytes, sccl_CUstream stream, char *err, size_t len) { + if (!sccl_packs_fit(nbytes)) { + put_error(err, len, "chain op of %" PRIu64 " bytes: its pack count exceeds the kernel's int32 argument", nbytes); + return ncclInvalidArgument; + } int32_t packs = (int32_t)(nbytes / PACK); sccl_chain_args a; memset(&a, 0, sizeof a); @@ -2957,6 +3172,11 @@ static ncclResult_t begin_call(sccl_engine *e, const char *what, sccl_CUstream s size_t len) { call->pushed = call->capturing = 0; pthread_mutex_lock(&e->lock); + if (e->closing) { + put_error(err, len, "%s: communicator is closing", what); + pthread_mutex_unlock(&e->lock); + return ncclInvalidUsage; + } sccl_CUcontext current = NULL; if (cu->CtxGetCurrent(¤t) != SCCL_CUDA_SUCCESS || current != e->ctx) { if (cuda_check(cu->CtxPushCurrent(e->ctx), "making the communicator's context current", err, len)) { @@ -2989,8 +3209,8 @@ static ncclResult_t begin_call(sccl_engine *e, const char *what, sccl_CUstream s (cuda_check(cu->EventRecord(e->order_event, e->last_stream), "recording the order event", err, len) || cuda_check(cu->StreamWaitEvent(stream, e->order_event, 0), "ordering the stream", err, len))) result = ncclUnhandledCudaError; - e->last_stream = stream; - e->have_last = 1; + e->have_last = result == ncclSuccess; + if (e->have_last) e->last_stream = stream; } if (result != ncclSuccess) { sccl_CUcontext popped; @@ -3383,17 +3603,17 @@ static ncclResult_t reduce_into(sccl_engine *e, const char *what, const void *se * ring op for the largest prefix of W equal chunks of whole packs; else under the chain schedule (or auto * from the chain minimum) one chain op for the 16-byte-aligned body; the rest in pieces, then the * zero-padded tail. Decided from agreed settings and sizes only, for calls that keep their result on - * every rank (not ncclReduce); unaligned buffers go through the staging buffer. */ + * every rank and for ncclReduce alike: off the root its destination is 0 and the link op runs in place in the + * staging buffer (reduce_link); unaligned buffers go through the staging buffer. */ if (body && e->shared.large_schedule != SCHEDULE_PIECES) { uint32_t schedule = effective_schedule(e, e->shared.large_schedule, nbytes, COLL_REDUCE); - uint64_t ring_bytes = body / (PACK * (uint64_t)e->world) * PACK * (uint64_t)e->world; - if (schedule == SCHEDULE_RING && e->link_on && ring_bytes && ring_bytes / (uint64_t)e->world < (1ull << 31)) { - result = reduce_link(e, 1, dtype, src, dst, ring_bytes, capturing, stream, err, len); - start = ring_bytes; - } else if (schedule != SCHEDULE_RING && e->chain_on && - (schedule == SCHEDULE_CHAIN || body >= e->shared.chain_mins[COLL_REDUCE])) { - result = reduce_link(e, 0, dtype, src, dst, body, capturing, stream, err, len); - start = body; + uint64_t link_bytes = 0; + int plan = sccl_large_reduce_plan(body, (uint64_t)e->world, schedule == SCHEDULE_RING, e->link_on, + e->chain_on && (schedule == SCHEDULE_CHAIN || + body >= e->shared.chain_mins[COLL_REDUCE]), &link_bytes); + if (plan != SCCL_LARGE_PIECES) { + result = reduce_link(e, plan == SCCL_LARGE_RING, dtype, src, dst, link_bytes, capturing, stream, err, len); + start = link_bytes; } } for (uint64_t offset = start; result == ncclSuccess && offset < body; offset += e->shared.large_piece) { @@ -3753,6 +3973,9 @@ ncclResult_t sccl_engine_mem_alloc(void **ptr, size_t size, char *err, size_t le return ncclSuccess; } +/* The memory is freed in the context it was allocated in (CU_POINTER_ATTRIBUTE_CONTEXT), made current for + * the call, whatever context the calling thread has, if any. */ +enum { SCCL_POINTER_ATTRIBUTE_CONTEXT = 1 }; ncclResult_t sccl_engine_mem_free(void *ptr, char *err, size_t len) { if (!ptr) return ncclSuccess; cu = sccl_cuda_get(); @@ -3760,8 +3983,23 @@ ncclResult_t sccl_engine_mem_free(void *ptr, char *err, size_t len) { put_error(err, len, "ncclMemFree: %s", sccl_cuda_error()); return ncclSystemError; } - return cuda_check(cu->MemFree((sccl_CUdeviceptr)(uintptr_t)ptr), "ncclMemFree", err, len) ? ncclUnhandledCudaError - : ncclSuccess; + sccl_CUdeviceptr memory = (sccl_CUdeviceptr)(uintptr_t)ptr; + sccl_CUcontext owner = NULL; + if (cuda_check(cu->PointerGetAttribute(&owner, SCCL_POINTER_ATTRIBUTE_CONTEXT, memory), + "ncclMemFree: finding the memory's context", err, len)) + return ncclInvalidArgument; + int pushed = owner && cu->CtxPushCurrent(owner) == SCCL_CUDA_SUCCESS; + if (owner && !pushed) { + put_error(err, len, "ncclMemFree: the memory's context cannot be made current"); + return ncclUnhandledCudaError; + } + ncclResult_t result = cuda_check(cu->MemFree(memory), "ncclMemFree", err, len) ? ncclUnhandledCudaError + : ncclSuccess; + if (pushed) { + sccl_CUcontext popped; + cu->CtxPopCurrent(&popped); + } + return result; } /* One pair exchange of `chunk` bytes each way: `sent` (0: nothing meaningful) to the peer, the peer's block @@ -4075,6 +4313,13 @@ ncclResult_t sccl_engine_p2p(sccl_engine *e, const sccl_p2p_op *ops, unsigned co ops[sends[1][k]].bytes, ops[recvs[1][k]].bytes); return ncclInvalidUsage; } + /* A channel item's message is a pack count of the point-to-point kernels' int32 arguments. */ + for (unsigned i = 0; e->world > 2 && i < count; ++i) + if (ops[i].peer != e->rank && !sccl_packs_fit(round_up(ops[i].bytes, PACK))) { + put_error(err, len, "%s of %zu bytes: a point-to-point channel message holds at most %" PRIu64 " bytes", + ops[i].send ? "ncclSend" : "ncclRecv", ops[i].bytes, (uint64_t)INT32_MAX * PACK); + return ncclInvalidArgument; + } char message[400]; ncclResult_t health = sccl_engine_async_error(e, message, sizeof message); if (health != ncclSuccess) { @@ -4150,16 +4395,15 @@ ncclResult_t sccl_engine_p2p(sccl_engine *e, const sccl_p2p_op *ops, unsigned co ncclResult_t sccl_engine_teardown_sync(sccl_engine *e) { if (!e || !cu) return ncclSuccess; + pthread_mutex_lock(&e->lock); + e->closing = 1; + pthread_mutex_unlock(&e->lock); ncclResult_t result = ncclSuccess; int pushed = e->ctx && cu->CtxPushCurrent(e->ctx) == SCCL_CUDA_SUCCESS; /* Without the creating context the enqueued work cannot be waited for: a failure, not a skipped wait. */ if (e->ctx && !pushed) result = ncclUnhandledCudaError; - if (pushed && e->have_last && cu->StreamSynchronize(e->last_stream) != SCCL_CUDA_SUCCESS) + if (pushed && cu->CtxSynchronize() != SCCL_CUDA_SUCCESS) result = ncclUnhandledCudaError; - for (int d = 0; pushed && d < 2; ++d) - for (int p = 0; p < e->world && p < MAX_WORLD; ++p) - if (e->p2p_ch[d][p].stream && cu->StreamSynchronize(e->p2p_ch[d][p].stream) != SCCL_CUDA_SUCCESS) - result = ncclUnhandledCudaError; if (pushed) { sccl_CUcontext popped; cu->CtxPopCurrent(&popped); @@ -4174,17 +4418,7 @@ int sccl_engine_wait_limit_ms(const sccl_engine *e) { } ncclResult_t sccl_engine_finalize(sccl_engine *e) { - if (!e || !cu) return ncclSuccess; - pthread_mutex_lock(&e->lock); - ncclResult_t result = ncclSuccess; - if (e->have_last) { - int pushed = cu->CtxPushCurrent(e->ctx) == SCCL_CUDA_SUCCESS; - if (cu->StreamSynchronize(e->last_stream) != SCCL_CUDA_SUCCESS) result = ncclUnhandledCudaError; - sccl_CUcontext popped; - if (pushed) cu->CtxPopCurrent(&popped); - } - pthread_mutex_unlock(&e->lock); - return result; + return sccl_engine_teardown_sync(e); } ncclResult_t sccl_engine_set_wait_regime(sccl_engine *e, const char *regime) { @@ -4295,6 +4529,8 @@ static void p2p_receipt(sccl_engine *e, char *out, size_t len) { c ? (int)t->stat(c, P2P_STAT_ABORT_FROM) - 1 : -1); } +/* The bytes quarantined engines hold (see quarantine_reclaim), reported in every receipt. */ +static _Atomic uint64_t quarantined_bytes; size_t sccl_engine_receipt(sccl_engine *e, char *out, size_t len) { static const char *const dt[SCCL_DT_COUNT] = {"float32", "float16", "bfloat16"}; char ops[512] = {0}; @@ -4342,7 +4578,7 @@ size_t sccl_engine_receipt(sccl_engine *e, char *out, size_t len) { out, len, "{\"schema\":\"libsircl-receipt/v1\",\"library\":\"libsircl " LIBSIRCL_VERSION "\",\"pid\":%d," "\"communicator\":%d,\"rank\":%d,\"world\":%d,\"position\":%d,\"lanes\":%d," - "\"device\":%d," + "\"device\":%d,\"quarantined_bytes\":%" PRIu64 "," "\"transport\":\"%s\",\"kernel_pack\":\"%s\",\"fold_pack\":\"%s\",\"links_pack\":\"%s\"," "\"slot_bytes\":%" PRIu64 ",\"capacity\":%" PRIu64 @@ -4367,6 +4603,7 @@ size_t sccl_engine_receipt(sccl_engine *e, char *out, size_t len) { "},\"forwarded\":0,\"native\":{\"ops_posted\":%" PRIu64 ",\"writes_completed\":%" PRIu64 ",\"phases_posted\":%" PRIu64 "},\"healthy\":%s,\"error\":%s}", (int)getpid(), e->receipt_id, e->rank, e->world, e->position, e->lanes, e->device, + atomic_load_explicit(&quarantined_bytes, memory_order_relaxed), e->world == 1 ? "none" : e->transport->name, sccl_kp_hash(), sccl_kp_fold_hash(), sccl_kp_links_hash(), @@ -4400,20 +4637,39 @@ size_t sccl_engine_receipt(sccl_engine *e, char *out, size_t len) { status == ncclSuccess ? "true" : "false", health_json); } +/* A diagnostic file of this library: created by open(2) with O_NOFOLLOW (a symbolic link at the path is + * refused, never followed) and close-on-exec; with `fresh`, also O_EXCL (the file must not exist), else + * appended to. NULL when the path did not fit its buffer (`fits` 0) or the open failed. */ +static FILE *open_diagnostic(const char *path, int fits, int fresh) { + if (!fits) return NULL; + int flags = O_WRONLY | O_CREAT | O_NOFOLLOW | O_CLOEXEC | (fresh ? O_EXCL : O_APPEND); + int fd = open(path, flags, 0644); + if (fd < 0) return NULL; + FILE *f = fdopen(fd, fresh ? "w" : "a"); + if (!f) close(fd); + return f; +} + /* .rank..c.json, n the communicator's number among the process's communicators, so * communicators of one process with the same rank keep files of their own. The file is replaced whole - * (written beside it, then renamed), so a reader never sees a partial receipt. */ + * (written beside it under a name of this writer's own, then renamed), so a reader never sees a partial + * receipt, and two writers (the fail-stop watcher and a library call) never share a partial file. A path + * that does not fit is not written. */ static void write_receipt(sccl_engine *e) { + static atomic_uint writes; const char *prefix = sccl_env("LIBSIRCL_RECEIPT"); if (!prefix || !*prefix) return; - char path[600], partial[620]; - snprintf(path, sizeof path, "%s.rank%d.%d.c%d.json", prefix, e->rank, (int)getpid(), e->receipt_id); - snprintf(partial, sizeof partial, "%s.partial", path); + char path[600], partial[640]; + int n = snprintf(path, sizeof path, "%s.rank%d.%d.c%d.json", prefix, e->rank, (int)getpid(), e->receipt_id); + int fits = n > 0 && (size_t)n < sizeof path; + n = snprintf(partial, sizeof partial, "%s.partial.%u", path, atomic_fetch_add(&writes, 1)); + fits = fits && n > 0 && (size_t)n < sizeof partial; + if (!fits) return; size_t need = sccl_engine_receipt(e, NULL, 0) + 1; char *text = malloc(need); if (!text) return; sccl_engine_receipt(e, text, need); - FILE *f = fopen(partial, "w"); + FILE *f = open_diagnostic(partial, 1, 1); if (f) { int ok = fprintf(f, "%s\n", text) > 0; ok = fclose(f) == 0 && ok; @@ -4444,9 +4700,7 @@ static void start_receipts(sccl_engine *e) { e->receipt_id = atomic_fetch_add(&communicators, 1) + 1; const char *prefix = sccl_env("LIBSIRCL_RECEIPT"); if (!prefix || !*prefix) return; - const char *text = sccl_env("LIBSIRCL_RECEIPT_INTERVAL_S"); - double seconds = text && *text ? strtod(text, NULL) : 10.0; - e->receipt_interval_ns = seconds > 0 ? (uint64_t)(seconds * 1e9) : 0; + /* e->receipt_interval_ns: LIBSIRCL_RECEIPT_INTERVAL_S, validated by read_settings (0: no refresh). */ e->receipt_next_ns = monotonic_ns() + e->receipt_interval_ns; write_receipt(e); } @@ -4471,8 +4725,8 @@ static void dump_links(sccl_engine *e, const char *why, int device) { const char *prefix = sccl_env("LIBSIRCL_LINK_DUMP"); if (!prefix || !*prefix || !e->link_on || !e->host) return; char path[640]; - snprintf(path, sizeof path, "%s.rank%d.%d.c%d.links.json", prefix, e->rank, (int)getpid(), e->receipt_id); - FILE *f = fopen(path, "a"); + int n = snprintf(path, sizeof path, "%s.rank%d.%d.c%d.links.json", prefix, e->rank, (int)getpid(), e->receipt_id); + FILE *f = open_diagnostic(path, n > 0 && (size_t)n < sizeof path, 0); if (!f) return; enum { LINKS = 4, STRIDE = 128 }; const uint8_t *area = e->host + e->link_off; @@ -4538,8 +4792,8 @@ static void dump_teardown(sccl_engine *e, uint64_t stop_ns, uint64_t stopped_ns, const char *prefix = sccl_env("LIBSIRCL_LINK_DUMP"); if (!prefix || !*prefix || !e->link_on || !e->host) return; char path[640]; - snprintf(path, sizeof path, "%s.rank%d.%d.c%d.links.json", prefix, e->rank, (int)getpid(), e->receipt_id); - FILE *f = fopen(path, "a"); + int n = snprintf(path, sizeof path, "%s.rank%d.%d.c%d.links.json", prefix, e->rank, (int)getpid(), e->receipt_id); + FILE *f = open_diagnostic(path, n > 0 && (size_t)n < sizeof path, 0); if (!f) return; fprintf(f, "{\"why\":\"proxy destroyed\",\"ns\":%" PRIu64 ",\"rank\":%d,\"position\":%d,\"stop_ns\":%" PRIu64 ",\"stopped_ns\":%" PRIu64 ",\"destroyed_ns\":%" PRIu64, @@ -4550,7 +4804,12 @@ static void dump_teardown(sccl_engine *e, uint64_t stop_ns, uint64_t stopped_ns, } void sccl_engine_teardown_stop(sccl_engine *e) { - if (!e || e->stopped) return; + if (!e) return; + pthread_mutex_lock(&e->lock); + e->closing = 1; + int stopped = e->stopped; + pthread_mutex_unlock(&e->lock); + if (stopped) return; write_receipt(e); int pushed = cu && e->ctx && cu->CtxPushCurrent(e->ctx) == SCCL_CUDA_SUCCESS; dump_links(e, "destroy", pushed); @@ -4565,30 +4824,104 @@ void sccl_engine_teardown_stop(sccl_engine *e) { e->stopped = 1; } +/* Engines whose arena a kernel may still read: an abort found a launch still running. Each holds events + * recorded behind its last launch and its channels' work; later creations and destroys retry them + * (quarantine_reclaim) and free every engine whose events completed, or whose context is gone (and with it + * every allocation and kernel). Engines kept because a queue pair or registration could not be released + * are not quarantined here and stay allocated: such an object may still be written. */ +static pthread_mutex_t quarantine_lock = PTHREAD_MUTEX_INITIALIZER; +static sccl_engine *quarantined; +static _Atomic uint64_t quarantined_bytes; + +static void quarantine_reclaim(void) { + if (!cu) return; + pthread_mutex_lock(&quarantine_lock); + for (sccl_engine **at = &quarantined; *at;) { + sccl_engine *e = *at; + int pushed = cu->CtxPushCurrent(e->ctx) == SCCL_CUDA_SUCCESS, done = 1; + for (int i = 0; pushed && done && i < e->quarantine_events; ++i) + done = cu->EventQuery(e->quarantine_done[i]) == SCCL_CUDA_SUCCESS; + if (!done) { + sccl_CUcontext popped; + cu->CtxPopCurrent(&popped); + at = &e->quarantine_next; + continue; + } + *at = e->quarantine_next; + atomic_fetch_sub_explicit(&quarantined_bytes, e->quarantine_bytes, memory_order_relaxed); + if (pushed) { + for (int i = 0; i < e->quarantine_events; ++i) cu->EventDestroy(e->quarantine_done[i]); + if (e->order_event) cu->EventDestroy(e->order_event); + p2p_close(e, 1); + release_memory(e); + if (e->kp_held) sccl_kp_release(e->ctx, 1); + sccl_CUcontext popped; + cu->CtxPopCurrent(&popped); + } else { + /* The context is gone, and with it every device allocation and kernel: only host bookkeeping. */ + free(e->retired_stage); + if (e->host && e->host_emulated) sccl_emu_segment_free(e->host); + if (e->p2p_host && e->p2p_host_emulated) sccl_emu_segment_free(e->p2p_host); + if (e->kp_held) sccl_kp_release(e->ctx, 0); + } + free(e); + } + pthread_mutex_unlock(&quarantine_lock); +} + +/* With the engine's context current: record the events quarantine_reclaim waits for (behind the last + * launch and every channel stream) and put the engine on the quarantine list. 0 when an event could not be + * made: the engine then stays allocated without a retry. */ +static int quarantine_add(sccl_engine *e) { + int n = 0; + sccl_CUstream streams[1 + 2 * MAX_WORLD]; + if (e->have_last) streams[n++] = e->last_stream; + for (int d = 0; d < 2; ++d) + for (int p = 0; p < e->world && p < MAX_WORLD; ++p) + if (e->p2p_ch[d][p].stream) streams[n++] = e->p2p_ch[d][p].stream; + e->quarantine_events = 0; + for (int i = 0; i < n; ++i) { + if (cu->EventCreate(&e->quarantine_done[i], SCCL_CU_EVENT_DISABLE_TIMING) != SCCL_CUDA_SUCCESS) return 0; + e->quarantine_events = i + 1; + if (cu->EventRecord(e->quarantine_done[i], streams[i]) != SCCL_CUDA_SUCCESS) return 0; + } + e->quarantine_bytes = e->region_bytes + e->shared.slot_bytes * (uint64_t)(1 + e->world); + atomic_fetch_add_explicit(&quarantined_bytes, e->quarantine_bytes, memory_order_relaxed); + pthread_mutex_lock(&quarantine_lock); + e->quarantine_next = quarantined; + quarantined = e; + pthread_mutex_unlock(&quarantine_lock); + return 1; +} + ncclResult_t sccl_engine_destroy(sccl_engine *e, int abort, char *err, size_t len) { if (!e) return ncclSuccess; + quarantine_reclaim(); + pthread_mutex_lock(&e->lock); + e->closing = 1; + pthread_mutex_unlock(&e->lock); watch_remove(e); ncclResult_t result = ncclSuccess; int stopped = e->stopped; if (!stopped) write_receipt(e); int pushed = cu && e->ctx && cu->CtxPushCurrent(e->ctx) == SCCL_CUDA_SUCCESS; - int idle = 1; - if (pushed && e->have_last) { - if (abort) { - idle = cu->StreamQuery(e->last_stream) == SCCL_CUDA_SUCCESS; - } else if (!stopped) { - cu->StreamSynchronize(e->last_stream); - } + int idle = pushed; + if (pushed && !abort && cu->CtxSynchronize() != SCCL_CUDA_SUCCESS) { + idle = 0; + result = ncclUnhandledCudaError; + put_error(err, len, "waiting for the communicator's context failed; its arena stays allocated"); + } + /* Abort stops the channels first: their kernels return once they see the poison word, and every peer gets + * the abort notice instead of waiting out its wait limit. A channel kernel still running at the idle check + * below keeps the arena allocated (quarantined). */ + if (abort) p2p_stop_with_notices(e, "ncclCommAbort"); + if (pushed && abort) { + if (e->have_last) idle = cu->StreamQuery(e->last_stream) == SCCL_CUDA_SUCCESS; + for (int d = 0; d < 2; ++d) + for (int p = 0; p < e->world && p < MAX_WORLD; ++p) + if (e->p2p_ch[d][p].stream) + idle = idle && cu->StreamQuery(e->p2p_ch[d][p].stream) == SCCL_CUDA_SUCCESS; } - for (int d = 0; pushed && d < 2; ++d) - for (int p = 0; p < e->world && p < MAX_WORLD; ++p) { - sccl_CUstream channel = e->p2p_ch[d][p].stream; - if (!channel) continue; - if (abort) - idle = idle && cu->StreamQuery(channel) == SCCL_CUDA_SUCCESS; - else if (!stopped) - cu->StreamSynchronize(channel); - } if (!stopped) { dump_links(e, "destroy", pushed && idle); e->stop_ns = realtime_ns(); @@ -4605,7 +4938,7 @@ ncclResult_t sccl_engine_destroy(sccl_engine *e, int abort, char *err, size_t le "verbs calls); their arena stays allocated", e->rank, p2p_unreleased); } /* A queue pair, completion queue or registration whose release failed may still be live: the arena - * and device memory stay allocated (quarantined) until the process exits. */ + * and device memory stay allocated until the process exits (no retry frees them). */ int unreleased = e->proxy ? e->transport->destroy(e->proxy) : 0; e->proxy = NULL; int released = unreleased == 0 && p2p_unreleased == 0; @@ -4616,16 +4949,35 @@ ncclResult_t sccl_engine_destroy(sccl_engine *e, int abort, char *err, size_t le } dump_teardown(e, e->stop_ns, e->stopped_ns, realtime_ns()); /* After an abort with a launch still running, its kernel may read the arena and counters - * until its wait limit; they stay allocated until the process exits. */ - if (pushed && idle && released) { + * until its wait limit; they stay allocated, quarantined, until a later creation or destroy finds that + * kernel finished (quarantine_reclaim). */ + int freed = pushed && idle && released; + if (freed) { if (e->order_event) cu->EventDestroy(e->order_event); release_memory(e); - } + /* No kernel of the packs runs any more: the context's pack entry may go (its last reference unloads). */ + if (e->kp_held) sccl_kp_release(e->ctx, 1); + } else if (!pushed && released) { + if (e->kp_held) sccl_kp_release(e->ctx, 0); + /* The communicator's context could not be made current: it was destroyed (or the driver is shutting + * down), and with it every device and pinned allocation and every kernel that could read them. Only the + * host bookkeeping is left: the retired staging list, the shared-memory segments of the emulation + * transport, and the engine itself. */ + free(e->retired_stage); + e->retired_stage = NULL; + if (e->host && e->host_emulated) sccl_emu_segment_free(e->host); + if (e->p2p_host && e->p2p_host_emulated) sccl_emu_segment_free(e->p2p_host); + e->host = e->p2p_host = NULL; + freed = 1; + } + /* The lock goes before quarantine_add: once the engine is on the quarantine list, another thread's + * quarantine_reclaim may free it, so nothing here may touch it after a successful add. */ + pthread_mutex_destroy(&e->lock); + int queued = !freed && pushed && released && quarantine_add(e); if (pushed) { sccl_CUcontext popped; cu->CtxPopCurrent(&popped); } - pthread_mutex_destroy(&e->lock); - if (idle && released) free(e); + if (freed && !queued) free(e); return result; } diff --git a/spark_transport/libsircl/src/kernelpack.c b/spark_transport/libsircl/src/kernelpack.c index 3b695e0a..e7454f3c 100644 --- a/spark_transport/libsircl/src/kernelpack.c +++ b/spark_transport/libsircl/src/kernelpack.c @@ -20,8 +20,14 @@ extern const unsigned char sccl_p2p_fatbin[]; extern const char sccl_p2p_fatbin_sha256[]; enum { MAX_CONTEXTS = 16 }; +/* The GPU architectures the packs hold cubins for (the build's nvcc -gencode code= values), named when a + * pack does not load, for example on a GPU of another architecture. */ +#ifndef SCCL_PACK_ARCHITECTURES +#define SCCL_PACK_ARCHITECTURES "sm_120 sm_121" +#endif typedef struct { sccl_CUcontext ctx; + int refs; /* sccl_kp_load calls not yet released (sccl_kp_release) */ sccl_CUmodule module, fold_module, links_module, p2p_module; sccl_CUfunction fn[SCCL_K_COUNT][SCCL_DT_COUNT][SCCL_KP_MAX_WORLD + 1]; sccl_CUfunction fold[SCCL_FOLD_DTYPES][SCCL_FOLD_OPS]; @@ -65,7 +71,9 @@ int sccl_kp_load(sccl_CUcontext ctx) { return -1; } pthread_mutex_lock(&lock); - if (find(ctx)) { + loaded_pack *held = find(ctx); + if (held) { + ++held->refs; pthread_mutex_unlock(&lock); return 0; } @@ -84,7 +92,8 @@ int sccl_kp_load(sccl_CUcontext ctx) { } r = cu->ModuleLoadData(&p->module, sccl_kernels_fatbin); int failed = r != SCCL_CUDA_SUCCESS; - if (failed) snprintf(error_text, sizeof error_text, "loading the kernel pack: %s", sccl_cuda_result_text(r)); + if (failed) snprintf(error_text, sizeof error_text, "loading the kernel pack (built for %s): %s", SCCL_PACK_ARCHITECTURES, + sccl_cuda_result_text(r)); for (int k = 0; !failed && k < SCCL_K_COUNT; ++k) for (int d = 0; !failed && d < SCCL_DT_COUNT; ++d) for (int w = SCCL_KP_MIN_WORLD; !failed && w <= SCCL_KP_MAX_WORLD; ++w) { @@ -104,7 +113,8 @@ int sccl_kp_load(sccl_CUcontext ctx) { if (!failed) { r = cu->ModuleLoadData(&p->fold_module, sccl_fold_fatbin); failed = r != SCCL_CUDA_SUCCESS; - if (failed) snprintf(error_text, sizeof error_text, "loading the fold pack: %s", sccl_cuda_result_text(r)); + if (failed) snprintf(error_text, sizeof error_text, "loading the fold pack (built for %s): %s", SCCL_PACK_ARCHITECTURES, + sccl_cuda_result_text(r)); } for (int d = 0; !failed && d < SCCL_FOLD_DTYPES; ++d) for (int o = 0; !failed && o < SCCL_FOLD_OPS; ++o) { @@ -119,7 +129,8 @@ int sccl_kp_load(sccl_CUcontext ctx) { if (!failed) { r = cu->ModuleLoadData(&p->links_module, sccl_links_fatbin); failed = r != SCCL_CUDA_SUCCESS; - if (failed) snprintf(error_text, sizeof error_text, "loading the link pack: %s", sccl_cuda_result_text(r)); + if (failed) snprintf(error_text, sizeof error_text, "loading the link pack (built for %s): %s", SCCL_PACK_ARCHITECTURES, + sccl_cuda_result_text(r)); } for (int d = 0; !failed && d < SCCL_DT_COUNT; ++d) for (int u = 1; !failed && u <= SCCL_CHAIN_MAX_UNROLL; ++u) { @@ -177,7 +188,8 @@ int sccl_kp_load(sccl_CUcontext ctx) { if (!failed) { r = cu->ModuleLoadData(&p->p2p_module, sccl_p2p_fatbin); failed = r != SCCL_CUDA_SUCCESS; - if (failed) snprintf(error_text, sizeof error_text, "loading the point-to-point pack: %s", sccl_cuda_result_text(r)); + if (failed) snprintf(error_text, sizeof error_text, "loading the point-to-point pack (built for %s): %s", + SCCL_PACK_ARCHITECTURES, sccl_cuda_result_text(r)); } for (int s = 0; !failed && s < 2; ++s) for (int u = 1; !failed && u <= SCCL_P2P_MAX_UNROLL; ++u) { @@ -198,12 +210,34 @@ int sccl_kp_load(sccl_CUcontext ctx) { cu->CtxPopCurrent(&popped); if (!failed) { p->ctx = ctx; + p->refs = 1; ++npacks; } pthread_mutex_unlock(&lock); return failed ? -1 : 0; } +void sccl_kp_release(sccl_CUcontext ctx, int unload) { + const sccl_cuda *cu = sccl_cuda_get(); + pthread_mutex_lock(&lock); + loaded_pack *p = find(ctx); + if (!p || --p->refs > 0) { + pthread_mutex_unlock(&lock); + return; + } + if (unload && cu && cu->CtxPushCurrent(ctx) == SCCL_CUDA_SUCCESS) { + sccl_CUmodule modules[4] = {p->p2p_module, p->links_module, p->fold_module, p->module}; + for (int i = 0; i < 4; ++i) + if (modules[i]) cu->ModuleUnload(modules[i]); + sccl_CUcontext popped; + cu->CtxPopCurrent(&popped); + } + /* The last entry takes the released one's place. */ + if (p != &packs[npacks - 1]) *p = packs[npacks - 1]; + --npacks; + pthread_mutex_unlock(&lock); +} + int sccl_kp_function(sccl_CUcontext ctx, int kind, int dtype, int world, sccl_CUfunction *out) { if (kind < 0 || kind >= SCCL_K_COUNT || dtype < 0 || dtype >= SCCL_DT_COUNT || world < SCCL_KP_MIN_WORLD || world > SCCL_KP_MAX_WORLD) { diff --git a/spark_transport/libsircl/src/kernelpack.h b/spark_transport/libsircl/src/kernelpack.h index 2ff88c6c..5f7df5eb 100644 --- a/spark_transport/libsircl/src/kernelpack.h +++ b/spark_transport/libsircl/src/kernelpack.h @@ -122,9 +122,14 @@ typedef struct { } sccl_p2p_args; _Static_assert(sizeof(sccl_p2p_args) == 120, "sccl_p2p_args must match the point-to-point pack's P2PParams"); -/* Load every pack into `ctx` once (pushing it current for the load); later - * calls for the same context return at once. 0 on success. */ +/* Load every pack into `ctx` once (pushing it current for the load); later calls for the same context + * return at once. Each successful call holds a reference on the context's entry, which sccl_kp_release + * drops; at most 16 contexts hold entries at once. 0 on success. */ int sccl_kp_load(sccl_CUcontext ctx); +/* Drop a reference sccl_kp_load took. The last one removes the context's entry, so a later context at the + * same address loads the packs afresh; with `unload` the modules are unloaded first (the context still + * exists and no kernel of the packs runs in it), without it they are left to the context's destruction. */ +void sccl_kp_release(sccl_CUcontext ctx, int unload); /* The entry of (kind, dtype, world) in a context the pack is loaded in. */ int sccl_kp_function(sccl_CUcontext ctx, int kind, int dtype, int world, sccl_CUfunction *out); /* One launch of `grid` blocks of `threads` threads on `stream`, from the @@ -147,8 +152,8 @@ sccl_CUresult sccl_kp_launch_chain(sccl_CUfunction function, const sccl_chain_ar /* The link collective entry of a kind (SCCL_LINK_*, SCCL_RING_*), dtype and unroll (1-8). */ int sccl_kp_link_function(sccl_CUcontext ctx, int kind, int dtype, int unroll, sccl_CUfunction *out); /* The ring all-reduce whose relay stores its result to link 3's slot, then copies the slot to the output - * (sircl_ring_reduce_two_pass__u): the same protocol and bytes as SCCL_RING_REDUCE's one-pass - * entry; LIBSIRCL_RING_REDUCE_PASSES=2 selects it, for comparisons. */ + * (sircl_ring_reduce_two_pass__u): the same protocol as SCCL_RING_REDUCE's one-pass entry. The + * library launches it by default (LIBSIRCL_RING_REDUCE_PASSES=2); the one-pass entry is research-only. */ int sccl_kp_ring_reduce_two_pass_function(sccl_CUcontext ctx, int dtype, int unroll, sccl_CUfunction *out); /* The pair exchange (sircl_ring_exchange_u): on the wire the ring all-gather of a ring of two, carrying * one block each way (input to the peer, the peer's block to output + peer * stride, the local block from diff --git a/spark_transport/libsircl/src/link_plan.h b/spark_transport/libsircl/src/link_plan.h new file mode 100644 index 00000000..e8887e9a --- /dev/null +++ b/spark_transport/libsircl/src/link_plan.h @@ -0,0 +1,36 @@ +/* Size limits of the link and chain kernels' arguments, and the plan of a large reduction that keeps to them. + * Header-only, so tests/test_link_plan.c checks the code the engine runs. */ +#ifndef SCCL_LINK_PLAN_H +#define SCCL_LINK_PLAN_H + +#include + +enum { SCCL_PLAN_PACK = 16 }; + +/* The link, chain and point-to-point kernels take pack counts (16-byte units) as int32_t. */ +static inline int sccl_packs_fit(uint64_t bytes) { return bytes / SCCL_PLAN_PACK <= (uint64_t)INT32_MAX; } + +enum { SCCL_LARGE_PIECES = 0, SCCL_LARGE_RING = 1, SCCL_LARGE_CHAIN = 2 }; + +/* The link op of a large all-reduce or reduce whose 16-byte-aligned body is `body` bytes, over `world` ranks + * (SIRCL's pieces.reduce_plan). `ring_schedule`: the ring schedule applies to this size; `link_on`: the link + * pack is loaded; `chain`: the chain schedule, or auto from the chain minimum, with the chain pack loaded. + * Under the ring schedule: one ring op over the largest prefix of `world` equal chunks of whole packs, each + * chunk below 2^31 bytes, else none. Otherwise: one chain op over the whole body when its pack count fits, + * else none. None means the body runs in pieces. *bytes is the link op's size (0 for none). */ +static inline int sccl_large_reduce_plan(uint64_t body, uint64_t world, int ring_schedule, int link_on, int chain, + uint64_t *bytes) { + *bytes = 0; + if (!body || world < 2) return SCCL_LARGE_PIECES; + if (ring_schedule) { + uint64_t ring_bytes = body / (SCCL_PLAN_PACK * world) * SCCL_PLAN_PACK * world; + if (!link_on || !ring_bytes || ring_bytes / world >= (1ull << 31)) return SCCL_LARGE_PIECES; + *bytes = ring_bytes; + return SCCL_LARGE_RING; + } + if (!chain || !sccl_packs_fit(body)) return SCCL_LARGE_PIECES; + *bytes = body; + return SCCL_LARGE_CHAIN; +} + +#endif diff --git a/spark_transport/libsircl/src/stubs.c b/spark_transport/libsircl/src/stubs.c index f2e395af..44c007aa 100644 --- a/spark_transport/libsircl/src/stubs.c +++ b/spark_transport/libsircl/src/stubs.c @@ -2,11 +2,13 @@ * 2015-2026 NVIDIA CORPORATION & AFFILIATES, Apache-2.0; see vendor/NCCL-LICENSE.txt). Do not edit. */ #include "internal.h" +#include + ncclResult_t ncclCommShrink(ncclComm_t comm, int* excludeRanksList, int excludeRanksCount, ncclComm_t* newcomm, ncclConfig_t* config, int shrinkFlags) { (void)comm; (void)excludeRanksList; (void)excludeRanksCount; - (void)newcomm; + if (newcomm) memset(newcomm, 0, sizeof(*newcomm)); (void)config; (void)shrinkFlags; return sccl_unsupported("ncclCommShrink"); @@ -14,7 +16,7 @@ ncclResult_t ncclCommShrink(ncclComm_t comm, int* excludeRanksList, int excludeR ncclResult_t ncclCommGetUniqueId(ncclComm_t comm, ncclUniqueId* uniqueId) { (void)comm; - (void)uniqueId; + if (uniqueId) memset(uniqueId, 0, sizeof(*uniqueId)); return sccl_unsupported("ncclCommGetUniqueId"); } @@ -23,13 +25,13 @@ ncclResult_t ncclCommGrow(ncclComm_t comm, int nRanks, const ncclUniqueId* uniqu (void)nRanks; (void)uniqueId; (void)rank; - (void)newcomm; + if (newcomm) memset(newcomm, 0, sizeof(*newcomm)); (void)config; return sccl_unsupported("ncclCommGrow"); } ncclResult_t ncclCommInitRankScalable(ncclComm_t* newcomm, int nranks, int myrank, int nId, ncclUniqueId* commIds, ncclConfig_t* config) { - (void)newcomm; + if (newcomm) memset(newcomm, 0, sizeof(*newcomm)); (void)nranks; (void)myrank; (void)nId; @@ -56,7 +58,7 @@ ncclResult_t ncclCommResume(ncclComm_t comm) { ncclResult_t ncclCommMemStats(ncclComm_t comm, ncclCommMemStat_t stat, uint64_t* value) { (void)comm; (void)stat; - (void)value; + if (value) memset(value, 0, sizeof(*value)); return sccl_unsupported("ncclCommMemStats"); } @@ -214,62 +216,62 @@ ncclResult_t ncclSetEncryption(const ncclEncryptionConfig_t* config) { } ncclResult_t ncclParamBind(ncclParamHandle_t* out, const char* key) { - (void)out; + if (out) memset(out, 0, sizeof(*out)); (void)key; return sccl_unsupported("ncclParamBind"); } ncclResult_t ncclParamGetI8(ncclParamHandle_t h, int8_t* out) { (void)h; - (void)out; + if (out) memset(out, 0, sizeof(*out)); return sccl_unsupported("ncclParamGetI8"); } ncclResult_t ncclParamGetI16(ncclParamHandle_t h, int16_t* out) { (void)h; - (void)out; + if (out) memset(out, 0, sizeof(*out)); return sccl_unsupported("ncclParamGetI16"); } ncclResult_t ncclParamGetI32(ncclParamHandle_t h, int32_t* out) { (void)h; - (void)out; + if (out) memset(out, 0, sizeof(*out)); return sccl_unsupported("ncclParamGetI32"); } ncclResult_t ncclParamGetI64(ncclParamHandle_t h, int64_t* out) { (void)h; - (void)out; + if (out) memset(out, 0, sizeof(*out)); return sccl_unsupported("ncclParamGetI64"); } ncclResult_t ncclParamGetU8(ncclParamHandle_t h, uint8_t* out) { (void)h; - (void)out; + if (out) memset(out, 0, sizeof(*out)); return sccl_unsupported("ncclParamGetU8"); } ncclResult_t ncclParamGetU16(ncclParamHandle_t h, uint16_t* out) { (void)h; - (void)out; + if (out) memset(out, 0, sizeof(*out)); return sccl_unsupported("ncclParamGetU16"); } ncclResult_t ncclParamGetU32(ncclParamHandle_t h, uint32_t* out) { (void)h; - (void)out; + if (out) memset(out, 0, sizeof(*out)); return sccl_unsupported("ncclParamGetU32"); } ncclResult_t ncclParamGetU64(ncclParamHandle_t h, uint64_t* out) { (void)h; - (void)out; + if (out) memset(out, 0, sizeof(*out)); return sccl_unsupported("ncclParamGetU64"); } ncclResult_t ncclParamGetStr(ncclParamHandle_t h, const char** out) { (void)h; - (void)out; + if (out) memset(out, 0, sizeof(*out)); return sccl_unsupported("ncclParamGetStr"); } @@ -277,20 +279,20 @@ ncclResult_t ncclParamGet(ncclParamHandle_t h, void* out, int maxLen, int* len) (void)h; (void)out; (void)maxLen; - (void)len; + if (len) memset(len, 0, sizeof(*len)); return sccl_unsupported("ncclParamGet"); } ncclResult_t ncclParamGetParameter(const char* key, const char** value, int* valueLen) { (void)key; - (void)value; - (void)valueLen; + if (value) memset(value, 0, sizeof(*value)); + if (valueLen) memset(valueLen, 0, sizeof(*valueLen)); return sccl_unsupported("ncclParamGetParameter"); } ncclResult_t ncclParamGetAllParameterKeys(const char*** table, int* tableLen) { - (void)table; - (void)tableLen; + if (table) memset(table, 0, sizeof(*table)); + if (tableLen) memset(tableLen, 0, sizeof(*tableLen)); return sccl_unsupported("ncclParamGetAllParameterKeys"); } diff --git a/spark_transport/libsircl/tests/data/site_routes_path0-3_l2.json b/spark_transport/libsircl/tests/data/site_routes_path0-3_l2.json index b18103e9..2bc679e4 100644 --- a/spark_transport/libsircl/tests/data/site_routes_path0-3_l2.json +++ b/spark_transport/libsircl/tests/data/site_routes_path0-3_l2.json @@ -20,6 +20,7 @@ "LIBSIRCL_POSITION": "0", "SIRCL_PEER_ROUTES": "1=rocep1s0f0/roceP2p1s0f0,2=rocep1s0f0/roceP2p1s0f0,3=rocep1s0f0/roceP2p1s0f0", "LIBSIRCL_CHAIN_ORDER": "0,1,2,3", + "LIBSIRCL_RELAYED_LANES": "2=1/1,3=1/1", "LIBSIRCL_FORWARD_WINDOWS": "2=131072/131072,3=131072/131072", "SIRCL_FORWARD_CHUNK_BYTES": "32768", "LIBSIRCL_RING_WINDOW": "0", @@ -33,6 +34,7 @@ "LIBSIRCL_POSITION": "1", "SIRCL_PEER_ROUTES": "0=rocep1s0f1/roceP2p1s0f1,2=rocep1s0f0/roceP2p1s0f0,3=rocep1s0f0/roceP2p1s0f0", "LIBSIRCL_CHAIN_ORDER": "0,1,2,3", + "LIBSIRCL_RELAYED_LANES": "3=1/1", "LIBSIRCL_FORWARD_WINDOWS": "3=131072/131072", "SIRCL_FORWARD_CHUNK_BYTES": "32768", "LIBSIRCL_RING_WINDOW": "0", @@ -46,6 +48,7 @@ "LIBSIRCL_POSITION": "2", "SIRCL_PEER_ROUTES": "0=rocep1s0f1/roceP2p1s0f1,1=rocep1s0f1/roceP2p1s0f1,3=rocep1s0f0/roceP2p1s0f0", "LIBSIRCL_CHAIN_ORDER": "0,1,2,3", + "LIBSIRCL_RELAYED_LANES": "0=1/1", "LIBSIRCL_FORWARD_WINDOWS": "0=131072/131072", "SIRCL_FORWARD_CHUNK_BYTES": "32768", "LIBSIRCL_RING_WINDOW": "0", @@ -59,6 +62,7 @@ "LIBSIRCL_POSITION": "3", "SIRCL_PEER_ROUTES": "0=rocep1s0f1/roceP2p1s0f1,1=rocep1s0f1/roceP2p1s0f1,2=rocep1s0f1/roceP2p1s0f1", "LIBSIRCL_CHAIN_ORDER": "0,1,2,3", + "LIBSIRCL_RELAYED_LANES": "0=1/1,1=1/1", "LIBSIRCL_FORWARD_WINDOWS": "0=131072/131072,1=131072/131072", "SIRCL_FORWARD_CHUNK_BYTES": "32768", "LIBSIRCL_RING_WINDOW": "393216", diff --git a/spark_transport/libsircl/tests/data/site_routes_ring8_l2.json b/spark_transport/libsircl/tests/data/site_routes_ring8_l2.json index e3bf1eab..72e7eb9d 100644 --- a/spark_transport/libsircl/tests/data/site_routes_ring8_l2.json +++ b/spark_transport/libsircl/tests/data/site_routes_ring8_l2.json @@ -225,6 +225,7 @@ "LIBSIRCL_POSITION": "0", "SIRCL_PEER_ROUTES": "1=rocep1s0f0/roceP2p1s0f0,2=rocep1s0f0/roceP2p1s0f0,3=rocep1s0f0/roceP2p1s0f0,4=rocep1s0f0/roceP2p1s0f1,5=rocep1s0f1/roceP2p1s0f1,6=rocep1s0f1/roceP2p1s0f1,7=rocep1s0f1/roceP2p1s0f1", "LIBSIRCL_CHAIN_ORDER": "0,1,2,3,4,5,6,7", + "LIBSIRCL_RELAYED_LANES": "2=1/1,3=1/1,4=1/1,5=1/1,6=1/1", "LIBSIRCL_FORWARD_WINDOWS": "2=98304/65536,3=65536/65536,4=65536/65536,5=98304/65536,6=131072/65536", "SIRCL_FORWARD_CHUNK_BYTES": "32768", "LIBSIRCL_RING_WINDOW": "0" @@ -236,6 +237,7 @@ "LIBSIRCL_POSITION": "1", "SIRCL_PEER_ROUTES": "0=rocep1s0f1/roceP2p1s0f1,2=rocep1s0f0/roceP2p1s0f0,3=rocep1s0f0/roceP2p1s0f0,4=rocep1s0f0/roceP2p1s0f0,5=rocep1s0f0/roceP2p1s0f1,6=rocep1s0f1/roceP2p1s0f1,7=rocep1s0f1/roceP2p1s0f1", "LIBSIRCL_CHAIN_ORDER": "0,1,2,3,4,5,6,7", + "LIBSIRCL_RELAYED_LANES": "3=1/1,4=1/1,5=1/1,6=1/1,7=1/1", "LIBSIRCL_FORWARD_WINDOWS": "3=65536/98304,4=65536/98304,5=65536/65536,6=131072/65536,7=131072/65536", "SIRCL_FORWARD_CHUNK_BYTES": "32768", "LIBSIRCL_RING_WINDOW": "0" @@ -247,6 +249,7 @@ "LIBSIRCL_POSITION": "2", "SIRCL_PEER_ROUTES": "0=rocep1s0f1/roceP2p1s0f1,1=rocep1s0f1/roceP2p1s0f1,3=rocep1s0f0/roceP2p1s0f0,4=rocep1s0f0/roceP2p1s0f0,5=rocep1s0f0/roceP2p1s0f0,6=rocep1s0f0/roceP2p1s0f1,7=rocep1s0f1/roceP2p1s0f1", "LIBSIRCL_CHAIN_ORDER": "0,1,2,3,4,5,6,7", + "LIBSIRCL_RELAYED_LANES": "0=1/1,4=1/1,5=1/1,6=1/1,7=1/1", "LIBSIRCL_FORWARD_WINDOWS": "0=98304/65536,4=65536/131072,5=65536/131072,6=65536/65536,7=98304/65536", "SIRCL_FORWARD_CHUNK_BYTES": "32768", "LIBSIRCL_RING_WINDOW": "0" @@ -258,6 +261,7 @@ "LIBSIRCL_POSITION": "3", "SIRCL_PEER_ROUTES": "0=rocep1s0f1/roceP2p1s0f1,1=rocep1s0f1/roceP2p1s0f1,2=rocep1s0f1/roceP2p1s0f1,4=rocep1s0f0/roceP2p1s0f0,5=rocep1s0f0/roceP2p1s0f0,6=rocep1s0f0/roceP2p1s0f0,7=rocep1s0f0/roceP2p1s0f1", "LIBSIRCL_CHAIN_ORDER": "0,1,2,3,4,5,6,7", + "LIBSIRCL_RELAYED_LANES": "0=1/1,1=1/1,5=1/1,6=1/1,7=1/1", "LIBSIRCL_FORWARD_WINDOWS": "0=65536/65536,1=65536/98304,5=65536/131072,6=65536/98304,7=65536/65536", "SIRCL_FORWARD_CHUNK_BYTES": "32768", "LIBSIRCL_RING_WINDOW": "0" @@ -269,6 +273,7 @@ "LIBSIRCL_POSITION": "4", "SIRCL_PEER_ROUTES": "0=rocep1s0f1/roceP2p1s0f0,1=rocep1s0f1/roceP2p1s0f1,2=rocep1s0f1/roceP2p1s0f1,3=rocep1s0f1/roceP2p1s0f1,5=rocep1s0f0/roceP2p1s0f0,6=rocep1s0f0/roceP2p1s0f0,7=rocep1s0f0/roceP2p1s0f0", "LIBSIRCL_CHAIN_ORDER": "0,1,2,3,4,5,6,7", + "LIBSIRCL_RELAYED_LANES": "0=1/1,1=1/1,2=1/1,6=1/1,7=1/1", "LIBSIRCL_FORWARD_WINDOWS": "0=65536/65536,1=65536/98304,2=65536/131072,6=65536/98304,7=65536/65536", "SIRCL_FORWARD_CHUNK_BYTES": "32768", "LIBSIRCL_RING_WINDOW": "0" @@ -280,6 +285,7 @@ "LIBSIRCL_POSITION": "5", "SIRCL_PEER_ROUTES": "0=rocep1s0f0/roceP2p1s0f0,1=rocep1s0f1/roceP2p1s0f0,2=rocep1s0f1/roceP2p1s0f1,3=rocep1s0f1/roceP2p1s0f1,4=rocep1s0f1/roceP2p1s0f1,6=rocep1s0f0/roceP2p1s0f0,7=rocep1s0f0/roceP2p1s0f0", "LIBSIRCL_CHAIN_ORDER": "0,1,2,3,4,5,6,7", + "LIBSIRCL_RELAYED_LANES": "0=1/1,1=1/1,2=1/1,3=1/1,7=1/1", "LIBSIRCL_FORWARD_WINDOWS": "0=98304/65536,1=65536/65536,2=65536/131072,3=65536/131072,7=98304/65536", "SIRCL_FORWARD_CHUNK_BYTES": "32768", "LIBSIRCL_RING_WINDOW": "0" @@ -291,6 +297,7 @@ "LIBSIRCL_POSITION": "6", "SIRCL_PEER_ROUTES": "0=rocep1s0f0/roceP2p1s0f0,1=rocep1s0f0/roceP2p1s0f0,2=rocep1s0f1/roceP2p1s0f0,3=rocep1s0f1/roceP2p1s0f1,4=rocep1s0f1/roceP2p1s0f1,5=rocep1s0f1/roceP2p1s0f1,7=rocep1s0f0/roceP2p1s0f0", "LIBSIRCL_CHAIN_ORDER": "0,1,2,3,4,5,6,7", + "LIBSIRCL_RELAYED_LANES": "0=1/1,1=1/1,2=1/1,3=1/1,4=1/1", "LIBSIRCL_FORWARD_WINDOWS": "0=131072/65536,1=131072/65536,2=65536/65536,3=65536/98304,4=65536/98304", "SIRCL_FORWARD_CHUNK_BYTES": "32768", "LIBSIRCL_RING_WINDOW": "0" @@ -302,6 +309,7 @@ "LIBSIRCL_POSITION": "7", "SIRCL_PEER_ROUTES": "0=rocep1s0f0/roceP2p1s0f0,1=rocep1s0f0/roceP2p1s0f0,2=rocep1s0f0/roceP2p1s0f0,3=rocep1s0f1/roceP2p1s0f0,4=rocep1s0f1/roceP2p1s0f1,5=rocep1s0f1/roceP2p1s0f1,6=rocep1s0f1/roceP2p1s0f1", "LIBSIRCL_CHAIN_ORDER": "0,1,2,3,4,5,6,7", + "LIBSIRCL_RELAYED_LANES": "1=1/1,2=1/1,3=1/1,4=1/1,5=1/1", "LIBSIRCL_FORWARD_WINDOWS": "1=131072/65536,2=98304/65536,3=65536/65536,4=65536/65536,5=98304/65536", "SIRCL_FORWARD_CHUNK_BYTES": "32768", "LIBSIRCL_RING_WINDOW": "0" diff --git a/spark_transport/libsircl/tests/data/site_routes_ring8_l2_p2p_alone.json b/spark_transport/libsircl/tests/data/site_routes_ring8_l2_p2p_alone.json index 5059e5bd..c5c95880 100644 --- a/spark_transport/libsircl/tests/data/site_routes_ring8_l2_p2p_alone.json +++ b/spark_transport/libsircl/tests/data/site_routes_ring8_l2_p2p_alone.json @@ -24,6 +24,7 @@ "LIBSIRCL_POSITION": "0", "SIRCL_PEER_ROUTES": "1=rocep1s0f0/roceP2p1s0f0,2=rocep1s0f0/roceP2p1s0f0,3=rocep1s0f0/roceP2p1s0f0,4=rocep1s0f0/roceP2p1s0f1,5=rocep1s0f1/roceP2p1s0f1,6=rocep1s0f1/roceP2p1s0f1,7=rocep1s0f1/roceP2p1s0f1", "LIBSIRCL_CHAIN_ORDER": "0,1,2,3,4,5,6,7", + "LIBSIRCL_RELAYED_LANES": "2=1/1,3=1/1,4=1/1,5=1/1,6=1/1", "LIBSIRCL_FORWARD_WINDOWS": "2=98304/65536,3=65536/65536,4=65536/65536,5=98304/65536,6=131072/65536", "SIRCL_FORWARD_CHUNK_BYTES": "32768", "LIBSIRCL_RING_WINDOW": "0", @@ -37,6 +38,7 @@ "LIBSIRCL_POSITION": "1", "SIRCL_PEER_ROUTES": "0=rocep1s0f1/roceP2p1s0f1,2=rocep1s0f0/roceP2p1s0f0,3=rocep1s0f0/roceP2p1s0f0,4=rocep1s0f0/roceP2p1s0f0,5=rocep1s0f0/roceP2p1s0f1,6=rocep1s0f1/roceP2p1s0f1,7=rocep1s0f1/roceP2p1s0f1", "LIBSIRCL_CHAIN_ORDER": "0,1,2,3,4,5,6,7", + "LIBSIRCL_RELAYED_LANES": "3=1/1,4=1/1,5=1/1,6=1/1,7=1/1", "LIBSIRCL_FORWARD_WINDOWS": "3=65536/98304,4=65536/98304,5=65536/65536,6=131072/65536,7=131072/65536", "SIRCL_FORWARD_CHUNK_BYTES": "32768", "LIBSIRCL_RING_WINDOW": "0", @@ -50,6 +52,7 @@ "LIBSIRCL_POSITION": "2", "SIRCL_PEER_ROUTES": "0=rocep1s0f1/roceP2p1s0f1,1=rocep1s0f1/roceP2p1s0f1,3=rocep1s0f0/roceP2p1s0f0,4=rocep1s0f0/roceP2p1s0f0,5=rocep1s0f0/roceP2p1s0f0,6=rocep1s0f0/roceP2p1s0f1,7=rocep1s0f1/roceP2p1s0f1", "LIBSIRCL_CHAIN_ORDER": "0,1,2,3,4,5,6,7", + "LIBSIRCL_RELAYED_LANES": "0=1/1,4=1/1,5=1/1,6=1/1,7=1/1", "LIBSIRCL_FORWARD_WINDOWS": "0=98304/65536,4=65536/131072,5=65536/131072,6=65536/65536,7=98304/65536", "SIRCL_FORWARD_CHUNK_BYTES": "32768", "LIBSIRCL_RING_WINDOW": "0", @@ -63,6 +66,7 @@ "LIBSIRCL_POSITION": "3", "SIRCL_PEER_ROUTES": "0=rocep1s0f1/roceP2p1s0f1,1=rocep1s0f1/roceP2p1s0f1,2=rocep1s0f1/roceP2p1s0f1,4=rocep1s0f0/roceP2p1s0f0,5=rocep1s0f0/roceP2p1s0f0,6=rocep1s0f0/roceP2p1s0f0,7=rocep1s0f0/roceP2p1s0f1", "LIBSIRCL_CHAIN_ORDER": "0,1,2,3,4,5,6,7", + "LIBSIRCL_RELAYED_LANES": "0=1/1,1=1/1,5=1/1,6=1/1,7=1/1", "LIBSIRCL_FORWARD_WINDOWS": "0=65536/65536,1=65536/98304,5=65536/131072,6=65536/98304,7=65536/65536", "SIRCL_FORWARD_CHUNK_BYTES": "32768", "LIBSIRCL_RING_WINDOW": "0", @@ -76,6 +80,7 @@ "LIBSIRCL_POSITION": "4", "SIRCL_PEER_ROUTES": "0=rocep1s0f1/roceP2p1s0f0,1=rocep1s0f1/roceP2p1s0f1,2=rocep1s0f1/roceP2p1s0f1,3=rocep1s0f1/roceP2p1s0f1,5=rocep1s0f0/roceP2p1s0f0,6=rocep1s0f0/roceP2p1s0f0,7=rocep1s0f0/roceP2p1s0f0", "LIBSIRCL_CHAIN_ORDER": "0,1,2,3,4,5,6,7", + "LIBSIRCL_RELAYED_LANES": "0=1/1,1=1/1,2=1/1,6=1/1,7=1/1", "LIBSIRCL_FORWARD_WINDOWS": "0=65536/65536,1=65536/98304,2=65536/131072,6=65536/98304,7=65536/65536", "SIRCL_FORWARD_CHUNK_BYTES": "32768", "LIBSIRCL_RING_WINDOW": "0", @@ -89,6 +94,7 @@ "LIBSIRCL_POSITION": "5", "SIRCL_PEER_ROUTES": "0=rocep1s0f0/roceP2p1s0f0,1=rocep1s0f1/roceP2p1s0f0,2=rocep1s0f1/roceP2p1s0f1,3=rocep1s0f1/roceP2p1s0f1,4=rocep1s0f1/roceP2p1s0f1,6=rocep1s0f0/roceP2p1s0f0,7=rocep1s0f0/roceP2p1s0f0", "LIBSIRCL_CHAIN_ORDER": "0,1,2,3,4,5,6,7", + "LIBSIRCL_RELAYED_LANES": "0=1/1,1=1/1,2=1/1,3=1/1,7=1/1", "LIBSIRCL_FORWARD_WINDOWS": "0=98304/65536,1=65536/65536,2=65536/131072,3=65536/131072,7=98304/65536", "SIRCL_FORWARD_CHUNK_BYTES": "32768", "LIBSIRCL_RING_WINDOW": "0", @@ -102,6 +108,7 @@ "LIBSIRCL_POSITION": "6", "SIRCL_PEER_ROUTES": "0=rocep1s0f0/roceP2p1s0f0,1=rocep1s0f0/roceP2p1s0f0,2=rocep1s0f1/roceP2p1s0f0,3=rocep1s0f1/roceP2p1s0f1,4=rocep1s0f1/roceP2p1s0f1,5=rocep1s0f1/roceP2p1s0f1,7=rocep1s0f0/roceP2p1s0f0", "LIBSIRCL_CHAIN_ORDER": "0,1,2,3,4,5,6,7", + "LIBSIRCL_RELAYED_LANES": "0=1/1,1=1/1,2=1/1,3=1/1,4=1/1", "LIBSIRCL_FORWARD_WINDOWS": "0=131072/65536,1=131072/65536,2=65536/65536,3=65536/98304,4=65536/98304", "SIRCL_FORWARD_CHUNK_BYTES": "32768", "LIBSIRCL_RING_WINDOW": "0", @@ -115,6 +122,7 @@ "LIBSIRCL_POSITION": "7", "SIRCL_PEER_ROUTES": "0=rocep1s0f0/roceP2p1s0f0,1=rocep1s0f0/roceP2p1s0f0,2=rocep1s0f0/roceP2p1s0f0,3=rocep1s0f1/roceP2p1s0f0,4=rocep1s0f1/roceP2p1s0f1,5=rocep1s0f1/roceP2p1s0f1,6=rocep1s0f1/roceP2p1s0f1", "LIBSIRCL_CHAIN_ORDER": "0,1,2,3,4,5,6,7", + "LIBSIRCL_RELAYED_LANES": "1=1/1,2=1/1,3=1/1,4=1/1,5=1/1", "LIBSIRCL_FORWARD_WINDOWS": "1=131072/65536,2=98304/65536,3=65536/65536,4=65536/65536,5=98304/65536", "SIRCL_FORWARD_CHUNK_BYTES": "32768", "LIBSIRCL_RING_WINDOW": "0", diff --git a/spark_transport/libsircl/tests/emulation/library_rank.py b/spark_transport/libsircl/tests/emulation/library_rank.py index db7bdfba..2c3800d8 100644 --- a/spark_transport/libsircl/tests/emulation/library_rank.py +++ b/spark_transport/libsircl/tests/emulation/library_rank.py @@ -34,7 +34,9 @@ collectives of every selection point with one rank's buffers unaligned or in place, eager and captured cold, launch and carry the same ops on every rank as with aligned buffers (receipt counts). On a pair, one-way exchanges whose empty direction alternates between the ranks across link-slot wrap, each round -followed by two-way calls. An op that is not built in refused and counted; the receipt, +followed by two-way calls. An op that is not built in refused and counted. Hostile arguments (overflowing counts, NULL buffers, roots +out of range) to every collective and to ncclSend and ncclRecv refused with ncclInvalidArgument, buffers +unchanged. The receipt, ncclCommGetAsyncError, finalize and destroy. """ from __future__ import annotations @@ -761,6 +763,55 @@ def grouped(): f"group end {results.get('end')}") +def run_hostile_argument_checks(torch, lib, comm, world, rank, stream, report) -> None: + """Hostile arguments, table-driven: for every collective and ncclSend/ncclRecv, a count whose bytes overflow + size_t, a NULL buffer where every rank needs one, and a root outside 0..W-1 for the rooted collectives. + Every rank makes the same calls, so every refusal happens on every rank before any communication; each + call must return ncclInvalidArgument and leave the buffers it was given unchanged.""" + v, i, z, c = ctypes.c_void_p, ctypes.c_int, ctypes.c_size_t, ctypes.c_void_p + signatures = {"ncclAllReduce": [v, v, z, i, i, c, v], "ncclAllGather": [v, v, z, i, c, v], + "ncclReduceScatter": [v, v, z, i, i, c, v], "ncclReduce": [v, v, z, i, i, i, c, v], + "ncclBroadcast": [v, v, z, i, i, c, v], "ncclAlltoAll": [v, v, z, i, c, v], + "ncclGather": [v, v, z, i, i, c, v], "ncclScatter": [v, v, z, i, i, c, v], + "ncclSend": [v, z, i, i, c, v], "ncclRecv": [v, z, i, i, c, v]} + for name, types in signatures.items(): + getattr(lib, name).argtypes = types + count = 1024 + send = torch.full((count * world,), 3.0, dtype=torch.float32, device="cuda") + recv = torch.full((count * world,), 7.0, dtype=torch.float32, device="cuda") + torch.cuda.synchronize() + snapshot = (send.cpu().clone(), recv.cpu().clone()) + s, r, h = ctypes.c_void_p(send.data_ptr()), ctypes.c_void_p(recv.data_ptr()), ctypes.c_void_p(stream.cuda_stream) + huge = (1 << 64) - 1 # size_t's largest value: its bytes overflow every element size + f32, total = 7, 0 + cases = [ + ("ncclAllReduce", (s, r, huge, f32, 0, comm, h)), ("ncclAllReduce", (None, r, count, f32, 0, comm, h)), + ("ncclAllReduce", (s, None, count, f32, 0, comm, h)), + ("ncclAllGather", (s, r, huge, f32, comm, h)), ("ncclAllGather", (None, r, count, f32, comm, h)), + ("ncclReduceScatter", (s, r, huge, f32, 0, comm, h)), ("ncclReduceScatter", (s, None, count, f32, 0, comm, h)), + ("ncclAlltoAll", (s, r, huge, f32, comm, h)), ("ncclAlltoAll", (None, r, count, f32, comm, h)), + ("ncclReduce", (s, r, huge, f32, 0, 0, comm, h)), ("ncclReduce", (None, r, count, f32, 0, 0, comm, h)), + ("ncclBroadcast", (s, r, huge, f32, 0, comm, h)), ("ncclBroadcast", (s, None, count, f32, 0, comm, h)), + ("ncclGather", (s, r, huge, f32, 0, comm, h)), ("ncclScatter", (s, r, huge, f32, 0, comm, h)), + ("ncclSend", (s, huge, f32, (rank + 1) % world, comm, h)), ("ncclSend", (None, count, f32, 0, comm, h)), + ("ncclRecv", (r, huge, f32, (rank + 1) % world, comm, h)), ("ncclRecv", (None, count, f32, 0, comm, h)), + ] + [(name, args) for root in (-1, world, 1 << 30) for name, args in ( + ("ncclReduce", (s, r, count, f32, 0, root, comm, h)), ("ncclBroadcast", (s, r, count, f32, root, comm, h)), + ("ncclGather", (s, r, count, f32, root, comm, h)), ("ncclScatter", (s, r, count, f32, root, comm, h)))] + wrong = [] + for name, args in cases: + total += 1 + code = getattr(lib, name)(*args) + if code != 4: + wrong.append(f"{name}{tuple('NULL' if a is None else a for a in args[2:-2])}: result {code}: " + f"{lib.ncclGetLastError(None).decode()}") + torch.cuda.synchronize() + unchanged = torch.equal(send.cpu(), snapshot[0]) and torch.equal(recv.cpu(), snapshot[1]) + report(f"{total} calls with hostile arguments (overflowing counts, NULL buffers, roots outside 0-{world - 1}) " + "refused with ncclInvalidArgument, buffers unchanged", not wrong and unchanged, + "; ".join(wrong[:4]) if wrong else "" if unchanged else "a buffer changed") + + def run_point_to_point_checks(torch, lib, comm, world, rank, max_piece, stream, seed_base, report) -> int: """ncclSend and ncclRecv: between the ranks of a two-rank communicator, outside and inside groups (both directions with unequal sizes, several calls matched in issue order, sends to the rank itself @@ -1956,6 +2007,7 @@ def reduce_scatter(src, dst, count, stream): alternating_alltoalls = run_alternating_grid_checks(torch, lib, comm, world, rank, stream_a, args.seed_base, report) run_split_checks(torch, lib, comm, world, rank, stream_a, args.seed_base, report) run_abort_with_queued_send_check(torch, lib, comm, world, rank, stream_a, report) + run_hostile_argument_checks(torch, lib, comm, world, rank, stream_a, report) run_capture_staging_check(torch, lib, comm, world, rank, args.seed_base, report) run_rank_local_staging_checks(torch, lib, comm, world, rank, args.seed_base, report) run_staging_growth_check(torch, lib, comm, world, rank, args.seed_base, report) diff --git a/spark_transport/libsircl/tests/emulation/p2p_channels.py b/spark_transport/libsircl/tests/emulation/p2p_channels.py index c0869dc2..44ffb5f6 100644 --- a/spark_transport/libsircl/tests/emulation/p2p_channels.py +++ b/spark_transport/libsircl/tests/emulation/p2p_channels.py @@ -39,7 +39,10 @@ the others rank 1 as its origin) and ncclCommAbort returns. ``--case gone``: four ranks with LIBSIRCL_FAIL_STOP=1 and a 3 s wait limit; rank 3 ends its process after setup, rank 0 receives from it: its wait times out, and ranks 0, 1 and 2 end with status 70 (rank 0 by its own timeout, 1 and 2 by the abort -notice). ``--case setup``: three groups of three ranks whose setup must fail on every rank, naming the +notice). ``--case abort``: four ranks with a 60 s wait limit; rank 1 enqueues a receive from rank 0 that no +send matches and calls ncclCommAbort 0.5 s later: the abort returns within 2 s, the receive's kernel stops +within 5 s, and ranks 0, 2 and 3 report the channels' failure naming rank 1 within 20 s, long before the wait +limit, because the abort poisons the channels and their progress thread posts abort notices. ``--case setup``: three groups of three ranks whose setup must fail on every rank, naming the reason: LIBSIRCL_P2P_CHANNELS on rank 0 only, SIRCL_P2P_THREADS=1024 on rank 1 (the pack runs at most 512), and a malformed LIBSIRCL_P2P_WINDOWS on rank 2. @@ -163,6 +166,35 @@ def call(send, view, peer): report("ncclCommAbort", lib.ncclCommAbort(comm) == 0, last_error()) save() return 0 + if args.case == "abort": + x = torch.zeros(4096, dtype=torch.uint8, device="cuda") + torch.cuda.synchronize() + if rank == 1: + code = call(False, x, 0) + report("a receive that no send matches enqueues", code == 0, last_error() if code else "") + time.sleep(0.5) + started = time.monotonic() + aborted = lib.ncclCommAbort(comm) + seconds = time.monotonic() - started + report(f"ncclCommAbort returned within {seconds:.2f} s", aborted == 0 and seconds < 2, last_error()) + started = time.monotonic() + stream.synchronize() + seconds = time.monotonic() - started + report(f"the receive's kernel stopped {seconds:.2f} s after the abort", seconds < 5) + save() + return 0 + status, message, started = ctypes.c_int(0), "", time.monotonic() + while time.monotonic() - started < 20: + lib.ncclCommGetAsyncError(comm, ctypes.byref(status)) + if status.value: + message = last_error(comm) + break + time.sleep(0.05) + report(f"rank 1's abort reported within {time.monotonic() - started:.2f} s", + status.value != 0 and "failure on rank 1" in message, f"result {status.value}: {message}") + report("ncclCommAbort", lib.ncclCommAbort(comm) == 0, last_error()) + save() + return 0 if args.case == "gone": save() if rank == 3: @@ -392,6 +424,8 @@ def run_group(args, extra_env=None, label="") -> int: }) if args.case == "gone": base.update({"SIRCL_STARTUP_WAIT_S": "3", "LIBSIRCL_FAIL_STOP": "1"}) + if args.case == "abort": + base.update({"SIRCL_STARTUP_WAIT_S": "60"}) base.update(item.split("=", 1) for item in args.env) started = time.perf_counter() processes = [] @@ -485,7 +519,7 @@ def main(argv=None) -> int: parser.add_argument("--world", type=int, default=4) parser.add_argument("--lanes", type=int, default=2) parser.add_argument("--layout", choices=("none", "ring8", "ring8-alone", "path4"), default="none") - parser.add_argument("--case", choices=("traffic", "size", "gone", "setup"), default="traffic") + parser.add_argument("--case", choices=("traffic", "size", "gone", "abort", "setup"), default="traffic") parser.add_argument("--rounds", type=int, default=3) parser.add_argument("--timeout", type=float, default=1200) parser.add_argument("--env", action="append", default=[], metavar="NAME=VALUE", diff --git a/spark_transport/libsircl/tests/emulation/setup_failures.py b/spark_transport/libsircl/tests/emulation/setup_failures.py index d8b4ced4..767f34a2 100644 --- a/spark_transport/libsircl/tests/emulation/setup_failures.py +++ b/spark_transport/libsircl/tests/emulation/setup_failures.py @@ -13,7 +13,9 @@ - a ring schedule without the layout's ring plan (``SIRCL_SCATTER_SCHEDULE=ring`` without ``LIBSIRCL_RING_WINDOW``, and ``SIRCL_LARGE_SCHEDULE=pieces``, which turns off the pair plan and with it the ring plan a cabled pair gets by default); -- a schedule differing between the ranks (``SIRCL_GATHER_SCHEDULE``). +- a schedule differing between the ranks (``SIRCL_GATHER_SCHEDULE``); +- fail-stop and the wait regime differing between the ranks (``LIBSIRCL_FAIL_STOP``, + ``LIBSIRCL_WAIT_REGIME``). """ from __future__ import annotations @@ -94,6 +96,12 @@ def main(argv=None) -> int: ("SIRCL_GATHER_SCHEDULE differing between ranks", [emulation, dict(emulation, SIRCL_GATHER_SCHEDULE="chain")], ["rank 1: SIRCL_GATHER_SCHEDULE differs from rank 0"]), + ("LIBSIRCL_FAIL_STOP differing between ranks", + [emulation, dict(emulation, LIBSIRCL_FAIL_STOP="1")], + ["rank 1: LIBSIRCL_FAIL_STOP differs from rank 0"]), + ("LIBSIRCL_WAIT_REGIME differing between ranks", + [emulation, dict(emulation, LIBSIRCL_WAIT_REGIME="serving")], + ["rank 1: LIBSIRCL_WAIT_REGIME differs from rank 0"]), ] failed = 0 for index, (name, environments, expected) in enumerate(cases): diff --git a/spark_transport/libsircl/tests/emulation/teardown_race.py b/spark_transport/libsircl/tests/emulation/teardown_race.py index 65cf75d5..b7a332cf 100644 --- a/spark_transport/libsircl/tests/emulation/teardown_race.py +++ b/spark_transport/libsircl/tests/emulation/teardown_race.py @@ -25,6 +25,17 @@ SIRCL_STARTUP_WAIT_S below S) or when releasing the transport fails (--env SIRCL_EMU_FAIL_DEREG=1: the stand-in fails every deregistration, and destroy keeps the arena allocated). +With --expect concurrent-close each round's child is finalized and destroyed while another thread of the +rank keeps enqueueing all-reduces on it. The parent communicator exchanges the threads' accepted call +counts after close. Both close calls must succeed when counts match; when counts differ, ncclRemoteError +is also allowed because unmatched collectives time out. Every thread must exit, its calls must return +success or a refusal (ncclInvalidArgument or ncclInvalidUsage), and a call after destroy must be refused. + +With --expect revoke-destroy each round revokes the child (ncclCommRevoke) right after its all-reduce and +then destroys it. Destroy after revoke runs the same two teardown rounds as destroy of a ready +communicator, so the rounds pass as with exact: every child all-reduce is exact, revoke and destroy succeed, +and the parent stays healthy. + With --expect fail-stop (and --env LIBSIRCL_FAIL_STOP=1, a late rank and SIRCL_STARTUP_WAIT_S below its delay) the rounds run as with exact, and the runner checks fail-stop instead: every rank but the late one must end with exit status 70 and the fail-stop line of a timed-out wait, whether the watcher or the library @@ -46,6 +57,7 @@ import subprocess import sys import tempfile +import threading import time from pathlib import Path @@ -58,6 +70,43 @@ } +def close_concurrently(enqueue, finalize, destroy, prepare_thread, exchange_counts, timeout=30): + """Race close against enqueue; exchange counts only after the worker has stopped changing them.""" + started_calls = threading.Event() + results, errors = [], [] + + def worker_main(): + try: + prepare_thread() + for _ in range(1000): + result = enqueue() + results.append(result) + started_calls.set() + if result: + break + except Exception as error: + errors.append(f"{type(error).__name__}: {error}") + finally: + started_calls.set() + + worker = threading.Thread(target=worker_main) + worker.start() + if not started_calls.wait(timeout): + raise TimeoutError("enqueue thread did not finish its first call before close") + finalized = finalize() + destroyed = destroy() + worker.join(timeout=timeout) + if worker.is_alive(): + raise TimeoutError("enqueue thread did not exit after close") + after = enqueue() + counts = exchange_counts(results.count(0)) + allowed = (0,) if len(set(counts)) == 1 else (0, 6) + return dict(ok=bool(results) and results[0] == 0 and not errors and finalized in allowed and destroyed in allowed + and all(result in (0, 4, 5) for result in results) and after in (4, 5), + finalize=finalized, destroy=destroyed, calls=results, accepted_counts=counts, + after_close=after, worker_exited=True, worker_errors=errors) + + def run_rank(args) -> int: import torch @@ -71,12 +120,15 @@ def run_rank(args) -> int: lib.ncclGetLastError.argtypes = [ctypes.c_void_p] lib.ncclAllReduce.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_size_t, ctypes.c_int, ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p] + lib.ncclAllGather.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_size_t, ctypes.c_int, + ctypes.c_void_p, ctypes.c_void_p] lib.ncclCommInitRank.argtypes = [ctypes.POINTER(ctypes.c_void_p), ctypes.c_int, unique_id.UniqueId, ctypes.c_int] lib.ncclCommSplit.argtypes = [ctypes.c_void_p, ctypes.c_int, ctypes.c_int, ctypes.POINTER(ctypes.c_void_p), ctypes.c_void_p] lib.ncclCommDestroy.argtypes = [ctypes.c_void_p] lib.ncclCommFinalize.argtypes = [ctypes.c_void_p] + lib.ncclCommRevoke.argtypes = [ctypes.c_void_p, ctypes.c_int] lib.ncclCommGetAsyncError.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_int)] lib.sirclGetReceipt.argtypes = [ctypes.c_void_p, ctypes.c_char_p, ctypes.c_size_t, ctypes.POINTER(ctypes.c_size_t)] @@ -134,14 +186,43 @@ def save(): rounds.append(entry) save() continue + if args.expect == "concurrent-close": + def exchange_counts(accepted): + # The parent already coordinates the splits and must survive every child close. + sent = torch.tensor([accepted], dtype=torch.int64, device="cuda") + received = torch.empty(world, dtype=torch.int64, device="cuda") + torch.cuda.synchronize() + gathered = lib.ncclAllGather(ctypes.c_void_p(sent.data_ptr()), + ctypes.c_void_p(received.data_ptr()), 1, 4, comm, + ctypes.c_void_p(stream.cuda_stream)) + torch.cuda.synchronize() + state = ctypes.c_int(0) + queried = lib.ncclCommGetAsyncError(comm, ctypes.byref(state)) + if gathered or queried or state.value: + raise RuntimeError(f"parent count exchange failed: {gathered}, {queried}, {state.value}") + return received.cpu().tolist() + + entry.update(close_concurrently( + lambda: lib.ncclAllReduce(ctypes.c_void_p(x.data_ptr()), ctypes.c_void_p(y.data_ptr()), + count, 9, 0, child, ctypes.c_void_p(stream.cuda_stream)), + lambda: lib.ncclCommFinalize(child), lambda: lib.ncclCommDestroy(child), + lambda: torch.cuda.set_device(0), exchange_counts)) + entry.update(ok=entry["ok"] and exact, exact=exact, + seconds=round(time.perf_counter() - started, 3)) + rounds.append(entry) + save() + continue + revoked = lib.ncclCommRevoke(child, 0) if args.expect == "revoke-destroy" else 0 + if revoked: + entry["revoke_error"] = lib.ncclGetLastError(None).decode(errors="replace") state = ctypes.c_int(0) lib.ncclCommGetAsyncError(child, ctypes.byref(state)) if not exact or state.value: entry["async_error"] = state.value entry["error"] = lib.ncclGetLastError(child).decode(errors="replace") destroyed = lib.ncclCommDestroy(child) - entry.update(ok=exact and not state.value and destroyed == 0, exact=exact, destroy=destroyed, - seconds=round(time.perf_counter() - started, 3)) + entry.update(ok=exact and not state.value and not revoked and destroyed == 0, exact=exact, destroy=destroyed, + revoke=revoked, seconds=round(time.perf_counter() - started, 3)) if destroyed: entry["destroy_error"] = lib.ncclGetLastError(None).decode(errors="replace") rounds.append(entry) @@ -230,6 +311,8 @@ def run_group(args) -> int: f"statuses {codes}, {len(problems)} problems, {time.perf_counter() - started:.1f} s; work {work}") return 1 if problems else 0 for rank in range(args.world): + if codes[rank] != 0: + problems.append(f"rank {rank}: exit {codes[rank]}; log {work / f'rank{rank}.log'}") path = work / f"rank{rank}.json" if not path.exists(): problems.append(f"rank {rank}: no result (exit {codes[rank]}); log {work / f'rank{rank}.log'}") @@ -246,6 +329,9 @@ def run_group(args) -> int: problems.append(f"rank {rank} round {entry['round']}: the close did not fail as expected: " f"finalize {entry.get('finalize')}, all-reduce after the close " f"{entry.get('after_close')}, destroy {entry.get('destroy')}: {entry.get('error')}") + elif not entry.get("ok") and args.expect == "concurrent-close": + failed_rounds.add(entry["round"]) + problems.append(f"rank {rank} round {entry['round']}: concurrent close {entry}") elif not entry.get("ok"): failed_rounds.add(entry["round"]) problems.append(f"rank {rank} round {entry['round']}: exact {entry.get('exact')}, destroy " @@ -275,7 +361,8 @@ def main(argv=None) -> int: parser.add_argument("--slow-rank", type=int, default=0) parser.add_argument("--slow-ns", type=int, default=5_000_000) parser.add_argument("--timeout", type=float, default=1800) - parser.add_argument("--expect", choices=("exact", "close-error", "fail-stop"), default="exact") + parser.add_argument("--expect", choices=("exact", "close-error", "fail-stop", "concurrent-close", + "revoke-destroy"), default="exact") parser.add_argument("--late-rank", type=int, default=-1, help="the rank that starts each all-reduce late") parser.add_argument("--late-s", type=float, default=0.0) parser.add_argument("--env", action="append", default=[], help="NAME=VALUE for every rank") diff --git a/spark_transport/libsircl/tests/fake_cuda.c b/spark_transport/libsircl/tests/fake_cuda.c index 144ecdae..5159d84f 100644 --- a/spark_transport/libsircl/tests/fake_cuda.c +++ b/spark_transport/libsircl/tests/fake_cuda.c @@ -19,6 +19,7 @@ enum { NOT_FOUND = 500, INVALID_IMAGE = 200, MAX_MODULES = 16, MAX_CUBINS = 8 }; typedef struct { + int in_use; int cubins; uint32_t flags[MAX_CUBINS]; char **names[MAX_CUBINS]; @@ -82,18 +83,26 @@ EXPORT int sccl_fake_cuda(void) { return 1; } /* Modules loaded, the fewest cubins of any module, and symbol lookups so far. */ EXPORT void sccl_fake_cuda_stats(int *loaded, int *fewest_cubins, int *looked_up) { - *loaded = nmodules; + /* The modules loaded and not unloaded, and the fewest cubins among them. */ + *loaded = 0; *fewest_cubins = 0; - for (int i = 0; i < nmodules; ++i) - if (!i || modules[i].cubins < *fewest_cubins) *fewest_cubins = modules[i].cubins; + for (int i = 0; i < nmodules; ++i) { + if (!modules[i].in_use) continue; + if (!*loaded || modules[i].cubins < *fewest_cubins) *fewest_cubins = modules[i].cubins; + ++*loaded; + } *looked_up = lookups; } EXPORT int cuModuleLoadData(void **module, const void *image) { const uint8_t *p = image; - if (nmodules == MAX_MODULES || u32(p) != 0xBA55ED50u) return INVALID_IMAGE; + const char *fail = getenv("FAKE_CUDA_FAIL_LOAD"); + if (fail && !strcmp(fail, "1")) return 209; /* CUDA_ERROR_NO_BINARY_FOR_GPU */ + int slot = 0; + while (slot < nmodules && modules[slot].in_use) ++slot; + if (slot == MAX_MODULES || u32(p) != 0xBA55ED50u) return INVALID_IMAGE; uint64_t total = (uint64_t)u16(p + 6) + u64(p + 8); - module_t *m = &modules[nmodules]; + module_t *m = &modules[slot]; memset(m, 0, sizeof *m); for (uint64_t at = 0; at + 4 <= total;) { if (memcmp(p + at, "\x7f" "ELF", 4) == 0) { @@ -107,7 +116,8 @@ EXPORT int cuModuleLoadData(void **module, const void *image) { } if (!m->cubins) return INVALID_IMAGE; *module = m; - ++nmodules; + m->in_use = 1; + if (slot == nmodules) ++nmodules; return 0; } @@ -174,14 +184,36 @@ EXPORT int cuCtxPopCurrent_v2(void **ctx) { return 0; } EXPORT int cuInit(unsigned flags) { return (int)(flags & 0); } -EXPORT int cuCtxPushCurrent_v2(void *ctx) { return ctx ? 0 : 201; } -EXPORT int cuModuleUnload(void *module) { return module ? 0 : 400; } +/* sccl_fake_cuda_fail_push(1): every later push fails, as for a destroyed context. */ +static int fail_push; +EXPORT void sccl_fake_cuda_fail_push(int on) { fail_push = on; } +static int pushes; +EXPORT int cuCtxPushCurrent_v2(void *ctx) { + if (!ctx || fail_push) return 201; + ++pushes; + return 0; +} +EXPORT int sccl_fake_cuda_pushes(void) { return pushes; } +/* CU_POINTER_ATTRIBUTE_CONTEXT (1) of any pointer: the stand-in's one context. */ +EXPORT int cuPointerGetAttribute(void *data, int attribute, unsigned long long pointer) { + if (attribute != 1 || !pointer) return 1; + *(void **)data = &context_token; + return 0; +} +EXPORT int cuModuleUnload(void *module) { + module_t *m = module; + if (!m || m < modules || m >= modules + MAX_MODULES || !m->in_use) return 400; + m->in_use = 0; + return 0; +} /* Every other entry point src/cuda_api.c binds: present, never used by the check. */ #define UNUSED_ENTRY(name) \ EXPORT int name(void) { return 0; } -UNUSED_ENTRY(cuCtxGetDevice) -UNUSED_ENTRY(cuCtxSynchronize) +EXPORT int cuCtxGetDevice(int *device) { *device = 0; return 0; } +static int context_syncs; +EXPORT int cuCtxSynchronize(void) { ++context_syncs; return 0; } +EXPORT int sccl_fake_cuda_context_syncs(void) { return context_syncs; } UNUSED_ENTRY(cuDeviceGetAttribute) UNUSED_ENTRY(cuLaunchKernel) UNUSED_ENTRY(cuMemAlloc_v2) @@ -206,5 +238,6 @@ UNUSED_ENTRY(cuEventCreate) UNUSED_ENTRY(cuEventRecord) UNUSED_ENTRY(cuEventDestroy_v2) UNUSED_ENTRY(cuEventSynchronize) +UNUSED_ENTRY(cuEventQuery) UNUSED_ENTRY(cuMemAllocAsync) UNUSED_ENTRY(cuMemFreeAsync) diff --git a/spark_transport/libsircl/tests/test_abi.py b/spark_transport/libsircl/tests/test_abi.py index 73a5ea19..c9ad1578 100644 --- a/spark_transport/libsircl/tests/test_abi.py +++ b/spark_transport/libsircl/tests/test_abi.py @@ -11,6 +11,7 @@ import sys import tempfile import unittest +from unittest.mock import patch ROOT = Path(__file__).resolve().parents[1] MANIFEST = json.loads((ROOT / "tests/api_manifest.json").read_text()) @@ -21,6 +22,20 @@ def run(*arguments: str, env: dict | None = None) -> str: return subprocess.check_output(arguments, text=True, stderr=subprocess.STDOUT, env=env) +def preload_environment() -> dict: + """Isolate the SONAME probe from inherited providers, keeping ASan first when requested.""" + env = os.environ.copy() + env.pop("LIBSIRCL_NCCL_API_VERSION", None) + # Keep the runtime selected by check-asan, including when CC names a different compiler. + preloads = [entry for entry in re.split(r"[:\s]+", env.get("LD_PRELOAD", "")) + if re.fullmatch(r"libasan\.so(?:\.\d+)*|libclang_rt\.asan.*\.so", Path(entry).name)] + if env.get("LIBSIRCL_TEST_ASAN") and not preloads: + raise unittest.SkipTest("LIBSIRCL_TEST_ASAN requires an identifiable ASan runtime in LD_PRELOAD " + "to isolate the SONAME probe from inherited NCCL providers") + env["LD_PRELOAD"] = ":".join([*preloads, str(LIBRARY)]) + return env + + # Host entry points of NCCL's device API (nccl_device/core.h and its barrier and LL all-to-all headers), # which programs built against NCCL 2.28 or later import; src/device_api.c defines them. DEVICE_API = {"ncclCommQueryProperties", "ncclDevCommCreate", "ncclDevCommDestroy", "ncclGetLsaMultimemDevicePointer", @@ -141,10 +156,11 @@ def test_preload_and_soname_reuse(self): (path / "libnccl.so").symlink_to("libnccl.so.2") (path / "preload.c").write_text(source) run("cc", "-std=c11", "-Wall", "-Wextra", "-Werror", str(path / "preload.c"), "-L", str(path), "-Wl,-rpath," + str(path), "-lnccl", "-ldl", "-o", str(path / "preload")) - env = os.environ.copy() - env.pop("LIBSIRCL_NCCL_API_VERSION", None) - env["LD_PRELOAD"] = str(LIBRARY) - self.assertIn("all 146 symbols", run(str(path / "preload"), env=env)) + inherited = os.environ.get("LD_PRELOAD", "") + for preload in (inherited, f"{path / 'libnccl.so.2'}:{inherited}", + f"{path / 'libnccl.so.2'} {inherited}"): + with self.subTest(preload=preload), patch.dict(os.environ, LD_PRELOAD=preload): + self.assertIn("all 146 symbols", run(str(path / "preload"), env=preload_environment())) if __name__ == "__main__": diff --git a/spark_transport/libsircl/tests/test_api.py b/spark_transport/libsircl/tests/test_api.py index 828c00b7..cddadc5b 100644 --- a/spark_transport/libsircl/tests/test_api.py +++ b/spark_transport/libsircl/tests/test_api.py @@ -4,6 +4,7 @@ import argparse import concurrent.futures +import importlib.util import ctypes as C import json import os @@ -53,6 +54,7 @@ def initialize_library() -> None: "CommInitRank": ([C.POINTER(C.c_void_p), C.c_int, UniqueId, C.c_int], C.c_int), "CommInitRankConfig": ([C.POINTER(C.c_void_p), C.c_int, UniqueId, C.c_int, C.POINTER(Config)], C.c_int), "CommInitAll": ([C.POINTER(C.c_void_p), C.c_int, C.POINTER(C.c_int)], C.c_int), + "CommSplit": ([C.c_void_p, C.c_int, C.c_int, C.POINTER(C.c_void_p), C.c_void_p], C.c_int), "CommRevoke": ([C.c_void_p, C.c_int], C.c_int), "CommGetAsyncError": ([C.c_void_p, C.POINTER(C.c_int)], C.c_int), "CommCount": ([C.c_void_p, C.POINTER(C.c_int)], C.c_int), @@ -153,6 +155,8 @@ class U(C.Structure): _fields_=[("b",C.c_ubyte*128)] child_uid=U(); assert l.ncclGetUniqueId(C.byref(child_uid))==5 child_comm=C.c_void_p(); assert l.ncclCommInitRank(C.byref(child_comm),1,uid,0)==5 assert child_comm.value is None + # The fork check comes first, before any argument check. + l.ncclCommRevoke.argtypes=[C.c_void_p,C.c_int]; assert l.ncclCommRevoke(None,1)==5 # libc.exit invokes DSO destructors, unlike os._exit. The inherited wake # pipe must not be written, because it would stop the parent's broker. libc=C.CDLL(None); libc.exit.argtypes=[C.c_int]; libc.exit(0) @@ -242,7 +246,11 @@ def test_revoke_then_finalize_and_abort(self): self.assertEqual(LIB.ncclCommFinalize(handle), INVALID_USAGE) self.assertEqual(LIB.ncclCommAbort(handle), SUCCESS) self.assertEqual(LIB.ncclCommAbort(handle), INVALID_ARGUMENT) - self.assertEqual(LIB.ncclCommAbort(None), INVALID_ARGUMENT) + # NULL is a no-op for destroy and abort, as in NVIDIA NCCL 2.32.3; finalize refuses it. + for prefix in ("", "p"): + self.assertEqual(getattr(LIB, prefix + "ncclCommAbort")(None), SUCCESS) + self.assertEqual(getattr(LIB, prefix + "ncclCommDestroy")(None), SUCCESS) + self.assertEqual(getattr(LIB, prefix + "ncclCommFinalize")(None), INVALID_ARGUMENT) def test_thread_local_nested_group_and_error(self): barrier = threading.Barrier(2) @@ -265,23 +273,33 @@ def separate(): f = LIB.ncclCommGetUniqueId f.argtypes = [C.c_void_p, C.c_void_p] self.assertEqual(f(None, None), INVALID_USAGE) - self.assertEqual(LIB.ncclGroupEnd(), INVALID_USAGE) - self.assertEqual(LIB.ncclGroupStart(), SUCCESS) + # An unsupported function enqueues nothing, so its refusal leaves the group's result alone. self.assertEqual(LIB.ncclGroupEnd(), SUCCESS) - - def test_unsupported_surface_preserves_every_output(self): - sentinel = C.create_string_buffer(b"Q" * 256) - before = bytes(sentinel) + self.assertIn(b"unsupported", LIB.ncclGetLastError(None)) + + def test_unsupported_surface_zeroes_its_outputs_and_preserves_every_input(self): + """Every unsupported function returns ncclInvalidUsage naming itself, zeroes the handle, value and + length outputs tools/generate_api.py lists (a NULL communicator, an all-zero unique id) and leaves every + other pointed-to byte as it was.""" + spec = importlib.util.spec_from_file_location("generate_api", ROOT / "tools/generate_api.py") + generator = importlib.util.module_from_spec(spec) + spec.loader.exec_module(generator) + sizes = {"ncclUniqueId*": 128, "int8_t*": 1, "uint8_t*": 1, "int16_t*": 2, "uint16_t*": 2, "int32_t*": 4, + "uint32_t*": 4, "int*": 4} for api in MANIFEST["functions"]: # ncclRedOpCreatePreMulSum writes its op (test_premulsum_refusal_returns_a_refused_op). if api["implemented"] or api["name"] == "ncclRedOpCreatePreMulSum": continue + zeroed = generator.ZEROED_OUTPUTS.get(api["name"], []) for prefix in ["", "p"]: with self.subTest(function=prefix + api["name"]): - args, types = [], [] + args, types, buffers = [], [], [] for argument in api["arguments"]: kind = argument["type"].removeprefix("const ") if "*" in kind or kind in ("ncclComm_t", "ncclWindow_t", "ncclParamHandle_t", "cudaStream_t"): + sentinel = C.create_string_buffer(b"Q" * 256) + size = sizes.get(kind.replace(" ", ""), 8) if argument["name"] in zeroed else 0 + buffers.append((argument["name"], sentinel, size)) types.append(C.c_void_p) args.append(C.cast(sentinel, C.c_void_p)) elif kind == "size_t": @@ -298,7 +316,8 @@ def test_unsupported_surface_preserves_every_output(self): function.restype = None if api["return"] == "void" else C.c_int result = function(*args) self.assertEqual(result, None if api["return"] == "void" else INVALID_USAGE) - self.assertEqual(bytes(sentinel), before) + for name, sentinel, size in buffers: + self.assertEqual(bytes(sentinel), bytes(size) + b"Q" * (256 - size) + b"\0", name) diagnostic = LIB.ncclGetLastError(None) self.assertTrue(diagnostic and api["name"].encode() in diagnostic) @@ -315,8 +334,8 @@ def test_premulsum_refusal_returns_a_refused_op(self): self.assertEqual(function(None, C.byref(scalar), 7, 1, None), INVALID_USAGE) def test_invalid_argument_inside_a_group_leaves_the_group(self): - # A refused call did nothing: ncclGroupEnd still succeeds (an unsupported call, ncclInvalidUsage, - # remains the group's result; test_thread_local_nested_group_and_error). + # A refused call did nothing: ncclGroupEnd still succeeds (as after an unsupported call, which + # enqueues nothing either; test_thread_local_nested_group_and_error). LIB.ncclAllReduce.argtypes = [C.c_void_p, C.c_void_p, C.c_size_t, C.c_int, C.c_int, C.c_void_p, C.c_void_p] LIB.ncclSend.argtypes = [C.c_void_p, C.c_size_t, C.c_int, C.c_int, C.c_void_p, C.c_void_p] diff --git a/spark_transport/libsircl/tests/test_bootstrap.py b/spark_transport/libsircl/tests/test_bootstrap.py index 61fb88dc..e686d58a 100644 --- a/spark_transport/libsircl/tests/test_bootstrap.py +++ b/spark_transport/libsircl/tests/test_bootstrap.py @@ -409,6 +409,89 @@ def joined_group(self, world, value=None): self.assertEqual([r for r, _ in results], [0] * world) return [handle for _, handle in results] + def test_every_descriptor_is_close_on_exec_from_its_creation(self): + """The bootstrap creates every socket and pipe close-on-exec atomically (SOCK_CLOEXEC, accept4 with + SOCK_CLOEXEC, pipe2 with O_CLOEXEC), never setting the flag after the fact, so a concurrent fork and + exec cannot inherit one. A build of the bootstrap whose socket, accept, accept4, pipe and pipe2 calls + are wrapped (ld --wrap) counts the creations made without the flag.""" + build = Path(self.build.name) + probe = build / "cloexec_probe.c" + probe.write_text(r''' +#define _GNU_SOURCE +#include +#include +#include +static int created, missed; +int __real_socket(int, int, int); +int __real_accept(int, struct sockaddr *, socklen_t *); +int __real_accept4(int, struct sockaddr *, socklen_t *, int); +int __real_pipe(int[2]); +int __real_pipe2(int[2], int); +int __wrap_socket(int d, int t, int p) { created++; missed += !(t & SOCK_CLOEXEC); return __real_socket(d, t, p); } +int __wrap_accept(int f, struct sockaddr *a, socklen_t *l) { created++; missed++; return __real_accept(f, a, l); } +int __wrap_accept4(int f, struct sockaddr *a, socklen_t *l, int x) { + created++; missed += !(x & SOCK_CLOEXEC); return __real_accept4(f, a, l, x); } +int __wrap_pipe(int p[2]) { created++; missed++; return __real_pipe(p); } +int __wrap_pipe2(int p[2], int x) { created++; missed += !(x & O_CLOEXEC); return __real_pipe2(p, x); } +int test_created(void) { return created; } +int test_missed(void) { return missed; } +''', encoding="utf-8") + source = Path(__file__).resolve().parents[1] / "src" / "bootstrap.c" + library = build / "bootstrap_wrapped.so" + wrap = ",".join(f"--wrap={name}" for name in ("socket", "accept", "accept4", "pipe", "pipe2")) + subprocess.run([os.environ.get("CC", "cc"), "-std=c11", "-O2", "-fPIC", "-shared", "-pthread", + "-I", str(source.parent), f"-Wl,{wrap}", str(source), str(probe), "-o", str(library)], + check=True) + lib = ctypes.CDLL(str(library)) + lib.sccl_bootstrap_id.argtypes = [ctypes.c_void_p] + lib.sccl_bootstrap_join.argtypes = [ctypes.c_void_p, ctypes.c_int, ctypes.c_int, ctypes.POINTER(ctypes.c_void_p)] + lib.sccl_bootstrap_close.argtypes = [ctypes.c_void_p] + value = ctypes.create_string_buffer(128) + self.assertEqual(lib.sccl_bootstrap_id(value), 0) + handle = ctypes.c_void_p() + self.assertEqual(lib.sccl_bootstrap_join(value.raw, 1, 0, ctypes.byref(handle)), 0) + lib.sccl_bootstrap_close(handle) + # The id's listener and wake pipe, the joining socket and the root's accepted connection. + self.assertGreaterEqual(lib.test_created(), 4) + self.assertEqual(lib.test_missed(), 0) + + def test_setup_timeout_must_be_valid(self): + """An invalid LIBSIRCL_SETUP_TIMEOUT_MS refuses the all-gather, naming the variable, instead of + silently waiting the 600 s default.""" + handles = self.joined_group(1) + saved = os.environ.get("LIBSIRCL_SETUP_TIMEOUT_MS") + try: + for invalid in ("abc", "0", "3600001", "-5"): + with self.subTest(value=invalid): + os.environ["LIBSIRCL_SETUP_TIMEOUT_MS"] = invalid + result, _ = self.bootstrap.allgather(handles[0], b"x", 1) + self.assertEqual(result, 4) + self.assertIn(b"LIBSIRCL_SETUP_TIMEOUT_MS", self.bootstrap.lib.sccl_bootstrap_error()) + os.environ["LIBSIRCL_SETUP_TIMEOUT_MS"] = "5000" + self.assertEqual(self.bootstrap.allgather(handles[0], b"ok", 1), (0, [b"ok"])) + finally: + if saved is None: + os.environ.pop("LIBSIRCL_SETUP_TIMEOUT_MS", None) + else: + os.environ["LIBSIRCL_SETUP_TIMEOUT_MS"] = saved + self.bootstrap.close(handles[0]) + + def test_idle_connections_do_not_hold_the_roots_slots(self): + """Thirty-two connections that never send a hello fill the root's pending slots; the root drops them + after its hello limit (1 s), so a rank that joins later is admitted within the join window.""" + os.environ["SIRCL_BOOTSTRAP_TIMEOUT_MS"] = "8000" + value = self.bootstrap.unique_id() + port = struct.unpack("!H", value[16:18])[0] + idle = [socket.create_connection(("127.0.0.1", port), timeout=1) for _ in range(32)] + try: + time.sleep(1.5) + result, handle = self.bootstrap.join(value, 1, 0) + self.assertEqual(result, 0, self.bootstrap.lib.sccl_bootstrap_error()) + self.bootstrap.close(handle) + finally: + for connection in idle: + connection.close() + def test_allgather_rounds(self): world = 3 handles = self.joined_group(world) @@ -426,6 +509,37 @@ def test_allgather_rounds(self): for handle in handles: self.bootstrap.close(handle) + def test_concurrent_allgathers_on_one_handle_run_one_after_another(self): + """Two threads start an all-gather on rank 0's handle at once while rank 1 runs two rounds: the two + rounds on rank 0 run one after another instead of interleaving their frames, so every round completes + and each of rank 1's rounds pairs with exactly one of rank 0's.""" + saved = os.environ.get("LIBSIRCL_SETUP_TIMEOUT_MS") + os.environ["LIBSIRCL_SETUP_TIMEOUT_MS"] = "3000" + try: + for attempt in range(20): + handles = self.joined_group(2) + try: + with concurrent.futures.ThreadPoolExecutor(max_workers=3) as pool: + zeros = [pool.submit(self.bootstrap.allgather, handles[0], payload * 3000, 2) + for payload in (b"a", b"b")] + ones = pool.submit(lambda: [self.bootstrap.allgather(handles[1], payload, 2) + for payload in (b"x", b"y")]) + zeros = [future.result(timeout=10) for future in zeros] + ones = ones.result(timeout=10) + self.assertEqual([result for result, _ in zeros + ones], [0] * 4, attempt) + seen_by_zero = {parts[1]: parts[0] for _, parts in zeros} + self.assertEqual(sorted(seen_by_zero), [b"x", b"y"], attempt) + for _, parts in ones: + self.assertEqual(parts[0], seen_by_zero[parts[1]], attempt) + finally: + for handle in handles: + self.bootstrap.close(handle) + finally: + if saved is None: + os.environ.pop("LIBSIRCL_SETUP_TIMEOUT_MS", None) + else: + os.environ["LIBSIRCL_SETUP_TIMEOUT_MS"] = saved + def test_allgather_fails_fast_when_a_rank_leaves(self): handles = self.joined_group(3) self.bootstrap.close(handles[2]) @@ -509,6 +623,32 @@ def test_interface_address(self): finally: os.environ.pop("SIRCL_BOOTSTRAP_ADDR") + def test_interface_exclusions_and_loopback(self): + """NCCL_SOCKET_IFNAME with exclusions only (``^docker``, ``^=lo``) never selects loopback, which a remote + rank cannot reach; ``^=name`` excludes exactly that interface; a list longer than 255 characters is + refused, naming the variable.""" + if not any(name != "lo" for _, name in socket.if_nameindex()): + self.skipTest("no interface besides loopback") + os.environ.pop("SIRCL_BOOTSTRAP_IFNAME", None) + try: + for value in ("^docker", "^=lo", "^=lo,^docker"): + with self.subTest(list=value): + os.environ["NCCL_SOCKET_IFNAME"] = value + buffer = ctypes.create_string_buffer(128) + result = self.bootstrap.lib.sccl_bootstrap_id(buffer) + if result == 4 and b"no IPv4 address" in self.bootstrap.lib.sccl_bootstrap_error(): + self.skipTest("no IPv4 interface besides loopback") + self.assertEqual(result, 0, self.bootstrap.lib.sccl_bootstrap_error()) + self.assertNotEqual(socket.inet_ntoa(buffer.raw[12:16]), "127.0.0.1") + for handle in self.joined_group(1, buffer.raw): + self.bootstrap.close(handle) + os.environ["NCCL_SOCKET_IFNAME"] = ",".join(["^docker"] * 40) + buffer = ctypes.create_string_buffer(128) + self.assertEqual(self.bootstrap.lib.sccl_bootstrap_id(buffer), 4) + self.assertIn(b"NCCL_SOCKET_IFNAME is longer than 255", self.bootstrap.lib.sccl_bootstrap_error()) + finally: + os.environ.pop("NCCL_SOCKET_IFNAME", None) + def test_invalid_env_and_arguments(self): for invalid in ["0", "-1", "nan", "600001"]: os.environ["SIRCL_BOOTSTRAP_TIMEOUT_MS"] = invalid diff --git a/spark_transport/libsircl/tests/test_build_flags.py b/spark_transport/libsircl/tests/test_build_flags.py new file mode 100644 index 00000000..e0fea72c --- /dev/null +++ b/spark_transport/libsircl/tests/test_build_flags.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +"""The build's flags and test lists. + +- The Makefile keeps its required compiler flags when CFLAGS or CPPFLAGS come from the command line: + ``make -n`` (nothing runs) of the library's objects with ``CFLAGS=-O1`` and ``CPPFLAGS=-DEXTRA`` must still + print -fPIC, -Werror, -std=c11 and -fstack-protector-strong (CFLAGS) and -Iinclude, -DLIBSIRCL_CPU_ONLY and + -D_FORTIFY_SOURCE=2 (CPPFLAGS) beside the given flags. +- The library (``--library``) is linked with full RELRO and immediate binding and a non-executable stack + (``readelf``: a GNU_RELRO segment, BIND_NOW, and a GNU_STACK segment without execute permission). +- ``make check`` and CMake's ``ctest`` run the same suites: the test names in CMakeLists.txt equal the + suites the Makefile's check recipe runs. +""" +import argparse +import os +import re +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +LIBRARY = ROOT / "build" / "libsircl.so" + + +class BuildFlagTests(unittest.TestCase): + def test_command_line_flags_keep_the_required_ones(self): + with tempfile.TemporaryDirectory() as build: + out = subprocess.run(["make", "-n", "-C", str(ROOT), f"BUILD={build}", "CFLAGS=-O1", + "CPPFLAGS=-DEXTRA", f"{build}/api.o"], + capture_output=True, text=True, timeout=60, + env={k: v for k, v in os.environ.items() if k not in ("CFLAGS", "CPPFLAGS")}) + self.assertEqual(out.returncode, 0, out.stderr) + line = next(row for row in out.stdout.splitlines() if "src/api.c" in row) + for flag in ("-O1", "-DEXTRA", "-fPIC", "-Werror", "-std=c11", "-Iinclude", "-DLIBSIRCL_CPU_ONLY", + "-fstack-protector-strong", "-D_FORTIFY_SOURCE=2"): + self.assertIn(flag, line.split(), line) + + def test_the_library_is_linked_hardened(self): + if not LIBRARY.exists(): + self.skipTest(f"{LIBRARY} is not built") + program = subprocess.run(["readelf", "-lW", str(LIBRARY)], capture_output=True, text=True, check=True).stdout + dynamic = subprocess.run(["readelf", "-dW", str(LIBRARY)], capture_output=True, text=True, check=True).stdout + self.assertIn("GNU_RELRO", program) + self.assertRegex(dynamic, r"\(FLAGS\)\s+BIND_NOW|\(FLAGS_1\)\s+Flags:.*\bNOW\b|\(BIND_NOW\)") + stack = next(row for row in program.splitlines() if "GNU_STACK" in row) + self.assertNotIn("E", stack.split()[-2], stack) + + def test_make_check_and_ctest_run_the_same_suites(self): + makefile = (ROOT / "Makefile").read_text() + recipe = makefile[makefile.index("\ncheck:"):makefile.index("exit $$status")] + from_make = set(re.findall(r"tests/test_(\w+)\.py", recipe)) + from_make |= {name.removeprefix("test_") for name in re.findall(r"\$\(BUILD\)/(test_\w+)", recipe)} + cmake = (ROOT / "CMakeLists.txt").read_text() + from_cmake = set(re.findall(r"add_test\(NAME (\w+)", cmake)) + for names in re.findall(r"foreach\(name ([\w ]+)\)\s*\n\s*add_test", cmake): + from_cmake |= set(names.split()) + from_cmake.discard("${name}") + self.assertEqual(from_make, from_cmake) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--library", type=Path, default=LIBRARY) + options, extra = parser.parse_known_args() + LIBRARY = options.library.resolve() + unittest.main(argv=[sys.argv[0], *extra], verbosity=2) diff --git a/spark_transport/libsircl/tests/test_docs_consistency.py b/spark_transport/libsircl/tests/test_docs_consistency.py new file mode 100644 index 00000000..6d745c88 --- /dev/null +++ b/spark_transport/libsircl/tests/test_docs_consistency.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +"""The counts libsircl's documents state agree with the code. + +- README.md states the number of the header's functions and of those implemented, which + tests/api_manifest.json records; STATUS.md lists every function the manifest marks not implemented. +- README.md, STATUS.md and KERNEL_ROUTE.md speak of the four kernel packs the loader embeds + (src/kernelpack.c's sccl_*_fatbin symbols), never of three or of "both" packs. +- README.md and STATUS.md name the version of this repository's SIRCL package. +""" +import json +import re +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +REPOSITORY = ROOT.parents[1] + + +def text(name): + return " ".join((ROOT / name).read_text(encoding="utf-8").split()) + + +class DocsConsistencyTests(unittest.TestCase): + def test_function_counts_match_the_manifest(self): + functions = json.loads((ROOT / "tests/api_manifest.json").read_text(encoding="utf-8"))["functions"] + implemented = sum(function["implemented"] for function in functions) + readme = text("README.md") + self.assertIn(f"implements {implemented} of the header's {len(functions)} functions", readme) + self.assertIn(f"the other {len(functions) - implemented} return `ncclInvalidUsage`", readme) + status = text("STATUS.md") + missing = [function["name"] for function in functions + if not function["implemented"] and f"`{function['name']}`" not in status] + self.assertEqual(missing, [], "STATUS.md does not list these unsupported functions") + + def test_the_documents_count_the_packs_the_loader_embeds(self): + loader = (ROOT / "src/kernelpack.c").read_text(encoding="utf-8") + packs = set(re.findall(r"extern const unsigned char (sccl_\w+_fatbin)\[\];", loader)) + self.assertEqual(len(packs), 4) + for name in ("README.md", "STATUS.md", "KERNEL_ROUTE.md"): + body = text(name) + with self.subTest(document=name): + self.assertNotRegex(body, r"\bthree (embedded |kernel )?packs\b") + self.assertNotRegex(body, r"\bboth packs\b") + self.assertIn("four kernel packs", text("STATUS.md")) + + def test_the_sircl_package_version(self): + version = re.search(r'^version = "([^"]+)"', (REPOSITORY / "spark_transport/sircl/pyproject.toml").read_text( + encoding="utf-8"), re.M).group(1) + for name in ("README.md", "STATUS.md"): + with self.subTest(document=name): + self.assertIn(f"`sparkring_sircl` {version}", text(name)) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/spark_transport/libsircl/tests/test_engine_cpu.py b/spark_transport/libsircl/tests/test_engine_cpu.py index bd1cc348..acdb6c86 100644 --- a/spark_transport/libsircl/tests/test_engine_cpu.py +++ b/spark_transport/libsircl/tests/test_engine_cpu.py @@ -10,10 +10,13 @@ import argparse import json import os +import re import subprocess import sys +import tempfile import unittest from pathlib import Path +from unittest.mock import patch LIBRARY = Path(__file__).resolve().parents[1] / "build" / "libsircl.so" FAKE_DRIVER = LIBRARY.parent / "fake-cuda" @@ -32,14 +35,328 @@ class U(C.Structure): _fields_=[("b",C.c_ubyte*128)] ''' -def run(script, env=None): +def run(script, env=None, *, isolate_preloads=False): environment = dict(os.environ) environment.update(env or {}) + if os.environ.get("LIBSIRCL_TEST_ASAN"): + environment["LD_PRELOAD"] = os.environ["LD_PRELOAD"] + if isolate_preloads: + # A runtime preloaded at startup wins RTLD_DEFAULT lookup over the fake driver's later dlopen. + # Preserve ASan first, but remove inherited CUDA/NCCL providers from these controlled probes. + preloads = [entry for entry in re.split(r"[:\s]+", environment.get("LD_PRELOAD", "")) + if re.fullmatch(r"libasan\.so(?:\.\d+)*|libclang_rt\.asan.*\.so", Path(entry).name)] + if environment.get("LIBSIRCL_TEST_ASAN") and not preloads: + raise unittest.SkipTest("LIBSIRCL_TEST_ASAN requires an identifiable ASan runtime in LD_PRELOAD " + "to isolate the CUDA context probe") + environment["LD_PRELOAD"] = ":".join(preloads) return subprocess.run([sys.executable, "-c", PRELUDE + script, str(LIBRARY)], env=environment, capture_output=True, text=True, timeout=60) class EngineCpuTests(unittest.TestCase): + def test_finalize_and_destroy_wait_for_the_context(self): + out = run('''g=C.CDLL("libcuda.so.1") +u=U(); assert l.ncclGetUniqueId(C.byref(u))==0 +c=C.c_void_p(); assert l.ncclCommInitRank(C.byref(c),1,u,0)==0 +assert l.ncclCommFinalize(c)==0 +before=g.sccl_fake_cuda_context_syncs() +assert l.ncclCommDestroy(c)==0 +print(json.dumps([before,g.sccl_fake_cuda_context_syncs()])) +''', {"LIBSIRCL_BOOTSTRAP_ONLY": "", "LIBSIRCL_TRANSPORT": "verbs", + "LD_LIBRARY_PATH": str(FAKE_DRIVER)}) + self.assertEqual(out.returncode, 0, out.stderr) + before, after = json.loads(out.stdout) + self.assertGreaterEqual(before, 1) + self.assertGreater(after, before) + + def test_one_pass_ring_relay_needs_the_research_opt_in(self): + """LIBSIRCL_RING_REDUCE_PASSES=1 selects the one-pass ring relay, which is research-only: creation is + refused, naming the opt-in, unless LIBSIRCL_RING_REDUCE_ONE_PASS_RESEARCH=1; 2 (the default) and the + opted-in 1 create a communicator on the stand-in driver.""" + script = '''u=U(); assert l.ncclGetUniqueId(C.byref(u))==0 +c=C.c_void_p(); r=l.ncclCommInitRank(C.byref(c),1,u,0) +print(json.dumps([r, l.ncclGetLastError(None).decode()])) +if c.value: l.ncclCommDestroy(c) +''' + base = {"LIBSIRCL_BOOTSTRAP_ONLY": "", "LIBSIRCL_TRANSPORT": "verbs", "LD_LIBRARY_PATH": str(FAKE_DRIVER)} + for passes, opt_in, refused in (("1", "", True), ("1", "0", True), ("1", "1", False), ("2", "", False)): + with self.subTest(passes=passes, opt_in=opt_in): + out = run(script, {**base, "LIBSIRCL_RING_REDUCE_PASSES": passes, + "LIBSIRCL_RING_REDUCE_ONE_PASS_RESEARCH": opt_in}) + self.assertEqual(out.returncode, 0, out.stderr) + result, message = json.loads(out.stdout) + if refused: + self.assertNotEqual(result, 0) + self.assertIn("LIBSIRCL_RING_REDUCE_ONE_PASS_RESEARCH=1", message) + else: + self.assertEqual(result, 0, message) + + def test_failed_creations_release_their_bootstrap(self): + """Seventy creations that fail in session setup (LIBSIRCL_TRANSPORT=bogus), whose handles are never + destroyed, leave no socket open and no bootstrap root held: a further unique id is still granted + (the process holds at most 64 roots).""" + out = run('''import os, time +before=len(os.listdir("/proc/self/fd")) +handles=[] +for _ in range(70): + u=U(); assert l.ncclGetUniqueId(C.byref(u))==0, l.ncclGetLastError(None) + c=C.c_void_p(); r=l.ncclCommInitRank(C.byref(c),1,u,0); assert r!=0 and c.value; handles.append(c) +time.sleep(0.3) +after=len(os.listdir("/proc/self/fd")) +u=U(); last=l.ncclGetUniqueId(C.byref(u)) +print(json.dumps([before, after, last, l.ncclGetLastError(None).decode()])) +''', {"LIBSIRCL_BOOTSTRAP_ONLY": "", "LIBSIRCL_TRANSPORT": "bogus", "LD_LIBRARY_PATH": str(FAKE_DRIVER)}) + self.assertEqual(out.returncode, 0, out.stderr[-2000:]) + before, after, last, message = json.loads(out.stdout) + self.assertEqual(last, 0, message) + self.assertEqual(after, before) + + def test_failed_initall_leaves_no_handles(self): + """An ncclCommInitAll whose communicators fail in session setup aborts them and clears every output.""" + out = run("c=(C.c_void_p*2)(); r=l.ncclCommInitAll(c,2,(C.c_int*2)(0,0)); " + "print(json.dumps([r,list(c),l.ncclGetLastError(None).decode()]))", + {"LIBSIRCL_BOOTSTRAP_ONLY": "", "LIBSIRCL_TRANSPORT": "bogus", "FAKE_CUDA_DEVICES": "1", + "LD_LIBRARY_PATH": str(FAKE_DRIVER)}) + self.assertEqual(out.returncode, 0, out.stderr) + result, handles, message = json.loads(out.stdout) + self.assertNotIn(result, (0, 7)) + self.assertEqual(handles, [None, None]) + self.assertIn("group initialization failed", message) + + def test_destroy_without_its_context_frees_the_host_bookkeeping(self): + """Destroying a communicator whose context can no longer be made current (the stand-in driver fails + every push, as for a destroyed context) frees its engine: 1,000 such destroys leave the process heap + within 1 MiB of where it started. Not under AddressSanitizer, whose allocator mallinfo2 does not see.""" + if os.environ.get("LIBSIRCL_TEST_ASAN"): + self.skipTest("mallinfo2 does not measure AddressSanitizer's allocator") + out = run('''g=C.CDLL("libcuda.so.1") +class Info(C.Structure): _fields_=[(n, C.c_size_t) for n in ("arena","ordblks","smblks","hblks","hblkhd","usmblks","fsmblks","uordblks","fordblks","keepcost")] +libc=C.CDLL(None); libc.mallinfo2.restype=Info +def used(): return libc.mallinfo2().uordblks + libc.mallinfo2().hblkhd +def cycle(): + u=U(); assert l.ncclGetUniqueId(C.byref(u))==0 + c=C.c_void_p(); assert l.ncclCommInitRank(C.byref(c),1,u,0)==0 + g.sccl_fake_cuda_fail_push(1); l.ncclCommDestroy(c); g.sccl_fake_cuda_fail_push(0) +for _ in range(5): cycle() +before=used() +for _ in range(1000): cycle() +print(json.dumps([before, used()])) +''', {"LIBSIRCL_BOOTSTRAP_ONLY": "", "LIBSIRCL_TRANSPORT": "verbs", "LD_LIBRARY_PATH": str(FAKE_DRIVER)}) + self.assertEqual(out.returncode, 0, out.stderr[-2000:]) + before, after = json.loads(out.stdout) + self.assertLess(after - before, 1 << 20, (before, after)) + + def test_initall_refuses_distinct_cuda_devices(self): + for devices in ("None", "(C.c_int*2)(0,1)"): + out = run("c=(C.c_void_p*2)(); r=l.ncclCommInitAll(c,2," + devices + "); " + "print(json.dumps([r,list(c),l.ncclGetLastError(None).decode()]))", + {"LIBSIRCL_BOOTSTRAP_ONLY": "", "LIBSIRCL_TRANSPORT": "bogus", + "FAKE_CUDA_DEVICES": "2", "LD_LIBRARY_PATH": str(FAKE_DRIVER)}) + self.assertEqual(out.returncode, 0, out.stderr) + result, handles, message = json.loads(out.stdout) + self.assertEqual(result, 5) + self.assertEqual(handles, [None, None]) + self.assertIn("distinct devices", message) + + def test_setup_refuses_out_of_range_peer_positions(self): + for position in (-1, 64, 2147483647): + with self.subTest(position=position): + out = run('''import socket, struct, threading +u=U(); assert l.ncclGetUniqueId(C.byref(u))==0 +server=socket.socket(); server.bind(("127.0.0.1", 0)); server.listen() +u.b[16:18]=struct.pack("!H",server.getsockname()[1]) +records=[] +def broker(): + peer,_=server.accept(); peer.settimeout(3) + def read(n): + data=b"" + while len(data).partial) is left as it + is and the file it names is not written: receipts are written under a name of the writer's own, + opened with O_EXCL and O_NOFOLLOW, then renamed over the receipt.""" + with tempfile.TemporaryDirectory() as directory: + out = run('''import os, sys +directory=sys.argv[2] +target=os.path.join(directory, "victim"); open(target, "w").write("untouched") +prefix=os.path.join(directory, "r") +receipt=f"{prefix}.rank0.{os.getpid()}.c1.json" +os.symlink(target, receipt + ".partial") +os.environ["LIBSIRCL_RECEIPT"]=prefix +u=U(); assert l.ncclGetUniqueId(C.byref(u))==0 +c=C.c_void_p(); assert l.ncclCommInitRank(C.byref(c),1,u,0)==0 +assert l.ncclCommDestroy(c)==0 +print(json.dumps([open(target).read(), os.path.islink(receipt + ".partial"), os.path.isfile(receipt), + sorted(name for name in os.listdir(directory) if ".partial." in name), + json.load(open(receipt)).get("quarantined_bytes")])) +'''.replace("sys.argv[2]", repr(directory)), {"LIBSIRCL_BOOTSTRAP_ONLY": "", "LIBSIRCL_TRANSPORT": "verbs", + "LD_LIBRARY_PATH": str(FAKE_DRIVER)}) + self.assertEqual(out.returncode, 0, out.stderr[-2000:]) + victim, still_link, written, leftovers, quarantined = json.loads(out.stdout) + # Every receipt reports the memory quarantined engines hold; none here. + self.assertEqual(quarantined, 0) + self.assertEqual(victim, "untouched") + self.assertTrue(still_link) + self.assertTrue(written) + self.assertEqual(leftovers, []) + + def test_settings_are_parsed_strictly(self): + """Settings that were taken loosely are refused, naming the variable: a signed integer, a traffic class + that is not 0-255, a receipt interval that is not a number, and a list variable longer than the + 1,024-byte buffer it is parsed from.""" + script = '''u=U(); assert l.ncclGetUniqueId(C.byref(u))==0 +c=C.c_void_p(); r=l.ncclCommInitRank(C.byref(c),1,u,0) +print(json.dumps([r, l.ncclGetLastError(None).decode()])) +if c.value: l.ncclCommDestroy(c) +''' + base = {"LIBSIRCL_BOOTSTRAP_ONLY": "", "LIBSIRCL_TRANSPORT": "verbs", "LD_LIBRARY_PATH": str(FAKE_DRIVER)} + cases = [({"LIBSIRCL_LANE_CHECK_MS": "+5000"}, "LIBSIRCL_LANE_CHECK_MS=+5000 must be an integer"), + ({"SIRCL_ONESHOT_MAX_BYTES": " 4096"}, "SIRCL_ONESHOT_MAX_BYTES= 4096 must be an integer"), + ({"SIRCL_TRAFFIC_CLASS": "abc"}, "SIRCL_TRAFFIC_CLASS=abc must be an integer from 0 to 255"), + ({"NCCL_IB_TC": "300"}, "NCCL_IB_TC=300 must be an integer from 0 to 255"), + ({"LIBSIRCL_RECEIPT_INTERVAL_S": "soon"}, "LIBSIRCL_RECEIPT_INTERVAL_S=soon must be a number"), + ({"LIBSIRCL_FORWARD_WINDOWS": ",".join(["1=32768/32768"] * 80)}, + "LIBSIRCL_FORWARD_WINDOWS is longer than 1023 characters")] + for extra, text in cases: + with self.subTest(**{k: v[:20] for k, v in extra.items()}): + out = run(script, {**base, **extra}) + self.assertEqual(out.returncode, 0, out.stderr) + result, message = json.loads(out.stdout) + self.assertNotEqual(result, 0) + self.assertIn(text, message) + out = run(script, {**base, "SIRCL_TRAFFIC_CLASS": "106", "LIBSIRCL_RECEIPT_INTERVAL_S": "2.5"}) + self.assertEqual(json.loads(out.stdout)[0], 0, out.stdout) + + def test_mem_free_makes_the_memorys_context_current(self): + """ncclMemFree on a thread without a current context frees in the context the memory belongs to: it + makes that context current for the call (the stand-in driver counts the pushes).""" + out = run('''g=C.CDLL("libcuda.so.1") +l.ncclMemFree.argtypes=[C.c_void_p] +before=g.sccl_fake_cuda_pushes() +r=l.ncclMemFree(C.c_void_p(0x10000)) +print(json.dumps([r, g.sccl_fake_cuda_pushes()-before])) +''', {"FAKE_CUDA_NO_CONTEXT": "1", "LD_LIBRARY_PATH": str(FAKE_DRIVER)}) + self.assertEqual(out.returncode, 0, out.stderr) + self.assertEqual(json.loads(out.stdout), [0, 1]) + + def test_an_ignored_config_field_is_logged_once_under_nccl_debug(self): + """libsircl honors only ncclConfig_t.blocking: under NCCL_DEBUG=WARN the first config that sets another + field writes one line naming it; without NCCL_DEBUG nothing is written.""" + script = '''class Config(C.Structure): + _fields_ = [("size", C.c_size_t), ("magic", C.c_uint), ("version", C.c_uint), ("blocking", C.c_int), + ("cgaClusterSize", C.c_int), ("minCTAs", C.c_int), ("maxCTAs", C.c_int), ("netName", C.c_char_p), + ("splitShare", C.c_int), ("trafficClass", C.c_int), ("commName", C.c_char_p)] + [ + (n, C.c_int) for n in ("collnetEnable", "CTAPolicy", "shrinkShare", "nvlsCTAs", + "nChannelsPerNetPeer", "nvlinkCentricSched", "graphUsageMode", "numRmaCtx", "maxP2pPeers", + "graphStreamOrdering", "launchOrderImplicit", "numRmaSig", "rmaEagerInit", "hostCftMode", + "nvlsHostMode")] +l.ncclCommInitRankConfig.argtypes=[C.POINTER(C.c_void_p),C.c_int,U,C.c_int,C.POINTER(Config)] +for name in ("commName", "commName"): + cfg=Config(); cfg.size=C.sizeof(Config); cfg.magic=0xcafebeef; cfg.version=23203 + for field, kind in Config._fields_[3:]: + if kind is C.c_int: setattr(cfg, field, -2147483648) + cfg.commName=b"tp0" + u=U(); assert l.ncclGetUniqueId(C.byref(u))==0 + c=C.c_void_p(); assert l.ncclCommInitRankConfig(C.byref(c),1,u,0,C.byref(cfg))==0 + assert l.ncclCommDestroy(c)==0 +''' + loud = run(script, {"LIBSIRCL_BOOTSTRAP_ONLY": "1", "NCCL_DEBUG": "WARN"}) + self.assertEqual(loud.returncode, 0, loud.stderr) + lines = [line for line in loud.stderr.splitlines() if "is set and ignored" in line] + self.assertEqual(lines, ["libsircl: ncclConfig_t.commName is set and ignored: libsircl honors only blocking"]) + quiet = run(script, {"LIBSIRCL_BOOTSTRAP_ONLY": "1", "NCCL_DEBUG": ""}) + self.assertEqual(quiet.returncode, 0, quiet.stderr) + self.assertNotIn("is set and ignored", quiet.stderr) + def test_info_names_the_collectives(self): out = run("print(l.sirclGetInfo().decode())") self.assertEqual(out.returncode, 0, out.stderr) @@ -48,6 +365,11 @@ def test_info_names_the_collectives(self): self.assertIn("reduce_scatter", info["collectives"]) self.assertIn("all_gather", info["collectives"]) self.assertEqual(info["transports"], ["verbs", "emulation"]) + # The documented point-to-point scope, the four packs and the plans that change the default schedule. + self.assertIn("point-to-point channels on communicators of three or more ranks", info["collectives"]["send_recv"]) + self.assertIn("transport, fold, link and point-to-point packs", info["kernels"]) + self.assertIn("pair plan", info["schedules"]["default"]) + self.assertIn("cycle plan", info["schedules"]["default"]) def test_device_api_host_entry_points(self): """ncclCommQueryProperties reports the communicator and no device API; the teams are the world, @@ -104,7 +426,8 @@ def creation(self, env, *, runtime=False): """One ncclCommInitRank of a one-rank communicator on the stand-in driver, on a thread without a current context; LIBSIRCL_TRANSPORT=bogus makes the engine refuse right after the context step, before any device work. ``runtime`` brings the stand-in's cudaGetDevice into the global scope, as a framework's CUDA runtime - is. Returns (result, ncclGetLastError, (retains, retained device, cuCtxSetCurrent calls), stderr).""" + is. Inherited preloads are removed except ASan, so only this stand-in can supply cudaGetDevice. + Returns (result, ncclGetLastError, (retains, retained device, cuCtxSetCurrent calls), stderr).""" load = (f"g=C.CDLL({str(FAKE_DRIVER / 'libcuda.so.1')!r}, mode=C.RTLD_GLOBAL)\n" if runtime else f"g=C.CDLL({str(FAKE_DRIVER / 'libcuda.so.1')!r})\n") environment = {"LIBSIRCL_BOOTSTRAP_ONLY": "", "LIBSIRCL_TRANSPORT": "bogus", "FAKE_CUDA_NO_CONTEXT": "1", @@ -114,7 +437,7 @@ def creation(self, env, *, runtime=False): c=C.c_void_p(); r=l.ncclCommInitRank(C.byref(c),1,u,0) n=[C.c_int(),C.c_int(),C.c_int()]; g.sccl_fake_cuda_contexts(*[C.byref(v) for v in n]) print(json.dumps({"result":r,"message":l.ncclGetLastError(None).decode(),"contexts":[v.value for v in n]}))''', - environment) + environment, isolate_preloads=True) self.assertEqual(out.returncode, 0, out.stderr) data = json.loads(out.stdout) return data["result"], data["message"], tuple(data["contexts"]), out.stderr @@ -136,6 +459,7 @@ def test_init_without_cuda_context_uses_the_runtimes_current_device(self): runtime=True) self.assertIn("LIBSIRCL_TRANSPORT=bogus", message) self.assertEqual(contexts, (1, 1, 1)) + self.assertNotEqual(result, 0) def test_init_without_cuda_context_is_refused_when_the_device_is_unknown(self): # Two devices and no CUDA runtime in the process: no device can be chosen, so the creation is refused, @@ -146,6 +470,31 @@ def test_init_without_cuda_context_is_refused_when_the_device_is_unknown(self): self.assertEqual(contexts, (0, -1, 0)) self.assertIn("libsircl: communicator creation failed (invalid usage): no current CUDA context", stderr) + def test_context_selection_ignores_inherited_cuda_runtime(self): + """A preloaded runtime must not replace the creation probe's controlled runtime or its absence.""" + source = '''#include +int cudaGetDevice(int *device) { + *device = 0; + return atoi(getenv("SHADOW_CUDA_RESULT")); +} +''' + with tempfile.TemporaryDirectory(prefix="sircl-cuda-preload-") as directory: + path = Path(directory) + (path / "runtime.c").write_text(source) + subprocess.run(["cc", "-shared", "-fPIC", "-Wall", "-Wextra", "-Werror", + str(path / "runtime.c"), "-o", str(path / "runtime.so")], check=True, + capture_output=True, text=True, timeout=60) + inherited = os.environ.get("LD_PRELOAD", "") + for separator in (":", " "): + for status in ("0", "100"): + # Keep an inherited ASan runtime first, as required by check-asan. + preload = separator.join(filter(None, (inherited, str(path / "runtime.so")))) + with self.subTest(separator=separator, runtime_status=status), patch.dict( + os.environ, LD_PRELOAD=preload, SHADOW_CUDA_RESULT=status): + self.test_init_without_cuda_context_uses_the_only_devices_primary_context() + self.test_init_without_cuda_context_uses_the_runtimes_current_device() + self.test_init_without_cuda_context_is_refused_when_the_device_is_unknown() + def test_init_without_a_cuda_driver_is_refused_with_reason(self): out = run('''u=U(); assert l.ncclGetUniqueId(C.byref(u))==0 c=C.c_void_p(); r=l.ncclCommInitRank(C.byref(c),1,u,0) @@ -211,6 +560,8 @@ class U(C.Structure): _fields_ = [("b", C.c_ubyte * 128)] print(json.dumps({"before": before, "after": state()})) ''' environment = {k: v for k, v in os.environ.items() if k != "LD_PRELOAD" and not k.startswith("PYTHON")} + if os.environ.get("LIBSIRCL_TEST_ASAN"): + environment["LD_PRELOAD"] = os.environ["LD_PRELOAD"] out = subprocess.run([sys.executable, "-S", "-E", "-c", script, str(LIBRARY)], env=environment, capture_output=True, text=True, timeout=60) self.assertEqual(out.returncode, 0, out.stderr) @@ -241,4 +592,5 @@ def test_setup_checks_the_native_proxy_feature_word(self): parser.add_argument("--library", type=Path, default=LIBRARY) options, extra = parser.parse_known_args() LIBRARY = options.library.resolve() + FAKE_DRIVER = LIBRARY.parent / "fake-cuda" unittest.main(argv=[sys.argv[0], *extra], verbosity=2) diff --git a/spark_transport/libsircl/tests/test_kernelpack_cache.c b/spark_transport/libsircl/tests/test_kernelpack_cache.c new file mode 100644 index 00000000..14b9ad6c --- /dev/null +++ b/spark_transport/libsircl/tests/test_kernelpack_cache.c @@ -0,0 +1,67 @@ +/* The kernel-pack cache (src/kernelpack.c) over the stand-in driver tests/fake_cuda.c: an entry per context, + * held by references. Packs loaded and released in turn in 20 distinct contexts (more than the 16 entries the + * cache holds at once) all load, and each last release unloads the context's four modules; a reference still + * held keeps the entry; a context address reused after its last release loads the packs afresh. + * + * LD_LIBRARY_PATH= test_kernelpack_cache + */ +#define _POSIX_C_SOURCE 200809L +#include +#include +#include +#include + +#include "kernelpack.h" + +static unsigned checks; +#define CHECK(expr) do { checks++; if (!(expr)) { \ + fprintf(stderr, "%s:%d: %s (%s)\n", __FILE__, __LINE__, #expr, sccl_kp_error()); exit(1); } } while (0) + +static void (*stats)(int *, int *, int *); +static int live_modules(void) { + int modules = 0, cubins = 0, lookups = 0; + stats(&modules, &cubins, &lookups); + return modules; +} +static int loaded(sccl_CUcontext ctx) { + sccl_CUfunction function; + return sccl_kp_function(ctx, SCCL_ALG_ONESHOT, SCCL_DT_F32, 2, &function) == 0; +} + +int main(void) { + void *driver = dlopen("libcuda.so.1", RTLD_NOW | RTLD_LOCAL); + if (driver) *(void **)(&stats) = dlsym(driver, "sccl_fake_cuda_stats"); + if (!driver || !stats) { + fprintf(stderr, "test_kernelpack_cache: put the stand-in driver's directory first on LD_LIBRARY_PATH\n"); + return 2; + } + static int contexts[20]; + for (int i = 0; i < 20; ++i) { + sccl_CUcontext ctx = (sccl_CUcontext)(void *)&contexts[i]; + CHECK(sccl_kp_load(ctx) == 0); + CHECK(loaded(ctx) && live_modules() == 4); + sccl_kp_release(ctx, 1); + CHECK(!loaded(ctx) && live_modules() == 0); + } + /* Two references: the first release keeps the entry, the second removes it. */ + sccl_CUcontext a = (sccl_CUcontext)(void *)&contexts[0], b = (sccl_CUcontext)(void *)&contexts[1]; + CHECK(sccl_kp_load(a) == 0 && sccl_kp_load(b) == 0 && sccl_kp_load(a) == 0); + CHECK(live_modules() == 8); + sccl_kp_release(a, 1); + CHECK(loaded(a) && loaded(b) && live_modules() == 8); + sccl_kp_release(a, 1); + CHECK(!loaded(a) && loaded(b) && live_modules() == 4); + /* A reused address loads afresh; a release without unloading leaves the modules to the context. */ + CHECK(sccl_kp_load(a) == 0 && loaded(a) && live_modules() == 8); + sccl_kp_release(a, 0); + CHECK(!loaded(a) && live_modules() == 8); + sccl_kp_release(b, 1); + CHECK(!loaded(b)); + /* A pack the driver cannot load (no cubin for the GPU) is refused, naming the architectures it holds. */ + setenv("FAKE_CUDA_FAIL_LOAD", "1", 1); + CHECK(sccl_kp_load(a) != 0); + CHECK(strstr(sccl_kp_error(), "built for sm_120 sm_121") != NULL); + unsetenv("FAKE_CUDA_FAIL_LOAD"); + printf("kernel-pack cache: %u checks passed\n", checks); + return 0; +} diff --git a/spark_transport/libsircl/tests/test_lifecycle.py b/spark_transport/libsircl/tests/test_lifecycle.py index 48d4c9c3..d331dbbf 100644 --- a/spark_transport/libsircl/tests/test_lifecycle.py +++ b/spark_transport/libsircl/tests/test_lifecycle.py @@ -5,15 +5,19 @@ import argparse import concurrent.futures import ctypes as C +import json import os from pathlib import Path import subprocess +import struct import sys +import tempfile import threading import time import unittest import test_api as api +from test_bootstrap import raw_peer, raw_reply SUCCESS, ARGUMENT, USAGE, PROGRESS, TIMEOUT = 0, 4, 5, 7, 8 @@ -21,6 +25,168 @@ class LifecycleTests(unittest.TestCase): + def test_abort_cancels_an_inflight_split(self): + handles = (C.c_void_p * 2)() + self.assertEqual(self.lib.ncclCommInitAll(handles, 2, None), SUCCESS) + old = os.environ.get("LIBSIRCL_SETUP_TIMEOUT_MS") + os.environ["LIBSIRCL_SETUP_TIMEOUT_MS"] = "2000" + child = C.c_void_p() + try: + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool: + split = pool.submit(self.lib.ncclCommSplit, handles[0], 0, 0, C.byref(child), None) + time.sleep(0.05) + self.assertFalse(split.done()) + start = time.monotonic() + self.assertEqual(self.lib.ncclCommAbort(handles[0]), SUCCESS) + elapsed = time.monotonic() - start + handles[0] = None + self.assertNotEqual(split.result(timeout=1), SUCCESS) + self.assertLess(elapsed, 1) + finally: + if old is None: + os.environ.pop("LIBSIRCL_SETUP_TIMEOUT_MS", None) + else: + os.environ["LIBSIRCL_SETUP_TIMEOUT_MS"] = old + for handle in handles: + if handle: + self.lib.ncclCommAbort(handle) + + def test_lifecycle_call_on_an_initializing_communicator_is_refused(self): + """ncclCommRevoke or ncclCommFinalize on a communicator whose initialization is still in progress is + refused with ncclInvalidUsage. ncclInProgress never becomes a group's result, so a group's pending + initialization never ends FAILED with ncclInProgress, which would read as still in progress forever.""" + os.environ["SIRCL_BOOTSTRAP_TIMEOUT_MS"] = "5000" + config = api.config() + config.blocking = 0 + pending, grouped = C.c_void_p(), C.c_void_p() + first, second = self.uid(), self.uid() + self.assertEqual(self.lib.ncclCommInitRankConfig(C.byref(pending), 2, first, 0, C.byref(config)), PROGRESS) + try: + self.assertEqual(self.lib.ncclGroupStart(), SUCCESS) + self.assertEqual(self.lib.ncclCommInitRank(C.byref(grouped), 1, second, 0), SUCCESS) + self.assertEqual(self.lib.ncclCommRevoke(pending, 0), USAGE) + self.assertEqual(self.lib.ncclCommFinalize(pending), USAGE) + self.assertEqual(self.lib.ncclGroupEnd(), SUCCESS) + self.assertEqual(self.asynchronous_error(grouped), SUCCESS) + finally: + self.lib.ncclCommAbort(pending) + if grouped.value: + self.lib.ncclCommDestroy(grouped) + + def test_sixty_four_live_roots_are_the_limit_and_the_refusal_names_it(self): + """Each id a process creates keeps its bootstrap root while its communicator lives: with 64 live one-rank + communicators a 65th id is refused with ncclInvalidUsage naming the limit, and destroying one + communicator makes room for an id again.""" + script = r''' +import ctypes as C, json, os, sys, time +sys.path.insert(0, sys.argv[2]); import test_api as api +api.LIBRARY=api.Path(sys.argv[1]); api.initialize_library() +os.environ["LIBSIRCL_BOOTSTRAP_ONLY"]="1" +handles=[api.new_comm() for _ in range(64)] +u=api.UniqueId(); refused=api.LIB.ncclGetUniqueId(C.byref(u)); message=api.LIB.ncclGetLastError(None).decode() +assert api.LIB.ncclCommDestroy(handles.pop())==0 +time.sleep(0.2) +again=api.LIB.ncclGetUniqueId(C.byref(u)) +print(json.dumps([refused, message, again])) +''' + out = subprocess.run([sys.executable, "-c", script, str(LIBRARY), str(Path(__file__).parent)], + capture_output=True, text=True, timeout=30) + self.assertEqual(out.returncode, 0, out.stderr[-2000:]) + refused, message, again = json.loads(out.stdout) + self.assertEqual(refused, USAGE) + self.assertIn("already holds 64 bootstrap roots", message) + self.assertEqual(again, SUCCESS) + + def test_split_leader_failure_fails_every_member_at_once(self): + """The split leader cannot create the new unique id (the process holds all 64 bootstrap roots). It still + takes part in the unique-id round with a failure record, so both ranks return its result at once + instead of the member waiting out LIBSIRCL_SETUP_TIMEOUT_MS, and the parent's bootstrap stays usable.""" + script = r''' +import concurrent.futures, ctypes as C, json, os, sys, time +sys.path.insert(0, sys.argv[2]); import test_api as api +api.LIBRARY=api.Path(sys.argv[1]); api.initialize_library() +os.environ.update(LIBSIRCL_BOOTSTRAP_ONLY="1", LIBSIRCL_SETUP_TIMEOUT_MS="5000", SIRCL_BOOTSTRAP_TIMEOUT_MS="30000") +handles=(C.c_void_p*2)(); assert api.LIB.ncclCommInitAll(handles,2,None)==0 +held=[] +while True: + u=api.UniqueId() + if api.LIB.ncclGetUniqueId(C.byref(u))!=0: break + held.append(u) +children=[C.c_void_p(), C.c_void_p()] +def split(r): + code=api.LIB.ncclCommSplit(handles[r],0,r,C.byref(children[r]),None) + return code, api.LIB.ncclGetLastError(None).decode() +start=time.monotonic() +with concurrent.futures.ThreadPoolExecutor(2) as pool: + results=list(pool.map(split, range(2))) +elapsed=time.monotonic()-start +none=[C.c_void_p(), C.c_void_p()] +with concurrent.futures.ThreadPoolExecutor(2) as pool: + again=list(pool.map(lambda r: api.LIB.ncclCommSplit(handles[r],-1,0,C.byref(none[r]),None), range(2))) +print(json.dumps({"results":results,"elapsed":elapsed,"held":len(held),"children":[c.value for c in children], + "again":again})) +''' + out = subprocess.run([sys.executable, "-c", script, str(LIBRARY), str(Path(__file__).parent)], + capture_output=True, text=True, timeout=30) + self.assertEqual(out.returncode, 0, out.stderr) + data = json.loads(out.stdout) + self.assertEqual(data["held"], 63) + self.assertEqual([code for code, _ in data["results"]], [USAGE, USAGE], data) + self.assertIn("64 bootstrap roots", data["results"][0][1]) + self.assertIn("split leader (parent rank 0)", data["results"][1][1]) + self.assertLess(data["elapsed"], 2) + self.assertEqual(data["children"], [None, None]) + self.assertEqual(data["again"], [SUCCESS, SUCCESS]) + + def test_split_whose_peer_never_arrives_is_a_remote_error(self): + """A split whose peer never takes part times out in its first all-gather and returns ncclRemoteError, the + result the engine's setup exchanges give a peer that does not arrive, naming the timeout.""" + handles = (C.c_void_p * 2)() + self.assertEqual(self.lib.ncclCommInitAll(handles, 2, None), SUCCESS) + saved = os.environ.get("LIBSIRCL_SETUP_TIMEOUT_MS") + os.environ["LIBSIRCL_SETUP_TIMEOUT_MS"] = "300" + try: + child = C.c_void_p() + self.assertEqual(self.lib.ncclCommSplit(handles[0], 0, 0, C.byref(child), None), 6) + self.assertIn(b"timed out", self.lib.ncclGetLastError(None)) + finally: + if saved is None: + os.environ.pop("LIBSIRCL_SETUP_TIMEOUT_MS", None) + else: + os.environ["LIBSIRCL_SETUP_TIMEOUT_MS"] = saved + for handle in handles: + self.lib.ncclCommAbort(handle) + + def test_split_refuses_short_records_in_both_rounds(self): + for short_round in (0, 1): + with self.subTest(round=short_round): + value = self.uid() + peer = raw_peer(bytes(value), 2, 1) + handle, child = C.c_void_p(), C.c_void_p() + try: + self.assertEqual(self.lib.ncclCommInitRank(C.byref(handle), 2, value, 0), SUCCESS) + self.assertEqual(raw_reply(peer), SUCCESS) + def respond(): + for round_number in range(short_round + 1): + payload = b"\x00" if round_number == short_round else struct.pack("=ii", 0, 1) + peer.sendall(b"SCF1" + struct.pack("!III", round_number, len(payload), 0) + payload) + header = peer.recv(16, __import__("socket").MSG_WAITALL) + self.assertEqual(len(header), 16) + size = struct.unpack("!I", header[12:])[0] + self.assertEqual(len(peer.recv(size, __import__("socket").MSG_WAITALL)), size) + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool: + response = pool.submit(respond) + result = self.lib.ncclCommSplit(handle, 0, 0, C.byref(child), None) + response.result(timeout=2) + self.assertEqual(result, 6) + self.assertFalse(child.value) + finally: + peer.close() + if child.value: + self.lib.ncclCommAbort(child) + if handle.value: + self.lib.ncclCommAbort(handle) + @classmethod def setUpClass(cls): api.LIBRARY = LIBRARY @@ -124,8 +290,50 @@ def test_nested_error_propagates_to_pending_handles(self): self.assertEqual(self.lib.ncclGroupEnd(), SUCCESS) self.assertEqual(self.lib.ncclGroupEnd(), ARGUMENT) try: + # The group's end and the pending handle name the call that failed the group. + self.assertIn(b"group initialization failed: negative device number", self.lib.ncclGetLastError(None)) self.assertEqual(self.asynchronous_error(handle), ARGUMENT) - self.assertIn(b"group", self.lib.ncclGetLastError(handle)) + self.assertIn(b"group initialization rejected: negative device number", self.lib.ncclGetLastError(handle)) + finally: + self.dispose_all([handle]) + + def test_thread_exiting_inside_a_group_releases_its_pending_handles(self): + """A thread opens a group, initializes a communicator in it and exits without ncclGroupEnd: the + handle fails with ncclInvalidUsage, naming the reason, and another thread can destroy it.""" + value = self.uid() + handle = C.c_void_p() + codes = [] + + def grouped(): + codes.append(self.lib.ncclGroupStart()) + codes.append(self.lib.ncclCommInitRank(C.byref(handle), 1, value, 0)) + thread = threading.Thread(target=grouped) + thread.start() + thread.join(timeout=5) + self.assertEqual(codes, [SUCCESS, SUCCESS]) + self.assertTrue(handle.value) + self.assertEqual(self.asynchronous_error(handle), USAGE) + self.assertIn(b"exited before its ncclGroupEnd", self.lib.ncclGetLastError(handle)) + self.assertEqual(self.lib.ncclCommDestroy(handle), SUCCESS) + + def test_calls_that_enqueue_nothing_leave_the_group_as_it_was(self): + """Inside a group, failed calls that enqueue nothing (a query of an unknown handle, a NULL output to + ncclGetUniqueId, an unsupported function, revoke flags) return their errors and leave the group's + initialization and result alone.""" + value = self.uid() + handle = C.c_void_p() + count = C.c_int(-1) + self.lib.ncclCommShrink.argtypes = [C.c_void_p, C.c_void_p, C.c_int, C.POINTER(C.c_void_p), C.c_void_p, + C.c_int] + self.assertEqual(self.lib.ncclGroupStart(), SUCCESS) + self.assertEqual(self.lib.ncclCommInitRank(C.byref(handle), 1, value, 0), SUCCESS) + self.assertEqual(self.lib.ncclCommCount(C.c_void_p(16), C.byref(count)), ARGUMENT) + self.assertEqual(self.lib.ncclGetUniqueId(None), ARGUMENT) + self.assertEqual(self.lib.ncclCommShrink(handle, None, 0, None, None, 0), USAGE) + self.assertEqual(self.lib.ncclCommRevoke(handle, 1), ARGUMENT) + try: + self.assertEqual(self.lib.ncclGroupEnd(), SUCCESS) + self.assertEqual(self.asynchronous_error(handle), SUCCESS) finally: self.dispose_all([handle]) @@ -213,6 +421,11 @@ def test_incomplete_blocking_init_retains_failure_for_cleanup(self): self.assertTrue(handle.value) self.assertEqual(self.asynchronous_error(handle), TIMEOUT) self.assertIn(b"timed out", self.lib.ncclGetLastError(handle)) + # A query of the failed handle returns its result with its text for this thread. + self.assertEqual(self.lib.ncclGetVersion(None), ARGUMENT) + count = C.c_int(-1) + self.assertEqual(self.lib.ncclCommCount(handle, C.byref(count)), TIMEOUT) + self.assertIn(b"timed out", self.lib.ncclGetLastError(None)) finally: self.dispose_all([handle]) @@ -316,6 +529,59 @@ def test_repeated_init_destroy_releases_file_descriptors(self): subprocess.run([sys.executable, "-c", script, str(LIBRARY), str(Path(__file__).parent)], check=True, capture_output=True, text=True, timeout=5) + def test_unload_waits_for_a_call_still_inside_the_library(self): + """Another thread is inside ncclCommSplit, waiting for a peer, when the library unloads (dlclose, or + the destructors that exit runs). The unload cancels the call and waits for it to leave before freeing + the communicator, so the call returns an error instead of using freed memory or unmapped code. The + exit case is decisive under `make check-asan`, where a use of the freed communicator is reported. + + The parent communicator's bootstrap root is a stand-in broker in the child process that admits rank 0 + and never answers its all-gather, so the split waits in its socket poll until cancellation reaches + it. In the exit case a helper library, loaded after libsircl and finalized after it, keeps the + exiting process alive for 0.3 s after libsircl's destructor, long enough for the split thread to wake.""" + script = r''' +import _ctypes, ctypes as C, os, socket, struct, sys, threading, time +sys.path.insert(0, sys.argv[2]); import test_api as api +api.LIBRARY=api.Path(sys.argv[1]); api.initialize_library() +if sys.argv[3]=="exit": C.CDLL(sys.argv[4]) +os.environ["LIBSIRCL_BOOTSTRAP_ONLY"]="1"; os.environ["LIBSIRCL_SETUP_TIMEOUT_MS"]="5000" +server=socket.socket(); server.bind(("127.0.0.1",0)); server.listen() +peers=[] +def broker(): + peer,_=server.accept(); peers.append(peer) + hello=b"" + while len(hello)<136: hello+=peer.recv(136-len(hello)) + peer.sendall(b"SCC1"+struct.pack("!I",0)) +threading.Thread(target=broker, daemon=True).start() +u=api.UniqueId(); assert api.LIB.ncclGetUniqueId(C.byref(u))==0 +u.internal[16:18]=struct.pack("!H",server.getsockname()[1]) +handle=C.c_void_p(); assert api.LIB.ncclCommInitRank(C.byref(handle),2,u,0)==0 +child=C.c_void_p(); result=[] +def split(): result.append(api.LIB.ncclCommSplit(handle,0,0,C.byref(child),None)) +thread=threading.Thread(target=split, daemon=True); thread.start(); time.sleep(0.1) +assert not result +if sys.argv[3]=="exit": + sys.stdout.flush(); C.CDLL(None).exit(0) +_ctypes.dlclose(api.LIB._handle) +thread.join(2) +assert result and result[0]!=0, result +print("split returned", result[0]) +''' + with tempfile.TemporaryDirectory() as directory: + source, helper = Path(directory) / "linger.c", Path(directory) / "liblinger.so" + source.write_text("#include \n__attribute__((destructor)) static void linger(void) " + "{ usleep(300000); }\n") + subprocess.run([os.environ.get("CC", "cc"), "-shared", "-fPIC", str(source), "-o", str(helper)], + check=True) + for mode in ("dlclose", "exit"): + with self.subTest(mode=mode): + out = subprocess.run([sys.executable, "-c", script, str(LIBRARY), str(Path(__file__).parent), + mode, str(helper)], capture_output=True, text=True, timeout=10) + self.assertEqual(out.returncode, 0, out.stderr) + self.assertNotIn("AddressSanitizer", out.stderr) + if mode == "dlclose": + self.assertIn("split returned", out.stdout) + def test_dso_unload_joins_workers_and_closes_roots(self): script = r''' import _ctypes, ctypes as C, os, sys, time diff --git a/spark_transport/libsircl/tests/test_link_plan.c b/spark_transport/libsircl/tests/test_link_plan.c new file mode 100644 index 00000000..990cf9c2 --- /dev/null +++ b/spark_transport/libsircl/tests/test_link_plan.c @@ -0,0 +1,39 @@ +/* The large-reduction plan (src/link_plan.h): no link or chain op whose pack count does not fit the kernels' + * int32_t arguments is chosen; such a body runs in pieces. */ +#include "link_plan.h" + +#include +#include + +static unsigned checks; +#define CHECK(expr) do { checks++; if (!(expr)) { \ + fprintf(stderr, "%s:%d: %s\n", __FILE__, __LINE__, #expr); exit(1); } } while (0) + +int main(void) { + const uint64_t limit = (uint64_t)INT32_MAX * SCCL_PLAN_PACK; /* the largest chain op: 32 GiB - 16 bytes */ + uint64_t bytes; + CHECK(sccl_packs_fit(limit)); + CHECK(!sccl_packs_fit(limit + SCCL_PLAN_PACK)); + CHECK(!sccl_packs_fit(32ull << 30)); + + /* The chain: up to the limit as one op; from 32 GiB in pieces, never a wrapped pack count. */ + CHECK(sccl_large_reduce_plan(8u << 20, 2, 0, 1, 1, &bytes) == SCCL_LARGE_CHAIN && bytes == 8u << 20); + CHECK(sccl_large_reduce_plan(limit, 2, 0, 1, 1, &bytes) == SCCL_LARGE_CHAIN && bytes == limit); + CHECK(sccl_large_reduce_plan(32ull << 30, 2, 0, 1, 1, &bytes) == SCCL_LARGE_PIECES && bytes == 0); + CHECK(sccl_large_reduce_plan(64ull << 30, 4, 0, 1, 1, &bytes) == SCCL_LARGE_PIECES && bytes == 0); + CHECK(sccl_large_reduce_plan(8u << 20, 2, 0, 1, 0, &bytes) == SCCL_LARGE_PIECES); + + /* The ring: the largest prefix of W equal chunks of whole packs, each below 2^31 bytes. */ + CHECK(sccl_large_reduce_plan((8u << 20) + 48, 4, 1, 1, 0, &bytes) == SCCL_LARGE_RING && bytes == 8u << 20); + CHECK(sccl_large_reduce_plan(8ull << 30, 4, 1, 1, 0, &bytes) == SCCL_LARGE_PIECES); + CHECK(sccl_large_reduce_plan((8ull << 30) - 64, 4, 1, 1, 0, &bytes) == SCCL_LARGE_RING && + bytes == (8ull << 30) - 64); + CHECK(sccl_large_reduce_plan(48, 4, 1, 1, 0, &bytes) == SCCL_LARGE_PIECES); + CHECK(sccl_large_reduce_plan(8u << 20, 4, 1, 0, 1, &bytes) == SCCL_LARGE_PIECES); + + /* One rank or an empty body: nothing to plan. */ + CHECK(sccl_large_reduce_plan(8u << 20, 1, 0, 1, 1, &bytes) == SCCL_LARGE_PIECES); + CHECK(sccl_large_reduce_plan(0, 2, 0, 1, 1, &bytes) == SCCL_LARGE_PIECES); + printf("link plan: %u checks passed\n", checks); + return 0; +} diff --git a/spark_transport/libsircl/tests/test_site_routes.py b/spark_transport/libsircl/tests/test_site_routes.py index 29f87529..2b994684 100644 --- a/spark_transport/libsircl/tests/test_site_routes.py +++ b/spark_transport/libsircl/tests/test_site_routes.py @@ -162,6 +162,47 @@ def test_session_shares_outside_zero_to_one_are_refused(self): self.assertEqual(process.returncode, 2) self.assertIn("--session-share", process.stderr) + def test_relayed_lanes_are_printed_without_any_window(self): + """With --max-window 0 no forward or point-to-point window is printed, and every rank with a relayed + lane still gets LIBSIRCL_RELAYED_LANES, marking exactly the lanes SIRCL's routes send through relays.""" + result = plan("--layout", "path:0-3", "--lanes", "2", "--max-window", "0") + group = self.routes.derive_routes(self.routes.Layout.parse("path:0-3"), 2) + for row in result["ranks"]: + rank, env = row["rank"], row["env"] + self.assertNotIn("LIBSIRCL_FORWARD_WINDOWS", env) + self.assertNotIn("LIBSIRCL_P2P_WINDOWS", env) + want = {peer: [1 if lane.relays else 0 for lane in group.lanes_to(rank, peer)] + for peer in range(4) if peer != rank} + want = {peer: flags for peer, flags in want.items() if any(flags)} + self.assertTrue(want, rank) + self.assertEqual(windows(env.get("LIBSIRCL_RELAYED_LANES", "")), want, rank) + + def test_windows_keep_to_the_native_caps(self): + """With small chunks SIRCL's planner gives windows of more chunks than the native layers accept; the + tool lowers them to the caps: forward windows to 60 chunks, point-to-point windows to 120.""" + def printed(result, name): + return [value for row in result["ranks"] for lanes in windows(row["env"].get(name, "")).values() + for value in lanes if value] + forward = printed(plan("--layout", "ring:8", "--lanes", "2", "--chunk", "1024"), "LIBSIRCL_FORWARD_WINDOWS") + self.assertTrue(forward) + self.assertLessEqual(max(forward), 60 * 1024) + self.assertTrue(all(value % 1024 == 0 for value in forward)) + p2p = printed(plan("--layout", "ring:8", "--lanes", "2", "--p2p-chunk", "1024", "--p2p-reserve", "none"), + "LIBSIRCL_P2P_WINDOWS") + self.assertTrue(p2p) + self.assertLessEqual(max(p2p), 120 * 1024) + + def test_a_plan_that_overfills_a_relay_queue_is_refused(self): + """SIRCL gives every relayed lane at least one chunk; with 64 KiB chunks on path:0-7 the lanes through + one relay queue then hold more than its share, and the tool refuses the plan, naming the queue.""" + process = subprocess.run([sys.executable, str(ROOT / "tools" / "site_routes.py"), "--layout", "path:0-7", + "--lanes", "2", "--chunk", "65536"], + env=dict(os.environ, PYTHONPATH=PACKAGE), capture_output=True, text=True, + timeout=120) + self.assertEqual(process.returncode, 1, process.stdout) + self.assertIn("above its share of 393216", process.stderr) + self.assertEqual(process.stdout, "") + def test_the_data_files_are_the_tools_output(self): for name, arguments in (("site_routes_ring8_l2.json", ("--layout", "ring:8")), ("site_routes_ring8_l2_p2p_alone.json", ("--layout", "ring:8", "--p2p-reserve", "none")), diff --git a/spark_transport/libsircl/tests/test_source_snapshot.py b/spark_transport/libsircl/tests/test_source_snapshot.py new file mode 100644 index 00000000..36efc6d6 --- /dev/null +++ b/spark_transport/libsircl/tests/test_source_snapshot.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +"""SOURCE_SNAPSHOT.json's schema, and what NOTICE says the file records. + +SOURCE_SNAPSHOT.json is a dated capture: the SHA-256 and size of every file of the SIRCL implementation tree +that STATUS.md's evidence was taken against. It is not the in-repository SIRCL package the library is built +against, and nothing enforces it at build time. These checks keep its shape readable (every entry a relative +path with a 64-digit lowercase SHA-256 and a byte count; the stated file count; the changes from the previous +capture as path lists) and keep NOTICE describing it as that capture. +""" +import json +import re +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +class SourceSnapshotTests(unittest.TestCase): + def setUp(self): + self.snapshot = json.loads((ROOT / "SOURCE_SNAPSHOT.json").read_text(encoding="utf-8")) + + def test_the_schema(self): + data = self.snapshot + self.assertEqual(set(data), {"description", "sources", "selected", "captured_utc", + "changes_from_previous_capture", "files"}) + self.assertIn(data["selected"], data["sources"]) + self.assertRegex(data["captured_utc"], r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$") + stated = re.search(r"(\d+) files\.?$", data["description"]) + self.assertIsNotNone(stated, "the description states the file count") + self.assertEqual(len(data["files"]), int(stated.group(1))) + for path, entry in data["files"].items(): + with self.subTest(path=path): + self.assertFalse(path.startswith("/") or ".." in Path(path).parts or "\\" in path) + self.assertEqual(set(entry), {"sha256", "bytes"}) + self.assertRegex(entry["sha256"], r"^[0-9a-f]{64}$") + self.assertIsInstance(entry["bytes"], int) + self.assertGreaterEqual(entry["bytes"], 0) + changes = data["changes_from_previous_capture"] + self.assertEqual(set(changes), {"changed", "added", "removed"}) + for kind, paths in changes.items(): + self.assertIsInstance(paths, list, kind) + self.assertTrue(all(isinstance(path, str) and path for path in paths), kind) + + def test_notice_describes_the_capture(self): + notice = " ".join((ROOT / "NOTICE").read_text(encoding="utf-8").split()) + self.assertIn("SOURCE_SNAPSHOT.json records the SHA-256 of every file of the SIRCL implementation tree " + "that STATUS.md's evidence was taken against", notice) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/spark_transport/libsircl/tests/test_teardown_race.py b/spark_transport/libsircl/tests/test_teardown_race.py new file mode 100644 index 00000000..ae597377 --- /dev/null +++ b/spark_transport/libsircl/tests/test_teardown_race.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +"""CPU checks of the teardown harness's verdicts, thread failures and process deadline.""" +from concurrent.futures import ThreadPoolExecutor +import contextlib +import io +import json +from pathlib import Path +import subprocess +import tempfile +import threading +import unittest +from unittest.mock import patch + +from emulation import teardown_race as race + + +class ConcurrentCloseTests(unittest.TestCase): + def run_ranks(self, accepted, close_codes, *, after_close=4, worker_code=5): + """Four ranks enqueue independently; a barrier publishes their actual accepted counts.""" + barrier = threading.Barrier(len(accepted), timeout=5) + counts = [None] * len(accepted) + + def rank_run(rank): + ready, closing, waiting = threading.Event(), threading.Event(), threading.Event() + posted, destroyed = 0, False + + def enqueue(): + nonlocal posted + if destroyed: + return after_close + if posted < accepted[rank]: + posted += 1 + if posted == accepted[rank]: + ready.set() + return 0 + waiting.set() + if not closing.wait(5): + raise TimeoutError("test close did not begin") + return worker_code + + def finalize(): + self.assertTrue(ready.wait(5)) + self.assertTrue(waiting.wait(5)) + closing.set() + return close_codes[0] + + def destroy(): + nonlocal destroyed + destroyed = True + return close_codes[1] + + def exchange(count): + counts[rank] = count + barrier.wait() + return list(counts) + + return race.close_concurrently(enqueue, finalize, destroy, lambda: None, exchange) + + with ThreadPoolExecutor(max_workers=len(accepted)) as pool: + return list(pool.map(rank_run, range(len(accepted)))) + + def test_equal_counts_require_success_from_both_close_calls(self): + for codes in ((0, 0), (6, 0), (0, 6), (6, 6)): + with self.subTest(codes=codes): + entries = self.run_ranks([2, 2, 2, 2], codes) + self.assertTrue(all(entry["ok"] == (codes == (0, 0)) for entry in entries)) + + def test_unequal_counts_allow_only_success_or_remote_error(self): + for codes in ((0, 0), (6, 0), (0, 6), (6, 6), (2, 6), (6, 5)): + with self.subTest(codes=codes): + entries = self.run_ranks([3, 19, 2, 4], codes) + self.assertTrue(all(entry["accepted_counts"] == [3, 19, 2, 4] for entry in entries)) + self.assertTrue(all(entry["ok"] == all(code in (0, 6) for code in codes) + for entry in entries)) + + def test_post_close_calls_must_be_refused(self): + for result in (0, 4, 5, 6): + with self.subTest(result=result): + entries = self.run_ranks([2, 2, 2, 2], (0, 0), after_close=result) + self.assertTrue(all(entry["ok"] == (result in (4, 5)) for entry in entries)) + + def test_worker_error_is_not_excused_by_unequal_counts(self): + entries = self.run_ranks([3, 19, 2, 4], (6, 6), worker_code=6) + self.assertTrue(all(not entry["ok"] for entry in entries)) + + def test_first_call_must_succeed_before_close_starts(self): + entry = race.close_concurrently(lambda: 5, lambda: 0, lambda: 0, lambda: None, lambda _: [0, 0]) + self.assertFalse(entry["ok"]) + + def test_worker_exception_fails_even_if_close_succeeds(self): + def prepare(): + raise RuntimeError("thread setup failed") + + entry = race.close_concurrently(lambda: 4, lambda: 0, lambda: 0, prepare, lambda _: [0, 0]) + self.assertFalse(entry["ok"]) + self.assertEqual(entry["worker_errors"], ["RuntimeError: thread setup failed"]) + + def test_stuck_worker_is_a_failure(self): + release = threading.Event() + exited = threading.Event() + calls = 0 + + def enqueue(): + nonlocal calls + calls += 1 + if calls == 1: + return 0 + release.wait(5) + exited.set() + return 5 + + try: + with self.assertRaisesRegex(TimeoutError, "did not exit"): + race.close_concurrently(enqueue, lambda: 0, lambda: 0, lambda: None, + lambda _: self.fail("must not exchange unstable counts"), timeout=0.01) + finally: + release.set() + self.assertTrue(exited.wait(5)) + + def test_failed_process_cannot_pass_with_successful_json(self): + class Process: + pid = 99999999 + + def __init__(self, timeout): + self.timeout = timeout + self.killed = False + + def wait(self, timeout=None): + if self.timeout and not self.killed: + raise subprocess.TimeoutExpired("rank", timeout) + return -9 if self.killed else 1 + + def kill(self): + self.killed = True + + for timeout in (False, True): + with self.subTest(timeout=timeout), tempfile.TemporaryDirectory() as work: + Path(work, "rank0.json").write_text(json.dumps({ + "rounds": [{"round": 0, "ok": True}], + "parent": {"exact": True, "async_error": 0}, "parent_destroy": 0, + })) + process = Process(timeout) + with patch.object(race.subprocess, "Popen", return_value=process), \ + contextlib.redirect_stdout(io.StringIO()) as output: + code = race.main(["--library", "unused.so", "--world", "1", "--rounds", "1", + "--expect", "concurrent-close", "--work", work]) + self.assertEqual(code, 1) + self.assertIn("rank 0: exit", output.getvalue()) + self.assertEqual(process.killed, timeout) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/spark_transport/libsircl/tools/generate_api.py b/spark_transport/libsircl/tools/generate_api.py index 3dc63235..3ebba3d6 100644 --- a/spark_transport/libsircl/tools/generate_api.py +++ b/spark_transport/libsircl/tools/generate_api.py @@ -68,12 +68,27 @@ def signature(api: dict, name: str | None = None) -> str: return f"{api['return']} {name or api['name']}({arguments})" -# Outputs that an unimplemented function sets before it returns ncclInvalidUsage, by function: the argument -# and its statement. ncclRedOpCreatePreMulSum returns ncclNumOps, the first value that is not a built-in op, -# so a caller that ignores the refusal and passes the op to a collective gets ncclInvalidArgument from the -# collective, never a plain sum from a handle left at 0 (ncclSum). +# Outputs that an unimplemented function sets before it returns ncclInvalidUsage, by function and argument: +# the statement. Handle, value and length outputs are zeroed (a NULL communicator or parameter handle, a zero +# value, an all-zero unique id, which every init refuses), so a caller that ignores the refusal never reads +# an uninitialized output. ncclRedOpCreatePreMulSum returns ncclNumOps, the first value that is not a +# built-in op, so a caller that ignores the refusal and passes the op to a collective gets +# ncclInvalidArgument from the collective, never a plain sum from a handle left at 0 (ncclSum). Buffers the +# caller sizes (ncclParamGet's `out`, ncclGroupSimulateEnd's `simInfo`) are left as they are. +def zeroed(name: str) -> str: + return f" if ({name}) memset({name}, 0, sizeof(*{name}));\n" + + +ZEROED_OUTPUTS = { + "ncclCommShrink": ["newcomm"], "ncclCommGrow": ["newcomm"], "ncclCommInitRankScalable": ["newcomm"], + "ncclCommGetUniqueId": ["uniqueId"], "ncclCommMemStats": ["value"], "ncclParamBind": ["out"], + **{f"ncclParamGet{kind}": ["out"] for kind in ("I8", "I16", "I32", "I64", "U8", "U16", "U32", "U64", "Str")}, + "ncclParamGet": ["len"], "ncclParamGetParameter": ["value", "valueLen"], + "ncclParamGetAllParameterKeys": ["table", "tableLen"], +} REFUSED_OUTPUTS = { - "ncclRedOpCreatePreMulSum": ("op", " if (op) *op = (ncclRedOp_t)ncclNumOps;\n"), + **{function: {name: zeroed(name) for name in names} for function, names in ZEROED_OUTPUTS.items()}, + "ncclRedOpCreatePreMulSum": {"op": " if (op) *op = (ncclRedOp_t)ncclNumOps;\n"}, } @@ -107,17 +122,17 @@ def artifacts() -> dict[Path, str]: ) + header[cuda_end:] generated = ("/* Generated by tools/generate_api.py from vendor/nccl.h.in (NVIDIA NCCL v2.32.3-1, Copyright (c)\n" " * 2015-2026 NVIDIA CORPORATION & AFFILIATES, Apache-2.0; see vendor/NCCL-LICENSE.txt). Do not edit. */\n") - stubs = generated + '#include "internal.h"\n\n' + stubs = generated + '#include "internal.h"\n\n#include \n\n' aliases = generated + '#include "internal.h"\n\n' for api in apis: if not api["implemented"]: stubs += signature(api) + " {\n" - output = REFUSED_OUTPUTS.get(api["name"]) + outputs = REFUSED_OUTPUTS.get(api["name"], {}) + unknown = set(outputs) - {argument["name"] for argument in api["arguments"]} + if unknown: + raise ValueError(f"{api['name']} has no argument {sorted(unknown)}") for argument in api["arguments"]: - if output and argument["name"] == output[0]: - stubs += output[1] - else: - stubs += f" (void){argument['name']};\n" + stubs += outputs.get(argument["name"], f" (void){argument['name']};\n") if api["return"] == "void": stubs += f" (void)sccl_unsupported(\"{api['name']}\");\n" else: diff --git a/spark_transport/libsircl/tools/site_routes.py b/spark_transport/libsircl/tools/site_routes.py index 88bee7ec..c141a9e0 100644 --- a/spark_transport/libsircl/tools/site_routes.py +++ b/spark_transport/libsircl/tools/site_routes.py @@ -9,8 +9,13 @@ ``allocate``) the window of every relayed lane of a point-to-point channel. This tool prints them as libsircl's environment, one line per rank (rank r of the layout runs at libsircl position r): - LIBSIRCL_POSITION=r SIRCL_PEER_ROUTES=... LIBSIRCL_CHAIN_ORDER=... LIBSIRCL_FORWARD_WINDOWS=... - LIBSIRCL_RING_WINDOW=... LIBSIRCL_P2P_WINDOWS=... + LIBSIRCL_POSITION=r SIRCL_PEER_ROUTES=... LIBSIRCL_CHAIN_ORDER=... LIBSIRCL_RELAYED_LANES=... + LIBSIRCL_FORWARD_WINDOWS=... LIBSIRCL_RING_WINDOW=... LIBSIRCL_P2P_WINDOWS=... + +LIBSIRCL_RELAYED_LANES appears on every rank with a lane that crosses relays: per peer position, 1 for each +such lane and 0 for a cable. libsircl takes a lane's relay status from it, not from whether a window was +printed, so a relayed lane without a forward window (``--max-window 0``) still counts as relayed: the +point-to-point channels refuse it unwindowed, and the cycle plan does not run across it. LIBSIRCL_RING_WINDOW appears on every rank when the ring can run (no relay hairpin queue carries two of its lanes): the bytes each of the rank's lanes toward the next rank of the ring keeps unacknowledged @@ -38,6 +43,16 @@ least one 32 KiB chunk from every relayed session lane (at 0.95: 32 to 96 KiB) and leaves every one of the 40 relayed ordered pairs a 32 KiB window; ``--max-window 32768`` (every session lane one chunk) does too. +Every window printed fits the native layers' caps: a forward window holds at most 60 chunks of +SIRCL_FORWARD_CHUNK_BYTES, the ring window at most 60 chunks of 32 KiB, and a point-to-point window at most +120 chunks of SIRCL_P2P_CHUNK_BYTES; a larger planned window is lowered to the cap. The tool then checks every +relay hairpin queue: the session's windows through it (its forward windows, and with ``--ring-schedules`` +the larger of those and its ring windows) plus, under ``--p2p-reserve session``, the channels' windows must +fit the queue's share (75 % of SIRCL_HAIRPIN_QUEUE_BYTES). A plan that overfills a queue, such as SIRCL's +one-chunk minimum with ``--chunk 65536`` on path:0-7, is refused (exit status 1, naming the queue). Under +``--p2p-reserve none`` the channels are sized as the only relayed traffic, so only the session's windows are +checked. + Limit: the windows are one plan per process, as libsircl reads them; communicators that run at once through the same relay queues are not budgeted against each other. @@ -51,6 +66,12 @@ import json import sys +# The native layers' window caps: SIRCL's native proxy accepts a forward or ring window of 1 to 60 chunks +# (ROCE_CREDIT - 4), its point-to-point library 1 to 120 chunks. +FORWARD_MAX_CHUNKS = 60 +RING_MAX_CHUNKS = 60 +P2P_MAX_CHUNKS = 120 + def share_value(text: str) -> float: value = float(text) @@ -96,6 +117,7 @@ def main(argv: list[str] | None = None) -> int: from sparkring_sircl import protocol ring_window, problems = routes.ring_window(layout, maps, order, chunk=protocol.LINK_WINDOW_CHUNK, queue_bytes=session_queue) + ring_window = min(ring_window, RING_MAX_CHUNKS * protocol.LINK_WINDOW_CHUNK) relayed = {rank for members in routes.ring_queues(layout, maps, order).values() for rank, _, _ in members} # Point-to-point windows of every ordered pair's relayed lanes (SIRCL's p2p/budget.py). p2p_max = p2p_protocol.DEFAULT_WINDOW_BYTES if args.p2p_max_window is None else args.p2p_max_window @@ -112,16 +134,45 @@ def main(argv: list[str] | None = None) -> int: p2p_unavailable = allocation.unavailable_pairs("layout") relayed_pairs = sum(1 for a in range(world) for b in range(world) if a != b and any(lane.relays for lane in group.lanes_to(a, b))) - rows = [] + tables = [] for rank in range(world): table = routes.forward_windows(layout, maps, rank, max_window=max_window, chunk=chunk, queue_bytes=session_queue) + tables.append([[min(window, FORWARD_MAX_CHUNKS * chunk) for window in lanes] for lanes in table]) + for peer in range(world): + if peer != rank: + p2p_table[rank][peer] = [min(window, P2P_MAX_CHUNKS * p2p_chunk) for window in p2p_table[rank][peer]] + # Every relay hairpin queue within its share: the session's windows through it, and the channels'. + share = int(routes.RELAY_QUEUE_SHARE * routes.DEFAULT_HAIRPIN_QUEUE) + ring_lanes = {} + if order is not None and args.ring_schedules and not problems: + ring_lanes = routes.ring_queues(layout, maps, order) + for key, members in sorted(routes.relay_queues(layout, maps).items()): + held = sum(tables[rank][peer][lane] for rank, peer, lane in members) + if key in ring_lanes: + held = max(held, sum(ring_window if rank in relayed else 0 for rank, _, _ in ring_lanes[key])) + if args.p2p_reserve == "session": + held += sum(p2p_table[rank][peer][lane] for rank, peer, lane in members) + if held > share: + relay, secondary, port = key + print(f"site_routes.py: the relay queue of position {relay} ({'secondary' if secondary else 'primary'} " + f"function, toward port {port}) would hold {held} bytes of windows, above its share of {share}; " + f"choose a smaller --chunk, --max-window or --session-share", file=sys.stderr) + return 1 + rows = [] + for rank in range(world): + table = tables[rank] env = { "LIBSIRCL_POSITION": str(rank), "SIRCL_PEER_ROUTES": ",".join(f"{peer}={'/'.join(devices)}" for peer, devices in sorted(maps[rank].items())), } if order is not None: env["LIBSIRCL_CHAIN_ORDER"] = ",".join(str(r) for r in order) + relays = {peer: [1 if lane.relays else 0 for lane in group.lanes_to(rank, peer)] + for peer in range(world) if peer != rank} + if any(any(flags) for flags in relays.values()): + env["LIBSIRCL_RELAYED_LANES"] = ",".join(f"{peer}={'/'.join(str(f) for f in flags)}" + for peer, flags in relays.items() if any(flags)) windows = [f"{peer}={'/'.join(str(w) for w in table[peer])}" for peer in range(world) if peer != rank and any(table[peer])] if windows: