From b5783ce2f9d9ab6725d8114f9bb356854bf2d674 Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Wed, 14 Oct 2026 23:10:23 -0500 Subject: [PATCH 01/65] Reject malformed split records before indexing them Both split rounds validate every contribution before fixed-stride reads or unique-id copies. Compatibility: Malformed peer records return ncclRemoteError; valid splits retain their wire format. Validation: Both short-frame regressions failed before the checks; all 15 lifecycle CPU tests pass. Request item: 1. NVIDIA NCCL source was neither read nor adapted. --- spark_transport/libsircl/src/api.c | 14 ++++++++ .../libsircl/tests/test_lifecycle.py | 32 +++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/spark_transport/libsircl/src/api.c b/spark_transport/libsircl/src/api.c index c7f504c6..491f5752 100644 --- a/spark_transport/libsircl/src/api.c +++ b/spark_transport/libsircl/src/api.c @@ -357,6 +357,13 @@ ncclResult_t ncclCommSplit(ncclComm_t handle, int color, int key, ncclComm_t *ne release(parent); return creation_failed(ncclSystemError, message); } + for (int r = 0; r < parent->nranks; ++r) { + if (lengths[r] != sizeof mine) { + free(all); + release(parent); + return creation_failed(ncclRemoteError, "ncclCommSplit: invalid color record length"); + } + } const int32_t *records = all; int members[64], count = 0, rank = -1; for (int r = 0; r < parent->nranks; ++r) @@ -386,6 +393,13 @@ ncclResult_t ncclCommSplit(ncclComm_t handle, int color, int key, ncclComm_t *ne release(parent); return creation_failed(ncclSystemError, message); } + for (int r = 0; r < parent->nranks; ++r) { + if (lengths[r] != sizeof id.internal) { + free(all); + release(parent); + return creation_failed(ncclRemoteError, "ncclCommSplit: invalid unique-id record length"); + } + } if (rank >= 0) memcpy(id.internal, (const unsigned char *)all + 128 * (size_t)members[0], 128); free(all); int position = parent->position, blocking = parent->blocking; diff --git a/spark_transport/libsircl/tests/test_lifecycle.py b/spark_transport/libsircl/tests/test_lifecycle.py index 48d4c9c3..5bbfb09a 100644 --- a/spark_transport/libsircl/tests/test_lifecycle.py +++ b/spark_transport/libsircl/tests/test_lifecycle.py @@ -8,12 +8,14 @@ import os from pathlib import Path import subprocess +import struct import sys import threading import time import unittest import test_api as api +from test_bootstrap import raw_peer, raw_reply SUCCESS, ARGUMENT, USAGE, PROGRESS, TIMEOUT = 0, 4, 5, 7, 8 @@ -21,6 +23,36 @@ class LifecycleTests(unittest.TestCase): + def test_split_refuses_short_records_in_both_rounds(self): + for short_round in (0, 1): + with self.subTest(round=short_round): + value = self.uid() + peer = raw_peer(bytes(value), 2, 1) + handle, child = C.c_void_p(), C.c_void_p() + try: + self.assertEqual(self.lib.ncclCommInitRank(C.byref(handle), 2, value, 0), SUCCESS) + self.assertEqual(raw_reply(peer), SUCCESS) + def respond(): + for round_number in range(short_round + 1): + payload = b"\x00" if round_number == short_round else struct.pack("=ii", 0, 1) + peer.sendall(b"SCF1" + struct.pack("!III", round_number, len(payload), 0) + payload) + header = peer.recv(16, __import__("socket").MSG_WAITALL) + self.assertEqual(len(header), 16) + size = struct.unpack("!I", header[12:])[0] + self.assertEqual(len(peer.recv(size, __import__("socket").MSG_WAITALL)), size) + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool: + response = pool.submit(respond) + result = self.lib.ncclCommSplit(handle, 0, 0, C.byref(child), None) + response.result(timeout=2) + self.assertEqual(result, 6) + self.assertFalse(child.value) + finally: + peer.close() + if child.value: + self.lib.ncclCommAbort(child) + if handle.value: + self.lib.ncclCommAbort(handle) + @classmethod def setUpClass(cls): api.LIBRARY = LIBRARY From b00bd146d48011df8d2184dd3707f3aef455101a Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Wed, 14 Oct 2026 23:10:47 -0500 Subject: [PATCH 02/65] Cancel bootstrap work before abort waits for references Abort publishes cancellation before waiting for acquired calls; teardown rounds observe the same flag. Compatibility: Abort interrupts split promptly; normal destroy retains its orderly teardown rounds. Validation: The split/abort test failed at 1.95 seconds before the fix; all 16 lifecycle CPU tests pass. Request item: 6. NVIDIA NCCL source was neither read nor adapted. --- spark_transport/libsircl/src/api.c | 6 ++--- .../libsircl/tests/test_lifecycle.py | 26 +++++++++++++++++++ 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/spark_transport/libsircl/src/api.c b/spark_transport/libsircl/src/api.c index 491f5752..ee4a01d8 100644 --- a/spark_transport/libsircl/src/api.c +++ b/spark_transport/libsircl/src/api.c @@ -872,7 +872,6 @@ static ncclResult_t quiesce(struct ncclComm *comm, const char *call) { snprintf(message, sizeof message, "%s: waiting for this rank's enqueued work failed", call); } if (result == ncclSuccess) result = sccl_engine_async_error(comm->engine, message, sizeof message); - const atomic_int never = 0; int limit_ms = sccl_engine_wait_limit_ms(comm->engine), arrived = 0; for (int round = 1; round <= 2; ++round) { if (round == 2) sccl_engine_teardown_stop(comm->engine); @@ -881,7 +880,7 @@ static ncclResult_t quiesce(struct ncclComm *comm, const char *call) { void *all = NULL; unsigned lengths[64] = {0}; arrived = sccl_bootstrap_allgather_within(comm->bootstrap, &mine, sizeof mine, &all, lengths, limit_ms, - &never) == 0; + &comm->cancelled) == 0; if (arrived && round == 1 && result == ncclSuccess) { for (int r = 0; r < comm->nranks; ++r) { int32_t theirs = 0; @@ -950,9 +949,10 @@ static ncclResult_t dispose(ncclComm_t handle, int abort) { return fail(ncclInvalidUsage, "communicator initialization is pending in a group"); } *link = comm->next; + if (abort) atomic_store(&comm->cancelled, 1); while (comm->refs) pthread_cond_wait(®istry_idle, ®istry_lock); pthread_mutex_unlock(®istry_lock); - atomic_store(&comm->cancelled, 1); + if (atomic_load(&comm->state) == INIT) atomic_store(&comm->cancelled, 1); if (comm->worker_started) pthread_join(comm->worker, NULL); ncclResult_t result = abort ? ncclSuccess : quiesce(comm, "ncclCommDestroy"); char message[sizeof comm->error_message]; diff --git a/spark_transport/libsircl/tests/test_lifecycle.py b/spark_transport/libsircl/tests/test_lifecycle.py index 5bbfb09a..ff99c9ad 100644 --- a/spark_transport/libsircl/tests/test_lifecycle.py +++ b/spark_transport/libsircl/tests/test_lifecycle.py @@ -23,6 +23,32 @@ class LifecycleTests(unittest.TestCase): + def test_abort_cancels_an_inflight_split(self): + handles = (C.c_void_p * 2)() + self.assertEqual(self.lib.ncclCommInitAll(handles, 2, None), SUCCESS) + old = os.environ.get("LIBSIRCL_SETUP_TIMEOUT_MS") + os.environ["LIBSIRCL_SETUP_TIMEOUT_MS"] = "2000" + child = C.c_void_p() + try: + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool: + split = pool.submit(self.lib.ncclCommSplit, handles[0], 0, 0, C.byref(child), None) + time.sleep(0.05) + self.assertFalse(split.done()) + start = time.monotonic() + self.assertEqual(self.lib.ncclCommAbort(handles[0]), SUCCESS) + elapsed = time.monotonic() - start + handles[0] = None + self.assertNotEqual(split.result(timeout=1), SUCCESS) + self.assertLess(elapsed, 1) + finally: + if old is None: + os.environ.pop("LIBSIRCL_SETUP_TIMEOUT_MS", None) + else: + os.environ["LIBSIRCL_SETUP_TIMEOUT_MS"] = old + for handle in handles: + if handle: + self.lib.ncclCommAbort(handle) + def test_split_refuses_short_records_in_both_rounds(self): for short_round in (0, 1): with self.subTest(round=short_round): From eefb074dbb1361501d7c6ffccbccb5e05702e8a2 Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Thu, 15 Oct 2026 07:46:34 -0500 Subject: [PATCH 03/65] Reject peer positions outside the route table Setup reports invalid positions through the common local verdict before route indexing or shifts. Compatibility: Invalid positions fail setup; valid position records retain their layout. Validation: The CPU socket-peer test failed for -1, 64 and INT_MAX before the check; all 13 engine CPU tests pass (one host-dependent skip). Request item: 2. NVIDIA NCCL source was neither read nor adapted. --- spark_transport/libsircl/src/engine.c | 4 ++ .../libsircl/tests/test_engine_cpu.py | 40 +++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/spark_transport/libsircl/src/engine.c b/spark_transport/libsircl/src/engine.c index e6800be2..4b28fdda 100644 --- a/spark_transport/libsircl/src/engine.c +++ b/spark_transport/libsircl/src/engine.c @@ -2064,6 +2064,10 @@ int sccl_engine_create(sccl_bootstrap *bootstrap, int nranks, int rank, int posi return ncclRemoteError; } for (int r = 0; r < nranks; ++r) { + if (all[r].position < 0 || all[r].position >= MAX_POSITIONS) { + if (!local[0]) put_error(local, sizeof local, "rank %d position %d is outside 0-63", r, all[r].position); + continue; + } e->positions[r] = all[r].position; e->relayed[r] = all[r].relayed; } diff --git a/spark_transport/libsircl/tests/test_engine_cpu.py b/spark_transport/libsircl/tests/test_engine_cpu.py index bd1cc348..f40b3900 100644 --- a/spark_transport/libsircl/tests/test_engine_cpu.py +++ b/spark_transport/libsircl/tests/test_engine_cpu.py @@ -40,6 +40,46 @@ def run(script, env=None): class EngineCpuTests(unittest.TestCase): + def test_setup_refuses_out_of_range_peer_positions(self): + for position in (-1, 64, 2147483647): + with self.subTest(position=position): + out = run('''import socket, struct, threading +u=U(); assert l.ncclGetUniqueId(C.byref(u))==0 +server=socket.socket(); server.bind(("127.0.0.1", 0)); server.listen() +u.b[16:18]=struct.pack("!H",server.getsockname()[1]) +records=[] +def broker(): + peer,_=server.accept(); peer.settimeout(3) + def read(n): + data=b"" + while len(data) Date: Thu, 15 Oct 2026 07:47:56 -0500 Subject: [PATCH 04/65] Refuse InitAll device selections that cannot be honored Distinct GPU devices are rejected before handles are allocated, and a single selected device must match the current context. Compatibility: Bootstrap-only logical device lists remain supported; unsupported GPU selections return ncclInvalidUsage. Validation: The two-device regression failed before the fix; 14 engine CPU and 16 lifecycle tests pass (one host-dependent skip). Request item: 3. NVIDIA NCCL source was neither read nor adapted. --- spark_transport/libsircl/src/api.c | 15 +++++++++++++++ spark_transport/libsircl/tests/test_engine_cpu.py | 12 ++++++++++++ 2 files changed, 27 insertions(+) diff --git a/spark_transport/libsircl/src/api.c b/spark_transport/libsircl/src/api.c index ee4a01d8..c202a5f1 100644 --- a/spark_transport/libsircl/src/api.c +++ b/spark_transport/libsircl/src/api.c @@ -561,6 +561,21 @@ ncclResult_t ncclCommInitAll(ncclComm_t *out, int ndev, const int *devlist) { for (int i = 0; i < ndev; ++i) out[i] = NULL; for (int i = 0; i < ndev; ++i) if (devlist && devlist[i] < 0) return fail(ncclInvalidArgument, "negative device number"); + const char *only = sccl_env("LIBSIRCL_BOOTSTRAP_ONLY"); + if (!only || strcmp(only, "1")) { + int device = devlist ? devlist[0] : 0; + for (int i = 1; i < ndev; ++i) + if ((devlist ? devlist[i] : i) != device) + return creation_failed(ncclInvalidUsage, "InitAll with distinct devices is unsupported; initialize each rank on its device"); + char why[512]; + sccl_CUcontext ctx = sccl_engine_current_context(); + if (!ctx) ctx = sccl_engine_device_context(why, sizeof why); + if (!ctx) return creation_failed(ncclInvalidUsage, why); + const sccl_cuda *driver = sccl_cuda_get(); + sccl_CUdevice current = -1; + if (!driver || driver->CtxGetDevice(¤t) != SCCL_CUDA_SUCCESS || current != device) + return creation_failed(ncclInvalidUsage, "InitAll requires the requested device's context to be current"); + } ncclUniqueId id; ncclResult_t result = ncclGetUniqueId(&id); if (result != ncclSuccess) return result; diff --git a/spark_transport/libsircl/tests/test_engine_cpu.py b/spark_transport/libsircl/tests/test_engine_cpu.py index f40b3900..0223890e 100644 --- a/spark_transport/libsircl/tests/test_engine_cpu.py +++ b/spark_transport/libsircl/tests/test_engine_cpu.py @@ -40,6 +40,18 @@ def run(script, env=None): class EngineCpuTests(unittest.TestCase): + def test_initall_refuses_distinct_cuda_devices(self): + for devices in ("None", "(C.c_int*2)(0,1)"): + out = run("c=(C.c_void_p*2)(); r=l.ncclCommInitAll(c,2," + devices + "); " + "print(json.dumps([r,list(c),l.ncclGetLastError(None).decode()]))", + {"LIBSIRCL_BOOTSTRAP_ONLY": "", "LIBSIRCL_TRANSPORT": "bogus", + "FAKE_CUDA_DEVICES": "2", "LD_LIBRARY_PATH": str(FAKE_DRIVER)}) + self.assertEqual(out.returncode, 0, out.stderr) + result, handles, message = json.loads(out.stdout) + self.assertEqual(result, 5) + self.assertEqual(handles, [None, None]) + self.assertIn("distinct devices", message) + def test_setup_refuses_out_of_range_peer_positions(self): for position in (-1, 64, 2147483647): with self.subTest(position=position): From 63764c79c68a612dc1a1fae9bb36e0bec4c2d85b Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Thu, 15 Oct 2026 07:49:42 -0500 Subject: [PATCH 05/65] Close engine submission before waiting for context work The engine closes under its submission lock, clears failed stream ordering, and synchronizes the creating context at finalize and every non-abort destroy, including after transport stop. Compatibility: Orderly teardown waits for all context work, including graph replay streams; failed waits preserve the arena. Validation: Context-sync regression failed before the fix; 15 engine and 16 lifecycle CPU tests pass. teardown_race.py --expect concurrent-close is written but not run. Request item: 4. NVIDIA NCCL source was neither read nor adapted. --- spark_transport/libsircl/src/engine.c | 70 +++++++++---------- .../libsircl/tests/emulation/teardown_race.py | 27 ++++++- spark_transport/libsircl/tests/fake_cuda.c | 6 +- .../libsircl/tests/test_engine_cpu.py | 15 ++++ 4 files changed, 79 insertions(+), 39 deletions(-) diff --git a/spark_transport/libsircl/src/engine.c b/spark_transport/libsircl/src/engine.c index 4b28fdda..46f1f458 100644 --- a/spark_transport/libsircl/src/engine.c +++ b/spark_transport/libsircl/src/engine.c @@ -433,7 +433,7 @@ struct sccl_engine { sccl_CUstream last_stream; /* Teardown: the progress thread stopped (sccl_engine_teardown_stop), when its stop was requested and * when it had stopped (CLOCK_REALTIME nanoseconds). */ - int stopped; + int stopped, closing; uint64_t stop_ns, stopped_ns; int have_last; sccl_CUevent order_event; @@ -2961,6 +2961,11 @@ static ncclResult_t begin_call(sccl_engine *e, const char *what, sccl_CUstream s size_t len) { call->pushed = call->capturing = 0; pthread_mutex_lock(&e->lock); + if (e->closing) { + put_error(err, len, "%s: communicator is closing", what); + pthread_mutex_unlock(&e->lock); + return ncclInvalidUsage; + } sccl_CUcontext current = NULL; if (cu->CtxGetCurrent(¤t) != SCCL_CUDA_SUCCESS || current != e->ctx) { if (cuda_check(cu->CtxPushCurrent(e->ctx), "making the communicator's context current", err, len)) { @@ -2993,8 +2998,8 @@ static ncclResult_t begin_call(sccl_engine *e, const char *what, sccl_CUstream s (cuda_check(cu->EventRecord(e->order_event, e->last_stream), "recording the order event", err, len) || cuda_check(cu->StreamWaitEvent(stream, e->order_event, 0), "ordering the stream", err, len))) result = ncclUnhandledCudaError; - e->last_stream = stream; - e->have_last = 1; + e->have_last = result == ncclSuccess; + if (e->have_last) e->last_stream = stream; } if (result != ncclSuccess) { sccl_CUcontext popped; @@ -4154,16 +4159,15 @@ ncclResult_t sccl_engine_p2p(sccl_engine *e, const sccl_p2p_op *ops, unsigned co ncclResult_t sccl_engine_teardown_sync(sccl_engine *e) { if (!e || !cu) return ncclSuccess; + pthread_mutex_lock(&e->lock); + e->closing = 1; + pthread_mutex_unlock(&e->lock); ncclResult_t result = ncclSuccess; int pushed = e->ctx && cu->CtxPushCurrent(e->ctx) == SCCL_CUDA_SUCCESS; /* Without the creating context the enqueued work cannot be waited for: a failure, not a skipped wait. */ if (e->ctx && !pushed) result = ncclUnhandledCudaError; - if (pushed && e->have_last && cu->StreamSynchronize(e->last_stream) != SCCL_CUDA_SUCCESS) + if (pushed && cu->CtxSynchronize() != SCCL_CUDA_SUCCESS) result = ncclUnhandledCudaError; - for (int d = 0; pushed && d < 2; ++d) - for (int p = 0; p < e->world && p < MAX_WORLD; ++p) - if (e->p2p_ch[d][p].stream && cu->StreamSynchronize(e->p2p_ch[d][p].stream) != SCCL_CUDA_SUCCESS) - result = ncclUnhandledCudaError; if (pushed) { sccl_CUcontext popped; cu->CtxPopCurrent(&popped); @@ -4178,17 +4182,7 @@ int sccl_engine_wait_limit_ms(const sccl_engine *e) { } ncclResult_t sccl_engine_finalize(sccl_engine *e) { - if (!e || !cu) return ncclSuccess; - pthread_mutex_lock(&e->lock); - ncclResult_t result = ncclSuccess; - if (e->have_last) { - int pushed = cu->CtxPushCurrent(e->ctx) == SCCL_CUDA_SUCCESS; - if (cu->StreamSynchronize(e->last_stream) != SCCL_CUDA_SUCCESS) result = ncclUnhandledCudaError; - sccl_CUcontext popped; - if (pushed) cu->CtxPopCurrent(&popped); - } - pthread_mutex_unlock(&e->lock); - return result; + return sccl_engine_teardown_sync(e); } ncclResult_t sccl_engine_set_wait_regime(sccl_engine *e, const char *regime) { @@ -4554,7 +4548,12 @@ static void dump_teardown(sccl_engine *e, uint64_t stop_ns, uint64_t stopped_ns, } void sccl_engine_teardown_stop(sccl_engine *e) { - if (!e || e->stopped) return; + if (!e) return; + pthread_mutex_lock(&e->lock); + e->closing = 1; + int stopped = e->stopped; + pthread_mutex_unlock(&e->lock); + if (stopped) return; write_receipt(e); int pushed = cu && e->ctx && cu->CtxPushCurrent(e->ctx) == SCCL_CUDA_SUCCESS; dump_links(e, "destroy", pushed); @@ -4571,28 +4570,27 @@ void sccl_engine_teardown_stop(sccl_engine *e) { ncclResult_t sccl_engine_destroy(sccl_engine *e, int abort, char *err, size_t len) { if (!e) return ncclSuccess; + pthread_mutex_lock(&e->lock); + e->closing = 1; + pthread_mutex_unlock(&e->lock); watch_remove(e); ncclResult_t result = ncclSuccess; int stopped = e->stopped; if (!stopped) write_receipt(e); int pushed = cu && e->ctx && cu->CtxPushCurrent(e->ctx) == SCCL_CUDA_SUCCESS; - int idle = 1; - if (pushed && e->have_last) { - if (abort) { - idle = cu->StreamQuery(e->last_stream) == SCCL_CUDA_SUCCESS; - } else if (!stopped) { - cu->StreamSynchronize(e->last_stream); - } + int idle = pushed; + if (pushed && !abort && cu->CtxSynchronize() != SCCL_CUDA_SUCCESS) { + idle = 0; + result = ncclUnhandledCudaError; + put_error(err, len, "waiting for the communicator's context failed; its arena stays allocated"); + } + if (pushed && abort) { + if (e->have_last) idle = cu->StreamQuery(e->last_stream) == SCCL_CUDA_SUCCESS; + for (int d = 0; d < 2; ++d) + for (int p = 0; p < e->world && p < MAX_WORLD; ++p) + if (e->p2p_ch[d][p].stream) + idle = idle && cu->StreamQuery(e->p2p_ch[d][p].stream) == SCCL_CUDA_SUCCESS; } - for (int d = 0; pushed && d < 2; ++d) - for (int p = 0; p < e->world && p < MAX_WORLD; ++p) { - sccl_CUstream channel = e->p2p_ch[d][p].stream; - if (!channel) continue; - if (abort) - idle = idle && cu->StreamQuery(channel) == SCCL_CUDA_SUCCESS; - else if (!stopped) - cu->StreamSynchronize(channel); - } if (!stopped) { dump_links(e, "destroy", pushed && idle); e->stop_ns = realtime_ns(); diff --git a/spark_transport/libsircl/tests/emulation/teardown_race.py b/spark_transport/libsircl/tests/emulation/teardown_race.py index 65cf75d5..3f28e37e 100644 --- a/spark_transport/libsircl/tests/emulation/teardown_race.py +++ b/spark_transport/libsircl/tests/emulation/teardown_race.py @@ -46,6 +46,7 @@ import subprocess import sys import tempfile +import threading import time from pathlib import Path @@ -134,6 +135,30 @@ def save(): rounds.append(entry) save() continue + if args.expect == "concurrent-close": + started_calls = threading.Event() + results = [] + def enqueue(): + torch.cuda.set_device(0) + for _ in range(1000): + result = lib.ncclAllReduce(ctypes.c_void_p(x.data_ptr()), ctypes.c_void_p(y.data_ptr()), + count, 9, 0, child, ctypes.c_void_p(stream.cuda_stream)) + results.append(result) + started_calls.set() + if result: + break + worker = threading.Thread(target=enqueue) + worker.start() + started_calls.wait() + finalized = lib.ncclCommFinalize(child) + destroyed = lib.ncclCommDestroy(child) + worker.join(timeout=30) + entry.update(ok=not worker.is_alive() and not finalized and not destroyed + and all(result in (0, 4, 5) for result in results), + finalize=finalized, destroy=destroyed, calls=results) + rounds.append(entry) + save() + continue state = ctypes.c_int(0) lib.ncclCommGetAsyncError(child, ctypes.byref(state)) if not exact or state.value: @@ -275,7 +300,7 @@ def main(argv=None) -> int: parser.add_argument("--slow-rank", type=int, default=0) parser.add_argument("--slow-ns", type=int, default=5_000_000) parser.add_argument("--timeout", type=float, default=1800) - parser.add_argument("--expect", choices=("exact", "close-error", "fail-stop"), default="exact") + parser.add_argument("--expect", choices=("exact", "close-error", "fail-stop", "concurrent-close"), default="exact") parser.add_argument("--late-rank", type=int, default=-1, help="the rank that starts each all-reduce late") parser.add_argument("--late-s", type=float, default=0.0) parser.add_argument("--env", action="append", default=[], help="NAME=VALUE for every rank") diff --git a/spark_transport/libsircl/tests/fake_cuda.c b/spark_transport/libsircl/tests/fake_cuda.c index 144ecdae..9d6cc67e 100644 --- a/spark_transport/libsircl/tests/fake_cuda.c +++ b/spark_transport/libsircl/tests/fake_cuda.c @@ -180,8 +180,10 @@ EXPORT int cuModuleUnload(void *module) { return module ? 0 : 400; } /* Every other entry point src/cuda_api.c binds: present, never used by the check. */ #define UNUSED_ENTRY(name) \ EXPORT int name(void) { return 0; } -UNUSED_ENTRY(cuCtxGetDevice) -UNUSED_ENTRY(cuCtxSynchronize) +EXPORT int cuCtxGetDevice(int *device) { *device = 0; return 0; } +static int context_syncs; +EXPORT int cuCtxSynchronize(void) { ++context_syncs; return 0; } +EXPORT int sccl_fake_cuda_context_syncs(void) { return context_syncs; } UNUSED_ENTRY(cuDeviceGetAttribute) UNUSED_ENTRY(cuLaunchKernel) UNUSED_ENTRY(cuMemAlloc_v2) diff --git a/spark_transport/libsircl/tests/test_engine_cpu.py b/spark_transport/libsircl/tests/test_engine_cpu.py index 0223890e..cbb386fa 100644 --- a/spark_transport/libsircl/tests/test_engine_cpu.py +++ b/spark_transport/libsircl/tests/test_engine_cpu.py @@ -40,6 +40,21 @@ def run(script, env=None): class EngineCpuTests(unittest.TestCase): + def test_finalize_and_destroy_wait_for_the_context(self): + out = run('''g=C.CDLL("libcuda.so.1") +u=U(); assert l.ncclGetUniqueId(C.byref(u))==0 +c=C.c_void_p(); assert l.ncclCommInitRank(C.byref(c),1,u,0)==0 +assert l.ncclCommFinalize(c)==0 +before=g.sccl_fake_cuda_context_syncs() +assert l.ncclCommDestroy(c)==0 +print(json.dumps([before,g.sccl_fake_cuda_context_syncs()])) +''', {"LIBSIRCL_BOOTSTRAP_ONLY": "", "LIBSIRCL_TRANSPORT": "verbs", + "LD_LIBRARY_PATH": str(FAKE_DRIVER)}) + self.assertEqual(out.returncode, 0, out.stderr) + before, after = json.loads(out.stdout) + self.assertGreaterEqual(before, 1) + self.assertGreater(after, before) + def test_initall_refuses_distinct_cuda_devices(self): for devices in ("None", "(C.c_int*2)(0,1)"): out = run("c=(C.c_void_p*2)(); r=l.ncclCommInitAll(c,2," + devices + "); " From 67e5042b7cc00298b2e228a1bcd530494912b6ee Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Thu, 15 Oct 2026 08:01:36 -0500 Subject: [PATCH 06/65] Add a sanitizer check target and run it with the exports check in CI `make check-asan` builds the host library with AddressSanitizer and UndefinedBehaviorSanitizer into $(BUILD)-asan and runs every `make check` suite with the ASan runtime preloaded. The sanitizer flags reach the sub-make through the environment, so the Makefile still appends its required flags. `make check` runs every suite and fails at the end when any failed, so each suite's result is reported even when another fails. Test harness changes needed for the sanitizer run: the ABI preload test appends the library to an existing LD_PRELOAD instead of replacing it; the engine CPU checks keep the ASan preload in children that set or clear LD_PRELOAD and locate the stand-in driver next to the library under test; ncclCommSplit gets its ctypes signature in test_api.initialize_library, so a heap address above 4 GiB (as under ASan) is not truncated to 32 bits. The CI job "libsircl C library" also runs tools/check_exports.py against the built library and `make check-asan`. STATUS.md states the sanitizer evidence as the target's result instead of fixed test counts. Compatibility: no library code changes. Validation: make check (12 Python suites and the fabric vectors) passes; tools/check_exports.py passes; make check-asan passes every suite; with a use-after-free injected into ncclCommDestroy (reverted before this commit) make check-asan fails with AddressSanitizer's heap-use-after-free report. Request item 41 (audit 11-6, 12-F3, 12-F5, 12-F9 items 2, 4 and 5). NVIDIA NCCL source was neither read nor adapted. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 2 + spark_transport/libsircl/Makefile | 39 ++++++++++++------- spark_transport/libsircl/STATUS.md | 9 +++-- spark_transport/libsircl/tests/test_abi.py | 2 +- spark_transport/libsircl/tests/test_api.py | 1 + .../libsircl/tests/test_engine_cpu.py | 5 +++ 6 files changed, 40 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 656f777d..a98ac1c4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -153,6 +153,8 @@ jobs: make -j BUILD=build NVCC="$NVCC" make check BUILD=build NVCC="$NVCC" sha256sum build/packs/*.fatbin + python tools/check_exports.py --include include --library build/libsircl.so + make check-asan BUILD=build NVCC="$NVCC" lil-bridge: name: pinned LIL bridge diff --git a/spark_transport/libsircl/Makefile b/spark_transport/libsircl/Makefile index 966942d5..5c26c0b1 100644 --- a/spark_transport/libsircl/Makefile +++ b/spark_transport/libsircl/Makefile @@ -167,19 +167,32 @@ $(BUILD)/mpi-shim/lib/libmpi.so: tools/mpi-shim/mpi_shim.c tools/mpi-shim/includ $(BUILD)/test_fabric: tests/test_fabric.c $(BUILD)/fabric.o $(CC) $(CPPFLAGS) $(CFLAGS) $^ -o $@ check: all $(BUILD)/test_fabric - $(BUILD)/test_fabric - $(PYTHON) tests/test_abi.py --library $(abspath $(BUILD)/libsircl.so) - $(PYTHON) tests/test_api.py --library $(abspath $(BUILD)/libsircl.so) - $(PYTHON) tests/test_bootstrap.py - $(PYTHON) tests/test_lifecycle.py --library $(abspath $(BUILD)/libsircl.so) - $(PYTHON) tests/test_engine_cpu.py --library $(abspath $(BUILD)/libsircl.so) - $(PYTHON) tests/test_shm_verbs.py - $(PYTHON) tests/test_vendored_sources.py - $(PYTHON) tests/test_p2p_native.py - $(PYTHON) tests/test_site_routes.py - $(PYTHON) tests/test_mpi_shim.py - $(PYTHON) tests/test_check_nccl_tests.py - $(PYTHON) tests/test_kernel_entries.py --build $(BUILD) + @status=0; \ + $(BUILD)/test_fabric || status=1; \ + $(PYTHON) tests/test_abi.py --library $(abspath $(BUILD)/libsircl.so) || status=1; \ + $(PYTHON) tests/test_api.py --library $(abspath $(BUILD)/libsircl.so) || status=1; \ + $(PYTHON) tests/test_bootstrap.py || status=1; \ + $(PYTHON) tests/test_lifecycle.py --library $(abspath $(BUILD)/libsircl.so) || status=1; \ + $(PYTHON) tests/test_engine_cpu.py --library $(abspath $(BUILD)/libsircl.so) || status=1; \ + $(PYTHON) tests/test_shm_verbs.py || status=1; \ + $(PYTHON) tests/test_vendored_sources.py || status=1; \ + $(PYTHON) tests/test_p2p_native.py || status=1; \ + $(PYTHON) tests/test_site_routes.py || status=1; \ + $(PYTHON) tests/test_mpi_shim.py || status=1; \ + $(PYTHON) tests/test_check_nccl_tests.py || status=1; \ + $(PYTHON) tests/test_kernel_entries.py --build $(BUILD) || status=1; \ + exit $$status + +# The check suites against a host library built with AddressSanitizer and UndefinedBehaviorSanitizer in +# $(BUILD)-asan. The sanitizer flags reach the sub-make through the environment, so the CFLAGS line above +# still appends the required flags. The interpreters that load the library preload the ASan runtime, which +# must come first in the process; leak detection is off because the interpreter's own allocations would +# dominate it. +.PHONY: check-asan +check-asan: + ASAN_OPTIONS=detect_leaks=0 CFLAGS="-O1 -g -fsanitize=address,undefined -fno-omit-frame-pointer" LDFLAGS="-fsanitize=address,undefined" $(MAKE) BUILD=$(BUILD)-asan all + LIBSIRCL_TEST_ASAN=1 LD_PRELOAD="$$($(CC) -print-file-name=libasan.so)" ASAN_OPTIONS=detect_leaks=0 UBSAN_OPTIONS=halt_on_error=1 CFLAGS="-O1 -g -fsanitize=address,undefined -fno-omit-frame-pointer" LDFLAGS="-fsanitize=address,undefined" $(MAKE) BUILD=$(BUILD)-asan check + generate: $(PYTHON) tools/generate_api.py clean: diff --git a/spark_transport/libsircl/STATUS.md b/spark_transport/libsircl/STATUS.md index fe7eefe7..13ae6a5d 100644 --- a/spark_transport/libsircl/STATUS.md +++ b/spark_transport/libsircl/STATUS.md @@ -58,7 +58,7 @@ The interface follows NVIDIA's public `nccl.h.in` v2.32.3-1 and the user guide o | Kernel-entry check: before the library links, its pack loader resolves every entry it names in the three embedded packs through a stand-in CUDA driver that reads every architecture's cubin offline (`tests/check_entries.c`, `tests/fake_cuda.c`; make and CMake) | implemented | kernel-entry suite (3 tests: every entry present; hiding one entry of each pack fails and names it; refused without the stand-in driver); a loader that names the two-pass ring all-reduce entries, built with link pack `5935b067...` (which has none), fails the build at `sircl_ring_reduce_two_pass_f32_u1` | | Loading: the library links only libc and the dynamic loader, has no constructor that touches CUDA, and opens `libcuda.so.1` only when a communicator is created | implemented | CPU test in an isolated interpreter, with a negative control that initializes CUDA in a constructor; `tools/probe_cuda_on_load.py` (`verification/cuda-on-load.json`) | | Bootstrap: unique id with root address, port, nonce and site hash; loopback, or a LAN address from `SIRCL_BOOTSTRAP_ADDR`, `SIRCL_BOOTSTRAP_IFNAME` or `NCCL_SOCKET_IFNAME`; all-gather rounds through the root; a rank leaving a complete group fails every later round at once | implemented | bootstrap suite (21 tests); between the two Sparks of the pair (hardware evidence) | -| Communicator lifecycle: blocking and non-blocking init, groups (a collective or point-to-point call refused with `ncclInvalidArgument` leaves its group as it was; a queued point-to-point call holds no reference on its communicator, so `ncclCommAbort` from another thread returns at once and that group's `ncclGroupEnd` fails the calls with `ncclInvalidUsage`), finalize and destroy (wait for every rank; see the teardown row), abort, revoke, a communicator's error and its text published together (one lock), `ncclRedOpCreatePreMulSum` (not implemented) returning `ncclNumOps`, which every collective refuses, async errors from the command ring and the progress thread; `ncclCommSplit` (members ordered by key, then parent rank; `NCCL_SPLIT_NOCOLOR` gets NULL; children keep the parent's route-map position; a NULL config, or a config whose `blocking` is undefined, takes the parent's blocking mode) | implemented | lifecycle (14) and API (17) suites, also under AddressSanitizer and UndefinedBehaviorSanitizer; GPU emulation (below): `library_rank.py`'s checks of an invalid-peer send inside a group (the group's valid send and receive complete), of `ncclCommAbort` with a send queued in another thread's open group (returns within 0.1 s; that group's end returns `ncclInvalidUsage`) and of `ncclRedOpCreatePreMulSum`'s op in an all-reduce of unequal inputs (refused with `ncclInvalidArgument`, the buffer unchanged). Abort cannot stop a kernel waiting for a dead peer before its wait limit (request AW) | +| Communicator lifecycle: blocking and non-blocking init, groups (a collective or point-to-point call refused with `ncclInvalidArgument` leaves its group as it was; a queued point-to-point call holds no reference on its communicator, so `ncclCommAbort` from another thread returns at once and that group's `ncclGroupEnd` fails the calls with `ncclInvalidUsage`), finalize and destroy (wait for every rank; see the teardown row), abort, revoke, a communicator's error and its text published together (one lock), `ncclRedOpCreatePreMulSum` (not implemented) returning `ncclNumOps`, which every collective refuses, async errors from the command ring and the progress thread; `ncclCommSplit` (members ordered by key, then parent rank; `NCCL_SPLIT_NOCOLOR` gets NULL; children keep the parent's route-map position; a NULL config, or a config whose `blocking` is undefined, takes the parent's blocking mode) | implemented | lifecycle and API suites, also under AddressSanitizer and UndefinedBehaviorSanitizer (`make check-asan`); GPU emulation (below): `library_rank.py`'s checks of an invalid-peer send inside a group (the group's valid send and receive complete), of `ncclCommAbort` with a send queued in another thread's open group (returns within 0.1 s; that group's end returns `ncclInvalidUsage`) and of `ncclRedOpCreatePreMulSum`'s op in an all-reduce of unequal inputs (refused with `ncclInvalidArgument`, the buffer unchanged). Abort cannot stop a kernel waiting for a dead peer before its wait limit (request AW) | | Communicator creation on a thread with no current CUDA context: the current CUDA device's primary context (the application's CUDA runtime's `cudaGetDevice`, else the only visible device), retained once per process and device and made current on the creating thread; refused with `ncclInvalidUsage`, naming the reason, when the device is unknown (several devices, no runtime) | implemented | Conditions: CPU, `tests/test_engine_cpu.py` on the stand-in driver (`tests/fake_cuda.c`): one device, the runtime's device 1 of two, and two devices without a runtime (refused); GPU on the workstation (RTX 5090, torch 2.12.0+cu130), `tests/emulation/init_without_context.py`: vLLM's PyNccl sequence (device selected with `torch.accelerator.set_device_index`, no allocation; `ncclCommInitRank` inside `torch.accelerator.device_index`; then a one-element float32 all-reduce of a tensor torch allocates afterwards). Measurement: the library of source tree `dbf36074` returned `ncclInvalidUsage` ("no current CUDA context") there, as the installer's `qwen38-flash-next-tp2 --transport libsircl` deployment on a cabled pair of Sparks did at PyNccl's first `ncclCommInitRank` on both ranks (image `1a8c10354eb0`, vLLM v0.1.dev21553+gab86b7073); with this change the call succeeds, torch's first allocation lands in the same primary context and the all-reduce returns 1.0. On the fabric (research run, 2026-10-10, a cabled pair of DGX Sparks at positions 0 and 1 of a ring of four; the installer's two containers of that deployment rerun unchanged except for a research library selected through `SPARKRING_LIBSIRCL_LIBRARY` and `SPARKRING_LIBSIRCL_SHA256`, each built in a container of image `1a8c10354eb0`, so with its nvcc 13.4.92 and kernel packs): source tree `dbf36074` with the creation diagnostic printed `libsircl: invalid usage: no current CUDA context: ...` on both ranks and vLLM stopped at PyNccl's first `ncclCommInitRank`; this tree's library (`cddcaa0e...`) let vLLM load, capture its CUDA graphs and serve: two known-answer chat requests at temperature 0 returned `42` (17 + 25) and `Paris`, and each rank's receipt of the tensor-parallel communicator recorded 7,783 all-reduces (3,420 captured in graphs, 5.43 GB), 133 all-gathers and one pair exchange on the verbs transport and the pair plan, healthy, no error, fail-stop on | | The reason of a failed communicator creation on stderr (`libsircl: communicator creation failed (): `, whatever `NCCL_DEBUG` says); with `NCCL_DEBUG=WARN`, `INFO` or `TRACE`, the reason of every failed call | implemented | CPU: `tests/test_engine_cpu.py` (a refused creation's line; `ncclGetVersion(NULL)` quiet without `NCCL_DEBUG`, named with `NCCL_DEBUG=WARN`) | | Session engine: settings and both packs' hashes agreed across ranks; route maps keyed by process position (`LIBSIRCL_POSITION`, exchanged at setup), so one map serves every communicator a process joins; pinned arena (cuMemHostAlloc on hardware, shared memory registered with CUDA in emulation), device counters, SIRCL's native proxy, queue-pair connection, lane check, progress thread per communicator | implemented | GPU emulation (below); setup-failure cases; verbs transport on the pair (hardware evidence) | @@ -629,9 +629,10 @@ Conditions: WSL2, GCC 13.3, Python 3.12, no GPU or RDMA device. Results: `SIRCL_PACKAGE` set (`verification/cmake-tests.log`). Libraries up to snapshot cddad8cd fail 3 of them there: rdma-core's inline `ibv_reg_mr` keeps its call to `ibv_reg_mr_iova2` unoptimized, which their verbs loader did not define, so the library did not load. -- AddressSanitizer and UndefinedBehaviorSanitizer build: lifecycle 13 and API 14 tests passed, engine 3 - of 4 and ABI 4 of 5; the other two replace the preload list, which the sanitizer runtime refuses, and - pass in the plain build (`verification/sanitizer-cpu.log`). +- AddressSanitizer and UndefinedBehaviorSanitizer: `make check-asan` builds the host library into + `$(BUILD)-asan` with both sanitizers and runs every `make check` suite with the sanitizer runtime preloaded + (the CI job "libsircl C library" runs it); every suite passes, and a use-after-free injected into + `ncclCommDestroy` makes the target fail with AddressSanitizer's report. - The kernel packs built from the sources by nvcc 13.3.73 (the workstation's CUDA pip packages, 2026-10-09) have the SHA-256 values the rows above name: transport `c2e6e5a1...`, fold `66da585d...`, link `dc9dd167...` and point-to-point `0f39a3b9...`. The library and its four packs built in 63 s diff --git a/spark_transport/libsircl/tests/test_abi.py b/spark_transport/libsircl/tests/test_abi.py index 73a5ea19..963d4a13 100644 --- a/spark_transport/libsircl/tests/test_abi.py +++ b/spark_transport/libsircl/tests/test_abi.py @@ -143,7 +143,7 @@ def test_preload_and_soname_reuse(self): run("cc", "-std=c11", "-Wall", "-Wextra", "-Werror", str(path / "preload.c"), "-L", str(path), "-Wl,-rpath," + str(path), "-lnccl", "-ldl", "-o", str(path / "preload")) env = os.environ.copy() env.pop("LIBSIRCL_NCCL_API_VERSION", None) - env["LD_PRELOAD"] = str(LIBRARY) + env["LD_PRELOAD"] = ":".join(filter(None, (env.get("LD_PRELOAD"), str(LIBRARY)))) self.assertIn("all 146 symbols", run(str(path / "preload"), env=env)) diff --git a/spark_transport/libsircl/tests/test_api.py b/spark_transport/libsircl/tests/test_api.py index 828c00b7..e7b2c365 100644 --- a/spark_transport/libsircl/tests/test_api.py +++ b/spark_transport/libsircl/tests/test_api.py @@ -53,6 +53,7 @@ def initialize_library() -> None: "CommInitRank": ([C.POINTER(C.c_void_p), C.c_int, UniqueId, C.c_int], C.c_int), "CommInitRankConfig": ([C.POINTER(C.c_void_p), C.c_int, UniqueId, C.c_int, C.POINTER(Config)], C.c_int), "CommInitAll": ([C.POINTER(C.c_void_p), C.c_int, C.POINTER(C.c_int)], C.c_int), + "CommSplit": ([C.c_void_p, C.c_int, C.c_int, C.POINTER(C.c_void_p), C.c_void_p], C.c_int), "CommRevoke": ([C.c_void_p, C.c_int], C.c_int), "CommGetAsyncError": ([C.c_void_p, C.POINTER(C.c_int)], C.c_int), "CommCount": ([C.c_void_p, C.POINTER(C.c_int)], C.c_int), diff --git a/spark_transport/libsircl/tests/test_engine_cpu.py b/spark_transport/libsircl/tests/test_engine_cpu.py index cbb386fa..f93311a6 100644 --- a/spark_transport/libsircl/tests/test_engine_cpu.py +++ b/spark_transport/libsircl/tests/test_engine_cpu.py @@ -35,6 +35,8 @@ class U(C.Structure): _fields_=[("b",C.c_ubyte*128)] def run(script, env=None): environment = dict(os.environ) environment.update(env or {}) + if os.environ.get("LIBSIRCL_TEST_ASAN"): + environment["LD_PRELOAD"] = os.environ["LD_PRELOAD"] return subprocess.run([sys.executable, "-c", PRELUDE + script, str(LIBRARY)], env=environment, capture_output=True, text=True, timeout=60) @@ -278,6 +280,8 @@ class U(C.Structure): _fields_ = [("b", C.c_ubyte * 128)] print(json.dumps({"before": before, "after": state()})) ''' environment = {k: v for k, v in os.environ.items() if k != "LD_PRELOAD" and not k.startswith("PYTHON")} + if os.environ.get("LIBSIRCL_TEST_ASAN"): + environment["LD_PRELOAD"] = os.environ["LD_PRELOAD"] out = subprocess.run([sys.executable, "-S", "-E", "-c", script, str(LIBRARY)], env=environment, capture_output=True, text=True, timeout=60) self.assertEqual(out.returncode, 0, out.stderr) @@ -308,4 +312,5 @@ def test_setup_checks_the_native_proxy_feature_word(self): parser.add_argument("--library", type=Path, default=LIBRARY) options, extra = parser.parse_known_args() LIBRARY = options.library.resolve() + FAKE_DRIVER = LIBRARY.parent / "fake-cuda" unittest.main(argv=[sys.argv[0], *extra], verbosity=2) From 78723e31ac20bac8742ff760526e6c74e42e1981 Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Thu, 15 Oct 2026 08:08:21 -0500 Subject: [PATCH 07/65] Wait for calls in progress before the unload frees communicators The library destructor (run by dlclose or by exit) cancels every communicator and then waits up to 1 s for calls that still hold a reference to leave before it frees them. Cancellation ends a bootstrap wait within one 50 ms poll slice, so a thread blocked in ncclCommSplit returns ncclSystemError instead of reading the freed communicator or returning into unmapped code. A communicator still referenced after the bound stays allocated with its bootstrap open. ncclCommSplit records its failure text before dropping the parent's reference, so the unload finds the call past its library work. Compatibility: a process unloading libsircl while another thread is inside a call can take up to 1 s longer to unload; communicators still in use at the bound are leaked instead of freed. dlclose concurrent with API calls stays outside the supported caller contract. Validation: tests/test_lifecycle.py's unload test (a split blocked on a stand-in broker, then dlclose or exit; in the exit case a helper library finalized after libsircl keeps the process alive 0.3 s) failed before the change: dlclose ended in SIGSEGV in the plain build, and under `make check-asan` both cases failed (a deadly signal for dlclose, an AddressSanitizer error report for exit). After the change, make check and make check-asan pass every suite. Request item 5 (audit 02-F16). NVIDIA NCCL source was neither read nor adapted. Co-Authored-By: Claude Opus 5.5 --- spark_transport/libsircl/STATUS.md | 2 +- spark_transport/libsircl/src/api.c | 46 ++++++++++++---- .../libsircl/tests/test_lifecycle.py | 54 +++++++++++++++++++ 3 files changed, 91 insertions(+), 11 deletions(-) diff --git a/spark_transport/libsircl/STATUS.md b/spark_transport/libsircl/STATUS.md index 13ae6a5d..958eadc7 100644 --- a/spark_transport/libsircl/STATUS.md +++ b/spark_transport/libsircl/STATUS.md @@ -85,7 +85,7 @@ The interface follows NVIDIA's public `nccl.h.in` v2.32.3-1 and the user guide o | The chain and link areas in the arena after the control line, the link counters, piece counters and decision words, SIRCL's link settings (`SIRCL_LINK_*`, staggers) agreed at setup; the ring plan from `LIBSIRCL_RING_WINDOW`; forward windows of relayed lanes from `LIBSIRCL_FORWARD_WINDOWS`; `tools/site_routes.py` prints every rank's route map, chain order, forward windows and ring plan from SIRCL's route planner | implemented | CPU suites; setup-failure cases; `tools/site_routes.py` reproduces the pair's route maps and gives `path:0-3` a ring window of 393,216 bytes on position 3 | | Teardown: `ncclCommDestroy` and `ncclCommFinalize` of a communicator of two or more ranks wait for this rank's work, then a bootstrap round of every rank that also carries each rank's health, then stop the progress thread, then a second round, before any queue pair, registration or arena is freed; each round bounded by the wait limit; the close is terminal (new work refused, its result kept); a failed release of the transport keeps the memory allocated; `ncclCommAbort` does not wait | implemented | Conditions: GPU emulation on the workstation and on GB10s (snapshot 7d1c6f19, built in the serving image); `tests/emulation/teardown_race.py`: a reversed `ncclCommSplit` child, one ring all-reduce, `ncclCommDestroy` at once, repeated, one rank's writes delayed 5 ms so its predecessor finishes first; with `--expect close-error`, a rank 4 s late under a 2 s wait limit, or the stand-in's deregistrations failed (`SIRCL_EMU_FAIL_DEREG=1`). Measurement: four ranks and two ranks at 8 MiB per rank, 20 rounds each, every round exact with no async error and the parent healthy (GB10: four ranks, 5 runs of 20 rounds, 0 failed); without the rounds (snapshot 6fe2882f) the same check failed 7 of 10 rounds on four ranks on the workstation and 98 of 100 on a GB10 (each failure poisons the parent): a rank's kernel had completed while its progress thread still held an outbound item gated by a peer's credit, destroy stopped that thread and freed its queue pairs, the peer waited out the wait limit, and the peer's late credit write failed (transport retry counter exceeded; the destination queue pair was gone). In close-error mode (workstation and GB10) every round's first finalize or destroy returned the error, a second finalize the same, and a collective after the close was refused. The full emulation suite on a GB10 (snapshot 7d1c6f19) passed every section, the teardown check on four ranks and on two at 8 MiB included. On the fabric (ConnectX-7, four Sparks on a path and a cabled pair, 8 MiB per rank, no delayed rank): 100 rounds each without the rounds and 60 with them (snapshot 7d1c6f19), all exact and healthy, and 10 runs of the relayed-group bit-exact check under the ring schedules (RUNBOOK 4.1) on the path, 0 failed (Sparks 0-3 and 0-1, `ccl-pair/done-hw-teardown-7d1-20261008T194629Z`); the same 60 rounds per layout and 10 checks at snapshot ba5a337b on Sparks 4-7 and 4-5, 0 failed (`ccl-pair/hw-teardown-ba5-47-20261008T225027Z`); the stranded item did not occur at natural timing there; on the cabled pair at positions 6-7 (snapshot 0f94c72b, the rounds of this tree, 2026-10-09): 6 runs of 20 rounds at 8 MiB, 3 with fail-stop off and 3 on, 0 failed, the parent exact and healthy. Result: round 1 proves that every item and flag a kernel waits for has landed; round 2 that no rank posts toward queue pairs or regions about to be freed. Writes no kernel waits for (credits) may still be in flight; each rank destroys its queue pairs before its registrations and memory, so they land in registered memory or are dropped. Limits: the rounds are bounded, but this rank's own wait for its work, the progress thread's join and the verbs destroys are not; in emulation the stand-in resolves a write under its registry lock and applies it after, so a write resolved before a deregistration still lands in the old segment, and emulation does not test a NIC's revocation of a deregistered region. Callers: torch 2.10's `destroy_process_group` shuts its groups down in one order on every rank (finalize, then destroy); vLLM 0.19.1's pynccl `destroy` calls `ncclCommDestroy`; a caller that tears down with `ncclCommAbort` runs no rounds | | Destroying or finalizing communicators in a different order on different ranks | unsupported | each teardown round waits for every rank of its communicator, so crossed orders wait out the wait limit on each round (20 s serving, 600 s startup) and return `ncclRemoteError`; they do not wait longer | -| Process exit without `ncclCommDestroy`, `ncclCommFinalize` or `ncclCommAbort` while a peer's collective on the communicator still runs | unsupported | at library unload only the native threads stop; a peer may wait out its wait limit | +| Process exit without `ncclCommDestroy`, `ncclCommFinalize` or `ncclCommAbort` while a peer's collective on the communicator still runs | unsupported | at library unload only the native threads stop; a peer may wait out its wait limit. Calls still running on other threads are cancelled, and the unload waits up to 1 s for them to leave before it frees their communicators; a communicator still in use after that stays allocated (CPU: `tests/test_lifecycle.py`, exit and `dlclose` with a split in progress, also under `make check-asan`) | | `ncclCommDestroy` while replays of captured CUDA graphs of the communicator's collectives still run | unsupported (caller contract) | teardown waits for the engine's last eager launch (a launch under capture records no stream), not for graph replays; a communicator used only under capture has nothing to wait for | | CUDA graph capture of a call that needs more staging than the communicator holds, on a driver or device without stream-ordered allocation (a driver before CUDA 11.2, or a device without memory pools; the GB10 and the workstation's RTX 5090 have them) | unsupported | the call is refused with `ncclInvalidUsage` on the rank that needs the staging while its peers capture theirs; it captures after one eager call of that size has run on that rank. Whether every rank has stream-ordered allocation is agreed at setup | | Fail-stop (`LIBSIRCL_FAIL_STOP=1` or `abort`, per process): at the first asynchronous error (a flag wait that timed out, a failed progress thread, a failed point-to-point channel) of a watched communicator of two or more ranks, found by the watcher thread (every `LIBSIRCL_FAIL_STOP_POLL_MS`, default 5) or by the first library call that would report it (`ncclCommGetAsyncError`, an enqueue's check, `ncclCommFinalize`, `ncclCommDestroy`, `ncclCommAbort`), the library writes the error, the time and what found it to stderr and the communicator's receipt, then ends the process, with `1` at once (exit status 70, no atexit handlers, no core dump), with `abort` by `abort()`; destroy and abort check once more, then take the communicator off the watch before releasing it, and library unload stops the watcher first | implemented | Conditions: the workstation, `tests/emulation/fail_stop.py`, two ranks, SIRCL_STARTUP_WAIT_S 2, rank 1 starting its second all-reduce 12 s late, rank 0 enqueuing its own at once, waiting for its stream, keeping the output and calling the library no more. Measurement (library `638cca1b...`, every time CLOCK_MONOTONIC): 15 runs of the exit and control cases and one of all three: with `1`, the fail-stop line 2.11 to 2.13 s and the process's end 2.43 to 2.53 s after the enqueue, exit status 70; with `abort`, the line 2.11 s after the enqueue and SIGABRT 20.8 s after it (WSL's crash capture ran first); without fail-stop, rank 0's stream wait returned 2.10 to 2.13 s after the enqueue with a wrong output, every call it made returned 0, and the process ran on past 7 s. With `LIBSIRCL_FAIL_STOP=1` the pair default's 533 library checks passed and no process ended; on a build of the same fail-stop code, so did the ring schedules' 989 checks on four ranks and 10 teardown rounds on four ranks. The wait limit itself (research build of this tree whose timed-out flag wait records its `%globaltimer` start and end and the limit it read): 34 timed-out 2 s waits lasted 2.00002 to 2.00095 s of `%globaltimer` with the limit read as 2,000,000 us; a 300 s kernel and five 10 s kernels that sampled `%globaltimer` every 1,024 polls showed no step beyond 11 ms (host-stamping jitter) and a rate of 0.943 to 0.948 of the workstation's CLOCK_MONOTONIC; the workstation's CLOCK_REALTIME stepped back by 1.2 to 1.7 s about every 30 s (WSL2's time synchronization), so two of 28 waits timed by the wall clock looked 0.40 s long while the kernel measured 2.0003 s; earlier runs of this check that used the wall clock showed the same. On the fabric (cabled pair at positions 6-7, snapshot 0f94c72b, 2026-10-09; `teardown_race.py`, rank 1 starting its child all-reduce 8 s late, SIRCL_STARTUP_WAIT_S 2): with `1`, rank 0 exited with status 70 3.9 s after it started, setup included, its fail-stop line naming the timed-out flag wait for the late rank; without fail-stop it exited 1 after 10.4 s with the round reported failed. Library calls (this tree, library `9776657f...`, GPU emulation, 2026-10-09, `verification/emulation/suite-s1/`): `fail_stop.py`'s cases in which rank 0 calls `ncclCommGetAsyncError`, `ncclCommDestroy` or `ncclCommAbort` right after its stream wait, with `LIBSIRCL_FAIL_STOP_POLL_MS=600000` so the watcher does not poll, end rank 0 with status 70 inside the call (2.31 to 2.38 s after the enqueue, the line 2.00 s after it, "found by a library call" or "found by ncclCommDestroy or ncclCommAbort"), the call never returning; against the library before this path the same cases had `ncclCommGetAsyncError` and `ncclCommAbort` return the error to rank 0, which ran on and exited 0. `teardown_race.py --expect fail-stop`, four ranks, rank 3's child all-reduce 8 s late, 2 s wait limit: every rank exited 70 in 6 of 6 runs; in 9 of the 15 non-late exits of 5 runs a library call found the error before the watcher (the gate's path of four lost one rank that way before this path). Conclusion: with fail-stop, a caller that checks only enqueue codes keeps a failed collective's output at most for the wait limit plus one poll; a read within that poll after its stream wait returns is not prevented | diff --git a/spark_transport/libsircl/src/api.c b/spark_transport/libsircl/src/api.c index c202a5f1..e3e72988 100644 --- a/spark_transport/libsircl/src/api.c +++ b/spark_transport/libsircl/src/api.c @@ -17,6 +17,7 @@ #include #include #include +#include enum comm_state { INIT, READY, FINALIZED, REVOKED, FAILED }; struct ncclComm { @@ -332,6 +333,13 @@ ncclResult_t ncclCommInitRank(ncclComm_t *out, int nranks, ncclUniqueId id, int return ncclCommInitRankConfig(out, nranks, id, rank, NULL); } +/* A failed split: its reason is recorded before the parent's reference is dropped, so that a library unload + * waiting for the reference finds the call already past its library work. */ +static ncclResult_t split_failed(struct ncclComm *parent, ncclResult_t result, const char *message) { + creation_failed(result, message); + release(parent); + return result; +} /* ncclCommSplit: two rounds over the parent's bootstrap. Round 1 gathers every rank's (color, key); the * ranks of one color form a communicator ordered by key, then parent rank. Round 2 gathers the unique id * each new rank 0 created; every member then joins its communicator with that id, keeping the parent's @@ -354,14 +362,12 @@ ncclResult_t ncclCommSplit(ncclComm_t handle, int color, int key, ncclComm_t *ne if (sccl_bootstrap_allgather(parent->bootstrap, mine, sizeof mine, &all, lengths, &parent->cancelled)) { char message[700]; snprintf(message, sizeof message, "ncclCommSplit: exchanging colors: %s", sccl_bootstrap_error()); - release(parent); - return creation_failed(ncclSystemError, message); + return split_failed(parent, ncclSystemError, message); } for (int r = 0; r < parent->nranks; ++r) { if (lengths[r] != sizeof mine) { free(all); - release(parent); - return creation_failed(ncclRemoteError, "ncclCommSplit: invalid color record length"); + return split_failed(parent, ncclRemoteError, "ncclCommSplit: invalid color record length"); } } const int32_t *records = all; @@ -390,14 +396,12 @@ ncclResult_t ncclCommSplit(ncclComm_t handle, int color, int key, ncclComm_t *ne if (sccl_bootstrap_allgather(parent->bootstrap, id.internal, sizeof id.internal, &all, lengths, &parent->cancelled)) { char message[700]; snprintf(message, sizeof message, "ncclCommSplit: exchanging unique ids: %s", sccl_bootstrap_error()); - release(parent); - return creation_failed(ncclSystemError, message); + return split_failed(parent, ncclSystemError, message); } for (int r = 0; r < parent->nranks; ++r) { if (lengths[r] != sizeof id.internal) { free(all); - release(parent); - return creation_failed(ncclRemoteError, "ncclCommSplit: invalid unique-id record length"); + return split_failed(parent, ncclRemoteError, "ncclCommSplit: invalid unique-id record length"); } } if (rank >= 0) memcpy(id.internal, (const unsigned char *)all + 128 * (size_t)members[0], 128); @@ -986,8 +990,19 @@ static ncclResult_t dispose(ncclComm_t handle, int abort) { ncclResult_t ncclCommDestroy(ncclComm_t comm) { return dispose(comm, 0); } ncclResult_t ncclCommAbort(ncclComm_t comm) { return dispose(comm, 1); } -/* A library unload must not leave a worker executing code in an unmapped DSO. - * dlclose concurrent with API calls is outside the supported caller contract. */ +/* A library unload (dlclose, or the destructors exit runs) must not leave a worker executing code in an + * unmapped DSO, nor free a communicator that a call on another thread still uses. Every communicator is + * cancelled first, which ends its bootstrap waits within one poll slice (50 ms); calls still inside the + * library hold references, and the unload waits up to UNLOAD_WAIT_MS for them to leave. A communicator still + * referenced after that stays allocated with its bootstrap open: freeing it would pull memory from under the + * call. dlclose concurrent with API calls remains outside the supported caller contract, since a call that + * has released its reference still returns through library code. */ +enum { UNLOAD_WAIT_MS = 1000 }; +static int referenced(const struct ncclComm *list) { + for (const struct ncclComm *entry = list; entry; entry = entry->next) + if (entry->refs) return 1; + return 0; +} __attribute__((destructor)) static void shutdown_communicators(void) { if (!sccl_bootstrap_process_valid()) return; sccl_engine_fail_stop_shutdown(); @@ -996,9 +1011,20 @@ __attribute__((destructor)) static void shutdown_communicators(void) { registry = NULL; for (struct ncclComm *entry = comm; entry; entry = entry->next) atomic_store(&entry->cancelled, 1); + struct timespec deadline; + clock_gettime(CLOCK_REALTIME, &deadline); + deadline.tv_sec += UNLOAD_WAIT_MS / 1000; + deadline.tv_nsec += (long)(UNLOAD_WAIT_MS % 1000) * 1000000L; + if (deadline.tv_nsec >= 1000000000L) { ++deadline.tv_sec; deadline.tv_nsec -= 1000000000L; } + while (referenced(comm)) + if (pthread_cond_timedwait(®istry_idle, ®istry_lock, &deadline) == ETIMEDOUT) break; pthread_mutex_unlock(®istry_lock); while (comm) { struct ncclComm *next = comm->next; + pthread_mutex_lock(®istry_lock); + int busy = comm->refs != 0; + pthread_mutex_unlock(®istry_lock); + if (busy) { comm = next; continue; } if (comm->worker_started) pthread_join(comm->worker, NULL); /* At unload only the native threads stop; CUDA may already be torn down. */ sccl_engine_shutdown(comm->engine); diff --git a/spark_transport/libsircl/tests/test_lifecycle.py b/spark_transport/libsircl/tests/test_lifecycle.py index ff99c9ad..43473183 100644 --- a/spark_transport/libsircl/tests/test_lifecycle.py +++ b/spark_transport/libsircl/tests/test_lifecycle.py @@ -10,6 +10,7 @@ import subprocess import struct import sys +import tempfile import threading import time import unittest @@ -374,6 +375,59 @@ def test_repeated_init_destroy_releases_file_descriptors(self): subprocess.run([sys.executable, "-c", script, str(LIBRARY), str(Path(__file__).parent)], check=True, capture_output=True, text=True, timeout=5) + def test_unload_waits_for_a_call_still_inside_the_library(self): + """Another thread is inside ncclCommSplit, waiting for a peer, when the library unloads (dlclose, or + the destructors that exit runs). The unload cancels the call and waits for it to leave before freeing + the communicator, so the call returns an error instead of using freed memory or unmapped code. The + exit case is decisive under `make check-asan`, where a use of the freed communicator is reported. + + The parent communicator's bootstrap root is a stand-in broker in the child process that admits rank 0 + and never answers its all-gather, so the split waits in its socket poll until cancellation reaches + it. In the exit case a helper library, loaded after libsircl and finalized after it, keeps the + exiting process alive for 0.3 s after libsircl's destructor, long enough for the split thread to wake.""" + script = r''' +import _ctypes, ctypes as C, os, socket, struct, sys, threading, time +sys.path.insert(0, sys.argv[2]); import test_api as api +api.LIBRARY=api.Path(sys.argv[1]); api.initialize_library() +if sys.argv[3]=="exit": C.CDLL(sys.argv[4]) +os.environ["LIBSIRCL_BOOTSTRAP_ONLY"]="1"; os.environ["LIBSIRCL_SETUP_TIMEOUT_MS"]="5000" +server=socket.socket(); server.bind(("127.0.0.1",0)); server.listen() +peers=[] +def broker(): + peer,_=server.accept(); peers.append(peer) + hello=b"" + while len(hello)<136: hello+=peer.recv(136-len(hello)) + peer.sendall(b"SCC1"+struct.pack("!I",0)) +threading.Thread(target=broker, daemon=True).start() +u=api.UniqueId(); assert api.LIB.ncclGetUniqueId(C.byref(u))==0 +u.internal[16:18]=struct.pack("!H",server.getsockname()[1]) +handle=C.c_void_p(); assert api.LIB.ncclCommInitRank(C.byref(handle),2,u,0)==0 +child=C.c_void_p(); result=[] +def split(): result.append(api.LIB.ncclCommSplit(handle,0,0,C.byref(child),None)) +thread=threading.Thread(target=split, daemon=True); thread.start(); time.sleep(0.1) +assert not result +if sys.argv[3]=="exit": + sys.stdout.flush(); C.CDLL(None).exit(0) +_ctypes.dlclose(api.LIB._handle) +thread.join(2) +assert result and result[0]!=0, result +print("split returned", result[0]) +''' + with tempfile.TemporaryDirectory() as directory: + source, helper = Path(directory) / "linger.c", Path(directory) / "liblinger.so" + source.write_text("#include \n__attribute__((destructor)) static void linger(void) " + "{ usleep(300000); }\n") + subprocess.run([os.environ.get("CC", "cc"), "-shared", "-fPIC", str(source), "-o", str(helper)], + check=True) + for mode in ("dlclose", "exit"): + with self.subTest(mode=mode): + out = subprocess.run([sys.executable, "-c", script, str(LIBRARY), str(Path(__file__).parent), + mode, str(helper)], capture_output=True, text=True, timeout=10) + self.assertEqual(out.returncode, 0, out.stderr) + self.assertNotIn("AddressSanitizer", out.stderr) + if mode == "dlclose": + self.assertIn("split returned", out.stdout) + def test_dso_unload_joins_workers_and_closes_roots(self): script = r''' import _ctypes, ctypes as C, os, sys, time From 0968a92c0b91fda8be87e18b076f89d21bdb4f1a Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Thu, 15 Oct 2026 08:11:03 -0500 Subject: [PATCH 08/65] Keep every split rank in the unique-id round when the leader fails A split leader whose ncclGetUniqueId fails still takes part in the unique-id all-gather. It sends a failure record (no id magic; bytes 8-11 carry its result) instead of returning before the round, so every member of its color returns the same result at once, with a reason naming the leader, and every rank's bootstrap round count stays equal. The leader's own failure goes through creation_failed like every other split failure. A bootstrap handle whose all-gather failed after it began sending is marked failed, and every later all-gather on it fails at once naming that cause: its round count can differ from the broker's, and a later round must not be matched with another round's records. Compatibility: the wire format of a successful split is unchanged. A member of a failed leader returns the leader's result immediately instead of a bootstrap timeout (ncclSystemError) after LIBSIRCL_SETUP_TIMEOUT_MS. Any all-gather on a parent bootstrap after a failed one fails with ncclRemoteError instead of waiting or desynchronizing. Validation: tests/test_lifecycle.py holds all 64 bootstrap roots so the leader's id creation fails. Before the change the two ranks returned ncclInvalidUsage and ncclSystemError after 5.1 s; after it both return ncclInvalidUsage within 0.3 s and a following NOCOLOR split on both ranks succeeds. make check and make check-asan pass. Request item 7 (audit 03-F9, 02-F4, 01-F12.5). NVIDIA NCCL source was neither read nor adapted. Co-Authored-By: Claude Opus 5.5 --- spark_transport/libsircl/src/api.c | 28 +++++++++++-- spark_transport/libsircl/src/bootstrap.c | 13 +++++- .../libsircl/tests/test_lifecycle.py | 42 +++++++++++++++++++ 3 files changed, 77 insertions(+), 6 deletions(-) diff --git a/spark_transport/libsircl/src/api.c b/spark_transport/libsircl/src/api.c index e3e72988..d681df50 100644 --- a/spark_transport/libsircl/src/api.c +++ b/spark_transport/libsircl/src/api.c @@ -385,13 +385,21 @@ ncclResult_t ncclCommSplit(ncclComm_t handle, int color, int key, ncclComm_t *ne for (int i = 0; i < count; ++i) if (members[i] == parent->rank) rank = i; free(all); + /* Every rank takes part in round 2, including a leader whose unique id could not be created: it sends a + * failure record instead (no id magic; bytes 8-11 its result, big-endian), so its members fail with the + * same result at once and every rank's bootstrap stays in step. */ ncclUniqueId id; memset(&id, 0, sizeof id); ncclResult_t result = ncclSuccess; - if (rank == 0) result = ncclGetUniqueId(&id); - if (result != ncclSuccess) { - release(parent); - return result; + char reason[512] = {0}; + if (rank == 0) { + result = ncclGetUniqueId(&id); + if (result != ncclSuccess) { + snprintf(reason, sizeof reason, "%s", last_error); + memset(&id, 0, sizeof id); + id.internal[8] = (unsigned char)((unsigned)result >> 24); id.internal[9] = (unsigned char)((unsigned)result >> 16); + id.internal[10] = (unsigned char)((unsigned)result >> 8); id.internal[11] = (unsigned char)result; + } } if (sccl_bootstrap_allgather(parent->bootstrap, id.internal, sizeof id.internal, &all, lengths, &parent->cancelled)) { char message[700]; @@ -406,6 +414,18 @@ ncclResult_t ncclCommSplit(ncclComm_t handle, int color, int key, ncclComm_t *ne } if (rank >= 0) memcpy(id.internal, (const unsigned char *)all + 128 * (size_t)members[0], 128); free(all); + if (rank > 0 && id.internal[0] == 0) { + unsigned code = (unsigned)id.internal[8] << 24 | (unsigned)id.internal[9] << 16 | (unsigned)id.internal[10] << 8 | + id.internal[11]; + result = code > ncclSuccess && code < ncclNumResults && code != ncclInProgress ? (ncclResult_t)code : ncclRemoteError; + snprintf(reason, sizeof reason, "the split leader (parent rank %d) could not create the unique id (%s)", + members[0], ncclGetErrorString(result)); + } + if (result != ncclSuccess) { + char message[700]; + snprintf(message, sizeof message, "ncclCommSplit: %s", reason); + return split_failed(parent, result, message); + } int position = parent->position, blocking = parent->blocking; release(parent); if (rank < 0) return ncclSuccess; diff --git a/spark_transport/libsircl/src/bootstrap.c b/spark_transport/libsircl/src/bootstrap.c index d15cb7fa..59f36702 100644 --- a/spark_transport/libsircl/src/bootstrap.c +++ b/spark_transport/libsircl/src/bootstrap.c @@ -35,7 +35,9 @@ static const unsigned char frame_magic[4] = {'S','C','F','1'}; static const unsigned char gather_magic[4] = {'S','C','R','1'}; static _Thread_local char last_error[192]; -struct sccl_bootstrap { int fd; int nranks; int rank; uint32_t round; }; +/* `failed`: an all-gather on this handle failed after it began sending, so this rank's round count may differ + * from the broker's; every later all-gather fails at once instead of matching another round's records. */ +struct sccl_bootstrap { int fd; int nranks; int rank; uint32_t round; int failed; }; struct peer { int fd; int rank; @@ -698,6 +700,7 @@ int sccl_bootstrap_join_cancel(const unsigned char id[ID_SIZE], int nranks, int handle->nranks = nranks; handle->rank = rank; handle->round = 0; + handle->failed = 0; *out = handle; return OK; failed: @@ -747,6 +750,8 @@ int sccl_bootstrap_allgather_within(sccl_bootstrap *handle, const void *data, ui return error_result(USAGE, "fork after library use is unsupported; exec is required in the child"); if (!handle || !out || !lengths || (bytes && !data) || bytes > MAX_FRAME || timeout_ms < 1) return error_result(ARGUMENT, "all-gather needs a handle, outputs, at most 1 MiB per rank and a timeout"); + if (handle->failed) + return error_result(REMOTE, "an earlier all-gather on this bootstrap failed, so its rounds are out of step"); int64_t deadline = now_ms() + timeout_ms; unsigned char header[FRAME_HEADER]; memcpy(header, frame_magic, 4); @@ -762,7 +767,10 @@ int sccl_bootstrap_allgather_within(sccl_bootstrap *handle, const void *data, ui get32(header + 8) != (uint32_t)handle->nranks || total > (uint32_t)handle->nranks * (4u + MAX_FRAME)) { result = REMOTE; goto failed; } unsigned char *body = malloc(total ? total : 1); - if (!body) return error_result(SYSTEM, "cannot allocate the all-gather answer"); + if (!body) { + handle->failed = 1; + return error_result(SYSTEM, "cannot allocate the all-gather answer"); + } result = recv_all(handle->fd, body, total, deadline, cancelled); if (result != OK) { free(body); goto failed; } /* Repack as the contributions back to back, without their length words. */ @@ -780,6 +788,7 @@ int sccl_bootstrap_allgather_within(sccl_bootstrap *handle, const void *data, ui *out = body; return OK; failed: + handle->failed = 1; switch (result) { case TIMEOUT: return error_result(TIMEOUT, "bootstrap all-gather timed out waiting for every rank"); case USAGE: return error_result(USAGE, "bootstrap all-gather cancelled"); diff --git a/spark_transport/libsircl/tests/test_lifecycle.py b/spark_transport/libsircl/tests/test_lifecycle.py index 43473183..b28fc878 100644 --- a/spark_transport/libsircl/tests/test_lifecycle.py +++ b/spark_transport/libsircl/tests/test_lifecycle.py @@ -5,6 +5,7 @@ import argparse import concurrent.futures import ctypes as C +import json import os from pathlib import Path import subprocess @@ -50,6 +51,47 @@ def test_abort_cancels_an_inflight_split(self): if handle: self.lib.ncclCommAbort(handle) + def test_split_leader_failure_fails_every_member_at_once(self): + """The split leader cannot create the new unique id (the process holds all 64 bootstrap roots). It still + takes part in the unique-id round with a failure record, so both ranks return its result at once + instead of the member waiting out LIBSIRCL_SETUP_TIMEOUT_MS, and the parent's bootstrap stays usable.""" + script = r''' +import concurrent.futures, ctypes as C, json, os, sys, time +sys.path.insert(0, sys.argv[2]); import test_api as api +api.LIBRARY=api.Path(sys.argv[1]); api.initialize_library() +os.environ.update(LIBSIRCL_BOOTSTRAP_ONLY="1", LIBSIRCL_SETUP_TIMEOUT_MS="5000", SIRCL_BOOTSTRAP_TIMEOUT_MS="30000") +handles=(C.c_void_p*2)(); assert api.LIB.ncclCommInitAll(handles,2,None)==0 +held=[] +while True: + u=api.UniqueId() + if api.LIB.ncclGetUniqueId(C.byref(u))!=0: break + held.append(u) +children=[C.c_void_p(), C.c_void_p()] +def split(r): + code=api.LIB.ncclCommSplit(handles[r],0,r,C.byref(children[r]),None) + return code, api.LIB.ncclGetLastError(None).decode() +start=time.monotonic() +with concurrent.futures.ThreadPoolExecutor(2) as pool: + results=list(pool.map(split, range(2))) +elapsed=time.monotonic()-start +none=[C.c_void_p(), C.c_void_p()] +with concurrent.futures.ThreadPoolExecutor(2) as pool: + again=list(pool.map(lambda r: api.LIB.ncclCommSplit(handles[r],-1,0,C.byref(none[r]),None), range(2))) +print(json.dumps({"results":results,"elapsed":elapsed,"held":len(held),"children":[c.value for c in children], + "again":again})) +''' + out = subprocess.run([sys.executable, "-c", script, str(LIBRARY), str(Path(__file__).parent)], + capture_output=True, text=True, timeout=30) + self.assertEqual(out.returncode, 0, out.stderr) + data = json.loads(out.stdout) + self.assertEqual(data["held"], 63) + self.assertEqual([code for code, _ in data["results"]], [USAGE, USAGE], data) + self.assertIn("64 active bootstrap IDs", data["results"][0][1]) + self.assertIn("split leader (parent rank 0)", data["results"][1][1]) + self.assertLess(data["elapsed"], 2) + self.assertEqual(data["children"], [None, None]) + self.assertEqual(data["again"], [SUCCESS, SUCCESS]) + def test_split_refuses_short_records_in_both_rounds(self): for short_round in (0, 1): with self.subTest(round=short_round): From 5085c5780492a1c7dbd1102ef483d0fa6a4ac5df Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Thu, 15 Oct 2026 08:12:26 -0500 Subject: [PATCH 09/65] Run the teardown rounds when destroying a revoked communicator ncclCommDestroy and ncclCommFinalize of a revoked communicator of two or more ranks run the same two bootstrap rounds as for a ready one before any queue pair, registration or arena is freed. Revoke refuses new work but does not stop kernels and forwards already in flight, so freeing without the rounds can remove regions a peer still writes; STATUS.md's teardown evidence records that skipping the rounds fails 7 of 10 rounds on the workstation and 98 of 100 on a GB10. README.md and STATUS.md name the revoked case in the teardown contract. tests/emulation/teardown_race.py gains --expect revoke-destroy (revoke each child right after its all-reduce, then destroy it) and documents --expect concurrent-close. Compatibility: destroy after revoke now waits for every rank, bounded by the wait limit per round, and returns ncclRemoteError when a rank does not arrive; abort remains the non-waiting path. Validation: the changed path needs a ready engine of two or more ranks, which the CPU harness cannot build. Written, not run: python tests/emulation/teardown_race.py --library build/libsircl.so \ --expect revoke-destroy make check and make check-asan pass. Request item 8 (audit 02-F5, revoke part of 01-F7). NVIDIA NCCL source was neither read nor adapted. Co-Authored-By: Claude Opus 5.5 --- spark_transport/libsircl/README.md | 4 ++-- spark_transport/libsircl/STATUS.md | 2 +- spark_transport/libsircl/src/api.c | 4 +++- .../libsircl/tests/emulation/teardown_race.py | 21 ++++++++++++++++--- 4 files changed, 24 insertions(+), 7 deletions(-) diff --git a/spark_transport/libsircl/README.md b/spark_transport/libsircl/README.md index cc550703..b5ab0f70 100644 --- a/spark_transport/libsircl/README.md +++ b/spark_transport/libsircl/README.md @@ -152,8 +152,8 @@ without it, such a capture is refused with `ncclInvalidUsage` on the rank that n eager call of that size has run there. Point-to-point channel messages stage through a stream-ordered allocation on their channel's stream where there is one, else through their channel's own staging buffer. -Teardown: `ncclCommDestroy` and `ncclCommFinalize` of a ready communicator of two or more ranks wait -for every rank. This rank's enqueued work completes; a first bootstrap round proves that every rank's work +Teardown: `ncclCommDestroy` and `ncclCommFinalize` of a ready or revoked communicator of two or more ranks +wait for every rank. This rank's enqueued work completes; a first bootstrap round proves that every rank's work completed, so every item and flag any kernel waits for has landed (a kernel completes once it has its inbound items, while its progress thread may still owe a forward or a last item to a peer); the progress thread then stops, and a second round proves that every rank's progress thread stopped before any queue diff --git a/spark_transport/libsircl/STATUS.md b/spark_transport/libsircl/STATUS.md index 958eadc7..b1465813 100644 --- a/spark_transport/libsircl/STATUS.md +++ b/spark_transport/libsircl/STATUS.md @@ -83,7 +83,7 @@ The interface follows NVIDIA's public `nccl.h.in` v2.32.3-1 and the user guide o | Op selection from shared state only: a call's schedule, op kinds and count, pieces, blocks per role and link op words follow from its shape, datatype, root and the settings agreed at setup; a rank's buffer alignment, in-place or overlapping buffers, a discarded output (off the root) and its staging state choose only local copies through scratch or staging around the same ops. The staging buffer grows outside capture with every earlier buffer kept until destroy (no limit on the number of sizes); under CUDA graph capture a call that needs more staging than the communicator holds takes a graph allocation on the capturing stream (`cuMemAllocAsync`), freed on that stream when the call returns, so the graph owns it (README.md, "Op selection and staging") | implemented | Conditions: GPU emulation on the workstation; `library_rank.py`'s rank-local staging check on a fresh split child: the collectives of every selection point (one-shot, pieces and padded tiles; chain, ring and link ops; pair exchanges of roots 0 and 1; `ncclAlltoAll`; on two ranks torch's all-to-all pattern and one-way sends each way), with every rank's buffers aligned, then rank 1's unaligned, then rank 1's in place, then captured cold into one graph on a second fresh child with rank 1's unaligned and replayed twice; and the staging growth check, 19 all-gathers of doubling shards from 16 B to 4 MiB with rank 1 unaligned. Measurement (the runs under "Op selection under rank-local staging, and alternating flags-only directions" below): in every run each rank's receipt counts of launched ops (pair exchanges and their bytes, link ops by kind and by blocks per role, transport, chain and point-to-point ops) and of carried transfers (native link ops and items, one-shot ops and phases) equal its aligned run's, and the launched counts are the same on every rank; every output exact; the cold capture is taken on every rank, each of rank 1's staged calls as a graph allocation (16 on the pair default, 20 under the ring schedule on two ranks), and both replays are exact; rank 1 keeps 19 staging buffers under the chain and ring schedules. The same checks against snapshot ba5a337b's library, with a 15 s wait limit: under the pair default, rank 1 refused its cold capture of the staged calls with `ncclInvalidUsage` while rank 0 captured them, rank 0's replays returned wrong outputs at the wait limit and the child's destroy failed; under the ring schedule, rank 1 also refused the 18th and 19th all-gathers of the growth check (`ncclInternalError`, too many staging buffer sizes) while rank 0 launched them, and rank 0's session was poisoned at the wait limit. Conclusion: in these runs a rank's alignment, in-place buffers, role and staging state change no rank's ops | | Progress thread placement: `SIRCL_PROGRESS_CPU` pins it; otherwise `LIBSIRCL_CPU_POLICY=performance` (default) starts it on the fastest CPU class the creating thread may use (GB10: the Cortex-X925 cores), `none` leaves it to the scheduler; the application's threads keep their affinity; receipts name the thread's CPUs (`progress_cpus`) | implemented | build and CPU suites (the workstation's CPUs are of one class); on GB10s (Sparks 6-7, snapshot 0f94c72b, hardware evidence) the default policy started the thread on CPUs 5-9 and 15-19 and `SIRCL_PROGRESS_CPU=19` pinned it to CPU 19, as each receipt's `progress_cpus` records | | The chain and link areas in the arena after the control line, the link counters, piece counters and decision words, SIRCL's link settings (`SIRCL_LINK_*`, staggers) agreed at setup; the ring plan from `LIBSIRCL_RING_WINDOW`; forward windows of relayed lanes from `LIBSIRCL_FORWARD_WINDOWS`; `tools/site_routes.py` prints every rank's route map, chain order, forward windows and ring plan from SIRCL's route planner | implemented | CPU suites; setup-failure cases; `tools/site_routes.py` reproduces the pair's route maps and gives `path:0-3` a ring window of 393,216 bytes on position 3 | -| Teardown: `ncclCommDestroy` and `ncclCommFinalize` of a communicator of two or more ranks wait for this rank's work, then a bootstrap round of every rank that also carries each rank's health, then stop the progress thread, then a second round, before any queue pair, registration or arena is freed; each round bounded by the wait limit; the close is terminal (new work refused, its result kept); a failed release of the transport keeps the memory allocated; `ncclCommAbort` does not wait | implemented | Conditions: GPU emulation on the workstation and on GB10s (snapshot 7d1c6f19, built in the serving image); `tests/emulation/teardown_race.py`: a reversed `ncclCommSplit` child, one ring all-reduce, `ncclCommDestroy` at once, repeated, one rank's writes delayed 5 ms so its predecessor finishes first; with `--expect close-error`, a rank 4 s late under a 2 s wait limit, or the stand-in's deregistrations failed (`SIRCL_EMU_FAIL_DEREG=1`). Measurement: four ranks and two ranks at 8 MiB per rank, 20 rounds each, every round exact with no async error and the parent healthy (GB10: four ranks, 5 runs of 20 rounds, 0 failed); without the rounds (snapshot 6fe2882f) the same check failed 7 of 10 rounds on four ranks on the workstation and 98 of 100 on a GB10 (each failure poisons the parent): a rank's kernel had completed while its progress thread still held an outbound item gated by a peer's credit, destroy stopped that thread and freed its queue pairs, the peer waited out the wait limit, and the peer's late credit write failed (transport retry counter exceeded; the destination queue pair was gone). In close-error mode (workstation and GB10) every round's first finalize or destroy returned the error, a second finalize the same, and a collective after the close was refused. The full emulation suite on a GB10 (snapshot 7d1c6f19) passed every section, the teardown check on four ranks and on two at 8 MiB included. On the fabric (ConnectX-7, four Sparks on a path and a cabled pair, 8 MiB per rank, no delayed rank): 100 rounds each without the rounds and 60 with them (snapshot 7d1c6f19), all exact and healthy, and 10 runs of the relayed-group bit-exact check under the ring schedules (RUNBOOK 4.1) on the path, 0 failed (Sparks 0-3 and 0-1, `ccl-pair/done-hw-teardown-7d1-20261008T194629Z`); the same 60 rounds per layout and 10 checks at snapshot ba5a337b on Sparks 4-7 and 4-5, 0 failed (`ccl-pair/hw-teardown-ba5-47-20261008T225027Z`); the stranded item did not occur at natural timing there; on the cabled pair at positions 6-7 (snapshot 0f94c72b, the rounds of this tree, 2026-10-09): 6 runs of 20 rounds at 8 MiB, 3 with fail-stop off and 3 on, 0 failed, the parent exact and healthy. Result: round 1 proves that every item and flag a kernel waits for has landed; round 2 that no rank posts toward queue pairs or regions about to be freed. Writes no kernel waits for (credits) may still be in flight; each rank destroys its queue pairs before its registrations and memory, so they land in registered memory or are dropped. Limits: the rounds are bounded, but this rank's own wait for its work, the progress thread's join and the verbs destroys are not; in emulation the stand-in resolves a write under its registry lock and applies it after, so a write resolved before a deregistration still lands in the old segment, and emulation does not test a NIC's revocation of a deregistered region. Callers: torch 2.10's `destroy_process_group` shuts its groups down in one order on every rank (finalize, then destroy); vLLM 0.19.1's pynccl `destroy` calls `ncclCommDestroy`; a caller that tears down with `ncclCommAbort` runs no rounds | +| Teardown: `ncclCommDestroy` and `ncclCommFinalize` of a communicator of two or more ranks (also after `ncclCommRevoke`) wait for this rank's work, then a bootstrap round of every rank that also carries each rank's health, then stop the progress thread, then a second round, before any queue pair, registration or arena is freed; each round bounded by the wait limit; the close is terminal (new work refused, its result kept); a failed release of the transport keeps the memory allocated; `ncclCommAbort` does not wait | implemented | Conditions: GPU emulation on the workstation and on GB10s (snapshot 7d1c6f19, built in the serving image); `tests/emulation/teardown_race.py`: a reversed `ncclCommSplit` child, one ring all-reduce, `ncclCommDestroy` at once, repeated, one rank's writes delayed 5 ms so its predecessor finishes first; with `--expect close-error`, a rank 4 s late under a 2 s wait limit, or the stand-in's deregistrations failed (`SIRCL_EMU_FAIL_DEREG=1`). Measurement: four ranks and two ranks at 8 MiB per rank, 20 rounds each, every round exact with no async error and the parent healthy (GB10: four ranks, 5 runs of 20 rounds, 0 failed); without the rounds (snapshot 6fe2882f) the same check failed 7 of 10 rounds on four ranks on the workstation and 98 of 100 on a GB10 (each failure poisons the parent): a rank's kernel had completed while its progress thread still held an outbound item gated by a peer's credit, destroy stopped that thread and freed its queue pairs, the peer waited out the wait limit, and the peer's late credit write failed (transport retry counter exceeded; the destination queue pair was gone). In close-error mode (workstation and GB10) every round's first finalize or destroy returned the error, a second finalize the same, and a collective after the close was refused. The full emulation suite on a GB10 (snapshot 7d1c6f19) passed every section, the teardown check on four ranks and on two at 8 MiB included. On the fabric (ConnectX-7, four Sparks on a path and a cabled pair, 8 MiB per rank, no delayed rank): 100 rounds each without the rounds and 60 with them (snapshot 7d1c6f19), all exact and healthy, and 10 runs of the relayed-group bit-exact check under the ring schedules (RUNBOOK 4.1) on the path, 0 failed (Sparks 0-3 and 0-1, `ccl-pair/done-hw-teardown-7d1-20261008T194629Z`); the same 60 rounds per layout and 10 checks at snapshot ba5a337b on Sparks 4-7 and 4-5, 0 failed (`ccl-pair/hw-teardown-ba5-47-20261008T225027Z`); the stranded item did not occur at natural timing there; on the cabled pair at positions 6-7 (snapshot 0f94c72b, the rounds of this tree, 2026-10-09): 6 runs of 20 rounds at 8 MiB, 3 with fail-stop off and 3 on, 0 failed, the parent exact and healthy. Result: round 1 proves that every item and flag a kernel waits for has landed; round 2 that no rank posts toward queue pairs or regions about to be freed. Writes no kernel waits for (credits) may still be in flight; each rank destroys its queue pairs before its registrations and memory, so they land in registered memory or are dropped. Limits: the rounds are bounded, but this rank's own wait for its work, the progress thread's join and the verbs destroys are not; in emulation the stand-in resolves a write under its registry lock and applies it after, so a write resolved before a deregistration still lands in the old segment, and emulation does not test a NIC's revocation of a deregistered region. Callers: torch 2.10's `destroy_process_group` shuts its groups down in one order on every rank (finalize, then destroy); vLLM 0.19.1's pynccl `destroy` calls `ncclCommDestroy`; a caller that tears down with `ncclCommAbort` runs no rounds | | Destroying or finalizing communicators in a different order on different ranks | unsupported | each teardown round waits for every rank of its communicator, so crossed orders wait out the wait limit on each round (20 s serving, 600 s startup) and return `ncclRemoteError`; they do not wait longer | | Process exit without `ncclCommDestroy`, `ncclCommFinalize` or `ncclCommAbort` while a peer's collective on the communicator still runs | unsupported | at library unload only the native threads stop; a peer may wait out its wait limit. Calls still running on other threads are cancelled, and the unload waits up to 1 s for them to leave before it frees their communicators; a communicator still in use after that stays allocated (CPU: `tests/test_lifecycle.py`, exit and `dlclose` with a split in progress, also under `make check-asan`) | | `ncclCommDestroy` while replays of captured CUDA graphs of the communicator's collectives still run | unsupported (caller contract) | teardown waits for the engine's last eager launch (a launch under capture records no stream), not for graph replays; a communicator used only under capture has nothing to wait for | diff --git a/spark_transport/libsircl/src/api.c b/spark_transport/libsircl/src/api.c index d681df50..b558c6bf 100644 --- a/spark_transport/libsircl/src/api.c +++ b/spark_transport/libsircl/src/api.c @@ -898,8 +898,10 @@ static ncclResult_t transition(ncclComm_t handle, int next) { static ncclResult_t quiesce(struct ncclComm *comm, const char *call) { if (comm->quiesced) return (ncclResult_t)comm->close_result; if (!comm->engine || comm->bootstrap_only || comm->nranks < 2 || !comm->bootstrap) return ncclSuccess; + /* A revoked communicator runs the rounds too: revoke stops new work, not the kernels and forwards already + * in flight, so freeing without the rounds could pull queue pairs and regions from under a peer's writes. */ int state = atomic_load(&comm->state); - if (state != READY && state != FINALIZED && state != FAILED) return ncclSuccess; + if (state == INIT) return ncclSuccess; comm->quiesced = 1; if (state == READY) atomic_store(&comm->state, FINALIZED); char message[700] = {0}; diff --git a/spark_transport/libsircl/tests/emulation/teardown_race.py b/spark_transport/libsircl/tests/emulation/teardown_race.py index 3f28e37e..6edfd994 100644 --- a/spark_transport/libsircl/tests/emulation/teardown_race.py +++ b/spark_transport/libsircl/tests/emulation/teardown_race.py @@ -25,6 +25,16 @@ SIRCL_STARTUP_WAIT_S below S) or when releasing the transport fails (--env SIRCL_EMU_FAIL_DEREG=1: the stand-in fails every deregistration, and destroy keeps the arena allocated). +With --expect concurrent-close each round's child is finalized and destroyed while another thread of the +rank keeps enqueueing all-reduces on it; the round passes when both calls succeed, the enqueueing thread +ends, and its calls returned success or a refusal (ncclInvalidArgument or ncclInvalidUsage) once the close +began. + +With --expect revoke-destroy each round revokes the child (ncclCommRevoke) right after its all-reduce and +then destroys it. Destroy after revoke runs the same two teardown rounds as destroy of a ready +communicator, so the rounds pass as with exact: every child all-reduce is exact, revoke and destroy succeed, +and the parent stays healthy. + With --expect fail-stop (and --env LIBSIRCL_FAIL_STOP=1, a late rank and SIRCL_STARTUP_WAIT_S below its delay) the rounds run as with exact, and the runner checks fail-stop instead: every rank but the late one must end with exit status 70 and the fail-stop line of a timed-out wait, whether the watcher or the library @@ -78,6 +88,7 @@ def run_rank(args) -> int: ctypes.c_void_p] lib.ncclCommDestroy.argtypes = [ctypes.c_void_p] lib.ncclCommFinalize.argtypes = [ctypes.c_void_p] + lib.ncclCommRevoke.argtypes = [ctypes.c_void_p, ctypes.c_int] lib.ncclCommGetAsyncError.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_int)] lib.sirclGetReceipt.argtypes = [ctypes.c_void_p, ctypes.c_char_p, ctypes.c_size_t, ctypes.POINTER(ctypes.c_size_t)] @@ -159,14 +170,17 @@ def enqueue(): rounds.append(entry) save() continue + revoked = lib.ncclCommRevoke(child, 0) if args.expect == "revoke-destroy" else 0 + if revoked: + entry["revoke_error"] = lib.ncclGetLastError(None).decode(errors="replace") state = ctypes.c_int(0) lib.ncclCommGetAsyncError(child, ctypes.byref(state)) if not exact or state.value: entry["async_error"] = state.value entry["error"] = lib.ncclGetLastError(child).decode(errors="replace") destroyed = lib.ncclCommDestroy(child) - entry.update(ok=exact and not state.value and destroyed == 0, exact=exact, destroy=destroyed, - seconds=round(time.perf_counter() - started, 3)) + entry.update(ok=exact and not state.value and not revoked and destroyed == 0, exact=exact, destroy=destroyed, + revoke=revoked, seconds=round(time.perf_counter() - started, 3)) if destroyed: entry["destroy_error"] = lib.ncclGetLastError(None).decode(errors="replace") rounds.append(entry) @@ -300,7 +314,8 @@ def main(argv=None) -> int: parser.add_argument("--slow-rank", type=int, default=0) parser.add_argument("--slow-ns", type=int, default=5_000_000) parser.add_argument("--timeout", type=float, default=1800) - parser.add_argument("--expect", choices=("exact", "close-error", "fail-stop", "concurrent-close"), default="exact") + parser.add_argument("--expect", choices=("exact", "close-error", "fail-stop", "concurrent-close", + "revoke-destroy"), default="exact") parser.add_argument("--late-rank", type=int, default=-1, help="the rank that starts each all-reduce late") parser.add_argument("--late-s", type=float, default=0.0) parser.add_argument("--env", action="append", default=[], help="NAME=VALUE for every rank") From f83438dbbb4e3f1f1c498f8a30c769dd75b980e2 Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Thu, 15 Oct 2026 08:13:50 -0500 Subject: [PATCH 10/65] Refuse lifecycle calls on an initializing communicator ncclCommRevoke and ncclCommFinalize on a communicator whose initialization is still in progress return ncclInvalidUsage, naming the reason, as an enqueue on it already does. ncclInProgress never becomes a group's result: ncclGroupEnd records the group's result as the terminal error of its pending initializations, and a terminal ncclInProgress reads as "still initializing" forever. Compatibility: these calls returned ncclInProgress before; callers that poll a non-blocking communicator with ncclCommGetAsyncError are unaffected. Validation: tests/test_lifecycle.py calls revoke and finalize on a non-blocking communicator still initializing, inside a group that also initializes a one-rank communicator. Before the change revoke returned ncclInProgress; after it both calls return ncclInvalidUsage, the group's end does not return ncclInProgress, and the grouped communicator's error is not ncclInProgress. make check and make check-asan pass. Request item 9 (audit 03-F13d). NVIDIA NCCL source was neither read nor adapted. Co-Authored-By: Claude Opus 5.5 --- spark_transport/libsircl/src/api.c | 13 ++++++++--- .../libsircl/tests/test_lifecycle.py | 22 +++++++++++++++++++ 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/spark_transport/libsircl/src/api.c b/spark_transport/libsircl/src/api.c index b558c6bf..a52d5a3c 100644 --- a/spark_transport/libsircl/src/api.c +++ b/spark_transport/libsircl/src/api.c @@ -76,9 +76,11 @@ static int debug_errors(void) { const char *debug = sccl_env("NCCL_DEBUG"); return debug && (!strcasecmp(debug, "WARN") || !strcasecmp(debug, "INFO") || !strcasecmp(debug, "TRACE")); } +/* ncclInProgress never becomes a group's result: ncclGroupEnd would record it as the terminal error of the + * group's pending initializations, which would then read as in progress forever. */ static ncclResult_t fail(ncclResult_t result, const char *message) { snprintf(last_error, sizeof(last_error), "%s", message); - if (group_depth && group_error == ncclSuccess) group_error = result; + if (group_depth && group_error == ncclSuccess && result != ncclInProgress) group_error = result; if (debug_errors()) fprintf(stderr, "libsircl: %s: %s\n", ncclGetErrorString(result), message); return result; } @@ -878,12 +880,17 @@ static ncclResult_t transition(ncclComm_t handle, int next) { if (!comm) return fail(ncclInvalidArgument, "unknown communicator handle"); int state = atomic_load(&comm->state); ncclResult_t result = (ncclResult_t)atomic_load(&comm->error); - if (result == ncclSuccess) { + const char *why = "communicator is not ready for this lifecycle operation"; + if (result == ncclInProgress) { + /* As collective_ready refuses an enqueue: the initialization has not finished. */ + result = ncclInvalidUsage; + why = "communicator initialization is still in progress"; + } else if (result == ncclSuccess) { if (state == READY || (state == next)) atomic_store(&comm->state, next); else result = ncclInvalidUsage; } release(comm); - return result == ncclSuccess ? result : fail(result, "communicator is not ready for this lifecycle operation"); + return result == ncclSuccess ? result : fail(result, why); } /* Teardown of a communicator of two or more ranks, so that no rank frees what a peer still writes. This rank's * enqueued work completes; round 1 over the bootstrap carries every rank's status (its wait, then its health: diff --git a/spark_transport/libsircl/tests/test_lifecycle.py b/spark_transport/libsircl/tests/test_lifecycle.py index b28fc878..9195f3b4 100644 --- a/spark_transport/libsircl/tests/test_lifecycle.py +++ b/spark_transport/libsircl/tests/test_lifecycle.py @@ -51,6 +51,28 @@ def test_abort_cancels_an_inflight_split(self): if handle: self.lib.ncclCommAbort(handle) + def test_lifecycle_call_on_an_initializing_communicator_is_refused(self): + """ncclCommRevoke or ncclCommFinalize on a communicator whose initialization is still in progress is + refused with ncclInvalidUsage. ncclInProgress never becomes a group's result, so a group's pending + initialization never ends FAILED with ncclInProgress, which would read as still in progress forever.""" + os.environ["SIRCL_BOOTSTRAP_TIMEOUT_MS"] = "5000" + config = api.config() + config.blocking = 0 + pending, grouped = C.c_void_p(), C.c_void_p() + first, second = self.uid(), self.uid() + self.assertEqual(self.lib.ncclCommInitRankConfig(C.byref(pending), 2, first, 0, C.byref(config)), PROGRESS) + try: + self.assertEqual(self.lib.ncclGroupStart(), SUCCESS) + self.assertEqual(self.lib.ncclCommInitRank(C.byref(grouped), 1, second, 0), SUCCESS) + self.assertEqual(self.lib.ncclCommRevoke(pending, 0), USAGE) + self.assertEqual(self.lib.ncclCommFinalize(pending), USAGE) + self.assertNotEqual(self.lib.ncclGroupEnd(), PROGRESS) + self.assertNotEqual(self.asynchronous_error(grouped), PROGRESS) + finally: + self.lib.ncclCommAbort(pending) + if grouped.value: + self.lib.ncclCommDestroy(grouped) + def test_split_leader_failure_fails_every_member_at_once(self): """The split leader cannot create the new unique id (the process holds all 64 bootstrap roots). It still takes part in the unique-id round with a failure record, so both ranks return its result at once From 8d7c2f210b153ab64e0b7a05cb6ddbd82dabc927 Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Thu, 15 Oct 2026 08:14:13 -0500 Subject: [PATCH 11/65] Document that ncclCommFinalize is synchronous for non-blocking communicators README.md's teardown contract states that ncclCommFinalize of a non-blocking communicator (config `blocking` 0) runs both teardown rounds before it returns and never returns ncclInProgress, and that a caller that must not block runs finalize on its own thread. Returning ncclInProgress would need a finalize worker whose lifetime interacts with abort, destroy, the unload destructor and the error word that the teardown round itself reports; that path is reachable only with a ready engine of two or more ranks, which no available test can run, so the behavior is documented instead of changed. Compatibility: documentation only. Validation: no code changed; the Markdown link check passes. Request item 10 (audit 01-F7, finalize part). NVIDIA NCCL source was neither read nor adapted. Co-Authored-By: Claude Opus 5.5 --- spark_transport/libsircl/README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/spark_transport/libsircl/README.md b/spark_transport/libsircl/README.md index b5ab0f70..00e1371e 100644 --- a/spark_transport/libsircl/README.md +++ b/spark_transport/libsircl/README.md @@ -165,7 +165,10 @@ naming it. The close is terminal: from its start the communicator refuses new wo its error, and a second `ncclCommFinalize` or the destroy that follows returns the same result. When the transport's queue pairs or registrations cannot be released, destroy keeps the communicator's memory allocated and returns `ncclSystemError`. Ranks destroy their communicators in the same order, as they issue -collectives. Finalize runs the rounds once and destroy then only frees; `ncclCommAbort` does not wait. +collectives. Finalize runs the rounds once and destroy then only frees; `ncclCommAbort` does not wait. Finalize is +synchronous for every communicator: a non-blocking one (`blocking` 0 in its config) also runs both rounds +before `ncclCommFinalize` returns, which never returns `ncclInProgress`; a caller that must not block runs +finalize on a thread of its own. Errors of single calls: a collective or point-to-point call refused with `ncclInvalidArgument` did nothing and leaves the calling thread's group as it was (the group's other calls still run at `ncclGroupEnd`); other From ae779929d602168a50a4208c3507e348ec7a6125 Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Thu, 15 Oct 2026 08:16:55 -0500 Subject: [PATCH 12/65] Poison the point-to-point channels and notify peers on abort ncclCommAbort on a communicator with point-to-point channels sets the channels' host-mapped poison word before the idle check. Channel kernels return once they see it, and SIRCL's progress thread takes its failure path, which writes an abort notice into every peer's abort word. The engine waits up to 50 ms for the progress thread to report the failure and then stops it; stopping joins the thread after the notices are posted. Peers report the channels' failure naming this rank instead of waiting out their wait limit, which is 600 s in the startup regime. Collective kernels are unaffected: stopping them needs an abort word in SIRCL's collective kernels, which is a kernel change outside this library's scope. Compatibility: abort of a communicator without channels is unchanged. Abort of one with channels takes up to 50 ms longer and makes every peer's channels fail with ncclRemoteError naming the aborting rank. Validation: the path needs live channels, which the CPU harness cannot build. Written, not run: python tests/emulation/p2p_channels.py --library build/libsircl.so \ --case abort make check and make check-asan pass. Request item 11 (audit 05-F2, notification part of 04-F5). NVIDIA NCCL source was neither read nor adapted. Co-Authored-By: Claude Opus 5.5 --- spark_transport/libsircl/src/engine.c | 22 +++++++++++ .../libsircl/tests/emulation/p2p_channels.py | 38 ++++++++++++++++++- 2 files changed, 58 insertions(+), 2 deletions(-) diff --git a/spark_transport/libsircl/src/engine.c b/spark_transport/libsircl/src/engine.c index 46f1f458..9ce8dcda 100644 --- a/spark_transport/libsircl/src/engine.c +++ b/spark_transport/libsircl/src/engine.c @@ -1739,6 +1739,24 @@ static int p2p_close(sccl_engine *e, int free_memory) { return 0; } +/* Stop the point-to-point channels so that every peer learns of it: poison the control line, so this rank's + * channel kernels return and its progress thread takes its failure path, which writes an abort notice into + * every peer's abort word (SIRCL's post_aborts); wait up to P2P_NOTICE_WAIT_NS for the thread to report the + * failure, then stop it, which joins it after the notices were posted. A thread that has not reported by + * then is stopped without notices; peers then fall back to their own wait limits. Not after the progress + * thread was stopped (teardown_stop), when no thread is left to post. */ +enum { P2P_NOTICE_WAIT_NS = 50000000 }; +static void p2p_fault(sccl_engine *e, const char *why); +static uint64_t monotonic_ns(void); +static void p2p_stop_with_notices(sccl_engine *e, const char *why) { + if (!e->p2p || !e->p2p_ctrl || e->stopped) return; + p2p_fault(e, why); + uint64_t until = monotonic_ns() + P2P_NOTICE_WAIT_NS; + const struct timespec pause = {0, 100000}; + while (!e->p2p_transport->failed(e->p2p) && monotonic_ns() < until) nanosleep(&pause, NULL); + e->p2p_transport->stop(e->p2p); +} + /* After the first setup exchange gave every rank's p2p_out: ranks a and b have a channel when each one's lanes * toward the other can carry items, so every rank derives the same table. A rank without a channel still * opens a context (every rank's connection record is validated together); when no pair of the group has one, @@ -4584,6 +4602,10 @@ ncclResult_t sccl_engine_destroy(sccl_engine *e, int abort, char *err, size_t le result = ncclUnhandledCudaError; put_error(err, len, "waiting for the communicator's context failed; its arena stays allocated"); } + /* Abort stops the channels first: their kernels return once they see the poison word, and every peer gets + * the abort notice instead of waiting out its wait limit. A channel kernel still running at the idle check + * below keeps the arena allocated (quarantined). */ + if (abort) p2p_stop_with_notices(e, "ncclCommAbort"); if (pushed && abort) { if (e->have_last) idle = cu->StreamQuery(e->last_stream) == SCCL_CUDA_SUCCESS; for (int d = 0; d < 2; ++d) diff --git a/spark_transport/libsircl/tests/emulation/p2p_channels.py b/spark_transport/libsircl/tests/emulation/p2p_channels.py index c0869dc2..44ffb5f6 100644 --- a/spark_transport/libsircl/tests/emulation/p2p_channels.py +++ b/spark_transport/libsircl/tests/emulation/p2p_channels.py @@ -39,7 +39,10 @@ the others rank 1 as its origin) and ncclCommAbort returns. ``--case gone``: four ranks with LIBSIRCL_FAIL_STOP=1 and a 3 s wait limit; rank 3 ends its process after setup, rank 0 receives from it: its wait times out, and ranks 0, 1 and 2 end with status 70 (rank 0 by its own timeout, 1 and 2 by the abort -notice). ``--case setup``: three groups of three ranks whose setup must fail on every rank, naming the +notice). ``--case abort``: four ranks with a 60 s wait limit; rank 1 enqueues a receive from rank 0 that no +send matches and calls ncclCommAbort 0.5 s later: the abort returns within 2 s, the receive's kernel stops +within 5 s, and ranks 0, 2 and 3 report the channels' failure naming rank 1 within 20 s, long before the wait +limit, because the abort poisons the channels and their progress thread posts abort notices. ``--case setup``: three groups of three ranks whose setup must fail on every rank, naming the reason: LIBSIRCL_P2P_CHANNELS on rank 0 only, SIRCL_P2P_THREADS=1024 on rank 1 (the pack runs at most 512), and a malformed LIBSIRCL_P2P_WINDOWS on rank 2. @@ -163,6 +166,35 @@ def call(send, view, peer): report("ncclCommAbort", lib.ncclCommAbort(comm) == 0, last_error()) save() return 0 + if args.case == "abort": + x = torch.zeros(4096, dtype=torch.uint8, device="cuda") + torch.cuda.synchronize() + if rank == 1: + code = call(False, x, 0) + report("a receive that no send matches enqueues", code == 0, last_error() if code else "") + time.sleep(0.5) + started = time.monotonic() + aborted = lib.ncclCommAbort(comm) + seconds = time.monotonic() - started + report(f"ncclCommAbort returned within {seconds:.2f} s", aborted == 0 and seconds < 2, last_error()) + started = time.monotonic() + stream.synchronize() + seconds = time.monotonic() - started + report(f"the receive's kernel stopped {seconds:.2f} s after the abort", seconds < 5) + save() + return 0 + status, message, started = ctypes.c_int(0), "", time.monotonic() + while time.monotonic() - started < 20: + lib.ncclCommGetAsyncError(comm, ctypes.byref(status)) + if status.value: + message = last_error(comm) + break + time.sleep(0.05) + report(f"rank 1's abort reported within {time.monotonic() - started:.2f} s", + status.value != 0 and "failure on rank 1" in message, f"result {status.value}: {message}") + report("ncclCommAbort", lib.ncclCommAbort(comm) == 0, last_error()) + save() + return 0 if args.case == "gone": save() if rank == 3: @@ -392,6 +424,8 @@ def run_group(args, extra_env=None, label="") -> int: }) if args.case == "gone": base.update({"SIRCL_STARTUP_WAIT_S": "3", "LIBSIRCL_FAIL_STOP": "1"}) + if args.case == "abort": + base.update({"SIRCL_STARTUP_WAIT_S": "60"}) base.update(item.split("=", 1) for item in args.env) started = time.perf_counter() processes = [] @@ -485,7 +519,7 @@ def main(argv=None) -> int: parser.add_argument("--world", type=int, default=4) parser.add_argument("--lanes", type=int, default=2) parser.add_argument("--layout", choices=("none", "ring8", "ring8-alone", "path4"), default="none") - parser.add_argument("--case", choices=("traffic", "size", "gone", "setup"), default="traffic") + parser.add_argument("--case", choices=("traffic", "size", "gone", "abort", "setup"), default="traffic") parser.add_argument("--rounds", type=int, default=3) parser.add_argument("--timeout", type=float, default=1200) parser.add_argument("--env", action="append", default=[], metavar="NAME=VALUE", From 39f31d9d4b01a57fcd4494682ead2bf9bb88b20a Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Thu, 15 Oct 2026 08:18:04 -0500 Subject: [PATCH 13/65] Post point-to-point abort notices before fail-stop ends the process fail_stop poisons a communicator's point-to-point channels, waits up to 50 ms for SIRCL's progress thread to report the failure, and stops the thread, which joins it after post_aborts wrote the abort notice into every peer's abort word. Only then does it write its line and receipt and end the process. When the failure came from the channels, the thread is already on its failure path, and the stop waits for the notices it is posting; when it came from a collective, the peers' channels learn of it too. SIRCL's progress thread stores `failed` and the poison word before it posts the notices (src/transport/sircl_p2p_proxy.c), so without the wait the watcher or a library call could end the process between the two. The vendored proxy is unchanged. Compatibility: fail-stop of a communicator with channels takes up to about 52 ms longer before the process exits. Communicators without channels, or whose progress thread already stopped at teardown, are unchanged. Validation: the window is microseconds against the watcher's 5 ms poll, so no available test reproduces the race. The emulation case that checks the notices reach the peers is python tests/emulation/p2p_channels.py --library build/libsircl.so \ --case gone (not run here; ranks 1 and 2 must end with status 70 through the notice). make check and make check-asan pass. Request item 12 (audit 04-F2). NVIDIA NCCL source was neither read nor adapted. Co-Authored-By: Claude Opus 5.5 --- spark_transport/libsircl/src/engine.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/spark_transport/libsircl/src/engine.c b/spark_transport/libsircl/src/engine.c index 9ce8dcda..43ec43a2 100644 --- a/spark_transport/libsircl/src/engine.c +++ b/spark_transport/libsircl/src/engine.c @@ -1929,6 +1929,9 @@ static ncclResult_t session_error(sccl_engine *e, char *message, size_t len); * timeouts and of Python's time.monotonic), which a stepped wall clock (a VM's time synchronization) does * not move; and what found the error. */ static void fail_stop(sccl_engine *e, const char *found_by, const char *message) { + /* Before the process ends, the channels' progress thread posts its abort notices, so the peers' channels + * stop now instead of at their wait limits (bounded; see p2p_stop_with_notices). */ + p2p_stop_with_notices(e, message); char line[1200]; uint64_t now = realtime_ns(), mono = monotonic_ns(); int n = snprintf(line, sizeof line, From 30520a47990d2ba3e7ff3b7313a3258ab3431c62 Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Thu, 15 Oct 2026 08:21:50 -0500 Subject: [PATCH 14/65] Keep large reductions within the kernels' int32 pack counts The chain, link and point-to-point kernels take pack counts (16-byte units) as int32_t, so an op of 32 GiB or more wrapped its count. A large all-reduce or reduce whose body exceeds the chain's limit (2^31 - 1 packs) now runs in pieces instead of one chain op; the plan lives in src/link_plan.h (sccl_large_reduce_plan), which the engine and a C unit test share. The reduce-scatter's link op is not chosen for a chunk whose pack count does not fit (the chunk is also its stride). Every remaining narrowing site refuses with ncclInvalidArgument and a message instead of launching with a wrapped count: launch_chain, launch_link_entry, and a point-to-point channel message above 32 GiB - 16 bytes. Compatibility: reductions below 32 GiB keep their plan and results. Larger ones run in pieces (correct, slower) where they returned success with a wrong amount reduced; a channel send or receive of 32 GiB or more is refused. Validation: tests/test_link_plan.c (make check and CMake's link_plan test) checks the plan at the limits; with the pack-count check removed from the plan it fails at the 32 GiB chain case. make check and make check-asan pass. Request item 13 (audit 10-F8a, 15-F12). NVIDIA NCCL source was neither read nor adapted. Co-Authored-By: Claude Opus 5.5 --- spark_transport/libsircl/CMakeLists.txt | 4 ++ spark_transport/libsircl/Makefile | 5 ++- spark_transport/libsircl/src/engine.c | 35 ++++++++++++----- spark_transport/libsircl/src/link_plan.h | 36 +++++++++++++++++ .../libsircl/tests/test_link_plan.c | 39 +++++++++++++++++++ 5 files changed, 109 insertions(+), 10 deletions(-) create mode 100644 spark_transport/libsircl/src/link_plan.h create mode 100644 spark_transport/libsircl/tests/test_link_plan.c diff --git a/spark_transport/libsircl/CMakeLists.txt b/spark_transport/libsircl/CMakeLists.txt index 77628c0a..a2bcc3f3 100644 --- a/spark_transport/libsircl/CMakeLists.txt +++ b/spark_transport/libsircl/CMakeLists.txt @@ -113,6 +113,10 @@ add_executable(test_fabric tests/test_fabric.c src/fabric.c) target_include_directories(test_fabric PRIVATE include tests) target_compile_options(test_fabric PRIVATE -Wall -Wextra -Werror) add_test(NAME fabric COMMAND test_fabric) +add_executable(test_link_plan tests/test_link_plan.c) +target_include_directories(test_link_plan PRIVATE src) +target_compile_options(test_link_plan PRIVATE -Wall -Wextra -Werror) +add_test(NAME link_plan COMMAND test_link_plan) foreach(name abi api lifecycle engine_cpu) add_test(NAME ${name} COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tests/test_${name}.py --library $) endforeach() diff --git a/spark_transport/libsircl/Makefile b/spark_transport/libsircl/Makefile index 5c26c0b1..f69c3dbb 100644 --- a/spark_transport/libsircl/Makefile +++ b/spark_transport/libsircl/Makefile @@ -166,9 +166,12 @@ $(BUILD)/mpi-shim/lib/libmpi.so: tools/mpi-shim/mpi_shim.c tools/mpi-shim/includ $(BUILD)/test_fabric: tests/test_fabric.c $(BUILD)/fabric.o $(CC) $(CPPFLAGS) $(CFLAGS) $^ -o $@ -check: all $(BUILD)/test_fabric +$(BUILD)/test_link_plan: tests/test_link_plan.c src/link_plan.h | $(BUILD) + $(CC) $(CPPFLAGS) $(CFLAGS) $< -o $@ +check: all $(BUILD)/test_fabric $(BUILD)/test_link_plan @status=0; \ $(BUILD)/test_fabric || status=1; \ + $(BUILD)/test_link_plan || status=1; \ $(PYTHON) tests/test_abi.py --library $(abspath $(BUILD)/libsircl.so) || status=1; \ $(PYTHON) tests/test_api.py --library $(abspath $(BUILD)/libsircl.so) || status=1; \ $(PYTHON) tests/test_bootstrap.py || status=1; \ diff --git a/spark_transport/libsircl/src/engine.c b/spark_transport/libsircl/src/engine.c index 43ec43a2..b60e4049 100644 --- a/spark_transport/libsircl/src/engine.c +++ b/spark_transport/libsircl/src/engine.c @@ -18,6 +18,7 @@ #include #include "kernelpack.h" +#include "link_plan.h" #include "transport/shm_verbs.h" /* -- SIRCL's native proxy (src/transport/sircl_roce_proxy.c), compiled once per transport ----- */ @@ -2656,6 +2657,11 @@ static ncclResult_t launch_link_entry(sccl_engine *e, sccl_CUfunction fn, unsign sccl_CUdeviceptr src, sccl_CUdeviceptr dst, sccl_CUdeviceptr scratch, uint64_t chunk, uint64_t stride, uint32_t piece, uint32_t blocks, int32_t flags, sccl_CUstream stream, char *err, size_t len) { + if (!sccl_packs_fit(chunk) || !sccl_packs_fit(stride) || !sccl_packs_fit(piece)) { + put_error(err, len, "%s: chunk %" PRIu64 " or stride %" PRIu64 " bytes: the pack count exceeds the kernel's int32 " + "argument", what, chunk, stride); + return ncclInvalidArgument; + } sccl_link_args a; memset(&a, 0, sizeof a); a.input = src; @@ -2827,7 +2833,8 @@ static int gather_link_kind(const sccl_engine *e, uint64_t shard) { * -1; chunks a multiple of 16 bytes. */ static int scatter_link_kind(const sccl_engine *e, uint64_t chunk) { uint64_t input = chunk * (uint64_t)e->world; - if (!e->link_on || chunk == 0 || chunk % PACK) return -1; + /* The chunk is also the ops' stride, a pack count of the kernels' int32 arguments. */ + if (!e->link_on || chunk == 0 || chunk % PACK || !sccl_packs_fit(chunk)) return -1; uint32_t schedule = effective_schedule(e, e->shared.scatter_schedule, input, COLL_SCATTER); if (schedule == SCHEDULE_RING) return SCCL_RING_SCATTER; if (schedule == SCHEDULE_PIECES) return -1; @@ -2918,6 +2925,10 @@ static ncclResult_t reduce_link(sccl_engine *e, int ring, int dtype, sccl_CUdevi /* One chain op over `nbytes` (a positive multiple of 16) from src to dst, both 16-byte aligned. */ static ncclResult_t launch_chain(sccl_engine *e, int dtype, sccl_CUdeviceptr src, sccl_CUdeviceptr dst, uint64_t nbytes, sccl_CUstream stream, char *err, size_t len) { + if (!sccl_packs_fit(nbytes)) { + put_error(err, len, "chain op of %" PRIu64 " bytes: its pack count exceeds the kernel's int32 argument", nbytes); + return ncclInvalidArgument; + } int32_t packs = (int32_t)(nbytes / PACK); sccl_chain_args a; memset(&a, 0, sizeof a); @@ -3416,14 +3427,13 @@ static ncclResult_t reduce_into(sccl_engine *e, const char *what, const void *se * every rank (not ncclReduce); unaligned buffers go through the staging buffer. */ if (body && e->shared.large_schedule != SCHEDULE_PIECES) { uint32_t schedule = effective_schedule(e, e->shared.large_schedule, nbytes, COLL_REDUCE); - uint64_t ring_bytes = body / (PACK * (uint64_t)e->world) * PACK * (uint64_t)e->world; - if (schedule == SCHEDULE_RING && e->link_on && ring_bytes && ring_bytes / (uint64_t)e->world < (1ull << 31)) { - result = reduce_link(e, 1, dtype, src, dst, ring_bytes, capturing, stream, err, len); - start = ring_bytes; - } else if (schedule != SCHEDULE_RING && e->chain_on && - (schedule == SCHEDULE_CHAIN || body >= e->shared.chain_mins[COLL_REDUCE])) { - result = reduce_link(e, 0, dtype, src, dst, body, capturing, stream, err, len); - start = body; + uint64_t link_bytes = 0; + int plan = sccl_large_reduce_plan(body, (uint64_t)e->world, schedule == SCHEDULE_RING, e->link_on, + e->chain_on && (schedule == SCHEDULE_CHAIN || + body >= e->shared.chain_mins[COLL_REDUCE]), &link_bytes); + if (plan != SCCL_LARGE_PIECES) { + result = reduce_link(e, plan == SCCL_LARGE_RING, dtype, src, dst, link_bytes, capturing, stream, err, len); + start = link_bytes; } } for (uint64_t offset = start; result == ncclSuccess && offset < body; offset += e->shared.large_piece) { @@ -4105,6 +4115,13 @@ ncclResult_t sccl_engine_p2p(sccl_engine *e, const sccl_p2p_op *ops, unsigned co ops[sends[1][k]].bytes, ops[recvs[1][k]].bytes); return ncclInvalidUsage; } + /* A channel item's message is a pack count of the point-to-point kernels' int32 arguments. */ + for (unsigned i = 0; e->world > 2 && i < count; ++i) + if (ops[i].peer != e->rank && !sccl_packs_fit(round_up(ops[i].bytes, PACK))) { + put_error(err, len, "%s of %zu bytes: a point-to-point channel message holds at most %" PRIu64 " bytes", + ops[i].send ? "ncclSend" : "ncclRecv", ops[i].bytes, (uint64_t)INT32_MAX * PACK); + return ncclInvalidArgument; + } char message[400]; ncclResult_t health = sccl_engine_async_error(e, message, sizeof message); if (health != ncclSuccess) { diff --git a/spark_transport/libsircl/src/link_plan.h b/spark_transport/libsircl/src/link_plan.h new file mode 100644 index 00000000..e8887e9a --- /dev/null +++ b/spark_transport/libsircl/src/link_plan.h @@ -0,0 +1,36 @@ +/* Size limits of the link and chain kernels' arguments, and the plan of a large reduction that keeps to them. + * Header-only, so tests/test_link_plan.c checks the code the engine runs. */ +#ifndef SCCL_LINK_PLAN_H +#define SCCL_LINK_PLAN_H + +#include + +enum { SCCL_PLAN_PACK = 16 }; + +/* The link, chain and point-to-point kernels take pack counts (16-byte units) as int32_t. */ +static inline int sccl_packs_fit(uint64_t bytes) { return bytes / SCCL_PLAN_PACK <= (uint64_t)INT32_MAX; } + +enum { SCCL_LARGE_PIECES = 0, SCCL_LARGE_RING = 1, SCCL_LARGE_CHAIN = 2 }; + +/* The link op of a large all-reduce or reduce whose 16-byte-aligned body is `body` bytes, over `world` ranks + * (SIRCL's pieces.reduce_plan). `ring_schedule`: the ring schedule applies to this size; `link_on`: the link + * pack is loaded; `chain`: the chain schedule, or auto from the chain minimum, with the chain pack loaded. + * Under the ring schedule: one ring op over the largest prefix of `world` equal chunks of whole packs, each + * chunk below 2^31 bytes, else none. Otherwise: one chain op over the whole body when its pack count fits, + * else none. None means the body runs in pieces. *bytes is the link op's size (0 for none). */ +static inline int sccl_large_reduce_plan(uint64_t body, uint64_t world, int ring_schedule, int link_on, int chain, + uint64_t *bytes) { + *bytes = 0; + if (!body || world < 2) return SCCL_LARGE_PIECES; + if (ring_schedule) { + uint64_t ring_bytes = body / (SCCL_PLAN_PACK * world) * SCCL_PLAN_PACK * world; + if (!link_on || !ring_bytes || ring_bytes / world >= (1ull << 31)) return SCCL_LARGE_PIECES; + *bytes = ring_bytes; + return SCCL_LARGE_RING; + } + if (!chain || !sccl_packs_fit(body)) return SCCL_LARGE_PIECES; + *bytes = body; + return SCCL_LARGE_CHAIN; +} + +#endif diff --git a/spark_transport/libsircl/tests/test_link_plan.c b/spark_transport/libsircl/tests/test_link_plan.c new file mode 100644 index 00000000..990cf9c2 --- /dev/null +++ b/spark_transport/libsircl/tests/test_link_plan.c @@ -0,0 +1,39 @@ +/* The large-reduction plan (src/link_plan.h): no link or chain op whose pack count does not fit the kernels' + * int32_t arguments is chosen; such a body runs in pieces. */ +#include "link_plan.h" + +#include +#include + +static unsigned checks; +#define CHECK(expr) do { checks++; if (!(expr)) { \ + fprintf(stderr, "%s:%d: %s\n", __FILE__, __LINE__, #expr); exit(1); } } while (0) + +int main(void) { + const uint64_t limit = (uint64_t)INT32_MAX * SCCL_PLAN_PACK; /* the largest chain op: 32 GiB - 16 bytes */ + uint64_t bytes; + CHECK(sccl_packs_fit(limit)); + CHECK(!sccl_packs_fit(limit + SCCL_PLAN_PACK)); + CHECK(!sccl_packs_fit(32ull << 30)); + + /* The chain: up to the limit as one op; from 32 GiB in pieces, never a wrapped pack count. */ + CHECK(sccl_large_reduce_plan(8u << 20, 2, 0, 1, 1, &bytes) == SCCL_LARGE_CHAIN && bytes == 8u << 20); + CHECK(sccl_large_reduce_plan(limit, 2, 0, 1, 1, &bytes) == SCCL_LARGE_CHAIN && bytes == limit); + CHECK(sccl_large_reduce_plan(32ull << 30, 2, 0, 1, 1, &bytes) == SCCL_LARGE_PIECES && bytes == 0); + CHECK(sccl_large_reduce_plan(64ull << 30, 4, 0, 1, 1, &bytes) == SCCL_LARGE_PIECES && bytes == 0); + CHECK(sccl_large_reduce_plan(8u << 20, 2, 0, 1, 0, &bytes) == SCCL_LARGE_PIECES); + + /* The ring: the largest prefix of W equal chunks of whole packs, each below 2^31 bytes. */ + CHECK(sccl_large_reduce_plan((8u << 20) + 48, 4, 1, 1, 0, &bytes) == SCCL_LARGE_RING && bytes == 8u << 20); + CHECK(sccl_large_reduce_plan(8ull << 30, 4, 1, 1, 0, &bytes) == SCCL_LARGE_PIECES); + CHECK(sccl_large_reduce_plan((8ull << 30) - 64, 4, 1, 1, 0, &bytes) == SCCL_LARGE_RING && + bytes == (8ull << 30) - 64); + CHECK(sccl_large_reduce_plan(48, 4, 1, 1, 0, &bytes) == SCCL_LARGE_PIECES); + CHECK(sccl_large_reduce_plan(8u << 20, 4, 1, 0, 1, &bytes) == SCCL_LARGE_PIECES); + + /* One rank or an empty body: nothing to plan. */ + CHECK(sccl_large_reduce_plan(8u << 20, 1, 0, 1, 1, &bytes) == SCCL_LARGE_PIECES); + CHECK(sccl_large_reduce_plan(0, 2, 0, 1, 1, &bytes) == SCCL_LARGE_PIECES); + printf("link plan: %u checks passed\n", checks); + return 0; +} From 65f613985e423985d7b9e7ffb22f4ba6ac885ede Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Thu, 15 Oct 2026 08:24:10 -0500 Subject: [PATCH 15/65] Require a research opt-in for the one-pass ring relay and agree it at setup LIBSIRCL_RING_REDUCE_PASSES=1 selects the one-pass ring all-reduce relay, which STATUS.md labels research-only: on a GB10 one run under forward windows on four ranks returned differing bits on one rank, and the cause is not isolated. Communicator creation refuses 1 with a message naming LIBSIRCL_RING_REDUCE_ONE_PASS_RESEARCH=1, which selects it. The setting joins the settings every rank compares at setup (in the record's reserved word after cycle_plan, so the record's size is unchanged), and a group whose ranks differ fails setup naming the variable. The comments in src/kernelpack.h and KERNEL_ROUTE.md state that the two-pass entry is the default and that the one-pass form's identical bytes are a design intent with one open contrary observation; README.md's variable table names the opt-in and the agreement. Compatibility: a process that sets LIBSIRCL_RING_REDUCE_PASSES=1 without the opt-in is refused at communicator creation; the default (2) keeps its behavior. Ranks that set different values fail setup instead of running different relay forms. Validation: tests/test_engine_cpu.py creates a one-rank communicator on the stand-in driver: before the change PASSES=1 without the opt-in (or with it set to 0) succeeded; after it both are refused naming the opt-in, and PASSES=1 with the opt-in and PASSES=2 succeed. make check and make check-asan pass. The strict-xfail emulation reproducer the audit asks for (12-F8) is not added: the mismatch was seen once on hardware, and an emulation case not known to reproduce it would pass and so fail as a strict xfail. Request item 14 (audit 10-F6; 12-F8 not done). NVIDIA NCCL source was neither read nor adapted. Co-Authored-By: Claude Opus 5.5 --- spark_transport/libsircl/KERNEL_ROUTE.md | 4 +++- spark_transport/libsircl/README.md | 2 +- spark_transport/libsircl/src/engine.c | 21 +++++++++++++----- spark_transport/libsircl/src/kernelpack.h | 4 ++-- .../libsircl/tests/test_engine_cpu.py | 22 +++++++++++++++++++ 5 files changed, 44 insertions(+), 9 deletions(-) diff --git a/spark_transport/libsircl/KERNEL_ROUTE.md b/spark_transport/libsircl/KERNEL_ROUTE.md index 16818fcf..04befd21 100644 --- a/spark_transport/libsircl/KERNEL_ROUTE.md +++ b/spark_transport/libsircl/KERNEL_ROUTE.md @@ -46,7 +46,9 @@ point-to-point library; they have not run in one group with SIRCL's DSL point-to `SIRCL_LINK_UNROLL`). The ring all-reduce's relay in `sircl_ring_reduce__u` stores each pack of this rank's finished piece to link 3's own slot and to the output from one load and one sum; the two-pass entries, which the library launches by default, store the slot, then copy it to the output - (`LIBSIRCL_RING_REDUCE_PASSES=1` selects the one-pass entries). Both write the same bytes and speak the same protocol, + (`LIBSIRCL_RING_REDUCE_PASSES=1` with `LIBSIRCL_RING_REDUCE_ONE_PASS_RESEARCH=1` selects the one-pass entries, and every + rank must agree). Both speak the same protocol and are designed to write the same bytes; one run of the one-pass + entries on a GB10 returned differing bits on one rank and is not isolated (`STATUS.md`), so they are research-only, and the ring all-gather's own role stores each pack to the slot and the output in one pass as well. The pair exchange entries run the ring all-gather's items and op word on a ring of two, carrying one block each way (or one way) with three roles: sending, receiving (or discarding), and copying the own block in diff --git a/spark_transport/libsircl/README.md b/spark_transport/libsircl/README.md index 00e1371e..89532b99 100644 --- a/spark_transport/libsircl/README.md +++ b/spark_transport/libsircl/README.md @@ -228,7 +228,7 @@ LF), and refuses a vendored copy that does not define the word; `P2P_FEATURES=0` | `LIBSIRCL_REDUCE_LINK_BLOCKS`, `LIBSIRCL_GATHER_LINK_BLOCKS`, `LIBSIRCL_SCATTER_LINK_BLOCKS` (or SIRCL's `SIRCL_REDUCE_LINK_BLOCKS`, `SIRCL_GATHER_LINK_BLOCKS`, `SIRCL_SCATTER_LINK_BLOCKS`) | blocks per link role (1-64) of every chain and ring link op of that collective (pair exchanges count as all-gathers); unset, `SIRCL_LINK_BLOCKS` when set, else the pair plan's on a pair, else 4. The chain all-reduce's blocks are `SIRCL_CHAIN_BLOCKS`. Ops of one kernel type may use different blocks: a launch's last block is the one whose arrival completes its own grid | | `LIBSIRCL_LINK_DUMP` | a diagnostic: a file prefix; one JSON line in `.rank..c.links.json` when a wait of a communicator first times out or its native progress thread fails, and at destroy: the time (CLOCK_REALTIME nanoseconds, the clock of the native event trace), the link area's words (credit, sent, ready, consumed and inbound flag words of every link), the native counters, the device link counters, the native event trace since the previous dump, whether the progress thread failed and its message, and on the emulation transport the stand-in's report (every queue pair's posted, executed and completed writes; every failed write with the resolution check that failed); after destroy, a line with the times the progress thread's stop was requested and completed and the transport was destroyed | | `LIBSIRCL_LINK_BLOCKS_CYCLE` | a test hook: `[,...]` gives successive link ops these blocks per role in turn, to exercise launches of one kernel type with different grids | -| `LIBSIRCL_RING_REDUCE_PASSES` | the ring all-reduce's relay: `2` (default) stores each pack of its finished piece to link 3's slot, then copies the slot to the output; `1` stores both in one pass (the same bytes and protocol, one read of the slot fewer; research-only, see `STATUS.md`); per rank | +| `LIBSIRCL_RING_REDUCE_PASSES` | the ring all-reduce's relay: `2` (default) stores each pack of its finished piece to link 3's slot, then copies the slot to the output; `1` stores both in one pass (the same protocol, one read of the slot fewer; research-only, see `STATUS.md`, and refused unless `LIBSIRCL_RING_REDUCE_ONE_PASS_RESEARCH=1`); agreed at setup, so every rank sets the same value | | `LIBSIRCL_LINK_TILE_BYTES` | bytes of every chunk one link reduce-scatter op carries (default 16 MiB); a call's chunks split into column tiles, which leave every element's arithmetic unchanged | | `LIBSIRCL_RING_WINDOW` | the ring plan: set on every rank when the ring that closes the chain can run, to the bytes this rank's ring lanes keep unacknowledged through relays (0 for cables); the ring schedules need it. One value per process serves every communicator it joins: a communicator whose ring next is reached through relays (a forward window toward it) while this gives 0, such as a pipeline pair of positions i and i + 4 of a ring of eight, keeps the smallest forward window toward that rank in flight on its ring lanes (the receipt's `ring_window_bytes`) | | `LIBSIRCL_FORWARD_WINDOWS`, `SIRCL_FORWARD_CHUNK_BYTES` | forward windows of relayed lanes: `=[/],...` per lane, 0 for a direct lane, and the chunk a windowed stripe posts in (default 32,768) | diff --git a/spark_transport/libsircl/src/engine.c b/spark_transport/libsircl/src/engine.c index b60e4049..a9cba208 100644 --- a/spark_transport/libsircl/src/engine.c +++ b/spark_transport/libsircl/src/engine.c @@ -316,8 +316,9 @@ typedef struct { * coll_blocks[c] from LIBSIRCL__LINK_BLOCKS (0: unset), blocks_set when SIRCL_LINK_BLOCKS * sets link_blocks; pair_plan when the pair plan chooses the rest by size. */ uint32_t chunk_set[COLLECTIVES], coll_blocks[COLLECTIVES], blocks_set, pair_plan; - /* The cycle plan (see CYCLE_RING_MIN_BYTES) applies. */ - uint32_t cycle_plan, cycle_reserved; + /* The cycle plan (see CYCLE_RING_MIN_BYTES) applies; the ring all-reduce's relay form + * (LIBSIRCL_RING_REDUCE_PASSES, see sccl_engine.ring_reduce_passes). */ + uint32_t cycle_plan, ring_reduce_passes; /* Point-to-point channels: on (LIBSIRCL_P2P_CHANNELS on a communicator of three or more ranks), the native * library's ABI, SIRCL's channel geometry (SIRCL_P2P_*) and the point-to-point pack. */ uint32_t p2p_on, p2p_abi, p2p_slots, p2p_blocks, p2p_threads, p2p_unroll, p2p_chunk, p2p_reserved; @@ -382,9 +383,10 @@ struct sccl_engine { uint32_t ring_window; /* The CPUs the transport's progress thread may run on (a CPU list, or "unpinned"). */ char progress_cpus[160]; - /* The ring all-reduce's form (LIBSIRCL_RING_REDUCE_PASSES): 2 (default), the relay stores its result to - * link 3's slot, then copies the slot to the output; 1, it stores both in one pass. Rank-local: both forms - * speak the same protocol and write the same bytes. */ + /* The ring all-reduce's form (LIBSIRCL_RING_REDUCE_PASSES, agreed at setup): 2 (default), the relay stores + * its result to link 3's slot, then copies the slot to the output; 1, it stores both in one pass. Both + * speak the same protocol and are designed to write the same bytes, but one run of the one-pass form on a + * GB10 returned differing bits on one rank (STATUS.md), so 1 needs LIBSIRCL_RING_REDUCE_ONE_PASS_RESEARCH=1. */ int ring_reduce_passes; uint64_t link_off; sccl_CUdeviceptr link_counters, piece_counters, link_scratch; @@ -921,7 +923,15 @@ static int read_settings(sccl_engine *e, int emulation, char *err, size_t len) { } e->fail_stop = !stop || !*stop || !strcmp(stop, "0") ? 0 : !strcmp(stop, "1") ? FAIL_STOP_EXIT : FAIL_STOP_ABORT; if (env_u64("LIBSIRCL_RING_REDUCE_PASSES", 2, 1, 2, &v, err, len)) return -1; + const char *one_pass = sccl_env("LIBSIRCL_RING_REDUCE_ONE_PASS_RESEARCH"); + if (v == 1 && (!one_pass || strcmp(one_pass, "1"))) { + put_error(err, len, "LIBSIRCL_RING_REDUCE_PASSES=1 (the one-pass ring relay) is research-only: one run returned " + "differing bits on one rank and the cause is not isolated; set LIBSIRCL_RING_REDUCE_ONE_PASS_RESEARCH=1 " + "to select it"); + return -1; + } e->ring_reduce_passes = (int)v; + s->ring_reduce_passes = (uint32_t)v; const char *policy = sccl_env("LIBSIRCL_CPU_POLICY"); if (policy && *policy && strcmp(policy, "performance") && strcmp(policy, "none")) { put_error(err, len, "LIBSIRCL_CPU_POLICY=%s: performance or none", policy); @@ -1114,6 +1124,7 @@ static const char *settings_difference(const shared_settings *a, const shared_se SAME(blocks_set, "SIRCL_LINK_BLOCKS"); SAME(pair_plan, "the pair plan (SIRCL_LARGE_SCHEDULE, LIBSIRCL_FORWARD_WINDOWS, LIBSIRCL_RING_WINDOW)"); SAME(cycle_plan, "the cycle plan (SIRCL_LARGE_SCHEDULE, LIBSIRCL_FORWARD_WINDOWS, LIBSIRCL_RING_WINDOW)"); + SAME(ring_reduce_passes, "LIBSIRCL_RING_REDUCE_PASSES"); SAME(p2p_on, "LIBSIRCL_P2P_CHANNELS"); SAME(p2p_abi, "point-to-point native ABI"); SAME(p2p_slots, "SIRCL_P2P_SLOTS"); diff --git a/spark_transport/libsircl/src/kernelpack.h b/spark_transport/libsircl/src/kernelpack.h index 2ff88c6c..6c664f57 100644 --- a/spark_transport/libsircl/src/kernelpack.h +++ b/spark_transport/libsircl/src/kernelpack.h @@ -147,8 +147,8 @@ sccl_CUresult sccl_kp_launch_chain(sccl_CUfunction function, const sccl_chain_ar /* The link collective entry of a kind (SCCL_LINK_*, SCCL_RING_*), dtype and unroll (1-8). */ int sccl_kp_link_function(sccl_CUcontext ctx, int kind, int dtype, int unroll, sccl_CUfunction *out); /* The ring all-reduce whose relay stores its result to link 3's slot, then copies the slot to the output - * (sircl_ring_reduce_two_pass__u): the same protocol and bytes as SCCL_RING_REDUCE's one-pass - * entry; LIBSIRCL_RING_REDUCE_PASSES=2 selects it, for comparisons. */ + * (sircl_ring_reduce_two_pass__u): the same protocol as SCCL_RING_REDUCE's one-pass entry. The + * library launches it by default (LIBSIRCL_RING_REDUCE_PASSES=2); the one-pass entry is research-only. */ int sccl_kp_ring_reduce_two_pass_function(sccl_CUcontext ctx, int dtype, int unroll, sccl_CUfunction *out); /* The pair exchange (sircl_ring_exchange_u): on the wire the ring all-gather of a ring of two, carrying * one block each way (input to the peer, the peer's block to output + peer * stride, the local block from diff --git a/spark_transport/libsircl/tests/test_engine_cpu.py b/spark_transport/libsircl/tests/test_engine_cpu.py index f93311a6..db701d3f 100644 --- a/spark_transport/libsircl/tests/test_engine_cpu.py +++ b/spark_transport/libsircl/tests/test_engine_cpu.py @@ -57,6 +57,28 @@ def test_finalize_and_destroy_wait_for_the_context(self): self.assertGreaterEqual(before, 1) self.assertGreater(after, before) + def test_one_pass_ring_relay_needs_the_research_opt_in(self): + """LIBSIRCL_RING_REDUCE_PASSES=1 selects the one-pass ring relay, which is research-only: creation is + refused, naming the opt-in, unless LIBSIRCL_RING_REDUCE_ONE_PASS_RESEARCH=1; 2 (the default) and the + opted-in 1 create a communicator on the stand-in driver.""" + script = '''u=U(); assert l.ncclGetUniqueId(C.byref(u))==0 +c=C.c_void_p(); r=l.ncclCommInitRank(C.byref(c),1,u,0) +print(json.dumps([r, l.ncclGetLastError(None).decode()])) +if c.value: l.ncclCommDestroy(c) +''' + base = {"LIBSIRCL_BOOTSTRAP_ONLY": "", "LIBSIRCL_TRANSPORT": "verbs", "LD_LIBRARY_PATH": str(FAKE_DRIVER)} + for passes, opt_in, refused in (("1", "", True), ("1", "0", True), ("1", "1", False), ("2", "", False)): + with self.subTest(passes=passes, opt_in=opt_in): + out = run(script, {**base, "LIBSIRCL_RING_REDUCE_PASSES": passes, + "LIBSIRCL_RING_REDUCE_ONE_PASS_RESEARCH": opt_in}) + self.assertEqual(out.returncode, 0, out.stderr) + result, message = json.loads(out.stdout) + if refused: + self.assertNotEqual(result, 0) + self.assertIn("LIBSIRCL_RING_REDUCE_ONE_PASS_RESEARCH=1", message) + else: + self.assertEqual(result, 0, message) + def test_initall_refuses_distinct_cuda_devices(self): for devices in ("None", "(C.c_int*2)(0,1)"): out = run("c=(C.c_void_p*2)(); r=l.ncclCommInitAll(c,2," + devices + "); " From f0113c37378c5e17cde38f28dcdc0c8aa07577d9 Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Thu, 15 Oct 2026 08:24:53 -0500 Subject: [PATCH 16/65] Confirm that vLLM resolves VLLM_NCCL_SO_PATH to the libsircl library The libsircl vLLM plugin's register sets VLLM_NCCL_SO_PATH and then reads vLLM's own value of it (vllm.envs.VLLM_NCCL_SO_PATH). When that value names another file, because the vLLM process read its environment before the plugin ran, register raises LibsirclSelectionError instead of letting PyNccl load the earlier library. A vllm.envs without the variable is refused the same way; outside vLLM (no vllm.envs) nothing more is checked. Compatibility: deployments whose vLLM reads the variable at each access are unaffected; a vLLM that cached its environment before the plugin ran fails at plugin registration instead of running on another NCCL. Validation: test_sparkring_libsircl.py installs stand-in vllm.envs modules: one reading the environment at each access passes; one holding the image's NVIDIA NCCL path, and one without the variable, raise. Before the change register returned in both failing cases. The plugin suite passes (13 passed, 1 skipped where symbolic links are unavailable). Request item 15 (audit 13-F1). NVIDIA NCCL source was neither read nor adapted. Co-Authored-By: Claude Opus 5.5 --- .../vllm/libsircl/sparkring_libsircl.py | 24 +++++++++- .../vllm/libsircl/test_sparkring_libsircl.py | 46 +++++++++++++++++++ 2 files changed, 68 insertions(+), 2 deletions(-) diff --git a/integrations/vllm/libsircl/sparkring_libsircl.py b/integrations/vllm/libsircl/sparkring_libsircl.py index 04d24a46..777eeb10 100644 --- a/integrations/vllm/libsircl/sparkring_libsircl.py +++ b/integrations/vllm/libsircl/sparkring_libsircl.py @@ -16,8 +16,11 @@ function PyNccl's ``NCCLLibrary`` binds. PyNccl disables itself, and vLLM falls back to torch's own NCCL, when that load or binding fails; this check makes such a fallback fail the process instead; -3. sets ``VLLM_NCCL_SO_PATH`` to the file; processes that vLLM starts later - inherit the setting. +3. sets ``VLLM_NCCL_SO_PATH`` to the file, and requires vLLM's own reading + of it (``vllm.envs.VLLM_NCCL_SO_PATH``) to name the same file, so a vLLM + whose environment values were read before the plugin ran fails instead of + loading the earlier library; processes that vLLM starts later inherit the + setting. Any other state fails the process, so a deployment that asked for libsircl never runs on another NCCL. The plugin changes nothing else: torch's @@ -91,8 +94,25 @@ def check_binding(path, *, load=None, functions=None): return info +def resolved_target(): + """vLLM's own value of ``VLLM_NCCL_SO_PATH`` (``vllm.envs``), or None outside vLLM.""" + try: + import vllm.envs as envs + except ImportError: + return None + try: + return getattr(envs, TARGET) + except AttributeError: + raise LibsirclSelectionError(f"vllm.envs has no {TARGET}, so this vLLM may not load the library it " + "names") from None + + def register(): """Point vLLM's PyNccl at the checked libsircl library; vLLM may call this more than once.""" path = selected() check_binding(path) os.environ[TARGET] = path + found = resolved_target() + if found is not None and found != path: + raise LibsirclSelectionError(f"vLLM resolves {TARGET} to {found!r}, not {path}: it read its environment " + "before the plugin set the variable, so PyNccl would load that library") diff --git a/integrations/vllm/libsircl/test_sparkring_libsircl.py b/integrations/vllm/libsircl/test_sparkring_libsircl.py index 1d3855f2..3b7b6eb9 100644 --- a/integrations/vllm/libsircl/test_sparkring_libsircl.py +++ b/integrations/vllm/libsircl/test_sparkring_libsircl.py @@ -2,6 +2,8 @@ import hashlib import importlib.util import os +import sys +import types from pathlib import Path import pytest @@ -104,3 +106,47 @@ def test_the_entry_point_names_the_register_function(): assert (group, name) == ("vllm.general_plugins", "libsircl") module, _, function = value.partition(":") assert module == SOURCE.stem and callable(getattr(plugin, function)) + + +class FrozenEnvs(types.ModuleType): + """A ``vllm.envs`` whose values were read before the plugin ran.""" + + def __init__(self, value): + super().__init__("vllm.envs") + self.VLLM_NCCL_SO_PATH = value + + +class LiveEnvs(types.ModuleType): + """A ``vllm.envs`` that reads the environment at each access, as vLLM does without its cache.""" + + def __init__(self): + super().__init__("vllm.envs") + + def __getattr__(self, name): + if name == plugin.TARGET: + return os.environ.get(name) + raise AttributeError(name) + + +@pytest.mark.parametrize("envs, message", [ + (LiveEnvs(), None), + (FrozenEnvs("/opt/sparkring/toolchain/nccl/lib/libnccl.so.2"), "vLLM resolves VLLM_NCCL_SO_PATH to"), + (types.ModuleType("vllm.envs"), "vllm.envs has no VLLM_NCCL_SO_PATH"), +]) +def test_register_confirms_vllm_resolves_the_library(tmp_path, monkeypatch, envs, message): + path, digest = library(tmp_path) + monkeypatch.setenv(plugin.LIBRARY, str(path)) + monkeypatch.setenv(plugin.DIGEST, digest) + monkeypatch.setenv(plugin.TARGET, "/opt/sparkring/toolchain/nccl/lib/libnccl.so.2") + monkeypatch.setattr(plugin, "LOADER", lambda name: Library()) + monkeypatch.setattr(plugin, "pynccl_functions", lambda: ["ncclAllReduce"]) + package = types.ModuleType("vllm") + package.envs = envs + monkeypatch.setitem(sys.modules, "vllm", package) + monkeypatch.setitem(sys.modules, "vllm.envs", envs) + if message is None: + plugin.register() + assert plugin.resolved_target() == str(path) + else: + with pytest.raises(plugin.LibsirclSelectionError, match=message): + plugin.register() From d126613a19ee5b67705912aca468d70a1ab79ecb Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Thu, 15 Oct 2026 08:26:30 -0500 Subject: [PATCH 17/65] Close a failed communicator's bootstrap connection at once When initialization fails after the bootstrap joined (session setup failed), the initializing thread closes the bootstrap connection and clears it before publishing FAILED. The failed handle is still returned for its result and reason and still needs ncclCommDestroy or ncclCommAbort, while holding no socket and, on the root's process, no broker thread or root slot (at most 64 per process) in the meantime. README.md states that contract. Compatibility: failed handles answer queries, destroy and abort unchanged; a caller that retries failed creations without destroying the handles keeps its descriptors and bootstrap roots. Validation: tests/test_engine_cpu.py makes 70 one-rank creations that fail in session setup (LIBSIRCL_TRANSPORT=bogus) without destroying the handles. Before the change the 65th unique id was refused ("at most 64 active bootstrap IDs"); after it every id is granted and the process's descriptor count returns to its starting value. make check and make check-asan pass. Request item 16 (audit 02-F1, 06-F1, 03-F4, 01-F12.4). NVIDIA NCCL source was neither read nor adapted. Co-Authored-By: Claude Opus 5.5 --- spark_transport/libsircl/README.md | 5 ++++- spark_transport/libsircl/src/api.c | 6 ++++++ .../libsircl/tests/test_engine_cpu.py | 20 +++++++++++++++++++ 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/spark_transport/libsircl/README.md b/spark_transport/libsircl/README.md index 89532b99..f386e773 100644 --- a/spark_transport/libsircl/README.md +++ b/spark_transport/libsircl/README.md @@ -39,7 +39,10 @@ gives the build, emulation and hardware commands. `ncclCommSplit`) writes one stderr line, `libsircl: communicator creation failed (): `, the text `ncclGetLastError` returns, whatever `NCCL_DEBUG` says: frameworks commonly report only `ncclGetErrorString`. With `NCCL_DEBUG` set to `WARN`, `INFO` or `TRACE`, every failed call writes - `libsircl: : `. + `libsircl: : `. A creation that fails after the handle was allocated (the bootstrap or the + session setup failed) still returns the handle, whose `ncclCommGetAsyncError` and `ncclGetLastError` give + the result and reason; the failed communicator holds no bootstrap connection or session, but the caller + must still destroy or abort the handle to free it. - `include/nccl.h` is NVIDIA's 2.32.3 header with marked changes (`NCCL_VERSION_CODE` 23203), used only to build libsircl. - The name `libsircl` without a suffix means this library; SIRCL's own test libraries are named diff --git a/spark_transport/libsircl/src/api.c b/spark_transport/libsircl/src/api.c index a52d5a3c..772a6591 100644 --- a/spark_transport/libsircl/src/api.c +++ b/spark_transport/libsircl/src/api.c @@ -200,6 +200,12 @@ static void *initialize(void *arg) { &comm->cancelled, &comm->engine, comm->error_message, sizeof(comm->error_message)); if (!result) comm->device = sccl_engine_device(comm->engine); } + if (result && comm->bootstrap) { + /* A failed communicator keeps no bootstrap connection: it would hold a socket, and on the root's process + * a broker thread and one of the 64 root slots, until the caller destroys the handle. */ + sccl_bootstrap_close(comm->bootstrap); + comm->bootstrap = NULL; + } atomic_store(&comm->state, result == 0 ? READY : FAILED); atomic_store(&comm->error, result); return NULL; diff --git a/spark_transport/libsircl/tests/test_engine_cpu.py b/spark_transport/libsircl/tests/test_engine_cpu.py index db701d3f..5f24874b 100644 --- a/spark_transport/libsircl/tests/test_engine_cpu.py +++ b/spark_transport/libsircl/tests/test_engine_cpu.py @@ -79,6 +79,26 @@ def test_one_pass_ring_relay_needs_the_research_opt_in(self): else: self.assertEqual(result, 0, message) + def test_failed_creations_release_their_bootstrap(self): + """Seventy creations that fail in session setup (LIBSIRCL_TRANSPORT=bogus), whose handles are never + destroyed, leave no socket open and no bootstrap root held: a further unique id is still granted + (the process holds at most 64 roots).""" + out = run('''import os, time +before=len(os.listdir("/proc/self/fd")) +handles=[] +for _ in range(70): + u=U(); assert l.ncclGetUniqueId(C.byref(u))==0, l.ncclGetLastError(None) + c=C.c_void_p(); r=l.ncclCommInitRank(C.byref(c),1,u,0); assert r!=0 and c.value; handles.append(c) +time.sleep(0.3) +after=len(os.listdir("/proc/self/fd")) +u=U(); last=l.ncclGetUniqueId(C.byref(u)) +print(json.dumps([before, after, last, l.ncclGetLastError(None).decode()])) +''', {"LIBSIRCL_BOOTSTRAP_ONLY": "", "LIBSIRCL_TRANSPORT": "bogus", "LD_LIBRARY_PATH": str(FAKE_DRIVER)}) + self.assertEqual(out.returncode, 0, out.stderr[-2000:]) + before, after, last, message = json.loads(out.stdout) + self.assertEqual(last, 0, message) + self.assertEqual(after, before) + def test_initall_refuses_distinct_cuda_devices(self): for devices in ("None", "(C.c_int*2)(0,1)"): out = run("c=(C.c_void_p*2)(); r=l.ncclCommInitAll(c,2," + devices + "); " From b01ab91034f9e28d16d9fc6206c0c4ba75298ca7 Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Thu, 15 Oct 2026 08:27:41 -0500 Subject: [PATCH 18/65] Abort and clear every handle of a failed ncclCommInitAll An ncclCommInitAll that fails outside the caller's group aborts every communicator it created and sets every output handle to NULL, keeping the failure's text for ncclGetLastError. Inside the caller's own group the handles belong to that group until its ncclGroupEnd, which reports their failure, so they are left in place. Compatibility: callers of a failed top-level InitAll receive NULL handles instead of FAILED handles that needed destroying. Validation: tests/test_engine_cpu.py runs InitAll of two ranks on one stand-in device with LIBSIRCL_TRANSPORT=bogus, so both fail in session setup. Before the change both outputs were non-NULL handles; after it both are NULL and the result and reason are kept. make check and make check-asan pass. Request item 17 (audit 03-F10, 02-F3 part a). NVIDIA NCCL source was neither read nor adapted. Co-Authored-By: Claude Opus 5.5 --- spark_transport/libsircl/src/api.c | 16 +++++++++++++++- .../libsircl/tests/test_engine_cpu.py | 12 ++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/spark_transport/libsircl/src/api.c b/spark_transport/libsircl/src/api.c index 772a6591..8b0573d8 100644 --- a/spark_transport/libsircl/src/api.c +++ b/spark_transport/libsircl/src/api.c @@ -611,6 +611,7 @@ ncclResult_t ncclCommInitAll(ncclComm_t *out, int ndev, const int *devlist) { ncclUniqueId id; ncclResult_t result = ncclGetUniqueId(&id); if (result != ncclSuccess) return result; + int nested = group_depth != 0; result = ncclGroupStart(); if (result != ncclSuccess) return result; for (int i = 0; i < ndev; ++i) { @@ -619,7 +620,20 @@ ncclResult_t ncclCommInitAll(ncclComm_t *out, int ndev, const int *devlist) { out[i]->device = devlist ? devlist[i] : i; } ncclResult_t end = ncclGroupEnd(); - return result != ncclSuccess ? result : end; + if (result == ncclSuccess) result = end; + /* A failed InitAll leaves no handle: every communicator it created is aborted and its output cleared. + * Inside the caller's own group the handles belong to that group until its ncclGroupEnd, which reports + * their failure. */ + if (result != ncclSuccess && result != ncclInProgress && !nested) { + char reason[sizeof last_error]; + snprintf(reason, sizeof reason, "%s", last_error); + for (int i = 0; i < ndev; ++i) { + if (out[i]) ncclCommAbort(out[i]); + out[i] = NULL; + } + snprintf(last_error, sizeof last_error, "%s", reason); + } + return result; } static ncclResult_t query(ncclComm_t handle, int *out, int field) { CHECK_PROCESS(); diff --git a/spark_transport/libsircl/tests/test_engine_cpu.py b/spark_transport/libsircl/tests/test_engine_cpu.py index 5f24874b..d61cd43f 100644 --- a/spark_transport/libsircl/tests/test_engine_cpu.py +++ b/spark_transport/libsircl/tests/test_engine_cpu.py @@ -99,6 +99,18 @@ def test_failed_creations_release_their_bootstrap(self): self.assertEqual(last, 0, message) self.assertEqual(after, before) + def test_failed_initall_leaves_no_handles(self): + """An ncclCommInitAll whose communicators fail in session setup aborts them and clears every output.""" + out = run("c=(C.c_void_p*2)(); r=l.ncclCommInitAll(c,2,(C.c_int*2)(0,0)); " + "print(json.dumps([r,list(c),l.ncclGetLastError(None).decode()]))", + {"LIBSIRCL_BOOTSTRAP_ONLY": "", "LIBSIRCL_TRANSPORT": "bogus", "FAKE_CUDA_DEVICES": "1", + "LD_LIBRARY_PATH": str(FAKE_DRIVER)}) + self.assertEqual(out.returncode, 0, out.stderr) + result, handles, message = json.loads(out.stdout) + self.assertNotIn(result, (0, 7)) + self.assertEqual(handles, [None, None]) + self.assertIn("group initialization failed", message) + def test_initall_refuses_distinct_cuda_devices(self): for devices in ("None", "(C.c_int*2)(0,1)"): out = run("c=(C.c_void_p*2)(); r=l.ncclCommInitAll(c,2," + devices + "); " From 37083b88d53c61d99a94a015b21dd1b2ca13f059 Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Thu, 15 Oct 2026 08:29:16 -0500 Subject: [PATCH 19/65] Release a thread's group state when the thread exits A thread that opens a group registers a thread-exit hook (a pthread key destructor). When the thread ends, the hook frees its point-to-point queue and fails every initialization still pending in its open group with ncclInvalidUsage ("the thread that initialized this communicator in a group exited before its ncclGroupEnd") and clears its grouped mark, so another thread can destroy the handle; queued point-to-point calls of that group are dropped. The library destructor deletes the key, so no hook runs into an unloaded library. Compatibility: a handle stranded by its thread's exit inside a group reports ncclInvalidUsage and can be destroyed, where every destroy of it returned ncclInvalidUsage forever. Threads that close their groups are unaffected apart from the per-thread key. Validation: tests/test_lifecycle.py starts a thread that calls ncclGroupStart and ncclCommInitRank and exits. Before the change the handle reported ncclInProgress; after it the handle reports ncclInvalidUsage with the reason and ncclCommDestroy succeeds. make check and make check-asan pass. The queue's release is not measured: the sanitizer target runs with leak detection off, since the interpreter's own allocations would dominate it. Request item 18 (audit 07-F8, 06-F8 bullet 1). NVIDIA NCCL source was neither read nor adapted. Co-Authored-By: Claude Opus 5.5 --- spark_transport/libsircl/src/api.c | 33 +++++++++++++++++++ .../libsircl/tests/test_lifecycle.py | 19 +++++++++++ 2 files changed, 52 insertions(+) diff --git a/spark_transport/libsircl/src/api.c b/spark_transport/libsircl/src/api.c index 8b0573d8..ce3cd454 100644 --- a/spark_transport/libsircl/src/api.c +++ b/spark_transport/libsircl/src/api.c @@ -55,6 +55,37 @@ typedef struct { static _Thread_local queued_p2p *p2p_queue; static _Thread_local unsigned p2p_count, p2p_capacity; +/* A thread that used group state gets a thread-exit hook (a pthread key's destructor): it frees the thread's + * point-to-point queue and fails the initializations still pending in the thread's open group, which no + * other thread may start or destroy, so their handles can be destroyed. The library destructor deletes the + * key, so no hook runs into an unloaded library. */ +static pthread_key_t group_key; +static pthread_once_t group_key_once = PTHREAD_ONCE_INIT; +static atomic_int group_key_ready; +static void group_thread_exit(void *unused) { + (void)unused; + for (unsigned i = 0; i < pending_count; ++i) { + struct ncclComm *comm = pending[i]; + snprintf(comm->error_message, sizeof(comm->error_message), + "the thread that initialized this communicator in a group exited before its ncclGroupEnd"); + atomic_store(&comm->state, FAILED); + atomic_store(&comm->error, ncclInvalidUsage); + atomic_store(&comm->grouped, 0); + } + pending_count = 0; + group_depth = 0; + free(p2p_queue); + p2p_queue = NULL; + p2p_count = p2p_capacity = 0; +} +static void group_key_create(void) { + if (!pthread_key_create(&group_key, group_thread_exit)) atomic_store(&group_key_ready, 1); +} +static void group_thread_watch(void) { + pthread_once(&group_key_once, group_key_create); + if (atomic_load(&group_key_ready) && !pthread_getspecific(group_key)) pthread_setspecific(group_key, &group_key); +} + /* Make `result` the communicator's terminal error with `message` as its text, unless it already has one; * 1 when this call set it. The text is written under the error lock, and readers copy it under the same * lock (read_error). */ @@ -549,6 +580,7 @@ ncclResult_t ncclGroupStart(void) { CHECK_PROCESS(); if (group_depth == 256) return fail(ncclInvalidUsage, "group nesting exceeds 256"); if (group_depth++ == 0) { group_error = ncclSuccess; pending_count = 0; } + group_thread_watch(); return ncclSuccess; } ncclResult_t ncclGroupEnd(void) { @@ -1054,6 +1086,7 @@ static int referenced(const struct ncclComm *list) { } __attribute__((destructor)) static void shutdown_communicators(void) { if (!sccl_bootstrap_process_valid()) return; + if (atomic_exchange(&group_key_ready, 0)) pthread_key_delete(group_key); sccl_engine_fail_stop_shutdown(); pthread_mutex_lock(®istry_lock); struct ncclComm *comm = registry; diff --git a/spark_transport/libsircl/tests/test_lifecycle.py b/spark_transport/libsircl/tests/test_lifecycle.py index 9195f3b4..6dca7449 100644 --- a/spark_transport/libsircl/tests/test_lifecycle.py +++ b/spark_transport/libsircl/tests/test_lifecycle.py @@ -252,6 +252,25 @@ def test_nested_error_propagates_to_pending_handles(self): finally: self.dispose_all([handle]) + def test_thread_exiting_inside_a_group_releases_its_pending_handles(self): + """A thread opens a group, initializes a communicator in it and exits without ncclGroupEnd: the + handle fails with ncclInvalidUsage, naming the reason, and another thread can destroy it.""" + value = self.uid() + handle = C.c_void_p() + codes = [] + + def grouped(): + codes.append(self.lib.ncclGroupStart()) + codes.append(self.lib.ncclCommInitRank(C.byref(handle), 1, value, 0)) + thread = threading.Thread(target=grouped) + thread.start() + thread.join(timeout=5) + self.assertEqual(codes, [SUCCESS, SUCCESS]) + self.assertTrue(handle.value) + self.assertEqual(self.asynchronous_error(handle), USAGE) + self.assertIn(b"exited before its ncclGroupEnd", self.lib.ncclGetLastError(handle)) + self.assertEqual(self.lib.ncclCommDestroy(handle), SUCCESS) + def test_pending_group_handle_cannot_be_destroyed_from_another_thread(self): value = self.uid() handle = C.c_void_p() From ef656f6c8c1fa481ae9e5fcff88c0a05b580919f Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Thu, 15 Oct 2026 08:32:29 -0500 Subject: [PATCH 20/65] Free a destroyed communicator's host bookkeeping when its context is gone When ncclCommDestroy or ncclCommAbort cannot make the communicator's context current, the context was destroyed (or the driver is shutting down), and with it every device and pinned allocation and every kernel that could read them. Destroy then frees the host bookkeeping that the context did not own: the retired staging list, the emulation transport's shared-memory segments, and the engine itself, provided the transports' queue pairs and registrations were released. It skips only the CUDA frees. The stand-in CUDA driver (tests/fake_cuda.c) gains sccl_fake_cuda_fail_push, which makes every later context push fail. Compatibility: no change while the context exists; the engine and its host allocations are freed instead of kept until the process exits. Validation: tests/test_engine_cpu.py creates and destroys a one-rank communicator 1,000 times with the push failing at destroy. Before the change the heap grew by 5.4 MB (5,424 bytes per destroy); after it the growth is 0. The check is skipped under make check-asan, whose allocator mallinfo2 does not measure. make check and make check-asan pass. Request item 20 (audit 02-F15). NVIDIA NCCL source was neither read nor adapted. Co-Authored-By: Claude Opus 5.5 --- spark_transport/libsircl/src/engine.c | 16 +++++++++++-- spark_transport/libsircl/tests/fake_cuda.c | 5 +++- .../libsircl/tests/test_engine_cpu.py | 23 +++++++++++++++++++ 3 files changed, 41 insertions(+), 3 deletions(-) diff --git a/spark_transport/libsircl/src/engine.c b/spark_transport/libsircl/src/engine.c index a9cba208..502b1098 100644 --- a/spark_transport/libsircl/src/engine.c +++ b/spark_transport/libsircl/src/engine.c @@ -4672,15 +4672,27 @@ ncclResult_t sccl_engine_destroy(sccl_engine *e, int abort, char *err, size_t le dump_teardown(e, e->stop_ns, e->stopped_ns, realtime_ns()); /* After an abort with a launch still running, its kernel may read the arena and counters * until its wait limit; they stay allocated until the process exits. */ - if (pushed && idle && released) { + int freed = pushed && idle && released; + if (freed) { if (e->order_event) cu->EventDestroy(e->order_event); release_memory(e); + } else if (!pushed && released) { + /* The communicator's context could not be made current: it was destroyed (or the driver is shutting + * down), and with it every device and pinned allocation and every kernel that could read them. Only the + * host bookkeeping is left: the retired staging list, the shared-memory segments of the emulation + * transport, and the engine itself. */ + free(e->retired_stage); + e->retired_stage = NULL; + if (e->host && e->host_emulated) sccl_emu_segment_free(e->host); + if (e->p2p_host && e->p2p_host_emulated) sccl_emu_segment_free(e->p2p_host); + e->host = e->p2p_host = NULL; + freed = 1; } if (pushed) { sccl_CUcontext popped; cu->CtxPopCurrent(&popped); } pthread_mutex_destroy(&e->lock); - if (idle && released) free(e); + if (freed) free(e); return result; } diff --git a/spark_transport/libsircl/tests/fake_cuda.c b/spark_transport/libsircl/tests/fake_cuda.c index 9d6cc67e..2c279dc0 100644 --- a/spark_transport/libsircl/tests/fake_cuda.c +++ b/spark_transport/libsircl/tests/fake_cuda.c @@ -174,7 +174,10 @@ EXPORT int cuCtxPopCurrent_v2(void **ctx) { return 0; } EXPORT int cuInit(unsigned flags) { return (int)(flags & 0); } -EXPORT int cuCtxPushCurrent_v2(void *ctx) { return ctx ? 0 : 201; } +/* sccl_fake_cuda_fail_push(1): every later push fails, as for a destroyed context. */ +static int fail_push; +EXPORT void sccl_fake_cuda_fail_push(int on) { fail_push = on; } +EXPORT int cuCtxPushCurrent_v2(void *ctx) { return ctx && !fail_push ? 0 : 201; } EXPORT int cuModuleUnload(void *module) { return module ? 0 : 400; } /* Every other entry point src/cuda_api.c binds: present, never used by the check. */ diff --git a/spark_transport/libsircl/tests/test_engine_cpu.py b/spark_transport/libsircl/tests/test_engine_cpu.py index d61cd43f..ce5dede9 100644 --- a/spark_transport/libsircl/tests/test_engine_cpu.py +++ b/spark_transport/libsircl/tests/test_engine_cpu.py @@ -111,6 +111,29 @@ def test_failed_initall_leaves_no_handles(self): self.assertEqual(handles, [None, None]) self.assertIn("group initialization failed", message) + def test_destroy_without_its_context_frees_the_host_bookkeeping(self): + """Destroying a communicator whose context can no longer be made current (the stand-in driver fails + every push, as for a destroyed context) frees its engine: 1,000 such destroys leave the process heap + within 1 MiB of where it started. Not under AddressSanitizer, whose allocator mallinfo2 does not see.""" + if os.environ.get("LIBSIRCL_TEST_ASAN"): + self.skipTest("mallinfo2 does not measure AddressSanitizer's allocator") + out = run('''g=C.CDLL("libcuda.so.1") +class Info(C.Structure): _fields_=[(n, C.c_size_t) for n in ("arena","ordblks","smblks","hblks","hblkhd","usmblks","fsmblks","uordblks","fordblks","keepcost")] +libc=C.CDLL(None); libc.mallinfo2.restype=Info +def used(): return libc.mallinfo2().uordblks + libc.mallinfo2().hblkhd +def cycle(): + u=U(); assert l.ncclGetUniqueId(C.byref(u))==0 + c=C.c_void_p(); assert l.ncclCommInitRank(C.byref(c),1,u,0)==0 + g.sccl_fake_cuda_fail_push(1); l.ncclCommDestroy(c); g.sccl_fake_cuda_fail_push(0) +for _ in range(5): cycle() +before=used() +for _ in range(1000): cycle() +print(json.dumps([before, used()])) +''', {"LIBSIRCL_BOOTSTRAP_ONLY": "", "LIBSIRCL_TRANSPORT": "verbs", "LD_LIBRARY_PATH": str(FAKE_DRIVER)}) + self.assertEqual(out.returncode, 0, out.stderr[-2000:]) + before, after = json.loads(out.stdout) + self.assertLess(after - before, 1 << 20, (before, after)) + def test_initall_refuses_distinct_cuda_devices(self): for devices in ("None", "(C.c_int*2)(0,1)"): out = run("c=(C.c_void_p*2)(); r=l.ncclCommInitAll(c,2," + devices + "); " From 8266a30fa776d21730ab5119b26de02def93f026 Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Thu, 15 Oct 2026 08:34:51 -0500 Subject: [PATCH 21/65] Keep failed calls that enqueue nothing out of a group's result Inside a group, only calls that belong to it make the group's result: a communicator creation, a collective or point-to-point call refused other than as an invalid argument, and the group calls themselves (group_fail). A failed call that enqueues nothing (a query of an unknown handle, a registration, ncclGetUniqueId, a lifecycle call, an unsupported function) returns its error and text and leaves the group's pending initializations and queued point-to-point calls to run at ncclGroupEnd. README.md states the rule. Compatibility: ncclGroupEnd returns success after such failures, where it returned the failed call's result, failed every pending initialization and dropped the queued point-to-point calls. tests/test_api.py's group test, which asserted the poisoning by an unsupported function, now asserts the group's success. Validation: tests/test_lifecycle.py initializes a one-rank communicator in a group, then fails a query on an unknown handle, ncclGetUniqueId with a NULL output, ncclCommShrink (unsupported) and ncclCommRevoke with flags. Before the change ncclGroupEnd returned ncclInvalidArgument; after it ncclGroupEnd succeeds and the communicator is ready. The revoke test of an initializing communicator now also asserts the group's success. make check and make check-asan pass. Request item 21 (audit 03-F8, 01-F2, 01-F12.3). NVIDIA NCCL source was neither read nor adapted. Co-Authored-By: Claude Opus 5.5 --- spark_transport/libsircl/README.md | 4 ++- spark_transport/libsircl/src/api.c | 28 +++++++++++++------ spark_transport/libsircl/tests/test_api.py | 4 +-- .../libsircl/tests/test_lifecycle.py | 25 +++++++++++++++-- 4 files changed, 47 insertions(+), 14 deletions(-) diff --git a/spark_transport/libsircl/README.md b/spark_transport/libsircl/README.md index f386e773..407dcd1d 100644 --- a/spark_transport/libsircl/README.md +++ b/spark_transport/libsircl/README.md @@ -175,7 +175,9 @@ finalize on a thread of its own. Errors of single calls: a collective or point-to-point call refused with `ncclInvalidArgument` did nothing and leaves the calling thread's group as it was (the group's other calls still run at `ncclGroupEnd`); other -errors inside a group become the group's result. A point-to-point call queued in a group holds no reference +errors of a collective, point-to-point or communicator-creation call inside a group become the group's +result. A failed call that enqueues nothing (a query, a registration, `ncclGetUniqueId`, a lifecycle call, an +unsupported function) returns its error and leaves the group as it was. A point-to-point call queued in a group holds no reference on its communicator, so `ncclCommAbort` from another thread returns at once, and that group's `ncclGroupEnd` then fails its calls with `ncclInvalidUsage`. `ncclCommSplit` with a NULL config (or a config whose `blocking` is undefined) takes the parent's blocking mode. diff --git a/spark_transport/libsircl/src/api.c b/spark_transport/libsircl/src/api.c index ce3cd454..8635e266 100644 --- a/spark_transport/libsircl/src/api.c +++ b/spark_transport/libsircl/src/api.c @@ -107,20 +107,29 @@ static int debug_errors(void) { const char *debug = sccl_env("NCCL_DEBUG"); return debug && (!strcasecmp(debug, "WARN") || !strcasecmp(debug, "INFO") || !strcasecmp(debug, "TRACE")); } -/* ncclInProgress never becomes a group's result: ncclGroupEnd would record it as the terminal error of the - * group's pending initializations, which would then read as in progress forever. */ +/* A failed call: its result and text for this thread. A call that enqueues nothing into the thread's group + * (a query, a registration, ncclGetUniqueId, a lifecycle call, an unsupported function) fails this way and + * leaves the group as it was. */ static ncclResult_t fail(ncclResult_t result, const char *message) { snprintf(last_error, sizeof(last_error), "%s", message); - if (group_depth && group_error == ncclSuccess && result != ncclInProgress) group_error = result; if (debug_errors()) fprintf(stderr, "libsircl: %s: %s\n", ncclGetErrorString(result), message); return result; } +/* A failed call that belongs to the thread's group (a communicator creation, a collective or point-to-point + * call refused other than as an invalid argument, the group calls themselves): its result also becomes the + * group's, unless the group already has one. ncclInProgress never becomes a group's result: ncclGroupEnd + * would record it as the terminal error of the group's pending initializations, which would then read as in + * progress forever. */ +static ncclResult_t group_fail(ncclResult_t result, const char *message) { + if (group_depth && group_error == ncclSuccess && result != ncclInProgress) group_error = result; + return fail(result, message); +} /* A communicator that cannot be created (ncclCommInitRank*, ncclCommSplit): its reason goes to stderr * whatever NCCL_DEBUG says, once, because the frameworks that create communicators commonly report only * ncclGetErrorString, and the text ncclGetLastError keeps is otherwise lost with the process. */ static ncclResult_t creation_failed(ncclResult_t result, const char *message) { int printed = debug_errors(); - fail(result, message); + group_fail(result, message); if (!printed) fprintf(stderr, "libsircl: communicator creation failed (%s): %s\n", ncclGetErrorString(result), message); return result; @@ -129,7 +138,7 @@ static ncclResult_t creation_failed(ncclResult_t result, const char *message) { * leaves the calling thread's group as it was (the call did nothing, and the work queued before it still * runs at ncclGroupEnd); any other error also becomes the group's result. */ static ncclResult_t refuse(ncclResult_t result, const char *message) { - if (result != ncclInvalidArgument) return fail(result, message); + if (result != ncclInvalidArgument) return group_fail(result, message); snprintf(last_error, sizeof(last_error), "%s", message); return result; } @@ -246,7 +255,7 @@ static ncclResult_t start(struct ncclComm *comm) { atomic_store(&comm->state, FAILED); snprintf(comm->error_message, sizeof(comm->error_message), "cannot start bootstrap worker"); atomic_store(&comm->error, ncclSystemError); - return fail(ncclSystemError, "cannot start bootstrap worker"); + return group_fail(ncclSystemError, "cannot start bootstrap worker"); } comm->worker_started = 1; return ncclSuccess; @@ -578,7 +587,7 @@ ncclResult_t ncclRecv(void *recvbuff, size_t count, ncclDataType_t datatype, int ncclResult_t ncclGroupStart(void) { CHECK_PROCESS(); - if (group_depth == 256) return fail(ncclInvalidUsage, "group nesting exceeds 256"); + if (group_depth == 256) return group_fail(ncclInvalidUsage, "group nesting exceeds 256"); if (group_depth++ == 0) { group_error = ncclSuccess; pending_count = 0; } group_thread_watch(); return ncclSuccess; @@ -621,10 +630,11 @@ ncclResult_t ncclGroupEnd(void) { } ncclResult_t ncclCommInitAll(ncclComm_t *out, int ndev, const int *devlist) { CHECK_PROCESS(); - if (!out || ndev < 1 || ndev > 8) return fail(ncclInvalidArgument, "InitAll requires 1 to 8 output handles"); + if (!out || ndev < 1 || ndev > 8) + return group_fail(ncclInvalidArgument, "InitAll requires 1 to 8 output handles"); for (int i = 0; i < ndev; ++i) out[i] = NULL; for (int i = 0; i < ndev; ++i) if (devlist && devlist[i] < 0) - return fail(ncclInvalidArgument, "negative device number"); + return group_fail(ncclInvalidArgument, "negative device number"); const char *only = sccl_env("LIBSIRCL_BOOTSTRAP_ONLY"); if (!only || strcmp(only, "1")) { int device = devlist ? devlist[0] : 0; diff --git a/spark_transport/libsircl/tests/test_api.py b/spark_transport/libsircl/tests/test_api.py index e7b2c365..92658b88 100644 --- a/spark_transport/libsircl/tests/test_api.py +++ b/spark_transport/libsircl/tests/test_api.py @@ -266,9 +266,9 @@ def separate(): f = LIB.ncclCommGetUniqueId f.argtypes = [C.c_void_p, C.c_void_p] self.assertEqual(f(None, None), INVALID_USAGE) - self.assertEqual(LIB.ncclGroupEnd(), INVALID_USAGE) - self.assertEqual(LIB.ncclGroupStart(), SUCCESS) + # An unsupported function enqueues nothing, so its refusal leaves the group's result alone. self.assertEqual(LIB.ncclGroupEnd(), SUCCESS) + self.assertIn(b"unsupported", LIB.ncclGetLastError(None)) def test_unsupported_surface_preserves_every_output(self): sentinel = C.create_string_buffer(b"Q" * 256) diff --git a/spark_transport/libsircl/tests/test_lifecycle.py b/spark_transport/libsircl/tests/test_lifecycle.py index 6dca7449..99faa71c 100644 --- a/spark_transport/libsircl/tests/test_lifecycle.py +++ b/spark_transport/libsircl/tests/test_lifecycle.py @@ -66,8 +66,8 @@ def test_lifecycle_call_on_an_initializing_communicator_is_refused(self): self.assertEqual(self.lib.ncclCommInitRank(C.byref(grouped), 1, second, 0), SUCCESS) self.assertEqual(self.lib.ncclCommRevoke(pending, 0), USAGE) self.assertEqual(self.lib.ncclCommFinalize(pending), USAGE) - self.assertNotEqual(self.lib.ncclGroupEnd(), PROGRESS) - self.assertNotEqual(self.asynchronous_error(grouped), PROGRESS) + self.assertEqual(self.lib.ncclGroupEnd(), SUCCESS) + self.assertEqual(self.asynchronous_error(grouped), SUCCESS) finally: self.lib.ncclCommAbort(pending) if grouped.value: @@ -271,6 +271,27 @@ def grouped(): self.assertIn(b"exited before its ncclGroupEnd", self.lib.ncclGetLastError(handle)) self.assertEqual(self.lib.ncclCommDestroy(handle), SUCCESS) + def test_calls_that_enqueue_nothing_leave_the_group_as_it_was(self): + """Inside a group, failed calls that enqueue nothing (a query of an unknown handle, a NULL output to + ncclGetUniqueId, an unsupported function, revoke flags) return their errors and leave the group's + initialization and result alone.""" + value = self.uid() + handle = C.c_void_p() + count = C.c_int(-1) + self.lib.ncclCommShrink.argtypes = [C.c_void_p, C.c_void_p, C.c_int, C.POINTER(C.c_void_p), C.c_void_p, + C.c_int] + self.assertEqual(self.lib.ncclGroupStart(), SUCCESS) + self.assertEqual(self.lib.ncclCommInitRank(C.byref(handle), 1, value, 0), SUCCESS) + self.assertEqual(self.lib.ncclCommCount(C.c_void_p(16), C.byref(count)), ARGUMENT) + self.assertEqual(self.lib.ncclGetUniqueId(None), ARGUMENT) + self.assertEqual(self.lib.ncclCommShrink(handle, None, 0, None, None, 0), USAGE) + self.assertEqual(self.lib.ncclCommRevoke(handle, 1), ARGUMENT) + try: + self.assertEqual(self.lib.ncclGroupEnd(), SUCCESS) + self.assertEqual(self.asynchronous_error(handle), SUCCESS) + finally: + self.dispose_all([handle]) + def test_pending_group_handle_cannot_be_destroyed_from_another_thread(self): value = self.uid() handle = C.c_void_p() From 8d6bdd5e015cf5c3dcaa4022dd5edae0f5a8b3b5 Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Thu, 15 Oct 2026 08:36:51 -0500 Subject: [PATCH 22/65] Zero the handle and value outputs of unsupported functions Every generated stub of an unsupported function zeroes its handle, value and length outputs before it returns ncclInvalidUsage: newcomm of ncclCommShrink, ncclCommGrow and ncclCommInitRankScalable (NULL), the unique id of ncclCommGetUniqueId (all zero, which every init refuses), ncclCommMemStats' value, and the outputs of the ncclParam* functions. Buffers the caller sizes (ncclParamGet's out, ncclGroupSimulateEnd's simInfo) and every input are left as they are. The table lives in tools/generate_api.py (ZEROED_OUTPUTS), which refuses an entry naming an argument the function does not have; src/stubs.c is regenerated from it. A test comment in tests/test_api.py now states that an unsupported call enqueues nothing and leaves its group's result alone. Compatibility: callers that ignore the refusal read NULL or zero instead of whatever their output held; return codes and messages are unchanged. Validation: tests/test_api.py calls every unsupported function and its pncclXxx twin with a separate 256-byte sentinel per pointer argument and requires the listed outputs zeroed and every other byte unchanged. Before the regeneration 36 subtests failed (every listed output kept its sentinel, for example newcomm after ncclCommShrink); after it all pass. generate_api.py --check reports no drift; make check and make check-asan pass. Request item 22 (audit 03-F12). NVIDIA NCCL source was neither read nor adapted. Co-Authored-By: Claude Opus 5.5 --- spark_transport/libsircl/src/stubs.c | 42 ++++++++++--------- spark_transport/libsircl/tests/test_api.py | 26 ++++++++---- .../libsircl/tools/generate_api.py | 37 +++++++++++----- 3 files changed, 67 insertions(+), 38 deletions(-) diff --git a/spark_transport/libsircl/src/stubs.c b/spark_transport/libsircl/src/stubs.c index f2e395af..44c007aa 100644 --- a/spark_transport/libsircl/src/stubs.c +++ b/spark_transport/libsircl/src/stubs.c @@ -2,11 +2,13 @@ * 2015-2026 NVIDIA CORPORATION & AFFILIATES, Apache-2.0; see vendor/NCCL-LICENSE.txt). Do not edit. */ #include "internal.h" +#include + ncclResult_t ncclCommShrink(ncclComm_t comm, int* excludeRanksList, int excludeRanksCount, ncclComm_t* newcomm, ncclConfig_t* config, int shrinkFlags) { (void)comm; (void)excludeRanksList; (void)excludeRanksCount; - (void)newcomm; + if (newcomm) memset(newcomm, 0, sizeof(*newcomm)); (void)config; (void)shrinkFlags; return sccl_unsupported("ncclCommShrink"); @@ -14,7 +16,7 @@ ncclResult_t ncclCommShrink(ncclComm_t comm, int* excludeRanksList, int excludeR ncclResult_t ncclCommGetUniqueId(ncclComm_t comm, ncclUniqueId* uniqueId) { (void)comm; - (void)uniqueId; + if (uniqueId) memset(uniqueId, 0, sizeof(*uniqueId)); return sccl_unsupported("ncclCommGetUniqueId"); } @@ -23,13 +25,13 @@ ncclResult_t ncclCommGrow(ncclComm_t comm, int nRanks, const ncclUniqueId* uniqu (void)nRanks; (void)uniqueId; (void)rank; - (void)newcomm; + if (newcomm) memset(newcomm, 0, sizeof(*newcomm)); (void)config; return sccl_unsupported("ncclCommGrow"); } ncclResult_t ncclCommInitRankScalable(ncclComm_t* newcomm, int nranks, int myrank, int nId, ncclUniqueId* commIds, ncclConfig_t* config) { - (void)newcomm; + if (newcomm) memset(newcomm, 0, sizeof(*newcomm)); (void)nranks; (void)myrank; (void)nId; @@ -56,7 +58,7 @@ ncclResult_t ncclCommResume(ncclComm_t comm) { ncclResult_t ncclCommMemStats(ncclComm_t comm, ncclCommMemStat_t stat, uint64_t* value) { (void)comm; (void)stat; - (void)value; + if (value) memset(value, 0, sizeof(*value)); return sccl_unsupported("ncclCommMemStats"); } @@ -214,62 +216,62 @@ ncclResult_t ncclSetEncryption(const ncclEncryptionConfig_t* config) { } ncclResult_t ncclParamBind(ncclParamHandle_t* out, const char* key) { - (void)out; + if (out) memset(out, 0, sizeof(*out)); (void)key; return sccl_unsupported("ncclParamBind"); } ncclResult_t ncclParamGetI8(ncclParamHandle_t h, int8_t* out) { (void)h; - (void)out; + if (out) memset(out, 0, sizeof(*out)); return sccl_unsupported("ncclParamGetI8"); } ncclResult_t ncclParamGetI16(ncclParamHandle_t h, int16_t* out) { (void)h; - (void)out; + if (out) memset(out, 0, sizeof(*out)); return sccl_unsupported("ncclParamGetI16"); } ncclResult_t ncclParamGetI32(ncclParamHandle_t h, int32_t* out) { (void)h; - (void)out; + if (out) memset(out, 0, sizeof(*out)); return sccl_unsupported("ncclParamGetI32"); } ncclResult_t ncclParamGetI64(ncclParamHandle_t h, int64_t* out) { (void)h; - (void)out; + if (out) memset(out, 0, sizeof(*out)); return sccl_unsupported("ncclParamGetI64"); } ncclResult_t ncclParamGetU8(ncclParamHandle_t h, uint8_t* out) { (void)h; - (void)out; + if (out) memset(out, 0, sizeof(*out)); return sccl_unsupported("ncclParamGetU8"); } ncclResult_t ncclParamGetU16(ncclParamHandle_t h, uint16_t* out) { (void)h; - (void)out; + if (out) memset(out, 0, sizeof(*out)); return sccl_unsupported("ncclParamGetU16"); } ncclResult_t ncclParamGetU32(ncclParamHandle_t h, uint32_t* out) { (void)h; - (void)out; + if (out) memset(out, 0, sizeof(*out)); return sccl_unsupported("ncclParamGetU32"); } ncclResult_t ncclParamGetU64(ncclParamHandle_t h, uint64_t* out) { (void)h; - (void)out; + if (out) memset(out, 0, sizeof(*out)); return sccl_unsupported("ncclParamGetU64"); } ncclResult_t ncclParamGetStr(ncclParamHandle_t h, const char** out) { (void)h; - (void)out; + if (out) memset(out, 0, sizeof(*out)); return sccl_unsupported("ncclParamGetStr"); } @@ -277,20 +279,20 @@ ncclResult_t ncclParamGet(ncclParamHandle_t h, void* out, int maxLen, int* len) (void)h; (void)out; (void)maxLen; - (void)len; + if (len) memset(len, 0, sizeof(*len)); return sccl_unsupported("ncclParamGet"); } ncclResult_t ncclParamGetParameter(const char* key, const char** value, int* valueLen) { (void)key; - (void)value; - (void)valueLen; + if (value) memset(value, 0, sizeof(*value)); + if (valueLen) memset(valueLen, 0, sizeof(*valueLen)); return sccl_unsupported("ncclParamGetParameter"); } ncclResult_t ncclParamGetAllParameterKeys(const char*** table, int* tableLen) { - (void)table; - (void)tableLen; + if (table) memset(table, 0, sizeof(*table)); + if (tableLen) memset(tableLen, 0, sizeof(*tableLen)); return sccl_unsupported("ncclParamGetAllParameterKeys"); } diff --git a/spark_transport/libsircl/tests/test_api.py b/spark_transport/libsircl/tests/test_api.py index 92658b88..91df07f5 100644 --- a/spark_transport/libsircl/tests/test_api.py +++ b/spark_transport/libsircl/tests/test_api.py @@ -4,6 +4,7 @@ import argparse import concurrent.futures +import importlib.util import ctypes as C import json import os @@ -270,19 +271,29 @@ def separate(): self.assertEqual(LIB.ncclGroupEnd(), SUCCESS) self.assertIn(b"unsupported", LIB.ncclGetLastError(None)) - def test_unsupported_surface_preserves_every_output(self): - sentinel = C.create_string_buffer(b"Q" * 256) - before = bytes(sentinel) + def test_unsupported_surface_zeroes_its_outputs_and_preserves_every_input(self): + """Every unsupported function returns ncclInvalidUsage naming itself, zeroes the handle, value and + length outputs tools/generate_api.py lists (a NULL communicator, an all-zero unique id) and leaves every + other pointed-to byte as it was.""" + spec = importlib.util.spec_from_file_location("generate_api", ROOT / "tools/generate_api.py") + generator = importlib.util.module_from_spec(spec) + spec.loader.exec_module(generator) + sizes = {"ncclUniqueId*": 128, "int8_t*": 1, "uint8_t*": 1, "int16_t*": 2, "uint16_t*": 2, "int32_t*": 4, + "uint32_t*": 4, "int*": 4} for api in MANIFEST["functions"]: # ncclRedOpCreatePreMulSum writes its op (test_premulsum_refusal_returns_a_refused_op). if api["implemented"] or api["name"] == "ncclRedOpCreatePreMulSum": continue + zeroed = generator.ZEROED_OUTPUTS.get(api["name"], []) for prefix in ["", "p"]: with self.subTest(function=prefix + api["name"]): - args, types = [], [] + args, types, buffers = [], [], [] for argument in api["arguments"]: kind = argument["type"].removeprefix("const ") if "*" in kind or kind in ("ncclComm_t", "ncclWindow_t", "ncclParamHandle_t", "cudaStream_t"): + sentinel = C.create_string_buffer(b"Q" * 256) + size = sizes.get(kind.replace(" ", ""), 8) if argument["name"] in zeroed else 0 + buffers.append((argument["name"], sentinel, size)) types.append(C.c_void_p) args.append(C.cast(sentinel, C.c_void_p)) elif kind == "size_t": @@ -299,7 +310,8 @@ def test_unsupported_surface_preserves_every_output(self): function.restype = None if api["return"] == "void" else C.c_int result = function(*args) self.assertEqual(result, None if api["return"] == "void" else INVALID_USAGE) - self.assertEqual(bytes(sentinel), before) + for name, sentinel, size in buffers: + self.assertEqual(bytes(sentinel), bytes(size) + b"Q" * (256 - size) + b"\0", name) diagnostic = LIB.ncclGetLastError(None) self.assertTrue(diagnostic and api["name"].encode() in diagnostic) @@ -316,8 +328,8 @@ def test_premulsum_refusal_returns_a_refused_op(self): self.assertEqual(function(None, C.byref(scalar), 7, 1, None), INVALID_USAGE) def test_invalid_argument_inside_a_group_leaves_the_group(self): - # A refused call did nothing: ncclGroupEnd still succeeds (an unsupported call, ncclInvalidUsage, - # remains the group's result; test_thread_local_nested_group_and_error). + # A refused call did nothing: ncclGroupEnd still succeeds (as after an unsupported call, which + # enqueues nothing either; test_thread_local_nested_group_and_error). LIB.ncclAllReduce.argtypes = [C.c_void_p, C.c_void_p, C.c_size_t, C.c_int, C.c_int, C.c_void_p, C.c_void_p] LIB.ncclSend.argtypes = [C.c_void_p, C.c_size_t, C.c_int, C.c_int, C.c_void_p, C.c_void_p] diff --git a/spark_transport/libsircl/tools/generate_api.py b/spark_transport/libsircl/tools/generate_api.py index 3dc63235..3ebba3d6 100644 --- a/spark_transport/libsircl/tools/generate_api.py +++ b/spark_transport/libsircl/tools/generate_api.py @@ -68,12 +68,27 @@ def signature(api: dict, name: str | None = None) -> str: return f"{api['return']} {name or api['name']}({arguments})" -# Outputs that an unimplemented function sets before it returns ncclInvalidUsage, by function: the argument -# and its statement. ncclRedOpCreatePreMulSum returns ncclNumOps, the first value that is not a built-in op, -# so a caller that ignores the refusal and passes the op to a collective gets ncclInvalidArgument from the -# collective, never a plain sum from a handle left at 0 (ncclSum). +# Outputs that an unimplemented function sets before it returns ncclInvalidUsage, by function and argument: +# the statement. Handle, value and length outputs are zeroed (a NULL communicator or parameter handle, a zero +# value, an all-zero unique id, which every init refuses), so a caller that ignores the refusal never reads +# an uninitialized output. ncclRedOpCreatePreMulSum returns ncclNumOps, the first value that is not a +# built-in op, so a caller that ignores the refusal and passes the op to a collective gets +# ncclInvalidArgument from the collective, never a plain sum from a handle left at 0 (ncclSum). Buffers the +# caller sizes (ncclParamGet's `out`, ncclGroupSimulateEnd's `simInfo`) are left as they are. +def zeroed(name: str) -> str: + return f" if ({name}) memset({name}, 0, sizeof(*{name}));\n" + + +ZEROED_OUTPUTS = { + "ncclCommShrink": ["newcomm"], "ncclCommGrow": ["newcomm"], "ncclCommInitRankScalable": ["newcomm"], + "ncclCommGetUniqueId": ["uniqueId"], "ncclCommMemStats": ["value"], "ncclParamBind": ["out"], + **{f"ncclParamGet{kind}": ["out"] for kind in ("I8", "I16", "I32", "I64", "U8", "U16", "U32", "U64", "Str")}, + "ncclParamGet": ["len"], "ncclParamGetParameter": ["value", "valueLen"], + "ncclParamGetAllParameterKeys": ["table", "tableLen"], +} REFUSED_OUTPUTS = { - "ncclRedOpCreatePreMulSum": ("op", " if (op) *op = (ncclRedOp_t)ncclNumOps;\n"), + **{function: {name: zeroed(name) for name in names} for function, names in ZEROED_OUTPUTS.items()}, + "ncclRedOpCreatePreMulSum": {"op": " if (op) *op = (ncclRedOp_t)ncclNumOps;\n"}, } @@ -107,17 +122,17 @@ def artifacts() -> dict[Path, str]: ) + header[cuda_end:] generated = ("/* Generated by tools/generate_api.py from vendor/nccl.h.in (NVIDIA NCCL v2.32.3-1, Copyright (c)\n" " * 2015-2026 NVIDIA CORPORATION & AFFILIATES, Apache-2.0; see vendor/NCCL-LICENSE.txt). Do not edit. */\n") - stubs = generated + '#include "internal.h"\n\n' + stubs = generated + '#include "internal.h"\n\n#include \n\n' aliases = generated + '#include "internal.h"\n\n' for api in apis: if not api["implemented"]: stubs += signature(api) + " {\n" - output = REFUSED_OUTPUTS.get(api["name"]) + outputs = REFUSED_OUTPUTS.get(api["name"], {}) + unknown = set(outputs) - {argument["name"] for argument in api["arguments"]} + if unknown: + raise ValueError(f"{api['name']} has no argument {sorted(unknown)}") for argument in api["arguments"]: - if output and argument["name"] == output[0]: - stubs += output[1] - else: - stubs += f" (void){argument['name']};\n" + stubs += outputs.get(argument["name"], f" (void){argument['name']};\n") if api["return"] == "void": stubs += f" (void)sccl_unsupported(\"{api['name']}\");\n" else: From 5192113a67896e9ba7e12b3ce43cbd389f781afe Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Thu, 15 Oct 2026 08:39:08 -0500 Subject: [PATCH 23/65] Serialize bootstrap rounds and close claims on one communicator Concurrent use of one communicator from several threads is outside NVIDIA NCCL's contract, but these guards make it fail safe: - A bootstrap handle holds a mutex across each all-gather round, so rounds that threads start on one handle at once run one after another instead of interleaving their frames on the socket. A round's deadline counts from the moment it holds the lock. - The teardown rounds of ncclCommFinalize and ncclCommDestroy run under a per-communicator close lock: a second, concurrent close waits for the first and returns its result instead of running the rounds again. - transition and the close's READY-to-FINALIZED step use compare-exchange, so a state another thread stored in between is seen, not overwritten. Compatibility: single-threaded use is unchanged. Validation: tests/test_bootstrap.py starts two all-gathers on rank 0's handle at once while rank 1 runs two rounds, 20 times. Before the change the interleaved frames made the broker drop the group (results [6, 0, 0, 6]); after it every round succeeds and each of rank 1's rounds pairs with exactly one of rank 0's. The close lock and compare-exchange need a ready engine of two or more ranks to exercise and are not tested on CPU. make check and make check-asan pass. Request item 23 (audit 02-F7, 03-F11, 07-F1, 07-F4). NVIDIA NCCL source was neither read nor adapted. Co-Authored-By: Claude Opus 5.5 --- spark_transport/libsircl/src/api.c | 19 +++++++++--- spark_transport/libsircl/src/bootstrap.c | 25 +++++++++++++-- .../libsircl/tests/test_bootstrap.py | 31 +++++++++++++++++++ 3 files changed, 68 insertions(+), 7 deletions(-) diff --git a/spark_transport/libsircl/src/api.c b/spark_transport/libsircl/src/api.c index 8635e266..35aae2a3 100644 --- a/spark_transport/libsircl/src/api.c +++ b/spark_transport/libsircl/src/api.c @@ -32,8 +32,9 @@ struct ncclComm { sccl_engine *engine; char error_message[1024]; pthread_mutex_t error_lock; /* an error published after the status (publish_error) with its text */ - int quiesced; /* the close began (ncclCommFinalize, or ncclCommDestroy) */ + int quiesced; /* the close ran (ncclCommFinalize, or ncclCommDestroy); under close_lock */ int close_result; /* its result, returned again by a repeated ncclCommFinalize or destroy */ + pthread_mutex_t close_lock; /* held across the close, so a concurrent second close waits for its result */ uint64_t serial; /* creation number in this process: tells a reused address apart */ atomic_int state, error, grouped, cancelled; }; @@ -346,6 +347,7 @@ static ncclResult_t init_rank(ncclComm_t *out, int nranks, ncclUniqueId id, int struct ncclComm *comm = calloc(1, sizeof(*comm)); if (!comm) return creation_failed(ncclSystemError, "communicator allocation failed"); pthread_mutex_init(&comm->error_lock, NULL); + pthread_mutex_init(&comm->close_lock, NULL); comm->nranks = nranks; comm->rank = rank; comm->blocking = blocking; comm->bootstrap_only = bootstrap_only; comm->ctx = ctx; comm->position = position; memcpy(comm->id, id.internal, 128); @@ -948,8 +950,9 @@ static ncclResult_t transition(ncclComm_t handle, int next) { result = ncclInvalidUsage; why = "communicator initialization is still in progress"; } else if (result == ncclSuccess) { - if (state == READY || (state == next)) atomic_store(&comm->state, next); - else result = ncclInvalidUsage; + /* Compare-exchange: a state another thread stored in between is seen, not overwritten. */ + while ((state == READY || state == next) && !atomic_compare_exchange_weak(&comm->state, &state, next)) {} + if (state != READY && state != next) result = ncclInvalidUsage; } release(comm); return result == ncclSuccess ? result : fail(result, why); @@ -964,7 +967,14 @@ static ncclResult_t transition(ncclComm_t handle, int next) { * result (this rank's own error first, else ncclRemoteError naming the first rank that reported one, or a * round that not every rank reached) is kept, becomes the communicator's error when it failed, and is what a * repeated ncclCommFinalize or the destroy returns. */ +static ncclResult_t quiesce_locked(struct ncclComm *comm, const char *call); static ncclResult_t quiesce(struct ncclComm *comm, const char *call) { + pthread_mutex_lock(&comm->close_lock); + ncclResult_t result = quiesce_locked(comm, call); + pthread_mutex_unlock(&comm->close_lock); + return result; +} +static ncclResult_t quiesce_locked(struct ncclComm *comm, const char *call) { if (comm->quiesced) return (ncclResult_t)comm->close_result; if (!comm->engine || comm->bootstrap_only || comm->nranks < 2 || !comm->bootstrap) return ncclSuccess; /* A revoked communicator runs the rounds too: revoke stops new work, not the kernels and forwards already @@ -972,7 +982,7 @@ static ncclResult_t quiesce(struct ncclComm *comm, const char *call) { int state = atomic_load(&comm->state); if (state == INIT) return ncclSuccess; comm->quiesced = 1; - if (state == READY) atomic_store(&comm->state, FINALIZED); + if (state == READY) atomic_compare_exchange_strong(&comm->state, &state, FINALIZED); char message[700] = {0}; ncclResult_t result = (ncclResult_t)atomic_load(&comm->error); if (result != ncclSuccess) read_error(comm, message, sizeof message); @@ -1075,6 +1085,7 @@ static ncclResult_t dispose(ncclComm_t handle, int abort) { } sccl_bootstrap_close(comm->bootstrap); pthread_mutex_destroy(&comm->error_lock); + pthread_mutex_destroy(&comm->close_lock); free(comm); return result == ncclSuccess ? ncclSuccess : fail(result, message); } diff --git a/spark_transport/libsircl/src/bootstrap.c b/spark_transport/libsircl/src/bootstrap.c index 59f36702..195f5247 100644 --- a/spark_transport/libsircl/src/bootstrap.c +++ b/spark_transport/libsircl/src/bootstrap.c @@ -36,8 +36,10 @@ static const unsigned char gather_magic[4] = {'S','C','R','1'}; static _Thread_local char last_error[192]; /* `failed`: an all-gather on this handle failed after it began sending, so this rank's round count may differ - * from the broker's; every later all-gather fails at once instead of matching another round's records. */ -struct sccl_bootstrap { int fd; int nranks; int rank; uint32_t round; int failed; }; + * from the broker's; every later all-gather fails at once instead of matching another round's records. + * `lock` is held across each all-gather round, so rounds that threads start on one handle at once run one + * after another instead of interleaving their frames on the socket. */ +struct sccl_bootstrap { int fd; int nranks; int rank; uint32_t round; int failed; pthread_mutex_t lock; }; struct peer { int fd; int rank; @@ -701,6 +703,7 @@ int sccl_bootstrap_join_cancel(const unsigned char id[ID_SIZE], int nranks, int handle->rank = rank; handle->round = 0; handle->failed = 0; + pthread_mutex_init(&handle->lock, NULL); *out = handle; return OK; failed: @@ -726,7 +729,11 @@ void sccl_bootstrap_close(sccl_bootstrap *handle) { (void)error_result(USAGE, "fork after library use is unsupported; exec is required in the child"); return; } - if (handle) { close(handle->fd); free(handle); } + if (handle) { + close(handle->fd); + pthread_mutex_destroy(&handle->lock); + free(handle); + } reap_finished(); } @@ -743,6 +750,9 @@ int sccl_bootstrap_allgather(sccl_bootstrap *handle, const void *data, uint32_t return sccl_bootstrap_allgather_within(handle, data, bytes, out, lengths, frame_deadline_ms(), cancelled); } +static int allgather_locked(sccl_bootstrap *handle, const void *data, uint32_t bytes, void **out, + uint32_t *lengths, int timeout_ms, const atomic_int *cancelled); + int sccl_bootstrap_allgather_within(sccl_bootstrap *handle, const void *data, uint32_t bytes, void **out, uint32_t *lengths, int timeout_ms, const atomic_int *cancelled) { if (out) *out = NULL; @@ -750,6 +760,15 @@ int sccl_bootstrap_allgather_within(sccl_bootstrap *handle, const void *data, ui return error_result(USAGE, "fork after library use is unsupported; exec is required in the child"); if (!handle || !out || !lengths || (bytes && !data) || bytes > MAX_FRAME || timeout_ms < 1) return error_result(ARGUMENT, "all-gather needs a handle, outputs, at most 1 MiB per rank and a timeout"); + pthread_mutex_lock(&handle->lock); + int result = allgather_locked(handle, data, bytes, out, lengths, timeout_ms, cancelled); + pthread_mutex_unlock(&handle->lock); + return result; +} + +/* One all-gather round with the handle's lock held; its deadline counts from the moment the lock is held. */ +static int allgather_locked(sccl_bootstrap *handle, const void *data, uint32_t bytes, void **out, + uint32_t *lengths, int timeout_ms, const atomic_int *cancelled) { if (handle->failed) return error_result(REMOTE, "an earlier all-gather on this bootstrap failed, so its rounds are out of step"); int64_t deadline = now_ms() + timeout_ms; diff --git a/spark_transport/libsircl/tests/test_bootstrap.py b/spark_transport/libsircl/tests/test_bootstrap.py index 61fb88dc..ea339f70 100644 --- a/spark_transport/libsircl/tests/test_bootstrap.py +++ b/spark_transport/libsircl/tests/test_bootstrap.py @@ -426,6 +426,37 @@ def test_allgather_rounds(self): for handle in handles: self.bootstrap.close(handle) + def test_concurrent_allgathers_on_one_handle_run_one_after_another(self): + """Two threads start an all-gather on rank 0's handle at once while rank 1 runs two rounds: the two + rounds on rank 0 run one after another instead of interleaving their frames, so every round completes + and each of rank 1's rounds pairs with exactly one of rank 0's.""" + saved = os.environ.get("LIBSIRCL_SETUP_TIMEOUT_MS") + os.environ["LIBSIRCL_SETUP_TIMEOUT_MS"] = "3000" + try: + for attempt in range(20): + handles = self.joined_group(2) + try: + with concurrent.futures.ThreadPoolExecutor(max_workers=3) as pool: + zeros = [pool.submit(self.bootstrap.allgather, handles[0], payload * 3000, 2) + for payload in (b"a", b"b")] + ones = pool.submit(lambda: [self.bootstrap.allgather(handles[1], payload, 2) + for payload in (b"x", b"y")]) + zeros = [future.result(timeout=10) for future in zeros] + ones = ones.result(timeout=10) + self.assertEqual([result for result, _ in zeros + ones], [0] * 4, attempt) + seen_by_zero = {parts[1]: parts[0] for _, parts in zeros} + self.assertEqual(sorted(seen_by_zero), [b"x", b"y"], attempt) + for _, parts in ones: + self.assertEqual(parts[0], seen_by_zero[parts[1]], attempt) + finally: + for handle in handles: + self.bootstrap.close(handle) + finally: + if saved is None: + os.environ.pop("LIBSIRCL_SETUP_TIMEOUT_MS", None) + else: + os.environ["LIBSIRCL_SETUP_TIMEOUT_MS"] = saved + def test_allgather_fails_fast_when_a_rank_leaves(self): handles = self.joined_group(3) self.bootstrap.close(handles[2]) From 4d91d9f9c423bc87734e446230dc6c0f3389f349 Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Thu, 15 Oct 2026 08:43:09 -0500 Subject: [PATCH 24/65] Carry relay status separately from the forward windows libsircl took a lane as relayed only when LIBSIRCL_FORWARD_WINDOWS gave it a nonzero window. With tools/site_routes.py --max-window 0 neither forward nor point-to-point windows are printed, so relayed pairs looked direct: they received unwindowed point-to-point channels, which SIRCL's budget refuses, and the cycle plan could run across them. tools/site_routes.py prints LIBSIRCL_RELAYED_LANES on every rank with a lane that crosses relays: per peer position, 1 for each relayed lane and 0 for a cable. The engine counts a lane as relayed when that variable marks it or a forward window covers it: the relayed positions of the first setup exchange (which decide the cycle plan), the pair plan's relay check, and the point-to-point channel table, where a relayed lane without a point-to-point window gives its pair no channel. A malformed value fails setup naming the variable. README.md documents it, and the route data files in tests/data are regenerated with it. Compatibility: layouts whose relayed lanes all have forward windows behave the same (the data files gain only the new variable). With --max-window 0 or hand-written settings, relayed pairs without windows get no point-to-point channel and keep the cycle plan off. Validation: tests/test_site_routes.py checks that path:0-3 with --max-window 0 prints the relayed lanes SIRCL's routes give on every rank and no window (failed before: nothing printed). tests/test_engine_cpu.py captures rank 0's first setup record through a stand-in broker: LIBSIRCL_RELAYED_LANES=1=1/0,5=0/1 marks positions 1 and 5 (failed before: mask 0), and a malformed value is refused naming the variable. The channel table itself needs a ready engine and is not run on CPU. make check and make check-asan pass; the Markdown link check passes. Request item 24 (audit 09-1). NVIDIA NCCL source was neither read nor adapted. Co-Authored-By: Claude Opus 5.5 --- spark_transport/libsircl/README.md | 3 +- spark_transport/libsircl/src/engine.c | 70 ++++++++++++++++--- .../tests/data/site_routes_path0-3_l2.json | 4 ++ .../tests/data/site_routes_ring8_l2.json | 8 +++ .../data/site_routes_ring8_l2_p2p_alone.json | 8 +++ .../libsircl/tests/test_engine_cpu.py | 44 ++++++++++++ .../libsircl/tests/test_site_routes.py | 15 ++++ spark_transport/libsircl/tools/site_routes.py | 14 +++- 8 files changed, 155 insertions(+), 11 deletions(-) diff --git a/spark_transport/libsircl/README.md b/spark_transport/libsircl/README.md index 407dcd1d..b2482715 100644 --- a/spark_transport/libsircl/README.md +++ b/spark_transport/libsircl/README.md @@ -236,10 +236,11 @@ LF), and refuses a vendored copy that does not define the word; `P2P_FEATURES=0` | `LIBSIRCL_RING_REDUCE_PASSES` | the ring all-reduce's relay: `2` (default) stores each pack of its finished piece to link 3's slot, then copies the slot to the output; `1` stores both in one pass (the same protocol, one read of the slot fewer; research-only, see `STATUS.md`, and refused unless `LIBSIRCL_RING_REDUCE_ONE_PASS_RESEARCH=1`); agreed at setup, so every rank sets the same value | | `LIBSIRCL_LINK_TILE_BYTES` | bytes of every chunk one link reduce-scatter op carries (default 16 MiB); a call's chunks split into column tiles, which leave every element's arithmetic unchanged | | `LIBSIRCL_RING_WINDOW` | the ring plan: set on every rank when the ring that closes the chain can run, to the bytes this rank's ring lanes keep unacknowledged through relays (0 for cables); the ring schedules need it. One value per process serves every communicator it joins: a communicator whose ring next is reached through relays (a forward window toward it) while this gives 0, such as a pipeline pair of positions i and i + 4 of a ring of eight, keeps the smallest forward window toward that rank in flight on its ring lanes (the receipt's `ring_window_bytes`) | +| `LIBSIRCL_RELAYED_LANES` | the lanes that cross relays: `=[/],...` per lane, flag 1 for a relayed lane and 0 for a cable; a lane counts as relayed when this marks it or `LIBSIRCL_FORWARD_WINDOWS` gives it a window, so a relayed lane without a forward window keeps the cycle plan off and gets no unwindowed point-to-point channel. `tools/site_routes.py` prints it on every rank of a layout with relays | | `LIBSIRCL_FORWARD_WINDOWS`, `SIRCL_FORWARD_CHUNK_BYTES` | forward windows of relayed lanes: `=[/],...` per lane, 0 for a direct lane, and the chunk a windowed stripe posts in (default 32,768) | | `LIBSIRCL_P2P_CHANNELS` | `off` (default) or `on`, the same on every rank: a communicator of three or more ranks created with `on` has point-to-point channels and carries `ncclSend` and `ncclRecv` between its ranks on them (above); without them such a call is refused | | `SIRCL_P2P_SLOTS`, `SIRCL_P2P_SLOT_BYTES`, `SIRCL_P2P_BLOCKS`, `SIRCL_P2P_THREADS`, `SIRCL_P2P_UNROLL`, `SIRCL_P2P_CHUNK_BYTES` | the channels' geometry, SIRCL's names and defaults, agreed at setup: 8 slots (a power of two, 2-32) of 512 KiB (a multiple of 4096) per channel end, 4 blocks per launch (each taking every fourth item), 512 threads per block (a multiple of 32 from 64 to 512; SIRCL's kernels allow 1024), 4 packs per thread per pass, and a windowed stripe posted in chunks of 32,768 bytes | -| `LIBSIRCL_P2P_WINDOWS` | windows of this process's channel lanes through relays: `=[/],...` per lane, 0 or a multiple of 16 holding 1 to 120 chunks; a lane counts as relayed when `LIBSIRCL_FORWARD_WINDOWS` gives it a window, and a pair with a relayed lane in either direction without one has no channel. `tools/site_routes.py` prints them from SIRCL's point-to-point budget: `--p2p-reserve session` (the default, SIRCL's rule) leaves the collective session's forward windows, and with `--ring-schedules` (the site runs the ring schedules) its ring windows, their room in every relay queue first; `--session-share F` (default 1) sizes the session against F of every queue's share, so the channels get the rest; `none` sizes the channels as the only relayed traffic. On ring:8 the session's forward windows at the default share leave no relayed pair a channel; windows are whole 32 KiB chunks, so any share below 1 (0.95 to 0.05 checked) takes at least one chunk from every relayed session lane (at 0.95: 64-128 KiB become 32-96 KiB) and gives all 40 relayed ordered pairs a channel, as does `--max-window 32768` (every session lane one chunk) | +| `LIBSIRCL_P2P_WINDOWS` | windows of this process's channel lanes through relays: `=[/],...` per lane, 0 or a multiple of 16 holding 1 to 120 chunks; a lane counts as relayed when `LIBSIRCL_RELAYED_LANES` marks it or `LIBSIRCL_FORWARD_WINDOWS` gives it a window, and a pair with a relayed lane in either direction without one has no channel. `tools/site_routes.py` prints them from SIRCL's point-to-point budget: `--p2p-reserve session` (the default, SIRCL's rule) leaves the collective session's forward windows, and with `--ring-schedules` (the site runs the ring schedules) its ring windows, their room in every relay queue first; `--session-share F` (default 1) sizes the session against F of every queue's share, so the channels get the rest; `none` sizes the channels as the only relayed traffic. On ring:8 the session's forward windows at the default share leave no relayed pair a channel; windows are whole 32 KiB chunks, so any share below 1 (0.95 to 0.05 checked) takes at least one chunk from every relayed session lane (at 0.95: 64-128 KiB become 32-96 KiB) and gives all 40 relayed ordered pairs a channel, as does `--max-window 32768` (every session lane one chunk) | | `SIRCL_P2P_PROGRESS_CPU` | read by the channels' progress thread, as in SIRCL: a CPU list that pins it; unset, it runs where `LIBSIRCL_CPU_POLICY` places the transport's progress thread | | `LIBSIRCL_STREAM_ORDERED_ALLOC` | `auto` (default): staging uses stream-ordered allocation (`cuMemAllocAsync`) where the driver and device have it; `off` declines it (the communicator's own staging buffers; captured calls that need more are refused), the same on every rank, for comparisons and tests | | `SIRCL_STARTUP_WAIT_S`, `SIRCL_SERVING_WAIT_S`, `LIBSIRCL_WAIT_REGIME` | flag-wait limits and the regime a communicator starts in (`startup`, default, or `serving`); `sirclSetWaitRegime` switches it | diff --git a/spark_transport/libsircl/src/engine.c b/spark_transport/libsircl/src/engine.c index 502b1098..ddff1675 100644 --- a/spark_transport/libsircl/src/engine.c +++ b/spark_transport/libsircl/src/engine.c @@ -764,12 +764,52 @@ typedef struct { uint64_t relayed; } position_record; -/* The positions LIBSIRCL_FORWARD_WINDOWS gives a nonzero window toward (reached through relays), as bits; - * a malformed entry counts as none here (forward_windows() reports it). */ +/* LIBSIRCL_RELAYED_LANES, "=<0|1>[/<0|1>],...": 1 for every lane toward that position that crosses + * relays, whatever its windows. tools/site_routes.py prints it on every rank of a layout with relays, so a + * relayed lane is known as relayed even when no forward or point-to-point window is printed for it (for + * example with --max-window 0). Fills `flags` (position by lane); -1 with `err` on a malformed value. */ +static int relayed_lanes(uint8_t flags[MAX_POSITIONS][MAX_LANES], char *err, size_t len) { + memset(flags, 0, sizeof(uint8_t[MAX_POSITIONS][MAX_LANES])); + const char *text = sccl_env("LIBSIRCL_RELAYED_LANES"); + if (!text || !*text) return 0; + if (strlen(text) >= 1024) { + put_error(err, len, "LIBSIRCL_RELAYED_LANES is longer than 1023 characters"); + return -1; + } + char copy[1024]; + snprintf(copy, sizeof copy, "%s", text); + for (char *save = NULL, *entry = strtok_r(copy, ",", &save); entry; entry = strtok_r(NULL, ",", &save)) { + char *equals = strchr(entry, '='), *end; + long position = equals ? strtol(entry, &end, 10) : -1; + if (!equals || end != equals || position < 0 || position >= MAX_POSITIONS) { + put_error(err, len, "LIBSIRCL_RELAYED_LANES entry '%s' is not =<0|1>[/<0|1>]", entry); + return -1; + } + int lane = 0; + for (char *save2 = NULL, *value = strtok_r(equals + 1, "/", &save2); value; + value = strtok_r(NULL, "/", &save2), ++lane) { + if (lane >= MAX_LANES || (strcmp(value, "0") && strcmp(value, "1"))) { + put_error(err, len, "LIBSIRCL_RELAYED_LANES entry for position %ld needs 1 or 2 flags of 0 or 1", position); + return -1; + } + flags[position][lane] = value[0] == '1'; + } + } + return 0; +} + +/* The positions reached through relays, as bits: those LIBSIRCL_FORWARD_WINDOWS gives a nonzero window + * toward, and those LIBSIRCL_RELAYED_LANES marks. A malformed entry counts as none here (read_settings and + * forward_windows() report it). */ static uint64_t relayed_positions(void) { - const char *text = sccl_env("LIBSIRCL_FORWARD_WINDOWS"); + uint8_t flags[MAX_POSITIONS][MAX_LANES]; uint64_t mask = 0; - if (!text || !*text) return 0; + if (relayed_lanes(flags, NULL, 0) == 0) + for (int p = 0; p < MAX_POSITIONS; ++p) + for (int l = 0; l < MAX_LANES; ++l) + if (flags[p][l]) mask |= 1ull << p; + const char *text = sccl_env("LIBSIRCL_FORWARD_WINDOWS"); + if (!text || !*text) return mask; char copy[1024]; snprintf(copy, sizeof copy, "%s", text); for (char *save = NULL, *entry = strtok_r(copy, ",", &save); entry; entry = strtok_r(NULL, ",", &save)) { @@ -782,9 +822,15 @@ static uint64_t relayed_positions(void) { return mask; } -/* Whether LIBSIRCL_FORWARD_WINDOWS gives a nonzero window on some lane toward rank `peer` (the peer is - * reached through relays). A malformed value counts as none here; forward_windows() reports it. */ +/* Whether rank `peer` is reached through relays: LIBSIRCL_RELAYED_LANES marks a lane toward it, or + * LIBSIRCL_FORWARD_WINDOWS gives a lane toward it a nonzero window. A malformed value counts as none here; + * read_settings and forward_windows() report it. */ static int relayed_toward(const sccl_engine *e, int peer) { + uint8_t flags[MAX_POSITIONS][MAX_LANES]; + int position = e->positions[peer]; + if (relayed_lanes(flags, NULL, 0) == 0 && position >= 0 && position < MAX_POSITIONS) + for (int l = 0; l < MAX_LANES; ++l) + if (flags[position][l]) return 1; const char *text = sccl_env("LIBSIRCL_FORWARD_WINDOWS"); if (!text || !*text) return 0; char copy[1024]; @@ -922,6 +968,8 @@ static int read_settings(sccl_engine *e, int emulation, char *err, size_t len) { return -1; } e->fail_stop = !stop || !*stop || !strcmp(stop, "0") ? 0 : !strcmp(stop, "1") ? FAIL_STOP_EXIT : FAIL_STOP_ABORT; + uint8_t relayed[MAX_POSITIONS][MAX_LANES]; + if (relayed_lanes(relayed, err, len)) return -1; if (env_u64("LIBSIRCL_RING_REDUCE_PASSES", 2, 1, 2, &v, err, len)) return -1; const char *one_pass = sccl_env("LIBSIRCL_RING_REDUCE_ONE_PASS_RESEARCH"); if (v == 1 && (!one_pass || strcmp(one_pass, "1"))) { @@ -1363,11 +1411,17 @@ static int p2p_windows(sccl_engine *e, char *err, size_t len) { } } } + /* A lane is relayed when it has a forward window or LIBSIRCL_RELAYED_LANES marks it; a relayed lane without + * a point-to-point window gives its pair no channel (SIRCL's budget refuses unwindowed relayed lanes). */ + uint8_t relayed[MAX_POSITIONS][MAX_LANES]; + if (relayed_lanes(relayed, err, len)) return -1; e->p2p_out = 0; for (int p = 0; p < e->world; ++p) { - int ok = p != e->rank; + int ok = p != e->rank, position = e->positions[p]; for (int l = 0; ok && l < e->lanes; ++l) - if (e->forward[p][l] && !e->p2p_window[p][l]) ok = 0; + if ((e->forward[p][l] || (position >= 0 && position < MAX_POSITIONS && relayed[position][l])) && + !e->p2p_window[p][l]) + ok = 0; if (ok) e->p2p_out |= 1u << p; } return 0; diff --git a/spark_transport/libsircl/tests/data/site_routes_path0-3_l2.json b/spark_transport/libsircl/tests/data/site_routes_path0-3_l2.json index b18103e9..2bc679e4 100644 --- a/spark_transport/libsircl/tests/data/site_routes_path0-3_l2.json +++ b/spark_transport/libsircl/tests/data/site_routes_path0-3_l2.json @@ -20,6 +20,7 @@ "LIBSIRCL_POSITION": "0", "SIRCL_PEER_ROUTES": "1=rocep1s0f0/roceP2p1s0f0,2=rocep1s0f0/roceP2p1s0f0,3=rocep1s0f0/roceP2p1s0f0", "LIBSIRCL_CHAIN_ORDER": "0,1,2,3", + "LIBSIRCL_RELAYED_LANES": "2=1/1,3=1/1", "LIBSIRCL_FORWARD_WINDOWS": "2=131072/131072,3=131072/131072", "SIRCL_FORWARD_CHUNK_BYTES": "32768", "LIBSIRCL_RING_WINDOW": "0", @@ -33,6 +34,7 @@ "LIBSIRCL_POSITION": "1", "SIRCL_PEER_ROUTES": "0=rocep1s0f1/roceP2p1s0f1,2=rocep1s0f0/roceP2p1s0f0,3=rocep1s0f0/roceP2p1s0f0", "LIBSIRCL_CHAIN_ORDER": "0,1,2,3", + "LIBSIRCL_RELAYED_LANES": "3=1/1", "LIBSIRCL_FORWARD_WINDOWS": "3=131072/131072", "SIRCL_FORWARD_CHUNK_BYTES": "32768", "LIBSIRCL_RING_WINDOW": "0", @@ -46,6 +48,7 @@ "LIBSIRCL_POSITION": "2", "SIRCL_PEER_ROUTES": "0=rocep1s0f1/roceP2p1s0f1,1=rocep1s0f1/roceP2p1s0f1,3=rocep1s0f0/roceP2p1s0f0", "LIBSIRCL_CHAIN_ORDER": "0,1,2,3", + "LIBSIRCL_RELAYED_LANES": "0=1/1", "LIBSIRCL_FORWARD_WINDOWS": "0=131072/131072", "SIRCL_FORWARD_CHUNK_BYTES": "32768", "LIBSIRCL_RING_WINDOW": "0", @@ -59,6 +62,7 @@ "LIBSIRCL_POSITION": "3", "SIRCL_PEER_ROUTES": "0=rocep1s0f1/roceP2p1s0f1,1=rocep1s0f1/roceP2p1s0f1,2=rocep1s0f1/roceP2p1s0f1", "LIBSIRCL_CHAIN_ORDER": "0,1,2,3", + "LIBSIRCL_RELAYED_LANES": "0=1/1,1=1/1", "LIBSIRCL_FORWARD_WINDOWS": "0=131072/131072,1=131072/131072", "SIRCL_FORWARD_CHUNK_BYTES": "32768", "LIBSIRCL_RING_WINDOW": "393216", diff --git a/spark_transport/libsircl/tests/data/site_routes_ring8_l2.json b/spark_transport/libsircl/tests/data/site_routes_ring8_l2.json index e3bf1eab..72e7eb9d 100644 --- a/spark_transport/libsircl/tests/data/site_routes_ring8_l2.json +++ b/spark_transport/libsircl/tests/data/site_routes_ring8_l2.json @@ -225,6 +225,7 @@ "LIBSIRCL_POSITION": "0", "SIRCL_PEER_ROUTES": "1=rocep1s0f0/roceP2p1s0f0,2=rocep1s0f0/roceP2p1s0f0,3=rocep1s0f0/roceP2p1s0f0,4=rocep1s0f0/roceP2p1s0f1,5=rocep1s0f1/roceP2p1s0f1,6=rocep1s0f1/roceP2p1s0f1,7=rocep1s0f1/roceP2p1s0f1", "LIBSIRCL_CHAIN_ORDER": "0,1,2,3,4,5,6,7", + "LIBSIRCL_RELAYED_LANES": "2=1/1,3=1/1,4=1/1,5=1/1,6=1/1", "LIBSIRCL_FORWARD_WINDOWS": "2=98304/65536,3=65536/65536,4=65536/65536,5=98304/65536,6=131072/65536", "SIRCL_FORWARD_CHUNK_BYTES": "32768", "LIBSIRCL_RING_WINDOW": "0" @@ -236,6 +237,7 @@ "LIBSIRCL_POSITION": "1", "SIRCL_PEER_ROUTES": "0=rocep1s0f1/roceP2p1s0f1,2=rocep1s0f0/roceP2p1s0f0,3=rocep1s0f0/roceP2p1s0f0,4=rocep1s0f0/roceP2p1s0f0,5=rocep1s0f0/roceP2p1s0f1,6=rocep1s0f1/roceP2p1s0f1,7=rocep1s0f1/roceP2p1s0f1", "LIBSIRCL_CHAIN_ORDER": "0,1,2,3,4,5,6,7", + "LIBSIRCL_RELAYED_LANES": "3=1/1,4=1/1,5=1/1,6=1/1,7=1/1", "LIBSIRCL_FORWARD_WINDOWS": "3=65536/98304,4=65536/98304,5=65536/65536,6=131072/65536,7=131072/65536", "SIRCL_FORWARD_CHUNK_BYTES": "32768", "LIBSIRCL_RING_WINDOW": "0" @@ -247,6 +249,7 @@ "LIBSIRCL_POSITION": "2", "SIRCL_PEER_ROUTES": "0=rocep1s0f1/roceP2p1s0f1,1=rocep1s0f1/roceP2p1s0f1,3=rocep1s0f0/roceP2p1s0f0,4=rocep1s0f0/roceP2p1s0f0,5=rocep1s0f0/roceP2p1s0f0,6=rocep1s0f0/roceP2p1s0f1,7=rocep1s0f1/roceP2p1s0f1", "LIBSIRCL_CHAIN_ORDER": "0,1,2,3,4,5,6,7", + "LIBSIRCL_RELAYED_LANES": "0=1/1,4=1/1,5=1/1,6=1/1,7=1/1", "LIBSIRCL_FORWARD_WINDOWS": "0=98304/65536,4=65536/131072,5=65536/131072,6=65536/65536,7=98304/65536", "SIRCL_FORWARD_CHUNK_BYTES": "32768", "LIBSIRCL_RING_WINDOW": "0" @@ -258,6 +261,7 @@ "LIBSIRCL_POSITION": "3", "SIRCL_PEER_ROUTES": "0=rocep1s0f1/roceP2p1s0f1,1=rocep1s0f1/roceP2p1s0f1,2=rocep1s0f1/roceP2p1s0f1,4=rocep1s0f0/roceP2p1s0f0,5=rocep1s0f0/roceP2p1s0f0,6=rocep1s0f0/roceP2p1s0f0,7=rocep1s0f0/roceP2p1s0f1", "LIBSIRCL_CHAIN_ORDER": "0,1,2,3,4,5,6,7", + "LIBSIRCL_RELAYED_LANES": "0=1/1,1=1/1,5=1/1,6=1/1,7=1/1", "LIBSIRCL_FORWARD_WINDOWS": "0=65536/65536,1=65536/98304,5=65536/131072,6=65536/98304,7=65536/65536", "SIRCL_FORWARD_CHUNK_BYTES": "32768", "LIBSIRCL_RING_WINDOW": "0" @@ -269,6 +273,7 @@ "LIBSIRCL_POSITION": "4", "SIRCL_PEER_ROUTES": "0=rocep1s0f1/roceP2p1s0f0,1=rocep1s0f1/roceP2p1s0f1,2=rocep1s0f1/roceP2p1s0f1,3=rocep1s0f1/roceP2p1s0f1,5=rocep1s0f0/roceP2p1s0f0,6=rocep1s0f0/roceP2p1s0f0,7=rocep1s0f0/roceP2p1s0f0", "LIBSIRCL_CHAIN_ORDER": "0,1,2,3,4,5,6,7", + "LIBSIRCL_RELAYED_LANES": "0=1/1,1=1/1,2=1/1,6=1/1,7=1/1", "LIBSIRCL_FORWARD_WINDOWS": "0=65536/65536,1=65536/98304,2=65536/131072,6=65536/98304,7=65536/65536", "SIRCL_FORWARD_CHUNK_BYTES": "32768", "LIBSIRCL_RING_WINDOW": "0" @@ -280,6 +285,7 @@ "LIBSIRCL_POSITION": "5", "SIRCL_PEER_ROUTES": "0=rocep1s0f0/roceP2p1s0f0,1=rocep1s0f1/roceP2p1s0f0,2=rocep1s0f1/roceP2p1s0f1,3=rocep1s0f1/roceP2p1s0f1,4=rocep1s0f1/roceP2p1s0f1,6=rocep1s0f0/roceP2p1s0f0,7=rocep1s0f0/roceP2p1s0f0", "LIBSIRCL_CHAIN_ORDER": "0,1,2,3,4,5,6,7", + "LIBSIRCL_RELAYED_LANES": "0=1/1,1=1/1,2=1/1,3=1/1,7=1/1", "LIBSIRCL_FORWARD_WINDOWS": "0=98304/65536,1=65536/65536,2=65536/131072,3=65536/131072,7=98304/65536", "SIRCL_FORWARD_CHUNK_BYTES": "32768", "LIBSIRCL_RING_WINDOW": "0" @@ -291,6 +297,7 @@ "LIBSIRCL_POSITION": "6", "SIRCL_PEER_ROUTES": "0=rocep1s0f0/roceP2p1s0f0,1=rocep1s0f0/roceP2p1s0f0,2=rocep1s0f1/roceP2p1s0f0,3=rocep1s0f1/roceP2p1s0f1,4=rocep1s0f1/roceP2p1s0f1,5=rocep1s0f1/roceP2p1s0f1,7=rocep1s0f0/roceP2p1s0f0", "LIBSIRCL_CHAIN_ORDER": "0,1,2,3,4,5,6,7", + "LIBSIRCL_RELAYED_LANES": "0=1/1,1=1/1,2=1/1,3=1/1,4=1/1", "LIBSIRCL_FORWARD_WINDOWS": "0=131072/65536,1=131072/65536,2=65536/65536,3=65536/98304,4=65536/98304", "SIRCL_FORWARD_CHUNK_BYTES": "32768", "LIBSIRCL_RING_WINDOW": "0" @@ -302,6 +309,7 @@ "LIBSIRCL_POSITION": "7", "SIRCL_PEER_ROUTES": "0=rocep1s0f0/roceP2p1s0f0,1=rocep1s0f0/roceP2p1s0f0,2=rocep1s0f0/roceP2p1s0f0,3=rocep1s0f1/roceP2p1s0f0,4=rocep1s0f1/roceP2p1s0f1,5=rocep1s0f1/roceP2p1s0f1,6=rocep1s0f1/roceP2p1s0f1", "LIBSIRCL_CHAIN_ORDER": "0,1,2,3,4,5,6,7", + "LIBSIRCL_RELAYED_LANES": "1=1/1,2=1/1,3=1/1,4=1/1,5=1/1", "LIBSIRCL_FORWARD_WINDOWS": "1=131072/65536,2=98304/65536,3=65536/65536,4=65536/65536,5=98304/65536", "SIRCL_FORWARD_CHUNK_BYTES": "32768", "LIBSIRCL_RING_WINDOW": "0" diff --git a/spark_transport/libsircl/tests/data/site_routes_ring8_l2_p2p_alone.json b/spark_transport/libsircl/tests/data/site_routes_ring8_l2_p2p_alone.json index 5059e5bd..c5c95880 100644 --- a/spark_transport/libsircl/tests/data/site_routes_ring8_l2_p2p_alone.json +++ b/spark_transport/libsircl/tests/data/site_routes_ring8_l2_p2p_alone.json @@ -24,6 +24,7 @@ "LIBSIRCL_POSITION": "0", "SIRCL_PEER_ROUTES": "1=rocep1s0f0/roceP2p1s0f0,2=rocep1s0f0/roceP2p1s0f0,3=rocep1s0f0/roceP2p1s0f0,4=rocep1s0f0/roceP2p1s0f1,5=rocep1s0f1/roceP2p1s0f1,6=rocep1s0f1/roceP2p1s0f1,7=rocep1s0f1/roceP2p1s0f1", "LIBSIRCL_CHAIN_ORDER": "0,1,2,3,4,5,6,7", + "LIBSIRCL_RELAYED_LANES": "2=1/1,3=1/1,4=1/1,5=1/1,6=1/1", "LIBSIRCL_FORWARD_WINDOWS": "2=98304/65536,3=65536/65536,4=65536/65536,5=98304/65536,6=131072/65536", "SIRCL_FORWARD_CHUNK_BYTES": "32768", "LIBSIRCL_RING_WINDOW": "0", @@ -37,6 +38,7 @@ "LIBSIRCL_POSITION": "1", "SIRCL_PEER_ROUTES": "0=rocep1s0f1/roceP2p1s0f1,2=rocep1s0f0/roceP2p1s0f0,3=rocep1s0f0/roceP2p1s0f0,4=rocep1s0f0/roceP2p1s0f0,5=rocep1s0f0/roceP2p1s0f1,6=rocep1s0f1/roceP2p1s0f1,7=rocep1s0f1/roceP2p1s0f1", "LIBSIRCL_CHAIN_ORDER": "0,1,2,3,4,5,6,7", + "LIBSIRCL_RELAYED_LANES": "3=1/1,4=1/1,5=1/1,6=1/1,7=1/1", "LIBSIRCL_FORWARD_WINDOWS": "3=65536/98304,4=65536/98304,5=65536/65536,6=131072/65536,7=131072/65536", "SIRCL_FORWARD_CHUNK_BYTES": "32768", "LIBSIRCL_RING_WINDOW": "0", @@ -50,6 +52,7 @@ "LIBSIRCL_POSITION": "2", "SIRCL_PEER_ROUTES": "0=rocep1s0f1/roceP2p1s0f1,1=rocep1s0f1/roceP2p1s0f1,3=rocep1s0f0/roceP2p1s0f0,4=rocep1s0f0/roceP2p1s0f0,5=rocep1s0f0/roceP2p1s0f0,6=rocep1s0f0/roceP2p1s0f1,7=rocep1s0f1/roceP2p1s0f1", "LIBSIRCL_CHAIN_ORDER": "0,1,2,3,4,5,6,7", + "LIBSIRCL_RELAYED_LANES": "0=1/1,4=1/1,5=1/1,6=1/1,7=1/1", "LIBSIRCL_FORWARD_WINDOWS": "0=98304/65536,4=65536/131072,5=65536/131072,6=65536/65536,7=98304/65536", "SIRCL_FORWARD_CHUNK_BYTES": "32768", "LIBSIRCL_RING_WINDOW": "0", @@ -63,6 +66,7 @@ "LIBSIRCL_POSITION": "3", "SIRCL_PEER_ROUTES": "0=rocep1s0f1/roceP2p1s0f1,1=rocep1s0f1/roceP2p1s0f1,2=rocep1s0f1/roceP2p1s0f1,4=rocep1s0f0/roceP2p1s0f0,5=rocep1s0f0/roceP2p1s0f0,6=rocep1s0f0/roceP2p1s0f0,7=rocep1s0f0/roceP2p1s0f1", "LIBSIRCL_CHAIN_ORDER": "0,1,2,3,4,5,6,7", + "LIBSIRCL_RELAYED_LANES": "0=1/1,1=1/1,5=1/1,6=1/1,7=1/1", "LIBSIRCL_FORWARD_WINDOWS": "0=65536/65536,1=65536/98304,5=65536/131072,6=65536/98304,7=65536/65536", "SIRCL_FORWARD_CHUNK_BYTES": "32768", "LIBSIRCL_RING_WINDOW": "0", @@ -76,6 +80,7 @@ "LIBSIRCL_POSITION": "4", "SIRCL_PEER_ROUTES": "0=rocep1s0f1/roceP2p1s0f0,1=rocep1s0f1/roceP2p1s0f1,2=rocep1s0f1/roceP2p1s0f1,3=rocep1s0f1/roceP2p1s0f1,5=rocep1s0f0/roceP2p1s0f0,6=rocep1s0f0/roceP2p1s0f0,7=rocep1s0f0/roceP2p1s0f0", "LIBSIRCL_CHAIN_ORDER": "0,1,2,3,4,5,6,7", + "LIBSIRCL_RELAYED_LANES": "0=1/1,1=1/1,2=1/1,6=1/1,7=1/1", "LIBSIRCL_FORWARD_WINDOWS": "0=65536/65536,1=65536/98304,2=65536/131072,6=65536/98304,7=65536/65536", "SIRCL_FORWARD_CHUNK_BYTES": "32768", "LIBSIRCL_RING_WINDOW": "0", @@ -89,6 +94,7 @@ "LIBSIRCL_POSITION": "5", "SIRCL_PEER_ROUTES": "0=rocep1s0f0/roceP2p1s0f0,1=rocep1s0f1/roceP2p1s0f0,2=rocep1s0f1/roceP2p1s0f1,3=rocep1s0f1/roceP2p1s0f1,4=rocep1s0f1/roceP2p1s0f1,6=rocep1s0f0/roceP2p1s0f0,7=rocep1s0f0/roceP2p1s0f0", "LIBSIRCL_CHAIN_ORDER": "0,1,2,3,4,5,6,7", + "LIBSIRCL_RELAYED_LANES": "0=1/1,1=1/1,2=1/1,3=1/1,7=1/1", "LIBSIRCL_FORWARD_WINDOWS": "0=98304/65536,1=65536/65536,2=65536/131072,3=65536/131072,7=98304/65536", "SIRCL_FORWARD_CHUNK_BYTES": "32768", "LIBSIRCL_RING_WINDOW": "0", @@ -102,6 +108,7 @@ "LIBSIRCL_POSITION": "6", "SIRCL_PEER_ROUTES": "0=rocep1s0f0/roceP2p1s0f0,1=rocep1s0f0/roceP2p1s0f0,2=rocep1s0f1/roceP2p1s0f0,3=rocep1s0f1/roceP2p1s0f1,4=rocep1s0f1/roceP2p1s0f1,5=rocep1s0f1/roceP2p1s0f1,7=rocep1s0f0/roceP2p1s0f0", "LIBSIRCL_CHAIN_ORDER": "0,1,2,3,4,5,6,7", + "LIBSIRCL_RELAYED_LANES": "0=1/1,1=1/1,2=1/1,3=1/1,4=1/1", "LIBSIRCL_FORWARD_WINDOWS": "0=131072/65536,1=131072/65536,2=65536/65536,3=65536/98304,4=65536/98304", "SIRCL_FORWARD_CHUNK_BYTES": "32768", "LIBSIRCL_RING_WINDOW": "0", @@ -115,6 +122,7 @@ "LIBSIRCL_POSITION": "7", "SIRCL_PEER_ROUTES": "0=rocep1s0f0/roceP2p1s0f0,1=rocep1s0f0/roceP2p1s0f0,2=rocep1s0f0/roceP2p1s0f0,3=rocep1s0f1/roceP2p1s0f0,4=rocep1s0f1/roceP2p1s0f1,5=rocep1s0f1/roceP2p1s0f1,6=rocep1s0f1/roceP2p1s0f1", "LIBSIRCL_CHAIN_ORDER": "0,1,2,3,4,5,6,7", + "LIBSIRCL_RELAYED_LANES": "1=1/1,2=1/1,3=1/1,4=1/1,5=1/1", "LIBSIRCL_FORWARD_WINDOWS": "1=131072/65536,2=98304/65536,3=65536/65536,4=65536/65536,5=98304/65536", "SIRCL_FORWARD_CHUNK_BYTES": "32768", "LIBSIRCL_RING_WINDOW": "0", diff --git a/spark_transport/libsircl/tests/test_engine_cpu.py b/spark_transport/libsircl/tests/test_engine_cpu.py index ce5dede9..972ea1f7 100644 --- a/spark_transport/libsircl/tests/test_engine_cpu.py +++ b/spark_transport/libsircl/tests/test_engine_cpu.py @@ -186,6 +186,50 @@ def read(n): self.assertIn("rank 1 position", data["local"]) self.assertIn("0-63", data["local"]) + def test_relayed_lanes_reach_the_setup_exchange(self): + """LIBSIRCL_RELAYED_LANES marks a peer position as relayed in the first setup exchange's record (which + the cycle plan reads) without any forward window; a malformed value fails creation naming it.""" + script = '''import socket, struct, threading +u=U(); assert l.ncclGetUniqueId(C.byref(u))==0 +server=socket.socket(); server.bind(("127.0.0.1", 0)); server.listen() +u.b[16:18]=struct.pack("!H",server.getsockname()[1]) +records=[] +def broker(): + peer,_=server.accept(); peer.settimeout(3) + def read(n): + data=b"" + while len(data) int: } if order is not None: env["LIBSIRCL_CHAIN_ORDER"] = ",".join(str(r) for r in order) + relays = {peer: [1 if lane.relays else 0 for lane in group.lanes_to(rank, peer)] + for peer in range(world) if peer != rank} + if any(any(flags) for flags in relays.values()): + env["LIBSIRCL_RELAYED_LANES"] = ",".join(f"{peer}={'/'.join(str(f) for f in flags)}" + for peer, flags in relays.items() if any(flags)) windows = [f"{peer}={'/'.join(str(w) for w in table[peer])}" for peer in range(world) if peer != rank and any(table[peer])] if windows: From 1799fda54e52fb9151f13180cc6c415969b2040f Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Thu, 15 Oct 2026 08:46:27 -0500 Subject: [PATCH 25/65] Keep route windows within native caps and relay queues within their share tools/site_routes.py lowers every printed window to the native layers' caps: a forward window to 60 chunks of SIRCL_FORWARD_CHUNK_BYTES, the ring window to 60 chunks of 32 KiB, a point-to-point window to 120 chunks of SIRCL_P2P_CHUNK_BYTES. It then checks every relay hairpin queue: the session's windows through it (forward windows, and with --ring-schedules the larger of those and its ring windows) plus, under --p2p-reserve session, the channels' windows must fit the queue's share (75 % of 512 KiB). A plan that overfills a queue is refused with exit status 1 and a message naming the queue; under --p2p-reserve none only the session's windows are checked, since the channels are then sized as the only relayed traffic. The engine validates the windows it reads, with errors naming the variable: a LIBSIRCL_FORWARD_WINDOWS window must hold 1 to 60 chunks of SIRCL_FORWARD_CHUNK_BYTES, and LIBSIRCL_RING_WINDOW 1 to 60 chunks of 32 KiB. A ring window derived from the forward windows is never rounded up past them: a forward window below one 32 KiB ring chunk refuses the communicator instead. Compatibility: plans within the caps and shares print the same values, and the route data files are unchanged. Plans that SIRCL's planner sizes past a cap print the capped window; plans that overfill a queue, such as --chunk 65536 on path:0-7, are refused instead of failing at setup or oversubscribing the queue. Validation: tests/test_site_routes.py checks --chunk 1024 on ring:8 (forward windows at most 60 chunks; before: 131,072-byte windows of 128 chunks), --p2p-chunk 1024 --p2p-reserve none (point-to-point windows at most 120 chunks), and --chunk 65536 on path:0-7 (refused naming the queue; before: printed). tests/test_engine_cpu.py checks that a LIBSIRCL_RING_WINDOW of 61 chunks is refused naming the variable. The forward-window and derived ring-window checks need a transport and are not reached on CPU. make check and make check-asan pass. Request item 25 (audit 09-2, 09-3, 09-4). NVIDIA NCCL source was neither read nor adapted. Co-Authored-By: Claude Opus 5.5 --- spark_transport/libsircl/src/engine.c | 39 +++++++++++++---- .../libsircl/tests/test_engine_cpu.py | 7 ++- .../libsircl/tests/test_site_routes.py | 26 +++++++++++ spark_transport/libsircl/tools/site_routes.py | 43 ++++++++++++++++++- 4 files changed, 104 insertions(+), 11 deletions(-) diff --git a/spark_transport/libsircl/src/engine.c b/spark_transport/libsircl/src/engine.c index ddff1675..5e77002b 100644 --- a/spark_transport/libsircl/src/engine.c +++ b/spark_transport/libsircl/src/engine.c @@ -285,6 +285,9 @@ enum { #define P2P_DEFAULT_UNROLL 4u #define P2P_DEFAULT_CHUNK_BYTES 32768u +/* The native layer's ring links post through relays in chunks of 32 KiB and accept a ring window of 1 to + * 60 such chunks (roce_set_links). */ +enum { RING_WINDOW_CHUNK = 32768, RING_WINDOW_MAX_CHUNKS = 60 }; enum { ALG_AUTO = -1 }; /* LIBSIRCL_FAIL_STOP's modes (see fail_stop). */ enum { FAIL_STOP_EXIT = 1, FAIL_STOP_ABORT = 2 }; @@ -752,6 +755,11 @@ static int link_settings(sccl_engine *e, shared_settings *s, int pair_plan, char e->ring_window = 0; if (ring && *ring) { if (env_u64("LIBSIRCL_RING_WINDOW", 0, 0, 0xFFFFFFFFu, &v, err, len)) return -1; + if (v && (v < RING_WINDOW_CHUNK || v / RING_WINDOW_CHUNK > RING_WINDOW_MAX_CHUNKS)) { + put_error(err, len, "LIBSIRCL_RING_WINDOW=%" PRIu64 " must be 0 or hold 1 to %d chunks of %d bytes", v, + RING_WINDOW_MAX_CHUNKS, RING_WINDOW_CHUNK); + return -1; + } e->ring_window = (uint32_t)v; } return 0; @@ -1296,6 +1304,8 @@ static int start_progress(sccl_engine *e) { * rank at that position (SIRCL's routes.forward_windows for the layout; tools/site_routes.py prints it), * 0 for a direct lane; positions outside the communicator are ignored. SIRCL_FORWARD_CHUNK_BYTES is the * chunk a windowed stripe is posted in (SIRCL's default 32,768). */ +/* The native proxy accepts a forward window of 1 to 60 chunks (roce_set_forward). */ +enum { FORWARD_MAX_CHUNKS = 60 }; static int forward_windows(sccl_engine *e, char *err, size_t len) { memset(e->forward, 0, sizeof e->forward); e->forward_lanes = 0; @@ -1330,6 +1340,12 @@ static int forward_windows(sccl_engine *e, char *err, size_t len) { put_error(err, len, "LIBSIRCL_FORWARD_WINDOWS entry '%s' needs 1 or 2 byte counts", entry); return -1; } + if (bytes && (bytes < chunk || bytes / chunk > FORWARD_MAX_CHUNKS)) { + put_error(err, len, "LIBSIRCL_FORWARD_WINDOWS: the window of %lu bytes toward position %ld must be 0 or " + "hold 1 to %d chunks of SIRCL_FORWARD_CHUNK_BYTES (%" PRIu64 ")", bytes, position, + FORWARD_MAX_CHUNKS, chunk); + return -1; + } if (peer < 0 || peer == e->rank || lane >= e->lanes) continue; e->forward[peer][lane] = (uint32_t)bytes; if (bytes) e->forward_lanes += 1; @@ -1338,9 +1354,6 @@ static int forward_windows(sccl_engine *e, char *err, size_t len) { return 0; } -/* The native layer's ring links post through relays in chunks of 32 KiB and accept a ring window of 1 to - * 60 such chunks (roce_set_links). */ -enum { RING_WINDOW_CHUNK = 32768, RING_WINDOW_MAX_CHUNKS = 60 }; /* The window of this communicator's ring lanes toward its ring's next rank. LIBSIRCL_RING_WINDOW is the * process's layout's ring plan, the same for every communicator the process joins; for the layout's own @@ -1349,18 +1362,25 @@ enum { RING_WINDOW_CHUNK = 32768, RING_WINDOW_MAX_CHUNKS = 60 }; * pairs i and i + 4 of a ring of eight) may have a ring next reached through relays, which the forward * windows name: its ring lanes then keep the smallest forward window toward that rank in flight, never * posting through relays unwindowed. A nonzero LIBSIRCL_RING_WINDOW is the planner's own value and is kept. */ -static void ring_lane_window(sccl_engine *e) { - if (e->ring_window || e->ring_next < 0) return; +static int ring_lane_window(sccl_engine *e, char *err, size_t len) { + if (e->ring_window || e->ring_next < 0) return 0; uint32_t window = 0; for (int l = 0; l < e->lanes; ++l) { uint32_t lane = e->forward[e->ring_next][l]; if (lane && (!window || lane < window)) window = lane; } - if (!window) return; + if (!window) return 0; + /* Never above the forward window: a forward window below one ring chunk leaves the ring lanes no window + * within it, and the communicator is refused rather than posting past it. */ + if (window < RING_WINDOW_CHUNK) { + put_error(err, len, "the ring lanes toward rank %d cross relays whose forward window (LIBSIRCL_FORWARD_WINDOWS) " + "is %u bytes, below one %d-byte ring chunk", e->ring_next, window, RING_WINDOW_CHUNK); + return -1; + } window = window / RING_WINDOW_CHUNK * RING_WINDOW_CHUNK; - if (window < RING_WINDOW_CHUNK) window = RING_WINDOW_CHUNK; if (window > RING_WINDOW_CHUNK * RING_WINDOW_MAX_CHUNKS) window = RING_WINDOW_CHUNK * RING_WINDOW_MAX_CHUNKS; e->ring_window = window; + return 0; } /* LIBSIRCL_P2P_WINDOWS: =[/],... the window of each of this process's lanes toward the @@ -2284,8 +2304,9 @@ int sccl_engine_create(sccl_bootstrap *bootstrap, int nranks, int rank, int posi put_error(local, sizeof local, "chain schedule: %s", e->transport->error(e->proxy)); } else if (forward_windows(e, local, sizeof local) == 0) { /* The forward windows first: the ring lanes' window may come from them (ring_lane_window). */ - if (e->link_on) ring_lane_window(e); - if (e->link_on && + if (e->link_on && ring_lane_window(e, local, sizeof local) != 0) { + /* refused: `local` says why */ + } else if (e->link_on && e->transport->set_links(e->proxy, e->chain_prev, e->chain_next, e->chain_index, e->ring_prev, e->ring_next, e->ring_window, (int)e->shared.link_slots, e->shared.link_slot_bytes, e->link_off) != 0) { diff --git a/spark_transport/libsircl/tests/test_engine_cpu.py b/spark_transport/libsircl/tests/test_engine_cpu.py index 972ea1f7..8235c6de 100644 --- a/spark_transport/libsircl/tests/test_engine_cpu.py +++ b/spark_transport/libsircl/tests/test_engine_cpu.py @@ -188,7 +188,8 @@ def read(n): def test_relayed_lanes_reach_the_setup_exchange(self): """LIBSIRCL_RELAYED_LANES marks a peer position as relayed in the first setup exchange's record (which - the cycle plan reads) without any forward window; a malformed value fails creation naming it.""" + the cycle plan reads) without any forward window; a malformed value, or a LIBSIRCL_RING_WINDOW above the + native cap, fails creation naming the variable.""" script = '''import socket, struct, threading u=U(); assert l.ncclGetUniqueId(C.byref(u))==0 server=socket.socket(); server.bind(("127.0.0.1", 0)); server.listen() @@ -229,6 +230,10 @@ def read(n): data = json.loads(out.stdout) self.assertNotEqual(data["result"], 0) self.assertIn("LIBSIRCL_RELAYED_LANES", data["local"]) + # A ring window above the native cap of 60 chunks of 32 KiB is refused, naming the variable. + out = run(script, {**base, "LIBSIRCL_TRANSPORT": "verbs", "LIBSIRCL_RING_WINDOW": str(61 * 32768)}) + self.assertEqual(out.returncode, 0, out.stderr) + self.assertIn("LIBSIRCL_RING_WINDOW=1998848 must be 0 or hold 1 to 60 chunks", json.loads(out.stdout)["local"]) def test_info_names_the_collectives(self): out = run("print(l.sirclGetInfo().decode())") diff --git a/spark_transport/libsircl/tests/test_site_routes.py b/spark_transport/libsircl/tests/test_site_routes.py index d680e158..2b994684 100644 --- a/spark_transport/libsircl/tests/test_site_routes.py +++ b/spark_transport/libsircl/tests/test_site_routes.py @@ -177,6 +177,32 @@ def test_relayed_lanes_are_printed_without_any_window(self): self.assertTrue(want, rank) self.assertEqual(windows(env.get("LIBSIRCL_RELAYED_LANES", "")), want, rank) + def test_windows_keep_to_the_native_caps(self): + """With small chunks SIRCL's planner gives windows of more chunks than the native layers accept; the + tool lowers them to the caps: forward windows to 60 chunks, point-to-point windows to 120.""" + def printed(result, name): + return [value for row in result["ranks"] for lanes in windows(row["env"].get(name, "")).values() + for value in lanes if value] + forward = printed(plan("--layout", "ring:8", "--lanes", "2", "--chunk", "1024"), "LIBSIRCL_FORWARD_WINDOWS") + self.assertTrue(forward) + self.assertLessEqual(max(forward), 60 * 1024) + self.assertTrue(all(value % 1024 == 0 for value in forward)) + p2p = printed(plan("--layout", "ring:8", "--lanes", "2", "--p2p-chunk", "1024", "--p2p-reserve", "none"), + "LIBSIRCL_P2P_WINDOWS") + self.assertTrue(p2p) + self.assertLessEqual(max(p2p), 120 * 1024) + + def test_a_plan_that_overfills_a_relay_queue_is_refused(self): + """SIRCL gives every relayed lane at least one chunk; with 64 KiB chunks on path:0-7 the lanes through + one relay queue then hold more than its share, and the tool refuses the plan, naming the queue.""" + process = subprocess.run([sys.executable, str(ROOT / "tools" / "site_routes.py"), "--layout", "path:0-7", + "--lanes", "2", "--chunk", "65536"], + env=dict(os.environ, PYTHONPATH=PACKAGE), capture_output=True, text=True, + timeout=120) + self.assertEqual(process.returncode, 1, process.stdout) + self.assertIn("above its share of 393216", process.stderr) + self.assertEqual(process.stdout, "") + def test_the_data_files_are_the_tools_output(self): for name, arguments in (("site_routes_ring8_l2.json", ("--layout", "ring:8")), ("site_routes_ring8_l2_p2p_alone.json", ("--layout", "ring:8", "--p2p-reserve", "none")), diff --git a/spark_transport/libsircl/tools/site_routes.py b/spark_transport/libsircl/tools/site_routes.py index d002ed0e..c141a9e0 100644 --- a/spark_transport/libsircl/tools/site_routes.py +++ b/spark_transport/libsircl/tools/site_routes.py @@ -43,6 +43,16 @@ least one 32 KiB chunk from every relayed session lane (at 0.95: 32 to 96 KiB) and leaves every one of the 40 relayed ordered pairs a 32 KiB window; ``--max-window 32768`` (every session lane one chunk) does too. +Every window printed fits the native layers' caps: a forward window holds at most 60 chunks of +SIRCL_FORWARD_CHUNK_BYTES, the ring window at most 60 chunks of 32 KiB, and a point-to-point window at most +120 chunks of SIRCL_P2P_CHUNK_BYTES; a larger planned window is lowered to the cap. The tool then checks every +relay hairpin queue: the session's windows through it (its forward windows, and with ``--ring-schedules`` +the larger of those and its ring windows) plus, under ``--p2p-reserve session``, the channels' windows must +fit the queue's share (75 % of SIRCL_HAIRPIN_QUEUE_BYTES). A plan that overfills a queue, such as SIRCL's +one-chunk minimum with ``--chunk 65536`` on path:0-7, is refused (exit status 1, naming the queue). Under +``--p2p-reserve none`` the channels are sized as the only relayed traffic, so only the session's windows are +checked. + Limit: the windows are one plan per process, as libsircl reads them; communicators that run at once through the same relay queues are not budgeted against each other. @@ -56,6 +66,12 @@ import json import sys +# The native layers' window caps: SIRCL's native proxy accepts a forward or ring window of 1 to 60 chunks +# (ROCE_CREDIT - 4), its point-to-point library 1 to 120 chunks. +FORWARD_MAX_CHUNKS = 60 +RING_MAX_CHUNKS = 60 +P2P_MAX_CHUNKS = 120 + def share_value(text: str) -> float: value = float(text) @@ -101,6 +117,7 @@ def main(argv: list[str] | None = None) -> int: from sparkring_sircl import protocol ring_window, problems = routes.ring_window(layout, maps, order, chunk=protocol.LINK_WINDOW_CHUNK, queue_bytes=session_queue) + ring_window = min(ring_window, RING_MAX_CHUNKS * protocol.LINK_WINDOW_CHUNK) relayed = {rank for members in routes.ring_queues(layout, maps, order).values() for rank, _, _ in members} # Point-to-point windows of every ordered pair's relayed lanes (SIRCL's p2p/budget.py). p2p_max = p2p_protocol.DEFAULT_WINDOW_BYTES if args.p2p_max_window is None else args.p2p_max_window @@ -117,10 +134,34 @@ def main(argv: list[str] | None = None) -> int: p2p_unavailable = allocation.unavailable_pairs("layout") relayed_pairs = sum(1 for a in range(world) for b in range(world) if a != b and any(lane.relays for lane in group.lanes_to(a, b))) - rows = [] + tables = [] for rank in range(world): table = routes.forward_windows(layout, maps, rank, max_window=max_window, chunk=chunk, queue_bytes=session_queue) + tables.append([[min(window, FORWARD_MAX_CHUNKS * chunk) for window in lanes] for lanes in table]) + for peer in range(world): + if peer != rank: + p2p_table[rank][peer] = [min(window, P2P_MAX_CHUNKS * p2p_chunk) for window in p2p_table[rank][peer]] + # Every relay hairpin queue within its share: the session's windows through it, and the channels'. + share = int(routes.RELAY_QUEUE_SHARE * routes.DEFAULT_HAIRPIN_QUEUE) + ring_lanes = {} + if order is not None and args.ring_schedules and not problems: + ring_lanes = routes.ring_queues(layout, maps, order) + for key, members in sorted(routes.relay_queues(layout, maps).items()): + held = sum(tables[rank][peer][lane] for rank, peer, lane in members) + if key in ring_lanes: + held = max(held, sum(ring_window if rank in relayed else 0 for rank, _, _ in ring_lanes[key])) + if args.p2p_reserve == "session": + held += sum(p2p_table[rank][peer][lane] for rank, peer, lane in members) + if held > share: + relay, secondary, port = key + print(f"site_routes.py: the relay queue of position {relay} ({'secondary' if secondary else 'primary'} " + f"function, toward port {port}) would hold {held} bytes of windows, above its share of {share}; " + f"choose a smaller --chunk, --max-window or --session-share", file=sys.stderr) + return 1 + rows = [] + for rank in range(world): + table = tables[rank] env = { "LIBSIRCL_POSITION": str(rank), "SIRCL_PEER_ROUTES": ",".join(f"{peer}={'/'.join(devices)}" for peer, devices in sorted(maps[rank].items())), From 842f3d92e35d250229c4854f6c5ca9b7440b2e88 Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Thu, 15 Oct 2026 08:51:04 -0500 Subject: [PATCH 26/65] Release a context's kernel-pack entry when its last engine is freed The kernel-pack cache keyed entries by the raw CUcontext value and never removed them: a context handle reused after its context was destroyed returned stale functions, and the 17th distinct context could not create a communicator. Entries now hold references: every sccl_kp_load takes one, and sccl_kp_release drops it. The engine releases its reference when it frees itself (at destroy once no kernel of the packs can run, and on a failed setup); the last release unloads the context's four modules while the context exists, or only removes the entry when the context is gone. A context address used again after its last release loads the packs afresh. At most 16 contexts hold entries at once. The stand-in CUDA driver (tests/fake_cuda.c) reuses the slots of unloaded modules and reports the modules still loaded, so the new C test tests/test_kernelpack_cache.c can load and release packs in turn; make check and CMake (kernelpack_cache) run it. Compatibility: a process that creates and destroys communicators on more than 16 contexts over its life keeps working; quarantined engines (a kernel still running at abort) keep their reference and modules. Validation: tests/test_kernelpack_cache.c (67 checks) loads and releases the packs in 20 distinct contexts, checks that a held reference keeps the entry, that the last release unloads the modules, and that a reused address reloads. Before the change no release existed, so entries could only accumulate. Engine-level creation of a communicator needs two or more ranks with a transport, which the CPU harness cannot build. make check and make check-asan pass. Request item 26 (audit 10-F2). NVIDIA NCCL source was neither read nor adapted. Co-Authored-By: Claude Opus 5.5 --- spark_transport/libsircl/CMakeLists.txt | 8 +++ spark_transport/libsircl/Makefile | 7 ++- spark_transport/libsircl/src/engine.c | 6 ++ spark_transport/libsircl/src/kernelpack.c | 27 ++++++++- spark_transport/libsircl/src/kernelpack.h | 9 ++- spark_transport/libsircl/tests/fake_cuda.c | 27 ++++++--- .../libsircl/tests/test_kernelpack_cache.c | 60 +++++++++++++++++++ 7 files changed, 133 insertions(+), 11 deletions(-) create mode 100644 spark_transport/libsircl/tests/test_kernelpack_cache.c diff --git a/spark_transport/libsircl/CMakeLists.txt b/spark_transport/libsircl/CMakeLists.txt index a2bcc3f3..6426c5b5 100644 --- a/spark_transport/libsircl/CMakeLists.txt +++ b/spark_transport/libsircl/CMakeLists.txt @@ -117,6 +117,14 @@ add_executable(test_link_plan tests/test_link_plan.c) target_include_directories(test_link_plan PRIVATE src) target_compile_options(test_link_plan PRIVATE -Wall -Wextra -Werror) add_test(NAME link_plan COMMAND test_link_plan) +add_executable(test_kernelpack_cache tests/test_kernelpack_cache.c src/kernelpack.c src/cuda_api.c + ${CMAKE_CURRENT_BINARY_DIR}/kernels_fatbin.c ${CMAKE_CURRENT_BINARY_DIR}/fold_fatbin.c + ${CMAKE_CURRENT_BINARY_DIR}/links_fatbin.c ${CMAKE_CURRENT_BINARY_DIR}/p2p_fatbin.c) +target_include_directories(test_kernelpack_cache PRIVATE include src) +target_compile_definitions(test_kernelpack_cache PRIVATE LIBSIRCL_CPU_ONLY NCCL_OS_LINUX) +target_link_libraries(test_kernelpack_cache PRIVATE Threads::Threads ${CMAKE_DL_LIBS}) +add_test(NAME kernelpack_cache COMMAND ${CMAKE_COMMAND} -E env + LD_LIBRARY_PATH=${CMAKE_CURRENT_BINARY_DIR}/fake-cuda $) foreach(name abi api lifecycle engine_cpu) add_test(NAME ${name} COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tests/test_${name}.py --library $) endforeach() diff --git a/spark_transport/libsircl/Makefile b/spark_transport/libsircl/Makefile index f69c3dbb..65af0719 100644 --- a/spark_transport/libsircl/Makefile +++ b/spark_transport/libsircl/Makefile @@ -168,10 +168,15 @@ $(BUILD)/test_fabric: tests/test_fabric.c $(BUILD)/fabric.o $(CC) $(CPPFLAGS) $(CFLAGS) $^ -o $@ $(BUILD)/test_link_plan: tests/test_link_plan.c src/link_plan.h | $(BUILD) $(CC) $(CPPFLAGS) $(CFLAGS) $< -o $@ -check: all $(BUILD)/test_fabric $(BUILD)/test_link_plan +$(BUILD)/test_kernelpack_cache: tests/test_kernelpack_cache.c $(BUILD)/kernelpack.o $(BUILD)/cuda_api.o \ + $(BUILD)/kernels_fatbin.o $(BUILD)/fold_fatbin.o $(BUILD)/links_fatbin.o \ + $(BUILD)/p2p_fatbin.o + $(CC) $(CPPFLAGS) $(CFLAGS) $^ -o $@ -ldl +check: all $(BUILD)/test_fabric $(BUILD)/test_link_plan $(BUILD)/test_kernelpack_cache @status=0; \ $(BUILD)/test_fabric || status=1; \ $(BUILD)/test_link_plan || status=1; \ + LD_LIBRARY_PATH=$(abspath $(BUILD)/fake-cuda) $(BUILD)/test_kernelpack_cache || status=1; \ $(PYTHON) tests/test_abi.py --library $(abspath $(BUILD)/libsircl.so) || status=1; \ $(PYTHON) tests/test_api.py --library $(abspath $(BUILD)/libsircl.so) || status=1; \ $(PYTHON) tests/test_bootstrap.py || status=1; \ diff --git a/spark_transport/libsircl/src/engine.c b/spark_transport/libsircl/src/engine.c index 5e77002b..791a9284 100644 --- a/spark_transport/libsircl/src/engine.c +++ b/spark_transport/libsircl/src/engine.c @@ -440,6 +440,7 @@ struct sccl_engine { /* Teardown: the progress thread stopped (sccl_engine_teardown_stop), when its stop was requested and * when it had stopped (CLOCK_REALTIME nanoseconds). */ int stopped, closing; + int kp_held; /* this engine holds a reference on its context's kernel-pack entry (sccl_kp_load) */ uint64_t stop_ns, stopped_ns; int have_last; sccl_CUevent order_event; @@ -2270,6 +2271,7 @@ int sccl_engine_create(sccl_bootstrap *bootstrap, int nranks, int rank, int posi put_error(local, sizeof local, "%s", sccl_kp_error()); break; } + e->kp_held = 1; for (int a = 0; a < SCCL_ALG_COUNT; ++a) for (int d = 0; d < SCCL_DT_COUNT; ++d) sccl_kp_function(ctx, a, d, nranks, &e->fn[a][d]); for (int k = SCCL_K_SCATTER; k < SCCL_K_COUNT; ++k) @@ -2465,6 +2467,7 @@ fail:; p2p_close(e, pushed); int unreleased = e->proxy ? e->transport->destroy(e->proxy) : 0; e->proxy = NULL; + if (e->kp_held) sccl_kp_release(e->ctx, pushed); if (pushed) { if (!unreleased) release_memory(e); cu->CtxPopCurrent(&ctx); @@ -4751,7 +4754,10 @@ ncclResult_t sccl_engine_destroy(sccl_engine *e, int abort, char *err, size_t le if (freed) { if (e->order_event) cu->EventDestroy(e->order_event); release_memory(e); + /* No kernel of the packs runs any more: the context's pack entry may go (its last reference unloads). */ + if (e->kp_held) sccl_kp_release(e->ctx, 1); } else if (!pushed && released) { + if (e->kp_held) sccl_kp_release(e->ctx, 0); /* The communicator's context could not be made current: it was destroyed (or the driver is shutting * down), and with it every device and pinned allocation and every kernel that could read them. Only the * host bookkeeping is left: the retired staging list, the shared-memory segments of the emulation diff --git a/spark_transport/libsircl/src/kernelpack.c b/spark_transport/libsircl/src/kernelpack.c index 3b695e0a..cbb91d8e 100644 --- a/spark_transport/libsircl/src/kernelpack.c +++ b/spark_transport/libsircl/src/kernelpack.c @@ -22,6 +22,7 @@ extern const char sccl_p2p_fatbin_sha256[]; enum { MAX_CONTEXTS = 16 }; typedef struct { sccl_CUcontext ctx; + int refs; /* sccl_kp_load calls not yet released (sccl_kp_release) */ sccl_CUmodule module, fold_module, links_module, p2p_module; sccl_CUfunction fn[SCCL_K_COUNT][SCCL_DT_COUNT][SCCL_KP_MAX_WORLD + 1]; sccl_CUfunction fold[SCCL_FOLD_DTYPES][SCCL_FOLD_OPS]; @@ -65,7 +66,9 @@ int sccl_kp_load(sccl_CUcontext ctx) { return -1; } pthread_mutex_lock(&lock); - if (find(ctx)) { + loaded_pack *held = find(ctx); + if (held) { + ++held->refs; pthread_mutex_unlock(&lock); return 0; } @@ -198,12 +201,34 @@ int sccl_kp_load(sccl_CUcontext ctx) { cu->CtxPopCurrent(&popped); if (!failed) { p->ctx = ctx; + p->refs = 1; ++npacks; } pthread_mutex_unlock(&lock); return failed ? -1 : 0; } +void sccl_kp_release(sccl_CUcontext ctx, int unload) { + const sccl_cuda *cu = sccl_cuda_get(); + pthread_mutex_lock(&lock); + loaded_pack *p = find(ctx); + if (!p || --p->refs > 0) { + pthread_mutex_unlock(&lock); + return; + } + if (unload && cu && cu->CtxPushCurrent(ctx) == SCCL_CUDA_SUCCESS) { + sccl_CUmodule modules[4] = {p->p2p_module, p->links_module, p->fold_module, p->module}; + for (int i = 0; i < 4; ++i) + if (modules[i]) cu->ModuleUnload(modules[i]); + sccl_CUcontext popped; + cu->CtxPopCurrent(&popped); + } + /* The last entry takes the released one's place. */ + if (p != &packs[npacks - 1]) *p = packs[npacks - 1]; + --npacks; + pthread_mutex_unlock(&lock); +} + int sccl_kp_function(sccl_CUcontext ctx, int kind, int dtype, int world, sccl_CUfunction *out) { if (kind < 0 || kind >= SCCL_K_COUNT || dtype < 0 || dtype >= SCCL_DT_COUNT || world < SCCL_KP_MIN_WORLD || world > SCCL_KP_MAX_WORLD) { diff --git a/spark_transport/libsircl/src/kernelpack.h b/spark_transport/libsircl/src/kernelpack.h index 6c664f57..5f7df5eb 100644 --- a/spark_transport/libsircl/src/kernelpack.h +++ b/spark_transport/libsircl/src/kernelpack.h @@ -122,9 +122,14 @@ typedef struct { } sccl_p2p_args; _Static_assert(sizeof(sccl_p2p_args) == 120, "sccl_p2p_args must match the point-to-point pack's P2PParams"); -/* Load every pack into `ctx` once (pushing it current for the load); later - * calls for the same context return at once. 0 on success. */ +/* Load every pack into `ctx` once (pushing it current for the load); later calls for the same context + * return at once. Each successful call holds a reference on the context's entry, which sccl_kp_release + * drops; at most 16 contexts hold entries at once. 0 on success. */ int sccl_kp_load(sccl_CUcontext ctx); +/* Drop a reference sccl_kp_load took. The last one removes the context's entry, so a later context at the + * same address loads the packs afresh; with `unload` the modules are unloaded first (the context still + * exists and no kernel of the packs runs in it), without it they are left to the context's destruction. */ +void sccl_kp_release(sccl_CUcontext ctx, int unload); /* The entry of (kind, dtype, world) in a context the pack is loaded in. */ int sccl_kp_function(sccl_CUcontext ctx, int kind, int dtype, int world, sccl_CUfunction *out); /* One launch of `grid` blocks of `threads` threads on `stream`, from the diff --git a/spark_transport/libsircl/tests/fake_cuda.c b/spark_transport/libsircl/tests/fake_cuda.c index 2c279dc0..1e7d8fb6 100644 --- a/spark_transport/libsircl/tests/fake_cuda.c +++ b/spark_transport/libsircl/tests/fake_cuda.c @@ -19,6 +19,7 @@ enum { NOT_FOUND = 500, INVALID_IMAGE = 200, MAX_MODULES = 16, MAX_CUBINS = 8 }; typedef struct { + int in_use; int cubins; uint32_t flags[MAX_CUBINS]; char **names[MAX_CUBINS]; @@ -82,18 +83,24 @@ EXPORT int sccl_fake_cuda(void) { return 1; } /* Modules loaded, the fewest cubins of any module, and symbol lookups so far. */ EXPORT void sccl_fake_cuda_stats(int *loaded, int *fewest_cubins, int *looked_up) { - *loaded = nmodules; + /* The modules loaded and not unloaded, and the fewest cubins among them. */ + *loaded = 0; *fewest_cubins = 0; - for (int i = 0; i < nmodules; ++i) - if (!i || modules[i].cubins < *fewest_cubins) *fewest_cubins = modules[i].cubins; + for (int i = 0; i < nmodules; ++i) { + if (!modules[i].in_use) continue; + if (!*loaded || modules[i].cubins < *fewest_cubins) *fewest_cubins = modules[i].cubins; + ++*loaded; + } *looked_up = lookups; } EXPORT int cuModuleLoadData(void **module, const void *image) { const uint8_t *p = image; - if (nmodules == MAX_MODULES || u32(p) != 0xBA55ED50u) return INVALID_IMAGE; + int slot = 0; + while (slot < nmodules && modules[slot].in_use) ++slot; + if (slot == MAX_MODULES || u32(p) != 0xBA55ED50u) return INVALID_IMAGE; uint64_t total = (uint64_t)u16(p + 6) + u64(p + 8); - module_t *m = &modules[nmodules]; + module_t *m = &modules[slot]; memset(m, 0, sizeof *m); for (uint64_t at = 0; at + 4 <= total;) { if (memcmp(p + at, "\x7f" "ELF", 4) == 0) { @@ -107,7 +114,8 @@ EXPORT int cuModuleLoadData(void **module, const void *image) { } if (!m->cubins) return INVALID_IMAGE; *module = m; - ++nmodules; + m->in_use = 1; + if (slot == nmodules) ++nmodules; return 0; } @@ -178,7 +186,12 @@ EXPORT int cuInit(unsigned flags) { return (int)(flags & 0); } static int fail_push; EXPORT void sccl_fake_cuda_fail_push(int on) { fail_push = on; } EXPORT int cuCtxPushCurrent_v2(void *ctx) { return ctx && !fail_push ? 0 : 201; } -EXPORT int cuModuleUnload(void *module) { return module ? 0 : 400; } +EXPORT int cuModuleUnload(void *module) { + module_t *m = module; + if (!m || m < modules || m >= modules + MAX_MODULES || !m->in_use) return 400; + m->in_use = 0; + return 0; +} /* Every other entry point src/cuda_api.c binds: present, never used by the check. */ #define UNUSED_ENTRY(name) \ diff --git a/spark_transport/libsircl/tests/test_kernelpack_cache.c b/spark_transport/libsircl/tests/test_kernelpack_cache.c new file mode 100644 index 00000000..d50a75ed --- /dev/null +++ b/spark_transport/libsircl/tests/test_kernelpack_cache.c @@ -0,0 +1,60 @@ +/* The kernel-pack cache (src/kernelpack.c) over the stand-in driver tests/fake_cuda.c: an entry per context, + * held by references. Packs loaded and released in turn in 20 distinct contexts (more than the 16 entries the + * cache holds at once) all load, and each last release unloads the context's four modules; a reference still + * held keeps the entry; a context address reused after its last release loads the packs afresh. + * + * LD_LIBRARY_PATH= test_kernelpack_cache + */ +#include +#include +#include + +#include "kernelpack.h" + +static unsigned checks; +#define CHECK(expr) do { checks++; if (!(expr)) { \ + fprintf(stderr, "%s:%d: %s (%s)\n", __FILE__, __LINE__, #expr, sccl_kp_error()); exit(1); } } while (0) + +static void (*stats)(int *, int *, int *); +static int live_modules(void) { + int modules = 0, cubins = 0, lookups = 0; + stats(&modules, &cubins, &lookups); + return modules; +} +static int loaded(sccl_CUcontext ctx) { + sccl_CUfunction function; + return sccl_kp_function(ctx, SCCL_ALG_ONESHOT, SCCL_DT_F32, 2, &function) == 0; +} + +int main(void) { + void *driver = dlopen("libcuda.so.1", RTLD_NOW | RTLD_LOCAL); + if (driver) *(void **)(&stats) = dlsym(driver, "sccl_fake_cuda_stats"); + if (!driver || !stats) { + fprintf(stderr, "test_kernelpack_cache: put the stand-in driver's directory first on LD_LIBRARY_PATH\n"); + return 2; + } + static int contexts[20]; + for (int i = 0; i < 20; ++i) { + sccl_CUcontext ctx = (sccl_CUcontext)(void *)&contexts[i]; + CHECK(sccl_kp_load(ctx) == 0); + CHECK(loaded(ctx) && live_modules() == 4); + sccl_kp_release(ctx, 1); + CHECK(!loaded(ctx) && live_modules() == 0); + } + /* Two references: the first release keeps the entry, the second removes it. */ + sccl_CUcontext a = (sccl_CUcontext)(void *)&contexts[0], b = (sccl_CUcontext)(void *)&contexts[1]; + CHECK(sccl_kp_load(a) == 0 && sccl_kp_load(b) == 0 && sccl_kp_load(a) == 0); + CHECK(live_modules() == 8); + sccl_kp_release(a, 1); + CHECK(loaded(a) && loaded(b) && live_modules() == 8); + sccl_kp_release(a, 1); + CHECK(!loaded(a) && loaded(b) && live_modules() == 4); + /* A reused address loads afresh; a release without unloading leaves the modules to the context. */ + CHECK(sccl_kp_load(a) == 0 && loaded(a) && live_modules() == 8); + sccl_kp_release(a, 0); + CHECK(!loaded(a) && live_modules() == 8); + sccl_kp_release(b, 1); + CHECK(!loaded(b)); + printf("kernel-pack cache: %u checks passed\n", checks); + return 0; +} From baadd4289a59d3af96c8f3cfb76c75e43859d41f Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Thu, 15 Oct 2026 08:52:37 -0500 Subject: [PATCH 27/65] Create bootstrap descriptors close-on-exec atomically The bootstrap creates its sockets with SOCK_CLOEXEC, accepts with accept4(SOCK_CLOEXEC | SOCK_NONBLOCK) and makes the root's wake pipe with pipe2(O_CLOEXEC | O_NONBLOCK), instead of setting FD_CLOEXEC after the call returned. A fork and exec on another thread between the two steps could inherit a broker connection and keep it open after this rank left, defeating the broker's detection of a departed rank. Compatibility: no behavior change apart from the closed window. Validation: tests/test_bootstrap.py builds the bootstrap with socket, accept, accept4, pipe and pipe2 wrapped (ld --wrap), creates an id and joins a one-rank group, and counts descriptors created without the close-on-exec flag: 5 before the change, 0 after it. make check and make check-asan pass. Request item 27 (audit 02-F10). NVIDIA NCCL source was neither read nor adapted. Co-Authored-By: Claude Opus 5.5 --- spark_transport/libsircl/src/bootstrap.c | 11 +++-- .../libsircl/tests/test_bootstrap.py | 46 +++++++++++++++++++ 2 files changed, 53 insertions(+), 4 deletions(-) diff --git a/spark_transport/libsircl/src/bootstrap.c b/spark_transport/libsircl/src/bootstrap.c index 195f5247..1f8a4612 100644 --- a/spark_transport/libsircl/src/bootstrap.c +++ b/spark_transport/libsircl/src/bootstrap.c @@ -157,6 +157,9 @@ static uint32_t get32(const unsigned char *p) { ((uint32_t)p[2] << 8) | (uint32_t)p[3]; } +/* Every descriptor the bootstrap creates is close-on-exec from its creation (SOCK_CLOEXEC, accept4, + * pipe2), so a concurrent fork and exec in another thread cannot inherit it and keep a broker connection + * open after this process's rank left. configure_fd sets the flags again for descriptors of any origin. */ static int configure_fd(int fd) { int flags = fcntl(fd, F_GETFL, 0); if (flags < 0 || fcntl(fd, F_SETFL, flags | O_NONBLOCK) < 0 || @@ -336,7 +339,7 @@ static int accept_hello(struct root *r, int index) { static void accept_connections(struct root *r) { /* Cap each batch so connections cannot starve the absolute deadline. */ for (int attempt = 0; attempt < MAX_PENDING; ++attempt) { - int fd = accept(r->listen_fd, NULL, NULL); + int fd = accept4(r->listen_fd, NULL, NULL, SOCK_CLOEXEC | SOCK_NONBLOCK); if (fd < 0) { if (errno == EINTR) continue; return; } if (configure_fd(fd) != 0) { close(fd); continue; } int slot; @@ -586,7 +589,7 @@ int sccl_bootstrap_id(unsigned char id[ID_SIZE]) { memcpy(r->id + IP_OFF, &bind_address.s_addr, 4); memcpy(r->id + HASH_OFF, hash, 32); if (random_nonce(r->id + NONCE_OFF) != 0) goto failed; - r->listen_fd = socket(AF_INET, SOCK_STREAM, 0); + r->listen_fd = socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, 0); if (r->listen_fd < 0 || configure_fd(r->listen_fd) != 0) goto failed; struct sockaddr_in address; memset(&address, 0, sizeof address); @@ -598,7 +601,7 @@ int sccl_bootstrap_id(unsigned char id[ID_SIZE]) { if (getsockname(r->listen_fd, (struct sockaddr *)&address, &length) < 0) goto failed; memcpy(r->id + PORT_OFF, &address.sin_port, 2); - if (pipe(r->wake) != 0 || configure_fd(r->wake[0]) != 0 || + if (pipe2(r->wake, O_CLOEXEC | O_NONBLOCK) != 0 || configure_fd(r->wake[0]) != 0 || configure_fd(r->wake[1]) != 0) goto failed; int64_t now = now_ms(); if (now < 0) goto failed; @@ -666,7 +669,7 @@ int sccl_bootstrap_join_cancel(const unsigned char id[ID_SIZE], int nranks, int int64_t now = now_ms(); if (now < 0) return error_result(SYSTEM, "cannot read monotonic clock"); int64_t deadline = now + timeout; - fd = socket(AF_INET, SOCK_STREAM, 0); + fd = socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, 0); if (fd < 0 || configure_fd(fd) != 0) { result = SYSTEM; goto failed; } struct sockaddr_in address; memset(&address, 0, sizeof address); diff --git a/spark_transport/libsircl/tests/test_bootstrap.py b/spark_transport/libsircl/tests/test_bootstrap.py index ea339f70..37a5f941 100644 --- a/spark_transport/libsircl/tests/test_bootstrap.py +++ b/spark_transport/libsircl/tests/test_bootstrap.py @@ -409,6 +409,52 @@ def joined_group(self, world, value=None): self.assertEqual([r for r, _ in results], [0] * world) return [handle for _, handle in results] + def test_every_descriptor_is_close_on_exec_from_its_creation(self): + """The bootstrap creates every socket and pipe close-on-exec atomically (SOCK_CLOEXEC, accept4 with + SOCK_CLOEXEC, pipe2 with O_CLOEXEC), never setting the flag after the fact, so a concurrent fork and + exec cannot inherit one. A build of the bootstrap whose socket, accept, accept4, pipe and pipe2 calls + are wrapped (ld --wrap) counts the creations made without the flag.""" + build = Path(self.build.name) + probe = build / "cloexec_probe.c" + probe.write_text(r''' +#define _GNU_SOURCE +#include +#include +#include +static int created, missed; +int __real_socket(int, int, int); +int __real_accept(int, struct sockaddr *, socklen_t *); +int __real_accept4(int, struct sockaddr *, socklen_t *, int); +int __real_pipe(int[2]); +int __real_pipe2(int[2], int); +int __wrap_socket(int d, int t, int p) { created++; missed += !(t & SOCK_CLOEXEC); return __real_socket(d, t, p); } +int __wrap_accept(int f, struct sockaddr *a, socklen_t *l) { created++; missed++; return __real_accept(f, a, l); } +int __wrap_accept4(int f, struct sockaddr *a, socklen_t *l, int x) { + created++; missed += !(x & SOCK_CLOEXEC); return __real_accept4(f, a, l, x); } +int __wrap_pipe(int p[2]) { created++; missed++; return __real_pipe(p); } +int __wrap_pipe2(int p[2], int x) { created++; missed += !(x & O_CLOEXEC); return __real_pipe2(p, x); } +int test_created(void) { return created; } +int test_missed(void) { return missed; } +''', encoding="utf-8") + source = Path(__file__).resolve().parents[1] / "src" / "bootstrap.c" + library = build / "bootstrap_wrapped.so" + wrap = ",".join(f"--wrap={name}" for name in ("socket", "accept", "accept4", "pipe", "pipe2")) + subprocess.run([os.environ.get("CC", "cc"), "-std=c11", "-O2", "-fPIC", "-shared", "-pthread", + "-I", str(source.parent), f"-Wl,{wrap}", str(source), str(probe), "-o", str(library)], + check=True) + lib = ctypes.CDLL(str(library)) + lib.sccl_bootstrap_id.argtypes = [ctypes.c_void_p] + lib.sccl_bootstrap_join.argtypes = [ctypes.c_void_p, ctypes.c_int, ctypes.c_int, ctypes.POINTER(ctypes.c_void_p)] + lib.sccl_bootstrap_close.argtypes = [ctypes.c_void_p] + value = ctypes.create_string_buffer(128) + self.assertEqual(lib.sccl_bootstrap_id(value), 0) + handle = ctypes.c_void_p() + self.assertEqual(lib.sccl_bootstrap_join(value.raw, 1, 0, ctypes.byref(handle)), 0) + lib.sccl_bootstrap_close(handle) + # The id's listener and wake pipe, the joining socket and the root's accepted connection. + self.assertGreaterEqual(lib.test_created(), 4) + self.assertEqual(lib.test_missed(), 0) + def test_allgather_rounds(self): world = 3 handles = self.joined_group(world) From cdb3676493b5234bb19a23ec9420694512104ecd Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Thu, 15 Oct 2026 08:54:49 -0500 Subject: [PATCH 28/65] Document the bootstrap root limit and name it in its refusal The bootstrap root of a unique id relays every all-gather round of its communicator (setup, splits and teardown), so the protocol needs it for the communicator's life: it lives until every rank's connection to it closes, or, before every rank joined, until its join window ends. A process holds at most 64 roots. README.md states this, and ncclGetUniqueId beyond the limit returns ncclInvalidUsage with a reason that names the 64 roots and what releases one. The text stays within the bootstrap's 192-byte error buffer. Compatibility: the result code is unchanged; the message text changes. Validation: tests/test_lifecycle.py creates 64 live one-rank communicators in a child process: the 65th id is refused with ncclInvalidUsage naming the limit (before: "at most 64 active bootstrap IDs are supported", without the cause), and after one communicator is destroyed an id is granted. The split-leader test matches the new text. make check and make check-asan pass; the Markdown link check passes. Request item 28 (audit 02-F11). NVIDIA NCCL source was neither read nor adapted. Co-Authored-By: Claude Opus 5.5 --- spark_transport/libsircl/README.md | 7 ++++- spark_transport/libsircl/src/bootstrap.c | 4 ++- .../libsircl/tests/test_lifecycle.py | 26 ++++++++++++++++++- 3 files changed, 34 insertions(+), 3 deletions(-) diff --git a/spark_transport/libsircl/README.md b/spark_transport/libsircl/README.md index b2482715..bff7d334 100644 --- a/spark_transport/libsircl/README.md +++ b/spark_transport/libsircl/README.md @@ -61,7 +61,12 @@ A communicator of W ranks (1 to 8) owns one SIRCL ring session: - **Bootstrap**: a 128-byte unique id carrying the root's address and port, a nonce and an optional site hash; TCP rendezvous over loopback or the wired LAN (`SIRCL_BOOTSTRAP_ADDR`, `SIRCL_BOOTSTRAP_IFNAME` - or `NCCL_SOCKET_IFNAME`); all-gather rounds through the root for setup. + or `NCCL_SOCKET_IFNAME`); all-gather rounds through the root for setup, splits and teardown. The root + runs in the process that created the id and lives until every rank's connection to it closes (each rank's + communicator is destroyed or aborted) or, before every rank joined, until its join window ends + (`SIRCL_BOOTSTRAP_TIMEOUT_MS`, default 10 s). A process holds at most 64 roots at once: `ncclGetUniqueId` + beyond that returns `ncclInvalidUsage`, naming the limit, until a communicator whose id it created is + destroyed. - **Session engine** (`src/engine.c`): settings agreed across ranks, the pinned arena, device counters, SIRCL's native progress thread (`src/transport/sircl_roce_proxy.c`, a byte-identical copy of SIRCL's `oneshot/_roce_proxy.c` with change LF, its SHA-256 checked by the build and its local feature word, diff --git a/spark_transport/libsircl/src/bootstrap.c b/spark_transport/libsircl/src/bootstrap.c index 1f8a4612..efe2c13e 100644 --- a/spark_transport/libsircl/src/bootstrap.c +++ b/spark_transport/libsircl/src/bootstrap.c @@ -611,7 +611,9 @@ int sccl_bootstrap_id(unsigned char id[ID_SIZE]) { for (struct root *entry = roots; entry; entry = entry->next) ++count; if (count >= MAX_ROOTS) { pthread_mutex_unlock(&roots_lock); - result = error_result(USAGE, "at most 64 active bootstrap IDs are supported"); + /* At most 191 characters: the error text buffer. */ + result = error_result(USAGE, "this process already holds 64 bootstrap roots, the limit: an id keeps its root until " + "its communicator is destroyed on every rank or its join window ends"); goto cleanup; } if (pthread_create(&r->thread, NULL, root_thread, r) != 0) { diff --git a/spark_transport/libsircl/tests/test_lifecycle.py b/spark_transport/libsircl/tests/test_lifecycle.py index 99faa71c..ce74fdef 100644 --- a/spark_transport/libsircl/tests/test_lifecycle.py +++ b/spark_transport/libsircl/tests/test_lifecycle.py @@ -73,6 +73,30 @@ def test_lifecycle_call_on_an_initializing_communicator_is_refused(self): if grouped.value: self.lib.ncclCommDestroy(grouped) + def test_sixty_four_live_roots_are_the_limit_and_the_refusal_names_it(self): + """Each id a process creates keeps its bootstrap root while its communicator lives: with 64 live one-rank + communicators a 65th id is refused with ncclInvalidUsage naming the limit, and destroying one + communicator makes room for an id again.""" + script = r''' +import ctypes as C, json, os, sys, time +sys.path.insert(0, sys.argv[2]); import test_api as api +api.LIBRARY=api.Path(sys.argv[1]); api.initialize_library() +os.environ["LIBSIRCL_BOOTSTRAP_ONLY"]="1" +handles=[api.new_comm() for _ in range(64)] +u=api.UniqueId(); refused=api.LIB.ncclGetUniqueId(C.byref(u)); message=api.LIB.ncclGetLastError(None).decode() +assert api.LIB.ncclCommDestroy(handles.pop())==0 +time.sleep(0.2) +again=api.LIB.ncclGetUniqueId(C.byref(u)) +print(json.dumps([refused, message, again])) +''' + out = subprocess.run([sys.executable, "-c", script, str(LIBRARY), str(Path(__file__).parent)], + capture_output=True, text=True, timeout=30) + self.assertEqual(out.returncode, 0, out.stderr[-2000:]) + refused, message, again = json.loads(out.stdout) + self.assertEqual(refused, USAGE) + self.assertIn("already holds 64 bootstrap roots", message) + self.assertEqual(again, SUCCESS) + def test_split_leader_failure_fails_every_member_at_once(self): """The split leader cannot create the new unique id (the process holds all 64 bootstrap roots). It still takes part in the unique-id round with a failure record, so both ranks return its result at once @@ -108,7 +132,7 @@ def split(r): data = json.loads(out.stdout) self.assertEqual(data["held"], 63) self.assertEqual([code for code, _ in data["results"]], [USAGE, USAGE], data) - self.assertIn("64 active bootstrap IDs", data["results"][0][1]) + self.assertIn("64 bootstrap roots", data["results"][0][1]) self.assertIn("split leader (parent rank 0)", data["results"][1][1]) self.assertLess(data["elapsed"], 2) self.assertEqual(data["children"], [None, None]) From 7b77bd844ed3e8de5db3894c9653f08be8f12c3b Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Thu, 15 Oct 2026 08:56:34 -0500 Subject: [PATCH 29/65] Parse NCCL_SOCKET_IFNAME exclusions fully and keep them off loopback The bootstrap's interface list follows NCCL_SOCKET_IFNAME's syntax in full: `prefix` and `=name` select, `^prefix` and `^=name` exclude. A list of exclusions only selects every interface it does not exclude except a loopback interface, which a remote rank cannot reach; only a selecting entry chooses loopback. A list longer than 255 characters is refused with ncclInvalidArgument naming the variable, instead of being truncated to its first 255 characters. README.md states the syntax. Compatibility: exclusion-only lists that picked 127.0.0.1 now pick the first non-loopback interface, or fail naming the variable when the host has none; lists with a selecting entry behave as before. Validation: tests/test_bootstrap.py creates ids with NCCL_SOCKET_IFNAME set to ^docker, ^=lo and ^=lo,^docker: before the change each advertised 127.0.0.1, after it a non-loopback address that a one-rank group joins; a 319-character list returned success before and is refused after, naming NCCL_SOCKET_IFNAME. make check and make check-asan pass. Request item 29 (audit 02-F9). NVIDIA NCCL source was neither read nor adapted; the list syntax follows NCCL's documented NCCL_SOCKET_IFNAME. Co-Authored-By: Claude Opus 5.5 --- spark_transport/libsircl/README.md | 2 +- spark_transport/libsircl/src/bootstrap.c | 31 +++++++++++++------ .../libsircl/tests/test_bootstrap.py | 26 ++++++++++++++++ 3 files changed, 48 insertions(+), 11 deletions(-) diff --git a/spark_transport/libsircl/README.md b/spark_transport/libsircl/README.md index bff7d334..96752523 100644 --- a/spark_transport/libsircl/README.md +++ b/spark_transport/libsircl/README.md @@ -253,7 +253,7 @@ LF), and refuses a vendored copy that does not define the word; `P2P_FEATURES=0` | `LIBSIRCL_FAIL_STOP_POLL_MS` | the fail-stop watcher's period, 1 to 3,600,000 ms (default 5); the tests set it long to show that the library calls alone end the process | | `SIRCL_POST_ORDER`, `SIRCL_PROGRESS_CPU`, `SIRCL_FORWARD_PROOF` | read by the native progress thread, as in SIRCL; `SIRCL_PROGRESS_CPU` pins it to a CPU list | | `LIBSIRCL_CPU_POLICY` | where the progress thread runs without `SIRCL_PROGRESS_CPU`: `performance` (default: on the fastest CPU class the creating thread may use, the Cortex-X925 cores of a GB10, found by `/proc/cpuinfo` part numbers or sysfs `cpu_capacity`) or `none` (where the scheduler puts it). The library never changes the affinity of the application's own threads; receipts name the progress thread's CPUs (`progress_cpus`) | -| `SIRCL_BOOTSTRAP_ADDR`, `SIRCL_BOOTSTRAP_IFNAME`, `NCCL_SOCKET_IFNAME` | the root's LAN address; a rank contacts a non-loopback root only when one is set | +| `SIRCL_BOOTSTRAP_ADDR`, `SIRCL_BOOTSTRAP_IFNAME`, `NCCL_SOCKET_IFNAME` | the root's LAN address; a rank contacts a non-loopback root only when one is set. An interface list (at most 255 characters) takes NCCL's syntax: `prefix` and `=name` select, `^prefix` and `^=name` exclude; a list of exclusions only never selects a loopback interface | | `SIRCL_BOOTSTRAP_TIMEOUT_MS`, `LIBSIRCL_SETUP_TIMEOUT_MS`, `LIBSIRCL_LANE_CHECK_MS` | rendezvous, setup-exchange and lane-check deadlines | | `LIBSIRCL_RECEIPT`, `LIBSIRCL_RECEIPT_INTERVAL_S` | prefix of the receipt files, one per communicator (`.rank..c.json`, n the communicator's number in the process), written when the communicator forms, refreshed after calls at most every `LIBSIRCL_RECEIPT_INTERVAL_S` seconds (default 10; 0: only at creation and destroy) and at destroy, each replaced whole; `sirclGetReceipt` returns the same JSON in process | | `LIBSIRCL_EMU_LANES`, `SIRCL_EMU_FABRIC`, `SIRCL_EMU_SEED`, `SIRCL_EMU_LATENCY_NS` | emulation transport | diff --git a/spark_transport/libsircl/src/bootstrap.c b/spark_transport/libsircl/src/bootstrap.c index efe2c13e..64c0416e 100644 --- a/spark_transport/libsircl/src/bootstrap.c +++ b/spark_transport/libsircl/src/bootstrap.c @@ -496,22 +496,29 @@ static int lan_configured(void) { return 0; } -/* Interface list match: comma-separated prefixes, "=name" for an exact name, - * "^prefix" entries exclude. */ -static int interface_matches(const char *list, const char *name) { +/* Interface list match, in NCCL_SOCKET_IFNAME's syntax: comma-separated entries; "prefix" selects the + * interfaces whose name starts with it and "=name" exactly that one; "^prefix" and "^=name" exclude them. + * Without any selecting entry every interface not excluded matches except a loopback interface, which only + * a selecting entry chooses (a remote rank cannot reach the root's loopback address). */ +static int interface_matches(const char *list, const char *name, int loopback) { char copy[256]; snprintf(copy, sizeof copy, "%s", list); int included = 0, any_include = 0; for (char *save = NULL, *item = strtok_r(copy, ",", &save); item; item = strtok_r(NULL, ",", &save)) { while (*item == ' ') ++item; - if (*item == '^') { - if (!strncmp(name, item + 1, strlen(item + 1))) return 0; + int exclude = *item == '^'; + if (exclude) ++item; + int exact = *item == '='; + if (exact) ++item; + int match = exact ? !strcmp(name, item) : !strncmp(name, item, strlen(item)); + if (exclude) { + if (match) return 0; continue; } any_include = 1; - if (*item == '=' ? !strcmp(name, item + 1) : !strncmp(name, item, strlen(item))) included = 1; + if (match) included = 1; } - return any_include ? included : 1; + return any_include ? included : !loopback; } /* The IPv4 address a new root listens on and advertises: SIRCL_BOOTSTRAP_ADDR, @@ -525,15 +532,19 @@ static int root_address(struct in_addr *out) { return error_result(ARGUMENT, "SIRCL_BOOTSTRAP_ADDR must be an IPv4 address"); return OK; } - const char *list = sccl_env("SIRCL_BOOTSTRAP_IFNAME"); - if (!list || !*list) list = sccl_env("NCCL_SOCKET_IFNAME"); + const char *name = "SIRCL_BOOTSTRAP_IFNAME", *list = sccl_env(name); + if (!list || !*list) list = sccl_env(name = "NCCL_SOCKET_IFNAME"); if (!list || !*list) return OK; + if (strlen(list) > 255) { + (void)snprintf(last_error, sizeof last_error, "%s is longer than 255 characters", name); + return ARGUMENT; + } struct ifaddrs *all = NULL; if (getifaddrs(&all) != 0) return error_result(SYSTEM, "cannot list network interfaces"); int found = 0; for (struct ifaddrs *a = all; a && !found; a = a->ifa_next) { if (!a->ifa_addr || a->ifa_addr->sa_family != AF_INET || !(a->ifa_flags & IFF_UP)) continue; - if (!interface_matches(list, a->ifa_name)) continue; + if (!interface_matches(list, a->ifa_name, (a->ifa_flags & IFF_LOOPBACK) != 0)) continue; *out = ((struct sockaddr_in *)a->ifa_addr)->sin_addr; found = 1; } diff --git a/spark_transport/libsircl/tests/test_bootstrap.py b/spark_transport/libsircl/tests/test_bootstrap.py index 37a5f941..4887ec6a 100644 --- a/spark_transport/libsircl/tests/test_bootstrap.py +++ b/spark_transport/libsircl/tests/test_bootstrap.py @@ -586,6 +586,32 @@ def test_interface_address(self): finally: os.environ.pop("SIRCL_BOOTSTRAP_ADDR") + def test_interface_exclusions_and_loopback(self): + """NCCL_SOCKET_IFNAME with exclusions only (``^docker``, ``^=lo``) never selects loopback, which a remote + rank cannot reach; ``^=name`` excludes exactly that interface; a list longer than 255 characters is + refused, naming the variable.""" + if not any(name != "lo" for _, name in socket.if_nameindex()): + self.skipTest("no interface besides loopback") + os.environ.pop("SIRCL_BOOTSTRAP_IFNAME", None) + try: + for value in ("^docker", "^=lo", "^=lo,^docker"): + with self.subTest(list=value): + os.environ["NCCL_SOCKET_IFNAME"] = value + buffer = ctypes.create_string_buffer(128) + result = self.bootstrap.lib.sccl_bootstrap_id(buffer) + if result == 4 and b"no IPv4 address" in self.bootstrap.lib.sccl_bootstrap_error(): + self.skipTest("no IPv4 interface besides loopback") + self.assertEqual(result, 0, self.bootstrap.lib.sccl_bootstrap_error()) + self.assertNotEqual(socket.inet_ntoa(buffer.raw[12:16]), "127.0.0.1") + for handle in self.joined_group(1, buffer.raw): + self.bootstrap.close(handle) + os.environ["NCCL_SOCKET_IFNAME"] = ",".join(["^docker"] * 40) + buffer = ctypes.create_string_buffer(128) + self.assertEqual(self.bootstrap.lib.sccl_bootstrap_id(buffer), 4) + self.assertIn(b"NCCL_SOCKET_IFNAME is longer than 255", self.bootstrap.lib.sccl_bootstrap_error()) + finally: + os.environ.pop("NCCL_SOCKET_IFNAME", None) + def test_invalid_env_and_arguments(self): for invalid in ["0", "-1", "nan", "600001"]: os.environ["SIRCL_BOOTSTRAP_TIMEOUT_MS"] = invalid From 8f271a82cba214d8be93b4f511508d0769d92444 Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Thu, 15 Oct 2026 08:58:18 -0500 Subject: [PATCH 30/65] Return success for NULL in ncclCommDestroy and ncclCommAbort NVIDIA NCCL 2.32.3's ncclCommDestroy and ncclCommAbort return ncclSuccess at once for a NULL communicator (src/init.cc of tag v2.32.3-1: `if (comm == NULL) ... return ncclSuccess;`). libsircl's destroy and abort do the same, so cleanup code that destroys a handle left NULL by a failed creation sees no error. ncclCommFinalize keeps refusing NULL with ncclInvalidArgument. NCCL 2.32.3's ncclCommFinalize also returns ncclSuccess for NULL (it jumps to its exit label with ret still ncclSuccess). libsircl keeps finalize's refusal as a documented difference; matching NCCL there is a separate compatibility decision. Compatibility: destroy and abort of NULL return ncclSuccess instead of ncclInvalidArgument. Validation: tests/test_api.py checks NULL for ncclCommAbort, ncclCommDestroy (success) and ncclCommFinalize (ncclInvalidArgument), for the nccl and pnccl names; before the change abort of NULL returned ncclInvalidArgument. make check and make check-asan pass. Request item 30 (audit 02-F12). NVIDIA NCCL source was read (init.cc of v2.32.3-1, the NULL checks of the three functions) and not adapted. Co-Authored-By: Claude Opus 5.5 --- spark_transport/libsircl/src/api.c | 2 ++ spark_transport/libsircl/tests/test_api.py | 6 +++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/spark_transport/libsircl/src/api.c b/spark_transport/libsircl/src/api.c index 35aae2a3..6e8e23b3 100644 --- a/spark_transport/libsircl/src/api.c +++ b/spark_transport/libsircl/src/api.c @@ -1053,6 +1053,8 @@ ncclResult_t ncclCommRevoke(ncclComm_t comm, int flags) { } static ncclResult_t dispose(ncclComm_t handle, int abort) { CHECK_PROCESS(); + /* NVIDIA NCCL 2.32.3's ncclCommDestroy and ncclCommAbort return ncclSuccess for a NULL communicator. */ + if (!handle) return ncclSuccess; /* Group-owned pending handles cannot be removed before GroupEnd consumes them. */ for (unsigned i = 0; i < pending_count; ++i) if (pending[i] == handle) return fail(ncclInvalidUsage, "communicator initialization is pending in a group"); diff --git a/spark_transport/libsircl/tests/test_api.py b/spark_transport/libsircl/tests/test_api.py index 91df07f5..e9cb7ac6 100644 --- a/spark_transport/libsircl/tests/test_api.py +++ b/spark_transport/libsircl/tests/test_api.py @@ -244,7 +244,11 @@ def test_revoke_then_finalize_and_abort(self): self.assertEqual(LIB.ncclCommFinalize(handle), INVALID_USAGE) self.assertEqual(LIB.ncclCommAbort(handle), SUCCESS) self.assertEqual(LIB.ncclCommAbort(handle), INVALID_ARGUMENT) - self.assertEqual(LIB.ncclCommAbort(None), INVALID_ARGUMENT) + # NULL is a no-op for destroy and abort, as in NVIDIA NCCL 2.32.3; finalize refuses it. + for prefix in ("", "p"): + self.assertEqual(getattr(LIB, prefix + "ncclCommAbort")(None), SUCCESS) + self.assertEqual(getattr(LIB, prefix + "ncclCommDestroy")(None), SUCCESS) + self.assertEqual(getattr(LIB, prefix + "ncclCommFinalize")(None), INVALID_ARGUMENT) def test_thread_local_nested_group_and_error(self): barrier = threading.Barrier(2) From 07951c8663b6cb6e2d067cddd5326ac1e7f49d11 Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Thu, 15 Oct 2026 08:59:43 -0500 Subject: [PATCH 31/65] Publish an initialization's result before its state The initializing thread stores the communicator's result before its state (READY or FAILED). Both stores are sequentially consistent, so a reader that sees READY or FAILED also sees the result that goes with it; storing the state first left a window in which a ready state carried ncclInProgress. The second half of the audit item, reading the environment on the caller's thread instead of the non-blocking initialization worker, is not part of this change: sccl_env is a header-inline accessor compiled into several translation units and into the separately built bootstrap tests, so a per-thread snapshot needs a shared definition; getenv on a worker races only with a concurrent setenv elsewhere in the process. Compatibility: no change visible to single-threaded callers. Validation: the window is two adjacent atomic stores and is not reproducible by a test; make check and make check-asan pass. Request item 31 (audit 02-F13, publication order only). NVIDIA NCCL source was neither read nor adapted. Co-Authored-By: Claude Opus 5.5 --- spark_transport/libsircl/src/api.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/spark_transport/libsircl/src/api.c b/spark_transport/libsircl/src/api.c index 6e8e23b3..4dfd9418 100644 --- a/spark_transport/libsircl/src/api.c +++ b/spark_transport/libsircl/src/api.c @@ -247,8 +247,10 @@ static void *initialize(void *arg) { sccl_bootstrap_close(comm->bootstrap); comm->bootstrap = NULL; } - atomic_store(&comm->state, result == 0 ? READY : FAILED); + /* The result first (both stores sequentially consistent): a reader that sees READY or FAILED also sees + * the result that goes with it. */ atomic_store(&comm->error, result); + atomic_store(&comm->state, result == 0 ? READY : FAILED); return NULL; } static ncclResult_t start(struct ncclComm *comm) { From 35365450abee22321a463c66515783e3c3a2234b Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Thu, 15 Oct 2026 09:01:16 -0500 Subject: [PATCH 32/65] Finish every local setup step before the last all-rank verdict LIBSIRCL_LANE_CHECK_MS is read with the other settings, so a bad value fails the first setup verdict on every rank, where it was parsed after an exchange and failed this rank alone while its peers waited for it. The order event is created inside the lane-check step, before the last verdict, so a failure to create it fails setup on every rank instead of failing this rank after its peers returned success. Compatibility: valid settings behave the same; an invalid LIBSIRCL_LANE_CHECK_MS fails every rank at the first verdict. Validation: tests/test_engine_cpu.py's setup-exchange test (renamed test_route_and_check_settings_reach_the_setup_exchange) captures rank 0's first verdict record through a stand-in broker: with LIBSIRCL_LANE_CHECK_MS=0 it names the variable (before the change the verdict carried an unrelated route-map error, and the bad value was found only later on this rank alone). The order-event step needs a transport and is not reached on CPU. make check and make check-asan pass. Request item 32 (audit 02-F14). NVIDIA NCCL source was neither read nor adapted. Co-Authored-By: Claude Opus 5.5 --- spark_transport/libsircl/src/engine.c | 23 +++++++++---------- .../libsircl/tests/test_engine_cpu.py | 8 +++++-- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/spark_transport/libsircl/src/engine.c b/spark_transport/libsircl/src/engine.c index 791a9284..a9d2f245 100644 --- a/spark_transport/libsircl/src/engine.c +++ b/spark_transport/libsircl/src/engine.c @@ -440,6 +440,7 @@ struct sccl_engine { /* Teardown: the progress thread stopped (sccl_engine_teardown_stop), when its stop was requested and * when it had stopped (CLOCK_REALTIME nanoseconds). */ int stopped, closing; + int lane_check_ms; /* LIBSIRCL_LANE_CHECK_MS: the lane check's bound */ int kp_held; /* this engine holds a reference on its context's kernel-pack entry (sccl_kp_load) */ uint64_t stop_ns, stopped_ns; int have_last; @@ -979,6 +980,9 @@ static int read_settings(sccl_engine *e, int emulation, char *err, size_t len) { e->fail_stop = !stop || !*stop || !strcmp(stop, "0") ? 0 : !strcmp(stop, "1") ? FAIL_STOP_EXIT : FAIL_STOP_ABORT; uint8_t relayed[MAX_POSITIONS][MAX_LANES]; if (relayed_lanes(relayed, err, len)) return -1; + /* Read with the other settings, so that a bad value fails the first verdict on every rank. */ + if (env_u64("LIBSIRCL_LANE_CHECK_MS", 10000, 1, 600000, &v, err, len)) return -1; + e->lane_check_ms = (int)v; if (env_u64("LIBSIRCL_RING_REDUCE_PASSES", 2, 1, 2, &v, err, len)) return -1; const char *one_pass = sccl_env("LIBSIRCL_RING_REDUCE_ONE_PASS_RESEARCH"); if (v == 1 && (!one_pass || strcmp(one_pass, "1"))) { @@ -2427,17 +2431,13 @@ int sccl_engine_create(sccl_bootstrap *bootstrap, int nranks, int rank, int posi } /* Exchange 3: prove every lane with one write, then start the progress thread. */ { - uint64_t check_ms; - if (env_u64("LIBSIRCL_LANE_CHECK_MS", 10000, 1, 600000, &check_ms, err, err_len)) { - status = ncclInvalidArgument; - goto fail; - } + int check_ms = e->lane_check_ms; char why[400] = {0}; - int rc = e->transport->lane_check(e->proxy, (int)check_ms); + int rc = e->transport->lane_check(e->proxy, check_ms); if (rc) snprintf(why, sizeof why, "lane check: %s", e->transport->error(e->proxy)); if (!rc && e->p2p) { uint64_t started = monotonic_ns(); - rc = e->p2p_transport->lane_check(e->p2p, (int)check_ms) || p2p_apply_windows(e); + rc = e->p2p_transport->lane_check(e->p2p, check_ms) || p2p_apply_windows(e); e->p2p_check_ns = monotonic_ns() - started; if (rc) snprintf(why, sizeof why, "point-to-point lane check: %s", e->p2p_transport->error(e->p2p)); } @@ -2447,16 +2447,15 @@ int sccl_engine_create(sccl_bootstrap *bootstrap, int nranks, int rank, int posi rc = start_progress(e); if (rc) snprintf(why, sizeof why, "%s", rc == 1 ? e->transport->error(e->proxy) : e->p2p_transport->error(e->p2p)); } + /* Every local step before the last verdict, so that no rank fails alone after its peers succeeded. */ + if (rc == 0 && cuda_check(cu->EventCreate(&e->order_event, SCCL_CU_EVENT_DISABLE_TIMING), + "creating the order event", why, sizeof why)) + rc = -1; if (verdict(bootstrap, nranks, rank, rc == 0, why, cancelled, err, err_len)) { status = ncclSystemError; goto fail; } } - if (cuda_check(cu->EventCreate(&e->order_event, SCCL_CU_EVENT_DISABLE_TIMING), "creating the order event", err, - err_len)) { - status = ncclUnhandledCudaError; - goto fail; - } cu->CtxPopCurrent(&ctx); start_receipts(e); if (e->fail_stop) watch_add(e); diff --git a/spark_transport/libsircl/tests/test_engine_cpu.py b/spark_transport/libsircl/tests/test_engine_cpu.py index 8235c6de..a0d670b7 100644 --- a/spark_transport/libsircl/tests/test_engine_cpu.py +++ b/spark_transport/libsircl/tests/test_engine_cpu.py @@ -186,10 +186,10 @@ def read(n): self.assertIn("rank 1 position", data["local"]) self.assertIn("0-63", data["local"]) - def test_relayed_lanes_reach_the_setup_exchange(self): + def test_route_and_check_settings_reach_the_setup_exchange(self): """LIBSIRCL_RELAYED_LANES marks a peer position as relayed in the first setup exchange's record (which the cycle plan reads) without any forward window; a malformed value, or a LIBSIRCL_RING_WINDOW above the - native cap, fails creation naming the variable.""" + native cap, or a LIBSIRCL_LANE_CHECK_MS out of range, fails the first verdict naming the variable.""" script = '''import socket, struct, threading u=U(); assert l.ncclGetUniqueId(C.byref(u))==0 server=socket.socket(); server.bind(("127.0.0.1", 0)); server.listen() @@ -234,6 +234,10 @@ def read(n): out = run(script, {**base, "LIBSIRCL_TRANSPORT": "verbs", "LIBSIRCL_RING_WINDOW": str(61 * 32768)}) self.assertEqual(out.returncode, 0, out.stderr) self.assertIn("LIBSIRCL_RING_WINDOW=1998848 must be 0 or hold 1 to 60 chunks", json.loads(out.stdout)["local"]) + # The lane check's bound is read with the settings, so a bad value fails the first verdict. + out = run(script, {**base, "LIBSIRCL_TRANSPORT": "verbs", "LIBSIRCL_LANE_CHECK_MS": "0"}) + self.assertEqual(out.returncode, 0, out.stderr) + self.assertIn("LIBSIRCL_LANE_CHECK_MS=0 must be an integer from 1", json.loads(out.stdout)["local"]) def test_info_names_the_collectives(self): out = run("print(l.sirclGetInfo().decode())") From 7ea9207f91b4bff072bc79eddd56988bee3e31ee Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Thu, 15 Oct 2026 09:03:03 -0500 Subject: [PATCH 33/65] Agree fail-stop and the starting wait regime across ranks at setup LIBSIRCL_FAIL_STOP's mode and the regime a communicator starts in (LIBSIRCL_WAIT_REGIME) join the settings every rank compares at setup; a communicator whose ranks differ fails setup naming the variable. A group in which one rank ends its process at the first asynchronous error while another keeps returning results, or in which ranks start under different wait limits, cannot form. The two values occupy reserved words of the settings record, so its size is unchanged. README.md states the agreement. Compatibility: ranks of one communicator must set LIBSIRCL_FAIL_STOP and LIBSIRCL_WAIT_REGIME alike; sirclSetWaitRegime still switches each rank afterwards. Validation: tests/test_engine_cpu.py adds CPU coverage of fail-stop's settings: an invalid LIBSIRCL_FAIL_STOP and LIBSIRCL_FAIL_STOP_POLL_MS=0 fail the first verdict naming the variable (both refusals predate this change; the audit noted no CPU test touched fail-stop). The agreement itself needs two ranks with a transport. Written, not run: python tests/emulation/setup_failures.py --library build/libsircl.so (cases "LIBSIRCL_FAIL_STOP differing between ranks" and "LIBSIRCL_WAIT_REGIME differing between ranks"). The fail_stop.py case of a peer killed with no collective in flight is not written. make check and make check-asan pass. Request item 33 (audit 04-F8; coverage parts of 04-F9 and 12-F4). NVIDIA NCCL source was neither read nor adapted. Co-Authored-By: Claude Opus 5.5 --- spark_transport/libsircl/README.md | 4 ++-- spark_transport/libsircl/src/engine.c | 12 ++++++++++-- .../libsircl/tests/emulation/setup_failures.py | 10 +++++++++- spark_transport/libsircl/tests/test_engine_cpu.py | 11 ++++++++++- 4 files changed, 31 insertions(+), 6 deletions(-) diff --git a/spark_transport/libsircl/README.md b/spark_transport/libsircl/README.md index 96752523..64941f60 100644 --- a/spark_transport/libsircl/README.md +++ b/spark_transport/libsircl/README.md @@ -248,8 +248,8 @@ LF), and refuses a vendored copy that does not define the word; `P2P_FEATURES=0` | `LIBSIRCL_P2P_WINDOWS` | windows of this process's channel lanes through relays: `=[/],...` per lane, 0 or a multiple of 16 holding 1 to 120 chunks; a lane counts as relayed when `LIBSIRCL_RELAYED_LANES` marks it or `LIBSIRCL_FORWARD_WINDOWS` gives it a window, and a pair with a relayed lane in either direction without one has no channel. `tools/site_routes.py` prints them from SIRCL's point-to-point budget: `--p2p-reserve session` (the default, SIRCL's rule) leaves the collective session's forward windows, and with `--ring-schedules` (the site runs the ring schedules) its ring windows, their room in every relay queue first; `--session-share F` (default 1) sizes the session against F of every queue's share, so the channels get the rest; `none` sizes the channels as the only relayed traffic. On ring:8 the session's forward windows at the default share leave no relayed pair a channel; windows are whole 32 KiB chunks, so any share below 1 (0.95 to 0.05 checked) takes at least one chunk from every relayed session lane (at 0.95: 64-128 KiB become 32-96 KiB) and gives all 40 relayed ordered pairs a channel, as does `--max-window 32768` (every session lane one chunk) | | `SIRCL_P2P_PROGRESS_CPU` | read by the channels' progress thread, as in SIRCL: a CPU list that pins it; unset, it runs where `LIBSIRCL_CPU_POLICY` places the transport's progress thread | | `LIBSIRCL_STREAM_ORDERED_ALLOC` | `auto` (default): staging uses stream-ordered allocation (`cuMemAllocAsync`) where the driver and device have it; `off` declines it (the communicator's own staging buffers; captured calls that need more are refused), the same on every rank, for comparisons and tests | -| `SIRCL_STARTUP_WAIT_S`, `SIRCL_SERVING_WAIT_S`, `LIBSIRCL_WAIT_REGIME` | flag-wait limits and the regime a communicator starts in (`startup`, default, or `serving`); `sirclSetWaitRegime` switches it | -| `LIBSIRCL_FAIL_STOP` | `0` (default), `1` or `abort`, per process: with `1` or `abort` the process ends at the first asynchronous error (a flag wait that timed out, a failed progress thread, a failed point-to-point channel) of any communicator of two or more ranks. The error is found by a watcher thread that checks every such communicator every `LIBSIRCL_FAIL_STOP_POLL_MS`, or by the first library call that would report it to the caller (`ncclCommGetAsyncError`, a collective or point-to-point enqueue, `ncclCommFinalize`, `ncclCommDestroy`, `ncclCommAbort`), whichever comes first. The library writes a line to stderr (`libsircl: LIBSIRCL_FAIL_STOP: ending the process at (Unix time; CLOCK_MONOTONIC s) on an asynchronous error of communicator (rank of , found by ): `) and the communicator's receipt, then ends the process: `1` at once with exit status 70 (no atexit handlers, no core dump), `abort` by `abort()` (SIGABRT, after the system's core-dump handling). A caller that only checks the codes of enqueue calls keeps a failed collective's output at most for the wait limit plus one poll, and a caller that reads the error and tears the communicator down never returns from that call; a read of the output within one poll after its stream wait returns, without a library call, is not prevented. Receipts read the error without ending the process | +| `SIRCL_STARTUP_WAIT_S`, `SIRCL_SERVING_WAIT_S`, `LIBSIRCL_WAIT_REGIME` | flag-wait limits and the regime a communicator starts in (`startup`, default, or `serving`; every rank of a communicator must start in the same regime, or setup fails naming the variable); `sirclSetWaitRegime` switches it | +| `LIBSIRCL_FAIL_STOP` | `0` (default), `1` or `abort`, per process and the same on every rank of a communicator (setup fails naming the variable otherwise): with `1` or `abort` the process ends at the first asynchronous error (a flag wait that timed out, a failed progress thread, a failed point-to-point channel) of any communicator of two or more ranks. The error is found by a watcher thread that checks every such communicator every `LIBSIRCL_FAIL_STOP_POLL_MS`, or by the first library call that would report it to the caller (`ncclCommGetAsyncError`, a collective or point-to-point enqueue, `ncclCommFinalize`, `ncclCommDestroy`, `ncclCommAbort`), whichever comes first. The library writes a line to stderr (`libsircl: LIBSIRCL_FAIL_STOP: ending the process at (Unix time; CLOCK_MONOTONIC s) on an asynchronous error of communicator (rank of , found by ): `) and the communicator's receipt, then ends the process: `1` at once with exit status 70 (no atexit handlers, no core dump), `abort` by `abort()` (SIGABRT, after the system's core-dump handling). A caller that only checks the codes of enqueue calls keeps a failed collective's output at most for the wait limit plus one poll, and a caller that reads the error and tears the communicator down never returns from that call; a read of the output within one poll after its stream wait returns, without a library call, is not prevented. Receipts read the error without ending the process | | `LIBSIRCL_FAIL_STOP_POLL_MS` | the fail-stop watcher's period, 1 to 3,600,000 ms (default 5); the tests set it long to show that the library calls alone end the process | | `SIRCL_POST_ORDER`, `SIRCL_PROGRESS_CPU`, `SIRCL_FORWARD_PROOF` | read by the native progress thread, as in SIRCL; `SIRCL_PROGRESS_CPU` pins it to a CPU list | | `LIBSIRCL_CPU_POLICY` | where the progress thread runs without `SIRCL_PROGRESS_CPU`: `performance` (default: on the fastest CPU class the creating thread may use, the Cortex-X925 cores of a GB10, found by `/proc/cpuinfo` part numbers or sysfs `cpu_capacity`) or `none` (where the scheduler puts it). The library never changes the affinity of the application's own threads; receipts name the progress thread's CPUs (`progress_cpus`) | diff --git a/spark_transport/libsircl/src/engine.c b/spark_transport/libsircl/src/engine.c index a9d2f245..bc3c9c2d 100644 --- a/spark_transport/libsircl/src/engine.c +++ b/spark_transport/libsircl/src/engine.c @@ -304,7 +304,9 @@ typedef struct { uint32_t startup_wait_us, serving_wait_us, proxy_abi, graph_staging; char kernel_pack[72], fold_pack[72], links_pack[72]; /* The chain schedule: SIRCL_LARGE_SCHEDULE, geometry, and the chain order (ranks by chain index). */ - uint32_t large_schedule, chain_slots, chain_chunk, chain_blocks, chain_unroll, chain_reserved; + uint32_t large_schedule, chain_slots, chain_chunk, chain_blocks, chain_unroll; + /* Fail-stop's mode (LIBSIRCL_FAIL_STOP, 0 for a communicator of one rank). */ + uint32_t fail_stop; uint64_t chain_slot_bytes; /* Smallest collective (COLL_*) that auto runs as a chain op, and that a ring schedule runs as a ring op. */ uint64_t chain_mins[COLLECTIVES], ring_mins[COLLECTIVES]; @@ -313,7 +315,9 @@ typedef struct { * collective (COLL_*), staggers, whether the ring that closes the chain runs (LIBSIRCL_RING_WINDOW * set), the reduce-scatter's column tile. */ uint32_t gather_schedule, scatter_schedule, link_slots, link_blocks, link_unroll, ring_on; - uint32_t ring_stagger, ring_gather_stagger, link_chunk[COLLECTIVES], link_reserved; + uint32_t ring_stagger, ring_gather_stagger, link_chunk[COLLECTIVES]; + /* The wait regime at setup (LIBSIRCL_WAIT_REGIME: 0 startup, 1 serving). */ + uint32_t wait_regime; uint64_t link_slot_bytes, link_tile; /* Blocks per role and pieces per collective: chunk_set[c] when a link chunk variable sets the piece, * coll_blocks[c] from LIBSIRCL__LINK_BLOCKS (0: unset), blocks_set when SIRCL_LINK_BLOCKS @@ -972,6 +976,7 @@ static int read_settings(sccl_engine *e, int emulation, char *err, size_t len) { return -1; } e->serving = regime && !strcmp(regime, "serving"); + s->wait_regime = (uint32_t)e->serving; const char *stop = sccl_env("LIBSIRCL_FAIL_STOP"); if (stop && *stop && strcmp(stop, "0") && strcmp(stop, "1") && strcmp(stop, "abort")) { put_error(err, len, "LIBSIRCL_FAIL_STOP=%s: 0, 1 or abort", stop); @@ -1166,6 +1171,8 @@ static const char *settings_difference(const shared_settings *a, const shared_se SAME(algorithm_plus_one, "SIRCL_ALLREDUCE_ALGORITHM"); SAME(startup_wait_us, "SIRCL_STARTUP_WAIT_S"); SAME(serving_wait_us, "SIRCL_SERVING_WAIT_S"); + SAME(wait_regime, "LIBSIRCL_WAIT_REGIME"); + SAME(fail_stop, "LIBSIRCL_FAIL_STOP"); SAME(proxy_abi, "native proxy ABI"); SAME(graph_staging, "stream-ordered allocation (cuMemAllocAsync and the device's memory pools), which " "stages unaligned buffers under CUDA graph capture"); @@ -2221,6 +2228,7 @@ int sccl_engine_create(sccl_bootstrap *bootstrap, int nranks, int rank, int posi e->device = device; if (read_settings(e, emulation, local, sizeof local)) break; if (nranks == 1) e->fail_stop = 0; + e->shared.fail_stop = (uint32_t)e->fail_stop; if (e->fail_stop && watch_start(local, sizeof local)) break; e->shared.proxy_abi = (uint32_t)e->transport->abi_version(); /* Before the setup verdict, so a rank whose native proxy lacks a feature fails setup with every rank. */ diff --git a/spark_transport/libsircl/tests/emulation/setup_failures.py b/spark_transport/libsircl/tests/emulation/setup_failures.py index d8b4ced4..767f34a2 100644 --- a/spark_transport/libsircl/tests/emulation/setup_failures.py +++ b/spark_transport/libsircl/tests/emulation/setup_failures.py @@ -13,7 +13,9 @@ - a ring schedule without the layout's ring plan (``SIRCL_SCATTER_SCHEDULE=ring`` without ``LIBSIRCL_RING_WINDOW``, and ``SIRCL_LARGE_SCHEDULE=pieces``, which turns off the pair plan and with it the ring plan a cabled pair gets by default); -- a schedule differing between the ranks (``SIRCL_GATHER_SCHEDULE``). +- a schedule differing between the ranks (``SIRCL_GATHER_SCHEDULE``); +- fail-stop and the wait regime differing between the ranks (``LIBSIRCL_FAIL_STOP``, + ``LIBSIRCL_WAIT_REGIME``). """ from __future__ import annotations @@ -94,6 +96,12 @@ def main(argv=None) -> int: ("SIRCL_GATHER_SCHEDULE differing between ranks", [emulation, dict(emulation, SIRCL_GATHER_SCHEDULE="chain")], ["rank 1: SIRCL_GATHER_SCHEDULE differs from rank 0"]), + ("LIBSIRCL_FAIL_STOP differing between ranks", + [emulation, dict(emulation, LIBSIRCL_FAIL_STOP="1")], + ["rank 1: LIBSIRCL_FAIL_STOP differs from rank 0"]), + ("LIBSIRCL_WAIT_REGIME differing between ranks", + [emulation, dict(emulation, LIBSIRCL_WAIT_REGIME="serving")], + ["rank 1: LIBSIRCL_WAIT_REGIME differs from rank 0"]), ] failed = 0 for index, (name, environments, expected) in enumerate(cases): diff --git a/spark_transport/libsircl/tests/test_engine_cpu.py b/spark_transport/libsircl/tests/test_engine_cpu.py index a0d670b7..15a3b53c 100644 --- a/spark_transport/libsircl/tests/test_engine_cpu.py +++ b/spark_transport/libsircl/tests/test_engine_cpu.py @@ -189,7 +189,8 @@ def read(n): def test_route_and_check_settings_reach_the_setup_exchange(self): """LIBSIRCL_RELAYED_LANES marks a peer position as relayed in the first setup exchange's record (which the cycle plan reads) without any forward window; a malformed value, or a LIBSIRCL_RING_WINDOW above the - native cap, or a LIBSIRCL_LANE_CHECK_MS out of range, fails the first verdict naming the variable.""" + native cap, a LIBSIRCL_LANE_CHECK_MS out of range, or an invalid LIBSIRCL_FAIL_STOP or LIBSIRCL_FAIL_STOP_POLL_MS + fails the first verdict naming the variable.""" script = '''import socket, struct, threading u=U(); assert l.ncclGetUniqueId(C.byref(u))==0 server=socket.socket(); server.bind(("127.0.0.1", 0)); server.listen() @@ -238,6 +239,14 @@ def read(n): out = run(script, {**base, "LIBSIRCL_TRANSPORT": "verbs", "LIBSIRCL_LANE_CHECK_MS": "0"}) self.assertEqual(out.returncode, 0, out.stderr) self.assertIn("LIBSIRCL_LANE_CHECK_MS=0 must be an integer from 1", json.loads(out.stdout)["local"]) + # Fail-stop's settings: refused before the first verdict, naming the variable. + for extra, text in (({"LIBSIRCL_FAIL_STOP": "yes"}, "LIBSIRCL_FAIL_STOP=yes: 0, 1 or abort"), + ({"LIBSIRCL_FAIL_STOP": "1", "LIBSIRCL_FAIL_STOP_POLL_MS": "0"}, + "LIBSIRCL_FAIL_STOP_POLL_MS=0 must be an integer from 1")): + with self.subTest(**extra): + out = run(script, {**base, "LIBSIRCL_TRANSPORT": "verbs", **extra}) + self.assertEqual(out.returncode, 0, out.stderr) + self.assertIn(text, json.loads(out.stdout)["local"]) def test_info_names_the_collectives(self): out = run("print(l.sirclGetInfo().decode())") From f19f4570b0eee57813a140a8c17607b407918527 Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Thu, 15 Oct 2026 09:04:25 -0500 Subject: [PATCH 34/65] Bound fail-stop's exit with a backstop thread fail_stop starts a detached backstop thread before any step that can block. Whatever those steps wait for (the point-to-point abort notices, the stderr line to a stalled pipe, the receipt or link dump on a stalled filesystem or into a FIFO without a reader), the process ends 1 s after fail_stop began with the fail-stop exit status 70. If the thread cannot be started the steps run unbounded, as without it. Compatibility: a fail-stop whose I/O completes within 1 s is unchanged; one whose I/O stalls ends at 1 s with status 70, possibly without its receipt or line. Validation: fail_stop runs only on an engine of two or more ranks, which the CPU harness cannot build. The emulation case the audit proposes (tests/emulation/fail_stop.py with the receipt in a FIFO that has no reader) is not written: the receipt file is also written before the fail-stop, and a case must first establish where a FIFO would block. make check and make check-asan pass. Request item 34 (audit 04-F7). NVIDIA NCCL source was neither read nor adapted. Co-Authored-By: Claude Opus 5.5 --- spark_transport/libsircl/src/engine.c | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/spark_transport/libsircl/src/engine.c b/spark_transport/libsircl/src/engine.c index bc3c9c2d..3921e423 100644 --- a/spark_transport/libsircl/src/engine.c +++ b/spark_transport/libsircl/src/engine.c @@ -2026,7 +2026,28 @@ static ncclResult_t session_error(sccl_engine *e, char *message, size_t len); /* The line names the time twice: the wall clock for logs, and CLOCK_MONOTONIC (the clock of the host's own * timeouts and of Python's time.monotonic), which a stepped wall clock (a VM's time synchronization) does * not move; and what found the error. */ +/* fail_stop's backstop: whatever the steps before _exit wait for (a stalled stderr pipe, a receipt or link + * dump on a stalled filesystem, a FIFO without a reader), the process ends FAIL_STOP_BACKSTOP_MS after + * fail_stop began, with the fail-stop status. */ +enum { FAIL_STOP_BACKSTOP_MS = 1000 }; +static void *fail_stop_backstop(void *unused) { + (void)unused; + const struct timespec delay = {FAIL_STOP_BACKSTOP_MS / 1000, (FAIL_STOP_BACKSTOP_MS % 1000) * 1000000L}; + nanosleep(&delay, NULL); + _exit(FAIL_STOP_STATUS); + return NULL; +} + static void fail_stop(sccl_engine *e, const char *found_by, const char *message) { + /* The backstop first, before any step that can block. A failure to start it leaves the steps unbounded, + * as they are without it. */ + pthread_t backstop; + pthread_attr_t attributes; + if (!pthread_attr_init(&attributes)) { + pthread_attr_setdetachstate(&attributes, PTHREAD_CREATE_DETACHED); + (void)pthread_create(&backstop, &attributes, fail_stop_backstop, NULL); + pthread_attr_destroy(&attributes); + } /* Before the process ends, the channels' progress thread posts its abort notices, so the peers' channels * stop now instead of at their wait limits (bounded; see p2p_stop_with_notices). */ p2p_stop_with_notices(e, message); From b53acb442acf4a5597c48bc37c2f9e01f98ac3e8 Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Thu, 15 Oct 2026 09:06:20 -0500 Subject: [PATCH 35/65] Write receipts and link dumps without following links or sharing a partial file Receipts are written under a partial name of the writer's own (.partial., n a per-process counter), so the fail-stop watcher and a library call writing the same communicator's receipt at once never share a file. Every receipt and link-dump file is opened with open(2) and O_NOFOLLOW (a symbolic link at the path is refused, never followed) and close-on-exec; a receipt's partial file also with O_EXCL. A path that does not fit its buffer is not written, instead of writing a truncated name. Compatibility: receipt and dump contents and final names are unchanged. A planted link where a diagnostic file goes makes that write fail instead of writing through the link. Validation: tests/test_engine_cpu.py plants a symbolic link at .partial pointing to a file, then creates and destroys a one-rank communicator with LIBSIRCL_RECEIPT set: before the change the linked file was overwritten with the receipt; after it the file and the link are untouched, the receipt is written and no partial file is left. make check and make check-asan pass. Request item 35 (audit 15-F08, 07-F9 item 4). NVIDIA NCCL source was neither read nor adapted. Co-Authored-By: Claude Opus 5.5 --- spark_transport/libsircl/src/engine.c | 38 ++++++++++++++----- .../libsircl/tests/test_engine_cpu.py | 27 +++++++++++++ 2 files changed, 56 insertions(+), 9 deletions(-) diff --git a/spark_transport/libsircl/src/engine.c b/spark_transport/libsircl/src/engine.c index 3921e423..8143252f 100644 --- a/spark_transport/libsircl/src/engine.c +++ b/spark_transport/libsircl/src/engine.c @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -4553,20 +4554,39 @@ size_t sccl_engine_receipt(sccl_engine *e, char *out, size_t len) { status == ncclSuccess ? "true" : "false", health_json); } +/* A diagnostic file of this library: created by open(2) with O_NOFOLLOW (a symbolic link at the path is + * refused, never followed) and close-on-exec; with `fresh`, also O_EXCL (the file must not exist), else + * appended to. NULL when the path did not fit its buffer (`fits` 0) or the open failed. */ +static FILE *open_diagnostic(const char *path, int fits, int fresh) { + if (!fits) return NULL; + int flags = O_WRONLY | O_CREAT | O_NOFOLLOW | O_CLOEXEC | (fresh ? O_EXCL : O_APPEND); + int fd = open(path, flags, 0644); + if (fd < 0) return NULL; + FILE *f = fdopen(fd, fresh ? "w" : "a"); + if (!f) close(fd); + return f; +} + /* .rank..c.json, n the communicator's number among the process's communicators, so * communicators of one process with the same rank keep files of their own. The file is replaced whole - * (written beside it, then renamed), so a reader never sees a partial receipt. */ + * (written beside it under a name of this writer's own, then renamed), so a reader never sees a partial + * receipt, and two writers (the fail-stop watcher and a library call) never share a partial file. A path + * that does not fit is not written. */ static void write_receipt(sccl_engine *e) { + static atomic_uint writes; const char *prefix = sccl_env("LIBSIRCL_RECEIPT"); if (!prefix || !*prefix) return; - char path[600], partial[620]; - snprintf(path, sizeof path, "%s.rank%d.%d.c%d.json", prefix, e->rank, (int)getpid(), e->receipt_id); - snprintf(partial, sizeof partial, "%s.partial", path); + char path[600], partial[640]; + int n = snprintf(path, sizeof path, "%s.rank%d.%d.c%d.json", prefix, e->rank, (int)getpid(), e->receipt_id); + int fits = n > 0 && (size_t)n < sizeof path; + n = snprintf(partial, sizeof partial, "%s.partial.%u", path, atomic_fetch_add(&writes, 1)); + fits = fits && n > 0 && (size_t)n < sizeof partial; + if (!fits) return; size_t need = sccl_engine_receipt(e, NULL, 0) + 1; char *text = malloc(need); if (!text) return; sccl_engine_receipt(e, text, need); - FILE *f = fopen(partial, "w"); + FILE *f = open_diagnostic(partial, 1, 1); if (f) { int ok = fprintf(f, "%s\n", text) > 0; ok = fclose(f) == 0 && ok; @@ -4624,8 +4644,8 @@ static void dump_links(sccl_engine *e, const char *why, int device) { const char *prefix = sccl_env("LIBSIRCL_LINK_DUMP"); if (!prefix || !*prefix || !e->link_on || !e->host) return; char path[640]; - snprintf(path, sizeof path, "%s.rank%d.%d.c%d.links.json", prefix, e->rank, (int)getpid(), e->receipt_id); - FILE *f = fopen(path, "a"); + int n = snprintf(path, sizeof path, "%s.rank%d.%d.c%d.links.json", prefix, e->rank, (int)getpid(), e->receipt_id); + FILE *f = open_diagnostic(path, n > 0 && (size_t)n < sizeof path, 0); if (!f) return; enum { LINKS = 4, STRIDE = 128 }; const uint8_t *area = e->host + e->link_off; @@ -4691,8 +4711,8 @@ static void dump_teardown(sccl_engine *e, uint64_t stop_ns, uint64_t stopped_ns, const char *prefix = sccl_env("LIBSIRCL_LINK_DUMP"); if (!prefix || !*prefix || !e->link_on || !e->host) return; char path[640]; - snprintf(path, sizeof path, "%s.rank%d.%d.c%d.links.json", prefix, e->rank, (int)getpid(), e->receipt_id); - FILE *f = fopen(path, "a"); + int n = snprintf(path, sizeof path, "%s.rank%d.%d.c%d.links.json", prefix, e->rank, (int)getpid(), e->receipt_id); + FILE *f = open_diagnostic(path, n > 0 && (size_t)n < sizeof path, 0); if (!f) return; fprintf(f, "{\"why\":\"proxy destroyed\",\"ns\":%" PRIu64 ",\"rank\":%d,\"position\":%d,\"stop_ns\":%" PRIu64 ",\"stopped_ns\":%" PRIu64 ",\"destroyed_ns\":%" PRIu64, diff --git a/spark_transport/libsircl/tests/test_engine_cpu.py b/spark_transport/libsircl/tests/test_engine_cpu.py index 15a3b53c..ddf48224 100644 --- a/spark_transport/libsircl/tests/test_engine_cpu.py +++ b/spark_transport/libsircl/tests/test_engine_cpu.py @@ -12,6 +12,7 @@ import os import subprocess import sys +import tempfile import unittest from pathlib import Path @@ -248,6 +249,32 @@ def read(n): self.assertEqual(out.returncode, 0, out.stderr) self.assertIn(text, json.loads(out.stdout)["local"]) + def test_receipts_never_follow_a_planted_link(self): + """A symbolic link planted where a receipt's partial file was named (.partial) is left as it + is and the file it names is not written: receipts are written under a name of the writer's own, + opened with O_EXCL and O_NOFOLLOW, then renamed over the receipt.""" + with tempfile.TemporaryDirectory() as directory: + out = run('''import os, sys +directory=sys.argv[2] +target=os.path.join(directory, "victim"); open(target, "w").write("untouched") +prefix=os.path.join(directory, "r") +receipt=f"{prefix}.rank0.{os.getpid()}.c1.json" +os.symlink(target, receipt + ".partial") +os.environ["LIBSIRCL_RECEIPT"]=prefix +u=U(); assert l.ncclGetUniqueId(C.byref(u))==0 +c=C.c_void_p(); assert l.ncclCommInitRank(C.byref(c),1,u,0)==0 +assert l.ncclCommDestroy(c)==0 +print(json.dumps([open(target).read(), os.path.islink(receipt + ".partial"), os.path.isfile(receipt), + sorted(name for name in os.listdir(directory) if ".partial." in name)])) +'''.replace("sys.argv[2]", repr(directory)), {"LIBSIRCL_BOOTSTRAP_ONLY": "", "LIBSIRCL_TRANSPORT": "verbs", + "LD_LIBRARY_PATH": str(FAKE_DRIVER)}) + self.assertEqual(out.returncode, 0, out.stderr[-2000:]) + victim, still_link, written, leftovers = json.loads(out.stdout) + self.assertEqual(victim, "untouched") + self.assertTrue(still_link) + self.assertTrue(written) + self.assertEqual(leftovers, []) + def test_info_names_the_collectives(self): out = run("print(l.sirclGetInfo().decode())") self.assertEqual(out.returncode, 0, out.stderr) From 044a56bc45e8d626803aabcb3d5380da0c0134c3 Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Thu, 15 Oct 2026 09:08:25 -0500 Subject: [PATCH 36/65] Validate the setup timeout and keep idle connections off the root - LIBSIRCL_SETUP_TIMEOUT_MS outside 1 to 3,600,000 (or not an integer) refuses the all-gather with ncclInvalidArgument naming the variable, where it fell back to 600 s silently. The broker's own answer deadline keeps the default for such a value. - The root drops a connection that has not sent its whole join request within 1 s of being accepted, so idle connections cannot hold its 32 pending slots for the whole join window. - The broker's answer to a round gives up at once when the library unloads (it observes the root's stop flag). - A rank that reaches a root after its join window ended is told the root rejected the id or disconnected, and the message names the window (SIRCL_BOOTSTRAP_TIMEOUT_MS, counted from ncclGetUniqueId). README.md states where the window starts, the hello limit and the setup timeout's range. Compatibility: valid settings behave the same. Invalid LIBSIRCL_SETUP_TIMEOUT_MS values fail the first all-gather; clients that connect without promptly sending their hello are dropped. Validation: tests/test_bootstrap.py checks abc, 0, 3600001 and -5 for the timeout (all accepted before the change; refused after, naming the variable) and opens 32 idle connections to a root before a rank joins 1.5 s later (rejected before; admitted after). make check and make check-asan pass; the Markdown link check passes. Request item 36 (audit 05-F8, 02-F8, 15-F04). NVIDIA NCCL source was neither read nor adapted. Co-Authored-By: Claude Opus 5.5 --- spark_transport/libsircl/README.md | 6 ++- spark_transport/libsircl/src/bootstrap.c | 48 ++++++++++++++----- .../libsircl/tests/test_bootstrap.py | 37 ++++++++++++++ 3 files changed, 78 insertions(+), 13 deletions(-) diff --git a/spark_transport/libsircl/README.md b/spark_transport/libsircl/README.md index 64941f60..84a293e5 100644 --- a/spark_transport/libsircl/README.md +++ b/spark_transport/libsircl/README.md @@ -64,7 +64,11 @@ A communicator of W ranks (1 to 8) owns one SIRCL ring session: or `NCCL_SOCKET_IFNAME`); all-gather rounds through the root for setup, splits and teardown. The root runs in the process that created the id and lives until every rank's connection to it closes (each rank's communicator is destroyed or aborted) or, before every rank joined, until its join window ends - (`SIRCL_BOOTSTRAP_TIMEOUT_MS`, default 10 s). A process holds at most 64 roots at once: `ncclGetUniqueId` + (`SIRCL_BOOTSTRAP_TIMEOUT_MS`, default 10 s, counted from `ncclGetUniqueId`, not from the first rank's + join; a rank that arrives later is told the root rejected the id or disconnected, and the message names + the window). A connection that has not sent its join request within 1 s of being accepted is dropped. + Each all-gather round waits at most `LIBSIRCL_SETUP_TIMEOUT_MS` (1 ms to 1 h, default 600 s); another + value is refused, naming the variable. A process holds at most 64 roots at once: `ncclGetUniqueId` beyond that returns `ncclInvalidUsage`, naming the limit, until a communicator whose id it created is destroyed. - **Session engine** (`src/engine.c`): settings agreed across ranks, the pinned arena, device counters, diff --git a/spark_transport/libsircl/src/bootstrap.c b/spark_transport/libsircl/src/bootstrap.c index 64c0416e..8dc61aac 100644 --- a/spark_transport/libsircl/src/bootstrap.c +++ b/spark_transport/libsircl/src/bootstrap.c @@ -21,8 +21,10 @@ #include enum { OK = 0, SYSTEM = 2, ARGUMENT = 4, USAGE = 5, REMOTE = 6, TIMEOUT = 8 }; +/* HELLO_LIMIT_MS: a connection that has not sent its whole hello this long after the root accepted it is + * dropped, so idle connections cannot hold the root's MAX_PENDING slots until the join window ends. */ enum { ID_SIZE = 128, HELLO_SIZE = 136, MAX_RANKS = 8, MAX_PENDING = 32, - MAX_ROOTS = 64, FRAME_HEADER = 16, MAX_FRAME = 1 << 20 }; + MAX_ROOTS = 64, FRAME_HEADER = 16, MAX_FRAME = 1 << 20, HELLO_LIMIT_MS = 1000 }; enum { VERSION_OFF = 8, IP_OFF = 12, PORT_OFF = 16, NONCE_OFF = 20, HASH_OFF = 36, RESERVED_OFF = 68 }; static const unsigned char id_magic[8] = {'S','C','C','L','B','O','0','1'}; @@ -43,6 +45,7 @@ struct sccl_bootstrap { int fd; int nranks; int rank; uint32_t round; int failed struct peer { int fd; int rank; + int64_t accepted; /* CLOCK_MONOTONIC ms when the root accepted it */ size_t used; unsigned char hello[HELLO_SIZE]; unsigned char frame[FRAME_HEADER]; @@ -112,16 +115,14 @@ static int timeout_ms(int *out) { /* Deadline of one all-gather round (LIBSIRCL_SETUP_TIMEOUT_MS, 1 ms to 1 h; * default 600 s, the startup wait regime: ranks may lag while they allocate, - * register and connect). */ + * register and connect). -1 for a value that is not an integer in range. */ static int frame_deadline_ms(void) { const char *value = sccl_env("LIBSIRCL_SETUP_TIMEOUT_MS"); char *end; - if (value && *value) { - errno = 0; - long parsed = strtol(value, &end, 10); - if (!errno && !*end && parsed >= 1 && parsed <= 3600000) return (int)parsed; - } - return 600000; + if (!value || !*value) return 600000; + errno = 0; + long parsed = strtol(value, &end, 10); + return !errno && !*end && parsed >= 1 && parsed <= 3600000 ? (int)parsed : -1; } static int hex_digit(unsigned char c) { @@ -308,9 +309,11 @@ static int finish_round(struct root *r) { at += 4 + by_rank[k]->payload_bytes; } int result = 0; - int64_t deadline = now_ms() + frame_deadline_ms(); + int limit = frame_deadline_ms(); + int64_t deadline = now_ms() + (limit > 0 ? limit : 600000); for (int k = 0; k < r->nranks; ++k) { - if (send_all(by_rank[k]->fd, answer, at, deadline, NULL) != OK) result = -1; + /* Bounded by the round's deadline, and given up at once when the library unloads (`stopped`). */ + if (send_all(by_rank[k]->fd, answer, at, deadline, &r->stopped) != OK) result = -1; free(by_rank[k]->payload); by_rank[k]->payload = NULL; by_rank[k]->payload_bytes = 0; by_rank[k]->payload_used = 0; @@ -349,6 +352,7 @@ static void accept_connections(struct root *r) { r->peers[slot].fd = fd; r->peers[slot].rank = -1; r->peers[slot].used = 0; + r->peers[slot].accepted = now_ms(); } } @@ -372,6 +376,20 @@ static void *root_thread(void *arg) { if (ready && !r->joined) break; int delay = ready ? 100 : remaining_ms(r->deadline); if (!ready && !delay) break; + /* Pending connections without a whole hello within HELLO_LIMIT_MS are dropped; wake for the next. */ + if (!ready) { + int64_t now = now_ms(); + for (int i = 0; i < MAX_PENDING; ++i) { + struct peer *p = &r->peers[i]; + if (p->fd < 0 || p->rank >= 0) continue; + int64_t left = p->accepted + HELLO_LIMIT_MS - now; + if (left <= 0) { + drop_peer(r, i, -1); + continue; + } + if (left < delay) delay = (int)left; + } + } int rc = poll(pfds, (nfds_t)n, delay); if (rc < 0) { if (errno == EINTR) continue; break; } for (int j = 0; j < n; ++j) { @@ -730,7 +748,8 @@ int sccl_bootstrap_join_cancel(const unsigned char id[ID_SIZE], int nranks, int case TIMEOUT: return error_result(result, "bootstrap timed out waiting for every rank"); case USAGE: return error_result(result, "bootstrap rejected a duplicate rank or mismatched world size"); case ARGUMENT: return error_result(result, "bootstrap rejected invalid rank arguments"); - case REMOTE: return error_result(result, "bootstrap endpoint rejected the ID or disconnected"); + case REMOTE: return error_result(result, "bootstrap root rejected the ID or disconnected (its join window, " + "SIRCL_BOOTSTRAP_TIMEOUT_MS from ncclGetUniqueId, may have ended)"); default: return error_result(SYSTEM, "bootstrap socket or allocation failure"); } } @@ -763,7 +782,12 @@ int sccl_bootstrap_fd(const sccl_bootstrap *handle) { int sccl_bootstrap_allgather(sccl_bootstrap *handle, const void *data, uint32_t bytes, void **out, uint32_t *lengths, const atomic_int *cancelled) { - return sccl_bootstrap_allgather_within(handle, data, bytes, out, lengths, frame_deadline_ms(), cancelled); + int limit = frame_deadline_ms(); + if (limit < 0) { + if (out) *out = NULL; + return error_result(ARGUMENT, "LIBSIRCL_SETUP_TIMEOUT_MS must be an integer from 1 to 3600000"); + } + return sccl_bootstrap_allgather_within(handle, data, bytes, out, lengths, limit, cancelled); } static int allgather_locked(sccl_bootstrap *handle, const void *data, uint32_t bytes, void **out, diff --git a/spark_transport/libsircl/tests/test_bootstrap.py b/spark_transport/libsircl/tests/test_bootstrap.py index 4887ec6a..e686d58a 100644 --- a/spark_transport/libsircl/tests/test_bootstrap.py +++ b/spark_transport/libsircl/tests/test_bootstrap.py @@ -455,6 +455,43 @@ def test_every_descriptor_is_close_on_exec_from_its_creation(self): self.assertGreaterEqual(lib.test_created(), 4) self.assertEqual(lib.test_missed(), 0) + def test_setup_timeout_must_be_valid(self): + """An invalid LIBSIRCL_SETUP_TIMEOUT_MS refuses the all-gather, naming the variable, instead of + silently waiting the 600 s default.""" + handles = self.joined_group(1) + saved = os.environ.get("LIBSIRCL_SETUP_TIMEOUT_MS") + try: + for invalid in ("abc", "0", "3600001", "-5"): + with self.subTest(value=invalid): + os.environ["LIBSIRCL_SETUP_TIMEOUT_MS"] = invalid + result, _ = self.bootstrap.allgather(handles[0], b"x", 1) + self.assertEqual(result, 4) + self.assertIn(b"LIBSIRCL_SETUP_TIMEOUT_MS", self.bootstrap.lib.sccl_bootstrap_error()) + os.environ["LIBSIRCL_SETUP_TIMEOUT_MS"] = "5000" + self.assertEqual(self.bootstrap.allgather(handles[0], b"ok", 1), (0, [b"ok"])) + finally: + if saved is None: + os.environ.pop("LIBSIRCL_SETUP_TIMEOUT_MS", None) + else: + os.environ["LIBSIRCL_SETUP_TIMEOUT_MS"] = saved + self.bootstrap.close(handles[0]) + + def test_idle_connections_do_not_hold_the_roots_slots(self): + """Thirty-two connections that never send a hello fill the root's pending slots; the root drops them + after its hello limit (1 s), so a rank that joins later is admitted within the join window.""" + os.environ["SIRCL_BOOTSTRAP_TIMEOUT_MS"] = "8000" + value = self.bootstrap.unique_id() + port = struct.unpack("!H", value[16:18])[0] + idle = [socket.create_connection(("127.0.0.1", port), timeout=1) for _ in range(32)] + try: + time.sleep(1.5) + result, handle = self.bootstrap.join(value, 1, 0) + self.assertEqual(result, 0, self.bootstrap.lib.sccl_bootstrap_error()) + self.bootstrap.close(handle) + finally: + for connection in idle: + connection.close() + def test_allgather_rounds(self): world = 3 handles = self.joined_group(world) From c7b3767c8206d20dca7da5d7b0bf3b40a3ce8b47 Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Thu, 15 Oct 2026 09:09:49 -0500 Subject: [PATCH 37/65] Give the teardown rounds 5 s of slack beyond the wait limit Each teardown round of ncclCommFinalize and ncclCommDestroy waits the session's wait limit plus 5 s for every rank. A rank whose own last work ended at the wait limit (a flag wait that timed out) reaches the round just after its peers began waiting; with a bound equal to the limit the peers' rounds could expire first and report ncclRemoteError although the late rank arrived. README.md, STATUS.md, the comment of quiesce and the round's error text state the bound. The other half of the audit item, finalize holding the engine's lock across its stream wait, does not apply to this tree: the engine marks itself closing under the lock and synchronizes its context without it (sccl_engine_teardown_sync). Compatibility: a teardown whose peer never arrives returns 5 s later. Validation: the rounds run only on an engine of two or more ranks; the existing emulation case that exercises them is python tests/emulation/teardown_race.py --library build/libsircl.so \ --expect close-error --late-rank 1 --late-s 25 (not run here). make check and make check-asan pass. Request item 37 (audit 05-F6). NVIDIA NCCL source was neither read nor adapted. Co-Authored-By: Claude Opus 5.5 --- spark_transport/libsircl/README.md | 2 +- spark_transport/libsircl/STATUS.md | 2 +- spark_transport/libsircl/src/api.c | 10 +++++++--- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/spark_transport/libsircl/README.md b/spark_transport/libsircl/README.md index 84a293e5..c6a34330 100644 --- a/spark_transport/libsircl/README.md +++ b/spark_transport/libsircl/README.md @@ -169,7 +169,7 @@ wait for every rank. This rank's enqueued work completes; a first bootstrap roun completed, so every item and flag any kernel waits for has landed (a kernel completes once it has its inbound items, while its progress thread may still owe a forward or a last item to a peer); the progress thread then stops, and a second round proves that every rank's progress thread stopped before any queue -pair, registration or arena is freed. Each round is bounded by the session's wait limit; a rank that does +pair, registration or arena is freed. Each round is bounded by the session's wait limit plus 5 s; a rank that does not arrive in time makes the call return `ncclRemoteError` after the local teardown. The first round also carries each rank's health: a rank whose own work failed (a flag wait that timed out, a failed progress thread, an earlier asynchronous error) returns that error, and every other rank returns `ncclRemoteError` diff --git a/spark_transport/libsircl/STATUS.md b/spark_transport/libsircl/STATUS.md index b1465813..e6f1d369 100644 --- a/spark_transport/libsircl/STATUS.md +++ b/spark_transport/libsircl/STATUS.md @@ -83,7 +83,7 @@ The interface follows NVIDIA's public `nccl.h.in` v2.32.3-1 and the user guide o | Op selection from shared state only: a call's schedule, op kinds and count, pieces, blocks per role and link op words follow from its shape, datatype, root and the settings agreed at setup; a rank's buffer alignment, in-place or overlapping buffers, a discarded output (off the root) and its staging state choose only local copies through scratch or staging around the same ops. The staging buffer grows outside capture with every earlier buffer kept until destroy (no limit on the number of sizes); under CUDA graph capture a call that needs more staging than the communicator holds takes a graph allocation on the capturing stream (`cuMemAllocAsync`), freed on that stream when the call returns, so the graph owns it (README.md, "Op selection and staging") | implemented | Conditions: GPU emulation on the workstation; `library_rank.py`'s rank-local staging check on a fresh split child: the collectives of every selection point (one-shot, pieces and padded tiles; chain, ring and link ops; pair exchanges of roots 0 and 1; `ncclAlltoAll`; on two ranks torch's all-to-all pattern and one-way sends each way), with every rank's buffers aligned, then rank 1's unaligned, then rank 1's in place, then captured cold into one graph on a second fresh child with rank 1's unaligned and replayed twice; and the staging growth check, 19 all-gathers of doubling shards from 16 B to 4 MiB with rank 1 unaligned. Measurement (the runs under "Op selection under rank-local staging, and alternating flags-only directions" below): in every run each rank's receipt counts of launched ops (pair exchanges and their bytes, link ops by kind and by blocks per role, transport, chain and point-to-point ops) and of carried transfers (native link ops and items, one-shot ops and phases) equal its aligned run's, and the launched counts are the same on every rank; every output exact; the cold capture is taken on every rank, each of rank 1's staged calls as a graph allocation (16 on the pair default, 20 under the ring schedule on two ranks), and both replays are exact; rank 1 keeps 19 staging buffers under the chain and ring schedules. The same checks against snapshot ba5a337b's library, with a 15 s wait limit: under the pair default, rank 1 refused its cold capture of the staged calls with `ncclInvalidUsage` while rank 0 captured them, rank 0's replays returned wrong outputs at the wait limit and the child's destroy failed; under the ring schedule, rank 1 also refused the 18th and 19th all-gathers of the growth check (`ncclInternalError`, too many staging buffer sizes) while rank 0 launched them, and rank 0's session was poisoned at the wait limit. Conclusion: in these runs a rank's alignment, in-place buffers, role and staging state change no rank's ops | | Progress thread placement: `SIRCL_PROGRESS_CPU` pins it; otherwise `LIBSIRCL_CPU_POLICY=performance` (default) starts it on the fastest CPU class the creating thread may use (GB10: the Cortex-X925 cores), `none` leaves it to the scheduler; the application's threads keep their affinity; receipts name the thread's CPUs (`progress_cpus`) | implemented | build and CPU suites (the workstation's CPUs are of one class); on GB10s (Sparks 6-7, snapshot 0f94c72b, hardware evidence) the default policy started the thread on CPUs 5-9 and 15-19 and `SIRCL_PROGRESS_CPU=19` pinned it to CPU 19, as each receipt's `progress_cpus` records | | The chain and link areas in the arena after the control line, the link counters, piece counters and decision words, SIRCL's link settings (`SIRCL_LINK_*`, staggers) agreed at setup; the ring plan from `LIBSIRCL_RING_WINDOW`; forward windows of relayed lanes from `LIBSIRCL_FORWARD_WINDOWS`; `tools/site_routes.py` prints every rank's route map, chain order, forward windows and ring plan from SIRCL's route planner | implemented | CPU suites; setup-failure cases; `tools/site_routes.py` reproduces the pair's route maps and gives `path:0-3` a ring window of 393,216 bytes on position 3 | -| Teardown: `ncclCommDestroy` and `ncclCommFinalize` of a communicator of two or more ranks (also after `ncclCommRevoke`) wait for this rank's work, then a bootstrap round of every rank that also carries each rank's health, then stop the progress thread, then a second round, before any queue pair, registration or arena is freed; each round bounded by the wait limit; the close is terminal (new work refused, its result kept); a failed release of the transport keeps the memory allocated; `ncclCommAbort` does not wait | implemented | Conditions: GPU emulation on the workstation and on GB10s (snapshot 7d1c6f19, built in the serving image); `tests/emulation/teardown_race.py`: a reversed `ncclCommSplit` child, one ring all-reduce, `ncclCommDestroy` at once, repeated, one rank's writes delayed 5 ms so its predecessor finishes first; with `--expect close-error`, a rank 4 s late under a 2 s wait limit, or the stand-in's deregistrations failed (`SIRCL_EMU_FAIL_DEREG=1`). Measurement: four ranks and two ranks at 8 MiB per rank, 20 rounds each, every round exact with no async error and the parent healthy (GB10: four ranks, 5 runs of 20 rounds, 0 failed); without the rounds (snapshot 6fe2882f) the same check failed 7 of 10 rounds on four ranks on the workstation and 98 of 100 on a GB10 (each failure poisons the parent): a rank's kernel had completed while its progress thread still held an outbound item gated by a peer's credit, destroy stopped that thread and freed its queue pairs, the peer waited out the wait limit, and the peer's late credit write failed (transport retry counter exceeded; the destination queue pair was gone). In close-error mode (workstation and GB10) every round's first finalize or destroy returned the error, a second finalize the same, and a collective after the close was refused. The full emulation suite on a GB10 (snapshot 7d1c6f19) passed every section, the teardown check on four ranks and on two at 8 MiB included. On the fabric (ConnectX-7, four Sparks on a path and a cabled pair, 8 MiB per rank, no delayed rank): 100 rounds each without the rounds and 60 with them (snapshot 7d1c6f19), all exact and healthy, and 10 runs of the relayed-group bit-exact check under the ring schedules (RUNBOOK 4.1) on the path, 0 failed (Sparks 0-3 and 0-1, `ccl-pair/done-hw-teardown-7d1-20261008T194629Z`); the same 60 rounds per layout and 10 checks at snapshot ba5a337b on Sparks 4-7 and 4-5, 0 failed (`ccl-pair/hw-teardown-ba5-47-20261008T225027Z`); the stranded item did not occur at natural timing there; on the cabled pair at positions 6-7 (snapshot 0f94c72b, the rounds of this tree, 2026-10-09): 6 runs of 20 rounds at 8 MiB, 3 with fail-stop off and 3 on, 0 failed, the parent exact and healthy. Result: round 1 proves that every item and flag a kernel waits for has landed; round 2 that no rank posts toward queue pairs or regions about to be freed. Writes no kernel waits for (credits) may still be in flight; each rank destroys its queue pairs before its registrations and memory, so they land in registered memory or are dropped. Limits: the rounds are bounded, but this rank's own wait for its work, the progress thread's join and the verbs destroys are not; in emulation the stand-in resolves a write under its registry lock and applies it after, so a write resolved before a deregistration still lands in the old segment, and emulation does not test a NIC's revocation of a deregistered region. Callers: torch 2.10's `destroy_process_group` shuts its groups down in one order on every rank (finalize, then destroy); vLLM 0.19.1's pynccl `destroy` calls `ncclCommDestroy`; a caller that tears down with `ncclCommAbort` runs no rounds | +| Teardown: `ncclCommDestroy` and `ncclCommFinalize` of a communicator of two or more ranks (also after `ncclCommRevoke`) wait for this rank's work, then a bootstrap round of every rank that also carries each rank's health, then stop the progress thread, then a second round, before any queue pair, registration or arena is freed; each round bounded by the wait limit plus 5 s; the close is terminal (new work refused, its result kept); a failed release of the transport keeps the memory allocated; `ncclCommAbort` does not wait | implemented | Conditions: GPU emulation on the workstation and on GB10s (snapshot 7d1c6f19, built in the serving image); `tests/emulation/teardown_race.py`: a reversed `ncclCommSplit` child, one ring all-reduce, `ncclCommDestroy` at once, repeated, one rank's writes delayed 5 ms so its predecessor finishes first; with `--expect close-error`, a rank 4 s late under a 2 s wait limit, or the stand-in's deregistrations failed (`SIRCL_EMU_FAIL_DEREG=1`). Measurement: four ranks and two ranks at 8 MiB per rank, 20 rounds each, every round exact with no async error and the parent healthy (GB10: four ranks, 5 runs of 20 rounds, 0 failed); without the rounds (snapshot 6fe2882f) the same check failed 7 of 10 rounds on four ranks on the workstation and 98 of 100 on a GB10 (each failure poisons the parent): a rank's kernel had completed while its progress thread still held an outbound item gated by a peer's credit, destroy stopped that thread and freed its queue pairs, the peer waited out the wait limit, and the peer's late credit write failed (transport retry counter exceeded; the destination queue pair was gone). In close-error mode (workstation and GB10) every round's first finalize or destroy returned the error, a second finalize the same, and a collective after the close was refused. The full emulation suite on a GB10 (snapshot 7d1c6f19) passed every section, the teardown check on four ranks and on two at 8 MiB included. On the fabric (ConnectX-7, four Sparks on a path and a cabled pair, 8 MiB per rank, no delayed rank): 100 rounds each without the rounds and 60 with them (snapshot 7d1c6f19), all exact and healthy, and 10 runs of the relayed-group bit-exact check under the ring schedules (RUNBOOK 4.1) on the path, 0 failed (Sparks 0-3 and 0-1, `ccl-pair/done-hw-teardown-7d1-20261008T194629Z`); the same 60 rounds per layout and 10 checks at snapshot ba5a337b on Sparks 4-7 and 4-5, 0 failed (`ccl-pair/hw-teardown-ba5-47-20261008T225027Z`); the stranded item did not occur at natural timing there; on the cabled pair at positions 6-7 (snapshot 0f94c72b, the rounds of this tree, 2026-10-09): 6 runs of 20 rounds at 8 MiB, 3 with fail-stop off and 3 on, 0 failed, the parent exact and healthy. Result: round 1 proves that every item and flag a kernel waits for has landed; round 2 that no rank posts toward queue pairs or regions about to be freed. Writes no kernel waits for (credits) may still be in flight; each rank destroys its queue pairs before its registrations and memory, so they land in registered memory or are dropped. Limits: the rounds are bounded, but this rank's own wait for its work, the progress thread's join and the verbs destroys are not; in emulation the stand-in resolves a write under its registry lock and applies it after, so a write resolved before a deregistration still lands in the old segment, and emulation does not test a NIC's revocation of a deregistered region. Callers: torch 2.10's `destroy_process_group` shuts its groups down in one order on every rank (finalize, then destroy); vLLM 0.19.1's pynccl `destroy` calls `ncclCommDestroy`; a caller that tears down with `ncclCommAbort` runs no rounds | | Destroying or finalizing communicators in a different order on different ranks | unsupported | each teardown round waits for every rank of its communicator, so crossed orders wait out the wait limit on each round (20 s serving, 600 s startup) and return `ncclRemoteError`; they do not wait longer | | Process exit without `ncclCommDestroy`, `ncclCommFinalize` or `ncclCommAbort` while a peer's collective on the communicator still runs | unsupported | at library unload only the native threads stop; a peer may wait out its wait limit. Calls still running on other threads are cancelled, and the unload waits up to 1 s for them to leave before it frees their communicators; a communicator still in use after that stays allocated (CPU: `tests/test_lifecycle.py`, exit and `dlclose` with a split in progress, also under `make check-asan`) | | `ncclCommDestroy` while replays of captured CUDA graphs of the communicator's collectives still run | unsupported (caller contract) | teardown waits for the engine's last eager launch (a launch under capture records no stream), not for graph replays; a communicator used only under capture has nothing to wait for | diff --git a/spark_transport/libsircl/src/api.c b/spark_transport/libsircl/src/api.c index 4dfd9418..2ca72b04 100644 --- a/spark_transport/libsircl/src/api.c +++ b/spark_transport/libsircl/src/api.c @@ -965,7 +965,7 @@ static ncclResult_t transition(ncclComm_t handle, int next) { * rank's work completed, so every item and flag any rank's kernels wait for has landed; this rank's progress * thread stops and posts nothing more; round 2 proves every progress thread stopped, so nothing is posted * toward this rank's queue pairs or regions any more and destroy may free them. Each round is bounded by the - * session's wait limit. The close is terminal: from its start the communicator refuses new work, and its + * session's wait limit plus 5 s. The close is terminal: from its start the communicator refuses new work, and its * result (this rank's own error first, else ncclRemoteError naming the first rank that reported one, or a * round that not every rank reached) is kept, becomes the communicator's error when it failed, and is what a * repeated ncclCommFinalize or the destroy returns. */ @@ -994,7 +994,11 @@ static ncclResult_t quiesce_locked(struct ncclComm *comm, const char *call) { snprintf(message, sizeof message, "%s: waiting for this rank's enqueued work failed", call); } if (result == ncclSuccess) result = sccl_engine_async_error(comm->engine, message, sizeof message); - int limit_ms = sccl_engine_wait_limit_ms(comm->engine), arrived = 0; + /* The wait limit plus TEARDOWN_SLACK_MS: a rank whose own work ended at its wait limit (a timed-out flag + * wait) reaches the round just after its peers began waiting, not before. */ + enum { TEARDOWN_SLACK_MS = 5000 }; + int wait_ms = sccl_engine_wait_limit_ms(comm->engine), arrived = 0; + int limit_ms = wait_ms > INT_MAX - TEARDOWN_SLACK_MS ? INT_MAX : wait_ms + TEARDOWN_SLACK_MS; for (int round = 1; round <= 2; ++round) { if (round == 2) sccl_engine_teardown_stop(comm->engine); if (round == 2 && !arrived) break; @@ -1020,7 +1024,7 @@ static ncclResult_t quiesce_locked(struct ncclComm *comm, const char *call) { if (result == ncclSuccess) { result = ncclRemoteError; snprintf(message, sizeof message, "%s: teardown round %d: not every rank arrived within the wait limit " - "(%d ms): %s", call, round, limit_ms, sccl_bootstrap_error()); + "plus 5 s (%d ms): %s", call, round, limit_ms, sccl_bootstrap_error()); } if (round == 1) sccl_engine_teardown_stop(comm->engine); break; From 7343c8041531bb1b27373c317c2e3a12a24e9a97 Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Thu, 15 Oct 2026 09:12:26 -0500 Subject: [PATCH 38/65] Parse engine settings strictly and name the variable in every refusal - Integer settings (env_u64) take digits only: a sign or leading space, which strtoull accepted ("-1" wrapped to a huge value), is refused. - SIRCL_TRAFFIC_CLASS and NCCL_IB_TC are read with the other settings and must be integers from 0 to 255; an invalid value became traffic class 0 silently. - LIBSIRCL_RECEIPT_INTERVAL_S is read with the settings and must be a number of seconds from 0 to 86,400; an unchecked strtod result was cast to uint64_t. - The list variables parsed from 1,024-byte copies (SIRCL_PEER_ROUTES, LIBSIRCL_FORWARD_WINDOWS, LIBSIRCL_P2P_WINDOWS, LIBSIRCL_RELAYED_LANES, LIBSIRCL_CHAIN_ORDER, SIRCL_GID_INDEX, SIRCL_PROGRESS_CPU, LIBSIRCL_LINK_BLOCKS_CYCLE) are refused when longer, instead of being cut. - The forward and point-to-point window errors say which value is not a byte count or which entry has too many lanes, instead of "needs 1 or 2 byte counts" for both. - A peer's error text in the setup verdict is printed with %.399s, so a record without a terminating zero is never read past its 400 bytes. Compatibility: valid settings behave the same; the refused forms above fail communicator creation, naming the variable. Validation: tests/test_engine_cpu.py creates one-rank communicators on the stand-in driver with LIBSIRCL_LANE_CHECK_MS=+5000, SIRCL_ONESHOT_MAX_BYTES=" 4096", SIRCL_TRAFFIC_CLASS=abc, NCCL_IB_TC=300, LIBSIRCL_RECEIPT_INTERVAL_S=soon and a 1,119-character LIBSIRCL_FORWARD_WINDOWS: all six were accepted before the change and are refused after it, naming the variable; a traffic class of 106 and an interval of 2.5 s are accepted. make check and make check-asan pass. Request item 38 (audit 15-F06, 15-F07, 15-F03, 09-5). NVIDIA NCCL source was neither read nor adapted. Co-Authored-By: Claude Opus 5.5 --- spark_transport/libsircl/src/engine.c | 71 +++++++++++++++---- .../libsircl/tests/test_engine_cpu.py | 27 +++++++ 2 files changed, 83 insertions(+), 15 deletions(-) diff --git a/spark_transport/libsircl/src/engine.c b/spark_transport/libsircl/src/engine.c index 8143252f..87fbf4ff 100644 --- a/spark_transport/libsircl/src/engine.c +++ b/spark_transport/libsircl/src/engine.c @@ -446,6 +446,7 @@ struct sccl_engine { * when it had stopped (CLOCK_REALTIME nanoseconds). */ int stopped, closing; int lane_check_ms; /* LIBSIRCL_LANE_CHECK_MS: the lane check's bound */ + int traffic_class; /* SIRCL_TRAFFIC_CLASS or NCCL_IB_TC */ int kp_held; /* this engine holds a reference on its context's kernel-pack entry (sccl_kp_load) */ uint64_t stop_ns, stopped_ns; int have_last; @@ -587,8 +588,9 @@ static int env_u64(const char *name, uint64_t fallback, uint64_t lo, uint64_t hi } char *end; errno = 0; - unsigned long long value = strtoull(text, &end, 10); - if (errno || *end || value < lo || value > hi) { + /* Digits only: strtoull would also take a sign ("-1" wraps to a huge value) and leading spaces. */ + unsigned long long value = text[0] >= '0' && text[0] <= '9' ? strtoull(text, &end, 10) : (end = (char *)text, 0); + if (errno || *end || end == text || value < lo || value > hi) { put_error(err, len, "%s=%s must be an integer from %" PRIu64 " to %" PRIu64, name, text, lo, hi); return -1; } @@ -905,6 +907,7 @@ static int p2p_settings(sccl_engine *e, shared_settings *s, char *err, size_t le return 0; } +static int traffic_class(int *out, char *err, size_t len); static int read_settings(sccl_engine *e, int emulation, char *err, size_t len) { shared_settings *s = &e->shared; uint64_t v; @@ -989,6 +992,29 @@ static int read_settings(sccl_engine *e, int emulation, char *err, size_t len) { /* Read with the other settings, so that a bad value fails the first verdict on every rank. */ if (env_u64("LIBSIRCL_LANE_CHECK_MS", 10000, 1, 600000, &v, err, len)) return -1; e->lane_check_ms = (int)v; + if (traffic_class(&e->traffic_class, err, len)) return -1; + /* The list variables are parsed from 1,024-byte copies: a longer value is refused, not cut. */ + static const char *const lists[] = {"SIRCL_PEER_ROUTES", "LIBSIRCL_FORWARD_WINDOWS", "LIBSIRCL_P2P_WINDOWS", + "LIBSIRCL_RELAYED_LANES", "LIBSIRCL_CHAIN_ORDER", "SIRCL_GID_INDEX", + "SIRCL_PROGRESS_CPU", "LIBSIRCL_LINK_BLOCKS_CYCLE"}; + for (size_t i = 0; i < sizeof lists / sizeof lists[0]; ++i) { + const char *text = sccl_env(lists[i]); + if (text && strlen(text) >= 1024) { + put_error(err, len, "%s is longer than 1023 characters", lists[i]); + return -1; + } + } + const char *interval = sccl_env("LIBSIRCL_RECEIPT_INTERVAL_S"); + double seconds = 10.0; + if (interval && *interval) { + char *end; + seconds = strtod(interval, &end); + if (*end || !(seconds >= 0 && seconds <= 86400)) { + put_error(err, len, "LIBSIRCL_RECEIPT_INTERVAL_S=%s must be a number of seconds from 0 to 86400", interval); + return -1; + } + } + e->receipt_interval_ns = (uint64_t)(seconds * 1e9); if (env_u64("LIBSIRCL_RING_REDUCE_PASSES", 2, 1, 2, &v, err, len)) return -1; const char *one_pass = sccl_env("LIBSIRCL_RING_REDUCE_ONE_PASS_RESEARCH"); if (v == 1 && (!one_pass || strcmp(one_pass, "1"))) { @@ -1349,8 +1375,13 @@ static int forward_windows(sccl_engine *e, char *err, size_t len) { value = strtok_r(NULL, "/", &save2), ++lane) { char *stop; unsigned long bytes = strtoul(value, &stop, 10); - if (*stop || lane >= MAX_LANES || bytes > 0xFFFFFFFFul) { - put_error(err, len, "LIBSIRCL_FORWARD_WINDOWS entry '%s' needs 1 or 2 byte counts", entry); + if (*stop || !*value || bytes > 0xFFFFFFFFul) { + put_error(err, len, "LIBSIRCL_FORWARD_WINDOWS: '%s' toward position %ld is not a byte count", value, position); + return -1; + } + if (lane >= MAX_LANES) { + put_error(err, len, "LIBSIRCL_FORWARD_WINDOWS: the entry for position %ld has more than %d lanes", position, + MAX_LANES); return -1; } if (bytes && (bytes < chunk || bytes / chunk > FORWARD_MAX_CHUNKS)) { @@ -1428,8 +1459,13 @@ static int p2p_windows(sccl_engine *e, char *err, size_t len) { value = strtok_r(NULL, "/", &save2), ++lane) { char *stop; unsigned long bytes = strtoul(value, &stop, 10); - if (*stop || lane >= MAX_LANES || bytes > 0xFFFFFFFFul) { - put_error(err, len, "LIBSIRCL_P2P_WINDOWS entry for position %ld needs 1 or 2 byte counts", position); + if (*stop || !*value || bytes > 0xFFFFFFFFul) { + put_error(err, len, "LIBSIRCL_P2P_WINDOWS: '%s' toward position %ld is not a byte count", value, position); + return -1; + } + if (lane >= MAX_LANES) { + put_error(err, len, "LIBSIRCL_P2P_WINDOWS: the entry for position %ld has more than %d lanes", position, + MAX_LANES); return -1; } if (bytes && (bytes % PACK || bytes < chunk || bytes / chunk > P2P_MAX_WINDOW_CHUNKS)) { @@ -1619,13 +1655,18 @@ static int gid_indices(route_table *t, char *err, size_t len) { return 0; } -static int traffic_class(void) { +/* The IB traffic class of the queue pairs: SIRCL_TRAFFIC_CLASS, else NCCL_IB_TC, else 0; a value that is not + * an integer from 0 to 255 is refused, naming the variable. */ +static int traffic_class(int *out, char *err, size_t len) { const char *names[] = {"SIRCL_TRAFFIC_CLASS", "NCCL_IB_TC"}; + *out = 0; for (int i = 0; i < 2; ++i) { const char *text = sccl_env(names[i]); if (text && *text) { - long value = strtol(text, NULL, 10); - return value >= 0 && value <= 255 ? (int)value : 0; + uint64_t value; + if (env_u64(names[i], 0, 0, 255, &value, err, len)) return -1; + *out = (int)value; + return 0; } } return 0; @@ -1918,7 +1959,7 @@ static int p2p_open(sccl_engine *e, const route_table *routes, char *err, size_t } char native[512] = {0}; e->p2p = e->p2p_transport->create(e->world, e->rank, names, routes->n_devices, lane_devices, e->lanes, - routes->gid_index, traffic_class(), channels, e->p2p_host, e->p2p_bytes, + routes->gid_index, e->traffic_class, channels, e->p2p_host, e->p2p_bytes, (int)s->p2p_slots, s->p2p_slot_bytes, native, sizeof native); if (!e->p2p) { put_error(err, len, "point-to-point native setup: %s", native); @@ -2330,7 +2371,7 @@ int sccl_engine_create(sccl_bootstrap *bootstrap, int nranks, int rank, int posi for (int l = 0; l < routes.lanes; ++l) lane_devices[p * routes.lanes + l] = routes.lane_device[p][l]; char proxy_error[512] = {0}; e->proxy = e->transport->create(nranks, rank, names, routes.n_devices, lane_devices, routes.lanes, - routes.gid_index, traffic_class(), e->host, e->region_bytes, + routes.gid_index, e->traffic_class, e->host, e->region_bytes, e->shared.slot_bytes, proxy_error, sizeof proxy_error); if (!e->proxy) { put_error(local, sizeof local, "native setup: %s", proxy_error); @@ -2393,7 +2434,9 @@ int sccl_engine_create(sccl_bootstrap *bootstrap, int nranks, int rank, int posi const char *why = records[r].error[0] ? records[r].error : NULL; const char *field = why || any_error ? NULL : settings_difference(&records[r].settings, &records[0].settings); if (!why && !field) continue; - used += (size_t)snprintf(text + used, used < sizeof text ? sizeof text - used : 0, "%srank %d: %s%s", + /* A peer's error field is a fixed 400-byte array from the wire: printed at most 399 bytes, so a + * record without a terminating zero is never read past its end. */ + used += (size_t)snprintf(text + used, used < sizeof text ? sizeof text - used : 0, "%srank %d: %.399s%s", used ? "; " : "", r, why ? why : field, why ? "" : " differs from rank 0"); if (used >= sizeof text) used = sizeof text - 1; } @@ -4617,9 +4660,7 @@ static void start_receipts(sccl_engine *e) { e->receipt_id = atomic_fetch_add(&communicators, 1) + 1; const char *prefix = sccl_env("LIBSIRCL_RECEIPT"); if (!prefix || !*prefix) return; - const char *text = sccl_env("LIBSIRCL_RECEIPT_INTERVAL_S"); - double seconds = text && *text ? strtod(text, NULL) : 10.0; - e->receipt_interval_ns = seconds > 0 ? (uint64_t)(seconds * 1e9) : 0; + /* e->receipt_interval_ns: LIBSIRCL_RECEIPT_INTERVAL_S, validated by read_settings (0: no refresh). */ e->receipt_next_ns = monotonic_ns() + e->receipt_interval_ns; write_receipt(e); } diff --git a/spark_transport/libsircl/tests/test_engine_cpu.py b/spark_transport/libsircl/tests/test_engine_cpu.py index ddf48224..4e63144b 100644 --- a/spark_transport/libsircl/tests/test_engine_cpu.py +++ b/spark_transport/libsircl/tests/test_engine_cpu.py @@ -275,6 +275,33 @@ def test_receipts_never_follow_a_planted_link(self): self.assertTrue(written) self.assertEqual(leftovers, []) + def test_settings_are_parsed_strictly(self): + """Settings that were taken loosely are refused, naming the variable: a signed integer, a traffic class + that is not 0-255, a receipt interval that is not a number, and a list variable longer than the + 1,024-byte buffer it is parsed from.""" + script = '''u=U(); assert l.ncclGetUniqueId(C.byref(u))==0 +c=C.c_void_p(); r=l.ncclCommInitRank(C.byref(c),1,u,0) +print(json.dumps([r, l.ncclGetLastError(None).decode()])) +if c.value: l.ncclCommDestroy(c) +''' + base = {"LIBSIRCL_BOOTSTRAP_ONLY": "", "LIBSIRCL_TRANSPORT": "verbs", "LD_LIBRARY_PATH": str(FAKE_DRIVER)} + cases = [({"LIBSIRCL_LANE_CHECK_MS": "+5000"}, "LIBSIRCL_LANE_CHECK_MS=+5000 must be an integer"), + ({"SIRCL_ONESHOT_MAX_BYTES": " 4096"}, "SIRCL_ONESHOT_MAX_BYTES= 4096 must be an integer"), + ({"SIRCL_TRAFFIC_CLASS": "abc"}, "SIRCL_TRAFFIC_CLASS=abc must be an integer from 0 to 255"), + ({"NCCL_IB_TC": "300"}, "NCCL_IB_TC=300 must be an integer from 0 to 255"), + ({"LIBSIRCL_RECEIPT_INTERVAL_S": "soon"}, "LIBSIRCL_RECEIPT_INTERVAL_S=soon must be a number"), + ({"LIBSIRCL_FORWARD_WINDOWS": ",".join(["1=32768/32768"] * 80)}, + "LIBSIRCL_FORWARD_WINDOWS is longer than 1023 characters")] + for extra, text in cases: + with self.subTest(**{k: v[:20] for k, v in extra.items()}): + out = run(script, {**base, **extra}) + self.assertEqual(out.returncode, 0, out.stderr) + result, message = json.loads(out.stdout) + self.assertNotEqual(result, 0) + self.assertIn(text, message) + out = run(script, {**base, "SIRCL_TRAFFIC_CLASS": "106", "LIBSIRCL_RECEIPT_INTERVAL_S": "2.5"}) + self.assertEqual(json.loads(out.stdout)[0], 0, out.stdout) + def test_info_names_the_collectives(self): out = run("print(l.sirclGetInfo().decode())") self.assertEqual(out.returncode, 0, out.stderr) From 3776bf28d2a7d2834a0b0da5f7595cef9bbc5ca8 Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Thu, 15 Oct 2026 09:13:38 -0500 Subject: [PATCH 39/65] Make the vLLM plugin's binding check and file identity strict - When vLLM is importable but its PyNccl wrapper is not, the list of functions PyNccl binds is unknown; pynccl_functions raises LibsirclSelectionError instead of returning None, which made the binding check pass with nothing to check. Outside vLLM it still returns None and only the library's identity is checked. - selected() refuses a library path that is not its own real path (a symbolic link in any directory of it, not only the last component, or an unnormalized path), hashes the open file and records its device and inode. register() requires the path to still name that file (same device and inode) right before it sets VLLM_NCCL_SO_PATH, so a file replaced between the hash and vLLM's load is refused. The plugin test that asserted `pynccl_functions() is None or a list` (true for every result) checks each case instead. Compatibility: deployments that name the library by its real path and leave it in place are unaffected. Validation: test_sparkring_libsircl.py adds an importable vLLM stub without its PyNccl wrapper (refused), a library replaced after its check (refused), a path through a linked directory (refused), and the outside case with the import failing and no vLLM (identity only). All four failed before the change; the suite passes in WSL (17 passed) and on Windows (15 passed, 2 skipped where symbolic links are unavailable). Request item 39 (audit 13-F2, 13-F8). NVIDIA NCCL source was neither read nor adapted. Co-Authored-By: Claude Opus 5.5 --- .../vllm/libsircl/sparkring_libsircl.py | 42 +++++++++++++---- .../vllm/libsircl/test_sparkring_libsircl.py | 46 +++++++++++++++++-- 2 files changed, 76 insertions(+), 12 deletions(-) diff --git a/integrations/vllm/libsircl/sparkring_libsircl.py b/integrations/vllm/libsircl/sparkring_libsircl.py index 777eeb10..dde22e36 100644 --- a/integrations/vllm/libsircl/sparkring_libsircl.py +++ b/integrations/vllm/libsircl/sparkring_libsircl.py @@ -29,6 +29,7 @@ """ import ctypes import hashlib +import importlib.util import json import os from pathlib import Path @@ -39,6 +40,7 @@ NAME = "libsircl" ENTRY_POINT = ("vllm.general_plugins", NAME, "sparkring_libsircl:register") _checked = {} +_identity = {} # How PyNccl loads its library (vllm/distributed/device_communicators/pynccl_wrapper.py). LOADER = ctypes.CDLL @@ -56,22 +58,43 @@ def selected(environ=None): library = Path(path) if not library.is_absolute() or library.is_symlink() or not library.is_file(): raise LibsirclSelectionError(f"{LIBRARY}={path} is not an absolute path to a regular file") - stat = library.stat() - key = (path, digest, stat.st_ino, stat.st_size, stat.st_mtime_ns) - if key not in _checked: - with library.open("rb") as stream: + if os.path.realpath(path) != path: + raise LibsirclSelectionError(f"{LIBRARY}={path} is not its own real path ({os.path.realpath(path)}): a " + "directory of it is a symbolic link, or the path is not normalized") + # The file hashed is the open file, and its device and inode identify it for identical(). + with library.open("rb") as stream: + stat = os.fstat(stream.fileno()) + key = (path, digest, stat.st_dev, stat.st_ino, stat.st_size, stat.st_mtime_ns) + if key not in _checked: found = hashlib.file_digest(stream, "sha256").hexdigest() - if found != digest: - raise LibsirclSelectionError(f"{path} has SHA-256 {found}, not {DIGEST}={digest}") - _checked[key] = True + if found != digest: + raise LibsirclSelectionError(f"{path} has SHA-256 {found}, not {DIGEST}={digest}") + _checked[key] = (stat.st_dev, stat.st_ino) + _identity[path] = _checked[key] return path +def identical(path): + """Require ``path`` to still name the file selected() hashed (same device and inode).""" + stat = os.stat(path) + if (stat.st_dev, stat.st_ino) != _identity.get(path): + raise LibsirclSelectionError(f"{path} was replaced after its SHA-256 was checked") + + def pynccl_functions(): - """The functions vLLM's PyNccl binds (``NCCLLibrary.exported_functions``), or None outside vLLM.""" + """The functions vLLM's PyNccl binds (``NCCLLibrary.exported_functions``), or None outside vLLM. Inside a + process where vLLM is importable but its PyNccl wrapper is not, the functions cannot be checked: + LibsirclSelectionError, not a check that passes with nothing to check.""" try: from vllm.distributed.device_communicators.pynccl_wrapper import NCCLLibrary - except ImportError: + except ImportError as error: + try: + importable = importlib.util.find_spec("vllm") is not None + except (ImportError, ValueError): + importable = "vllm" in __import__("sys").modules + if importable: + raise LibsirclSelectionError(f"vLLM is importable but its PyNccl wrapper is not ({error}), so the " + "functions PyNccl binds cannot be checked") from None return None return [function.name for function in NCCLLibrary.exported_functions] @@ -111,6 +134,7 @@ def register(): """Point vLLM's PyNccl at the checked libsircl library; vLLM may call this more than once.""" path = selected() check_binding(path) + identical(path) os.environ[TARGET] = path found = resolved_target() if found is not None and found != path: diff --git a/integrations/vllm/libsircl/test_sparkring_libsircl.py b/integrations/vllm/libsircl/test_sparkring_libsircl.py index 3b7b6eb9..275523b2 100644 --- a/integrations/vllm/libsircl/test_sparkring_libsircl.py +++ b/integrations/vllm/libsircl/test_sparkring_libsircl.py @@ -84,12 +84,52 @@ def load(path): plugin.check_binding("/opt/sparkring/libsircl/lib/libsircl.so.0.6.0", load=load, functions=lambda: functions) -def test_outside_vllm_only_the_identity_is_checked(): - assert plugin.pynccl_functions() is None or isinstance(plugin.pynccl_functions(), list) - info = plugin.check_binding("/lib", load=lambda path: Library(functions=()), functions=lambda: None) +WRAPPER = "vllm.distributed.device_communicators.pynccl_wrapper" + + +def test_outside_vllm_only_the_identity_is_checked(monkeypatch): + monkeypatch.delitem(sys.modules, "vllm", raising=False) + monkeypatch.setitem(sys.modules, WRAPPER, None) + monkeypatch.setattr(plugin.importlib.util, "find_spec", lambda name: None) + assert plugin.pynccl_functions() is None + info = plugin.check_binding("/lib", load=lambda path: Library(functions=()), functions=plugin.pynccl_functions) assert info["library"] == "libsircl" +def test_an_importable_vllm_whose_pynccl_wrapper_fails_to_import_is_refused(monkeypatch): + # The functions PyNccl binds cannot be listed, so the binding check must not pass with nothing to check. + monkeypatch.setitem(sys.modules, "vllm", types.ModuleType("vllm")) + monkeypatch.setitem(sys.modules, WRAPPER, None) + with pytest.raises(plugin.LibsirclSelectionError, match="PyNccl wrapper is not"): + plugin.pynccl_functions() + with pytest.raises(plugin.LibsirclSelectionError, match="PyNccl wrapper is not"): + plugin.check_binding("/lib", load=lambda path: Library(), functions=plugin.pynccl_functions) + + +def test_a_library_replaced_after_its_check_is_refused(tmp_path): + path, digest = library(tmp_path) + assert plugin.selected({plugin.LIBRARY: str(path), plugin.DIGEST: digest}) == str(path) + plugin.identical(str(path)) + other = tmp_path / "other.so" + other.write_bytes(b"ELF something else") + os.replace(other, path) + with pytest.raises(plugin.LibsirclSelectionError, match="replaced after its SHA-256 was checked"): + plugin.identical(str(path)) + + +def test_a_path_through_a_linked_directory_is_not_the_library(tmp_path): + real = tmp_path / "real" + real.mkdir() + path, digest = library(real) + linked = tmp_path / "linked" + try: + linked.symlink_to(real, target_is_directory=True) + except OSError: + pytest.skip("this host cannot create symbolic links") + with pytest.raises(plugin.LibsirclSelectionError, match="not its own real path"): + plugin.selected({plugin.LIBRARY: str(linked / path.name), plugin.DIGEST: digest}) + + def test_a_symbolic_link_is_not_the_library(tmp_path): path, digest = library(tmp_path) link = tmp_path / "link.so" From 644a5615d3fbfe89fa725126810002ab70c683ae Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Thu, 15 Oct 2026 09:15:14 -0500 Subject: [PATCH 40/65] Report a split's failed exchange with the engine's result codes ncclCommSplit reported every failed all-gather as ncclSystemError. It reports them as the engine's setup exchanges do: a peer that does not arrive in time or a lost broker is ncclRemoteError, a cancellation (an abort on another thread) ncclInvalidUsage, an invalid setting such as LIBSIRCL_SETUP_TIMEOUT_MS ncclInvalidArgument, and only a local socket or allocation failure ncclSystemError. Compatibility: a split whose peers time out returns ncclRemoteError instead of ncclSystemError; the message is unchanged. Validation: tests/test_lifecycle.py splits rank 0 of a two-rank communicator alone with LIBSIRCL_SETUP_TIMEOUT_MS=300: ncclSystemError (2) before the change, ncclRemoteError (6) after, naming the timeout. make check and make check-asan pass. Request item 40, fifth check (audit 03-F6). NVIDIA NCCL source was neither read nor adapted. Co-Authored-By: Claude Opus 5.5 --- spark_transport/libsircl/src/api.c | 22 +++++++++++++++---- .../libsircl/tests/test_lifecycle.py | 19 ++++++++++++++++ 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/spark_transport/libsircl/src/api.c b/spark_transport/libsircl/src/api.c index 2ca72b04..bbf2ed1a 100644 --- a/spark_transport/libsircl/src/api.c +++ b/spark_transport/libsircl/src/api.c @@ -385,6 +385,17 @@ ncclResult_t ncclCommInitRank(ncclComm_t *out, int nranks, ncclUniqueId id, int return ncclCommInitRankConfig(out, nranks, id, rank, NULL); } +/* The result of a split whose all-gather failed, as the engine's setup exchanges report theirs: a peer that + * did not arrive in time or a lost broker is ncclRemoteError; a cancellation (abort) ncclInvalidUsage; an + * invalid setting ncclInvalidArgument; a local socket or allocation failure ncclSystemError. */ +static ncclResult_t split_exchange_result(int bootstrap_result) { + switch (bootstrap_result) { + case ncclInvalidUsage: return ncclInvalidUsage; + case ncclInvalidArgument: return ncclInvalidArgument; + case ncclSystemError: return ncclSystemError; + default: return ncclRemoteError; + } +} /* A failed split: its reason is recorded before the parent's reference is dropped, so that a library unload * waiting for the reference finds the call already past its library work. */ static ncclResult_t split_failed(struct ncclComm *parent, ncclResult_t result, const char *message) { @@ -411,10 +422,11 @@ ncclResult_t ncclCommSplit(ncclComm_t handle, int color, int key, ncclComm_t *ne int32_t mine[2] = {color, key}; void *all = NULL; unsigned lengths[64]; - if (sccl_bootstrap_allgather(parent->bootstrap, mine, sizeof mine, &all, lengths, &parent->cancelled)) { + int gathered = sccl_bootstrap_allgather(parent->bootstrap, mine, sizeof mine, &all, lengths, &parent->cancelled); + if (gathered) { char message[700]; snprintf(message, sizeof message, "ncclCommSplit: exchanging colors: %s", sccl_bootstrap_error()); - return split_failed(parent, ncclSystemError, message); + return split_failed(parent, split_exchange_result(gathered), message); } for (int r = 0; r < parent->nranks; ++r) { if (lengths[r] != sizeof mine) { @@ -453,10 +465,12 @@ ncclResult_t ncclCommSplit(ncclComm_t handle, int color, int key, ncclComm_t *ne id.internal[10] = (unsigned char)((unsigned)result >> 8); id.internal[11] = (unsigned char)result; } } - if (sccl_bootstrap_allgather(parent->bootstrap, id.internal, sizeof id.internal, &all, lengths, &parent->cancelled)) { + gathered = sccl_bootstrap_allgather(parent->bootstrap, id.internal, sizeof id.internal, &all, lengths, + &parent->cancelled); + if (gathered) { char message[700]; snprintf(message, sizeof message, "ncclCommSplit: exchanging unique ids: %s", sccl_bootstrap_error()); - return split_failed(parent, ncclSystemError, message); + return split_failed(parent, split_exchange_result(gathered), message); } for (int r = 0; r < parent->nranks; ++r) { if (lengths[r] != sizeof id.internal) { diff --git a/spark_transport/libsircl/tests/test_lifecycle.py b/spark_transport/libsircl/tests/test_lifecycle.py index ce74fdef..bd20f666 100644 --- a/spark_transport/libsircl/tests/test_lifecycle.py +++ b/spark_transport/libsircl/tests/test_lifecycle.py @@ -138,6 +138,25 @@ def split(r): self.assertEqual(data["children"], [None, None]) self.assertEqual(data["again"], [SUCCESS, SUCCESS]) + def test_split_whose_peer_never_arrives_is_a_remote_error(self): + """A split whose peer never takes part times out in its first all-gather and returns ncclRemoteError, the + result the engine's setup exchanges give a peer that does not arrive, naming the timeout.""" + handles = (C.c_void_p * 2)() + self.assertEqual(self.lib.ncclCommInitAll(handles, 2, None), SUCCESS) + saved = os.environ.get("LIBSIRCL_SETUP_TIMEOUT_MS") + os.environ["LIBSIRCL_SETUP_TIMEOUT_MS"] = "300" + try: + child = C.c_void_p() + self.assertEqual(self.lib.ncclCommSplit(handles[0], 0, 0, C.byref(child), None), 6) + self.assertIn(b"timed out", self.lib.ncclGetLastError(None)) + finally: + if saved is None: + os.environ.pop("LIBSIRCL_SETUP_TIMEOUT_MS", None) + else: + os.environ["LIBSIRCL_SETUP_TIMEOUT_MS"] = saved + for handle in handles: + self.lib.ncclCommAbort(handle) + def test_split_refuses_short_records_in_both_rounds(self): for short_round in (0, 1): with self.subTest(round=short_round): From 13d3ce10fdea4f7df3b5d5aac958c6132c703e8b Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Thu, 15 Oct 2026 09:17:09 -0500 Subject: [PATCH 41/65] Free ncclMemFree memory in the context it belongs to ncclMemFree asks the driver for the memory's context (cuPointerGetAttribute with CU_POINTER_ATTRIBUTE_CONTEXT), makes it current for the free and restores the caller's context afterwards. A thread without a current context, or with another device's context, frees the memory correctly instead of depending on the driver's handling of a free outside the allocating context. A pointer the driver does not know is refused with ncclInvalidArgument. The library binds cuPointerGetAttribute in its driver table (present in every CUDA driver since 4.0), and the stand-in driver (tests/fake_cuda.c) provides it and counts context pushes. Compatibility: frees from the allocating context behave the same; an unknown pointer returns ncclInvalidArgument instead of the driver's free error (ncclUnhandledCudaError). Validation: tests/test_engine_cpu.py frees a pointer on a thread without a current context: before the change no context was pushed; after it the memory's context is pushed once and the free succeeds. make check and make check-asan pass. Request item 40, first check (audit 06-F5). NVIDIA NCCL source was neither read nor adapted. Co-Authored-By: Claude Opus 5.5 --- spark_transport/libsircl/src/cuda_api.c | 1 + spark_transport/libsircl/src/cuda_api.h | 1 + spark_transport/libsircl/src/engine.c | 22 +++++++++++++++++-- spark_transport/libsircl/tests/fake_cuda.c | 14 +++++++++++- .../libsircl/tests/test_engine_cpu.py | 12 ++++++++++ 5 files changed, 47 insertions(+), 3 deletions(-) diff --git a/spark_transport/libsircl/src/cuda_api.c b/spark_transport/libsircl/src/cuda_api.c index 204fa14d..0d7aa45d 100644 --- a/spark_transport/libsircl/src/cuda_api.c +++ b/spark_transport/libsircl/src/cuda_api.c @@ -45,6 +45,7 @@ static void load(void) { SYM(LaunchKernel, "cuLaunchKernel"); SYM(MemAlloc, "cuMemAlloc_v2"); SYM(MemFree, "cuMemFree_v2"); + SYM(PointerGetAttribute, "cuPointerGetAttribute"); SYM(MemsetD32Async, "cuMemsetD32Async"); SYM(MemsetD8Async, "cuMemsetD8Async"); SYM(MemcpyDtoDAsync, "cuMemcpyDtoDAsync_v2"); diff --git a/spark_transport/libsircl/src/cuda_api.h b/spark_transport/libsircl/src/cuda_api.h index c7bffb3b..17002d6d 100644 --- a/spark_transport/libsircl/src/cuda_api.h +++ b/spark_transport/libsircl/src/cuda_api.h @@ -55,6 +55,7 @@ typedef struct sccl_cuda { unsigned, sccl_CUstream, void **, void **); sccl_CUresult (*MemAlloc)(sccl_CUdeviceptr *, size_t); sccl_CUresult (*MemFree)(sccl_CUdeviceptr); + sccl_CUresult (*PointerGetAttribute)(void *, int, sccl_CUdeviceptr); sccl_CUresult (*MemsetD32Async)(sccl_CUdeviceptr, unsigned, size_t, sccl_CUstream); sccl_CUresult (*MemsetD8Async)(sccl_CUdeviceptr, unsigned char, size_t, sccl_CUstream); sccl_CUresult (*MemcpyDtoDAsync)(sccl_CUdeviceptr, sccl_CUdeviceptr, size_t, sccl_CUstream); diff --git a/spark_transport/libsircl/src/engine.c b/spark_transport/libsircl/src/engine.c index 87fbf4ff..b4de58c7 100644 --- a/spark_transport/libsircl/src/engine.c +++ b/spark_transport/libsircl/src/engine.c @@ -3954,6 +3954,9 @@ ncclResult_t sccl_engine_mem_alloc(void **ptr, size_t size, char *err, size_t le return ncclSuccess; } +/* The memory is freed in the context it was allocated in (CU_POINTER_ATTRIBUTE_CONTEXT), made current for + * the call, whatever context the calling thread has, if any. */ +enum { SCCL_POINTER_ATTRIBUTE_CONTEXT = 1 }; ncclResult_t sccl_engine_mem_free(void *ptr, char *err, size_t len) { if (!ptr) return ncclSuccess; cu = sccl_cuda_get(); @@ -3961,8 +3964,23 @@ ncclResult_t sccl_engine_mem_free(void *ptr, char *err, size_t len) { put_error(err, len, "ncclMemFree: %s", sccl_cuda_error()); return ncclSystemError; } - return cuda_check(cu->MemFree((sccl_CUdeviceptr)(uintptr_t)ptr), "ncclMemFree", err, len) ? ncclUnhandledCudaError - : ncclSuccess; + sccl_CUdeviceptr memory = (sccl_CUdeviceptr)(uintptr_t)ptr; + sccl_CUcontext owner = NULL; + if (cuda_check(cu->PointerGetAttribute(&owner, SCCL_POINTER_ATTRIBUTE_CONTEXT, memory), + "ncclMemFree: finding the memory's context", err, len)) + return ncclInvalidArgument; + int pushed = owner && cu->CtxPushCurrent(owner) == SCCL_CUDA_SUCCESS; + if (owner && !pushed) { + put_error(err, len, "ncclMemFree: the memory's context cannot be made current"); + return ncclUnhandledCudaError; + } + ncclResult_t result = cuda_check(cu->MemFree(memory), "ncclMemFree", err, len) ? ncclUnhandledCudaError + : ncclSuccess; + if (pushed) { + sccl_CUcontext popped; + cu->CtxPopCurrent(&popped); + } + return result; } /* One pair exchange of `chunk` bytes each way: `sent` (0: nothing meaningful) to the peer, the peer's block diff --git a/spark_transport/libsircl/tests/fake_cuda.c b/spark_transport/libsircl/tests/fake_cuda.c index 1e7d8fb6..96fffbb6 100644 --- a/spark_transport/libsircl/tests/fake_cuda.c +++ b/spark_transport/libsircl/tests/fake_cuda.c @@ -185,7 +185,19 @@ EXPORT int cuInit(unsigned flags) { return (int)(flags & 0); } /* sccl_fake_cuda_fail_push(1): every later push fails, as for a destroyed context. */ static int fail_push; EXPORT void sccl_fake_cuda_fail_push(int on) { fail_push = on; } -EXPORT int cuCtxPushCurrent_v2(void *ctx) { return ctx && !fail_push ? 0 : 201; } +static int pushes; +EXPORT int cuCtxPushCurrent_v2(void *ctx) { + if (!ctx || fail_push) return 201; + ++pushes; + return 0; +} +EXPORT int sccl_fake_cuda_pushes(void) { return pushes; } +/* CU_POINTER_ATTRIBUTE_CONTEXT (1) of any pointer: the stand-in's one context. */ +EXPORT int cuPointerGetAttribute(void *data, int attribute, unsigned long long pointer) { + if (attribute != 1 || !pointer) return 1; + *(void **)data = &context_token; + return 0; +} EXPORT int cuModuleUnload(void *module) { module_t *m = module; if (!m || m < modules || m >= modules + MAX_MODULES || !m->in_use) return 400; diff --git a/spark_transport/libsircl/tests/test_engine_cpu.py b/spark_transport/libsircl/tests/test_engine_cpu.py index 4e63144b..d7a1e4ec 100644 --- a/spark_transport/libsircl/tests/test_engine_cpu.py +++ b/spark_transport/libsircl/tests/test_engine_cpu.py @@ -302,6 +302,18 @@ def test_settings_are_parsed_strictly(self): out = run(script, {**base, "SIRCL_TRAFFIC_CLASS": "106", "LIBSIRCL_RECEIPT_INTERVAL_S": "2.5"}) self.assertEqual(json.loads(out.stdout)[0], 0, out.stdout) + def test_mem_free_makes_the_memorys_context_current(self): + """ncclMemFree on a thread without a current context frees in the context the memory belongs to: it + makes that context current for the call (the stand-in driver counts the pushes).""" + out = run('''g=C.CDLL("libcuda.so.1") +l.ncclMemFree.argtypes=[C.c_void_p] +before=g.sccl_fake_cuda_pushes() +r=l.ncclMemFree(C.c_void_p(0x10000)) +print(json.dumps([r, g.sccl_fake_cuda_pushes()-before])) +''', {"FAKE_CUDA_NO_CONTEXT": "1", "LD_LIBRARY_PATH": str(FAKE_DRIVER)}) + self.assertEqual(out.returncode, 0, out.stderr) + self.assertEqual(json.loads(out.stdout), [0, 1]) + def test_info_names_the_collectives(self): out = run("print(l.sirclGetInfo().decode())") self.assertEqual(out.returncode, 0, out.stderr) From 8f954de456fb6cec64188aff6c7e81388e05af73 Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Thu, 15 Oct 2026 09:18:50 -0500 Subject: [PATCH 42/65] Check every kernel-pack lookup at communicator setup Session setup looks up every transport, fold, chain, link and pair exchange entry it launches and refuses setup with the loader's message (sccl_kp_error, naming the entry) when one lookup fails, instead of ignoring the results and leaving a NULL function for the first launch. Every argument is range-checked before the lookups, so none fails with the packs this tree builds; the check guards a change that widens a dtype, kind or unroll range without the matching entry. The point-to-point lookups were already checked. Compatibility: no change with the packs this tree builds. Validation: the lookups run only during setup of two or more ranks, which the CPU harness cannot build (a one-rank communicator loads no pack); the build-time kernel-entry check (tests/check_entries.c) keeps verifying that every entry the loader names is in every cubin. make check and make check-asan pass. Request item 40, third check (audit 10-F8b). NVIDIA NCCL source was neither read nor adapted. Co-Authored-By: Claude Opus 5.5 --- spark_transport/libsircl/src/engine.c | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/spark_transport/libsircl/src/engine.c b/spark_transport/libsircl/src/engine.c index b4de58c7..ea8c9297 100644 --- a/spark_transport/libsircl/src/engine.c +++ b/spark_transport/libsircl/src/engine.c @@ -2347,21 +2347,30 @@ int sccl_engine_create(sccl_bootstrap *bootstrap, int nranks, int rank, int posi break; } e->kp_held = 1; + /* Every entry the engine launches, looked up once; a lookup that fails refuses setup here, naming the + * entry, instead of leaving a NULL function for the first launch. */ + int missing = 0; for (int a = 0; a < SCCL_ALG_COUNT; ++a) - for (int d = 0; d < SCCL_DT_COUNT; ++d) sccl_kp_function(ctx, a, d, nranks, &e->fn[a][d]); + for (int d = 0; d < SCCL_DT_COUNT; ++d) missing |= sccl_kp_function(ctx, a, d, nranks, &e->fn[a][d]); for (int k = SCCL_K_SCATTER; k < SCCL_K_COUNT; ++k) - for (int d = 0; d < SCCL_DT_COUNT; ++d) sccl_kp_function(ctx, k, d, nranks, &e->fn[k][d]); + for (int d = 0; d < SCCL_DT_COUNT; ++d) missing |= sccl_kp_function(ctx, k, d, nranks, &e->fn[k][d]); for (int d = 0; d < SCCL_FOLD_DTYPES; ++d) - for (int o = 0; o < SCCL_FOLD_OPS; ++o) sccl_kp_fold_function(ctx, d, o, &e->fold[d][o]); - for (int d = 0; d < SCCL_DT_COUNT; ++d) sccl_kp_chain_function(ctx, d, (int)e->shared.chain_unroll, &e->chain_fn[d]); + for (int o = 0; o < SCCL_FOLD_OPS; ++o) missing |= sccl_kp_fold_function(ctx, d, o, &e->fold[d][o]); + for (int d = 0; d < SCCL_DT_COUNT; ++d) + missing |= sccl_kp_chain_function(ctx, d, (int)e->shared.chain_unroll, &e->chain_fn[d]); for (int k = 0; k < SCCL_LINK_KINDS; ++k) for (int d = 0; d < SCCL_DT_COUNT; ++d) - sccl_kp_link_function(ctx, k, d, (int)e->shared.link_unroll, &e->link_fn[k][d]); + missing |= sccl_kp_link_function(ctx, k, d, (int)e->shared.link_unroll, &e->link_fn[k][d]); for (int d = 0; e->ring_reduce_passes == 2 && d < SCCL_DT_COUNT; ++d) - sccl_kp_ring_reduce_two_pass_function(ctx, d, (int)e->shared.link_unroll, &e->link_fn[SCCL_RING_REDUCE][d]); - sccl_kp_ring_exchange_function(ctx, (int)e->shared.link_unroll, &e->exchange_fn); + missing |= sccl_kp_ring_reduce_two_pass_function(ctx, d, (int)e->shared.link_unroll, + &e->link_fn[SCCL_RING_REDUCE][d]); + missing |= sccl_kp_ring_exchange_function(ctx, (int)e->shared.link_unroll, &e->exchange_fn); for (int d = 0; d < SCCL_DT_COUNT; ++d) - sccl_kp_ring_exchange_reduce_function(ctx, d, (int)e->shared.link_unroll, &e->exchange_reduce_fn[d]); + missing |= sccl_kp_ring_exchange_reduce_function(ctx, d, (int)e->shared.link_unroll, &e->exchange_reduce_fn[d]); + if (missing) { + put_error(local, sizeof local, "kernel pack lookup: %s", sccl_kp_error()); + break; + } if (allocate(e, local, sizeof local)) break; e->ctrl[CTRL_WAIT_LIMIT_US] = e->serving ? e->shared.serving_wait_us : e->shared.startup_wait_us; const char *names[MAX_DEVICES]; From 2b3ec301f8d255ba0fd34cc2c8f4622b00341c93 Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Thu, 15 Oct 2026 09:20:07 -0500 Subject: [PATCH 43/65] Refuse point-to-point group-rule violations at the call that breaks them Inside a group, ncclSend and ncclRecv on one communicator must use one stream and number at most 1,024. Both rules were checked only at ncclGroupEnd, after every call had returned success. The call that breaks a rule is now refused with ncclInvalidUsage naming the rule (and, as a refusal other than an invalid argument, becomes the group's result); the engine keeps its own check at ncclGroupEnd. README.md states the rules. Compatibility: groups within the rules are unchanged; a violating group fails at the violating call instead of at ncclGroupEnd. Validation: the check runs after the call's communicator passed its readiness check, which a CPU-only communicator never does (it carries no collectives or point-to-point calls), so no CPU test reaches it; no emulation case is written. make check and make check-asan pass; the Markdown link check passes. Request item 40, fourth check (audit 01-F11, code part). NVIDIA NCCL source was neither read nor adapted. Co-Authored-By: Claude Opus 5.5 --- spark_transport/libsircl/README.md | 4 +++- spark_transport/libsircl/src/api.c | 22 ++++++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/spark_transport/libsircl/README.md b/spark_transport/libsircl/README.md index c6a34330..13e10b09 100644 --- a/spark_transport/libsircl/README.md +++ b/spark_transport/libsircl/README.md @@ -186,7 +186,9 @@ Errors of single calls: a collective or point-to-point call refused with `ncclIn and leaves the calling thread's group as it was (the group's other calls still run at `ncclGroupEnd`); other errors of a collective, point-to-point or communicator-creation call inside a group become the group's result. A failed call that enqueues nothing (a query, a registration, `ncclGetUniqueId`, a lifecycle call, an -unsupported function) returns its error and leaves the group as it was. A point-to-point call queued in a group holds no reference +unsupported function) returns its error and leaves the group as it was. Inside a group the point-to-point +calls on one communicator must use one stream and number at most 1,024; the call that breaks either rule is +refused with `ncclInvalidUsage`, which becomes the group's result. A point-to-point call queued in a group holds no reference on its communicator, so `ncclCommAbort` from another thread returns at once, and that group's `ncclGroupEnd` then fails its calls with `ncclInvalidUsage`. `ncclCommSplit` with a NULL config (or a config whose `blocking` is undefined) takes the parent's blocking mode. diff --git a/spark_transport/libsircl/src/api.c b/spark_transport/libsircl/src/api.c index bbf2ed1a..6a8754b2 100644 --- a/spark_transport/libsircl/src/api.c +++ b/spark_transport/libsircl/src/api.c @@ -571,6 +571,28 @@ static ncclResult_t point_to_point(const char *what, int send, const void *buff, } sccl_p2p_op op = {send, peer, buff, count * item, stream}; if (group_depth) { + /* The group rules the engine applies at ncclGroupEnd, refused at the call that breaks them: the calls of + * one group on one communicator use one stream, and there are at most MAX_GROUP_P2P of them. */ + enum { MAX_GROUP_P2P = 1024 }; + unsigned same = 0; + const queued_p2p *first = NULL; + for (unsigned i = 0; i < p2p_count; ++i) + if (p2p_queue[i].comm == comm && p2p_queue[i].serial == comm->serial) { + if (!first) first = &p2p_queue[i]; + ++same; + } + if (first && first->op.stream != op.stream) { + release(comm); + snprintf(message, sizeof message, "%s: the point-to-point calls of one group on one communicator must use " + "one stream", what); + return refuse(ncclInvalidUsage, message); + } + if (same >= MAX_GROUP_P2P) { + release(comm); + snprintf(message, sizeof message, "%s: a group carries at most %d point-to-point calls on one communicator", + what, MAX_GROUP_P2P); + return refuse(ncclInvalidUsage, message); + } if (p2p_count == p2p_capacity) { unsigned capacity = p2p_capacity ? 2 * p2p_capacity : 64; queued_p2p *grown = realloc(p2p_queue, sizeof(*grown) * capacity); From abe298d3a82fafefd53a09ba28af60108dbb18ac Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Thu, 15 Oct 2026 09:21:35 -0500 Subject: [PATCH 44/65] Keep the Makefile's required flags under command-line CFLAGS and CPPFLAGS The Makefile appends its required compiler and preprocessor flags with `override`, so `make CFLAGS=-O1` or `make CPPFLAGS=-DX` replaces only the optional part (the default -O2 -g) and keeps -std=c11, -fPIC, -Werror, -pthread, the dependency flags, the include directories and the build's definitions. A command-line value used to replace the whole variable and build a library without position-independent code or the warnings as errors. Values from the environment (as `make check-asan` passes them) behave as before. Compatibility: builds without command-line CFLAGS or CPPFLAGS are unchanged. Validation: tests/test_build_flags.py (in make check and CMake's build_flags test) runs `make -n` for src/api.c with CFLAGS=-O1 and CPPFLAGS=-DEXTRA and requires the required flags beside the given ones: before the change the line was `cc -DEXTRA -O1 -c src/api.c ...`; after it every required flag is present. make check and make check-asan pass. Request item 42 (audit 11-7). NVIDIA NCCL source was neither read nor adapted. Co-Authored-By: Claude Opus 5.5 --- spark_transport/libsircl/CMakeLists.txt | 2 +- spark_transport/libsircl/Makefile | 10 +++--- .../libsircl/tests/test_build_flags.py | 31 +++++++++++++++++++ 3 files changed, 38 insertions(+), 5 deletions(-) create mode 100644 spark_transport/libsircl/tests/test_build_flags.py diff --git a/spark_transport/libsircl/CMakeLists.txt b/spark_transport/libsircl/CMakeLists.txt index 6426c5b5..4ad2437c 100644 --- a/spark_transport/libsircl/CMakeLists.txt +++ b/spark_transport/libsircl/CMakeLists.txt @@ -128,7 +128,7 @@ add_test(NAME kernelpack_cache COMMAND ${CMAKE_COMMAND} -E env foreach(name abi api lifecycle engine_cpu) add_test(NAME ${name} COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tests/test_${name}.py --library $) endforeach() -foreach(name bootstrap shm_verbs vendored_sources p2p_native site_routes mpi_shim check_nccl_tests) +foreach(name bootstrap shm_verbs vendored_sources p2p_native site_routes mpi_shim check_nccl_tests build_flags) add_test(NAME ${name} COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tests/test_${name}.py) endforeach() add_test(NAME kernel_entries COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tests/test_kernel_entries.py diff --git a/spark_transport/libsircl/Makefile b/spark_transport/libsircl/Makefile index 65af0719..ab842f4c 100644 --- a/spark_transport/libsircl/Makefile +++ b/spark_transport/libsircl/Makefile @@ -5,10 +5,12 @@ NVCC ?= nvcc BUILD ?= build # The library's version, from the one version source (VERSION). VERSION := $(shell cat VERSION) -CPPFLAGS += -Iinclude -Isrc -DLIBSIRCL_CPU_ONLY -DNCCL_OS_LINUX -DLIBSIRCL_VERSION='"$(VERSION)"' +# The flags the build needs are appended with `override`, so that CFLAGS or CPPFLAGS given on the command line +# (make CFLAGS=-O1) or in the environment replace only the optional part (the default -O2 -g) and keep them. +override CPPFLAGS += -Iinclude -Isrc -DLIBSIRCL_CPU_ONLY -DNCCL_OS_LINUX -DLIBSIRCL_VERSION='"$(VERSION)"' CFLAGS ?= -O2 -g # Diagnostic messages are bounded copies that truncate by design. -CFLAGS += -std=c11 -fPIC -Wall -Wextra -Werror -Wno-format-truncation -pthread -MMD -MP +override CFLAGS += -std=c11 -fPIC -Wall -Wextra -Werror -Wno-format-truncation -pthread -MMD -MP # The library: API layer, bootstrap, fabric arithmetic, session engine, CUDA # driver loader and the embedded kernel packs, plus SIRCL's native proxy and @@ -52,7 +54,7 @@ PACKS = $(FATBIN) $(FOLD_FATBIN) $(LINKS_FATBIN) $(P2P_FATBIN) P2P_PROXY = src/transport/sircl_p2p_proxy.c P2P_FEATURES ?= 1 ifeq ($(P2P_FEATURES),1) -CPPFLAGS += -DSCCL_P2P_LOCAL_FEATURES +override CPPFLAGS += -DSCCL_P2P_LOCAL_FEATURES endif NVCCFLAGS = -O3 -std=c++17 -gencode arch=compute_120,code=sm_120 -gencode arch=compute_121,code=sm_121 @@ -188,7 +190,7 @@ check: all $(BUILD)/test_fabric $(BUILD)/test_link_plan $(BUILD)/test_kernelpack $(PYTHON) tests/test_site_routes.py || status=1; \ $(PYTHON) tests/test_mpi_shim.py || status=1; \ $(PYTHON) tests/test_check_nccl_tests.py || status=1; \ - $(PYTHON) tests/test_kernel_entries.py --build $(BUILD) || status=1; \ + $(PYTHON) tests/test_kernel_entries.py --build $(BUILD) || status=1; $(PYTHON) tests/test_build_flags.py || status=1; \ exit $$status # The check suites against a host library built with AddressSanitizer and UndefinedBehaviorSanitizer in diff --git a/spark_transport/libsircl/tests/test_build_flags.py b/spark_transport/libsircl/tests/test_build_flags.py new file mode 100644 index 00000000..97c14485 --- /dev/null +++ b/spark_transport/libsircl/tests/test_build_flags.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 +"""The Makefile keeps its required compiler flags when CFLAGS or CPPFLAGS come from the command line. + +``make -n`` (nothing runs) of the library's objects with ``CFLAGS=-O1`` and ``CPPFLAGS=-DEXTRA`` must still +print -fPIC, -Werror and -std=c11 (CFLAGS) and -Iinclude and -DLIBSIRCL_CPU_ONLY (CPPFLAGS) beside the given +flags. Without ``override`` in the Makefile a command-line value replaces the whole variable. +""" +import os +import subprocess +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +class BuildFlagTests(unittest.TestCase): + def test_command_line_flags_keep_the_required_ones(self): + with tempfile.TemporaryDirectory() as build: + out = subprocess.run(["make", "-n", "-C", str(ROOT), f"BUILD={build}", "CFLAGS=-O1", + "CPPFLAGS=-DEXTRA", f"{build}/api.o"], + capture_output=True, text=True, timeout=60, + env={k: v for k, v in os.environ.items() if k not in ("CFLAGS", "CPPFLAGS")}) + self.assertEqual(out.returncode, 0, out.stderr) + line = next(row for row in out.stdout.splitlines() if "src/api.c" in row) + for flag in ("-O1", "-DEXTRA", "-fPIC", "-Werror", "-std=c11", "-Iinclude", "-DLIBSIRCL_CPU_ONLY"): + self.assertIn(flag, line.split(), line) + + +if __name__ == "__main__": + unittest.main(verbosity=2) From e152b5d9d86b345f30b57dfdd4180875b8542d29 Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Thu, 15 Oct 2026 09:27:54 -0500 Subject: [PATCH 45/65] Harden both builds, align CMake with the Makefile, and build CMake in CI Both builds compile with -fstack-protector-strong and -D_FORTIFY_SOURCE=2 (after -U, so the compiler's own default does not conflict) and link the library with -z relro, -z now and -z noexecstack, whatever the image compiler's defaults. Fortification needs an optimized build; CMake's Debug configuration goes without it. CMake: - an unset build type defaults to RelWithDebInfo (the Makefile's -O2 -g); - the kernel-entry check takes the architecture count from the -gencode flags instead of a hard-coded 2; - the vendored-source guard (SHA-256, LF, SIRCL change LF's feature words) lives in cmake/check_vendored.cmake, runs at configuration and again before every build of the library and its transport objects; - LIBSIRCL_NVCC_EXTRA_FLAGS adds nvcc flags such as -DCCCL_DISABLE_CTK_COMPATIBILITY_CHECK; - the build_flags suite is registered beside the others. Makefile: the objects and the library depend on the Makefile, so a flag change rebuilds them. The CI job "libsircl C library" adds a CMake configure, build and ctest step. tests/test_build_flags.py checks the hardening flags under command-line CFLAGS and CPPFLAGS, reads the built library with readelf (GNU_RELRO, BIND_NOW, a GNU_STACK segment without execute permission), and requires make check's suites to equal ctest's test names. Compatibility: builds at -O0 through the Makefile now fail on glibc's _FORTIFY_SOURCE warning under -Werror; every documented build is optimized. Validation: before the change test_build_flags failed on the missing -fstack-protector-strong and on BIND_NOW; after it make check and make check-asan pass, and the readelf checks pass on both libraries. CMake is not installed on this workstation, so the CMake changes and the new CI step were not run; they run in CI. Request item 43 (audit 11-5, 11-8, 11-2). NVIDIA NCCL source was neither read nor adapted. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 10 +++ spark_transport/libsircl/CMakeLists.txt | 68 +++++++++++-------- spark_transport/libsircl/Makefile | 15 ++-- .../libsircl/cmake/check_vendored.cmake | 35 ++++++++++ .../libsircl/tests/test_build_flags.py | 48 +++++++++++-- 5 files changed, 136 insertions(+), 40 deletions(-) create mode 100644 spark_transport/libsircl/cmake/check_vendored.cmake diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a98ac1c4..4ca1d1a1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -155,6 +155,16 @@ jobs: sha256sum build/packs/*.fatbin python tools/check_exports.py --include include --library build/libsircl.so make check-asan BUILD=build NVCC="$NVCC" + # The CMake build of the same sources and its tests (the same suites as make check). + - name: Build libsircl with CMake and run ctest + working-directory: spark_transport/libsircl + env: + SIRCL_PACKAGE: ${{ github.workspace }}/spark_transport/sircl + run: | + cmake -S . -B build-cmake -DLIBSIRCL_NVCC="$LIBSIRCL_NVCC" \ + -DLIBSIRCL_NVCC_EXTRA_FLAGS=-DCCCL_DISABLE_CTK_COMPATIBILITY_CHECK + cmake --build build-cmake -j + ctest --test-dir build-cmake --output-on-failure lil-bridge: name: pinned LIL bridge diff --git a/spark_transport/libsircl/CMakeLists.txt b/spark_transport/libsircl/CMakeLists.txt index 4ad2437c..f63b0403 100644 --- a/spark_transport/libsircl/CMakeLists.txt +++ b/spark_transport/libsircl/CMakeLists.txt @@ -1,4 +1,8 @@ cmake_minimum_required(VERSION 3.20) +# Optimized with debug information unless a build type is given, as the Makefile's -O2 -g. +if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) + set(CMAKE_BUILD_TYPE RelWithDebInfo CACHE STRING "CMake build type" FORCE) +endif() file(STRINGS ${CMAKE_CURRENT_SOURCE_DIR}/VERSION LIBSIRCL_VERSION LIMIT_COUNT 1) project(libsircl VERSION ${LIBSIRCL_VERSION} LANGUAGES C) if(NOT UNIX OR APPLE) @@ -14,14 +18,24 @@ find_program(LIBSIRCL_NVCC NAMES nvcc HINTS ENV CUDA_HOME PATH_SUFFIXES bin PATH if(NOT LIBSIRCL_NVCC) message(FATAL_ERROR "libsircl's kernel packs need nvcc (CUDA 13.3 or later): set LIBSIRCL_NVCC") endif() +set(LIBSIRCL_NVCC_EXTRA_FLAGS "" CACHE STRING "flags added to nvcc's (for example -DCCCL_DISABLE_CTK_COMPATIBILITY_CHECK)") set(LIBSIRCL_NVCCFLAGS -O3 -std=c++17 -gencode arch=compute_120,code=sm_120 -gencode arch=compute_121,code=sm_121) +# The architectures each pack holds, for the kernel-entry check: the number of -gencode flags. +set(LIBSIRCL_ARCHITECTURES 0) +foreach(flag ${LIBSIRCL_NVCCFLAGS}) + if(flag STREQUAL "-gencode") + math(EXPR LIBSIRCL_ARCHITECTURES "${LIBSIRCL_ARCHITECTURES} + 1") + endif() +endforeach() +separate_arguments(LIBSIRCL_NVCC_EXTRA UNIX_COMMAND "${LIBSIRCL_NVCC_EXTRA_FLAGS}") file(GLOB LIBSIRCL_KERNEL_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/kernels/*.cuh) foreach(pack kernels fold links p2p) set(FATBIN ${CMAKE_CURRENT_BINARY_DIR}/packs/sircl_${pack}.fatbin) add_custom_command( OUTPUT ${FATBIN} COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_CURRENT_BINARY_DIR}/packs - COMMAND ${LIBSIRCL_NVCC} ${LIBSIRCL_NVCCFLAGS} -fatbin -o ${FATBIN} ${CMAKE_CURRENT_SOURCE_DIR}/kernels/sircl_${pack}.cu + COMMAND ${LIBSIRCL_NVCC} ${LIBSIRCL_NVCC_EXTRA} ${LIBSIRCL_NVCCFLAGS} -fatbin -o ${FATBIN} + ${CMAKE_CURRENT_SOURCE_DIR}/kernels/sircl_${pack}.cu DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/kernels/sircl_${pack}.cu ${LIBSIRCL_KERNEL_HEADERS}) add_custom_command( OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${pack}_fatbin.c @@ -30,32 +44,23 @@ foreach(pack kernels fold links p2p) DEPENDS ${FATBIN} ${CMAKE_CURRENT_SOURCE_DIR}/tools/embed_fatbin.py) endforeach() -# SIRCL's point-to-point native library is vendored byte-identical; configuration refuses a copy whose -# SHA-256 differs from the one recorded beside it. -file(SHA256 ${CMAKE_CURRENT_SOURCE_DIR}/src/transport/sircl_p2p_proxy.c P2P_PROXY_SHA256) -file(STRINGS ${CMAKE_CURRENT_SOURCE_DIR}/src/transport/sircl_p2p_proxy.c.sha256 P2P_PROXY_RECORDED LIMIT_COUNT 1) -if(NOT P2P_PROXY_SHA256 STREQUAL P2P_PROXY_RECORDED) - message(FATAL_ERROR "src/transport/sircl_p2p_proxy.c differs from the vendored SIRCL source recorded in its .sha256") -endif() -# SIRCL's native proxy, vendored byte-identical the same way. -file(SHA256 ${CMAKE_CURRENT_SOURCE_DIR}/src/transport/sircl_roce_proxy.c ROCE_PROXY_SHA256) -file(STRINGS ${CMAKE_CURRENT_SOURCE_DIR}/src/transport/sircl_roce_proxy.c.sha256 ROCE_PROXY_RECORDED LIMIT_COUNT 1) -if(NOT ROCE_PROXY_SHA256 STREQUAL ROCE_PROXY_RECORDED) - message(FATAL_ERROR "src/transport/sircl_roce_proxy.c differs from the vendored SIRCL source recorded in its .sha256") -endif() -file(STRINGS ${CMAKE_CURRENT_SOURCE_DIR}/src/transport/sircl_roce_proxy.c ROCE_LOCAL_FEATURES_LINES REGEX "roce_local_features") -if(NOT ROCE_LOCAL_FEATURES_LINES) - message(FATAL_ERROR "src/transport/sircl_roce_proxy.c predates SIRCL change LF (no roce_local_features)") -endif() # The engine's setup check of the library's local feature word (p2p_local_features bit 0: p2p_destroy counts # failed verbs calls; SIRCL change LF), on by default as in the Makefile (P2P_FEATURES). option(LIBSIRCL_P2P_FEATURES "check the point-to-point library's local feature word at setup" ON) -if(LIBSIRCL_P2P_FEATURES) - file(STRINGS ${CMAKE_CURRENT_SOURCE_DIR}/src/transport/sircl_p2p_proxy.c P2P_LOCAL_FEATURES_LINES REGEX "p2p_local_features") - if(NOT P2P_LOCAL_FEATURES_LINES) - message(FATAL_ERROR "LIBSIRCL_P2P_FEATURES: src/transport/sircl_p2p_proxy.c predates SIRCL change LF (no p2p_local_features)") - endif() -endif() +# SIRCL's native proxy and point-to-point library are vendored byte-identical (LF, SHA-256 recorded beside +# each, with change LF's feature words): checked at configuration and again before every build of the library. +include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/check_vendored.cmake) +add_custom_target(vendored_sources_guard + COMMAND ${CMAKE_COMMAND} -DSOURCE_DIR=${CMAKE_CURRENT_SOURCE_DIR} + -DP2P_FEATURES=${LIBSIRCL_P2P_FEATURES} -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/check_vendored.cmake + COMMENT "checking the vendored SIRCL native sources") + +# Hardening of every object and of the library, as in the Makefile: stack protector, fortified libc calls, +# full RELRO with immediate binding, and a non-executable stack. +# _FORTIFY_SOURCE needs an optimized build, so a Debug build (-O0) goes without it. +set(LIBSIRCL_HARDENING_COMPILE -fstack-protector-strong $<$>:-U_FORTIFY_SOURCE> + $<$>:-D_FORTIFY_SOURCE=2>) +set(LIBSIRCL_HARDENING_LINK -Wl,-z,relro,-z,now,-z,noexecstack) # SIRCL's native proxy and point-to-point library twice: over the real libibverbs (loaded at run time) and # over the shared-memory verbs stand-in (emulation), each with its own include path. @@ -64,7 +69,8 @@ add_library(transport_emu OBJECT src/transport/proxy_emu.c src/transport/p2p_emu target_include_directories(transport_emu BEFORE PRIVATE src/transport/fake_verbs src/transport) foreach(target transport_hw transport_emu) set_target_properties(${target} PROPERTIES POSITION_INDEPENDENT_CODE ON C_STANDARD 11) - target_compile_options(${target} PRIVATE -Wall -Wextra -Werror -Wno-format-truncation) + target_compile_options(${target} PRIVATE -Wall -Wextra -Werror -Wno-format-truncation ${LIBSIRCL_HARDENING_COMPILE}) + add_dependencies(${target} vendored_sources_guard) endforeach() add_library(sircl SHARED src/api.c src/bootstrap.c src/stubs.c src/aliases.c src/device_api.c src/fabric.c src/engine.c @@ -78,13 +84,15 @@ if(LIBSIRCL_P2P_FEATURES) target_compile_definitions(sircl PRIVATE SCCL_P2P_LOCAL_FEATURES) endif() target_include_directories(sircl PUBLIC include PRIVATE src) -target_compile_options(sircl PRIVATE -Wall -Wextra -Werror -Wno-format-truncation) +target_compile_options(sircl PRIVATE -Wall -Wextra -Werror -Wno-format-truncation ${LIBSIRCL_HARDENING_COMPILE}) set_source_files_properties(${CMAKE_CURRENT_BINARY_DIR}/kernels_fatbin.c ${CMAKE_CURRENT_BINARY_DIR}/fold_fatbin.c ${CMAKE_CURRENT_BINARY_DIR}/links_fatbin.c ${CMAKE_CURRENT_BINARY_DIR}/p2p_fatbin.c PROPERTIES COMPILE_OPTIONS "-O0;-w") target_link_libraries(sircl PRIVATE Threads::Threads ${CMAKE_DL_LIBS}) set_target_properties(sircl PROPERTIES NO_SONAME TRUE SUFFIX ".so.${PROJECT_VERSION}") -target_link_options(sircl PRIVATE -Wl,-soname,libnccl.so.2 -Wl,-Bsymbolic-functions -Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/exports.map) +target_link_options(sircl PRIVATE -Wl,-soname,libnccl.so.2 -Wl,-Bsymbolic-functions -Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/exports.map + ${LIBSIRCL_HARDENING_LINK}) +add_dependencies(sircl vendored_sources_guard) # The kernel-entry check (tests/check_entries.c): the pack loader runs on the embedded fatbins through a # stand-in CUDA driver (tests/fake_cuda.c) that reads their cubins offline; the library links after it @@ -102,7 +110,7 @@ target_compile_definitions(check_entries PRIVATE LIBSIRCL_CPU_ONLY NCCL_OS_LINUX target_link_libraries(check_entries PRIVATE Threads::Threads ${CMAKE_DL_LIBS}) add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/entries.ok COMMAND ${CMAKE_COMMAND} -E env LD_LIBRARY_PATH=${CMAKE_CURRENT_BINARY_DIR}/fake-cuda - $ 2 + $ ${LIBSIRCL_ARCHITECTURES} COMMAND ${CMAKE_COMMAND} -E touch ${CMAKE_CURRENT_BINARY_DIR}/entries.ok DEPENDS check_entries fake_cuda) add_custom_target(kernel_entries DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/entries.ok) @@ -125,10 +133,10 @@ target_compile_definitions(test_kernelpack_cache PRIVATE LIBSIRCL_CPU_ONLY NCCL_ target_link_libraries(test_kernelpack_cache PRIVATE Threads::Threads ${CMAKE_DL_LIBS}) add_test(NAME kernelpack_cache COMMAND ${CMAKE_COMMAND} -E env LD_LIBRARY_PATH=${CMAKE_CURRENT_BINARY_DIR}/fake-cuda $) -foreach(name abi api lifecycle engine_cpu) +foreach(name abi api lifecycle engine_cpu build_flags) add_test(NAME ${name} COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tests/test_${name}.py --library $) endforeach() -foreach(name bootstrap shm_verbs vendored_sources p2p_native site_routes mpi_shim check_nccl_tests build_flags) +foreach(name bootstrap shm_verbs vendored_sources p2p_native site_routes mpi_shim check_nccl_tests) add_test(NAME ${name} COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tests/test_${name}.py) endforeach() add_test(NAME kernel_entries COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tests/test_kernel_entries.py diff --git a/spark_transport/libsircl/Makefile b/spark_transport/libsircl/Makefile index ab842f4c..2d18f32e 100644 --- a/spark_transport/libsircl/Makefile +++ b/spark_transport/libsircl/Makefile @@ -11,6 +11,11 @@ override CPPFLAGS += -Iinclude -Isrc -DLIBSIRCL_CPU_ONLY -DNCCL_OS_LINUX -DLIBSI CFLAGS ?= -O2 -g # Diagnostic messages are bounded copies that truncate by design. override CFLAGS += -std=c11 -fPIC -Wall -Wextra -Werror -Wno-format-truncation -pthread -MMD -MP +# Hardening, whatever the compiler's own defaults: stack protector and fortified libc calls (which need an +# optimized build, -O1 or above), full RELRO with immediate binding and a non-executable stack for the library. +override CFLAGS += -fstack-protector-strong +override CPPFLAGS += -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2 +HARDENING_LDFLAGS = -Wl,-z,relro,-z,now,-z,noexecstack # The library: API layer, bootstrap, fabric arithmetic, session engine, CUDA # driver loader and the embedded kernel packs, plus SIRCL's native proxy and @@ -63,7 +68,7 @@ all: $(BUILD)/libsircl.so $(BUILD) $(BUILD)/transport: mkdir -p $@ -$(BUILD)/%.o: src/%.c include/nccl.h src/internal.h | $(BUILD) +$(BUILD)/%.o: src/%.c include/nccl.h src/internal.h Makefile | $(BUILD) $(CC) $(CPPFLAGS) $(CFLAGS) -c $< -o $@ # SIRCL's native proxy is vendored byte-identical as well (src/transport/sircl_roce_proxy.c, SHA-256 in its # .sha256 file); the build refuses a copy of other contents. The engine checks its local feature word @@ -134,8 +139,10 @@ $(BUILD)/kernels_fatbin.o $(BUILD)/fold_fatbin.o $(BUILD)/links_fatbin.o $(BUILD # and ctypes.CDLL("libnccl.so.2") resolve. libsircl.so is a development link to it. $(BUILD)/libsircl.so: $(BUILD)/libsircl.so.$(VERSION) ln -sf libsircl.so.$(VERSION) $@ -$(BUILD)/libsircl.so.$(VERSION): $(OBJECTS) exports.map $(BUILD)/entries.ok - $(CC) -shared -pthread $(LDFLAGS) -Wl,-soname,libnccl.so.2 -Wl,-Bsymbolic-functions -Wl,--version-script=exports.map -o $@ $(OBJECTS) -ldl +# The Makefile is a prerequisite of the objects and the library, so a change of their flags rebuilds them. +$(filter $(BUILD)/transport/%.o,$(OBJECTS)): Makefile +$(BUILD)/libsircl.so.$(VERSION): $(OBJECTS) exports.map $(BUILD)/entries.ok Makefile + $(CC) -shared -pthread $(LDFLAGS) $(HARDENING_LDFLAGS) -Wl,-soname,libnccl.so.2 -Wl,-Bsymbolic-functions -Wl,--version-script=exports.map -o $@ $(OBJECTS) -ldl # The kernel-entry check (tests/check_entries.c): the library's pack loader runs on the embedded fatbins # through a stand-in CUDA driver (tests/fake_cuda.c) that reads their cubins offline, so every entry the @@ -190,7 +197,7 @@ check: all $(BUILD)/test_fabric $(BUILD)/test_link_plan $(BUILD)/test_kernelpack $(PYTHON) tests/test_site_routes.py || status=1; \ $(PYTHON) tests/test_mpi_shim.py || status=1; \ $(PYTHON) tests/test_check_nccl_tests.py || status=1; \ - $(PYTHON) tests/test_kernel_entries.py --build $(BUILD) || status=1; $(PYTHON) tests/test_build_flags.py || status=1; \ + $(PYTHON) tests/test_kernel_entries.py --build $(BUILD) || status=1; $(PYTHON) tests/test_build_flags.py --library $(abspath $(BUILD)/libsircl.so) || status=1; \ exit $$status # The check suites against a host library built with AddressSanitizer and UndefinedBehaviorSanitizer in diff --git a/spark_transport/libsircl/cmake/check_vendored.cmake b/spark_transport/libsircl/cmake/check_vendored.cmake new file mode 100644 index 00000000..cbbfe5aa --- /dev/null +++ b/spark_transport/libsircl/cmake/check_vendored.cmake @@ -0,0 +1,35 @@ +# The vendored SIRCL native sources (src/transport/sircl_roce_proxy.c and sircl_p2p_proxy.c) must be the +# byte-identical LF copies whose SHA-256 is recorded beside them, with SIRCL change LF's local feature words. +# CMakeLists.txt includes this file at configure time and runs it as a script (cmake -P, with SOURCE_DIR and +# P2P_FEATURES set) before every build of the library, as the Makefile's roce_source.ok and p2p_source.ok +# rules do, so a copy changed after configuration fails the build. +if(NOT SOURCE_DIR) + set(SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) +endif() +if(NOT DEFINED P2P_FEATURES) + set(P2P_FEATURES ${LIBSIRCL_P2P_FEATURES}) +endif() +foreach(name roce p2p) + set(source ${SOURCE_DIR}/src/transport/sircl_${name}_proxy.c) + file(SHA256 ${source} found) + file(STRINGS ${source}.sha256 recorded LIMIT_COUNT 1) + if(NOT found STREQUAL recorded) + message(FATAL_ERROR "src/transport/sircl_${name}_proxy.c differs from the vendored SIRCL source recorded in its .sha256") + endif() + # file(STRINGS) drops carriage returns, so the raw contents are searched. + file(READ ${source} contents) + string(FIND "${contents}" "\r" carriage_return) + if(NOT carriage_return EQUAL -1) + message(FATAL_ERROR "src/transport/sircl_${name}_proxy.c has CRLF line endings; vendor SIRCL's LF bytes") + endif() +endforeach() +file(STRINGS ${SOURCE_DIR}/src/transport/sircl_roce_proxy.c roce_features REGEX "roce_local_features") +if(NOT roce_features) + message(FATAL_ERROR "src/transport/sircl_roce_proxy.c predates SIRCL change LF (no roce_local_features)") +endif() +if(P2P_FEATURES) + file(STRINGS ${SOURCE_DIR}/src/transport/sircl_p2p_proxy.c p2p_features REGEX "p2p_local_features") + if(NOT p2p_features) + message(FATAL_ERROR "LIBSIRCL_P2P_FEATURES: src/transport/sircl_p2p_proxy.c predates SIRCL change LF (no p2p_local_features)") + endif() +endif() diff --git a/spark_transport/libsircl/tests/test_build_flags.py b/spark_transport/libsircl/tests/test_build_flags.py index 97c14485..e0fea72c 100644 --- a/spark_transport/libsircl/tests/test_build_flags.py +++ b/spark_transport/libsircl/tests/test_build_flags.py @@ -1,17 +1,26 @@ #!/usr/bin/env python3 -"""The Makefile keeps its required compiler flags when CFLAGS or CPPFLAGS come from the command line. +"""The build's flags and test lists. -``make -n`` (nothing runs) of the library's objects with ``CFLAGS=-O1`` and ``CPPFLAGS=-DEXTRA`` must still -print -fPIC, -Werror and -std=c11 (CFLAGS) and -Iinclude and -DLIBSIRCL_CPU_ONLY (CPPFLAGS) beside the given -flags. Without ``override`` in the Makefile a command-line value replaces the whole variable. +- The Makefile keeps its required compiler flags when CFLAGS or CPPFLAGS come from the command line: + ``make -n`` (nothing runs) of the library's objects with ``CFLAGS=-O1`` and ``CPPFLAGS=-DEXTRA`` must still + print -fPIC, -Werror, -std=c11 and -fstack-protector-strong (CFLAGS) and -Iinclude, -DLIBSIRCL_CPU_ONLY and + -D_FORTIFY_SOURCE=2 (CPPFLAGS) beside the given flags. +- The library (``--library``) is linked with full RELRO and immediate binding and a non-executable stack + (``readelf``: a GNU_RELRO segment, BIND_NOW, and a GNU_STACK segment without execute permission). +- ``make check`` and CMake's ``ctest`` run the same suites: the test names in CMakeLists.txt equal the + suites the Makefile's check recipe runs. """ +import argparse import os +import re import subprocess +import sys import tempfile import unittest from pathlib import Path ROOT = Path(__file__).resolve().parents[1] +LIBRARY = ROOT / "build" / "libsircl.so" class BuildFlagTests(unittest.TestCase): @@ -23,9 +32,36 @@ def test_command_line_flags_keep_the_required_ones(self): env={k: v for k, v in os.environ.items() if k not in ("CFLAGS", "CPPFLAGS")}) self.assertEqual(out.returncode, 0, out.stderr) line = next(row for row in out.stdout.splitlines() if "src/api.c" in row) - for flag in ("-O1", "-DEXTRA", "-fPIC", "-Werror", "-std=c11", "-Iinclude", "-DLIBSIRCL_CPU_ONLY"): + for flag in ("-O1", "-DEXTRA", "-fPIC", "-Werror", "-std=c11", "-Iinclude", "-DLIBSIRCL_CPU_ONLY", + "-fstack-protector-strong", "-D_FORTIFY_SOURCE=2"): self.assertIn(flag, line.split(), line) + def test_the_library_is_linked_hardened(self): + if not LIBRARY.exists(): + self.skipTest(f"{LIBRARY} is not built") + program = subprocess.run(["readelf", "-lW", str(LIBRARY)], capture_output=True, text=True, check=True).stdout + dynamic = subprocess.run(["readelf", "-dW", str(LIBRARY)], capture_output=True, text=True, check=True).stdout + self.assertIn("GNU_RELRO", program) + self.assertRegex(dynamic, r"\(FLAGS\)\s+BIND_NOW|\(FLAGS_1\)\s+Flags:.*\bNOW\b|\(BIND_NOW\)") + stack = next(row for row in program.splitlines() if "GNU_STACK" in row) + self.assertNotIn("E", stack.split()[-2], stack) + + def test_make_check_and_ctest_run_the_same_suites(self): + makefile = (ROOT / "Makefile").read_text() + recipe = makefile[makefile.index("\ncheck:"):makefile.index("exit $$status")] + from_make = set(re.findall(r"tests/test_(\w+)\.py", recipe)) + from_make |= {name.removeprefix("test_") for name in re.findall(r"\$\(BUILD\)/(test_\w+)", recipe)} + cmake = (ROOT / "CMakeLists.txt").read_text() + from_cmake = set(re.findall(r"add_test\(NAME (\w+)", cmake)) + for names in re.findall(r"foreach\(name ([\w ]+)\)\s*\n\s*add_test", cmake): + from_cmake |= set(names.split()) + from_cmake.discard("${name}") + self.assertEqual(from_make, from_cmake) + if __name__ == "__main__": - unittest.main(verbosity=2) + parser = argparse.ArgumentParser() + parser.add_argument("--library", type=Path, default=LIBRARY) + options, extra = parser.parse_known_args() + LIBRARY = options.library.resolve() + unittest.main(argv=[sys.argv[0], *extra], verbosity=2) From d7589c3c6369347cb889cbaa65ffd93157ecd389 Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Thu, 15 Oct 2026 09:29:39 -0500 Subject: [PATCH 46/65] Add a table-driven hostile-argument case to the library emulation rank tests/emulation/library_rank.py gains run_hostile_argument_checks, run on every rank after the abort check: 31 calls covering every collective and ncclSend and ncclRecv with a count whose bytes overflow size_t, a NULL buffer where every rank needs one, and roots -1, W and 2^30 for ncclReduce, ncclBroadcast, ncclGather and ncclScatter. Every rank makes the same calls, so each refusal happens on every rank before any communication; the case passes when every call returns ncclInvalidArgument and the send and receive buffers it was given are unchanged. The module docstring lists the case. Compatibility: test-only. Validation: needs a GPU; written, not run: python tests/emulation/run_library.py --library build/libsircl.so --world 2 (library_rank.py runs once per rank under run_library.py). The module compiles and passes ruff. Request item 44 (audit 12-F2). NVIDIA NCCL source was neither read nor adapted. Co-Authored-By: Claude Opus 5.5 --- .../libsircl/tests/emulation/library_rank.py | 54 ++++++++++++++++++- 1 file changed, 53 insertions(+), 1 deletion(-) diff --git a/spark_transport/libsircl/tests/emulation/library_rank.py b/spark_transport/libsircl/tests/emulation/library_rank.py index db7bdfba..2c3800d8 100644 --- a/spark_transport/libsircl/tests/emulation/library_rank.py +++ b/spark_transport/libsircl/tests/emulation/library_rank.py @@ -34,7 +34,9 @@ collectives of every selection point with one rank's buffers unaligned or in place, eager and captured cold, launch and carry the same ops on every rank as with aligned buffers (receipt counts). On a pair, one-way exchanges whose empty direction alternates between the ranks across link-slot wrap, each round -followed by two-way calls. An op that is not built in refused and counted; the receipt, +followed by two-way calls. An op that is not built in refused and counted. Hostile arguments (overflowing counts, NULL buffers, roots +out of range) to every collective and to ncclSend and ncclRecv refused with ncclInvalidArgument, buffers +unchanged. The receipt, ncclCommGetAsyncError, finalize and destroy. """ from __future__ import annotations @@ -761,6 +763,55 @@ def grouped(): f"group end {results.get('end')}") +def run_hostile_argument_checks(torch, lib, comm, world, rank, stream, report) -> None: + """Hostile arguments, table-driven: for every collective and ncclSend/ncclRecv, a count whose bytes overflow + size_t, a NULL buffer where every rank needs one, and a root outside 0..W-1 for the rooted collectives. + Every rank makes the same calls, so every refusal happens on every rank before any communication; each + call must return ncclInvalidArgument and leave the buffers it was given unchanged.""" + v, i, z, c = ctypes.c_void_p, ctypes.c_int, ctypes.c_size_t, ctypes.c_void_p + signatures = {"ncclAllReduce": [v, v, z, i, i, c, v], "ncclAllGather": [v, v, z, i, c, v], + "ncclReduceScatter": [v, v, z, i, i, c, v], "ncclReduce": [v, v, z, i, i, i, c, v], + "ncclBroadcast": [v, v, z, i, i, c, v], "ncclAlltoAll": [v, v, z, i, c, v], + "ncclGather": [v, v, z, i, i, c, v], "ncclScatter": [v, v, z, i, i, c, v], + "ncclSend": [v, z, i, i, c, v], "ncclRecv": [v, z, i, i, c, v]} + for name, types in signatures.items(): + getattr(lib, name).argtypes = types + count = 1024 + send = torch.full((count * world,), 3.0, dtype=torch.float32, device="cuda") + recv = torch.full((count * world,), 7.0, dtype=torch.float32, device="cuda") + torch.cuda.synchronize() + snapshot = (send.cpu().clone(), recv.cpu().clone()) + s, r, h = ctypes.c_void_p(send.data_ptr()), ctypes.c_void_p(recv.data_ptr()), ctypes.c_void_p(stream.cuda_stream) + huge = (1 << 64) - 1 # size_t's largest value: its bytes overflow every element size + f32, total = 7, 0 + cases = [ + ("ncclAllReduce", (s, r, huge, f32, 0, comm, h)), ("ncclAllReduce", (None, r, count, f32, 0, comm, h)), + ("ncclAllReduce", (s, None, count, f32, 0, comm, h)), + ("ncclAllGather", (s, r, huge, f32, comm, h)), ("ncclAllGather", (None, r, count, f32, comm, h)), + ("ncclReduceScatter", (s, r, huge, f32, 0, comm, h)), ("ncclReduceScatter", (s, None, count, f32, 0, comm, h)), + ("ncclAlltoAll", (s, r, huge, f32, comm, h)), ("ncclAlltoAll", (None, r, count, f32, comm, h)), + ("ncclReduce", (s, r, huge, f32, 0, 0, comm, h)), ("ncclReduce", (None, r, count, f32, 0, 0, comm, h)), + ("ncclBroadcast", (s, r, huge, f32, 0, comm, h)), ("ncclBroadcast", (s, None, count, f32, 0, comm, h)), + ("ncclGather", (s, r, huge, f32, 0, comm, h)), ("ncclScatter", (s, r, huge, f32, 0, comm, h)), + ("ncclSend", (s, huge, f32, (rank + 1) % world, comm, h)), ("ncclSend", (None, count, f32, 0, comm, h)), + ("ncclRecv", (r, huge, f32, (rank + 1) % world, comm, h)), ("ncclRecv", (None, count, f32, 0, comm, h)), + ] + [(name, args) for root in (-1, world, 1 << 30) for name, args in ( + ("ncclReduce", (s, r, count, f32, 0, root, comm, h)), ("ncclBroadcast", (s, r, count, f32, root, comm, h)), + ("ncclGather", (s, r, count, f32, root, comm, h)), ("ncclScatter", (s, r, count, f32, root, comm, h)))] + wrong = [] + for name, args in cases: + total += 1 + code = getattr(lib, name)(*args) + if code != 4: + wrong.append(f"{name}{tuple('NULL' if a is None else a for a in args[2:-2])}: result {code}: " + f"{lib.ncclGetLastError(None).decode()}") + torch.cuda.synchronize() + unchanged = torch.equal(send.cpu(), snapshot[0]) and torch.equal(recv.cpu(), snapshot[1]) + report(f"{total} calls with hostile arguments (overflowing counts, NULL buffers, roots outside 0-{world - 1}) " + "refused with ncclInvalidArgument, buffers unchanged", not wrong and unchanged, + "; ".join(wrong[:4]) if wrong else "" if unchanged else "a buffer changed") + + def run_point_to_point_checks(torch, lib, comm, world, rank, max_piece, stream, seed_base, report) -> int: """ncclSend and ncclRecv: between the ranks of a two-rank communicator, outside and inside groups (both directions with unequal sizes, several calls matched in issue order, sends to the rank itself @@ -1956,6 +2007,7 @@ def reduce_scatter(src, dst, count, stream): alternating_alltoalls = run_alternating_grid_checks(torch, lib, comm, world, rank, stream_a, args.seed_base, report) run_split_checks(torch, lib, comm, world, rank, stream_a, args.seed_base, report) run_abort_with_queued_send_check(torch, lib, comm, world, rank, stream_a, report) + run_hostile_argument_checks(torch, lib, comm, world, rank, stream_a, report) run_capture_staging_check(torch, lib, comm, world, rank, args.seed_base, report) run_rank_local_staging_checks(torch, lib, comm, world, rank, args.seed_base, report) run_staging_growth_check(torch, lib, comm, world, rank, args.seed_base, report) From 1f83ca7d53d612a2f2d444302240aa38c1f3137d Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Thu, 15 Oct 2026 09:31:18 -0500 Subject: [PATCH 47/65] Describe the point-to-point scope, default schedules and four packs in sirclGetInfo sirclGetInfo's JSON states the point-to-point scope README.md documents (two-rank communicators, a rank to itself, and point-to-point channels on communicators of three or more ranks with LIBSIRCL_P2P_CHANNELS=on), the schedules' default (pieces, the pair plan on a cabled pair, the cycle plan on a ring that closes over cables) and the four kernel packs (transport, fold, link and point-to-point). Compatibility: the JSON keys are unchanged; only these three strings change. Nothing in this repository parses them. Validation: tests/test_engine_cpu.py checks the three strings (the send_recv text failed before the change). make check and make check-asan pass. Request item 48 (audit 14-F10, part of 01-F11). NVIDIA NCCL source was neither read nor adapted. Co-Authored-By: Claude Opus 5.5 --- spark_transport/libsircl/src/api.c | 8 +++++--- spark_transport/libsircl/tests/test_engine_cpu.py | 5 +++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/spark_transport/libsircl/src/api.c b/spark_transport/libsircl/src/api.c index 6a8754b2..a27a901e 100644 --- a/spark_transport/libsircl/src/api.c +++ b/spark_transport/libsircl/src/api.c @@ -170,12 +170,14 @@ const char *sirclGetInfo(void) { "\"ops\":[\"sum\",\"prod\",\"max\",\"min\",\"avg\"],\"transport_reductions\":[\"float16/sum\"," "\"bfloat16/sum\",\"float32/sum\"],\"all_gather\":\"every datatype\",\"broadcast\":" "\"every datatype\",\"all_to_all\":\"every datatype\",\"gather\":\"every datatype\",\"scatter\":" - "\"every datatype\",\"send_recv\":\"two-rank communicators, and a rank to itself\",\"split\":true}," + "\"every datatype\",\"send_recv\":\"two-rank communicators, a rank to itself, and point-to-point " + "channels on communicators of three or more ranks (LIBSIRCL_P2P_CHANNELS=on)\",\"split\":true}," "\"schedules\":{\"large_all_reduce\":[\"pieces\",\"chain\",\"auto\",\"ring\"],\"all_gather\":" "[\"pieces\",\"chain\",\"auto\",\"ring\"],\"reduce_scatter\":[\"pieces\",\"chain\",\"auto\",\"ring\"]," - "\"default\":\"pieces\"}," + "\"default\":\"pieces; the pair plan on a cabled pair; the cycle plan on a ring that closes over " + "cables\"}," "\"transports\":[\"verbs\",\"emulation\"],\"kernels\":\"ahead-of-time CUDA C++ " - "(sm_120, sm_121): transport, fold and link packs\",\"bootstrap\":" + "(sm_120, sm_121): transport, fold, link and point-to-point packs\",\"bootstrap\":" "\"tcp (loopback or SIRCL_BOOTSTRAP_ADDR/IFNAME)\",\"world\":\"1-8\"}"; } ncclResult_t ncclGetVersion(int *version) { diff --git a/spark_transport/libsircl/tests/test_engine_cpu.py b/spark_transport/libsircl/tests/test_engine_cpu.py index d7a1e4ec..45e8849c 100644 --- a/spark_transport/libsircl/tests/test_engine_cpu.py +++ b/spark_transport/libsircl/tests/test_engine_cpu.py @@ -322,6 +322,11 @@ def test_info_names_the_collectives(self): self.assertIn("reduce_scatter", info["collectives"]) self.assertIn("all_gather", info["collectives"]) self.assertEqual(info["transports"], ["verbs", "emulation"]) + # The documented point-to-point scope, the four packs and the plans that change the default schedule. + self.assertIn("point-to-point channels on communicators of three or more ranks", info["collectives"]["send_recv"]) + self.assertIn("transport, fold, link and point-to-point packs", info["kernels"]) + self.assertIn("pair plan", info["schedules"]["default"]) + self.assertIn("cycle plan", info["schedules"]["default"]) def test_device_api_host_entry_points(self): """ncclCommQueryProperties reports the communicator and no device API; the teams are the world, From 5734519a930d3e27a4bb3be6c617e678a718d18a Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Thu, 15 Oct 2026 09:32:50 -0500 Subject: [PATCH 48/65] Give a failed ncclCommFinalize its own text without an overlapping copy ncclCommFinalize passed the thread's error buffer as the source of the snprintf that writes the same buffer (formally undefined), and when the engine's wait for this rank's work failed it reported whatever text an earlier call had left. It now reports its own text: "waiting for this rank's enqueued work failed" for a failed wait, else the close's result as the communicator recorded it (so a repeated finalize returns the first close's text with its result), copied into a local buffer before fail() writes the thread's text. Compatibility: result codes are unchanged; the message text is the failure's own. Validation: a failed finalize needs an engine of two or more ranks, which the CPU harness cannot build; make check and make check-asan pass. Request item 50, second fix (audit 03-F5). NVIDIA NCCL source was neither read nor adapted. Co-Authored-By: Claude Opus 5.5 --- spark_transport/libsircl/src/api.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/spark_transport/libsircl/src/api.c b/spark_transport/libsircl/src/api.c index a27a901e..5c105a6f 100644 --- a/spark_transport/libsircl/src/api.c +++ b/spark_transport/libsircl/src/api.c @@ -1086,9 +1086,16 @@ ncclResult_t ncclCommFinalize(ncclComm_t handle) { if (!comm) return fail(ncclInvalidArgument, "unknown communicator handle"); ncclResult_t flushed = comm->engine && atomic_load(&comm->state) == READY ? sccl_engine_finalize(comm->engine) : ncclSuccess; - if (flushed == ncclSuccess) flushed = quiesce(comm, "ncclCommFinalize"); + /* The failure's own text: this rank's wait, else the close's result as the communicator recorded it (a + * repeated finalize returns the first close's result and text). Copied before fail() writes the thread's + * text, which it must not read from. */ + char reason[sizeof last_error] = "ncclCommFinalize: waiting for this rank's enqueued work failed"; + if (flushed == ncclSuccess) { + flushed = quiesce(comm, "ncclCommFinalize"); + if (flushed != ncclSuccess) read_error(comm, reason, sizeof reason); + } release(comm); - if (flushed != ncclSuccess) return fail(flushed, last_error[0] ? last_error : "ncclCommFinalize failed"); + if (flushed != ncclSuccess) return fail(flushed, reason); return transition(handle, FINALIZED); } ncclResult_t ncclCommRevoke(ncclComm_t comm, int flags) { From b67cac1a7fd6bae7e511845fbc3b0cff8c601389 Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Thu, 15 Oct 2026 09:33:25 -0500 Subject: [PATCH 49/65] Check for a forked process first in ncclCommRevoke ncclCommRevoke runs the fork check before it checks its flags, as every other entry point does, so a forked child that calls it with nonzero flags gets ncclInvalidUsage naming the fork instead of ncclInvalidArgument for the flags. Compatibility: only the error code and text of that combination change. Validation: tests/test_api.py's fork test calls ncclCommRevoke(NULL, 1) in the child: ncclInvalidArgument before the change, ncclInvalidUsage after it. Request item 50, fourth fix (audit 03-F13a). NVIDIA NCCL source was neither read nor adapted. Co-Authored-By: Claude Opus 5.5 --- spark_transport/libsircl/src/api.c | 1 + spark_transport/libsircl/tests/test_api.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/spark_transport/libsircl/src/api.c b/spark_transport/libsircl/src/api.c index 5c105a6f..7dc7d28a 100644 --- a/spark_transport/libsircl/src/api.c +++ b/spark_transport/libsircl/src/api.c @@ -1099,6 +1099,7 @@ ncclResult_t ncclCommFinalize(ncclComm_t handle) { return transition(handle, FINALIZED); } ncclResult_t ncclCommRevoke(ncclComm_t comm, int flags) { + CHECK_PROCESS(); if (flags) return fail(ncclInvalidArgument, "revoke flags must be zero"); return transition(comm, REVOKED); } diff --git a/spark_transport/libsircl/tests/test_api.py b/spark_transport/libsircl/tests/test_api.py index e9cb7ac6..cddadc5b 100644 --- a/spark_transport/libsircl/tests/test_api.py +++ b/spark_transport/libsircl/tests/test_api.py @@ -155,6 +155,8 @@ class U(C.Structure): _fields_=[("b",C.c_ubyte*128)] child_uid=U(); assert l.ncclGetUniqueId(C.byref(child_uid))==5 child_comm=C.c_void_p(); assert l.ncclCommInitRank(C.byref(child_comm),1,uid,0)==5 assert child_comm.value is None + # The fork check comes first, before any argument check. + l.ncclCommRevoke.argtypes=[C.c_void_p,C.c_int]; assert l.ncclCommRevoke(None,1)==5 # libc.exit invokes DSO destructors, unlike os._exit. The inherited wake # pipe must not be written, because it would stop the parent's broker. libc=C.CDLL(None); libc.exit.argtypes=[C.c_int]; libc.exit(0) From 7815d5464082acffb52bd9f56f0d3851527f5fec Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Thu, 15 Oct 2026 09:34:38 -0500 Subject: [PATCH 50/65] Give the calling thread a failed communicator's text on queries ncclCommCount, ncclCommUserRank and ncclCommCuDevice of a failed communicator return its result together with its text for the calling thread, so ncclGetLastError(NULL) says why instead of holding whatever an earlier call left. A communicator still initializing returns ncclInProgress without text, as before; the failure does not touch the thread's group. Compatibility: result codes are unchanged; ncclGetLastError(NULL) text changes after such a query. Validation: tests/test_lifecycle.py queries a communicator whose blocking initialization timed out after another call set the thread's text: before the change ncclGetLastError(NULL) still read "version output is NULL"; after it, the timeout. make check and make check-asan pass. Request item 50, fifth fix (audit 03-F13b). NVIDIA NCCL source was neither read nor adapted. Co-Authored-By: Claude Opus 5.5 --- spark_transport/libsircl/src/api.c | 5 ++++- spark_transport/libsircl/tests/test_lifecycle.py | 5 +++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/spark_transport/libsircl/src/api.c b/spark_transport/libsircl/src/api.c index 7dc7d28a..02baae94 100644 --- a/spark_transport/libsircl/src/api.c +++ b/spark_transport/libsircl/src/api.c @@ -726,8 +726,11 @@ static ncclResult_t query(ncclComm_t handle, int *out, int field) { if (!comm) return fail(ncclInvalidArgument, "unknown communicator handle"); ncclResult_t result = (ncclResult_t)atomic_load(&comm->error); if (result == ncclSuccess) *out = field == 0 ? comm->nranks : field == 1 ? comm->rank : comm->device; + /* A failed communicator's result comes with its text, so ncclGetLastError(NULL) says why. */ + char text[sizeof comm->error_message] = {0}; + if (result != ncclSuccess && result != ncclInProgress) read_error(comm, text, sizeof text); release(comm); - return result; + return text[0] ? fail(result, text) : result; } ncclResult_t ncclCommCount(ncclComm_t comm, int *out) { return query(comm, out, 0); } diff --git a/spark_transport/libsircl/tests/test_lifecycle.py b/spark_transport/libsircl/tests/test_lifecycle.py index bd20f666..a2f196d3 100644 --- a/spark_transport/libsircl/tests/test_lifecycle.py +++ b/spark_transport/libsircl/tests/test_lifecycle.py @@ -419,6 +419,11 @@ def test_incomplete_blocking_init_retains_failure_for_cleanup(self): self.assertTrue(handle.value) self.assertEqual(self.asynchronous_error(handle), TIMEOUT) self.assertIn(b"timed out", self.lib.ncclGetLastError(handle)) + # A query of the failed handle returns its result with its text for this thread. + self.assertEqual(self.lib.ncclGetVersion(None), ARGUMENT) + count = C.c_int(-1) + self.assertEqual(self.lib.ncclCommCount(handle, C.byref(count)), TIMEOUT) + self.assertIn(b"timed out", self.lib.ncclGetLastError(None)) finally: self.dispose_all([handle]) From f2ad11d4601d66b6e7eb43acdbe998a70a28d6c3 Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Thu, 15 Oct 2026 09:35:41 -0500 Subject: [PATCH 51/65] Poll the engine's asynchronous error in ncclGetLastError(comm) ncclGetLastError(comm) on a communicator that has not recorded an error asks the engine for an asynchronous failure, as ncclCommGetAsyncError does; a failure found becomes the communicator's terminal error with its description, and the call returns that text instead of the thread's earlier text. Compatibility: no change for communicators without an asynchronous failure. Validation: an asynchronous failure needs an engine of two or more ranks, which the CPU harness cannot build; make check and make check-asan pass. Request item 50, first fix (audit 03-F3). NVIDIA NCCL source was neither read nor adapted. Co-Authored-By: Claude Opus 5.5 --- spark_transport/libsircl/src/api.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/spark_transport/libsircl/src/api.c b/spark_transport/libsircl/src/api.c index 02baae94..4f1595f5 100644 --- a/spark_transport/libsircl/src/api.c +++ b/spark_transport/libsircl/src/api.c @@ -228,6 +228,14 @@ const char *ncclGetLastError(ncclComm_t handle) { struct ncclComm *comm = acquire(handle); if (comm) { int error = atomic_load(&comm->error); + if (error == ncclSuccess && comm->engine) { + /* As ncclCommGetAsyncError: an asynchronous failure the engine found and the communicator has not yet + * recorded becomes its terminal error, with its description. */ + char message[512]; + ncclResult_t failure = sccl_engine_async_error(comm->engine, message, sizeof message); + if (failure != ncclSuccess && publish_error(comm, failure, message)) atomic_store(&comm->state, FAILED); + error = atomic_load(&comm->error); + } if (error != ncclSuccess && error != ncclInProgress) read_error(comm, last_error, sizeof(last_error)); release(comm); } From 93e0ddf4b6e1f368d4ce42e101e9b1b7bdfa7624 Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Thu, 15 Oct 2026 09:37:16 -0500 Subject: [PATCH 52/65] Name the call that failed a group, and report failed deferred creations The first call that fails a group keeps its text with the group's result. ncclGroupEnd's own failure reads "group initialization failed: ", and each pending initialization the group fails records "group initialization rejected: " (its result first, then its state, as initialization publishes them), where both texts were fixed placeholders. A blocking initialization deferred to ncclGroupEnd that fails on its own (not cancelled by the group's failure) writes the creation-failed line and sets the thread's text, as an ungrouped creation does, and becomes the group's reason when no call failed it. Compatibility: result codes are unchanged; the texts name the cause, and a failed deferred creation writes one stderr line. Validation: tests/test_lifecycle.py's nested-group test checks both texts after an InitAll with a negative device fails the outer group; before the change ncclGetLastError(NULL) read only "group initialization failed". make check and make check-asan pass. Request item 50, third fix (audit 03-F1). NVIDIA NCCL source was neither read nor adapted. Co-Authored-By: Claude Opus 5.5 --- spark_transport/libsircl/src/api.c | 27 +++++++++++++++---- .../libsircl/tests/test_lifecycle.py | 4 ++- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/spark_transport/libsircl/src/api.c b/spark_transport/libsircl/src/api.c index 4f1595f5..054f9300 100644 --- a/spark_transport/libsircl/src/api.c +++ b/spark_transport/libsircl/src/api.c @@ -121,8 +121,12 @@ static ncclResult_t fail(ncclResult_t result, const char *message) { * group's, unless the group already has one. ncclInProgress never becomes a group's result: ncclGroupEnd * would record it as the terminal error of the group's pending initializations, which would then read as in * progress forever. */ +static _Thread_local char group_reason[400]; /* the text of the call that set group_error */ static ncclResult_t group_fail(ncclResult_t result, const char *message) { - if (group_depth && group_error == ncclSuccess && result != ncclInProgress) group_error = result; + if (group_depth && group_error == ncclSuccess && result != ncclInProgress) { + group_error = result; + snprintf(group_reason, sizeof group_reason, "%s", message); + } return fail(result, message); } /* A communicator that cannot be created (ncclCommInitRank*, ncclCommSplit): its reason goes to stderr @@ -638,7 +642,7 @@ ncclResult_t ncclRecv(void *recvbuff, size_t count, ncclDataType_t datatype, int ncclResult_t ncclGroupStart(void) { CHECK_PROCESS(); if (group_depth == 256) return group_fail(ncclInvalidUsage, "group nesting exceeds 256"); - if (group_depth++ == 0) { group_error = ncclSuccess; pending_count = 0; } + if (group_depth++ == 0) { group_error = ncclSuccess; group_reason[0] = 0; pending_count = 0; } group_thread_watch(); return ncclSuccess; } @@ -647,11 +651,15 @@ ncclResult_t ncclGroupEnd(void) { if (!group_depth) return fail(ncclInvalidUsage, "GroupEnd without GroupStart"); if (--group_depth) return ncclSuccess; ncclResult_t result = group_error; + /* The reason the group failed: the first failing call's text, else the first initialization's. */ + char reason[sizeof group_reason + 64]; + snprintf(reason, sizeof reason, "%s", group_reason); for (unsigned i = 0; i < pending_count; ++i) { if (result != ncclSuccess) { - atomic_store(&pending[i]->state, FAILED); - snprintf(pending[i]->error_message, sizeof(pending[i]->error_message), "group initialization rejected"); + snprintf(pending[i]->error_message, sizeof(pending[i]->error_message), "group initialization rejected: %s", + reason); atomic_store(&pending[i]->error, result); + atomic_store(&pending[i]->state, FAILED); } else { ncclResult_t r = start(pending[i]); if (r != ncclSuccess) result = r; @@ -666,6 +674,13 @@ ncclResult_t ncclGroupEnd(void) { if (comm->blocking && comm->worker_started) { pthread_join(comm->worker, NULL); comm->worker_started = 0; ncclResult_t r = (ncclResult_t)atomic_load(&comm->error); + if (r != ncclSuccess && !atomic_load(&comm->cancelled)) { + /* A group-deferred creation that failed reports its reason as an ungrouped one does. */ + char message[1200]; + snprintf(message, sizeof message, "communicator initialization failed: %s", comm->error_message); + creation_failed(r, message); + if (!reason[0]) snprintf(reason, sizeof reason, "%s", message); + } if ((result == ncclSuccess || result == ncclInProgress) && r != ncclSuccess) result = r; } else if (!comm->blocking && result == ncclSuccess) result = ncclInProgress; atomic_store(&comm->grouped, 0); @@ -673,7 +688,9 @@ ncclResult_t ncclGroupEnd(void) { pending_count = 0; if (result != ncclSuccess && result != ncclInProgress) { flush_p2p(0); - return fail(result, "group initialization failed"); + char text[sizeof reason + 64]; + snprintf(text, sizeof text, "group initialization failed%s%s", reason[0] ? ": " : "", reason); + return fail(result, text); } ncclResult_t sent = flush_p2p(1); return sent != ncclSuccess ? sent : result; diff --git a/spark_transport/libsircl/tests/test_lifecycle.py b/spark_transport/libsircl/tests/test_lifecycle.py index a2f196d3..d331dbbf 100644 --- a/spark_transport/libsircl/tests/test_lifecycle.py +++ b/spark_transport/libsircl/tests/test_lifecycle.py @@ -290,8 +290,10 @@ def test_nested_error_propagates_to_pending_handles(self): self.assertEqual(self.lib.ncclGroupEnd(), SUCCESS) self.assertEqual(self.lib.ncclGroupEnd(), ARGUMENT) try: + # The group's end and the pending handle name the call that failed the group. + self.assertIn(b"group initialization failed: negative device number", self.lib.ncclGetLastError(None)) self.assertEqual(self.asynchronous_error(handle), ARGUMENT) - self.assertIn(b"group", self.lib.ncclGetLastError(handle)) + self.assertIn(b"group initialization rejected: negative device number", self.lib.ncclGetLastError(handle)) finally: self.dispose_all([handle]) From fa58c8708fae5a93f3efa07d471e1c838c8954ec Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Thu, 15 Oct 2026 09:37:35 -0500 Subject: [PATCH 53/65] Correct the large-reduction plan comment about ncclReduce The comment above the large-reduction plan in reduce_into said the plan applies to calls that keep their result on every rank "(not ncclReduce)". ncclReduce reaches the same chain and ring ops: off the root its destination is 0 and reduce_link runs the op in place in the staging buffer. The comment says so. Compatibility: comment only. Validation: no code changed. Request item 50, eighth fix (audit 10-F8c). NVIDIA NCCL source was neither read nor adapted. Co-Authored-By: Claude Opus 5.5 --- spark_transport/libsircl/src/engine.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/spark_transport/libsircl/src/engine.c b/spark_transport/libsircl/src/engine.c index ea8c9297..4e1bcebe 100644 --- a/spark_transport/libsircl/src/engine.c +++ b/spark_transport/libsircl/src/engine.c @@ -3594,7 +3594,8 @@ static ncclResult_t reduce_into(sccl_engine *e, const char *what, const void *se * ring op for the largest prefix of W equal chunks of whole packs; else under the chain schedule (or auto * from the chain minimum) one chain op for the 16-byte-aligned body; the rest in pieces, then the * zero-padded tail. Decided from agreed settings and sizes only, for calls that keep their result on - * every rank (not ncclReduce); unaligned buffers go through the staging buffer. */ + * every rank and for ncclReduce alike: off the root its destination is 0 and the link op runs in place in the + * staging buffer (reduce_link); unaligned buffers go through the staging buffer. */ if (body && e->shared.large_schedule != SCHEDULE_PIECES) { uint32_t schedule = effective_schedule(e, e->shared.large_schedule, nbytes, COLL_REDUCE); uint64_t link_bytes = 0; From 8ff37d25680ec323c99dc08d2034e1de1f06c074 Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Thu, 15 Oct 2026 09:39:16 -0500 Subject: [PATCH 54/65] Name the packs' architectures when a pack does not load A kernel pack that the driver cannot load (for example on a GPU whose architecture the build did not target) is refused with "loading the pack (built for sm_120 sm_121): ". The list is the build's nvcc code= values: the Makefile derives it from NVCCFLAGS and CMake from LIBSIRCL_NVCCFLAGS, and kernelpack.c falls back to the default pair when built without either. The stand-in driver (tests/fake_cuda.c) fails module loads under FAKE_CUDA_FAIL_LOAD=1 with CUDA_ERROR_NO_BINARY_FOR_GPU. Compatibility: the result is unchanged; the message names the architectures. Validation: tests/test_kernelpack_cache.c (69 checks) makes a load fail and requires the error to name "built for sm_120 sm_121" (the message had no architecture list before the change). `make -n` shows the definition on kernelpack.o. make check and make check-asan pass. Request item 50, seventh fix (audit 10-F3). NVIDIA NCCL source was neither read nor adapted. Co-Authored-By: Claude Opus 5.5 --- spark_transport/libsircl/CMakeLists.txt | 7 ++++++- spark_transport/libsircl/Makefile | 4 ++++ spark_transport/libsircl/src/kernelpack.c | 17 +++++++++++++---- spark_transport/libsircl/tests/fake_cuda.c | 2 ++ .../libsircl/tests/test_kernelpack_cache.c | 7 +++++++ 5 files changed, 32 insertions(+), 5 deletions(-) diff --git a/spark_transport/libsircl/CMakeLists.txt b/spark_transport/libsircl/CMakeLists.txt index f63b0403..3442b482 100644 --- a/spark_transport/libsircl/CMakeLists.txt +++ b/spark_transport/libsircl/CMakeLists.txt @@ -28,6 +28,10 @@ foreach(flag ${LIBSIRCL_NVCCFLAGS}) endif() endforeach() separate_arguments(LIBSIRCL_NVCC_EXTRA UNIX_COMMAND "${LIBSIRCL_NVCC_EXTRA_FLAGS}") +# The architectures the packs hold (the code= values), which a pack-load error names. +string(REGEX MATCHALL "code=[a-z0-9_]+" LIBSIRCL_PACK_CODES "${LIBSIRCL_NVCCFLAGS}") +string(REPLACE "code=" "" LIBSIRCL_PACK_CODES "${LIBSIRCL_PACK_CODES}") +string(REPLACE ";" " " LIBSIRCL_PACK_ARCHITECTURES "${LIBSIRCL_PACK_CODES}") file(GLOB LIBSIRCL_KERNEL_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/kernels/*.cuh) foreach(pack kernels fold links p2p) set(FATBIN ${CMAKE_CURRENT_BINARY_DIR}/packs/sircl_${pack}.fatbin) @@ -79,7 +83,8 @@ add_library(sircl SHARED src/api.c src/bootstrap.c src/stubs.c src/aliases.c src ${CMAKE_CURRENT_BINARY_DIR}/p2p_fatbin.c $ $) target_compile_features(sircl PRIVATE c_std_11) target_compile_definitions(sircl PUBLIC LIBSIRCL_CPU_ONLY NCCL_OS_LINUX - PRIVATE LIBSIRCL_VERSION="${PROJECT_VERSION}") + PRIVATE LIBSIRCL_VERSION="${PROJECT_VERSION}" + SCCL_PACK_ARCHITECTURES="${LIBSIRCL_PACK_ARCHITECTURES}") if(LIBSIRCL_P2P_FEATURES) target_compile_definitions(sircl PRIVATE SCCL_P2P_LOCAL_FEATURES) endif() diff --git a/spark_transport/libsircl/Makefile b/spark_transport/libsircl/Makefile index 2d18f32e..9c0842b0 100644 --- a/spark_transport/libsircl/Makefile +++ b/spark_transport/libsircl/Makefile @@ -62,6 +62,10 @@ ifeq ($(P2P_FEATURES),1) override CPPFLAGS += -DSCCL_P2P_LOCAL_FEATURES endif NVCCFLAGS = -O3 -std=c++17 -gencode arch=compute_120,code=sm_120 -gencode arch=compute_121,code=sm_121 +# The architectures the packs hold (the code= values), which a pack-load error names. +comma := , +PACK_ARCHITECTURES = $(patsubst code=%,%,$(filter code=%,$(subst $(comma), ,$(NVCCFLAGS)))) +$(BUILD)/kernelpack.o: override CPPFLAGS += -DSCCL_PACK_ARCHITECTURES='"$(PACK_ARCHITECTURES)"' .PHONY: all check clean generate kernels emulation-tools mpi-shim all: $(BUILD)/libsircl.so diff --git a/spark_transport/libsircl/src/kernelpack.c b/spark_transport/libsircl/src/kernelpack.c index cbb91d8e..e7454f3c 100644 --- a/spark_transport/libsircl/src/kernelpack.c +++ b/spark_transport/libsircl/src/kernelpack.c @@ -20,6 +20,11 @@ extern const unsigned char sccl_p2p_fatbin[]; extern const char sccl_p2p_fatbin_sha256[]; enum { MAX_CONTEXTS = 16 }; +/* The GPU architectures the packs hold cubins for (the build's nvcc -gencode code= values), named when a + * pack does not load, for example on a GPU of another architecture. */ +#ifndef SCCL_PACK_ARCHITECTURES +#define SCCL_PACK_ARCHITECTURES "sm_120 sm_121" +#endif typedef struct { sccl_CUcontext ctx; int refs; /* sccl_kp_load calls not yet released (sccl_kp_release) */ @@ -87,7 +92,8 @@ int sccl_kp_load(sccl_CUcontext ctx) { } r = cu->ModuleLoadData(&p->module, sccl_kernels_fatbin); int failed = r != SCCL_CUDA_SUCCESS; - if (failed) snprintf(error_text, sizeof error_text, "loading the kernel pack: %s", sccl_cuda_result_text(r)); + if (failed) snprintf(error_text, sizeof error_text, "loading the kernel pack (built for %s): %s", SCCL_PACK_ARCHITECTURES, + sccl_cuda_result_text(r)); for (int k = 0; !failed && k < SCCL_K_COUNT; ++k) for (int d = 0; !failed && d < SCCL_DT_COUNT; ++d) for (int w = SCCL_KP_MIN_WORLD; !failed && w <= SCCL_KP_MAX_WORLD; ++w) { @@ -107,7 +113,8 @@ int sccl_kp_load(sccl_CUcontext ctx) { if (!failed) { r = cu->ModuleLoadData(&p->fold_module, sccl_fold_fatbin); failed = r != SCCL_CUDA_SUCCESS; - if (failed) snprintf(error_text, sizeof error_text, "loading the fold pack: %s", sccl_cuda_result_text(r)); + if (failed) snprintf(error_text, sizeof error_text, "loading the fold pack (built for %s): %s", SCCL_PACK_ARCHITECTURES, + sccl_cuda_result_text(r)); } for (int d = 0; !failed && d < SCCL_FOLD_DTYPES; ++d) for (int o = 0; !failed && o < SCCL_FOLD_OPS; ++o) { @@ -122,7 +129,8 @@ int sccl_kp_load(sccl_CUcontext ctx) { if (!failed) { r = cu->ModuleLoadData(&p->links_module, sccl_links_fatbin); failed = r != SCCL_CUDA_SUCCESS; - if (failed) snprintf(error_text, sizeof error_text, "loading the link pack: %s", sccl_cuda_result_text(r)); + if (failed) snprintf(error_text, sizeof error_text, "loading the link pack (built for %s): %s", SCCL_PACK_ARCHITECTURES, + sccl_cuda_result_text(r)); } for (int d = 0; !failed && d < SCCL_DT_COUNT; ++d) for (int u = 1; !failed && u <= SCCL_CHAIN_MAX_UNROLL; ++u) { @@ -180,7 +188,8 @@ int sccl_kp_load(sccl_CUcontext ctx) { if (!failed) { r = cu->ModuleLoadData(&p->p2p_module, sccl_p2p_fatbin); failed = r != SCCL_CUDA_SUCCESS; - if (failed) snprintf(error_text, sizeof error_text, "loading the point-to-point pack: %s", sccl_cuda_result_text(r)); + if (failed) snprintf(error_text, sizeof error_text, "loading the point-to-point pack (built for %s): %s", + SCCL_PACK_ARCHITECTURES, sccl_cuda_result_text(r)); } for (int s = 0; !failed && s < 2; ++s) for (int u = 1; !failed && u <= SCCL_P2P_MAX_UNROLL; ++u) { diff --git a/spark_transport/libsircl/tests/fake_cuda.c b/spark_transport/libsircl/tests/fake_cuda.c index 96fffbb6..c80180f7 100644 --- a/spark_transport/libsircl/tests/fake_cuda.c +++ b/spark_transport/libsircl/tests/fake_cuda.c @@ -96,6 +96,8 @@ EXPORT void sccl_fake_cuda_stats(int *loaded, int *fewest_cubins, int *looked_up EXPORT int cuModuleLoadData(void **module, const void *image) { const uint8_t *p = image; + const char *fail = getenv("FAKE_CUDA_FAIL_LOAD"); + if (fail && !strcmp(fail, "1")) return 209; /* CUDA_ERROR_NO_BINARY_FOR_GPU */ int slot = 0; while (slot < nmodules && modules[slot].in_use) ++slot; if (slot == MAX_MODULES || u32(p) != 0xBA55ED50u) return INVALID_IMAGE; diff --git a/spark_transport/libsircl/tests/test_kernelpack_cache.c b/spark_transport/libsircl/tests/test_kernelpack_cache.c index d50a75ed..14b9ad6c 100644 --- a/spark_transport/libsircl/tests/test_kernelpack_cache.c +++ b/spark_transport/libsircl/tests/test_kernelpack_cache.c @@ -5,9 +5,11 @@ * * LD_LIBRARY_PATH= test_kernelpack_cache */ +#define _POSIX_C_SOURCE 200809L #include #include #include +#include #include "kernelpack.h" @@ -55,6 +57,11 @@ int main(void) { CHECK(!loaded(a) && live_modules() == 8); sccl_kp_release(b, 1); CHECK(!loaded(b)); + /* A pack the driver cannot load (no cubin for the GPU) is refused, naming the architectures it holds. */ + setenv("FAKE_CUDA_FAIL_LOAD", "1", 1); + CHECK(sccl_kp_load(a) != 0); + CHECK(strstr(sccl_kp_error(), "built for sm_120 sm_121") != NULL); + unsetenv("FAKE_CUDA_FAIL_LOAD"); printf("kernel-pack cache: %u checks passed\n", checks); return 0; } From c21fe1b95a54805e800b56ca54da5ecdacb50a55 Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Thu, 15 Oct 2026 09:41:13 -0500 Subject: [PATCH 55/65] Document the threading, stream and graph contract, and log ignored config fields README.md gains a section "Threading, streams and graphs" stating the caller contracts: one communicator is not used from several threads at once without ordering; a stream passed to a call stays valid until the next call or teardown; graph replays do not overlap eager calls or each other; the last-error text is per thread; only ncclConfig_t.blocking is honored; the group rules; what SIRCL_SITE_HASH does; and how an unsupported function refuses. Under NCCL_DEBUG (WARN, INFO or TRACE), the first ncclConfig_t of the process that sets a field other than blocking (an int that is not NCCL_CONFIG_UNDEF_INT, a name that is not NULL, within the config's size) writes one stderr line naming the field. Compatibility: the log line appears only under NCCL_DEBUG; behavior is otherwise unchanged. Validation: tests/test_engine_cpu.py creates two communicators with a config whose commName is set: under NCCL_DEBUG=WARN exactly one line names ncclConfig_t.commName (none before the change); without NCCL_DEBUG none. make check and make check-asan pass; the Markdown link check passes. Request item 47 (audit 07-F6 = 08-F2, 08-F3, 08-F4, 07-F5, 03-F2, 01-F9 = 08-F9, 01-F11, 14-F11, 14-F13, the cap of 02-F11). NVIDIA NCCL source was neither read nor adapted. Co-Authored-By: Claude Opus 5.5 --- spark_transport/libsircl/README.md | 27 +++++++++++++ spark_transport/libsircl/src/api.c | 39 +++++++++++++++++++ .../libsircl/tests/test_engine_cpu.py | 29 ++++++++++++++ 3 files changed, 95 insertions(+) diff --git a/spark_transport/libsircl/README.md b/spark_transport/libsircl/README.md index 13e10b09..a086863a 100644 --- a/spark_transport/libsircl/README.md +++ b/spark_transport/libsircl/README.md @@ -203,6 +203,33 @@ gets an error, never a plain sum); a reduction op that is not built in returns ` the receipt. Nothing is forwarded to another NCCL. `tests/api_manifest.json` marks every function of the header implemented or not. +## Threading, streams and graphs + +These are caller contracts; libsircl does not detect every violation. + +- One communicator is not used from several threads at once without ordering between them. The guards + for concurrent use (bootstrap rounds that run one after another, a second close that waits for the + first) make misuse fail safe, not correct. +- A stream passed to a collective or point-to-point call stays valid until the next call on that + communicator or its teardown: libsircl orders later calls after it and waits for the communicator's + work at finalize and destroy. +- CUDA graph replays of one communicator do not overlap its eager calls or each other; they share the + communicator's sequence and scratch. +- `ncclGetLastError(NULL)` returns the calling thread's last error text (NVIDIA NCCL's is per process); + `ncclGetLastError(comm)` returns the communicator's. +- `ncclConfig_t`: only `blocking` is honored. Every other field is accepted and ignored; under + `NCCL_DEBUG` (`WARN`, `INFO` or `TRACE`) the first config of the process that sets one writes one line + naming it. The `nccl*Config` collectives are unsupported. +- Groups: collectives inside a group run at the call; point-to-point calls are queued and run at the + outermost `ncclGroupEnd`, at most 1,024 per communicator, all on one stream; a group's pending + initializations start at `ncclGroupEnd`. A thread that ends inside its group fails its pending + initializations. +- `SIRCL_SITE_HASH` (64 hexadecimal digits) goes into every unique id this process creates, and a rank + refuses to join an id whose hash differs from its own, so processes of different sites never form a + communicator. +- An unsupported function returns `ncclInvalidUsage`, writes `libsircl: unsupported by this + build` to stderr once per process, and zeroes its handle and value outputs. + ## Build and test ```sh diff --git a/spark_transport/libsircl/src/api.c b/spark_transport/libsircl/src/api.c index 054f9300..72a6a435 100644 --- a/spark_transport/libsircl/src/api.c +++ b/spark_transport/libsircl/src/api.c @@ -277,6 +277,44 @@ static ncclResult_t start(struct ncclComm *comm) { comm->worker_started = 1; return ncclSuccess; } +/* libsircl honors only `blocking` of ncclConfig_t. Under NCCL_DEBUG (WARN, INFO or TRACE) the first config + * of the process that sets another field (an int not NCCL_CONFIG_UNDEF_INT, a name not NULL) within its + * `size` writes one stderr line naming that field. */ +static void log_ignored_config(const ncclConfig_t *config, size_t size) { + static atomic_int logged; + if (!debug_errors() || atomic_load(&logged)) return; + static const struct { const char *name; size_t offset; int pointer; } fields[] = { +#define INT_FIELD(f) {#f, offsetof(ncclConfig_t, f), 0} +#define NAME_FIELD(f) {#f, offsetof(ncclConfig_t, f), 1} + INT_FIELD(cgaClusterSize), INT_FIELD(minCTAs), INT_FIELD(maxCTAs), NAME_FIELD(netName), + INT_FIELD(splitShare), INT_FIELD(trafficClass), NAME_FIELD(commName), INT_FIELD(collnetEnable), + INT_FIELD(CTAPolicy), INT_FIELD(shrinkShare), INT_FIELD(nvlsCTAs), INT_FIELD(nChannelsPerNetPeer), + INT_FIELD(nvlinkCentricSched), INT_FIELD(graphUsageMode), INT_FIELD(numRmaCtx), INT_FIELD(maxP2pPeers), + INT_FIELD(graphStreamOrdering), INT_FIELD(launchOrderImplicit), INT_FIELD(numRmaSig), + INT_FIELD(rmaEagerInit), INT_FIELD(hostCftMode), INT_FIELD(nvlsHostMode), +#undef INT_FIELD +#undef NAME_FIELD + }; + for (size_t i = 0; i < sizeof fields / sizeof fields[0]; ++i) { + size_t width = fields[i].pointer ? sizeof(const char *) : sizeof(int); + if (fields[i].offset + width > size) continue; + int set; + if (fields[i].pointer) { + const char *name; + memcpy(&name, (const char *)config + fields[i].offset, sizeof name); + set = name != NULL; + } else { + int value; + memcpy(&value, (const char *)config + fields[i].offset, sizeof value); + set = value != NCCL_CONFIG_UNDEF_INT; + } + if (set && !atomic_exchange(&logged, 1)) { + fprintf(stderr, "libsircl: ncclConfig_t.%s is set and ignored: libsircl honors only blocking\n", + fields[i].name); + return; + } + } +} /* The blocking mode of a new communicator: the config's, or `inherited` when the config is NULL or leaves the * field undefined (1 for ncclCommInitRank*, the parent's for ncclCommSplit). */ static ncclResult_t config_blocking(const ncclConfig_t *config, int inherited, int *blocking) { @@ -293,6 +331,7 @@ static ncclResult_t config_blocking(const ncclConfig_t *config, int inherited, i if (magic != NCCL_API_MAGIC || version < 21400) return fail(ncclInvalidArgument, "config magic or version is invalid"); memcpy(blocking, (const char *)config + offsetof(ncclConfig_t, blocking), sizeof(*blocking)); + log_ignored_config(config, size); if (*blocking == NCCL_CONFIG_UNDEF_INT) *blocking = inherited; if (*blocking != 0 && *blocking != 1) return fail(ncclInvalidArgument, "config blocking must be 0, 1, or UNDEF"); diff --git a/spark_transport/libsircl/tests/test_engine_cpu.py b/spark_transport/libsircl/tests/test_engine_cpu.py index 45e8849c..d63d8d67 100644 --- a/spark_transport/libsircl/tests/test_engine_cpu.py +++ b/spark_transport/libsircl/tests/test_engine_cpu.py @@ -314,6 +314,35 @@ def test_mem_free_makes_the_memorys_context_current(self): self.assertEqual(out.returncode, 0, out.stderr) self.assertEqual(json.loads(out.stdout), [0, 1]) + def test_an_ignored_config_field_is_logged_once_under_nccl_debug(self): + """libsircl honors only ncclConfig_t.blocking: under NCCL_DEBUG=WARN the first config that sets another + field writes one line naming it; without NCCL_DEBUG nothing is written.""" + script = '''class Config(C.Structure): + _fields_ = [("size", C.c_size_t), ("magic", C.c_uint), ("version", C.c_uint), ("blocking", C.c_int), + ("cgaClusterSize", C.c_int), ("minCTAs", C.c_int), ("maxCTAs", C.c_int), ("netName", C.c_char_p), + ("splitShare", C.c_int), ("trafficClass", C.c_int), ("commName", C.c_char_p)] + [ + (n, C.c_int) for n in ("collnetEnable", "CTAPolicy", "shrinkShare", "nvlsCTAs", + "nChannelsPerNetPeer", "nvlinkCentricSched", "graphUsageMode", "numRmaCtx", "maxP2pPeers", + "graphStreamOrdering", "launchOrderImplicit", "numRmaSig", "rmaEagerInit", "hostCftMode", + "nvlsHostMode")] +l.ncclCommInitRankConfig.argtypes=[C.POINTER(C.c_void_p),C.c_int,U,C.c_int,C.POINTER(Config)] +for name in ("commName", "commName"): + cfg=Config(); cfg.size=C.sizeof(Config); cfg.magic=0xcafebeef; cfg.version=23203 + for field, kind in Config._fields_[3:]: + if kind is C.c_int: setattr(cfg, field, -2147483648) + cfg.commName=b"tp0" + u=U(); assert l.ncclGetUniqueId(C.byref(u))==0 + c=C.c_void_p(); assert l.ncclCommInitRankConfig(C.byref(c),1,u,0,C.byref(cfg))==0 + assert l.ncclCommDestroy(c)==0 +''' + loud = run(script, {"LIBSIRCL_BOOTSTRAP_ONLY": "1", "NCCL_DEBUG": "WARN"}) + self.assertEqual(loud.returncode, 0, loud.stderr) + lines = [line for line in loud.stderr.splitlines() if "is set and ignored" in line] + self.assertEqual(lines, ["libsircl: ncclConfig_t.commName is set and ignored: libsircl honors only blocking"]) + quiet = run(script, {"LIBSIRCL_BOOTSTRAP_ONLY": "1", "NCCL_DEBUG": ""}) + self.assertEqual(quiet.returncode, 0, quiet.stderr) + self.assertNotIn("is set and ignored", quiet.stderr) + def test_info_names_the_collectives(self): out = run("print(l.sirclGetInfo().decode())") self.assertEqual(out.returncode, 0, out.stderr) From 9effe190d3870f809966b313943c307de9ca27e5 Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Thu, 15 Oct 2026 09:41:30 -0500 Subject: [PATCH 56/65] Document how libsircl's error strings and NULL handling differ from NVIDIA NCCL README.md states that libsircl's result codes have NVIDIA NCCL's values and meaning, that ncclGetErrorString returns short texts without NVIDIA NCCL's hints to rerun with NCCL_DEBUG (the reason is in ncclGetLastError), and how a NULL communicator is handled: ncclCommDestroy and ncclCommAbort return ncclSuccess as NVIDIA NCCL 2.32.3 does, while ncclCommFinalize returns ncclInvalidArgument where NVIDIA NCCL 2.32.3 returns ncclSuccess. Compatibility: documentation only. Validation: the Markdown link check passes. Request item 50, sixth fix (audit 01-F8 = 03-F7). NVIDIA NCCL source was read for the NULL behavior (init.cc of v2.32.3-1); its error-string texts were not copied. Co-Authored-By: Claude Opus 5.5 --- spark_transport/libsircl/README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/spark_transport/libsircl/README.md b/spark_transport/libsircl/README.md index a086863a..dd3edeff 100644 --- a/spark_transport/libsircl/README.md +++ b/spark_transport/libsircl/README.md @@ -43,6 +43,11 @@ gives the build, emulation and hardware commands. session setup failed) still returns the handle, whose `ncclCommGetAsyncError` and `ncclGetLastError` give the result and reason; the failed communicator holds no bootstrap connection or session, but the caller must still destroy or abort the handle to free it. +- Result codes have NVIDIA NCCL's values and meaning. `ncclGetErrorString` returns short texts ("invalid + usage", "remote error", ...) without NVIDIA NCCL's hints to rerun with `NCCL_DEBUG`; the reason of a + failed call is in `ncclGetLastError`. `ncclCommDestroy(NULL)` and `ncclCommAbort(NULL)` return + `ncclSuccess`, as in NVIDIA NCCL 2.32.3; `ncclCommFinalize(NULL)` returns `ncclInvalidArgument`, where + NVIDIA NCCL 2.32.3 returns `ncclSuccess`. - `include/nccl.h` is NVIDIA's 2.32.3 header with marked changes (`NCCL_VERSION_CODE` 23203), used only to build libsircl. - The name `libsircl` without a suffix means this library; SIRCL's own test libraries are named From 562317bec4caaa861cd6d0f3d1f6bc39ae3cca72 Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Thu, 15 Oct 2026 09:51:40 -0500 Subject: [PATCH 57/65] Carry LIBSIRCL_RELAYED_LANES through the installer's libsircl lane tools/site_routes.py prints LIBSIRCL_RELAYED_LANES on every rank of a layout with relays, and the installer's libsircl transport (runtime/common/libsircl.py) requires each rank's routing settings to equal the tool's output. The variable joins ROUTE_VARIABLES, so the lane records it per rank in the deployment lock and sets it in the rank's container, and a profile cannot set it. docs/architecture/libsircl.md's container-settings table lists it. Compatibility: deployments planned on layouts with relays gain the variable in their locks and containers; layouts without relays are unchanged. Validation: runtime/common/test_libsircl.py: the five tests that compare the lane's routing settings with the tool's output failed after the tool began printing the variable and pass with it carried (46 passed). The Markdown link check passes. Follow-up to request item 24 (audit 09-1). NVIDIA NCCL source was neither read nor adapted. Co-Authored-By: Claude Opus 5.5 --- docs/architecture/libsircl.md | 2 +- runtime/common/libsircl.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/architecture/libsircl.md b/docs/architecture/libsircl.md index faa03dee..bbc515e7 100644 --- a/docs/architecture/libsircl.md +++ b/docs/architecture/libsircl.md @@ -236,7 +236,7 @@ Each rank's container gets the profile's settings with these changes | `LIBSIRCL_FAIL_STOP` | `1` | | `LIBSIRCL_NCCL_API_VERSION` | `22705`, libsircl's default: it passes vLLM's and torch's feature gates without opening newer ones | | `LIBSIRCL_TRANSPORT` | `verbs` | -| `LIBSIRCL_POSITION`, `SIRCL_PEER_ROUTES`, `LIBSIRCL_CHAIN_ORDER`, `LIBSIRCL_FORWARD_WINDOWS`, `SIRCL_FORWARD_CHUNK_BYTES`, `LIBSIRCL_RING_WINDOW`, `LIBSIRCL_P2P_WINDOWS`, `SIRCL_P2P_CHUNK_BYTES` | the rank's routing settings, recorded per rank in the deployment lock | +| `LIBSIRCL_POSITION`, `SIRCL_PEER_ROUTES`, `LIBSIRCL_CHAIN_ORDER`, `LIBSIRCL_RELAYED_LANES`, `LIBSIRCL_FORWARD_WINDOWS`, `SIRCL_FORWARD_CHUNK_BYTES`, `LIBSIRCL_RING_WINDOW`, `LIBSIRCL_P2P_WINDOWS`, `SIRCL_P2P_CHUNK_BYTES` | the rank's routing settings, recorded per rank in the deployment lock | | `SIRCL_BOOTSTRAP_ADDR` | the rank's `VLLM_HOST_IP` | | `SIRCL_GID_INDEX` | the profile's `NCCL_IB_GID_INDEX`, else 3 | | `LIBSIRCL_RECEIPT` | `/run/sparkring/sircl/receipts/libsircl`: one receipt per communicator in the deployment's receipt directory on each Spark | diff --git a/runtime/common/libsircl.py b/runtime/common/libsircl.py index e0a7be96..2e475e70 100644 --- a/runtime/common/libsircl.py +++ b/runtime/common/libsircl.py @@ -89,8 +89,9 @@ # The kernel packs every libsircl build carries (runtime/images/libsircl_layer.py builds each from its CUDA # source); a layer receipt that records another set describes another libsircl build. KERNEL_PACKS = ("sircl_kernels", "sircl_fold", "sircl_links", "sircl_p2p") -ROUTE_VARIABLES = ("LIBSIRCL_POSITION", "SIRCL_PEER_ROUTES", "LIBSIRCL_CHAIN_ORDER", "LIBSIRCL_FORWARD_WINDOWS", - "SIRCL_FORWARD_CHUNK_BYTES", "LIBSIRCL_RING_WINDOW", "LIBSIRCL_P2P_WINDOWS", "SIRCL_P2P_CHUNK_BYTES") +ROUTE_VARIABLES = ("LIBSIRCL_POSITION", "SIRCL_PEER_ROUTES", "LIBSIRCL_CHAIN_ORDER", "LIBSIRCL_RELAYED_LANES", + "LIBSIRCL_FORWARD_WINDOWS", "SIRCL_FORWARD_CHUNK_BYTES", "LIBSIRCL_RING_WINDOW", "LIBSIRCL_P2P_WINDOWS", + "SIRCL_P2P_CHUNK_BYTES") LIBRARY_VARIABLE = "SPARKRING_LIBSIRCL_LIBRARY" DIGEST_VARIABLE = "SPARKRING_LIBSIRCL_SHA256" # Every communicator's receipt, .rank..c.json, in the deployment's receipt directory. From 5fe15fb189c04db40fcd70d3eab0fecf07d077b1 Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Thu, 15 Oct 2026 09:51:59 -0500 Subject: [PATCH 58/65] State the libsircl lane's wait regime and dead-peer detection time docs/architecture/libsircl.md's fail-stop section states that the installer's libsircl lane sets no wait variable, so every communicator runs in the startup wait regime at libsircl's default limit of 600 s for its whole life (nothing calls sirclSetWaitRegime), and a rank whose peer died learns of it only when a wait reaches that limit; fail-stop then ends the process. runtime/common/test_libsircl.py pins this: the lane's library environment sets none of LIBSIRCL_WAIT_REGIME, SIRCL_STARTUP_WAIT_S, SIRCL_SERVING_WAIT_S and LIBSIRCL_SETUP_TIMEOUT_MS, the vLLM plugin does not call sirclSetWaitRegime, and the section names the startup wait regime and the limit that src/engine.c's DEFAULT_STARTUP_WAIT_S defines, so the document, the adapter and the library change together. Compatibility: documentation and test only. Validation: the test fails against the section without the paragraph and passes with it; runtime/common/test_libsircl.py passes (46 passed); the Markdown link check passes. Request item 49 (audit 16-F5, documentation part of 04-F1 and 05-F5). NVIDIA NCCL source was neither read nor adapted. Co-Authored-By: Claude Opus 5.5 --- docs/architecture/libsircl.md | 7 +++++++ runtime/common/test_libsircl.py | 20 ++++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/docs/architecture/libsircl.md b/docs/architecture/libsircl.md index bbc515e7..b9ea3ba2 100644 --- a/docs/architecture/libsircl.md +++ b/docs/architecture/libsircl.md @@ -224,6 +224,13 @@ refuses a library without the mode. The layer records `fail_stop` in the lock when the library's bytes name `LIBSIRCL_FAIL_STOP`; the stock-image preflight reads the same mark from the host library. +The lane sets no wait variable, so every communicator runs in the startup +wait regime with libsircl's default limit of 600 s (`SIRCL_STARTUP_WAIT_S`) +for its whole life: nothing switches it to the serving regime +(`sirclSetWaitRegime`). A rank whose peer died learns of it only when one of +its waits reaches that limit, 600 s after the wait began; fail-stop then ends +the process. + ### Container settings Each rank's container gets the profile's settings with these changes diff --git a/runtime/common/test_libsircl.py b/runtime/common/test_libsircl.py index 8acc4fcb..4a433b04 100644 --- a/runtime/common/test_libsircl.py +++ b/runtime/common/test_libsircl.py @@ -8,6 +8,7 @@ import json import os from pathlib import Path +import re import subprocess import sys @@ -251,6 +252,25 @@ def test_two_pipeline_stages_and_expert_parallelism_on_grouped_all_gathers_pass( "allgather_reducescatter"]) == [] +def test_the_lane_runs_in_the_startup_wait_regime_at_the_librarys_limit(): + """The lane sets no wait variable, so every communicator runs in the startup regime at libsircl's default + limit; the architecture document's fail-stop section states that regime and limit, so the document, the + adapter and the library change together.""" + _, section = deployment(TP2, "pair", 2, [0, 1]) + environment = libsircl.library_environment(section["libsircl"]) + assert not {"LIBSIRCL_WAIT_REGIME", "SIRCL_STARTUP_WAIT_S", "SIRCL_SERVING_WAIT_S", + "LIBSIRCL_SETUP_TIMEOUT_MS"} & set(environment) + root = Path(__file__).resolve().parents[2] + engine = (root / "spark_transport/libsircl/src/engine.c").read_text(encoding="utf-8") + limit = re.search(r"#define DEFAULT_STARTUP_WAIT_S (\d+)\.0", engine).group(1) + text = (root / "docs/architecture/libsircl.md").read_text(encoding="utf-8") + fail_stop = text[text.index("### Fail-stop"):text.index("### Container settings")] + words = " ".join(fail_stop.split()) + assert "startup wait regime" in words and f"limit of {limit} s" in words + plugin = (root / "integrations/vllm/libsircl/sparkring_libsircl.py").read_text(encoding="utf-8") + assert "sirclSetWaitRegime" not in plugin + + def test_a_library_without_the_fail_stop_mode_is_refused(): value = image() value["libsircl"]["fail_stop"] = False From a5e0a50b8545b009efd9b58f83821210a3347a11 Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Thu, 15 Oct 2026 09:54:11 -0500 Subject: [PATCH 59/65] Describe SOURCE_SNAPSHOT.json and the rdma-core notice as they are NOTICE said SOURCE_SNAPSHOT.json records "the SIRCL files this library follows". It records the SHA-256 of every file of the SIRCL implementation tree that STATUS.md's evidence was taken against, a dated capture of a development workspace, while the library is built against this repository's spark_transport/sircl and the vendored copies under src/transport. NOTICE now says so. LICENSES/rdma-core-verbs.txt said a release built in another image copies that image's header. Every libsircl image layer ships this file as it is in the source tree (runtime/images/libsircl_layer.py copies the notices runtime/common/libsircl.py lists); the file says so. tests/test_source_snapshot.py (make check, CMake's source_snapshot) checks the snapshot's schema (its keys, the selected source, the capture time, the stated file count, every entry a relative path with a 64-digit SHA-256 and a byte count, the change lists) and NOTICE's description of it. Compatibility: documentation and a test; the notice files the layer ships change text. Validation: the NOTICE check failed before the change and passes after it; the schema check passes; make check and make check-asan pass; runtime/images/test_libsircl_layer.py passes (19 passed). Request item 46 (audit 11-1, 11-11). NVIDIA NCCL source was neither read nor adapted. Co-Authored-By: Claude Opus 5.5 --- spark_transport/libsircl/CMakeLists.txt | 2 +- .../libsircl/LICENSES/rdma-core-verbs.txt | 5 +- spark_transport/libsircl/Makefile | 2 +- spark_transport/libsircl/NOTICE | 6 ++- .../libsircl/tests/test_source_snapshot.py | 51 +++++++++++++++++++ 5 files changed, 60 insertions(+), 6 deletions(-) create mode 100644 spark_transport/libsircl/tests/test_source_snapshot.py diff --git a/spark_transport/libsircl/CMakeLists.txt b/spark_transport/libsircl/CMakeLists.txt index 3442b482..6a4dd41b 100644 --- a/spark_transport/libsircl/CMakeLists.txt +++ b/spark_transport/libsircl/CMakeLists.txt @@ -141,7 +141,7 @@ add_test(NAME kernelpack_cache COMMAND ${CMAKE_COMMAND} -E env foreach(name abi api lifecycle engine_cpu build_flags) add_test(NAME ${name} COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tests/test_${name}.py --library $) endforeach() -foreach(name bootstrap shm_verbs vendored_sources p2p_native site_routes mpi_shim check_nccl_tests) +foreach(name bootstrap shm_verbs vendored_sources p2p_native site_routes mpi_shim check_nccl_tests source_snapshot) add_test(NAME ${name} COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tests/test_${name}.py) endforeach() add_test(NAME kernel_entries COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tests/test_kernel_entries.py diff --git a/spark_transport/libsircl/LICENSES/rdma-core-verbs.txt b/spark_transport/libsircl/LICENSES/rdma-core-verbs.txt index 626aea21..08b8715e 100644 --- a/spark_transport/libsircl/LICENSES/rdma-core-verbs.txt +++ b/spark_transport/libsircl/LICENSES/rdma-core-verbs.txt @@ -1,7 +1,8 @@ The RDMA transport of libsircl is compiled against from rdma-core and contains code from that header's inline functions (among them ibv_post_send and ibv_poll_cq), used under the OpenIB.org -BSD license option. The notices below are copied from the header of rdma-core 50.0 on the build -workstation; a release built in another image copies that image's header. +BSD license option. The notices below are copied from the header of rdma-core 50.0. Every libsircl +image layer ships this file as it is in the source tree (runtime/images/libsircl_layer.py copies the +notices listed in runtime/common/libsircl.py), whichever rdma-core header its build compiled against. Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. Copyright (c) 2004, 2011-2012 Intel Corporation. All rights reserved. diff --git a/spark_transport/libsircl/Makefile b/spark_transport/libsircl/Makefile index 9c0842b0..b8e17b6f 100644 --- a/spark_transport/libsircl/Makefile +++ b/spark_transport/libsircl/Makefile @@ -201,7 +201,7 @@ check: all $(BUILD)/test_fabric $(BUILD)/test_link_plan $(BUILD)/test_kernelpack $(PYTHON) tests/test_site_routes.py || status=1; \ $(PYTHON) tests/test_mpi_shim.py || status=1; \ $(PYTHON) tests/test_check_nccl_tests.py || status=1; \ - $(PYTHON) tests/test_kernel_entries.py --build $(BUILD) || status=1; $(PYTHON) tests/test_build_flags.py --library $(abspath $(BUILD)/libsircl.so) || status=1; \ + $(PYTHON) tests/test_kernel_entries.py --build $(BUILD) || status=1; $(PYTHON) tests/test_build_flags.py --library $(abspath $(BUILD)/libsircl.so) || status=1; $(PYTHON) tests/test_source_snapshot.py || status=1; \ exit $$status # The check suites against a host library built with AddressSanitizer and UndefinedBehaviorSanitizer in diff --git a/spark_transport/libsircl/NOTICE b/spark_transport/libsircl/NOTICE index 3c00b48a..b94ffa4a 100644 --- a/spark_transport/libsircl/NOTICE +++ b/spark_transport/libsircl/NOTICE @@ -23,8 +23,10 @@ SIRCL components kernels, device intrinsics and progress-thread protocol derive from RoCEnante, the b12x one-shot RoCE collectives by Local Inference Lab and its contributors (https://github.com/local-inference-lab, b12x commit 236ddff04a5f5064084b46464800b70b34ec7d8c), licensed under the Apache License, - Version 2.0. vendor/SIRCL-NOTICE reproduces SIRCL's NOTICE; SOURCE_SNAPSHOT.json records the - SIRCL files this library follows. + Version 2.0. vendor/SIRCL-NOTICE reproduces SIRCL's NOTICE. SOURCE_SNAPSHOT.json records the SHA-256 + of every file of the SIRCL implementation tree that STATUS.md's evidence was taken against, a dated + capture of a development workspace; the library is built against this repository's + spark_transport/sircl and the vendored copies under src/transport. libibverbs header code The RDMA transport is compiled against from rdma-core and contains code from diff --git a/spark_transport/libsircl/tests/test_source_snapshot.py b/spark_transport/libsircl/tests/test_source_snapshot.py new file mode 100644 index 00000000..36efc6d6 --- /dev/null +++ b/spark_transport/libsircl/tests/test_source_snapshot.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +"""SOURCE_SNAPSHOT.json's schema, and what NOTICE says the file records. + +SOURCE_SNAPSHOT.json is a dated capture: the SHA-256 and size of every file of the SIRCL implementation tree +that STATUS.md's evidence was taken against. It is not the in-repository SIRCL package the library is built +against, and nothing enforces it at build time. These checks keep its shape readable (every entry a relative +path with a 64-digit lowercase SHA-256 and a byte count; the stated file count; the changes from the previous +capture as path lists) and keep NOTICE describing it as that capture. +""" +import json +import re +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +class SourceSnapshotTests(unittest.TestCase): + def setUp(self): + self.snapshot = json.loads((ROOT / "SOURCE_SNAPSHOT.json").read_text(encoding="utf-8")) + + def test_the_schema(self): + data = self.snapshot + self.assertEqual(set(data), {"description", "sources", "selected", "captured_utc", + "changes_from_previous_capture", "files"}) + self.assertIn(data["selected"], data["sources"]) + self.assertRegex(data["captured_utc"], r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$") + stated = re.search(r"(\d+) files\.?$", data["description"]) + self.assertIsNotNone(stated, "the description states the file count") + self.assertEqual(len(data["files"]), int(stated.group(1))) + for path, entry in data["files"].items(): + with self.subTest(path=path): + self.assertFalse(path.startswith("/") or ".." in Path(path).parts or "\\" in path) + self.assertEqual(set(entry), {"sha256", "bytes"}) + self.assertRegex(entry["sha256"], r"^[0-9a-f]{64}$") + self.assertIsInstance(entry["bytes"], int) + self.assertGreaterEqual(entry["bytes"], 0) + changes = data["changes_from_previous_capture"] + self.assertEqual(set(changes), {"changed", "added", "removed"}) + for kind, paths in changes.items(): + self.assertIsInstance(paths, list, kind) + self.assertTrue(all(isinstance(path, str) and path for path in paths), kind) + + def test_notice_describes_the_capture(self): + notice = " ".join((ROOT / "NOTICE").read_text(encoding="utf-8").split()) + self.assertIn("SOURCE_SNAPSHOT.json records the SHA-256 of every file of the SIRCL implementation tree " + "that STATUS.md's evidence was taken against", notice) + + +if __name__ == "__main__": + unittest.main(verbosity=2) From b29b4d188367268964cee9ecede4907a558ffa8d Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Thu, 15 Oct 2026 09:58:13 -0500 Subject: [PATCH 60/65] Bring libsircl's status documents in line with the code and the records README.md and STATUS.md state the research-only status first. README names the API coverage the manifest records (36 of the header's 73 functions; the other 37 return ncclInvalidUsage) instead of saying STATUS.md lists each function, and STATUS.md's unsupported row lists all 37 by name. Both name this repository's SIRCL package as 0.3.2. STATUS.md: - the headline's bit-identity claim is narrowed to what the body shows: float16, bfloat16 and float32 sum all-reduces, all-gathers and reduce-scatters against the SIRCL Python session, every other datatype and op against the fold model and the rank-order reference; - four kernel packs, not three or "both"; - rows that said "Not run on Sparks" or "in emulation only" cite the Spark runs the evidence sections record (point-to-point channels and the LF setup checks on snapshots 0f94c72b and a3477af2, the cycle plan in the ring-of-eight image record, vLLM's PyNccl on a cabled pair); - the evidence section says the cited verification/ records stay on the machines that made them, and the external artifacts include the gate script gate_multi_0f94c72b.sh; - hard-coded test counts are removed, and a reference to a document outside the repository is described instead of cited by section. RUNBOOK.md states which sections ran, drops the undefined "ring measurement lock", the stale "pending 3.4" and "stays pieces" (the pair and cycle defaults), and names what another image changes. KERNEL_ROUTE.md labels the older link-pack hash, replaces the dangling report and section references, and counts all four packs in the setup agreement. tests/test_docs_consistency.py (make check, CMake's docs_consistency) checks README's function counts and STATUS's unsupported list against tests/api_manifest.json, the absence of "three" or "both" packs against the loader's four, and the SIRCL package version against its pyproject.toml. Compatibility: documentation and a test. Validation: the new test failed on the documents before the change (the unsupported list, "three embedded packs", the version) and passes after it; make check and make check-asan pass; the Markdown link check passes. Request item 45 (audit 14-F1 to 14-F9, 14-F12, 16-F1, 16-F2, 16-F4, 16-F9, 10-F7). NVIDIA NCCL source was neither read nor adapted. Co-Authored-By: Claude Opus 5.5 --- spark_transport/libsircl/CMakeLists.txt | 2 +- spark_transport/libsircl/KERNEL_ROUTE.md | 8 +-- spark_transport/libsircl/Makefile | 1 + spark_transport/libsircl/README.md | 10 ++- spark_transport/libsircl/RUNBOOK.md | 17 +++-- spark_transport/libsircl/STATUS.md | 64 +++++++++---------- .../libsircl/tests/test_docs_consistency.py | 55 ++++++++++++++++ 7 files changed, 111 insertions(+), 46 deletions(-) create mode 100644 spark_transport/libsircl/tests/test_docs_consistency.py diff --git a/spark_transport/libsircl/CMakeLists.txt b/spark_transport/libsircl/CMakeLists.txt index 6a4dd41b..ed9a6976 100644 --- a/spark_transport/libsircl/CMakeLists.txt +++ b/spark_transport/libsircl/CMakeLists.txt @@ -141,7 +141,7 @@ add_test(NAME kernelpack_cache COMMAND ${CMAKE_COMMAND} -E env foreach(name abi api lifecycle engine_cpu build_flags) add_test(NAME ${name} COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tests/test_${name}.py --library $) endforeach() -foreach(name bootstrap shm_verbs vendored_sources p2p_native site_routes mpi_shim check_nccl_tests source_snapshot) +foreach(name bootstrap shm_verbs vendored_sources p2p_native site_routes mpi_shim check_nccl_tests source_snapshot docs_consistency) add_test(NAME ${name} COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tests/test_${name}.py) endforeach() add_test(NAME kernel_entries COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tests/test_kernel_entries.py diff --git a/spark_transport/libsircl/KERNEL_ROUTE.md b/spark_transport/libsircl/KERNEL_ROUTE.md index 04befd21..cb41e66b 100644 --- a/spark_transport/libsircl/KERNEL_ROUTE.md +++ b/spark_transport/libsircl/KERNEL_ROUTE.md @@ -87,13 +87,13 @@ The source holds no compiled pack. |---|---|---|---| | nvcc 13.3.73, `-O3 -std=c++17`, sm_120 + sm_121 SASS, WSL2 x86_64 | Build time, registers, spills; rebuilds from two source paths | 6.1 s for 154 cubins; 34-64 registers, 0 spill bytes; fatbin 2,714,984 bytes, SHA-256 `c2e6e5a1...be4eaa25`, identical on every rebuild | The pack is reproducible and path independent: one nvcc and one source give one pack, whatever the build directory. | | Same compiler and flags, fold pack | Build time, registers, spills; rebuilds from two source paths | 4.0 s for 120 cubins; 46-64 registers, 0 spill bytes, 0-byte stack frames; fatbin 1,388,776 bytes, SHA-256 `66da585d...9f11eee9`, identical on every rebuild | The fold pack is reproducible like the transport pack. | -| Same compiler and flags, link pack | Build time, registers, spills; rebuilds from two source paths | 55 s for 336 cubins; 56-128 registers, 0 spill bytes, 0-byte stack frames (the pair reduce exchange at unroll 4: 96 registers); fatbin 16,734,040 bytes, SHA-256 `2fb2fb0b...f9b340b7`, identical from both paths | The link pack is reproducible like the others. With a 1,024-thread launch bound (64 registers) the unroll-4 chain reduce-scatter spilled 20-44 bytes and unrolls 6-8 more; the 512-thread bound removes every spill. | +| Same compiler and flags, link pack | Build time, registers, spills; rebuilds from two source paths | 55 s for 336 cubins; 56-128 registers, 0 spill bytes, 0-byte stack frames (the pair reduce exchange at unroll 4: 96 registers); fatbin 16,734,040 bytes, SHA-256 `2fb2fb0b...f9b340b7` (an earlier revision of the link pack's source; `STATUS.md` names the packs the current sources build), identical from both paths | The link pack is reproducible like the others. With a 1,024-thread launch bound (64 registers) the unroll-4 chain reduce-scatter spilled 20-44 bytes and unrolls 6-8 more; the 512-thread bound removes every spill. | | SIRCL's DSL (nvidia-cutlass-dsl 4.5.0.dev0, NVVM 12.9) with `CUTE_DSL_KEEP_PTX`, sm_120a | Arithmetic of the dumped one-shot PTX | `add.f32` (round to nearest, no flush to zero), conversions by SIRCL's own inline PTX | The port uses `add.rn.f32` and the identical conversion PTX, so each reduction is the same IEEE computation. | | SIRCL's emulation harness unmodified (`EmulatedGroup`: ranks as threads of one process, RTX 5090, SIRCL's in-process verbs stand-in and native progress thread); per rank either SIRCL's DSL launchers or the pack launched from C with the session's own arena, counters and stream (`tests/emulation/mixed_group.py`) | 181 checks per run: all-reduce one-shot and two-shot, auto and forced, 16 B to 256 KiB, bf16, fp16, fp32; `all_reduce_large` with padded tails; reduce-scatter (scatter ops) with chunks of 16 B to 256 KiB; all-gather along dimension 0 and the last dimension, direct, padded and tiled; all-to-all; one CUDA graph per rank holding all of them, replayed for two seeds; all-DSL, mixed (even ranks C++, odd ranks DSL) and all-C++ back to back on the same sessions; a C++ rank alone under a 0.5 s serving limit | 905 of 905 passed: `path:0-1` (1 lane), `ring:3` (1 lane), `path:0-3` (2 lanes; one-block and every-block polling) and `ring:8` (2 lanes). Every output equals the host reference and the DSL output of the same inputs bit for bit, eager and in graph replay; the lone rank poisoned its session after 0.51 s and named peer 1, lane 0 and the sequence | The C++ kernels are wire-compatible with SIRCL's and reproduce the SIRCL Python session's outputs bit for bit. | | Same harness, DSL launchers | Compile time of `prepare()` | 104 s (pair) to 512 s (`ring:8`, eight ranks in one process) | The DSL compiles per (dtype, world, rank, lanes, polling) at run time; the pack compiles nothing at run time. | | DSL specializations for 2-8 ranks | Count of one-shot and two-shot cubins per architecture (rank, world, lanes and polling are compile-time constants there) | 35 (world, rank) pairs x 3 dtypes x 2 lanes x 2 polling modes x 2 algorithms = 840, before all-gather, scatter and link kernels | Cubin extraction multiplies files and ties each to one DSL version's undocumented parameter packing. | -| RTX 5090, WSL2, driver 595.79; pack kernel on a poisoned session so each launch returns at once (`tests/emulation/launch_bench.c`) | Host time per `cuLaunchKernel` from C, 20,000 launches in batches of 100 | one-shot 4.25 us mean (3.48 us best batch), two-shot 4.19 us (3.67 us) | A C launch costs what `research/nccl-compat/REPORT.md` section 6.4 measured for an empty kernel with the same parameter count on this machine (3.9 us), against 49.7 us through the DSL executor called from Python. | -| `nvidia-cutlass-dsl` licence as installed (section 1.1) | Distribution grant | Python files of the package in source form; no grant names compiled output; developer tools are for internal use unless identified as distributable | Distributing DSL-generated cubins has no stated permission; the CUDA C++ pack carries only SIRCL's Apache-2.0 terms and the b12x RoCEnante attribution (NOTICE). | +| RTX 5090, WSL2, driver 595.79; pack kernel on a poisoned session so each launch returns at once (`tests/emulation/launch_bench.c`) | Host time per `cuLaunchKernel` from C, 20,000 launches in batches of 100 | one-shot 4.25 us mean (3.48 us best batch), two-shot 4.19 us (3.67 us) | A C launch costs what an earlier measurement on this machine, outside this repository's evidence, found for an empty kernel with the same parameter count (3.9 us), against 49.7 us through the DSL executor called from Python. | +| `nvidia-cutlass-dsl` licence as installed | Distribution grant | Python files of the package in source form; no grant names compiled output; developer tools are for internal use unless identified as distributable | Distributing DSL-generated cubins has no stated permission; the CUDA C++ pack carries only SIRCL's Apache-2.0 terms and the b12x RoCEnante attribution (NOTICE). | ## Consequences @@ -101,7 +101,7 @@ The source holds no compiled pack. - Bit-exactness of every further kernel is proven the same way: the mixed-group check runs the ported kernel against SIRCL's DSL kernel in one group, on the same sessions, eager and captured. - Kernel changes inside the SIRCL package (an abort word in the timed waits, new collectives) are - ported here after they land there; both packs' hashes join every communicator's setup agreement, so + ported here after they land there; all four packs' hashes join every communicator's setup agreement, so ranks with different packs refuse to form a session. - Every kernel of SIRCL's session that a collective of the NCCL API reaches is ported; kernels SIRCL adds later (an abort word in the timed waits, new collectives) are ported after they land there. diff --git a/spark_transport/libsircl/Makefile b/spark_transport/libsircl/Makefile index b8e17b6f..66a7f7dd 100644 --- a/spark_transport/libsircl/Makefile +++ b/spark_transport/libsircl/Makefile @@ -202,6 +202,7 @@ check: all $(BUILD)/test_fabric $(BUILD)/test_link_plan $(BUILD)/test_kernelpack $(PYTHON) tests/test_mpi_shim.py || status=1; \ $(PYTHON) tests/test_check_nccl_tests.py || status=1; \ $(PYTHON) tests/test_kernel_entries.py --build $(BUILD) || status=1; $(PYTHON) tests/test_build_flags.py --library $(abspath $(BUILD)/libsircl.so) || status=1; $(PYTHON) tests/test_source_snapshot.py || status=1; \ + $(PYTHON) tests/test_docs_consistency.py || status=1; \ exit $$status # The check suites against a host library built with AddressSanitizer and UndefinedBehaviorSanitizer in diff --git a/spark_transport/libsircl/README.md b/spark_transport/libsircl/README.md index dd3edeff..77445d14 100644 --- a/spark_transport/libsircl/README.md +++ b/spark_transport/libsircl/README.md @@ -1,13 +1,17 @@ # libsircl +Status: **research-only**. No serving or training claim rests on libsircl; `STATUS.md` states what is +implemented and the evidence for it. + libsircl is SIRCL's NCCL-compatible C API: a Linux shared library that implements the public C interface of NVIDIA NCCL 2.32 (the 73 functions of the NCCL 2.32.3 header and their `pnccl` profiling twins) on SIRCL, SparkRing's Switchless Inference RDMA Collective Layer for clusters of NVIDIA DGX Spark systems. A program written against NCCL's C API, such as PyTorch, vLLM, SGLang or nccl-tests, loads libsircl in place of NCCL's `libnccl.so.2` without modification. -Compatibility is at the C API and ABI. libsircl implements a subset of the API (`STATUS.md` lists -each function) and does not exchange data with NVIDIA NCCL processes. libsircl is an independent +Compatibility is at the C API and ABI. libsircl implements 36 of the header's 73 functions +(`tests/api_manifest.json` marks each one implemented or not; the other 37 return `ncclInvalidUsage`) and +does not exchange data with NVIDIA NCCL processes. libsircl is an independent implementation: it contains no NCCL implementation source, it is not NVIDIA NCCL, and it is not sponsored or endorsed by NVIDIA. `STATUS.md` states what is implemented and the evidence; `RUNBOOK.md` gives the build, emulation and hardware commands. @@ -366,7 +370,7 @@ and their terms, and every source or binary copy carries `LICENSE`, `NOTICE`, `v unmodified in `vendor/`, adapted with marked changes in `include/nccl.h`) and the NCCL user guide; no NCCL implementation source is used. - The SIRCL implementation this library follows is SIRCL's package in this repository - (`spark_transport/sircl`, `sparkring_sircl` 0.3.1); its tests run against that package + (`spark_transport/sircl`, `sparkring_sircl` 0.3.2); its tests run against that package (`SIRCL_PACKAGE`). `SOURCE_SNAPSHOT.json` records the SHA-256 of each of the 200 files of the SIRCL tree that the evidence in `STATUS.md` was taken against (captured 2026-10-08 06:00 UTC). A change the library needs inside SIRCL is a change to `spark_transport/sircl`. diff --git a/spark_transport/libsircl/RUNBOOK.md b/spark_transport/libsircl/RUNBOOK.md index fb597393..5df82aac 100644 --- a/spark_transport/libsircl/RUNBOOK.md +++ b/spark_transport/libsircl/RUNBOOK.md @@ -1,9 +1,11 @@ # libsircl runbook Commands to build libsircl, check it in GPU emulation on the workstation, run the pair milestone on a -cabled pair of NVIDIA DGX Spark systems, and run relayed groups (a path of four, the cycle of eight). Status: sections -3.1 to 3.3 ran on the pair at positions 0-1 (`STATUS.md`, hardware evidence); sections 3.4, 3.5 and 4 -have not run. Only the operator who holds the ring measurement lock runs sections 3 and 4. +cabled pair of NVIDIA DGX Spark systems, and run relayed groups (a path of four, the cycle of eight). Status +(research-only): sections 3.1 to 3.5 ran on the cabled pairs at positions 0-1 and 6-7, and section 4's +bit-exact check, nccl-tests and timing sweeps on the path of four at positions 4-7 and the cycle of eight +(`STATUS.md`, hardware evidence). Sections 3 and 4 change the Sparks: only the ring's operator, who +schedules the ring's time, runs them. Safety classes, as in SIRCL's ring runbook: @@ -144,7 +146,8 @@ NCCL_TESTS_BUILD=/opt/nccl-tests/build bash tools/emulation_suite.sh /tmp/libsir ## 3. Pair milestone on Sparks at positions 0 and 1 (MUTATES HOST; operator only) -Conditions: the serving image (the image whose ID starts `aba309e4610c`, or its successor), host +Conditions: the serving image (the image whose ID starts `aba309e4610c`; another image gives other library +and pack bytes, which the run's records then name), host networking, GPU 0, the RDMA devices of the cabled pair, containers started the way SIRCL's ring harness starts them. `LAN0` is the wired-LAN address of the Spark at position 0 and `IFACE` the wired-LAN interface name (the site file's `lan_interface`). The route maps are SIRCL's for `path:0-1` with two @@ -237,7 +240,7 @@ python3 tests/emulation/perf_rank.py --library $LIB --world 2 --rank --id-se ``` Exit: every size checked is correct; the eager 8192-byte rows give the milestone's eager latency -measurement (criterion: at or below 20 us) and the graph rows its graph measurement, pending 3.4. +measurement (criterion: at or below 20 us) and the graph rows its graph measurement. ### 3.4 nccl-tests, unmodified, through `LD_PRELOAD` @@ -473,4 +476,6 @@ env [schedule variables] python3 tests/emulation/perf_rank.py --library --id-server $LAN0:29724 --out /tmp/sircl-ccl/perf--rank.json ``` -The library's default large-message schedule stays `pieces` until these measurements choose another. +Without schedule variables the library takes the pair plan on a cabled pair and the cycle plan on a ring that +closes over cables (`README.md`, "Pair default" and "Cycle default"), else `pieces`; these measurements +compare the schedules. diff --git a/spark_transport/libsircl/STATUS.md b/spark_transport/libsircl/STATUS.md index e6f1d369..cd6ef0e6 100644 --- a/spark_transport/libsircl/STATUS.md +++ b/spark_transport/libsircl/STATUS.md @@ -1,7 +1,9 @@ # libsircl status -Status of the library as a whole: **implemented; verified in GPU emulation; on a cabled pair of NVIDIA -DGX Spark systems, bit-identical to the SIRCL Python session's outputs and timed.** The NCCL-compatible +Status of the library as a whole: **research-only. Implemented; verified in GPU emulation; on a cabled +pair of NVIDIA DGX Spark systems, its float16, bfloat16 and float32 sum all-reduces, all-gathers and +reduce-scatters bit-identical to the SIRCL Python session's outputs (every other datatype and op checked +against the fold model and the rank-order reference), and timed.** The NCCL-compatible API layer, bootstrap, session engine, the four kernel packs, both transports and the point-to-point channels are implemented. The pair at ring positions 0-1 ran `RUNBOOK.md` sections 3.1 to 3.5 on snapshot fb63329a, nccl-tests v2.21.1 included, @@ -24,9 +26,9 @@ hash). Identifiers and logs. A snapshot named by eight hexadecimal digits (`fb63329a`, `0f94c72b`, `a3477af2`, `e31abc5c` and others) is a copy of the library's files that its development workspace wrote, named by the SHA-256 of its file list; this directory's first commit in the repository holds snapshot -`e31abc5c`'s files. Paths under `verification/`, `../sircl-ccl-snapshots/` and `ccl-pair/` name run -logs, snapshot builds and the ring operator's gate outputs that the machines which made them keep; they -are not in this repository. +`e31abc5c`'s files. Paths under `verification/`, `../sircl-ccl-snapshots/` and `ccl-pair/`, and the ring operator's gate +script `gate_multi_0f94c72b.sh`, name run logs, snapshot builds, gate outputs and the script that the +machines which made them keep; they are not in this repository. ## Sources @@ -36,8 +38,8 @@ and the outputs of SIRCL's ring harness runs excluded) and the 39 files that cha the capture before it; at 03:00 CDT (08:00 UTC) every source file of that tree still matched the capture and no source file had been added. SIRCL's own CPU tests passed on it (503 passed, 16 skipped; `verification/sircl-cpu-tests.log`). This repository's SIRCL package is `spark_transport/sircl` -(`sparkring_sircl` 0.3.1). The library -carries byte-identical copies of two native libraries of SIRCL 0.3.1 (with change LF), in the LF-ending +(`sparkring_sircl` 0.3.2). The library +carries byte-identical copies of two native libraries of SIRCL 0.3.2 (with change LF), in the LF-ending bytes SIRCL's release builds its natives from, each SHA-256 recorded beside it and checked by the build (which also refuses carriage returns; `tests/test_vendored_sources.py`): the native proxy `oneshot/_roce_proxy.c` (`src/transport/sircl_roce_proxy.c`, `09db8715...`, the source of the serving @@ -54,14 +56,14 @@ The interface follows NVIDIA's public `nccl.h.in` v2.32.3-1 and the user guide o | Component | Status | Evidence | |---|---|---| -| Symbol surface: 73 `ncclX` host functions and 73 `pnccl` twins of the 2.32.3 header, and the 16 host entry points of NCCL's device API (`nccl_device/core.h` and its barrier and LL all-to-all headers, NCCL 2.28 and later) with their twins, which programs built against current headers import (`src/device_api.c`: `ncclCommQueryProperties` and the team queries implemented, the device communicator, device pointers and requirement builders refused); SONAME `libnccl.so.2`, version 22705; only `nccl*`, `pnccl*` and the three extension functions `sirclGetInfo`, `sirclSetWaitRegime` and `sirclGetReceipt` exported, each by exact name; `tests/api_manifest.json` marks each function of the header implemented or not; `tools/check_exports.py` lists any function a header set declares or a program imports that the library lacks | implemented | ABI suite (5 tests); engine suite's device-API test; generator check; `tools/check_exports.py`: 0 missing against the NCCL 2.29.7 headers and the nccl-tests v2.21.1 binaries built on the workstation; PyTorch and vLLM bind to it (`verification/framework-loading.json`) | -| Kernel-entry check: before the library links, its pack loader resolves every entry it names in the three embedded packs through a stand-in CUDA driver that reads every architecture's cubin offline (`tests/check_entries.c`, `tests/fake_cuda.c`; make and CMake) | implemented | kernel-entry suite (3 tests: every entry present; hiding one entry of each pack fails and names it; refused without the stand-in driver); a loader that names the two-pass ring all-reduce entries, built with link pack `5935b067...` (which has none), fails the build at `sircl_ring_reduce_two_pass_f32_u1` | +| Symbol surface: 73 `ncclX` host functions and 73 `pnccl` twins of the 2.32.3 header, and the 16 host entry points of NCCL's device API (`nccl_device/core.h` and its barrier and LL all-to-all headers, NCCL 2.28 and later) with their twins, which programs built against current headers import (`src/device_api.c`: `ncclCommQueryProperties` and the team queries implemented, the device communicator, device pointers and requirement builders refused); SONAME `libnccl.so.2`, version 22705; only `nccl*`, `pnccl*` and the three extension functions `sirclGetInfo`, `sirclSetWaitRegime` and `sirclGetReceipt` exported, each by exact name; `tests/api_manifest.json` marks each function of the header implemented or not; `tools/check_exports.py` lists any function a header set declares or a program imports that the library lacks | implemented | ABI suite; engine suite's device-API test; generator check; `tools/check_exports.py`: 0 missing against the NCCL 2.29.7 headers and the nccl-tests v2.21.1 binaries built on the workstation; PyTorch and vLLM bind to it (`verification/framework-loading.json`) | +| Kernel-entry check: before the library links, its pack loader resolves every entry it names in the four embedded packs through a stand-in CUDA driver that reads every architecture's cubin offline (`tests/check_entries.c`, `tests/fake_cuda.c`; make and CMake) | implemented | kernel-entry suite (3 tests: every entry present; hiding one entry of each pack fails and names it; refused without the stand-in driver); a loader that names the two-pass ring all-reduce entries, built with link pack `5935b067...` (which has none), fails the build at `sircl_ring_reduce_two_pass_f32_u1` | | Loading: the library links only libc and the dynamic loader, has no constructor that touches CUDA, and opens `libcuda.so.1` only when a communicator is created | implemented | CPU test in an isolated interpreter, with a negative control that initializes CUDA in a constructor; `tools/probe_cuda_on_load.py` (`verification/cuda-on-load.json`) | -| Bootstrap: unique id with root address, port, nonce and site hash; loopback, or a LAN address from `SIRCL_BOOTSTRAP_ADDR`, `SIRCL_BOOTSTRAP_IFNAME` or `NCCL_SOCKET_IFNAME`; all-gather rounds through the root; a rank leaving a complete group fails every later round at once | implemented | bootstrap suite (21 tests); between the two Sparks of the pair (hardware evidence) | +| Bootstrap: unique id with root address, port, nonce and site hash; loopback, or a LAN address from `SIRCL_BOOTSTRAP_ADDR`, `SIRCL_BOOTSTRAP_IFNAME` or `NCCL_SOCKET_IFNAME`; all-gather rounds through the root; a rank leaving a complete group fails every later round at once | implemented | bootstrap suite; between the two Sparks of the pair (hardware evidence) | | Communicator lifecycle: blocking and non-blocking init, groups (a collective or point-to-point call refused with `ncclInvalidArgument` leaves its group as it was; a queued point-to-point call holds no reference on its communicator, so `ncclCommAbort` from another thread returns at once and that group's `ncclGroupEnd` fails the calls with `ncclInvalidUsage`), finalize and destroy (wait for every rank; see the teardown row), abort, revoke, a communicator's error and its text published together (one lock), `ncclRedOpCreatePreMulSum` (not implemented) returning `ncclNumOps`, which every collective refuses, async errors from the command ring and the progress thread; `ncclCommSplit` (members ordered by key, then parent rank; `NCCL_SPLIT_NOCOLOR` gets NULL; children keep the parent's route-map position; a NULL config, or a config whose `blocking` is undefined, takes the parent's blocking mode) | implemented | lifecycle and API suites, also under AddressSanitizer and UndefinedBehaviorSanitizer (`make check-asan`); GPU emulation (below): `library_rank.py`'s checks of an invalid-peer send inside a group (the group's valid send and receive complete), of `ncclCommAbort` with a send queued in another thread's open group (returns within 0.1 s; that group's end returns `ncclInvalidUsage`) and of `ncclRedOpCreatePreMulSum`'s op in an all-reduce of unequal inputs (refused with `ncclInvalidArgument`, the buffer unchanged). Abort cannot stop a kernel waiting for a dead peer before its wait limit (request AW) | | Communicator creation on a thread with no current CUDA context: the current CUDA device's primary context (the application's CUDA runtime's `cudaGetDevice`, else the only visible device), retained once per process and device and made current on the creating thread; refused with `ncclInvalidUsage`, naming the reason, when the device is unknown (several devices, no runtime) | implemented | Conditions: CPU, `tests/test_engine_cpu.py` on the stand-in driver (`tests/fake_cuda.c`): one device, the runtime's device 1 of two, and two devices without a runtime (refused); GPU on the workstation (RTX 5090, torch 2.12.0+cu130), `tests/emulation/init_without_context.py`: vLLM's PyNccl sequence (device selected with `torch.accelerator.set_device_index`, no allocation; `ncclCommInitRank` inside `torch.accelerator.device_index`; then a one-element float32 all-reduce of a tensor torch allocates afterwards). Measurement: the library of source tree `dbf36074` returned `ncclInvalidUsage` ("no current CUDA context") there, as the installer's `qwen38-flash-next-tp2 --transport libsircl` deployment on a cabled pair of Sparks did at PyNccl's first `ncclCommInitRank` on both ranks (image `1a8c10354eb0`, vLLM v0.1.dev21553+gab86b7073); with this change the call succeeds, torch's first allocation lands in the same primary context and the all-reduce returns 1.0. On the fabric (research run, 2026-10-10, a cabled pair of DGX Sparks at positions 0 and 1 of a ring of four; the installer's two containers of that deployment rerun unchanged except for a research library selected through `SPARKRING_LIBSIRCL_LIBRARY` and `SPARKRING_LIBSIRCL_SHA256`, each built in a container of image `1a8c10354eb0`, so with its nvcc 13.4.92 and kernel packs): source tree `dbf36074` with the creation diagnostic printed `libsircl: invalid usage: no current CUDA context: ...` on both ranks and vLLM stopped at PyNccl's first `ncclCommInitRank`; this tree's library (`cddcaa0e...`) let vLLM load, capture its CUDA graphs and serve: two known-answer chat requests at temperature 0 returned `42` (17 + 25) and `Paris`, and each rank's receipt of the tensor-parallel communicator recorded 7,783 all-reduces (3,420 captured in graphs, 5.43 GB), 133 all-gathers and one pair exchange on the verbs transport and the pair plan, healthy, no error, fail-stop on | | The reason of a failed communicator creation on stderr (`libsircl: communicator creation failed (): `, whatever `NCCL_DEBUG` says); with `NCCL_DEBUG=WARN`, `INFO` or `TRACE`, the reason of every failed call | implemented | CPU: `tests/test_engine_cpu.py` (a refused creation's line; `ncclGetVersion(NULL)` quiet without `NCCL_DEBUG`, named with `NCCL_DEBUG=WARN`) | -| Session engine: settings and both packs' hashes agreed across ranks; route maps keyed by process position (`LIBSIRCL_POSITION`, exchanged at setup), so one map serves every communicator a process joins; pinned arena (cuMemHostAlloc on hardware, shared memory registered with CUDA in emulation), device counters, SIRCL's native proxy, queue-pair connection, lane check, progress thread per communicator | implemented | GPU emulation (below); setup-failure cases; verbs transport on the pair (hardware evidence) | +| Session engine: settings and the four kernel packs' hashes agreed across ranks; route maps keyed by process position (`LIBSIRCL_POSITION`, exchanged at setup), so one map serves every communicator a process joins; pinned arena (cuMemHostAlloc on hardware, shared memory registered with CUDA in emulation), device counters, SIRCL's native proxy, queue-pair connection, lane check, progress thread per communicator | implemented | GPU emulation (below); setup-failure cases; verbs transport on the pair (hardware evidence) | | Transport kernel pack: one-shot and two-shot all-reduce, all-gather, reduce-scatter and all-to-all scatter ops in CUDA C++, sm_120 and sm_121, compiled by nvcc when the library is built; its hash, the fold pack's and the link pack's join the setup agreement | implemented | `KERNEL_ROUTE.md`: 905 of 905 mixed-group checks against SIRCL's DSL kernels; on the pair (hardware evidence) | | Fold kernel pack: rank-ordered local reduction of gathered rows, 12 datatypes x 5 built-in ops, sm_120 and sm_121, compiled when the library is built | implemented | library emulation and the pair: every datatype and op equal to the host model | | `ncclAllReduce`, every datatype, `ncclSum`/`ncclProd`/`ncclMax`/`ncclMin`/`ncclAvg`: float16, bfloat16 and float32 sums in the transport kernels (one-shot to `SIRCL_ONESHOT_MAX_BYTES`, two-shot above, pieces of `SIRCL_LARGE_PIECE_BYTES`, zero-padded tails below 16 bytes, unaligned buffers through scratch, in place); every other datatype and op as an all-gather of slot-sized tiles and the fold | implemented | library emulation, PyTorch emulation, the pair | @@ -75,10 +77,10 @@ The interface follows NVIDIA's public `nccl.h.in` v2.32.3-1 and the user guide o | One-way pair exchanges post the empty direction's items as flags only: the exchange kernel sets bit 24 of the link op word when its input is 0, and the native proxy then posts those items without payload; the peer discards them as before | implemented | Conditions: snapshot ca09d28c (SIRCL's landed proxy with this change, link pack `dc9dd167...`) against snapshot 7d1c6f19 and NVIDIA NCCL 2.32.3 on ConnectX-7 between two Sparks (pair 4-5), nccl-tests v2.21.1, bfloat16, root 0, 512 KiB to 256 MiB, out of place, two runs of this tree and of NVIDIA. Measurement (us at 256 MiB and 4 MiB): broadcast 11,008 and 11,000 / 177 and 173 against 7d1c6f19's 11,976 / 216 and NVIDIA's 11,033 and 11,029 / 208 and 190; reduce 10,993 and 10,993 / 177 and 180 against 12,447 and NVIDIA's 11,025 and 10,990 / 205 and 220; gather 5,505 and 5,495 / 90 and 90 against NVIDIA's 5,531 and 5,543 / 109 and 115; scatter 5,483 and 5,494 / 88 and 86 against NVIDIA's 5,545 and 5,548 / 99 and 104; every job exact; NIC bytes per direction over the five rows 133.1 GB against 232.7 GB without the change (NVIDIA 133.2 GB), as in research snapshot bcff23bc. The all-to-all at API level 22705 (grouped point-to-point) does not use this path: 7,229 / 157 us against NVIDIA's 6,228 and 6,215 / 166 and 176. GPU emulation on the workstation: every one-way exchange of the two-rank library run bit-exact, the same link items with fewer link bytes on both ranks; `library_rank.py`'s flags-only check on a fresh split child, three rounds of ten one-way exchanges of 1.1 to 2.8 MiB (10 to 13 pieces each against 8 link slots, so every exchange wraps the slots at a shifting place) whose empty direction alternates between the ranks (broadcasts, gathers, scatters and ncclReduce sums of roots 0 and 1, sends each way), each round followed by a two-way all-to-all and all-gather: every output exact and 33 pair exchanges for 33 calls under the pair plan on one and two lanes, and on a relayed pair with forward windows and a ring window of 393,216 bytes, which carried them in 1,151 window chunks. Conclusion: the four rooted one-way collectives match NVIDIA within 1% at 256 MiB and are faster from 4 MiB | | The all-to-all of two ranks as grouped point-to-point calls (torch's pattern; nccl-tests' below NCCL API level 22800): a send to and a receive from the peer and a send to and a receive from the rank itself of one size, the receive blocks adjacent in rank order and neither send block overlapping them, carried as one pair exchange whose kernel copies the own block in parallel with the transfer (otherwise a copy ahead of the exchange) | implemented | GPU emulation on the workstation: `library_rank.py` on two ranks, torch's pattern of 1 MiB blocks under the pair plan exact with one pair exchange and no separate local copy, and exact under the pieces schedule (no pair plan, the copy ahead); the emulation suite's nccl-tests lines (11, `alltoall_perf` included) exit 0 with `#wrong` 0. On ConnectX-7 between two Sparks (pair 4-5; snapshot b75d4b26 = this change on ca09d28c, against ca09d28c and NVIDIA NCCL 2.32.3; nccl-tests v2.21.1 `alltoall_perf` at API level 22705, bfloat16, 512 KiB to 256 MiB, out of place, two runs each; us at 256 / 16 / 4 MiB): 6,147 and 6,139 / 428 and 423 / 141 and 142 against ca09d28c's 7,217 and 7,266 / 493 and 493 / 159 and 159 and NVIDIA's 6,216 and 6,220 / 455 and 447 / 163 and 177; every job exact; broadcast, reduce, gather and scatter unchanged within the runs' spread. Conclusion: the all-to-all through torch's grouped point-to-point path is 1% faster than NVIDIA at 256 MiB, 5-7% at 16 MiB and 13-20% at 4 MiB | | Shared-memory verbs stand-in (`src/transport/shm_verbs.c`): the verbs subset of SIRCL's proxy across processes, with seeded interleaving across queue pairs; segments named by process id and process start time, so containers that share the host's `/dev/shm` with reused process ids never share a name, and a name that exists already is skipped (never removed); a report of every queue pair and failed write for link dumps; the emulation runners remove the segments of their ended rank processes | research-only (test infrastructure) | CPU suite across processes (5 tests, one with stale segments under the names the ranks try first: the group runs exact and leaves them untouched; with the skip disabled the same test fails with `shm_open: File exists`); every library emulation run | -| MPI shim for nccl-tests (`tools/mpi-shim`: every MPI call and constant of nccl-tests v2.21.1, including `MPI_Allgatherv`, `MPI_Reduce`, `MPI_Error_string`, splits of any color and `MPI_UNDEFINED`, splits of splits, collectives among a communicator's members over a full TCP mesh; a job tag, `SIRCL_MPI_JOB`, in every hello, so processes of different jobs refuse each other; a watchdog that ends the process when a peer's process ends outside `MPI_Finalize`; with `SIRCL_MPI_DISTINCT_HOSTS=1`, a host name per rank, so nccl-tests on one host in emulation counts one rank per host and every rank uses device 0), nccl-tests driver and evaluator (`tools/nccl_tests_pair.sh`: one shim job and a time limit per test line, the ring harness's CPU placement, a job manifest `jobs.tsv` per rank and exit status 1 when any line failed; `tools/check_nccl_tests.py`: the manifests of every rank, the same jobs on every rank and, given the lines file, every expected job, exit 0 per job, every job's sweep complete, `#wrong` 0 with an all-to-all's in-place `N/A` counted as not covered, receipts whose all-reduce ops (transport, fold, chain, ring) cover their calls), all-reduce timing sweep (`tests/emulation/perf_rank.py`) | research-only (test infrastructure) | MPI shim CPU suite (6 tests, 1 to 4 processes: a peer that exits after `MPI_Init` ends the other within 15 s, ranks of different jobs never join, each rank's host name under `SIRCL_MPI_DISTINCT_HOSTS`); the emulation suite's nccl-tests section on the workstation (two processes, one GPU, 11 lines, every line exit 0 on both ranks and `Out of bounds values : 0 OK`); nccl-tests v2.21.1 (commit `afd59ab`) compiles and links against the shim on the workstation with every test binary, `comm_ops_perf` included (nvcc 13.3, NCCL 2.29.7 headers, sm_121; its GIN device-API tests need NCCL 2.30.7 headers and were not built, and their MPI calls compile and run as C++ against the shim on two processes); evaluator on synthetic outputs (`tests/test_check_nccl_tests.py`, 6 cases: a complete pair, a missing manifest, a partial log with a failed exit, a job missing from the lines file, ranks with different jobs, an in-place `N/A` outside all-to-all); timing sweep in emulation (below) | +| MPI shim for nccl-tests (`tools/mpi-shim`: every MPI call and constant of nccl-tests v2.21.1, including `MPI_Allgatherv`, `MPI_Reduce`, `MPI_Error_string`, splits of any color and `MPI_UNDEFINED`, splits of splits, collectives among a communicator's members over a full TCP mesh; a job tag, `SIRCL_MPI_JOB`, in every hello, so processes of different jobs refuse each other; a watchdog that ends the process when a peer's process ends outside `MPI_Finalize`; with `SIRCL_MPI_DISTINCT_HOSTS=1`, a host name per rank, so nccl-tests on one host in emulation counts one rank per host and every rank uses device 0), nccl-tests driver and evaluator (`tools/nccl_tests_pair.sh`: one shim job and a time limit per test line, the ring harness's CPU placement, a job manifest `jobs.tsv` per rank and exit status 1 when any line failed; `tools/check_nccl_tests.py`: the manifests of every rank, the same jobs on every rank and, given the lines file, every expected job, exit 0 per job, every job's sweep complete, `#wrong` 0 with an all-to-all's in-place `N/A` counted as not covered, receipts whose all-reduce ops (transport, fold, chain, ring) cover their calls), all-reduce timing sweep (`tests/emulation/perf_rank.py`) | research-only (test infrastructure) | MPI shim CPU suite (6 tests, 1 to 4 processes: a peer that exits after `MPI_Init` ends the other within 15 s, ranks of different jobs never join, each rank's host name under `SIRCL_MPI_DISTINCT_HOSTS`); the emulation suite's nccl-tests section on the workstation (two processes, one GPU, 11 lines, every line exit 0 on both ranks and `Out of bounds values : 0 OK`); nccl-tests v2.21.1 (commit `afd59ab`) compiles and links against the shim on the workstation with every test binary, `comm_ops_perf` included (nvcc 13.3, NCCL 2.29.7 headers, sm_121; its GIN device-API tests need NCCL 2.30.7 headers and were not built, and their MPI calls compile and run as C++ against the shim on two processes); evaluator on synthetic outputs (`tests/test_check_nccl_tests.py`, among them a complete pair, a missing manifest, a partial log with a failed exit, a job missing from the lines file, ranks with different jobs, an in-place `N/A` outside all-to-all); timing sweep in emulation (below) | | Link kernel pack (`kernels/sircl_links.cu`, `dc9dd167...`): SIRCL's chain all-reduce and link collectives in CUDA C++ (chain all-gather and reduce-scatter over links 0 and 1; ring all-gather, reduce-scatter and all-reduce over links 2 and 3 with both staggers), chain position and rank order as launch parameters, unroll 1-8, at most 512 threads per block (up to 128 registers per thread, no spills), sm_120 and sm_121, compiled when the library is built; the ring all-reduce's relay in the two-pass form (default: the slot, then the output) and in one pass (`LIBSIRCL_RING_REDUCE_PASSES=1`); the pair exchange entries; a launch's last block is the one whose arrival completes its own grid, so link ops of one kernel type may differ in blocks per role | implemented; the one-pass relay: research-only (on a GB10, one run under forward windows on four ranks returned differing bits on one rank of a split communicator; not yet isolated) | mixed groups against SIRCL's DSL kernels on `path:0-1`, `path:0-3` and `ring:8` of the two-pass pack `5935b067...`: chain and ring schedules and SIRCL's own link checks, every check passed (below); the one-pass pack's mixed groups are queued | | Point-to-point kernel pack (`kernels/sircl_p2p.cu`, `0f39a3b9...`): the send and receive kernels of SIRCL's point-to-point channels (`p2p/_kernels.py`) in CUDA C++, one entry per direction and unroll 1-8, threads, lanes, slots, slot bytes and the native layer's block offsets as launch parameters (`sccl_p2p_args`), at most 512 threads per block (44 to 96 registers, no spills), sm_120 and sm_121, compiled when the library is built; its hash joins the setup agreement | implemented | GPU emulation against SIRCL's native point-to-point library (below, "Point-to-point channels"); not run in one group with SIRCL's DSL point-to-point kernels | -| The cycle plan (README.md, "Cycle default"): a communicator of three or more ranks without `SIRCL_LARGE_SCHEDULE`, given a ring plan, whose ring closes over cables only (every rank's relayed positions exchanged with the ranks' positions, the decision compared with the settings), runs all-reduces, all-gathers and reduce-scatters from 8 MiB as one ring op each and the pieces below; the receipt's `cycle_plan` | implemented | GPU emulation (`tools/emulation_suite.sh`): four and eight ranks with a ring plan and no relays take the plan and every check of `library_rank.py` passes against the pieces digests below 8 MiB and the ring digests from it; four ranks whose ring closes through relays, and four with `SIRCL_LARGE_SCHEDULE=pieces`, do not take it; with every rank's own planner settings the channel test's ring:8 layouts take it and path:0-3 does not. The measurement behind it: the ring schedules overtake the pieces between 4 and 8 MiB on the cycle of eight Sparks (snapshot a3477af2, hardware evidence). The plan itself has not run on Sparks | +| The cycle plan (README.md, "Cycle default"): a communicator of three or more ranks without `SIRCL_LARGE_SCHEDULE`, given a ring plan, whose ring closes over cables only (every rank's relayed positions exchanged with the ranks' positions, the decision compared with the settings), runs all-reduces, all-gathers and reduce-scatters from 8 MiB as one ring op each and the pieces below; the receipt's `cycle_plan` | implemented | GPU emulation (`tools/emulation_suite.sh`): four and eight ranks with a ring plan and no relays take the plan and every check of `library_rank.py` passes against the pieces digests below 8 MiB and the ring digests from it; four ranks whose ring closes through relays, and four with `SIRCL_LARGE_SCHEDULE=pieces`, do not take it; with every rank's own planner settings the channel test's ring:8 layouts take it and path:0-3 does not. The measurement behind it: the ring schedules overtake the pieces between 4 and 8 MiB on the cycle of eight Sparks (snapshot a3477af2, hardware evidence). On Sparks, the image gate of image `1a8c10354eb0`'s library on the cycle of eight recorded every eight-rank communicator without a schedule setting taking the plan (96 communicators; [record](../../performance/records/transport/libsircl-ring8-image-1a8c10354eb0-20261009.md)) | | Schedules of large messages and relayed groups, SIRCL's plan: `SIRCL_LARGE_SCHEDULE` (all-reduce: one ring op for the largest prefix of W equal chunks, or one chain op for the 16-byte-aligned body), `SIRCL_GATHER_SCHEDULE` (one chain or ring all-gather for 16-byte-multiple shards), `SIRCL_SCATTER_SCHEDULE` (chain or ring reduce-scatter of 16-byte-multiple chunks, in column tiles of `LIBSIRCL_LINK_TILE_BYTES`); each `pieces`, `chain`, `auto` (from SIRCL's per-collective chain minimums) or `ring` (from its ring minimums, else as `auto`); unset, `pieces` on groups of three or more and the pair plan on two ranks (a cabled pair, one block per link role: ring all-reduces from 2 MiB in 256 KiB pieces, ring all-gathers from 1 MiB shards with pieces by shard size, ring reduce-scatters from 4 MiB of input, and pair exchanges for all-to-alls, point-to-point exchanges of equal or one-way sizes, broadcasts, scatters, gathers and ncclReduce sums from 1 MiB; a relayed pair: the chain from 8 MiB; README.md, "Pair default"); blocks per link role per collective and per call (`LIBSIRCL_*_LINK_BLOCKS`, `SIRCL_LINK_BLOCKS`, the plan), agreed at setup; decided from agreed settings and sizes only, with unaligned and in-place buffers staged, so every rank runs the same ops; chain order from `LIBSIRCL_CHAIN_ORDER` or the ranks by position | implemented | CPU suites; GPU emulation (below); the pair default on the cabled pair at positions 6-7 (snapshot 0f94c72b, hardware evidence): every nccl-tests row of `RUNBOOK.md` sections 3.4 and 3.5 correct. Its sizes follow SIRCL's ring harness on two cabled pairs (section 3.6); libsircl's large-message times under the pair default have not been compared on Sparks with its other schedules, the ring harness or NVIDIA NCCL | | Op selection from shared state only: a call's schedule, op kinds and count, pieces, blocks per role and link op words follow from its shape, datatype, root and the settings agreed at setup; a rank's buffer alignment, in-place or overlapping buffers, a discarded output (off the root) and its staging state choose only local copies through scratch or staging around the same ops. The staging buffer grows outside capture with every earlier buffer kept until destroy (no limit on the number of sizes); under CUDA graph capture a call that needs more staging than the communicator holds takes a graph allocation on the capturing stream (`cuMemAllocAsync`), freed on that stream when the call returns, so the graph owns it (README.md, "Op selection and staging") | implemented | Conditions: GPU emulation on the workstation; `library_rank.py`'s rank-local staging check on a fresh split child: the collectives of every selection point (one-shot, pieces and padded tiles; chain, ring and link ops; pair exchanges of roots 0 and 1; `ncclAlltoAll`; on two ranks torch's all-to-all pattern and one-way sends each way), with every rank's buffers aligned, then rank 1's unaligned, then rank 1's in place, then captured cold into one graph on a second fresh child with rank 1's unaligned and replayed twice; and the staging growth check, 19 all-gathers of doubling shards from 16 B to 4 MiB with rank 1 unaligned. Measurement (the runs under "Op selection under rank-local staging, and alternating flags-only directions" below): in every run each rank's receipt counts of launched ops (pair exchanges and their bytes, link ops by kind and by blocks per role, transport, chain and point-to-point ops) and of carried transfers (native link ops and items, one-shot ops and phases) equal its aligned run's, and the launched counts are the same on every rank; every output exact; the cold capture is taken on every rank, each of rank 1's staged calls as a graph allocation (16 on the pair default, 20 under the ring schedule on two ranks), and both replays are exact; rank 1 keeps 19 staging buffers under the chain and ring schedules. The same checks against snapshot ba5a337b's library, with a 15 s wait limit: under the pair default, rank 1 refused its cold capture of the staged calls with `ncclInvalidUsage` while rank 0 captured them, rank 0's replays returned wrong outputs at the wait limit and the child's destroy failed; under the ring schedule, rank 1 also refused the 18th and 19th all-gathers of the growth check (`ncclInternalError`, too many staging buffer sizes) while rank 0 launched them, and rank 0's session was poisoned at the wait limit. Conclusion: in these runs a rank's alignment, in-place buffers, role and staging state change no rank's ops | | Progress thread placement: `SIRCL_PROGRESS_CPU` pins it; otherwise `LIBSIRCL_CPU_POLICY=performance` (default) starts it on the fastest CPU class the creating thread may use (GB10: the Cortex-X925 cores), `none` leaves it to the scheduler; the application's threads keep their affinity; receipts name the thread's CPUs (`progress_cpus`) | implemented | build and CPU suites (the workstation's CPUs are of one class); on GB10s (Sparks 6-7, snapshot 0f94c72b, hardware evidence) the default policy started the thread on CPUs 5-9 and 15-19 and `SIRCL_PROGRESS_CPU=19` pinned it to CPU 19, as each receipt's `progress_cpus` records | @@ -90,26 +92,27 @@ The interface follows NVIDIA's public `nccl.h.in` v2.32.3-1 and the user guide o | CUDA graph capture of a call that needs more staging than the communicator holds, on a driver or device without stream-ordered allocation (a driver before CUDA 11.2, or a device without memory pools; the GB10 and the workstation's RTX 5090 have them) | unsupported | the call is refused with `ncclInvalidUsage` on the rank that needs the staging while its peers capture theirs; it captures after one eager call of that size has run on that rank. Whether every rank has stream-ordered allocation is agreed at setup | | Fail-stop (`LIBSIRCL_FAIL_STOP=1` or `abort`, per process): at the first asynchronous error (a flag wait that timed out, a failed progress thread, a failed point-to-point channel) of a watched communicator of two or more ranks, found by the watcher thread (every `LIBSIRCL_FAIL_STOP_POLL_MS`, default 5) or by the first library call that would report it (`ncclCommGetAsyncError`, an enqueue's check, `ncclCommFinalize`, `ncclCommDestroy`, `ncclCommAbort`), the library writes the error, the time and what found it to stderr and the communicator's receipt, then ends the process, with `1` at once (exit status 70, no atexit handlers, no core dump), with `abort` by `abort()`; destroy and abort check once more, then take the communicator off the watch before releasing it, and library unload stops the watcher first | implemented | Conditions: the workstation, `tests/emulation/fail_stop.py`, two ranks, SIRCL_STARTUP_WAIT_S 2, rank 1 starting its second all-reduce 12 s late, rank 0 enqueuing its own at once, waiting for its stream, keeping the output and calling the library no more. Measurement (library `638cca1b...`, every time CLOCK_MONOTONIC): 15 runs of the exit and control cases and one of all three: with `1`, the fail-stop line 2.11 to 2.13 s and the process's end 2.43 to 2.53 s after the enqueue, exit status 70; with `abort`, the line 2.11 s after the enqueue and SIGABRT 20.8 s after it (WSL's crash capture ran first); without fail-stop, rank 0's stream wait returned 2.10 to 2.13 s after the enqueue with a wrong output, every call it made returned 0, and the process ran on past 7 s. With `LIBSIRCL_FAIL_STOP=1` the pair default's 533 library checks passed and no process ended; on a build of the same fail-stop code, so did the ring schedules' 989 checks on four ranks and 10 teardown rounds on four ranks. The wait limit itself (research build of this tree whose timed-out flag wait records its `%globaltimer` start and end and the limit it read): 34 timed-out 2 s waits lasted 2.00002 to 2.00095 s of `%globaltimer` with the limit read as 2,000,000 us; a 300 s kernel and five 10 s kernels that sampled `%globaltimer` every 1,024 polls showed no step beyond 11 ms (host-stamping jitter) and a rate of 0.943 to 0.948 of the workstation's CLOCK_MONOTONIC; the workstation's CLOCK_REALTIME stepped back by 1.2 to 1.7 s about every 30 s (WSL2's time synchronization), so two of 28 waits timed by the wall clock looked 0.40 s long while the kernel measured 2.0003 s; earlier runs of this check that used the wall clock showed the same. On the fabric (cabled pair at positions 6-7, snapshot 0f94c72b, 2026-10-09; `teardown_race.py`, rank 1 starting its child all-reduce 8 s late, SIRCL_STARTUP_WAIT_S 2): with `1`, rank 0 exited with status 70 3.9 s after it started, setup included, its fail-stop line naming the timed-out flag wait for the late rank; without fail-stop it exited 1 after 10.4 s with the round reported failed. Library calls (this tree, library `9776657f...`, GPU emulation, 2026-10-09, `verification/emulation/suite-s1/`): `fail_stop.py`'s cases in which rank 0 calls `ncclCommGetAsyncError`, `ncclCommDestroy` or `ncclCommAbort` right after its stream wait, with `LIBSIRCL_FAIL_STOP_POLL_MS=600000` so the watcher does not poll, end rank 0 with status 70 inside the call (2.31 to 2.38 s after the enqueue, the line 2.00 s after it, "found by a library call" or "found by ncclCommDestroy or ncclCommAbort"), the call never returning; against the library before this path the same cases had `ncclCommGetAsyncError` and `ncclCommAbort` return the error to rank 0, which ran on and exited 0. `teardown_race.py --expect fail-stop`, four ranks, rank 3's child all-reduce 8 s late, 2 s wait limit: every rank exited 70 in 6 of 6 runs; in 9 of the 15 non-late exits of 5 runs a library call found the error before the watcher (the gate's path of four lost one rank that way before this path). Conclusion: with fail-stop, a caller that checks only enqueue codes keeps a failed collective's output at most for the wait limit plus one poll; a read within that poll after its stream wait returns is not prevented | | `ncclCommSuspend` and `ncclCommResume` | unsupported | refused with `ncclInvalidUsage` (logged once); a communicator's arena and device memory stay allocated while it exists | -| Point-to-point between two ranks of a communicator of three or more ranks on SIRCL's point-to-point channels (`LIBSIRCL_P2P_CHANNELS=on`, off by default; README.md, "Point-to-point channels"): SIRCL's native point-to-point library vendored byte-identical (`src/transport/sircl_p2p_proxy.c`, `c8ccfcb9...`, SIRCL 0.3.1's copy with change LF, checked by the build, its symbols prefixed per transport and not exported), created inside the communicator's setup exchanges (each rank's lanes and `LIBSIRCL_P2P_WINDOWS` give a channel table every rank derives alike; connection records validated together; lane check; windows of relayed lanes), first-in first-out matching per ordered pair, one stream per channel direction joined to the caller's stream, sends queued before receives, staging of messages not 16-byte aligned or not whole packs (stream-ordered allocation, or each channel's own buffer), refusal on both ranks of a pair without a channel, asynchronous errors and fail-stop on every rank after a kernel's timeout or size mismatch (abort notices), the channels' progress thread stopped in the teardown rounds and their verbs objects counted at destroy, receipts (`channels`) | implemented | GPU emulation on the workstation (below, "Point-to-point channels"): four and eight ranks, ring:8 under the route planner's settings, torch initialized eagerly, nccl-tests' sendrecv, hypercube and alltoallv on four processes; CPU: the native library across processes over the verbs stand-in. Not run on Sparks | +| Point-to-point between two ranks of a communicator of three or more ranks on SIRCL's point-to-point channels (`LIBSIRCL_P2P_CHANNELS=on`, off by default; README.md, "Point-to-point channels"): SIRCL's native point-to-point library vendored byte-identical (`src/transport/sircl_p2p_proxy.c`, `c8ccfcb9...`, SIRCL 0.3.2's copy with change LF, checked by the build, its symbols prefixed per transport and not exported), created inside the communicator's setup exchanges (each rank's lanes and `LIBSIRCL_P2P_WINDOWS` give a channel table every rank derives alike; connection records validated together; lane check; windows of relayed lanes), first-in first-out matching per ordered pair, one stream per channel direction joined to the caller's stream, sends queued before receives, staging of messages not 16-byte aligned or not whole packs (stream-ordered allocation, or each channel's own buffer), refusal on both ranks of a pair without a channel, asynchronous errors and fail-stop on every rank after a kernel's timeout or size mismatch (abort notices), the channels' progress thread stopped in the teardown rounds and their verbs objects counted at destroy, receipts (`channels`) | implemented | GPU emulation on the workstation (below, "Point-to-point channels"): four and eight ranks, ring:8 under the route planner's settings, torch initialized eagerly, nccl-tests' sendrecv, hypercube and alltoallv on four processes; CPU: the native library across processes over the verbs stand-in. Sparks: the path of four and the cycle of eight with channels on, snapshot a3477af2 (hardware evidence below) | | Point-to-point channel items under CUDA graph capture | unsupported | refused with `ncclInvalidUsage`: a replay would reuse the item numbers fixed at enqueue | | Point-to-point channels created at the first `ncclSend` or `ncclRecv` | unsupported | connecting the channels is collective over every rank, and a first send involves two; the channels are created with the communicator when `LIBSIRCL_P2P_CHANNELS=on`, and a communicator without them holds none of their memory, queue pairs or thread | | Point-to-point windows of the route planner (`tools/site_routes.py`, SIRCL's `p2p/budget.py`): `--p2p-reserve session` reserves the collective session's forward windows first, and its ring windows only with `--ring-schedules` (libsircl's ring lanes carry traffic only under the ring schedules); `--session-share F` (default 1) sizes the session's forward and ring windows against F of every relay queue's share and gives the channels the rest; a relayed pair without a window on every lane in both directions has no channel, named in the JSON output | implemented | CPU (`tests/test_site_routes.py`, 8 tests, with `SIRCL_PACKAGE` naming the SIRCL reference tree's `spark_transport/sircl`): path:4-7 gives all 6 relayed ordered pairs a window without `--ring-schedules` and leaves 3 without one with it (rank 3's 393,216-byte ring window reserved); ring:8 leaves all 40 without one at the default plan and gives all 40 one with `--max-window 32768` and with `--session-share` 0.95, 0.5 and 0.25 (at 0.95 every relayed session lane one 32 KiB chunk smaller, the channels' windows 32 KiB); a share of 1 prints the default's output; shares outside (0, 1] are refused; in every case no relay queue holds more than its 393,216-byte share (session plus channels), and `--p2p-reserve none` on ring:8, as the check's negative control, overfills one; the ring:8 files of `tests/data` equal the tool's output | -| The setup check of the native proxy's local feature word (`roce_local_features`, SIRCL change LF): communicator setup fails on every rank, naming both bits, unless bit 0 (`roce_destroy` counts failed verbs calls) and bit 1 (flags-only own items, which the pair exchange without input uses) are set; the build refuses a vendored copy that differs from its recorded SHA-256 or does not define the word | implemented | CPU: the native probe reads bits 0 and 1 from the vendored copy (`tests/test_shm_verbs.py`); the library carries the check (`tests/test_engine_cpu.py`); a build against a modified copy, a copy with CRLF endings, or the copy before LF with its own hash, stops with the reason. GPU emulation: every communicator of the suite passed the check. Not run on Sparks | -| The setup check of the point-to-point library's local feature word (`p2p_local_features` bit 0: `p2p_destroy` counts failed verbs calls; SIRCL change LF): channel setup fails with a named reason when the bit is absent; compiled by default (`P2P_FEATURES=1`, CMake `LIBSIRCL_P2P_FEATURES`), and the build refuses a vendored copy without the word | implemented | CPU: the native probe reads bit 0 from the vendored copy (`tests/test_p2p_native.py`); the default build carries the check (`tests/test_engine_cpu.py`); a default build against the copy before LF stops with the reason, and `P2P_FEATURES=0` builds it. GPU emulation: every channel setup of the suite below passed the check. Not run on Sparks | +| The setup check of the native proxy's local feature word (`roce_local_features`, SIRCL change LF): communicator setup fails on every rank, naming both bits, unless bit 0 (`roce_destroy` counts failed verbs calls) and bit 1 (flags-only own items, which the pair exchange without input uses) are set; the build refuses a vendored copy that differs from its recorded SHA-256 or does not define the word | implemented | CPU: the native probe reads bits 0 and 1 from the vendored copy (`tests/test_shm_verbs.py`); the library carries the check (`tests/test_engine_cpu.py`); a build against a modified copy, a copy with CRLF endings, or the copy before LF with its own hash, stops with the reason. GPU emulation: every communicator of the suite passed the check. Sparks: snapshots 0f94c72b and a3477af2 carry the check, and every communicator of their runs passed it (hardware evidence below) | +| The setup check of the point-to-point library's local feature word (`p2p_local_features` bit 0: `p2p_destroy` counts failed verbs calls; SIRCL change LF): channel setup fails with a named reason when the bit is absent; compiled by default (`P2P_FEATURES=1`, CMake `LIBSIRCL_P2P_FEATURES`), and the build refuses a vendored copy without the word | implemented | CPU: the native probe reads bit 0 from the vendored copy (`tests/test_p2p_native.py`); the default build carries the check (`tests/test_engine_cpu.py`); a default build against the copy before LF stops with the reason, and `P2P_FEATURES=0` builds it. GPU emulation: every channel setup of the suite below passed the check. Sparks: snapshot a3477af2's channel runs passed it (hardware evidence below) | | Pipeline parallelism on two-rank communicators: a pipeline group of two ranks (TP 4 x PP 2 on a ring of eight: positions i and i + 4, every lane through three relays), and torch's lazily initialized ProcessGroupNCCL, which gives each stage pair of a larger pipeline group a two-rank communicator of its own (vLLM 0.19.1's V1 pipeline exchange: `isend_tensor_dict` and `irecv_tensor_dict`, one `torch.distributed.isend` or `irecv` per tensor) | implemented | Conditions: GPU emulation on the workstation (library `02573b7c...`), every rank with the settings `tools/site_routes.py --layout ring:8 --lanes 2` emits for its position (`tests/data/site_routes_ring8_l2.json`). Measurement: `tests/emulation/pp_pairs.py` (eight ranks; tensor-parallel splits of positions 0-3 and 4-7, pair splits of positions i and i + 4; six microbatches per pair at its own pace of a tensor dictionary of 2 MiB, 4 MiB, 6 KiB and 1,000,003 B sent as torch's batch_isend_irecv groups them, one tensor back, and every third microbatch both ways in one group): 64 of 64 checks, every received byte exact; each pair ran 20 pair exchanges through its ring lanes' window of 65,536 B (640 to 1,408 window chunks) and its small transfers through the forward windows; without the ring plan 64 of 64, every transfer through the forward windows. `tests/emulation/torch_pp.py`: vLLM's exchange through torch.distributed, initialized lazily: TP 4 x PP 2 under the same ring:8 settings (an all-reduce on the default group, `new_group` tensor-parallel and pipeline groups, four microbatches) 56 of 56 checks exact over four two-rank point-to-point communicators, each pair's ring lanes windowed; a chain of four stages 12 of 12 over three two-rank communicators. Not run on Sparks | | Communicators of some of a layout's positions (split children; two-rank communicators such as torch's per-pair ones) under the route planner's per-process settings, one `LIBSIRCL_CHAIN_ORDER`, `LIBSIRCL_FORWARD_WINDOWS` and `LIBSIRCL_RING_WINDOW` for every communicator a process joins: the chain order's listed members in its order, the other positions skipped; a ring next reached through relays keeps the smallest forward window toward it when the layout's ring plan gives 0 | implemented | Libraries up to and including snapshot db529218 (the serving image's) refuse every such communicator at setup ("LIBSIRCL_CHAIN_ORDER=0,1,2,3,4,5,6,7 must list the positions of the communicator's ranks once"): `torch_pp.py --shape tp4pp2` failed there with 17 problems (the tensor-parallel groups did not form) and `pp_pairs.py` 8 of 8 (the splits); this tree's library forms them (the pipeline row's runs). With the ring lanes' window not derived (a research build of this tree), every pipeline pair ran its 20 pair exchanges through relays unwindowed (ring window 0 B, 0 window chunks; emulation, which has no relay queues, still exact). Unchanged where the ring next is a cable or the plan gives a window: cabled pairs, `path:0-3` (393,216 B on position 3) and ring:8; the pair default, windowed pair, relayed pair, ring schedules on two, four and eight ranks, the four-rank ring with windows and pieces on four and eight ranks passed every check of `library_rank.py` (533, 533, 531, 533, 989, 1,977, 989, 985 and 1,969) | | `ncclMemAlloc` and `ncclMemFree` (device memory of the current CUDA context, ordinary buffers for every collective, eager and in graphs); buffer and window registration (`ncclCommRegister`, `ncclCommWindowRegister` and their deregistrations) accepted as hints, the handle and the window being the buffer's address, and `ncclWinGetUserPtr` | implemented | API suite (registration and windows on a CPU communicator); nccl-tests v2.21.1 allocates its buffers with `ncclMemAlloc` | -| `ncclRedOpCreatePreMulSum` and other ops beyond the five built in, the per-call `nccl*Config` collectives, shrink, grow, the scalable initialization, host RMA signals, the device communicator and device pointers | unsupported | refused with `ncclInvalidUsage` (an op number beyond the built-in five: `ncclInvalidArgument`, counted in the receipt) and logged once | +| The 37 functions of the header that `tests/api_manifest.json` marks not implemented: `ncclCommShrink`, `ncclCommGetUniqueId`, `ncclCommGrow`, `ncclCommInitRankScalable`, `ncclResetDebugInit`, `ncclCommSuspend`, `ncclCommResume`, `ncclCommMemStats`, `ncclRedOpCreatePreMulSum`, `ncclRedOpDestroy`, `ncclAllReduceConfig`, `ncclBroadcastConfig`, `ncclReduceConfig`, `ncclAllGatherConfig`, `ncclReduceScatterConfig`, `ncclAlltoAllConfig`, `ncclGatherConfig`, `ncclScatterConfig`, `ncclPutSignal`, `ncclSignal`, `ncclWaitSignal`, `ncclGroupSimulateEnd`, `ncclSetEncryption`, `ncclParamBind`, `ncclParamGetI8`, `ncclParamGetI16`, `ncclParamGetI32`, `ncclParamGetI64`, `ncclParamGetU8`, `ncclParamGetU16`, `ncclParamGetU32`, `ncclParamGetU64`, `ncclParamGetStr`, `ncclParamGet`, `ncclParamGetParameter`, `ncclParamGetAllParameterKeys`, `ncclParamDumpAll`; ops beyond the five built in; the device communicator and device pointers | unsupported | refused with `ncclInvalidUsage` (an op number beyond the built-in five: `ncclInvalidArgument`, counted in the receipt) and logged once | | One progress thread per process; an abort word watched by every kernel wait | unsupported | each needs a change to SIRCL's package (`spark_transport/sircl`) that it does not have: an externally driven progress loop with which one thread serves every session of a process (request PO), and an abort word in the command ring that every timed flag wait watches (request AW) | | All-reduce timing on a cabled pair (`tests/emulation/perf_rank.py`: bf16, fp16, fp32, 8 B to 256 MiB, eager and graph) | qualified | the pair (hardware evidence): fp32 8 KiB 9.73 us eager, 9.61 us in graphs | | nccl-tests v2.21.1, unmodified, through `LD_PRELOAD` on a cabled pair: every collective test and point-to-point | qualified | snapshot fb63329a on Sparks 0-1 (hardware evidence): `#wrong 0` on every line of `RUNBOOK.md` sections 3.4 and 3.5; snapshot 0f94c72b, this tree's library and pair plan, on Sparks 6-7, 2026-10-09 (hardware evidence): `#wrong 0` on every row of sections 3.4 and 3.5 and of `sendrecv_perf` to 256 MiB (in-place `N/A` on all-to-all and sendrecv, which have no in-place result) | -| Frameworks on the ring | unsupported | PyTorch in emulation only | +| Frameworks on the ring | research-only | vLLM's PyNccl on a cabled pair of Sparks (the communicator-creation row above; the installer's libsircl lane, `docs/architecture/libsircl.md`); torch's own process groups in GPU emulation only | ## Evidence -The files behind each result are in `verification/` (CPU, build and loading records), -`verification/emulation/` (GPU emulation logs and per-rank results) and `verification/hardware/` (the -cabled pairs). +The results below cite their run records by paths under `verification/` (CPU, build and loading +records), `verification/emulation/` (GPU emulation logs and per-rank results) and `verification/hardware/` +(the Spark runs); as the scope above says, those records stay on the machines that made them and are not +in this repository. ### Hardware: the cabled pair at ring positions 0-1 @@ -387,7 +390,7 @@ DSL or from the pack (`tests/emulation/mixed_group.py`). Measurement, result and `KERNEL_ROUTE.md`. In short: 905 of 905 checks on `path:0-1`, `ring:3`, `path:0-3` (one-block and every-block polling) and `ring:8`, every output bit-exact against the host reference and the DSL's output, eager and in graph replay, mixed groups included; pack `c2e6e5a1...be4eaa25`. The test shim, -rebuilt to load both packs, repeats the `path:0-1` run with 181 of 181 checks passed +rebuilt to load the packs of that build, repeats the `path:0-1` run with 181 of 181 checks passed (`verification/emulation/mixed-pair-rebuilt-shim.log`). ### Library end to end: one process per rank on one GPU @@ -513,7 +516,7 @@ captured in a CUDA graph and replayed twice; `destroy_process_group`; every tens reference bit for bit. Result: every check passed with 2 ranks (46 checks) and with 4 ranks (79 checks; point-to-point runs with two ranks only, since torch then issues it on the four-rank communicator); torch reports `comm_split_count` 1, so `new_group` split the group's communicator through `ncclCommSplit`; every -receipt reports both packs' hashes, `"forwarded":0` and no refusals +receipt reports the hashes of its library's packs, `"forwarded":0` and no refusals (`verification/emulation/torch-pg-w2.log`, `torch-pg-w4.log`). Conclusion: torch's NCCL backend runs unmodified on libsircl in emulation, including object collectives, integer and averaging reductions, subgroups and, on two ranks, point-to-point. @@ -609,19 +612,16 @@ Conditions: RTX 5090, WSL2; pack kernels launched on a poisoned session so each (`tests/emulation/launch_bench.c`); 20,000 launches in batches of 100. Result: one-shot 4.25 us mean (3.48 us best batch), two-shot 4.19 us (3.67 us) per `cuLaunchKernel`. Conclusion: a C launch of the pack costs about 4 us on the workstation; GB10 measured 2.75 us for a DSL kernel through a prebuilt -argument block (handoff section 4), so the eager 8 KiB target of 20 us on a pair keeps its margin. Not +argument block (a measurement outside this repository's evidence), so the eager 8 KiB target of 20 us on a pair keeps its margin. Not measured on GB10. ### CPU suites, builds and loading Conditions: WSL2, GCC 13.3, Python 3.12, no GPU or RDMA device. Results: -- `make check` with `SIRCL_PACKAGE` naming the SIRCL reference's `spark_transport/sircl`: ABI 5, - API 17, bootstrap 22, lifecycle 14, engine 8, shared-memory verbs 5, vendored native sources 4, - point-to-point native library 5, route planner 8, MPI shim 6, nccl-tests evaluator 12 and kernel entries - 3 tests passed; 10,057 fabric - vector checks passed (`verification/library-tests.log`). Without `SIRCL_PACKAGE` the route planner's 8 - tests are skipped. +- `make check` with `SIRCL_PACKAGE` naming the SIRCL reference's `spark_transport/sircl`: every suite + passed, and the fabric vector checks passed (`verification/library-tests.log`). Without `SIRCL_PACKAGE` + the route planner's tests are skipped. - The CUDA-on-load test passes against the library and fails against a control library that initializes CUDA in a constructor; `tools/probe_cuda_on_load.py` on the workstation finds that the library maps and initializes nothing of CUDA, with and without site packages (`verification/cuda-on-load.json`). @@ -678,8 +678,8 @@ Conditions: WSL2, GCC 13.3, Python 3.12, no GPU or RDMA device. Results: `tools/site_routes.py --session-share` below 1 (0.95 to 0.05 checked) takes at least one chunk from every relayed session lane (at 0.95: 64-128 KiB become 32-96 KiB) and gives all 40 relayed ordered pairs a 32 KiB channel window, as `--max-window 32768` does (every session lane one chunk); the session's cost is not - yet measured on Sparks. The channels have run in emulation only, where no lane crosses - a relay. + yet measured on Sparks. In emulation no lane crosses a relay; on Sparks the channels ran on the path of + four and the cycle of eight (snapshot a3477af2, hardware evidence above). - The library orders eager calls across streams, not CUDA graph replays: a graph that holds a communicator's collectives must not run while that communicator's other work runs. - The fold path sends every rank's whole message to every rank (an all-gather or all-to-all, then a diff --git a/spark_transport/libsircl/tests/test_docs_consistency.py b/spark_transport/libsircl/tests/test_docs_consistency.py new file mode 100644 index 00000000..6d745c88 --- /dev/null +++ b/spark_transport/libsircl/tests/test_docs_consistency.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +"""The counts libsircl's documents state agree with the code. + +- README.md states the number of the header's functions and of those implemented, which + tests/api_manifest.json records; STATUS.md lists every function the manifest marks not implemented. +- README.md, STATUS.md and KERNEL_ROUTE.md speak of the four kernel packs the loader embeds + (src/kernelpack.c's sccl_*_fatbin symbols), never of three or of "both" packs. +- README.md and STATUS.md name the version of this repository's SIRCL package. +""" +import json +import re +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +REPOSITORY = ROOT.parents[1] + + +def text(name): + return " ".join((ROOT / name).read_text(encoding="utf-8").split()) + + +class DocsConsistencyTests(unittest.TestCase): + def test_function_counts_match_the_manifest(self): + functions = json.loads((ROOT / "tests/api_manifest.json").read_text(encoding="utf-8"))["functions"] + implemented = sum(function["implemented"] for function in functions) + readme = text("README.md") + self.assertIn(f"implements {implemented} of the header's {len(functions)} functions", readme) + self.assertIn(f"the other {len(functions) - implemented} return `ncclInvalidUsage`", readme) + status = text("STATUS.md") + missing = [function["name"] for function in functions + if not function["implemented"] and f"`{function['name']}`" not in status] + self.assertEqual(missing, [], "STATUS.md does not list these unsupported functions") + + def test_the_documents_count_the_packs_the_loader_embeds(self): + loader = (ROOT / "src/kernelpack.c").read_text(encoding="utf-8") + packs = set(re.findall(r"extern const unsigned char (sccl_\w+_fatbin)\[\];", loader)) + self.assertEqual(len(packs), 4) + for name in ("README.md", "STATUS.md", "KERNEL_ROUTE.md"): + body = text(name) + with self.subTest(document=name): + self.assertNotRegex(body, r"\bthree (embedded |kernel )?packs\b") + self.assertNotRegex(body, r"\bboth packs\b") + self.assertIn("four kernel packs", text("STATUS.md")) + + def test_the_sircl_package_version(self): + version = re.search(r'^version = "([^"]+)"', (REPOSITORY / "spark_transport/sircl/pyproject.toml").read_text( + encoding="utf-8"), re.M).group(1) + for name in ("README.md", "STATUS.md"): + with self.subTest(document=name): + self.assertIn(f"`sparkring_sircl` {version}", text(name)) + + +if __name__ == "__main__": + unittest.main(verbosity=2) From 8168a0db8eabd16643641c5907503c446754f726 Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Thu, 15 Oct 2026 10:01:44 -0500 Subject: [PATCH 61/65] Reclaim engines quarantined by an abort once their kernels finish When ncclCommAbort (or a destroy whose context wait failed) finds a kernel of the communicator still running, the engine's arena, device memory and channels stay allocated, since the kernel may still read them. Such an engine records events behind its last launch and every channel stream and joins a quarantine list. Every later communicator creation and destroy in the process retries the list: an engine whose events completed is freed (memory, channels, order event, its kernel-pack reference), and one whose context is gone has its host bookkeeping freed. Every receipt reports quarantined_bytes, the memory the list holds. Engines kept because a queue pair or registration could not be released are not reclaimed: such an object may still be written, and the native layer that created it has already been destroyed. The library binds cuEventQuery (present in every CUDA driver), and the stand-in driver provides it. Compatibility: no change for teardowns without a running kernel; quarantined memory is returned during the process instead of at its exit. Receipts gain the quarantined_bytes field. Validation: a running kernel at abort needs an engine of two or more ranks, which the CPU harness cannot build, and the reclaim path has not run. tests/test_engine_cpu.py checks that a one-rank communicator's receipt reports quarantined_bytes 0. The emulation case the request proposes (SIRCL_EMU_FAIL_DEREG=1 with the fault cleared later) does not apply, because a failed verbs release is never reclaimed; no emulation case is written. make check and make check-asan pass; README.md states the behavior. Request item 19 (audit 06-F2, 06-F3, leak part of 04-F5). NVIDIA NCCL source was neither read nor adapted. Co-Authored-By: Claude Opus 5.5 --- spark_transport/libsircl/README.md | 5 +- spark_transport/libsircl/src/cuda_api.c | 1 + spark_transport/libsircl/src/cuda_api.h | 1 + spark_transport/libsircl/src/engine.c | 93 ++++++++++++++++++- spark_transport/libsircl/tests/fake_cuda.c | 1 + .../libsircl/tests/test_engine_cpu.py | 7 +- 6 files changed, 101 insertions(+), 7 deletions(-) diff --git a/spark_transport/libsircl/README.md b/spark_transport/libsircl/README.md index 77445d14..6816407f 100644 --- a/spark_transport/libsircl/README.md +++ b/spark_transport/libsircl/README.md @@ -186,7 +186,10 @@ naming it. The close is terminal: from its start the communicator refuses new wo its error, and a second `ncclCommFinalize` or the destroy that follows returns the same result. When the transport's queue pairs or registrations cannot be released, destroy keeps the communicator's memory allocated and returns `ncclSystemError`. Ranks destroy their communicators in the same order, as they issue -collectives. Finalize runs the rounds once and destroy then only frees; `ncclCommAbort` does not wait. Finalize is +collectives. Finalize runs the rounds once and destroy then only frees; `ncclCommAbort` does not wait. When an abort finds a +kernel of the communicator still running, its memory stays allocated (quarantined) until a later communicator +creation or destroy in the process finds that kernel finished and frees it; every receipt's `quarantined_bytes` +says how much memory is held so. Finalize is synchronous for every communicator: a non-blocking one (`blocking` 0 in its config) also runs both rounds before `ncclCommFinalize` returns, which never returns `ncclInProgress`; a caller that must not block runs finalize on a thread of its own. diff --git a/spark_transport/libsircl/src/cuda_api.c b/spark_transport/libsircl/src/cuda_api.c index 0d7aa45d..56d9c5b7 100644 --- a/spark_transport/libsircl/src/cuda_api.c +++ b/spark_transport/libsircl/src/cuda_api.c @@ -66,6 +66,7 @@ static void load(void) { SYM(EventRecord, "cuEventRecord"); SYM(EventDestroy, "cuEventDestroy_v2"); SYM(EventSynchronize, "cuEventSynchronize"); + SYM(EventQuery, "cuEventQuery"); #undef SYM *(void **)(&table.MemAllocAsync) = dlsym(handle, "cuMemAllocAsync"); *(void **)(&table.MemFreeAsync) = dlsym(handle, "cuMemFreeAsync"); diff --git a/spark_transport/libsircl/src/cuda_api.h b/spark_transport/libsircl/src/cuda_api.h index 17002d6d..3454ac98 100644 --- a/spark_transport/libsircl/src/cuda_api.h +++ b/spark_transport/libsircl/src/cuda_api.h @@ -76,6 +76,7 @@ typedef struct sccl_cuda { sccl_CUresult (*EventRecord)(sccl_CUevent, sccl_CUstream); sccl_CUresult (*EventDestroy)(sccl_CUevent); sccl_CUresult (*EventSynchronize)(sccl_CUevent); + sccl_CUresult (*EventQuery)(sccl_CUevent); /* Optional (NULL when the driver lacks them): stream-ordered allocation, which inside a CUDA graph * capture adds allocation and free nodes to the graph. */ sccl_CUresult (*MemAllocAsync)(sccl_CUdeviceptr *, size_t, sccl_CUstream); diff --git a/spark_transport/libsircl/src/engine.c b/spark_transport/libsircl/src/engine.c index 4e1bcebe..84e6c5a4 100644 --- a/spark_transport/libsircl/src/engine.c +++ b/spark_transport/libsircl/src/engine.c @@ -448,6 +448,12 @@ struct sccl_engine { int lane_check_ms; /* LIBSIRCL_LANE_CHECK_MS: the lane check's bound */ int traffic_class; /* SIRCL_TRAFFIC_CLASS or NCCL_IB_TC */ int kp_held; /* this engine holds a reference on its context's kernel-pack entry (sccl_kp_load) */ + /* Quarantine (sccl_engine_destroy after an abort with a launch still running): the events recorded + * behind the last launch and the channels' work, the bytes held, the next quarantined engine. */ + sccl_CUevent quarantine_done[1 + 2 * MAX_WORLD]; + int quarantine_events; + uint64_t quarantine_bytes; + struct sccl_engine *quarantine_next; uint64_t stop_ns, stopped_ns; int have_last; sccl_CUevent order_event; @@ -2219,10 +2225,13 @@ void sccl_engine_fail_stop_shutdown(void) { if (started) pthread_join(watcher, NULL); } +static void quarantine_reclaim(void); int sccl_engine_create(sccl_bootstrap *bootstrap, int nranks, int rank, int position, sccl_CUcontext ctx, const atomic_int *cancelled, sccl_engine **out, char *err, size_t err_len) { *out = NULL; cu = sccl_cuda_get(); + /* Engines quarantined by earlier aborts whose kernels have finished since are freed first. */ + quarantine_reclaim(); sccl_engine *e = calloc(1, sizeof *e); if (!e) { put_error(err, err_len, "out of memory"); @@ -4520,6 +4529,8 @@ static void p2p_receipt(sccl_engine *e, char *out, size_t len) { c ? (int)t->stat(c, P2P_STAT_ABORT_FROM) - 1 : -1); } +/* The bytes quarantined engines hold (see quarantine_reclaim), reported in every receipt. */ +static _Atomic uint64_t quarantined_bytes; size_t sccl_engine_receipt(sccl_engine *e, char *out, size_t len) { static const char *const dt[SCCL_DT_COUNT] = {"float32", "float16", "bfloat16"}; char ops[512] = {0}; @@ -4567,7 +4578,7 @@ size_t sccl_engine_receipt(sccl_engine *e, char *out, size_t len) { out, len, "{\"schema\":\"libsircl-receipt/v1\",\"library\":\"libsircl " LIBSIRCL_VERSION "\",\"pid\":%d," "\"communicator\":%d,\"rank\":%d,\"world\":%d,\"position\":%d,\"lanes\":%d," - "\"device\":%d," + "\"device\":%d,\"quarantined_bytes\":%" PRIu64 "," "\"transport\":\"%s\",\"kernel_pack\":\"%s\",\"fold_pack\":\"%s\",\"links_pack\":\"%s\"," "\"slot_bytes\":%" PRIu64 ",\"capacity\":%" PRIu64 @@ -4592,6 +4603,7 @@ size_t sccl_engine_receipt(sccl_engine *e, char *out, size_t len) { "},\"forwarded\":0,\"native\":{\"ops_posted\":%" PRIu64 ",\"writes_completed\":%" PRIu64 ",\"phases_posted\":%" PRIu64 "},\"healthy\":%s,\"error\":%s}", (int)getpid(), e->receipt_id, e->rank, e->world, e->position, e->lanes, e->device, + atomic_load_explicit(&quarantined_bytes, memory_order_relaxed), e->world == 1 ? "none" : e->transport->name, sccl_kp_hash(), sccl_kp_fold_hash(), sccl_kp_links_hash(), @@ -4812,8 +4824,79 @@ void sccl_engine_teardown_stop(sccl_engine *e) { e->stopped = 1; } +/* Engines whose arena a kernel may still read: an abort found a launch still running. Each holds events + * recorded behind its last launch and its channels' work; later creations and destroys retry them + * (quarantine_reclaim) and free every engine whose events completed, or whose context is gone (and with it + * every allocation and kernel). Engines kept because a queue pair or registration could not be released + * are not quarantined here and stay allocated: such an object may still be written. */ +static pthread_mutex_t quarantine_lock = PTHREAD_MUTEX_INITIALIZER; +static sccl_engine *quarantined; +static _Atomic uint64_t quarantined_bytes; + +static void quarantine_reclaim(void) { + if (!cu) return; + pthread_mutex_lock(&quarantine_lock); + for (sccl_engine **at = &quarantined; *at;) { + sccl_engine *e = *at; + int pushed = cu->CtxPushCurrent(e->ctx) == SCCL_CUDA_SUCCESS, done = 1; + for (int i = 0; pushed && done && i < e->quarantine_events; ++i) + done = cu->EventQuery(e->quarantine_done[i]) == SCCL_CUDA_SUCCESS; + if (!done) { + sccl_CUcontext popped; + cu->CtxPopCurrent(&popped); + at = &e->quarantine_next; + continue; + } + *at = e->quarantine_next; + atomic_fetch_sub_explicit(&quarantined_bytes, e->quarantine_bytes, memory_order_relaxed); + if (pushed) { + for (int i = 0; i < e->quarantine_events; ++i) cu->EventDestroy(e->quarantine_done[i]); + if (e->order_event) cu->EventDestroy(e->order_event); + p2p_close(e, 1); + release_memory(e); + if (e->kp_held) sccl_kp_release(e->ctx, 1); + sccl_CUcontext popped; + cu->CtxPopCurrent(&popped); + } else { + /* The context is gone, and with it every device allocation and kernel: only host bookkeeping. */ + free(e->retired_stage); + if (e->host && e->host_emulated) sccl_emu_segment_free(e->host); + if (e->p2p_host && e->p2p_host_emulated) sccl_emu_segment_free(e->p2p_host); + if (e->kp_held) sccl_kp_release(e->ctx, 0); + } + free(e); + } + pthread_mutex_unlock(&quarantine_lock); +} + +/* With the engine's context current: record the events quarantine_reclaim waits for (behind the last + * launch and every channel stream) and put the engine on the quarantine list. 0 when an event could not be + * made: the engine then stays allocated without a retry. */ +static int quarantine_add(sccl_engine *e) { + int n = 0; + sccl_CUstream streams[1 + 2 * MAX_WORLD]; + if (e->have_last) streams[n++] = e->last_stream; + for (int d = 0; d < 2; ++d) + for (int p = 0; p < e->world && p < MAX_WORLD; ++p) + if (e->p2p_ch[d][p].stream) streams[n++] = e->p2p_ch[d][p].stream; + e->quarantine_events = 0; + for (int i = 0; i < n; ++i) { + if (cu->EventCreate(&e->quarantine_done[i], SCCL_CU_EVENT_DISABLE_TIMING) != SCCL_CUDA_SUCCESS) return 0; + e->quarantine_events = i + 1; + if (cu->EventRecord(e->quarantine_done[i], streams[i]) != SCCL_CUDA_SUCCESS) return 0; + } + e->quarantine_bytes = e->region_bytes + e->shared.slot_bytes * (uint64_t)(1 + e->world); + atomic_fetch_add_explicit(&quarantined_bytes, e->quarantine_bytes, memory_order_relaxed); + pthread_mutex_lock(&quarantine_lock); + e->quarantine_next = quarantined; + quarantined = e; + pthread_mutex_unlock(&quarantine_lock); + return 1; +} + ncclResult_t sccl_engine_destroy(sccl_engine *e, int abort, char *err, size_t len) { if (!e) return ncclSuccess; + quarantine_reclaim(); pthread_mutex_lock(&e->lock); e->closing = 1; pthread_mutex_unlock(&e->lock); @@ -4855,7 +4938,7 @@ ncclResult_t sccl_engine_destroy(sccl_engine *e, int abort, char *err, size_t le "verbs calls); their arena stays allocated", e->rank, p2p_unreleased); } /* A queue pair, completion queue or registration whose release failed may still be live: the arena - * and device memory stay allocated (quarantined) until the process exits. */ + * and device memory stay allocated until the process exits (no retry frees them). */ int unreleased = e->proxy ? e->transport->destroy(e->proxy) : 0; e->proxy = NULL; int released = unreleased == 0 && p2p_unreleased == 0; @@ -4866,7 +4949,8 @@ ncclResult_t sccl_engine_destroy(sccl_engine *e, int abort, char *err, size_t le } dump_teardown(e, e->stop_ns, e->stopped_ns, realtime_ns()); /* After an abort with a launch still running, its kernel may read the arena and counters - * until its wait limit; they stay allocated until the process exits. */ + * until its wait limit; they stay allocated, quarantined, until a later creation or destroy finds that + * kernel finished (quarantine_reclaim). */ int freed = pushed && idle && released; if (freed) { if (e->order_event) cu->EventDestroy(e->order_event); @@ -4886,11 +4970,12 @@ ncclResult_t sccl_engine_destroy(sccl_engine *e, int abort, char *err, size_t le e->host = e->p2p_host = NULL; freed = 1; } + int queued = !freed && pushed && released && quarantine_add(e); if (pushed) { sccl_CUcontext popped; cu->CtxPopCurrent(&popped); } pthread_mutex_destroy(&e->lock); - if (freed) free(e); + if (freed && !queued) free(e); return result; } diff --git a/spark_transport/libsircl/tests/fake_cuda.c b/spark_transport/libsircl/tests/fake_cuda.c index c80180f7..5159d84f 100644 --- a/spark_transport/libsircl/tests/fake_cuda.c +++ b/spark_transport/libsircl/tests/fake_cuda.c @@ -238,5 +238,6 @@ UNUSED_ENTRY(cuEventCreate) UNUSED_ENTRY(cuEventRecord) UNUSED_ENTRY(cuEventDestroy_v2) UNUSED_ENTRY(cuEventSynchronize) +UNUSED_ENTRY(cuEventQuery) UNUSED_ENTRY(cuMemAllocAsync) UNUSED_ENTRY(cuMemFreeAsync) diff --git a/spark_transport/libsircl/tests/test_engine_cpu.py b/spark_transport/libsircl/tests/test_engine_cpu.py index d63d8d67..af9a1460 100644 --- a/spark_transport/libsircl/tests/test_engine_cpu.py +++ b/spark_transport/libsircl/tests/test_engine_cpu.py @@ -265,11 +265,14 @@ def test_receipts_never_follow_a_planted_link(self): c=C.c_void_p(); assert l.ncclCommInitRank(C.byref(c),1,u,0)==0 assert l.ncclCommDestroy(c)==0 print(json.dumps([open(target).read(), os.path.islink(receipt + ".partial"), os.path.isfile(receipt), - sorted(name for name in os.listdir(directory) if ".partial." in name)])) + sorted(name for name in os.listdir(directory) if ".partial." in name), + json.load(open(receipt)).get("quarantined_bytes")])) '''.replace("sys.argv[2]", repr(directory)), {"LIBSIRCL_BOOTSTRAP_ONLY": "", "LIBSIRCL_TRANSPORT": "verbs", "LD_LIBRARY_PATH": str(FAKE_DRIVER)}) self.assertEqual(out.returncode, 0, out.stderr[-2000:]) - victim, still_link, written, leftovers = json.loads(out.stdout) + victim, still_link, written, leftovers, quarantined = json.loads(out.stdout) + # Every receipt reports the memory quarantined engines hold; none here. + self.assertEqual(quarantined, 0) self.assertEqual(victim, "untouched") self.assertTrue(still_link) self.assertTrue(written) From a1bebb84cc3a9f2e2dfd41e7b5ef40bbc22f843c Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Thu, 15 Oct 2026 16:23:31 -0500 Subject: [PATCH 62/65] Judge concurrent close against accepted collective counts The teardown race exchanges each worker's successful enqueue count through the surviving parent communicator. Finalize and destroy must succeed when all ranks posted equal counts; unequal counts also permit ncclRemoteError because unmatched collectives time out. The initial collective must remain exact, the first worker call must succeed before close, subsequent calls must return success or 4/5, and a call after destroy must return 4/5. Bound worker startup and join, record worker exceptions, and reject failed rank processes even when they left successful JSON. Keep the process deadline and kill behavior. Add CPU harness regressions to make check and CTest for count agreement, refusal codes, thread failures and process exits. Validation: make check and make check-asan pass; the final eight harness tests pass with and without ASan. Ruff and repository links pass. GPU execution remains pending. No library implementation changes. --- spark_transport/libsircl/CMakeLists.txt | 2 +- spark_transport/libsircl/Makefile | 1 + .../libsircl/tests/emulation/teardown_race.py | 93 ++++++++--- .../libsircl/tests/test_teardown_race.py | 153 ++++++++++++++++++ 4 files changed, 225 insertions(+), 24 deletions(-) create mode 100644 spark_transport/libsircl/tests/test_teardown_race.py diff --git a/spark_transport/libsircl/CMakeLists.txt b/spark_transport/libsircl/CMakeLists.txt index ed9a6976..4d75bf84 100644 --- a/spark_transport/libsircl/CMakeLists.txt +++ b/spark_transport/libsircl/CMakeLists.txt @@ -141,7 +141,7 @@ add_test(NAME kernelpack_cache COMMAND ${CMAKE_COMMAND} -E env foreach(name abi api lifecycle engine_cpu build_flags) add_test(NAME ${name} COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tests/test_${name}.py --library $) endforeach() -foreach(name bootstrap shm_verbs vendored_sources p2p_native site_routes mpi_shim check_nccl_tests source_snapshot docs_consistency) +foreach(name bootstrap shm_verbs vendored_sources p2p_native site_routes mpi_shim check_nccl_tests source_snapshot docs_consistency teardown_race) add_test(NAME ${name} COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tests/test_${name}.py) endforeach() add_test(NAME kernel_entries COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tests/test_kernel_entries.py diff --git a/spark_transport/libsircl/Makefile b/spark_transport/libsircl/Makefile index 66a7f7dd..94984f32 100644 --- a/spark_transport/libsircl/Makefile +++ b/spark_transport/libsircl/Makefile @@ -203,6 +203,7 @@ check: all $(BUILD)/test_fabric $(BUILD)/test_link_plan $(BUILD)/test_kernelpack $(PYTHON) tests/test_check_nccl_tests.py || status=1; \ $(PYTHON) tests/test_kernel_entries.py --build $(BUILD) || status=1; $(PYTHON) tests/test_build_flags.py --library $(abspath $(BUILD)/libsircl.so) || status=1; $(PYTHON) tests/test_source_snapshot.py || status=1; \ $(PYTHON) tests/test_docs_consistency.py || status=1; \ + $(PYTHON) tests/test_teardown_race.py || status=1; \ exit $$status # The check suites against a host library built with AddressSanitizer and UndefinedBehaviorSanitizer in diff --git a/spark_transport/libsircl/tests/emulation/teardown_race.py b/spark_transport/libsircl/tests/emulation/teardown_race.py index 6edfd994..b7a332cf 100644 --- a/spark_transport/libsircl/tests/emulation/teardown_race.py +++ b/spark_transport/libsircl/tests/emulation/teardown_race.py @@ -26,9 +26,10 @@ stand-in fails every deregistration, and destroy keeps the arena allocated). With --expect concurrent-close each round's child is finalized and destroyed while another thread of the -rank keeps enqueueing all-reduces on it; the round passes when both calls succeed, the enqueueing thread -ends, and its calls returned success or a refusal (ncclInvalidArgument or ncclInvalidUsage) once the close -began. +rank keeps enqueueing all-reduces on it. The parent communicator exchanges the threads' accepted call +counts after close. Both close calls must succeed when counts match; when counts differ, ncclRemoteError +is also allowed because unmatched collectives time out. Every thread must exit, its calls must return +success or a refusal (ncclInvalidArgument or ncclInvalidUsage), and a call after destroy must be refused. With --expect revoke-destroy each round revokes the child (ncclCommRevoke) right after its all-reduce and then destroys it. Destroy after revoke runs the same two teardown rounds as destroy of a ready @@ -69,6 +70,43 @@ } +def close_concurrently(enqueue, finalize, destroy, prepare_thread, exchange_counts, timeout=30): + """Race close against enqueue; exchange counts only after the worker has stopped changing them.""" + started_calls = threading.Event() + results, errors = [], [] + + def worker_main(): + try: + prepare_thread() + for _ in range(1000): + result = enqueue() + results.append(result) + started_calls.set() + if result: + break + except Exception as error: + errors.append(f"{type(error).__name__}: {error}") + finally: + started_calls.set() + + worker = threading.Thread(target=worker_main) + worker.start() + if not started_calls.wait(timeout): + raise TimeoutError("enqueue thread did not finish its first call before close") + finalized = finalize() + destroyed = destroy() + worker.join(timeout=timeout) + if worker.is_alive(): + raise TimeoutError("enqueue thread did not exit after close") + after = enqueue() + counts = exchange_counts(results.count(0)) + allowed = (0,) if len(set(counts)) == 1 else (0, 6) + return dict(ok=bool(results) and results[0] == 0 and not errors and finalized in allowed and destroyed in allowed + and all(result in (0, 4, 5) for result in results) and after in (4, 5), + finalize=finalized, destroy=destroyed, calls=results, accepted_counts=counts, + after_close=after, worker_exited=True, worker_errors=errors) + + def run_rank(args) -> int: import torch @@ -82,6 +120,8 @@ def run_rank(args) -> int: lib.ncclGetLastError.argtypes = [ctypes.c_void_p] lib.ncclAllReduce.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_size_t, ctypes.c_int, ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p] + lib.ncclAllGather.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_size_t, ctypes.c_int, + ctypes.c_void_p, ctypes.c_void_p] lib.ncclCommInitRank.argtypes = [ctypes.POINTER(ctypes.c_void_p), ctypes.c_int, unique_id.UniqueId, ctypes.c_int] lib.ncclCommSplit.argtypes = [ctypes.c_void_p, ctypes.c_int, ctypes.c_int, ctypes.POINTER(ctypes.c_void_p), @@ -147,26 +187,28 @@ def save(): save() continue if args.expect == "concurrent-close": - started_calls = threading.Event() - results = [] - def enqueue(): - torch.cuda.set_device(0) - for _ in range(1000): - result = lib.ncclAllReduce(ctypes.c_void_p(x.data_ptr()), ctypes.c_void_p(y.data_ptr()), - count, 9, 0, child, ctypes.c_void_p(stream.cuda_stream)) - results.append(result) - started_calls.set() - if result: - break - worker = threading.Thread(target=enqueue) - worker.start() - started_calls.wait() - finalized = lib.ncclCommFinalize(child) - destroyed = lib.ncclCommDestroy(child) - worker.join(timeout=30) - entry.update(ok=not worker.is_alive() and not finalized and not destroyed - and all(result in (0, 4, 5) for result in results), - finalize=finalized, destroy=destroyed, calls=results) + def exchange_counts(accepted): + # The parent already coordinates the splits and must survive every child close. + sent = torch.tensor([accepted], dtype=torch.int64, device="cuda") + received = torch.empty(world, dtype=torch.int64, device="cuda") + torch.cuda.synchronize() + gathered = lib.ncclAllGather(ctypes.c_void_p(sent.data_ptr()), + ctypes.c_void_p(received.data_ptr()), 1, 4, comm, + ctypes.c_void_p(stream.cuda_stream)) + torch.cuda.synchronize() + state = ctypes.c_int(0) + queried = lib.ncclCommGetAsyncError(comm, ctypes.byref(state)) + if gathered or queried or state.value: + raise RuntimeError(f"parent count exchange failed: {gathered}, {queried}, {state.value}") + return received.cpu().tolist() + + entry.update(close_concurrently( + lambda: lib.ncclAllReduce(ctypes.c_void_p(x.data_ptr()), ctypes.c_void_p(y.data_ptr()), + count, 9, 0, child, ctypes.c_void_p(stream.cuda_stream)), + lambda: lib.ncclCommFinalize(child), lambda: lib.ncclCommDestroy(child), + lambda: torch.cuda.set_device(0), exchange_counts)) + entry.update(ok=entry["ok"] and exact, exact=exact, + seconds=round(time.perf_counter() - started, 3)) rounds.append(entry) save() continue @@ -269,6 +311,8 @@ def run_group(args) -> int: f"statuses {codes}, {len(problems)} problems, {time.perf_counter() - started:.1f} s; work {work}") return 1 if problems else 0 for rank in range(args.world): + if codes[rank] != 0: + problems.append(f"rank {rank}: exit {codes[rank]}; log {work / f'rank{rank}.log'}") path = work / f"rank{rank}.json" if not path.exists(): problems.append(f"rank {rank}: no result (exit {codes[rank]}); log {work / f'rank{rank}.log'}") @@ -285,6 +329,9 @@ def run_group(args) -> int: problems.append(f"rank {rank} round {entry['round']}: the close did not fail as expected: " f"finalize {entry.get('finalize')}, all-reduce after the close " f"{entry.get('after_close')}, destroy {entry.get('destroy')}: {entry.get('error')}") + elif not entry.get("ok") and args.expect == "concurrent-close": + failed_rounds.add(entry["round"]) + problems.append(f"rank {rank} round {entry['round']}: concurrent close {entry}") elif not entry.get("ok"): failed_rounds.add(entry["round"]) problems.append(f"rank {rank} round {entry['round']}: exact {entry.get('exact')}, destroy " diff --git a/spark_transport/libsircl/tests/test_teardown_race.py b/spark_transport/libsircl/tests/test_teardown_race.py new file mode 100644 index 00000000..ae597377 --- /dev/null +++ b/spark_transport/libsircl/tests/test_teardown_race.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +"""CPU checks of the teardown harness's verdicts, thread failures and process deadline.""" +from concurrent.futures import ThreadPoolExecutor +import contextlib +import io +import json +from pathlib import Path +import subprocess +import tempfile +import threading +import unittest +from unittest.mock import patch + +from emulation import teardown_race as race + + +class ConcurrentCloseTests(unittest.TestCase): + def run_ranks(self, accepted, close_codes, *, after_close=4, worker_code=5): + """Four ranks enqueue independently; a barrier publishes their actual accepted counts.""" + barrier = threading.Barrier(len(accepted), timeout=5) + counts = [None] * len(accepted) + + def rank_run(rank): + ready, closing, waiting = threading.Event(), threading.Event(), threading.Event() + posted, destroyed = 0, False + + def enqueue(): + nonlocal posted + if destroyed: + return after_close + if posted < accepted[rank]: + posted += 1 + if posted == accepted[rank]: + ready.set() + return 0 + waiting.set() + if not closing.wait(5): + raise TimeoutError("test close did not begin") + return worker_code + + def finalize(): + self.assertTrue(ready.wait(5)) + self.assertTrue(waiting.wait(5)) + closing.set() + return close_codes[0] + + def destroy(): + nonlocal destroyed + destroyed = True + return close_codes[1] + + def exchange(count): + counts[rank] = count + barrier.wait() + return list(counts) + + return race.close_concurrently(enqueue, finalize, destroy, lambda: None, exchange) + + with ThreadPoolExecutor(max_workers=len(accepted)) as pool: + return list(pool.map(rank_run, range(len(accepted)))) + + def test_equal_counts_require_success_from_both_close_calls(self): + for codes in ((0, 0), (6, 0), (0, 6), (6, 6)): + with self.subTest(codes=codes): + entries = self.run_ranks([2, 2, 2, 2], codes) + self.assertTrue(all(entry["ok"] == (codes == (0, 0)) for entry in entries)) + + def test_unequal_counts_allow_only_success_or_remote_error(self): + for codes in ((0, 0), (6, 0), (0, 6), (6, 6), (2, 6), (6, 5)): + with self.subTest(codes=codes): + entries = self.run_ranks([3, 19, 2, 4], codes) + self.assertTrue(all(entry["accepted_counts"] == [3, 19, 2, 4] for entry in entries)) + self.assertTrue(all(entry["ok"] == all(code in (0, 6) for code in codes) + for entry in entries)) + + def test_post_close_calls_must_be_refused(self): + for result in (0, 4, 5, 6): + with self.subTest(result=result): + entries = self.run_ranks([2, 2, 2, 2], (0, 0), after_close=result) + self.assertTrue(all(entry["ok"] == (result in (4, 5)) for entry in entries)) + + def test_worker_error_is_not_excused_by_unequal_counts(self): + entries = self.run_ranks([3, 19, 2, 4], (6, 6), worker_code=6) + self.assertTrue(all(not entry["ok"] for entry in entries)) + + def test_first_call_must_succeed_before_close_starts(self): + entry = race.close_concurrently(lambda: 5, lambda: 0, lambda: 0, lambda: None, lambda _: [0, 0]) + self.assertFalse(entry["ok"]) + + def test_worker_exception_fails_even_if_close_succeeds(self): + def prepare(): + raise RuntimeError("thread setup failed") + + entry = race.close_concurrently(lambda: 4, lambda: 0, lambda: 0, prepare, lambda _: [0, 0]) + self.assertFalse(entry["ok"]) + self.assertEqual(entry["worker_errors"], ["RuntimeError: thread setup failed"]) + + def test_stuck_worker_is_a_failure(self): + release = threading.Event() + exited = threading.Event() + calls = 0 + + def enqueue(): + nonlocal calls + calls += 1 + if calls == 1: + return 0 + release.wait(5) + exited.set() + return 5 + + try: + with self.assertRaisesRegex(TimeoutError, "did not exit"): + race.close_concurrently(enqueue, lambda: 0, lambda: 0, lambda: None, + lambda _: self.fail("must not exchange unstable counts"), timeout=0.01) + finally: + release.set() + self.assertTrue(exited.wait(5)) + + def test_failed_process_cannot_pass_with_successful_json(self): + class Process: + pid = 99999999 + + def __init__(self, timeout): + self.timeout = timeout + self.killed = False + + def wait(self, timeout=None): + if self.timeout and not self.killed: + raise subprocess.TimeoutExpired("rank", timeout) + return -9 if self.killed else 1 + + def kill(self): + self.killed = True + + for timeout in (False, True): + with self.subTest(timeout=timeout), tempfile.TemporaryDirectory() as work: + Path(work, "rank0.json").write_text(json.dumps({ + "rounds": [{"round": 0, "ok": True}], + "parent": {"exact": True, "async_error": 0}, "parent_destroy": 0, + })) + process = Process(timeout) + with patch.object(race.subprocess, "Popen", return_value=process), \ + contextlib.redirect_stdout(io.StringIO()) as output: + code = race.main(["--library", "unused.so", "--world", "1", "--rounds", "1", + "--expect", "concurrent-close", "--work", work]) + self.assertEqual(code, 1) + self.assertIn("rank 0: exit", output.getvalue()) + self.assertEqual(process.killed, timeout) + + +if __name__ == "__main__": + unittest.main(verbosity=2) From 4349361951d1725245a0ad5abd78a2d890913594 Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Thu, 15 Oct 2026 16:23:39 -0500 Subject: [PATCH 63/65] Isolate the ABI preload probe from inherited NCCL providers Give the SONAME probe a controlled LD_PRELOAD containing identifiable ASan runtimes followed by the library under test. Skip with a precise reason when LIBSIRCL_TEST_ASAN requires a runtime that cannot be identified. Inherited NCCL and CUDA preloads no longer shadow the tested symbols. Exercise inherited conflicting libnccl.so.2 entries with both colon and space delimiters. Retain all 146 symbol checks, the version and alias checks, and linked versus dlopen SONAME reuse. Validation: a stand-in NCCL preload fails the baseline probe and passes the fixed probe, both plain and with ASan. make check, make check-asan, Ruff and repository links pass. The serving image rerun remains pending. No library implementation changes. --- spark_transport/libsircl/tests/test_abi.py | 24 ++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/spark_transport/libsircl/tests/test_abi.py b/spark_transport/libsircl/tests/test_abi.py index 963d4a13..c9ad1578 100644 --- a/spark_transport/libsircl/tests/test_abi.py +++ b/spark_transport/libsircl/tests/test_abi.py @@ -11,6 +11,7 @@ import sys import tempfile import unittest +from unittest.mock import patch ROOT = Path(__file__).resolve().parents[1] MANIFEST = json.loads((ROOT / "tests/api_manifest.json").read_text()) @@ -21,6 +22,20 @@ def run(*arguments: str, env: dict | None = None) -> str: return subprocess.check_output(arguments, text=True, stderr=subprocess.STDOUT, env=env) +def preload_environment() -> dict: + """Isolate the SONAME probe from inherited providers, keeping ASan first when requested.""" + env = os.environ.copy() + env.pop("LIBSIRCL_NCCL_API_VERSION", None) + # Keep the runtime selected by check-asan, including when CC names a different compiler. + preloads = [entry for entry in re.split(r"[:\s]+", env.get("LD_PRELOAD", "")) + if re.fullmatch(r"libasan\.so(?:\.\d+)*|libclang_rt\.asan.*\.so", Path(entry).name)] + if env.get("LIBSIRCL_TEST_ASAN") and not preloads: + raise unittest.SkipTest("LIBSIRCL_TEST_ASAN requires an identifiable ASan runtime in LD_PRELOAD " + "to isolate the SONAME probe from inherited NCCL providers") + env["LD_PRELOAD"] = ":".join([*preloads, str(LIBRARY)]) + return env + + # Host entry points of NCCL's device API (nccl_device/core.h and its barrier and LL all-to-all headers), # which programs built against NCCL 2.28 or later import; src/device_api.c defines them. DEVICE_API = {"ncclCommQueryProperties", "ncclDevCommCreate", "ncclDevCommDestroy", "ncclGetLsaMultimemDevicePointer", @@ -141,10 +156,11 @@ def test_preload_and_soname_reuse(self): (path / "libnccl.so").symlink_to("libnccl.so.2") (path / "preload.c").write_text(source) run("cc", "-std=c11", "-Wall", "-Wextra", "-Werror", str(path / "preload.c"), "-L", str(path), "-Wl,-rpath," + str(path), "-lnccl", "-ldl", "-o", str(path / "preload")) - env = os.environ.copy() - env.pop("LIBSIRCL_NCCL_API_VERSION", None) - env["LD_PRELOAD"] = ":".join(filter(None, (env.get("LD_PRELOAD"), str(LIBRARY)))) - self.assertIn("all 146 symbols", run(str(path / "preload"), env=env)) + inherited = os.environ.get("LD_PRELOAD", "") + for preload in (inherited, f"{path / 'libnccl.so.2'}:{inherited}", + f"{path / 'libnccl.so.2'} {inherited}"): + with self.subTest(preload=preload), patch.dict(os.environ, LD_PRELOAD=preload): + self.assertIn("all 146 symbols", run(str(path / "preload"), env=preload_environment())) if __name__ == "__main__": From 8258bfaf9850df6f997560ad1301b4b372f0d32c Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Thu, 15 Oct 2026 17:42:15 -0500 Subject: [PATCH 64/65] Isolate CPU context-selection probes from inherited CUDA runtimes Preloaded cudaGetDevice providers take precedence over the fake driver's later RTLD_GLOBAL load. A failing provider makes the two-device probe refuse context selection; a successful one can select the wrong device. Remove inherited preloads only for the controlled context probes, keeping ASan runtimes first. Cover successful and failing competing runtimes with colon and space preload lists across all three device-selection cases. Library behavior, check ordering and public compatibility are unchanged. Validation: baseline reproduces the reported diagnostic with a failing runtime preload. The regression fails before isolation and passes after. Local WSL make check passes with that preload retained and cleared; make check-asan, the regression with ASan plus the conflicting preload, ruff check ., Markdown links and optional site-route checks pass. The serving image was not run; no DGX Spark was accessed. --- .../libsircl/tests/test_engine_cpu.py | 44 +++++++++++++++++-- 1 file changed, 41 insertions(+), 3 deletions(-) diff --git a/spark_transport/libsircl/tests/test_engine_cpu.py b/spark_transport/libsircl/tests/test_engine_cpu.py index af9a1460..acdb6c86 100644 --- a/spark_transport/libsircl/tests/test_engine_cpu.py +++ b/spark_transport/libsircl/tests/test_engine_cpu.py @@ -10,11 +10,13 @@ import argparse import json import os +import re import subprocess import sys import tempfile import unittest from pathlib import Path +from unittest.mock import patch LIBRARY = Path(__file__).resolve().parents[1] / "build" / "libsircl.so" FAKE_DRIVER = LIBRARY.parent / "fake-cuda" @@ -33,11 +35,20 @@ class U(C.Structure): _fields_=[("b",C.c_ubyte*128)] ''' -def run(script, env=None): +def run(script, env=None, *, isolate_preloads=False): environment = dict(os.environ) environment.update(env or {}) if os.environ.get("LIBSIRCL_TEST_ASAN"): environment["LD_PRELOAD"] = os.environ["LD_PRELOAD"] + if isolate_preloads: + # A runtime preloaded at startup wins RTLD_DEFAULT lookup over the fake driver's later dlopen. + # Preserve ASan first, but remove inherited CUDA/NCCL providers from these controlled probes. + preloads = [entry for entry in re.split(r"[:\s]+", environment.get("LD_PRELOAD", "")) + if re.fullmatch(r"libasan\.so(?:\.\d+)*|libclang_rt\.asan.*\.so", Path(entry).name)] + if environment.get("LIBSIRCL_TEST_ASAN") and not preloads: + raise unittest.SkipTest("LIBSIRCL_TEST_ASAN requires an identifiable ASan runtime in LD_PRELOAD " + "to isolate the CUDA context probe") + environment["LD_PRELOAD"] = ":".join(preloads) return subprocess.run([sys.executable, "-c", PRELUDE + script, str(LIBRARY)], env=environment, capture_output=True, text=True, timeout=60) @@ -415,7 +426,8 @@ def creation(self, env, *, runtime=False): """One ncclCommInitRank of a one-rank communicator on the stand-in driver, on a thread without a current context; LIBSIRCL_TRANSPORT=bogus makes the engine refuse right after the context step, before any device work. ``runtime`` brings the stand-in's cudaGetDevice into the global scope, as a framework's CUDA runtime - is. Returns (result, ncclGetLastError, (retains, retained device, cuCtxSetCurrent calls), stderr).""" + is. Inherited preloads are removed except ASan, so only this stand-in can supply cudaGetDevice. + Returns (result, ncclGetLastError, (retains, retained device, cuCtxSetCurrent calls), stderr).""" load = (f"g=C.CDLL({str(FAKE_DRIVER / 'libcuda.so.1')!r}, mode=C.RTLD_GLOBAL)\n" if runtime else f"g=C.CDLL({str(FAKE_DRIVER / 'libcuda.so.1')!r})\n") environment = {"LIBSIRCL_BOOTSTRAP_ONLY": "", "LIBSIRCL_TRANSPORT": "bogus", "FAKE_CUDA_NO_CONTEXT": "1", @@ -425,7 +437,7 @@ def creation(self, env, *, runtime=False): c=C.c_void_p(); r=l.ncclCommInitRank(C.byref(c),1,u,0) n=[C.c_int(),C.c_int(),C.c_int()]; g.sccl_fake_cuda_contexts(*[C.byref(v) for v in n]) print(json.dumps({"result":r,"message":l.ncclGetLastError(None).decode(),"contexts":[v.value for v in n]}))''', - environment) + environment, isolate_preloads=True) self.assertEqual(out.returncode, 0, out.stderr) data = json.loads(out.stdout) return data["result"], data["message"], tuple(data["contexts"]), out.stderr @@ -447,6 +459,7 @@ def test_init_without_cuda_context_uses_the_runtimes_current_device(self): runtime=True) self.assertIn("LIBSIRCL_TRANSPORT=bogus", message) self.assertEqual(contexts, (1, 1, 1)) + self.assertNotEqual(result, 0) def test_init_without_cuda_context_is_refused_when_the_device_is_unknown(self): # Two devices and no CUDA runtime in the process: no device can be chosen, so the creation is refused, @@ -457,6 +470,31 @@ def test_init_without_cuda_context_is_refused_when_the_device_is_unknown(self): self.assertEqual(contexts, (0, -1, 0)) self.assertIn("libsircl: communicator creation failed (invalid usage): no current CUDA context", stderr) + def test_context_selection_ignores_inherited_cuda_runtime(self): + """A preloaded runtime must not replace the creation probe's controlled runtime or its absence.""" + source = '''#include +int cudaGetDevice(int *device) { + *device = 0; + return atoi(getenv("SHADOW_CUDA_RESULT")); +} +''' + with tempfile.TemporaryDirectory(prefix="sircl-cuda-preload-") as directory: + path = Path(directory) + (path / "runtime.c").write_text(source) + subprocess.run(["cc", "-shared", "-fPIC", "-Wall", "-Wextra", "-Werror", + str(path / "runtime.c"), "-o", str(path / "runtime.so")], check=True, + capture_output=True, text=True, timeout=60) + inherited = os.environ.get("LD_PRELOAD", "") + for separator in (":", " "): + for status in ("0", "100"): + # Keep an inherited ASan runtime first, as required by check-asan. + preload = separator.join(filter(None, (inherited, str(path / "runtime.so")))) + with self.subTest(separator=separator, runtime_status=status), patch.dict( + os.environ, LD_PRELOAD=preload, SHADOW_CUDA_RESULT=status): + self.test_init_without_cuda_context_uses_the_only_devices_primary_context() + self.test_init_without_cuda_context_uses_the_runtimes_current_device() + self.test_init_without_cuda_context_is_refused_when_the_device_is_unknown() + def test_init_without_a_cuda_driver_is_refused_with_reason(self): out = run('''u=U(); assert l.ncclGetUniqueId(C.byref(u))==0 c=C.c_void_p(); r=l.ncclCommInitRank(C.byref(c),1,u,0) From 598a9c2279951f4ff1d12bb81c37601ceeae3bd3 Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Sat, 31 Oct 2026 18:18:06 -0500 Subject: [PATCH 65/65] Destroy a quarantined engine's lock before it joins the quarantine list sccl_engine_destroy put an engine whose kernel might still be running on the quarantine list and then called pthread_mutex_destroy on the engine's lock. Once the engine is on the list, another thread's quarantine_reclaim (run by any engine creation or destroy) may find its events complete and free it. The mutex destroy then wrote to freed memory. The lock is now destroyed before quarantine_add, and nothing touches the engine after a successful add. Compatibility: no behaviour change apart from the removed race. Validation: make check, check_exports.py and make check-asan pass on a CPU build (nvcc 13.3 from pip, as in CI). The race needs two threads and a GPU kernel that outlives an abort, so no CPU test exercises it; nothing ran on hardware. Co-Authored-By: Claude Opus 5.5 --- spark_transport/libsircl/src/engine.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/spark_transport/libsircl/src/engine.c b/spark_transport/libsircl/src/engine.c index 84e6c5a4..8a77c39f 100644 --- a/spark_transport/libsircl/src/engine.c +++ b/spark_transport/libsircl/src/engine.c @@ -4970,12 +4970,14 @@ ncclResult_t sccl_engine_destroy(sccl_engine *e, int abort, char *err, size_t le e->host = e->p2p_host = NULL; freed = 1; } + /* The lock goes before quarantine_add: once the engine is on the quarantine list, another thread's + * quarantine_reclaim may free it, so nothing here may touch it after a successful add. */ + pthread_mutex_destroy(&e->lock); int queued = !freed && pushed && released && quarantine_add(e); if (pushed) { sccl_CUcontext popped; cu->CtxPopCurrent(&popped); } - pthread_mutex_destroy(&e->lock); if (freed && !queued) free(e); return result; }