Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
67 commits
Select commit Hold shift + click to select a range
b5783ce
Reject malformed split records before indexing them
FujitsuPolycom Oct 15, 2026
b00bd14
Cancel bootstrap work before abort waits for references
FujitsuPolycom Oct 15, 2026
eefb074
Reject peer positions outside the route table
FujitsuPolycom Oct 15, 2026
bb5decb
Refuse InitAll device selections that cannot be honored
FujitsuPolycom Oct 15, 2026
63764c7
Close engine submission before waiting for context work
FujitsuPolycom Oct 15, 2026
67e5042
Add a sanitizer check target and run it with the exports check in CI
FujitsuPolycom Oct 15, 2026
78723e3
Wait for calls in progress before the unload frees communicators
FujitsuPolycom Oct 15, 2026
0968a92
Keep every split rank in the unique-id round when the leader fails
FujitsuPolycom Oct 15, 2026
5085c57
Run the teardown rounds when destroying a revoked communicator
FujitsuPolycom Oct 15, 2026
f83438d
Refuse lifecycle calls on an initializing communicator
FujitsuPolycom Oct 15, 2026
8d7c2f2
Document that ncclCommFinalize is synchronous for non-blocking commun…
FujitsuPolycom Oct 15, 2026
ae77992
Poison the point-to-point channels and notify peers on abort
FujitsuPolycom Oct 15, 2026
39f31d9
Post point-to-point abort notices before fail-stop ends the process
FujitsuPolycom Oct 15, 2026
30520a4
Keep large reductions within the kernels' int32 pack counts
FujitsuPolycom Oct 15, 2026
65f6139
Require a research opt-in for the one-pass ring relay and agree it at…
FujitsuPolycom Oct 15, 2026
f0113c3
Confirm that vLLM resolves VLLM_NCCL_SO_PATH to the libsircl library
FujitsuPolycom Oct 15, 2026
d126613
Close a failed communicator's bootstrap connection at once
FujitsuPolycom Oct 15, 2026
b01ab91
Abort and clear every handle of a failed ncclCommInitAll
FujitsuPolycom Oct 15, 2026
37083b8
Release a thread's group state when the thread exits
FujitsuPolycom Oct 15, 2026
ef656f6
Free a destroyed communicator's host bookkeeping when its context is …
FujitsuPolycom Oct 15, 2026
8266a30
Keep failed calls that enqueue nothing out of a group's result
FujitsuPolycom Oct 15, 2026
8d6bdd5
Zero the handle and value outputs of unsupported functions
FujitsuPolycom Oct 15, 2026
5192113
Serialize bootstrap rounds and close claims on one communicator
FujitsuPolycom Oct 15, 2026
4d91d9f
Carry relay status separately from the forward windows
FujitsuPolycom Oct 15, 2026
1799fda
Keep route windows within native caps and relay queues within their s…
FujitsuPolycom Oct 15, 2026
842f3d9
Release a context's kernel-pack entry when its last engine is freed
FujitsuPolycom Oct 15, 2026
baadd42
Create bootstrap descriptors close-on-exec atomically
FujitsuPolycom Oct 15, 2026
cdb3676
Document the bootstrap root limit and name it in its refusal
FujitsuPolycom Oct 15, 2026
7b77bd8
Parse NCCL_SOCKET_IFNAME exclusions fully and keep them off loopback
FujitsuPolycom Oct 15, 2026
8f271a8
Return success for NULL in ncclCommDestroy and ncclCommAbort
FujitsuPolycom Oct 15, 2026
07951c8
Publish an initialization's result before its state
FujitsuPolycom Oct 15, 2026
3536545
Finish every local setup step before the last all-rank verdict
FujitsuPolycom Oct 15, 2026
7ea9207
Agree fail-stop and the starting wait regime across ranks at setup
FujitsuPolycom Oct 15, 2026
f19f457
Bound fail-stop's exit with a backstop thread
FujitsuPolycom Oct 15, 2026
b53acb4
Write receipts and link dumps without following links or sharing a pa…
FujitsuPolycom Oct 15, 2026
044a56b
Validate the setup timeout and keep idle connections off the root
FujitsuPolycom Oct 15, 2026
c7b3767
Give the teardown rounds 5 s of slack beyond the wait limit
FujitsuPolycom Oct 15, 2026
7343c80
Parse engine settings strictly and name the variable in every refusal
FujitsuPolycom Oct 15, 2026
3776bf2
Make the vLLM plugin's binding check and file identity strict
FujitsuPolycom Oct 15, 2026
644a561
Report a split's failed exchange with the engine's result codes
FujitsuPolycom Oct 15, 2026
13d3ce1
Free ncclMemFree memory in the context it belongs to
FujitsuPolycom Oct 15, 2026
8f954de
Check every kernel-pack lookup at communicator setup
FujitsuPolycom Oct 15, 2026
2b3ec30
Refuse point-to-point group-rule violations at the call that breaks them
FujitsuPolycom Oct 15, 2026
abe298d
Keep the Makefile's required flags under command-line CFLAGS and CPPF…
FujitsuPolycom Oct 15, 2026
e152b5d
Harden both builds, align CMake with the Makefile, and build CMake in CI
FujitsuPolycom Oct 15, 2026
d7589c3
Add a table-driven hostile-argument case to the library emulation rank
FujitsuPolycom Oct 15, 2026
1f83ca7
Describe the point-to-point scope, default schedules and four packs i…
FujitsuPolycom Oct 15, 2026
5734519
Give a failed ncclCommFinalize its own text without an overlapping copy
FujitsuPolycom Oct 15, 2026
b67cac1
Check for a forked process first in ncclCommRevoke
FujitsuPolycom Oct 15, 2026
7815d54
Give the calling thread a failed communicator's text on queries
FujitsuPolycom Oct 15, 2026
f2ad11d
Poll the engine's asynchronous error in ncclGetLastError(comm)
FujitsuPolycom Oct 15, 2026
93e0ddf
Name the call that failed a group, and report failed deferred creations
FujitsuPolycom Oct 15, 2026
fa58c87
Correct the large-reduction plan comment about ncclReduce
FujitsuPolycom Oct 15, 2026
8ff37d2
Name the packs' architectures when a pack does not load
FujitsuPolycom Oct 15, 2026
c21fe1b
Document the threading, stream and graph contract, and log ignored co…
FujitsuPolycom Oct 15, 2026
9effe19
Document how libsircl's error strings and NULL handling differ from N…
FujitsuPolycom Oct 15, 2026
562317b
Carry LIBSIRCL_RELAYED_LANES through the installer's libsircl lane
FujitsuPolycom Oct 15, 2026
5fe15fb
State the libsircl lane's wait regime and dead-peer detection time
FujitsuPolycom Oct 15, 2026
a5e0a50
Describe SOURCE_SNAPSHOT.json and the rdma-core notice as they are
FujitsuPolycom Oct 15, 2026
b29b4d1
Bring libsircl's status documents in line with the code and the records
FujitsuPolycom Oct 15, 2026
8168a0d
Reclaim engines quarantined by an abort once their kernels finish
FujitsuPolycom Oct 15, 2026
a1bebb8
Judge concurrent close against accepted collective counts
FujitsuPolycom Oct 15, 2026
4349361
Isolate the ABI preload probe from inherited NCCL providers
FujitsuPolycom Oct 15, 2026
8258bfa
Isolate CPU context-selection probes from inherited CUDA runtimes
FujitsuPolycom Oct 15, 2026
598a9c2
Destroy a quarantined engine's lock before it joins the quarantine list
FujitsuPolycom Oct 31, 2026
22378d2
Merge remote-tracking branch 'origin/main' into pr382-update
FujitsuPolycom Nov 2, 2026
dbffb93
Merge remote-tracking branch 'origin/main' into pr382-update
Nov 2, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,18 @@ jobs:
make -j BUILD=build NVCC="$NVCC"
make check BUILD=build NVCC="$NVCC"
sha256sum build/packs/*.fatbin
python tools/check_exports.py --include include --library build/libsircl.so
make check-asan BUILD=build NVCC="$NVCC"
# The CMake build of the same sources and its tests (the same suites as make check).
- name: Build libsircl with CMake and run ctest
working-directory: spark_transport/libsircl
env:
SIRCL_PACKAGE: ${{ github.workspace }}/spark_transport/sircl
run: |
cmake -S . -B build-cmake -DLIBSIRCL_NVCC="$LIBSIRCL_NVCC" \
-DLIBSIRCL_NVCC_EXTRA_FLAGS=-DCCCL_DISABLE_CTK_COMPATIBILITY_CHECK
cmake --build build-cmake -j
ctest --test-dir build-cmake --output-on-failure

lil-bridge:
name: pinned LIL bridge
Expand Down
9 changes: 8 additions & 1 deletion docs/architecture/libsircl.md
Original file line number Diff line number Diff line change
Expand Up @@ -225,6 +225,13 @@ refuses a library without the mode. The layer records `fail_stop` in the lock
when the library's bytes name `LIBSIRCL_FAIL_STOP`; the stock-image preflight
reads the same mark from the host library.

The lane sets no wait variable, so every communicator runs in the startup
wait regime with libsircl's default limit of 600 s (`SIRCL_STARTUP_WAIT_S`)
for its whole life: nothing switches it to the serving regime
(`sirclSetWaitRegime`). A rank whose peer died learns of it only when one of
its waits reaches that limit, 600 s after the wait began; fail-stop then ends
the process.

### Container settings

Each rank's container gets the profile's settings with these changes
Expand All @@ -237,7 +244,7 @@ Each rank's container gets the profile's settings with these changes
| `LIBSIRCL_FAIL_STOP` | `1` |
| `LIBSIRCL_NCCL_API_VERSION` | `22705`, libsircl's default: it passes vLLM's and torch's feature gates without opening newer ones |
| `LIBSIRCL_TRANSPORT` | `verbs` |
| `LIBSIRCL_POSITION`, `SIRCL_PEER_ROUTES`, `LIBSIRCL_CHAIN_ORDER`, `LIBSIRCL_FORWARD_WINDOWS`, `SIRCL_FORWARD_CHUNK_BYTES`, `LIBSIRCL_RING_WINDOW`, `LIBSIRCL_P2P_WINDOWS`, `SIRCL_P2P_CHUNK_BYTES` | the rank's routing settings, recorded per rank in the deployment lock |
| `LIBSIRCL_POSITION`, `SIRCL_PEER_ROUTES`, `LIBSIRCL_CHAIN_ORDER`, `LIBSIRCL_RELAYED_LANES`, `LIBSIRCL_FORWARD_WINDOWS`, `SIRCL_FORWARD_CHUNK_BYTES`, `LIBSIRCL_RING_WINDOW`, `LIBSIRCL_P2P_WINDOWS`, `SIRCL_P2P_CHUNK_BYTES` | the rank's routing settings, recorded per rank in the deployment lock |
| `SIRCL_BOOTSTRAP_ADDR` | the rank's `VLLM_HOST_IP` |
| `SIRCL_GID_INDEX` | the profile's `NCCL_IB_GID_INDEX`, else 3 |
| `LIBSIRCL_RECEIPT` | `/run/sparkring/sircl/receipts/libsircl`: one receipt per communicator in the deployment's receipt directory on each Spark |
Expand Down
66 changes: 55 additions & 11 deletions integrations/vllm/libsircl/sparkring_libsircl.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,11 @@
function PyNccl's ``NCCLLibrary`` binds. PyNccl disables itself, and vLLM
falls back to torch's own NCCL, when that load or binding fails; this
check makes such a fallback fail the process instead;
3. sets ``VLLM_NCCL_SO_PATH`` to the file; processes that vLLM starts later
inherit the setting.
3. sets ``VLLM_NCCL_SO_PATH`` to the file, and requires vLLM's own reading
of it (``vllm.envs.VLLM_NCCL_SO_PATH``) to name the same file, so a vLLM
whose environment values were read before the plugin ran fails instead of
loading the earlier library; processes that vLLM starts later inherit the
setting.

Any other state fails the process, so a deployment that asked for libsircl
never runs on another NCCL. The plugin changes nothing else: torch's
Expand All @@ -26,6 +29,7 @@
"""
import ctypes
import hashlib
import importlib.util
import json
import os
from pathlib import Path
Expand All @@ -36,6 +40,7 @@
NAME = "libsircl"
ENTRY_POINT = ("vllm.general_plugins", NAME, "sparkring_libsircl:register")
_checked = {}
_identity = {}
# How PyNccl loads its library (vllm/distributed/device_communicators/pynccl_wrapper.py).
LOADER = ctypes.CDLL

Expand All @@ -53,22 +58,43 @@ def selected(environ=None):
library = Path(path)
if not library.is_absolute() or library.is_symlink() or not library.is_file():
raise LibsirclSelectionError(f"{LIBRARY}={path} is not an absolute path to a regular file")
stat = library.stat()
key = (path, digest, stat.st_ino, stat.st_size, stat.st_mtime_ns)
if key not in _checked:
with library.open("rb") as stream:
if os.path.realpath(path) != path:
raise LibsirclSelectionError(f"{LIBRARY}={path} is not its own real path ({os.path.realpath(path)}): a "
"directory of it is a symbolic link, or the path is not normalized")
# The file hashed is the open file, and its device and inode identify it for identical().
with library.open("rb") as stream:
stat = os.fstat(stream.fileno())
key = (path, digest, stat.st_dev, stat.st_ino, stat.st_size, stat.st_mtime_ns)
if key not in _checked:
found = hashlib.file_digest(stream, "sha256").hexdigest()
if found != digest:
raise LibsirclSelectionError(f"{path} has SHA-256 {found}, not {DIGEST}={digest}")
_checked[key] = True
if found != digest:
raise LibsirclSelectionError(f"{path} has SHA-256 {found}, not {DIGEST}={digest}")
_checked[key] = (stat.st_dev, stat.st_ino)
_identity[path] = _checked[key]
return path


def identical(path):
"""Require ``path`` to still name the file selected() hashed (same device and inode)."""
stat = os.stat(path)
if (stat.st_dev, stat.st_ino) != _identity.get(path):
raise LibsirclSelectionError(f"{path} was replaced after its SHA-256 was checked")


def pynccl_functions():
"""The functions vLLM's PyNccl binds (``NCCLLibrary.exported_functions``), or None outside vLLM."""
"""The functions vLLM's PyNccl binds (``NCCLLibrary.exported_functions``), or None outside vLLM. Inside a
process where vLLM is importable but its PyNccl wrapper is not, the functions cannot be checked:
LibsirclSelectionError, not a check that passes with nothing to check."""
try:
from vllm.distributed.device_communicators.pynccl_wrapper import NCCLLibrary
except ImportError:
except ImportError as error:
try:
importable = importlib.util.find_spec("vllm") is not None
except (ImportError, ValueError):
importable = "vllm" in __import__("sys").modules
if importable:
raise LibsirclSelectionError(f"vLLM is importable but its PyNccl wrapper is not ({error}), so the "
"functions PyNccl binds cannot be checked") from None
return None
return [function.name for function in NCCLLibrary.exported_functions]

Expand All @@ -91,8 +117,26 @@ def check_binding(path, *, load=None, functions=None):
return info


def resolved_target():
"""vLLM's own value of ``VLLM_NCCL_SO_PATH`` (``vllm.envs``), or None outside vLLM."""
try:
import vllm.envs as envs
except ImportError:
return None
try:
return getattr(envs, TARGET)
except AttributeError:
raise LibsirclSelectionError(f"vllm.envs has no {TARGET}, so this vLLM may not load the library it "
"names") from None


def register():
"""Point vLLM's PyNccl at the checked libsircl library; vLLM may call this more than once."""
path = selected()
check_binding(path)
identical(path)
os.environ[TARGET] = path
found = resolved_target()
if found is not None and found != path:
raise LibsirclSelectionError(f"vLLM resolves {TARGET} to {found!r}, not {path}: it read its environment "
"before the plugin set the variable, so PyNccl would load that library")
92 changes: 89 additions & 3 deletions integrations/vllm/libsircl/test_sparkring_libsircl.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
import hashlib
import importlib.util
import os
import sys
import types
from pathlib import Path

import pytest
Expand Down Expand Up @@ -82,12 +84,52 @@ def load(path):
plugin.check_binding("/opt/sparkring/libsircl/lib/libsircl.so.0.6.0", load=load, functions=lambda: functions)


def test_outside_vllm_only_the_identity_is_checked():
assert plugin.pynccl_functions() is None or isinstance(plugin.pynccl_functions(), list)
info = plugin.check_binding("/lib", load=lambda path: Library(functions=()), functions=lambda: None)
WRAPPER = "vllm.distributed.device_communicators.pynccl_wrapper"


def test_outside_vllm_only_the_identity_is_checked(monkeypatch):
monkeypatch.delitem(sys.modules, "vllm", raising=False)
monkeypatch.setitem(sys.modules, WRAPPER, None)
monkeypatch.setattr(plugin.importlib.util, "find_spec", lambda name: None)
assert plugin.pynccl_functions() is None
info = plugin.check_binding("/lib", load=lambda path: Library(functions=()), functions=plugin.pynccl_functions)
assert info["library"] == "libsircl"


def test_an_importable_vllm_whose_pynccl_wrapper_fails_to_import_is_refused(monkeypatch):
# The functions PyNccl binds cannot be listed, so the binding check must not pass with nothing to check.
monkeypatch.setitem(sys.modules, "vllm", types.ModuleType("vllm"))
monkeypatch.setitem(sys.modules, WRAPPER, None)
with pytest.raises(plugin.LibsirclSelectionError, match="PyNccl wrapper is not"):
plugin.pynccl_functions()
with pytest.raises(plugin.LibsirclSelectionError, match="PyNccl wrapper is not"):
plugin.check_binding("/lib", load=lambda path: Library(), functions=plugin.pynccl_functions)


def test_a_library_replaced_after_its_check_is_refused(tmp_path):
path, digest = library(tmp_path)
assert plugin.selected({plugin.LIBRARY: str(path), plugin.DIGEST: digest}) == str(path)
plugin.identical(str(path))
other = tmp_path / "other.so"
other.write_bytes(b"ELF something else")
os.replace(other, path)
with pytest.raises(plugin.LibsirclSelectionError, match="replaced after its SHA-256 was checked"):
plugin.identical(str(path))


def test_a_path_through_a_linked_directory_is_not_the_library(tmp_path):
real = tmp_path / "real"
real.mkdir()
path, digest = library(real)
linked = tmp_path / "linked"
try:
linked.symlink_to(real, target_is_directory=True)
except OSError:
pytest.skip("this host cannot create symbolic links")
with pytest.raises(plugin.LibsirclSelectionError, match="not its own real path"):
plugin.selected({plugin.LIBRARY: str(linked / path.name), plugin.DIGEST: digest})


def test_a_symbolic_link_is_not_the_library(tmp_path):
path, digest = library(tmp_path)
link = tmp_path / "link.so"
Expand All @@ -104,3 +146,47 @@ def test_the_entry_point_names_the_register_function():
assert (group, name) == ("vllm.general_plugins", "libsircl")
module, _, function = value.partition(":")
assert module == SOURCE.stem and callable(getattr(plugin, function))


class FrozenEnvs(types.ModuleType):
"""A ``vllm.envs`` whose values were read before the plugin ran."""

def __init__(self, value):
super().__init__("vllm.envs")
self.VLLM_NCCL_SO_PATH = value


class LiveEnvs(types.ModuleType):
"""A ``vllm.envs`` that reads the environment at each access, as vLLM does without its cache."""

def __init__(self):
super().__init__("vllm.envs")

def __getattr__(self, name):
if name == plugin.TARGET:
return os.environ.get(name)
raise AttributeError(name)


@pytest.mark.parametrize("envs, message", [
(LiveEnvs(), None),
(FrozenEnvs("/opt/sparkring/toolchain/nccl/lib/libnccl.so.2"), "vLLM resolves VLLM_NCCL_SO_PATH to"),
(types.ModuleType("vllm.envs"), "vllm.envs has no VLLM_NCCL_SO_PATH"),
])
def test_register_confirms_vllm_resolves_the_library(tmp_path, monkeypatch, envs, message):
path, digest = library(tmp_path)
monkeypatch.setenv(plugin.LIBRARY, str(path))
monkeypatch.setenv(plugin.DIGEST, digest)
monkeypatch.setenv(plugin.TARGET, "/opt/sparkring/toolchain/nccl/lib/libnccl.so.2")
monkeypatch.setattr(plugin, "LOADER", lambda name: Library())
monkeypatch.setattr(plugin, "pynccl_functions", lambda: ["ncclAllReduce"])
package = types.ModuleType("vllm")
package.envs = envs
monkeypatch.setitem(sys.modules, "vllm", package)
monkeypatch.setitem(sys.modules, "vllm.envs", envs)
if message is None:
plugin.register()
assert plugin.resolved_target() == str(path)
else:
with pytest.raises(plugin.LibsirclSelectionError, match=message):
plugin.register()
5 changes: 3 additions & 2 deletions runtime/common/libsircl.py
Original file line number Diff line number Diff line change
Expand Up @@ -89,8 +89,9 @@
# The kernel packs every libsircl build carries (runtime/images/libsircl_layer.py builds each from its CUDA
# source); a layer receipt that records another set describes another libsircl build.
KERNEL_PACKS = ("sircl_kernels", "sircl_fold", "sircl_links", "sircl_p2p")
ROUTE_VARIABLES = ("LIBSIRCL_POSITION", "SIRCL_PEER_ROUTES", "LIBSIRCL_CHAIN_ORDER", "LIBSIRCL_FORWARD_WINDOWS",
"SIRCL_FORWARD_CHUNK_BYTES", "LIBSIRCL_RING_WINDOW", "LIBSIRCL_P2P_WINDOWS", "SIRCL_P2P_CHUNK_BYTES")
ROUTE_VARIABLES = ("LIBSIRCL_POSITION", "SIRCL_PEER_ROUTES", "LIBSIRCL_CHAIN_ORDER", "LIBSIRCL_RELAYED_LANES",
"LIBSIRCL_FORWARD_WINDOWS", "SIRCL_FORWARD_CHUNK_BYTES", "LIBSIRCL_RING_WINDOW", "LIBSIRCL_P2P_WINDOWS",
"SIRCL_P2P_CHUNK_BYTES")
LIBRARY_VARIABLE = "SPARKRING_LIBSIRCL_LIBRARY"
DIGEST_VARIABLE = "SPARKRING_LIBSIRCL_SHA256"
# Every communicator's receipt, <prefix>.rank<r>.<pid>.c<n>.json, in the deployment's receipt directory.
Expand Down
20 changes: 20 additions & 0 deletions runtime/common/test_libsircl.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import json
import os
from pathlib import Path
import re
import subprocess
import sys

Expand Down Expand Up @@ -251,6 +252,25 @@ def test_two_pipeline_stages_and_expert_parallelism_on_grouped_all_gathers_pass(
"allgather_reducescatter"]) == []


def test_the_lane_runs_in_the_startup_wait_regime_at_the_librarys_limit():
"""The lane sets no wait variable, so every communicator runs in the startup regime at libsircl's default
limit; the architecture document's fail-stop section states that regime and limit, so the document, the
adapter and the library change together."""
_, section = deployment(TP2, "pair", 2, [0, 1])
environment = libsircl.library_environment(section["libsircl"])
assert not {"LIBSIRCL_WAIT_REGIME", "SIRCL_STARTUP_WAIT_S", "SIRCL_SERVING_WAIT_S",
"LIBSIRCL_SETUP_TIMEOUT_MS"} & set(environment)
root = Path(__file__).resolve().parents[2]
engine = (root / "spark_transport/libsircl/src/engine.c").read_text(encoding="utf-8")
limit = re.search(r"#define DEFAULT_STARTUP_WAIT_S (\d+)\.0", engine).group(1)
text = (root / "docs/architecture/libsircl.md").read_text(encoding="utf-8")
fail_stop = text[text.index("### Fail-stop"):text.index("### Container settings")]
words = " ".join(fail_stop.split())
assert "startup wait regime" in words and f"limit of {limit} s" in words
plugin = (root / "integrations/vllm/libsircl/sparkring_libsircl.py").read_text(encoding="utf-8")
assert "sirclSetWaitRegime" not in plugin


def test_a_library_without_the_fail_stop_mode_is_refused():
value = image()
value["libsircl"]["fail_stop"] = False
Expand Down
Loading
Loading