From ed4603a9f8cb3b258e492d04f9e2843c3347c0cb Mon Sep 17 00:00:00 2001 From: Noppanat Wadlom Date: Fri, 11 Sep 2026 22:36:08 +0800 Subject: [PATCH 01/11] refactor: move WorkerType onto the adapter base The CPU/GPU distinction is not Docker-specific; the Kubernetes provider added later in this branch needs it too, and importing it from the Docker adapter would pull the Docker SDK in with it. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Noppanat Wadlom --- src/server/supervisor/adapters/base.py | 6 ++++++ src/server/supervisor/adapters/docker.py | 7 +------ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/server/supervisor/adapters/base.py b/src/server/supervisor/adapters/base.py index b5c8aeba..a9645e98 100644 --- a/src/server/supervisor/adapters/base.py +++ b/src/server/supervisor/adapters/base.py @@ -1,6 +1,7 @@ import os from abc import ABC, abstractmethod from dataclasses import dataclass +from enum import StrEnum from typing import NewType from pydantic import BaseModel, ConfigDict, SecretStr @@ -11,6 +12,11 @@ from .utils import env_to_secret_str, to_env_str +class WorkerType(StrEnum): + CPU = "cpu" + GPU = "gpu" + + class WorkerConfig(BaseModel): model_config = ConfigDict(frozen=True) diff --git a/src/server/supervisor/adapters/docker.py b/src/server/supervisor/adapters/docker.py index 18327930..9d621706 100644 --- a/src/server/supervisor/adapters/docker.py +++ b/src/server/supervisor/adapters/docker.py @@ -5,7 +5,6 @@ import re import threading from collections import Counter -from enum import StrEnum from typing import Any from docker import DockerClient @@ -29,6 +28,7 @@ WorkerConfig, WorkerFactory, WorkerTokenType, + WorkerType, ) from .utils import get_worker_image_name, to_env_str @@ -95,11 +95,6 @@ def _prepare_volume( ) -class WorkerType(StrEnum): - CPU = "cpu" - GPU = "gpu" - - class SSHConfig(BaseModel): default_image: str | None = env.SSH_DEFAULT_IMAGE """Default container image for SSH sessions""" From a4bccab5e2872c058b8ccd87ead1863528db09de Mon Sep 17 00:00:00 2001 From: Noppanat Wadlom Date: Tue, 8 Sep 2026 22:45:50 +0800 Subject: [PATCH 02/11] chore: add kubernetes client to the server runtime The relock also moves pyyaml from 6.0.2 to 6.0.3. Signed-off-by: Noppanat Wadlom --- pyproject.toml | 3 +- src/server/requirements.txt | 3 +- uv.lock | 122 +++++++++++++++++++++++++++++------- 3 files changed, 103 insertions(+), 25 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 30f3c4ac..73568226 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,7 @@ runtime-server = [ "flowmesh-hook", "grpcio>=1.76.0", "httpx>=0.28.1", + "kubernetes>=31.0.0", "lumid-hooks>=0.2.0", "protobuf>=5.29.6", "pydantic>=2.12.3", @@ -212,7 +213,7 @@ module = [ ignore_missing_imports = true [[tool.mypy.overrides]] -module = ["pynvml", "vllm_omni.*", "deepspeed.*"] +module = ["pynvml", "vllm_omni.*", "deepspeed.*", "kubernetes.*"] follow_untyped_imports = true [tool.codespell] diff --git a/src/server/requirements.txt b/src/server/requirements.txt index 28d2191b..c968afa3 100644 --- a/src/server/requirements.txt +++ b/src/server/requirements.txt @@ -6,11 +6,12 @@ docker==7.1.0 fastapi==0.136.1 grpcio==1.83.1 httpx==0.28.1 +kubernetes==36.0.3 lumid-hooks==0.2.0 protobuf==6.33.6 pydantic==2.12.3 python-multipart==0.0.32 -pyyaml==6.0.2 +pyyaml==6.0.3 redis==7.0.1 requests==2.33.1 uvicorn[standard]==0.32.1 diff --git a/uv.lock b/uv.lock index 0f7ab1a8..530d81d3 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 2 requires-python = ">=3.12" resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'darwin'", @@ -1499,6 +1499,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/55/e2/2537ebcff11c1ee1ff17d8d0b6f4db75873e3b0fb32c2d4a2ee31ecb310a/docstring_parser-0.17.0-py3-none-any.whl", hash = "sha256:cf2569abd23dce8099b300f9b4fa8191e9582dda731fd533daf54c4551658708", size = 36896, upload-time = "2025-07-21T07:35:00.684Z" }, ] +[[package]] +name = "durationpy" +version = "0.11" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b0/5d/5f8571bd5dedc80863191621ac4be001f3f3dd8315d2ec078705dab7dec1/durationpy-0.11.tar.gz", hash = "sha256:181898e1ae282e288f0a2291829656bf1b6b3aadf30a97993b85db4943642905", size = 3582, upload-time = "2026-08-26T13:56:00.991Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e6/c4/ebdf7837bc4ef6fd98cfb013c28855bb358467bf86c1af011bbc21e21df0/durationpy-0.11-py3-none-any.whl", hash = "sha256:a739fe2b8972c250ff72f8e2c488d18cf25f7b852f49ee76048775d5171df30c", size = 4133, upload-time = "2026-08-26T13:55:59.456Z" }, +] + [[package]] name = "einops" version = "0.8.1" @@ -1917,6 +1926,7 @@ ci = [ { name = "ipython" }, { name = "isort" }, { name = "jinja2" }, + { name = "kubernetes" }, { name = "lumid-hooks" }, { name = "matplotlib" }, { name = "mcp" }, @@ -2080,6 +2090,7 @@ runtime-server = [ { name = "flowmesh-hook" }, { name = "grpcio" }, { name = "httpx" }, + { name = "kubernetes" }, { name = "lumid-hooks" }, { name = "protobuf" }, { name = "pydantic" }, @@ -2225,6 +2236,7 @@ ci = [ { name = "ipython", specifier = ">=9.5.0" }, { name = "isort", specifier = ">=7.0.0" }, { name = "jinja2", specifier = ">=3.1.6" }, + { name = "kubernetes", specifier = ">=31.0.0" }, { name = "lumid-hooks", specifier = ">=0.2.0" }, { name = "matplotlib", specifier = ">=3.10.6" }, { name = "mcp", specifier = ">=1.28.1" }, @@ -2386,6 +2398,7 @@ runtime-server = [ { name = "flowmesh-hook", editable = "hook" }, { name = "grpcio", specifier = ">=1.76.0" }, { name = "httpx", specifier = ">=0.28.1" }, + { name = "kubernetes", specifier = ">=31.0.0" }, { name = "lumid-hooks", specifier = ">=0.2.0" }, { name = "protobuf", specifier = ">=5.29.6" }, { name = "pydantic", specifier = ">=2.12.3" }, @@ -3632,6 +3645,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/80/be/3578e8afd18c88cdf9cb4cffde75a96d2be38c5a903f1ed0ceec061bd09e/kiwisolver-1.4.9-cp314-cp314t-win_arm64.whl", hash = "sha256:4a48a2ce79d65d363597ef7b567ce3d14d68783d2b2263d98db3d9477805ba32", size = 70260, upload-time = "2025-08-10T21:27:36.606Z" }, ] +[[package]] +name = "kubernetes" +version = "36.0.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohttp" }, + { name = "certifi" }, + { name = "durationpy" }, + { name = "python-dateutil" }, + { name = "pyyaml" }, + { name = "requests" }, + { name = "requests-oauthlib" }, + { name = "six" }, + { name = "urllib3" }, + { name = "websocket-client" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ca/57/b07b96353f902aa1bdbe00e878e3a12a137977d03a962479785576aa8ec9/kubernetes-36.0.3.tar.gz", hash = "sha256:36993ed25ce59b789c9341473a228fcf268504a2fec7c2b2b1531d73072e5ce7", size = 2337528, upload-time = "2026-07-13T20:38:12.128Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5b/30/a96d47df739689ac0001ade0afefc16e3b477fc2fb426b568515fdc8afce/kubernetes-36.0.3-py2.py3-none-any.whl", hash = "sha256:8fde9241c4b298e6374a069dcf728359b4e72c2fb29489a975ba4e1c047cf10f", size = 4618066, upload-time = "2026-07-13T20:38:10.172Z" }, +] + [[package]] name = "lark" version = "1.2.2" @@ -5106,6 +5140,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/14/5d/e000de781d92b732d52c572517db0e9e3a0085795f8bdc18201713c52d1f/nvtx-0.2.15-cp314-cp314t-win_amd64.whl", hash = "sha256:9d1d10db4fb4a3b0ffd6ed37bf25f0a966a3b4d34b3c9abb1f6572732959a6e5", size = 149109, upload-time = "2026-03-18T10:03:21.615Z" }, ] +[[package]] +name = "oauthlib" +version = "3.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0b/5f/19930f824ffeb0ad4372da4812c50edbd1434f678c90c2733e1188edfc63/oauthlib-3.3.1.tar.gz", hash = "sha256:0f0f8aa759826a193cf66c12ea1af1637f87b9b4622d46e866952bb022e538c9", size = 185918, upload-time = "2025-06-19T22:48:08.269Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/be/9c/92789c596b8df838baa98fa71844d84283302f7604ed565dafe5a6b5041a/oauthlib-3.3.1-py3-none-any.whl", hash = "sha256:88119c938d2b8fb88561af5f6ee0eec8cc8d552b7bb1f712743136eb7523b7a1", size = 160065, upload-time = "2025-06-19T22:48:06.508Z" }, +] + [[package]] name = "omegaconf" version = "2.3.0" @@ -6727,28 +6770,48 @@ wheels = [ [[package]] name = "pyyaml" -version = "6.0.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/54/ed/79a089b6be93607fa5cdaedf301d7dfb23af5f25c398d5ead2525b063e17/pyyaml-6.0.2.tar.gz", hash = "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e", size = 130631, upload-time = "2024-08-06T20:33:50.674Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/86/0c/c581167fc46d6d6d7ddcfb8c843a4de25bdd27e4466938109ca68492292c/PyYAML-6.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab", size = 183873, upload-time = "2024-08-06T20:32:25.131Z" }, - { url = "https://files.pythonhosted.org/packages/a8/0c/38374f5bb272c051e2a69281d71cba6fdb983413e6758b84482905e29a5d/PyYAML-6.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725", size = 173302, upload-time = "2024-08-06T20:32:26.511Z" }, - { url = "https://files.pythonhosted.org/packages/c3/93/9916574aa8c00aa06bbac729972eb1071d002b8e158bd0e83a3b9a20a1f7/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5", size = 739154, upload-time = "2024-08-06T20:32:28.363Z" }, - { url = "https://files.pythonhosted.org/packages/95/0f/b8938f1cbd09739c6da569d172531567dbcc9789e0029aa070856f123984/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425", size = 766223, upload-time = "2024-08-06T20:32:30.058Z" }, - { url = "https://files.pythonhosted.org/packages/b9/2b/614b4752f2e127db5cc206abc23a8c19678e92b23c3db30fc86ab731d3bd/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476", size = 767542, upload-time = "2024-08-06T20:32:31.881Z" }, - { url = "https://files.pythonhosted.org/packages/d4/00/dd137d5bcc7efea1836d6264f049359861cf548469d18da90cd8216cf05f/PyYAML-6.0.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48", size = 731164, upload-time = "2024-08-06T20:32:37.083Z" }, - { url = "https://files.pythonhosted.org/packages/c9/1f/4f998c900485e5c0ef43838363ba4a9723ac0ad73a9dc42068b12aaba4e4/PyYAML-6.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b", size = 756611, upload-time = "2024-08-06T20:32:38.898Z" }, - { url = "https://files.pythonhosted.org/packages/df/d1/f5a275fdb252768b7a11ec63585bc38d0e87c9e05668a139fea92b80634c/PyYAML-6.0.2-cp312-cp312-win32.whl", hash = "sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4", size = 140591, upload-time = "2024-08-06T20:32:40.241Z" }, - { url = "https://files.pythonhosted.org/packages/0c/e8/4f648c598b17c3d06e8753d7d13d57542b30d56e6c2dedf9c331ae56312e/PyYAML-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8", size = 156338, upload-time = "2024-08-06T20:32:41.93Z" }, - { url = "https://files.pythonhosted.org/packages/ef/e3/3af305b830494fa85d95f6d95ef7fa73f2ee1cc8ef5b495c7c3269fb835f/PyYAML-6.0.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba", size = 181309, upload-time = "2024-08-06T20:32:43.4Z" }, - { url = "https://files.pythonhosted.org/packages/45/9f/3b1c20a0b7a3200524eb0076cc027a970d320bd3a6592873c85c92a08731/PyYAML-6.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1", size = 171679, upload-time = "2024-08-06T20:32:44.801Z" }, - { url = "https://files.pythonhosted.org/packages/7c/9a/337322f27005c33bcb656c655fa78325b730324c78620e8328ae28b64d0c/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133", size = 733428, upload-time = "2024-08-06T20:32:46.432Z" }, - { url = "https://files.pythonhosted.org/packages/a3/69/864fbe19e6c18ea3cc196cbe5d392175b4cf3d5d0ac1403ec3f2d237ebb5/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484", size = 763361, upload-time = "2024-08-06T20:32:51.188Z" }, - { url = "https://files.pythonhosted.org/packages/04/24/b7721e4845c2f162d26f50521b825fb061bc0a5afcf9a386840f23ea19fa/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5", size = 759523, upload-time = "2024-08-06T20:32:53.019Z" }, - { url = "https://files.pythonhosted.org/packages/2b/b2/e3234f59ba06559c6ff63c4e10baea10e5e7df868092bf9ab40e5b9c56b6/PyYAML-6.0.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc", size = 726660, upload-time = "2024-08-06T20:32:54.708Z" }, - { url = "https://files.pythonhosted.org/packages/fe/0f/25911a9f080464c59fab9027482f822b86bf0608957a5fcc6eaac85aa515/PyYAML-6.0.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652", size = 751597, upload-time = "2024-08-06T20:32:56.985Z" }, - { url = "https://files.pythonhosted.org/packages/14/0d/e2c3b43bbce3cf6bd97c840b46088a3031085179e596d4929729d8d68270/PyYAML-6.0.2-cp313-cp313-win32.whl", hash = "sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183", size = 140527, upload-time = "2024-08-06T20:33:03.001Z" }, - { url = "https://files.pythonhosted.org/packages/fa/de/02b54f42487e3d3c6efb3f89428677074ca7bf43aae402517bc7cca949f3/PyYAML-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563", size = 156446, upload-time = "2024-08-06T20:33:04.33Z" }, +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, + { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, + { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, + { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, ] [[package]] @@ -6975,6 +7038,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d7/8e/7540e8a2036f79a125c1d2ebadf69ed7901608859186c856fa0388ef4197/requests-2.33.1-py3-none-any.whl", hash = "sha256:4e6d1ef462f3626a1f0a0a9c42dd93c63bad33f9f1c1937509b8c5c8718ab56a", size = 64947, upload-time = "2026-03-30T16:09:13.83Z" }, ] +[[package]] +name = "requests-oauthlib" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "oauthlib" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/42/f2/05f29bc3913aea15eb670be136045bf5c5bbf4b99ecb839da9b422bb2c85/requests-oauthlib-2.0.0.tar.gz", hash = "sha256:b3dffaebd884d8cd778494369603a9e7b58d29111bf6b41bdc2dcd87203af4e9", size = 55650, upload-time = "2024-03-22T20:32:29.939Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3b/5d/63d4ae3b9daea098d5d6f5da83984853c1bbacd5dc826764b249fe119d24/requests_oauthlib-2.0.0-py2.py3-none-any.whl", hash = "sha256:7dd8a5c40426b779b0868c404bdef9768deccf22749cde15852df527e6269b36", size = 24179, upload-time = "2024-03-22T20:32:28.055Z" }, +] + [[package]] name = "respx" version = "0.22.0" From ef39e2f617b43ec2d4905f9acdd8b38cc1554d2f Mon Sep 17 00:00:00 2001 From: Noppanat Wadlom Date: Tue, 8 Sep 2026 22:46:11 +0800 Subject: [PATCH 03/11] feat: add kubernetes worker provider The supervisor creates and deletes worker Pods through the Kubernetes API, so a cluster schedules workers onto GPU nodes instead of the supervisor placing containers on its own host. Credentials travel in a per-worker Secret consumed via envFrom rather than inline in the pod spec, and the secret set is derived from the SecretStr fields on WorkerConfig so a newly added credential cannot silently land in the manifest. GPUs are requested as extended resources and left to the device plugin to assign. Object names are sanitized to RFC 1123, which Docker container names are not; the default node alias alone would otherwise be rejected by the API. Signed-off-by: Noppanat Wadlom --- src/server/env.py | 15 + src/server/supervisor/adapters/kubernetes.py | 681 +++++++++++++++++++ src/server/supervisor/manager.py | 7 + tests/server/test_k8s_worker_adapter.py | 543 +++++++++++++++ 4 files changed, 1246 insertions(+) create mode 100644 src/server/supervisor/adapters/kubernetes.py create mode 100644 tests/server/test_k8s_worker_adapter.py diff --git a/src/server/env.py b/src/server/env.py index 11561e1c..d40a314a 100644 --- a/src/server/env.py +++ b/src/server/env.py @@ -107,6 +107,21 @@ ) DOCKER_GPU_RUNTIME: str | None = os.getenv("DOCKER_GPU_RUNTIME", "").strip() or None +K8S_NAMESPACE: str = ( + os.getenv("K8S_NAMESPACE") or os.getenv("POD_NAMESPACE") or "default" +).strip() +K8S_WORKER_NAMESPACE: str = ( + os.getenv("K8S_WORKER_NAMESPACE") or "" +).strip() or K8S_NAMESPACE +K8S_SUPERVISOR_SERVICE: str = ( + os.getenv("K8S_SUPERVISOR_SERVICE") or "flowmesh-supervisor" +).strip() +K8S_SERVER_SERVICE: str = (os.getenv("K8S_SERVER_SERVICE") or "flowmesh-server").strip() +K8S_CLUSTER_DOMAIN: str = (os.getenv("K8S_CLUSTER_DOMAIN") or "cluster.local").strip() +K8S_GPU_RESOURCE_NAME: str = ( + os.getenv("K8S_GPU_RESOURCE_NAME") or "nvidia.com/gpu" +).strip() + WORKER_CONFIG_PATH: str = os.getenv("WORKER_CONFIG_PATH", "configs/worker_config.yaml") CUDA_VISIBLE_DEVICES: str | None = os.getenv("CUDA_VISIBLE_DEVICES") if CUDA_VISIBLE_DEVICES is not None: diff --git a/src/server/supervisor/adapters/kubernetes.py b/src/server/supervisor/adapters/kubernetes.py new file mode 100644 index 00000000..19905180 --- /dev/null +++ b/src/server/supervisor/adapters/kubernetes.py @@ -0,0 +1,681 @@ +import asyncio +import hashlib +import logging +import re +from collections import Counter +from pathlib import PurePosixPath +from typing import Any, get_args + +from kubernetes import client, config +from kubernetes.client.exceptions import ApiException +from pydantic import Field, SecretStr + +from shared.utils import parse_mem_to_bytes + +from ... import env +from ...hooks import PrincipalContext +from ...schemas.node import CPUInfo, GpuInfo, GpuPlatformInfo, MemoryInfo +from ..resource_manager import GpuArch +from ..schemas import WorkerHardware, WorkerInfo, WorkerStatus +from .base import ( + ProviderSpec, + WorkerAdapter, + WorkerConfig, + WorkerFactory, + WorkerTokenType, + WorkerType, +) +from .utils import get_worker_image_name + +PROVIDER_NAME = "kubernetes" + +MANAGED_LABEL = "flowmesh.io/managed" +NODE_ALIAS_LABEL = "flowmesh.io/node-alias" +NODE_ID_LABEL = "flowmesh.io/node-id" +WORKER_NAME_LABEL = "flowmesh.io/worker-name" + +_GPU_PRODUCT_LABEL = "nvidia.com/gpu.product" +_RFC1123_MAX_LEN = 63 +_RFC1123_INVALID_RE = re.compile(r"[^a-z0-9-]+") +_LABEL_VALUE_INVALID_RE = re.compile(r"[^A-Za-z0-9_.-]+") +_LABEL_VALUE_MAX_LEN = 63 +_CPU_MILLI_RE = re.compile(r"^([0-9]+)m$") + +logger = logging.getLogger("supervisor") + + +def _short_digest(value: str) -> str: + return hashlib.md5(value.encode("utf-8"), usedforsecurity=False).hexdigest()[:8] + + +def sanitize_object_name(name: str, maxlen: int = _RFC1123_MAX_LEN) -> str: + """Return an RFC 1123 DNS label derived from ``name``. + + Kubernetes object names are far stricter than Docker container names — + lowercase alphanumerics and dashes only — so names that are valid + elsewhere in FlowMesh (``flowmesh_node``, alias-derived names carrying + underscores) are rejected by the API. A truncated name carries a digest of + the original so two long names cannot collapse onto one object. + """ + sanitized = _RFC1123_INVALID_RE.sub("-", name.strip().lower()) + sanitized = re.sub(r"-{2,}", "-", sanitized).strip("-") + if len(sanitized) > maxlen: + head = sanitized[: maxlen - 9].rstrip("-") + sanitized = f"{head}-{_short_digest(name)}" + if not sanitized or not sanitized[0].isalnum(): + sanitized = f"w-{_short_digest(name)}" + return sanitized + + +def sanitize_label_value(value: str) -> str: + """Return a value accepted by the Kubernetes label-value grammar.""" + sanitized = _LABEL_VALUE_INVALID_RE.sub("-", value.strip())[:_LABEL_VALUE_MAX_LEN] + return sanitized.strip("-_.") + + +def _secret_field_names(config_cls: type[WorkerConfig]) -> frozenset[str]: + """Return the config fields declared as ``SecretStr``. + + Deriving the credential set from the model keeps it correct as fields are + added; a hand-maintained list would silently leak the next secret someone + adds to the worker environment into the pod spec. + """ + names: set[str] = set() + for field_name, field in config_cls.model_fields.items(): + annotation = field.annotation + if any(arg is SecretStr for arg in (annotation, *get_args(annotation))): + names.add(field_name) + return frozenset(names) + + +def _parse_cpu_quantity(value: str) -> int | None: + raw = value.strip() + if match := _CPU_MILLI_RE.match(raw): + return max(1, int(match.group(1)) // 1000) + try: + return int(float(raw)) + except ValueError: + return None + + +def _deep_merge(base: dict[str, Any], overlay: dict[str, Any]) -> dict[str, Any]: + """Merge ``overlay`` onto ``base``; mappings merge, everything else replaces.""" + merged = dict(base) + for key, value in overlay.items(): + current = merged.get(key) + if isinstance(current, dict) and isinstance(value, dict): + merged[key] = _deep_merge(current, value) + else: + merged[key] = value + return merged + + +def _prune(value: dict[str, Any]) -> dict[str, Any]: + return {k: v for k, v in value.items() if v is not None} + + +class KubernetesWorkerConfig(WorkerConfig): + supervisor_grpc_target: str = ( + f"{env.K8S_SUPERVISOR_SERVICE}.{env.K8S_NAMESPACE}" + f".svc.{env.K8S_CLUSTER_DOMAIN}:{env.SERVER_GRPC_PORT}" + ) + """Supervisor gRPC target""" + flowmesh_url: str = ( + f"http://{env.K8S_SERVER_SERVICE}.{env.K8S_NAMESPACE}" + f".svc.{env.K8S_CLUSTER_DOMAIN}:{env.SERVER_APP_PORT}" + ) + """FlowMesh HTTP base URL workers use to fetch and upload artifacts. + + Defaults to the in-cluster Service rather than the externally advertised + ``FLOWMESH_BASE_URL``, which need not resolve from inside the cluster.""" + namespace: str = env.K8S_WORKER_NAMESPACE + """Namespace worker pods are created in""" + pod_name: str | None = None + """Optional explicit pod name""" + worker_type: WorkerType = WorkerType.CPU + """Type of worker (cpu or gpu)""" + gpu_count: int = 1 + """Number of GPUs requested for GPU workers""" + gpu_resource_name: str = env.K8S_GPU_RESOURCE_NAME + """Extended resource name used to request GPUs""" + gpu_arch: GpuArch | None = None + """GPU architecture selecting the worker image variant""" + node_selector: dict[str, str] | None = None + """Node labels a worker pod must match""" + tolerations: list[dict[str, Any]] | None = None + """Tolerations applied to the worker pod""" + service_account_name: str | None = None + """Service account the worker pod runs as""" + image_pull_secrets: list[str] = Field(default_factory=list) + """Image pull secrets referenced by the worker pod""" + image_pull_policy: str = "IfNotPresent" + """Image pull policy for the worker container""" + cpu_request: str | None = None + """CPU request for the worker container""" + cpu_limit: str | None = None + """CPU limit for the worker container""" + memory_request: str | None = None + """Memory request for the worker container""" + memory_limit: str | None = None + """Memory limit for the worker container""" + shm_size: str | None = None + """Size of the ``/dev/shm`` in-memory volume""" + results_pvc: str | None = None + """PersistentVolumeClaim backing the results directory""" + results_mount_path: str = "/var/lib/flowmesh-results" + """Path the results volume is mounted at inside the worker container""" + hf_cache_pvc: str | None = None + """PersistentVolumeClaim backing the Hugging Face cache""" + runtime_class_name: str | None = None + """RuntimeClass for the worker pod""" + priority_class_name: str | None = None + """PriorityClass for the worker pod""" + pod_labels: dict[str, str] | None = None + """Extra labels applied to the worker pod""" + pod_annotations: dict[str, str] | None = None + """Annotations applied to the worker pod""" + pod_overrides: dict[str, Any] | None = None + """Overlay merged onto the generated pod manifest. + + Mappings merge recursively; sequences and scalars replace.""" + stop_grace_period_sec: int = 30 + """Termination grace period applied when deleting the worker pod""" + docker_registry: str = env.FLOWMESH_REGISTRY + """Registry to pull worker images from""" + version: str = env.FLOWMESH_VERSION + """Worker image version tag""" + + def model_post_init(self, __context: object) -> None: + super().model_post_init(__context) + if self.worker_type == WorkerType.GPU and self.gpu_count < 1: + raise ValueError("Expected at least one GPU for GPU worker.") + + +class KubernetesWorkerInfo(WorkerInfo): + pass + + +class KubernetesWorkerAdapter(WorkerAdapter): + CONTAINER_HF_CACHE_DIR: str = "/home/appuser/.cache/huggingface" + SHM_MOUNT_PATH: str = PurePosixPath("/", "dev", "shm").as_posix() + CONTAINER_NAME: str = "worker" + WORKER_GID: int = 10001 + + def __init__( + self, + token: WorkerTokenType, + name: str, + pod_name: str, + config: KubernetesWorkerConfig, + core_api: client.CoreV1Api, + node_alias: str, + owner: PrincipalContext, + ) -> None: + super().__init__(token, name, config, owner) + + self.config: KubernetesWorkerConfig + self.pod_name = pod_name + self.secret_name = f"{pod_name}-env" + self.node_alias = node_alias + + self._core = core_api + self._status: WorkerStatus = WorkerStatus.STOPPED + self._hardware: dict[str, Any] | WorkerHardware | None = None + self._is_started = False + + @property + def status(self) -> WorkerStatus: + return self._status + + def set_status(self, status: WorkerStatus) -> None: + self._status = status + + def get_info(self) -> KubernetesWorkerInfo: + hardware = self._hardware + if isinstance(hardware, dict): + hardware = WorkerHardware.model_validate(hardware) + self._hardware = hardware + return KubernetesWorkerInfo( + id=self.worker_id, + name=self.name, + provider=PROVIDER_NAME, + status=self.status, + hardware=hardware, + ) + + async def start(self) -> bool: + self.set_status(WorkerStatus.STARTING) + try: + ok = await asyncio.to_thread(self._start) + if not ok: + self.set_status(WorkerStatus.STOPPED) + return ok + except Exception: + self.set_status(WorkerStatus.STOPPED) + raise + + async def prepare(self) -> None: + self._hardware = await asyncio.to_thread(self._probe_hardware) + + async def stop(self) -> bool: + prev_status = self.status + if prev_status in (WorkerStatus.STOPPING, WorkerStatus.STOPPED): + return True + self.set_status(WorkerStatus.STOPPING) + try: + ok = await asyncio.to_thread(self._stop) + if not ok: + self.set_status(prev_status) + return ok + except Exception: + self.set_status(prev_status) + raise + + def get_image_name(self) -> str: + return get_worker_image_name( + self.config.docker_registry, self.config.version, self._gpu_arch() + ) + + def _gpu_arch(self) -> GpuArch | None: + if self.config.worker_type != WorkerType.GPU: + return None + return self.config.gpu_arch or GpuArch.UNKNOWN + + def _base_environment(self) -> dict[str, str]: + environment = super()._base_environment() + environment["RESULTS_DIR"] = self.config.results_mount_path + return environment + + def _split_environment(self) -> tuple[dict[str, str], dict[str, str]]: + """Partition the worker environment into inline and secret-held values.""" + secret_keys = {"WORKER_TOKEN", "FLOWMESH_API_KEY"} | { + name.upper() for name in _secret_field_names(type(self.config)) + } + inline: dict[str, str] = {} + secret: dict[str, str] = {} + for key, value in self._base_environment().items(): + if key in secret_keys: + secret[key] = value + else: + inline[key] = value + return inline, secret + + def _labels(self) -> dict[str, str]: + labels = { + MANAGED_LABEL: "true", + NODE_ALIAS_LABEL: sanitize_label_value(self.node_alias), + WORKER_NAME_LABEL: sanitize_label_value(self.name), + } + if worker_id := self.worker_id: + labels[NODE_ID_LABEL] = sanitize_label_value(worker_id) + if self.config.pod_labels: + labels.update(self.config.pod_labels) + return labels + + def _resources(self) -> dict[str, Any]: + config = self.config + requests = _prune({"cpu": config.cpu_request, "memory": config.memory_request}) + limits = _prune({"cpu": config.cpu_limit, "memory": config.memory_limit}) + if config.worker_type == WorkerType.GPU: + limits[config.gpu_resource_name] = str(config.gpu_count) + return _prune({"requests": requests or None, "limits": limits or None}) + + def _volumes(self) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + config = self.config + volumes: list[dict[str, Any]] = [] + mounts: list[dict[str, Any]] = [] + + if config.results_pvc: + volumes.append( + { + "name": "results", + "persistentVolumeClaim": {"claimName": config.results_pvc}, + } + ) + else: + volumes.append({"name": "results", "emptyDir": {}}) + mounts.append({"name": "results", "mountPath": config.results_mount_path}) + + if config.hf_cache_pvc: + volumes.append( + { + "name": "hf-cache", + "persistentVolumeClaim": {"claimName": config.hf_cache_pvc}, + } + ) + mounts.append( + {"name": "hf-cache", "mountPath": self.CONTAINER_HF_CACHE_DIR} + ) + + if config.shm_size: + volumes.append( + { + "name": "dshm", + "emptyDir": {"medium": "Memory", "sizeLimit": config.shm_size}, + } + ) + mounts.append({"name": "dshm", "mountPath": self.SHM_MOUNT_PATH}) + + return volumes, mounts + + def build_pod_manifest(self) -> dict[str, Any]: + config = self.config + inline_env, _ = self._split_environment() + volumes, mounts = self._volumes() + + container = _prune( + { + "name": self.CONTAINER_NAME, + "image": self.get_image_name(), + "imagePullPolicy": config.image_pull_policy, + "env": [{"name": k, "value": v} for k, v in sorted(inline_env.items())], + "envFrom": [{"secretRef": {"name": self.secret_name}}], + "resources": self._resources() or None, + "volumeMounts": mounts or None, + } + ) + + spec = _prune( + { + "restartPolicy": "Always", + "terminationGracePeriodSeconds": config.stop_grace_period_sec, + "containers": [container], + "volumes": volumes or None, + "nodeSelector": config.node_selector or None, + "tolerations": config.tolerations or None, + "serviceAccountName": config.service_account_name, + "runtimeClassName": config.runtime_class_name, + "priorityClassName": config.priority_class_name, + "securityContext": {"fsGroup": self.WORKER_GID}, + "imagePullSecrets": ( + [{"name": name} for name in config.image_pull_secrets] or None + ), + } + ) + + manifest: dict[str, Any] = { + "apiVersion": "v1", + "kind": "Pod", + "metadata": _prune( + { + "name": self.pod_name, + "namespace": config.namespace, + "labels": self._labels(), + "annotations": config.pod_annotations or None, + } + ), + "spec": spec, + } + if config.pod_overrides: + manifest = _deep_merge(manifest, config.pod_overrides) + return manifest + + def build_secret_manifest(self) -> dict[str, Any]: + _, secret_env = self._split_environment() + return { + "apiVersion": "v1", + "kind": "Secret", + "type": "Opaque", + "metadata": { + "name": self.secret_name, + "namespace": self.config.namespace, + "labels": self._labels(), + }, + "stringData": secret_env, + } + + def _start(self) -> bool: + existing = self._read_pod() + if existing is not None: + phase = self._pod_phase(existing) + if phase in ("Running", "Pending"): + self._is_started = True + logger.warning("Pod %s is already running.", self.pod_name) + return True + if not self._delete_pod(): + return False + + if not self._apply_secret(): + return False + + try: + self._core.create_namespaced_pod( + namespace=self.config.namespace, body=self.build_pod_manifest() + ) + except ApiException as exc: + logger.error("Failed to create pod %s: %s", self.pod_name, repr(exc)) + self._delete_secret() + return False + + self._is_started = True + return True + + def _stop(self) -> bool: + deleted = self._delete_pod() + self._delete_secret() + if deleted: + self._is_started = False + return deleted + + def _read_pod(self) -> Any | None: + try: + return self._core.read_namespaced_pod( + name=self.pod_name, namespace=self.config.namespace + ) + except ApiException as exc: + if exc.status == 404: + return None + logger.warning("Failed to read pod %s: %s", self.pod_name, repr(exc)) + return None + + @staticmethod + def _pod_phase(pod: Any) -> str | None: + status = pod.status if hasattr(pod, "status") else None + if status is None: + return None + return status.phase if hasattr(status, "phase") else None + + def _apply_secret(self) -> bool: + body = self.build_secret_manifest() + try: + self._core.create_namespaced_secret( + namespace=self.config.namespace, body=body + ) + return True + except ApiException as exc: + if exc.status != 409: + logger.error( + "Failed to create secret %s: %s", self.secret_name, repr(exc) + ) + return False + try: + self._core.replace_namespaced_secret( + name=self.secret_name, namespace=self.config.namespace, body=body + ) + return True + except ApiException as exc: + logger.error("Failed to update secret %s: %s", self.secret_name, repr(exc)) + return False + + def _delete_pod(self) -> bool: + try: + self._core.delete_namespaced_pod( + name=self.pod_name, + namespace=self.config.namespace, + grace_period_seconds=self.config.stop_grace_period_sec, + ) + return True + except ApiException as exc: + if exc.status == 404: + return True + logger.error("Failed to delete pod %s: %s", self.pod_name, repr(exc)) + return False + + def _delete_secret(self) -> bool: + try: + self._core.delete_namespaced_secret( + name=self.secret_name, namespace=self.config.namespace + ) + return True + except ApiException as exc: + if exc.status == 404: + return True + logger.warning( + "Failed to delete secret %s: %s", self.secret_name, repr(exc) + ) + return False + + def _probe_hardware(self) -> dict[str, Any] | None: + selector = ",".join( + f"{key}={value}" for key, value in (self.config.node_selector or {}).items() + ) + try: + nodes = self._core.list_node(label_selector=selector or None) + except ApiException as exc: + if exc.status == 403: + logger.debug( + "No permission to list nodes; skipping hardware probe for %s", + self.name, + ) + else: + logger.warning( + "Failed to list nodes for worker %s: %s", self.name, repr(exc) + ) + return None + + items = nodes.items if hasattr(nodes, "items") else [] + if not items: + return None + return self._hardware_from_node(items[0]).model_dump() + + def _hardware_from_node(self, node: Any) -> WorkerHardware: + status = node.status if hasattr(node, "status") else None + allocatable = dict(getattr(status, "allocatable", None) or {}) + metadata = node.metadata if hasattr(node, "metadata") else None + node_labels = dict(getattr(metadata, "labels", None) or {}) + + cpu = CPUInfo(logical_cores=_parse_cpu_quantity(allocatable.get("cpu", ""))) + memory = MemoryInfo( + total_bytes=parse_mem_to_bytes(allocatable.get("memory", "")) + ) + + gpu = GpuPlatformInfo() + if self.config.worker_type == WorkerType.GPU: + product = node_labels.get(_GPU_PRODUCT_LABEL, "") + arch = self.config.gpu_arch or GpuArch.from_name(product) + gpu = GpuPlatformInfo( + gpu_arch=arch.value, + devices=[ + GpuInfo(index=index, name=product or None) + for index in range(self.config.gpu_count) + ], + ) + + return WorkerHardware(cpu=cpu, memory=memory, gpu=gpu) + + +class KubernetesWorkerFactory(WorkerFactory): + def __init__(self, system_principal: PrincipalContext) -> None: + super().__init__(system_principal) + self._api_client: client.ApiClient | None = None + self._core: client.CoreV1Api | None = None + self._worker_id_registry: Counter[str] = Counter() + self._reaped = False + + def create_worker( + self, token: WorkerTokenType, config: KubernetesWorkerConfig + ) -> KubernetesWorkerAdapter: + core = self.core_api() + self._reap_orphans(core, config.namespace) + + name = config.worker_alias or self._next_worker_name(config.worker_type) + pod_name = config.pod_name or sanitize_object_name(f"{env.NODE_ALIAS}-{name}") + return KubernetesWorkerAdapter( + token=token, + name=name, + pod_name=pod_name, + config=config, + core_api=core, + node_alias=env.NODE_ALIAS, + owner=self.system_principal, + ) + + def destroy_worker(self, worker: WorkerAdapter) -> None: + if not isinstance(worker, KubernetesWorkerAdapter): + raise ValueError("Invalid worker type") + + def cleanup(self) -> None: + if self._api_client is not None: + self._api_client.close() + self._api_client = None + self._core = None + + def core_api(self) -> client.CoreV1Api: + if self._core is None: + try: + config.load_incluster_config() + except config.ConfigException: + config.load_kube_config() + self._api_client = client.ApiClient() + self._core = client.CoreV1Api(self._api_client) + return self._core + + def _next_worker_name(self, worker_type: WorkerType) -> str: + match worker_type: + case WorkerType.CPU: + prefix = "flowmesh_server_worker_cpu_" + case WorkerType.GPU: + prefix = "flowmesh_server_worker_gpu_" + case _: + raise ValueError(f"Unsupported worker type: {worker_type}") + self._worker_id_registry[prefix] += 1 + return f"{prefix}{self._worker_id_registry[prefix]}" + + def _reap_orphans(self, core: client.CoreV1Api, namespace: str) -> None: + """Delete worker pods left behind by a previous supervisor incarnation. + + Their tokens died with that supervisor's in-memory registry, so they can + never register again; a clean shutdown drains its own workers, so + anything matching here is a zombie from an unclean exit. + """ + if self._reaped: + return + self._reaped = True + + selector = ( + f"{MANAGED_LABEL}=true," + f"{NODE_ALIAS_LABEL}={sanitize_label_value(env.NODE_ALIAS)}" + ) + try: + pods = core.list_namespaced_pod( + namespace=namespace, label_selector=selector + ) + except ApiException as exc: + logger.warning("Failed to list orphaned worker pods: %s", repr(exc)) + return + + for pod in pods.items if hasattr(pods, "items") else []: + pod_name = pod.metadata.name + logger.info("Reaping orphaned worker pod %s", pod_name) + try: + core.delete_namespaced_pod(name=pod_name, namespace=namespace) + except ApiException as exc: + if exc.status != 404: + logger.warning( + "Failed to delete orphaned pod %s: %s", pod_name, repr(exc) + ) + try: + core.delete_collection_namespaced_secret( + namespace=namespace, label_selector=selector + ) + except ApiException as exc: + logger.warning("Failed to delete orphaned worker secrets: %s", repr(exc)) + + +def get_provider_spec(system_principal: PrincipalContext) -> ProviderSpec: + return ProviderSpec( + name=PROVIDER_NAME, + config_cls=KubernetesWorkerConfig, + adapter_cls=KubernetesWorkerAdapter, + factory=KubernetesWorkerFactory(system_principal), + ) diff --git a/src/server/supervisor/manager.py b/src/server/supervisor/manager.py index 681f9dd2..f6aa7be4 100644 --- a/src/server/supervisor/manager.py +++ b/src/server/supervisor/manager.py @@ -12,6 +12,7 @@ from .adapters.docker import get_provider_spec as docker_provider_spec from .adapters.external import get_provider_spec as external_provider_spec from .adapters.external import verify_external_token +from .adapters.kubernetes import get_provider_spec as kubernetes_provider_spec from .adapters.vastai import get_provider_spec as vastai_provider_spec from .registry import WorkerRegistry from .schemas import WorkerInfo, WorkerStatus @@ -89,6 +90,12 @@ def __init__( logger.warning( "Docker worker provider unavailable, continuing without it: %s", exc ) + try: + specs.append(kubernetes_provider_spec(system_principal)) + except Exception as exc: + logger.warning( + "Kubernetes worker provider unavailable, continuing without it: %s", exc + ) try: specs.append(vastai_provider_spec(system_principal)) except Exception as exc: diff --git a/tests/server/test_k8s_worker_adapter.py b/tests/server/test_k8s_worker_adapter.py new file mode 100644 index 00000000..b63bb31b --- /dev/null +++ b/tests/server/test_k8s_worker_adapter.py @@ -0,0 +1,543 @@ +"""Kubernetes worker provider: pod construction, lifecycle, and hardware probe.""" + +import asyncio +from typing import Any +from unittest.mock import MagicMock, patch + +import pytest +from kubernetes.client.exceptions import ApiException + +from server.hooks import PrincipalContext +from server.supervisor.adapters.base import WorkerTokenType, WorkerType +from server.supervisor.adapters.kubernetes import ( + MANAGED_LABEL, + NODE_ALIAS_LABEL, + WORKER_NAME_LABEL, + KubernetesWorkerAdapter, + KubernetesWorkerConfig, + KubernetesWorkerFactory, + _secret_field_names, + sanitize_label_value, + sanitize_object_name, +) +from server.supervisor.resource_manager import GpuArch +from server.supervisor.schemas import WorkerStatus + +# ------------------------------------------------------------------ # +# Helpers +# ------------------------------------------------------------------ # + + +def _token(value: str) -> WorkerTokenType: + return WorkerTokenType(value) + + +def _principal() -> PrincipalContext: + return PrincipalContext( + principal_id="p-test", + org_id="org", + external_id="ext", + principal_type="user", + scopes=[], + ) + + +def _adapter( + core: MagicMock | None = None, **config_kwargs: Any +) -> KubernetesWorkerAdapter: + return KubernetesWorkerAdapter( + token=_token("tok-123"), + name="worker-1", + pod_name="fm-worker-1", + config=KubernetesWorkerConfig(**config_kwargs), + core_api=core or MagicMock(), + node_alias="flowmesh_node", + owner=_principal(), + ) + + +def _api_error(status: int) -> ApiException: + return ApiException(status=status, reason="test") + + +def _container(manifest: dict[str, Any]) -> dict[str, Any]: + containers: list[dict[str, Any]] = manifest["spec"]["containers"] + return containers[0] + + +def _env_names(manifest: dict[str, Any]) -> set[str]: + return {entry["name"] for entry in _container(manifest).get("env", [])} + + +def _node(allocatable: dict[str, str], labels: dict[str, str]) -> MagicMock: + node = MagicMock() + node.status.allocatable = allocatable + node.metadata.labels = labels + return node + + +# ------------------------------------------------------------------ # +# Object naming +# ------------------------------------------------------------------ # + + +class TestObjectNaming: + def test_underscores_and_case_are_normalized(self) -> None: + assert sanitize_object_name("flowmesh_node-Worker_CPU_1") == ( + "flowmesh-node-worker-cpu-1" + ) + + def test_long_names_are_truncated_with_a_digest(self) -> None: + name = sanitize_object_name("x" * 200) + assert len(name) <= 63 + assert name != sanitize_object_name("y" * 200) + + def test_distinct_long_names_do_not_collide(self) -> None: + base = "flowmesh-server-worker-gpu-" + "a" * 60 + assert sanitize_object_name(base + "-one") != sanitize_object_name( + base + "-two" + ) + + def test_name_starting_with_a_separator_is_replaced(self) -> None: + assert sanitize_object_name("___").startswith("w-") + + def test_label_values_keep_underscores(self) -> None: + assert sanitize_label_value("flowmesh_node") == "flowmesh_node" + + +# ------------------------------------------------------------------ # +# Environment split +# ------------------------------------------------------------------ # + + +class TestEnvironmentSplit: + def test_credentials_go_to_the_secret_and_not_the_pod(self) -> None: + adapter = _adapter() + inline, secret = adapter._split_environment() + + assert "WORKER_TOKEN" in secret + assert "FLOWMESH_API_KEY" in secret + assert "WORKER_TOKEN" not in inline + assert "FLOWMESH_API_KEY" not in inline + + def test_secret_str_fields_are_all_treated_as_secret(self) -> None: + adapter = _adapter() + _, secret = adapter._split_environment() + + for field in _secret_field_names(KubernetesWorkerConfig): + assert field.upper() in secret + + def test_derived_secret_keys_exist_in_the_worker_environment(self) -> None: + """Guards the field-name -> env-key assumption behind the split.""" + adapter = _adapter() + base_env = adapter._base_environment() + + for field in _secret_field_names(KubernetesWorkerConfig): + assert field.upper() in base_env + + def test_non_secret_values_stay_inline(self) -> None: + adapter = _adapter() + inline, secret = adapter._split_environment() + + assert "SUPERVISOR_GRPC_TARGET" in inline + assert "SUPERVISOR_GRPC_TLS_CA_B64" in inline + assert not set(inline) & set(secret) + + def test_results_dir_points_at_the_mount_path(self) -> None: + adapter = _adapter(results_mount_path="/mnt/results") + assert adapter._base_environment()["RESULTS_DIR"] == "/mnt/results" + + def test_flowmesh_url_defaults_to_the_in_cluster_service(self) -> None: + adapter = _adapter() + url = adapter._base_environment()["FLOWMESH_BASE_URL"] + assert url.startswith("http://flowmesh-server.") + assert ".svc." in url + + +# ------------------------------------------------------------------ # +# Pod manifest +# ------------------------------------------------------------------ # + + +class TestPodManifest: + def test_labels_identify_the_managing_node(self) -> None: + manifest = _adapter().build_pod_manifest() + labels = manifest["metadata"]["labels"] + + assert labels[MANAGED_LABEL] == "true" + assert labels[NODE_ALIAS_LABEL] == "flowmesh_node" + assert labels[WORKER_NAME_LABEL] == "worker-1" + + def test_pod_restarts_in_place(self) -> None: + manifest = _adapter().build_pod_manifest() + assert manifest["spec"]["restartPolicy"] == "Always" + + def test_secret_is_consumed_via_env_from(self) -> None: + adapter = _adapter() + manifest = adapter.build_pod_manifest() + env_from = _container(manifest)["envFrom"] + + assert env_from == [{"secretRef": {"name": adapter.secret_name}}] + + def test_cpu_worker_requests_no_gpu(self) -> None: + manifest = _adapter().build_pod_manifest() + limits = _container(manifest).get("resources", {}).get("limits", {}) + assert "nvidia.com/gpu" not in limits + + def test_gpu_worker_requests_the_gpu_resource(self) -> None: + manifest = _adapter( + worker_type=WorkerType.GPU, gpu_count=2 + ).build_pod_manifest() + limits = _container(manifest)["resources"]["limits"] + assert limits["nvidia.com/gpu"] == "2" + + def test_gpu_resource_name_is_configurable(self) -> None: + manifest = _adapter( + worker_type=WorkerType.GPU, gpu_resource_name="amd.com/gpu" + ).build_pod_manifest() + assert "amd.com/gpu" in _container(manifest)["resources"]["limits"] + + def test_device_indices_are_never_pinned_server_side(self) -> None: + """The device plugin owns assignment; a server-set index would fight it.""" + manifest = _adapter(worker_type=WorkerType.GPU).build_pod_manifest() + assert "CUDA_VISIBLE_DEVICES" not in _env_names(manifest) + + def test_results_default_to_an_ephemeral_volume(self) -> None: + manifest = _adapter().build_pod_manifest() + volumes = {v["name"]: v for v in manifest["spec"]["volumes"]} + assert volumes["results"]["emptyDir"] == {} + + def test_results_pvc_is_mounted_when_configured(self) -> None: + manifest = _adapter(results_pvc="fm-results").build_pod_manifest() + volumes = {v["name"]: v for v in manifest["spec"]["volumes"]} + assert volumes["results"]["persistentVolumeClaim"]["claimName"] == "fm-results" + + def test_shm_volume_is_added_when_sized(self) -> None: + manifest = _adapter(shm_size="2Gi").build_pod_manifest() + volumes = {v["name"]: v for v in manifest["spec"]["volumes"]} + mounts = {m["name"]: m for m in _container(manifest)["volumeMounts"]} + + assert volumes["dshm"]["emptyDir"] == {"medium": "Memory", "sizeLimit": "2Gi"} + assert mounts["dshm"]["mountPath"] == "/dev/shm" + + def test_no_shm_volume_by_default(self) -> None: + manifest = _adapter().build_pod_manifest() + assert "dshm" not in {v["name"] for v in manifest["spec"]["volumes"]} + + def test_scheduling_hints_are_carried_through(self) -> None: + manifest = _adapter( + node_selector={"gpu": "true"}, + tolerations=[{"key": "gpu", "operator": "Exists"}], + service_account_name="fm-worker", + image_pull_secrets=["regcred"], + runtime_class_name="nvidia", + priority_class_name="high", + ).build_pod_manifest() + spec = manifest["spec"] + + assert spec["nodeSelector"] == {"gpu": "true"} + assert spec["tolerations"] == [{"key": "gpu", "operator": "Exists"}] + assert spec["serviceAccountName"] == "fm-worker" + assert spec["imagePullSecrets"] == [{"name": "regcred"}] + assert spec["runtimeClassName"] == "nvidia" + assert spec["priorityClassName"] == "high" + + def test_unset_optional_fields_are_omitted(self) -> None: + spec = _adapter().build_pod_manifest()["spec"] + for key in ("nodeSelector", "tolerations", "runtimeClassName"): + assert key not in spec + + def test_pod_overrides_deep_merge_mappings(self) -> None: + manifest = _adapter( + pod_overrides={"metadata": {"annotations": {"team": "ml"}}} + ).build_pod_manifest() + + assert manifest["metadata"]["annotations"] == {"team": "ml"} + assert manifest["metadata"]["name"] == "fm-worker-1" + + def test_pod_overrides_replace_sequences(self) -> None: + manifest = _adapter( + pod_overrides={"spec": {"tolerations": [{"key": "override"}]}} + ).build_pod_manifest() + + assert manifest["spec"]["tolerations"] == [{"key": "override"}] + + def test_secret_manifest_carries_only_credentials(self) -> None: + adapter = _adapter() + secret = adapter.build_secret_manifest() + _, expected = adapter._split_environment() + + assert secret["kind"] == "Secret" + assert secret["stringData"] == expected + + +# ------------------------------------------------------------------ # +# Lifecycle +# ------------------------------------------------------------------ # + + +class TestLifecycle: + def test_start_creates_the_secret_before_the_pod(self) -> None: + core = MagicMock() + core.read_namespaced_pod.side_effect = _api_error(404) + adapter = _adapter(core) + + assert asyncio.run(adapter.start()) is True + core.create_namespaced_secret.assert_called_once() + core.create_namespaced_pod.assert_called_once() + + def test_start_adopts_a_running_pod(self) -> None: + core = MagicMock() + core.read_namespaced_pod.return_value.status.phase = "Running" + adapter = _adapter(core) + + assert asyncio.run(adapter.start()) is True + core.create_namespaced_pod.assert_not_called() + + def test_start_replaces_a_terminal_pod(self) -> None: + core = MagicMock() + core.read_namespaced_pod.return_value.status.phase = "Failed" + adapter = _adapter(core) + + assert asyncio.run(adapter.start()) is True + core.delete_namespaced_pod.assert_called_once() + core.create_namespaced_pod.assert_called_once() + + def test_failed_pod_creation_removes_the_orphaned_secret(self) -> None: + core = MagicMock() + core.read_namespaced_pod.side_effect = _api_error(404) + core.create_namespaced_pod.side_effect = _api_error(422) + adapter = _adapter(core) + + assert asyncio.run(adapter.start()) is False + core.delete_namespaced_secret.assert_called_once() + assert adapter.status is WorkerStatus.STOPPED + + def test_existing_secret_is_replaced(self) -> None: + core = MagicMock() + core.read_namespaced_pod.side_effect = _api_error(404) + core.create_namespaced_secret.side_effect = _api_error(409) + adapter = _adapter(core) + + assert asyncio.run(adapter.start()) is True + core.replace_namespaced_secret.assert_called_once() + + def test_stop_deletes_pod_and_secret(self) -> None: + core = MagicMock() + adapter = _adapter(core) + adapter.set_status(WorkerStatus.RUNNING) + + assert asyncio.run(adapter.stop()) is True + core.delete_namespaced_pod.assert_called_once() + core.delete_namespaced_secret.assert_called_once() + + def test_stop_tolerates_an_already_deleted_pod(self) -> None: + core = MagicMock() + core.delete_namespaced_pod.side_effect = _api_error(404) + core.delete_namespaced_secret.side_effect = _api_error(404) + adapter = _adapter(core) + adapter.set_status(WorkerStatus.RUNNING) + + assert asyncio.run(adapter.stop()) is True + + def test_stop_reports_failure_and_restores_status(self) -> None: + core = MagicMock() + core.delete_namespaced_pod.side_effect = _api_error(500) + adapter = _adapter(core) + adapter.set_status(WorkerStatus.RUNNING) + + assert asyncio.run(adapter.stop()) is False + assert adapter.status is WorkerStatus.RUNNING + + def test_stopping_a_stopped_worker_is_a_no_op(self) -> None: + core = MagicMock() + adapter = _adapter(core) + + assert asyncio.run(adapter.stop()) is True + core.delete_namespaced_pod.assert_not_called() + + +# ------------------------------------------------------------------ # +# Hardware probe +# ------------------------------------------------------------------ # + + +class TestHardwareProbe: + def test_probe_degrades_without_node_permission(self) -> None: + core = MagicMock() + core.list_node.side_effect = _api_error(403) + adapter = _adapter(core) + + asyncio.run(adapter.prepare()) + assert adapter.get_info().hardware is None + + def test_probe_reads_allocatable_capacity(self) -> None: + core = MagicMock() + core.list_node.return_value.items = [ + _node({"cpu": "32", "memory": "65536Ki"}, {}) + ] + adapter = _adapter(core) + + asyncio.run(adapter.prepare()) + hardware = adapter.get_info().hardware + + assert hardware is not None + assert hardware.cpu.logical_cores == 32 + assert hardware.memory.total_bytes == 65536 * 1024 + + def test_probe_parses_millicore_quantities(self) -> None: + core = MagicMock() + core.list_node.return_value.items = [_node({"cpu": "7800m"}, {})] + adapter = _adapter(core) + + asyncio.run(adapter.prepare()) + hardware = adapter.get_info().hardware + + assert hardware is not None + assert hardware.cpu.logical_cores == 7 + + def test_probe_derives_gpu_arch_from_node_labels(self) -> None: + core = MagicMock() + core.list_node.return_value.items = [ + _node({"cpu": "8"}, {"nvidia.com/gpu.product": "NVIDIA-H100-80GB-HBM3"}) + ] + adapter = _adapter(core, worker_type=WorkerType.GPU, gpu_count=2) + + asyncio.run(adapter.prepare()) + hardware = adapter.get_info().hardware + + assert hardware is not None + assert hardware.gpu.gpu_arch == GpuArch.HOPPER.value + assert len(hardware.gpu.devices) == 2 + + def test_probe_without_matching_nodes_returns_nothing(self) -> None: + core = MagicMock() + core.list_node.return_value.items = [] + adapter = _adapter(core) + + asyncio.run(adapter.prepare()) + assert adapter.get_info().hardware is None + + def test_probe_filters_by_node_selector(self) -> None: + core = MagicMock() + core.list_node.return_value.items = [] + adapter = _adapter(core, node_selector={"gpu": "true", "zone": "a"}) + + asyncio.run(adapter.prepare()) + selector = core.list_node.call_args.kwargs["label_selector"] + + assert set(selector.split(",")) == {"gpu=true", "zone=a"} + + +# ------------------------------------------------------------------ # +# Image selection +# ------------------------------------------------------------------ # + + +class TestImageSelection: + def test_cpu_worker_uses_the_cpu_image(self) -> None: + assert _adapter().get_image_name().endswith("-cpu") + + def test_gpu_worker_uses_the_gpu_image(self) -> None: + adapter = _adapter(worker_type=WorkerType.GPU) + assert adapter.get_image_name().endswith("-gpu") + + +# ------------------------------------------------------------------ # +# Factory +# ------------------------------------------------------------------ # + + +class TestFactory: + def _factory(self, core: MagicMock) -> KubernetesWorkerFactory: + factory = KubernetesWorkerFactory(_principal()) + factory._core = core + return factory + + def test_orphaned_pods_are_reaped_once(self) -> None: + core = MagicMock() + pod = MagicMock() + pod.metadata.name = "fm-stale" + core.list_namespaced_pod.return_value.items = [pod] + factory = self._factory(core) + + factory.create_worker(_token("t1"), KubernetesWorkerConfig()) + factory.create_worker(_token("t2"), KubernetesWorkerConfig()) + + core.list_namespaced_pod.assert_called_once() + core.delete_namespaced_pod.assert_called_once_with( + name="fm-stale", namespace="default" + ) + + def test_reaping_selects_this_node_only(self) -> None: + core = MagicMock() + core.list_namespaced_pod.return_value.items = [] + factory = self._factory(core) + + factory.create_worker(_token("t1"), KubernetesWorkerConfig()) + selector = core.list_namespaced_pod.call_args.kwargs["label_selector"] + + assert f"{MANAGED_LABEL}=true" in selector + assert NODE_ALIAS_LABEL in selector + + def test_reaping_survives_a_listing_failure(self) -> None: + core = MagicMock() + core.list_namespaced_pod.side_effect = _api_error(403) + factory = self._factory(core) + + worker = factory.create_worker(_token("t1"), KubernetesWorkerConfig()) + assert worker.name + + def test_pod_names_are_api_safe(self) -> None: + core = MagicMock() + core.list_namespaced_pod.return_value.items = [] + factory = self._factory(core) + + worker = factory.create_worker( + _token("t1"), KubernetesWorkerConfig(worker_alias="Train_Worker_01") + ) + + assert worker.pod_name == sanitize_object_name(worker.pod_name) + + def test_worker_names_increment_per_type(self) -> None: + core = MagicMock() + core.list_namespaced_pod.return_value.items = [] + factory = self._factory(core) + + first = factory.create_worker(_token("t1"), KubernetesWorkerConfig()) + second = factory.create_worker(_token("t2"), KubernetesWorkerConfig()) + + assert first.name != second.name + + def test_cleanup_closes_the_api_client(self) -> None: + factory = KubernetesWorkerFactory(_principal()) + api_client = MagicMock() + factory._api_client = api_client + factory._core = MagicMock() + + factory.cleanup() + + api_client.close.assert_called_once_with() + assert factory._core is None + + def test_client_falls_back_to_kubeconfig_outside_a_cluster(self) -> None: + from kubernetes import config as k8s_config + + factory = KubernetesWorkerFactory(_principal()) + with ( + patch.object( + k8s_config, + "load_incluster_config", + side_effect=k8s_config.ConfigException("not in cluster"), + ), + patch.object(k8s_config, "load_kube_config") as load_kube_config, + ): + factory.core_api() + + load_kube_config.assert_called_once() + factory.cleanup() + + def test_destroy_rejects_a_foreign_worker(self) -> None: + factory = KubernetesWorkerFactory(_principal()) + with pytest.raises(ValueError, match="Invalid worker type"): + factory.destroy_worker(MagicMock()) From da5f238ddf8b6ff9c577d9a1d9764bbbbd6d3477 Mon Sep 17 00:00:00 2001 From: Noppanat Wadlom Date: Tue, 8 Sep 2026 23:00:02 +0800 Subject: [PATCH 04/11] feat: render kubernetes manifests from stack environment values The manifests are ordinary YAML assets carrying compose-compatible ${VAR} substitution plus a few directives for what substitution alone cannot express: conditional inclusion, integers for fields the API rejects as strings, file contents for configuration compose bind-mounts, and the environment file's own values for what compose passes through env_file. One .env therefore drives both backends. Values handed to the cluster come from the environment file rather than the process environment, so nothing else in the operator's shell is copied into the namespace. Signed-off-by: Noppanat Wadlom --- cli/stack/pyproject.toml | 1 + .../assets/k8s/00-namespace.yaml | 6 + .../assets/k8s/10-rbac.yaml | 77 ++++ .../assets/k8s/20-redis.yaml | 201 ++++++++++ .../assets/k8s/30-server.yaml | 181 +++++++++ .../assets/k8s/worker_config.k8s.yaml | 26 ++ sdk/stack/src/flowmesh_stack/kubernetes.py | 154 ++++++++ sdk/stack/src/flowmesh_stack/manifests.py | 199 ++++++++++ tests/sdk/test_manifests.py | 349 ++++++++++++++++++ 9 files changed, 1194 insertions(+) create mode 100644 cli/stack/src/flowmesh_cli_stack/assets/k8s/00-namespace.yaml create mode 100644 cli/stack/src/flowmesh_cli_stack/assets/k8s/10-rbac.yaml create mode 100644 cli/stack/src/flowmesh_cli_stack/assets/k8s/20-redis.yaml create mode 100644 cli/stack/src/flowmesh_cli_stack/assets/k8s/30-server.yaml create mode 100644 cli/stack/src/flowmesh_cli_stack/assets/k8s/worker_config.k8s.yaml create mode 100644 sdk/stack/src/flowmesh_stack/kubernetes.py create mode 100644 sdk/stack/src/flowmesh_stack/manifests.py create mode 100644 tests/sdk/test_manifests.py diff --git a/cli/stack/pyproject.toml b/cli/stack/pyproject.toml index 66cebc91..988deaf1 100644 --- a/cli/stack/pyproject.toml +++ b/cli/stack/pyproject.toml @@ -28,4 +28,5 @@ where = ["src"] "flowmesh_cli_stack" = [ "assets/**", "assets/.env.example", + "assets/k8s/**", ] diff --git a/cli/stack/src/flowmesh_cli_stack/assets/k8s/00-namespace.yaml b/cli/stack/src/flowmesh_cli_stack/assets/k8s/00-namespace.yaml new file mode 100644 index 00000000..eadd4c80 --- /dev/null +++ b/cli/stack/src/flowmesh_cli_stack/assets/k8s/00-namespace.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: ${K8S_NAMESPACE:-flowmesh} + labels: + app.kubernetes.io/part-of: flowmesh diff --git a/cli/stack/src/flowmesh_cli_stack/assets/k8s/10-rbac.yaml b/cli/stack/src/flowmesh_cli_stack/assets/k8s/10-rbac.yaml new file mode 100644 index 00000000..e15649ac --- /dev/null +++ b/cli/stack/src/flowmesh_cli_stack/assets/k8s/10-rbac.yaml @@ -0,0 +1,77 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: flowmesh-server + namespace: ${K8S_NAMESPACE:-flowmesh} + labels: + app.kubernetes.io/part-of: flowmesh +--- +# The supervisor creates worker pods and the per-worker secret carrying their +# credentials, and reads pod logs for diagnostics. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: flowmesh-server + namespace: ${K8S_WORKER_NAMESPACE:-flowmesh} + labels: + app.kubernetes.io/part-of: flowmesh +rules: + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch", "create", "delete", "deletecollection"] + - apiGroups: [""] + resources: ["pods/log"] + verbs: ["get"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list", "create", "update", "delete", "deletecollection"] + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: flowmesh-server + namespace: ${K8S_WORKER_NAMESPACE:-flowmesh} + labels: + app.kubernetes.io/part-of: flowmesh +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: flowmesh-server +subjects: + - kind: ServiceAccount + name: flowmesh-server + namespace: ${K8S_NAMESPACE:-flowmesh} +--- +# Optional. Lets the supervisor report a worker's hardware before the worker +# starts, by reading node allocatable capacity and GPU labels. Without it the +# hardware preview is empty and everything else works unchanged. +x-flowmesh-when: K8S_ENABLE_NODE_RBAC +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: flowmesh-server-nodes-${K8S_NAMESPACE:-flowmesh} + labels: + app.kubernetes.io/part-of: flowmesh +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list"] +--- +x-flowmesh-when: K8S_ENABLE_NODE_RBAC +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: flowmesh-server-nodes-${K8S_NAMESPACE:-flowmesh} + labels: + app.kubernetes.io/part-of: flowmesh +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: flowmesh-server-nodes-${K8S_NAMESPACE:-flowmesh} +subjects: + - kind: ServiceAccount + name: flowmesh-server + namespace: ${K8S_NAMESPACE:-flowmesh} diff --git a/cli/stack/src/flowmesh_cli_stack/assets/k8s/20-redis.yaml b/cli/stack/src/flowmesh_cli_stack/assets/k8s/20-redis.yaml new file mode 100644 index 00000000..bd6db7b0 --- /dev/null +++ b/cli/stack/src/flowmesh_cli_stack/assets/k8s/20-redis.yaml @@ -0,0 +1,201 @@ +# Redis runs on root nodes only. A worker node connects to the root node's +# Redis through REDIS_CONTROL_URL / REDIS_TELEMETRY_URL. +x-flowmesh-when: NODE_ROLE==root +apiVersion: v1 +kind: Secret +metadata: + name: flowmesh-redis-acl + namespace: ${K8S_NAMESPACE:-flowmesh} + labels: + app.kubernetes.io/part-of: flowmesh +type: Opaque +stringData: + users.acl: | + user default off + user ${REDIS_USERNAME:-admin} on >${REDIS_PASSWORD:-} ~* &* +@all +--- +x-flowmesh-when: NODE_ROLE==root +apiVersion: v1 +kind: Service +metadata: + name: flowmesh-redis-control + namespace: ${K8S_NAMESPACE:-flowmesh} + labels: + app.kubernetes.io/part-of: flowmesh + app.kubernetes.io/component: redis-control +spec: + clusterIP: None + selector: + app.kubernetes.io/part-of: flowmesh + app.kubernetes.io/component: redis-control + ports: + - name: redis + port: 6379 + targetPort: 6379 +--- +x-flowmesh-when: NODE_ROLE==root +apiVersion: v1 +kind: Service +metadata: + name: flowmesh-redis-telemetry + namespace: ${K8S_NAMESPACE:-flowmesh} + labels: + app.kubernetes.io/part-of: flowmesh + app.kubernetes.io/component: redis-telemetry +spec: + clusterIP: None + selector: + app.kubernetes.io/part-of: flowmesh + app.kubernetes.io/component: redis-telemetry + ports: + - name: redis + port: 6379 + targetPort: 6379 +--- +x-flowmesh-when: NODE_ROLE==root +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: flowmesh-redis-control + namespace: ${K8S_NAMESPACE:-flowmesh} + labels: + app.kubernetes.io/part-of: flowmesh + app.kubernetes.io/component: redis-control +spec: + serviceName: flowmesh-redis-control + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/part-of: flowmesh + app.kubernetes.io/component: redis-control + template: + metadata: + labels: + app.kubernetes.io/part-of: flowmesh + app.kubernetes.io/component: redis-control + spec: + containers: + - name: redis + image: redis:7-alpine + args: + - redis-server + - --save + - "60 1" + - --loglevel + - warning + # Control-plane pubsub clients are dropped under load without a + # raised output buffer limit. + - --client-output-buffer-limit + - pubsub 1gb 512mb 60 + - x-flowmesh-when: REDIS_ACL_ENABLED + x-flowmesh-value: --aclfile + - x-flowmesh-when: REDIS_ACL_ENABLED + x-flowmesh-value: /etc/redis/acl/users.acl + ports: + - name: redis + containerPort: 6379 + volumeMounts: + - name: data + mountPath: /data + - x-flowmesh-when: REDIS_ACL_ENABLED + name: acl + mountPath: /etc/redis/acl + readOnly: true + readinessProbe: + exec: + command: ["redis-cli", "-p", "6379", "ping"] + initialDelaySeconds: 5 + periodSeconds: 10 + livenessProbe: + exec: + command: ["redis-cli", "-p", "6379", "ping"] + initialDelaySeconds: 15 + periodSeconds: 20 + volumes: + - x-flowmesh-when: REDIS_ACL_ENABLED + name: acl + secret: + secretName: flowmesh-redis-acl + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: ["ReadWriteOnce"] + storageClassName: + x-flowmesh-when: K8S_STORAGE_CLASS + x-flowmesh-value: ${K8S_STORAGE_CLASS:-} + resources: + requests: + storage: ${K8S_REDIS_STORAGE_SIZE:-8Gi} +--- +x-flowmesh-when: NODE_ROLE==root +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: flowmesh-redis-telemetry + namespace: ${K8S_NAMESPACE:-flowmesh} + labels: + app.kubernetes.io/part-of: flowmesh + app.kubernetes.io/component: redis-telemetry +spec: + serviceName: flowmesh-redis-telemetry + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/part-of: flowmesh + app.kubernetes.io/component: redis-telemetry + template: + metadata: + labels: + app.kubernetes.io/part-of: flowmesh + app.kubernetes.io/component: redis-telemetry + spec: + containers: + - name: redis + image: redis:7-alpine + args: + - redis-server + - --save + - "300 1" + - --loglevel + - warning + - x-flowmesh-when: REDIS_ACL_ENABLED + x-flowmesh-value: --aclfile + - x-flowmesh-when: REDIS_ACL_ENABLED + x-flowmesh-value: /etc/redis/acl/users.acl + ports: + - name: redis + containerPort: 6379 + volumeMounts: + - name: data + mountPath: /data + - x-flowmesh-when: REDIS_ACL_ENABLED + name: acl + mountPath: /etc/redis/acl + readOnly: true + readinessProbe: + exec: + command: ["redis-cli", "-p", "6379", "ping"] + initialDelaySeconds: 5 + periodSeconds: 10 + livenessProbe: + exec: + command: ["redis-cli", "-p", "6379", "ping"] + initialDelaySeconds: 15 + periodSeconds: 20 + volumes: + - x-flowmesh-when: REDIS_ACL_ENABLED + name: acl + secret: + secretName: flowmesh-redis-acl + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: ["ReadWriteOnce"] + storageClassName: + x-flowmesh-when: K8S_STORAGE_CLASS + x-flowmesh-value: ${K8S_STORAGE_CLASS:-} + resources: + requests: + storage: ${K8S_REDIS_STORAGE_SIZE:-8Gi} diff --git a/cli/stack/src/flowmesh_cli_stack/assets/k8s/30-server.yaml b/cli/stack/src/flowmesh_cli_stack/assets/k8s/30-server.yaml new file mode 100644 index 00000000..84b3cf58 --- /dev/null +++ b/cli/stack/src/flowmesh_cli_stack/assets/k8s/30-server.yaml @@ -0,0 +1,181 @@ +apiVersion: v1 +kind: Secret +metadata: + name: flowmesh-server-env + namespace: ${K8S_NAMESPACE:-flowmesh} + labels: + app.kubernetes.io/part-of: flowmesh +type: Opaque +stringData: + x-flowmesh-env-values: true +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: flowmesh-worker-config + namespace: ${K8S_NAMESPACE:-flowmesh} + labels: + app.kubernetes.io/part-of: flowmesh +data: + worker_config.yaml: + x-flowmesh-file: SERVER_WORKER_CONFIG +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: flowmesh-results + namespace: ${K8S_NAMESPACE:-flowmesh} + labels: + app.kubernetes.io/part-of: flowmesh +spec: + accessModes: + - ${K8S_RESULTS_ACCESS_MODE:-ReadWriteOnce} + storageClassName: + x-flowmesh-when: K8S_STORAGE_CLASS + x-flowmesh-value: ${K8S_STORAGE_CLASS:-} + resources: + requests: + storage: ${K8S_RESULTS_STORAGE_SIZE:-20Gi} +--- +apiVersion: v1 +kind: Service +metadata: + name: flowmesh-server + namespace: ${K8S_NAMESPACE:-flowmesh} + labels: + app.kubernetes.io/part-of: flowmesh + app.kubernetes.io/component: server +spec: + type: ${K8S_SERVER_SERVICE_TYPE:-ClusterIP} + selector: + app.kubernetes.io/part-of: flowmesh + app.kubernetes.io/component: server + ports: + - name: http + port: + x-flowmesh-int: ${SERVER_HTTP_PORT:-8000} + targetPort: http +--- +# Workers dial this name. A gRPC TLS certificate must carry it as a SAN. +apiVersion: v1 +kind: Service +metadata: + name: ${K8S_SUPERVISOR_SERVICE:-flowmesh-supervisor} + namespace: ${K8S_NAMESPACE:-flowmesh} + labels: + app.kubernetes.io/part-of: flowmesh + app.kubernetes.io/component: supervisor +spec: + clusterIP: None + selector: + app.kubernetes.io/part-of: flowmesh + app.kubernetes.io/component: server + ports: + - name: grpc + port: + x-flowmesh-int: ${SERVER_GRPC_PORT:-50051} + targetPort: grpc +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flowmesh-server + namespace: ${K8S_NAMESPACE:-flowmesh} + labels: + app.kubernetes.io/part-of: flowmesh + app.kubernetes.io/component: server +spec: + # A worker's token lives in the registry of the supervisor that minted it, so + # a second replica behind the Service would reject those registrations. + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app.kubernetes.io/part-of: flowmesh + app.kubernetes.io/component: server + template: + metadata: + labels: + app.kubernetes.io/part-of: flowmesh + app.kubernetes.io/component: server + spec: + serviceAccountName: flowmesh-server + containers: + - name: server + image: ${FLOWMESH_REGISTRY:-ghcr.io/mlsys-io}/flowmesh_server:${FLOWMESH_VERSION:-dev} + imagePullPolicy: ${K8S_IMAGE_PULL_POLICY:-IfNotPresent} + envFrom: + - secretRef: + name: flowmesh-server-env + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: SERVER_APP_PORT + value: "${SERVER_HTTP_PORT:-8000}" + - name: WORKER_CONFIG_PATH + value: /etc/flowmesh/worker_config.yaml + - name: RESULTS_DIR + value: /mnt/flowmesh-results + - name: SERVER_METRICS_DIR + value: /mnt/flowmesh-metrics + - name: LOG_FILE + value: /var/log/flowmesh-server/server.log + ports: + - name: http + containerPort: + x-flowmesh-int: ${SERVER_HTTP_PORT:-8000} + - name: grpc + containerPort: + x-flowmesh-int: ${SERVER_GRPC_PORT:-50051} + volumeMounts: + - name: worker-config + mountPath: /etc/flowmesh + readOnly: true + - name: results + mountPath: /mnt/flowmesh-results + - name: metrics + mountPath: /mnt/flowmesh-metrics + - name: logs + mountPath: /var/log/flowmesh-server + - x-flowmesh-when: SERVER_GRPC_TLS_SECRET + name: server-tls + mountPath: /etc/ssl/server + readOnly: true + - x-flowmesh-when: REDIS_TLS_SECRET + name: redis-tls + mountPath: /etc/ssl/redis + readOnly: true + readinessProbe: + httpGet: + path: /healthz + port: http + initialDelaySeconds: 10 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /healthz + port: http + initialDelaySeconds: 40 + periodSeconds: 30 + volumes: + - name: worker-config + configMap: + name: flowmesh-worker-config + - name: results + persistentVolumeClaim: + claimName: flowmesh-results + - name: metrics + emptyDir: {} + - name: logs + emptyDir: {} + - x-flowmesh-when: SERVER_GRPC_TLS_SECRET + name: server-tls + secret: + secretName: ${SERVER_GRPC_TLS_SECRET:-} + - x-flowmesh-when: REDIS_TLS_SECRET + name: redis-tls + secret: + secretName: ${REDIS_TLS_SECRET:-} diff --git a/cli/stack/src/flowmesh_cli_stack/assets/k8s/worker_config.k8s.yaml b/cli/stack/src/flowmesh_cli_stack/assets/k8s/worker_config.k8s.yaml new file mode 100644 index 00000000..4de64684 --- /dev/null +++ b/cli/stack/src/flowmesh_cli_stack/assets/k8s/worker_config.k8s.yaml @@ -0,0 +1,26 @@ +# Worker configuration for the Kubernetes backend. Point SERVER_WORKER_CONFIG +# at a copy of this file and adjust the pools to match your cluster. +default_worker_config: + image_pull_policy: IfNotPresent + +workers: + - provider: kubernetes + init_on_start: true + worker_config: + worker_type: cpu + cpu_request: "2" + memory_request: 8Gi + + - provider: kubernetes + init_on_start: false + worker_config: + worker_type: gpu + gpu_count: 1 + # Torch dataloaders need more than the 64Mi default /dev/shm. + shm_size: 8Gi + node_selector: + nvidia.com/gpu.present: "true" + tolerations: + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule diff --git a/sdk/stack/src/flowmesh_stack/kubernetes.py b/sdk/stack/src/flowmesh_stack/kubernetes.py new file mode 100644 index 00000000..5107186d --- /dev/null +++ b/sdk/stack/src/flowmesh_stack/kubernetes.py @@ -0,0 +1,154 @@ +"""kubectl helpers for the FlowMesh Kubernetes stack backend.""" + +import os +import shutil +import subprocess +from collections.abc import Callable, Mapping +from dataclasses import dataclass, field +from pathlib import Path + +from .env import parse_env_file +from .manifests import render_manifests + +MANAGED_LABEL = "flowmesh.io/managed" +"""Label carried by worker pods the supervisor created.""" + +STACK_LABEL = "app.kubernetes.io/part-of" +"""Label carried by every stack resource.""" + +STACK_LABEL_VALUE = "flowmesh" + + +class KubectlError(RuntimeError): + """Raised when a kubectl command fails.""" + + +def ensure_kubectl_available() -> str: + """Return the absolute kubectl path, raising when it is not installed.""" + path = shutil.which("kubectl") + if path is None: + raise KubectlError("kubectl is required but was not found in PATH") + return path + + +def kubectl( + args: list[str], + namespace: str | None = None, + context: str | None = None, + kubeconfig: str | None = None, + stdin: str | None = None, + env: Mapping[str, str] | None = None, + capture_output: bool = False, +) -> subprocess.CompletedProcess[str]: + """Run kubectl with the provided arguments.""" + cmd = [ensure_kubectl_available()] + if kubeconfig: + cmd += ["--kubeconfig", kubeconfig] + if context: + cmd += ["--context", context] + if namespace: + cmd += ["--namespace", namespace] + cmd += args + + merged_env = dict(os.environ) + if env: + merged_env.update(env) + + return ( + subprocess.run( # nosec B603: argv list, no shell, absolute path via which(). + cmd, + check=False, + input=stdin, + capture_output=capture_output, + text=True, + env=merged_env, + ) + ) + + +@dataclass +class KubernetesStack: + """Applies and inspects the FlowMesh stack in a Kubernetes namespace.""" + + manifests: list[Path] + """Manifest assets rendered for every apply.""" + namespace: str + """Namespace the stack is deployed into.""" + load_env: Callable[[Path], None] + """Callback that loads and resolves env-file values before operations run.""" + context: str | None = None + """kubectl context to target.""" + kubeconfig: str | None = None + """kubeconfig file to use instead of the default.""" + rollout_targets: list[str] = field(default_factory=list) + """Workloads that ``up`` waits on before reporting success.""" + + def render(self, env_file: Path) -> str: + """Render the stack manifests using the resolved environment. + + Substitution reads the resolved process environment; the values handed + to the cluster come from the environment file alone, so nothing else in + the operator's shell is copied into the namespace. + """ + self.load_env(env_file) + return render_manifests( + self.manifests, dict(os.environ), parse_env_file(env_file) + ) + + def _run( + self, args: list[str], stdin: str | None = None, capture_output: bool = False + ) -> subprocess.CompletedProcess[str]: + return kubectl( + args, + namespace=self.namespace, + context=self.context, + kubeconfig=self.kubeconfig, + stdin=stdin, + capture_output=capture_output, + ) + + def apply(self, env_file: Path) -> subprocess.CompletedProcess[str]: + """Apply the rendered stack manifests.""" + return self._run(["apply", "-f", "-"], stdin=self.render(env_file)) + + def delete( + self, env_file: Path, ignore_not_found: bool = True + ) -> subprocess.CompletedProcess[str]: + """Delete the resources described by the rendered stack manifests.""" + args = ["delete", "-f", "-"] + if ignore_not_found: + args.append("--ignore-not-found") + return self._run(args, stdin=self.render(env_file)) + + def rollout_status(self, target: str, timeout: str) -> subprocess.CompletedProcess: + """Wait for a workload to finish rolling out.""" + return self._run(["rollout", "status", target, f"--timeout={timeout}"]) + + def rollout_restart(self, target: str) -> subprocess.CompletedProcess[str]: + """Restart a workload in place.""" + return self._run(["rollout", "restart", target]) + + def logs(self, target: str, follow: bool = True) -> subprocess.CompletedProcess: + """Stream logs for a workload or pod.""" + args = ["logs", target, "--all-containers"] + if follow: + args.append("-f") + return self._run(args) + + def status(self) -> subprocess.CompletedProcess[str]: + """Show stack pods.""" + return self._run( + ["get", "pods", "-l", f"{STACK_LABEL}={STACK_LABEL_VALUE}", "-o", "wide"] + ) + + def worker_status(self) -> subprocess.CompletedProcess[str]: + """Show pods the supervisor created for workers.""" + return self._run(["get", "pods", "-l", f"{MANAGED_LABEL}=true", "-o", "wide"]) + + def delete_workers(self) -> subprocess.CompletedProcess[str]: + """Delete every supervisor-managed worker pod in the namespace.""" + return self._run(["delete", "pods", "-l", f"{MANAGED_LABEL}=true"]) + + def delete_volumes(self) -> subprocess.CompletedProcess[str]: + """Delete the stack's persistent volume claims.""" + return self._run(["delete", "pvc", "-l", f"{STACK_LABEL}={STACK_LABEL_VALUE}"]) diff --git a/sdk/stack/src/flowmesh_stack/manifests.py b/sdk/stack/src/flowmesh_stack/manifests.py new file mode 100644 index 00000000..5dcb772b --- /dev/null +++ b/sdk/stack/src/flowmesh_stack/manifests.py @@ -0,0 +1,199 @@ +"""Rendering of Kubernetes manifest assets from stack environment values. + +Assets are ordinary multi-document YAML carrying a small set of extensions: + +``${VAR}`` / ``${VAR:-default}`` + Compose-compatible substitution, so one ``.env`` drives both stack + backends. A reference with neither a value nor a default is an error, so a + misconfigured stack fails while rendering rather than while applying. + +``x-flowmesh-when: `` + Conditional inclusion. The mapping carrying the key is dropped when the + expression is falsey. A dropped document leaves the stream, a dropped + sequence entry leaves its sequence, and a dropped mapping value takes its + key with it. Expressions are ``VAR``, ``!VAR``, or ``VAR==value``. + +``x-flowmesh-value: `` + Renders to the node itself, so ``x-flowmesh-when`` can make a single scalar + conditional inside a list of arguments. + +``x-flowmesh-int: `` + Renders to an integer. Ports and replica counts are rejected by the API as + strings, and substitution otherwise always yields a string. + +``x-flowmesh-file: `` + Renders to the contents of the file named by the environment variable + ``VAR``, for configuration the compose backend bind-mounts. + +``x-flowmesh-env-values`` + The mapping carrying the key is replaced by the environment file's own + key/value pairs, which is how the server receives the configuration compose + passes through ``env_file``. Values come from the environment file alone, + never from the caller's process environment. +""" + +import re +from collections.abc import Mapping +from pathlib import Path +from typing import Any + +import yaml + +WHEN_KEY = "x-flowmesh-when" +VALUE_KEY = "x-flowmesh-value" +INT_KEY = "x-flowmesh-int" +FILE_KEY = "x-flowmesh-file" +ENV_VALUES_KEY = "x-flowmesh-env-values" + +_VAR_PATTERN = re.compile( + r""" + \$\{ + (?P[A-Za-z_][A-Za-z0-9_]*) + (?::-(?P[^}]*))? + \} + """, + re.VERBOSE, +) +_FALSEY = {"", "0", "false", "no", "off"} + +_DROP = object() +"""Sentinel marking a node whose condition evaluated false.""" + + +class ManifestError(RuntimeError): + """Raised when a manifest cannot be rendered.""" + + +def substitute(value: str, env: Mapping[str, str]) -> str: + """Expand ``${VAR}`` and ``${VAR:-default}`` references in ``value``.""" + + def replace(match: re.Match[str]) -> str: + name = match.group("name") + default = match.group("default") + if name in env: + return env[name] + if default is not None: + return default + raise ManifestError( + f"{name} is not set and has no default in the manifest reference " + f"{match.group(0)}" + ) + + return _VAR_PATTERN.sub(replace, value) + + +def evaluate_condition(expression: str, env: Mapping[str, str]) -> bool: + """Evaluate an ``x-flowmesh-when`` expression against ``env``.""" + expr = expression.strip() + if not expr: + raise ManifestError("Empty x-flowmesh-when expression") + + if expr.startswith("!"): + return not evaluate_condition(expr[1:], env) + + if "==" in expr: + name, _, expected = expr.partition("==") + return env.get(name.strip(), "").strip() == expected.strip() + + return env.get(expr, "").strip().lower() not in _FALSEY + + +def _render_int(node: Any, env: Mapping[str, str]) -> int: + rendered = substitute(str(node), env).strip() + try: + return int(rendered) + except ValueError: + raise ManifestError( + f"{INT_KEY} expected an integer but {node!r} rendered to {rendered!r}" + ) from None + + +def _render_file(node: Any, env: Mapping[str, str]) -> str: + name = substitute(str(node), env).strip() + path_value = env.get(name, "").strip() + if not path_value: + raise ManifestError(f"{FILE_KEY} requires {name} to name a readable file") + try: + return Path(path_value).read_text(encoding="utf-8") + except OSError as exc: + raise ManifestError(f"Failed to read {name} file {path_value}: {exc}") from exc + + +def _render_node( + node: Any, env: Mapping[str, str], env_values: Mapping[str, str] +) -> Any: + if isinstance(node, dict): + condition = node.get(WHEN_KEY) + if condition is not None and not evaluate_condition( + substitute(str(condition), env), env + ): + return _DROP + if VALUE_KEY in node: + return _render_node(node[VALUE_KEY], env, env_values) + if INT_KEY in node: + return _render_int(node[INT_KEY], env) + if FILE_KEY in node: + return _render_file(node[FILE_KEY], env) + if ENV_VALUES_KEY in node: + return dict(env_values) + + rendered: dict[str, Any] = {} + for key, value in node.items(): + if key == WHEN_KEY: + continue + child = _render_node(value, env, env_values) + if child is _DROP: + continue + rendered[key] = child + return rendered + + if isinstance(node, list): + items = [_render_node(item, env, env_values) for item in node] + return [item for item in items if item is not _DROP] + + if isinstance(node, str): + return substitute(node, env) + + return node + + +def render_documents( + source: str, + env: Mapping[str, str], + env_values: Mapping[str, str] | None = None, +) -> list[dict[str, Any]]: + """Render a multi-document manifest string into resource dictionaries. + + Document order is preserved because ``kubectl apply`` processes a stream in + order and later resources depend on earlier ones. + """ + documents: list[dict[str, Any]] = [] + for raw in yaml.safe_load_all(source): + if raw is None: + continue + rendered = _render_node(raw, env, env_values or {}) + if rendered is _DROP or not rendered: + continue + documents.append(rendered) + return documents + + +def render_manifests( + paths: list[Path], + env: Mapping[str, str], + env_values: Mapping[str, str] | None = None, +) -> str: + """Render manifest assets into a single YAML stream.""" + documents: list[dict[str, Any]] = [] + for path in paths: + try: + source = path.read_text(encoding="utf-8") + except OSError as exc: + raise ManifestError(f"Failed to read manifest {path}: {exc}") from exc + try: + documents.extend(render_documents(source, env, env_values)) + except yaml.YAMLError as exc: + raise ManifestError(f"Failed to parse manifest {path}: {exc}") from exc + if not documents: + raise ManifestError("No manifests to apply") + return yaml.safe_dump_all(documents, sort_keys=False, default_flow_style=False) diff --git a/tests/sdk/test_manifests.py b/tests/sdk/test_manifests.py new file mode 100644 index 00000000..1a0863f7 --- /dev/null +++ b/tests/sdk/test_manifests.py @@ -0,0 +1,349 @@ +"""Kubernetes manifest rendering: substitution, directives, and shipped assets.""" + +from pathlib import Path + +import pytest +import yaml +from flowmesh_cli.core.assets import asset_path +from flowmesh_stack.manifests import ( + ManifestError, + evaluate_condition, + render_documents, + render_manifests, + substitute, +) + +MANIFEST_ASSETS = ( + "00-namespace.yaml", + "10-rbac.yaml", + "20-redis.yaml", + "30-server.yaml", +) + + +def _asset_paths() -> list[Path]: + return [ + asset_path("flowmesh_cli_stack.assets", "k8s", name) for name in MANIFEST_ASSETS + ] + + +def _base_env(**overrides: str) -> dict[str, str]: + env = { + "K8S_NAMESPACE": "flowmesh", + "K8S_WORKER_NAMESPACE": "flowmesh", + "NODE_ROLE": "root", + } + env.update(overrides) + return env + + +def _by_kind(documents: list[dict]) -> dict[tuple[str, str], dict]: + return {(doc["kind"], doc["metadata"]["name"]): doc for doc in documents} + + +# ------------------------------------------------------------------ # +# Substitution +# ------------------------------------------------------------------ # + + +class TestSubstitution: + def test_value_is_expanded(self) -> None: + assert substitute("ns/${NAME}", {"NAME": "flowmesh"}) == "ns/flowmesh" + + def test_default_is_used_when_unset(self) -> None: + assert substitute("${NAME:-fallback}", {}) == "fallback" + + def test_set_value_beats_the_default(self) -> None: + assert substitute("${NAME:-fallback}", {"NAME": "real"}) == "real" + + def test_empty_default_renders_empty(self) -> None: + assert substitute("${NAME:-}", {}) == "" + + def test_multiple_references_in_one_value(self) -> None: + rendered = substitute("${A}:${B}", {"A": "host", "B": "8000"}) + assert rendered == "host:8000" + + def test_missing_value_without_a_default_is_an_error(self) -> None: + with pytest.raises(ManifestError, match="NAME is not set"): + substitute("${NAME}", {}) + + +# ------------------------------------------------------------------ # +# Conditions +# ------------------------------------------------------------------ # + + +class TestConditions: + @pytest.mark.parametrize("value", ["1", "true", "TRUE", "yes", "on"]) + def test_truthy_values(self, value: str) -> None: + assert evaluate_condition("FLAG", {"FLAG": value}) is True + + @pytest.mark.parametrize("value", ["", "0", "false", "no", "off"]) + def test_falsey_values(self, value: str) -> None: + assert evaluate_condition("FLAG", {"FLAG": value}) is False + + def test_unset_is_falsey(self) -> None: + assert evaluate_condition("FLAG", {}) is False + + def test_negation(self) -> None: + assert evaluate_condition("!FLAG", {}) is True + assert evaluate_condition("!FLAG", {"FLAG": "1"}) is False + + def test_equality(self) -> None: + assert evaluate_condition("ROLE==root", {"ROLE": "root"}) is True + assert evaluate_condition("ROLE==root", {"ROLE": "worker"}) is False + + def test_empty_expression_is_an_error(self) -> None: + with pytest.raises(ManifestError, match="Empty"): + evaluate_condition(" ", {}) + + +# ------------------------------------------------------------------ # +# Directives +# ------------------------------------------------------------------ # + + +class TestDirectives: + def test_falsey_document_leaves_the_stream(self) -> None: + source = """ + x-flowmesh-when: ENABLED + kind: ConfigMap + """ + assert render_documents(source, {}) == [] + + def test_truthy_document_drops_only_the_marker(self) -> None: + source = """ + x-flowmesh-when: ENABLED + kind: ConfigMap + """ + assert render_documents(source, {"ENABLED": "1"}) == [{"kind": "ConfigMap"}] + + def test_falsey_sequence_entry_leaves_the_sequence(self) -> None: + source = """ + kind: Pod + items: + - name: keep + - x-flowmesh-when: TLS + name: drop + """ + document = render_documents(source, {})[0] + assert document["items"] == [{"name": "keep"}] + + def test_falsey_mapping_value_takes_its_key(self) -> None: + source = """ + kind: Pod + spec: + storageClassName: + x-flowmesh-when: CLASS + x-flowmesh-value: ${CLASS:-} + """ + document = render_documents(source, {})[0] + assert "storageClassName" not in document["spec"] + + def test_conditional_scalar_renders_bare(self) -> None: + source = """ + kind: Pod + args: + - x-flowmesh-when: ACL + x-flowmesh-value: --aclfile + """ + document = render_documents(source, {"ACL": "1"})[0] + assert document["args"] == ["--aclfile"] + + def test_int_directive_yields_an_integer(self) -> None: + source = """ + kind: Service + port: + x-flowmesh-int: ${PORT:-8000} + """ + document = render_documents(source, {})[0] + assert document["port"] == 8000 + assert isinstance(document["port"], int) + + def test_int_directive_rejects_non_numeric_values(self) -> None: + source = """ + kind: Service + port: + x-flowmesh-int: ${PORT:-http} + """ + with pytest.raises(ManifestError, match="expected an integer"): + render_documents(source, {}) + + def test_file_directive_inlines_contents(self, tmp_path: Path) -> None: + config = tmp_path / "worker_config.yaml" + config.write_text("workers: []\n") + source = """ + kind: ConfigMap + data: + worker_config.yaml: + x-flowmesh-file: SERVER_WORKER_CONFIG + """ + document = render_documents(source, {"SERVER_WORKER_CONFIG": str(config)})[0] + assert document["data"]["worker_config.yaml"] == "workers: []\n" + + def test_file_directive_requires_the_variable(self) -> None: + source = """ + kind: ConfigMap + data: + x-flowmesh-file: SERVER_WORKER_CONFIG + """ + with pytest.raises(ManifestError, match="readable file"): + render_documents(source, {}) + + def test_env_values_directive_uses_the_env_file_only(self) -> None: + source = """ + kind: Secret + stringData: + x-flowmesh-env-values: true + """ + document = render_documents( + source, {"PROCESS_ONLY": "leaked"}, {"FROM_FILE": "kept"} + )[0] + assert document["stringData"] == {"FROM_FILE": "kept"} + + +# ------------------------------------------------------------------ # +# Rendering assets +# ------------------------------------------------------------------ # + + +class TestRenderManifests: + def test_document_order_is_preserved(self, tmp_path: Path) -> None: + first = tmp_path / "a.yaml" + first.write_text("kind: Namespace\n---\nkind: Role\n") + second = tmp_path / "b.yaml" + second.write_text("kind: Deployment\n") + + documents = list(yaml.safe_load_all(render_manifests([first, second], {}))) + assert [doc["kind"] for doc in documents] == [ + "Namespace", + "Role", + "Deployment", + ] + + def test_empty_output_is_an_error(self, tmp_path: Path) -> None: + empty = tmp_path / "empty.yaml" + empty.write_text("x-flowmesh-when: NEVER\nkind: ConfigMap\n") + with pytest.raises(ManifestError, match="No manifests"): + render_manifests([empty], {}) + + def test_unreadable_asset_is_an_error(self, tmp_path: Path) -> None: + with pytest.raises(ManifestError, match="Failed to read"): + render_manifests([tmp_path / "missing.yaml"], {}) + + +# ------------------------------------------------------------------ # +# Shipped assets +# ------------------------------------------------------------------ # + + +class TestShippedAssets: + def _documents(self, **overrides: str) -> list[dict]: + env = _base_env(**overrides) + env.setdefault( + "SERVER_WORKER_CONFIG", + asset_path( + "flowmesh_cli_stack.assets", "k8s", "worker_config.k8s.yaml" + ).as_posix(), + ) + return list(yaml.safe_load_all(render_manifests(_asset_paths(), env, env))) + + def test_namespace_is_applied_first(self) -> None: + assert self._documents()[0]["kind"] == "Namespace" + + def test_root_node_ships_both_redis_workloads(self) -> None: + names = { + doc["metadata"]["name"] + for doc in self._documents() + if doc["kind"] == "StatefulSet" + } + assert names == {"flowmesh-redis-control", "flowmesh-redis-telemetry"} + + def test_worker_node_ships_no_redis(self) -> None: + documents = self._documents(NODE_ROLE="worker") + assert not [doc for doc in documents if doc["kind"] == "StatefulSet"] + assert [doc for doc in documents if doc["kind"] == "Deployment"] + + def test_server_runs_a_single_replica(self) -> None: + deployment = _by_kind(self._documents())[("Deployment", "flowmesh-server")] + assert deployment["spec"]["replicas"] == 1 + assert deployment["spec"]["strategy"]["type"] == "Recreate" + + def test_node_rbac_is_opt_in(self) -> None: + kinds = {doc["kind"] for doc in self._documents()} + assert "ClusterRole" not in kinds + + enabled = {doc["kind"] for doc in self._documents(K8S_ENABLE_NODE_RBAC="true")} + assert {"ClusterRole", "ClusterRoleBinding"} <= enabled + + def test_service_ports_are_integers(self) -> None: + service = _by_kind(self._documents())[("Service", "flowmesh-server")] + assert isinstance(service["spec"]["ports"][0]["port"], int) + + def test_storage_class_is_omitted_when_unset(self) -> None: + claim = _by_kind(self._documents())[ + ("PersistentVolumeClaim", "flowmesh-results") + ] + assert "storageClassName" not in claim["spec"] + + def test_storage_class_is_set_when_configured(self) -> None: + claim = _by_kind(self._documents(K8S_STORAGE_CLASS="fast"))[ + ("PersistentVolumeClaim", "flowmesh-results") + ] + assert claim["spec"]["storageClassName"] == "fast" + + def test_redis_keeps_the_pubsub_buffer_limit(self) -> None: + statefulset = _by_kind(self._documents())[ + ("StatefulSet", "flowmesh-redis-control") + ] + args = statefulset["spec"]["template"]["spec"]["containers"][0]["args"] + assert "--client-output-buffer-limit" in args + + def test_redis_acl_is_wired_only_when_enabled(self) -> None: + without = _by_kind(self._documents())[("StatefulSet", "flowmesh-redis-control")] + assert ( + "--aclfile" + not in without["spec"]["template"]["spec"]["containers"][0]["args"] + ) + + with_acl = _by_kind(self._documents(REDIS_ACL_ENABLED="true"))[ + ("StatefulSet", "flowmesh-redis-control") + ] + spec = with_acl["spec"]["template"]["spec"] + assert "--aclfile" in spec["containers"][0]["args"] + assert [volume["name"] for volume in spec["volumes"]] == ["acl"] + + def test_tls_secrets_are_mounted_only_when_named(self) -> None: + deployment = _by_kind(self._documents(SERVER_GRPC_TLS_SECRET="fm-tls"))[ + ("Deployment", "flowmesh-server") + ] + volumes = {v["name"] for v in deployment["spec"]["template"]["spec"]["volumes"]} + assert "server-tls" in volumes + + default = _by_kind(self._documents())[("Deployment", "flowmesh-server")] + default_volumes = { + v["name"] for v in default["spec"]["template"]["spec"]["volumes"] + } + assert "server-tls" not in default_volumes + + def test_server_receives_the_namespace_from_the_downward_api(self) -> None: + deployment = _by_kind(self._documents())[("Deployment", "flowmesh-server")] + container = deployment["spec"]["template"]["spec"]["containers"][0] + namespace_env = next( + entry for entry in container["env"] if entry["name"] == "POD_NAMESPACE" + ) + assert ( + namespace_env["valueFrom"]["fieldRef"]["fieldPath"] == "metadata.namespace" + ) + + def test_worker_config_is_carried_in_a_config_map(self) -> None: + config_map = _by_kind(self._documents())[ + ("ConfigMap", "flowmesh-worker-config") + ] + assert "provider: kubernetes" in config_map["data"]["worker_config.yaml"] + + def test_env_file_values_reach_the_server_secret(self) -> None: + secret = _by_kind(self._documents(FLOWMESH_API_KEY="secret-key"))[ + ("Secret", "flowmesh-server-env") + ] + assert secret["stringData"]["FLOWMESH_API_KEY"] == "secret-key" From 61b2c96022ef96fe5a1061735753aed4289f204d Mon Sep 17 00:00:00 2001 From: Noppanat Wadlom Date: Tue, 8 Sep 2026 23:00:35 +0800 Subject: [PATCH 05/11] feat: add a kubernetes backend to the stack commands `flowmesh stack up/down/restart/logs/ps/clean --backend k8s` renders the manifests and drives them with kubectl, while the compose path is unchanged when the backend is compose. The backend also reads from STACK_BACKEND, so an environment file selects it once. Restarting with an image tag applies rather than rolls, because a rollout restart leaves the pod template's image as it was and the tag would silently have no effect. The worker drain moves into utils so both backends run it before the supervisor goes away. Signed-off-by: Noppanat Wadlom --- .../flowmesh_cli_stack/assets/.env.example | 32 ++ .../src/flowmesh_cli_stack/env_schema.py | 68 +++++ cli/stack/src/flowmesh_cli_stack/k8s.py | 224 ++++++++++++++ cli/stack/src/flowmesh_cli_stack/stack.py | 68 ++++- cli/stack/src/flowmesh_cli_stack/utils.py | 9 + tests/cli/test_stack_k8s.py | 288 ++++++++++++++++++ tests/cli/test_stack_restart.py | 21 +- 7 files changed, 687 insertions(+), 23 deletions(-) create mode 100644 cli/stack/src/flowmesh_cli_stack/k8s.py create mode 100644 tests/cli/test_stack_k8s.py diff --git a/cli/stack/src/flowmesh_cli_stack/assets/.env.example b/cli/stack/src/flowmesh_cli_stack/assets/.env.example index 9564abf0..ec0f1a11 100644 --- a/cli/stack/src/flowmesh_cli_stack/assets/.env.example +++ b/cli/stack/src/flowmesh_cli_stack/assets/.env.example @@ -33,6 +33,38 @@ SERVER_GRPC_TLS_CA_FILE=/etc/ssl/server/server-ca.pem SERVER_GRPC_TLS_CERT_FILE=/etc/ssl/server/server.pem SERVER_GRPC_TLS_KEY_FILE=/etc/ssl/server/server.key +# ==== Kubernetes Backend ==== +# Used when STACK_BACKEND=k8s. The stack is deployed into one +# namespace; the Kubernetes scheduler places workers across nodes. +# A gRPC TLS certificate must carry the supervisor Service name +# as a SAN, since workers dial the Service, not a host. +STACK_BACKEND=compose +K8S_NAMESPACE=flowmesh +# Namespace for worker pods; defaults to K8S_NAMESPACE. +K8S_WORKER_NAMESPACE= +# kubectl context; empty uses current. +K8S_CONTEXT= +# kubeconfig path; empty uses default. +K8S_KUBECONFIG= +K8S_SUPERVISOR_SERVICE=flowmesh-supervisor +K8S_SERVER_SERVICE=flowmesh-server +K8S_CLUSTER_DOMAIN=cluster.local +K8S_GPU_RESOURCE_NAME=nvidia.com/gpu +K8S_SERVER_SERVICE_TYPE=ClusterIP +K8S_IMAGE_PULL_POLICY=IfNotPresent +# Storage class for stack volumes; empty uses default. +K8S_STORAGE_CLASS= +K8S_REDIS_STORAGE_SIZE=8Gi +K8S_RESULTS_STORAGE_SIZE=20Gi +K8S_RESULTS_ACCESS_MODE=ReadWriteOnce +# Grant cluster-scoped node read access so worker hardware +# can be reported before a worker starts. +K8S_ENABLE_NODE_RBAC=false +# Secret holding the server gRPC TLS files. +SERVER_GRPC_TLS_SECRET= +# Secret holding the Redis TLS files. +REDIS_TLS_SECRET= + # ==== Supervisor gRPC ==== # Tuning for the supervisor's gRPC server and worker connections. # Leave SUPERVISOR_GRPC_EXTERNAL_PORT empty unless workers connect diff --git a/cli/stack/src/flowmesh_cli_stack/env_schema.py b/cli/stack/src/flowmesh_cli_stack/env_schema.py index 445485f4..f76c0ec5 100644 --- a/cli/stack/src/flowmesh_cli_stack/env_schema.py +++ b/cli/stack/src/flowmesh_cli_stack/env_schema.py @@ -125,6 +125,74 @@ def _warn_reaper_without_watchdog( ), ], ), + EnvSection( + title="Kubernetes Backend", + description=[ + "Used when STACK_BACKEND=k8s. The stack is deployed into one", + "namespace; the Kubernetes scheduler places workers across nodes.", + "A gRPC TLS certificate must carry the supervisor Service name", + "as a SAN, since workers dial the Service, not a host.", + ], + vars=[ + EnvVar( + "STACK_BACKEND", + "compose", + var_type=EnvVarType.ENUM, + choices=("compose", "k8s"), + ), + EnvVar("K8S_NAMESPACE", "flowmesh"), + EnvVar( + "K8S_WORKER_NAMESPACE", + description=[ + "Namespace for worker pods; defaults to K8S_NAMESPACE." + ], + ), + EnvVar( + "K8S_CONTEXT", description=["kubectl context; empty uses current."] + ), + EnvVar( + "K8S_KUBECONFIG", + description=["kubeconfig path; empty uses default."], + ), + EnvVar("K8S_SUPERVISOR_SERVICE", "flowmesh-supervisor"), + EnvVar("K8S_SERVER_SERVICE", "flowmesh-server"), + EnvVar("K8S_CLUSTER_DOMAIN", "cluster.local"), + EnvVar("K8S_GPU_RESOURCE_NAME", "nvidia.com/gpu"), + EnvVar( + "K8S_SERVER_SERVICE_TYPE", + "ClusterIP", + var_type=EnvVarType.ENUM, + choices=("ClusterIP", "NodePort", "LoadBalancer"), + ), + EnvVar("K8S_IMAGE_PULL_POLICY", "IfNotPresent"), + EnvVar( + "K8S_STORAGE_CLASS", + description=[ + "Storage class for stack volumes; empty uses default." + ], + ), + EnvVar("K8S_REDIS_STORAGE_SIZE", "8Gi"), + EnvVar("K8S_RESULTS_STORAGE_SIZE", "20Gi"), + EnvVar("K8S_RESULTS_ACCESS_MODE", "ReadWriteOnce"), + EnvVar( + "K8S_ENABLE_NODE_RBAC", + "false", + var_type=EnvVarType.BOOL, + description=[ + "Grant cluster-scoped node read access so worker hardware", + "can be reported before a worker starts.", + ], + ), + EnvVar( + "SERVER_GRPC_TLS_SECRET", + description=["Secret holding the server gRPC TLS files."], + ), + EnvVar( + "REDIS_TLS_SECRET", + description=["Secret holding the Redis TLS files."], + ), + ], + ), EnvSection( title="Supervisor gRPC", description=[ diff --git a/cli/stack/src/flowmesh_cli_stack/k8s.py b/cli/stack/src/flowmesh_cli_stack/k8s.py new file mode 100644 index 00000000..c8118220 --- /dev/null +++ b/cli/stack/src/flowmesh_cli_stack/k8s.py @@ -0,0 +1,224 @@ +"""Kubernetes backend for the FlowMesh stack lifecycle commands.""" + +import os +import subprocess +from collections.abc import Iterator +from contextlib import contextmanager +from enum import StrEnum +from pathlib import Path + +import typer +from flowmesh.models.nodes import NodeRole +from flowmesh_cli.core import logging +from flowmesh_cli.core.assets import asset_path +from flowmesh_stack.env import ensure_env_file, load_env, parse_env_file +from flowmesh_stack.kubernetes import KubectlError, KubernetesStack +from flowmesh_stack.manifests import ManifestError +from flowmesh_stack.paths import resolve_path + +from .utils import DEFAULT_ENV_FILE, STACK_PATH_KEYS, drain_workers, stack_env_example + +MANIFEST_ASSETS = ( + "00-namespace.yaml", + "10-rbac.yaml", + "20-redis.yaml", + "30-server.yaml", +) +"""Manifest assets applied in order; the namespace must come first.""" + +SERVER_WORKLOAD = "deployment/flowmesh-server" +REDIS_WORKLOADS = ( + "statefulset/flowmesh-redis-control", + "statefulset/flowmesh-redis-telemetry", +) +STACK_WORKLOADS = { + "server": SERVER_WORKLOAD, + "redis_control": REDIS_WORKLOADS[0], + "redis_telemetry": REDIS_WORKLOADS[1], +} +ROLLOUT_TIMEOUT = "300s" + +DEFAULT_NAMESPACE = "flowmesh" +DEFAULT_WORKER_CONFIG = "./configs/worker_config.yaml" + + +class StackBackend(StrEnum): + COMPOSE = "compose" + K8S = "k8s" + + +def resolve_backend(value: str | None, env_file: Path) -> StackBackend: + """Resolve the stack backend from the option, falling back to the env file.""" + raw = (value or "").strip() + if not raw: + raw = parse_env_file(env_file).get("STACK_BACKEND", "").strip() + if not raw: + return StackBackend.COMPOSE + try: + return StackBackend(raw.lower()) + except ValueError: + logging.error( + f"Unknown stack backend {raw!r}; " + f"expected one of {', '.join(StackBackend)}." + ) + raise typer.Exit(code=1) from None + + +def manifest_paths() -> list[Path]: + return [ + asset_path("flowmesh_cli_stack.assets", "k8s", name) for name in MANIFEST_ASSETS + ] + + +def apply_k8s_env(base_dir: Path) -> None: + """Fill in the derived Kubernetes values the manifests reference. + + ``K8S_WORKER_NAMESPACE`` defaults to the stack namespace and + ``SERVER_WORKER_CONFIG`` to the path the compose backend bind-mounts, + because manifest substitution has no nested defaults. + """ + namespace = os.environ.get("K8S_NAMESPACE", "").strip() or DEFAULT_NAMESPACE + os.environ["K8S_NAMESPACE"] = namespace + if not os.environ.get("K8S_WORKER_NAMESPACE", "").strip(): + os.environ["K8S_WORKER_NAMESPACE"] = namespace + os.environ["SERVER_WORKER_CONFIG"] = resolve_path( + os.environ.get("SERVER_WORKER_CONFIG", ""), + default=DEFAULT_WORKER_CONFIG, + base_dir=base_dir, + ).as_posix() + + +def node_role() -> NodeRole: + raw = os.environ.get("NODE_ROLE", "").strip() + return NodeRole(raw.lower()) if raw else NodeRole.ROOT + + +def stack(env_file: Path, image_tag: str | None = None) -> KubernetesStack: + def _load(path: Path) -> None: + ensure_env_file(path, stack_env_example()) + load_env(path, base_dir=Path.cwd(), path_keys=STACK_PATH_KEYS) + apply_k8s_env(Path.cwd()) + if image_tag: + os.environ["FLOWMESH_VERSION"] = image_tag + + _load(env_file) + return KubernetesStack( + manifests=manifest_paths(), + namespace=os.environ["K8S_NAMESPACE"], + load_env=_load, + context=os.environ.get("K8S_CONTEXT", "").strip() or None, + kubeconfig=os.environ.get("K8S_KUBECONFIG", "").strip() or None, + ) + + +def workloads() -> list[str]: + """Return the workloads this node runs; Redis only on a root node.""" + if node_role() is NodeRole.ROOT: + return [*REDIS_WORKLOADS, SERVER_WORKLOAD] + return [SERVER_WORKLOAD] + + +def _check(result: subprocess.CompletedProcess) -> None: + if result.returncode != 0: + raise typer.Exit(code=result.returncode) + + +@contextmanager +def _reporting(action: str) -> Iterator[None]: + try: + yield + except (KubectlError, ManifestError) as exc: + logging.error(f"Failed to {action}: {exc}") + raise typer.Exit(code=1) from exc + + +def _resolve_workloads(services: list[str] | None) -> list[str]: + if not services: + return workloads() + + requested = list(dict.fromkeys(services)) + unknown = [name for name in requested if name not in STACK_WORKLOADS] + if unknown: + logging.error( + f"Unknown service(s): {', '.join(unknown)}. " + f"Known services: {', '.join(STACK_WORKLOADS)}." + ) + raise typer.Exit(code=1) + return [STACK_WORKLOADS[name] for name in requested] + + +def up(env_file: Path = DEFAULT_ENV_FILE, image_tag: str | None = None) -> None: + """Apply the stack manifests and wait for the workloads to become ready.""" + with _reporting("apply the stack"): + target = stack(env_file, image_tag) + _check(target.apply(env_file)) + for workload in workloads(): + _check(target.rollout_status(workload, ROLLOUT_TIMEOUT)) + logging.success("FlowMesh stack is up.") + + +def down(env_file: Path = DEFAULT_ENV_FILE, image_tag: str | None = None) -> None: + """Drain workers and delete the stack resources.""" + logging.info("Draining workers...") + drain_workers(env_file) + logging.info("Shutting down the FlowMesh stack...") + with _reporting("delete the stack"): + target = stack(env_file, image_tag) + _check(target.delete_workers()) + _check(target.delete(env_file)) + logging.success("FlowMesh stack stopped.") + + +def restart( + services: list[str] | None = None, + env_file: Path = DEFAULT_ENV_FILE, + image_tag: str | None = None, +) -> None: + """Restart the stack, or the named workloads, in place. + + An image tag change is applied rather than rolled, because a rollout + restart does not change the pod template's image. + """ + with _reporting("restart the stack"): + target = stack(env_file, image_tag) + requested = _resolve_workloads(services) + + if SERVER_WORKLOAD in requested: + logging.info("Draining workers...") + drain_workers(env_file) + + if image_tag: + _check(target.apply(env_file)) + else: + for workload in requested: + _check(target.rollout_restart(workload)) + + for workload in requested: + _check(target.rollout_status(workload, ROLLOUT_TIMEOUT)) + logging.success("FlowMesh stack restarted.") + + +def logs(service: str | None = None, env_file: Path = DEFAULT_ENV_FILE) -> None: + """Stream logs from a stack workload.""" + with _reporting("stream logs"): + target = stack(env_file) + workload = _resolve_workloads([service])[0] if service else SERVER_WORKLOAD + _check(target.logs(workload)) + + +def ps(env_file: Path = DEFAULT_ENV_FILE) -> None: + """Show stack pods and supervisor-managed worker pods.""" + with _reporting("read stack status"): + target = stack(env_file) + _check(target.status()) + logging.log("\nWorkers:") + _check(target.worker_status()) + + +def clean(env_file: Path = DEFAULT_ENV_FILE, image_tag: str | None = None) -> None: + """Drain workers, delete the stack, and remove its persistent volumes.""" + down(env_file, image_tag) + logging.info("Removing stack volumes...") + with _reporting("remove stack volumes"): + _check(stack(env_file, image_tag).delete_volumes()) + logging.success("FlowMesh stack cleaned.") diff --git a/cli/stack/src/flowmesh_cli_stack/stack.py b/cli/stack/src/flowmesh_cli_stack/stack.py index bce6361d..b716d47e 100644 --- a/cli/stack/src/flowmesh_cli_stack/stack.py +++ b/cli/stack/src/flowmesh_cli_stack/stack.py @@ -28,19 +28,21 @@ get_push_platforms, ) +from . import k8s from .env_schema import STACK_ENV_SCHEMA, deploy_overrides, role_overrides +from .k8s import StackBackend, resolve_backend from .utils import ( DEFAULT_ENV_FILE, STACK_PATH_KEYS, apply_plugin_data_env, apply_stack_resource_env, + drain_workers, ensure_deploy_paths, parse_node_role, resolve_package_version, stack_bake_file, stack_compose_file, stack_env_example, - stack_node_client, ) from .worker import worker_pull @@ -478,6 +480,9 @@ def up( image_tag: str | None = typer.Option( None, "--image-tag", help="Override FLOWMESH_VERSION" ), + backend: str | None = typer.Option( + None, "--backend", help="Stack backend to target (compose|k8s)" + ), ) -> None: """Start the stack. @@ -486,6 +491,10 @@ def up( services are skipped — the worker is expected to connect to the root node's Redis via REDIS_CONTROL_URL / REDIS_TELEMETRY_URL. """ + if resolve_backend(backend, env_file) is StackBackend.K8S: + k8s.up(env_file=env_file, image_tag=image_tag) + return + profile = "root" if _node_role(env_file) == NodeRole.ROOT else None _compose( ["up", "-d", "--wait"], @@ -497,15 +506,6 @@ def up( logging.success("FlowMesh stack is up.") -def _drain_workers(env_file: Path) -> None: - """Destroy all dynamically spawned workers before stopping the server.""" - try: - client = stack_node_client(env_file, base_url=None, token=None) - client.destroy_all_workers() - except Exception as exc: - logging.warning(f"Unable to drain workers; continuing shutdown. {exc}") - - @app.command() def down( env_file: Path = typer.Option( @@ -514,10 +514,17 @@ def down( image_tag: str | None = typer.Option( None, "--image-tag", help="Override FLOWMESH_VERSION" ), + backend: str | None = typer.Option( + None, "--backend", help="Stack backend to target (compose|k8s)" + ), ) -> None: """Drain workers and stop the stack.""" + if resolve_backend(backend, env_file) is StackBackend.K8S: + k8s.down(env_file=env_file, image_tag=image_tag) + return + logging.info("Draining workers...") - _drain_workers(env_file) + drain_workers(env_file) logging.info("Shutting down the FlowMesh stack...") _compose( ["down"], @@ -554,6 +561,9 @@ def restart( pull: bool = typer.Option( True, "--pull/--no-pull", help="Pull the target image before recreating." ), + backend: str | None = typer.Option( + None, "--backend", help="Stack backend to target (compose|k8s)" + ), ) -> None: """Drain workers and restart the stack, or recreate specific services in place. @@ -561,9 +571,13 @@ def restart( are recreated; when any of them manages workers (the server / supervisor) its workers are drained first so their in-flight tasks requeue onto other nodes. """ + if resolve_backend(backend, env_file) is StackBackend.K8S: + k8s.restart(services=services, env_file=env_file, image_tag=image_tag) + return + if not services: logging.info("Draining workers...") - _drain_workers(env_file) + drain_workers(env_file) _compose( ["down"], env_file=env_file, @@ -592,7 +606,7 @@ def restart( if any(svc in WORKER_MANAGING_SERVICES for svc in requested): logging.info("Draining workers...") - _drain_workers(env_file) + drain_workers(env_file) profile = "root" if _node_role(env_file) == NodeRole.ROOT else None up_args = ["up", "-d", "--no-deps", "--force-recreate", "--wait"] @@ -617,8 +631,15 @@ def logs( env_file: Path = typer.Option( DEFAULT_ENV_FILE, "--env-file", help="Env file for compose" ), + backend: str | None = typer.Option( + None, "--backend", help="Stack backend to target (compose|k8s)" + ), ) -> None: """Stream logs from stack services or a specific service container.""" + if resolve_backend(backend, env_file) is StackBackend.K8S: + k8s.logs(service=service, env_file=env_file) + return + code = _stack().stream_logs(env_file=env_file, service=service, profile="root") if code != 0: raise typer.Exit(code=code) @@ -629,8 +650,15 @@ def ps( env_file: Path = typer.Option( DEFAULT_ENV_FILE, "--env-file", help="Env file for compose" ), + backend: str | None = typer.Option( + None, "--backend", help="Stack backend to target (compose|k8s)" + ), ) -> None: """Display running status of stack containers and worker containers.""" + if resolve_backend(backend, env_file) is StackBackend.K8S: + k8s.ps(env_file=env_file) + return + _compose(["ps"], env_file=env_file, env=None, profile="root") logging.log("\nWorkers:") docker_bin = _require_bin("docker") @@ -653,9 +681,12 @@ def status_cmd( env_file: Path = typer.Option( DEFAULT_ENV_FILE, "--env-file", help="Env file for compose" ), + backend: str | None = typer.Option( + None, "--backend", help="Stack backend to target (compose|k8s)" + ), ) -> None: """Display running status of stack containers (alias for ps).""" - ps(env_file=env_file) + ps(env_file=env_file, backend=backend) @app.command() @@ -666,10 +697,17 @@ def clean( image_tag: str | None = typer.Option( None, "--image-tag", help="Override FLOWMESH_VERSION" ), + backend: str | None = typer.Option( + None, "--backend", help="Stack backend to target (compose|k8s)" + ), ) -> None: """Drain workers, stop the stack, and remove all containers and volumes.""" + if resolve_backend(backend, env_file) is StackBackend.K8S: + k8s.clean(env_file=env_file, image_tag=image_tag) + return + logging.info("Draining workers...") - _drain_workers(env_file) + drain_workers(env_file) logging.info("Removing stack containers and volumes...") _compose( ["down", "-v"], diff --git a/cli/stack/src/flowmesh_cli_stack/utils.py b/cli/stack/src/flowmesh_cli_stack/utils.py index 011eb14c..d8997e4e 100644 --- a/cli/stack/src/flowmesh_cli_stack/utils.py +++ b/cli/stack/src/flowmesh_cli_stack/utils.py @@ -98,6 +98,15 @@ def stack_node_client( return NodeClient(resolved_base, token=resolved_token) +def drain_workers(env_file: Path) -> None: + """Destroy all dynamically spawned workers before stopping the server.""" + try: + client = stack_node_client(env_file, base_url=None, token=None) + client.destroy_all_workers() + except Exception as exc: + logging.warning(f"Unable to drain workers; continuing shutdown. {exc}") + + def flowmesh_client( env_file: Path, base_url: str | None, api_key: str | None ) -> FlowMesh: diff --git a/tests/cli/test_stack_k8s.py b/tests/cli/test_stack_k8s.py new file mode 100644 index 00000000..52a96185 --- /dev/null +++ b/tests/cli/test_stack_k8s.py @@ -0,0 +1,288 @@ +"""`flowmesh stack --backend k8s` dispatch and Kubernetes backend behavior.""" + +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest +import typer +from flowmesh.models.nodes import NodeRole +from flowmesh_cli_stack import k8s as k8s_module +from flowmesh_cli_stack import stack as stack_module +from flowmesh_cli_stack.k8s import StackBackend + +ENV_FILE = Path(".env") + + +# ------------------------------------------------------------------ # +# Backend resolution +# ------------------------------------------------------------------ # + + +class TestResolveBackend: + def test_option_wins_over_the_env_file(self) -> None: + with patch.object( + k8s_module, "parse_env_file", return_value={"STACK_BACKEND": "compose"} + ): + assert k8s_module.resolve_backend("k8s", ENV_FILE) is StackBackend.K8S + + def test_env_file_is_used_without_an_option(self) -> None: + with patch.object( + k8s_module, "parse_env_file", return_value={"STACK_BACKEND": "k8s"} + ): + assert k8s_module.resolve_backend(None, ENV_FILE) is StackBackend.K8S + + def test_compose_is_the_default(self) -> None: + with patch.object(k8s_module, "parse_env_file", return_value={}): + assert k8s_module.resolve_backend(None, ENV_FILE) is StackBackend.COMPOSE + + def test_unknown_backend_exits(self) -> None: + with patch.object(k8s_module, "parse_env_file", return_value={}): + with pytest.raises(typer.Exit): + k8s_module.resolve_backend("nomad", ENV_FILE) + + +# ------------------------------------------------------------------ # +# Command dispatch +# ------------------------------------------------------------------ # + + +class TestDispatch: + def test_up_routes_to_kubernetes(self) -> None: + with ( + patch.object( + stack_module, "resolve_backend", return_value=StackBackend.K8S + ), + patch.object(stack_module.k8s, "up") as up, + patch.object(stack_module, "_compose") as compose, + ): + stack_module.up(env_file=ENV_FILE, image_tag=None, backend="k8s") + + up.assert_called_once() + compose.assert_not_called() + + def test_up_leaves_the_compose_path_untouched(self) -> None: + with ( + patch.object( + stack_module, "resolve_backend", return_value=StackBackend.COMPOSE + ), + patch.object(stack_module.k8s, "up") as up, + patch.object(stack_module, "_compose") as compose, + patch.object(stack_module, "_node_role", return_value=NodeRole.ROOT), + patch.object(stack_module, "image_env_overrides", return_value={}), + ): + stack_module.up(env_file=ENV_FILE, image_tag=None, backend=None) + + up.assert_not_called() + compose.assert_called_once() + + def test_down_routes_to_kubernetes(self) -> None: + with ( + patch.object( + stack_module, "resolve_backend", return_value=StackBackend.K8S + ), + patch.object(stack_module.k8s, "down") as down, + patch.object(stack_module, "_compose") as compose, + patch.object(stack_module, "drain_workers") as drain, + ): + stack_module.down(env_file=ENV_FILE, image_tag=None, backend="k8s") + + down.assert_called_once() + compose.assert_not_called() + drain.assert_not_called() + + def test_restart_routes_to_kubernetes(self) -> None: + with ( + patch.object( + stack_module, "resolve_backend", return_value=StackBackend.K8S + ), + patch.object(stack_module.k8s, "restart") as restart, + patch.object(stack_module, "_compose") as compose, + ): + stack_module.restart( + services=["server"], + env_file=ENV_FILE, + image_tag=None, + pull=True, + backend="k8s", + ) + + restart.assert_called_once() + assert restart.call_args.kwargs["services"] == ["server"] + compose.assert_not_called() + + def test_ps_routes_to_kubernetes(self) -> None: + with ( + patch.object( + stack_module, "resolve_backend", return_value=StackBackend.K8S + ), + patch.object(stack_module.k8s, "ps") as ps, + patch.object(stack_module, "_compose") as compose, + ): + stack_module.ps(env_file=ENV_FILE, backend="k8s") + + ps.assert_called_once() + compose.assert_not_called() + + +# ------------------------------------------------------------------ # +# Kubernetes lifecycle +# ------------------------------------------------------------------ # + + +def _stack(role: NodeRole = NodeRole.ROOT) -> MagicMock: + target = MagicMock() + target.apply.return_value.returncode = 0 + target.delete.return_value.returncode = 0 + target.delete_workers.return_value.returncode = 0 + target.delete_volumes.return_value.returncode = 0 + target.rollout_status.return_value.returncode = 0 + target.rollout_restart.return_value.returncode = 0 + target.logs.return_value.returncode = 0 + target.status.return_value.returncode = 0 + target.worker_status.return_value.returncode = 0 + return target + + +class TestKubernetesLifecycle: + def test_up_waits_for_every_workload(self) -> None: + target = _stack() + with ( + patch.object(k8s_module, "stack", return_value=target), + patch.object(k8s_module, "node_role", return_value=NodeRole.ROOT), + ): + k8s_module.up(ENV_FILE) + + target.apply.assert_called_once() + waited = [call.args[0] for call in target.rollout_status.call_args_list] + assert waited == [ + "statefulset/flowmesh-redis-control", + "statefulset/flowmesh-redis-telemetry", + "deployment/flowmesh-server", + ] + + def test_worker_node_waits_only_for_the_server(self) -> None: + target = _stack() + with ( + patch.object(k8s_module, "stack", return_value=target), + patch.object(k8s_module, "node_role", return_value=NodeRole.WORKER), + ): + k8s_module.up(ENV_FILE) + + waited = [call.args[0] for call in target.rollout_status.call_args_list] + assert waited == ["deployment/flowmesh-server"] + + def test_down_drains_workers_first(self) -> None: + target = _stack() + with ( + patch.object(k8s_module, "stack", return_value=target), + patch.object(k8s_module, "drain_workers") as drain, + ): + k8s_module.down(ENV_FILE) + + drain.assert_called_once() + target.delete_workers.assert_called_once() + target.delete.assert_called_once() + + def test_restart_of_the_server_drains_workers(self) -> None: + target = _stack() + with ( + patch.object(k8s_module, "stack", return_value=target), + patch.object(k8s_module, "drain_workers") as drain, + ): + k8s_module.restart(services=["server"], env_file=ENV_FILE) + + drain.assert_called_once() + target.rollout_restart.assert_called_once_with("deployment/flowmesh-server") + + def test_restart_of_redis_does_not_drain_workers(self) -> None: + target = _stack() + with ( + patch.object(k8s_module, "stack", return_value=target), + patch.object(k8s_module, "drain_workers") as drain, + ): + k8s_module.restart(services=["redis_control"], env_file=ENV_FILE) + + drain.assert_not_called() + + def test_image_tag_is_applied_rather_than_rolled(self) -> None: + """A rollout restart alone would leave the pod template's image unchanged.""" + target = _stack() + with ( + patch.object(k8s_module, "stack", return_value=target), + patch.object(k8s_module, "drain_workers"), + ): + k8s_module.restart(services=["server"], env_file=ENV_FILE, image_tag="v2") + + target.apply.assert_called_once() + target.rollout_restart.assert_not_called() + + def test_unknown_service_exits_without_acting(self) -> None: + target = _stack() + with ( + patch.object(k8s_module, "stack", return_value=target), + patch.object(k8s_module, "drain_workers") as drain, + ): + with pytest.raises(typer.Exit): + k8s_module.restart(services=["nope"], env_file=ENV_FILE) + + drain.assert_not_called() + target.rollout_restart.assert_not_called() + + def test_ps_lists_stack_and_worker_pods(self) -> None: + target = _stack() + with patch.object(k8s_module, "stack", return_value=target): + k8s_module.ps(ENV_FILE) + + target.status.assert_called_once() + target.worker_status.assert_called_once() + + def test_failed_kubectl_call_exits_with_its_code(self) -> None: + target = _stack() + target.apply.return_value.returncode = 3 + with ( + patch.object(k8s_module, "stack", return_value=target), + patch.object(k8s_module, "node_role", return_value=NodeRole.ROOT), + ): + with pytest.raises(typer.Exit) as exit_info: + k8s_module.up(ENV_FILE) + + assert exit_info.value.exit_code == 3 + + +# ------------------------------------------------------------------ # +# Environment derivation +# ------------------------------------------------------------------ # + + +class TestEnvDerivation: + def test_worker_namespace_defaults_to_the_stack_namespace( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + monkeypatch.setenv("K8S_NAMESPACE", "ml-team") + monkeypatch.delenv("K8S_WORKER_NAMESPACE", raising=False) + monkeypatch.delenv("SERVER_WORKER_CONFIG", raising=False) + + k8s_module.apply_k8s_env(tmp_path) + + assert k8s_module.os.environ["K8S_WORKER_NAMESPACE"] == "ml-team" + + def test_explicit_worker_namespace_is_kept( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + monkeypatch.setenv("K8S_NAMESPACE", "ml-team") + monkeypatch.setenv("K8S_WORKER_NAMESPACE", "gpu-pool") + + k8s_module.apply_k8s_env(tmp_path) + + assert k8s_module.os.environ["K8S_WORKER_NAMESPACE"] == "gpu-pool" + + def test_worker_config_resolves_to_an_absolute_path( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + monkeypatch.delenv("SERVER_WORKER_CONFIG", raising=False) + + k8s_module.apply_k8s_env(tmp_path) + + resolved = Path(k8s_module.os.environ["SERVER_WORKER_CONFIG"]) + assert resolved.is_absolute() + assert resolved.name == "worker_config.yaml" diff --git a/tests/cli/test_stack_restart.py b/tests/cli/test_stack_restart.py index eb58b8e9..f2681756 100644 --- a/tests/cli/test_stack_restart.py +++ b/tests/cli/test_stack_restart.py @@ -14,15 +14,20 @@ def _restart( env_file: Path = Path(".env"), image_tag: str | None = None, pull: bool = True, + backend: str | None = None, ) -> None: stack_module.restart( - services=services, env_file=env_file, image_tag=image_tag, pull=pull + services=services, + env_file=env_file, + image_tag=image_tag, + pull=pull, + backend=backend, ) def test_restart_server_drains_then_recreates_only_server() -> None: with ( - patch.object(stack_module, "_drain_workers") as drain, + patch.object(stack_module, "drain_workers") as drain, patch.object(stack_module, "_compose") as compose, patch.object(stack_module, "_node_role", return_value=NodeRole.ROOT), patch.object(stack_module, "image_env_overrides", return_value={}), @@ -38,7 +43,7 @@ def test_restart_server_drains_then_recreates_only_server() -> None: def test_restart_multiple_services_drains_once_and_recreates_all() -> None: with ( - patch.object(stack_module, "_drain_workers") as drain, + patch.object(stack_module, "drain_workers") as drain, patch.object(stack_module, "_compose") as compose, patch.object(stack_module, "_node_role", return_value=NodeRole.ROOT), patch.object(stack_module, "image_env_overrides", return_value={}), @@ -56,7 +61,7 @@ def test_restart_multiple_services_drains_once_and_recreates_all() -> None: def test_restart_dedupes_repeated_services() -> None: with ( - patch.object(stack_module, "_drain_workers"), + patch.object(stack_module, "drain_workers"), patch.object(stack_module, "_compose") as compose, patch.object(stack_module, "_node_role", return_value=NodeRole.ROOT), patch.object(stack_module, "image_env_overrides", return_value={}), @@ -69,7 +74,7 @@ def test_restart_dedupes_repeated_services() -> None: def test_restart_no_pull_omits_pull_flag() -> None: with ( - patch.object(stack_module, "_drain_workers"), + patch.object(stack_module, "drain_workers"), patch.object(stack_module, "_compose") as compose, patch.object(stack_module, "_node_role", return_value=NodeRole.ROOT), patch.object(stack_module, "image_env_overrides", return_value={}), @@ -81,7 +86,7 @@ def test_restart_no_pull_omits_pull_flag() -> None: def test_restart_redis_service_does_not_drain_workers() -> None: with ( - patch.object(stack_module, "_drain_workers") as drain, + patch.object(stack_module, "drain_workers") as drain, patch.object(stack_module, "_compose"), patch.object(stack_module, "_node_role", return_value=NodeRole.ROOT), patch.object(stack_module, "image_env_overrides", return_value={}), @@ -93,7 +98,7 @@ def test_restart_redis_service_does_not_drain_workers() -> None: def test_restart_unknown_service_exits_without_acting() -> None: with ( - patch.object(stack_module, "_drain_workers") as drain, + patch.object(stack_module, "drain_workers") as drain, patch.object(stack_module, "_compose") as compose, ): with pytest.raises(typer.Exit): @@ -105,7 +110,7 @@ def test_restart_unknown_service_exits_without_acting() -> None: def test_restart_unknown_in_a_set_exits_without_acting() -> None: with ( - patch.object(stack_module, "_drain_workers") as drain, + patch.object(stack_module, "drain_workers") as drain, patch.object(stack_module, "_compose") as compose, ): with pytest.raises(typer.Exit): From e2e48a63fcdf669501991730d23c3c51e54edae5 Mon Sep 17 00:00:00 2001 From: Noppanat Wadlom Date: Tue, 8 Sep 2026 23:02:06 +0800 Subject: [PATCH 06/11] docs: document the kubernetes deployment path Signed-off-by: Noppanat Wadlom --- AGENTS.md | 3 + docs/ARCHITECTURE.md | 28 +++++++- docs/CLI.md | 15 +++++ docs/ENV.md | 25 ++++++++ docs/KUBERNETES.md | 149 +++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 219 insertions(+), 1 deletion(-) create mode 100644 docs/KUBERNETES.md diff --git a/AGENTS.md b/AGENTS.md index e0560d78..f25a5d4f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -39,6 +39,9 @@ and artifacts. - **[`docs/ENV.md`](docs/ENV.md)** — curated server / worker / supervisor env var tables (the knobs you actually tune). Full schema in `cli/stack/src/flowmesh_cli_stack/env_schema.py`. +- **[`docs/KUBERNETES.md`](docs/KUBERNETES.md)** — deploying the stack on + Kubernetes: `--backend k8s`, the `kubernetes` worker provider, RBAC, TLS, + and the single-replica server constraint. - **[`docs/PLUGINS.md`](docs/PLUGINS.md)** — plugin extension contract, loader semantics (`FLOWMESH_PLUGINS`), and a worked example. diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 03058154..4c8ed1d7 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -40,11 +40,32 @@ The runtime is two top-level processes: workflow / task / dispatch logic and the **Supervisor subsystem** (`src/server/supervisor/`), which manages per-node worker lifecycle, runs the worker-facing gRPC server (`:50051`), and drives the - Docker / Vast.ai worker adapters. + Docker / Kubernetes / Vast.ai worker adapters. 2. **Worker** (`src/worker/`) — stateless executor. Connects to a supervisor via gRPC, receives tasks, runs the matching executor, reports results. +### Kubernetes + +On Kubernetes one namespace is one FlowMesh node: a single server Deployment +runs `NODE_ROLE=root` with its supervisor, and the cluster scheduler places +workers across machines. + +``` +Client ──▶ Service flowmesh-server ──▶ Deployment flowmesh-server (replicas: 1) + │ ServiceAccount flowmesh-server + ├─▶ StatefulSet redis-control / redis-telemetry + └─▶ Pod flowmesh-worker-* (nvidia.com/gpu: N) +``` + +The server runs as a single replica with the `Recreate` strategy. A worker's +token lives in the registry of the supervisor that minted it, so a second +replica behind the Service would reject those registrations. Workers dial the +supervisor through a headless Service, which a gRPC TLS certificate must name +as a SAN. + +See [`KUBERNETES.md`](KUBERNETES.md). + ## Communication - **server ↔ supervisor (same node)** — `multiprocessing.Queue`. @@ -127,6 +148,11 @@ scripts/dev/ compile_protos, sync_requirements, check_env_examples `WorkerHardware`. The dispatcher's `_cached_worker_candidates` filters to workers whose cache covers the task's references; entries older than `WORKER_CACHE_TTL_SEC` are ignored. +- **Worker providers.** A worker is created through the provider named in + `worker_config.yaml`: `docker` (containers on the node's own daemon), + `kubernetes` (pods through the cluster API), or `vastai` (rented + instances). Provider backends are contacted only when a provider is used, + so a deployment needs just its own backend to be reachable. - **Worker capabilities.** Beyond hardware fit, each worker advertises the set of task types it can service, and the dispatcher routes a task only to workers that advertise its type. A worker advertises a type only when its executor came diff --git a/docs/CLI.md b/docs/CLI.md index 84fa6aca..21d6ff83 100644 --- a/docs/CLI.md +++ b/docs/CLI.md @@ -95,6 +95,21 @@ flowmesh stack worker up cpu 2 --name-template '{slug}-run-{idx}' # {slug}-run- The template must keep names unique within one `up` invocation — include `{idx}` or `{gpu}` when creating more than one worker. +### Kubernetes backend + +`--backend k8s` targets a Kubernetes cluster instead of the local Docker +daemon; `STACK_BACKEND=k8s` in the env file makes the flag unnecessary. It +applies to `up`, `down`, `restart`, `logs`, `ps`, and `clean`. + +```bash +flowmesh stack up --backend k8s # apply manifests, wait for rollout +flowmesh stack ps --backend k8s # stack pods and worker pods +flowmesh stack down --backend k8s +``` + +`build`, `push`, `pull`, and `pullall` remain Docker-only — each node's kubelet +pulls images itself. See [`KUBERNETES.md`](KUBERNETES.md). + `flowmesh stack up` reads `NODE_ROLE` from the env file (default `root`). On a root node, both local Redis services are deployed alongside the server. On a worker node (`NODE_ROLE=worker`), Redis services are skipped — the worker diff --git a/docs/ENV.md b/docs/ENV.md index 482f3a94..1d30fd42 100644 --- a/docs/ENV.md +++ b/docs/ENV.md @@ -109,3 +109,28 @@ cap)`. A task that requests more than the worker cap is dispatched to another worker if one has a larger cap; otherwise the dispatcher follows its standard requeue/retry behavior. The worker logs a startup warning if SSH is enabled with no cap configured. + +## Kubernetes backend + +Used when `STACK_BACKEND=k8s`. See [`KUBERNETES.md`](KUBERNETES.md). + +| Variable | Default | Purpose | +|----------|---------|---------| +| `STACK_BACKEND` | `compose` | Stack backend: `compose` or `k8s`. | +| `K8S_NAMESPACE` | `flowmesh` | Namespace the stack is deployed into. | +| `K8S_WORKER_NAMESPACE` | `K8S_NAMESPACE` | Namespace for worker pods. | +| `K8S_CONTEXT` | current | kubectl context to target. | +| `K8S_KUBECONFIG` | default | kubeconfig file to use. | +| `K8S_SUPERVISOR_SERVICE` | `flowmesh-supervisor` | Headless Service workers dial for gRPC. | +| `K8S_SERVER_SERVICE` | `flowmesh-server` | Service workers use for artifact transfer. | +| `K8S_CLUSTER_DOMAIN` | `cluster.local` | Cluster DNS domain. | +| `K8S_GPU_RESOURCE_NAME` | `nvidia.com/gpu` | Extended resource requested for GPU workers. | +| `K8S_SERVER_SERVICE_TYPE` | `ClusterIP` | `ClusterIP`, `NodePort`, or `LoadBalancer`. | +| `K8S_IMAGE_PULL_POLICY` | `IfNotPresent` | Pull policy for the server image. | +| `K8S_STORAGE_CLASS` | cluster default | Storage class for stack volumes. | +| `K8S_REDIS_STORAGE_SIZE` | `8Gi` | Size of each Redis volume. | +| `K8S_RESULTS_STORAGE_SIZE` | `20Gi` | Size of the server results volume. | +| `K8S_RESULTS_ACCESS_MODE` | `ReadWriteOnce` | Access mode for the results volume. | +| `K8S_ENABLE_NODE_RBAC` | `false` | Grant cluster-scoped node reads for pre-start hardware reporting. | +| `SERVER_GRPC_TLS_SECRET` | — | Secret holding the server gRPC TLS files. | +| `REDIS_TLS_SECRET` | — | Secret holding the Redis TLS files. | diff --git a/docs/KUBERNETES.md b/docs/KUBERNETES.md new file mode 100644 index 00000000..a06c2870 --- /dev/null +++ b/docs/KUBERNETES.md @@ -0,0 +1,149 @@ +# Kubernetes deployment + +FlowMesh deploys into an existing Kubernetes cluster with +`flowmesh stack ... --backend k8s`. One namespace holds one FlowMesh node: a +server Deployment with its supervisor, two Redis StatefulSets, and the worker +pods the supervisor creates. + +## Prerequisites + +- A reachable cluster and `kubectl` on `PATH`. +- A storage class for the Redis and results volumes, or `K8S_STORAGE_CLASS` + pointing at one. +- For GPU workers: the NVIDIA device plugin, so nodes advertise + `nvidia.com/gpu`. + +## Deploying + +```bash +flowmesh stack init # scaffold .env +# set STACK_BACKEND=k8s and K8S_NAMESPACE in .env +flowmesh stack up --backend k8s # apply manifests, wait for rollout +flowmesh stack ps --backend k8s # stack pods and worker pods +flowmesh stack logs server --backend k8s +flowmesh stack down --backend k8s +``` + +`STACK_BACKEND=k8s` in the env file makes `--backend` unnecessary on every +call. The same `.env` drives both backends. + +`flowmesh stack up` applies the namespace, RBAC, Redis, and server manifests in +that order, then waits on each workload's rollout. `down` drains workers before +deleting the stack; `clean` also removes its persistent volume claims. + +`restart` rolls the workloads in place. With `--image-tag` it applies instead, +because a rollout restart does not change the pod template's image. + +`build`, `push`, `pull`, and `pullall` act on a local Docker daemon and take no +`--backend`; on Kubernetes each node's kubelet pulls images itself. + +## Server topology + +The server Deployment runs `replicas: 1` with the `Recreate` strategy. A +worker's token lives in the registry of the supervisor that minted it, so a +second replica behind the Service would reject registrations from workers the +other replica started. Scheduling state is persisted to Redis and rebuilt on +startup, so restarting the pod is safe. + +Two Services front the Deployment: + +| Service | Purpose | +|---------|---------| +| `flowmesh-server` | REST API on `SERVER_HTTP_PORT`. `K8S_SERVER_SERVICE_TYPE` selects `ClusterIP`, `NodePort`, or `LoadBalancer`. | +| `flowmesh-supervisor` | Headless; worker pods dial it for gRPC on `SERVER_GRPC_PORT`. | + +The whole env file is carried into the pod as the `flowmesh-server-env` Secret, +which is the Kubernetes equivalent of compose's `env_file`. Values come from +the env file alone, so nothing else in the operator's shell reaches the +cluster. + +## Workers + +Worker pods are created by the supervisor through the Kubernetes API, not by a +separate manifest. Point `SERVER_WORKER_CONFIG` at a worker config declaring +the `kubernetes` provider; a starting point ships as +`cli/stack/src/flowmesh_cli_stack/assets/k8s/worker_config.k8s.yaml`. + +```yaml +workers: + - provider: kubernetes + init_on_start: true + worker_config: + worker_type: gpu + gpu_count: 1 + shm_size: 8Gi + node_selector: + nvidia.com/gpu.present: "true" +``` + +Each worker pod: + +- runs with `restartPolicy: Always` and carries the labels + `flowmesh.io/managed`, `flowmesh.io/node-alias`, and + `flowmesh.io/worker-name`; +- receives its credentials from a per-worker Secret through `envFrom`, never + inline in the pod spec; +- requests GPUs as `nvidia.com/gpu` (or `gpu_resource_name`) and leaves device + assignment to the device plugin; +- has no owner reference to the server pod, so a server rollout does not take + running workers with it. Pods left by an unclean exit are reaped at + supervisor startup, because their tokens died with the previous registry. + +Set `shm_size` for training and inference workloads; the 64 MiB default +`/dev/shm` is too small for torch dataloaders. + +### Configuration reference + +`results_pvc`, `results_mount_path`, `hf_cache_pvc`, `cpu_request`, +`cpu_limit`, `memory_request`, `memory_limit`, `tolerations`, +`service_account_name`, `image_pull_secrets`, `runtime_class_name`, +`priority_class_name`, `pod_labels`, and `pod_annotations` map onto their pod +spec equivalents. `pod_overrides` is merged onto the generated manifest for +anything else — mappings merge recursively, sequences and scalars replace. + +### Results + +Workflows declare where their output goes. Without a `results_pvc` a worker +writes to an ephemeral volume, so a workflow that keeps results either sets +`output.destination`, enables `WORKER_UPLOAD_RESULTS` to upload them to the +server, or names a claim through `results_pvc`. + +### SSH tasks + +Worker pods have no Docker socket, so the SSH executor does not load and +workers never advertise the `ssh` task type. The dispatcher routes SSH tasks +only to workers that advertise it. + +## RBAC + +`flowmesh stack up` creates a `flowmesh-server` ServiceAccount and a namespaced +Role granting pods, pod logs, secrets, and events — everything the supervisor +needs to run workers. + +Setting `K8S_ENABLE_NODE_RBAC=true` additionally binds a cluster-scoped +ClusterRole granting `get` and `list` on nodes. This is the one cluster-scoped +grant, and it is optional: it lets the supervisor report a worker's hardware +from node allocatable capacity and GPU labels before the worker starts. +Without it that preview is empty, and hardware still arrives from the worker +itself at registration. + +## TLS + +`SERVER_GRPC_TLS_SECRET` names a Secret mounted at `/etc/ssl/server`, and +`REDIS_TLS_SECRET` one mounted at `/etc/ssl/redis`. The certificate must carry +the supervisor Service DNS name +(`flowmesh-supervisor..svc.cluster.local`) as a SAN, because that is +the name workers dial. The CA reaches workers automatically. + +## Joining an external root node + +A cluster can also run as a worker node against a root node deployed elsewhere. +Set `NODE_ROLE=worker` and point `REDIS_CONTROL_URL` and `REDIS_TELEMETRY_URL` +at the root node's Redis; the Redis StatefulSets are then not deployed. + +## Not yet supported + +Workers deployed directly with `kubectl` — a Deployment or DaemonSet that +enrolls itself rather than being created by the supervisor — need an enrollment +flow that mints a token at registration rather than at spawn. Today every +worker's token is minted by the supervisor before it starts the pod. From b3a18ab75573b917f477e100057c8749b32e5175 Mon Sep 17 00:00:00 2001 From: Noppanat Wadlom Date: Tue, 8 Sep 2026 23:25:54 +0800 Subject: [PATCH 07/11] fix: correct kubernetes stack teardown and worker namespace Teardown applied the whole rendered stream to `kubectl delete`, which includes the Namespace and the results PersistentVolumeClaim. Deleting a namespace cascades to every claim inside it, so `stack down` destroyed task results and Redis state, while the compose backend's `down` leaves volumes alone. `down` now keeps the namespace and the claims and `clean` removes them, matching the split compose has between `down` and `down -v`. Separately, the RBAC the server needs is bound in the worker namespace, so setting K8S_WORKER_NAMESPACE to anything other than the stack namespace made the apply target a namespace that was never created. Signed-off-by: Noppanat Wadlom --- .../assets/k8s/00-namespace.yaml | 10 +++ cli/stack/src/flowmesh_cli_stack/k8s.py | 11 ++- docs/KUBERNETES.md | 12 ++- sdk/stack/src/flowmesh_stack/kubernetes.py | 28 ++++-- tests/cli/test_stack_k8s.py | 31 +++++++ tests/sdk/test_kubernetes_stack.py | 85 +++++++++++++++++++ tests/sdk/test_manifests.py | 20 +++++ 7 files changed, 186 insertions(+), 11 deletions(-) create mode 100644 tests/sdk/test_kubernetes_stack.py diff --git a/cli/stack/src/flowmesh_cli_stack/assets/k8s/00-namespace.yaml b/cli/stack/src/flowmesh_cli_stack/assets/k8s/00-namespace.yaml index eadd4c80..8f291bac 100644 --- a/cli/stack/src/flowmesh_cli_stack/assets/k8s/00-namespace.yaml +++ b/cli/stack/src/flowmesh_cli_stack/assets/k8s/00-namespace.yaml @@ -4,3 +4,13 @@ metadata: name: ${K8S_NAMESPACE:-flowmesh} labels: app.kubernetes.io/part-of: flowmesh +--- +# Only when worker pods live outside the stack namespace; the RBAC below is +# bound there and would otherwise apply into a namespace that does not exist. +x-flowmesh-when: K8S_WORKER_NAMESPACE_DISTINCT +apiVersion: v1 +kind: Namespace +metadata: + name: ${K8S_WORKER_NAMESPACE:-flowmesh} + labels: + app.kubernetes.io/part-of: flowmesh diff --git a/cli/stack/src/flowmesh_cli_stack/k8s.py b/cli/stack/src/flowmesh_cli_stack/k8s.py index c8118220..c371d6fd 100644 --- a/cli/stack/src/flowmesh_cli_stack/k8s.py +++ b/cli/stack/src/flowmesh_cli_stack/k8s.py @@ -75,12 +75,17 @@ def apply_k8s_env(base_dir: Path) -> None: ``K8S_WORKER_NAMESPACE`` defaults to the stack namespace and ``SERVER_WORKER_CONFIG`` to the path the compose backend bind-mounts, - because manifest substitution has no nested defaults. + because manifest substitution has no nested defaults. The distinctness of + the worker namespace is precomputed for the same reason: the manifest + conditionals compare against a value, not against another variable. """ namespace = os.environ.get("K8S_NAMESPACE", "").strip() or DEFAULT_NAMESPACE os.environ["K8S_NAMESPACE"] = namespace - if not os.environ.get("K8S_WORKER_NAMESPACE", "").strip(): - os.environ["K8S_WORKER_NAMESPACE"] = namespace + worker_namespace = os.environ.get("K8S_WORKER_NAMESPACE", "").strip() or namespace + os.environ["K8S_WORKER_NAMESPACE"] = worker_namespace + os.environ["K8S_WORKER_NAMESPACE_DISTINCT"] = ( + "true" if worker_namespace != namespace else "false" + ) os.environ["SERVER_WORKER_CONFIG"] = resolve_path( os.environ.get("SERVER_WORKER_CONFIG", ""), default=DEFAULT_WORKER_CONFIG, diff --git a/docs/KUBERNETES.md b/docs/KUBERNETES.md index a06c2870..63ff49da 100644 --- a/docs/KUBERNETES.md +++ b/docs/KUBERNETES.md @@ -28,8 +28,12 @@ flowmesh stack down --backend k8s call. The same `.env` drives both backends. `flowmesh stack up` applies the namespace, RBAC, Redis, and server manifests in -that order, then waits on each workload's rollout. `down` drains workers before -deleting the stack; `clean` also removes its persistent volume claims. +that order, then waits on each workload's rollout. + +`down` drains workers and deletes the workloads, leaving the namespace and the +persistent volume claims in place, so task results and Redis state survive a +teardown. `clean` does that and then removes the claims — the same split +compose has between `down` and `down -v`. `restart` rolls the workloads in place. With `--image-tag` it applies instead, because a rollout restart does not change the pod template's image. @@ -52,6 +56,10 @@ Two Services front the Deployment: | `flowmesh-server` | REST API on `SERVER_HTTP_PORT`. `K8S_SERVER_SERVICE_TYPE` selects `ClusterIP`, `NodePort`, or `LoadBalancer`. | | `flowmesh-supervisor` | Headless; worker pods dial it for gRPC on `SERVER_GRPC_PORT`. | +Setting `K8S_WORKER_NAMESPACE` to something other than `K8S_NAMESPACE` puts +worker pods in their own namespace; that namespace and the Role binding the +server needs in it are created alongside the stack. + The whole env file is carried into the pod as the `flowmesh-server-env` Secret, which is the Kubernetes equivalent of compose's `env_file`. Values come from the env file alone, so nothing else in the operator's shell reaches the diff --git a/sdk/stack/src/flowmesh_stack/kubernetes.py b/sdk/stack/src/flowmesh_stack/kubernetes.py index 5107186d..7aecf13e 100644 --- a/sdk/stack/src/flowmesh_stack/kubernetes.py +++ b/sdk/stack/src/flowmesh_stack/kubernetes.py @@ -7,6 +7,8 @@ from dataclasses import dataclass, field from pathlib import Path +import yaml + from .env import parse_env_file from .manifests import render_manifests @@ -18,6 +20,9 @@ STACK_LABEL_VALUE = "flowmesh" +DATA_KINDS = frozenset({"Namespace", "PersistentVolumeClaim"}) +"""Kinds a stack teardown leaves in place, so results and Redis state survive it.""" + class KubectlError(RuntimeError): """Raised when a kubectl command fails.""" @@ -112,13 +117,24 @@ def apply(self, env_file: Path) -> subprocess.CompletedProcess[str]: return self._run(["apply", "-f", "-"], stdin=self.render(env_file)) def delete( - self, env_file: Path, ignore_not_found: bool = True + self, env_file: Path, keep_data: bool = True ) -> subprocess.CompletedProcess[str]: - """Delete the resources described by the rendered stack manifests.""" - args = ["delete", "-f", "-"] - if ignore_not_found: - args.append("--ignore-not-found") - return self._run(args, stdin=self.render(env_file)) + """Delete the resources described by the rendered stack manifests. + + The namespace and persistent volume claims are kept by default: + deleting the namespace cascades to every claim in it, which would make + a teardown destroy task results and Redis state. Removing them is + ``clean``'s job, matching what compose volumes do. + """ + stream = self.render(env_file) + if keep_data: + kept = [ + document + for document in yaml.safe_load_all(stream) + if document and document.get("kind") not in DATA_KINDS + ] + stream = yaml.safe_dump_all(kept, sort_keys=False) + return self._run(["delete", "-f", "-", "--ignore-not-found"], stdin=stream) def rollout_status(self, target: str, timeout: str) -> subprocess.CompletedProcess: """Wait for a workload to finish rolling out.""" diff --git a/tests/cli/test_stack_k8s.py b/tests/cli/test_stack_k8s.py index 52a96185..dedd0743 100644 --- a/tests/cli/test_stack_k8s.py +++ b/tests/cli/test_stack_k8s.py @@ -228,6 +228,17 @@ def test_unknown_service_exits_without_acting(self) -> None: drain.assert_not_called() target.rollout_restart.assert_not_called() + def test_clean_removes_volumes_after_teardown(self) -> None: + target = _stack() + with ( + patch.object(k8s_module, "stack", return_value=target), + patch.object(k8s_module, "drain_workers"), + ): + k8s_module.clean(ENV_FILE) + + target.delete.assert_called_once() + target.delete_volumes.assert_called_once() + def test_ps_lists_stack_and_worker_pods(self) -> None: target = _stack() with patch.object(k8s_module, "stack", return_value=target): @@ -276,6 +287,26 @@ def test_explicit_worker_namespace_is_kept( assert k8s_module.os.environ["K8S_WORKER_NAMESPACE"] == "gpu-pool" + def test_distinct_worker_namespace_is_flagged( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + monkeypatch.setenv("K8S_NAMESPACE", "flowmesh") + monkeypatch.setenv("K8S_WORKER_NAMESPACE", "gpu-pool") + + k8s_module.apply_k8s_env(tmp_path) + + assert k8s_module.os.environ["K8S_WORKER_NAMESPACE_DISTINCT"] == "true" + + def test_shared_worker_namespace_is_not_flagged( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + monkeypatch.setenv("K8S_NAMESPACE", "flowmesh") + monkeypatch.delenv("K8S_WORKER_NAMESPACE", raising=False) + + k8s_module.apply_k8s_env(tmp_path) + + assert k8s_module.os.environ["K8S_WORKER_NAMESPACE_DISTINCT"] == "false" + def test_worker_config_resolves_to_an_absolute_path( self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: diff --git a/tests/sdk/test_kubernetes_stack.py b/tests/sdk/test_kubernetes_stack.py new file mode 100644 index 00000000..75d0823b --- /dev/null +++ b/tests/sdk/test_kubernetes_stack.py @@ -0,0 +1,85 @@ +"""KubernetesStack: what a teardown deletes, and what it must leave behind.""" + +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest +import yaml +from flowmesh_stack.kubernetes import ( + DATA_KINDS, + KubectlError, + KubernetesStack, + ensure_kubectl_available, +) + +MANIFEST = """ +apiVersion: v1 +kind: Namespace +metadata: + name: flowmesh +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flowmesh-server +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: flowmesh-results +""" + + +def _stack(tmp_path: Path) -> KubernetesStack: + manifest = tmp_path / "stack.yaml" + manifest.write_text(MANIFEST) + return KubernetesStack( + manifests=[manifest], + namespace="flowmesh", + load_env=lambda _: None, + ) + + +def _stdin_kinds(run: MagicMock) -> list[str]: + stream = run.call_args.kwargs["stdin"] + return [doc["kind"] for doc in yaml.safe_load_all(stream) if doc] + + +class TestTeardown: + def test_delete_keeps_the_namespace_and_claims(self, tmp_path: Path) -> None: + """Deleting the namespace would cascade to every claim inside it.""" + stack = _stack(tmp_path) + with patch.object(stack, "_run") as run: + stack.delete(tmp_path / ".env") + + kinds = _stdin_kinds(run) + assert kinds == ["Deployment"] + assert not DATA_KINDS & set(kinds) + + def test_delete_can_be_asked_for_everything(self, tmp_path: Path) -> None: + stack = _stack(tmp_path) + with patch.object(stack, "_run") as run: + stack.delete(tmp_path / ".env", keep_data=False) + + assert _stdin_kinds(run) == ["Namespace", "Deployment", "PersistentVolumeClaim"] + + def test_delete_tolerates_missing_resources(self, tmp_path: Path) -> None: + stack = _stack(tmp_path) + with patch.object(stack, "_run") as run: + stack.delete(tmp_path / ".env") + + assert "--ignore-not-found" in run.call_args.args[0] + + def test_apply_sends_every_document(self, tmp_path: Path) -> None: + stack = _stack(tmp_path) + with patch.object(stack, "_run") as run: + stack.apply(tmp_path / ".env") + + assert _stdin_kinds(run) == ["Namespace", "Deployment", "PersistentVolumeClaim"] + + +class TestKubectl: + def test_missing_kubectl_is_reported(self) -> None: + with patch("flowmesh_stack.kubernetes.shutil.which", return_value=None): + with pytest.raises(KubectlError, match="kubectl is required"): + ensure_kubectl_available() diff --git a/tests/sdk/test_manifests.py b/tests/sdk/test_manifests.py index 1a0863f7..38d45ef4 100644 --- a/tests/sdk/test_manifests.py +++ b/tests/sdk/test_manifests.py @@ -251,6 +251,26 @@ def _documents(self, **overrides: str) -> list[dict]: def test_namespace_is_applied_first(self) -> None: assert self._documents()[0]["kind"] == "Namespace" + def test_only_one_namespace_when_workers_share_it(self) -> None: + namespaces = [ + doc["metadata"]["name"] + for doc in self._documents() + if doc["kind"] == "Namespace" + ] + assert namespaces == ["flowmesh"] + + def test_a_separate_worker_namespace_is_created(self) -> None: + """RBAC is bound in the worker namespace, so it has to exist.""" + namespaces = [ + doc["metadata"]["name"] + for doc in self._documents( + K8S_WORKER_NAMESPACE="gpu-pool", + K8S_WORKER_NAMESPACE_DISTINCT="true", + ) + if doc["kind"] == "Namespace" + ] + assert namespaces == ["flowmesh", "gpu-pool"] + def test_root_node_ships_both_redis_workloads(self) -> None: names = { doc["metadata"]["name"] From c0da4ce7992c2f1cd9fbd52e6914819fab4cc473 Mon Sep 17 00:00:00 2001 From: Noppanat Wadlom Date: Tue, 8 Sep 2026 23:26:09 +0800 Subject: [PATCH 08/11] fix: wait for a replaced worker pod to terminate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pod deletion is accepted asynchronously and the object outlives the call while it terminates, so recreating a worker under the same name raced the pod being replaced and would fail with a conflict. Replacement now deletes with no grace period and waits for the name to come free. The `flowmesh.io/node-id` label also carried a worker id, which is assigned only once the worker registers — after the pod exists — so it was both misnamed and never actually set. It is dropped; the node alias remains the stable key the orphan reaper selects on. Signed-off-by: Noppanat Wadlom --- src/server/supervisor/adapters/kubernetes.py | 30 ++++++++++--- tests/server/test_k8s_worker_adapter.py | 45 +++++++++++++++++++- 2 files changed, 68 insertions(+), 7 deletions(-) diff --git a/src/server/supervisor/adapters/kubernetes.py b/src/server/supervisor/adapters/kubernetes.py index 19905180..9084f31b 100644 --- a/src/server/supervisor/adapters/kubernetes.py +++ b/src/server/supervisor/adapters/kubernetes.py @@ -2,6 +2,7 @@ import hashlib import logging import re +import time from collections import Counter from pathlib import PurePosixPath from typing import Any, get_args @@ -31,7 +32,6 @@ MANAGED_LABEL = "flowmesh.io/managed" NODE_ALIAS_LABEL = "flowmesh.io/node-alias" -NODE_ID_LABEL = "flowmesh.io/node-id" WORKER_NAME_LABEL = "flowmesh.io/worker-name" _GPU_PRODUCT_LABEL = "nvidia.com/gpu.product" @@ -40,6 +40,8 @@ _LABEL_VALUE_INVALID_RE = re.compile(r"[^A-Za-z0-9_.-]+") _LABEL_VALUE_MAX_LEN = 63 _CPU_MILLI_RE = re.compile(r"^([0-9]+)m$") +_DELETE_TIMEOUT_SEC = 120.0 +_DELETE_POLL_SEC = 0.5 logger = logging.getLogger("supervisor") @@ -306,8 +308,6 @@ def _labels(self) -> dict[str, str]: NODE_ALIAS_LABEL: sanitize_label_value(self.node_alias), WORKER_NAME_LABEL: sanitize_label_value(self.name), } - if worker_id := self.worker_id: - labels[NODE_ID_LABEL] = sanitize_label_value(worker_id) if self.config.pod_labels: labels.update(self.config.pod_labels) return labels @@ -432,7 +432,9 @@ def _start(self) -> bool: self._is_started = True logger.warning("Pod %s is already running.", self.pod_name) return True - if not self._delete_pod(): + if not self._delete_pod(grace_period_seconds=0): + return False + if not self._await_pod_deletion(): return False if not self._apply_secret(): @@ -497,12 +499,28 @@ def _apply_secret(self) -> bool: logger.error("Failed to update secret %s: %s", self.secret_name, repr(exc)) return False - def _delete_pod(self) -> bool: + def _await_pod_deletion(self) -> bool: + """Block until the pod name is free again. + + Deletion is accepted asynchronously and the object outlives the call + while it terminates, so reusing the name immediately would collide. + """ + deadline = time.monotonic() + _DELETE_TIMEOUT_SEC + while time.monotonic() < deadline: + if self._read_pod() is None: + return True + time.sleep(_DELETE_POLL_SEC) + logger.error("Pod %s was still terminating after deletion", self.pod_name) + return False + + def _delete_pod(self, grace_period_seconds: int | None = None) -> bool: + if grace_period_seconds is None: + grace_period_seconds = self.config.stop_grace_period_sec try: self._core.delete_namespaced_pod( name=self.pod_name, namespace=self.config.namespace, - grace_period_seconds=self.config.stop_grace_period_sec, + grace_period_seconds=grace_period_seconds, ) return True except ApiException as exc: diff --git a/tests/server/test_k8s_worker_adapter.py b/tests/server/test_k8s_worker_adapter.py index b63bb31b..d70fcfaf 100644 --- a/tests/server/test_k8s_worker_adapter.py +++ b/tests/server/test_k8s_worker_adapter.py @@ -8,6 +8,7 @@ from kubernetes.client.exceptions import ApiException from server.hooks import PrincipalContext +from server.supervisor.adapters import kubernetes as k8s_adapter from server.supervisor.adapters.base import WorkerTokenType, WorkerType from server.supervisor.adapters.kubernetes import ( MANAGED_LABEL, @@ -168,6 +169,13 @@ def test_labels_identify_the_managing_node(self) -> None: assert labels[NODE_ALIAS_LABEL] == "flowmesh_node" assert labels[WORKER_NAME_LABEL] == "worker-1" + def test_reaping_selector_matches_the_pod_labels(self) -> None: + """The reaper keys on the node alias, which survives a restart.""" + labels = _adapter().build_pod_manifest()["metadata"]["labels"] + + assert labels[MANAGED_LABEL] == "true" + assert NODE_ALIAS_LABEL in labels + def test_pod_restarts_in_place(self) -> None: manifest = _adapter().build_pod_manifest() assert manifest["spec"]["restartPolicy"] == "Always" @@ -296,13 +304,48 @@ def test_start_adopts_a_running_pod(self) -> None: def test_start_replaces_a_terminal_pod(self) -> None: core = MagicMock() - core.read_namespaced_pod.return_value.status.phase = "Failed" + terminal = MagicMock() + terminal.status.phase = "Failed" + core.read_namespaced_pod.side_effect = [terminal, _api_error(404)] adapter = _adapter(core) assert asyncio.run(adapter.start()) is True core.delete_namespaced_pod.assert_called_once() core.create_namespaced_pod.assert_called_once() + def test_replacing_a_pod_waits_for_the_name_to_free(self) -> None: + """Deletion is asynchronous; reusing the name too early collides.""" + core = MagicMock() + terminal = MagicMock() + terminal.status.phase = "Failed" + terminating = MagicMock() + terminating.status.phase = "Failed" + core.read_namespaced_pod.side_effect = [ + terminal, + terminating, + _api_error(404), + ] + adapter = _adapter(core) + + with patch.object(k8s_adapter, "_DELETE_POLL_SEC", 0): + assert asyncio.run(adapter.start()) is True + + assert core.read_namespaced_pod.call_count == 3 + core.create_namespaced_pod.assert_called_once() + assert core.delete_namespaced_pod.call_args.kwargs["grace_period_seconds"] == 0 + + def test_replacement_gives_up_when_the_pod_never_goes_away(self) -> None: + core = MagicMock() + terminal = MagicMock() + terminal.status.phase = "Failed" + core.read_namespaced_pod.return_value = terminal + adapter = _adapter(core) + + with patch.object(k8s_adapter, "_DELETE_TIMEOUT_SEC", 0): + assert asyncio.run(adapter.start()) is False + + core.create_namespaced_pod.assert_not_called() + def test_failed_pod_creation_removes_the_orphaned_secret(self) -> None: core = MagicMock() core.read_namespaced_pod.side_effect = _api_error(404) From cd86ed597b6d74b74113db7798ee3a278cb9988b Mon Sep 17 00:00:00 2001 From: Noppanat Wadlom Date: Tue, 8 Sep 2026 23:27:11 +0800 Subject: [PATCH 09/11] fix: label redis volumes so cleanup can find them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `clean` deletes persistent volume claims by label, but the Redis claims are created by the StatefulSet controller from volumeClaimTemplates, which carried no labels — so cleanup silently left both Redis volumes behind. Signed-off-by: Noppanat Wadlom --- .../src/flowmesh_cli_stack/assets/k8s/20-redis.yaml | 4 ++++ tests/sdk/test_manifests.py | 9 +++++++++ 2 files changed, 13 insertions(+) diff --git a/cli/stack/src/flowmesh_cli_stack/assets/k8s/20-redis.yaml b/cli/stack/src/flowmesh_cli_stack/assets/k8s/20-redis.yaml index bd6db7b0..46eb034a 100644 --- a/cli/stack/src/flowmesh_cli_stack/assets/k8s/20-redis.yaml +++ b/cli/stack/src/flowmesh_cli_stack/assets/k8s/20-redis.yaml @@ -119,6 +119,8 @@ spec: volumeClaimTemplates: - metadata: name: data + labels: + app.kubernetes.io/part-of: flowmesh spec: accessModes: ["ReadWriteOnce"] storageClassName: @@ -191,6 +193,8 @@ spec: volumeClaimTemplates: - metadata: name: data + labels: + app.kubernetes.io/part-of: flowmesh spec: accessModes: ["ReadWriteOnce"] storageClassName: diff --git a/tests/sdk/test_manifests.py b/tests/sdk/test_manifests.py index 38d45ef4..6104dcab 100644 --- a/tests/sdk/test_manifests.py +++ b/tests/sdk/test_manifests.py @@ -312,6 +312,15 @@ def test_storage_class_is_set_when_configured(self) -> None: ] assert claim["spec"]["storageClassName"] == "fast" + def test_redis_volumes_are_labelled_for_cleanup(self) -> None: + """`clean` selects volumes by label; controller-made claims need it too.""" + for name in ("flowmesh-redis-control", "flowmesh-redis-telemetry"): + statefulset = _by_kind(self._documents())[("StatefulSet", name)] + claim = statefulset["spec"]["volumeClaimTemplates"][0] + assert ( + claim["metadata"]["labels"]["app.kubernetes.io/part-of"] == "flowmesh" + ) + def test_redis_keeps_the_pubsub_buffer_limit(self) -> None: statefulset = _by_kind(self._documents())[ ("StatefulSet", "flowmesh-redis-control") From 2e4a529512e6b17f4b314e7fc591d80a3ad6d813 Mon Sep 17 00:00:00 2001 From: Noppanat Wadlom Date: Fri, 11 Sep 2026 22:42:00 +0800 Subject: [PATCH 10/11] fix: report the kubernetes provider unavailable without a cluster MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The factory resolved cluster access on first use, so it always constructed and the node advertised `kubernetes` even with no kubeconfig and no cluster — the create then failed late instead of being refused by the provider registry. Resolving access in the initializer lets an unreachable cluster drop the provider the same way an absent Docker daemon drops that one. The dockerless-host test now stubs every spawning provider rather than two of them, so its assertion about the surviving provider set no longer depends on whether the machine running it happens to have a kubeconfig. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Noppanat Wadlom --- src/server/supervisor/adapters/kubernetes.py | 32 ++++++------ tests/server/test_external_worker.py | 1 + tests/server/test_k8s_worker_adapter.py | 52 ++++++++++++++------ 3 files changed, 52 insertions(+), 33 deletions(-) diff --git a/src/server/supervisor/adapters/kubernetes.py b/src/server/supervisor/adapters/kubernetes.py index 9084f31b..22b2b849 100644 --- a/src/server/supervisor/adapters/kubernetes.py +++ b/src/server/supervisor/adapters/kubernetes.py @@ -594,16 +594,27 @@ def _hardware_from_node(self, node: Any) -> WorkerHardware: class KubernetesWorkerFactory(WorkerFactory): def __init__(self, system_principal: PrincipalContext) -> None: + """Resolve cluster access up front. + + Construction is what decides whether this node reports the provider as + available, so an unreachable cluster has to fail here rather than at + first use — otherwise the node would advertise a provider it cannot + serve and reject the create only once a worker was requested. + """ super().__init__(system_principal) - self._api_client: client.ApiClient | None = None - self._core: client.CoreV1Api | None = None + try: + config.load_incluster_config() + except config.ConfigException: + config.load_kube_config() + self._api_client = client.ApiClient() + self._core = client.CoreV1Api(self._api_client) self._worker_id_registry: Counter[str] = Counter() self._reaped = False def create_worker( self, token: WorkerTokenType, config: KubernetesWorkerConfig ) -> KubernetesWorkerAdapter: - core = self.core_api() + core = self._core self._reap_orphans(core, config.namespace) name = config.worker_alias or self._next_worker_name(config.worker_type) @@ -623,20 +634,7 @@ def destroy_worker(self, worker: WorkerAdapter) -> None: raise ValueError("Invalid worker type") def cleanup(self) -> None: - if self._api_client is not None: - self._api_client.close() - self._api_client = None - self._core = None - - def core_api(self) -> client.CoreV1Api: - if self._core is None: - try: - config.load_incluster_config() - except config.ConfigException: - config.load_kube_config() - self._api_client = client.ApiClient() - self._core = client.CoreV1Api(self._api_client) - return self._core + self._api_client.close() def _next_worker_name(self, worker_type: WorkerType) -> str: match worker_type: diff --git a/tests/server/test_external_worker.py b/tests/server/test_external_worker.py index cbce2a46..4fda2b59 100644 --- a/tests/server/test_external_worker.py +++ b/tests/server/test_external_worker.py @@ -157,6 +157,7 @@ def _explode(_principal): raise RuntimeError("Error while fetching server API version") monkeypatch.setattr(manager_mod, "docker_provider_spec", _explode) + monkeypatch.setattr(manager_mod, "kubernetes_provider_spec", _explode) monkeypatch.setattr(manager_mod, "vastai_provider_spec", _explode) mgr = manager_mod.WorkerManager( diff --git a/tests/server/test_k8s_worker_adapter.py b/tests/server/test_k8s_worker_adapter.py index d70fcfaf..74827a19 100644 --- a/tests/server/test_k8s_worker_adapter.py +++ b/tests/server/test_k8s_worker_adapter.py @@ -493,9 +493,29 @@ def test_gpu_worker_uses_the_gpu_image(self) -> None: class TestFactory: def _factory(self, core: MagicMock) -> KubernetesWorkerFactory: - factory = KubernetesWorkerFactory(_principal()) - factory._core = core - return factory + with ( + patch.object(k8s_adapter.config, "load_incluster_config"), + patch.object(k8s_adapter.client, "ApiClient"), + patch.object(k8s_adapter.client, "CoreV1Api", return_value=core), + ): + return KubernetesWorkerFactory(_principal()) + + def test_construction_without_a_cluster_raises(self) -> None: + """An unreachable cluster must make the node report the provider absent.""" + with ( + patch.object( + k8s_adapter.config, + "load_incluster_config", + side_effect=k8s_adapter.config.ConfigException("not in cluster"), + ), + patch.object( + k8s_adapter.config, + "load_kube_config", + side_effect=k8s_adapter.config.ConfigException("no kubeconfig"), + ), + ): + with pytest.raises(k8s_adapter.config.ConfigException): + KubernetesWorkerFactory(_principal()) def test_orphaned_pods_are_reaped_once(self) -> None: core = MagicMock() @@ -553,34 +573,34 @@ def test_worker_names_increment_per_type(self) -> None: assert first.name != second.name def test_cleanup_closes_the_api_client(self) -> None: - factory = KubernetesWorkerFactory(_principal()) api_client = MagicMock() - factory._api_client = api_client - factory._core = MagicMock() + with ( + patch.object(k8s_adapter.config, "load_incluster_config"), + patch.object(k8s_adapter.client, "ApiClient", return_value=api_client), + patch.object(k8s_adapter.client, "CoreV1Api"), + ): + factory = KubernetesWorkerFactory(_principal()) factory.cleanup() api_client.close.assert_called_once_with() - assert factory._core is None def test_client_falls_back_to_kubeconfig_outside_a_cluster(self) -> None: - from kubernetes import config as k8s_config - - factory = KubernetesWorkerFactory(_principal()) with ( patch.object( - k8s_config, + k8s_adapter.config, "load_incluster_config", - side_effect=k8s_config.ConfigException("not in cluster"), + side_effect=k8s_adapter.config.ConfigException("not in cluster"), ), - patch.object(k8s_config, "load_kube_config") as load_kube_config, + patch.object(k8s_adapter.config, "load_kube_config") as load_kube_config, + patch.object(k8s_adapter.client, "ApiClient"), + patch.object(k8s_adapter.client, "CoreV1Api"), ): - factory.core_api() + KubernetesWorkerFactory(_principal()) load_kube_config.assert_called_once() - factory.cleanup() def test_destroy_rejects_a_foreign_worker(self) -> None: - factory = KubernetesWorkerFactory(_principal()) + factory = self._factory(MagicMock()) with pytest.raises(ValueError, match="Invalid worker type"): factory.destroy_worker(MagicMock()) From 5ba8d645f9b685c082f07eb0c5e5d6974e4d334e Mon Sep 17 00:00:00 2001 From: Noppanat Wadlom Date: Fri, 11 Sep 2026 22:42:12 +0800 Subject: [PATCH 11/11] docs: describe choosing between the external and kubernetes providers The `external` provider covers workers deployed with kubectl that enroll themselves, which this guide had listed as unsupported. Contrast the two Kubernetes paths by who owns the pod lifecycle, and record that a `kubernetes` worker is a bare Pod and so is not rescheduled when its node is lost. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Noppanat Wadlom --- docs/ARCHITECTURE.md | 7 ++++--- docs/KUBERNETES.md | 26 ++++++++++++++++++++------ 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 4c8ed1d7..fbcc47b5 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -150,9 +150,10 @@ scripts/dev/ compile_protos, sync_requirements, check_env_examples than `WORKER_CACHE_TTL_SEC` are ignored. - **Worker providers.** A worker is created through the provider named in `worker_config.yaml`: `docker` (containers on the node's own daemon), - `kubernetes` (pods through the cluster API), or `vastai` (rented - instances). Provider backends are contacted only when a provider is used, - so a deployment needs just its own backend to be reachable. + `kubernetes` (pods through the cluster API), `vastai` (rented instances), or + `external` (workers this supervisor does not launch, admitted by a shared + secret). A spawning provider whose backend is unreachable at startup is + dropped, so a node advertises only what it can actually serve. - **Worker capabilities.** Beyond hardware fit, each worker advertises the set of task types it can service, and the dispatcher routes a task only to workers that advertise its type. A worker advertises a type only when its executor came diff --git a/docs/KUBERNETES.md b/docs/KUBERNETES.md index 63ff49da..e248f829 100644 --- a/docs/KUBERNETES.md +++ b/docs/KUBERNETES.md @@ -149,9 +149,23 @@ A cluster can also run as a worker node against a root node deployed elsewhere. Set `NODE_ROLE=worker` and point `REDIS_CONTROL_URL` and `REDIS_TELEMETRY_URL` at the root node's Redis; the Redis StatefulSets are then not deployed. -## Not yet supported - -Workers deployed directly with `kubectl` — a Deployment or DaemonSet that -enrolls itself rather than being created by the supervisor — need an enrollment -flow that mints a token at registration rather than at spawn. Today every -worker's token is minted by the supervisor before it starts the pod. +## Choosing a worker provider + +Two providers serve Kubernetes, differing in who owns the pod lifecycle. + +| | `external` | `kubernetes` | +|---|---|---| +| Creates workers | you, via your own Deployment or controller | the supervisor, through the cluster API | +| Scaling | `kubectl scale`, HPA, your controller | `flowmesh worker create` / `destroy` | +| Pod spec | your manifest, the full API | this provider's config fields plus `pod_overrides` | +| Server RBAC | none | pods and secrets in the worker namespace | +| Survives a supervisor restart | yes — tokens re-derive from the shared secret | no — supervisor-minted tokens are lost and the pods are reaped | + +Use `external` when the cluster or your own controller owns worker lifecycle +and the pool is stable. Use `kubernetes` when FlowMesh should create and destroy +workers itself — bursty workloads, or per-workflow GPU shapes that would +otherwise need a Deployment each. + +A `kubernetes` worker is a bare Pod, so it restarts in place but is not +rescheduled if its node is lost. Pools on preemptible or frequently drained +nodes are better served by `external`.