diff --git a/AGENTS.md b/AGENTS.md index e0560d78d..f25a5d4fd 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -39,6 +39,9 @@ and artifacts. - **[`docs/ENV.md`](docs/ENV.md)** — curated server / worker / supervisor env var tables (the knobs you actually tune). Full schema in `cli/stack/src/flowmesh_cli_stack/env_schema.py`. +- **[`docs/KUBERNETES.md`](docs/KUBERNETES.md)** — deploying the stack on + Kubernetes: `--backend k8s`, the `kubernetes` worker provider, RBAC, TLS, + and the single-replica server constraint. - **[`docs/PLUGINS.md`](docs/PLUGINS.md)** — plugin extension contract, loader semantics (`FLOWMESH_PLUGINS`), and a worked example. diff --git a/cli/stack/pyproject.toml b/cli/stack/pyproject.toml index 66cebc917..988deaf13 100644 --- a/cli/stack/pyproject.toml +++ b/cli/stack/pyproject.toml @@ -28,4 +28,5 @@ where = ["src"] "flowmesh_cli_stack" = [ "assets/**", "assets/.env.example", + "assets/k8s/**", ] diff --git a/cli/stack/src/flowmesh_cli_stack/assets/.env.example b/cli/stack/src/flowmesh_cli_stack/assets/.env.example index 9564abf0d..ec0f1a112 100644 --- a/cli/stack/src/flowmesh_cli_stack/assets/.env.example +++ b/cli/stack/src/flowmesh_cli_stack/assets/.env.example @@ -33,6 +33,38 @@ SERVER_GRPC_TLS_CA_FILE=/etc/ssl/server/server-ca.pem SERVER_GRPC_TLS_CERT_FILE=/etc/ssl/server/server.pem SERVER_GRPC_TLS_KEY_FILE=/etc/ssl/server/server.key +# ==== Kubernetes Backend ==== +# Used when STACK_BACKEND=k8s. The stack is deployed into one +# namespace; the Kubernetes scheduler places workers across nodes. +# A gRPC TLS certificate must carry the supervisor Service name +# as a SAN, since workers dial the Service, not a host. +STACK_BACKEND=compose +K8S_NAMESPACE=flowmesh +# Namespace for worker pods; defaults to K8S_NAMESPACE. +K8S_WORKER_NAMESPACE= +# kubectl context; empty uses current. +K8S_CONTEXT= +# kubeconfig path; empty uses default. +K8S_KUBECONFIG= +K8S_SUPERVISOR_SERVICE=flowmesh-supervisor +K8S_SERVER_SERVICE=flowmesh-server +K8S_CLUSTER_DOMAIN=cluster.local +K8S_GPU_RESOURCE_NAME=nvidia.com/gpu +K8S_SERVER_SERVICE_TYPE=ClusterIP +K8S_IMAGE_PULL_POLICY=IfNotPresent +# Storage class for stack volumes; empty uses default. +K8S_STORAGE_CLASS= +K8S_REDIS_STORAGE_SIZE=8Gi +K8S_RESULTS_STORAGE_SIZE=20Gi +K8S_RESULTS_ACCESS_MODE=ReadWriteOnce +# Grant cluster-scoped node read access so worker hardware +# can be reported before a worker starts. +K8S_ENABLE_NODE_RBAC=false +# Secret holding the server gRPC TLS files. +SERVER_GRPC_TLS_SECRET= +# Secret holding the Redis TLS files. +REDIS_TLS_SECRET= + # ==== Supervisor gRPC ==== # Tuning for the supervisor's gRPC server and worker connections. # Leave SUPERVISOR_GRPC_EXTERNAL_PORT empty unless workers connect diff --git a/cli/stack/src/flowmesh_cli_stack/assets/k8s/00-namespace.yaml b/cli/stack/src/flowmesh_cli_stack/assets/k8s/00-namespace.yaml new file mode 100644 index 000000000..8f291bac2 --- /dev/null +++ b/cli/stack/src/flowmesh_cli_stack/assets/k8s/00-namespace.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: ${K8S_NAMESPACE:-flowmesh} + labels: + app.kubernetes.io/part-of: flowmesh +--- +# Only when worker pods live outside the stack namespace; the RBAC below is +# bound there and would otherwise apply into a namespace that does not exist. +x-flowmesh-when: K8S_WORKER_NAMESPACE_DISTINCT +apiVersion: v1 +kind: Namespace +metadata: + name: ${K8S_WORKER_NAMESPACE:-flowmesh} + labels: + app.kubernetes.io/part-of: flowmesh diff --git a/cli/stack/src/flowmesh_cli_stack/assets/k8s/10-rbac.yaml b/cli/stack/src/flowmesh_cli_stack/assets/k8s/10-rbac.yaml new file mode 100644 index 000000000..e15649ace --- /dev/null +++ b/cli/stack/src/flowmesh_cli_stack/assets/k8s/10-rbac.yaml @@ -0,0 +1,77 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: flowmesh-server + namespace: ${K8S_NAMESPACE:-flowmesh} + labels: + app.kubernetes.io/part-of: flowmesh +--- +# The supervisor creates worker pods and the per-worker secret carrying their +# credentials, and reads pod logs for diagnostics. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: flowmesh-server + namespace: ${K8S_WORKER_NAMESPACE:-flowmesh} + labels: + app.kubernetes.io/part-of: flowmesh +rules: + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch", "create", "delete", "deletecollection"] + - apiGroups: [""] + resources: ["pods/log"] + verbs: ["get"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list", "create", "update", "delete", "deletecollection"] + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: flowmesh-server + namespace: ${K8S_WORKER_NAMESPACE:-flowmesh} + labels: + app.kubernetes.io/part-of: flowmesh +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: flowmesh-server +subjects: + - kind: ServiceAccount + name: flowmesh-server + namespace: ${K8S_NAMESPACE:-flowmesh} +--- +# Optional. Lets the supervisor report a worker's hardware before the worker +# starts, by reading node allocatable capacity and GPU labels. Without it the +# hardware preview is empty and everything else works unchanged. +x-flowmesh-when: K8S_ENABLE_NODE_RBAC +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: flowmesh-server-nodes-${K8S_NAMESPACE:-flowmesh} + labels: + app.kubernetes.io/part-of: flowmesh +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list"] +--- +x-flowmesh-when: K8S_ENABLE_NODE_RBAC +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: flowmesh-server-nodes-${K8S_NAMESPACE:-flowmesh} + labels: + app.kubernetes.io/part-of: flowmesh +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: flowmesh-server-nodes-${K8S_NAMESPACE:-flowmesh} +subjects: + - kind: ServiceAccount + name: flowmesh-server + namespace: ${K8S_NAMESPACE:-flowmesh} diff --git a/cli/stack/src/flowmesh_cli_stack/assets/k8s/20-redis.yaml b/cli/stack/src/flowmesh_cli_stack/assets/k8s/20-redis.yaml new file mode 100644 index 000000000..46eb034a0 --- /dev/null +++ b/cli/stack/src/flowmesh_cli_stack/assets/k8s/20-redis.yaml @@ -0,0 +1,205 @@ +# Redis runs on root nodes only. A worker node connects to the root node's +# Redis through REDIS_CONTROL_URL / REDIS_TELEMETRY_URL. +x-flowmesh-when: NODE_ROLE==root +apiVersion: v1 +kind: Secret +metadata: + name: flowmesh-redis-acl + namespace: ${K8S_NAMESPACE:-flowmesh} + labels: + app.kubernetes.io/part-of: flowmesh +type: Opaque +stringData: + users.acl: | + user default off + user ${REDIS_USERNAME:-admin} on >${REDIS_PASSWORD:-} ~* &* +@all +--- +x-flowmesh-when: NODE_ROLE==root +apiVersion: v1 +kind: Service +metadata: + name: flowmesh-redis-control + namespace: ${K8S_NAMESPACE:-flowmesh} + labels: + app.kubernetes.io/part-of: flowmesh + app.kubernetes.io/component: redis-control +spec: + clusterIP: None + selector: + app.kubernetes.io/part-of: flowmesh + app.kubernetes.io/component: redis-control + ports: + - name: redis + port: 6379 + targetPort: 6379 +--- +x-flowmesh-when: NODE_ROLE==root +apiVersion: v1 +kind: Service +metadata: + name: flowmesh-redis-telemetry + namespace: ${K8S_NAMESPACE:-flowmesh} + labels: + app.kubernetes.io/part-of: flowmesh + app.kubernetes.io/component: redis-telemetry +spec: + clusterIP: None + selector: + app.kubernetes.io/part-of: flowmesh + app.kubernetes.io/component: redis-telemetry + ports: + - name: redis + port: 6379 + targetPort: 6379 +--- +x-flowmesh-when: NODE_ROLE==root +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: flowmesh-redis-control + namespace: ${K8S_NAMESPACE:-flowmesh} + labels: + app.kubernetes.io/part-of: flowmesh + app.kubernetes.io/component: redis-control +spec: + serviceName: flowmesh-redis-control + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/part-of: flowmesh + app.kubernetes.io/component: redis-control + template: + metadata: + labels: + app.kubernetes.io/part-of: flowmesh + app.kubernetes.io/component: redis-control + spec: + containers: + - name: redis + image: redis:7-alpine + args: + - redis-server + - --save + - "60 1" + - --loglevel + - warning + # Control-plane pubsub clients are dropped under load without a + # raised output buffer limit. + - --client-output-buffer-limit + - pubsub 1gb 512mb 60 + - x-flowmesh-when: REDIS_ACL_ENABLED + x-flowmesh-value: --aclfile + - x-flowmesh-when: REDIS_ACL_ENABLED + x-flowmesh-value: /etc/redis/acl/users.acl + ports: + - name: redis + containerPort: 6379 + volumeMounts: + - name: data + mountPath: /data + - x-flowmesh-when: REDIS_ACL_ENABLED + name: acl + mountPath: /etc/redis/acl + readOnly: true + readinessProbe: + exec: + command: ["redis-cli", "-p", "6379", "ping"] + initialDelaySeconds: 5 + periodSeconds: 10 + livenessProbe: + exec: + command: ["redis-cli", "-p", "6379", "ping"] + initialDelaySeconds: 15 + periodSeconds: 20 + volumes: + - x-flowmesh-when: REDIS_ACL_ENABLED + name: acl + secret: + secretName: flowmesh-redis-acl + volumeClaimTemplates: + - metadata: + name: data + labels: + app.kubernetes.io/part-of: flowmesh + spec: + accessModes: ["ReadWriteOnce"] + storageClassName: + x-flowmesh-when: K8S_STORAGE_CLASS + x-flowmesh-value: ${K8S_STORAGE_CLASS:-} + resources: + requests: + storage: ${K8S_REDIS_STORAGE_SIZE:-8Gi} +--- +x-flowmesh-when: NODE_ROLE==root +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: flowmesh-redis-telemetry + namespace: ${K8S_NAMESPACE:-flowmesh} + labels: + app.kubernetes.io/part-of: flowmesh + app.kubernetes.io/component: redis-telemetry +spec: + serviceName: flowmesh-redis-telemetry + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/part-of: flowmesh + app.kubernetes.io/component: redis-telemetry + template: + metadata: + labels: + app.kubernetes.io/part-of: flowmesh + app.kubernetes.io/component: redis-telemetry + spec: + containers: + - name: redis + image: redis:7-alpine + args: + - redis-server + - --save + - "300 1" + - --loglevel + - warning + - x-flowmesh-when: REDIS_ACL_ENABLED + x-flowmesh-value: --aclfile + - x-flowmesh-when: REDIS_ACL_ENABLED + x-flowmesh-value: /etc/redis/acl/users.acl + ports: + - name: redis + containerPort: 6379 + volumeMounts: + - name: data + mountPath: /data + - x-flowmesh-when: REDIS_ACL_ENABLED + name: acl + mountPath: /etc/redis/acl + readOnly: true + readinessProbe: + exec: + command: ["redis-cli", "-p", "6379", "ping"] + initialDelaySeconds: 5 + periodSeconds: 10 + livenessProbe: + exec: + command: ["redis-cli", "-p", "6379", "ping"] + initialDelaySeconds: 15 + periodSeconds: 20 + volumes: + - x-flowmesh-when: REDIS_ACL_ENABLED + name: acl + secret: + secretName: flowmesh-redis-acl + volumeClaimTemplates: + - metadata: + name: data + labels: + app.kubernetes.io/part-of: flowmesh + spec: + accessModes: ["ReadWriteOnce"] + storageClassName: + x-flowmesh-when: K8S_STORAGE_CLASS + x-flowmesh-value: ${K8S_STORAGE_CLASS:-} + resources: + requests: + storage: ${K8S_REDIS_STORAGE_SIZE:-8Gi} diff --git a/cli/stack/src/flowmesh_cli_stack/assets/k8s/30-server.yaml b/cli/stack/src/flowmesh_cli_stack/assets/k8s/30-server.yaml new file mode 100644 index 000000000..84b3cf58d --- /dev/null +++ b/cli/stack/src/flowmesh_cli_stack/assets/k8s/30-server.yaml @@ -0,0 +1,181 @@ +apiVersion: v1 +kind: Secret +metadata: + name: flowmesh-server-env + namespace: ${K8S_NAMESPACE:-flowmesh} + labels: + app.kubernetes.io/part-of: flowmesh +type: Opaque +stringData: + x-flowmesh-env-values: true +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: flowmesh-worker-config + namespace: ${K8S_NAMESPACE:-flowmesh} + labels: + app.kubernetes.io/part-of: flowmesh +data: + worker_config.yaml: + x-flowmesh-file: SERVER_WORKER_CONFIG +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: flowmesh-results + namespace: ${K8S_NAMESPACE:-flowmesh} + labels: + app.kubernetes.io/part-of: flowmesh +spec: + accessModes: + - ${K8S_RESULTS_ACCESS_MODE:-ReadWriteOnce} + storageClassName: + x-flowmesh-when: K8S_STORAGE_CLASS + x-flowmesh-value: ${K8S_STORAGE_CLASS:-} + resources: + requests: + storage: ${K8S_RESULTS_STORAGE_SIZE:-20Gi} +--- +apiVersion: v1 +kind: Service +metadata: + name: flowmesh-server + namespace: ${K8S_NAMESPACE:-flowmesh} + labels: + app.kubernetes.io/part-of: flowmesh + app.kubernetes.io/component: server +spec: + type: ${K8S_SERVER_SERVICE_TYPE:-ClusterIP} + selector: + app.kubernetes.io/part-of: flowmesh + app.kubernetes.io/component: server + ports: + - name: http + port: + x-flowmesh-int: ${SERVER_HTTP_PORT:-8000} + targetPort: http +--- +# Workers dial this name. A gRPC TLS certificate must carry it as a SAN. +apiVersion: v1 +kind: Service +metadata: + name: ${K8S_SUPERVISOR_SERVICE:-flowmesh-supervisor} + namespace: ${K8S_NAMESPACE:-flowmesh} + labels: + app.kubernetes.io/part-of: flowmesh + app.kubernetes.io/component: supervisor +spec: + clusterIP: None + selector: + app.kubernetes.io/part-of: flowmesh + app.kubernetes.io/component: server + ports: + - name: grpc + port: + x-flowmesh-int: ${SERVER_GRPC_PORT:-50051} + targetPort: grpc +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flowmesh-server + namespace: ${K8S_NAMESPACE:-flowmesh} + labels: + app.kubernetes.io/part-of: flowmesh + app.kubernetes.io/component: server +spec: + # A worker's token lives in the registry of the supervisor that minted it, so + # a second replica behind the Service would reject those registrations. + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app.kubernetes.io/part-of: flowmesh + app.kubernetes.io/component: server + template: + metadata: + labels: + app.kubernetes.io/part-of: flowmesh + app.kubernetes.io/component: server + spec: + serviceAccountName: flowmesh-server + containers: + - name: server + image: ${FLOWMESH_REGISTRY:-ghcr.io/mlsys-io}/flowmesh_server:${FLOWMESH_VERSION:-dev} + imagePullPolicy: ${K8S_IMAGE_PULL_POLICY:-IfNotPresent} + envFrom: + - secretRef: + name: flowmesh-server-env + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: SERVER_APP_PORT + value: "${SERVER_HTTP_PORT:-8000}" + - name: WORKER_CONFIG_PATH + value: /etc/flowmesh/worker_config.yaml + - name: RESULTS_DIR + value: /mnt/flowmesh-results + - name: SERVER_METRICS_DIR + value: /mnt/flowmesh-metrics + - name: LOG_FILE + value: /var/log/flowmesh-server/server.log + ports: + - name: http + containerPort: + x-flowmesh-int: ${SERVER_HTTP_PORT:-8000} + - name: grpc + containerPort: + x-flowmesh-int: ${SERVER_GRPC_PORT:-50051} + volumeMounts: + - name: worker-config + mountPath: /etc/flowmesh + readOnly: true + - name: results + mountPath: /mnt/flowmesh-results + - name: metrics + mountPath: /mnt/flowmesh-metrics + - name: logs + mountPath: /var/log/flowmesh-server + - x-flowmesh-when: SERVER_GRPC_TLS_SECRET + name: server-tls + mountPath: /etc/ssl/server + readOnly: true + - x-flowmesh-when: REDIS_TLS_SECRET + name: redis-tls + mountPath: /etc/ssl/redis + readOnly: true + readinessProbe: + httpGet: + path: /healthz + port: http + initialDelaySeconds: 10 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /healthz + port: http + initialDelaySeconds: 40 + periodSeconds: 30 + volumes: + - name: worker-config + configMap: + name: flowmesh-worker-config + - name: results + persistentVolumeClaim: + claimName: flowmesh-results + - name: metrics + emptyDir: {} + - name: logs + emptyDir: {} + - x-flowmesh-when: SERVER_GRPC_TLS_SECRET + name: server-tls + secret: + secretName: ${SERVER_GRPC_TLS_SECRET:-} + - x-flowmesh-when: REDIS_TLS_SECRET + name: redis-tls + secret: + secretName: ${REDIS_TLS_SECRET:-} diff --git a/cli/stack/src/flowmesh_cli_stack/assets/k8s/worker_config.k8s.yaml b/cli/stack/src/flowmesh_cli_stack/assets/k8s/worker_config.k8s.yaml new file mode 100644 index 000000000..4de646840 --- /dev/null +++ b/cli/stack/src/flowmesh_cli_stack/assets/k8s/worker_config.k8s.yaml @@ -0,0 +1,26 @@ +# Worker configuration for the Kubernetes backend. Point SERVER_WORKER_CONFIG +# at a copy of this file and adjust the pools to match your cluster. +default_worker_config: + image_pull_policy: IfNotPresent + +workers: + - provider: kubernetes + init_on_start: true + worker_config: + worker_type: cpu + cpu_request: "2" + memory_request: 8Gi + + - provider: kubernetes + init_on_start: false + worker_config: + worker_type: gpu + gpu_count: 1 + # Torch dataloaders need more than the 64Mi default /dev/shm. + shm_size: 8Gi + node_selector: + nvidia.com/gpu.present: "true" + tolerations: + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule diff --git a/cli/stack/src/flowmesh_cli_stack/env_schema.py b/cli/stack/src/flowmesh_cli_stack/env_schema.py index 445485f48..f76c0ec5b 100644 --- a/cli/stack/src/flowmesh_cli_stack/env_schema.py +++ b/cli/stack/src/flowmesh_cli_stack/env_schema.py @@ -125,6 +125,74 @@ def _warn_reaper_without_watchdog( ), ], ), + EnvSection( + title="Kubernetes Backend", + description=[ + "Used when STACK_BACKEND=k8s. The stack is deployed into one", + "namespace; the Kubernetes scheduler places workers across nodes.", + "A gRPC TLS certificate must carry the supervisor Service name", + "as a SAN, since workers dial the Service, not a host.", + ], + vars=[ + EnvVar( + "STACK_BACKEND", + "compose", + var_type=EnvVarType.ENUM, + choices=("compose", "k8s"), + ), + EnvVar("K8S_NAMESPACE", "flowmesh"), + EnvVar( + "K8S_WORKER_NAMESPACE", + description=[ + "Namespace for worker pods; defaults to K8S_NAMESPACE." + ], + ), + EnvVar( + "K8S_CONTEXT", description=["kubectl context; empty uses current."] + ), + EnvVar( + "K8S_KUBECONFIG", + description=["kubeconfig path; empty uses default."], + ), + EnvVar("K8S_SUPERVISOR_SERVICE", "flowmesh-supervisor"), + EnvVar("K8S_SERVER_SERVICE", "flowmesh-server"), + EnvVar("K8S_CLUSTER_DOMAIN", "cluster.local"), + EnvVar("K8S_GPU_RESOURCE_NAME", "nvidia.com/gpu"), + EnvVar( + "K8S_SERVER_SERVICE_TYPE", + "ClusterIP", + var_type=EnvVarType.ENUM, + choices=("ClusterIP", "NodePort", "LoadBalancer"), + ), + EnvVar("K8S_IMAGE_PULL_POLICY", "IfNotPresent"), + EnvVar( + "K8S_STORAGE_CLASS", + description=[ + "Storage class for stack volumes; empty uses default." + ], + ), + EnvVar("K8S_REDIS_STORAGE_SIZE", "8Gi"), + EnvVar("K8S_RESULTS_STORAGE_SIZE", "20Gi"), + EnvVar("K8S_RESULTS_ACCESS_MODE", "ReadWriteOnce"), + EnvVar( + "K8S_ENABLE_NODE_RBAC", + "false", + var_type=EnvVarType.BOOL, + description=[ + "Grant cluster-scoped node read access so worker hardware", + "can be reported before a worker starts.", + ], + ), + EnvVar( + "SERVER_GRPC_TLS_SECRET", + description=["Secret holding the server gRPC TLS files."], + ), + EnvVar( + "REDIS_TLS_SECRET", + description=["Secret holding the Redis TLS files."], + ), + ], + ), EnvSection( title="Supervisor gRPC", description=[ diff --git a/cli/stack/src/flowmesh_cli_stack/k8s.py b/cli/stack/src/flowmesh_cli_stack/k8s.py new file mode 100644 index 000000000..c371d6fd7 --- /dev/null +++ b/cli/stack/src/flowmesh_cli_stack/k8s.py @@ -0,0 +1,229 @@ +"""Kubernetes backend for the FlowMesh stack lifecycle commands.""" + +import os +import subprocess +from collections.abc import Iterator +from contextlib import contextmanager +from enum import StrEnum +from pathlib import Path + +import typer +from flowmesh.models.nodes import NodeRole +from flowmesh_cli.core import logging +from flowmesh_cli.core.assets import asset_path +from flowmesh_stack.env import ensure_env_file, load_env, parse_env_file +from flowmesh_stack.kubernetes import KubectlError, KubernetesStack +from flowmesh_stack.manifests import ManifestError +from flowmesh_stack.paths import resolve_path + +from .utils import DEFAULT_ENV_FILE, STACK_PATH_KEYS, drain_workers, stack_env_example + +MANIFEST_ASSETS = ( + "00-namespace.yaml", + "10-rbac.yaml", + "20-redis.yaml", + "30-server.yaml", +) +"""Manifest assets applied in order; the namespace must come first.""" + +SERVER_WORKLOAD = "deployment/flowmesh-server" +REDIS_WORKLOADS = ( + "statefulset/flowmesh-redis-control", + "statefulset/flowmesh-redis-telemetry", +) +STACK_WORKLOADS = { + "server": SERVER_WORKLOAD, + "redis_control": REDIS_WORKLOADS[0], + "redis_telemetry": REDIS_WORKLOADS[1], +} +ROLLOUT_TIMEOUT = "300s" + +DEFAULT_NAMESPACE = "flowmesh" +DEFAULT_WORKER_CONFIG = "./configs/worker_config.yaml" + + +class StackBackend(StrEnum): + COMPOSE = "compose" + K8S = "k8s" + + +def resolve_backend(value: str | None, env_file: Path) -> StackBackend: + """Resolve the stack backend from the option, falling back to the env file.""" + raw = (value or "").strip() + if not raw: + raw = parse_env_file(env_file).get("STACK_BACKEND", "").strip() + if not raw: + return StackBackend.COMPOSE + try: + return StackBackend(raw.lower()) + except ValueError: + logging.error( + f"Unknown stack backend {raw!r}; " + f"expected one of {', '.join(StackBackend)}." + ) + raise typer.Exit(code=1) from None + + +def manifest_paths() -> list[Path]: + return [ + asset_path("flowmesh_cli_stack.assets", "k8s", name) for name in MANIFEST_ASSETS + ] + + +def apply_k8s_env(base_dir: Path) -> None: + """Fill in the derived Kubernetes values the manifests reference. + + ``K8S_WORKER_NAMESPACE`` defaults to the stack namespace and + ``SERVER_WORKER_CONFIG`` to the path the compose backend bind-mounts, + because manifest substitution has no nested defaults. The distinctness of + the worker namespace is precomputed for the same reason: the manifest + conditionals compare against a value, not against another variable. + """ + namespace = os.environ.get("K8S_NAMESPACE", "").strip() or DEFAULT_NAMESPACE + os.environ["K8S_NAMESPACE"] = namespace + worker_namespace = os.environ.get("K8S_WORKER_NAMESPACE", "").strip() or namespace + os.environ["K8S_WORKER_NAMESPACE"] = worker_namespace + os.environ["K8S_WORKER_NAMESPACE_DISTINCT"] = ( + "true" if worker_namespace != namespace else "false" + ) + os.environ["SERVER_WORKER_CONFIG"] = resolve_path( + os.environ.get("SERVER_WORKER_CONFIG", ""), + default=DEFAULT_WORKER_CONFIG, + base_dir=base_dir, + ).as_posix() + + +def node_role() -> NodeRole: + raw = os.environ.get("NODE_ROLE", "").strip() + return NodeRole(raw.lower()) if raw else NodeRole.ROOT + + +def stack(env_file: Path, image_tag: str | None = None) -> KubernetesStack: + def _load(path: Path) -> None: + ensure_env_file(path, stack_env_example()) + load_env(path, base_dir=Path.cwd(), path_keys=STACK_PATH_KEYS) + apply_k8s_env(Path.cwd()) + if image_tag: + os.environ["FLOWMESH_VERSION"] = image_tag + + _load(env_file) + return KubernetesStack( + manifests=manifest_paths(), + namespace=os.environ["K8S_NAMESPACE"], + load_env=_load, + context=os.environ.get("K8S_CONTEXT", "").strip() or None, + kubeconfig=os.environ.get("K8S_KUBECONFIG", "").strip() or None, + ) + + +def workloads() -> list[str]: + """Return the workloads this node runs; Redis only on a root node.""" + if node_role() is NodeRole.ROOT: + return [*REDIS_WORKLOADS, SERVER_WORKLOAD] + return [SERVER_WORKLOAD] + + +def _check(result: subprocess.CompletedProcess) -> None: + if result.returncode != 0: + raise typer.Exit(code=result.returncode) + + +@contextmanager +def _reporting(action: str) -> Iterator[None]: + try: + yield + except (KubectlError, ManifestError) as exc: + logging.error(f"Failed to {action}: {exc}") + raise typer.Exit(code=1) from exc + + +def _resolve_workloads(services: list[str] | None) -> list[str]: + if not services: + return workloads() + + requested = list(dict.fromkeys(services)) + unknown = [name for name in requested if name not in STACK_WORKLOADS] + if unknown: + logging.error( + f"Unknown service(s): {', '.join(unknown)}. " + f"Known services: {', '.join(STACK_WORKLOADS)}." + ) + raise typer.Exit(code=1) + return [STACK_WORKLOADS[name] for name in requested] + + +def up(env_file: Path = DEFAULT_ENV_FILE, image_tag: str | None = None) -> None: + """Apply the stack manifests and wait for the workloads to become ready.""" + with _reporting("apply the stack"): + target = stack(env_file, image_tag) + _check(target.apply(env_file)) + for workload in workloads(): + _check(target.rollout_status(workload, ROLLOUT_TIMEOUT)) + logging.success("FlowMesh stack is up.") + + +def down(env_file: Path = DEFAULT_ENV_FILE, image_tag: str | None = None) -> None: + """Drain workers and delete the stack resources.""" + logging.info("Draining workers...") + drain_workers(env_file) + logging.info("Shutting down the FlowMesh stack...") + with _reporting("delete the stack"): + target = stack(env_file, image_tag) + _check(target.delete_workers()) + _check(target.delete(env_file)) + logging.success("FlowMesh stack stopped.") + + +def restart( + services: list[str] | None = None, + env_file: Path = DEFAULT_ENV_FILE, + image_tag: str | None = None, +) -> None: + """Restart the stack, or the named workloads, in place. + + An image tag change is applied rather than rolled, because a rollout + restart does not change the pod template's image. + """ + with _reporting("restart the stack"): + target = stack(env_file, image_tag) + requested = _resolve_workloads(services) + + if SERVER_WORKLOAD in requested: + logging.info("Draining workers...") + drain_workers(env_file) + + if image_tag: + _check(target.apply(env_file)) + else: + for workload in requested: + _check(target.rollout_restart(workload)) + + for workload in requested: + _check(target.rollout_status(workload, ROLLOUT_TIMEOUT)) + logging.success("FlowMesh stack restarted.") + + +def logs(service: str | None = None, env_file: Path = DEFAULT_ENV_FILE) -> None: + """Stream logs from a stack workload.""" + with _reporting("stream logs"): + target = stack(env_file) + workload = _resolve_workloads([service])[0] if service else SERVER_WORKLOAD + _check(target.logs(workload)) + + +def ps(env_file: Path = DEFAULT_ENV_FILE) -> None: + """Show stack pods and supervisor-managed worker pods.""" + with _reporting("read stack status"): + target = stack(env_file) + _check(target.status()) + logging.log("\nWorkers:") + _check(target.worker_status()) + + +def clean(env_file: Path = DEFAULT_ENV_FILE, image_tag: str | None = None) -> None: + """Drain workers, delete the stack, and remove its persistent volumes.""" + down(env_file, image_tag) + logging.info("Removing stack volumes...") + with _reporting("remove stack volumes"): + _check(stack(env_file, image_tag).delete_volumes()) + logging.success("FlowMesh stack cleaned.") diff --git a/cli/stack/src/flowmesh_cli_stack/stack.py b/cli/stack/src/flowmesh_cli_stack/stack.py index bce6361da..b716d47e7 100644 --- a/cli/stack/src/flowmesh_cli_stack/stack.py +++ b/cli/stack/src/flowmesh_cli_stack/stack.py @@ -28,19 +28,21 @@ get_push_platforms, ) +from . import k8s from .env_schema import STACK_ENV_SCHEMA, deploy_overrides, role_overrides +from .k8s import StackBackend, resolve_backend from .utils import ( DEFAULT_ENV_FILE, STACK_PATH_KEYS, apply_plugin_data_env, apply_stack_resource_env, + drain_workers, ensure_deploy_paths, parse_node_role, resolve_package_version, stack_bake_file, stack_compose_file, stack_env_example, - stack_node_client, ) from .worker import worker_pull @@ -478,6 +480,9 @@ def up( image_tag: str | None = typer.Option( None, "--image-tag", help="Override FLOWMESH_VERSION" ), + backend: str | None = typer.Option( + None, "--backend", help="Stack backend to target (compose|k8s)" + ), ) -> None: """Start the stack. @@ -486,6 +491,10 @@ def up( services are skipped — the worker is expected to connect to the root node's Redis via REDIS_CONTROL_URL / REDIS_TELEMETRY_URL. """ + if resolve_backend(backend, env_file) is StackBackend.K8S: + k8s.up(env_file=env_file, image_tag=image_tag) + return + profile = "root" if _node_role(env_file) == NodeRole.ROOT else None _compose( ["up", "-d", "--wait"], @@ -497,15 +506,6 @@ def up( logging.success("FlowMesh stack is up.") -def _drain_workers(env_file: Path) -> None: - """Destroy all dynamically spawned workers before stopping the server.""" - try: - client = stack_node_client(env_file, base_url=None, token=None) - client.destroy_all_workers() - except Exception as exc: - logging.warning(f"Unable to drain workers; continuing shutdown. {exc}") - - @app.command() def down( env_file: Path = typer.Option( @@ -514,10 +514,17 @@ def down( image_tag: str | None = typer.Option( None, "--image-tag", help="Override FLOWMESH_VERSION" ), + backend: str | None = typer.Option( + None, "--backend", help="Stack backend to target (compose|k8s)" + ), ) -> None: """Drain workers and stop the stack.""" + if resolve_backend(backend, env_file) is StackBackend.K8S: + k8s.down(env_file=env_file, image_tag=image_tag) + return + logging.info("Draining workers...") - _drain_workers(env_file) + drain_workers(env_file) logging.info("Shutting down the FlowMesh stack...") _compose( ["down"], @@ -554,6 +561,9 @@ def restart( pull: bool = typer.Option( True, "--pull/--no-pull", help="Pull the target image before recreating." ), + backend: str | None = typer.Option( + None, "--backend", help="Stack backend to target (compose|k8s)" + ), ) -> None: """Drain workers and restart the stack, or recreate specific services in place. @@ -561,9 +571,13 @@ def restart( are recreated; when any of them manages workers (the server / supervisor) its workers are drained first so their in-flight tasks requeue onto other nodes. """ + if resolve_backend(backend, env_file) is StackBackend.K8S: + k8s.restart(services=services, env_file=env_file, image_tag=image_tag) + return + if not services: logging.info("Draining workers...") - _drain_workers(env_file) + drain_workers(env_file) _compose( ["down"], env_file=env_file, @@ -592,7 +606,7 @@ def restart( if any(svc in WORKER_MANAGING_SERVICES for svc in requested): logging.info("Draining workers...") - _drain_workers(env_file) + drain_workers(env_file) profile = "root" if _node_role(env_file) == NodeRole.ROOT else None up_args = ["up", "-d", "--no-deps", "--force-recreate", "--wait"] @@ -617,8 +631,15 @@ def logs( env_file: Path = typer.Option( DEFAULT_ENV_FILE, "--env-file", help="Env file for compose" ), + backend: str | None = typer.Option( + None, "--backend", help="Stack backend to target (compose|k8s)" + ), ) -> None: """Stream logs from stack services or a specific service container.""" + if resolve_backend(backend, env_file) is StackBackend.K8S: + k8s.logs(service=service, env_file=env_file) + return + code = _stack().stream_logs(env_file=env_file, service=service, profile="root") if code != 0: raise typer.Exit(code=code) @@ -629,8 +650,15 @@ def ps( env_file: Path = typer.Option( DEFAULT_ENV_FILE, "--env-file", help="Env file for compose" ), + backend: str | None = typer.Option( + None, "--backend", help="Stack backend to target (compose|k8s)" + ), ) -> None: """Display running status of stack containers and worker containers.""" + if resolve_backend(backend, env_file) is StackBackend.K8S: + k8s.ps(env_file=env_file) + return + _compose(["ps"], env_file=env_file, env=None, profile="root") logging.log("\nWorkers:") docker_bin = _require_bin("docker") @@ -653,9 +681,12 @@ def status_cmd( env_file: Path = typer.Option( DEFAULT_ENV_FILE, "--env-file", help="Env file for compose" ), + backend: str | None = typer.Option( + None, "--backend", help="Stack backend to target (compose|k8s)" + ), ) -> None: """Display running status of stack containers (alias for ps).""" - ps(env_file=env_file) + ps(env_file=env_file, backend=backend) @app.command() @@ -666,10 +697,17 @@ def clean( image_tag: str | None = typer.Option( None, "--image-tag", help="Override FLOWMESH_VERSION" ), + backend: str | None = typer.Option( + None, "--backend", help="Stack backend to target (compose|k8s)" + ), ) -> None: """Drain workers, stop the stack, and remove all containers and volumes.""" + if resolve_backend(backend, env_file) is StackBackend.K8S: + k8s.clean(env_file=env_file, image_tag=image_tag) + return + logging.info("Draining workers...") - _drain_workers(env_file) + drain_workers(env_file) logging.info("Removing stack containers and volumes...") _compose( ["down", "-v"], diff --git a/cli/stack/src/flowmesh_cli_stack/utils.py b/cli/stack/src/flowmesh_cli_stack/utils.py index 011eb14c5..d8997e4e3 100644 --- a/cli/stack/src/flowmesh_cli_stack/utils.py +++ b/cli/stack/src/flowmesh_cli_stack/utils.py @@ -98,6 +98,15 @@ def stack_node_client( return NodeClient(resolved_base, token=resolved_token) +def drain_workers(env_file: Path) -> None: + """Destroy all dynamically spawned workers before stopping the server.""" + try: + client = stack_node_client(env_file, base_url=None, token=None) + client.destroy_all_workers() + except Exception as exc: + logging.warning(f"Unable to drain workers; continuing shutdown. {exc}") + + def flowmesh_client( env_file: Path, base_url: str | None, api_key: str | None ) -> FlowMesh: diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 03058154b..fbcc47b5b 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -40,11 +40,32 @@ The runtime is two top-level processes: workflow / task / dispatch logic and the **Supervisor subsystem** (`src/server/supervisor/`), which manages per-node worker lifecycle, runs the worker-facing gRPC server (`:50051`), and drives the - Docker / Vast.ai worker adapters. + Docker / Kubernetes / Vast.ai worker adapters. 2. **Worker** (`src/worker/`) — stateless executor. Connects to a supervisor via gRPC, receives tasks, runs the matching executor, reports results. +### Kubernetes + +On Kubernetes one namespace is one FlowMesh node: a single server Deployment +runs `NODE_ROLE=root` with its supervisor, and the cluster scheduler places +workers across machines. + +``` +Client ──▶ Service flowmesh-server ──▶ Deployment flowmesh-server (replicas: 1) + │ ServiceAccount flowmesh-server + ├─▶ StatefulSet redis-control / redis-telemetry + └─▶ Pod flowmesh-worker-* (nvidia.com/gpu: N) +``` + +The server runs as a single replica with the `Recreate` strategy. A worker's +token lives in the registry of the supervisor that minted it, so a second +replica behind the Service would reject those registrations. Workers dial the +supervisor through a headless Service, which a gRPC TLS certificate must name +as a SAN. + +See [`KUBERNETES.md`](KUBERNETES.md). + ## Communication - **server ↔ supervisor (same node)** — `multiprocessing.Queue`. @@ -127,6 +148,12 @@ scripts/dev/ compile_protos, sync_requirements, check_env_examples `WorkerHardware`. The dispatcher's `_cached_worker_candidates` filters to workers whose cache covers the task's references; entries older than `WORKER_CACHE_TTL_SEC` are ignored. +- **Worker providers.** A worker is created through the provider named in + `worker_config.yaml`: `docker` (containers on the node's own daemon), + `kubernetes` (pods through the cluster API), `vastai` (rented instances), or + `external` (workers this supervisor does not launch, admitted by a shared + secret). A spawning provider whose backend is unreachable at startup is + dropped, so a node advertises only what it can actually serve. - **Worker capabilities.** Beyond hardware fit, each worker advertises the set of task types it can service, and the dispatcher routes a task only to workers that advertise its type. A worker advertises a type only when its executor came diff --git a/docs/CLI.md b/docs/CLI.md index 84fa6acac..21d6ff83b 100644 --- a/docs/CLI.md +++ b/docs/CLI.md @@ -95,6 +95,21 @@ flowmesh stack worker up cpu 2 --name-template '{slug}-run-{idx}' # {slug}-run- The template must keep names unique within one `up` invocation — include `{idx}` or `{gpu}` when creating more than one worker. +### Kubernetes backend + +`--backend k8s` targets a Kubernetes cluster instead of the local Docker +daemon; `STACK_BACKEND=k8s` in the env file makes the flag unnecessary. It +applies to `up`, `down`, `restart`, `logs`, `ps`, and `clean`. + +```bash +flowmesh stack up --backend k8s # apply manifests, wait for rollout +flowmesh stack ps --backend k8s # stack pods and worker pods +flowmesh stack down --backend k8s +``` + +`build`, `push`, `pull`, and `pullall` remain Docker-only — each node's kubelet +pulls images itself. See [`KUBERNETES.md`](KUBERNETES.md). + `flowmesh stack up` reads `NODE_ROLE` from the env file (default `root`). On a root node, both local Redis services are deployed alongside the server. On a worker node (`NODE_ROLE=worker`), Redis services are skipped — the worker diff --git a/docs/ENV.md b/docs/ENV.md index 482f3a944..1d30fd42d 100644 --- a/docs/ENV.md +++ b/docs/ENV.md @@ -109,3 +109,28 @@ cap)`. A task that requests more than the worker cap is dispatched to another worker if one has a larger cap; otherwise the dispatcher follows its standard requeue/retry behavior. The worker logs a startup warning if SSH is enabled with no cap configured. + +## Kubernetes backend + +Used when `STACK_BACKEND=k8s`. See [`KUBERNETES.md`](KUBERNETES.md). + +| Variable | Default | Purpose | +|----------|---------|---------| +| `STACK_BACKEND` | `compose` | Stack backend: `compose` or `k8s`. | +| `K8S_NAMESPACE` | `flowmesh` | Namespace the stack is deployed into. | +| `K8S_WORKER_NAMESPACE` | `K8S_NAMESPACE` | Namespace for worker pods. | +| `K8S_CONTEXT` | current | kubectl context to target. | +| `K8S_KUBECONFIG` | default | kubeconfig file to use. | +| `K8S_SUPERVISOR_SERVICE` | `flowmesh-supervisor` | Headless Service workers dial for gRPC. | +| `K8S_SERVER_SERVICE` | `flowmesh-server` | Service workers use for artifact transfer. | +| `K8S_CLUSTER_DOMAIN` | `cluster.local` | Cluster DNS domain. | +| `K8S_GPU_RESOURCE_NAME` | `nvidia.com/gpu` | Extended resource requested for GPU workers. | +| `K8S_SERVER_SERVICE_TYPE` | `ClusterIP` | `ClusterIP`, `NodePort`, or `LoadBalancer`. | +| `K8S_IMAGE_PULL_POLICY` | `IfNotPresent` | Pull policy for the server image. | +| `K8S_STORAGE_CLASS` | cluster default | Storage class for stack volumes. | +| `K8S_REDIS_STORAGE_SIZE` | `8Gi` | Size of each Redis volume. | +| `K8S_RESULTS_STORAGE_SIZE` | `20Gi` | Size of the server results volume. | +| `K8S_RESULTS_ACCESS_MODE` | `ReadWriteOnce` | Access mode for the results volume. | +| `K8S_ENABLE_NODE_RBAC` | `false` | Grant cluster-scoped node reads for pre-start hardware reporting. | +| `SERVER_GRPC_TLS_SECRET` | — | Secret holding the server gRPC TLS files. | +| `REDIS_TLS_SECRET` | — | Secret holding the Redis TLS files. | diff --git a/docs/KUBERNETES.md b/docs/KUBERNETES.md new file mode 100644 index 000000000..e248f829f --- /dev/null +++ b/docs/KUBERNETES.md @@ -0,0 +1,171 @@ +# Kubernetes deployment + +FlowMesh deploys into an existing Kubernetes cluster with +`flowmesh stack ... --backend k8s`. One namespace holds one FlowMesh node: a +server Deployment with its supervisor, two Redis StatefulSets, and the worker +pods the supervisor creates. + +## Prerequisites + +- A reachable cluster and `kubectl` on `PATH`. +- A storage class for the Redis and results volumes, or `K8S_STORAGE_CLASS` + pointing at one. +- For GPU workers: the NVIDIA device plugin, so nodes advertise + `nvidia.com/gpu`. + +## Deploying + +```bash +flowmesh stack init # scaffold .env +# set STACK_BACKEND=k8s and K8S_NAMESPACE in .env +flowmesh stack up --backend k8s # apply manifests, wait for rollout +flowmesh stack ps --backend k8s # stack pods and worker pods +flowmesh stack logs server --backend k8s +flowmesh stack down --backend k8s +``` + +`STACK_BACKEND=k8s` in the env file makes `--backend` unnecessary on every +call. The same `.env` drives both backends. + +`flowmesh stack up` applies the namespace, RBAC, Redis, and server manifests in +that order, then waits on each workload's rollout. + +`down` drains workers and deletes the workloads, leaving the namespace and the +persistent volume claims in place, so task results and Redis state survive a +teardown. `clean` does that and then removes the claims — the same split +compose has between `down` and `down -v`. + +`restart` rolls the workloads in place. With `--image-tag` it applies instead, +because a rollout restart does not change the pod template's image. + +`build`, `push`, `pull`, and `pullall` act on a local Docker daemon and take no +`--backend`; on Kubernetes each node's kubelet pulls images itself. + +## Server topology + +The server Deployment runs `replicas: 1` with the `Recreate` strategy. A +worker's token lives in the registry of the supervisor that minted it, so a +second replica behind the Service would reject registrations from workers the +other replica started. Scheduling state is persisted to Redis and rebuilt on +startup, so restarting the pod is safe. + +Two Services front the Deployment: + +| Service | Purpose | +|---------|---------| +| `flowmesh-server` | REST API on `SERVER_HTTP_PORT`. `K8S_SERVER_SERVICE_TYPE` selects `ClusterIP`, `NodePort`, or `LoadBalancer`. | +| `flowmesh-supervisor` | Headless; worker pods dial it for gRPC on `SERVER_GRPC_PORT`. | + +Setting `K8S_WORKER_NAMESPACE` to something other than `K8S_NAMESPACE` puts +worker pods in their own namespace; that namespace and the Role binding the +server needs in it are created alongside the stack. + +The whole env file is carried into the pod as the `flowmesh-server-env` Secret, +which is the Kubernetes equivalent of compose's `env_file`. Values come from +the env file alone, so nothing else in the operator's shell reaches the +cluster. + +## Workers + +Worker pods are created by the supervisor through the Kubernetes API, not by a +separate manifest. Point `SERVER_WORKER_CONFIG` at a worker config declaring +the `kubernetes` provider; a starting point ships as +`cli/stack/src/flowmesh_cli_stack/assets/k8s/worker_config.k8s.yaml`. + +```yaml +workers: + - provider: kubernetes + init_on_start: true + worker_config: + worker_type: gpu + gpu_count: 1 + shm_size: 8Gi + node_selector: + nvidia.com/gpu.present: "true" +``` + +Each worker pod: + +- runs with `restartPolicy: Always` and carries the labels + `flowmesh.io/managed`, `flowmesh.io/node-alias`, and + `flowmesh.io/worker-name`; +- receives its credentials from a per-worker Secret through `envFrom`, never + inline in the pod spec; +- requests GPUs as `nvidia.com/gpu` (or `gpu_resource_name`) and leaves device + assignment to the device plugin; +- has no owner reference to the server pod, so a server rollout does not take + running workers with it. Pods left by an unclean exit are reaped at + supervisor startup, because their tokens died with the previous registry. + +Set `shm_size` for training and inference workloads; the 64 MiB default +`/dev/shm` is too small for torch dataloaders. + +### Configuration reference + +`results_pvc`, `results_mount_path`, `hf_cache_pvc`, `cpu_request`, +`cpu_limit`, `memory_request`, `memory_limit`, `tolerations`, +`service_account_name`, `image_pull_secrets`, `runtime_class_name`, +`priority_class_name`, `pod_labels`, and `pod_annotations` map onto their pod +spec equivalents. `pod_overrides` is merged onto the generated manifest for +anything else — mappings merge recursively, sequences and scalars replace. + +### Results + +Workflows declare where their output goes. Without a `results_pvc` a worker +writes to an ephemeral volume, so a workflow that keeps results either sets +`output.destination`, enables `WORKER_UPLOAD_RESULTS` to upload them to the +server, or names a claim through `results_pvc`. + +### SSH tasks + +Worker pods have no Docker socket, so the SSH executor does not load and +workers never advertise the `ssh` task type. The dispatcher routes SSH tasks +only to workers that advertise it. + +## RBAC + +`flowmesh stack up` creates a `flowmesh-server` ServiceAccount and a namespaced +Role granting pods, pod logs, secrets, and events — everything the supervisor +needs to run workers. + +Setting `K8S_ENABLE_NODE_RBAC=true` additionally binds a cluster-scoped +ClusterRole granting `get` and `list` on nodes. This is the one cluster-scoped +grant, and it is optional: it lets the supervisor report a worker's hardware +from node allocatable capacity and GPU labels before the worker starts. +Without it that preview is empty, and hardware still arrives from the worker +itself at registration. + +## TLS + +`SERVER_GRPC_TLS_SECRET` names a Secret mounted at `/etc/ssl/server`, and +`REDIS_TLS_SECRET` one mounted at `/etc/ssl/redis`. The certificate must carry +the supervisor Service DNS name +(`flowmesh-supervisor..svc.cluster.local`) as a SAN, because that is +the name workers dial. The CA reaches workers automatically. + +## Joining an external root node + +A cluster can also run as a worker node against a root node deployed elsewhere. +Set `NODE_ROLE=worker` and point `REDIS_CONTROL_URL` and `REDIS_TELEMETRY_URL` +at the root node's Redis; the Redis StatefulSets are then not deployed. + +## Choosing a worker provider + +Two providers serve Kubernetes, differing in who owns the pod lifecycle. + +| | `external` | `kubernetes` | +|---|---|---| +| Creates workers | you, via your own Deployment or controller | the supervisor, through the cluster API | +| Scaling | `kubectl scale`, HPA, your controller | `flowmesh worker create` / `destroy` | +| Pod spec | your manifest, the full API | this provider's config fields plus `pod_overrides` | +| Server RBAC | none | pods and secrets in the worker namespace | +| Survives a supervisor restart | yes — tokens re-derive from the shared secret | no — supervisor-minted tokens are lost and the pods are reaped | + +Use `external` when the cluster or your own controller owns worker lifecycle +and the pool is stable. Use `kubernetes` when FlowMesh should create and destroy +workers itself — bursty workloads, or per-workflow GPU shapes that would +otherwise need a Deployment each. + +A `kubernetes` worker is a bare Pod, so it restarts in place but is not +rescheduled if its node is lost. Pools on preemptible or frequently drained +nodes are better served by `external`. diff --git a/pyproject.toml b/pyproject.toml index 30f3c4acb..735682266 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,7 @@ runtime-server = [ "flowmesh-hook", "grpcio>=1.76.0", "httpx>=0.28.1", + "kubernetes>=31.0.0", "lumid-hooks>=0.2.0", "protobuf>=5.29.6", "pydantic>=2.12.3", @@ -212,7 +213,7 @@ module = [ ignore_missing_imports = true [[tool.mypy.overrides]] -module = ["pynvml", "vllm_omni.*", "deepspeed.*"] +module = ["pynvml", "vllm_omni.*", "deepspeed.*", "kubernetes.*"] follow_untyped_imports = true [tool.codespell] diff --git a/sdk/stack/src/flowmesh_stack/kubernetes.py b/sdk/stack/src/flowmesh_stack/kubernetes.py new file mode 100644 index 000000000..7aecf13e3 --- /dev/null +++ b/sdk/stack/src/flowmesh_stack/kubernetes.py @@ -0,0 +1,170 @@ +"""kubectl helpers for the FlowMesh Kubernetes stack backend.""" + +import os +import shutil +import subprocess +from collections.abc import Callable, Mapping +from dataclasses import dataclass, field +from pathlib import Path + +import yaml + +from .env import parse_env_file +from .manifests import render_manifests + +MANAGED_LABEL = "flowmesh.io/managed" +"""Label carried by worker pods the supervisor created.""" + +STACK_LABEL = "app.kubernetes.io/part-of" +"""Label carried by every stack resource.""" + +STACK_LABEL_VALUE = "flowmesh" + +DATA_KINDS = frozenset({"Namespace", "PersistentVolumeClaim"}) +"""Kinds a stack teardown leaves in place, so results and Redis state survive it.""" + + +class KubectlError(RuntimeError): + """Raised when a kubectl command fails.""" + + +def ensure_kubectl_available() -> str: + """Return the absolute kubectl path, raising when it is not installed.""" + path = shutil.which("kubectl") + if path is None: + raise KubectlError("kubectl is required but was not found in PATH") + return path + + +def kubectl( + args: list[str], + namespace: str | None = None, + context: str | None = None, + kubeconfig: str | None = None, + stdin: str | None = None, + env: Mapping[str, str] | None = None, + capture_output: bool = False, +) -> subprocess.CompletedProcess[str]: + """Run kubectl with the provided arguments.""" + cmd = [ensure_kubectl_available()] + if kubeconfig: + cmd += ["--kubeconfig", kubeconfig] + if context: + cmd += ["--context", context] + if namespace: + cmd += ["--namespace", namespace] + cmd += args + + merged_env = dict(os.environ) + if env: + merged_env.update(env) + + return ( + subprocess.run( # nosec B603: argv list, no shell, absolute path via which(). + cmd, + check=False, + input=stdin, + capture_output=capture_output, + text=True, + env=merged_env, + ) + ) + + +@dataclass +class KubernetesStack: + """Applies and inspects the FlowMesh stack in a Kubernetes namespace.""" + + manifests: list[Path] + """Manifest assets rendered for every apply.""" + namespace: str + """Namespace the stack is deployed into.""" + load_env: Callable[[Path], None] + """Callback that loads and resolves env-file values before operations run.""" + context: str | None = None + """kubectl context to target.""" + kubeconfig: str | None = None + """kubeconfig file to use instead of the default.""" + rollout_targets: list[str] = field(default_factory=list) + """Workloads that ``up`` waits on before reporting success.""" + + def render(self, env_file: Path) -> str: + """Render the stack manifests using the resolved environment. + + Substitution reads the resolved process environment; the values handed + to the cluster come from the environment file alone, so nothing else in + the operator's shell is copied into the namespace. + """ + self.load_env(env_file) + return render_manifests( + self.manifests, dict(os.environ), parse_env_file(env_file) + ) + + def _run( + self, args: list[str], stdin: str | None = None, capture_output: bool = False + ) -> subprocess.CompletedProcess[str]: + return kubectl( + args, + namespace=self.namespace, + context=self.context, + kubeconfig=self.kubeconfig, + stdin=stdin, + capture_output=capture_output, + ) + + def apply(self, env_file: Path) -> subprocess.CompletedProcess[str]: + """Apply the rendered stack manifests.""" + return self._run(["apply", "-f", "-"], stdin=self.render(env_file)) + + def delete( + self, env_file: Path, keep_data: bool = True + ) -> subprocess.CompletedProcess[str]: + """Delete the resources described by the rendered stack manifests. + + The namespace and persistent volume claims are kept by default: + deleting the namespace cascades to every claim in it, which would make + a teardown destroy task results and Redis state. Removing them is + ``clean``'s job, matching what compose volumes do. + """ + stream = self.render(env_file) + if keep_data: + kept = [ + document + for document in yaml.safe_load_all(stream) + if document and document.get("kind") not in DATA_KINDS + ] + stream = yaml.safe_dump_all(kept, sort_keys=False) + return self._run(["delete", "-f", "-", "--ignore-not-found"], stdin=stream) + + def rollout_status(self, target: str, timeout: str) -> subprocess.CompletedProcess: + """Wait for a workload to finish rolling out.""" + return self._run(["rollout", "status", target, f"--timeout={timeout}"]) + + def rollout_restart(self, target: str) -> subprocess.CompletedProcess[str]: + """Restart a workload in place.""" + return self._run(["rollout", "restart", target]) + + def logs(self, target: str, follow: bool = True) -> subprocess.CompletedProcess: + """Stream logs for a workload or pod.""" + args = ["logs", target, "--all-containers"] + if follow: + args.append("-f") + return self._run(args) + + def status(self) -> subprocess.CompletedProcess[str]: + """Show stack pods.""" + return self._run( + ["get", "pods", "-l", f"{STACK_LABEL}={STACK_LABEL_VALUE}", "-o", "wide"] + ) + + def worker_status(self) -> subprocess.CompletedProcess[str]: + """Show pods the supervisor created for workers.""" + return self._run(["get", "pods", "-l", f"{MANAGED_LABEL}=true", "-o", "wide"]) + + def delete_workers(self) -> subprocess.CompletedProcess[str]: + """Delete every supervisor-managed worker pod in the namespace.""" + return self._run(["delete", "pods", "-l", f"{MANAGED_LABEL}=true"]) + + def delete_volumes(self) -> subprocess.CompletedProcess[str]: + """Delete the stack's persistent volume claims.""" + return self._run(["delete", "pvc", "-l", f"{STACK_LABEL}={STACK_LABEL_VALUE}"]) diff --git a/sdk/stack/src/flowmesh_stack/manifests.py b/sdk/stack/src/flowmesh_stack/manifests.py new file mode 100644 index 000000000..5dcb772bc --- /dev/null +++ b/sdk/stack/src/flowmesh_stack/manifests.py @@ -0,0 +1,199 @@ +"""Rendering of Kubernetes manifest assets from stack environment values. + +Assets are ordinary multi-document YAML carrying a small set of extensions: + +``${VAR}`` / ``${VAR:-default}`` + Compose-compatible substitution, so one ``.env`` drives both stack + backends. A reference with neither a value nor a default is an error, so a + misconfigured stack fails while rendering rather than while applying. + +``x-flowmesh-when: `` + Conditional inclusion. The mapping carrying the key is dropped when the + expression is falsey. A dropped document leaves the stream, a dropped + sequence entry leaves its sequence, and a dropped mapping value takes its + key with it. Expressions are ``VAR``, ``!VAR``, or ``VAR==value``. + +``x-flowmesh-value: `` + Renders to the node itself, so ``x-flowmesh-when`` can make a single scalar + conditional inside a list of arguments. + +``x-flowmesh-int: `` + Renders to an integer. Ports and replica counts are rejected by the API as + strings, and substitution otherwise always yields a string. + +``x-flowmesh-file: `` + Renders to the contents of the file named by the environment variable + ``VAR``, for configuration the compose backend bind-mounts. + +``x-flowmesh-env-values`` + The mapping carrying the key is replaced by the environment file's own + key/value pairs, which is how the server receives the configuration compose + passes through ``env_file``. Values come from the environment file alone, + never from the caller's process environment. +""" + +import re +from collections.abc import Mapping +from pathlib import Path +from typing import Any + +import yaml + +WHEN_KEY = "x-flowmesh-when" +VALUE_KEY = "x-flowmesh-value" +INT_KEY = "x-flowmesh-int" +FILE_KEY = "x-flowmesh-file" +ENV_VALUES_KEY = "x-flowmesh-env-values" + +_VAR_PATTERN = re.compile( + r""" + \$\{ + (?P[A-Za-z_][A-Za-z0-9_]*) + (?::-(?P[^}]*))? + \} + """, + re.VERBOSE, +) +_FALSEY = {"", "0", "false", "no", "off"} + +_DROP = object() +"""Sentinel marking a node whose condition evaluated false.""" + + +class ManifestError(RuntimeError): + """Raised when a manifest cannot be rendered.""" + + +def substitute(value: str, env: Mapping[str, str]) -> str: + """Expand ``${VAR}`` and ``${VAR:-default}`` references in ``value``.""" + + def replace(match: re.Match[str]) -> str: + name = match.group("name") + default = match.group("default") + if name in env: + return env[name] + if default is not None: + return default + raise ManifestError( + f"{name} is not set and has no default in the manifest reference " + f"{match.group(0)}" + ) + + return _VAR_PATTERN.sub(replace, value) + + +def evaluate_condition(expression: str, env: Mapping[str, str]) -> bool: + """Evaluate an ``x-flowmesh-when`` expression against ``env``.""" + expr = expression.strip() + if not expr: + raise ManifestError("Empty x-flowmesh-when expression") + + if expr.startswith("!"): + return not evaluate_condition(expr[1:], env) + + if "==" in expr: + name, _, expected = expr.partition("==") + return env.get(name.strip(), "").strip() == expected.strip() + + return env.get(expr, "").strip().lower() not in _FALSEY + + +def _render_int(node: Any, env: Mapping[str, str]) -> int: + rendered = substitute(str(node), env).strip() + try: + return int(rendered) + except ValueError: + raise ManifestError( + f"{INT_KEY} expected an integer but {node!r} rendered to {rendered!r}" + ) from None + + +def _render_file(node: Any, env: Mapping[str, str]) -> str: + name = substitute(str(node), env).strip() + path_value = env.get(name, "").strip() + if not path_value: + raise ManifestError(f"{FILE_KEY} requires {name} to name a readable file") + try: + return Path(path_value).read_text(encoding="utf-8") + except OSError as exc: + raise ManifestError(f"Failed to read {name} file {path_value}: {exc}") from exc + + +def _render_node( + node: Any, env: Mapping[str, str], env_values: Mapping[str, str] +) -> Any: + if isinstance(node, dict): + condition = node.get(WHEN_KEY) + if condition is not None and not evaluate_condition( + substitute(str(condition), env), env + ): + return _DROP + if VALUE_KEY in node: + return _render_node(node[VALUE_KEY], env, env_values) + if INT_KEY in node: + return _render_int(node[INT_KEY], env) + if FILE_KEY in node: + return _render_file(node[FILE_KEY], env) + if ENV_VALUES_KEY in node: + return dict(env_values) + + rendered: dict[str, Any] = {} + for key, value in node.items(): + if key == WHEN_KEY: + continue + child = _render_node(value, env, env_values) + if child is _DROP: + continue + rendered[key] = child + return rendered + + if isinstance(node, list): + items = [_render_node(item, env, env_values) for item in node] + return [item for item in items if item is not _DROP] + + if isinstance(node, str): + return substitute(node, env) + + return node + + +def render_documents( + source: str, + env: Mapping[str, str], + env_values: Mapping[str, str] | None = None, +) -> list[dict[str, Any]]: + """Render a multi-document manifest string into resource dictionaries. + + Document order is preserved because ``kubectl apply`` processes a stream in + order and later resources depend on earlier ones. + """ + documents: list[dict[str, Any]] = [] + for raw in yaml.safe_load_all(source): + if raw is None: + continue + rendered = _render_node(raw, env, env_values or {}) + if rendered is _DROP or not rendered: + continue + documents.append(rendered) + return documents + + +def render_manifests( + paths: list[Path], + env: Mapping[str, str], + env_values: Mapping[str, str] | None = None, +) -> str: + """Render manifest assets into a single YAML stream.""" + documents: list[dict[str, Any]] = [] + for path in paths: + try: + source = path.read_text(encoding="utf-8") + except OSError as exc: + raise ManifestError(f"Failed to read manifest {path}: {exc}") from exc + try: + documents.extend(render_documents(source, env, env_values)) + except yaml.YAMLError as exc: + raise ManifestError(f"Failed to parse manifest {path}: {exc}") from exc + if not documents: + raise ManifestError("No manifests to apply") + return yaml.safe_dump_all(documents, sort_keys=False, default_flow_style=False) diff --git a/src/server/env.py b/src/server/env.py index 11561e1c7..d40a314a8 100644 --- a/src/server/env.py +++ b/src/server/env.py @@ -107,6 +107,21 @@ ) DOCKER_GPU_RUNTIME: str | None = os.getenv("DOCKER_GPU_RUNTIME", "").strip() or None +K8S_NAMESPACE: str = ( + os.getenv("K8S_NAMESPACE") or os.getenv("POD_NAMESPACE") or "default" +).strip() +K8S_WORKER_NAMESPACE: str = ( + os.getenv("K8S_WORKER_NAMESPACE") or "" +).strip() or K8S_NAMESPACE +K8S_SUPERVISOR_SERVICE: str = ( + os.getenv("K8S_SUPERVISOR_SERVICE") or "flowmesh-supervisor" +).strip() +K8S_SERVER_SERVICE: str = (os.getenv("K8S_SERVER_SERVICE") or "flowmesh-server").strip() +K8S_CLUSTER_DOMAIN: str = (os.getenv("K8S_CLUSTER_DOMAIN") or "cluster.local").strip() +K8S_GPU_RESOURCE_NAME: str = ( + os.getenv("K8S_GPU_RESOURCE_NAME") or "nvidia.com/gpu" +).strip() + WORKER_CONFIG_PATH: str = os.getenv("WORKER_CONFIG_PATH", "configs/worker_config.yaml") CUDA_VISIBLE_DEVICES: str | None = os.getenv("CUDA_VISIBLE_DEVICES") if CUDA_VISIBLE_DEVICES is not None: diff --git a/src/server/requirements.txt b/src/server/requirements.txt index 28d2191b0..c968afa31 100644 --- a/src/server/requirements.txt +++ b/src/server/requirements.txt @@ -6,11 +6,12 @@ docker==7.1.0 fastapi==0.136.1 grpcio==1.83.1 httpx==0.28.1 +kubernetes==36.0.3 lumid-hooks==0.2.0 protobuf==6.33.6 pydantic==2.12.3 python-multipart==0.0.32 -pyyaml==6.0.2 +pyyaml==6.0.3 redis==7.0.1 requests==2.33.1 uvicorn[standard]==0.32.1 diff --git a/src/server/supervisor/adapters/base.py b/src/server/supervisor/adapters/base.py index b5c8aeba0..a9645e98d 100644 --- a/src/server/supervisor/adapters/base.py +++ b/src/server/supervisor/adapters/base.py @@ -1,6 +1,7 @@ import os from abc import ABC, abstractmethod from dataclasses import dataclass +from enum import StrEnum from typing import NewType from pydantic import BaseModel, ConfigDict, SecretStr @@ -11,6 +12,11 @@ from .utils import env_to_secret_str, to_env_str +class WorkerType(StrEnum): + CPU = "cpu" + GPU = "gpu" + + class WorkerConfig(BaseModel): model_config = ConfigDict(frozen=True) diff --git a/src/server/supervisor/adapters/docker.py b/src/server/supervisor/adapters/docker.py index 18327930e..9d6217067 100644 --- a/src/server/supervisor/adapters/docker.py +++ b/src/server/supervisor/adapters/docker.py @@ -5,7 +5,6 @@ import re import threading from collections import Counter -from enum import StrEnum from typing import Any from docker import DockerClient @@ -29,6 +28,7 @@ WorkerConfig, WorkerFactory, WorkerTokenType, + WorkerType, ) from .utils import get_worker_image_name, to_env_str @@ -95,11 +95,6 @@ def _prepare_volume( ) -class WorkerType(StrEnum): - CPU = "cpu" - GPU = "gpu" - - class SSHConfig(BaseModel): default_image: str | None = env.SSH_DEFAULT_IMAGE """Default container image for SSH sessions""" diff --git a/src/server/supervisor/adapters/kubernetes.py b/src/server/supervisor/adapters/kubernetes.py new file mode 100644 index 000000000..22b2b8496 --- /dev/null +++ b/src/server/supervisor/adapters/kubernetes.py @@ -0,0 +1,697 @@ +import asyncio +import hashlib +import logging +import re +import time +from collections import Counter +from pathlib import PurePosixPath +from typing import Any, get_args + +from kubernetes import client, config +from kubernetes.client.exceptions import ApiException +from pydantic import Field, SecretStr + +from shared.utils import parse_mem_to_bytes + +from ... import env +from ...hooks import PrincipalContext +from ...schemas.node import CPUInfo, GpuInfo, GpuPlatformInfo, MemoryInfo +from ..resource_manager import GpuArch +from ..schemas import WorkerHardware, WorkerInfo, WorkerStatus +from .base import ( + ProviderSpec, + WorkerAdapter, + WorkerConfig, + WorkerFactory, + WorkerTokenType, + WorkerType, +) +from .utils import get_worker_image_name + +PROVIDER_NAME = "kubernetes" + +MANAGED_LABEL = "flowmesh.io/managed" +NODE_ALIAS_LABEL = "flowmesh.io/node-alias" +WORKER_NAME_LABEL = "flowmesh.io/worker-name" + +_GPU_PRODUCT_LABEL = "nvidia.com/gpu.product" +_RFC1123_MAX_LEN = 63 +_RFC1123_INVALID_RE = re.compile(r"[^a-z0-9-]+") +_LABEL_VALUE_INVALID_RE = re.compile(r"[^A-Za-z0-9_.-]+") +_LABEL_VALUE_MAX_LEN = 63 +_CPU_MILLI_RE = re.compile(r"^([0-9]+)m$") +_DELETE_TIMEOUT_SEC = 120.0 +_DELETE_POLL_SEC = 0.5 + +logger = logging.getLogger("supervisor") + + +def _short_digest(value: str) -> str: + return hashlib.md5(value.encode("utf-8"), usedforsecurity=False).hexdigest()[:8] + + +def sanitize_object_name(name: str, maxlen: int = _RFC1123_MAX_LEN) -> str: + """Return an RFC 1123 DNS label derived from ``name``. + + Kubernetes object names are far stricter than Docker container names — + lowercase alphanumerics and dashes only — so names that are valid + elsewhere in FlowMesh (``flowmesh_node``, alias-derived names carrying + underscores) are rejected by the API. A truncated name carries a digest of + the original so two long names cannot collapse onto one object. + """ + sanitized = _RFC1123_INVALID_RE.sub("-", name.strip().lower()) + sanitized = re.sub(r"-{2,}", "-", sanitized).strip("-") + if len(sanitized) > maxlen: + head = sanitized[: maxlen - 9].rstrip("-") + sanitized = f"{head}-{_short_digest(name)}" + if not sanitized or not sanitized[0].isalnum(): + sanitized = f"w-{_short_digest(name)}" + return sanitized + + +def sanitize_label_value(value: str) -> str: + """Return a value accepted by the Kubernetes label-value grammar.""" + sanitized = _LABEL_VALUE_INVALID_RE.sub("-", value.strip())[:_LABEL_VALUE_MAX_LEN] + return sanitized.strip("-_.") + + +def _secret_field_names(config_cls: type[WorkerConfig]) -> frozenset[str]: + """Return the config fields declared as ``SecretStr``. + + Deriving the credential set from the model keeps it correct as fields are + added; a hand-maintained list would silently leak the next secret someone + adds to the worker environment into the pod spec. + """ + names: set[str] = set() + for field_name, field in config_cls.model_fields.items(): + annotation = field.annotation + if any(arg is SecretStr for arg in (annotation, *get_args(annotation))): + names.add(field_name) + return frozenset(names) + + +def _parse_cpu_quantity(value: str) -> int | None: + raw = value.strip() + if match := _CPU_MILLI_RE.match(raw): + return max(1, int(match.group(1)) // 1000) + try: + return int(float(raw)) + except ValueError: + return None + + +def _deep_merge(base: dict[str, Any], overlay: dict[str, Any]) -> dict[str, Any]: + """Merge ``overlay`` onto ``base``; mappings merge, everything else replaces.""" + merged = dict(base) + for key, value in overlay.items(): + current = merged.get(key) + if isinstance(current, dict) and isinstance(value, dict): + merged[key] = _deep_merge(current, value) + else: + merged[key] = value + return merged + + +def _prune(value: dict[str, Any]) -> dict[str, Any]: + return {k: v for k, v in value.items() if v is not None} + + +class KubernetesWorkerConfig(WorkerConfig): + supervisor_grpc_target: str = ( + f"{env.K8S_SUPERVISOR_SERVICE}.{env.K8S_NAMESPACE}" + f".svc.{env.K8S_CLUSTER_DOMAIN}:{env.SERVER_GRPC_PORT}" + ) + """Supervisor gRPC target""" + flowmesh_url: str = ( + f"http://{env.K8S_SERVER_SERVICE}.{env.K8S_NAMESPACE}" + f".svc.{env.K8S_CLUSTER_DOMAIN}:{env.SERVER_APP_PORT}" + ) + """FlowMesh HTTP base URL workers use to fetch and upload artifacts. + + Defaults to the in-cluster Service rather than the externally advertised + ``FLOWMESH_BASE_URL``, which need not resolve from inside the cluster.""" + namespace: str = env.K8S_WORKER_NAMESPACE + """Namespace worker pods are created in""" + pod_name: str | None = None + """Optional explicit pod name""" + worker_type: WorkerType = WorkerType.CPU + """Type of worker (cpu or gpu)""" + gpu_count: int = 1 + """Number of GPUs requested for GPU workers""" + gpu_resource_name: str = env.K8S_GPU_RESOURCE_NAME + """Extended resource name used to request GPUs""" + gpu_arch: GpuArch | None = None + """GPU architecture selecting the worker image variant""" + node_selector: dict[str, str] | None = None + """Node labels a worker pod must match""" + tolerations: list[dict[str, Any]] | None = None + """Tolerations applied to the worker pod""" + service_account_name: str | None = None + """Service account the worker pod runs as""" + image_pull_secrets: list[str] = Field(default_factory=list) + """Image pull secrets referenced by the worker pod""" + image_pull_policy: str = "IfNotPresent" + """Image pull policy for the worker container""" + cpu_request: str | None = None + """CPU request for the worker container""" + cpu_limit: str | None = None + """CPU limit for the worker container""" + memory_request: str | None = None + """Memory request for the worker container""" + memory_limit: str | None = None + """Memory limit for the worker container""" + shm_size: str | None = None + """Size of the ``/dev/shm`` in-memory volume""" + results_pvc: str | None = None + """PersistentVolumeClaim backing the results directory""" + results_mount_path: str = "/var/lib/flowmesh-results" + """Path the results volume is mounted at inside the worker container""" + hf_cache_pvc: str | None = None + """PersistentVolumeClaim backing the Hugging Face cache""" + runtime_class_name: str | None = None + """RuntimeClass for the worker pod""" + priority_class_name: str | None = None + """PriorityClass for the worker pod""" + pod_labels: dict[str, str] | None = None + """Extra labels applied to the worker pod""" + pod_annotations: dict[str, str] | None = None + """Annotations applied to the worker pod""" + pod_overrides: dict[str, Any] | None = None + """Overlay merged onto the generated pod manifest. + + Mappings merge recursively; sequences and scalars replace.""" + stop_grace_period_sec: int = 30 + """Termination grace period applied when deleting the worker pod""" + docker_registry: str = env.FLOWMESH_REGISTRY + """Registry to pull worker images from""" + version: str = env.FLOWMESH_VERSION + """Worker image version tag""" + + def model_post_init(self, __context: object) -> None: + super().model_post_init(__context) + if self.worker_type == WorkerType.GPU and self.gpu_count < 1: + raise ValueError("Expected at least one GPU for GPU worker.") + + +class KubernetesWorkerInfo(WorkerInfo): + pass + + +class KubernetesWorkerAdapter(WorkerAdapter): + CONTAINER_HF_CACHE_DIR: str = "/home/appuser/.cache/huggingface" + SHM_MOUNT_PATH: str = PurePosixPath("/", "dev", "shm").as_posix() + CONTAINER_NAME: str = "worker" + WORKER_GID: int = 10001 + + def __init__( + self, + token: WorkerTokenType, + name: str, + pod_name: str, + config: KubernetesWorkerConfig, + core_api: client.CoreV1Api, + node_alias: str, + owner: PrincipalContext, + ) -> None: + super().__init__(token, name, config, owner) + + self.config: KubernetesWorkerConfig + self.pod_name = pod_name + self.secret_name = f"{pod_name}-env" + self.node_alias = node_alias + + self._core = core_api + self._status: WorkerStatus = WorkerStatus.STOPPED + self._hardware: dict[str, Any] | WorkerHardware | None = None + self._is_started = False + + @property + def status(self) -> WorkerStatus: + return self._status + + def set_status(self, status: WorkerStatus) -> None: + self._status = status + + def get_info(self) -> KubernetesWorkerInfo: + hardware = self._hardware + if isinstance(hardware, dict): + hardware = WorkerHardware.model_validate(hardware) + self._hardware = hardware + return KubernetesWorkerInfo( + id=self.worker_id, + name=self.name, + provider=PROVIDER_NAME, + status=self.status, + hardware=hardware, + ) + + async def start(self) -> bool: + self.set_status(WorkerStatus.STARTING) + try: + ok = await asyncio.to_thread(self._start) + if not ok: + self.set_status(WorkerStatus.STOPPED) + return ok + except Exception: + self.set_status(WorkerStatus.STOPPED) + raise + + async def prepare(self) -> None: + self._hardware = await asyncio.to_thread(self._probe_hardware) + + async def stop(self) -> bool: + prev_status = self.status + if prev_status in (WorkerStatus.STOPPING, WorkerStatus.STOPPED): + return True + self.set_status(WorkerStatus.STOPPING) + try: + ok = await asyncio.to_thread(self._stop) + if not ok: + self.set_status(prev_status) + return ok + except Exception: + self.set_status(prev_status) + raise + + def get_image_name(self) -> str: + return get_worker_image_name( + self.config.docker_registry, self.config.version, self._gpu_arch() + ) + + def _gpu_arch(self) -> GpuArch | None: + if self.config.worker_type != WorkerType.GPU: + return None + return self.config.gpu_arch or GpuArch.UNKNOWN + + def _base_environment(self) -> dict[str, str]: + environment = super()._base_environment() + environment["RESULTS_DIR"] = self.config.results_mount_path + return environment + + def _split_environment(self) -> tuple[dict[str, str], dict[str, str]]: + """Partition the worker environment into inline and secret-held values.""" + secret_keys = {"WORKER_TOKEN", "FLOWMESH_API_KEY"} | { + name.upper() for name in _secret_field_names(type(self.config)) + } + inline: dict[str, str] = {} + secret: dict[str, str] = {} + for key, value in self._base_environment().items(): + if key in secret_keys: + secret[key] = value + else: + inline[key] = value + return inline, secret + + def _labels(self) -> dict[str, str]: + labels = { + MANAGED_LABEL: "true", + NODE_ALIAS_LABEL: sanitize_label_value(self.node_alias), + WORKER_NAME_LABEL: sanitize_label_value(self.name), + } + if self.config.pod_labels: + labels.update(self.config.pod_labels) + return labels + + def _resources(self) -> dict[str, Any]: + config = self.config + requests = _prune({"cpu": config.cpu_request, "memory": config.memory_request}) + limits = _prune({"cpu": config.cpu_limit, "memory": config.memory_limit}) + if config.worker_type == WorkerType.GPU: + limits[config.gpu_resource_name] = str(config.gpu_count) + return _prune({"requests": requests or None, "limits": limits or None}) + + def _volumes(self) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + config = self.config + volumes: list[dict[str, Any]] = [] + mounts: list[dict[str, Any]] = [] + + if config.results_pvc: + volumes.append( + { + "name": "results", + "persistentVolumeClaim": {"claimName": config.results_pvc}, + } + ) + else: + volumes.append({"name": "results", "emptyDir": {}}) + mounts.append({"name": "results", "mountPath": config.results_mount_path}) + + if config.hf_cache_pvc: + volumes.append( + { + "name": "hf-cache", + "persistentVolumeClaim": {"claimName": config.hf_cache_pvc}, + } + ) + mounts.append( + {"name": "hf-cache", "mountPath": self.CONTAINER_HF_CACHE_DIR} + ) + + if config.shm_size: + volumes.append( + { + "name": "dshm", + "emptyDir": {"medium": "Memory", "sizeLimit": config.shm_size}, + } + ) + mounts.append({"name": "dshm", "mountPath": self.SHM_MOUNT_PATH}) + + return volumes, mounts + + def build_pod_manifest(self) -> dict[str, Any]: + config = self.config + inline_env, _ = self._split_environment() + volumes, mounts = self._volumes() + + container = _prune( + { + "name": self.CONTAINER_NAME, + "image": self.get_image_name(), + "imagePullPolicy": config.image_pull_policy, + "env": [{"name": k, "value": v} for k, v in sorted(inline_env.items())], + "envFrom": [{"secretRef": {"name": self.secret_name}}], + "resources": self._resources() or None, + "volumeMounts": mounts or None, + } + ) + + spec = _prune( + { + "restartPolicy": "Always", + "terminationGracePeriodSeconds": config.stop_grace_period_sec, + "containers": [container], + "volumes": volumes or None, + "nodeSelector": config.node_selector or None, + "tolerations": config.tolerations or None, + "serviceAccountName": config.service_account_name, + "runtimeClassName": config.runtime_class_name, + "priorityClassName": config.priority_class_name, + "securityContext": {"fsGroup": self.WORKER_GID}, + "imagePullSecrets": ( + [{"name": name} for name in config.image_pull_secrets] or None + ), + } + ) + + manifest: dict[str, Any] = { + "apiVersion": "v1", + "kind": "Pod", + "metadata": _prune( + { + "name": self.pod_name, + "namespace": config.namespace, + "labels": self._labels(), + "annotations": config.pod_annotations or None, + } + ), + "spec": spec, + } + if config.pod_overrides: + manifest = _deep_merge(manifest, config.pod_overrides) + return manifest + + def build_secret_manifest(self) -> dict[str, Any]: + _, secret_env = self._split_environment() + return { + "apiVersion": "v1", + "kind": "Secret", + "type": "Opaque", + "metadata": { + "name": self.secret_name, + "namespace": self.config.namespace, + "labels": self._labels(), + }, + "stringData": secret_env, + } + + def _start(self) -> bool: + existing = self._read_pod() + if existing is not None: + phase = self._pod_phase(existing) + if phase in ("Running", "Pending"): + self._is_started = True + logger.warning("Pod %s is already running.", self.pod_name) + return True + if not self._delete_pod(grace_period_seconds=0): + return False + if not self._await_pod_deletion(): + return False + + if not self._apply_secret(): + return False + + try: + self._core.create_namespaced_pod( + namespace=self.config.namespace, body=self.build_pod_manifest() + ) + except ApiException as exc: + logger.error("Failed to create pod %s: %s", self.pod_name, repr(exc)) + self._delete_secret() + return False + + self._is_started = True + return True + + def _stop(self) -> bool: + deleted = self._delete_pod() + self._delete_secret() + if deleted: + self._is_started = False + return deleted + + def _read_pod(self) -> Any | None: + try: + return self._core.read_namespaced_pod( + name=self.pod_name, namespace=self.config.namespace + ) + except ApiException as exc: + if exc.status == 404: + return None + logger.warning("Failed to read pod %s: %s", self.pod_name, repr(exc)) + return None + + @staticmethod + def _pod_phase(pod: Any) -> str | None: + status = pod.status if hasattr(pod, "status") else None + if status is None: + return None + return status.phase if hasattr(status, "phase") else None + + def _apply_secret(self) -> bool: + body = self.build_secret_manifest() + try: + self._core.create_namespaced_secret( + namespace=self.config.namespace, body=body + ) + return True + except ApiException as exc: + if exc.status != 409: + logger.error( + "Failed to create secret %s: %s", self.secret_name, repr(exc) + ) + return False + try: + self._core.replace_namespaced_secret( + name=self.secret_name, namespace=self.config.namespace, body=body + ) + return True + except ApiException as exc: + logger.error("Failed to update secret %s: %s", self.secret_name, repr(exc)) + return False + + def _await_pod_deletion(self) -> bool: + """Block until the pod name is free again. + + Deletion is accepted asynchronously and the object outlives the call + while it terminates, so reusing the name immediately would collide. + """ + deadline = time.monotonic() + _DELETE_TIMEOUT_SEC + while time.monotonic() < deadline: + if self._read_pod() is None: + return True + time.sleep(_DELETE_POLL_SEC) + logger.error("Pod %s was still terminating after deletion", self.pod_name) + return False + + def _delete_pod(self, grace_period_seconds: int | None = None) -> bool: + if grace_period_seconds is None: + grace_period_seconds = self.config.stop_grace_period_sec + try: + self._core.delete_namespaced_pod( + name=self.pod_name, + namespace=self.config.namespace, + grace_period_seconds=grace_period_seconds, + ) + return True + except ApiException as exc: + if exc.status == 404: + return True + logger.error("Failed to delete pod %s: %s", self.pod_name, repr(exc)) + return False + + def _delete_secret(self) -> bool: + try: + self._core.delete_namespaced_secret( + name=self.secret_name, namespace=self.config.namespace + ) + return True + except ApiException as exc: + if exc.status == 404: + return True + logger.warning( + "Failed to delete secret %s: %s", self.secret_name, repr(exc) + ) + return False + + def _probe_hardware(self) -> dict[str, Any] | None: + selector = ",".join( + f"{key}={value}" for key, value in (self.config.node_selector or {}).items() + ) + try: + nodes = self._core.list_node(label_selector=selector or None) + except ApiException as exc: + if exc.status == 403: + logger.debug( + "No permission to list nodes; skipping hardware probe for %s", + self.name, + ) + else: + logger.warning( + "Failed to list nodes for worker %s: %s", self.name, repr(exc) + ) + return None + + items = nodes.items if hasattr(nodes, "items") else [] + if not items: + return None + return self._hardware_from_node(items[0]).model_dump() + + def _hardware_from_node(self, node: Any) -> WorkerHardware: + status = node.status if hasattr(node, "status") else None + allocatable = dict(getattr(status, "allocatable", None) or {}) + metadata = node.metadata if hasattr(node, "metadata") else None + node_labels = dict(getattr(metadata, "labels", None) or {}) + + cpu = CPUInfo(logical_cores=_parse_cpu_quantity(allocatable.get("cpu", ""))) + memory = MemoryInfo( + total_bytes=parse_mem_to_bytes(allocatable.get("memory", "")) + ) + + gpu = GpuPlatformInfo() + if self.config.worker_type == WorkerType.GPU: + product = node_labels.get(_GPU_PRODUCT_LABEL, "") + arch = self.config.gpu_arch or GpuArch.from_name(product) + gpu = GpuPlatformInfo( + gpu_arch=arch.value, + devices=[ + GpuInfo(index=index, name=product or None) + for index in range(self.config.gpu_count) + ], + ) + + return WorkerHardware(cpu=cpu, memory=memory, gpu=gpu) + + +class KubernetesWorkerFactory(WorkerFactory): + def __init__(self, system_principal: PrincipalContext) -> None: + """Resolve cluster access up front. + + Construction is what decides whether this node reports the provider as + available, so an unreachable cluster has to fail here rather than at + first use — otherwise the node would advertise a provider it cannot + serve and reject the create only once a worker was requested. + """ + super().__init__(system_principal) + try: + config.load_incluster_config() + except config.ConfigException: + config.load_kube_config() + self._api_client = client.ApiClient() + self._core = client.CoreV1Api(self._api_client) + self._worker_id_registry: Counter[str] = Counter() + self._reaped = False + + def create_worker( + self, token: WorkerTokenType, config: KubernetesWorkerConfig + ) -> KubernetesWorkerAdapter: + core = self._core + self._reap_orphans(core, config.namespace) + + name = config.worker_alias or self._next_worker_name(config.worker_type) + pod_name = config.pod_name or sanitize_object_name(f"{env.NODE_ALIAS}-{name}") + return KubernetesWorkerAdapter( + token=token, + name=name, + pod_name=pod_name, + config=config, + core_api=core, + node_alias=env.NODE_ALIAS, + owner=self.system_principal, + ) + + def destroy_worker(self, worker: WorkerAdapter) -> None: + if not isinstance(worker, KubernetesWorkerAdapter): + raise ValueError("Invalid worker type") + + def cleanup(self) -> None: + self._api_client.close() + + def _next_worker_name(self, worker_type: WorkerType) -> str: + match worker_type: + case WorkerType.CPU: + prefix = "flowmesh_server_worker_cpu_" + case WorkerType.GPU: + prefix = "flowmesh_server_worker_gpu_" + case _: + raise ValueError(f"Unsupported worker type: {worker_type}") + self._worker_id_registry[prefix] += 1 + return f"{prefix}{self._worker_id_registry[prefix]}" + + def _reap_orphans(self, core: client.CoreV1Api, namespace: str) -> None: + """Delete worker pods left behind by a previous supervisor incarnation. + + Their tokens died with that supervisor's in-memory registry, so they can + never register again; a clean shutdown drains its own workers, so + anything matching here is a zombie from an unclean exit. + """ + if self._reaped: + return + self._reaped = True + + selector = ( + f"{MANAGED_LABEL}=true," + f"{NODE_ALIAS_LABEL}={sanitize_label_value(env.NODE_ALIAS)}" + ) + try: + pods = core.list_namespaced_pod( + namespace=namespace, label_selector=selector + ) + except ApiException as exc: + logger.warning("Failed to list orphaned worker pods: %s", repr(exc)) + return + + for pod in pods.items if hasattr(pods, "items") else []: + pod_name = pod.metadata.name + logger.info("Reaping orphaned worker pod %s", pod_name) + try: + core.delete_namespaced_pod(name=pod_name, namespace=namespace) + except ApiException as exc: + if exc.status != 404: + logger.warning( + "Failed to delete orphaned pod %s: %s", pod_name, repr(exc) + ) + try: + core.delete_collection_namespaced_secret( + namespace=namespace, label_selector=selector + ) + except ApiException as exc: + logger.warning("Failed to delete orphaned worker secrets: %s", repr(exc)) + + +def get_provider_spec(system_principal: PrincipalContext) -> ProviderSpec: + return ProviderSpec( + name=PROVIDER_NAME, + config_cls=KubernetesWorkerConfig, + adapter_cls=KubernetesWorkerAdapter, + factory=KubernetesWorkerFactory(system_principal), + ) diff --git a/src/server/supervisor/manager.py b/src/server/supervisor/manager.py index 681f9dd20..f6aa7be4d 100644 --- a/src/server/supervisor/manager.py +++ b/src/server/supervisor/manager.py @@ -12,6 +12,7 @@ from .adapters.docker import get_provider_spec as docker_provider_spec from .adapters.external import get_provider_spec as external_provider_spec from .adapters.external import verify_external_token +from .adapters.kubernetes import get_provider_spec as kubernetes_provider_spec from .adapters.vastai import get_provider_spec as vastai_provider_spec from .registry import WorkerRegistry from .schemas import WorkerInfo, WorkerStatus @@ -89,6 +90,12 @@ def __init__( logger.warning( "Docker worker provider unavailable, continuing without it: %s", exc ) + try: + specs.append(kubernetes_provider_spec(system_principal)) + except Exception as exc: + logger.warning( + "Kubernetes worker provider unavailable, continuing without it: %s", exc + ) try: specs.append(vastai_provider_spec(system_principal)) except Exception as exc: diff --git a/tests/cli/test_stack_k8s.py b/tests/cli/test_stack_k8s.py new file mode 100644 index 000000000..dedd07431 --- /dev/null +++ b/tests/cli/test_stack_k8s.py @@ -0,0 +1,319 @@ +"""`flowmesh stack --backend k8s` dispatch and Kubernetes backend behavior.""" + +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest +import typer +from flowmesh.models.nodes import NodeRole +from flowmesh_cli_stack import k8s as k8s_module +from flowmesh_cli_stack import stack as stack_module +from flowmesh_cli_stack.k8s import StackBackend + +ENV_FILE = Path(".env") + + +# ------------------------------------------------------------------ # +# Backend resolution +# ------------------------------------------------------------------ # + + +class TestResolveBackend: + def test_option_wins_over_the_env_file(self) -> None: + with patch.object( + k8s_module, "parse_env_file", return_value={"STACK_BACKEND": "compose"} + ): + assert k8s_module.resolve_backend("k8s", ENV_FILE) is StackBackend.K8S + + def test_env_file_is_used_without_an_option(self) -> None: + with patch.object( + k8s_module, "parse_env_file", return_value={"STACK_BACKEND": "k8s"} + ): + assert k8s_module.resolve_backend(None, ENV_FILE) is StackBackend.K8S + + def test_compose_is_the_default(self) -> None: + with patch.object(k8s_module, "parse_env_file", return_value={}): + assert k8s_module.resolve_backend(None, ENV_FILE) is StackBackend.COMPOSE + + def test_unknown_backend_exits(self) -> None: + with patch.object(k8s_module, "parse_env_file", return_value={}): + with pytest.raises(typer.Exit): + k8s_module.resolve_backend("nomad", ENV_FILE) + + +# ------------------------------------------------------------------ # +# Command dispatch +# ------------------------------------------------------------------ # + + +class TestDispatch: + def test_up_routes_to_kubernetes(self) -> None: + with ( + patch.object( + stack_module, "resolve_backend", return_value=StackBackend.K8S + ), + patch.object(stack_module.k8s, "up") as up, + patch.object(stack_module, "_compose") as compose, + ): + stack_module.up(env_file=ENV_FILE, image_tag=None, backend="k8s") + + up.assert_called_once() + compose.assert_not_called() + + def test_up_leaves_the_compose_path_untouched(self) -> None: + with ( + patch.object( + stack_module, "resolve_backend", return_value=StackBackend.COMPOSE + ), + patch.object(stack_module.k8s, "up") as up, + patch.object(stack_module, "_compose") as compose, + patch.object(stack_module, "_node_role", return_value=NodeRole.ROOT), + patch.object(stack_module, "image_env_overrides", return_value={}), + ): + stack_module.up(env_file=ENV_FILE, image_tag=None, backend=None) + + up.assert_not_called() + compose.assert_called_once() + + def test_down_routes_to_kubernetes(self) -> None: + with ( + patch.object( + stack_module, "resolve_backend", return_value=StackBackend.K8S + ), + patch.object(stack_module.k8s, "down") as down, + patch.object(stack_module, "_compose") as compose, + patch.object(stack_module, "drain_workers") as drain, + ): + stack_module.down(env_file=ENV_FILE, image_tag=None, backend="k8s") + + down.assert_called_once() + compose.assert_not_called() + drain.assert_not_called() + + def test_restart_routes_to_kubernetes(self) -> None: + with ( + patch.object( + stack_module, "resolve_backend", return_value=StackBackend.K8S + ), + patch.object(stack_module.k8s, "restart") as restart, + patch.object(stack_module, "_compose") as compose, + ): + stack_module.restart( + services=["server"], + env_file=ENV_FILE, + image_tag=None, + pull=True, + backend="k8s", + ) + + restart.assert_called_once() + assert restart.call_args.kwargs["services"] == ["server"] + compose.assert_not_called() + + def test_ps_routes_to_kubernetes(self) -> None: + with ( + patch.object( + stack_module, "resolve_backend", return_value=StackBackend.K8S + ), + patch.object(stack_module.k8s, "ps") as ps, + patch.object(stack_module, "_compose") as compose, + ): + stack_module.ps(env_file=ENV_FILE, backend="k8s") + + ps.assert_called_once() + compose.assert_not_called() + + +# ------------------------------------------------------------------ # +# Kubernetes lifecycle +# ------------------------------------------------------------------ # + + +def _stack(role: NodeRole = NodeRole.ROOT) -> MagicMock: + target = MagicMock() + target.apply.return_value.returncode = 0 + target.delete.return_value.returncode = 0 + target.delete_workers.return_value.returncode = 0 + target.delete_volumes.return_value.returncode = 0 + target.rollout_status.return_value.returncode = 0 + target.rollout_restart.return_value.returncode = 0 + target.logs.return_value.returncode = 0 + target.status.return_value.returncode = 0 + target.worker_status.return_value.returncode = 0 + return target + + +class TestKubernetesLifecycle: + def test_up_waits_for_every_workload(self) -> None: + target = _stack() + with ( + patch.object(k8s_module, "stack", return_value=target), + patch.object(k8s_module, "node_role", return_value=NodeRole.ROOT), + ): + k8s_module.up(ENV_FILE) + + target.apply.assert_called_once() + waited = [call.args[0] for call in target.rollout_status.call_args_list] + assert waited == [ + "statefulset/flowmesh-redis-control", + "statefulset/flowmesh-redis-telemetry", + "deployment/flowmesh-server", + ] + + def test_worker_node_waits_only_for_the_server(self) -> None: + target = _stack() + with ( + patch.object(k8s_module, "stack", return_value=target), + patch.object(k8s_module, "node_role", return_value=NodeRole.WORKER), + ): + k8s_module.up(ENV_FILE) + + waited = [call.args[0] for call in target.rollout_status.call_args_list] + assert waited == ["deployment/flowmesh-server"] + + def test_down_drains_workers_first(self) -> None: + target = _stack() + with ( + patch.object(k8s_module, "stack", return_value=target), + patch.object(k8s_module, "drain_workers") as drain, + ): + k8s_module.down(ENV_FILE) + + drain.assert_called_once() + target.delete_workers.assert_called_once() + target.delete.assert_called_once() + + def test_restart_of_the_server_drains_workers(self) -> None: + target = _stack() + with ( + patch.object(k8s_module, "stack", return_value=target), + patch.object(k8s_module, "drain_workers") as drain, + ): + k8s_module.restart(services=["server"], env_file=ENV_FILE) + + drain.assert_called_once() + target.rollout_restart.assert_called_once_with("deployment/flowmesh-server") + + def test_restart_of_redis_does_not_drain_workers(self) -> None: + target = _stack() + with ( + patch.object(k8s_module, "stack", return_value=target), + patch.object(k8s_module, "drain_workers") as drain, + ): + k8s_module.restart(services=["redis_control"], env_file=ENV_FILE) + + drain.assert_not_called() + + def test_image_tag_is_applied_rather_than_rolled(self) -> None: + """A rollout restart alone would leave the pod template's image unchanged.""" + target = _stack() + with ( + patch.object(k8s_module, "stack", return_value=target), + patch.object(k8s_module, "drain_workers"), + ): + k8s_module.restart(services=["server"], env_file=ENV_FILE, image_tag="v2") + + target.apply.assert_called_once() + target.rollout_restart.assert_not_called() + + def test_unknown_service_exits_without_acting(self) -> None: + target = _stack() + with ( + patch.object(k8s_module, "stack", return_value=target), + patch.object(k8s_module, "drain_workers") as drain, + ): + with pytest.raises(typer.Exit): + k8s_module.restart(services=["nope"], env_file=ENV_FILE) + + drain.assert_not_called() + target.rollout_restart.assert_not_called() + + def test_clean_removes_volumes_after_teardown(self) -> None: + target = _stack() + with ( + patch.object(k8s_module, "stack", return_value=target), + patch.object(k8s_module, "drain_workers"), + ): + k8s_module.clean(ENV_FILE) + + target.delete.assert_called_once() + target.delete_volumes.assert_called_once() + + def test_ps_lists_stack_and_worker_pods(self) -> None: + target = _stack() + with patch.object(k8s_module, "stack", return_value=target): + k8s_module.ps(ENV_FILE) + + target.status.assert_called_once() + target.worker_status.assert_called_once() + + def test_failed_kubectl_call_exits_with_its_code(self) -> None: + target = _stack() + target.apply.return_value.returncode = 3 + with ( + patch.object(k8s_module, "stack", return_value=target), + patch.object(k8s_module, "node_role", return_value=NodeRole.ROOT), + ): + with pytest.raises(typer.Exit) as exit_info: + k8s_module.up(ENV_FILE) + + assert exit_info.value.exit_code == 3 + + +# ------------------------------------------------------------------ # +# Environment derivation +# ------------------------------------------------------------------ # + + +class TestEnvDerivation: + def test_worker_namespace_defaults_to_the_stack_namespace( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + monkeypatch.setenv("K8S_NAMESPACE", "ml-team") + monkeypatch.delenv("K8S_WORKER_NAMESPACE", raising=False) + monkeypatch.delenv("SERVER_WORKER_CONFIG", raising=False) + + k8s_module.apply_k8s_env(tmp_path) + + assert k8s_module.os.environ["K8S_WORKER_NAMESPACE"] == "ml-team" + + def test_explicit_worker_namespace_is_kept( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + monkeypatch.setenv("K8S_NAMESPACE", "ml-team") + monkeypatch.setenv("K8S_WORKER_NAMESPACE", "gpu-pool") + + k8s_module.apply_k8s_env(tmp_path) + + assert k8s_module.os.environ["K8S_WORKER_NAMESPACE"] == "gpu-pool" + + def test_distinct_worker_namespace_is_flagged( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + monkeypatch.setenv("K8S_NAMESPACE", "flowmesh") + monkeypatch.setenv("K8S_WORKER_NAMESPACE", "gpu-pool") + + k8s_module.apply_k8s_env(tmp_path) + + assert k8s_module.os.environ["K8S_WORKER_NAMESPACE_DISTINCT"] == "true" + + def test_shared_worker_namespace_is_not_flagged( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + monkeypatch.setenv("K8S_NAMESPACE", "flowmesh") + monkeypatch.delenv("K8S_WORKER_NAMESPACE", raising=False) + + k8s_module.apply_k8s_env(tmp_path) + + assert k8s_module.os.environ["K8S_WORKER_NAMESPACE_DISTINCT"] == "false" + + def test_worker_config_resolves_to_an_absolute_path( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + monkeypatch.delenv("SERVER_WORKER_CONFIG", raising=False) + + k8s_module.apply_k8s_env(tmp_path) + + resolved = Path(k8s_module.os.environ["SERVER_WORKER_CONFIG"]) + assert resolved.is_absolute() + assert resolved.name == "worker_config.yaml" diff --git a/tests/cli/test_stack_restart.py b/tests/cli/test_stack_restart.py index eb58b8e97..f2681756d 100644 --- a/tests/cli/test_stack_restart.py +++ b/tests/cli/test_stack_restart.py @@ -14,15 +14,20 @@ def _restart( env_file: Path = Path(".env"), image_tag: str | None = None, pull: bool = True, + backend: str | None = None, ) -> None: stack_module.restart( - services=services, env_file=env_file, image_tag=image_tag, pull=pull + services=services, + env_file=env_file, + image_tag=image_tag, + pull=pull, + backend=backend, ) def test_restart_server_drains_then_recreates_only_server() -> None: with ( - patch.object(stack_module, "_drain_workers") as drain, + patch.object(stack_module, "drain_workers") as drain, patch.object(stack_module, "_compose") as compose, patch.object(stack_module, "_node_role", return_value=NodeRole.ROOT), patch.object(stack_module, "image_env_overrides", return_value={}), @@ -38,7 +43,7 @@ def test_restart_server_drains_then_recreates_only_server() -> None: def test_restart_multiple_services_drains_once_and_recreates_all() -> None: with ( - patch.object(stack_module, "_drain_workers") as drain, + patch.object(stack_module, "drain_workers") as drain, patch.object(stack_module, "_compose") as compose, patch.object(stack_module, "_node_role", return_value=NodeRole.ROOT), patch.object(stack_module, "image_env_overrides", return_value={}), @@ -56,7 +61,7 @@ def test_restart_multiple_services_drains_once_and_recreates_all() -> None: def test_restart_dedupes_repeated_services() -> None: with ( - patch.object(stack_module, "_drain_workers"), + patch.object(stack_module, "drain_workers"), patch.object(stack_module, "_compose") as compose, patch.object(stack_module, "_node_role", return_value=NodeRole.ROOT), patch.object(stack_module, "image_env_overrides", return_value={}), @@ -69,7 +74,7 @@ def test_restart_dedupes_repeated_services() -> None: def test_restart_no_pull_omits_pull_flag() -> None: with ( - patch.object(stack_module, "_drain_workers"), + patch.object(stack_module, "drain_workers"), patch.object(stack_module, "_compose") as compose, patch.object(stack_module, "_node_role", return_value=NodeRole.ROOT), patch.object(stack_module, "image_env_overrides", return_value={}), @@ -81,7 +86,7 @@ def test_restart_no_pull_omits_pull_flag() -> None: def test_restart_redis_service_does_not_drain_workers() -> None: with ( - patch.object(stack_module, "_drain_workers") as drain, + patch.object(stack_module, "drain_workers") as drain, patch.object(stack_module, "_compose"), patch.object(stack_module, "_node_role", return_value=NodeRole.ROOT), patch.object(stack_module, "image_env_overrides", return_value={}), @@ -93,7 +98,7 @@ def test_restart_redis_service_does_not_drain_workers() -> None: def test_restart_unknown_service_exits_without_acting() -> None: with ( - patch.object(stack_module, "_drain_workers") as drain, + patch.object(stack_module, "drain_workers") as drain, patch.object(stack_module, "_compose") as compose, ): with pytest.raises(typer.Exit): @@ -105,7 +110,7 @@ def test_restart_unknown_service_exits_without_acting() -> None: def test_restart_unknown_in_a_set_exits_without_acting() -> None: with ( - patch.object(stack_module, "_drain_workers") as drain, + patch.object(stack_module, "drain_workers") as drain, patch.object(stack_module, "_compose") as compose, ): with pytest.raises(typer.Exit): diff --git a/tests/sdk/test_kubernetes_stack.py b/tests/sdk/test_kubernetes_stack.py new file mode 100644 index 000000000..75d0823b9 --- /dev/null +++ b/tests/sdk/test_kubernetes_stack.py @@ -0,0 +1,85 @@ +"""KubernetesStack: what a teardown deletes, and what it must leave behind.""" + +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest +import yaml +from flowmesh_stack.kubernetes import ( + DATA_KINDS, + KubectlError, + KubernetesStack, + ensure_kubectl_available, +) + +MANIFEST = """ +apiVersion: v1 +kind: Namespace +metadata: + name: flowmesh +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flowmesh-server +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: flowmesh-results +""" + + +def _stack(tmp_path: Path) -> KubernetesStack: + manifest = tmp_path / "stack.yaml" + manifest.write_text(MANIFEST) + return KubernetesStack( + manifests=[manifest], + namespace="flowmesh", + load_env=lambda _: None, + ) + + +def _stdin_kinds(run: MagicMock) -> list[str]: + stream = run.call_args.kwargs["stdin"] + return [doc["kind"] for doc in yaml.safe_load_all(stream) if doc] + + +class TestTeardown: + def test_delete_keeps_the_namespace_and_claims(self, tmp_path: Path) -> None: + """Deleting the namespace would cascade to every claim inside it.""" + stack = _stack(tmp_path) + with patch.object(stack, "_run") as run: + stack.delete(tmp_path / ".env") + + kinds = _stdin_kinds(run) + assert kinds == ["Deployment"] + assert not DATA_KINDS & set(kinds) + + def test_delete_can_be_asked_for_everything(self, tmp_path: Path) -> None: + stack = _stack(tmp_path) + with patch.object(stack, "_run") as run: + stack.delete(tmp_path / ".env", keep_data=False) + + assert _stdin_kinds(run) == ["Namespace", "Deployment", "PersistentVolumeClaim"] + + def test_delete_tolerates_missing_resources(self, tmp_path: Path) -> None: + stack = _stack(tmp_path) + with patch.object(stack, "_run") as run: + stack.delete(tmp_path / ".env") + + assert "--ignore-not-found" in run.call_args.args[0] + + def test_apply_sends_every_document(self, tmp_path: Path) -> None: + stack = _stack(tmp_path) + with patch.object(stack, "_run") as run: + stack.apply(tmp_path / ".env") + + assert _stdin_kinds(run) == ["Namespace", "Deployment", "PersistentVolumeClaim"] + + +class TestKubectl: + def test_missing_kubectl_is_reported(self) -> None: + with patch("flowmesh_stack.kubernetes.shutil.which", return_value=None): + with pytest.raises(KubectlError, match="kubectl is required"): + ensure_kubectl_available() diff --git a/tests/sdk/test_manifests.py b/tests/sdk/test_manifests.py new file mode 100644 index 000000000..6104dcab5 --- /dev/null +++ b/tests/sdk/test_manifests.py @@ -0,0 +1,378 @@ +"""Kubernetes manifest rendering: substitution, directives, and shipped assets.""" + +from pathlib import Path + +import pytest +import yaml +from flowmesh_cli.core.assets import asset_path +from flowmesh_stack.manifests import ( + ManifestError, + evaluate_condition, + render_documents, + render_manifests, + substitute, +) + +MANIFEST_ASSETS = ( + "00-namespace.yaml", + "10-rbac.yaml", + "20-redis.yaml", + "30-server.yaml", +) + + +def _asset_paths() -> list[Path]: + return [ + asset_path("flowmesh_cli_stack.assets", "k8s", name) for name in MANIFEST_ASSETS + ] + + +def _base_env(**overrides: str) -> dict[str, str]: + env = { + "K8S_NAMESPACE": "flowmesh", + "K8S_WORKER_NAMESPACE": "flowmesh", + "NODE_ROLE": "root", + } + env.update(overrides) + return env + + +def _by_kind(documents: list[dict]) -> dict[tuple[str, str], dict]: + return {(doc["kind"], doc["metadata"]["name"]): doc for doc in documents} + + +# ------------------------------------------------------------------ # +# Substitution +# ------------------------------------------------------------------ # + + +class TestSubstitution: + def test_value_is_expanded(self) -> None: + assert substitute("ns/${NAME}", {"NAME": "flowmesh"}) == "ns/flowmesh" + + def test_default_is_used_when_unset(self) -> None: + assert substitute("${NAME:-fallback}", {}) == "fallback" + + def test_set_value_beats_the_default(self) -> None: + assert substitute("${NAME:-fallback}", {"NAME": "real"}) == "real" + + def test_empty_default_renders_empty(self) -> None: + assert substitute("${NAME:-}", {}) == "" + + def test_multiple_references_in_one_value(self) -> None: + rendered = substitute("${A}:${B}", {"A": "host", "B": "8000"}) + assert rendered == "host:8000" + + def test_missing_value_without_a_default_is_an_error(self) -> None: + with pytest.raises(ManifestError, match="NAME is not set"): + substitute("${NAME}", {}) + + +# ------------------------------------------------------------------ # +# Conditions +# ------------------------------------------------------------------ # + + +class TestConditions: + @pytest.mark.parametrize("value", ["1", "true", "TRUE", "yes", "on"]) + def test_truthy_values(self, value: str) -> None: + assert evaluate_condition("FLAG", {"FLAG": value}) is True + + @pytest.mark.parametrize("value", ["", "0", "false", "no", "off"]) + def test_falsey_values(self, value: str) -> None: + assert evaluate_condition("FLAG", {"FLAG": value}) is False + + def test_unset_is_falsey(self) -> None: + assert evaluate_condition("FLAG", {}) is False + + def test_negation(self) -> None: + assert evaluate_condition("!FLAG", {}) is True + assert evaluate_condition("!FLAG", {"FLAG": "1"}) is False + + def test_equality(self) -> None: + assert evaluate_condition("ROLE==root", {"ROLE": "root"}) is True + assert evaluate_condition("ROLE==root", {"ROLE": "worker"}) is False + + def test_empty_expression_is_an_error(self) -> None: + with pytest.raises(ManifestError, match="Empty"): + evaluate_condition(" ", {}) + + +# ------------------------------------------------------------------ # +# Directives +# ------------------------------------------------------------------ # + + +class TestDirectives: + def test_falsey_document_leaves_the_stream(self) -> None: + source = """ + x-flowmesh-when: ENABLED + kind: ConfigMap + """ + assert render_documents(source, {}) == [] + + def test_truthy_document_drops_only_the_marker(self) -> None: + source = """ + x-flowmesh-when: ENABLED + kind: ConfigMap + """ + assert render_documents(source, {"ENABLED": "1"}) == [{"kind": "ConfigMap"}] + + def test_falsey_sequence_entry_leaves_the_sequence(self) -> None: + source = """ + kind: Pod + items: + - name: keep + - x-flowmesh-when: TLS + name: drop + """ + document = render_documents(source, {})[0] + assert document["items"] == [{"name": "keep"}] + + def test_falsey_mapping_value_takes_its_key(self) -> None: + source = """ + kind: Pod + spec: + storageClassName: + x-flowmesh-when: CLASS + x-flowmesh-value: ${CLASS:-} + """ + document = render_documents(source, {})[0] + assert "storageClassName" not in document["spec"] + + def test_conditional_scalar_renders_bare(self) -> None: + source = """ + kind: Pod + args: + - x-flowmesh-when: ACL + x-flowmesh-value: --aclfile + """ + document = render_documents(source, {"ACL": "1"})[0] + assert document["args"] == ["--aclfile"] + + def test_int_directive_yields_an_integer(self) -> None: + source = """ + kind: Service + port: + x-flowmesh-int: ${PORT:-8000} + """ + document = render_documents(source, {})[0] + assert document["port"] == 8000 + assert isinstance(document["port"], int) + + def test_int_directive_rejects_non_numeric_values(self) -> None: + source = """ + kind: Service + port: + x-flowmesh-int: ${PORT:-http} + """ + with pytest.raises(ManifestError, match="expected an integer"): + render_documents(source, {}) + + def test_file_directive_inlines_contents(self, tmp_path: Path) -> None: + config = tmp_path / "worker_config.yaml" + config.write_text("workers: []\n") + source = """ + kind: ConfigMap + data: + worker_config.yaml: + x-flowmesh-file: SERVER_WORKER_CONFIG + """ + document = render_documents(source, {"SERVER_WORKER_CONFIG": str(config)})[0] + assert document["data"]["worker_config.yaml"] == "workers: []\n" + + def test_file_directive_requires_the_variable(self) -> None: + source = """ + kind: ConfigMap + data: + x-flowmesh-file: SERVER_WORKER_CONFIG + """ + with pytest.raises(ManifestError, match="readable file"): + render_documents(source, {}) + + def test_env_values_directive_uses_the_env_file_only(self) -> None: + source = """ + kind: Secret + stringData: + x-flowmesh-env-values: true + """ + document = render_documents( + source, {"PROCESS_ONLY": "leaked"}, {"FROM_FILE": "kept"} + )[0] + assert document["stringData"] == {"FROM_FILE": "kept"} + + +# ------------------------------------------------------------------ # +# Rendering assets +# ------------------------------------------------------------------ # + + +class TestRenderManifests: + def test_document_order_is_preserved(self, tmp_path: Path) -> None: + first = tmp_path / "a.yaml" + first.write_text("kind: Namespace\n---\nkind: Role\n") + second = tmp_path / "b.yaml" + second.write_text("kind: Deployment\n") + + documents = list(yaml.safe_load_all(render_manifests([first, second], {}))) + assert [doc["kind"] for doc in documents] == [ + "Namespace", + "Role", + "Deployment", + ] + + def test_empty_output_is_an_error(self, tmp_path: Path) -> None: + empty = tmp_path / "empty.yaml" + empty.write_text("x-flowmesh-when: NEVER\nkind: ConfigMap\n") + with pytest.raises(ManifestError, match="No manifests"): + render_manifests([empty], {}) + + def test_unreadable_asset_is_an_error(self, tmp_path: Path) -> None: + with pytest.raises(ManifestError, match="Failed to read"): + render_manifests([tmp_path / "missing.yaml"], {}) + + +# ------------------------------------------------------------------ # +# Shipped assets +# ------------------------------------------------------------------ # + + +class TestShippedAssets: + def _documents(self, **overrides: str) -> list[dict]: + env = _base_env(**overrides) + env.setdefault( + "SERVER_WORKER_CONFIG", + asset_path( + "flowmesh_cli_stack.assets", "k8s", "worker_config.k8s.yaml" + ).as_posix(), + ) + return list(yaml.safe_load_all(render_manifests(_asset_paths(), env, env))) + + def test_namespace_is_applied_first(self) -> None: + assert self._documents()[0]["kind"] == "Namespace" + + def test_only_one_namespace_when_workers_share_it(self) -> None: + namespaces = [ + doc["metadata"]["name"] + for doc in self._documents() + if doc["kind"] == "Namespace" + ] + assert namespaces == ["flowmesh"] + + def test_a_separate_worker_namespace_is_created(self) -> None: + """RBAC is bound in the worker namespace, so it has to exist.""" + namespaces = [ + doc["metadata"]["name"] + for doc in self._documents( + K8S_WORKER_NAMESPACE="gpu-pool", + K8S_WORKER_NAMESPACE_DISTINCT="true", + ) + if doc["kind"] == "Namespace" + ] + assert namespaces == ["flowmesh", "gpu-pool"] + + def test_root_node_ships_both_redis_workloads(self) -> None: + names = { + doc["metadata"]["name"] + for doc in self._documents() + if doc["kind"] == "StatefulSet" + } + assert names == {"flowmesh-redis-control", "flowmesh-redis-telemetry"} + + def test_worker_node_ships_no_redis(self) -> None: + documents = self._documents(NODE_ROLE="worker") + assert not [doc for doc in documents if doc["kind"] == "StatefulSet"] + assert [doc for doc in documents if doc["kind"] == "Deployment"] + + def test_server_runs_a_single_replica(self) -> None: + deployment = _by_kind(self._documents())[("Deployment", "flowmesh-server")] + assert deployment["spec"]["replicas"] == 1 + assert deployment["spec"]["strategy"]["type"] == "Recreate" + + def test_node_rbac_is_opt_in(self) -> None: + kinds = {doc["kind"] for doc in self._documents()} + assert "ClusterRole" not in kinds + + enabled = {doc["kind"] for doc in self._documents(K8S_ENABLE_NODE_RBAC="true")} + assert {"ClusterRole", "ClusterRoleBinding"} <= enabled + + def test_service_ports_are_integers(self) -> None: + service = _by_kind(self._documents())[("Service", "flowmesh-server")] + assert isinstance(service["spec"]["ports"][0]["port"], int) + + def test_storage_class_is_omitted_when_unset(self) -> None: + claim = _by_kind(self._documents())[ + ("PersistentVolumeClaim", "flowmesh-results") + ] + assert "storageClassName" not in claim["spec"] + + def test_storage_class_is_set_when_configured(self) -> None: + claim = _by_kind(self._documents(K8S_STORAGE_CLASS="fast"))[ + ("PersistentVolumeClaim", "flowmesh-results") + ] + assert claim["spec"]["storageClassName"] == "fast" + + def test_redis_volumes_are_labelled_for_cleanup(self) -> None: + """`clean` selects volumes by label; controller-made claims need it too.""" + for name in ("flowmesh-redis-control", "flowmesh-redis-telemetry"): + statefulset = _by_kind(self._documents())[("StatefulSet", name)] + claim = statefulset["spec"]["volumeClaimTemplates"][0] + assert ( + claim["metadata"]["labels"]["app.kubernetes.io/part-of"] == "flowmesh" + ) + + def test_redis_keeps_the_pubsub_buffer_limit(self) -> None: + statefulset = _by_kind(self._documents())[ + ("StatefulSet", "flowmesh-redis-control") + ] + args = statefulset["spec"]["template"]["spec"]["containers"][0]["args"] + assert "--client-output-buffer-limit" in args + + def test_redis_acl_is_wired_only_when_enabled(self) -> None: + without = _by_kind(self._documents())[("StatefulSet", "flowmesh-redis-control")] + assert ( + "--aclfile" + not in without["spec"]["template"]["spec"]["containers"][0]["args"] + ) + + with_acl = _by_kind(self._documents(REDIS_ACL_ENABLED="true"))[ + ("StatefulSet", "flowmesh-redis-control") + ] + spec = with_acl["spec"]["template"]["spec"] + assert "--aclfile" in spec["containers"][0]["args"] + assert [volume["name"] for volume in spec["volumes"]] == ["acl"] + + def test_tls_secrets_are_mounted_only_when_named(self) -> None: + deployment = _by_kind(self._documents(SERVER_GRPC_TLS_SECRET="fm-tls"))[ + ("Deployment", "flowmesh-server") + ] + volumes = {v["name"] for v in deployment["spec"]["template"]["spec"]["volumes"]} + assert "server-tls" in volumes + + default = _by_kind(self._documents())[("Deployment", "flowmesh-server")] + default_volumes = { + v["name"] for v in default["spec"]["template"]["spec"]["volumes"] + } + assert "server-tls" not in default_volumes + + def test_server_receives_the_namespace_from_the_downward_api(self) -> None: + deployment = _by_kind(self._documents())[("Deployment", "flowmesh-server")] + container = deployment["spec"]["template"]["spec"]["containers"][0] + namespace_env = next( + entry for entry in container["env"] if entry["name"] == "POD_NAMESPACE" + ) + assert ( + namespace_env["valueFrom"]["fieldRef"]["fieldPath"] == "metadata.namespace" + ) + + def test_worker_config_is_carried_in_a_config_map(self) -> None: + config_map = _by_kind(self._documents())[ + ("ConfigMap", "flowmesh-worker-config") + ] + assert "provider: kubernetes" in config_map["data"]["worker_config.yaml"] + + def test_env_file_values_reach_the_server_secret(self) -> None: + secret = _by_kind(self._documents(FLOWMESH_API_KEY="secret-key"))[ + ("Secret", "flowmesh-server-env") + ] + assert secret["stringData"]["FLOWMESH_API_KEY"] == "secret-key" diff --git a/tests/server/test_external_worker.py b/tests/server/test_external_worker.py index cbce2a466..4fda2b592 100644 --- a/tests/server/test_external_worker.py +++ b/tests/server/test_external_worker.py @@ -157,6 +157,7 @@ def _explode(_principal): raise RuntimeError("Error while fetching server API version") monkeypatch.setattr(manager_mod, "docker_provider_spec", _explode) + monkeypatch.setattr(manager_mod, "kubernetes_provider_spec", _explode) monkeypatch.setattr(manager_mod, "vastai_provider_spec", _explode) mgr = manager_mod.WorkerManager( diff --git a/tests/server/test_k8s_worker_adapter.py b/tests/server/test_k8s_worker_adapter.py new file mode 100644 index 000000000..74827a190 --- /dev/null +++ b/tests/server/test_k8s_worker_adapter.py @@ -0,0 +1,606 @@ +"""Kubernetes worker provider: pod construction, lifecycle, and hardware probe.""" + +import asyncio +from typing import Any +from unittest.mock import MagicMock, patch + +import pytest +from kubernetes.client.exceptions import ApiException + +from server.hooks import PrincipalContext +from server.supervisor.adapters import kubernetes as k8s_adapter +from server.supervisor.adapters.base import WorkerTokenType, WorkerType +from server.supervisor.adapters.kubernetes import ( + MANAGED_LABEL, + NODE_ALIAS_LABEL, + WORKER_NAME_LABEL, + KubernetesWorkerAdapter, + KubernetesWorkerConfig, + KubernetesWorkerFactory, + _secret_field_names, + sanitize_label_value, + sanitize_object_name, +) +from server.supervisor.resource_manager import GpuArch +from server.supervisor.schemas import WorkerStatus + +# ------------------------------------------------------------------ # +# Helpers +# ------------------------------------------------------------------ # + + +def _token(value: str) -> WorkerTokenType: + return WorkerTokenType(value) + + +def _principal() -> PrincipalContext: + return PrincipalContext( + principal_id="p-test", + org_id="org", + external_id="ext", + principal_type="user", + scopes=[], + ) + + +def _adapter( + core: MagicMock | None = None, **config_kwargs: Any +) -> KubernetesWorkerAdapter: + return KubernetesWorkerAdapter( + token=_token("tok-123"), + name="worker-1", + pod_name="fm-worker-1", + config=KubernetesWorkerConfig(**config_kwargs), + core_api=core or MagicMock(), + node_alias="flowmesh_node", + owner=_principal(), + ) + + +def _api_error(status: int) -> ApiException: + return ApiException(status=status, reason="test") + + +def _container(manifest: dict[str, Any]) -> dict[str, Any]: + containers: list[dict[str, Any]] = manifest["spec"]["containers"] + return containers[0] + + +def _env_names(manifest: dict[str, Any]) -> set[str]: + return {entry["name"] for entry in _container(manifest).get("env", [])} + + +def _node(allocatable: dict[str, str], labels: dict[str, str]) -> MagicMock: + node = MagicMock() + node.status.allocatable = allocatable + node.metadata.labels = labels + return node + + +# ------------------------------------------------------------------ # +# Object naming +# ------------------------------------------------------------------ # + + +class TestObjectNaming: + def test_underscores_and_case_are_normalized(self) -> None: + assert sanitize_object_name("flowmesh_node-Worker_CPU_1") == ( + "flowmesh-node-worker-cpu-1" + ) + + def test_long_names_are_truncated_with_a_digest(self) -> None: + name = sanitize_object_name("x" * 200) + assert len(name) <= 63 + assert name != sanitize_object_name("y" * 200) + + def test_distinct_long_names_do_not_collide(self) -> None: + base = "flowmesh-server-worker-gpu-" + "a" * 60 + assert sanitize_object_name(base + "-one") != sanitize_object_name( + base + "-two" + ) + + def test_name_starting_with_a_separator_is_replaced(self) -> None: + assert sanitize_object_name("___").startswith("w-") + + def test_label_values_keep_underscores(self) -> None: + assert sanitize_label_value("flowmesh_node") == "flowmesh_node" + + +# ------------------------------------------------------------------ # +# Environment split +# ------------------------------------------------------------------ # + + +class TestEnvironmentSplit: + def test_credentials_go_to_the_secret_and_not_the_pod(self) -> None: + adapter = _adapter() + inline, secret = adapter._split_environment() + + assert "WORKER_TOKEN" in secret + assert "FLOWMESH_API_KEY" in secret + assert "WORKER_TOKEN" not in inline + assert "FLOWMESH_API_KEY" not in inline + + def test_secret_str_fields_are_all_treated_as_secret(self) -> None: + adapter = _adapter() + _, secret = adapter._split_environment() + + for field in _secret_field_names(KubernetesWorkerConfig): + assert field.upper() in secret + + def test_derived_secret_keys_exist_in_the_worker_environment(self) -> None: + """Guards the field-name -> env-key assumption behind the split.""" + adapter = _adapter() + base_env = adapter._base_environment() + + for field in _secret_field_names(KubernetesWorkerConfig): + assert field.upper() in base_env + + def test_non_secret_values_stay_inline(self) -> None: + adapter = _adapter() + inline, secret = adapter._split_environment() + + assert "SUPERVISOR_GRPC_TARGET" in inline + assert "SUPERVISOR_GRPC_TLS_CA_B64" in inline + assert not set(inline) & set(secret) + + def test_results_dir_points_at_the_mount_path(self) -> None: + adapter = _adapter(results_mount_path="/mnt/results") + assert adapter._base_environment()["RESULTS_DIR"] == "/mnt/results" + + def test_flowmesh_url_defaults_to_the_in_cluster_service(self) -> None: + adapter = _adapter() + url = adapter._base_environment()["FLOWMESH_BASE_URL"] + assert url.startswith("http://flowmesh-server.") + assert ".svc." in url + + +# ------------------------------------------------------------------ # +# Pod manifest +# ------------------------------------------------------------------ # + + +class TestPodManifest: + def test_labels_identify_the_managing_node(self) -> None: + manifest = _adapter().build_pod_manifest() + labels = manifest["metadata"]["labels"] + + assert labels[MANAGED_LABEL] == "true" + assert labels[NODE_ALIAS_LABEL] == "flowmesh_node" + assert labels[WORKER_NAME_LABEL] == "worker-1" + + def test_reaping_selector_matches_the_pod_labels(self) -> None: + """The reaper keys on the node alias, which survives a restart.""" + labels = _adapter().build_pod_manifest()["metadata"]["labels"] + + assert labels[MANAGED_LABEL] == "true" + assert NODE_ALIAS_LABEL in labels + + def test_pod_restarts_in_place(self) -> None: + manifest = _adapter().build_pod_manifest() + assert manifest["spec"]["restartPolicy"] == "Always" + + def test_secret_is_consumed_via_env_from(self) -> None: + adapter = _adapter() + manifest = adapter.build_pod_manifest() + env_from = _container(manifest)["envFrom"] + + assert env_from == [{"secretRef": {"name": adapter.secret_name}}] + + def test_cpu_worker_requests_no_gpu(self) -> None: + manifest = _adapter().build_pod_manifest() + limits = _container(manifest).get("resources", {}).get("limits", {}) + assert "nvidia.com/gpu" not in limits + + def test_gpu_worker_requests_the_gpu_resource(self) -> None: + manifest = _adapter( + worker_type=WorkerType.GPU, gpu_count=2 + ).build_pod_manifest() + limits = _container(manifest)["resources"]["limits"] + assert limits["nvidia.com/gpu"] == "2" + + def test_gpu_resource_name_is_configurable(self) -> None: + manifest = _adapter( + worker_type=WorkerType.GPU, gpu_resource_name="amd.com/gpu" + ).build_pod_manifest() + assert "amd.com/gpu" in _container(manifest)["resources"]["limits"] + + def test_device_indices_are_never_pinned_server_side(self) -> None: + """The device plugin owns assignment; a server-set index would fight it.""" + manifest = _adapter(worker_type=WorkerType.GPU).build_pod_manifest() + assert "CUDA_VISIBLE_DEVICES" not in _env_names(manifest) + + def test_results_default_to_an_ephemeral_volume(self) -> None: + manifest = _adapter().build_pod_manifest() + volumes = {v["name"]: v for v in manifest["spec"]["volumes"]} + assert volumes["results"]["emptyDir"] == {} + + def test_results_pvc_is_mounted_when_configured(self) -> None: + manifest = _adapter(results_pvc="fm-results").build_pod_manifest() + volumes = {v["name"]: v for v in manifest["spec"]["volumes"]} + assert volumes["results"]["persistentVolumeClaim"]["claimName"] == "fm-results" + + def test_shm_volume_is_added_when_sized(self) -> None: + manifest = _adapter(shm_size="2Gi").build_pod_manifest() + volumes = {v["name"]: v for v in manifest["spec"]["volumes"]} + mounts = {m["name"]: m for m in _container(manifest)["volumeMounts"]} + + assert volumes["dshm"]["emptyDir"] == {"medium": "Memory", "sizeLimit": "2Gi"} + assert mounts["dshm"]["mountPath"] == "/dev/shm" + + def test_no_shm_volume_by_default(self) -> None: + manifest = _adapter().build_pod_manifest() + assert "dshm" not in {v["name"] for v in manifest["spec"]["volumes"]} + + def test_scheduling_hints_are_carried_through(self) -> None: + manifest = _adapter( + node_selector={"gpu": "true"}, + tolerations=[{"key": "gpu", "operator": "Exists"}], + service_account_name="fm-worker", + image_pull_secrets=["regcred"], + runtime_class_name="nvidia", + priority_class_name="high", + ).build_pod_manifest() + spec = manifest["spec"] + + assert spec["nodeSelector"] == {"gpu": "true"} + assert spec["tolerations"] == [{"key": "gpu", "operator": "Exists"}] + assert spec["serviceAccountName"] == "fm-worker" + assert spec["imagePullSecrets"] == [{"name": "regcred"}] + assert spec["runtimeClassName"] == "nvidia" + assert spec["priorityClassName"] == "high" + + def test_unset_optional_fields_are_omitted(self) -> None: + spec = _adapter().build_pod_manifest()["spec"] + for key in ("nodeSelector", "tolerations", "runtimeClassName"): + assert key not in spec + + def test_pod_overrides_deep_merge_mappings(self) -> None: + manifest = _adapter( + pod_overrides={"metadata": {"annotations": {"team": "ml"}}} + ).build_pod_manifest() + + assert manifest["metadata"]["annotations"] == {"team": "ml"} + assert manifest["metadata"]["name"] == "fm-worker-1" + + def test_pod_overrides_replace_sequences(self) -> None: + manifest = _adapter( + pod_overrides={"spec": {"tolerations": [{"key": "override"}]}} + ).build_pod_manifest() + + assert manifest["spec"]["tolerations"] == [{"key": "override"}] + + def test_secret_manifest_carries_only_credentials(self) -> None: + adapter = _adapter() + secret = adapter.build_secret_manifest() + _, expected = adapter._split_environment() + + assert secret["kind"] == "Secret" + assert secret["stringData"] == expected + + +# ------------------------------------------------------------------ # +# Lifecycle +# ------------------------------------------------------------------ # + + +class TestLifecycle: + def test_start_creates_the_secret_before_the_pod(self) -> None: + core = MagicMock() + core.read_namespaced_pod.side_effect = _api_error(404) + adapter = _adapter(core) + + assert asyncio.run(adapter.start()) is True + core.create_namespaced_secret.assert_called_once() + core.create_namespaced_pod.assert_called_once() + + def test_start_adopts_a_running_pod(self) -> None: + core = MagicMock() + core.read_namespaced_pod.return_value.status.phase = "Running" + adapter = _adapter(core) + + assert asyncio.run(adapter.start()) is True + core.create_namespaced_pod.assert_not_called() + + def test_start_replaces_a_terminal_pod(self) -> None: + core = MagicMock() + terminal = MagicMock() + terminal.status.phase = "Failed" + core.read_namespaced_pod.side_effect = [terminal, _api_error(404)] + adapter = _adapter(core) + + assert asyncio.run(adapter.start()) is True + core.delete_namespaced_pod.assert_called_once() + core.create_namespaced_pod.assert_called_once() + + def test_replacing_a_pod_waits_for_the_name_to_free(self) -> None: + """Deletion is asynchronous; reusing the name too early collides.""" + core = MagicMock() + terminal = MagicMock() + terminal.status.phase = "Failed" + terminating = MagicMock() + terminating.status.phase = "Failed" + core.read_namespaced_pod.side_effect = [ + terminal, + terminating, + _api_error(404), + ] + adapter = _adapter(core) + + with patch.object(k8s_adapter, "_DELETE_POLL_SEC", 0): + assert asyncio.run(adapter.start()) is True + + assert core.read_namespaced_pod.call_count == 3 + core.create_namespaced_pod.assert_called_once() + assert core.delete_namespaced_pod.call_args.kwargs["grace_period_seconds"] == 0 + + def test_replacement_gives_up_when_the_pod_never_goes_away(self) -> None: + core = MagicMock() + terminal = MagicMock() + terminal.status.phase = "Failed" + core.read_namespaced_pod.return_value = terminal + adapter = _adapter(core) + + with patch.object(k8s_adapter, "_DELETE_TIMEOUT_SEC", 0): + assert asyncio.run(adapter.start()) is False + + core.create_namespaced_pod.assert_not_called() + + def test_failed_pod_creation_removes_the_orphaned_secret(self) -> None: + core = MagicMock() + core.read_namespaced_pod.side_effect = _api_error(404) + core.create_namespaced_pod.side_effect = _api_error(422) + adapter = _adapter(core) + + assert asyncio.run(adapter.start()) is False + core.delete_namespaced_secret.assert_called_once() + assert adapter.status is WorkerStatus.STOPPED + + def test_existing_secret_is_replaced(self) -> None: + core = MagicMock() + core.read_namespaced_pod.side_effect = _api_error(404) + core.create_namespaced_secret.side_effect = _api_error(409) + adapter = _adapter(core) + + assert asyncio.run(adapter.start()) is True + core.replace_namespaced_secret.assert_called_once() + + def test_stop_deletes_pod_and_secret(self) -> None: + core = MagicMock() + adapter = _adapter(core) + adapter.set_status(WorkerStatus.RUNNING) + + assert asyncio.run(adapter.stop()) is True + core.delete_namespaced_pod.assert_called_once() + core.delete_namespaced_secret.assert_called_once() + + def test_stop_tolerates_an_already_deleted_pod(self) -> None: + core = MagicMock() + core.delete_namespaced_pod.side_effect = _api_error(404) + core.delete_namespaced_secret.side_effect = _api_error(404) + adapter = _adapter(core) + adapter.set_status(WorkerStatus.RUNNING) + + assert asyncio.run(adapter.stop()) is True + + def test_stop_reports_failure_and_restores_status(self) -> None: + core = MagicMock() + core.delete_namespaced_pod.side_effect = _api_error(500) + adapter = _adapter(core) + adapter.set_status(WorkerStatus.RUNNING) + + assert asyncio.run(adapter.stop()) is False + assert adapter.status is WorkerStatus.RUNNING + + def test_stopping_a_stopped_worker_is_a_no_op(self) -> None: + core = MagicMock() + adapter = _adapter(core) + + assert asyncio.run(adapter.stop()) is True + core.delete_namespaced_pod.assert_not_called() + + +# ------------------------------------------------------------------ # +# Hardware probe +# ------------------------------------------------------------------ # + + +class TestHardwareProbe: + def test_probe_degrades_without_node_permission(self) -> None: + core = MagicMock() + core.list_node.side_effect = _api_error(403) + adapter = _adapter(core) + + asyncio.run(adapter.prepare()) + assert adapter.get_info().hardware is None + + def test_probe_reads_allocatable_capacity(self) -> None: + core = MagicMock() + core.list_node.return_value.items = [ + _node({"cpu": "32", "memory": "65536Ki"}, {}) + ] + adapter = _adapter(core) + + asyncio.run(adapter.prepare()) + hardware = adapter.get_info().hardware + + assert hardware is not None + assert hardware.cpu.logical_cores == 32 + assert hardware.memory.total_bytes == 65536 * 1024 + + def test_probe_parses_millicore_quantities(self) -> None: + core = MagicMock() + core.list_node.return_value.items = [_node({"cpu": "7800m"}, {})] + adapter = _adapter(core) + + asyncio.run(adapter.prepare()) + hardware = adapter.get_info().hardware + + assert hardware is not None + assert hardware.cpu.logical_cores == 7 + + def test_probe_derives_gpu_arch_from_node_labels(self) -> None: + core = MagicMock() + core.list_node.return_value.items = [ + _node({"cpu": "8"}, {"nvidia.com/gpu.product": "NVIDIA-H100-80GB-HBM3"}) + ] + adapter = _adapter(core, worker_type=WorkerType.GPU, gpu_count=2) + + asyncio.run(adapter.prepare()) + hardware = adapter.get_info().hardware + + assert hardware is not None + assert hardware.gpu.gpu_arch == GpuArch.HOPPER.value + assert len(hardware.gpu.devices) == 2 + + def test_probe_without_matching_nodes_returns_nothing(self) -> None: + core = MagicMock() + core.list_node.return_value.items = [] + adapter = _adapter(core) + + asyncio.run(adapter.prepare()) + assert adapter.get_info().hardware is None + + def test_probe_filters_by_node_selector(self) -> None: + core = MagicMock() + core.list_node.return_value.items = [] + adapter = _adapter(core, node_selector={"gpu": "true", "zone": "a"}) + + asyncio.run(adapter.prepare()) + selector = core.list_node.call_args.kwargs["label_selector"] + + assert set(selector.split(",")) == {"gpu=true", "zone=a"} + + +# ------------------------------------------------------------------ # +# Image selection +# ------------------------------------------------------------------ # + + +class TestImageSelection: + def test_cpu_worker_uses_the_cpu_image(self) -> None: + assert _adapter().get_image_name().endswith("-cpu") + + def test_gpu_worker_uses_the_gpu_image(self) -> None: + adapter = _adapter(worker_type=WorkerType.GPU) + assert adapter.get_image_name().endswith("-gpu") + + +# ------------------------------------------------------------------ # +# Factory +# ------------------------------------------------------------------ # + + +class TestFactory: + def _factory(self, core: MagicMock) -> KubernetesWorkerFactory: + with ( + patch.object(k8s_adapter.config, "load_incluster_config"), + patch.object(k8s_adapter.client, "ApiClient"), + patch.object(k8s_adapter.client, "CoreV1Api", return_value=core), + ): + return KubernetesWorkerFactory(_principal()) + + def test_construction_without_a_cluster_raises(self) -> None: + """An unreachable cluster must make the node report the provider absent.""" + with ( + patch.object( + k8s_adapter.config, + "load_incluster_config", + side_effect=k8s_adapter.config.ConfigException("not in cluster"), + ), + patch.object( + k8s_adapter.config, + "load_kube_config", + side_effect=k8s_adapter.config.ConfigException("no kubeconfig"), + ), + ): + with pytest.raises(k8s_adapter.config.ConfigException): + KubernetesWorkerFactory(_principal()) + + def test_orphaned_pods_are_reaped_once(self) -> None: + core = MagicMock() + pod = MagicMock() + pod.metadata.name = "fm-stale" + core.list_namespaced_pod.return_value.items = [pod] + factory = self._factory(core) + + factory.create_worker(_token("t1"), KubernetesWorkerConfig()) + factory.create_worker(_token("t2"), KubernetesWorkerConfig()) + + core.list_namespaced_pod.assert_called_once() + core.delete_namespaced_pod.assert_called_once_with( + name="fm-stale", namespace="default" + ) + + def test_reaping_selects_this_node_only(self) -> None: + core = MagicMock() + core.list_namespaced_pod.return_value.items = [] + factory = self._factory(core) + + factory.create_worker(_token("t1"), KubernetesWorkerConfig()) + selector = core.list_namespaced_pod.call_args.kwargs["label_selector"] + + assert f"{MANAGED_LABEL}=true" in selector + assert NODE_ALIAS_LABEL in selector + + def test_reaping_survives_a_listing_failure(self) -> None: + core = MagicMock() + core.list_namespaced_pod.side_effect = _api_error(403) + factory = self._factory(core) + + worker = factory.create_worker(_token("t1"), KubernetesWorkerConfig()) + assert worker.name + + def test_pod_names_are_api_safe(self) -> None: + core = MagicMock() + core.list_namespaced_pod.return_value.items = [] + factory = self._factory(core) + + worker = factory.create_worker( + _token("t1"), KubernetesWorkerConfig(worker_alias="Train_Worker_01") + ) + + assert worker.pod_name == sanitize_object_name(worker.pod_name) + + def test_worker_names_increment_per_type(self) -> None: + core = MagicMock() + core.list_namespaced_pod.return_value.items = [] + factory = self._factory(core) + + first = factory.create_worker(_token("t1"), KubernetesWorkerConfig()) + second = factory.create_worker(_token("t2"), KubernetesWorkerConfig()) + + assert first.name != second.name + + def test_cleanup_closes_the_api_client(self) -> None: + api_client = MagicMock() + with ( + patch.object(k8s_adapter.config, "load_incluster_config"), + patch.object(k8s_adapter.client, "ApiClient", return_value=api_client), + patch.object(k8s_adapter.client, "CoreV1Api"), + ): + factory = KubernetesWorkerFactory(_principal()) + + factory.cleanup() + + api_client.close.assert_called_once_with() + + def test_client_falls_back_to_kubeconfig_outside_a_cluster(self) -> None: + with ( + patch.object( + k8s_adapter.config, + "load_incluster_config", + side_effect=k8s_adapter.config.ConfigException("not in cluster"), + ), + patch.object(k8s_adapter.config, "load_kube_config") as load_kube_config, + patch.object(k8s_adapter.client, "ApiClient"), + patch.object(k8s_adapter.client, "CoreV1Api"), + ): + KubernetesWorkerFactory(_principal()) + + load_kube_config.assert_called_once() + + def test_destroy_rejects_a_foreign_worker(self) -> None: + factory = self._factory(MagicMock()) + with pytest.raises(ValueError, match="Invalid worker type"): + factory.destroy_worker(MagicMock()) diff --git a/uv.lock b/uv.lock index 0f7ab1a8b..530d81d31 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 2 requires-python = ">=3.12" resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'darwin'", @@ -1499,6 +1499,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/55/e2/2537ebcff11c1ee1ff17d8d0b6f4db75873e3b0fb32c2d4a2ee31ecb310a/docstring_parser-0.17.0-py3-none-any.whl", hash = "sha256:cf2569abd23dce8099b300f9b4fa8191e9582dda731fd533daf54c4551658708", size = 36896, upload-time = "2025-07-21T07:35:00.684Z" }, ] +[[package]] +name = "durationpy" +version = "0.11" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b0/5d/5f8571bd5dedc80863191621ac4be001f3f3dd8315d2ec078705dab7dec1/durationpy-0.11.tar.gz", hash = "sha256:181898e1ae282e288f0a2291829656bf1b6b3aadf30a97993b85db4943642905", size = 3582, upload-time = "2026-08-26T13:56:00.991Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e6/c4/ebdf7837bc4ef6fd98cfb013c28855bb358467bf86c1af011bbc21e21df0/durationpy-0.11-py3-none-any.whl", hash = "sha256:a739fe2b8972c250ff72f8e2c488d18cf25f7b852f49ee76048775d5171df30c", size = 4133, upload-time = "2026-08-26T13:55:59.456Z" }, +] + [[package]] name = "einops" version = "0.8.1" @@ -1917,6 +1926,7 @@ ci = [ { name = "ipython" }, { name = "isort" }, { name = "jinja2" }, + { name = "kubernetes" }, { name = "lumid-hooks" }, { name = "matplotlib" }, { name = "mcp" }, @@ -2080,6 +2090,7 @@ runtime-server = [ { name = "flowmesh-hook" }, { name = "grpcio" }, { name = "httpx" }, + { name = "kubernetes" }, { name = "lumid-hooks" }, { name = "protobuf" }, { name = "pydantic" }, @@ -2225,6 +2236,7 @@ ci = [ { name = "ipython", specifier = ">=9.5.0" }, { name = "isort", specifier = ">=7.0.0" }, { name = "jinja2", specifier = ">=3.1.6" }, + { name = "kubernetes", specifier = ">=31.0.0" }, { name = "lumid-hooks", specifier = ">=0.2.0" }, { name = "matplotlib", specifier = ">=3.10.6" }, { name = "mcp", specifier = ">=1.28.1" }, @@ -2386,6 +2398,7 @@ runtime-server = [ { name = "flowmesh-hook", editable = "hook" }, { name = "grpcio", specifier = ">=1.76.0" }, { name = "httpx", specifier = ">=0.28.1" }, + { name = "kubernetes", specifier = ">=31.0.0" }, { name = "lumid-hooks", specifier = ">=0.2.0" }, { name = "protobuf", specifier = ">=5.29.6" }, { name = "pydantic", specifier = ">=2.12.3" }, @@ -3632,6 +3645,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/80/be/3578e8afd18c88cdf9cb4cffde75a96d2be38c5a903f1ed0ceec061bd09e/kiwisolver-1.4.9-cp314-cp314t-win_arm64.whl", hash = "sha256:4a48a2ce79d65d363597ef7b567ce3d14d68783d2b2263d98db3d9477805ba32", size = 70260, upload-time = "2025-08-10T21:27:36.606Z" }, ] +[[package]] +name = "kubernetes" +version = "36.0.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohttp" }, + { name = "certifi" }, + { name = "durationpy" }, + { name = "python-dateutil" }, + { name = "pyyaml" }, + { name = "requests" }, + { name = "requests-oauthlib" }, + { name = "six" }, + { name = "urllib3" }, + { name = "websocket-client" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ca/57/b07b96353f902aa1bdbe00e878e3a12a137977d03a962479785576aa8ec9/kubernetes-36.0.3.tar.gz", hash = "sha256:36993ed25ce59b789c9341473a228fcf268504a2fec7c2b2b1531d73072e5ce7", size = 2337528, upload-time = "2026-07-13T20:38:12.128Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5b/30/a96d47df739689ac0001ade0afefc16e3b477fc2fb426b568515fdc8afce/kubernetes-36.0.3-py2.py3-none-any.whl", hash = "sha256:8fde9241c4b298e6374a069dcf728359b4e72c2fb29489a975ba4e1c047cf10f", size = 4618066, upload-time = "2026-07-13T20:38:10.172Z" }, +] + [[package]] name = "lark" version = "1.2.2" @@ -5106,6 +5140,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/14/5d/e000de781d92b732d52c572517db0e9e3a0085795f8bdc18201713c52d1f/nvtx-0.2.15-cp314-cp314t-win_amd64.whl", hash = "sha256:9d1d10db4fb4a3b0ffd6ed37bf25f0a966a3b4d34b3c9abb1f6572732959a6e5", size = 149109, upload-time = "2026-03-18T10:03:21.615Z" }, ] +[[package]] +name = "oauthlib" +version = "3.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0b/5f/19930f824ffeb0ad4372da4812c50edbd1434f678c90c2733e1188edfc63/oauthlib-3.3.1.tar.gz", hash = "sha256:0f0f8aa759826a193cf66c12ea1af1637f87b9b4622d46e866952bb022e538c9", size = 185918, upload-time = "2025-06-19T22:48:08.269Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/be/9c/92789c596b8df838baa98fa71844d84283302f7604ed565dafe5a6b5041a/oauthlib-3.3.1-py3-none-any.whl", hash = "sha256:88119c938d2b8fb88561af5f6ee0eec8cc8d552b7bb1f712743136eb7523b7a1", size = 160065, upload-time = "2025-06-19T22:48:06.508Z" }, +] + [[package]] name = "omegaconf" version = "2.3.0" @@ -6727,28 +6770,48 @@ wheels = [ [[package]] name = "pyyaml" -version = "6.0.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/54/ed/79a089b6be93607fa5cdaedf301d7dfb23af5f25c398d5ead2525b063e17/pyyaml-6.0.2.tar.gz", hash = "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e", size = 130631, upload-time = "2024-08-06T20:33:50.674Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/86/0c/c581167fc46d6d6d7ddcfb8c843a4de25bdd27e4466938109ca68492292c/PyYAML-6.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab", size = 183873, upload-time = "2024-08-06T20:32:25.131Z" }, - { url = "https://files.pythonhosted.org/packages/a8/0c/38374f5bb272c051e2a69281d71cba6fdb983413e6758b84482905e29a5d/PyYAML-6.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725", size = 173302, upload-time = "2024-08-06T20:32:26.511Z" }, - { url = "https://files.pythonhosted.org/packages/c3/93/9916574aa8c00aa06bbac729972eb1071d002b8e158bd0e83a3b9a20a1f7/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5", size = 739154, upload-time = "2024-08-06T20:32:28.363Z" }, - { url = "https://files.pythonhosted.org/packages/95/0f/b8938f1cbd09739c6da569d172531567dbcc9789e0029aa070856f123984/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425", size = 766223, upload-time = "2024-08-06T20:32:30.058Z" }, - { url = "https://files.pythonhosted.org/packages/b9/2b/614b4752f2e127db5cc206abc23a8c19678e92b23c3db30fc86ab731d3bd/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476", size = 767542, upload-time = "2024-08-06T20:32:31.881Z" }, - { url = "https://files.pythonhosted.org/packages/d4/00/dd137d5bcc7efea1836d6264f049359861cf548469d18da90cd8216cf05f/PyYAML-6.0.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48", size = 731164, upload-time = "2024-08-06T20:32:37.083Z" }, - { url = "https://files.pythonhosted.org/packages/c9/1f/4f998c900485e5c0ef43838363ba4a9723ac0ad73a9dc42068b12aaba4e4/PyYAML-6.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b", size = 756611, upload-time = "2024-08-06T20:32:38.898Z" }, - { url = "https://files.pythonhosted.org/packages/df/d1/f5a275fdb252768b7a11ec63585bc38d0e87c9e05668a139fea92b80634c/PyYAML-6.0.2-cp312-cp312-win32.whl", hash = "sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4", size = 140591, upload-time = "2024-08-06T20:32:40.241Z" }, - { url = "https://files.pythonhosted.org/packages/0c/e8/4f648c598b17c3d06e8753d7d13d57542b30d56e6c2dedf9c331ae56312e/PyYAML-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8", size = 156338, upload-time = "2024-08-06T20:32:41.93Z" }, - { url = "https://files.pythonhosted.org/packages/ef/e3/3af305b830494fa85d95f6d95ef7fa73f2ee1cc8ef5b495c7c3269fb835f/PyYAML-6.0.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba", size = 181309, upload-time = "2024-08-06T20:32:43.4Z" }, - { url = "https://files.pythonhosted.org/packages/45/9f/3b1c20a0b7a3200524eb0076cc027a970d320bd3a6592873c85c92a08731/PyYAML-6.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1", size = 171679, upload-time = "2024-08-06T20:32:44.801Z" }, - { url = "https://files.pythonhosted.org/packages/7c/9a/337322f27005c33bcb656c655fa78325b730324c78620e8328ae28b64d0c/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133", size = 733428, upload-time = "2024-08-06T20:32:46.432Z" }, - { url = "https://files.pythonhosted.org/packages/a3/69/864fbe19e6c18ea3cc196cbe5d392175b4cf3d5d0ac1403ec3f2d237ebb5/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484", size = 763361, upload-time = "2024-08-06T20:32:51.188Z" }, - { url = "https://files.pythonhosted.org/packages/04/24/b7721e4845c2f162d26f50521b825fb061bc0a5afcf9a386840f23ea19fa/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5", size = 759523, upload-time = "2024-08-06T20:32:53.019Z" }, - { url = "https://files.pythonhosted.org/packages/2b/b2/e3234f59ba06559c6ff63c4e10baea10e5e7df868092bf9ab40e5b9c56b6/PyYAML-6.0.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc", size = 726660, upload-time = "2024-08-06T20:32:54.708Z" }, - { url = "https://files.pythonhosted.org/packages/fe/0f/25911a9f080464c59fab9027482f822b86bf0608957a5fcc6eaac85aa515/PyYAML-6.0.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652", size = 751597, upload-time = "2024-08-06T20:32:56.985Z" }, - { url = "https://files.pythonhosted.org/packages/14/0d/e2c3b43bbce3cf6bd97c840b46088a3031085179e596d4929729d8d68270/PyYAML-6.0.2-cp313-cp313-win32.whl", hash = "sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183", size = 140527, upload-time = "2024-08-06T20:33:03.001Z" }, - { url = "https://files.pythonhosted.org/packages/fa/de/02b54f42487e3d3c6efb3f89428677074ca7bf43aae402517bc7cca949f3/PyYAML-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563", size = 156446, upload-time = "2024-08-06T20:33:04.33Z" }, +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, + { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, + { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, + { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, ] [[package]] @@ -6975,6 +7038,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d7/8e/7540e8a2036f79a125c1d2ebadf69ed7901608859186c856fa0388ef4197/requests-2.33.1-py3-none-any.whl", hash = "sha256:4e6d1ef462f3626a1f0a0a9c42dd93c63bad33f9f1c1937509b8c5c8718ab56a", size = 64947, upload-time = "2026-03-30T16:09:13.83Z" }, ] +[[package]] +name = "requests-oauthlib" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "oauthlib" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/42/f2/05f29bc3913aea15eb670be136045bf5c5bbf4b99ecb839da9b422bb2c85/requests-oauthlib-2.0.0.tar.gz", hash = "sha256:b3dffaebd884d8cd778494369603a9e7b58d29111bf6b41bdc2dcd87203af4e9", size = 55650, upload-time = "2024-03-22T20:32:29.939Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3b/5d/63d4ae3b9daea098d5d6f5da83984853c1bbacd5dc826764b249fe119d24/requests_oauthlib-2.0.0-py2.py3-none-any.whl", hash = "sha256:7dd8a5c40426b779b0868c404bdef9768deccf22749cde15852df527e6269b36", size = 24179, upload-time = "2024-03-22T20:32:28.055Z" }, +] + [[package]] name = "respx" version = "0.22.0"