Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,9 @@ and artifacts.
- **[`docs/ENV.md`](docs/ENV.md)** — curated server / worker /
supervisor env var tables (the knobs you actually tune). Full schema
in `cli/stack/src/flowmesh_cli_stack/env_schema.py`.
- **[`docs/KUBERNETES.md`](docs/KUBERNETES.md)** — deploying the stack on
Kubernetes: `--backend k8s`, the `kubernetes` worker provider, RBAC, TLS,
and the single-replica server constraint.
- **[`docs/PLUGINS.md`](docs/PLUGINS.md)** — plugin extension contract,
loader semantics (`FLOWMESH_PLUGINS`), and a worked example.

Expand Down
1 change: 1 addition & 0 deletions cli/stack/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -28,4 +28,5 @@ where = ["src"]
"flowmesh_cli_stack" = [
"assets/**",
"assets/.env.example",
"assets/k8s/**",
]
32 changes: 32 additions & 0 deletions cli/stack/src/flowmesh_cli_stack/assets/.env.example
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,38 @@ SERVER_GRPC_TLS_CA_FILE=/etc/ssl/server/server-ca.pem
SERVER_GRPC_TLS_CERT_FILE=/etc/ssl/server/server.pem
SERVER_GRPC_TLS_KEY_FILE=/etc/ssl/server/server.key

# ==== Kubernetes Backend ====
# Used when STACK_BACKEND=k8s. The stack is deployed into one
# namespace; the Kubernetes scheduler places workers across nodes.
# A gRPC TLS certificate must carry the supervisor Service name
# as a SAN, since workers dial the Service, not a host.
STACK_BACKEND=compose
K8S_NAMESPACE=flowmesh
# Namespace for worker pods; defaults to K8S_NAMESPACE.
K8S_WORKER_NAMESPACE=
# kubectl context; empty uses current.
K8S_CONTEXT=
# kubeconfig path; empty uses default.
K8S_KUBECONFIG=
K8S_SUPERVISOR_SERVICE=flowmesh-supervisor
K8S_SERVER_SERVICE=flowmesh-server
K8S_CLUSTER_DOMAIN=cluster.local
K8S_GPU_RESOURCE_NAME=nvidia.com/gpu
K8S_SERVER_SERVICE_TYPE=ClusterIP
K8S_IMAGE_PULL_POLICY=IfNotPresent
# Storage class for stack volumes; empty uses default.
K8S_STORAGE_CLASS=
K8S_REDIS_STORAGE_SIZE=8Gi
K8S_RESULTS_STORAGE_SIZE=20Gi
K8S_RESULTS_ACCESS_MODE=ReadWriteOnce
# Grant cluster-scoped node read access so worker hardware
# can be reported before a worker starts.
K8S_ENABLE_NODE_RBAC=false
# Secret holding the server gRPC TLS files.
SERVER_GRPC_TLS_SECRET=
# Secret holding the Redis TLS files.
REDIS_TLS_SECRET=

# ==== Supervisor gRPC ====
# Tuning for the supervisor's gRPC server and worker connections.
# Leave SUPERVISOR_GRPC_EXTERNAL_PORT empty unless workers connect
Expand Down
16 changes: 16 additions & 0 deletions cli/stack/src/flowmesh_cli_stack/assets/k8s/00-namespace.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
apiVersion: v1
kind: Namespace
metadata:
name: ${K8S_NAMESPACE:-flowmesh}
labels:
app.kubernetes.io/part-of: flowmesh
---
# Only when worker pods live outside the stack namespace; the RBAC below is
# bound there and would otherwise apply into a namespace that does not exist.
x-flowmesh-when: K8S_WORKER_NAMESPACE_DISTINCT
apiVersion: v1
kind: Namespace
metadata:
name: ${K8S_WORKER_NAMESPACE:-flowmesh}
labels:
app.kubernetes.io/part-of: flowmesh
77 changes: 77 additions & 0 deletions cli/stack/src/flowmesh_cli_stack/assets/k8s/10-rbac.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: flowmesh-server
namespace: ${K8S_NAMESPACE:-flowmesh}
labels:
app.kubernetes.io/part-of: flowmesh
---
# The supervisor creates worker pods and the per-worker secret carrying their
# credentials, and reads pod logs for diagnostics.
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: flowmesh-server
namespace: ${K8S_WORKER_NAMESPACE:-flowmesh}
labels:
app.kubernetes.io/part-of: flowmesh
rules:
- apiGroups: [""]
resources: ["pods"]
verbs: ["get", "list", "watch", "create", "delete", "deletecollection"]
- apiGroups: [""]
resources: ["pods/log"]
verbs: ["get"]
- apiGroups: [""]
resources: ["secrets"]
verbs: ["get", "list", "create", "update", "delete", "deletecollection"]
- apiGroups: [""]
resources: ["events"]
verbs: ["get", "list"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: flowmesh-server
namespace: ${K8S_WORKER_NAMESPACE:-flowmesh}
labels:
app.kubernetes.io/part-of: flowmesh
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: flowmesh-server
subjects:
- kind: ServiceAccount
name: flowmesh-server
namespace: ${K8S_NAMESPACE:-flowmesh}
---
# Optional. Lets the supervisor report a worker's hardware before the worker
# starts, by reading node allocatable capacity and GPU labels. Without it the
# hardware preview is empty and everything else works unchanged.
x-flowmesh-when: K8S_ENABLE_NODE_RBAC
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: flowmesh-server-nodes-${K8S_NAMESPACE:-flowmesh}
labels:
app.kubernetes.io/part-of: flowmesh
rules:
- apiGroups: [""]
resources: ["nodes"]
verbs: ["get", "list"]
---
x-flowmesh-when: K8S_ENABLE_NODE_RBAC
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: flowmesh-server-nodes-${K8S_NAMESPACE:-flowmesh}
labels:
app.kubernetes.io/part-of: flowmesh
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: flowmesh-server-nodes-${K8S_NAMESPACE:-flowmesh}
subjects:
- kind: ServiceAccount
name: flowmesh-server
namespace: ${K8S_NAMESPACE:-flowmesh}
205 changes: 205 additions & 0 deletions cli/stack/src/flowmesh_cli_stack/assets/k8s/20-redis.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,205 @@
# Redis runs on root nodes only. A worker node connects to the root node's
# Redis through REDIS_CONTROL_URL / REDIS_TELEMETRY_URL.
x-flowmesh-when: NODE_ROLE==root
apiVersion: v1
kind: Secret
metadata:
name: flowmesh-redis-acl
namespace: ${K8S_NAMESPACE:-flowmesh}
labels:
app.kubernetes.io/part-of: flowmesh
type: Opaque
stringData:
users.acl: |
user default off
user ${REDIS_USERNAME:-admin} on >${REDIS_PASSWORD:-} ~* &* +@all
---
x-flowmesh-when: NODE_ROLE==root
apiVersion: v1
kind: Service
metadata:
name: flowmesh-redis-control
namespace: ${K8S_NAMESPACE:-flowmesh}
labels:
app.kubernetes.io/part-of: flowmesh
app.kubernetes.io/component: redis-control
spec:
clusterIP: None
selector:
app.kubernetes.io/part-of: flowmesh
app.kubernetes.io/component: redis-control
ports:
- name: redis
port: 6379
targetPort: 6379
---
x-flowmesh-when: NODE_ROLE==root
apiVersion: v1
kind: Service
metadata:
name: flowmesh-redis-telemetry
namespace: ${K8S_NAMESPACE:-flowmesh}
labels:
app.kubernetes.io/part-of: flowmesh
app.kubernetes.io/component: redis-telemetry
spec:
clusterIP: None
selector:
app.kubernetes.io/part-of: flowmesh
app.kubernetes.io/component: redis-telemetry
ports:
- name: redis
port: 6379
targetPort: 6379
---
x-flowmesh-when: NODE_ROLE==root
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: flowmesh-redis-control
namespace: ${K8S_NAMESPACE:-flowmesh}
labels:
app.kubernetes.io/part-of: flowmesh
app.kubernetes.io/component: redis-control
spec:
serviceName: flowmesh-redis-control
replicas: 1
selector:
matchLabels:
app.kubernetes.io/part-of: flowmesh
app.kubernetes.io/component: redis-control
template:
metadata:
labels:
app.kubernetes.io/part-of: flowmesh
app.kubernetes.io/component: redis-control
spec:
containers:
- name: redis
image: redis:7-alpine
args:
- redis-server
- --save
- "60 1"
- --loglevel
- warning
# Control-plane pubsub clients are dropped under load without a
# raised output buffer limit.
- --client-output-buffer-limit
- pubsub 1gb 512mb 60
- x-flowmesh-when: REDIS_ACL_ENABLED
x-flowmesh-value: --aclfile
- x-flowmesh-when: REDIS_ACL_ENABLED
x-flowmesh-value: /etc/redis/acl/users.acl
ports:
- name: redis
containerPort: 6379
volumeMounts:
- name: data
mountPath: /data
- x-flowmesh-when: REDIS_ACL_ENABLED
name: acl
mountPath: /etc/redis/acl
readOnly: true
readinessProbe:
exec:
command: ["redis-cli", "-p", "6379", "ping"]
initialDelaySeconds: 5
periodSeconds: 10
livenessProbe:
exec:
command: ["redis-cli", "-p", "6379", "ping"]
initialDelaySeconds: 15
periodSeconds: 20
volumes:
- x-flowmesh-when: REDIS_ACL_ENABLED
name: acl
secret:
secretName: flowmesh-redis-acl
volumeClaimTemplates:
- metadata:
name: data
labels:
app.kubernetes.io/part-of: flowmesh
spec:
accessModes: ["ReadWriteOnce"]
storageClassName:
x-flowmesh-when: K8S_STORAGE_CLASS
x-flowmesh-value: ${K8S_STORAGE_CLASS:-}
resources:
requests:
storage: ${K8S_REDIS_STORAGE_SIZE:-8Gi}
---
x-flowmesh-when: NODE_ROLE==root
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: flowmesh-redis-telemetry
namespace: ${K8S_NAMESPACE:-flowmesh}
labels:
app.kubernetes.io/part-of: flowmesh
app.kubernetes.io/component: redis-telemetry
spec:
serviceName: flowmesh-redis-telemetry
replicas: 1
selector:
matchLabels:
app.kubernetes.io/part-of: flowmesh
app.kubernetes.io/component: redis-telemetry
template:
metadata:
labels:
app.kubernetes.io/part-of: flowmesh
app.kubernetes.io/component: redis-telemetry
spec:
containers:
- name: redis
image: redis:7-alpine
args:
- redis-server
- --save
- "300 1"
- --loglevel
- warning
- x-flowmesh-when: REDIS_ACL_ENABLED
x-flowmesh-value: --aclfile
- x-flowmesh-when: REDIS_ACL_ENABLED
x-flowmesh-value: /etc/redis/acl/users.acl
ports:
- name: redis
containerPort: 6379
volumeMounts:
- name: data
mountPath: /data
- x-flowmesh-when: REDIS_ACL_ENABLED
name: acl
mountPath: /etc/redis/acl
readOnly: true
readinessProbe:
exec:
command: ["redis-cli", "-p", "6379", "ping"]
initialDelaySeconds: 5
periodSeconds: 10
livenessProbe:
exec:
command: ["redis-cli", "-p", "6379", "ping"]
initialDelaySeconds: 15
periodSeconds: 20
volumes:
- x-flowmesh-when: REDIS_ACL_ENABLED
name: acl
secret:
secretName: flowmesh-redis-acl
volumeClaimTemplates:
- metadata:
name: data
labels:
app.kubernetes.io/part-of: flowmesh
spec:
accessModes: ["ReadWriteOnce"]
storageClassName:
x-flowmesh-when: K8S_STORAGE_CLASS
x-flowmesh-value: ${K8S_STORAGE_CLASS:-}
resources:
requests:
storage: ${K8S_REDIS_STORAGE_SIZE:-8Gi}
Loading