From 62a6a8a00619f69aa30b7afc6152e335fe5f242c Mon Sep 17 00:00:00 2001 From: Mike Camp Date: Thu, 30 Jul 2026 11:17:44 -0400 Subject: [PATCH 1/4] feat(self-managed): install the managed LLM PKI issuer Signed-off-by: Mike Camp --- .../self-managed/environments/base.yaml | 34 +- deploy/stacks/self-managed/global.yaml.gotmpl | 34 +- .../helmfile.d/01-dependencies.yaml.gotmpl | 45 +++ .../tests/check-llm-pki-issuer.sh | 336 ++++++++++++++++++ 4 files changed, 425 insertions(+), 24 deletions(-) create mode 100755 deploy/stacks/self-managed/tests/check-llm-pki-issuer.sh diff --git a/deploy/stacks/self-managed/environments/base.yaml b/deploy/stacks/self-managed/environments/base.yaml index 21db9b8f5..9fac4c86f 100644 --- a/deploy/stacks/self-managed/environments/base.yaml +++ b/deploy/stacks/self-managed/environments/base.yaml @@ -9,13 +9,13 @@ global: # ============================================================================= # Helm Chart Sources Configuration # ============================================================================= - # Option A — HTTP Helm repository (e.g. NGC public catalog): + # Option A: HTTP Helm repository (e.g. NGC public catalog): # sources: # url: "https://helm.ngc.nvidia.com/nvidia/nvcf" # username: "$oauthtoken" # password: "YOUR_NGC_API_KEY" # - # Option B — OCI registry (e.g. ECR or a private NGC org after mirroring): + # Option B: OCI registry (e.g. ECR or a private NGC org after mirroring): # sources: # registry: "nvcr.io" # or .dkr.ecr..amazonaws.com # repository: "YOUR_ORG/YOUR_TEAM" @@ -199,9 +199,12 @@ cassandra: certManager: # cert-manager controller required to reconcile Certificate resources for - # the self-managed PKI stack (ClusterIssuer/nvcf-openbao-pki + per-service - # Certificates land in subsequent MRs). The bundled helm-nvcf-cert-manager - # chart ships installCRDs=true by default. + # the self-managed PKI stack. Enabling addons.llm.pki with the managed + # issuer installs ClusterIssuer/nvcf-openbao-pki in this namespace. The + # bundled helm-nvcf-cert-manager chart ships installCRDs=true by default. + # Set this to false when cert-manager is installed outside this stack; the + # external installation must still provide ServiceAccount/cert-manager in + # the cert-manager namespace for the managed issuer to authenticate. enabled: true openbao: @@ -252,16 +255,16 @@ addons: # LLM addon: gateway + request router (stargate) for LLM function invocation llm: enabled: false - # OpenBao-issued QUIC TLS certificate for the request router (Stargate). - # When enabled, the chart provisions an NVCF service-issuing PKI hierarchy - # in OpenBao via a Helm pre-install/pre-upgrade Job, requests a Certificate - # against the nvcf-openbao-pki ClusterIssuer, and mounts the resulting - # Secret into the request-router pod. Disabled by default; opt in per env. + # QUIC TLS certificate for the request router (Stargate). When enabled, + # the chart requests a Certificate from the configured issuer and mounts + # the resulting Secret. Managed mode also provisions the OpenBao + # service-issuing hierarchy and defaults to + # ClusterIssuer/nvcf-openbao-pki. Disabled by default; opt in per env. pki: enabled: false - # REQUIRED when enabled. Comma-separated DNS suffixes the OpenBao PKI - # role accepts. Typically the customer domain plus cluster.local for - # in-cluster service identity. + # REQUIRED for a managed issuer. Comma-separated DNS suffixes the + # OpenBao PKI role accepts. Typically the customer domain plus + # cluster.local for in-cluster service identity. allowedDomains: "" # REQUIRED when enabled. SANs requested on the issued certificate. dnsNames: [] @@ -269,6 +272,11 @@ addons: # secretName: stargate-quic-tls # issuerKind: ClusterIssuer # issuerName: nvcf-openbao-pki + # Stack management defaults to true only for the default + # ClusterIssuer/nvcf-openbao-pki configuration. Set this explicitly to + # manage a custom ClusterIssuer. Leave it false for an external issuer. + # clusterIssuer: + # enabled: true # namespace: vault-system # mountPath: /etc/stargate/tls # certPath: /etc/stargate/tls/tls.crt diff --git a/deploy/stacks/self-managed/global.yaml.gotmpl b/deploy/stacks/self-managed/global.yaml.gotmpl index c8dc86618..58a232f21 100644 --- a/deploy/stacks/self-managed/global.yaml.gotmpl +++ b/deploy/stacks/self-managed/global.yaml.gotmpl @@ -865,24 +865,26 @@ llmRequestRouter: should not be punished with confusing required errors during render. */ -}} {{- if and (dig "addons" "llm" "enabled" false .Values) (dig "addons" "llm" "pki" "enabled" false .Values) }} {{- $secretName := dig "addons" "llm" "pki" "secretName" "stargate-quic-tls" .Values }} - {{- $allowedDomains := required "addons.llm.pki.allowedDomains is required when addons.llm.pki.enabled is true" (dig "addons" "llm" "pki" "allowedDomains" "" .Values) }} + {{- $issuerKind := dig "addons" "llm" "pki" "issuerKind" "ClusterIssuer" .Values }} + {{- $issuerName := dig "addons" "llm" "pki" "issuerName" "nvcf-openbao-pki" .Values }} + {{- $manageSetting := dig "addons" "llm" "pki" "clusterIssuer" "enabled" "__default__" .Values }} + {{- $managedIssuer := and (eq $issuerKind "ClusterIssuer") (eq $issuerName "nvcf-openbao-pki") }} + {{- if ne (toString $manageSetting) "__default__" }} + {{- $managedIssuer = eq (toString $manageSetting) "true" }} + {{- end }} + {{- if and $managedIssuer (ne $issuerKind "ClusterIssuer") }} + {{- fail "addons.llm.pki.clusterIssuer management supports only issuerKind=ClusterIssuer" }} + {{- end }} {{- $dnsNames := dig "addons" "llm" "pki" "dnsNames" (list) .Values }} {{- if eq (len $dnsNames) 0 }} {{- fail "addons.llm.pki.dnsNames must contain at least one DNS name when addons.llm.pki.enabled is true" }} {{- end }} - {{- /* Provisioning hook reuses the same image the k8s-openbao chart drives; - fall back to openbao.migrations.image.tag so operators set the version - in one place. */ -}} - {{- $pkiImageTag := dig "addons" "llm" "pki" "image" "tag" (dig "openbao" "migrations" "image" "tag" "" .Values) .Values }} - {{- if not $pkiImageTag }} - {{- fail "addons.llm.pki.image.tag (or openbao.migrations.image.tag) is required when addons.llm.pki.enabled is true" }} - {{- end }} certificate: enabled: true secretName: {{ $secretName | quote }} issuerRef: - kind: {{ dig "addons" "llm" "pki" "issuerKind" "ClusterIssuer" .Values | quote }} - name: {{ dig "addons" "llm" "pki" "issuerName" "nvcf-openbao-pki" .Values | quote }} + kind: {{ $issuerKind | quote }} + name: {{ $issuerName | quote }} dnsNames: {{- toYaml $dnsNames | nindent 6 }} tls: @@ -891,6 +893,15 @@ llmRequestRouter: certPath: {{ dig "addons" "llm" "pki" "certPath" "/etc/stargate/tls/tls.crt" .Values | quote }} keyPath: {{ dig "addons" "llm" "pki" "keyPath" "/etc/stargate/tls/tls.key" .Values | quote }} quicInsecure: false + {{- if $managedIssuer }} + {{- $allowedDomains := required "addons.llm.pki.allowedDomains is required when addons.llm.pki.clusterIssuer management is enabled" (dig "addons" "llm" "pki" "allowedDomains" "" .Values) }} + {{- /* Provisioning hook reuses the same image the k8s-openbao chart drives; + fall back to openbao.migrations.image.tag so operators set the version + in one place. */ -}} + {{- $pkiImageTag := dig "addons" "llm" "pki" "image" "tag" (dig "openbao" "migrations" "image" "tag" "" .Values) .Values }} + {{- if not $pkiImageTag }} + {{- fail "addons.llm.pki.image.tag (or openbao.migrations.image.tag) is required when addons.llm.pki.clusterIssuer management is enabled" }} + {{- end }} pki: enabled: true namespace: {{ dig "addons" "llm" "pki" "namespace" "vault-system" .Values | quote }} @@ -905,6 +916,7 @@ llmRequestRouter: {{- toYaml .Values.global.imagePullSecrets | nindent 6 }} {{- end }} {{- end }} + {{- end }} {{- with dig "llmRequestRouter" "podDisruptionBudget" dict .Values }} podDisruptionBudget: {{- toYaml . | nindent 4 }} @@ -992,7 +1004,7 @@ vanityGateway: {{- if dig "addons" "nvcfUi" "enabled" false .Values }} {{/* Shared backend coordinates for the services the nvcf-ui proxies to and the control-plane health monitor probes. Read the same way gateway-routes - is fed — from ingress.gatewayApi.routes..backend — so the config URLs and + is fed from ingress.gatewayApi.routes..backend so the config URLs and component endpoints track any operator override; the dict defaults fill the usual case where a route pins no explicit backend. merge lets a partial override (e.g. namespace only) keep the remaining default fields. */}} diff --git a/deploy/stacks/self-managed/helmfile.d/01-dependencies.yaml.gotmpl b/deploy/stacks/self-managed/helmfile.d/01-dependencies.yaml.gotmpl index 1fcb5bb33..d243461ec 100644 --- a/deploy/stacks/self-managed/helmfile.d/01-dependencies.yaml.gotmpl +++ b/deploy/stacks/self-managed/helmfile.d/01-dependencies.yaml.gotmpl @@ -6,6 +6,23 @@ environments: --- +{{- $llmEnabled := dig "addons" "llm" "enabled" false .Values }} +{{- $pkiEnabled := dig "addons" "llm" "pki" "enabled" false .Values }} +{{- $issuerKind := dig "addons" "llm" "pki" "issuerKind" "ClusterIssuer" .Values }} +{{- $issuerName := dig "addons" "llm" "pki" "issuerName" "nvcf-openbao-pki" .Values }} +{{- $manageSetting := dig "addons" "llm" "pki" "clusterIssuer" "enabled" "__default__" .Values }} +{{- $manageIssuer := and (eq $issuerKind "ClusterIssuer") (eq $issuerName "nvcf-openbao-pki") }} +{{- if ne (toString $manageSetting) "__default__" }} +{{- $manageIssuer = eq (toString $manageSetting) "true" }} +{{- end }} +{{- $managedIssuer := and $llmEnabled $pkiEnabled $manageIssuer }} +{{- if and $managedIssuer (ne $issuerKind "ClusterIssuer") }} +{{- fail "addons.llm.pki.clusterIssuer management supports only issuerKind=ClusterIssuer" }} +{{- end }} +{{- if and $managedIssuer (not (dig "openbao" "enabled" true .Values)) }} +{{- fail "openbao.enabled must be true when addons.llm.pki.clusterIssuer management is enabled" }} +{{- end }} + repositories: - name: nvcf {{- if dig "global" "helm" "sources" "url" "" .Values }} @@ -64,6 +81,34 @@ releases: needs: - nats-system/nats +{{- if $managedIssuer }} + - name: nvcf-pki + chart: nvcf/helm-nvcf-pki + version: 0.1.0 + namespace: cert-manager + values: + - clusterIssuer: + enabled: true + name: {{ $issuerName | quote }} + server: "http://openbao-server.vault-system.svc.cluster.local:8200" + path: "services/all/pki/nvcf-service-issuing/sign/nvcf-service-server" + auth: + mountPath: "/v1/auth/jwt" + role: "cert-manager" + serviceAccount: + name: "cert-manager" + audience: "http://openbao-server.vault-system.svc.cluster.local:8200" + wait: true + waitForJobs: true + labels: + release-group: dependencies + needs: + - vault-system/openbao-server +{{- if dig "certManager" "enabled" true .Values }} + - cert-manager/cert-manager +{{- end }} +{{- end }} + - name: cassandra version: 0.17.0 condition: cassandra.enabled # From defaults.yaml or env overrides diff --git a/deploy/stacks/self-managed/tests/check-llm-pki-issuer.sh b/deploy/stacks/self-managed/tests/check-llm-pki-issuer.sh new file mode 100755 index 000000000..cddd196a7 --- /dev/null +++ b/deploy/stacks/self-managed/tests/check-llm-pki-issuer.sh @@ -0,0 +1,336 @@ +#!/usr/bin/env bash +set -euo pipefail + +stack_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +work_dir="$(mktemp -d)" +trap 'rm -rf "$work_dir"' EXIT + +fail() { + echo "check-llm-pki-issuer: $*" >&2 + exit 1 +} + +helmfile_args=( + --file "$stack_dir/helmfile.d/01-dependencies.yaml.gotmpl" + --environment default +) + +# The router's default advertised hostname is the stable service name at +# replicaCount=1 and the per-pod headless name otherwise. Request both SANs, +# matching the documented operator configuration, so these cases stay valid +# at any replica count. +router_dns_names=( + --state-values-set-string + 'addons.llm.pki.dnsNames[0]=llm-request-router.nvcf.svc.cluster.local' + --state-values-set-string + 'addons.llm.pki.dnsNames[1]=*.llm-request-router-headless.nvcf.svc.cluster.local' +) + +core_state_values=( + --state-values-set ingress.gatewayApi.gateways.shared.name=shared + --state-values-set ingress.gatewayApi.gateways.shared.namespace=envoy + --state-values-set ingress.gatewayApi.gateways.grpc.name=grpc + --state-values-set ingress.gatewayApi.gateways.grpc.namespace=envoy +) + +render_list() { + local case_name="$1" + shift + + HELMFILE_ENV=base helmfile \ + "${helmfile_args[@]}" \ + "$@" \ + list --skip-charts --output json \ + >"$work_dir/$case_name.json" +} + +render_debug() { + local case_name="$1" + shift + + HELMFILE_ENV=base helmfile \ + --log-level debug \ + "${helmfile_args[@]}" \ + "$@" \ + list --skip-charts --output json \ + >"$work_dir/$case_name.debug" 2>&1 +} + +expect_enabled() { + local case_name="$1" + local expected="$2" + + local actual + actual="$( + jq -r \ + 'any(.[]; .name == "nvcf-pki" and .enabled == true)' \ + "$work_dir/$case_name.json" + )" + test "$actual" = "$expected" || + fail "$case_name expected nvcf-pki enabled=$expected, got $actual" +} + +expect_failure() { + local case_name="$1" + local expected_error="$2" + shift 2 + + if HELMFILE_ENV=base helmfile \ + "${helmfile_args[@]}" \ + "$@" \ + list --skip-charts --output json \ + >"$work_dir/$case_name.log" 2>&1; then + fail "$case_name rendered successfully" + fi + + grep -Fq "$expected_error" "$work_dir/$case_name.log" || + fail "$case_name did not return the expected error: $expected_error" +} + +render_router() { + local case_name="$1" + shift + local values_file="$work_dir/$case_name.router-values.yaml" + local manifests_file="$work_dir/$case_name.router-manifests.yaml" + local router_chart="$stack_dir/../../helm/llm-request-router/llm-request-router" + + HELMFILE_ENV=base HELMFILE_CACHE_HOME="$work_dir/helmfile-cache" helmfile \ + --file "$stack_dir/helmfile.d/02-core.yaml.gotmpl" \ + --environment default \ + --selector name=llm-request-router \ + --chart "$router_chart" \ + --skip-deps \ + "${core_state_values[@]}" \ + --state-values-set addons.llm.enabled=true \ + --state-values-set addons.llm.pki.enabled=true \ + "${router_dns_names[@]}" \ + "$@" \ + write-values \ + --output-file-template "$values_file" >/dev/null + + helm template llm-request-router "$router_chart" \ + --namespace nvcf \ + --values "$values_file" \ + >"$manifests_file" +} + +expect_external_router() { + local case_name="$1" + local issuer_kind="$2" + local issuer_name="$3" + local manage_mode="${4:-explicit}" + local manifests_file="$work_dir/$case_name.router-manifests.yaml" + local certificate_file="$work_dir/$case_name.certificate.yaml" + local router_overrides=( + --state-values-set openbao.enabled=false + --state-values-set-string "addons.llm.pki.issuerKind=$issuer_kind" + --state-values-set-string "addons.llm.pki.issuerName=$issuer_name" + ) + + if test "$manage_mode" = "explicit"; then + router_overrides+=( + --state-values-set addons.llm.pki.clusterIssuer.enabled=false + ) + fi + + render_router "$case_name" \ + "${router_overrides[@]}" + + sed -n '/^kind: Certificate$/,/^---$/p' \ + "$manifests_file" \ + >"$certificate_file" + if ! grep -Fq "kind: \"$issuer_kind\"" "$certificate_file" && + ! grep -Fq "kind: $issuer_kind" "$certificate_file"; then + fail "$case_name did not render Certificate issuer kind $issuer_kind" + fi + if ! grep -Fq "name: \"$issuer_name\"" "$certificate_file" && + ! grep -Fq "name: $issuer_name" "$certificate_file"; then + fail "$case_name did not render Certificate issuer name $issuer_name" + fi + grep -Fq -- '--tls-cert-path=/etc/stargate/tls/tls.crt' "$manifests_file" || + fail "$case_name did not enable the request-router TLS certificate" + grep -Fq -- '--tls-key-path=/etc/stargate/tls/tls.key' "$manifests_file" || + fail "$case_name did not enable the request-router TLS key" + if grep -Fq -- '--quic-insecure' "$manifests_file"; then + fail "$case_name enabled insecure request-router transport" + fi + if grep -Fq 'name: addons-llm-migrations' "$manifests_file"; then + fail "$case_name rendered the managed OpenBao provisioning hook" + fi +} + +expect_managed_router() { + local case_name="${1:-managed-defaults}" + local issuer_name="${2:-nvcf-openbao-pki}" + local manage_mode="${3:-default}" + local manifests_file="$work_dir/$case_name.router-manifests.yaml" + local certificate_file="$work_dir/$case_name.certificate.yaml" + local router_overrides=( + --state-values-set-string addons.llm.pki.allowedDomains=nvcf.svc.cluster.local + --state-values-set-string addons.llm.pki.image.tag=test + ) + + if test "$manage_mode" = "explicit"; then + router_overrides+=( + --state-values-set addons.llm.pki.clusterIssuer.enabled=true + --state-values-set-string "addons.llm.pki.issuerName=$issuer_name" + ) + fi + + render_router "$case_name" \ + "${router_overrides[@]}" + + sed -n '/^kind: Certificate$/,/^---$/p' \ + "$manifests_file" \ + >"$certificate_file" + grep -Fq 'kind: "ClusterIssuer"' "$certificate_file" || + fail "$case_name did not render the managed ClusterIssuer kind" + grep -Fq "name: \"$issuer_name\"" "$certificate_file" || + fail "$case_name did not render ClusterIssuer name $issuer_name" + grep -Fq 'name: addons-llm-migrations' "$manifests_file" || + fail "$case_name did not render the managed OpenBao provisioning hook" + grep -Fq -- '--tls-cert-path=/etc/stargate/tls/tls.crt' "$manifests_file" || + fail "$case_name did not enable the request-router TLS certificate" + if grep -Fq -- '--quic-insecure' "$manifests_file"; then + fail "$case_name enabled insecure request-router transport" + fi +} + +# Case 1: LLM disabled. +render_list llm-disabled \ + --state-values-set addons.llm.pki.enabled=true \ + --state-values-set addons.llm.pki.clusterIssuer.enabled=true +expect_enabled llm-disabled false + +# Case 2: LLM enabled, PKI disabled. +render_list pki-disabled \ + --state-values-set addons.llm.enabled=true \ + --state-values-set addons.llm.pki.clusterIssuer.enabled=true +expect_enabled pki-disabled false + +# Case 3: LLM and PKI enabled with the default managed ClusterIssuer. +managed_defaults=( + --state-values-set addons.llm.enabled=true + --state-values-set addons.llm.pki.enabled=true +) +render_list managed-defaults "${managed_defaults[@]}" +expect_enabled managed-defaults true +jq -e ' + any(.[]; + .name == "nvcf-pki" and + .namespace == "cert-manager" and + .chart == "nvcf/helm-nvcf-pki" and + .version == "0.1.0" and + .enabled == true and + .installed == true + ) +' "$work_dir/managed-defaults.json" >/dev/null || + fail "managed defaults did not render the published nvcf-pki release contract" + +render_debug managed-defaults "${managed_defaults[@]}" +sed -n '/- name: nvcf-pki/,/- name: cassandra/p' \ + "$work_dir/managed-defaults.debug" \ + >"$work_dir/managed-defaults.release" + +managed_release="$work_dir/managed-defaults.release" +for expected in \ + 'enabled: true' \ + 'name: "nvcf-openbao-pki"' \ + 'server: "http://openbao-server.vault-system.svc.cluster.local:8200"' \ + 'path: "services/all/pki/nvcf-service-issuing/sign/nvcf-service-server"' \ + 'mountPath: "/v1/auth/jwt"' \ + 'role: "cert-manager"' \ + 'name: "cert-manager"' \ + 'audience: "http://openbao-server.vault-system.svc.cluster.local:8200"' \ + '- vault-system/openbao-server' \ + '- cert-manager/cert-manager'; do + grep -Fq -- "$expected" "$managed_release" || + fail "managed defaults did not render: $expected" +done +if grep -Fq -- '- nats-system/nats' "$managed_release"; then + fail "nvcf-pki rendered a redundant direct NATS dependency" +fi +expect_managed_router + +# Case 4: A managed issuer requires stack-managed OpenBao. +expect_failure managed-without-openbao \ + 'openbao.enabled must be true when addons.llm.pki.clusterIssuer management is enabled' \ + "${managed_defaults[@]}" \ + --state-values-set openbao.enabled=false + +# Case 5: Explicit external ownership overrides default managed-issuer detection. +render_list external-clusterissuer \ + "${managed_defaults[@]}" \ + --state-values-set openbao.enabled=false \ + --state-values-set addons.llm.pki.clusterIssuer.enabled=false +expect_enabled external-clusterissuer false +expect_external_router \ + external-clusterissuer \ + ClusterIssuer \ + nvcf-openbao-pki + +# Case 6: An external namespaced Issuer remains external and does not require OpenBao. +render_list external-issuer \ + "${managed_defaults[@]}" \ + --state-values-set openbao.enabled=false \ + --state-values-set-string addons.llm.pki.issuerKind=Issuer \ + --state-values-set-string addons.llm.pki.issuerName=external-pki.example.invalid +expect_enabled external-issuer false +expect_external_router \ + external-issuer \ + Issuer \ + external-pki.example.invalid \ + default + +# Case 7: A managed issuer with external cert-manager keeps only the OpenBao dependency. +external_cert_manager=( + "${managed_defaults[@]}" + --state-values-set certManager.enabled=false +) +render_list external-cert-manager "${external_cert_manager[@]}" +expect_enabled external-cert-manager true +render_debug external-cert-manager "${external_cert_manager[@]}" +sed -n '/- name: nvcf-pki/,/- name: cassandra/p' \ + "$work_dir/external-cert-manager.debug" \ + >"$work_dir/external-cert-manager.release" +grep -Fq -- '- vault-system/openbao-server' "$work_dir/external-cert-manager.release" || + fail "external cert-manager mode lost the OpenBao dependency" +if grep -Fq -- '- cert-manager/cert-manager' "$work_dir/external-cert-manager.release"; then + fail "external cert-manager mode rendered a dangling cert-manager dependency" +fi + +# Case 8: A custom managed ClusterIssuer requires explicit management. +render_list custom-unmanaged \ + "${managed_defaults[@]}" \ + --state-values-set-string addons.llm.pki.issuerName=custom-managed-pki +expect_enabled custom-unmanaged false +expect_external_router \ + custom-unmanaged \ + ClusterIssuer \ + custom-managed-pki \ + default + +custom_managed=( + "${managed_defaults[@]}" + --state-values-set addons.llm.pki.clusterIssuer.enabled=true + --state-values-set-string addons.llm.pki.issuerName=custom-managed-pki +) +render_list custom-managed "${custom_managed[@]}" +expect_enabled custom-managed true +render_debug custom-managed "${custom_managed[@]}" +sed -n '/- name: nvcf-pki/,/- name: cassandra/p' \ + "$work_dir/custom-managed.debug" \ + >"$work_dir/custom-managed.release" +grep -Fq 'name: "custom-managed-pki"' "$work_dir/custom-managed.release" || + fail "custom managed ClusterIssuer name was not passed to nvcf-pki" +expect_managed_router custom-managed custom-managed-pki explicit + +# Case 9: The cluster-scoped chart cannot manage a namespaced Issuer. +expect_failure managed-namespaced-issuer \ + 'addons.llm.pki.clusterIssuer management supports only issuerKind=ClusterIssuer' \ + "${managed_defaults[@]}" \ + --state-values-set addons.llm.pki.clusterIssuer.enabled=true \ + --state-values-set-string addons.llm.pki.issuerKind=Issuer \ + --state-values-set-string addons.llm.pki.issuerName=custom-managed-pki + +echo "check-llm-pki-issuer: all checks passed" From 07e7017d7901ec11e52c6bc72c80ed0b06f45856 Mon Sep 17 00:00:00 2001 From: Mike Camp Date: Thu, 30 Jul 2026 11:24:57 -0400 Subject: [PATCH 2/4] docs(self-managed): document LLM transport TLS and rollback Signed-off-by: Mike Camp --- docs/user/llm-function-enablement.md | 543 +++++++++++++++++++++++---- 1 file changed, 474 insertions(+), 69 deletions(-) diff --git a/docs/user/llm-function-enablement.md b/docs/user/llm-function-enablement.md index 1903ed500..7c091d8b7 100644 --- a/docs/user/llm-function-enablement.md +++ b/docs/user/llm-function-enablement.md @@ -31,39 +31,112 @@ When enabled, the stack creates: - LLM worker pods with a `pylon` sidecar that forwards requests to the function container on the configured `inferencePort`. -## Helmfile Configuration +## Production TLS Configuration -Add the LLM addon and `agentConfig` block to your Helmfile environment file -before applying the stack: +Production deployments must secure the QUIC transport between each LLM worker +and the request router. The request router presents a certificate issued by +cert-manager. Each compute plane receives the public root CA certificate and +uses the combined system and private trust bundle in the `llm-worker` sidecar. + +The request-router address configured for the compute plane must use a DNS name +listed in the certificate SANs. For a single-cluster deployment, use +`llm-request-router.nvcf.svc.cluster.local:50071`. Use an address reachable from +each compute cluster when the control and compute planes use separate networks. + +### Managed OpenBao issuer + +The default managed configuration installs `ClusterIssuer/nvcf-openbao-pki` +through the `helm-nvcf-pki` chart. It uses the stack's OpenBao service as the +signing backend. + +Add the following values to the Helmfile environment: ```yaml +certManager: + enabled: true + +openbao: + enabled: true + addons: llm: enabled: true + pki: + enabled: true + allowedDomains: nvcf.svc.cluster.local + dnsNames: + - llm-request-router.nvcf.svc.cluster.local + - "*.llm-request-router-headless.nvcf.svc.cluster.local" gateway: - replicaCount: 1 - auth: - grpcInsecure: true - metrics: - serviceMonitor: - enabled: false + replicaCount: 2 requestRouter: - replicaCount: 1 - metrics: - serviceMonitor: - enabled: false + replicaCount: 2 +``` -agentConfig: - mergeConfig: | - cluster: - validationPolicy: - name: Unrestricted - workload: - stargateQUICInsecure: true +The stable service name covers a single request-router replica. The wildcard +SAN covers the pod-specific headless service names advertised when the +request-router StatefulSet has multiple replicas. + +To use a custom managed `ClusterIssuer`, set both the issuer identity and the +management override: + +```yaml +addons: + llm: + pki: + issuerKind: ClusterIssuer + issuerName: custom-openbao-pki + clusterIssuer: + enabled: true ``` -Use `replicaCount: 1` for local or single-node test clusters. Increase -replica counts for shared or production environments. +Issuer management supports only `issuerKind: ClusterIssuer`. OpenBao must be +enabled when the stack manages the issuer. + +### External issuer + +Set `clusterIssuer.enabled: false` when the issuer is managed outside this +stack. For an external `ClusterIssuer`, use: + +```yaml +addons: + llm: + enabled: true + pki: + enabled: true + issuerKind: ClusterIssuer + issuerName: external-llm-pki + clusterIssuer: + enabled: false + dnsNames: + - llm-request-router.nvcf.svc.cluster.local + - "*.llm-request-router-headless.nvcf.svc.cluster.local" +``` + +For a namespaced issuer, create the `Issuer` in the `nvcf` namespace and use: + +```yaml +addons: + llm: + enabled: true + pki: + enabled: true + issuerKind: Issuer + issuerName: external-llm-pki + clusterIssuer: + enabled: false + dnsNames: + - llm-request-router.nvcf.svc.cluster.local + - "*.llm-request-router-headless.nvcf.svc.cluster.local" +``` + +The external issuer must allow the requested DNS names and issue a server +certificate from the root CA distributed to the compute planes. You can set +`openbao.enabled: false` when no other stack component requires OpenBao. + +`addons.llm.pki.allowedDomains` constrains the managed OpenBao signing role +only. The stack ignores it for an external issuer. Apply the equivalent +constraint in the external issuer's own configuration. When `addons.llm.enabled` is `true`, the stack defaults `global.workerEndpoints.llmRequestRouterAddress` to @@ -83,89 +156,267 @@ set, and accepts any supported `routingMethod` from a function. When a load-balancer configuration is set, a function can only select an algorithm that the configuration enables. -If you mirror images to a registry that does not use the stack's default -`global.image.registry` and `global.image.repository`, override the -`pylon` sidecar image passed to generated LLM workers: +### External cert-manager + +Set `certManager.enabled: false` when cert-manager is installed and managed +outside this stack: ```yaml -api: - env: - NVCF_SIDECARS_LLM_ROUTER_CLIENT_IMAGE: //pylon:0.2.1 +certManager: + enabled: false ``` -The LLM API Gateway and request router images are resolved from the same stack -artifact registry settings as the other control plane services. +Install the cert-manager CRDs and controller before applying the NVCF stack. +When the stack manages the OpenBao issuer, the external installation must also +provide `ServiceAccount/cert-manager` in the `cert-manager` namespace because +the managed issuer uses Kubernetes authentication with that identity. -## Local Plaintext Transport +### Compute-plane trust -Local development clusters commonly run the LLM API Gateway to NVCF API gRPC -hop and the worker `pylon` sidecar to request-router QUIC tunnel without TLS. -In that case, add both plaintext controls. +The control-plane profile carries the public transport CA: + +```yaml +transportTls: + trustMode: bundle + trustBundleFingerprint: sha256:<64-lowercase-hex-digits> + trustBundlePem: | + -----BEGIN CERTIFICATE----- + + -----END CERTIFICATE----- +``` -The complete Helmfile example above includes these settings. If you already -have an LLM block, include these plaintext-specific fields: +For the managed OpenBao issuer, export a refreshed profile after the managed +PKI hierarchy is available: + +```bash +nvcf-cli self-hosted \ + --control-plane-stack deploy/stacks/self-managed \ + --env \ + --control-plane-context \ + --compute-plane-context \ + control-plane profile export \ + --cluster-name \ + --nca-id \ + --region +``` + +The exporter cannot infer the worker-facing request-router endpoint. Add the +reachable `host:port` before validating or registering the profile: + +```yaml +controlPlane: + addons: + llm: + requestRouterAddress: llm-router.example.com:443 +``` + +Use the endpoint that compute-plane workers can resolve and reach. Its +hostname must match a SAN on the request-router certificate. Registration +converts this field to `agent.llm.requestRouterAddress`, which supplies the +worker's `--stargate-address` argument. + +Add that hostname to `addons.llm.pki.dnsNames`. The list accepts any number of +additional names; the stack only requires that one entry covers the router's +advertised hostname. For the managed issuer, also extend `allowedDomains` with +the parent domain, because the OpenBao signing role allows subdomains and +wildcards but not bare domains. To issue for `llm-router.example.com`, use: ```yaml addons: llm: - enabled: true - gateway: - replicaCount: 1 - auth: - grpcInsecure: true - requestRouter: - replicaCount: 1 + pki: + allowedDomains: nvcf.svc.cluster.local,example.com + dnsNames: + - llm-request-router.nvcf.svc.cluster.local + - "*.llm-request-router-headless.nvcf.svc.cluster.local" + - llm-router.example.com +``` + +`allowedDomains: llm-router.example.com` does not work for that name. The role +sets `allow_bare_domains=false`, so the entry must be the parent domain. + +The managed export reads the public root CA certificate from +`services/all/pki/root` in the stack's OpenBao service and calculates the +canonical NVCF trust-bundle fingerprint. It does not discover the CA for an +external issuer. For an external issuer, add the issuer owner's public CA +bundle and canonical NVCF fingerprint to `transportTls` in the existing +profile. + +Calculate the canonical fingerprint for a public CA bundle with: + +```bash +set -euo pipefail + +bundle_file=path/to/public-ca-bundle.pem +fingerprint_tmp="$(mktemp -d)" +trap 'rm -rf "$fingerprint_tmp"' EXIT + +awk -v output_dir="$fingerprint_tmp" ' + /-----BEGIN CERTIFICATE-----/ { certificate++; in_certificate=1 } + in_certificate { + print > (output_dir "/certificate-" certificate ".pem") + } + /-----END CERTIFICATE-----/ { in_certificate=0 } +' "$bundle_file" + +for certificate_file in "$fingerprint_tmp"/certificate-*.pem; do + openssl x509 -in "$certificate_file" -outform DER \ + | openssl dgst -sha256 -r \ + | awk '{print $1}' +done | sort -u >"$fingerprint_tmp/certificate-hashes" + +{ + printf 'nvcf-trust-bundle-v1\n' + cat "$fingerprint_tmp/certificate-hashes" +} | openssl dgst -sha256 -r \ + | awk '{print "sha256:" $1}' +``` + +The procedure hashes each certificate's DER bytes, removes duplicates, sorts +the certificate hashes, and hashes the versioned canonical text. Use the +result for `trustBundleFingerprint`. + +Validate the profile after updating it: +```bash +nvcf-cli self-hosted control-plane profile validate \ + --file deploy/stacks/self-managed/out/control-plane-profile.yaml \ + --require compute-reachable +``` + +Compute-plane registration converts `transportTls` into this exact generated +NVCA fragment: + +```yaml agentConfig: mergeConfig: | workload: - stargateQUICInsecure: true + transportTLS: + trustMode: bundle + trustBundleFingerprint: sha256:<64-lowercase-hex-digits> + trustBundlePem: | + -----BEGIN CERTIFICATE----- + + -----END CERTIFICATE----- ``` -`addons.llm.gateway.auth.grpcInsecure: true` configures the LLM API Gateway to -talk to the NVCF API over plaintext gRPC. +Use only public CA certificates in `trustBundlePem`. Do not add a private key, +leaf certificate, or OpenBao token. -`workload.stargateQUICInsecure: true` configures generated LLM workers to pass -the plaintext QUIC setting to the `pylon` sidecar. +When replacing a local plaintext configuration, also set this in the +compute-plane Helmfile environment: - -Use these plaintext settings only for local or isolated test clusters. -Production environments should use TLS-capable service configuration instead. +```yaml +agentConfig: + mergeConfig: | + workload: + stargateQUICInsecure: false +``` - +The compute-plane Helmfile merges the environment fragment over the generated +registration fragment, so the TLS trust configuration is retained and an old +plaintext override is disabled. + +If you mirror images to a registry that does not use the stack's default +`global.image.registry` and `global.image.repository`, override the `pylon` +sidecar image passed to generated LLM workers: + +```yaml +api: + env: + NVCF_SIDECARS_LLM_ROUTER_CLIENT_IMAGE: //pylon:0.2.1 +``` + +The LLM API Gateway and request router images are resolved from the same stack +artifact registry settings as the other control-plane services. ## Apply and Verify -Apply the updated control plane environment before creating LLM functions: +Apply the updated control-plane environment: ```bash cd path/to/nvcf-self-managed-stack make apply HELMFILE_ENV= ``` -Apply or refresh the compute-plane stack for each registered GPU cluster so the -NVCA operator receives `agentConfig.mergeConfig`: +For the managed issuer, wait for the `ClusterIssuer`: + +```bash +kubectl wait --for=condition=Ready \ + clusterissuer/nvcf-openbao-pki \ + --timeout=2m +kubectl get clusterissuer nvcf-openbao-pki +``` + +For an external issuer, wait for the configured resource instead: ```bash -make -C deploy/stacks/nvcf-compute-plane install \ - HELMFILE_ENV= \ - CLUSTER_NAME= +kubectl wait --for=condition=Ready \ + clusterissuer/ \ + --timeout=2m +# For a namespaced Issuer: +kubectl -n nvcf wait --for=condition=Ready \ + issuer/ \ + --timeout=2m +``` + +Verify the request-router `Certificate`, its issuer reference, and the SANs. +The commands read only the public certificate: + +```bash +kubectl -n nvcf wait --for=condition=Ready \ + certificate/stargate-quic-tls \ + --timeout=2m +kubectl -n nvcf get certificate stargate-quic-tls \ + -o jsonpath='{.spec.issuerRef.kind}{"/"}{.spec.issuerRef.name}{"\n"}' +kubectl -n nvcf get secret stargate-quic-tls \ + -o jsonpath='{.data.tls\.crt}' \ + | base64 --decode \ + | openssl x509 -noout -subject -issuer -dates -ext subjectAltName +``` + +Regenerate and apply the registration values for each compute cluster: + +```bash +nvcf-cli self-hosted \ + --control-plane-stack deploy/stacks/self-managed \ + --compute-plane-stack deploy/stacks/nvcf-compute-plane \ + --env \ + --control-plane-context \ + --compute-plane-context \ + compute-plane register \ + --control-plane-profile deploy/stacks/self-managed/out/control-plane-profile.yaml \ + --cluster-name \ + --kube-context \ + --region \ + --output deploy/stacks/nvcf-compute-plane/out/-register-values.yaml + +nvcf-cli self-hosted \ + --compute-plane-stack deploy/stacks/nvcf-compute-plane \ + --env \ + compute-plane install \ + --values deploy/stacks/nvcf-compute-plane/out/-register-values.yaml \ + --kube-context \ + --cluster-name ``` Existing LLM function pods keep their current sidecar arguments. Recreate or -redeploy those functions after refreshing the compute plane so new pods get the -updated worker transport settings. +redeploy those functions after refreshing the compute plane. -Verify the LLM control plane components: +After deploying an LLM function, verify the workload trust bundle. Compare +only `.data.fingerprint` with `transportTls.trustBundleFingerprint` in the +profile. `openssl x509 -fingerprint` does not calculate the canonical NVCF +bundle fingerprint. ```bash -kubectl get deployment -n nvcf llm-api-gateway -kubectl get statefulset -n nvcf llm-request-router -kubectl get pods -n nvcf | grep -E 'llm-api-gateway|llm-request-router' -kubectl get httproute -A | grep llm +kubectl -n nvcf-backend get configmap nvcf-transport-trust-bundle \ + -o jsonpath='{.data.fingerprint}{"\n"}' +kubectl -n nvcf-backend get configmap nvcf-transport-trust-bundle \ + -o jsonpath='{.data.nvcf-ca-bundle\.pem}' \ + | openssl x509 -noout -subject -issuer ``` -After deploying an LLM function, verify the worker sidecar: +Verify the worker sidecar: ```bash kubectl get pods -n nvcf-backend -L FUNCTION_ID @@ -173,11 +424,151 @@ kubectl -n nvcf-backend get pod \ -o jsonpath='{range .spec.containers[*]}{.name}{"\t"}{.image}{"\n"}{end}' kubectl -n nvcf-backend get pod \ -o jsonpath='{range .spec.containers[?(@.name=="llm-worker")].args[*]}{.}{"\n"}{end}' +kubectl -n nvcf-backend get pod \ + -o jsonpath='{range .spec.containers[?(@.name=="llm-worker")].env[?(@.name=="STARGATE_TLS_CERT_PATH")]}{.name}{"="}{.value}{"\n"}{end}' +``` + +The worker args must contain +`--stargate-address=llm-request-router.nvcf.svc.cluster.local:50071`, or the +configured routable DNS name, and must not contain `--quic-insecure`. The +address hostname must match a certificate SAN. The environment must contain: + +```text +STARGATE_TLS_CERT_PATH=/etc/ssl/certs/ca-certificates.crt ``` -The function pod should include an `llm-worker` container using `pylon`. For -local plaintext clusters, the `llm-worker` args should include -`--quic-insecure`. +Also verify the control-plane components: + +```bash +kubectl get deployment -n nvcf llm-api-gateway +kubectl get statefulset -n nvcf llm-request-router +kubectl get pods -n nvcf | grep -E 'llm-api-gateway|llm-request-router' +kubectl get httproute -A | grep llm +``` + +## Certificate Renewal + +cert-manager renews the request-router certificate and updates +`Secret/stargate-quic-tls`. The request router loads its certificate when the +pod starts. Restart the StatefulSet after renewal so every replica uses the +updated certificate: + +```bash +kubectl -n nvcf rollout restart statefulset/llm-request-router +kubectl -n nvcf rollout status statefulset/llm-request-router --timeout=5m +``` + +Run the certificate and worker checks again after the restart. Do not assume +that the request router hot reloads certificate changes. + +## Upgrade and Rollback + +Use this order for an upgrade from plaintext transport: + +1. Make the `helm-nvcf-pki` chart available from the configured chart source. +2. Remove the old plaintext setting or set + `workload.stargateQUICInsecure: false`. +3. Apply the dependency Helmfile stage to install cert-manager, OpenBao, and + the managed issuer, or prepare the external cert-manager and issuer: + + ```bash + HELMFILE_ENV= helmfile \ + --file deploy/stacks/self-managed/helmfile.d/01-dependencies.yaml.gotmpl \ + --environment default \ + apply + ``` + +4. Schedule a maintenance window and undeploy the existing LLM functions. + The router does not support a mixed plaintext and TLS transition. +5. Apply the remaining control-plane stack. The managed router hook prepares + the OpenBao signing path before cert-manager reconciles the request-router + `Certificate`. +6. Wait for the issuer and `Certificate/stargate-quic-tls` to become ready. +7. Export or update the control-plane profile, register each compute plane + again, and install the refreshed registration values. +8. Recreate the LLM functions and verify the certificate SAN, trust-bundle + fingerprint, worker address, and worker arguments. + +Use this order for a safe rollback: + +1. Create and verify the replacement `ClusterIssuer` or namespaced `Issuer`. +2. Add both the current and replacement public roots to the compute-plane + profile, calculate the canonical bundle fingerprint, register each compute + plane again, and recreate the LLM workers with the combined trust bundle. +3. Set `addons.llm.pki.issuerKind` and `issuerName` to the replacement, set + `clusterIssuer.enabled: false`, and apply the control-plane stack. +4. Wait for `Certificate/stargate-quic-tls` to become ready, restart the + request-router StatefulSet, and verify the replacement TLS data path. +5. Remove the old root from the compute-plane profile, register each compute + plane again, and recreate the workers. +6. Confirm that no `Certificate` references the old issuer: + + ```bash + kubectl --context get certificate -A \ + -o custom-columns=NAMESPACE:.metadata.namespace,NAME:.metadata.name,KIND:.spec.issuerRef.kind,ISSUER:.spec.issuerRef.name + ``` + +7. Remove the old managed release after it is no longer part of the Helmfile + state: + + ```bash + helm status nvcf-pki \ + --namespace cert-manager \ + --kube-context + helm uninstall nvcf-pki \ + --namespace cert-manager \ + --kube-context + ``` + +8. Delete the retained old `ClusterIssuer` only after all references are gone: + + ```bash + kubectl --context \ + delete clusterissuer + ``` + +If no replacement issuer is available, undeploy the LLM functions and disable +the LLM addon before removing the issuer. Keep LLM traffic stopped until a +secure issuer and trust path are available. + +The managed `ClusterIssuer` is retained when its Helm release is removed. Do +not delete it before its certificates and consumers have moved to the +replacement trust path. Do not use `stargateQUICInsecure` as a production +rollback path. + +## Local Plaintext Transport + +Use plaintext transport only in local or isolated test clusters. Set both +plaintext controls: + +```yaml +addons: + llm: + enabled: true + gateway: + replicaCount: 1 + auth: + grpcInsecure: true + requestRouter: + replicaCount: 1 + +agentConfig: + mergeConfig: | + workload: + stargateQUICInsecure: true +``` + +`addons.llm.gateway.auth.grpcInsecure: true` configures the LLM API Gateway to +talk to the NVCF API over plaintext gRPC. + +`workload.stargateQUICInsecure: true` configures generated LLM workers to pass +the plaintext QUIC setting to the `pylon` sidecar. + + +Use these settings only for local or isolated test clusters. Do not enable +plaintext worker transport in production. + + ## Troubleshooting @@ -198,6 +589,20 @@ mean the router knows the target but has no active eligible backend. Check: - Local clusters using plaintext transport include both `grpcInsecure` and `stargateQUICInsecure`. +For transport TLS failures, check: + +- Unknown issuer: inspect the `Certificate` Ready condition and verify + `issuerRef.kind`, `issuerRef.name`, and the issuer namespace. A namespaced + `Issuer` must be in `nvcf`. +- SAN mismatch: compare the hostname in `--stargate-address` with the SANs in + `Secret/stargate-quic-tls`. Do not replace the hostname with an IP address. +- Expired or not-yet-valid certificate: inspect the certificate dates and the + cluster clock. Renew the certificate and restart the request-router + StatefulSet. +- Missing trust bundle: verify `ConfigMap/nvcf-transport-trust-bundle`, compare + its fingerprint with the compute-plane profile, and confirm + `STARGATE_TLS_CERT_PATH` in the `llm-worker` container. + Useful logs: ```bash From 3834aa27d85ef6d1fc3500e8319be3c9100a4346 Mon Sep 17 00:00:00 2001 From: Mike Camp Date: Mon, 17 Aug 2026 17:41:04 -0400 Subject: [PATCH 3/4] fix(self-managed): reject empty LLM PKI issuer references dig only falls back for a missing path, so an explicit empty addons.llm.pki.issuerKind survived into managed-issuer detection. The stack then skipped the nvcf-pki release while the request-router chart re-defaulted the kind to ClusterIssuer, producing a Certificate that referenced an issuer nothing created. That is the failure mode #502 exists to prevent. Reject empty issuerKind and issuerName in both the values template and the dependency stage, and correct the cluster-scoped ClusterIssuer comment. Signed-off-by: Mike Camp --- .../self-managed/environments/base.yaml | 12 ++++--- deploy/stacks/self-managed/global.yaml.gotmpl | 8 +++-- .../helmfile.d/01-dependencies.yaml.gotmpl | 8 +++++ .../tests/check-llm-pki-issuer.sh | 31 +++++++++++++++++++ docs/user/llm-function-enablement.md | 10 +++++- 5 files changed, 61 insertions(+), 8 deletions(-) diff --git a/deploy/stacks/self-managed/environments/base.yaml b/deploy/stacks/self-managed/environments/base.yaml index 9fac4c86f..4b97eb8f9 100644 --- a/deploy/stacks/self-managed/environments/base.yaml +++ b/deploy/stacks/self-managed/environments/base.yaml @@ -200,11 +200,13 @@ cassandra: certManager: # cert-manager controller required to reconcile Certificate resources for # the self-managed PKI stack. Enabling addons.llm.pki with the managed - # issuer installs ClusterIssuer/nvcf-openbao-pki in this namespace. The - # bundled helm-nvcf-cert-manager chart ships installCRDs=true by default. - # Set this to false when cert-manager is installed outside this stack; the - # external installation must still provide ServiceAccount/cert-manager in - # the cert-manager namespace for the managed issuer to authenticate. + # issuer creates ClusterIssuer/nvcf-openbao-pki, which is cluster-scoped + # and has no namespace. The nvcf-pki Helm release that creates it and the + # cert-manager ServiceAccount it authenticates as both live in the + # cert-manager namespace. The bundled helm-nvcf-cert-manager chart ships + # installCRDs=true by default. Set this to false when cert-manager is + # installed outside this stack; that external installation must still + # provide ServiceAccount/cert-manager in the cert-manager namespace. enabled: true openbao: diff --git a/deploy/stacks/self-managed/global.yaml.gotmpl b/deploy/stacks/self-managed/global.yaml.gotmpl index 58a232f21..cb82eeecc 100644 --- a/deploy/stacks/self-managed/global.yaml.gotmpl +++ b/deploy/stacks/self-managed/global.yaml.gotmpl @@ -865,8 +865,12 @@ llmRequestRouter: should not be punished with confusing required errors during render. */ -}} {{- if and (dig "addons" "llm" "enabled" false .Values) (dig "addons" "llm" "pki" "enabled" false .Values) }} {{- $secretName := dig "addons" "llm" "pki" "secretName" "stargate-quic-tls" .Values }} - {{- $issuerKind := dig "addons" "llm" "pki" "issuerKind" "ClusterIssuer" .Values }} - {{- $issuerName := dig "addons" "llm" "pki" "issuerName" "nvcf-openbao-pki" .Values }} + {{- /* dig only falls back for a missing path, so an explicit empty value + survives. An empty issuerKind would drop out of managed-issuer + detection here while the router chart re-defaults it to ClusterIssuer, + leaving a Certificate that references an issuer nothing creates. */ -}} + {{- $issuerKind := required "addons.llm.pki.issuerKind must not be empty when addons.llm.pki.enabled is true" (dig "addons" "llm" "pki" "issuerKind" "ClusterIssuer" .Values) }} + {{- $issuerName := required "addons.llm.pki.issuerName must not be empty when addons.llm.pki.enabled is true" (dig "addons" "llm" "pki" "issuerName" "nvcf-openbao-pki" .Values) }} {{- $manageSetting := dig "addons" "llm" "pki" "clusterIssuer" "enabled" "__default__" .Values }} {{- $managedIssuer := and (eq $issuerKind "ClusterIssuer") (eq $issuerName "nvcf-openbao-pki") }} {{- if ne (toString $manageSetting) "__default__" }} diff --git a/deploy/stacks/self-managed/helmfile.d/01-dependencies.yaml.gotmpl b/deploy/stacks/self-managed/helmfile.d/01-dependencies.yaml.gotmpl index d243461ec..3b93158d1 100644 --- a/deploy/stacks/self-managed/helmfile.d/01-dependencies.yaml.gotmpl +++ b/deploy/stacks/self-managed/helmfile.d/01-dependencies.yaml.gotmpl @@ -11,6 +11,14 @@ environments: {{- $issuerKind := dig "addons" "llm" "pki" "issuerKind" "ClusterIssuer" .Values }} {{- $issuerName := dig "addons" "llm" "pki" "issuerName" "nvcf-openbao-pki" .Values }} {{- $manageSetting := dig "addons" "llm" "pki" "clusterIssuer" "enabled" "__default__" .Values }} +{{- /* dig only falls back for a missing path, so an explicit empty value + survives and would silently disable managed-issuer detection. */ -}} +{{- if and $llmEnabled $pkiEnabled (not $issuerKind) }} +{{- fail "addons.llm.pki.issuerKind must not be empty when addons.llm.pki.enabled is true" }} +{{- end }} +{{- if and $llmEnabled $pkiEnabled (not $issuerName) }} +{{- fail "addons.llm.pki.issuerName must not be empty when addons.llm.pki.enabled is true" }} +{{- end }} {{- $manageIssuer := and (eq $issuerKind "ClusterIssuer") (eq $issuerName "nvcf-openbao-pki") }} {{- if ne (toString $manageSetting) "__default__" }} {{- $manageIssuer = eq (toString $manageSetting) "true" }} diff --git a/deploy/stacks/self-managed/tests/check-llm-pki-issuer.sh b/deploy/stacks/self-managed/tests/check-llm-pki-issuer.sh index cddd196a7..ba8d336cb 100755 --- a/deploy/stacks/self-managed/tests/check-llm-pki-issuer.sh +++ b/deploy/stacks/self-managed/tests/check-llm-pki-issuer.sh @@ -70,6 +70,22 @@ expect_enabled() { fail "$case_name expected nvcf-pki enabled=$expected, got $actual" } +# An explicit empty value cannot be expressed with --state-values-set-string, +# which leaves the key unset so dig falls back to its default. Write a real +# override file so the empty string reaches the template. +empty_issuer_override() { + local field="$1" + local override_file="$work_dir/empty-$field.yaml" + + cat >"$override_file" < + ``` + + After confirming, remove the release: + + ```bash helm uninstall nvcf-pki \ --namespace cert-manager \ --kube-context From a6eeeae8c10875f8cd4e8f14198beafa16c54062 Mon Sep 17 00:00:00 2001 From: Mike Camp Date: Mon, 17 Aug 2026 18:44:51 -0400 Subject: [PATCH 4/4] fix(self-managed): reject malformed LLM PKI issuer configuration The issuer-management flag was compared as a string, so only the exact value "true" enabled management. Explicit null, "", "TRUE", and other non-boolean values silently resolved to unmanaged: the stack skipped the nvcf-pki release while global.yaml.gotmpl still rendered a request-router Certificate naming ClusterIssuer/nvcf-openbao-pki. Whitespace-only and wrongly cased issuerKind, and invalid or uppercase issuerName, failed the same way. Each produced the dangling-issuer state this stack exists to prevent, with no diagnostic. Both the dependency stage and the values template now apply one type-aware contract: clusterIssuer.enabled must be a YAML boolean when present and is otherwise inferred as before, issuerKind must be exactly ClusterIssuer or Issuer, and issuerName must be a lowercase RFC 1123 DNS subdomain. Malformed input fails rendering with an actionable message instead of being normalized. Extend the render regressions to assert the diagnostic for each invalid input rather than only the absence of the release, and add an invariant check that no successful configuration leaves a Certificate pointing at an issuer the stack was expected to install. Every new assertion was mutation-tested. Correct the worker-address documentation and the profile, CLI values, and nvca-operator comments that claimed agent.llm.requestRouterAddress supplies --stargate-address. At this commit workers require LLM_REQUEST_ROUTER_ADDRESS or the legacy STARGATE_ADDRESS in the launch environment, and translation rejects a launch that supplies neither. TranslateConfig.DefaultStargateAddress is set by callers but never read, so it is not a fallback. Signed-off-by: Mike Camp --- deploy/stacks/self-managed/global.yaml.gotmpl | 44 +++- .../helmfile.d/01-dependencies.yaml.gotmpl | 48 +++- .../tests/check-llm-pki-issuer.sh | 230 ++++++++++++++++-- docs/user/llm-function-enablement.md | 14 +- .../selfhosted/controlplaneprofile/profile.go | 8 +- .../internal/selfhosted/nvca/values.go | 8 +- .../nvca/deployments/nvca-operator/README.md | 2 +- .../deployments/nvca-operator/values.yaml | 2 +- 8 files changed, 301 insertions(+), 55 deletions(-) diff --git a/deploy/stacks/self-managed/global.yaml.gotmpl b/deploy/stacks/self-managed/global.yaml.gotmpl index cb82eeecc..d5cfa926d 100644 --- a/deploy/stacks/self-managed/global.yaml.gotmpl +++ b/deploy/stacks/self-managed/global.yaml.gotmpl @@ -865,16 +865,42 @@ llmRequestRouter: should not be punished with confusing required errors during render. */ -}} {{- if and (dig "addons" "llm" "enabled" false .Values) (dig "addons" "llm" "pki" "enabled" false .Values) }} {{- $secretName := dig "addons" "llm" "pki" "secretName" "stargate-quic-tls" .Values }} - {{- /* dig only falls back for a missing path, so an explicit empty value - survives. An empty issuerKind would drop out of managed-issuer - detection here while the router chart re-defaults it to ClusterIssuer, - leaving a Certificate that references an issuer nothing creates. */ -}} - {{- $issuerKind := required "addons.llm.pki.issuerKind must not be empty when addons.llm.pki.enabled is true" (dig "addons" "llm" "pki" "issuerKind" "ClusterIssuer" .Values) }} - {{- $issuerName := required "addons.llm.pki.issuerName must not be empty when addons.llm.pki.enabled is true" (dig "addons" "llm" "pki" "issuerName" "nvcf-openbao-pki" .Values) }} - {{- $manageSetting := dig "addons" "llm" "pki" "clusterIssuer" "enabled" "__default__" .Values }} + {{- /* dig only falls back for a missing path, so explicit null, empty, and + wrongly typed values reach these checks. Reject them instead of coercing + them: a malformed management flag that silently resolves to false skips + the nvcf-pki release in 01-dependencies.yaml.gotmpl while the block + below still renders a Certificate referencing the default issuer. Keep + this contract identical to the dependency stage. */ -}} + {{- $issuerKind := dig "addons" "llm" "pki" "issuerKind" "ClusterIssuer" .Values }} + {{- $issuerName := dig "addons" "llm" "pki" "issuerName" "nvcf-openbao-pki" .Values }} + {{- if not (kindIs "string" $issuerKind) }} + {{- fail "addons.llm.pki.issuerKind must be the string \"ClusterIssuer\" or \"Issuer\"" }} + {{- end }} + {{- if not (has $issuerKind (list "ClusterIssuer" "Issuer")) }} + {{- fail (printf "addons.llm.pki.issuerKind must be exactly \"ClusterIssuer\" or \"Issuer\", got %q" $issuerKind) }} + {{- end }} + {{- if not (kindIs "string" $issuerName) }} + {{- fail "addons.llm.pki.issuerName must be a string" }} + {{- end }} + {{- if not (regexMatch "^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$" $issuerName) }} + {{- fail (printf "addons.llm.pki.issuerName must be a lowercase RFC 1123 DNS subdomain, got %q" $issuerName) }} + {{- end }} + {{- if gt (len $issuerName) 253 }} + {{- fail "addons.llm.pki.issuerName must be at most 253 characters" }} + {{- end }} {{- $managedIssuer := and (eq $issuerKind "ClusterIssuer") (eq $issuerName "nvcf-openbao-pki") }} - {{- if ne (toString $manageSetting) "__default__" }} - {{- $managedIssuer = eq (toString $manageSetting) "true" }} + {{- $pkiValues := dig "addons" "llm" "pki" dict .Values }} + {{- if kindIs "map" $pkiValues }} + {{- $clusterIssuerValues := dig "clusterIssuer" dict $pkiValues }} + {{- if and (kindIs "map" $clusterIssuerValues) (hasKey $clusterIssuerValues "enabled") }} + {{- /* index, not get: helmfile overrides sprig's get with a path-based + signature that rejects a map as its first argument. */ -}} + {{- $manageSetting := index $clusterIssuerValues "enabled" }} + {{- if not (kindIs "bool" $manageSetting) }} + {{- fail (printf "addons.llm.pki.clusterIssuer.enabled must be a YAML boolean, true or false. Quoted strings, null, and empty values are rejected. Got %q" (toString $manageSetting)) }} + {{- end }} + {{- $managedIssuer = $manageSetting }} + {{- end }} {{- end }} {{- if and $managedIssuer (ne $issuerKind "ClusterIssuer") }} {{- fail "addons.llm.pki.clusterIssuer management supports only issuerKind=ClusterIssuer" }} diff --git a/deploy/stacks/self-managed/helmfile.d/01-dependencies.yaml.gotmpl b/deploy/stacks/self-managed/helmfile.d/01-dependencies.yaml.gotmpl index 3b93158d1..3c84d7cef 100644 --- a/deploy/stacks/self-managed/helmfile.d/01-dependencies.yaml.gotmpl +++ b/deploy/stacks/self-managed/helmfile.d/01-dependencies.yaml.gotmpl @@ -8,22 +8,48 @@ environments: {{- $llmEnabled := dig "addons" "llm" "enabled" false .Values }} {{- $pkiEnabled := dig "addons" "llm" "pki" "enabled" false .Values }} +{{- $llmPkiActive := and $llmEnabled $pkiEnabled }} {{- $issuerKind := dig "addons" "llm" "pki" "issuerKind" "ClusterIssuer" .Values }} {{- $issuerName := dig "addons" "llm" "pki" "issuerName" "nvcf-openbao-pki" .Values }} -{{- $manageSetting := dig "addons" "llm" "pki" "clusterIssuer" "enabled" "__default__" .Values }} -{{- /* dig only falls back for a missing path, so an explicit empty value - survives and would silently disable managed-issuer detection. */ -}} -{{- if and $llmEnabled $pkiEnabled (not $issuerKind) }} -{{- fail "addons.llm.pki.issuerKind must not be empty when addons.llm.pki.enabled is true" }} +{{- $manageIssuer := false }} +{{- if $llmPkiActive }} +{{- /* dig only falls back for a missing path, so explicit null, empty, and + wrongly typed values reach these checks. Reject them instead of coercing + them: a malformed management flag that silently resolves to false skips + the nvcf-pki release while global.yaml.gotmpl still renders a Certificate + referencing ClusterIssuer/nvcf-openbao-pki, which is the dangling-issuer + failure this stack exists to prevent. */ -}} +{{- if not (kindIs "string" $issuerKind) }} +{{- fail "addons.llm.pki.issuerKind must be the string \"ClusterIssuer\" or \"Issuer\"" }} {{- end }} -{{- if and $llmEnabled $pkiEnabled (not $issuerName) }} -{{- fail "addons.llm.pki.issuerName must not be empty when addons.llm.pki.enabled is true" }} +{{- if not (has $issuerKind (list "ClusterIssuer" "Issuer")) }} +{{- fail (printf "addons.llm.pki.issuerKind must be exactly \"ClusterIssuer\" or \"Issuer\", got %q" $issuerKind) }} {{- end }} -{{- $manageIssuer := and (eq $issuerKind "ClusterIssuer") (eq $issuerName "nvcf-openbao-pki") }} -{{- if ne (toString $manageSetting) "__default__" }} -{{- $manageIssuer = eq (toString $manageSetting) "true" }} +{{- if not (kindIs "string" $issuerName) }} +{{- fail "addons.llm.pki.issuerName must be a string" }} {{- end }} -{{- $managedIssuer := and $llmEnabled $pkiEnabled $manageIssuer }} +{{- if not (regexMatch "^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$" $issuerName) }} +{{- fail (printf "addons.llm.pki.issuerName must be a lowercase RFC 1123 DNS subdomain, got %q" $issuerName) }} +{{- end }} +{{- if gt (len $issuerName) 253 }} +{{- fail "addons.llm.pki.issuerName must be at most 253 characters" }} +{{- end }} +{{- $manageIssuer = and (eq $issuerKind "ClusterIssuer") (eq $issuerName "nvcf-openbao-pki") }} +{{- $pkiValues := dig "addons" "llm" "pki" dict .Values }} +{{- if kindIs "map" $pkiValues }} +{{- $clusterIssuerValues := dig "clusterIssuer" dict $pkiValues }} +{{- if and (kindIs "map" $clusterIssuerValues) (hasKey $clusterIssuerValues "enabled") }} +{{- /* index, not get: helmfile overrides sprig's get with a path-based + signature that rejects a map as its first argument. */ -}} +{{- $manageSetting := index $clusterIssuerValues "enabled" }} +{{- if not (kindIs "bool" $manageSetting) }} +{{- fail (printf "addons.llm.pki.clusterIssuer.enabled must be a YAML boolean, true or false. Quoted strings, null, and empty values are rejected. Got %q" (toString $manageSetting)) }} +{{- end }} +{{- $manageIssuer = $manageSetting }} +{{- end }} +{{- end }} +{{- end }} +{{- $managedIssuer := and $llmPkiActive $manageIssuer }} {{- if and $managedIssuer (ne $issuerKind "ClusterIssuer") }} {{- fail "addons.llm.pki.clusterIssuer management supports only issuerKind=ClusterIssuer" }} {{- end }} diff --git a/deploy/stacks/self-managed/tests/check-llm-pki-issuer.sh b/deploy/stacks/self-managed/tests/check-llm-pki-issuer.sh index ba8d336cb..86969d2d0 100755 --- a/deploy/stacks/self-managed/tests/check-llm-pki-issuer.sh +++ b/deploy/stacks/self-managed/tests/check-llm-pki-issuer.sh @@ -70,22 +70,57 @@ expect_enabled() { fail "$case_name expected nvcf-pki enabled=$expected, got $actual" } -# An explicit empty value cannot be expressed with --state-values-set-string, -# which leaves the key unset so dig falls back to its default. Write a real -# override file so the empty string reaches the template. -empty_issuer_override() { - local field="$1" - local override_file="$work_dir/empty-$field.yaml" - - cat >"$override_file" <"$override_file" printf '%s' "$override_file" } +# Guards the defect this stack exists to prevent: a Certificate that names an +# issuer no release creates. Dangling is only acceptable when the operator +# explicitly declined management, so pass explicitly_external=true for those. +expect_no_dangling_issuer() { + local case_name="$1" + local explicitly_external="$2" + local manifests_file="$work_dir/$case_name.router-manifests.yaml" + + local issuer_kind issuer_name pki_enabled + issuer_kind="$(yq -rN 'select(.kind == "Certificate") | .spec.issuerRef.kind' "$manifests_file" | head -1)" + issuer_name="$(yq -rN 'select(.kind == "Certificate") | .spec.issuerRef.name' "$manifests_file" | head -1)" + pki_enabled="$(jq -r 'any(.[]; .name == "nvcf-pki" and .enabled == true)' "$work_dir/$case_name.json")" + + # Without this the check is vacuous: if the manifest shape changes and the + # queries stop resolving, every case would take the "not the default issuer" + # exit below and silently pass. + if test -z "$issuer_kind" || test "$issuer_kind" = "null"; then + fail "$case_name could not read .spec.issuerRef.kind from the rendered Certificate" + fi + if test -z "$issuer_name" || test "$issuer_name" = "null"; then + fail "$case_name could not read .spec.issuerRef.name from the rendered Certificate" + fi + + if test "$issuer_kind" != "ClusterIssuer" || test "$issuer_name" != "nvcf-openbao-pki"; then + return 0 + fi + if test "$pki_enabled" = "true"; then + return 0 + fi + if test "$explicitly_external" = "true"; then + return 0 + fi + fail "$case_name renders a Certificate for ClusterIssuer/nvcf-openbao-pki while the nvcf-pki release is absent and management was not explicitly declined" +} + expect_failure() { local case_name="$1" local expected_error="$2" @@ -349,19 +384,166 @@ expect_failure managed-namespaced-issuer \ --state-values-set-string addons.llm.pki.issuerKind=Issuer \ --state-values-set-string addons.llm.pki.issuerName=custom-managed-pki -# Case 10: An explicit empty issuerKind must not silently fall out of managed -# mode. Without this guard the release is skipped while the router chart -# re-defaults the kind, leaving a Certificate pointing at an absent issuer. -expect_failure empty-issuer-kind \ - 'addons.llm.pki.issuerKind must not be empty when addons.llm.pki.enabled is true' \ +# Cases 10 to 19: malformed issuer-management values must fail rendering. +# +# Each of these previously resolved to "not managed" without an error, which +# skipped the nvcf-pki release while the request-router Certificate still +# named ClusterIssuer/nvcf-openbao-pki. Asserting only that nvcf-pki is absent +# would pass against that defect, so every case asserts the diagnostic. +bool_error='addons.llm.pki.clusterIssuer.enabled must be a YAML boolean' +kind_error='addons.llm.pki.issuerKind must be exactly' +kind_type_error='addons.llm.pki.issuerKind must be the string' +name_error='addons.llm.pki.issuerName must be a lowercase RFC 1123 DNS subdomain' +name_type_error='addons.llm.pki.issuerName must be a string' + +expect_failure cluster-issuer-null "$bool_error" \ + "${managed_defaults[@]}" \ + --state-values-file "$(pki_override cluster-issuer-null <<'YAML' +clusterIssuer: + enabled: null +YAML +)" + +expect_failure cluster-issuer-empty-string "$bool_error" \ + "${managed_defaults[@]}" \ + --state-values-file "$(pki_override cluster-issuer-empty-string <<'YAML' +clusterIssuer: + enabled: "" +YAML +)" + +expect_failure cluster-issuer-string-true "$bool_error" \ + "${managed_defaults[@]}" \ + --state-values-file "$(pki_override cluster-issuer-string-true <<'YAML' +clusterIssuer: + enabled: "true" +YAML +)" + +expect_failure cluster-issuer-string-upper-true "$bool_error" \ + "${managed_defaults[@]}" \ + --state-values-file "$(pki_override cluster-issuer-string-upper-true <<'YAML' +clusterIssuer: + enabled: "TRUE" +YAML +)" + +expect_failure issuer-kind-whitespace "$kind_error" \ + "${managed_defaults[@]}" \ + --state-values-file "$(pki_override issuer-kind-whitespace <<'YAML' +issuerKind: " " +YAML +)" + +expect_failure issuer-kind-wrong-case "$kind_error" \ + "${managed_defaults[@]}" \ + --state-values-file "$(pki_override issuer-kind-wrong-case <<'YAML' +issuerKind: clusterissuer +YAML +)" + +expect_failure issuer-kind-null "$kind_type_error" \ + "${managed_defaults[@]}" \ + --state-values-file "$(pki_override issuer-kind-null <<'YAML' +issuerKind: null +YAML +)" + +expect_failure issuer-name-whitespace "$name_error" \ + "${managed_defaults[@]}" \ + --state-values-file "$(pki_override issuer-name-whitespace <<'YAML' +issuerName: " " +YAML +)" + +expect_failure issuer-name-uppercase "$name_error" \ + "${managed_defaults[@]}" \ + --state-values-file "$(pki_override issuer-name-uppercase <<'YAML' +issuerName: Custom-PKI +YAML +)" + +expect_failure issuer-name-null "$name_type_error" \ "${managed_defaults[@]}" \ - --state-values-file "$(empty_issuer_override issuerKind)" + --state-values-file "$(pki_override issuer-name-null <<'YAML' +issuerName: null +YAML +)" + +# Cases 20 to 24: the valid forms must keep working, and none of them may +# leave a Certificate pointing at an issuer the stack was expected to install. +explicit_true_file="$(pki_override cluster-issuer-bool-true <<'YAML' +clusterIssuer: + enabled: true +YAML +)" +render_list cluster-issuer-bool-true \ + "${managed_defaults[@]}" --state-values-file "$explicit_true_file" +expect_enabled cluster-issuer-bool-true true +render_router cluster-issuer-bool-true \ + --state-values-set-string addons.llm.pki.allowedDomains=nvcf.svc.cluster.local \ + --state-values-set-string addons.llm.pki.image.tag=test \ + --state-values-file "$explicit_true_file" +expect_no_dangling_issuer cluster-issuer-bool-true false + +explicit_false_file="$(pki_override cluster-issuer-bool-false <<'YAML' +clusterIssuer: + enabled: false +YAML +)" +render_list cluster-issuer-bool-false \ + "${managed_defaults[@]}" \ + --state-values-set openbao.enabled=false \ + --state-values-file "$explicit_false_file" +expect_enabled cluster-issuer-bool-false false +render_router cluster-issuer-bool-false \ + --state-values-set openbao.enabled=false \ + --state-values-file "$explicit_false_file" +expect_no_dangling_issuer cluster-issuer-bool-false true + +render_list omitted-cluster-issuer "${managed_defaults[@]}" +expect_enabled omitted-cluster-issuer true +render_router omitted-cluster-issuer \ + --state-values-set-string addons.llm.pki.allowedDomains=nvcf.svc.cluster.local \ + --state-values-set-string addons.llm.pki.image.tag=test +expect_no_dangling_issuer omitted-cluster-issuer false + +# A valid custom external issuer must remain supported end to end. +external_valid_file="$(pki_override external-issuer-valid <<'YAML' +issuerKind: Issuer +issuerName: external-pki.example.invalid +clusterIssuer: + enabled: false +YAML +)" +render_list external-issuer-valid \ + "${managed_defaults[@]}" \ + --state-values-set openbao.enabled=false \ + --state-values-file "$external_valid_file" +expect_enabled external-issuer-valid false +render_router external-issuer-valid \ + --state-values-set openbao.enabled=false \ + --state-values-file "$external_valid_file" +expect_no_dangling_issuer external-issuer-valid true -# Case 11: An explicit empty issuerName must fail in the stack, not later in -# the request-router chart. -expect_failure empty-issuer-name \ - 'addons.llm.pki.issuerName must not be empty when addons.llm.pki.enabled is true' \ +# Managed mode with external cert-manager must still install the issuer. +render_list managed-external-cert-manager \ "${managed_defaults[@]}" \ - --state-values-file "$(empty_issuer_override issuerName)" + --state-values-set certManager.enabled=false \ + --state-values-file "$explicit_true_file" +expect_enabled managed-external-cert-manager true +render_router managed-external-cert-manager \ + --state-values-set certManager.enabled=false \ + --state-values-set-string addons.llm.pki.allowedDomains=nvcf.svc.cluster.local \ + --state-values-set-string addons.llm.pki.image.tag=test \ + --state-values-file "$explicit_true_file" +expect_no_dangling_issuer managed-external-cert-manager false + +# Managed mode still requires stack-managed OpenBao under an explicit boolean. +expect_failure managed-bool-without-openbao \ + 'openbao.enabled must be true when addons.llm.pki.clusterIssuer management is enabled' \ + "${managed_defaults[@]}" \ + --state-values-set openbao.enabled=false \ + --state-values-file "$explicit_true_file" echo "check-llm-pki-issuer: all checks passed" diff --git a/docs/user/llm-function-enablement.md b/docs/user/llm-function-enablement.md index 34f5259c6..4265665e9 100644 --- a/docs/user/llm-function-enablement.md +++ b/docs/user/llm-function-enablement.md @@ -211,9 +211,17 @@ controlPlane: ``` Use the endpoint that compute-plane workers can resolve and reach. Its -hostname must match a SAN on the request-router certificate. Registration -converts this field to `agent.llm.requestRouterAddress`, which supplies the -worker's `--stargate-address` argument. +hostname must match a SAN on the request-router certificate. + +Registration renders this field as `agent.llm.requestRouterAddress` in the +compute-plane values. That is operator configuration, not a runtime fallback +for workers. At this release the worker sidecar takes its `--stargate-address` +from the `LLM_REQUEST_ROUTER_ADDRESS` variable in its launch environment, with +`STARGATE_ADDRESS` accepted as a legacy alias. The NVCF API injects +`LLM_REQUEST_ROUTER_ADDRESS` on the normal launch path, derived from +`global.workerEndpoints.llmRequestRouterAddress`. If neither variable reaches +the workload, translation rejects the launch instead of falling back to the +registered address. Add that hostname to `addons.llm.pki.dnsNames`. The list accepts any number of additional names; the stack only requires that one entry covers the router's diff --git a/src/clis/nvcf-cli/internal/selfhosted/controlplaneprofile/profile.go b/src/clis/nvcf-cli/internal/selfhosted/controlplaneprofile/profile.go index 7d85b4a3f..3b985acd2 100644 --- a/src/clis/nvcf-cli/internal/selfhosted/controlplaneprofile/profile.go +++ b/src/clis/nvcf-cli/internal/selfhosted/controlplaneprofile/profile.go @@ -80,9 +80,11 @@ type Addons struct { // LLMAddon advertises the LLM add-on coordinates. RequestRouterAddress is the // host:port the LLM request router (Stargate QUIC endpoint) is reachable at -// from the compute plane; register renders it as the default -// agent.llm.requestRouterAddress so LLM workloads that do not set -// STARGATE_ADDRESS bootstrap against it. +// from the compute plane; register renders it as +// agent.llm.requestRouterAddress in the compute-plane values. That is operator +// configuration only. LLM workloads read their address from the +// LLM_REQUEST_ROUTER_ADDRESS launch environment variable (STARGATE_ADDRESS is +// a legacy alias), and translation rejects a launch that supplies neither. type LLMAddon struct { RequestRouterAddress string `yaml:"requestRouterAddress,omitempty"` } diff --git a/src/clis/nvcf-cli/internal/selfhosted/nvca/values.go b/src/clis/nvcf-cli/internal/selfhosted/nvca/values.go index 003cbb9b3..5443cb34f 100644 --- a/src/clis/nvcf-cli/internal/selfhosted/nvca/values.go +++ b/src/clis/nvcf-cli/internal/selfhosted/nvca/values.go @@ -42,9 +42,11 @@ type AgentValues struct { LLM *AgentLLMValues `yaml:"llm,omitempty"` } -// AgentLLMValues carries agent.llm.* values. RequestRouterAddress is the default -// host:port LLM request router (Stargate) address for LLM workloads, consumed by -// the nvca-operator chart under agent.llm.requestRouterAddress. +// AgentLLMValues carries agent.llm.* values. RequestRouterAddress is the +// host:port LLM request router (Stargate) address consumed by the +// nvca-operator chart under agent.llm.requestRouterAddress. It configures the +// operator; it is not a runtime fallback for LLM workloads, which require +// LLM_REQUEST_ROUTER_ADDRESS in their launch environment. type AgentLLMValues struct { RequestRouterAddress string `yaml:"requestRouterAddress,omitempty"` } diff --git a/src/compute-plane-services/nvca/deployments/nvca-operator/README.md b/src/compute-plane-services/nvca/deployments/nvca-operator/README.md index 9685f6129..20acd0894 100644 --- a/src/compute-plane-services/nvca/deployments/nvca-operator/README.md +++ b/src/compute-plane-services/nvca/deployments/nvca-operator/README.md @@ -80,7 +80,7 @@ used in Kubernetes Clusters to run NVCF Workloads. | `agent.functionEnvOverrides` | Map of environment variable overrides for function workloads (e.g., {"INIT_CONTAINER": "nvcr.io/custom/init:v1.0", "UTILS_CONTAINER": "nvcr.io/custom/utils:v1.0"}) | `{}` | | `agent.taskEnvOverrides` | Map of environment variable overrides for task workloads (e.g., {"INIT_CONTAINER": "nvcr.io/custom/init:v1.0", "ESS_AGENT_CONTAINER": "nvcr.io/custom/ess:v1.0"}) | `{}` | | `agent.overrideEnvironmentVariables` | Map of environment variables to override on the NVCA agent container. These take precedence over default values. Example: {"LOG_LEVEL": "debug", "CUSTOM_FLAG": "enabled"} | `{}` | -| `agent.llm.requestRouterAddress` | Default LLM request-router worker address rendered as STARGATE_ADDRESS for LLM workers | `""` | +| `agent.llm.requestRouterAddress` | Operator default LLM request-router address. Workers read LLM_REQUEST_ROUTER_ADDRESS from the launch environment; not a runtime fallback | `""` | | `agent.serviceOAuth` | OAuth token and JWKS endpoints used by dependent services | See `values.yaml` | ### Webhook Container Resource configuration diff --git a/src/compute-plane-services/nvca/deployments/nvca-operator/values.yaml b/src/compute-plane-services/nvca/deployments/nvca-operator/values.yaml index b40efa108..89da88093 100644 --- a/src/compute-plane-services/nvca/deployments/nvca-operator/values.yaml +++ b/src/compute-plane-services/nvca/deployments/nvca-operator/values.yaml @@ -189,7 +189,7 @@ resources: ## @param agent.taskEnvOverrides Map of environment variable overrides for task workloads (e.g., {"INIT_CONTAINER": "nvcr.io/custom/init:v1.0", "ESS_AGENT_CONTAINER": "nvcr.io/custom/ess:v1.0"}) ## @param agent.overrideEnvironmentVariables Map of environment variables to override on the NVCA agent container. These take precedence over default values. Example: {"LOG_LEVEL": "debug", "CUSTOM_FLAG": "enabled"} ## @param agent.tolerations K8s tolerations for the NVCA agent pod -## @param agent.llm.requestRouterAddress Default LLM request-router worker address rendered as STARGATE_ADDRESS for LLM workers +## @param agent.llm.requestRouterAddress Operator default LLM request-router address. Workers read LLM_REQUEST_ROUTER_ADDRESS from the launch environment; not a runtime fallback ## @param agent.serviceOAuth OAuth token and JWKS endpoints used by dependent services agent: cacheMountOptionsEnabled: true