From 67cd68a3b4373e1003aef2a68919f0b3b2b6d900 Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Mon, 27 Apr 2026 17:28:38 -0600 Subject: [PATCH 01/27] fix: replace yaml.load with yaml.safe_load to prevent deserialization RCE (NVBug 6119291) Fixes insecure YAML deserialization vulnerability reported via Intigriti (NVIDIA-WHT18LD0). yaml.load() with FullLoader (or no Loader) allows Python object construction tags (!!python/object/new:...) enabling arbitrary code execution. Affected files: - scripts/k8s/update_kubeflow_config.py (2 sinks: outer + inner yaml.load) - workloads/examples/k8s/kubeflow-pipeline-deploy/triton_ops.py (1 sink: no Loader) Fix: replace all instances with yaml.safe_load() which disables Python-specific type constructors entirely. --- scripts/k8s/update_kubeflow_config.py | 4 ++-- workloads/examples/k8s/kubeflow-pipeline-deploy/triton_ops.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/k8s/update_kubeflow_config.py b/scripts/k8s/update_kubeflow_config.py index 13be90e1b..304b106c5 100755 --- a/scripts/k8s/update_kubeflow_config.py +++ b/scripts/k8s/update_kubeflow_config.py @@ -63,8 +63,8 @@ def get_images(url='https://api.ngc.nvidia.com/v2/repos', number_tags=5): def update_yaml(images, yaml_file): with open(yaml_file, 'r') as fname: - config = yaml.load(fname.read(), Loader=yaml.FullLoader) - ui_config = yaml.load(config['data']['spawner_ui_config.yaml'], Loader=yaml.FullLoader) + config = yaml.safe_load(fname.read()) + ui_config = yaml.safe_load(config['data']['spawner_ui_config.yaml']) # XXX: the yaml file doesn't read in properly due to the line 'spawner_ui_config.yaml: |'. So we pull it out and put it back later. config['data']['spawner_ui_config.yaml'] = ui_config diff --git a/workloads/examples/k8s/kubeflow-pipeline-deploy/triton_ops.py b/workloads/examples/k8s/kubeflow-pipeline-deploy/triton_ops.py index f94fcd435..33996c0a2 100644 --- a/workloads/examples/k8s/kubeflow-pipeline-deploy/triton_ops.py +++ b/workloads/examples/k8s/kubeflow-pipeline-deploy/triton_ops.py @@ -100,6 +100,6 @@ def __init__(self, name): super(TritonService, self).__init__( name=name, - k8s_resource=yaml.load(__TRITON_SERVICE_MANIFEST___), + k8s_resource=yaml.safe_load(__TRITON_SERVICE_MANIFEST___), action='create' ) From e9bcfa44f008f269dd682483d3a6f0abe826f096 Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Thu, 14 May 2026 17:00:04 -0600 Subject: [PATCH 02/27] chore(release): bump component versions for 26.05 --- roles/alertmanager/defaults/main.yml | 2 +- roles/grafana/defaults/main.yml | 2 +- roles/nginx-docker-registry-cache/defaults/main.yml | 2 +- roles/nvidia-gpu-operator/defaults/main.yml | 4 ++-- roles/nvidia-k8s-gpu-device-plugin/defaults/main.yml | 2 +- roles/nvidia-k8s-gpu-feature-discovery/defaults/main.yml | 4 ++-- roles/nvidia-mig-manager/defaults/main.yml | 4 ++-- roles/nvidia-network-operator/vars/main.yaml | 4 ++-- roles/prometheus-node-exporter/defaults/main.yml | 2 +- roles/prometheus/defaults/main.yml | 2 +- roles/slurm/defaults/main.yml | 2 +- roles/spack/defaults/main.yml | 2 +- roles/standalone-container-registry/defaults/main.yml | 2 +- scripts/k8s/deploy_ingress.sh | 2 +- scripts/k8s/deploy_monitoring.sh | 2 +- submodules/kubespray | 2 +- 16 files changed, 20 insertions(+), 20 deletions(-) diff --git a/roles/alertmanager/defaults/main.yml b/roles/alertmanager/defaults/main.yml index bcbed59c7..8bfdbc49d 100644 --- a/roles/alertmanager/defaults/main.yml +++ b/roles/alertmanager/defaults/main.yml @@ -1,6 +1,6 @@ alertmanager_config_dir: /etc/alertmanager alertmanager_config_src: templates/alertmanager.yml.j2 -alertmanager_container: "prom/alertmanager:v0.23.0" +alertmanager_container: "prom/alertmanager:v0.32.1" alertmanager_svc_name: "docker.alertmanager.service" alertmanager_docker_volume_name: "deepops_alertmanager_metrics" alertmanager_state: started diff --git a/roles/grafana/defaults/main.yml b/roles/grafana/defaults/main.yml index 5156ce0c6..920a37943 100644 --- a/roles/grafana/defaults/main.yml +++ b/roles/grafana/defaults/main.yml @@ -2,7 +2,7 @@ grafana_config_dir: /etc/grafana grafana_config_template: templates/grafana.ini.j2 grafana_data_dir: /var/lib/grafana grafana_user_id: 472 -grafana_container: "grafana/grafana:8.5.10" +grafana_container: "grafana/grafana:13.0.1" grafana_svc_name: "docker.grafana.service" grafana_state: started grafana_enabled: yes diff --git a/roles/nginx-docker-registry-cache/defaults/main.yml b/roles/nginx-docker-registry-cache/defaults/main.yml index dba99e9ac..6824bc7d5 100644 --- a/roles/nginx-docker-registry-cache/defaults/main.yml +++ b/roles/nginx-docker-registry-cache/defaults/main.yml @@ -1,6 +1,6 @@ --- nginx_docker_cache_name: "deepops-nginx-docker-cache" -nginx_docker_cache_image: "rpardini/docker-registry-proxy:0.6.4" +nginx_docker_cache_image: "rpardini/docker-registry-proxy:0.6.5" nginx_docker_cache_mirror_path: "/opt/deepops/nginx-docker-cache/mirror" nginx_docker_cache_ca_path: "/opt/deepops/nginx-docker-cache/ca" diff --git a/roles/nvidia-gpu-operator/defaults/main.yml b/roles/nvidia-gpu-operator/defaults/main.yml index 0cd7dad47..ebecc5d60 100644 --- a/roles/nvidia-gpu-operator/defaults/main.yml +++ b/roles/nvidia-gpu-operator/defaults/main.yml @@ -12,7 +12,7 @@ gpu_operator_nvaie_helm_repo: "https://helm.ngc.nvidia.com/nvaie" gpu_operator_nvaie_chart_name: "nvaie/gpu-operator" # NVAIE GPU Operator may require different version, check NGC enterprise collection. -gpu_operator_chart_version: "v23.3.2" +gpu_operator_chart_version: "v26.3.1" k8s_gpu_mig_strategy: "mixed" @@ -33,7 +33,7 @@ gpu_operator_grid_config_dir: "{{ deepops_dir }}/gpu_operator" # Defaults from https://github.com/NVIDIA/gpu-operator/blob/master/deployments/gpu-operator/values.yaml gpu_operator_default_runtime: "containerd" gpu_operator_driver_registry: "nvcr.io/nvidia" -gpu_operator_driver_version: "525.105.17" +gpu_operator_driver_version: "580.126.20" # This enables/disables NVAIE gpu_operator_nvaie_enable: false diff --git a/roles/nvidia-k8s-gpu-device-plugin/defaults/main.yml b/roles/nvidia-k8s-gpu-device-plugin/defaults/main.yml index a1214ec17..021d9a3f2 100644 --- a/roles/nvidia-k8s-gpu-device-plugin/defaults/main.yml +++ b/roles/nvidia-k8s-gpu-device-plugin/defaults/main.yml @@ -2,6 +2,6 @@ k8s_gpu_plugin_helm_repo: "https://nvidia.github.io/k8s-device-plugin" k8s_gpu_plugin_chart_name: "nvdp/nvidia-device-plugin" k8s_gpu_plugin_release_name: "nvidia-device-plugin" -k8s_gpu_plugin_chart_version: "0.14.0" +k8s_gpu_plugin_chart_version: "0.19.1" k8s_gpu_plugin_init_error: "false" k8s_gpu_mig_strategy: "mixed" diff --git a/roles/nvidia-k8s-gpu-feature-discovery/defaults/main.yml b/roles/nvidia-k8s-gpu-feature-discovery/defaults/main.yml index d1cced31e..77111b3ff 100644 --- a/roles/nvidia-k8s-gpu-feature-discovery/defaults/main.yml +++ b/roles/nvidia-k8s-gpu-feature-discovery/defaults/main.yml @@ -1,6 +1,6 @@ # Vars needed to install feature discovery -k8s_gpu_feature_discovery_helm_repo: "https://nvidia.github.io/gpu-feature-discovery" +k8s_gpu_feature_discovery_helm_repo: "https://nvidia.github.io/k8s-device-plugin" k8s_gpu_feature_discovery_chart_name: "nvgfd/gpu-feature-discovery" k8s_gpu_feature_discovery_release_name: "gpu-feature-discovery" -k8s_gpu_feature_discovery_chart_version: "0.8.0" +k8s_gpu_feature_discovery_chart_version: "0.19.1" k8s_gpu_mig_strategy: "mixed" diff --git a/roles/nvidia-mig-manager/defaults/main.yml b/roles/nvidia-mig-manager/defaults/main.yml index 1139c4608..80d0c60f4 100644 --- a/roles/nvidia-mig-manager/defaults/main.yml +++ b/roles/nvidia-mig-manager/defaults/main.yml @@ -1,3 +1,3 @@ --- -mig_manager_url_deb: https://github.com/NVIDIA/mig-parted/releases/download/v0.4.2/nvidia-mig-manager_0.4.2-1_amd64.deb -mig_manager_url_rpm: https://github.com/NVIDIA/mig-parted/releases/download/v0.4.2/nvidia-mig-manager-0.4.2-1.x86_64.rpm +mig_manager_url_deb: https://github.com/NVIDIA/mig-parted/releases/download/v0.14.1/nvidia-mig-manager_0.14.1-1_amd64.deb +mig_manager_url_rpm: https://github.com/NVIDIA/mig-parted/releases/download/v0.14.1/nvidia-mig-manager-0.14.1-1.x86_64.rpm diff --git a/roles/nvidia-network-operator/vars/main.yaml b/roles/nvidia-network-operator/vars/main.yaml index 2d3322081..95a420b99 100644 --- a/roles/nvidia-network-operator/vars/main.yaml +++ b/roles/nvidia-network-operator/vars/main.yaml @@ -6,8 +6,8 @@ # if_name must match k8s network annotation name # -nvidia_network_operator_version: "1.2.0" -nvidia_network_operator_url: "https://mellanox.github.io/network-operator" +nvidia_network_operator_version: "26.1.1" +nvidia_network_operator_url: "https://helm.ngc.nvidia.com/nvidia" mpi_operator_version: "v2beta1" mpi_raw_url: "https://raw.githubusercontent.com/kubeflow/mpi-operator/master/deploy/v2beta1" diff --git a/roles/prometheus-node-exporter/defaults/main.yml b/roles/prometheus-node-exporter/defaults/main.yml index 9d6116c8e..925f4baee 100644 --- a/roles/prometheus-node-exporter/defaults/main.yml +++ b/roles/prometheus-node-exporter/defaults/main.yml @@ -1,4 +1,4 @@ -node_exporter_container: "quay.io/prometheus/node-exporter:v1.3.1" +node_exporter_container: "quay.io/prometheus/node-exporter:v1.11.1" node_exporter_prom_dir: "/run/prometheus" node_exporter_svc_name: "docker.node-exporter.service" node_exporter_state: started diff --git a/roles/prometheus/defaults/main.yml b/roles/prometheus/defaults/main.yml index 6e7afbd21..7349da1b3 100644 --- a/roles/prometheus/defaults/main.yml +++ b/roles/prometheus/defaults/main.yml @@ -1,7 +1,7 @@ prometheus_config_dir: /etc/prometheus prometheus_config_src: templates/prometheus.yml.j2 prometheus_alert_rules_src: templates/alert_rules.yml.j2 -prometheus_container: "prom/prometheus:v2.37.0" +prometheus_container: "prom/prometheus:v3.11.3" prometheus_svc_name: "docker.prometheus.service" prometheus_docker_volume_name: "deepops_prometheus_metrics" prometheus_state: started diff --git a/roles/slurm/defaults/main.yml b/roles/slurm/defaults/main.yml index 4573301db..735ae16ee 100644 --- a/roles/slurm/defaults/main.yml +++ b/roles/slurm/defaults/main.yml @@ -7,7 +7,7 @@ hwloc_build_dir: /opt/deepops/build/hwloc pmix_build_dir: /opt/deepops/build/pmix slurm_workflow_build: yes -slurm_version: "23.02.4" +slurm_version: "25.11.6" slurm_src_url: "https://download.schedmd.com/slurm/slurm-{{ slurm_version }}.tar.bz2" slurm_build_make_clean: no slurm_build_dir_cleanup: no diff --git a/roles/spack/defaults/main.yml b/roles/spack/defaults/main.yml index 26540b781..76e356c63 100644 --- a/roles/spack/defaults/main.yml +++ b/roles/spack/defaults/main.yml @@ -1,7 +1,7 @@ --- spack_repo: "https://github.com/spack/spack.git" spack_install_dir: "/sw/spack" -spack_version: "v0.18.1" +spack_version: "v1.1.1" spack_user: "root" spack_group: "root" diff --git a/roles/standalone-container-registry/defaults/main.yml b/roles/standalone-container-registry/defaults/main.yml index 383f6e9e4..562524707 100644 --- a/roles/standalone-container-registry/defaults/main.yml +++ b/roles/standalone-container-registry/defaults/main.yml @@ -2,7 +2,7 @@ epel_package: "https://dl.fedoraproject.org/pub/epel/epel-release-latest-{{ ansible_distribution_major_version }}.noarch.rpm" epel_key_url: "https://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-{{ ansible_distribution_major_version }}" -standalone_container_registry_image: "registry:2.8" +standalone_container_registry_image: "registry:3.1.1" standalone_container_registry_port: "5000" standalone_container_registry_name: "deepops-registry" diff --git a/scripts/k8s/deploy_ingress.sh b/scripts/k8s/deploy_ingress.sh index f311f4d84..b1ad2143e 100755 --- a/scripts/k8s/deploy_ingress.sh +++ b/scripts/k8s/deploy_ingress.sh @@ -9,7 +9,7 @@ ROOT_DIR="${SCRIPT_DIR}/../.." source ${ROOT_DIR}/scripts/common.sh HELM_CHARTS_REPO_INGRESS="${HELM_CHARTS_REPO_INGRESS:-https://kubernetes.github.io/ingress-nginx}" -HELM_INGRESS_CHART_VERSION="${HELM_INGRESS_CHART_VERSION:-4.2.1}" +HELM_INGRESS_CHART_VERSION="${HELM_INGRESS_CHART_VERSION:-4.15.1}" # HELM_INGRESS_CONFIG, defaults below based on presence of metallb ${SCRIPT_DIR}/k8s/install_helm.sh diff --git a/scripts/k8s/deploy_monitoring.sh b/scripts/k8s/deploy_monitoring.sh index d73fdc5f5..05a5851a7 100755 --- a/scripts/k8s/deploy_monitoring.sh +++ b/scripts/k8s/deploy_monitoring.sh @@ -24,7 +24,7 @@ if [ ! -d "${DEEPOPS_CONFIG_DIR}" ]; then fi HELM_CHARTS_REPO_PROMETHEUS="${HELM_CHARTS_REPO_PROMETHEUS:-https://prometheus-community.github.io/helm-charts}" -HELM_PROMETHEUS_CHART_VERSION="${HELM_PROMETHEUS_CHART_VERSION:-39.5.0}" +HELM_PROMETHEUS_CHART_VERSION="${HELM_PROMETHEUS_CHART_VERSION:-85.0.3}" ingress_name="ingress-nginx" PROMETHEUS_YAML_CONFIG="${PROMETHEUS_YAML_CONFIG:-${DEEPOPS_CONFIG_DIR}/helm/monitoring.yml}" diff --git a/submodules/kubespray b/submodules/kubespray index f4ccdb5e7..1c9add489 160000 --- a/submodules/kubespray +++ b/submodules/kubespray @@ -1 +1 @@ -Subproject commit f4ccdb5e72395eaf9f3444056ebd1a6625ddb89a +Subproject commit 1c9add48975060f45396b34d8e022c30d7f80dab From 252dd33f97c4d50aa50aab84787d725e8106bb0b Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Thu, 14 May 2026 17:01:54 -0600 Subject: [PATCH 03/27] docs: update Kubernetes inventory group names --- docs/deepops/configuration.md | 2 +- docs/k8s-cluster/README.md | 12 ++++++------ docs/k8s-cluster/kubeflow.md | 2 +- docs/k8s-cluster/nvidia-network-operator.md | 4 ++-- docs/k8s-cluster/roce-perf-k8s.md | 6 +++--- docs/pxe/maas.md | 8 ++++---- roles/netapp-trident/README.md | 6 +++--- 7 files changed, 20 insertions(+), 20 deletions(-) diff --git a/docs/deepops/configuration.md b/docs/deepops/configuration.md index 70c1475e6..3884b8137 100644 --- a/docs/deepops/configuration.md +++ b/docs/deepops/configuration.md @@ -62,7 +62,7 @@ my-cluster-compute-02 ansible_host=10.0.0.3 (Note that, by default, DeepOps will set the hostname of these machines to match the inventory hostname! If you don't want this, you can set `deepops_set_hostname: false` using the instructions in [the next section](#modifying-ansible-variables).) -The example DeepOps inventory also includes groups for the different components of Kubernetes clusters (`kube-master`, `etcd`, and `kube-node`), +The example DeepOps inventory also includes groups for the different components of Kubernetes clusters (`kube_control_plane`, `etcd`, and `kube_node`), and groups for the different components of Slurm clusters (`slurm-master` and `slurm-node`). These groups are used by DeepOps to determine which playbooks run on which nodes for each type of cluster, and you should add nodes to these groups based on how you want to lay out your cluster. diff --git a/docs/k8s-cluster/README.md b/docs/k8s-cluster/README.md index ee6bfe66c..9fb71364b 100644 --- a/docs/k8s-cluster/README.md +++ b/docs/k8s-cluster/README.md @@ -123,7 +123,7 @@ Run the following script to create an administrative user and print out the dash #### NFS Client Provisioner -The default behavior of DeepOps is to setup an NFS server on the first `kube-master` node. This temporary NFS server is used by the `nfs-client-provisioner` which is installed as the default StorageClass of a standard DeepOps deployment. +The default behavior of DeepOps is to setup an NFS server on the first `kube_control_plane` node. This temporary NFS server is used by the `nfs-client-provisioner` which is installed as the default StorageClass of a standard DeepOps deployment. To use an existing nfs server server update the `k8s_nfs_server` and `k8s_nfs_export_path` variables in `config/group_vars/k8s-cluster.yml` and set the `k8s_deploy_nfs_server` to false in `config/group_vars/k8s-cluster.yml`. Additionally, the `k8s_nfs_mkdir` variable can be set to `false` if the export directory is already configured on the server. @@ -207,9 +207,9 @@ delete Legacy positional argument for delete. Same as -d flag. The services can be reached from the following addresses: -- Grafana: http://\:30200 -- Prometheus: http://\:30500 -- Alertmanager: http://\:30400 +- Grafana: http://\:30200 +- Prometheus: http://\:30500 +- Alertmanager: http://\:30400 We deploy our monitoring services using the [prometheus-operator](https://github.com/prometheus-operator/prometheus-operator) project. For documentation on configuring and managing the monitoring services, please see the [prometheus-operator user guides](https://github.com/prometheus-operator/prometheus-operator/tree/master/Documentation/user-guides). @@ -234,7 +234,7 @@ Follow the [ELK logging Guide](logging.md) to setup logging in the cluster. The service can be reached from the following address: -- Kibana: http://\:30700 +- Kibana: http://\:30700 ### Container Registry @@ -264,7 +264,7 @@ DeepOps uses [Kubespray](https://github.com/kubernetes-sigs/kubespray) to deploy ### Adding Nodes -To add K8s nodes, modify the `config/inventory` file to include the new nodes under `[all]`. Then list the nodes as relevant under the `[kube-master]`, `[etcd]`, and `[kube-node]` sections. For example, if adding a new master node, list it under kube-master and etcd. A new worker node would go under kube-node. +To add K8s nodes, modify the `config/inventory` file to include the new nodes under `[all]`. Then list the nodes as relevant under the `[kube_control_plane]`, `[etcd]`, and `[kube_node]` sections. For example, if adding a new control-plane node, list it under `kube_control_plane` and `etcd`. A new worker node would go under `kube_node`. Then run the Kubespray `scale.yml` playbook... diff --git a/docs/k8s-cluster/kubeflow.md b/docs/k8s-cluster/kubeflow.md index 02394fc86..59f4d6797 100644 --- a/docs/k8s-cluster/kubeflow.md +++ b/docs/k8s-cluster/kubeflow.md @@ -43,7 +43,7 @@ A local checkout of the [Kubeflow manifests](https://github.com/kubeflow/manifes The services can be reached from the following address: -- Kubeflow: http://\:31380 +- Kubeflow: http://\:31380 ## Login information diff --git a/docs/k8s-cluster/nvidia-network-operator.md b/docs/k8s-cluster/nvidia-network-operator.md index 972211aa3..ea0636653 100644 --- a/docs/k8s-cluster/nvidia-network-operator.md +++ b/docs/k8s-cluster/nvidia-network-operator.md @@ -83,9 +83,9 @@ This playbook is developed and tested in following environments: gpu01 ansible_host=192.168.2.11 gpu02 ansible_host=192.168.3.11 ... - [kube-master] + [kube_control_plane] mgmt01 - [kube-node] + [kube_node] gpu01 gpu02 ``` diff --git a/docs/k8s-cluster/roce-perf-k8s.md b/docs/k8s-cluster/roce-perf-k8s.md index b80a6f41c..fd7de1ce6 100644 --- a/docs/k8s-cluster/roce-perf-k8s.md +++ b/docs/k8s-cluster/roce-perf-k8s.md @@ -142,7 +142,7 @@ add switch PFC, ECN configuration ```bash # Modify the Ansible inventory file - # Especially the 'all', 'kube-master', 'etcd', 'kube-node' and 'k8s-cluster' sections + # Especially the 'all', 'kube_control_plane', 'etcd', 'kube_node' and 'k8s-cluster' sections vi config/inventory ``` @@ -159,10 +159,10 @@ add switch PFC, ECN configuration gpu02 ansible_host=192.168.2.11 ... - [kube-master] + [kube_control_plane] mgmt01 - [kube-node] + [kube_node] gpu01 gpu02 diff --git a/docs/pxe/maas.md b/docs/pxe/maas.md index ca95671c9..1deb6dad1 100644 --- a/docs/pxe/maas.md +++ b/docs/pxe/maas.md @@ -258,8 +258,8 @@ only need to tag leaf groups. | Tag | Ansible Group | Used By | |-----|--------------|---------| -| `kube-master` | `[kube-master]` | K8s control plane | -| `kube-node` | `[kube-node]` | K8s worker nodes | +| `kube_control_plane` | `[kube_control_plane]` | K8s control plane | +| `kube_node` | `[kube_node]` | K8s worker nodes | | `slurm-master` | `[slurm-master]` | Slurm head node | | `slurm-node` | `[slurm-node]` | Slurm compute nodes | | `slurm-nfs` | `[slurm-nfs]` | Slurm NFS server | @@ -276,8 +276,8 @@ ansible-playbook -i scripts/maas_inventory.py playbooks/slurm-cluster.yml # Later, retag for K8s maas admin tag update-nodes slurm-master remove= -maas admin tag update-nodes kube-master add= -maas admin tag update-nodes kube-node add= add= +maas admin tag update-nodes kube_control_plane add= +maas admin tag update-nodes kube_node add= add= # Run K8s deployment ansible-playbook -i scripts/maas_inventory.py playbooks/k8s-cluster.yml diff --git a/roles/netapp-trident/README.md b/roles/netapp-trident/README.md index 87b5557c0..7911d097b 100644 --- a/roles/netapp-trident/README.md +++ b/roles/netapp-trident/README.md @@ -41,7 +41,7 @@ Example A: Example B: - name: "Deploy NetApp Trident" - hosts: kube-master + hosts: kube_control_plane become: true become_method: sudo roles: @@ -50,7 +50,7 @@ Example B: Example C: - name: "Deploy NetApp Trident" - hosts: kube-master + hosts: kube_control_plane become: true become_method: sudo vars_files: @@ -88,7 +88,7 @@ all: ip: 192.168.1.215 access_ip: 192.168.1.215 children: - kube-master: + kube_control_plane: hosts: mgmt01: mgmt02: From ff1d89d382a2d51cbeb93419ebb6b8abaeb7beff Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Thu, 14 May 2026 17:18:22 -0600 Subject: [PATCH 04/27] chore(maas): update role dependency for Ansible 10 --- config.example/group_vars/all.yml | 5 ++--- roles/requirements.yml | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/config.example/group_vars/all.yml b/config.example/group_vars/all.yml index 6b69f98ba..b6c47707e 100644 --- a/config.example/group_vars/all.yml +++ b/config.example/group_vars/all.yml @@ -249,7 +249,7 @@ maas_adminusers: maas_dns_domain: 'deepops.local' maas_region_controller: '192.168.1.1' maas_region_controller_url: 'http://{{ maas_region_controller }}:5240/MAAS' -maas_repo: 'ppa:maas/2.8' +maas_repo: 'ppa:maas/3.5' # Defines if maas user should generate ssh keys # Usable for remote KVM/libvirt power actions @@ -257,7 +257,7 @@ maas_setup_user: false maas_single_node_install: true -maas_kvm: false +maas_kvm_management: false ################################################################################ # NVIDIA Datacenter GPU Manager # @@ -305,4 +305,3 @@ standalone_container_registry_port: "5000" ngc_ready_cuda_container: "nvcr.io/nvidia/cuda:12.4.1-base-ubuntu22.04" ngc_ready_pytorch: "nvcr.io/nvidia/pytorch:24.04-py3" ngc_ready_tensorflow: "nvcr.io/nvidia/tensorflow:24.04-tf2-py3" - diff --git a/roles/requirements.yml b/roles/requirements.yml index b6a0795aa..77f97ef7b 100644 --- a/roles/requirements.yml +++ b/roles/requirements.yml @@ -56,9 +56,9 @@ roles: - src: robertdebock.kibana version: "1.2.6" -- src: https://github.com/DeepOps/ansible-maas.git +- src: https://github.com/mrlesmithjr/ansible-maas.git name: ansible-maas - version: '632fe9bd1e048b9abb717621dc2d76b19614327b' + version: '178a999c9bfc979ef32c42f4f59c034664df10d0' - src: https://github.com/DeepOps/ansible-role-chrony name: DeepOps.chrony From ed9cda3c0470f2140407787a372bcc063b2e49f5 Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Fri, 15 May 2026 07:50:16 -0600 Subject: [PATCH 05/27] fix(nvidia): support Ubuntu open kernel modules --- config.example/group_vars/all.yml | 4 ++++ docs/deepops/update-deepops.md | 24 ++++++++++++++----- playbooks/nvidia-software/nvidia-cuda.yml | 4 ++++ playbooks/nvidia-software/nvidia-driver.yml | 4 ++++ .../tasks/nvidia-driver-ubuntu-packages.yml | 12 ++++++++++ 5 files changed, 42 insertions(+), 6 deletions(-) create mode 100644 playbooks/nvidia-software/tasks/nvidia-driver-ubuntu-packages.yml diff --git a/config.example/group_vars/all.yml b/config.example/group_vars/all.yml index b6c47707e..a7eab54fa 100644 --- a/config.example/group_vars/all.yml +++ b/config.example/group_vars/all.yml @@ -140,6 +140,10 @@ gpu_clock_lock: "1507,1507" # Debugging var: force install NVIDIA driver even if GPU not detected nvidia_driver_force_install: false +# Default NVIDIA driver branch for generic host driver installs. +nvidia_driver_branch: "580" +# Set true on Ubuntu systems that require NVIDIA open kernel modules. +nvidia_driver_ubuntu_use_open_kernel_modules: false ################################################################################ diff --git a/docs/deepops/update-deepops.md b/docs/deepops/update-deepops.md index 3e081c18c..8d2bd73f2 100644 --- a/docs/deepops/update-deepops.md +++ b/docs/deepops/update-deepops.md @@ -267,7 +267,7 @@ DeepOps offers the option to configure each of the necessary NVIDIA components i ##### Updating the NVIDIA driver -**Important**: Note that upgrading the NVIDIA driver will reboot the node, unless you set `nvidia_driver_skip_reboot` to false. +**Important**: Note that upgrading the NVIDIA driver will reboot the node, unless you set `nvidia_driver_skip_reboot` to true. If you are using MIG-enabled GPUs ensure that your MIG configuration is persistent by using the [nvidia-mig-manager systemd](https://github.com/NVIDIA/mig-parted/tree/master/deployments/systemd) service or the [nvidia-mig-manager Kubernetes GPU Operator-included DaemonSet](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/gpu-operator-mig.html). @@ -277,7 +277,7 @@ To update the driver on a DGX system, we recommend following the instructions in ###### On Ubuntu -On Ubuntu, the default behavior in DeepOps is to use the LTS release branch distributed through the Ubuntu repositories. In this mode, the driver is generally pinned to a particular release branch such as 450 or 470. +On Ubuntu, the default behavior in DeepOps is to use the LTS release branch distributed through the Ubuntu repositories. In this mode, the driver is generally pinned to a particular release branch such as 580. To upgrade to the latest driver within your current release branch, run: @@ -288,7 +288,13 @@ ansible-playbook playbooks/nvidia-software/nvidia-driver.yml -e nvidia_driver_pa To upgrade the driver to a new release branch, set the following parameter in your DeepOps configuration: ```bash -nvidia_driver_ubuntu_branch: "470" +nvidia_driver_ubuntu_branch: "580" +``` + +Some newer GPUs require NVIDIA open kernel modules. To install the Ubuntu open kernel module packages for the selected branch, set: + +```bash +nvidia_driver_ubuntu_use_open_kernel_modules: true ``` Then run: @@ -433,7 +439,7 @@ Note that this can take a long time, as we download and build Slurm from source #### Updating the NVIDIA driver -**Important**: Note that upgrading the NVIDIA driver will reboot the node, unless you set `nvidia_driver_skip_reboot` to false. +**Important**: Note that upgrading the NVIDIA driver will reboot the node, unless you set `nvidia_driver_skip_reboot` to true. ##### On DGX @@ -441,7 +447,7 @@ To update the driver on a DGX system, we recommend following the instructions in ##### On Ubuntu -On Ubuntu, the default behavior in DeepOps is to use the LTS release branch distributed through the Ubuntu repositories. In this mode, the driver is generally pinned to a particular release branch such as 450 or 470. +On Ubuntu, the default behavior in DeepOps is to use the LTS release branch distributed through the Ubuntu repositories. In this mode, the driver is generally pinned to a particular release branch such as 580. To upgrade to the latest driver within your current release branch, run: @@ -452,7 +458,13 @@ ansible-playbook playbooks/nvidia-software/nvidia-driver.yml -e nvidia_driver_pa To upgrade the driver to a new release branch, set the following parameter in your DeepOps configuration: ```bash -nvidia_driver_ubuntu_branch: "510" +nvidia_driver_ubuntu_branch: "580" +``` + +Some newer GPUs require NVIDIA open kernel modules. To install the Ubuntu open kernel module packages for the selected branch, set: + +```bash +nvidia_driver_ubuntu_use_open_kernel_modules: true ``` Then run: diff --git a/playbooks/nvidia-software/nvidia-cuda.yml b/playbooks/nvidia-software/nvidia-cuda.yml index f7e704e2e..7945f671c 100644 --- a/playbooks/nvidia-software/nvidia-cuda.yml +++ b/playbooks/nvidia-software/nvidia-cuda.yml @@ -13,6 +13,10 @@ include_role: name: facts + - name: configure Ubuntu NVIDIA driver packages + include_tasks: tasks/nvidia-driver-ubuntu-packages.yml + when: ansible_distribution == 'Ubuntu' + - name: install nvidia driver include_role: name: nvidia.nvidia_driver diff --git a/playbooks/nvidia-software/nvidia-driver.yml b/playbooks/nvidia-software/nvidia-driver.yml index c9a05b687..c445e5574 100644 --- a/playbooks/nvidia-software/nvidia-driver.yml +++ b/playbooks/nvidia-software/nvidia-driver.yml @@ -14,6 +14,10 @@ include_role: name: facts + - name: configure Ubuntu NVIDIA driver packages + include_tasks: tasks/nvidia-driver-ubuntu-packages.yml + when: ansible_distribution == 'Ubuntu' + - name: install nvidia driver include_role: name: nvidia.nvidia_driver diff --git a/playbooks/nvidia-software/tasks/nvidia-driver-ubuntu-packages.yml b/playbooks/nvidia-software/tasks/nvidia-driver-ubuntu-packages.yml new file mode 100644 index 000000000..1a987eac4 --- /dev/null +++ b/playbooks/nvidia-software/tasks/nvidia-driver-ubuntu-packages.yml @@ -0,0 +1,12 @@ +--- +- name: Select Ubuntu NVIDIA open kernel module packages + set_fact: + nvidia_driver_ubuntu_packages: + - "nvidia-headless-{{ _nvidia_driver_ubuntu_branch }}{{ _nvidia_driver_ubuntu_packages_suffix }}-open" + - "nvidia-utils-{{ _nvidia_driver_ubuntu_branch }}{{ _nvidia_driver_ubuntu_packages_suffix }}" + - "nvidia-headless-no-dkms-{{ _nvidia_driver_ubuntu_branch }}{{ _nvidia_driver_ubuntu_packages_suffix }}-open" + - "nvidia-kernel-source-{{ _nvidia_driver_ubuntu_branch }}{{ _nvidia_driver_ubuntu_packages_suffix }}-open" + vars: + _nvidia_driver_ubuntu_branch: "{{ nvidia_driver_ubuntu_branch | default(nvidia_driver_branch | default('515')) }}" + _nvidia_driver_ubuntu_packages_suffix: "{{ nvidia_driver_ubuntu_packages_suffix | default('-server') }}" + when: nvidia_driver_ubuntu_use_open_kernel_modules | default(false) | bool From 56be7831af84114a8593f8bf121b59b95acfee1d Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Fri, 15 May 2026 09:05:57 -0600 Subject: [PATCH 06/27] fix(slurm): make controller setup rerunnable --- roles/slurm/tasks/controller.yml | 22 ++++++++++++++++++---- roles/slurm/tasks/login-compute-setup.yml | 1 + 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/roles/slurm/tasks/controller.yml b/roles/slurm/tasks/controller.yml index 733e32823..8ebe48fac 100644 --- a/roles/slurm/tasks/controller.yml +++ b/roles/slurm/tasks/controller.yml @@ -149,16 +149,30 @@ - name: create account command: sacctmgr -i add account compute-account Description="Compute Accounts" Organization="Prestige" register: create_account_result - failed_when: "create_account_result.rc != 0 and 'Nothing new added' not in create_account_result.stdout" - changed_when: "'Nothing new added' not in create_account_result.stdout" + failed_when: > + create_account_result.rc != 0 and + 'Nothing new added' not in create_account_result.stdout and + 'Nothing added' not in create_account_result.stdout and + 'Already existing' not in create_account_result.stdout + changed_when: > + 'Nothing new added' not in create_account_result.stdout and + 'Nothing added' not in create_account_result.stdout and + 'Already existing' not in create_account_result.stdout environment: PATH: '{{ slurm_install_prefix }}/bin:{{ ansible_env.PATH }}' - name: create users command: sacctmgr -i create user {{ item }} account=compute-account adminlevel=None register: create_user_result - failed_when: "create_user_result.rc != 0 and 'Nothing new added' not in create_user_result.stdout" - changed_when: "'Nothing new added' not in create_user_result.stdout" + failed_when: > + create_user_result.rc != 0 and + 'Nothing new added' not in create_user_result.stdout and + 'Nothing added' not in create_user_result.stdout and + 'Already existing' not in create_user_result.stdout + changed_when: > + 'Nothing new added' not in create_user_result.stdout and + 'Nothing added' not in create_user_result.stdout and + 'Already existing' not in create_user_result.stdout with_items: - "{{ user }}" environment: diff --git a/roles/slurm/tasks/login-compute-setup.yml b/roles/slurm/tasks/login-compute-setup.yml index 69c2769a9..2644c9bc3 100644 --- a/roles/slurm/tasks/login-compute-setup.yml +++ b/roles/slurm/tasks/login-compute-setup.yml @@ -9,6 +9,7 @@ systemctl set-property sshd.service DeviceAllow="/dev/nvidiactl" fi args: + executable: /bin/bash creates: "{{ '/etc/systemd/system.control/sshd.service.d/50-DeviceAllow.conf' \ if ansible_os_family == 'RedHat' else \ '/etc/systemd/system.control/ssh.service.d/50-DeviceAllow.conf' }}" From a5c9a97479f0fa3a6649e0d6426bc6b75f608fff Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Fri, 15 May 2026 15:19:10 -0600 Subject: [PATCH 07/27] fix(k8s): use mapping vars for NFS role include --- playbooks/k8s-cluster/nfs-client-provisioner.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/k8s-cluster/nfs-client-provisioner.yml b/playbooks/k8s-cluster/nfs-client-provisioner.yml index 93592e85b..200f27f99 100644 --- a/playbooks/k8s-cluster/nfs-client-provisioner.yml +++ b/playbooks/k8s-cluster/nfs-client-provisioner.yml @@ -22,7 +22,7 @@ include_role: name: nfs vars: - - nfs_is_server: yes + nfs_is_server: yes when: k8s_deploy_nfs_server - hosts: "k8s_cluster" From 908555ea7e330a34201edfa1f9749c5adbe307b1 Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Fri, 15 May 2026 17:04:53 -0600 Subject: [PATCH 08/27] fix(k8s): update network operator role for current chart --- docs/k8s-cluster/README.md | 6 +-- docs/k8s-cluster/nvidia-network-operator.md | 2 +- docs/k8s-cluster/roce-perf-k8s.md | 6 +-- docs/k8s-cluster/roce_backend.md | 2 +- roles/nvidia-network-operator/tasks/main.yaml | 40 +++++++++++++++++-- .../tasks/sriovibnetwork.yaml | 10 +---- .../tasks/sriovnetworknodepolicy.yaml | 2 +- .../templates/nicclusterpolicy.yaml | 22 ++++++++++ .../templates/values.yaml | 18 --------- roles/nvidia-network-operator/vars/main.yaml | 4 ++ virtual/vars_files/virt_k8s.yml | 2 +- virtual/virtual_inventory | 10 ++--- virtual/virtual_inventory_full | 10 ++--- 13 files changed, 85 insertions(+), 49 deletions(-) create mode 100644 roles/nvidia-network-operator/templates/nicclusterpolicy.yaml diff --git a/docs/k8s-cluster/README.md b/docs/k8s-cluster/README.md index 9fb71364b..b7f1e32b1 100644 --- a/docs/k8s-cluster/README.md +++ b/docs/k8s-cluster/README.md @@ -84,7 +84,7 @@ Instructions for deploying a GPU cluster with Kubernetes # NOTE: If SSH requires a password, add: `-k` # NOTE: If sudo on remote machine requires a password, add: `-K` # NOTE: If SSH user is different than current user, add: `-u ubuntu` - ansible-playbook -l k8s-cluster playbooks/k8s-cluster.yml + ansible-playbook -l k8s_cluster playbooks/k8s-cluster.yml ``` More information on Kubespray can be found in the official [Getting Started Guide](https://github.com/kubernetes-sigs/kubespray/blob/master/docs/getting-started.md) @@ -165,7 +165,7 @@ Deploy NetApp Astra Trident for services that require persistent storage (such a # NOTE: If SSH requires a password, add: `-k` # NOTE: If sudo on remote machine requires a password, add: `-K` # NOTE: If SSH user is different than current user, add: `-u ubuntu` - ansible-playbook -l k8s-cluster playbooks/k8s-cluster/netapp-trident.yml + ansible-playbook -l k8s_cluster playbooks/k8s-cluster/netapp-trident.yml ``` 3. Verify that Astra Trident is running. @@ -272,7 +272,7 @@ Then run the Kubespray `scale.yml` playbook... # NOTE: If SSH requires a password, add: `-k` # NOTE: If sudo on remote machine requires a password, add: `-K` # NOTE: If SSH user is different than current user, add: `-u ubuntu` -ansible-playbook -l k8s-cluster submodules/kubespray/scale.yml +ansible-playbook -l k8s_cluster submodules/kubespray/scale.yml ``` More information on this topic may be found in the [Kubespray docs](https://github.com/kubernetes-sigs/kubespray/blob/master/docs/getting-started.md#adding-nodes). diff --git a/docs/k8s-cluster/nvidia-network-operator.md b/docs/k8s-cluster/nvidia-network-operator.md index ea0636653..3817c14f5 100644 --- a/docs/k8s-cluster/nvidia-network-operator.md +++ b/docs/k8s-cluster/nvidia-network-operator.md @@ -123,7 +123,7 @@ This playbook is developed and tested in following environments: # NOTE: If SSH requires a password, add: `-k` # NOTE: If sudo on remote machine requires a password, add: `-K` # NOTE: If SSH user is different than current user, add: `-u ubuntu` - ansible-playbook -l k8s-cluster playbooks/k8s-cluster.yml + ansible-playbook -l k8s_cluster playbooks/k8s-cluster.yml ``` Please refer to [DeepOps Kubernetes Deployment Guidehere](https://github.com/NVIDIA/deepops/blob/master/docs/kubernetes-cluster.md) for more information. diff --git a/docs/k8s-cluster/roce-perf-k8s.md b/docs/k8s-cluster/roce-perf-k8s.md index fd7de1ce6..6f5d12c1b 100644 --- a/docs/k8s-cluster/roce-perf-k8s.md +++ b/docs/k8s-cluster/roce-perf-k8s.md @@ -142,7 +142,7 @@ add switch PFC, ECN configuration ```bash # Modify the Ansible inventory file - # Especially the 'all', 'kube_control_plane', 'etcd', 'kube_node' and 'k8s-cluster' sections + # Especially the 'all', 'kube_control_plane', 'etcd', 'kube_node' and 'k8s_cluster' sections vi config/inventory ``` @@ -203,7 +203,7 @@ add switch PFC, ECN configuration # NOTE: If SSH requires a password, add: `-k` # NOTE: If sudo on remote machine requires a password, add: `-K` # NOTE: If SSH user is different than current user, add: `-u ubuntu` - ansible-playbook -l k8s-cluster playbooks/k8s-cluster.yml + ansible-playbook -l k8s_cluster playbooks/k8s-cluster.yml ``` Please refer to [DeepOps Kubernetes Deployment Guidehere](https://github.com/NVIDIA/deepops/blob/master/docs/kubernetes-cluster.md) for more information. @@ -252,7 +252,7 @@ add switch PFC, ECN configuration Run following script to deploy SRIOV RoCE functions: ```bash - nvidia@mgmt01:~/deepops_0322$ ansible-playbook -l k8s-cluster playbooks/k8s-cluster/roce.yaml + nvidia@mgmt01:~/deepops_0322$ ansible-playbook -l k8s_cluster playbooks/k8s-cluster/roce.yaml ``` If using a different username and SSH key-based authentication haven't set up, try to use `-u -k -K` when you run the script. diff --git a/docs/k8s-cluster/roce_backend.md b/docs/k8s-cluster/roce_backend.md index 254007217..aa5c9dba6 100644 --- a/docs/k8s-cluster/roce_backend.md +++ b/docs/k8s-cluster/roce_backend.md @@ -106,7 +106,7 @@ The Role installing following components: ## Role deployment ```bash -ansible-playbook -l k8s-cluster playbooks/k8s-cluster/roce.yaml +ansible-playbook -l k8s_cluster playbooks/k8s-cluster/roce.yaml ``` ## License diff --git a/roles/nvidia-network-operator/tasks/main.yaml b/roles/nvidia-network-operator/tasks/main.yaml index d2c9e409e..09e7fa8dc 100644 --- a/roles/nvidia-network-operator/tasks/main.yaml +++ b/roles/nvidia-network-operator/tasks/main.yaml @@ -4,7 +4,7 @@ - name: label the nodes # noqa command-instead-of-shell shell: kubectl label --overwrite nodes {{ item }} node-role.kubernetes.io/worker= - with_items: "{{ groups['kube-node'] }}" + with_items: "{{ groups['kube_node'] }}" changed_when: false ## required as the DeepOps openshift role doesn't work @@ -20,7 +20,7 @@ - name: Deploy network operator helm chart kubernetes.core.helm: name: network-operator - release_namespace: network-operator + release_namespace: "{{ nvidia_network_operator_namespace }}" chart_version: "{{ nvidia_network_operator_version }}" chart_ref: mellanox/network-operator create_namespace: true @@ -28,7 +28,41 @@ wait: true values: "{{ lookup('template', 'values.yaml') | from_yaml }}" -- name: Create network node poliy +- name: Create NicClusterPolicy + kubernetes.core.k8s: + state: present + definition: "{{ lookup('template', 'nicclusterpolicy.yaml') | from_yaml }}" + run_once: true + +- name: Wait for NVIDIA IPAM CRD + kubernetes.core.k8s_info: + api_version: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + name: ippools.nv-ipam.nvidia.com + register: nvidia_ipam_crd + until: nvidia_ipam_crd.resources | length > 0 + retries: 30 + delay: 10 + when: nvidia_network_operator_ipam_type == "nv-ipam" + run_once: true + +- name: Create NVIDIA IPAM pools + kubernetes.core.k8s: + state: present + definition: + apiVersion: nv-ipam.nvidia.com/v1alpha1 + kind: IPPool + metadata: + name: "{{ item.ip_pool_name | default(item.res_name) }}" + namespace: "{{ nvidia_network_operator_namespace }}" + spec: + subnet: "{{ item.ip_addr }}" + perNodeBlockSize: "{{ nvidia_network_operator_ipam_per_node_block_size | int }}" + with_items: "{{ intf_resources }}" + when: nvidia_network_operator_ipam_type == "nv-ipam" + run_once: true + +- name: Create network node policy include_tasks: sriovnetworknodepolicy.yaml with_items: "{{ intf_resources }}" diff --git a/roles/nvidia-network-operator/tasks/sriovibnetwork.yaml b/roles/nvidia-network-operator/tasks/sriovibnetwork.yaml index fe6626fc7..ae08cde6b 100644 --- a/roles/nvidia-network-operator/tasks/sriovibnetwork.yaml +++ b/roles/nvidia-network-operator/tasks/sriovibnetwork.yaml @@ -9,14 +9,8 @@ networkNamespace: "default" ipam: | { - "type": "whereabouts", - "datastore": "kubernetes", - "kubernetes": { - "kubeconfig": "/etc/cni/net.d/whereabouts.d/whereabouts.kubeconfig" - }, - "range": "{{ item.ip_addr }}", - "log_file": "/var/log/whereabouts.log", - "log_level": "info" + "type": "{{ nvidia_network_operator_ipam_type }}", + "poolName": "{{ item.ip_pool_name | default(item.res_name) }}" } k8s: state: present diff --git a/roles/nvidia-network-operator/tasks/sriovnetworknodepolicy.yaml b/roles/nvidia-network-operator/tasks/sriovnetworknodepolicy.yaml index c41f16bcd..9cecfdbb1 100644 --- a/roles/nvidia-network-operator/tasks/sriovnetworknodepolicy.yaml +++ b/roles/nvidia-network-operator/tasks/sriovnetworknodepolicy.yaml @@ -7,7 +7,7 @@ deviceType: netdevice mtu: {{ mtu |int }} nodeSelector: - feature.node.kubernetes.io/network-sriov.capable: "true" + feature.node.kubernetes.io/pci-{{ vendor_id }}.present: "true" nicSelector: vendor: "{{ vendor_id }}" pfNames: ["{{ item.pf_name }}"] diff --git a/roles/nvidia-network-operator/templates/nicclusterpolicy.yaml b/roles/nvidia-network-operator/templates/nicclusterpolicy.yaml new file mode 100644 index 000000000..81834316b --- /dev/null +++ b/roles/nvidia-network-operator/templates/nicclusterpolicy.yaml @@ -0,0 +1,22 @@ +--- +apiVersion: mellanox.com/v1alpha1 +kind: NicClusterPolicy +metadata: + name: nic-cluster-policy +spec: + secondaryNetwork: + cniPlugins: + image: plugins + repository: nvcr.io/nvidia/mellanox + version: {{ nvidia_network_operator_image_tag }} + multus: + image: multus-cni + repository: nvcr.io/nvidia/mellanox + version: {{ nvidia_network_operator_image_tag }} +{% if nvidia_network_operator_ipam_type == "nv-ipam" %} + nvIpam: + image: nvidia-k8s-ipam + repository: nvcr.io/nvidia/mellanox + version: {{ nvidia_network_operator_image_tag }} + enableWebhook: false +{% endif %} diff --git a/roles/nvidia-network-operator/templates/values.yaml b/roles/nvidia-network-operator/templates/values.yaml index 9de7fbab2..4b63ac7b5 100644 --- a/roles/nvidia-network-operator/templates/values.yaml +++ b/roles/nvidia-network-operator/templates/values.yaml @@ -7,21 +7,3 @@ nfd: enabled: true sriovNetworkOperator: enabled: true - -# NicClusterPolicy CR values: -deployCR: true -ofedDriver: - deploy: false -rdmaSharedDevicePlugin: - deploy: false -sriovDevicePlugin: - deploy: false - -secondaryNetwork: - deploy: true - multus: - deploy: true - cniPlugins: - deploy: true - ipamPlugin: - deploy: true diff --git a/roles/nvidia-network-operator/vars/main.yaml b/roles/nvidia-network-operator/vars/main.yaml index 95a420b99..7be8ee253 100644 --- a/roles/nvidia-network-operator/vars/main.yaml +++ b/roles/nvidia-network-operator/vars/main.yaml @@ -7,6 +7,10 @@ # nvidia_network_operator_version: "26.1.1" +nvidia_network_operator_image_tag: "network-operator-v{{ nvidia_network_operator_version }}" +nvidia_network_operator_namespace: "network-operator" +nvidia_network_operator_ipam_type: "nv-ipam" +nvidia_network_operator_ipam_per_node_block_size: "{{ num_vf }}" nvidia_network_operator_url: "https://helm.ngc.nvidia.com/nvidia" mpi_operator_version: "v2beta1" mpi_raw_url: "https://raw.githubusercontent.com/kubeflow/mpi-operator/master/deploy/v2beta1" diff --git a/virtual/vars_files/virt_k8s.yml b/virtual/vars_files/virt_k8s.yml index c393e9a3e..089f52add 100644 --- a/virtual/vars_files/virt_k8s.yml +++ b/virtual/vars_files/virt_k8s.yml @@ -1,3 +1,3 @@ --- container_registry_persistence_enabled: false -rsyslog_client_tcp_host: "{{ groups['kube-master'][0] }}" +rsyslog_client_tcp_host: "{{ groups['kube_control_plane'][0] }}" diff --git a/virtual/virtual_inventory b/virtual/virtual_inventory index 71e0b72ba..d5a39bb8d 100644 --- a/virtual/virtual_inventory +++ b/virtual/virtual_inventory @@ -15,19 +15,19 @@ virtual-gpu01 ansible_host=10.0.0.6 ip=10.0.0.6 ###### # KUBERNETES ###### -[kube-master] +[kube_control_plane] virtual-mgmt01 [etcd] virtual-mgmt01 -[kube-node] +[kube_node] virtual-mgmt01 virtual-gpu01 -[k8s-cluster:children] -kube-master -kube-node +[k8s_cluster:children] +kube_control_plane +kube_node ###### # SLURM diff --git a/virtual/virtual_inventory_full b/virtual/virtual_inventory_full index f639e0e78..1f1c70b27 100644 --- a/virtual/virtual_inventory_full +++ b/virtual/virtual_inventory_full @@ -19,7 +19,7 @@ virtual-gpu02 ansible_host=10.0.0.7 ip=10.0.0.7 ###### # KUBERNETES ###### -[kube-master] +[kube_control_plane] virtual-mgmt01 virtual-mgmt02 virtual-mgmt03 @@ -29,16 +29,16 @@ virtual-mgmt01 virtual-mgmt02 virtual-mgmt03 -[kube-node] +[kube_node] virtual-mgmt01 virtual-mgmt02 virtual-mgmt03 virtual-gpu01 virtual-gpu02 -[k8s-cluster:children] -kube-master -kube-node +[k8s_cluster:children] +kube_control_plane +kube_node ###### # SLURM From e6ae743b983c389a3ed70a9da86a92e50d227e11 Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Fri, 15 May 2026 18:03:07 -0600 Subject: [PATCH 09/27] fix(k8s): run Helm installer with bash --- playbooks/k8s-cluster.yml | 2 +- scripts/k8s/install_helm.sh | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/playbooks/k8s-cluster.yml b/playbooks/k8s-cluster.yml index 742eed77d..983e9c750 100644 --- a/playbooks/k8s-cluster.yml +++ b/playbooks/k8s-cluster.yml @@ -282,7 +282,7 @@ ansible_become: no tasks: - name: Install Helm on admin node - command: "sh {{ playbook_dir }}/../scripts/k8s/install_helm.sh" + command: "bash {{ playbook_dir }}/../scripts/k8s/install_helm.sh" delegate_to: localhost - name: Globally update the deprecated "stable" helm repo command: "/usr/local/bin/helm repo add 'stable' 'https://charts.helm.sh/stable' --force-update" diff --git a/scripts/k8s/install_helm.sh b/scripts/k8s/install_helm.sh index 1efd217de..b3c2bb4e7 100755 --- a/scripts/k8s/install_helm.sh +++ b/scripts/k8s/install_helm.sh @@ -35,7 +35,7 @@ case "$ID" in ;; esac -helm_version=$(helm version --short) +helm_version=$(helm version --short 2>/dev/null || true) helm_min_installed=$(echo -e "${HELM_MINIMUM_VERSION}\n${helm_version}"| sort -V | head -n 1) if [ "${HELM_MINIMUM_VERSION}" != "${helm_min_installed}" ]; then if [ "${helm_version}" != "" ]; then @@ -49,5 +49,7 @@ if [ "${HELM_MINIMUM_VERSION}" != "${helm_min_installed}" ]; then HELM_INSTALL_DIR=${HELM_INSTALL_DIR} DESIRED_VERSION=v3.17.1 /var/tmp/get_helm.sh # Should match: config/group_vars/k8s-cluster.yml:helm_version: fi +sudo chmod 0755 "${HELM_INSTALL_DIR}/helm" + # Display the helm version for better debug helm version From f09134804e1d31e025454f51a965d5bc3f0f6119 Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Mon, 18 May 2026 20:13:45 -0600 Subject: [PATCH 10/27] Fix MAAS requirements and document staged upgrades --- config.example/group_vars/all.yml | 7 +++++++ docs/deepops/update-deepops.md | 2 ++ 2 files changed, 9 insertions(+) diff --git a/config.example/group_vars/all.yml b/config.example/group_vars/all.yml index a7eab54fa..b7de1ed73 100644 --- a/config.example/group_vars/all.yml +++ b/config.example/group_vars/all.yml @@ -263,6 +263,13 @@ maas_single_node_install: true maas_kvm_management: false +# Avoid installing python-libmaas via pip: it shadows the packaged MAAS CLI +# with /usr/local/bin/maas, which breaks DeepOps' MAAS operational workflow. +maas_python_reqs: + - jinja2 + - oauth + - pyyaml + ################################################################################ # NVIDIA Datacenter GPU Manager # ################################################################################ diff --git a/docs/deepops/update-deepops.md b/docs/deepops/update-deepops.md index 8d2bd73f2..a1f7af047 100644 --- a/docs/deepops/update-deepops.md +++ b/docs/deepops/update-deepops.md @@ -160,6 +160,8 @@ In particular, Additionally, please note that Kubespray can only upgrade between one minor version of Kubernetes at a time. This means that you may need to upgrade multiple times between your current version and your desired version of Kubernetes. +Other cluster components managed by Kubespray may have similar staged-upgrade requirements. +For example, the network plugin version installed by an older DeepOps release may need to be upgraded through an intermediate DeepOps/Kubespray release before a newer Kubespray release will accept it. For example, to upgrade from Kubernetes version 1.19.9 and 1.21.1, you might use a workflow like this: From 8d4797874ab339bca77041cb288b46144791e4cb Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Tue, 26 May 2026 09:35:17 -0600 Subject: [PATCH 11/27] release: 26.05 notes and README tag --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4476ffc6e..6f4dbdb42 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ The DeepOps project encapsulates best practices in the deployment of GPU server - An existing cluster that needs a resource manager / batch scheduler, where DeepOps is used to install Slurm or Kubernetes - A single machine where no scheduler is desired, only NVIDIA drivers, Docker, and the NVIDIA Container Runtime -Latest release: [DeepOps 23.08 Release](https://github.com/NVIDIA/deepops/releases/tag/23.08) +Latest release: [DeepOps 26.05 Release](https://github.com/NVIDIA/deepops/releases/tag/26.05) It is recommended to use the latest release branch for stable code (linked above). All development takes place on the master branch, which is generally [functional](docs/deepops/testing.md) but may change significantly between releases. From e6bf1f6e83a8ff33a490220e717fff6bd9f76aa5 Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Wed, 27 May 2026 09:21:35 -0600 Subject: [PATCH 12/27] feat(dgx): update DGX software stack role --- README.md | 12 +- docs/deepops/dgx-software-stack.md | 131 +++++++ docs/deepops/testing.md | 4 +- roles/nvidia-dgx/tasks/main.yml | 4 +- roles/nvidia-dgx/tasks/redhat-el8-plus.yml | 139 +++++++ roles/nvidia-dgx/tasks/redhat-legacy-el7.yml | 105 +++++ roles/nvidia-dgx/tasks/redhat.yml | 111 +----- roles/nvidia-dgx/tasks/ubuntu-22.04.yml | 124 ++++++ roles/nvidia-dgx/tasks/ubuntu-24.04.yml | 92 +++++ roles/nvidia-dgx/tasks/ubuntu-legacy.yml | 384 ++++++++++++++++++ roles/nvidia-dgx/tasks/ubuntu.yml | 391 +------------------ roles/nvidia-dgx/vars/redhat.yml | 112 ++++++ roles/nvidia-dgx/vars/ubuntu-22.04.yml | 115 ++++++ roles/nvidia-dgx/vars/ubuntu-24.04.yml | 102 +++++ 14 files changed, 1338 insertions(+), 488 deletions(-) create mode 100644 docs/deepops/dgx-software-stack.md create mode 100644 roles/nvidia-dgx/tasks/redhat-el8-plus.yml create mode 100644 roles/nvidia-dgx/tasks/redhat-legacy-el7.yml create mode 100644 roles/nvidia-dgx/tasks/ubuntu-22.04.yml create mode 100644 roles/nvidia-dgx/tasks/ubuntu-24.04.yml create mode 100644 roles/nvidia-dgx/tasks/ubuntu-legacy.yml create mode 100644 roles/nvidia-dgx/vars/ubuntu-22.04.yml create mode 100644 roles/nvidia-dgx/vars/ubuntu-24.04.yml diff --git a/README.md b/README.md index 6f4dbdb42..68df95a20 100644 --- a/README.md +++ b/README.md @@ -38,19 +38,21 @@ It is recommended to use the latest release branch for stable code (linked above The provisioning system is used to orchestrate the running of all playbooks and one will be needed when instantiating Kubernetes or Slurm clusters. Supported operating systems which are tested and supported include: -- NVIDIA DGX OS 4, 5 -- Ubuntu 18.04 LTS, 20.04, 22.04 LTS +- NVIDIA DGX OS 4, 5, 6, 7 +- Ubuntu 18.04 LTS, 20.04, 22.04 LTS, 24.04 LTS - CentOS 7, 8 ### Cluster System The cluster nodes will follow the requirements described by Slurm or Kubernetes. You may also use a cluster node as a provisioning system but it is not required. -- NVIDIA DGX OS 4, 5 -- Ubuntu 18.04 LTS, 20.04, 22.04 LTS +- NVIDIA DGX OS 4, 5, 6, 7 +- Ubuntu 18.04 LTS, 20.04, 22.04 LTS, 24.04 LTS - CentOS 7, 8 +- Red Hat Enterprise Linux / Rocky Linux 8 and 9 for the DGX software stack through the `nvidia-dgx` role You may also install a supported operating system on all servers via a 3rd-party solution (i.e. [MAAS](https://maas.io/), [Foreman](https://www.theforeman.org/)) or utilize the provided [OS install container](docs/pxe/minimal-pxe-container.md). +For DGX platform software installation on top of vanilla Ubuntu or Red Hat family operating systems, see the [DGX software stack role guide](docs/deepops/dgx-software-stack.md). ### Kubernetes @@ -77,7 +79,7 @@ For more information on Slurm in general, refer to the [official Slurm docs](htt [NVIDIA Bright Cluster Manager](https://www.brightcomputing.com/brightclustermanager) is recommended as an enterprise solution which enables managing multiple workload managers within a single cluster, including Kubernetes, Slurm, Univa Grid Engine, and PBS Pro. **DeepOps does not test or support a configuration where nodes have a heterogenous OS running.** -Additional modifications are needed if you plan to use unsupported operating systems such as RHEL. +The `nvidia-dgx` role can install NVIDIA DGX platform software on supported DGX systems running Red Hat Enterprise Linux / Rocky Linux 8 or 9; broader Kubernetes or Slurm cluster support on RHEL still requires site-specific validation. ### Virtual diff --git a/docs/deepops/dgx-software-stack.md b/docs/deepops/dgx-software-stack.md new file mode 100644 index 000000000..9164c1a0d --- /dev/null +++ b/docs/deepops/dgx-software-stack.md @@ -0,0 +1,131 @@ +# DGX Software Stack Role + +The `nvidia-dgx` role installs NVIDIA DGX platform software on supported DGX +systems after a base operating system has been installed. + +This role is intended for DGX hardware only. It checks the system product name +and stops on non-DGX systems. + +## Supported Paths + +The role has two implementation paths: + +| Base OS | DGX software path | Notes | +| ------- | ----------------- | ----- | +| Ubuntu 18.04 | DGX OS 4 legacy packages | Existing legacy role path. | +| Ubuntu 20.04 | DGX OS 5 legacy packages | Existing legacy role path. | +| Ubuntu 22.04 | DGX OS 6 software stack | Uses the official DGX OS 6 repository archive and system-specific packages. | +| Ubuntu 24.04 | DGX OS 7 software stack | Uses the official DGX OS 7 repository archive and unified `nvidia-system-*` packages. | +| Red Hat Enterprise Linux 7 | Legacy DGX EL7 packages | Existing legacy role path. | +| Red Hat Enterprise Linux 8 / Rocky Linux 8 | DGX Software for RHEL 8 | Uses the official NVIDIA repository setup RPM and DGX configuration groups. | +| Red Hat Enterprise Linux 9 / Rocky Linux 9 | DGX Software for RHEL 9 | Uses the official NVIDIA repository setup RPM and DGX configuration groups. | + +The EL8 work addresses GitHub issue +[#1120](https://github.com/NVIDIA/deepops/issues/1120). + +## Official References + +- [Installing DGX Software on Ubuntu](https://docs.nvidia.com/dgx/dgx-os-6-user-guide/installing_on_ubuntu.html) +- [Customizing Ubuntu Installation with DGX Software](https://docs.nvidia.com/dgx/dgx-os-7-user-guide/installing_on_ubuntu.html) +- [DGX Software for Red Hat Enterprise Linux 8 Installation Guide](https://docs.nvidia.com/dgx/dgx-rhel8-install-guide/index.html) +- [DGX Software for Red Hat Enterprise Linux 8 Release Notes](https://docs.nvidia.com/dgx/dgx-rhel8-sw-release-notes/index.html) +- [DGX Software for Red Hat Enterprise Linux 9 User Guide](https://docs.nvidia.com/dgx/dgx-el9-user-guide/index.html) + +## Ubuntu 22.04 / DGX OS 6 + +The role follows the DGX OS 6 guide: + +1. Install the DGX repository files from + `https://repo.download.nvidia.com/baseos/ubuntu/jammy/dgx-repo-files.tgz`. +2. Install the system-specific DGX configuration and tools packages. +3. Install `linux-tools-nvidia` and `nvidia-peermem-loader`. +4. Optionally install the NVIDIA driver, Docker/NVIDIA Container Toolkit, NVSM, + serial-over-LAN, logrotate, and additional DGX OS administration/development + packages. + +The default driver branch is `550`, matching the DGX OS 6 examples. Override it +when needed: + +```yaml +dgx_os6_driver_branch: "580" +``` + +Disruptive package upgrades are opt-in: + +```yaml +dgx_os6_upgrade_packages: true +``` + +## Red Hat Enterprise Linux 8 and 9 + +The role follows the official Red Hat DGX software guides: + +1. Optionally enable the required Red Hat subscription repositories on RHEL. + This is skipped automatically on Rocky Linux. +2. Install the NVIDIA DGX repository setup RPM for EL8 or EL9. +3. Install the DGX configuration group for the detected DGX platform. +4. Optionally install the NVIDIA driver module and support packages. +5. Optionally install Docker CE and the NVIDIA Container Runtime group. + +The default driver stream follows the official examples: `525` on most EL8 +systems, `535-dkms` on EL8 DGX H100, and `580` on EL9. EL9 NVSwitch systems +install the open-kernel-module stream by default. Override the branch when a +validated DGX release note calls for another stream: + +```yaml +dgx_redhat_driver_branch: "580" +``` + +RHEL subscription repository management is enabled by default only when +`ansible_distribution == 'RedHat'`. Disable it if subscriptions are managed +outside DeepOps: + +```yaml +dgx_redhat_manage_subscription_repos: false +``` + +Disruptive `dnf update --nobest` behavior is opt-in: + +```yaml +dgx_redhat_upgrade_packages: true +``` + +## Ubuntu 24.04 / DGX OS 7 + +The role follows the DGX OS 7 guide: + +1. Install the architecture-specific DGX OS 7 repository archive from + `https://repo.download.nvidia.com/baseos/ubuntu/noble/`. +2. Install the unified DGX OS 7 metapackages: `nvidia-system-core`, + `nvidia-system-utils`, and `nvidia-system-extra`. +3. Install `nvidia-system-station` for DGX Station and DGX Spark systems. +4. Install kernel tools and `nvidia-peermem-loader`. +5. Optionally install the Release 580 open GPU kernel module driver packages, + including Fabric Manager, NVLSM/NVSDM, or IMEX packages for the DGX platform + that requires them. + +Disruptive package upgrades are opt-in: + +```yaml +dgx_os7_upgrade_packages: true +``` + +## Validation + +Full validation requires real DGX hardware and access to NVIDIA/OS package +repositories. At minimum, run syntax validation before opening a PR: + +```bash +ansible-playbook --syntax-check playbooks/nvidia-dgx/nvidia-dgx.yml +``` + +On hardware, validate the role with the target OS and DGX model, reboot if the +driver was installed, then verify: + +```bash +nvidia-smi +sudo docker run --gpus=all --rm nvcr.io/nvidia/cuda:12.3.2-base-ubuntu22.04 nvidia-smi +``` + +Use the RHEL UBI CUDA image from the official guide when validating the RHEL +path. diff --git a/docs/deepops/testing.md b/docs/deepops/testing.md index e0d0b6765..7f2b1731a 100644 --- a/docs/deepops/testing.md +++ b/docs/deepops/testing.md @@ -53,8 +53,8 @@ A short description of the nightly testing is outlined below. The full suit of t | Ubuntu 20.04 | | x | x | | | CentOS 7 | | x | x | | | CentOS | | | x | | -| DGX OS | | | | No automated testing support | -| RHEL | | | | No testing support | +| DGX OS | | | | Syntax-checked only; full validation requires DGX hardware | +| RHEL | | | | DGX software-stack role syntax-checked only; full validation requires DGX hardware and subscriptions | | 1 mgmt node | x | x | | | | 3 mgmt nodes | | | x | | | 1 gpu node | x | x | | | diff --git a/roles/nvidia-dgx/tasks/main.yml b/roles/nvidia-dgx/tasks/main.yml index 64198524e..e95f03fa2 100644 --- a/roles/nvidia-dgx/tasks/main.yml +++ b/roles/nvidia-dgx/tasks/main.yml @@ -4,12 +4,12 @@ msg: "Role supports DGX systems only" when: ansible_product_name is not search("DGX") -- name: Ubuntu tasks for DGX OS 4/5 +- name: Ubuntu tasks for DGX Software Stack include_tasks: ubuntu.yml when: - ansible_distribution == 'Ubuntu' -- name: redhat family tasks +- name: Red Hat family tasks for DGX Software Stack include_tasks: redhat.yml when: ansible_os_family == 'RedHat' diff --git a/roles/nvidia-dgx/tasks/redhat-el8-plus.yml b/roles/nvidia-dgx/tasks/redhat-el8-plus.yml new file mode 100644 index 000000000..ce4b1997c --- /dev/null +++ b/roles/nvidia-dgx/tasks/redhat-el8-plus.yml @@ -0,0 +1,139 @@ +--- +- name: Determine Red Hat DGX platform package set + set_fact: + dgx_redhat_platform: "{{ item }}" + loop: "{{ dgx_redhat_platforms }}" + when: + - ansible_product_name is search(item.match) + - ansible_distribution_major_version in item.supported_major_versions + +- name: Fail if Red Hat DGX platform package set is unknown + fail: + msg: "Unsupported DGX model for EL{{ ansible_distribution_major_version }} DGX role path: {{ ansible_product_name }}" + when: dgx_redhat_platform is not defined + +- name: Enable Red Hat subscription repositories for DGX Software Stack + command: "subscription-manager repos --enable={{ item }}" + loop: "{{ dgx_redhat_subscription_repos[ansible_distribution_major_version] }}" + changed_when: false + when: + - dgx_redhat_manage_subscription_repos + - ansible_distribution == 'RedHat' + +- name: Enable Rocky Linux CRB repository for DGX Software Stack + command: dnf config-manager --set-enabled crb + changed_when: false + when: + - ansible_distribution == 'Rocky' + - ansible_distribution_major_version == '9' + +- name: Install NVIDIA DGX repository setup package + dnf: + name: "{{ dgx_redhat_repo_setup_rpms[ansible_distribution_major_version] }}" + state: present + disable_gpg_check: yes + +- name: Upgrade Red Hat DGX Software Stack packages + dnf: + name: "*" + state: latest + nobest: yes + when: dgx_redhat_upgrade_packages + tags: skip_ansible_lint + +- name: Install EL9 kernel development packages for DGX driver builds + dnf: + name: + - "kernel-devel-{{ ansible_kernel }}" + - "kernel-headers-{{ ansible_kernel }}" + state: present + when: ansible_distribution_major_version == '9' + +- name: Install Red Hat DGX configuration group + dnf: + name: "@{{ dgx_redhat_platform.configuration_group }}" + state: present + +- name: Configure Red Hat DGX driver facts + set_fact: + dgx_redhat_driver_profile: "{{ 'fm' if dgx_redhat_platform.nvswitch else 'default' }}" + dgx_redhat_driver_stream: "{{ dgx_redhat_driver_branch }}{{ '-open' if dgx_redhat_use_open_kernel_modules else '' }}" + +- name: Configure Red Hat DGX driver profile list + set_fact: + dgx_redhat_driver_profiles: >- + {{ + [dgx_redhat_driver_profile, 'src'] + if ansible_distribution_major_version == '8' + and not dgx_redhat_use_open_kernel_modules + and 'dkms' not in dgx_redhat_driver_stream + else [dgx_redhat_driver_profile] + }} + +- name: Configure Red Hat DGX driver module specification + set_fact: + dgx_redhat_driver_module_spec: >- + nvidia-driver:{{ dgx_redhat_driver_stream }}/{{ '{' ~ (dgx_redhat_driver_profiles | join(',')) ~ '}' if (dgx_redhat_driver_profiles | length) > 1 else dgx_redhat_driver_profiles[0] }} + +- name: Install Red Hat DGX NVIDIA driver module + command: "dnf module install --nobest -y {{ dgx_redhat_driver_module_spec }}" + register: dgx_redhat_driver_module_install + changed_when: "'Nothing to do' not in dgx_redhat_driver_module_install.stdout" + notify: reboot after driver install + when: dgx_redhat_install_driver + +- name: Mark Red Hat DGX driver install state for reboot handler + set_fact: + install_driver: + changed: "{{ dgx_redhat_driver_module_install.changed | default(false) }}" + when: dgx_redhat_install_driver + +- name: Install Red Hat DGX driver support packages + dnf: + name: "{{ dgx_redhat_driver_support_packages + (dgx_redhat_fabricmanager_packages if dgx_redhat_platform.nvswitch else dgx_redhat_non_nvswitch_packages) + (dgx_redhat_nvlink5_packages if dgx_redhat_platform.nvlink5 else []) }}" + state: present + when: dgx_redhat_install_driver + +- name: Install Red Hat DGX Station extra driver packages + dnf: + name: "{{ dgx_redhat_station_packages }}" + state: present + when: + - dgx_redhat_install_driver + - dgx_redhat_platform.station + +- name: Install Red Hat DGX Docker CE + dnf: + name: docker-ce + state: present + allowerasing: yes + when: + - dgx_redhat_install_container_runtime + - dgx_redhat_install_docker_ce + +- name: Install Red Hat DGX NVIDIA Container Runtime group + dnf: + name: "@NVIDIA Container Runtime" + state: present + allowerasing: yes + notify: + - restart docker + when: dgx_redhat_install_container_runtime + +- name: Install Red Hat DGX optional cachefilesd configuration + dnf: + name: nvidia-conf-cachefilesd + state: present + when: dgx_redhat_install_cachefilesd + +- name: Populate Red Hat DGX service facts + service_facts: + +- name: Enable available Red Hat DGX services + systemd: + name: "{{ item }}" + state: started + enabled: yes + daemon_reload: yes + loop: "{{ dgx_redhat_services }}" + when: item in ansible_facts.services diff --git a/roles/nvidia-dgx/tasks/redhat-legacy-el7.yml b/roles/nvidia-dgx/tasks/redhat-legacy-el7.yml new file mode 100644 index 000000000..e1ab122f8 --- /dev/null +++ b/roles/nvidia-dgx/tasks/redhat-legacy-el7.yml @@ -0,0 +1,105 @@ +--- +- name: include os specific variables + include_vars: redhat.yml + +- name: trust GPG key for EPEL + rpm_key: + key: "{{ epel_key_url }}" + state: present + +- name: Add epel repo + yum: + name: + - "{{ epel_package }}" + state: latest + +- name: Add DGX repo + yum_repository: + name: nvidia-dgx-7 + description: NVIDIA DGX EL7 + gpgkey: "{{ nvidia_dgx_rhel_gpgkey }}" + baseurl: "{{ nvidia_dgx_rhel_baseurl }}" + +- name: Install packages + yum: + name: "@DGX-1 Configurations" + state: present + update_cache: yes + when: ansible_product_name is search("DGX-1") + +- name: Install packages + yum: + name: "@DGX-2 Configurations" + state: present + update_cache: yes + when: ansible_product_name is search("DGX-2") + +- name: Install extra packages + yum: + name: "{{ dgx_extra_packages }}" + state: present + update_cache: yes + +- name: check kernel versions + yum: + list: kernel + register: yum_list + +- name: register installed kernel version + debug: + msg: "{{ yum_list.results | selectattr('yumstate', 'equalto', 'installed') | list }}" + register: kernel_version + +- name: check kernel-headers versions + yum: + list: kernel-headers + register: yum_list + +- name: register installed kernel-headers version + debug: + msg: "{{ yum_list.results | selectattr('yumstate', 'equalto', 'installed') | list }}" + register: kernel_headers_version + +- name: update kernel if headers don't match + yum: + name: + - kernel + - kernel-tools + - kernel-tools-libs + - kernel-devel + - kernel-debug-devel + - kernel-headers + state: latest + register: kernel_update + when: kernel_version.msg[0].release != kernel_headers_version.msg[0].release + tags: skip_ansible_lint + +- name: Install driver packages + package: + name: "{{ item }}" + with_items: + - cuda-drivers + - cuda-drivers-diagnostic + - dgx-persistence-mode + register: install_driver + notify: reboot after driver install + +- name: Install Docker and the NVIDIA container runtime + yum: + name: "{{ item }}" + state: present + update_cache: yes + with_items: + - docker + - "@NVIDIA container runtime" + notify: + - restart docker + +- name: Install DGX System Management Tools + yum: + name: "{{ item }}" + state: present + update_cache: yes + with_items: + - python36 + - "@DGX System Management" diff --git a/roles/nvidia-dgx/tasks/redhat.yml b/roles/nvidia-dgx/tasks/redhat.yml index 61ead034a..e4c2794a4 100644 --- a/roles/nvidia-dgx/tasks/redhat.yml +++ b/roles/nvidia-dgx/tasks/redhat.yml @@ -1,105 +1,16 @@ --- -- name: include os specific variables +- name: Include Red Hat DGX variables include_vars: redhat.yml -- name: trust GPG key for EPEL - rpm_key: - key: "{{ epel_key_url }}" - state: present +- name: Check for supported Red Hat family DGX Software Stack version + fail: + msg: "Role supports Red Hat family major versions 7, 8, and 9 only" + when: ansible_distribution_major_version not in ['7', '8', '9'] -- name: Add epel repo - yum: - name: - - "{{ epel_package }}" - state: latest +- name: Run legacy EL7 DGX tasks + include_tasks: redhat-legacy-el7.yml + when: ansible_distribution_major_version == '7' -- name: Add DGX repo - yum_repository: - name: nvidia-dgx-7 - description: NVIDIA DGX EL7 - gpgkey: "{{ nvidia_dgx_rhel_gpgkey }}" - baseurl: "{{ nvidia_dgx_rhel_baseurl }}" - -- name: Install packages - yum: - name: "@DGX-1 Configurations" - state: present - update_cache: yes - when: ansible_product_name is search("DGX-1") - -- name: Install packages - yum: - name: "@DGX-2 Configurations" - state: present - update_cache: yes - when: ansible_product_name is search("DGX-2") - -- name: Install extra packages - yum: - name: "{{ dgx_extra_packages }}" - state: present - update_cache: yes - -- name: check kernel versions - yum: - list: kernel - register: yum_list - -- name: register installed kernel version - debug: - msg: "{{ yum_list.results | selectattr('yumstate', 'equalto', 'installed') | list }}" - register: kernel_version - -- name: check kernel-headers versions - yum: - list: kernel-headers - register: yum_list - -- name: register installed kernel-headers version - debug: - msg: "{{ yum_list.results | selectattr('yumstate', 'equalto', 'installed') | list }}" - register: kernel_headers_version - -- name: update kernel if headers don't match - yum: - name: - - kernel - - kernel-tools - - kernel-tools-libs - - kernel-devel - - kernel-debug-devel - - kernel-headers - state: latest - register: kernel_update - when: kernel_version.msg[0].release != kernel_headers_version.msg[0].release - tags: skip_ansible_lint - -- name: Install driver packages - package: - name: "{{ item }}" - with_items: - - cuda-drivers - - cuda-drivers-diagnostic - - dgx-persistence-mode - register: install_driver - notify: reboot after driver install - -- name: Install Docker and the NVIDIA container runtime - yum: - name: "{{ item }}" - state: present - update_cache: yes - with_items: - - docker - - "@NVIDIA container runtime" - notify: - - restart docker - -- name: Install DGX System Management Tools - yum: - name: "{{ item }}" - state: present - update_cache: yes - with_items: - - python36 - - "@DGX System Management" +- name: Run EL8 and EL9 DGX Software Stack tasks + include_tasks: redhat-el8-plus.yml + when: ansible_distribution_major_version in ['8', '9'] diff --git a/roles/nvidia-dgx/tasks/ubuntu-22.04.yml b/roles/nvidia-dgx/tasks/ubuntu-22.04.yml new file mode 100644 index 000000000..2240b4338 --- /dev/null +++ b/roles/nvidia-dgx/tasks/ubuntu-22.04.yml @@ -0,0 +1,124 @@ +--- +- name: Include DGX OS 6 variables + include_vars: ubuntu-22.04.yml + +- name: Determine DGX OS 6 platform package set + set_fact: + dgx_os6_platform: "{{ item }}" + loop: "{{ dgx_os6_platforms }}" + when: ansible_product_name is search(item.match) + +- name: Fail if DGX OS 6 platform package set is unknown + fail: + msg: "Unsupported DGX model for Ubuntu 22.04 DGX OS 6 role path: {{ ansible_product_name }}" + when: dgx_os6_platform is not defined + +- name: Create temporary DGX OS 6 repository archive path + tempfile: + state: file + suffix: -dgx-repo-files.tgz + register: dgx_os6_repo_archive + +- name: Download DGX OS 6 repository files + get_url: + url: "{{ dgx_os6_repo_files_url }}" + dest: "{{ dgx_os6_repo_archive.path }}" + mode: "0644" + +- name: Install DGX OS 6 repository files + unarchive: + src: "{{ dgx_os6_repo_archive.path }}" + dest: / + remote_src: true + +- name: Remove temporary DGX OS 6 repository archive + file: + path: "{{ dgx_os6_repo_archive.path }}" + state: absent + +- name: Update apt cache for DGX OS 6 repositories + apt: + update_cache: yes + +- name: Upgrade packages for DGX OS 6 + apt: + upgrade: dist + when: dgx_os6_upgrade_packages + +- name: Install DGX OS 6 system configurations and tools + apt: + name: "{{ dgx_os6_platform.config_packages + dgx_os6_common_packages }}" + state: present + update_cache: yes + +- name: Install DGX OS 6 NVIDIA driver packages + apt: + name: "{{ dgx_os6_driver_packages + (dgx_os6_fabricmanager_packages if dgx_os6_platform.nvswitch else []) }}" + state: present + update_cache: yes + register: install_driver + notify: reboot after driver install + when: dgx_os6_install_driver + +- name: Install DGX OS 6 Docker and NVIDIA container toolkit + apt: + name: "{{ dgx_os6_container_packages }}" + state: present + update_cache: yes + notify: + - restart docker + when: dgx_os6_install_container_runtime + +- name: Install DGX OS 6 NVSM packages + apt: + name: "{{ dgx_os6_nvsm_packages }}" + state: present + update_cache: yes + when: dgx_os6_install_nvsm + +- name: Install additional DGX OS 6 administration packages + apt: + name: "{{ dgx_os6_admin_packages }}" + state: present + update_cache: yes + when: dgx_os6_install_admin_packages + +- name: Install additional DGX OS 6 development packages + apt: + name: "{{ dgx_os6_development_packages }}" + state: present + update_cache: yes + when: dgx_os6_install_development_packages + +- name: Disable unattended upgrades for DGX OS 6 + apt: + name: unattended-upgrades + state: absent + purge: yes + when: dgx_os6_disable_unattended_upgrades + +- name: Install DGX OS 6 serial-over-LAN console package + apt: + name: nvidia-ipmisol + state: present + update_cache: yes + when: dgx_os6_install_ipmisol + +- name: Install DGX OS 6 logrotate package + apt: + name: nvidia-logrotate + state: present + update_cache: yes + when: dgx_os6_install_logrotate + +- name: Populate DGX OS 6 service facts + service_facts: + +- name: Enable available DGX OS 6 services + systemd: + name: "{{ item }}" + state: started + enabled: yes + daemon_reload: yes + loop: "{{ dgx_os6_services }}" + when: item in ansible_facts.services diff --git a/roles/nvidia-dgx/tasks/ubuntu-24.04.yml b/roles/nvidia-dgx/tasks/ubuntu-24.04.yml new file mode 100644 index 000000000..b2c8161a1 --- /dev/null +++ b/roles/nvidia-dgx/tasks/ubuntu-24.04.yml @@ -0,0 +1,92 @@ +--- +- name: Include DGX OS 7 variables + include_vars: ubuntu-24.04.yml + +- name: Determine DGX OS 7 platform package set + set_fact: + dgx_os7_platform: "{{ item }}" + loop: "{{ dgx_os7_platforms }}" + when: ansible_product_name is search(item.match) + +- name: Fail if DGX OS 7 platform package set is unknown + fail: + msg: "Unsupported DGX model for Ubuntu 24.04 DGX OS 7 role path: {{ ansible_product_name }}" + when: dgx_os7_platform is not defined + +- name: Create temporary DGX OS 7 repository archive path + tempfile: + state: file + suffix: -dgx-repo-files.tgz + register: dgx_os7_repo_archive + +- name: Download DGX OS 7 repository files + get_url: + url: "{{ dgx_os7_repo_files_url }}" + dest: "{{ dgx_os7_repo_archive.path }}" + mode: "0644" + +- name: Install DGX OS 7 repository files + unarchive: + src: "{{ dgx_os7_repo_archive.path }}" + dest: / + remote_src: true + +- name: Remove temporary DGX OS 7 repository archive + file: + path: "{{ dgx_os7_repo_archive.path }}" + state: absent + +- name: Update apt cache for DGX OS 7 repositories + apt: + update_cache: yes + +- name: Upgrade packages for DGX OS 7 + apt: + upgrade: dist + when: dgx_os7_upgrade_packages + +- name: Install DGX OS 7 system packages + apt: + name: "{{ dgx_os7_system_packages + ([dgx_os7_station_package] if dgx_os7_platform.station else []) }}" + state: present + update_cache: yes + +- name: Install DGX OS 7 kernel tools and peer memory loader + apt: + name: + - "{{ dgx_os7_platform.linux_tools_package }}" + - nvidia-peermem-loader + state: present + update_cache: yes + +- name: Install DGX OS 7 NVIDIA driver packages + apt: + name: "{{ dgx_os7_driver_packages + (dgx_os7_fabricmanager_packages if dgx_os7_platform.nvswitch else []) + (dgx_os7_nvlink5_packages if dgx_os7_platform.nvlink5 else []) + (dgx_os7_imex_packages if dgx_os7_platform.imex else []) }}" + state: present + update_cache: yes + register: install_driver + notify: reboot after driver install + when: dgx_os7_install_driver + +- name: Disable unattended upgrades for DGX OS 7 + apt: + name: unattended-upgrades + state: absent + purge: yes + when: dgx_os7_disable_unattended_upgrades + +- name: Populate DGX OS 7 service facts + service_facts: + +- name: Build DGX OS 7 service list + set_fact: + dgx_os7_enabled_services: "{{ dgx_os7_base_services + (dgx_os7_fabricmanager_services if dgx_os7_platform.nvswitch else []) + (dgx_os7_imex_services if dgx_os7_platform.imex else []) }}" + +- name: Enable available DGX OS 7 services + systemd: + name: "{{ item }}" + state: started + enabled: yes + daemon_reload: yes + loop: "{{ dgx_os7_enabled_services }}" + when: item in ansible_facts.services diff --git a/roles/nvidia-dgx/tasks/ubuntu-legacy.yml b/roles/nvidia-dgx/tasks/ubuntu-legacy.yml new file mode 100644 index 000000000..cdfb95f96 --- /dev/null +++ b/roles/nvidia-dgx/tasks/ubuntu-legacy.yml @@ -0,0 +1,384 @@ +--- +- name: Check for Ubuntu version + fail: + msg: "Role supports Ubuntu 18.04 and 20.04 only" + when: + - ansible_distribution_version != "20.04" + - ansible_distribution_version != "18.04" + +- name: include 18.04 specific variables for DGX OS 4 + include_vars: ubuntu-18.04.yml + when: ansible_distribution_version == "18.04" + +- name: include DGXA100 specific variables for DGX OS 4.99 + include_vars: dgxa100.yml + when: + - ansible_distribution_version == "18.04" + - ansible_product_name is search("DGXA100") + +- name: include 20.04 specific variables for DGX OS 5 + include_vars: ubuntu-20.04.yml + when: ansible_distribution_version == "20.04" + +- name: install dmidecode + apt: + name: dmidecode + state: present + update_cache: yes + +# Housekeeping +- name: register DGX product name + command: "dmidecode --string system-product-name" + register: dgx_name + +- name: register DGX-1 serial number + command: "dmidecode --string system-serial-number" + register: dgx1_serial + when: ansible_product_name is search("DGX-1") + +- name: register DGX-2 serial number + command: "dmidecode --string chassis-serial-number" + register: dgx2_serial + when: ansible_product_name is search("DGX-2") + +- name: register DGXA100 serial number + command: "dmidecode --string chassis-serial-number" + register: dgxa100_serial + when: ansible_product_name is search("DGXA100") + +- name: figure out which serial number we ended up with + set_fact: + dgx_serial: "{{ dgx1_serial.stdout }}" + when: dgx1_serial.skipped is not defined + +- name: figure out which serial number we ended up with + set_fact: + dgx_serial: "{{ dgx2_serial.stdout }}" + when: dgx2_serial.skipped is not defined + +- name: figure out which serial number we ended up with + set_fact: + dgx_serial: "{{ dgxa100_serial.stdout }}" + when: dgxa100_serial.skipped is not defined + +- name: fail if we don't recognize the DGX system + fail: + msg: "Unknown DGX model: {{ ansible_product_name }}" + when: dgx_serial is undefined + +- name: update dgx platform file + blockinfile: + path: /etc/dgx-release + create: yes + block: | + DGX_NAME="DGX Server" + DGX_PRETTY_NAME="NVIDIA DGX Server" + DGX_PLATFORM="DGX Server for {{ dgx_name.stdout }}" + DGX_SERIAL_NUMBER="{{ dgx_serial }}" + +- name: update dgx platform file with additional info for DGX OS 4 + blockinfile: + path: /etc/dgx-release + create: yes + block: | + DGX_SWBUILD_DATE="{{ DGX_SWBUILD_DATE }}" + DGX_SWBUILD_VERSION="{{ DGX_SWBUILD_VERSION }}" + DGX_COMMIT_ID="{{ DGX_COMMIT_ID }}" + when: ansible_distribution_version == "18.04" + +# Repos and installs +- name: remove ubuntu nvidia driver ppa if installed + apt_repository: + repo: ppa:graphics-drivers/ppa + state: absent + +- name: remove packages when using Bright Computing source image + apt: + name: "{{ item }}" + state: absent + purge: yes + with_items: + - cuda-dcgm + - cuda-driver + - libumad2sim0 + - libsim-utils + - ibutils + +- name: remove apt mirror when using Bright Computing source image + replace: + path: /etc/apt/sources.list + regexp: 'nl\.' + +- name: unload drivers when using Bright Computing source image + modprobe: + name: "{{ item }}" + state: absent + with_items: + - ib_srp + - ib_iser + - rdma_ucm + - rpcrdma + - rdma_cm + +- name: add DGX repo keys for DGX OS 4 + apt: + deb: "{{ nvidia_dgx_ubuntu_gpgkey }}" + when: ansible_distribution_version == "18.04" + +- name: add DGX repo key for DGX OS 5 + apt_key: + url: "{{ nvidia_dgx_os5_ubuntu_gpgkey }}" + state: present + when: ansible_distribution_version == "20.04" + +- name: add DGX repo for DGX OS 4 + template: + src: dgx.list.j2 + dest: /etc/apt/sources.list.d/dgx.list + mode: 0644 + when: ansible_distribution_version == "18.04" + +- name: add DGX repo for DGX OS 5 + template: + src: dgxos5.list.j2 + dest: /etc/apt/sources.list.d/dgx-temp.list + mode: 0644 + when: ansible_distribution_version == "20.04" + +- name: update apt cache + apt: + update_cache: yes + +- name: add DGX cuda 10.1 repo for DGX OS 4 + apt: + name: dgx-bionic-r418+cuda10.1-repo + update_cache: yes + dpkg_options: "force-confdef,force-confold" + when: ansible_distribution_version == "18.04" + +- name: add DGX cuda 11.0 repo for DGX OS 4 + apt: + name: dgx-bionic-r450+cuda11.0-repo + update_cache: yes + dpkg_options: "force-confdef,force-confold" + when: ansible_distribution_version == "18.04" + +- name: add DGXA100 repo for DGX OS 4 + apt: + name: dgx-bionic-4.99-repo + update_cache: yes + dpkg_options: "force-confdef,force-confold" + when: + - ansible_distribution_version == "18.04" + - ansible_product_name is search("DGXA100") + +- name: install prerequisites for DGX OS 5 + apt: + name: "{{ item }}" + state: present + update_cache: yes + with_items: + - "{{ G_PREREQS }}" + - "{{ G_SETUP_PKGS }}" + when: ansible_distribution_version == "20.04" + +- name: install DGX repo package for DGX OS 5 + apt: + name: "{{ item }}" + update_cache: yes + dpkg_options: "force-confdef,force-confold" + with_items: + - "{{ G_DGX_REPO_PKG }}" + when: ansible_distribution_version == "20.04" + +- name: remove temporary repo package for DGX OS 5 + file: + path: /etc/apt/sources.list.d/dgx-temp.list + state: absent + when: ansible_distribution_version == "20.04" + +- name: install more packages for DGX OS 5 + apt: + name: "{{ item }}" + update_cache: yes + dpkg_options: "force-confdef,force-confold" + with_items: + - "{{ G_PKGS_DEFAULT }}" + when: ansible_distribution_version == "20.04" + +- name: install DGX-1 packages for DGX OS 4 + apt: + name: "{{ item }}" + state: present + update_cache: yes + with_items: + - "{{ PKGS_DGX1_ALL }}" + when: + - ansible_distribution_version == "18.04" + - ansible_product_name is search("DGX-1") + notify: + - restart docker + +- name: install DGX-1 packages for DGX OS 5 + apt: + name: "{{ item }}" + state: present + update_cache: yes + with_items: + - "{{ G_DGX_COMMON_PKGS }}" + - "{{ G_DGX1_PKGS }}" + when: + - ansible_distribution_version == "20.04" + - ansible_product_name is search("DGX-1") + notify: + - restart docker + +- name: install DGX-2 packages for DGX OS 4 + apt: + name: "{{ item }}" + state: present + update_cache: yes + with_items: + - "{{ PKGS_DGX2_ALL }}" + when: + - ansible_distribution_version == "18.04" + - ansible_product_name is search("DGX-2") + notify: + - restart docker + +- name: install DGX-2 packages for DGX OS 5 + apt: + name: "{{ item }}" + state: present + update_cache: yes + with_items: + - "{{ G_DGX_COMMON_PKGS }}" + - "{{ G_DGX2_PKGS }}" + when: + - ansible_distribution_version == "20.04" + - ansible_product_name is search("DGX-2") + notify: + - restart docker + +- name: install DGXA100 packages for DGX OS 4 + apt: + name: "{{ item }}" + state: present + update_cache: yes + with_items: + - "{{ PKGS_DGXA100_ALL }}" + when: + - ansible_distribution_version == "18.04" + - ansible_product_name is search("DGXA100") + notify: + - restart docker + +- name: install DGXA100 packages for DGX OS 5 + apt: + name: "{{ item }}" + state: present + update_cache: yes + with_items: + - "{{ G_DGX_COMMON_PKGS }}" + - "{{ G_DGX_A100_PKGS }}" + when: + - ansible_distribution_version == "20.04" + - ansible_product_name is search("DGXA100") + notify: + - restart docker + +- name: install extra DGX packages for DGX OS 4 + apt: + name: "{{ item }}" + state: present + update_cache: yes + with_items: + - "{{ dgx_extra_packages }}" + when: ansible_distribution_version == "18.04" + +- name: touch grub ipmisol.cfg for DGX OS 4 + file: + path: /etc/default/grub.d/ipmisol.cfg + state: touch + mode: u=rw,g=r,o=r + when: ansible_distribution_version == "18.04" + +- name: add DGX IPMI SOL package for DGX OS 4 + apt: + name: dgx-ipmisol + state: present + when: + - ansible_distribution_version == "18.04" + - dgx_name is search("DGX") + +- name: disable release update prompt for DGX OS 5 + lineinfile: + path: /etc/update-manager/release-upgrades + regexp: '^Prompt=.*' + line: Prompt=never + when: ansible_distribution_version == "20.04" + +- name: add ipmi kernel module at boot + lineinfile: + path: /etc/modules + line: ipmi_devintf + +- name: populate service facts + service_facts: + +- name: enable services + systemd: + name: "{{ item }}" + state: started + enabled: yes + daemon_reload: yes + loop: + - openibd.service + - dcgm.service + - nvidia-dcgm.service + - nvidia-persistenced.service + - nvidia-fabricmanager.service + when: item in services + +- name: enable PXE/UEFI on MLNX interfaces for DGX OS 5 + command: /usr/sbin/mlnx_pxe_setup.bash + when: ansible_distribution_version == "20.04" + +- name: configure nv peer mem service startup + command: /usr/sbin/update-rc.d nv_peer_mem defaults + +- name: configure default ubuntu repos for DGX OS 4 + template: + src: sources.list.j2 + dest: /etc/apt/sources.list + mode: 0644 + when: ansible_distribution_version == "18.04" + +- name: disable srp services + systemd: + name: "{{ item }}" + state: stopped + enabled: no + when: ansible_product_name is search("DGXA100") + with_items: + - srp_daemon + - srptools + +# setup_data_drive "/dev/sdb1" "dgx1cache" +- name: create raid directory for raid cache mount point + file: + path: "{{ cachefilesd_cache_dir }}" + state: directory + mode: "{{ cachefilesd_cache_dir_mode }}" + +- name: configure cachefilesd + template: + src: cachefilesd.conf.j2 + dest: /etc/cachefilesd.conf + +# Misc stuff +- name: restart docker service just in case... + systemd: + name: docker + state: restarted + enabled: yes diff --git a/roles/nvidia-dgx/tasks/ubuntu.yml b/roles/nvidia-dgx/tasks/ubuntu.yml index cdfb95f96..7a2821ccb 100644 --- a/roles/nvidia-dgx/tasks/ubuntu.yml +++ b/roles/nvidia-dgx/tasks/ubuntu.yml @@ -1,384 +1,17 @@ --- -- name: Check for Ubuntu version +- name: Check for supported Ubuntu DGX Software Stack version fail: - msg: "Role supports Ubuntu 18.04 and 20.04 only" - when: - - ansible_distribution_version != "20.04" - - ansible_distribution_version != "18.04" + msg: "Role supports Ubuntu 18.04, 20.04, 22.04, and 24.04 only" + when: ansible_distribution_version not in ['18.04', '20.04', '22.04', '24.04'] -- name: include 18.04 specific variables for DGX OS 4 - include_vars: ubuntu-18.04.yml - when: ansible_distribution_version == "18.04" +- name: Run legacy DGX OS 4/5 tasks + include_tasks: ubuntu-legacy.yml + when: ansible_distribution_version in ['18.04', '20.04'] -- name: include DGXA100 specific variables for DGX OS 4.99 - include_vars: dgxa100.yml - when: - - ansible_distribution_version == "18.04" - - ansible_product_name is search("DGXA100") +- name: Run DGX OS 6 tasks + include_tasks: ubuntu-22.04.yml + when: ansible_distribution_version == '22.04' -- name: include 20.04 specific variables for DGX OS 5 - include_vars: ubuntu-20.04.yml - when: ansible_distribution_version == "20.04" - -- name: install dmidecode - apt: - name: dmidecode - state: present - update_cache: yes - -# Housekeeping -- name: register DGX product name - command: "dmidecode --string system-product-name" - register: dgx_name - -- name: register DGX-1 serial number - command: "dmidecode --string system-serial-number" - register: dgx1_serial - when: ansible_product_name is search("DGX-1") - -- name: register DGX-2 serial number - command: "dmidecode --string chassis-serial-number" - register: dgx2_serial - when: ansible_product_name is search("DGX-2") - -- name: register DGXA100 serial number - command: "dmidecode --string chassis-serial-number" - register: dgxa100_serial - when: ansible_product_name is search("DGXA100") - -- name: figure out which serial number we ended up with - set_fact: - dgx_serial: "{{ dgx1_serial.stdout }}" - when: dgx1_serial.skipped is not defined - -- name: figure out which serial number we ended up with - set_fact: - dgx_serial: "{{ dgx2_serial.stdout }}" - when: dgx2_serial.skipped is not defined - -- name: figure out which serial number we ended up with - set_fact: - dgx_serial: "{{ dgxa100_serial.stdout }}" - when: dgxa100_serial.skipped is not defined - -- name: fail if we don't recognize the DGX system - fail: - msg: "Unknown DGX model: {{ ansible_product_name }}" - when: dgx_serial is undefined - -- name: update dgx platform file - blockinfile: - path: /etc/dgx-release - create: yes - block: | - DGX_NAME="DGX Server" - DGX_PRETTY_NAME="NVIDIA DGX Server" - DGX_PLATFORM="DGX Server for {{ dgx_name.stdout }}" - DGX_SERIAL_NUMBER="{{ dgx_serial }}" - -- name: update dgx platform file with additional info for DGX OS 4 - blockinfile: - path: /etc/dgx-release - create: yes - block: | - DGX_SWBUILD_DATE="{{ DGX_SWBUILD_DATE }}" - DGX_SWBUILD_VERSION="{{ DGX_SWBUILD_VERSION }}" - DGX_COMMIT_ID="{{ DGX_COMMIT_ID }}" - when: ansible_distribution_version == "18.04" - -# Repos and installs -- name: remove ubuntu nvidia driver ppa if installed - apt_repository: - repo: ppa:graphics-drivers/ppa - state: absent - -- name: remove packages when using Bright Computing source image - apt: - name: "{{ item }}" - state: absent - purge: yes - with_items: - - cuda-dcgm - - cuda-driver - - libumad2sim0 - - libsim-utils - - ibutils - -- name: remove apt mirror when using Bright Computing source image - replace: - path: /etc/apt/sources.list - regexp: 'nl\.' - -- name: unload drivers when using Bright Computing source image - modprobe: - name: "{{ item }}" - state: absent - with_items: - - ib_srp - - ib_iser - - rdma_ucm - - rpcrdma - - rdma_cm - -- name: add DGX repo keys for DGX OS 4 - apt: - deb: "{{ nvidia_dgx_ubuntu_gpgkey }}" - when: ansible_distribution_version == "18.04" - -- name: add DGX repo key for DGX OS 5 - apt_key: - url: "{{ nvidia_dgx_os5_ubuntu_gpgkey }}" - state: present - when: ansible_distribution_version == "20.04" - -- name: add DGX repo for DGX OS 4 - template: - src: dgx.list.j2 - dest: /etc/apt/sources.list.d/dgx.list - mode: 0644 - when: ansible_distribution_version == "18.04" - -- name: add DGX repo for DGX OS 5 - template: - src: dgxos5.list.j2 - dest: /etc/apt/sources.list.d/dgx-temp.list - mode: 0644 - when: ansible_distribution_version == "20.04" - -- name: update apt cache - apt: - update_cache: yes - -- name: add DGX cuda 10.1 repo for DGX OS 4 - apt: - name: dgx-bionic-r418+cuda10.1-repo - update_cache: yes - dpkg_options: "force-confdef,force-confold" - when: ansible_distribution_version == "18.04" - -- name: add DGX cuda 11.0 repo for DGX OS 4 - apt: - name: dgx-bionic-r450+cuda11.0-repo - update_cache: yes - dpkg_options: "force-confdef,force-confold" - when: ansible_distribution_version == "18.04" - -- name: add DGXA100 repo for DGX OS 4 - apt: - name: dgx-bionic-4.99-repo - update_cache: yes - dpkg_options: "force-confdef,force-confold" - when: - - ansible_distribution_version == "18.04" - - ansible_product_name is search("DGXA100") - -- name: install prerequisites for DGX OS 5 - apt: - name: "{{ item }}" - state: present - update_cache: yes - with_items: - - "{{ G_PREREQS }}" - - "{{ G_SETUP_PKGS }}" - when: ansible_distribution_version == "20.04" - -- name: install DGX repo package for DGX OS 5 - apt: - name: "{{ item }}" - update_cache: yes - dpkg_options: "force-confdef,force-confold" - with_items: - - "{{ G_DGX_REPO_PKG }}" - when: ansible_distribution_version == "20.04" - -- name: remove temporary repo package for DGX OS 5 - file: - path: /etc/apt/sources.list.d/dgx-temp.list - state: absent - when: ansible_distribution_version == "20.04" - -- name: install more packages for DGX OS 5 - apt: - name: "{{ item }}" - update_cache: yes - dpkg_options: "force-confdef,force-confold" - with_items: - - "{{ G_PKGS_DEFAULT }}" - when: ansible_distribution_version == "20.04" - -- name: install DGX-1 packages for DGX OS 4 - apt: - name: "{{ item }}" - state: present - update_cache: yes - with_items: - - "{{ PKGS_DGX1_ALL }}" - when: - - ansible_distribution_version == "18.04" - - ansible_product_name is search("DGX-1") - notify: - - restart docker - -- name: install DGX-1 packages for DGX OS 5 - apt: - name: "{{ item }}" - state: present - update_cache: yes - with_items: - - "{{ G_DGX_COMMON_PKGS }}" - - "{{ G_DGX1_PKGS }}" - when: - - ansible_distribution_version == "20.04" - - ansible_product_name is search("DGX-1") - notify: - - restart docker - -- name: install DGX-2 packages for DGX OS 4 - apt: - name: "{{ item }}" - state: present - update_cache: yes - with_items: - - "{{ PKGS_DGX2_ALL }}" - when: - - ansible_distribution_version == "18.04" - - ansible_product_name is search("DGX-2") - notify: - - restart docker - -- name: install DGX-2 packages for DGX OS 5 - apt: - name: "{{ item }}" - state: present - update_cache: yes - with_items: - - "{{ G_DGX_COMMON_PKGS }}" - - "{{ G_DGX2_PKGS }}" - when: - - ansible_distribution_version == "20.04" - - ansible_product_name is search("DGX-2") - notify: - - restart docker - -- name: install DGXA100 packages for DGX OS 4 - apt: - name: "{{ item }}" - state: present - update_cache: yes - with_items: - - "{{ PKGS_DGXA100_ALL }}" - when: - - ansible_distribution_version == "18.04" - - ansible_product_name is search("DGXA100") - notify: - - restart docker - -- name: install DGXA100 packages for DGX OS 5 - apt: - name: "{{ item }}" - state: present - update_cache: yes - with_items: - - "{{ G_DGX_COMMON_PKGS }}" - - "{{ G_DGX_A100_PKGS }}" - when: - - ansible_distribution_version == "20.04" - - ansible_product_name is search("DGXA100") - notify: - - restart docker - -- name: install extra DGX packages for DGX OS 4 - apt: - name: "{{ item }}" - state: present - update_cache: yes - with_items: - - "{{ dgx_extra_packages }}" - when: ansible_distribution_version == "18.04" - -- name: touch grub ipmisol.cfg for DGX OS 4 - file: - path: /etc/default/grub.d/ipmisol.cfg - state: touch - mode: u=rw,g=r,o=r - when: ansible_distribution_version == "18.04" - -- name: add DGX IPMI SOL package for DGX OS 4 - apt: - name: dgx-ipmisol - state: present - when: - - ansible_distribution_version == "18.04" - - dgx_name is search("DGX") - -- name: disable release update prompt for DGX OS 5 - lineinfile: - path: /etc/update-manager/release-upgrades - regexp: '^Prompt=.*' - line: Prompt=never - when: ansible_distribution_version == "20.04" - -- name: add ipmi kernel module at boot - lineinfile: - path: /etc/modules - line: ipmi_devintf - -- name: populate service facts - service_facts: - -- name: enable services - systemd: - name: "{{ item }}" - state: started - enabled: yes - daemon_reload: yes - loop: - - openibd.service - - dcgm.service - - nvidia-dcgm.service - - nvidia-persistenced.service - - nvidia-fabricmanager.service - when: item in services - -- name: enable PXE/UEFI on MLNX interfaces for DGX OS 5 - command: /usr/sbin/mlnx_pxe_setup.bash - when: ansible_distribution_version == "20.04" - -- name: configure nv peer mem service startup - command: /usr/sbin/update-rc.d nv_peer_mem defaults - -- name: configure default ubuntu repos for DGX OS 4 - template: - src: sources.list.j2 - dest: /etc/apt/sources.list - mode: 0644 - when: ansible_distribution_version == "18.04" - -- name: disable srp services - systemd: - name: "{{ item }}" - state: stopped - enabled: no - when: ansible_product_name is search("DGXA100") - with_items: - - srp_daemon - - srptools - -# setup_data_drive "/dev/sdb1" "dgx1cache" -- name: create raid directory for raid cache mount point - file: - path: "{{ cachefilesd_cache_dir }}" - state: directory - mode: "{{ cachefilesd_cache_dir_mode }}" - -- name: configure cachefilesd - template: - src: cachefilesd.conf.j2 - dest: /etc/cachefilesd.conf - -# Misc stuff -- name: restart docker service just in case... - systemd: - name: docker - state: restarted - enabled: yes +- name: Run DGX OS 7 tasks + include_tasks: ubuntu-24.04.yml + when: ansible_distribution_version == '24.04' diff --git a/roles/nvidia-dgx/vars/redhat.yml b/roles/nvidia-dgx/vars/redhat.yml index 5eaa1621d..1ed547e5d 100644 --- a/roles/nvidia-dgx/vars/redhat.yml +++ b/roles/nvidia-dgx/vars/redhat.yml @@ -3,3 +3,115 @@ dgx_repo_dir: "rhel{{ ansible_distribution_major_version }}" dgx_extra_packages: - dgx-conf-cachefilesd - kernel-headers + +dgx_redhat_manage_subscription_repos: true +dgx_redhat_upgrade_packages: false +dgx_redhat_install_driver: true +dgx_redhat_install_container_runtime: true +dgx_redhat_install_docker_ce: "{{ ansible_distribution_major_version == '8' }}" +dgx_redhat_install_cachefilesd: false +dgx_redhat_driver_branch: "{{ '580' if ansible_distribution_major_version == '9' else ('535-dkms' if dgx_redhat_platform.configuration_group == 'DGX H100 Configurations' else '525') }}" +dgx_redhat_use_open_kernel_modules: "{{ ansible_distribution_major_version == '9' and dgx_redhat_platform.nvswitch | default(false) }}" + +dgx_redhat_repo_setup_rpms: + "8": https://repo.download.nvidia.com/baseos/el/el-files/8/nvidia-repo-setup-21.06-1.el8.x86_64.rpm + "9": https://repo.download.nvidia.com/baseos/el/el-files/9/nvidia-repo-setup-25.02-1.el9.x86_64.rpm + +dgx_redhat_subscription_repos: + "8": + - rhel-8-for-x86_64-appstream-rpms + - rhel-8-for-x86_64-baseos-rpms + - codeready-builder-for-rhel-8-x86_64-rpms + "9": + - rhel-9-for-x86_64-appstream-rpms + - rhel-9-for-x86_64-baseos-rpms + - codeready-builder-for-rhel-9-x86_64-rpms + +dgx_redhat_platforms: + - match: "DGX-1" + configuration_group: "DGX-1 Configurations" + supported_major_versions: ["8"] + nvswitch: false + nvlink5: false + station: false + - match: "DGX-2" + configuration_group: "DGX-2 Configurations" + supported_major_versions: ["8"] + nvswitch: true + nvlink5: false + station: false + - match: "DGX[ -]?A100|DGXA100" + configuration_group: "DGX A100 Configurations" + supported_major_versions: ["8", "9"] + nvswitch: true + nvlink5: false + station: false + - match: "DGX[ -]?A800|DGXA800" + configuration_group: "DGX A800 Configurations" + supported_major_versions: ["8"] + nvswitch: true + nvlink5: false + station: false + - match: "DGX[ -]?H100|DGXH100" + configuration_group: "DGX H100 Configurations" + supported_major_versions: ["8", "9"] + nvswitch: true + nvlink5: false + station: false + - match: "DGX[ -]?H200|DGXH200" + configuration_group: "DGX H200 Configurations" + supported_major_versions: ["9"] + nvswitch: true + nvlink5: false + station: false + - match: "DGX[ -]?B200|DGXB200" + configuration_group: "DGX B200 Configurations" + supported_major_versions: ["9"] + nvswitch: true + nvlink5: true + station: false + - match: "DGX[ -]?B300|DGXB300" + configuration_group: "DGX B300 Configurations" + supported_major_versions: ["9"] + nvswitch: true + nvlink5: true + station: false + - match: "DGX Station A100" + configuration_group: "DGX Station A100 Configurations" + supported_major_versions: ["8", "9"] + nvswitch: false + nvlink5: false + station: true + - match: "DGX Station A800" + configuration_group: "DGX Station A800 Configurations" + supported_major_versions: ["8"] + nvswitch: false + nvlink5: false + station: true + - match: "^DGX Station$" + configuration_group: "DGX Station Configurations" + supported_major_versions: ["8"] + nvswitch: false + nvlink5: false + station: true + +dgx_redhat_driver_support_packages: + - nv-persistence-mode + +dgx_redhat_fabricmanager_packages: + - nvidia-fm-enable + +dgx_redhat_non_nvswitch_packages: + - "libnvidia-nscq-{{ dgx_redhat_driver_branch }}" + +dgx_redhat_nvlink5_packages: + - nvlsm + +dgx_redhat_station_packages: + - nvidia-conf-xconfig + - nv-docker-gpus + +dgx_redhat_services: + - docker.service + - nvidia-persistenced.service + - nvidia-fabricmanager.service diff --git a/roles/nvidia-dgx/vars/ubuntu-22.04.yml b/roles/nvidia-dgx/vars/ubuntu-22.04.yml new file mode 100644 index 000000000..a17da2b91 --- /dev/null +++ b/roles/nvidia-dgx/vars/ubuntu-22.04.yml @@ -0,0 +1,115 @@ +--- +dgx_os6_repo_files_url: https://repo.download.nvidia.com/baseos/ubuntu/jammy/dgx-repo-files.tgz + +dgx_os6_upgrade_packages: false +dgx_os6_install_driver: true +dgx_os6_install_container_runtime: true +dgx_os6_install_nvsm: true +dgx_os6_install_admin_packages: true +dgx_os6_install_development_packages: true +dgx_os6_disable_unattended_upgrades: true +dgx_os6_install_ipmisol: true +dgx_os6_install_logrotate: true + +dgx_os6_driver_branch: "550" + +dgx_os6_platforms: + - match: "DGX-1" + nvswitch: false + config_packages: + - dgx1-system-configurations + - dgx1-system-tools + - dgx1-system-tools-extra + - match: "DGX-2" + nvswitch: true + config_packages: + - dgx2-system-configurations + - dgx2-system-tools + - dgx2-system-tools-extra + - match: "DGX[ -]?A100|DGXA100" + nvswitch: true + config_packages: + - dgx-a100-system-configurations + - dgx-a100-system-tools + - dgx-a100-system-tools-extra + - "nvidia-utils-{{ dgx_os6_driver_branch }}-server" + - match: "DGX[ -]?H100|DGXH100" + nvswitch: true + config_packages: + - dgx-h100-system-configurations + - dgx-h100-system-tools + - dgx-h100-system-tools-extra + - "nvidia-utils-{{ dgx_os6_driver_branch }}-server" + - nvfwupd + - match: "DGX[ -]?H200|DGXH200" + nvswitch: true + config_packages: + - dgx-h200-system-configurations + - dgx-h200-system-tools + - dgx-h200-system-tools-extra + - "nvidia-utils-{{ dgx_os6_driver_branch }}-server" + - nvfwupd + +dgx_os6_common_packages: + - linux-tools-nvidia + - nvidia-peermem-loader + +dgx_os6_driver_packages: + - "nvidia-driver-{{ dgx_os6_driver_branch }}-server" + - "linux-modules-nvidia-{{ dgx_os6_driver_branch }}-server-nvidia" + - "libnvidia-nscq-{{ dgx_os6_driver_branch }}" + - nvidia-modprobe + - nv-persistence-mode + +dgx_os6_fabricmanager_packages: + - "nvidia-fabricmanager-{{ dgx_os6_driver_branch }}" + - nvidia-fm-enable + +dgx_os6_container_packages: + - docker-ce + - nvidia-container-toolkit + - nv-docker-options + +dgx_os6_nvsm_packages: + - nvsm + - nvidia-motd + +dgx_os6_admin_packages: + - chrpath + - cifs-utils + - fping + - gdisk + - iperf + - ipmitool + - lsscsi + - net-tools + - nfs-common + - quota + - rasdaemon + - pm-utils + - samba-common + - samba-libs + - sysstat + - vlan + +dgx_os6_development_packages: + - build-essential + - automake + - bison + - cmake + - dpatch + - flex + - gcc-multilib + - gdb + - g++-multilib + - libelf-dev + - libltdl-dev + - linux-tools-generic + - m4 + - swig + +dgx_os6_services: + - dcgm.service + - nvidia-dcgm.service + - nvidia-persistenced.service + - nvidia-fabricmanager.service diff --git a/roles/nvidia-dgx/vars/ubuntu-24.04.yml b/roles/nvidia-dgx/vars/ubuntu-24.04.yml new file mode 100644 index 000000000..bd05fa49d --- /dev/null +++ b/roles/nvidia-dgx/vars/ubuntu-24.04.yml @@ -0,0 +1,102 @@ +--- +dgx_os7_repo_files_url: "https://repo.download.nvidia.com/baseos/ubuntu/noble/{{ 'arm64' if ansible_architecture in ['aarch64', 'arm64'] else 'x86_64' }}/dgx-repo-files.tgz" + +dgx_os7_upgrade_packages: false +dgx_os7_install_driver: true +dgx_os7_disable_unattended_upgrades: true + +dgx_os7_platforms: + - match: "DGX Station A100" + station: true + nvswitch: false + nvlink5: false + imex: false + linux_tools_package: linux-tools-generic + - match: "DGX Station A800" + station: true + nvswitch: false + nvlink5: false + imex: false + linux_tools_package: linux-tools-generic + - match: "DGX Spark" + station: true + nvswitch: false + nvlink5: false + imex: false + linux_tools_package: linux-tools-nvidia-hwe-24.04 + - match: "DGX[ -]?A100|DGXA100" + station: false + nvswitch: true + nvlink5: false + imex: false + linux_tools_package: linux-tools-generic + - match: "DGX[ -]?H100|DGXH100" + station: false + nvswitch: true + nvlink5: false + imex: false + linux_tools_package: linux-tools-generic + - match: "DGX[ -]?H200|DGXH200" + station: false + nvswitch: true + nvlink5: false + imex: false + linux_tools_package: linux-tools-generic + - match: "DGX[ -]?B200|DGXB200" + station: false + nvswitch: true + nvlink5: true + imex: false + linux_tools_package: linux-tools-generic + - match: "DGX[ -]?B300|DGXB300" + station: false + nvswitch: true + nvlink5: true + imex: false + linux_tools_package: linux-tools-generic + - match: "DGX[ -]?GB200|DGXGB200" + station: false + nvswitch: false + nvlink5: false + imex: true + linux_tools_package: linux-tools-nvidia-64k-hwe-24.04 + - match: "DGX[ -]?GB300|DGXGB300" + station: false + nvswitch: false + nvlink5: false + imex: true + linux_tools_package: linux-tools-nvidia-64k-hwe-24.04 + +dgx_os7_system_packages: + - nvidia-system-core + - nvidia-system-utils + - nvidia-system-extra + +dgx_os7_station_package: nvidia-system-station + +dgx_os7_driver_packages: + - nvidia-driver-580-open + - libnvidia-nscq + - nvidia-modprobe + - datacenter-gpu-manager-4-cuda13 + - nv-persistence-mode + +dgx_os7_fabricmanager_packages: + - nvidia-fabricmanager + +dgx_os7_nvlink5_packages: + - nvlsm + - libnvsdm + +dgx_os7_imex_packages: + - nvidia-imex + +dgx_os7_base_services: + - nvidia-persistenced.service + - nvidia-dcgm.service + +dgx_os7_fabricmanager_services: + - nvidia-fabricmanager.service + +dgx_os7_imex_services: + - nvidia-imex.service From c8c74ed36ae992a9a56588d567496900162c2d6b Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Wed, 27 May 2026 10:24:23 -0600 Subject: [PATCH 13/27] fix(dgx): harden EL8 driver install --- docs/deepops/dgx-software-stack.md | 9 +++++---- roles/nvidia-dgx/tasks/redhat-el8-plus.yml | 22 +++++++++++++++------- roles/nvidia-dgx/vars/redhat.yml | 7 +++++-- 3 files changed, 25 insertions(+), 13 deletions(-) diff --git a/docs/deepops/dgx-software-stack.md b/docs/deepops/dgx-software-stack.md index 9164c1a0d..ffd5db302 100644 --- a/docs/deepops/dgx-software-stack.md +++ b/docs/deepops/dgx-software-stack.md @@ -67,10 +67,11 @@ The role follows the official Red Hat DGX software guides: 4. Optionally install the NVIDIA driver module and support packages. 5. Optionally install Docker CE and the NVIDIA Container Runtime group. -The default driver stream follows the official examples: `525` on most EL8 -systems, `535-dkms` on EL8 DGX H100, and `580` on EL9. EL9 NVSwitch systems -install the open-kernel-module stream by default. Override the branch when a -validated DGX release note calls for another stream: +The default driver stream uses DKMS on EL8 so current EL8 minor kernels can +build a matching NVIDIA kernel module: `525-dkms` on most EL8 systems, +`535-dkms` on EL8 DGX H100, and `580` on EL9. EL9 NVSwitch systems install the +open-kernel-module stream by default. Override the branch when a validated DGX +release note calls for another stream: ```yaml dgx_redhat_driver_branch: "580" diff --git a/roles/nvidia-dgx/tasks/redhat-el8-plus.yml b/roles/nvidia-dgx/tasks/redhat-el8-plus.yml index ce4b1997c..e3137e2ae 100644 --- a/roles/nvidia-dgx/tasks/redhat-el8-plus.yml +++ b/roles/nvidia-dgx/tasks/redhat-el8-plus.yml @@ -41,13 +41,13 @@ when: dgx_redhat_upgrade_packages tags: skip_ansible_lint -- name: Install EL9 kernel development packages for DGX driver builds +- name: Install kernel development packages for DGX driver builds dnf: name: - "kernel-devel-{{ ansible_kernel }}" - "kernel-headers-{{ ansible_kernel }}" state: present - when: ansible_distribution_major_version == '9' + when: ansible_distribution_major_version == '9' or 'dkms' in dgx_redhat_driver_branch - name: Install Red Hat DGX configuration group dnf: @@ -70,13 +70,21 @@ else [dgx_redhat_driver_profile] }} -- name: Configure Red Hat DGX driver module specification +- name: Configure Red Hat DGX driver module specifications set_fact: - dgx_redhat_driver_module_spec: >- - nvidia-driver:{{ dgx_redhat_driver_stream }}/{{ '{' ~ (dgx_redhat_driver_profiles | join(',')) ~ '}' if (dgx_redhat_driver_profiles | length) > 1 else dgx_redhat_driver_profiles[0] }} + dgx_redhat_driver_module_specs: "{{ dgx_redhat_driver_profiles | map('regex_replace', '^(.*)$', 'nvidia-driver:' ~ dgx_redhat_driver_stream ~ '/\\1') | list }}" + +- name: Remove precompiled NVIDIA kmod headers before DKMS driver install + dnf: + name: nvidia-kmod-headers + state: absent + when: + - dgx_redhat_install_driver + - "'dkms' in dgx_redhat_driver_stream" - name: Install Red Hat DGX NVIDIA driver module - command: "dnf module install --nobest -y {{ dgx_redhat_driver_module_spec }}" + command: "dnf module install --nobest -y {{ item }}" + loop: "{{ dgx_redhat_driver_module_specs }}" register: dgx_redhat_driver_module_install changed_when: "'Nothing to do' not in dgx_redhat_driver_module_install.stdout" notify: reboot after driver install @@ -135,5 +143,5 @@ state: started enabled: yes daemon_reload: yes - loop: "{{ dgx_redhat_services }}" + loop: "{{ dgx_redhat_services + (dgx_redhat_fabricmanager_services if dgx_redhat_platform.nvswitch else []) }}" when: item in ansible_facts.services diff --git a/roles/nvidia-dgx/vars/redhat.yml b/roles/nvidia-dgx/vars/redhat.yml index 1ed547e5d..27a386c12 100644 --- a/roles/nvidia-dgx/vars/redhat.yml +++ b/roles/nvidia-dgx/vars/redhat.yml @@ -10,7 +10,8 @@ dgx_redhat_install_driver: true dgx_redhat_install_container_runtime: true dgx_redhat_install_docker_ce: "{{ ansible_distribution_major_version == '8' }}" dgx_redhat_install_cachefilesd: false -dgx_redhat_driver_branch: "{{ '580' if ansible_distribution_major_version == '9' else ('535-dkms' if dgx_redhat_platform.configuration_group == 'DGX H100 Configurations' else '525') }}" +dgx_redhat_driver_branch: "{{ '580' if ansible_distribution_major_version == '9' else ('535-dkms' if dgx_redhat_platform.configuration_group == 'DGX H100 Configurations' else '525-dkms') }}" +dgx_redhat_driver_package_branch: "{{ dgx_redhat_driver_branch | regex_replace('-dkms$', '') }}" dgx_redhat_use_open_kernel_modules: "{{ ansible_distribution_major_version == '9' and dgx_redhat_platform.nvswitch | default(false) }}" dgx_redhat_repo_setup_rpms: @@ -102,7 +103,7 @@ dgx_redhat_fabricmanager_packages: - nvidia-fm-enable dgx_redhat_non_nvswitch_packages: - - "libnvidia-nscq-{{ dgx_redhat_driver_branch }}" + - "libnvidia-nscq-{{ dgx_redhat_driver_package_branch }}" dgx_redhat_nvlink5_packages: - nvlsm @@ -114,4 +115,6 @@ dgx_redhat_station_packages: dgx_redhat_services: - docker.service - nvidia-persistenced.service + +dgx_redhat_fabricmanager_services: - nvidia-fabricmanager.service From 40d707fe3f0cc97d19840923abf3e9e628975978 Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Wed, 27 May 2026 18:37:45 -0600 Subject: [PATCH 14/27] feat: support Ubuntu 24.04 container toolkit --- docs/airgap/mirror-apt-repos.md | 108 +++++++++-------- docs/airgap/ngc-ready.md | 6 +- docs/deepops/testing.md | 11 +- docs/ngc-ready/README.md | 2 + docs/slurm-cluster/slurm-single-node.md | 2 +- playbooks/container/nvidia-docker.yml | 9 ++ .../defaults/main.yml | 10 ++ .../handlers/main.yml | 6 + roles/nvidia_container_toolkit/tasks/main.yml | 114 ++++++++++++++++++ 9 files changed, 213 insertions(+), 55 deletions(-) create mode 100644 roles/nvidia_container_toolkit/defaults/main.yml create mode 100644 roles/nvidia_container_toolkit/handlers/main.yml create mode 100644 roles/nvidia_container_toolkit/tasks/main.yml diff --git a/docs/airgap/mirror-apt-repos.md b/docs/airgap/mirror-apt-repos.md index ccb10ec11..93064f704 100644 --- a/docs/airgap/mirror-apt-repos.md +++ b/docs/airgap/mirror-apt-repos.md @@ -15,7 +15,7 @@ Set up offline repositoriy mirrors for Aptitude - [NVIDIA CUDA repository](#nvidia-cuda-repository) - [APT Configuration](#apt-configuration-1) - [GPG Key Validation](#gpg-key-validation-1) - - [nvidia-docker](#nvidia-docker) + - [NVIDIA Container Toolkit](#nvidia-container-toolkit) - [APT Configuration](#apt-configuration-2) - [GPG Key Validation](#gpg-key-validation-2) - [Additional DEB packages](#additional-deb-packages) @@ -57,7 +57,7 @@ deb http://archive.ubuntu.com/ubuntu/ -updates main multiverse uni ``` where `` is the name of the Ubuntu release you want to mirror. -This is `bionic` for Ubuntu 18.04, and `focal` for Ubuntu 20.04. +This is `bionic` for Ubuntu 18.04, `focal` for Ubuntu 20.04, `jammy` for Ubuntu 22.04, and `noble` for Ubuntu 24.04. ### Docker repository @@ -74,7 +74,7 @@ https://download.docker.com/linux/ubuntu/gpg ``` where `` is the name of the Ubuntu release you want to mirror. -This is `bionic` for Ubuntu 18.04, and `focal` for Ubuntu 20.04. +This is `bionic` for Ubuntu 18.04, `focal` for Ubuntu 20.04, `jammy` for Ubuntu 22.04, and `noble` for Ubuntu 24.04. ### NVIDIA CUDA repository @@ -92,6 +92,18 @@ deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804 deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004 ``` +**Ubuntu 22.04** + +```bash +deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64 / +``` + +**Ubuntu 24.04** + +```bash +deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64 / +``` + #### GPG Key Validation **Ubuntu 18.04** @@ -106,30 +118,30 @@ https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/7fa2a https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/7fa2af80.pub ``` -### nvidia-docker +**Ubuntu 22.04** -#### APT Configuration +```bash +https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/3bf863cc.pub +``` -**Ubuntu 18.04** +**Ubuntu 24.04** ```bash -deb https://nvidia.github.io/libnvidia-container/stable/ubuntu18.04/$(ARCH) / -deb https://nvidia.github.io/nvidia-container-runtime/stable/ubuntu18.04/$(ARCH) / -deb https://nvidia.github.io/nvidia-docker/ubuntu18.04/$(ARCH) +https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/3bf863cc.pub ``` -**Ubuntu 20.04** +### NVIDIA Container Toolkit + +#### APT Configuration ```bash -deb https://nvidia.github.io/libnvidia-container/stable/ubuntu18.04/$(ARCH) / -deb https://nvidia.github.io/nvidia-container-runtime/stable/ubuntu18.04/$(ARCH) / -deb https://nvidia.github.io/nvidia-docker/ubuntu18.04/$(ARCH) +deb https://nvidia.github.io/libnvidia-container/stable/deb/$(ARCH) / ``` #### GPG Key Validation ```bash -https://nvidia.github.io/nvidia-docker/gpgkey +https://nvidia.github.io/libnvidia-container/gpgkey ``` ### Additional DEB packages @@ -161,7 +173,7 @@ After installing `apt-mirror`, edit the `/etc/apt/mirror.list` file make the fol - Set the `base_path` to the desired download path for your mirror (here, `/var/repos`) - Add a list of APT configuration lines for each repo you wish to mirror -For example, if we just want to mirror the Docker and NVIDIA Docker repositories, this configuration would work: +For example, if we just want to mirror the Docker and NVIDIA Container Toolkit repositories, this configuration would work: ``` ############# config ################## @@ -172,11 +184,11 @@ set _tilde 0 # ############# end config ############## -deb https://download.docker.com/linux/ubuntu bionic stable -deb https://nvidia.github.io/nvidia-docker/ubuntu20.04/amd64 / +deb https://download.docker.com/linux/ubuntu noble stable +deb https://nvidia.github.io/libnvidia-container/stable/deb/amd64 / ``` -The full mirror.list file for Deepops: +The full mirror.list file for DeepOps: ``` ############# config ################## @@ -195,29 +207,27 @@ set _tilde 0 # ############# end config ############## -deb http://archive.ubuntu.com/ubuntu focal main restricted universe multiverse -deb http://archive.ubuntu.com/ubuntu focal-security main restricted universe multiverse -deb http://archive.ubuntu.com/ubuntu focal-updates main restricted universe multiverse -deb http://archive.ubuntu.com/ubuntu focal-proposed main restricted universe multiverse -deb http://archive.ubuntu.com/ubuntu focal-backports main restricted universe multiverse -deb http://ppa.launchpad.net/maas/2.9/ubuntu focal main -deb http://archive.canonical.com/ubuntu focal partner +deb http://archive.ubuntu.com/ubuntu noble main restricted universe multiverse +deb http://archive.ubuntu.com/ubuntu noble-security main restricted universe multiverse +deb http://archive.ubuntu.com/ubuntu noble-updates main restricted universe multiverse +deb http://archive.ubuntu.com/ubuntu noble-proposed main restricted universe multiverse +deb http://archive.ubuntu.com/ubuntu noble-backports main restricted universe multiverse +deb http://ppa.launchpad.net/maas/3.5/ubuntu noble main +deb http://archive.canonical.com/ubuntu noble partner -deb-src http://archive.ubuntu.com/ubuntu focal main restricted universe multiverse -deb-src http://archive.ubuntu.com/ubuntu focal-security main restricted universe multiverse -deb-src http://archive.ubuntu.com/ubuntu focal-updates main restricted universe multiverse -deb-src http://archive.ubuntu.com/ubuntu focal-proposed main restricted universe multiverse -deb-src http://archive.ubuntu.com/ubuntu focal-backports main restricted universe multiverse +deb-src http://archive.ubuntu.com/ubuntu noble main restricted universe multiverse +deb-src http://archive.ubuntu.com/ubuntu noble-security main restricted universe multiverse +deb-src http://archive.ubuntu.com/ubuntu noble-updates main restricted universe multiverse +deb-src http://archive.ubuntu.com/ubuntu noble-proposed main restricted universe multiverse +deb-src http://archive.ubuntu.com/ubuntu noble-backports main restricted universe multiverse -deb https://download.docker.com/linux/ubuntu focal stable -deb https://nvidia.github.io/nvidia-docker/ubuntu20.04/amd64 / -deb https://nvidia.github.io/libnvidia-container/stable/ubuntu20.04/amd64 / -deb https://nvidia.github.io/nvidia-container-runtime/stable/ubuntu20.04/amd64 / +deb https://download.docker.com/linux/ubuntu noble stable +deb https://nvidia.github.io/libnvidia-container/stable/deb/amd64 / -deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64 / +deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64 / -deb http://repo.download.nvidia.com/baseos/ubuntu/focal/x86_64/ focal common dgx -deb http://repo.download.nvidia.com/baseos/ubuntu/focal/x86_64/ focal-updates common dgx +deb http://repo.download.nvidia.com/baseos/ubuntu/noble/x86_64/ noble common dgx +deb http://repo.download.nvidia.com/baseos/ubuntu/noble/x86_64/ noble-updates common dgx clean http://archive.ubuntu.com/ubuntu clean https://download.docker.com @@ -266,10 +276,10 @@ sudo mkdir /var/www/html/repos Then, from the extracted mirror directory, copy the directories for each repository into the web root. -For example, assuming the extracted mirror directory is `/var/repos` and the repository is `nvidia-docker`: +For example, assuming the extracted mirror directory is `/var/repos` and the repository is `libnvidia-container`: ```bash -sudo cp -r /var/repos/mirror/nvidia.github.com/nvidia-docker/ /var/www/html/repos/nvidia-docker/ +sudo cp -r /var/repos/mirror/nvidia.github.io/libnvidia-container/ /var/www/html/repos/libnvidia-container/ ``` At this point, the downloaded package repositories should be available on your offline network via the package server. @@ -278,29 +288,27 @@ You can then add these downloaded repos to the `/etc/apt/sources.list` configura Line added to `/etc/apt/sources.list`: ``` -deb http://repo-server/ubuntu focal main restricted universe multiverse -deb http://repo-server/ubuntu focal-updates main restricted universe multiverse -deb http://repo-server/ubuntu focal-backports main restricted universe multiverse -deb http://repo-server/ubuntu focal-security main restricted universe multiverse +deb http://repo-server/ubuntu noble main restricted universe multiverse +deb http://repo-server/ubuntu noble-updates main restricted universe multiverse +deb http://repo-server/ubuntu noble-backports main restricted universe multiverse +deb http://repo-server/ubuntu noble-security main restricted universe multiverse ``` Lines added to `/etc/apt/sources.list.d/dgx.list`: ``` -deb http://repo-server/baseos/ubuntu/focal/x86_64/ focal common dgx -deb http://repo-server/baseos/ubuntu/focal/x86_64/ focal-updates common dgx +deb http://repo-server/baseos/ubuntu/noble/x86_64/ noble common dgx +deb http://repo-server/baseos/ubuntu/noble/x86_64/ noble-updates common dgx ``` Lines added to `/etc/apt/sources.list.d/cuda-compute-repo.list`: ``` -deb http://repo-server/cuda/repos/ubuntu2004/x86_64/ / +deb http://repo-server/cuda/repos/ubuntu2404/x86_64/ / ``` -Lines add to `/etc/apt/sources.list.d/nvidia-docker.list`: +Lines add to `/etc/apt/sources.list.d/nvidia-container-toolkit.list`: ``` -deb [trusted=yes] http://repo-server/libnvidia-container/stable/ubuntu20.04/amd64 / -deb [trusted=yes] http://repo-server/nvidia-container-runtime/stable/ubuntu20.04/amd64 / -deb [trusted=yes] http://repo-server/nvidia-docker/ubuntu20.04/amd64 / +deb [trusted=yes] http://repo-server/libnvidia-container/stable/deb/amd64 / ``` diff --git a/docs/airgap/ngc-ready.md b/docs/airgap/ngc-ready.md index 1acebe12a..0c6de8db9 100644 --- a/docs/airgap/ngc-ready.md +++ b/docs/airgap/ngc-ready.md @@ -24,7 +24,7 @@ The following Apt repositories will need to be mirrored in the offline environme - Ubuntu distribution repositories - Docker CE repository -- nvidia-docker repositories +- NVIDIA container runtime repositories For instructions on mirroring these repositories, see the [doc on Apt mirrors](./mirror-apt-repos.md). @@ -49,7 +49,7 @@ The following RPM repositories will need to be mirrored in the offline environme - Enterprise Linux distribution repositories (RHEL or CentOS, depending on your distro) - Docker CE repository -- nvidia-docker repositories +- NVIDIA container runtime repositories For instructions on mirroring these repositories, see the [doc on RPM mirrors](./mirror-rpm-repos.md). @@ -123,7 +123,7 @@ In all cases, you should edit the URLs appropriately to ensure they can download ### Configure DeepOps to use your mirrors for non-distribution package repositories -The NGC-Ready playbook depends on the Docker CE and nvidia-docker package repositories. +The NGC-Ready playbook depends on the Docker CE and NVIDIA container runtime package repositories. DeepOps sets up these repositories automatically during the installation. To configure alternate URLs for these repositories, set the following variables in your DeepOps configuration: diff --git a/docs/deepops/testing.md b/docs/deepops/testing.md index 7f2b1731a..c42b116f6 100644 --- a/docs/deepops/testing.md +++ b/docs/deepops/testing.md @@ -51,6 +51,8 @@ A short description of the nightly testing is outlined below. The full suit of t | --------------------------------------------------- | ----------------------------------------- | ------------------------------------------------------ | ----------------------------------------------------------------------- | ------------------------------------ | | Ubuntu 18.04 | x | x | x | | | Ubuntu 20.04 | | x | x | | +| Ubuntu 22.04 | | | | setup.sh and Molecule GitHub Actions | +| Ubuntu 24.04 | | | | setup.sh and Molecule GitHub Actions | | CentOS 7 | | x | x | | | CentOS | | | x | | | DGX OS | | | | Syntax-checked only; full validation requires DGX hardware | @@ -118,7 +120,8 @@ molecule init scenario -r --driver-name docker ``` 4. In the file `molecule/default/molecule.yml`, define the list of platforms to be tested. - DeepOps currently supports operating systems based on Ubuntu 18.04, Ubuntu 20.04, EL7, and EL8. + DeepOps currently supports operating systems based on Ubuntu 18.04, Ubuntu 20.04, Ubuntu 22.04, Ubuntu 24.04, EL7, and EL8. + The DGX software stack role also supports Red Hat Enterprise Linux / Rocky Linux 8 and 9 for DGX platform software installation. To test these stacks, the following `platforms` stanza can be used. ```yaml @@ -129,6 +132,12 @@ platforms: - name: ubuntu-2004 image: geerlingguy/docker-ubuntu2004-ansible pre_build_image: true + - name: ubuntu-2204 + image: geerlingguy/docker-ubuntu2204-ansible + pre_build_image: true + - name: ubuntu-2404 + image: geerlingguy/docker-ubuntu2404-ansible + pre_build_image: true - name: centos-7 image: geerlingguy/docker-centos7-ansible pre_build_image: true diff --git a/docs/ngc-ready/README.md b/docs/ngc-ready/README.md index d0929cca9..04b0e2e61 100644 --- a/docs/ngc-ready/README.md +++ b/docs/ngc-ready/README.md @@ -15,6 +15,8 @@ These instructions assume the following: - You have a NGC-Ready server. To determine if your server is NGC-Ready, please review the list of validated servers at the NGC-Ready Server documentation page - https://docs.nvidia.com/certification-programs/ngc-ready-systems/index.html - Your NGC-Ready Server has a compatible Linux distribution installed: - Ubuntu Server 20.04 LTS + - Ubuntu Server 22.04 LTS + - Ubuntu Server 24.04 LTS - CentOS 7 ## Setup diff --git a/docs/slurm-cluster/slurm-single-node.md b/docs/slurm-cluster/slurm-single-node.md index b048b614e..18174b52d 100644 --- a/docs/slurm-cluster/slurm-single-node.md +++ b/docs/slurm-cluster/slurm-single-node.md @@ -15,7 +15,7 @@ Single Node Slurm Deployment Guide ## Introduction -The general requirements and procedure for Slurm setup via deepops is documented in the [README.md](README.md) for the slurm-cluster. The instructions below outline the steps to deviate from the general setup to enable single node DeepOps Slurm setup. The machine on which Slurm is being deployed should be up to date in a stable state with GPU drivers already installed and functional. The supported Operating Systems are Ubuntu (version 18 and 20), CentOS and RHEL (version 7 and 8 albeit version 8 is preferred). +The general requirements and procedure for Slurm setup via deepops is documented in the [README.md](README.md) for the slurm-cluster. The instructions below outline the steps to deviate from the general setup to enable single node DeepOps Slurm setup. The machine on which Slurm is being deployed should be up to date in a stable state with GPU drivers already installed and functional. The supported operating systems are Ubuntu 18.04, 20.04, 22.04, and 24.04; CentOS 7 and 8; and RHEL 7 and 8, with RHEL 8 preferred among the RHEL paths. ## Deployment Procedure diff --git a/playbooks/container/nvidia-docker.yml b/playbooks/container/nvidia-docker.yml index 3ea34972c..b15edc99f 100644 --- a/playbooks/container/nvidia-docker.yml +++ b/playbooks/container/nvidia-docker.yml @@ -16,10 +16,19 @@ state: absent when: docker_install | default('yes') + - name: install NVIDIA Container Toolkit on Ubuntu 24.04 and newer + include_role: + name: nvidia_container_toolkit + when: + - ansible_local['gpus']['count'] and ansible_distribution == "Ubuntu" + - ansible_distribution_version is version('24.04', '>=') + - docker_install | default('yes') + - name: install nvidia-docker include_role: name: nvidia.nvidia_docker when: - ansible_local['gpus']['count'] and (ansible_distribution == "Ubuntu" or ansible_os_family == "RedHat") + - not (ansible_distribution == "Ubuntu" and ansible_distribution_version is version('24.04', '>=')) - docker_install | default('yes') environment: "{{ proxy_env if proxy_env is defined else {} }}" diff --git a/roles/nvidia_container_toolkit/defaults/main.yml b/roles/nvidia_container_toolkit/defaults/main.yml new file mode 100644 index 000000000..87577f72c --- /dev/null +++ b/roles/nvidia_container_toolkit/defaults/main.yml @@ -0,0 +1,10 @@ +--- +nvidia_container_toolkit_repo_base_url: "https://nvidia.github.io/libnvidia-container" +nvidia_container_toolkit_repo_gpg_url: "{{ nvidia_container_toolkit_repo_base_url }}/gpgkey" +nvidia_container_toolkit_keyring_ascii_path: "/usr/share/keyrings/nvidia-container-toolkit-keyring.asc" +nvidia_container_toolkit_keyring_path: "/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg" +nvidia_container_toolkit_apt_source_path: "/etc/apt/sources.list.d/nvidia-container-toolkit.list" +nvidia_container_toolkit_package: "nvidia-container-toolkit" +nvidia_container_toolkit_configure_docker: true +nvidia_container_toolkit_set_as_default_runtime: true +nvidia_container_toolkit_restart_docker: true diff --git a/roles/nvidia_container_toolkit/handlers/main.yml b/roles/nvidia_container_toolkit/handlers/main.yml new file mode 100644 index 000000000..1e67b7367 --- /dev/null +++ b/roles/nvidia_container_toolkit/handlers/main.yml @@ -0,0 +1,6 @@ +--- +- name: restart docker + ansible.builtin.service: + name: docker + state: restarted + when: nvidia_container_toolkit_restart_docker | bool diff --git a/roles/nvidia_container_toolkit/tasks/main.yml b/roles/nvidia_container_toolkit/tasks/main.yml new file mode 100644 index 000000000..c681d88e6 --- /dev/null +++ b/roles/nvidia_container_toolkit/tasks/main.yml @@ -0,0 +1,114 @@ +--- +- name: Ubuntu | verify supported distribution + ansible.builtin.assert: + that: + - ansible_distribution == "Ubuntu" + fail_msg: "The nvidia_container_toolkit role currently supports Ubuntu only." + +- name: Ubuntu | set package architecture + ansible.builtin.set_fact: + nvidia_container_toolkit_deb_arch: "{{ _nvidia_container_toolkit_arch_map.get(ansible_architecture, ansible_architecture) }}" + vars: + _nvidia_container_toolkit_arch_map: + aarch64: arm64 + arm64: arm64 + x86_64: amd64 + +- name: Ubuntu | install repository prerequisites + ansible.builtin.apt: + name: + - ca-certificates + - gnupg + state: present + update_cache: true + when: nvidia_container_toolkit_repo_base_url | length > 0 + +- name: Ubuntu | ensure keyring directory exists + ansible.builtin.file: + path: "{{ nvidia_container_toolkit_keyring_path | dirname }}" + state: directory + owner: root + group: root + mode: "0755" + +- name: Ubuntu | check NVIDIA Container Toolkit keyring + ansible.builtin.stat: + path: "{{ nvidia_container_toolkit_keyring_path }}" + register: nvidia_container_toolkit_keyring + +- name: Ubuntu | download NVIDIA Container Toolkit GPG key + ansible.builtin.get_url: + url: "{{ nvidia_container_toolkit_repo_gpg_url }}" + dest: "{{ nvidia_container_toolkit_keyring_ascii_path }}" + owner: root + group: root + mode: "0644" + register: nvidia_container_toolkit_key + environment: "{{ proxy_env if proxy_env is defined else {} }}" + +- name: Ubuntu | install NVIDIA Container Toolkit GPG keyring + ansible.builtin.command: + cmd: "gpg --dearmor --yes -o {{ nvidia_container_toolkit_keyring_path }} {{ nvidia_container_toolkit_keyring_ascii_path }}" + when: nvidia_container_toolkit_key.changed or not nvidia_container_toolkit_keyring.stat.exists + register: nvidia_container_toolkit_keyring_install + changed_when: true + +- name: Ubuntu | set NVIDIA Container Toolkit GPG keyring permissions + ansible.builtin.file: + path: "{{ nvidia_container_toolkit_keyring_path }}" + owner: root + group: root + mode: "0644" + +- name: Ubuntu | configure NVIDIA Container Toolkit APT repository + ansible.builtin.copy: + content: | + deb [signed-by={{ nvidia_container_toolkit_keyring_path }}] {{ nvidia_container_toolkit_repo_base_url }}/stable/deb/{{ nvidia_container_toolkit_deb_arch }} / + dest: "{{ nvidia_container_toolkit_apt_source_path }}" + owner: root + group: root + mode: "0644" + register: nvidia_container_toolkit_apt_source + +- name: Ubuntu | install NVIDIA Container Toolkit + ansible.builtin.apt: + name: "{{ nvidia_container_toolkit_package }}" + state: present + update_cache: "{{ nvidia_container_toolkit_apt_source.changed or nvidia_container_toolkit_keyring_install.changed | default(false) }}" + environment: "{{ proxy_env if proxy_env is defined else {} }}" + +- name: Docker | ensure Docker configuration directory exists + ansible.builtin.file: + path: /etc/docker + state: directory + owner: root + group: root + mode: "0755" + when: nvidia_container_toolkit_configure_docker | bool + +- name: Docker | check NVIDIA runtime configuration + ansible.builtin.command: + cmd: >- + python3 -c 'import json, pathlib, sys; + p = pathlib.Path("/etc/docker/daemon.json"); + required_default = {{ nvidia_container_toolkit_set_as_default_runtime | bool | ternary("True", "False") }}; + data = {}; + text = p.read_text().strip() if p.exists() else ""; + data = json.loads(text) if text else {}; + runtime = data.get("runtimes", {}).get("nvidia", {}); + ok = runtime.get("path") in ("nvidia-container-runtime", "/usr/bin/nvidia-container-runtime"); + ok = ok and (not required_default or data.get("default-runtime") == "nvidia"); + sys.exit(0 if ok else 1)' + register: nvidia_container_toolkit_docker_runtime + failed_when: false + changed_when: false + when: nvidia_container_toolkit_configure_docker | bool + +- name: Docker | configure NVIDIA runtime + ansible.builtin.command: + cmd: "nvidia-ctk runtime configure --runtime=docker{{ ' --set-as-default' if nvidia_container_toolkit_set_as_default_runtime | bool else '' }}" + when: + - nvidia_container_toolkit_configure_docker | bool + - nvidia_container_toolkit_docker_runtime.rc != 0 + changed_when: true + notify: restart docker From a6242f6a4696d1c06304db988a123d22b457267c Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Wed, 27 May 2026 22:04:56 -0600 Subject: [PATCH 15/27] Refresh DCGM exporter --- .../files/k8s-cluster/dcgm-custom-metrics.csv | 89 +++++++++++-------- docs/deepops/update-deepops.md | 2 +- docs/slurm-cluster/slurm-monitor.md | 4 +- roles/nvidia-dcgm-exporter/defaults/main.yml | 2 +- .../files/dcgm-custom-metrics.csv | 89 +++++++++++-------- workloads/services/k8s/dcgm-exporter.yml | 6 +- 6 files changed, 116 insertions(+), 76 deletions(-) diff --git a/config.example/files/k8s-cluster/dcgm-custom-metrics.csv b/config.example/files/k8s-cluster/dcgm-custom-metrics.csv index 66875527f..2eed7a651 100644 --- a/config.example/files/k8s-cluster/dcgm-custom-metrics.csv +++ b/config.example/files/k8s-cluster/dcgm-custom-metrics.csv @@ -1,55 +1,63 @@ -# Format,, -# If line starts with a '#' it is considered a comment,, +# Format +# If line starts with a '#' it is considered a comment # DCGM FIELD, Prometheus metric type, help message -# Clocks,, +# Clocks DCGM_FI_DEV_SM_CLOCK, gauge, SM clock frequency (in MHz). DCGM_FI_DEV_MEM_CLOCK, gauge, Memory clock frequency (in MHz). -# Temperature,, +# Temperature DCGM_FI_DEV_MEMORY_TEMP, gauge, Memory temperature (in C). DCGM_FI_DEV_GPU_TEMP, gauge, GPU temperature (in C). -# Power,, +# Power DCGM_FI_DEV_POWER_USAGE, gauge, Power draw (in W). DCGM_FI_DEV_TOTAL_ENERGY_CONSUMPTION, counter, Total energy consumption since boot (in mJ). -# PCIE,, -DCGM_FI_DEV_PCIE_TX_THROUGHPUT, counter, Total number of bytes transmitted through PCIe TX (in KB) via NVML. -DCGM_FI_DEV_PCIE_RX_THROUGHPUT, counter, Total number of bytes received through PCIe RX (in KB) via NVML. +# PCIE +# DCGM_FI_PROF_PCIE_TX_BYTES, counter, Total number of bytes transmitted through PCIe TX via NVML. +# DCGM_FI_PROF_PCIE_RX_BYTES, counter, Total number of bytes received through PCIe RX via NVML. DCGM_FI_DEV_PCIE_REPLAY_COUNTER, counter, Total number of PCIe retries. -# Utilization (the sample period varies depending on the product),, +# Utilization (the sample period varies depending on the product) DCGM_FI_DEV_GPU_UTIL, gauge, GPU utilization (in %). DCGM_FI_DEV_MEM_COPY_UTIL, gauge, Memory utilization (in %). DCGM_FI_DEV_ENC_UTIL, gauge, Encoder utilization (in %). DCGM_FI_DEV_DEC_UTIL , gauge, Decoder utilization (in %). -# Errors and violations,, -DCGM_FI_DEV_XID_ERRORS, gauge, Value of the last XID error encountered. -# DCGM_FI_DEV_POWER_VIOLATION, counter, Throttling duration due to power constraints (in us). -# DCGM_FI_DEV_THERMAL_VIOLATION, counter, Throttling duration due to thermal constraints (in us). -# DCGM_FI_DEV_SYNC_BOOST_VIOLATION, counter, Throttling duration due to sync-boost constraints (in us). -# DCGM_FI_DEV_BOARD_LIMIT_VIOLATION, counter, Throttling duration due to board limit constraints (in us). -# DCGM_FI_DEV_LOW_UTIL_VIOLATION, counter, Throttling duration due to low utilization (in us). -# DCGM_FI_DEV_RELIABILITY_VIOLATION, counter, Throttling duration due to reliability constraints (in us). +# Errors and violations +DCGM_FI_DEV_XID_ERRORS, gauge, Value of the last XID error encountered. +# DCGM_FI_DEV_POWER_VIOLATION, counter, Throttling duration due to power constraints (in ns). +# DCGM_FI_DEV_THERMAL_VIOLATION, counter, Throttling duration due to thermal constraints (in ns). +# DCGM_FI_DEV_SYNC_BOOST_VIOLATION, counter, Throttling duration due to sync-boost constraints (in ns). +# DCGM_FI_DEV_BOARD_LIMIT_VIOLATION, counter, Throttling duration due to board limit constraints (in ns). +# DCGM_FI_DEV_LOW_UTIL_VIOLATION, counter, Throttling duration due to low utilization (in ns). +# DCGM_FI_DEV_RELIABILITY_VIOLATION, counter, Throttling duration due to reliability constraints (in ns). -# Memory usage,, +# DCGM Exporter fields + +# DCGM_EXP_CLOCK_EVENTS_COUNT, counter, reported clock events +# DCGM_EXP_XID_ERRORS_COUNT, counter, reported XIDs during last window +# DCGM_EXP_GPU_HEALTH_STATUS, counter, DCGM reported health status +# DCGM_EXP_P2P_STATUS, counter, P2P NvLink status + +# Memory usage DCGM_FI_DEV_FB_FREE, gauge, Framebuffer memory free (in MiB). DCGM_FI_DEV_FB_USED, gauge, Framebuffer memory used (in MiB). +DCGM_FI_DEV_FB_RESERVED, gauge, Framebuffer memory reserved (in MiB). -# ECC,, +# ECC # DCGM_FI_DEV_ECC_SBE_VOL_TOTAL, counter, Total number of single-bit volatile ECC errors. # DCGM_FI_DEV_ECC_DBE_VOL_TOTAL, counter, Total number of double-bit volatile ECC errors. # DCGM_FI_DEV_ECC_SBE_AGG_TOTAL, counter, Total number of single-bit persistent ECC errors. # DCGM_FI_DEV_ECC_DBE_AGG_TOTAL, counter, Total number of double-bit persistent ECC errors. -# Retired pages,, +# Retired pages # DCGM_FI_DEV_RETIRED_SBE, counter, Total number of retired pages due to single-bit errors. # DCGM_FI_DEV_RETIRED_DBE, counter, Total number of retired pages due to double-bit errors. # DCGM_FI_DEV_RETIRED_PENDING, counter, Total number of pages pending retirement. -# NVLink,, +# NVLink # DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_TOTAL, counter, Total number of NVLink flow-control CRC errors. # DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_TOTAL, counter, Total number of NVLink data CRC errors. # DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_TOTAL, counter, Total number of NVLink retries. @@ -57,23 +65,34 @@ DCGM_FI_DEV_FB_USED, gauge, Framebuffer memory used (in MiB). DCGM_FI_DEV_NVLINK_BANDWIDTH_TOTAL, counter, Total number of NVLink bandwidth counters for all lanes. # DCGM_FI_DEV_NVLINK_BANDWIDTH_L0, counter, The number of bytes of active NVLink rx or tx data including both header and payload. -# VGPU License status,, -# DCGM_FI_DEV_VGPU_LICENSE_STATUS, gauge, vGPU License status +# VGPU License status +DCGM_FI_DEV_VGPU_LICENSE_STATUS, gauge, vGPU License status -# Remapped rows,, +# Remapped rows DCGM_FI_DEV_UNCORRECTABLE_REMAPPED_ROWS, counter, Number of remapped rows for uncorrectable errors DCGM_FI_DEV_CORRECTABLE_REMAPPED_ROWS, counter, Number of remapped rows for correctable errors DCGM_FI_DEV_ROW_REMAP_FAILURE, gauge, Whether remapping of rows has failed -# DCP metrics,, -DCGM_FI_PROF_GR_ENGINE_ACTIVE, gauge, Ratio of time the graphics engine is active (in %). -DCGM_FI_PROF_SM_ACTIVE, gauge, The ratio of cycles an SM has at least 1 warp assigned (in %). -DCGM_FI_PROF_SM_OCCUPANCY, gauge, The ratio of number of warps resident on an SM (in %). -DCGM_FI_PROF_PIPE_TENSOR_ACTIVE, gauge, Ratio of cycles the tensor (HMMA) pipe is active (in %). -DCGM_FI_PROF_DRAM_ACTIVE, gauge, Ratio of cycles the device memory interface is active sending or receiving data (in %). -DCGM_FI_PROF_PIPE_FP64_ACTIVE, gauge, Ratio of cycles the fp64 pipes are active (in %). -DCGM_FI_PROF_PIPE_FP32_ACTIVE, gauge, Ratio of cycles the fp32 pipes are active (in %). -DCGM_FI_PROF_PIPE_FP16_ACTIVE, gauge, Ratio of cycles the fp16 pipes are active (in %). -DCGM_FI_PROF_PCIE_TX_BYTES, counter, The number of bytes of active pcie tx data including both header and payload. -DCGM_FI_PROF_PCIE_RX_BYTES, counter, The number of bytes of active pcie rx data including both header and payload. +# Static configuration information. These appear as labels on the other metrics +DCGM_FI_DRIVER_VERSION, label, Driver Version +# DCGM_FI_NVML_VERSION, label, NVML Version +# DCGM_FI_DEV_BRAND, label, Device Brand +# DCGM_FI_DEV_SERIAL, label, Device Serial Number +# DCGM_FI_DEV_OEM_INFOROM_VER, label, OEM inforom version +# DCGM_FI_DEV_ECC_INFOROM_VER, label, ECC inforom version +# DCGM_FI_DEV_POWER_INFOROM_VER, label, Power management object inforom version +# DCGM_FI_DEV_INFOROM_IMAGE_VER, label, Inforom image version +# DCGM_FI_DEV_VBIOS_VERSION, label, VBIOS version of the device +# Datacenter Profiling (DCP) metrics +# NOTE: supported on Nvidia datacenter Volta GPUs and newer +DCGM_FI_PROF_GR_ENGINE_ACTIVE, gauge, Ratio of time the graphics engine is active. +# DCGM_FI_PROF_SM_ACTIVE, gauge, The ratio of cycles an SM has at least 1 warp assigned. +# DCGM_FI_PROF_SM_OCCUPANCY, gauge, The ratio of number of warps resident on an SM. +DCGM_FI_PROF_PIPE_TENSOR_ACTIVE, gauge, Ratio of cycles the tensor (HMMA) pipe is active. +DCGM_FI_PROF_DRAM_ACTIVE, gauge, Ratio of cycles the device memory interface is active sending or receiving data. +# DCGM_FI_PROF_PIPE_FP64_ACTIVE, gauge, Ratio of cycles the fp64 pipes are active. +# DCGM_FI_PROF_PIPE_FP32_ACTIVE, gauge, Ratio of cycles the fp32 pipes are active. +# DCGM_FI_PROF_PIPE_FP16_ACTIVE, gauge, Ratio of cycles the fp16 pipes are active. +DCGM_FI_PROF_PCIE_TX_BYTES, gauge, The rate of data transmitted over the PCIe bus - including both protocol headers and data payloads - in bytes per second. +DCGM_FI_PROF_PCIE_RX_BYTES, gauge, The rate of data received over the PCIe bus - including both protocol headers and data payloads - in bytes per second. diff --git a/docs/deepops/update-deepops.md b/docs/deepops/update-deepops.md index a1f7af047..63d0a75d5 100644 --- a/docs/deepops/update-deepops.md +++ b/docs/deepops/update-deepops.md @@ -524,7 +524,7 @@ systemctl restart docker.node-exporter For the NVIDIA DCGM Exporter, we do pin a particular version of the container. To update to a newer version, edit your DeepOps configuration to specify a new container tag: ``` -nvidia_dcgm_container_version: "2.1.8-2.4.0-rc.2-ubuntu20.04" +nvidia_dcgm_container_version: "4.5.3-4.8.2-distroless" ``` Then re-run the playbook: diff --git a/docs/slurm-cluster/slurm-monitor.md b/docs/slurm-cluster/slurm-monitor.md index 88847a74f..9c9a717af 100644 --- a/docs/slurm-cluster/slurm-monitor.md +++ b/docs/slurm-cluster/slurm-monitor.md @@ -22,7 +22,7 @@ $ sudo docker ps CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES 139fe402640f quay.io/prometheus/node-exporter "/bin/node_exporter …" 56 minutes ago Up 56 minutes docker.node-exporter.service -0da9e3f1a7c8 nvidia/dcgm-exporter "/usr/bin/dcgm-expor…" 56 minutes ago Up 56 minutes 0.0.0.0:9400->9400/tcp docker.dcgm-exporter.service +0da9e3f1a7c8 nvcr.io/nvidia/k8s/dcgm-exporter "/usr/bin/dcgm-expor…" 56 minutes ago Up 56 minutes 0.0.0.0:9400->9400/tcp docker.dcgm-exporter.service ``` ## Grafana @@ -102,4 +102,4 @@ Describes how to configure Alertmanager. Visit http://\:9093 for checking and setting about alerting. ![Alertmanager](../img/slurm_monitoring_alertmanager02.png) -For more information on constructing Alertmanager alerting see the official [Alertmanager](https://prometheus.io/docs/alerting/latest/alertmanager/). \ No newline at end of file +For more information on constructing Alertmanager alerting see the official [Alertmanager](https://prometheus.io/docs/alerting/latest/alertmanager/). diff --git a/roles/nvidia-dcgm-exporter/defaults/main.yml b/roles/nvidia-dcgm-exporter/defaults/main.yml index 63e0c8785..123658d96 100644 --- a/roles/nvidia-dcgm-exporter/defaults/main.yml +++ b/roles/nvidia-dcgm-exporter/defaults/main.yml @@ -1,4 +1,4 @@ -nvidia_dcgm_container_version: "2.1.8-2.4.0-rc.2-ubuntu20.04" +nvidia_dcgm_container_version: "4.5.3-4.8.2-distroless" nvidia_dcgm_container: "nvcr.io/nvidia/k8s/dcgm-exporter:{{ nvidia_dcgm_container_version }}" nvidia_dcgm_container_config_dir: "/opt/deepops/nvidia-dcgm-exporter" nvidia_dcgm_container_custom_metrics_file: "dcgm-custom-metrics.csv" diff --git a/roles/nvidia-dcgm-exporter/files/dcgm-custom-metrics.csv b/roles/nvidia-dcgm-exporter/files/dcgm-custom-metrics.csv index 66875527f..2eed7a651 100644 --- a/roles/nvidia-dcgm-exporter/files/dcgm-custom-metrics.csv +++ b/roles/nvidia-dcgm-exporter/files/dcgm-custom-metrics.csv @@ -1,55 +1,63 @@ -# Format,, -# If line starts with a '#' it is considered a comment,, +# Format +# If line starts with a '#' it is considered a comment # DCGM FIELD, Prometheus metric type, help message -# Clocks,, +# Clocks DCGM_FI_DEV_SM_CLOCK, gauge, SM clock frequency (in MHz). DCGM_FI_DEV_MEM_CLOCK, gauge, Memory clock frequency (in MHz). -# Temperature,, +# Temperature DCGM_FI_DEV_MEMORY_TEMP, gauge, Memory temperature (in C). DCGM_FI_DEV_GPU_TEMP, gauge, GPU temperature (in C). -# Power,, +# Power DCGM_FI_DEV_POWER_USAGE, gauge, Power draw (in W). DCGM_FI_DEV_TOTAL_ENERGY_CONSUMPTION, counter, Total energy consumption since boot (in mJ). -# PCIE,, -DCGM_FI_DEV_PCIE_TX_THROUGHPUT, counter, Total number of bytes transmitted through PCIe TX (in KB) via NVML. -DCGM_FI_DEV_PCIE_RX_THROUGHPUT, counter, Total number of bytes received through PCIe RX (in KB) via NVML. +# PCIE +# DCGM_FI_PROF_PCIE_TX_BYTES, counter, Total number of bytes transmitted through PCIe TX via NVML. +# DCGM_FI_PROF_PCIE_RX_BYTES, counter, Total number of bytes received through PCIe RX via NVML. DCGM_FI_DEV_PCIE_REPLAY_COUNTER, counter, Total number of PCIe retries. -# Utilization (the sample period varies depending on the product),, +# Utilization (the sample period varies depending on the product) DCGM_FI_DEV_GPU_UTIL, gauge, GPU utilization (in %). DCGM_FI_DEV_MEM_COPY_UTIL, gauge, Memory utilization (in %). DCGM_FI_DEV_ENC_UTIL, gauge, Encoder utilization (in %). DCGM_FI_DEV_DEC_UTIL , gauge, Decoder utilization (in %). -# Errors and violations,, -DCGM_FI_DEV_XID_ERRORS, gauge, Value of the last XID error encountered. -# DCGM_FI_DEV_POWER_VIOLATION, counter, Throttling duration due to power constraints (in us). -# DCGM_FI_DEV_THERMAL_VIOLATION, counter, Throttling duration due to thermal constraints (in us). -# DCGM_FI_DEV_SYNC_BOOST_VIOLATION, counter, Throttling duration due to sync-boost constraints (in us). -# DCGM_FI_DEV_BOARD_LIMIT_VIOLATION, counter, Throttling duration due to board limit constraints (in us). -# DCGM_FI_DEV_LOW_UTIL_VIOLATION, counter, Throttling duration due to low utilization (in us). -# DCGM_FI_DEV_RELIABILITY_VIOLATION, counter, Throttling duration due to reliability constraints (in us). +# Errors and violations +DCGM_FI_DEV_XID_ERRORS, gauge, Value of the last XID error encountered. +# DCGM_FI_DEV_POWER_VIOLATION, counter, Throttling duration due to power constraints (in ns). +# DCGM_FI_DEV_THERMAL_VIOLATION, counter, Throttling duration due to thermal constraints (in ns). +# DCGM_FI_DEV_SYNC_BOOST_VIOLATION, counter, Throttling duration due to sync-boost constraints (in ns). +# DCGM_FI_DEV_BOARD_LIMIT_VIOLATION, counter, Throttling duration due to board limit constraints (in ns). +# DCGM_FI_DEV_LOW_UTIL_VIOLATION, counter, Throttling duration due to low utilization (in ns). +# DCGM_FI_DEV_RELIABILITY_VIOLATION, counter, Throttling duration due to reliability constraints (in ns). -# Memory usage,, +# DCGM Exporter fields + +# DCGM_EXP_CLOCK_EVENTS_COUNT, counter, reported clock events +# DCGM_EXP_XID_ERRORS_COUNT, counter, reported XIDs during last window +# DCGM_EXP_GPU_HEALTH_STATUS, counter, DCGM reported health status +# DCGM_EXP_P2P_STATUS, counter, P2P NvLink status + +# Memory usage DCGM_FI_DEV_FB_FREE, gauge, Framebuffer memory free (in MiB). DCGM_FI_DEV_FB_USED, gauge, Framebuffer memory used (in MiB). +DCGM_FI_DEV_FB_RESERVED, gauge, Framebuffer memory reserved (in MiB). -# ECC,, +# ECC # DCGM_FI_DEV_ECC_SBE_VOL_TOTAL, counter, Total number of single-bit volatile ECC errors. # DCGM_FI_DEV_ECC_DBE_VOL_TOTAL, counter, Total number of double-bit volatile ECC errors. # DCGM_FI_DEV_ECC_SBE_AGG_TOTAL, counter, Total number of single-bit persistent ECC errors. # DCGM_FI_DEV_ECC_DBE_AGG_TOTAL, counter, Total number of double-bit persistent ECC errors. -# Retired pages,, +# Retired pages # DCGM_FI_DEV_RETIRED_SBE, counter, Total number of retired pages due to single-bit errors. # DCGM_FI_DEV_RETIRED_DBE, counter, Total number of retired pages due to double-bit errors. # DCGM_FI_DEV_RETIRED_PENDING, counter, Total number of pages pending retirement. -# NVLink,, +# NVLink # DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_TOTAL, counter, Total number of NVLink flow-control CRC errors. # DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_TOTAL, counter, Total number of NVLink data CRC errors. # DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_TOTAL, counter, Total number of NVLink retries. @@ -57,23 +65,34 @@ DCGM_FI_DEV_FB_USED, gauge, Framebuffer memory used (in MiB). DCGM_FI_DEV_NVLINK_BANDWIDTH_TOTAL, counter, Total number of NVLink bandwidth counters for all lanes. # DCGM_FI_DEV_NVLINK_BANDWIDTH_L0, counter, The number of bytes of active NVLink rx or tx data including both header and payload. -# VGPU License status,, -# DCGM_FI_DEV_VGPU_LICENSE_STATUS, gauge, vGPU License status +# VGPU License status +DCGM_FI_DEV_VGPU_LICENSE_STATUS, gauge, vGPU License status -# Remapped rows,, +# Remapped rows DCGM_FI_DEV_UNCORRECTABLE_REMAPPED_ROWS, counter, Number of remapped rows for uncorrectable errors DCGM_FI_DEV_CORRECTABLE_REMAPPED_ROWS, counter, Number of remapped rows for correctable errors DCGM_FI_DEV_ROW_REMAP_FAILURE, gauge, Whether remapping of rows has failed -# DCP metrics,, -DCGM_FI_PROF_GR_ENGINE_ACTIVE, gauge, Ratio of time the graphics engine is active (in %). -DCGM_FI_PROF_SM_ACTIVE, gauge, The ratio of cycles an SM has at least 1 warp assigned (in %). -DCGM_FI_PROF_SM_OCCUPANCY, gauge, The ratio of number of warps resident on an SM (in %). -DCGM_FI_PROF_PIPE_TENSOR_ACTIVE, gauge, Ratio of cycles the tensor (HMMA) pipe is active (in %). -DCGM_FI_PROF_DRAM_ACTIVE, gauge, Ratio of cycles the device memory interface is active sending or receiving data (in %). -DCGM_FI_PROF_PIPE_FP64_ACTIVE, gauge, Ratio of cycles the fp64 pipes are active (in %). -DCGM_FI_PROF_PIPE_FP32_ACTIVE, gauge, Ratio of cycles the fp32 pipes are active (in %). -DCGM_FI_PROF_PIPE_FP16_ACTIVE, gauge, Ratio of cycles the fp16 pipes are active (in %). -DCGM_FI_PROF_PCIE_TX_BYTES, counter, The number of bytes of active pcie tx data including both header and payload. -DCGM_FI_PROF_PCIE_RX_BYTES, counter, The number of bytes of active pcie rx data including both header and payload. +# Static configuration information. These appear as labels on the other metrics +DCGM_FI_DRIVER_VERSION, label, Driver Version +# DCGM_FI_NVML_VERSION, label, NVML Version +# DCGM_FI_DEV_BRAND, label, Device Brand +# DCGM_FI_DEV_SERIAL, label, Device Serial Number +# DCGM_FI_DEV_OEM_INFOROM_VER, label, OEM inforom version +# DCGM_FI_DEV_ECC_INFOROM_VER, label, ECC inforom version +# DCGM_FI_DEV_POWER_INFOROM_VER, label, Power management object inforom version +# DCGM_FI_DEV_INFOROM_IMAGE_VER, label, Inforom image version +# DCGM_FI_DEV_VBIOS_VERSION, label, VBIOS version of the device +# Datacenter Profiling (DCP) metrics +# NOTE: supported on Nvidia datacenter Volta GPUs and newer +DCGM_FI_PROF_GR_ENGINE_ACTIVE, gauge, Ratio of time the graphics engine is active. +# DCGM_FI_PROF_SM_ACTIVE, gauge, The ratio of cycles an SM has at least 1 warp assigned. +# DCGM_FI_PROF_SM_OCCUPANCY, gauge, The ratio of number of warps resident on an SM. +DCGM_FI_PROF_PIPE_TENSOR_ACTIVE, gauge, Ratio of cycles the tensor (HMMA) pipe is active. +DCGM_FI_PROF_DRAM_ACTIVE, gauge, Ratio of cycles the device memory interface is active sending or receiving data. +# DCGM_FI_PROF_PIPE_FP64_ACTIVE, gauge, Ratio of cycles the fp64 pipes are active. +# DCGM_FI_PROF_PIPE_FP32_ACTIVE, gauge, Ratio of cycles the fp32 pipes are active. +# DCGM_FI_PROF_PIPE_FP16_ACTIVE, gauge, Ratio of cycles the fp16 pipes are active. +DCGM_FI_PROF_PCIE_TX_BYTES, gauge, The rate of data transmitted over the PCIe bus - including both protocol headers and data payloads - in bytes per second. +DCGM_FI_PROF_PCIE_RX_BYTES, gauge, The rate of data received over the PCIe bus - including both protocol headers and data payloads - in bytes per second. diff --git a/workloads/services/k8s/dcgm-exporter.yml b/workloads/services/k8s/dcgm-exporter.yml index d5b9d268f..165adb448 100644 --- a/workloads/services/k8s/dcgm-exporter.yml +++ b/workloads/services/k8s/dcgm-exporter.yml @@ -23,9 +23,9 @@ spec: nodeSelector: hardware-type: NVIDIAGPU containers: - - image: "nvcr.io/nvidia/k8s/dcgm-exporter:2.3.2-2.6.3-ubuntu20.04" + - image: "nvcr.io/nvidia/k8s/dcgm-exporter:4.5.3-4.8.2-distroless" name: nvidia-dcgm-exporter - command: ["/usr/bin/dcgm-exporter", "-f", "/etc/dcgm-config/dcgm-custom-metrics.csv"] + args: ["-f", "/etc/dcgm-config/dcgm-custom-metrics.csv"] env: - name: "DCGM_EXPORTER_LISTEN" value: ":9400" @@ -36,6 +36,8 @@ spec: runAsUser: 0 capabilities: add: ["SYS_ADMIN"] + drop: ["ALL"] + allowPrivilegeEscalation: false volumeMounts: - name: "pod-gpu-resources" readOnly: true From d4632fe4f5946d932117d177fa82ed49049aa9da Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Wed, 27 May 2026 22:11:58 -0600 Subject: [PATCH 16/27] Refresh CUDA example images --- docs/airgap/mirror-docker-images.md | 20 +++++++------------ docs/airgap/ngc-ready.md | 18 ++++++++--------- virtual/scripts/setup_k8s.sh | 2 +- .../examples/k8s/cluster-gpu-test-job.yml | 2 +- workloads/examples/k8s/gpu-test-job.yml | 3 +-- workloads/jenkins/scripts/run-gpu-job.sh | 2 +- .../jenkins/scripts/test-slurm-enroot-job.sh | 2 +- 7 files changed, 21 insertions(+), 28 deletions(-) diff --git a/docs/airgap/mirror-docker-images.md b/docs/airgap/mirror-docker-images.md index b65d29dcc..ccd04df47 100644 --- a/docs/airgap/mirror-docker-images.md +++ b/docs/airgap/mirror-docker-images.md @@ -42,9 +42,8 @@ Then, for each image you want to download, you should pull the image from the re In this example, we're saving all our Docker images to `/tmp/images`: ``` -$ docker pull nvidia/cuda:11.1-devel-ubuntu20.04 -$ docker save -o /tmp/images/nvidia-cuda-11.1-devel-ubuntu20.04.tar nvidia/cuda:11.1-devel-ubuntu20.04 ->>>>>>> e1d0a775 (Airgap documentation update.) +$ docker pull nvcr.io/nvidia/cuda:12.4.1-base-ubuntu22.04 +$ docker save -o /tmp/images/nvidia-cuda-12.4.1-base-ubuntu22.04.tar nvcr.io/nvidia/cuda:12.4.1-base-ubuntu22.04 ``` Additionally, you should download and save the [`registry` image](https://hub.docker.com/_/registry) so that you can deploy a local registry on the offline network. @@ -76,13 +75,8 @@ Additionally, we assume that the `registry` image was included when you transfer Load the registry image into the Docker image cache of your container registry host: -<<<<<<< HEAD ```bash -docker load < /tmp/images/registry-2.7.tar -======= -``` -$ docker load -i /tmp/images/registry-2.7.tar ->>>>>>> e1d0a775 (Airgap documentation update.) +$ docker load -i /tmp/images/registry-3.1.1.tar ``` Then create a Docker volume to store your container images: @@ -99,7 +93,7 @@ docker run -d \ --restart=always \ --name registry \ -v registry-images:/var/lib/registry \ - registry:2.7 + registry:3.1.1 ``` ## Configuring your hosts to use the offline container registry @@ -121,7 +115,7 @@ docker_insecure_registries: Once your registry is running and you've configured your hosts to access it, you can load additional images and push them to the offline registry: ```bash -docker load < /tmp/images/nvidia-cuda-11.1-devel-ubuntu20.04.tar -docker tag nvidia/cuda:11.1-devel-ubuntu20.04 registry-host:5000/nvidia/cuda:11.1-devel-ubuntu20.04 -docker push registry-host:5000/nvidia/cuda:11.1-devel-ubuntu20.04 +docker load -i /tmp/images/nvidia-cuda-12.4.1-base-ubuntu22.04.tar +docker tag nvcr.io/nvidia/cuda:12.4.1-base-ubuntu22.04 registry-host:5000/nvidia/cuda:12.4.1-base-ubuntu22.04 +docker push registry-host:5000/nvidia/cuda:12.4.1-base-ubuntu22.04 ``` diff --git a/docs/airgap/ngc-ready.md b/docs/airgap/ngc-ready.md index 1acebe12a..5917d3e75 100644 --- a/docs/airgap/ngc-ready.md +++ b/docs/airgap/ngc-ready.md @@ -37,9 +37,9 @@ For instructions on setting up an HTTP mirror, see the [doc on HTTP mirrors](./m Container images are only needed if you want to run the tests built into the playbook: -- nvcr.io/nvidia/cuda:10.1-base-ubuntu18.04 -- nvcr.io/nvidia/pytorch:18.10-py3 -- nvcr.io/nvidia/tensorflow:18.10-py3 +- nvcr.io/nvidia/cuda:12.4.1-base-ubuntu22.04 +- nvcr.io/nvidia/pytorch:24.04-py3 +- nvcr.io/nvidia/tensorflow:24.04-tf2-py3 For instructions on setting up a Docker registry mirror, see the [doc on Docker mirrors](./mirror-docker-images.md). @@ -63,9 +63,9 @@ For instructions on setting up an HTTP mirror, see the [doc on HTTP mirrors](./m Container images (how to mirror) are only needed if you want to run the tests built into the playbook: -- nvcr.io/nvidia/cuda:10.1-base-ubuntu18.04 -- nvcr.io/nvidia/pytorch:18.10-py3 -- nvcr.io/nvidia/tensorflow:18.10-py3 +- nvcr.io/nvidia/cuda:12.4.1-base-ubuntu22.04 +- nvcr.io/nvidia/pytorch:24.04-py3 +- nvcr.io/nvidia/tensorflow:24.04-tf2-py3 For instructions on setting up a Docker registry mirror, see the [doc on Docker mirrors](./mirror-docker-images.md). @@ -182,9 +182,9 @@ dcgm_rpm_package: "/path/to/datacenter-gpu-manager.rpm" If running the container tests as part of the NGC-Ready playbook, set the following variables in your DeepOps configuration: ```bash -ngc_ready_cuda_container: "/nvidia/cuda:10.1-base-ubuntu18.04" -ngc_ready_pytorch: "/nvidia/pytorch:18.10-py3" -ngc_ready_tensorflow: "/nvidia/tensorflow:18.10-py3" +ngc_ready_cuda_container: "/nvidia/cuda:12.4.1-base-ubuntu22.04" +ngc_ready_pytorch: "/nvidia/pytorch:24.04-py3" +ngc_ready_tensorflow: "/nvidia/tensorflow:24.04-tf2-py3" ``` ## Running the NGC-Ready playbook diff --git a/virtual/scripts/setup_k8s.sh b/virtual/scripts/setup_k8s.sh index 20860d8b3..a38777eab 100755 --- a/virtual/scripts/setup_k8s.sh +++ b/virtual/scripts/setup_k8s.sh @@ -40,7 +40,7 @@ source "${VIRT_DIR}/k8s_environment.sh" # Verify that the cluster is up file ${K8S_CONFIG_DIR}/artifacts/kubectl && chmod +x ${K8S_CONFIG_DIR}/artifacts/kubectl kubectl get nodes -#kubectl run gpu-test --rm -t -i --restart=Never --image=nvcr.io/nvidia/cuda:10.1-base-ubuntu18.04 --limits=nvidia.com/gpu=1 -- nvidia-smi +#kubectl run gpu-test --rm -t -i --restart=Never --image=nvcr.io/nvidia/cuda:12.4.1-base-ubuntu22.04 --limits=nvidia.com/gpu=1 -- nvidia-smi # Install helm "${ROOT_DIR}/scripts/k8s/install_helm.sh" diff --git a/workloads/examples/k8s/cluster-gpu-test-job.yml b/workloads/examples/k8s/cluster-gpu-test-job.yml index fd4af4904..8946601c7 100644 --- a/workloads/examples/k8s/cluster-gpu-test-job.yml +++ b/workloads/examples/k8s/cluster-gpu-test-job.yml @@ -10,7 +10,7 @@ spec: spec: containers: - name: cluster-gpu-tests - image: nvcr.io/nvidia/cuda:9.0-base + image: nvcr.io/nvidia/cuda:12.4.1-base-ubuntu22.04 command: ["/bin/bash","-c","nvidia-smi && sleep 10"] args: resources: diff --git a/workloads/examples/k8s/gpu-test-job.yml b/workloads/examples/k8s/gpu-test-job.yml index 7a0abdb71..63da1a375 100644 --- a/workloads/examples/k8s/gpu-test-job.yml +++ b/workloads/examples/k8s/gpu-test-job.yml @@ -5,11 +5,10 @@ metadata: spec: containers: - name: cuda-container - image: nvcr.io/nvidia/cuda:10.0-devel + image: nvcr.io/nvidia/cuda:12.4.1-base-ubuntu22.04 command: ["sleep", "6000"] args: resources: limits: nvidia.com/gpu: 1 restartPolicy: Never - diff --git a/workloads/jenkins/scripts/run-gpu-job.sh b/workloads/jenkins/scripts/run-gpu-job.sh index b1e2be648..470109ff1 100644 --- a/workloads/jenkins/scripts/run-gpu-job.sh +++ b/workloads/jenkins/scripts/run-gpu-job.sh @@ -32,7 +32,7 @@ fi # Occassionally this gpu-test fails and/or hangs. To ease debugging of this we run a describe several seconds into the launch. sleep 10 && kubectl describe pods gpu-test & -timeout 300 kubectl run gpu-test --rm -t -i --restart=Never --image=nvcr.io/nvidia/cuda:10.1-base-ubuntu18.04 --limits=nvidia.com/gpu=1 -- nvidia-smi +timeout 300 kubectl run gpu-test --rm -t -i --restart=Never --image=nvcr.io/nvidia/cuda:12.4.1-base-ubuntu22.04 --limits=nvidia.com/gpu=1 -- nvidia-smi # Run multi-GPU test if [ ${DEEPOPS_FULL_INSTALL} ]; then diff --git a/workloads/jenkins/scripts/test-slurm-enroot-job.sh b/workloads/jenkins/scripts/test-slurm-enroot-job.sh index d8168ec94..2504bbce7 100644 --- a/workloads/jenkins/scripts/test-slurm-enroot-job.sh +++ b/workloads/jenkins/scripts/test-slurm-enroot-job.sh @@ -9,5 +9,5 @@ ssh -v \ -i "${HOME}/.ssh/id_rsa" \ "10.0.0.5${GPU01}" \ srun -N1 -G1 \ - --container-image="nvcr.io#nvidia/cuda:10.2-base-ubuntu18.04" \ + --container-image="nvcr.io#nvidia/cuda:12.4.1-base-ubuntu22.04" \ nvidia-smi -L From 617e72ba9bddedacd4f7c6607a1507946339f959 Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Wed, 27 May 2026 21:09:04 -0600 Subject: [PATCH 17/27] feat: support Red Hat container toolkit path --- docs/airgap/mirror-rpm-repos.md | 20 +-- playbooks/container/nvidia-docker.yml | 16 ++- .../defaults/main.yml | 4 + .../nvidia_container_toolkit/tasks/docker.yml | 32 +++++ roles/nvidia_container_toolkit/tasks/main.yml | 116 ++---------------- .../nvidia_container_toolkit/tasks/redhat.yml | 23 ++++ .../nvidia_container_toolkit/tasks/ubuntu.yml | 72 +++++++++++ 7 files changed, 163 insertions(+), 120 deletions(-) create mode 100644 roles/nvidia_container_toolkit/tasks/docker.yml create mode 100644 roles/nvidia_container_toolkit/tasks/redhat.yml create mode 100644 roles/nvidia_container_toolkit/tasks/ubuntu.yml diff --git a/docs/airgap/mirror-rpm-repos.md b/docs/airgap/mirror-rpm-repos.md index bef7a7b28..653f327f2 100644 --- a/docs/airgap/mirror-rpm-repos.md +++ b/docs/airgap/mirror-rpm-repos.md @@ -25,17 +25,16 @@ If you do not already have mirrors of the distribution repositories available, p The following additional RPM repositories are commonly used for GPU-enabled systems deployed by DeepOps: - [Fedora Extra Packages for Enterprise Linux (EPEL)](https://fedoraproject.org/wiki/EPEL) -- NVIDIA CUDA repository: [repo file for EL7](https://developer.download.nvidia.com/compute/cuda/repos/rhel7/x86_64/cuda-rhel7.repo), [repo file for EL8](https://developer.download.nvidia.com/compute/cuda/repos/rhel7/x86_64/cuda-rhel8.repo) -- NVIDIA container repositories: [repo file for EL7](https://raw.githubusercontent.com/NVIDIA/nvidia-docker/gh-pages/centos7/nvidia-docker.repo), [repo file for EL8](https://raw.githubusercontent.com/NVIDIA/nvidia-docker/gh-pages/centos8/nvidia-docker.repo) +- NVIDIA CUDA repository: [repo file for EL8](https://developer.download.nvidia.com/compute/cuda/repos/rhel8/x86_64/cuda-rhel8.repo), [repo file for EL9](https://developer.download.nvidia.com/compute/cuda/repos/rhel9/x86_64/cuda-rhel9.repo) +- NVIDIA Container Toolkit repository: [repo file for RPM-based distributions](https://nvidia.github.io/libnvidia-container/stable/rpm/nvidia-container-toolkit.repo) - Docker CE repository: [repo file](https://download.docker.com/linux/centos/docker-ce.repo) These repo files provide the following repository IDs, which will be needed by `reposync` below: - epel -- cuda-rhel7-x86_64 or cuda-rhel8-x86_64 +- cuda-rhel8-x86_64 or cuda-rhel9-x86_64 - libnvidia-container -- nvidia-container-runtime -- nvidia-docker +- nvidia-container-toolkit - docker-ce-stable To discover a complete list of repositories needed for your particular workload, @@ -49,10 +48,11 @@ On a RHEL or CentOS machine with Internet access, install the `yum-utils` and `c sudo yum install yum-utils createrepo ``` -Then install the EPEL repository: +Then install the EPEL repository if your workload requires EPEL packages. +For example, on EL9: ```bash -sudo yum install https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm +sudo yum install https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm ``` Then, for each of the other repo files, install the file into the `/etc/yum.repos.d` directory. @@ -61,8 +61,8 @@ For example, if using the list of repositories from the previous section: ```bash cd /etc/yum.repos.d sudo wget https://download.docker.com/linux/centos/docker-ce.repo -sudo wget https://developer.download.nvidia.com/compute/cuda/repos/rhel7/x86_64/cuda-rhel7.repo -sudo wget https://raw.githubusercontent.com/NVIDIA/nvidia-docker/gh-pages/centos7/nvidia-docker.repo +sudo wget https://developer.download.nvidia.com/compute/cuda/repos/rhel9/x86_64/cuda-rhel9.repo +sudo wget https://nvidia.github.io/libnvidia-container/stable/rpm/nvidia-container-toolkit.repo ``` For each of the repositories you wish to mirror, run the `reposync` command to download the contents of the repository. @@ -80,7 +80,7 @@ At this point, you should have one subdirectory for each of the repositories you ```bash ls /var/repos/ -docker-ce-stable nvidia-docker +docker-ce-stable libnvidia-container nvidia-container-toolkit ``` For each of these directories, run the `createrepo` command to generate repository metadata: diff --git a/playbooks/container/nvidia-docker.yml b/playbooks/container/nvidia-docker.yml index b15edc99f..5c2b9733d 100644 --- a/playbooks/container/nvidia-docker.yml +++ b/playbooks/container/nvidia-docker.yml @@ -16,12 +16,15 @@ state: absent when: docker_install | default('yes') - - name: install NVIDIA Container Toolkit on Ubuntu 24.04 and newer + - name: install NVIDIA Container Toolkit on current OS releases include_role: name: nvidia_container_toolkit when: - - ansible_local['gpus']['count'] and ansible_distribution == "Ubuntu" - - ansible_distribution_version is version('24.04', '>=') + - ansible_local['gpus']['count'] + - > + (ansible_distribution == "Ubuntu" and ansible_distribution_version is version('24.04', '>=')) + or + (ansible_os_family == "RedHat" and ansible_distribution_major_version is version('8', '>=')) - docker_install | default('yes') - name: install nvidia-docker @@ -29,6 +32,11 @@ name: nvidia.nvidia_docker when: - ansible_local['gpus']['count'] and (ansible_distribution == "Ubuntu" or ansible_os_family == "RedHat") - - not (ansible_distribution == "Ubuntu" and ansible_distribution_version is version('24.04', '>=')) + - > + not ( + (ansible_distribution == "Ubuntu" and ansible_distribution_version is version('24.04', '>=')) + or + (ansible_os_family == "RedHat" and ansible_distribution_major_version is version('8', '>=')) + ) - docker_install | default('yes') environment: "{{ proxy_env if proxy_env is defined else {} }}" diff --git a/roles/nvidia_container_toolkit/defaults/main.yml b/roles/nvidia_container_toolkit/defaults/main.yml index 87577f72c..6159a7a68 100644 --- a/roles/nvidia_container_toolkit/defaults/main.yml +++ b/roles/nvidia_container_toolkit/defaults/main.yml @@ -4,6 +4,10 @@ nvidia_container_toolkit_repo_gpg_url: "{{ nvidia_container_toolkit_repo_base_ur nvidia_container_toolkit_keyring_ascii_path: "/usr/share/keyrings/nvidia-container-toolkit-keyring.asc" nvidia_container_toolkit_keyring_path: "/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg" nvidia_container_toolkit_apt_source_path: "/etc/apt/sources.list.d/nvidia-container-toolkit.list" +nvidia_container_toolkit_rpm_repo_url: "{{ nvidia_container_toolkit_repo_base_url }}/stable/rpm/nvidia-container-toolkit.repo" +nvidia_container_toolkit_yum_repo_path: "/etc/yum.repos.d/nvidia-container-toolkit.repo" +nvidia_container_toolkit_rpm_prerequisites: + - ca-certificates nvidia_container_toolkit_package: "nvidia-container-toolkit" nvidia_container_toolkit_configure_docker: true nvidia_container_toolkit_set_as_default_runtime: true diff --git a/roles/nvidia_container_toolkit/tasks/docker.yml b/roles/nvidia_container_toolkit/tasks/docker.yml new file mode 100644 index 000000000..432bc3e1c --- /dev/null +++ b/roles/nvidia_container_toolkit/tasks/docker.yml @@ -0,0 +1,32 @@ +--- +- name: Docker | ensure Docker configuration directory exists + ansible.builtin.file: + path: /etc/docker + state: directory + owner: root + group: root + mode: "0755" + +- name: Docker | check NVIDIA runtime configuration + ansible.builtin.command: + cmd: >- + python3 -c 'import json, pathlib, sys; + p = pathlib.Path("/etc/docker/daemon.json"); + required_default = {{ nvidia_container_toolkit_set_as_default_runtime | bool | ternary("True", "False") }}; + data = {}; + text = p.read_text().strip() if p.exists() else ""; + data = json.loads(text) if text else {}; + runtime = data.get("runtimes", {}).get("nvidia", {}); + ok = runtime.get("path") in ("nvidia-container-runtime", "/usr/bin/nvidia-container-runtime"); + ok = ok and (not required_default or data.get("default-runtime") == "nvidia"); + sys.exit(0 if ok else 1)' + register: nvidia_container_toolkit_docker_runtime + failed_when: false + changed_when: false + +- name: Docker | configure NVIDIA runtime + ansible.builtin.command: + cmd: "nvidia-ctk runtime configure --runtime=docker{{ ' --set-as-default' if nvidia_container_toolkit_set_as_default_runtime | bool else '' }}" + when: nvidia_container_toolkit_docker_runtime.rc != 0 + changed_when: true + notify: restart docker diff --git a/roles/nvidia_container_toolkit/tasks/main.yml b/roles/nvidia_container_toolkit/tasks/main.yml index c681d88e6..202a08390 100644 --- a/roles/nvidia_container_toolkit/tasks/main.yml +++ b/roles/nvidia_container_toolkit/tasks/main.yml @@ -1,114 +1,18 @@ --- -- name: Ubuntu | verify supported distribution +- name: Verify supported distribution ansible.builtin.assert: that: - - ansible_distribution == "Ubuntu" - fail_msg: "The nvidia_container_toolkit role currently supports Ubuntu only." - -- name: Ubuntu | set package architecture - ansible.builtin.set_fact: - nvidia_container_toolkit_deb_arch: "{{ _nvidia_container_toolkit_arch_map.get(ansible_architecture, ansible_architecture) }}" - vars: - _nvidia_container_toolkit_arch_map: - aarch64: arm64 - arm64: arm64 - x86_64: amd64 - -- name: Ubuntu | install repository prerequisites - ansible.builtin.apt: - name: - - ca-certificates - - gnupg - state: present - update_cache: true - when: nvidia_container_toolkit_repo_base_url | length > 0 - -- name: Ubuntu | ensure keyring directory exists - ansible.builtin.file: - path: "{{ nvidia_container_toolkit_keyring_path | dirname }}" - state: directory - owner: root - group: root - mode: "0755" - -- name: Ubuntu | check NVIDIA Container Toolkit keyring - ansible.builtin.stat: - path: "{{ nvidia_container_toolkit_keyring_path }}" - register: nvidia_container_toolkit_keyring - -- name: Ubuntu | download NVIDIA Container Toolkit GPG key - ansible.builtin.get_url: - url: "{{ nvidia_container_toolkit_repo_gpg_url }}" - dest: "{{ nvidia_container_toolkit_keyring_ascii_path }}" - owner: root - group: root - mode: "0644" - register: nvidia_container_toolkit_key - environment: "{{ proxy_env if proxy_env is defined else {} }}" - -- name: Ubuntu | install NVIDIA Container Toolkit GPG keyring - ansible.builtin.command: - cmd: "gpg --dearmor --yes -o {{ nvidia_container_toolkit_keyring_path }} {{ nvidia_container_toolkit_keyring_ascii_path }}" - when: nvidia_container_toolkit_key.changed or not nvidia_container_toolkit_keyring.stat.exists - register: nvidia_container_toolkit_keyring_install - changed_when: true - -- name: Ubuntu | set NVIDIA Container Toolkit GPG keyring permissions - ansible.builtin.file: - path: "{{ nvidia_container_toolkit_keyring_path }}" - owner: root - group: root - mode: "0644" - -- name: Ubuntu | configure NVIDIA Container Toolkit APT repository - ansible.builtin.copy: - content: | - deb [signed-by={{ nvidia_container_toolkit_keyring_path }}] {{ nvidia_container_toolkit_repo_base_url }}/stable/deb/{{ nvidia_container_toolkit_deb_arch }} / - dest: "{{ nvidia_container_toolkit_apt_source_path }}" - owner: root - group: root - mode: "0644" - register: nvidia_container_toolkit_apt_source + - ansible_distribution == "Ubuntu" or ansible_os_family == "RedHat" + fail_msg: "The nvidia_container_toolkit role supports Ubuntu and Red Hat family hosts only." - name: Ubuntu | install NVIDIA Container Toolkit - ansible.builtin.apt: - name: "{{ nvidia_container_toolkit_package }}" - state: present - update_cache: "{{ nvidia_container_toolkit_apt_source.changed or nvidia_container_toolkit_keyring_install.changed | default(false) }}" - environment: "{{ proxy_env if proxy_env is defined else {} }}" - -- name: Docker | ensure Docker configuration directory exists - ansible.builtin.file: - path: /etc/docker - state: directory - owner: root - group: root - mode: "0755" - when: nvidia_container_toolkit_configure_docker | bool + ansible.builtin.include_tasks: ubuntu.yml + when: ansible_distribution == "Ubuntu" -- name: Docker | check NVIDIA runtime configuration - ansible.builtin.command: - cmd: >- - python3 -c 'import json, pathlib, sys; - p = pathlib.Path("/etc/docker/daemon.json"); - required_default = {{ nvidia_container_toolkit_set_as_default_runtime | bool | ternary("True", "False") }}; - data = {}; - text = p.read_text().strip() if p.exists() else ""; - data = json.loads(text) if text else {}; - runtime = data.get("runtimes", {}).get("nvidia", {}); - ok = runtime.get("path") in ("nvidia-container-runtime", "/usr/bin/nvidia-container-runtime"); - ok = ok and (not required_default or data.get("default-runtime") == "nvidia"); - sys.exit(0 if ok else 1)' - register: nvidia_container_toolkit_docker_runtime - failed_when: false - changed_when: false - when: nvidia_container_toolkit_configure_docker | bool +- name: Red Hat | install NVIDIA Container Toolkit + ansible.builtin.include_tasks: redhat.yml + when: ansible_os_family == "RedHat" - name: Docker | configure NVIDIA runtime - ansible.builtin.command: - cmd: "nvidia-ctk runtime configure --runtime=docker{{ ' --set-as-default' if nvidia_container_toolkit_set_as_default_runtime | bool else '' }}" - when: - - nvidia_container_toolkit_configure_docker | bool - - nvidia_container_toolkit_docker_runtime.rc != 0 - changed_when: true - notify: restart docker + ansible.builtin.include_tasks: docker.yml + when: nvidia_container_toolkit_configure_docker | bool diff --git a/roles/nvidia_container_toolkit/tasks/redhat.yml b/roles/nvidia_container_toolkit/tasks/redhat.yml new file mode 100644 index 000000000..74744e41e --- /dev/null +++ b/roles/nvidia_container_toolkit/tasks/redhat.yml @@ -0,0 +1,23 @@ +--- +- name: Red Hat | install repository prerequisites + ansible.builtin.dnf: + name: "{{ nvidia_container_toolkit_rpm_prerequisites }}" + state: present + when: nvidia_container_toolkit_repo_base_url | length > 0 + +- name: Red Hat | configure NVIDIA Container Toolkit Yum repository + ansible.builtin.get_url: + url: "{{ nvidia_container_toolkit_rpm_repo_url }}" + dest: "{{ nvidia_container_toolkit_yum_repo_path }}" + owner: root + group: root + mode: "0644" + register: nvidia_container_toolkit_yum_repo + environment: "{{ proxy_env if proxy_env is defined else {} }}" + +- name: Red Hat | install NVIDIA Container Toolkit + ansible.builtin.dnf: + name: "{{ nvidia_container_toolkit_package }}" + state: present + update_cache: "{{ nvidia_container_toolkit_yum_repo.changed }}" + environment: "{{ proxy_env if proxy_env is defined else {} }}" diff --git a/roles/nvidia_container_toolkit/tasks/ubuntu.yml b/roles/nvidia_container_toolkit/tasks/ubuntu.yml new file mode 100644 index 000000000..e446a8ced --- /dev/null +++ b/roles/nvidia_container_toolkit/tasks/ubuntu.yml @@ -0,0 +1,72 @@ +--- +- name: Ubuntu | set package architecture + ansible.builtin.set_fact: + nvidia_container_toolkit_deb_arch: "{{ _nvidia_container_toolkit_arch_map.get(ansible_architecture, ansible_architecture) }}" + vars: + _nvidia_container_toolkit_arch_map: + aarch64: arm64 + arm64: arm64 + x86_64: amd64 + +- name: Ubuntu | install repository prerequisites + ansible.builtin.apt: + name: + - ca-certificates + - gnupg + state: present + update_cache: true + when: nvidia_container_toolkit_repo_base_url | length > 0 + +- name: Ubuntu | ensure keyring directory exists + ansible.builtin.file: + path: "{{ nvidia_container_toolkit_keyring_path | dirname }}" + state: directory + owner: root + group: root + mode: "0755" + +- name: Ubuntu | check NVIDIA Container Toolkit keyring + ansible.builtin.stat: + path: "{{ nvidia_container_toolkit_keyring_path }}" + register: nvidia_container_toolkit_keyring + +- name: Ubuntu | download NVIDIA Container Toolkit GPG key + ansible.builtin.get_url: + url: "{{ nvidia_container_toolkit_repo_gpg_url }}" + dest: "{{ nvidia_container_toolkit_keyring_ascii_path }}" + owner: root + group: root + mode: "0644" + register: nvidia_container_toolkit_key + environment: "{{ proxy_env if proxy_env is defined else {} }}" + +- name: Ubuntu | install NVIDIA Container Toolkit GPG keyring + ansible.builtin.command: + cmd: "gpg --dearmor --yes -o {{ nvidia_container_toolkit_keyring_path }} {{ nvidia_container_toolkit_keyring_ascii_path }}" + when: nvidia_container_toolkit_key.changed or not nvidia_container_toolkit_keyring.stat.exists + register: nvidia_container_toolkit_keyring_install + changed_when: true + +- name: Ubuntu | set NVIDIA Container Toolkit GPG keyring permissions + ansible.builtin.file: + path: "{{ nvidia_container_toolkit_keyring_path }}" + owner: root + group: root + mode: "0644" + +- name: Ubuntu | configure NVIDIA Container Toolkit APT repository + ansible.builtin.copy: + content: | + deb [signed-by={{ nvidia_container_toolkit_keyring_path }}] {{ nvidia_container_toolkit_repo_base_url }}/stable/deb/{{ nvidia_container_toolkit_deb_arch }} / + dest: "{{ nvidia_container_toolkit_apt_source_path }}" + owner: root + group: root + mode: "0644" + register: nvidia_container_toolkit_apt_source + +- name: Ubuntu | install NVIDIA Container Toolkit + ansible.builtin.apt: + name: "{{ nvidia_container_toolkit_package }}" + state: present + update_cache: "{{ nvidia_container_toolkit_apt_source.changed or nvidia_container_toolkit_keyring_install.changed | default(false) }}" + environment: "{{ proxy_env if proxy_env is defined else {} }}" From ad241ba76267cdd14b7ac844584744ad83aa0236 Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Thu, 28 May 2026 11:05:59 -0600 Subject: [PATCH 18/27] Retire legacy PXE provisioning paths --- README.md | 3 +- config.example/group_vars/all.yml | 4 +- config.example/helm/dgxie.yml | 62 --- config.example/pxe/dnsmasq.extra.conf | 9 - config.example/pxe/env | 49 -- config.example/pxe/ipmi.conf | 4 - config.example/pxe/ipmi_host_list | 4 - config.example/pxe/machines/machines.json | 50 -- docs/airgap/mirror-apt-repos.md | 2 +- docs/k8s-cluster/README.md | 3 +- docs/k8s-cluster/roce-perf-k8s.md | 3 +- docs/pxe/README.md | 10 +- docs/pxe/dgxie-container.md | 62 --- docs/pxe/dgxie-on-k8s.md | 87 ---- docs/pxe/maas.md | 27 +- docs/pxe/minimal-pxe-container.md | 69 --- docs/slurm-cluster/README.md | 3 +- docs/slurm-cluster/slurm-perf-cluster.md | 3 +- scripts/pxe/build_and_restart_dgxie.sh | 25 - scripts/pxe/dgxctl.sh | 270 ---------- scripts/pxe/install_dgxie_prereqs.sh | 43 -- scripts/pxe/setup_nat.sh | 21 - src/containers/dgxie/Dockerfile | 28 - src/containers/dgxie/README.md | 461 ---------------- src/containers/dgxie/api.py | 48 -- src/containers/dgxie/dnsmasq.conf | 24 - src/containers/dgxie/docker-compose.yml | 49 -- src/containers/dgxie/get_hosts.py | 258 --------- src/containers/dgxie/kickstart.cfg | 97 ---- src/containers/dgxie/mboot.efi | Bin 168632 -> 0 bytes src/containers/dgxie/nginx.conf | 43 -- src/containers/dgxie/start | 127 ----- src/containers/dgxie/vsftpd.conf | 123 ----- src/containers/pixiecore/Dockerfile | 26 - src/containers/pxe/dhcp/Dockerfile | 11 - src/containers/pxe/dhcp/dnsmasq.conf | 18 - src/containers/pxe/docker-compose.yml | 28 - src/containers/pxe/preseed | 491 ------------------ .../examples/k8s/services/nfs-dgx-iso.yml | 24 - workloads/examples/k8s/services/pxe.yml | 27 - workloads/services/k8s/dgxie/.helmignore | 21 - workloads/services/k8s/dgxie/Chart.yaml | 5 - .../services/k8s/dgxie/templates/NOTES.txt | 0 .../services/k8s/dgxie/templates/_helpers.tpl | 32 -- .../k8s/dgxie/templates/deployment.yaml | 139 ----- workloads/services/k8s/dgxie/values.yaml | 56 -- workloads/services/k8s/iso-loader.yml | 32 -- 47 files changed, 43 insertions(+), 2938 deletions(-) delete mode 100644 config.example/helm/dgxie.yml delete mode 100644 config.example/pxe/dnsmasq.extra.conf delete mode 100644 config.example/pxe/env delete mode 100644 config.example/pxe/ipmi.conf delete mode 100644 config.example/pxe/ipmi_host_list delete mode 100644 config.example/pxe/machines/machines.json delete mode 100644 docs/pxe/dgxie-container.md delete mode 100644 docs/pxe/dgxie-on-k8s.md delete mode 100644 docs/pxe/minimal-pxe-container.md delete mode 100755 scripts/pxe/build_and_restart_dgxie.sh delete mode 100755 scripts/pxe/dgxctl.sh delete mode 100755 scripts/pxe/install_dgxie_prereqs.sh delete mode 100755 scripts/pxe/setup_nat.sh delete mode 100644 src/containers/dgxie/Dockerfile delete mode 100644 src/containers/dgxie/README.md delete mode 100755 src/containers/dgxie/api.py delete mode 100644 src/containers/dgxie/dnsmasq.conf delete mode 100644 src/containers/dgxie/docker-compose.yml delete mode 100755 src/containers/dgxie/get_hosts.py delete mode 100644 src/containers/dgxie/kickstart.cfg delete mode 100644 src/containers/dgxie/mboot.efi delete mode 100644 src/containers/dgxie/nginx.conf delete mode 100755 src/containers/dgxie/start delete mode 100644 src/containers/dgxie/vsftpd.conf delete mode 100644 src/containers/pixiecore/Dockerfile delete mode 100644 src/containers/pxe/dhcp/Dockerfile delete mode 100644 src/containers/pxe/dhcp/dnsmasq.conf delete mode 100644 src/containers/pxe/docker-compose.yml delete mode 100644 src/containers/pxe/preseed delete mode 100644 workloads/examples/k8s/services/nfs-dgx-iso.yml delete mode 100644 workloads/examples/k8s/services/pxe.yml delete mode 100644 workloads/services/k8s/dgxie/.helmignore delete mode 100644 workloads/services/k8s/dgxie/Chart.yaml delete mode 100644 workloads/services/k8s/dgxie/templates/NOTES.txt delete mode 100644 workloads/services/k8s/dgxie/templates/_helpers.tpl delete mode 100644 workloads/services/k8s/dgxie/templates/deployment.yaml delete mode 100644 workloads/services/k8s/dgxie/values.yaml delete mode 100644 workloads/services/k8s/iso-loader.yml diff --git a/README.md b/README.md index 68df95a20..84edf7657 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,8 @@ The cluster nodes will follow the requirements described by Slurm or Kubernetes. - CentOS 7, 8 - Red Hat Enterprise Linux / Rocky Linux 8 and 9 for the DGX software stack through the `nvidia-dgx` role -You may also install a supported operating system on all servers via a 3rd-party solution (i.e. [MAAS](https://maas.io/), [Foreman](https://www.theforeman.org/)) or utilize the provided [OS install container](docs/pxe/minimal-pxe-container.md). +You may also install a supported operating system on all servers via a 3rd-party solution such as [MAAS](https://maas.io/) or [Foreman](https://www.theforeman.org/), or via an existing site-standard automated installer. +For new Ubuntu 24.04 or DGX OS 7 deployments, prefer Ubuntu autoinstall/cloud-init or MAAS and then apply DeepOps roles after the OS is present. For DGX platform software installation on top of vanilla Ubuntu or Red Hat family operating systems, see the [DGX software stack role guide](docs/deepops/dgx-software-stack.md). ### Kubernetes diff --git a/config.example/group_vars/all.yml b/config.example/group_vars/all.yml index b7de1ed73..d98664bdc 100644 --- a/config.example/group_vars/all.yml +++ b/config.example/group_vars/all.yml @@ -253,7 +253,9 @@ maas_adminusers: maas_dns_domain: 'deepops.local' maas_region_controller: '192.168.1.1' maas_region_controller_url: 'http://{{ maas_region_controller }}:5240/MAAS' -maas_repo: 'ppa:maas/3.5' +# MAAS 3.7 is the current Ubuntu 24.04 line. Keep the 3.5 PPA when the MAAS +# controller itself still runs Ubuntu 22.04. +maas_repo: "{{ 'ppa:maas/3.7' if ansible_distribution_version is version('24.04', '>=') else 'ppa:maas/3.5' }}" # Defines if maas user should generate ssh keys # Usable for remote KVM/libvirt power actions diff --git a/config.example/helm/dgxie.yml b/config.example/helm/dgxie.yml deleted file mode 100644 index 8417e9c22..000000000 --- a/config.example/helm/dgxie.yml +++ /dev/null @@ -1,62 +0,0 @@ -# -# DGXie configuration: -# - -# Default DGX boot mode -# Options: -# "DGX": boot DGX to install media -# "local": boot DGX to local disk -bootMode: DGX - -# DGX network interface to use during install -dgxNetInt: enp1s0f0 - -# DGX disk to use during install -dgxDisk: sda - -# DGX Keyboard layout to use -dgxKbd: us - -# Extra kernel parameters to pass during DGX OS install -# i.e. "rebuild-raid" -dgxKernExtra: '' - -# Network interface of public network on management servers -mgmtIntPub: eth0 - -# Network interface of private network where DGX are connected on management servers -mgmtIntPrv: eth1 - -# Network domain -netDomain: local - -# IP address of private network interface on management server -netPrvIp: 192.168.1.1 - -# Private Network -netPrvNet: 192.168.1.0 - -# Private network netmask -netPrvNetmask: 255.255.255.0 - -# Private network gateway -netPrvGateway: 192.168.1.1 - -# DNS nameservers -netPrvDns1: 8.8.8.8 -netPrvDns2: 8.8.4.4 - -# DHCP dynamic address range -netPrvDhcpStart: 192.168.1.100 -netPrvDhcpEnd: 192.168.1.199 - -#ntp: 192.168.1.10 - -# DHCP lease time -netPrvDhcpLease: 7200 - -# HTTPS proxy to use in preseed -#httpsProxy: "http://192.168.2.1:3128" - -# Install extra packages -extraPackages: "aptitude ubuntu-minimal dirmngr" diff --git a/config.example/pxe/dnsmasq.extra.conf b/config.example/pxe/dnsmasq.extra.conf deleted file mode 100644 index b46aeb66b..000000000 --- a/config.example/pxe/dnsmasq.extra.conf +++ /dev/null @@ -1,9 +0,0 @@ -# -# Additional DNSMASQ configuration -# - -# If the dhcp-ignore flag is specified in this fashion, only hosts configured with dhcp-host will be given DHCP -#dhcp-ignore=tag:!known - -# Example static IP; note this will not work for bonded interfaces -#dhcp-host=12:34:56:78,server-01,192.168.1.23 diff --git a/config.example/pxe/env b/config.example/pxe/env deleted file mode 100644 index dafa98528..000000000 --- a/config.example/pxe/env +++ /dev/null @@ -1,49 +0,0 @@ -## Docker Compose settings -COMPOSE_PROJECT_NAME=deepops -COMPOSE_FILE=src/containers/dgxie/docker-compose.yml - -## DHCP/PXE server settings -DHCP_ENABLE=1 -DHCP_INT=eth1 -NETWORK=192.168.1.0 -NETMASK=255.255.255.0 -GATEWAY=192.168.1.1 -DNS1=8.8.8.8 -DNS2=8.8.4.4 -DHCP_START=192.168.1.2 -DHCP_END=192.168.1.254 -LEASETIME=7200 -DOMAIN=local -NTP=pool.ntp.org - -## NAT settings -NAT_ENABLE=1 -NAT_INT_PUB=eth0 -NAT_INT_PRV=eth1 - -## HTTP server settings -HTTP_INT=eth1 -HTTP_PORT=13370 - -## NFS server settings -# Directory on host containing ISOs, drivers, etc. -DATA_DIR=/home/ubuntu - -## VMware install settings -VMW_NFS_IP=192.168.1.1 -VMW_GPU_VIB=NVIDIA-VMware_ESXi_6.7_Host_Driver-418.66-1OEM.670.0.0.8169922.x86_64.vib -VMW_VM_SIZE=100g - -## DGX OS install settings -# DGX boot interface -DGX_INT=enp1s0f0 -# DGX boot disk -DGX_DISK=sda -# DGX keyboard -DGX_KBD=us -# DGX extra kernel params -DGX_KERN_EXTRA="" -# DGX extra packages to install -DGX_EXTRA_PACKAGES="" -# DGX install proxy -DGX_HTTPS_PROXY="" diff --git a/config.example/pxe/ipmi.conf b/config.example/pxe/ipmi.conf deleted file mode 100644 index 7cd5ea95a..000000000 --- a/config.example/pxe/ipmi.conf +++ /dev/null @@ -1,4 +0,0 @@ -# This configuration file is used while rebooting DGX servers into PXE boot -# This information is used to connect to the DGX BMC -IPMI_USERNAME=dgxuser -IPMI_PASSWORD=dgxuser \ No newline at end of file diff --git a/config.example/pxe/ipmi_host_list b/config.example/pxe/ipmi_host_list deleted file mode 100644 index be3af39a6..000000000 --- a/config.example/pxe/ipmi_host_list +++ /dev/null @@ -1,4 +0,0 @@ -# This configuration file is used while rebooting DGX servers into PXE boot -# This information is used to connect to the DGX BMC -10.0.0.1 -10.0.0.2 \ No newline at end of file diff --git a/config.example/pxe/machines/machines.json b/config.example/pxe/machines/machines.json deleted file mode 100644 index b03b94ed7..000000000 --- a/config.example/pxe/machines/machines.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "esxi-example": { - "mac": "54:ab:3a.*", - "kernel": "http://localhost:$HTTP_PORT/vmware/mboot.efi", - "cmdline": "-c {{ URL \"http://localhost:$HTTP_PORT/vmware/boot.cfg\" }} rdblacklist=nouveau ip=dhcp nomodeset rw console=tty0 console=ttyS0,115200n8", - "message": "VMware ESXi" - }, - "dgx-station-example": { - "mac": "2c:56:dc.*", - "kernel": "http://localhost:$HTTP_PORT/iso/install/netboot/ubuntu-installer/amd64/linux", - "initrd": ["http://localhost:$HTTP_PORT/iso/install/netboot/ubuntu-installer/amd64/initrd.gz"], - "cmdline": "quiet url={{ URL \"http://localhost:$HTTP_PORT/dgx.seed\" }} ramdisk_size=100000 locale=en_US.UTF-8 auto=true priority=critical kbd-chooser/method=us netcfg/choose_interface=enp1s0f0 netcfg/get_hostname=dgx-server netcfg/dhcp_timeout=120", - "message": "DGX Station 4.1.0" - }, - "dgx-1-example": { - "mac": "d8:c4:97.*", - "kernel": "http://localhost:$HTTP_PORT/iso/install/netboot/ubuntu-installer/amd64/linux", - "initrd": ["http://localhost:$HTTP_PORT/iso/install/netboot/ubuntu-installer/amd64/initrd.gz"], - "cmdline": "quiet url={{ URL \"http://localhost:$HTTP_PORT/dgx.seed\" }} ramdisk_size=100000 locale=en_US.UTF-8 auto=true priority=critical kbd-chooser/method=us netcfg/choose_interface=enp1s0f0 netcfg/get_hostname=dgx-server netcfg/dhcp_timeout=120", - "message": "DGX-1 Server 4.1.0" - }, - "dgx-2-example": { - "mac": "5c:ff:35.*", - "kernel": "http://localhost:$HTTP_PORT/iso/install/netboot/ubuntu-installer/amd64/linux", - "initrd": ["http://localhost:$HTTP_PORT/iso/install/netboot/ubuntu-installer/amd64/initrd.gz"], - "cmdline": "quiet url={{ URL \"http://localhost:$HTTP_PORT/dgx.seed\" }} ramdisk_size=100000 locale=en_US.UTF-8 auto=true priority=critical kbd-chooser/method=us netcfg/choose_interface=enp1s0f0 netcfg/get_hostname=dgx-server netcfg/dhcp_timeout=120", - "message": "DGX-2 Server 4.1.0" - }, - "64-bit-ubuntu-example": { - "mac": "2c:56:dc:47:f5:1f", - "kernel": "http://archive.ubuntu.com/ubuntu/dists/xenial-updates/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/linux", - "initrd": ["http://archive.ubuntu.com/ubuntu/dists/xenial-updates/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/initrd.gz"], - "cmdline": "locale=en_US.UTF-8 priority=critical kbd-chooser/method=us", - "message": "Ubuntu 16.04" - }, - "64-bit-coreos-example": { - "mac": "2c:56:dc:47:f5:1f", - "kernel": "https://alpha.release.core-os.net/amd64-usr/current/coreos_production_pxe.vmlinuz", - "initrd": ["https://alpha.release.core-os.net/amd64-usr/current/coreos_production_pxe_image.cpio.gz"], - "cmdline": "coreos.autologin", - "message": "CoreOS Alpha" - }, - "32-bit-ubuntu-example": { - "mac": "00:24:e8:de:bf:53", - "kernel": "http://archive.ubuntu.com/ubuntu/dists/xenial-updates/main/installer-i386/current/images/netboot/ubuntu-installer/i386/linux", - "initrd": ["http://archive.ubuntu.com/ubuntu/dists/xenial-updates/main/installer-i386/current/images/netboot/ubuntu-installer/i386/initrd.gz"], - "cmdline": "locale=en_US.UTF-8 priority=critical kbd-chooser/method=us", - "message": "all your netbooks are belong to us" - } -} diff --git a/docs/airgap/mirror-apt-repos.md b/docs/airgap/mirror-apt-repos.md index 93064f704..d6d6d685e 100644 --- a/docs/airgap/mirror-apt-repos.md +++ b/docs/airgap/mirror-apt-repos.md @@ -212,7 +212,7 @@ deb http://archive.ubuntu.com/ubuntu noble-security main restricted universe mul deb http://archive.ubuntu.com/ubuntu noble-updates main restricted universe multiverse deb http://archive.ubuntu.com/ubuntu noble-proposed main restricted universe multiverse deb http://archive.ubuntu.com/ubuntu noble-backports main restricted universe multiverse -deb http://ppa.launchpad.net/maas/3.5/ubuntu noble main +deb http://ppa.launchpad.net/maas/3.7/ubuntu noble main deb http://archive.canonical.com/ubuntu noble partner deb-src http://archive.ubuntu.com/ubuntu noble main restricted universe multiverse diff --git a/docs/k8s-cluster/README.md b/docs/k8s-cluster/README.md index b7f1e32b1..3230f24a5 100644 --- a/docs/k8s-cluster/README.md +++ b/docs/k8s-cluster/README.md @@ -35,7 +35,8 @@ Instructions for deploying a GPU cluster with Kubernetes 1. Install a supported operating system on all nodes. - Install a supported operating system on all servers via a 3rd-party solution (i.e. [MAAS](https://maas.io/), [Foreman](https://www.theforeman.org/)) or utilize the provided [OS install container](../pxe). + Install a supported operating system on all servers via a 3rd-party solution such as [MAAS](https://maas.io/) or [Foreman](https://www.theforeman.org/), or via an existing site-standard automated installer. + For new Ubuntu 24.04 or DGX OS 7 deployments, prefer Ubuntu autoinstall/cloud-init or MAAS and then apply DeepOps after the OS is present. 2. Set up your provisioning machine. diff --git a/docs/k8s-cluster/roce-perf-k8s.md b/docs/k8s-cluster/roce-perf-k8s.md index 6f5d12c1b..13eb0911d 100644 --- a/docs/k8s-cluster/roce-perf-k8s.md +++ b/docs/k8s-cluster/roce-perf-k8s.md @@ -71,7 +71,8 @@ add switch PFC, ECN configuration 2. Install a supported operating system on all nodes. - Install a supported operating system on all servers utilizing the [DGXie](/docs/pxe/dgxie-container.md) provisioning container, via a 3rd-party solution (i.e. [MAAS](https://maas.io/), [Foreman](https://www.theforeman.org/)), or server BMC/console. + Install a supported operating system on all servers via a 3rd-party solution such as [MAAS](https://maas.io/) or [Foreman](https://www.theforeman.org/), via an existing site-standard automated installer, or through server BMC/console. + For new Ubuntu 24.04 or DGX OS 7 deployments, prefer Ubuntu autoinstall/cloud-init or MAAS. > NOTE: During OS installation, it is ideal if the identical user/password is configured. Otherwise, follow step 4 below to create an identical user across all nodes in the cluster. diff --git a/docs/pxe/README.md b/docs/pxe/README.md index cf0ea0b5a..3c22bd47c 100644 --- a/docs/pxe/README.md +++ b/docs/pxe/README.md @@ -14,10 +14,10 @@ Most of the playbooks in DeepOps are agnostic to the OS install tooling, assumin For example, DeepOps can be used to deploy a [Slurm cluster](../slurm-cluster/) or a [Kubernetes cluster](../k8s-cluster) regardless of how the OS was installed. This makes it relatively easy to integrate with an existing datacenter environment. -However, DeepOps does provide tooling for several PXE installation mechanisms which can be used if an existing tool isn't already deployed. -These include: +DeepOps does not try to replace a site provisioning system. +For environments without an existing bare-metal provisioning workflow, DeepOps provides MAAS setup guidance: - [MAAS](./maas.md), an open-source bare-metal provisioning tool developed by [Canonical](https://canonical.com/) -- [DGXIE](./dgxie-container.md), a containerized deployment tool developed specifically to deploy NVIDIA DGX OS - - [DGXIE on Kubernetes](./dgxie-on-k8s.md) -- A minimal [PXE container](./minimal-pxe-container.md) which wraps [Pixiecore](https://github.com/danderson/netboot/tree/master/pixiecore), an open source tool for network booting + +For new Ubuntu 24.04 or DGX OS 7 cluster deployments, prefer MAAS, an existing site provisioning system, or Ubuntu autoinstall/cloud-init. +NVIDIA DGX OS 7 supports installing the DGX Software Stack on regular Ubuntu 24.04 for cluster deployments, which is a better fit for current automated installation tooling than the retired legacy DGX OS installer workflows. diff --git a/docs/pxe/dgxie-container.md b/docs/pxe/dgxie-container.md deleted file mode 100644 index 98640da0c..000000000 --- a/docs/pxe/dgxie-container.md +++ /dev/null @@ -1,62 +0,0 @@ -# DGXIE Container - -DGXie is an all-in-one container for DHCP, DNS, and PXE, specifically tailored to the DGX Base OS. - -- [DGXIE Container](#dgxie-container) - - [Download DGX ISO](#download-dgx-iso) - - [Configure](#configure) - - [Deploy DGXie container](#deploy-dgxie-container) - - [Testing the DGXie PXE service](#testing-the-dgxie-pxe-service) - - [PXE booting the DGX](#pxe-booting-the-dgx) - - [Making updates](#making-updates) - -## Download DGX ISO - -You will need to download the official DGX Base OS ISO image to your provisioning machine. The latest DGX Base OS is available via the NVIDIA Enterprise Support Portal (ESP). - -Update the `DATA_DIR` specified in `config/pxe/env` and copy the DGX Base OS ISO there. - -## Configure - -Configuration information for DGXie is located in `config/pxe`. - -Update the `config/pxe/dnsmasq.extra.conf` with additional options, such as assigning static IPs by MAC address. - -DGXie uses docker-compose to build and run. The `src/containers/dgxie/docker-compose-yml` file consumes several environment variables that are defined in `config/pxe/env`. Changes to the DHCP range, network used for serving up PXE files, and other values can be updated there. Be sure to update the `eth1` and `eth0` values to match your machine interfaces or the DGXie will fail to start. - -> Note: This assumes you have run the setup.sh script. If you have not, you must manually copy the example config and install docker/docker-compose. - -## Deploy DGXie container - -```bash -./scripts/pxe/build_and_restart_dgxie.sh -``` - -## Testing the DGXie PXE service - -If the default HTTP_PORT or machines.json file have not been changed, the below curl call should verify that the PXE API is responding: - -```bash -curl localhost:13370/v1/boot/d8:c4:97:00:00:00 -``` - -## PXE booting the DGX - -The DGX servers can be PXE booted manually through the console. The DeepOps repo also provides the `dgxctl.sh` tool to automate this process using IPMI. - -Update the `config/pxe/ipmi_host_list` file with a list of BMC IPs. -Update the `config/pxe/ipmi.conf` file with the proper username and password. - -Run: - -```bash -./scripts/pxe/dgxctl.sh -i -``` - -> Note: This tool assumes all DGX systems are configured with the same username and password. - -## Making updates - -To make configuration changes or ISO updates, update the config files or ISO followed by re-running `./scripts/pxe/build_and_restart_dgxie.sh`. This will tear down the old DGXie and start a new one with the configuration changes. - -Updates to the machines.json file do not require a restart. diff --git a/docs/pxe/dgxie-on-k8s.md b/docs/pxe/dgxie-on-k8s.md deleted file mode 100644 index 5aadfc772..000000000 --- a/docs/pxe/dgxie-on-k8s.md +++ /dev/null @@ -1,87 +0,0 @@ -# DGXIE on Kubernetes - -DGXie is an all-in-one container for DHCP, DNS, and PXE, specifically tailored to the DGX Base OS. - -- [DGXIE on Kubernetes](#dgxie-on-kubernetes) - - [Setup](#setup) - - [Configure](#configure) - - [Deploy DGXie service](#deploy-dgxie-service) - - [Updating DHCP Configuration](#updating-dhcp-configuration) - - [Updating PXE Machines](#updating-pxe-machines) - -## Setup - -You will need to download the official DGX Base OS ISO image to your provisioning machine. The latest DGX Base OS is available via the NVIDIA Enterprise Support Portal (ESP). - -Copy the DGX Base OS ISO to shared storage via a container running in Kubernetes, substituting the path to the DGX ISO you downloaded (be sure to wait for the `iso-loader` POD to be in the _Running_ state before attempting to copy the ISO): - -```bash -kubectl apply -f workloads/services/k8s/iso-loader.yml -kubectl cp /local/DGXServer-4.0.2.180925_6acd9c.iso $(kubectl get pod -l app=iso-loader -o custom-columns=:metadata.name --no-headers):/data/iso/ -``` - -> Note: If the `iso-loader` POD fails to mount the CephFS volume, you may need to restart the kubelet service on the master node(s): `ansible mgmt -b -a "systemctl restart kubelet"` -> You may see an error that looks like this in your syslog file: `failed to get Plugin from volumeSpec for volume "cephfs" err=no volume plugin matched` - -## Configure - -Modify the DGXie configuration in `config/helm/dgxie.yml` to set values for the DHCP server and DGX install process. - -Modify `config/dhcpd.hosts.conf` to add a static IP lease for each login node and DGX server in the cluster if required. IP addresses should match those used in the `config/inventory` file. You may also add other valid configuration options for dnsmasq to this file. - -```bash -grep TODO config/* -``` - -> Note: There are several `TODO` comments in these configuration files that will likely need to be changed. Depending on the system architecture there may be additional required config changes. - -You can get the MAC address of DGX system interfaces via the BMC, for example: - -```bash -# interface 1 -ipmitool -I lanplus -U -P -H raw 0x30 0x19 0x00 0x02 | tail -c 18 | tr ' ' ':' -# interface 2 -ipmitool -I lanplus -U -P -H raw 0x30 0x19 0x00 0x12 | tail -c 18 | tr ' ' ':' -``` - -Modify `config/machines.json` to add a PXE entry for each DGX. Copy the `dgx-example` section and modify the MAC address for each DGX you would like to boot. You can modify boot parameters or install alternate operating systems if required. - -Store the config files as config-maps in Kubernetes, even if you have not made any changes (the DGXie container will try to mount these config maps): - -```bash -kubectl create configmap dhcpd --from-file=config/dhcpd.hosts.conf -kubectl create configmap pxe-machines --from-file=config/machines.json -``` - -## Deploy DGXie service - -Launch the DGXie service: - -```bash -helm install --values config/helm/dgxie.yml workloads/services/k8s/dgxie -``` - -Check the DGXie logs to make sure the services were started without errors: - -```bash -kubectl logs -l app=dgxie -``` - -> NOTE: If you later make changes to `config/dhcpd.hosts.conf` or `machines.json` you can follow the [steps](#updating-pxe-machines) to update the dgxie service. - -## Updating DHCP Configuration - -If you make changes to `config/dhcpd.hosts.conf`, you can update the file in Kubernetes and restart the service with: - -```bash -kubectl create configmap dhcpd --from-file=config/dhcpd.hosts.conf -o yaml --dry-run | kubectl replace -f - -kubectl delete pod -l app=dgxie -``` - -## Updating PXE Machines - -If you make changes to `machines.json`, you can update the file without having to restart the DGXie POD: - -```bash -kubectl create configmap pxe-machines --from-file=config/machines.json -o yaml --dry-run | kubectl replace -f - -``` diff --git a/docs/pxe/maas.md b/docs/pxe/maas.md index 1deb6dad1..08a22418b 100644 --- a/docs/pxe/maas.md +++ b/docs/pxe/maas.md @@ -3,7 +3,8 @@ OS Provisioning with MAAS - [MAAS](#maas) - - [Summary](#summary) + - [Introduction](#introduction) + - [DeepOps and MAAS operating model](#deepops-and-maas-operating-model) - [Pre-requisites](#pre-requisites) - [Installing MAAS with DeepOps](#installing-maas-with-deepops) - [Configuring MAAS](#configuring-maas) @@ -33,11 +34,29 @@ This guide was originally written using MAAS 2.8; current MAAS releases are 3.x. MAAS has a lot of different configuration options which are outside the scope of this guide. For the best reference on how to use MAAS in general, see the [documentation on maas.io](https://maas.io/docs). +## DeepOps and MAAS operating model + +DeepOps should stay modular and simple: MAAS is the source of truth for bare-metal provisioning state, while DeepOps consumes that state through small, understandable scripts and Ansible inventory. +MAAS owns machine lifecycle, power control, PXE/DHCP behavior, OS images, commissioning, deployment, release, pools, zones, and machine tags. +DeepOps should not run a second reconciliation loop for those responsibilities or become a replacement for BCM. + +The DeepOps-owned state should stay deliberately small: + +- MAAS tags that map deployed machines into DeepOps inventory groups, such as `kube_control_plane`, `kube_node`, `slurm-master`, and `slurm-node`. +- A dynamic inventory view from `scripts/maas_inventory.py`, derived from deployed MAAS machines and tags. +- Explicit deploy, tag, status, and release operations through `scripts/maas_deploy.sh`. +- Ansible run artifacts and validation results that record what DeepOps last applied and observed. + +This keeps DeepOps useful for lightweight cluster setup without competing with larger cluster managers. +It also keeps the contribution surface accessible: tags, inventory output, and playbook artifacts are easy for new contributors to inspect, reproduce, and improve. +If a site already uses BCM or another fleet manager, keep that system authoritative and use DeepOps only for the roles and playbooks the site intentionally delegates. + ## Pre-requisites In order to set up and use MAAS, you should at minimum have the following components: -- An Ubuntu 22.04 or 24.04 server which you can use to run MAAS +- An Ubuntu 22.04 or 24.04 server which you can use to run MAAS. + MAAS 3.7 is the current Ubuntu 24.04 line; use MAAS 3.5 if the controller itself remains on Ubuntu 22.04. - One or more servers which you will manage using MAAS - A network connection between all the servers on which you can safely run DHCP. This is needed so that MAAS can provision IP addresses to the nodes it manages. - A network connection which you can use to log into the MAAS server. This may be the same network as the inter-node network, or it may be a separate network. @@ -73,8 +92,10 @@ Please consult your hypervisor documentation for instructions on doing this. maas_dns_domain: 'deepops.local' maas_region_controller: '192.168.1.1' maas_region_controller_url: 'http://{{ maas_region_controller }}:5240/MAAS' - maas_repo: 'ppa:maas/3.5' + maas_repo: "{{ 'ppa:maas/3.7' if ansible_distribution_version is version('24.04', '>=') else 'ppa:maas/3.5' }}" ``` + If you know every MAAS controller host is running Ubuntu 24.04, you can set this directly to `ppa:maas/3.7`. + If the controller is still Ubuntu 22.04, keep `ppa:maas/3.5`. 1. Run the Ansible playbook to install: ```bash ansible-playbook -l playbooks/provisioning/maas.yml diff --git a/docs/pxe/minimal-pxe-container.md b/docs/pxe/minimal-pxe-container.md deleted file mode 100644 index bf68ab39e..000000000 --- a/docs/pxe/minimal-pxe-container.md +++ /dev/null @@ -1,69 +0,0 @@ -# Minimal PXE Container - -Minimal containers for OS installation - -- [Minimal PXE Container](#minimal-pxe-container) - - [Requirements](#requirements) - - [Installation Steps](#installation-steps) - - [IPMI Command Reference](#ipmi-command-reference) - -## Requirements - -- Control machine connected to the same VLAN/subnet as target machines -- Docker installed on control machine - -## Installation Steps - -This process should run from a Linux system on the same network segment as the target nodes. - -1. Install docker. - - ```bash - ./scripts/generic/install_docker.sh - ``` - -2. (Optional) Start DHCP server. - - If you have an existing DHCP server, skip this step - - ```bash - # Modify listen interface, DHCP range, and network gateway IP - docker-compose -f src/containers/pxe/docker-compose.yml run -d dhcp dnsmasq -d --interface=ens192 --dhcp-range=192.168.1.100,192.168.1.199,7200 --dhcp-option=6,8.8.8.8 --dhcp-option=3,192.168.1.1 - ``` - -3. (Optional) Configure NAT routing. - - If you have an existing network gateway, skip this step - - ```bash - # Set eth0 and eth1 to your public and private interfaces, respectively - ./scripts/pxe/setup_nat.sh eth0 eth1 - ``` - -4. Start PXE server. - - ```bash - docker-compose -f src/containers/pxe/docker-compose.yml up -d pxe - ``` - -5. Install OS. - - Set servers to boot from the network for the next boot only (to avoid re-install loops) and reboot them to install the OS. - - The default credentials are: - - - Username: `ubuntu` - - Password: `deepops` - -## IPMI Command Reference - -```bash -# Set to boot from disk, always -# Dell -chassis bootdev disk options=persistent -# DGX-1 -raw 0x00 0x08 0x05 0xe0 0x08 0x00 0x00 0x00 - -# Set to boot from the network, next boot only -chassis bootdev pxe options=efiboot -``` diff --git a/docs/slurm-cluster/README.md b/docs/slurm-cluster/README.md index 3db80e343..e5842cd4f 100644 --- a/docs/slurm-cluster/README.md +++ b/docs/slurm-cluster/README.md @@ -27,7 +27,8 @@ Instructions for deploying a GPU cluster with Slurm 1. Install a supported operating system on all nodes. - Install a supported operating system on all servers via a 3rd-party solution (i.e. [MAAS](https://maas.io/), [Foreman](https://www.theforeman.org/)) or utilize the provided [OS install container](../pxe). + Install a supported operating system on all servers via a 3rd-party solution such as [MAAS](https://maas.io/) or [Foreman](https://www.theforeman.org/), or via an existing site-standard automated installer. + For new Ubuntu 24.04 or DGX OS 7 deployments, prefer Ubuntu autoinstall/cloud-init or MAAS and then apply DeepOps after the OS is present. 2. Set up your provisioning machine. diff --git a/docs/slurm-cluster/slurm-perf-cluster.md b/docs/slurm-cluster/slurm-perf-cluster.md index 804fcc985..6b252f0d8 100644 --- a/docs/slurm-cluster/slurm-perf-cluster.md +++ b/docs/slurm-cluster/slurm-perf-cluster.md @@ -44,7 +44,8 @@ These packages have been installed and tested with the following Linux distribut 1. Install a supported operating system on all nodes. - Install a supported operating system on all servers utilizing the [DGXie](/docs/pxe/dgxie-container.md) provisioning container, via a 3rd-party solution (i.e. [MAAS](https://maas.io/), [Foreman](https://www.theforeman.org/)), or server BMC/console. + Install a supported operating system on all servers via a 3rd-party solution such as [MAAS](https://maas.io/) or [Foreman](https://www.theforeman.org/), via an existing site-standard automated installer, or through server BMC/console. + For new Ubuntu 24.04 or DGX OS 7 deployments, prefer Ubuntu autoinstall/cloud-init or MAAS. > NOTE: During OS installation, it is ideal if the identical user/password is configured. Otherwise, follow step 4 below to create an identical user across all nodes in the cluster. diff --git a/scripts/pxe/build_and_restart_dgxie.sh b/scripts/pxe/build_and_restart_dgxie.sh deleted file mode 100755 index f0d0ca46c..000000000 --- a/scripts/pxe/build_and_restart_dgxie.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -set -xe - -source config/pxe/env - -compose_directory_cmd="" #"--project-directory ." -compose_cmd="docker-compose --env-file ./config/pxe/env ${compose_directory} -f ${COMPOSE_FILE}" - - -function tear_down() { - ${compose_cmd} down -} - -function build() { - ${compose_cmd} build -} - -function bring_up() { - ${compose_cmd} up -d -} - - -tear_down -build -bring_up diff --git a/scripts/pxe/dgxctl.sh b/scripts/pxe/dgxctl.sh deleted file mode 100755 index c875d6218..000000000 --- a/scripts/pxe/dgxctl.sh +++ /dev/null @@ -1,270 +0,0 @@ -#!/usr/bin/env bash - -# Set configuration -. config/pxe/ipmi.conf - - -OPTIND=1 -IPMI_HOST_LIST="config/pxe/ipmi_host_list" -install=0 -progress=0 -upgrade=0 -config_host= -host_list=0 -install_log=0 -ssh_check=0 -show_bmc=0 -validate=0 -ipmi= -retry=0 -fw_target=all - -usage () { - echo "Manage DGX install/upgrade/configure/check process" - echo - echo "Usage: $0 [arguments]" - echo - echo "General Arguments:" - echo " -h Show help" - echo " -x Show list of DGX host IP addresses from DHCP server" - echo " -y Show DGX server install log from provisioning container" - echo " -z Verify DGX are available via SSH connection" - echo " -b Show DGX BMC IP matched with host IP (use with -z)" - echo " -w Control DGX power status via IPMI (options: on, off)" - echo - echo "Install Arguments:" - echo " -i Run install process" - echo " -p Show install progress" - echo " -f Use alternate IP file (default: ${IPMI_HOST_LIST})" - echo - echo "Update/Configuration arguments:" - echo " -u Run upgrade and configure process on all hosts" - echo " -l Run upgrade and configure process on single host" - echo " -q Re-run only failed upgrade/configuration tasks" - echo " -r Update FRU (default: only update SBIOS and BMC)" - echo - echo "Validation/check arguments:" - echo " -v Run the validation checks on all hosts" - echo " -l Run the validation checks on a single host" - exit 0 -} - -while getopts "h?pif:ul:rxyzbvw:q" opt; do - case "$opt" in - h|\?) - usage - ;; - i) install=1 - ;; - p) progress=1 - ;; - f) IPMI_HOST_LIST="${OPTARG}" - ;; - u) upgrade=1 - ;; - l) config_host="${OPTARG}" - ;; - r) fw_target="FRU" - ;; - x) host_list=1 - ;; - y) install_log=1 - ;; - z) ssh_check=1 - ;; - b) show_bmc=1 - ;; - v) validate=1 - ;; - w) ipmi="${OPTARG}" - ;; - q) retry=1 - ;; - *) - usage - ;; - esac -done -shift $((OPTIND-1)) -[ "$1" == "--" ] && shift - -test -f "${IPMI_HOST_LIST}" -if [ $? -ne 0 ] ; then - echo File not found: "${IPMI_HOST_LIST}" - exit 1 -fi - -if [ "${progress}" -eq 1 ] ; then - # Chassis identify information: - # $ sudo ipmitool -I lanplus -H -U dgxuser -P dgxuser chassis identify force - # Chassis identify interval: indefinite - # $ sudo ipmitool -I lanplus -H -U dgxuser -P dgxuser raw 0x00 0x01 - # 41 10 60 10 - # $ sudo ipmitool -I lanplus -H -U dgxuser -P dgxuser chassis identify 0 - # Chassis identify interval: off - # $ sudo ipmitool -I lanplus -H -U dgxuser -P dgxuser raw 0x00 0x01 - # 41 10 40 10 - echo "Install progress (host list: ${IPMI_HOST_LIST})": - echo - while read -u10 IPMI_HOST_IP ; do - echo -n "${IPMI_HOST_IP}: " - chassis_ident_state=$(sudo ipmitool -I lanplus -U ${IPMI_USERNAME} -P ${IPMI_PASSWORD} -H ${IPMI_HOST_IP} raw 0x00 0x01 | awk '{print $3}') - if [ "${chassis_ident_state}" == "60" ] ; then - echo installing... - elif [ "${chassis_ident_state}" == "40" ] ; then - echo finished - else - echo status unknown... - fi - done 10<${IPMI_HOST_LIST} -elif [ "${install}" -eq 1 ] ; then - echo Initiating PXE install process via BMC host list: ${IPMI_HOST_LIST} - while read -u10 IPMI_HOST_IP ; do - echo -n "${IPMI_HOST_IP}: " - - # make sure BMC is reachable - sudo ipmitool -I lanplus -U ${IPMI_USERNAME} -P ${IPMI_PASSWORD} -H ${IPMI_HOST_IP} bmc info >/dev/null 2>&1 - if [ $? -ne 0 ] ; then - echo "Error communicating with BMC" - continue - fi - echo -n "available | " - - # disable IPMI boot device selection 60s timeout - sudo ipmitool -I lanplus -U ${IPMI_USERNAME} -P ${IPMI_PASSWORD} -H ${IPMI_HOST_IP} raw 0x00 0x08 0x03 0x08 >/dev/null 2>&1 - if [ $? -ne 0 ] ; then - echo -n "config ERROR" - continue - fi - echo -n "config(1) | " - - # set baseline (boot to disk, efi, persistent) - sudo ipmitool -I lanplus -U ${IPMI_USERNAME} -P ${IPMI_PASSWORD} -H ${IPMI_HOST_IP} raw 0x00 0x08 0x05 0xe0 0x08 0x00 0x00 0x00 >/dev/null 2>&1 - if [ $? -ne 0 ] ; then - echo -n "config ERROR" - continue - fi - echo -n "config(2) | " - - # set boot device to PXE, EFI, next boot only. Needed when defaulting to install vs boot local disk - sudo ipmitool -I lanplus -U ${IPMI_USERNAME} -P ${IPMI_PASSWORD} -H ${IPMI_HOST_IP} chassis bootdev pxe options=efiboot >/dev/null 2>&1 - if [ $? -ne 0 ] ; then - echo -n "pxe ERROR" - continue - fi - echo -n "pxe | " - - # check that we have the correct bitmask - a004000000 - BOOT_CODE=a004000000 - boot_param=$(sudo ipmitool -I lanplus -U ${IPMI_USERNAME} -P ${IPMI_PASSWORD} -H ${IPMI_HOST_IP} chassis bootparam get 0x05 | egrep "^Boot parameter data:" | awk '{print $4}') - if [ "${boot_param}" != "${BOOT_CODE}" ] ; then - echo "Error: boot parameter incorrect (${boot_param})" - continue - fi - - # power off/on host - sudo ipmitool -I lanplus -U ${IPMI_USERNAME} -P ${IPMI_PASSWORD} -H ${IPMI_HOST_IP} power off >/dev/null 2>&1 - if [ $? -ne 0 ] ; then - echo "power off ERROR" - continue - fi - sleep 5 - sudo ipmitool -I lanplus -U ${IPMI_USERNAME} -P ${IPMI_PASSWORD} -H ${IPMI_HOST_IP} power on >/dev/null 2>&1 - if [ $? -ne 0 ] ; then - echo "power on ERROR" - continue - fi - echo "reset | installing..." - - # turn on chassis identifier light to indicate installation is in progress - sudo ipmitool -I lanplus -U ${IPMI_USERNAME} -P ${IPMI_PASSWORD} -H ${IPMI_HOST_IP} chassis identify force >/dev/null 2>&1 - - done 10<${IPMI_HOST_LIST} - echo The install process will take approximately 15 minutes -elif [ "${upgrade}" -eq 1 ] || [ "${validate}" -eq 1 ] ; then - # Get host list and generate ansible inventory file - host_list=$(curl -s localhost/hosts | grep 54:ab:3a | awk '{print $2}') - - inventory_file=$(mktemp) - echo "[all:vars]" > "${inventory_file}" - echo "ansible_user=${DGX_USERNAME}" >> "${inventory_file}" - echo "ansible_ssh_pass=${DGX_PASSWORD}" >> "${inventory_file}" - echo "ansible_sudo_pass=${DGX_PASSWORD}" >> "${inventory_file}" - echo "[hosts]" >> "${inventory_file}" >> "${inventory_file}" - - for host in ${host_list} ; do - echo "${host}" >> "${inventory_file}" - # remove stale host key in case we did a re-install - ssh-keygen -f "~/.ssh/known_hosts" -R "${host}" >/dev/null 2>&1 - done - - # Run configuration scripts - if [ "${upgrade}" -eq 1 ] ; then - echo "The upgrade process will take upwards of 30 minutes" - echo "Start time: $(date)" - - if [ "${retry}" -eq 1 ] ; then - ansible-playbook -i "${inventory_file}" -l "@${ANSIBLE_REPO}/playbook.retry" "${ANSIBLE_REPO}/playbook.yml" - elif [ "x${config_host}" != "x" ] ; then - ansible-playbook -i "${inventory_file}" -l "${config_host}" "${ANSIBLE_REPO}/playbook.yml" --extra-vars \"target_fw=${fw_target}\" - else - ansible-playbook -i "${inventory_file}" -l hosts "${ANSIBLE_REPO}/playbook.yml" --extra-vars \"target_fw=${fw_target}\" - fi - elif [ "${validate}" -eq 1 ] ; then - echo "The validation process will take approximately 10 minutes" - echo "Start time: $(date)" - - if [ "x${config_host}" != "x" ] ; then - echo ansible-playbook -i "${inventory_file}" -l "${config_host}" "${ANSIBLE_REPO}/playbook.yml" - else - echo ansible-playbook -i "${inventory_file}" -l hosts "${ANSIBLE_REPO}/playbook.yml" - fi - fi - - rm -f "${inventory_file}" - echo "End time: $(date)" -elif [ "${host_list}" -eq 1 ] ; then - curl localhost/hosts -elif [ "${install_log}" -eq 1 ] ; then - curl localhost/log -elif [ "${ssh_check}" -eq 1 ] ; then - # Get host list - host_list=$(curl -s localhost/hosts | grep 54:ab:3a | awk '{print $2}') - # Check hosts - for host in ${host_list} ; do - echo -n "${host}: " - ping -c1 "${host}" >/dev/null 2>&1 - if [ $? -ne 0 ] ; then - echo "unavailable" - continue - fi - # remove stale host key in case we did a re-install - ssh-keygen -f "${HOME}/.ssh/known_hosts" -R "${host}" >/dev/null 2>&1 - if [ "${show_bmc}" -eq 1 ] ; then - sshpass -p "${DGX_PASSWORD}" ssh -oStrictHostKeyChecking=no dgxuser@"${host}" "echo ${DGX_PASSWORD} | sudo -S ipmitool lan print 1 2>/dev/null | egrep '^IP Address' | tail -1 | awk '{print \$4}'" 2>&1 | grep -v Warning - else - sshpass -p "${DGX_PASSWORD}" ssh -oStrictHostKeyChecking=no dgxuser@"${host}" uptime 2>&1 | grep -v Warning - fi - done -elif [ "x${ipmi}" != "x" ] ; then - # Power on/off DGX via IPMI - if [ "x${config_host}" != "x" ] ; then - echo -n "${config_host}: " - if [ "${ipmi}" == "on" ] ; then - sudo ipmitool -I lanplus -U ${IPMI_USERNAME} -P ${IPMI_PASSWORD} -H "${config_host}" power on - elif [ "${ipmi}" == "off" ] ; then - sudo ipmitool -I lanplus -U ${IPMI_USERNAME} -P ${IPMI_PASSWORD} -H "${config_host}" power off - fi - else - while read -u10 IPMI_HOST_IP ; do - echo -n "${IPMI_HOST_IP}: " - if [ "${ipmi}" == "on" ] ; then - sudo ipmitool -I lanplus -U ${IPMI_USERNAME} -P ${IPMI_PASSWORD} -H ${IPMI_HOST_IP} power on - elif [ "${ipmi}" == "off" ] ; then - sudo ipmitool -I lanplus -U ${IPMI_USERNAME} -P ${IPMI_PASSWORD} -H ${IPMI_HOST_IP} power off - fi - done 10<${IPMI_HOST_LIST} - fi -else - usage -fi diff --git a/scripts/pxe/install_dgxie_prereqs.sh b/scripts/pxe/install_dgxie_prereqs.sh deleted file mode 100755 index 1895c4e92..000000000 --- a/scripts/pxe/install_dgxie_prereqs.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash - -# install required software -sudo apt-get update -sudo apt-get install -y git ipmitool vim software-properties-common sshpass - -# install docker -type docker >/dev/null 2>&1 -if [ $? -ne 0 ] ; then - sudo apt-get install -y apt-transport-https ca-certificates curl software-properties-common - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - sudo apt-get update - sudo apt-get install -y docker-ce - docker --version -fi - -# install docker-compose -type docker-compose >/dev/null 2>&1 -if [ $? -ne 0 ] ; then - sudo curl -L https://github.com/docker/compose/releases/download/1.17.0/docker-compose-`uname -s`-`uname -m` -o /usr/local/bin/docker-compose - sudo chmod +x /usr/local/bin/docker-compose - docker-compose --version -fi - -# install ansible -type ansible >/dev/null 2>&1 -if [ $? -ne 0 ] ; then - sudo apt-add-repository -y ppa:ansible/ansible - sudo apt-get update - sudo apt-get -y install ansible - ansible --version -fi - -# configure ISO mount -grep DGXServer /etc/fstab -if [ $? -ne 0 ] ; then - mkdir -p /mnt/3.1.2 - echo "${HOME}/DGXServer-3.1.2.170902_f8777e.iso /mnt/3.1.2 iso9660 loop 0 0" | sudo tee -a /etc/fstab -fi - -# remove SUDO password requirement -sudo sed -i "s/^\%sudo.*/\%sudo ALL=\(ALL:ALL\) NOPASSWD: ALL/g" /etc/sudoers diff --git a/scripts/pxe/setup_nat.sh b/scripts/pxe/setup_nat.sh deleted file mode 100755 index 9d6183579..000000000 --- a/scripts/pxe/setup_nat.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env bash - -export HOST_INT_PUB="${1}" -export HOST_INT_PRV="${2}" - -ip a show dev "${HOST_INT_PUB}" -if [ $? -ne 0 ] ; then - exit 1 -fi - -ip a show dev "${HOST_INT_PRV}" -if [ $? -ne 0 ] ; then - exit 1 -fi - -set -x -sudo /sbin/iptables -t nat -A POSTROUTING -o ${HOST_INT_PUB} -j MASQUERADE -sudo /sbin/iptables -A FORWARD -i ${HOST_INT_PUB} -o ${HOST_INT_PRV} -m state --state RELATED,ESTABLISHED -j ACCEPT -sudo /sbin/iptables -A FORWARD -i ${HOST_INT_PRV} -o ${HOST_INT_PUB} -j ACCEPT -sudo sysctl -w net.ipv4.ip_forward=1 -set +x diff --git a/src/containers/dgxie/Dockerfile b/src/containers/dgxie/Dockerfile deleted file mode 100644 index b5498f9c3..000000000 --- a/src/containers/dgxie/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -FROM ubuntu:16.04 - -MAINTAINER Douglas Holt - -RUN apt-get update && \ - apt-get -y install apt-transport-https curl && \ - curl -L https://packagecloud.io/danderson/pixiecore/gpgkey | apt-key add - && \ - echo "deb https://packagecloud.io/danderson/pixiecore/debian stretch main" >/etc/apt/sources.list.d/pixiecore.list && \ - apt-get update && \ - apt-get -y install pixiecore nginx vsftpd iptables dnsmasq python-flask - -RUN mkdir -p /www /var/run/vsftpd/empty /www/vmware - -COPY get_hosts.py /usr/local/bin -COPY rest_api.py /usr/local/bin -COPY api.py /api.py -COPY start /usr/sbin/start -COPY nginx.conf /etc/nginx/nginx.conf -COPY dnsmasq.conf /etc/dnsmasq.conf -COPY vsftpd.conf /etc/vsftpd.conf - -COPY kickstart.cfg /www/vmware -COPY mboot.efi /www/vmware - -VOLUME /etc/dnsmasq.d - -ENTRYPOINT ["/bin/bash"] -CMD ["/usr/sbin/start"] diff --git a/src/containers/dgxie/README.md b/src/containers/dgxie/README.md deleted file mode 100644 index 046388588..000000000 --- a/src/containers/dgxie/README.md +++ /dev/null @@ -1,461 +0,0 @@ - -DGXie: PXE Boot DGX Install Environment -=== - -This repo is intended to be used to automate provisioning of DGX-1 Servers over the network. The tools will automatically configure, reboot, re-install and re-configure the DGX with minimal commands. - -You will need a computer (i.e. a laptop) running Ubuntu 16.04 which has two network connections, i.e built-in and USB/Ethernet adapter. - -If you are setting up a laptop from scratch, see the "Setting up a new laptop" section at the end of the README - -Otherwise, follow each of the sections below in order: - -## Connect the laptop to the network - -Connect the built-in ethernet to the private network where the DGX are connected, and the usb-ethernet dongle to a public, internet accessible network, if available. - The install process does not require an internet connection, but it may be useful later for updating or installing extra software. - -This assumes the built-in ethernet interface is the "public" network and the usb-ethernet dongle is the "private" network where the DGX are connected - -In addition to the network connection to the DGX data network, the laptop will need to have access to the management network where the DGX BMCs are connected. - This can be shared through the private interface connected to the DGX data network or through the second interface connected to the public network. - -In some cases you may need to: -* Disconnect the network uplink from DGX data networks -* Set the DHCP helper IP in the ToR switches to point to the private interface on the laptop where DGXie runs its servers - -The private interface on the laptop should have a static IP/netmask/etc. (i.e 192.168.1.1/24) - -To configure the private interface on the laptop, you can either use the Network Manager GUI in Ubuntu Desktop, or configure - via `nmcli` in a terminal: - -List network devices: - -```console -nmcli con show -``` - -Modify settings of private interface: - -```console -sudo nmcli c modify "Belkin USB-ethernet adapter" ipv4.addresses 192.168.1.1/24 ipv4.dns 8.8.8.8 ipv4.gateway 192.168.1.1 -``` - -Show device properties: - -```console -nmcli con show "Belkin USB-ethernet adapter" -``` - -Activate interface: - -```console -sudo nmcli con up "Belkin USB-ethernet adapter" -``` - -### BMC considerations - -You will need a list of BMC IP addresses for each DGX Server to be provisioned. - -The laptop will likely need a second network connection to the BMC network, this can be accomplished by adding a second IP address to the network interface - used to connect to the DGX network, or by connecting a second network interface, such as the USB-ethernet adapter, to the network and assigning an IP in - the BMC subnet. - -If the DHCP server for the BMC network is disconnected, a logical choice for IP address would be the gateway IP for the BMC subnet. You can get this IP - from Penguin or from a DGX BMC via IPMI. - -## Run the Dgxie service container - -If you need to modify the default network settings, modify `docker-compose.yml` to edit the environment variables to configure DGXie. - See the section at the end of the README for all configuration options. You may need to configure the DGXie container with - public and private interface names, and IP information to configure the DHCP server if you are not using 192.168.1.0/24. - -Run containers: - -```console -sudo docker-compose build -sudo docker-compose up -d -``` - -Make sure the containers are running with: `sudo docker ps -a` - -You can check the container logs with (you may have to substitute a different container name): `sudo docker logs deepops_dgxie_1` - -## Provision DGX - -The DGX install process will take approximately **15** minutes. - -**Steps:** -* Modify the *ipmi_host_list* script to contain the BMC IP address of each DGX which needs to be re-imaged, one per line -* Modify the *configuration* file to contain the username and password of the DGX BMCs. This file also contains the default username and password of the DGX ISO, - which should not need to be changed. -* Run *dgxctl.sh* with the `-i` flag to start the install process (see example below) - -The *dgxctl.sh* script will iterate over each BMC IP address, one at a time, making sure it's up, disabling the boot order timeout, setting the system to PXE boot - and power-cycling the system via BMC. - -The DGX will immediately power-cycle and attempt to boot from the connected network interface. The DGXie container provides the DHCP and PXE server which the DGX - will use to automatically run the install process without user intervention. When the install process is finished, the DGX will automatically reboot and boot to - the first hard disk, which now contains the DGX Server OS. - -The *dgxctl.sh* script will turn on the DGX chassis identification light during the install. If there is a DGX with a suspected install problem, you can either - check the DGX via virtual console on the BMC IP address or connect a physical console to the affected DGX, identified by a lit chassis identification light. - -The default BMC credentials are: - -```console -Username: dgxuser -Password: dgxuser -``` - -The default DGX OS user login credentials are: - -```console -Username: dgxuser -Password: DgxUser123 -``` - -Here's an example of running *dgxctl.sh* and the expected output: - -```console -$ ./dgxctl.sh -i -Initiating PXE install process via BMC host list: ipmi_host_list - -10.0.1.1: available | config(1) | config(2) | pxe | reset | installing... -10.0.1.2: available | config(1) | config(2) | pxe | reset | installing... -``` - -If a system fails, you'll see an error: - -```console -$ ./dgxctl.sh -i -Initiating PXE install process via BMC host list: ipmi_host_list - -10.0.1.1: Error communicating with BMC on host: 10.0.1.1 -``` - -You can check the install progress with the `-p` flag: - -```console -$ ./dgxctl.sh -p -Install progress (host list: ipmi_host_list): - -10.0.1.1: installing... -10.0.1.2: installing... - -$ ./dgxctl.sh -p -Install progress (host list: ipmi_host_list): - -10.0.1.1: finished -10.0.1.2: finished -``` - -If you want to specify a different BMC IP file, use the `-f ` flag. You can see the help options with the `-h` flag. - -### Provisioning API - -DGXie provides several methods to obtain data, these should be run from the laptop outside of the DGXie container - -Get list of hosts: - -```console -$ ./dgxctl.sh -x -+------------------------------------------------------------------------------ -| DHCPD ACTIVE LEASES REPORT -+-----------------+-------------------+----------------------+----------------- -| IP Address | MAC Address | Expires (days,H:M:S) | Client Hostname -+-----------------+-------------------+----------------------+----------------- -| 192.168.1.3 | 54:ab:3a:d6:61:9d | 11:43:56 | -| 192.168.1.4 | 54:ab:3a:da:c4:8b | 11:43:53 | -+-----------------+-------------------+----------------------+----------------- -| Total Active Leases: 2 -| Report generated (UTC): 2017-11-09 15:16:51 -+------------------------------------------------------------------------------ -``` - -Get list of finished installs: - -```console -$ ./dgxctl.sh -y -== LOG OPENED == -2017-11-09 15:02:23.464795: start - 192.168.1.4 -2017-11-09 15:02:23.798555: start - 192.168.1.3 -2017-11-09 15:08:43.272009: end - 192.168.1.4 -2017-11-09 15:08:52.148569: end - 192.168.1.3 -``` - -### Monitoring the DGX install and confirming it has completed - -Once the output of `./dgxctl.sh -y` and `./dgxctl.sh -p` show that the install has ended/finished, it will take a few additional minutes for the DGX - to reboot and boot into the new operating system on the disk. - -You can run the command below to check whether the DGX are ready and available to move on to the next steps: - -```console -$ ./dgxctl.sh -z -192.168.1.3: 07:29:51 up 10:44, 0 users, load average: 0.16, 0.16, 0.11 -192.168.1.4: 07:29:51 up 10:45, 0 users, load average: 0.19, 0.14, 0.15 -``` - -## End - -## Misc Tasks - -**Power on/off all DGX via IPMI** - -```console -$ ./dgxctl.sh -w on -10.0.1.1: Chassis Power Control: Up/On -10.0.1.2: Chassis Power Control: Up/On -``` - -**Power on/off a single DGX via IPMI** - -```console -$ ./dgxctl.sh -w off -l 10.0.1.1 -10.0.1.1: Chassis Power Control: Down/Off -``` - -# Optional information - -This information is not required if you are using a pre-configured laptop. It's left here for reference if you are starting from scratch. - -## Setting up a new laptop - -### Setting up a system to run the DGXie tools: - -You will need a laptop running Ubuntu 16.04, which has two network connections, i.e wireless and USB/Ethernet adapter. - One connection should have access to the internet (public), - while the other connection should be to a dedicated network (private) containing the DGX to be provisioned. - -Download the DGX Server ISO from the Enterprise Support Portal: https://nvidia-esp.custhelp.com - -*Currently tested with DGX Server 3.1.2 170902 f8777e* - -Place the ISO in the user home directory, e.g. `${HOME}/DGXServer-3.1.2.170902_f8777e.iso` - -Run the *install_prereqs.sh* script on the Ubuntu 16.04 Linux laptop: - -```console -./install_prereqs.sh -``` - -DGXie: PXE Boot DGX Install Environment - container components -=== - -DGXie is a Docker container application for remotely installing the official DGX Server operating system over the network - -DGXie contains: - * DHCP server - * Provides PXE boot environment and DGX network settings - * TFTP server - * Provides PXE bootstrap files - * FTP server - * Provides a repo for the official DGX install ISO - * HTTP server - * Provides additional files such as a modified install pre-seed - * NAT setup - * Provides internet access to the DGX network through the system running DGXie - * REST API - * Provides list of host IP addresses from DHCP leases - -DGXie should be run on a system connected to a network of DGX servers. - The DGX servers are set to boot from the network interface connected to this network and will present a menu of boot options. - Current boot options are to boot to the local disk (default) or to install the DGX operating system on the DGX. - __There are no additional prompts after the menu, and the DGX will be completely erased during the install process.__ - -## Network topology - -(public network)------[DGXie system]------(private network)------[DGX server systems] - -The computer running this container can be on either a single network or two networks (public/private). - -The container will attempt to set up NAT routing from a private to public subnet. - -## Prerequisites - -The computer running DGXie needs Docker and should be capable of running IPtables for NAT to work (Linux) - -Download the DGX Server ISO from the Enterprise Support Portal: https://nvidia-esp.custhelp.com - -Disable the boot order update timeout (required): - -```console -# disable IPMI boot device selection 60s timeout -ipmitool -I lanplus -U -P -H raw 0x00 0x08 0x03 0x08 -``` - -Set your DGX to boot from the network (PXE) for the next boot only: - -```console -# set boot device to PXE, EFI, next boot only. Needed when defaulting to install vs boot local disk -ipmitool -I lanplus -U -P -H chassis bootdev pxe options=efiboot -``` - -### Optional/Misc - -You can also run the IPMI commands directly on the DGX via `ipmitool`: - -```console -sudo ipmitool raw ... -``` - -To set the DGX to boot from disk first: - -```console -# set boot device to first disk, EFI, persistent -ipmitool -I lanplus -U -P -H raw 0x00 0x08 0x05 0xe0 0x08 0x00 0x00 0x00 -``` - -You can set the DGX to boot from the network every time, but if DGXie is set to default to install, this can create a re-install loop - -```console -# set boot device to PXE, EFI, persistent -ipmitool -I lanplus -U -P -H raw 0x00 0x08 0x05 0xe0 0x04 0x00 0x00 0x00 -``` - -## Setup and running - -Mount the DGX Server ISO as a volume when running the container. - -*Tested with DGX Base OS 3.1.2* - -```console -sudo mkdir -p /mnt/3.1.2 -sudo mount -o loop DGXServer-3.1.2.170902_f8777e.iso /mnt/3.1.2 -``` - -Add an IP to your private interface (DGX network) if required - -```console -sudo ip addr add 192.168.1.1/24 broadcast 192.168.1.255 dev ens192 -``` - -Build and run the container - -```console -docker build -t dgxie . -docker run -d --privileged --net=host -v /mnt/3.1.2:/iso:ro --name dgxie dgxie -``` - -The `--privileged` and `--net=host` flags are required to manipulate IPTABLES on the host. - -### DGXie configuration options - -#### Provisioning host configuration - -Specify DGXie server public network interface - -```console -# default: eth0 --e HOST_INT_PUB=ens160 -``` - -Specify DGXie server private network interface - -```console -# default: eth1 --e HOST_INT_PRV=ens192 -``` - -#### Provisioning network configuration - -Options to configure the DHCP server subnet, these options are probably required - -Specify DHCP/PXE server IP address (IP of machine running DGXie on DGX network) - -```console -# default: 192.168.1.1 --e IP=10.0.0.1 -``` - -Specify DHCP/PXE server network subnet - -```console -# default: 192.168.1.0 --e NETWORK=10.0.0.0 -``` - -Specify DHCP/PXE server subnet netmask - -```console -# default: 255.255.255.0 --e NETMASK=255.255.255.0 -``` - -Specify DHCP/PXE server subnet gateway - -```console -# default: (192.168.1.1) --e GATEWAY=10.0.0.254 -``` - -Specify DHCP/PXE server DNS - -```console -# default: 8.8.8.8, 8.8.4.4 --e DNS1=10.1.1.1 -e DNS2=10.2.2.2 -``` - -Specify DHCP/PXE server lease range start/end - -```console -# default: 192.168.1.2 192.168.1.254 --e DHCP_START=10.0.0.2 -e DHCP_END=10.0.0.254 -``` - -#### DGX install options - -These options are probably optional unless you're installing on something other than a DGX - -Use a different interface on DGX clients - -```console -# default: enp1s0f0 --e INT=eth0 -``` - -Use a different disk for the root partition on DGX clients - -```console -# default: sda --e DISK=xvda -``` - -### Examples: - -DGXie container running on a VM and provisioning DGX with the default subnet options. The VM has a public and private interface on two different VLANs. - -```console -sudo docker run --rm -ti --net=host --privileged -v /mnt/3.1.2:/iso:ro -e HOST_INT_PUB=ens160 -e HOST_INT_PRV=ens192 --name dgxie dgxie -``` - -Attach to a running container: - -```console -docker exec -ti dgxie /bin/sh -``` - -Show current DGX boot flags: - -```console -sudo ipmitool chassis bootparam get 0x05 -``` - -## Default user/pass - -DGX installations will default to these login credentials: - -user: dgxuser - -pass: DgxUser123 - -## REST API - -DGXie will output a list of uniq IP address of DGX-1 servers via REST API. Adjust `localhost` to the host running DGXie if not running from the same machine: - -```console -curl http://localhost/hosts -``` - -DHCPD lease file parse script source: https://askubuntu.com/questions/219609/how-do-i-show-active-dhcp-leases diff --git a/src/containers/dgxie/api.py b/src/containers/dgxie/api.py deleted file mode 100755 index f839bb685..000000000 --- a/src/containers/dgxie/api.py +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/python -from flask import Flask, abort, request -import json -import datetime -import re -import os - -app = Flask(__name__) - -@app.route('/v1/boot/') -def pxe(mac): - '''See https://github.com/danderson/netboot/blob/master/pixiecore/README.api.md for API specs''' - # load machine profiles for each call so we can re-load changes from disk - jf = open('/etc/machines/machines.json', 'r') - machines = json.load(jf) - jf.close() - - if "HTTP_PORT" in os.environ.keys(): - http_port = os.environ['HTTP_PORT'] - else: - http_port = "13370" - - # return profile in json for matching machine - for machine in machines: - if 'mac' in machines[machine] and re.match(machines[machine]['mac'], mac): - machines[machine]['mac'] = mac - - machines[machine]['kernel'] = machines[machine]['kernel'].replace("$HTTP_PORT", http_port) - if 'cmdline' in machines[machine]: - machines[machine]['cmdline'] = machines[machine]['cmdline'].replace("$HTTP_PORT", http_port) - if 'initrd' in machines[machine]: - for i in range(len(machines[machine]['initrd'])): - machines[machine]['initrd'][i] = machines[machine]['initrd'][i].replace("$HTTP_PORT", http_port) - - return json.dumps(machines[machine]) - abort(404) - -@app.route('/install', methods=['POST']) -def install(): - if request.method == 'POST': - timestamp = datetime.datetime.now() - ip = request.environ.get('HTTP_X_REAL_IP', request.remote_addr) - action = request.form['action'] - print timestamp, ip - return 'done' - -if __name__ == '__main__': - app.run(port=9090, threaded=True) diff --git a/src/containers/dgxie/dnsmasq.conf b/src/containers/dgxie/dnsmasq.conf deleted file mode 100644 index 2d434177b..000000000 --- a/src/containers/dgxie/dnsmasq.conf +++ /dev/null @@ -1,24 +0,0 @@ -domain-needed -bogus-priv -strict-order -no-resolv -no-poll -expand-hosts -cache-size=2048 -bind-interfaces - -server=#DNS1# -server=#DNS2# -domain=#DOMAIN# -interface=#DHCP_INT# - -log-queries -log-dhcp -log-facility=/var/log/dnsmasq.log - -dhcp-authoritative -dhcp-range=#DHCP_START#,#DHCP_END#,#LEASETIME# -dhcp-option=tag:green,option:domain-search,#DOMAIN# -dhcp-option=3,#GATEWAY# - -conf-dir=/etc/dnsmasq.d,*.conf diff --git a/src/containers/dgxie/docker-compose.yml b/src/containers/dgxie/docker-compose.yml deleted file mode 100644 index 2b018fcd1..000000000 --- a/src/containers/dgxie/docker-compose.yml +++ /dev/null @@ -1,49 +0,0 @@ -# sudo docker run --rm -ti --net=host --privileged -v /mnt/:/iso:ro -e HOST_INT_PUB=ens160 -e HOST_INT_PRV=ens192 --name dgxie dgxie -# sudo docker run -d --name nfs --privileged --net=host -v /home/ubuntu/:/master -e SHARED_DIRECTORY=/master nfs-server:latest -version: '3.4' -services: - dgxie: - build: - context: . - network: "host" - network_mode: "host" - privileged: true - volumes: - - ${DATA_DIR}:/data:ro # path to ISO files - - ../../config/pxe/machines:/etc/machines:ro - - ../../config/pxe/dnsmasq.extra.conf:/etc/dnsmasq.d/dnsmasq.extra.conf:ro - environment: - - DHCP_ENABLE=${DHCP_ENABLE} - - DHCP_INT=${DHCP_INT} - - NETWORK=${NETWORK} - - NETMASK=${NETMASK} - - GATEWAY=${GATEWAY} - - DNS1=${DNS1} - - DNS2=${DNS2} - - DHCP_START=${DHCP_START} - - DHCP_END=${DHCP_END} - - LEASETIME=${LEASETIME} - - DOMAIN=${DOMAIN} - - NTP=${NTP} - - NAT_ENABLE=${NAT_ENABLE} - - NAT_INT_PUB=${NAT_INT_PUB} - - NAT_INT_PRV=${NAT_INT_PRV} - - HTTP_INT=${HTTP_INT} - - HTTP_PORT=${HTTP_PORT} - - DGX_INT=${DGX_INT} - - DGX_DISK=${DGX_DISK} - - DGX_KBD=${DGX_KBD} - - DGX_KERN_EXTRA=${DGX_KERN_EXTRA} - - DGX_EXTRA_PACKAGES=${DGX_EXTRA_PACKAGES} - - DGX_HTTPS_PROXY=${DGX_HTTPS_PROXY} - - VMW_NFS_IP=${VMW_NFS_IP} - - VMW_GPU_VIB=${VMW_GPU_VIB} - - VMW_VM_SIZE=${VMW_VM_SIZE} - nfs-server: - image: itsthenetwork/nfs-server-alpine:latest - network_mode: "host" - privileged: true - volumes: - - ${DATA_DIR}:/shared - environment: - - SHARED_DIRECTORY=/shared diff --git a/src/containers/dgxie/get_hosts.py b/src/containers/dgxie/get_hosts.py deleted file mode 100755 index 2b594bcc1..000000000 --- a/src/containers/dgxie/get_hosts.py +++ /dev/null @@ -1,258 +0,0 @@ -#!/usr/bin/python -import datetime, bisect - -def parse_timestamp(raw_str): - tokens = raw_str.split() - - if len(tokens) == 1: - if tokens[0].lower() == 'never': - return 'never'; - - else: - raise Exception('Parse error in timestamp') - - elif len(tokens) == 3: - return datetime.datetime.strptime(' '.join(tokens[1:]), - '%Y/%m/%d %H:%M:%S') - - else: - raise Exception('Parse error in timestamp') - - -def timestamp_is_ge(t1, t2): - if t1 == 'never': - return True - - elif t2 == 'never': - return False - - else: - return t1 >= t2 - - -def timestamp_is_lt(t1, t2): - if t1 == 'never': - return False - - elif t2 == 'never': - return t1 != 'never' - - else: - return t1 < t2 - - -def timestamp_is_between(t, tstart, tend): - return timestamp_is_ge(t, tstart) and timestamp_is_lt(t, tend) - - -def parse_hardware(raw_str): - tokens = raw_str.split() - - if len(tokens) == 2: - return tokens[1] - - else: - raise Exception('Parse error in hardware') - - -def strip_endquotes(raw_str): - return raw_str.strip('"') - - -def identity(raw_str): - return raw_str - - -def parse_binding_state(raw_str): - tokens = raw_str.split() - - if len(tokens) == 2: - return tokens[1] - - else: - raise Exception('Parse error in binding state') - - -def parse_next_binding_state(raw_str): - tokens = raw_str.split() - - if len(tokens) == 3: - return tokens[2] - - else: - raise Exception('Parse error in next binding state') - - -def parse_rewind_binding_state(raw_str): - tokens = raw_str.split() - - if len(tokens) == 3: - return tokens[2] - - else: - raise Exception('Parse error in next binding state') - - -def parse_leases_file(leases_file): - valid_keys = { - 'starts': parse_timestamp, - 'ends': parse_timestamp, - 'tstp': parse_timestamp, - 'tsfp': parse_timestamp, - 'atsfp': parse_timestamp, - 'cltt': parse_timestamp, - 'hardware': parse_hardware, - 'binding': parse_binding_state, - 'next': parse_next_binding_state, - 'rewind': parse_rewind_binding_state, - 'uid': strip_endquotes, - 'client-hostname': strip_endquotes, - 'option': identity, - 'set': identity, - 'on': identity, - 'abandoned': None, - 'bootp': None, - 'reserved': None, - } - - leases_db = {} - - lease_rec = {} - in_lease = False - in_failover = False - - for line in leases_file: - if line.lstrip().startswith('#'): - continue - - tokens = line.split() - - if len(tokens) == 0: - continue - - key = tokens[0].lower() - - if key == 'lease': - if not in_lease: - ip_address = tokens[1] - - lease_rec = {'ip_address' : ip_address} - in_lease = True - - else: - raise Exception('Parse error in leases file') - - elif key == 'failover': - in_failover = True - elif key == '}': - if in_lease: - for k in valid_keys: - if callable(valid_keys[k]): - lease_rec[k] = lease_rec.get(k, '') - else: - lease_rec[k] = False - - ip_address = lease_rec['ip_address'] - - if ip_address in leases_db: - leases_db[ip_address].insert(0, lease_rec) - - else: - leases_db[ip_address] = [lease_rec] - - lease_rec = {} - in_lease = False - - elif in_failover: - in_failover = False - continue - else: - raise Exception('Parse error in leases file') - - elif key in valid_keys: - if in_lease: - value = line[(line.index(key) + len(key)):] - value = value.strip().rstrip(';').rstrip() - - if callable(valid_keys[key]): - lease_rec[key] = valid_keys[key](value) - else: - lease_rec[key] = True - - else: - raise Exception('Parse error in leases file') - - else: - if in_lease: - raise Exception('Parse error in leases file') - - if in_lease: - raise Exception('Parse error in leases file') - - return leases_db - - -def round_timedelta(tdelta): - return datetime.timedelta(tdelta.days, - tdelta.seconds + (0 if tdelta.microseconds < 500000 else 1)) - - -def timestamp_now(): - n = datetime.datetime.utcnow() - return datetime.datetime(n.year, n.month, n.day, n.hour, n.minute, - n.second + (0 if n.microsecond < 500000 else 1)) - - -def lease_is_active(lease_rec, as_of_ts): - return timestamp_is_between(as_of_ts, lease_rec['starts'], - lease_rec['ends']) - - -def ipv4_to_int(ipv4_addr): - parts = ipv4_addr.split('.') - return (int(parts[0]) << 24) + (int(parts[1]) << 16) + \ - (int(parts[2]) << 8) + int(parts[3]) - - -def select_active_leases(leases_db, as_of_ts): - retarray = [] - sortedarray = [] - - for ip_address in leases_db: - lease_rec = leases_db[ip_address][0] - - if lease_is_active(lease_rec, as_of_ts): - ip_as_int = ipv4_to_int(ip_address) - insertpos = bisect.bisect(sortedarray, ip_as_int) - sortedarray.insert(insertpos, ip_as_int) - retarray.insert(insertpos, lease_rec) - - return retarray - - -############################################################################## - - -myfile = open('/var/lib/dhcp/dhcpd.leases', 'r') -leases = parse_leases_file(myfile) -myfile.close() - -now = timestamp_now() -report_dataset = select_active_leases(leases, now) - -print('+------------------------------------------------------------------------------') -print('| DHCPD ACTIVE LEASES REPORT') -print('+-----------------+-------------------+----------------------+-----------------') -print('| IP Address | MAC Address | Expires (days,H:M:S) | Client Hostname ') -print('+-----------------+-------------------+----------------------+-----------------') - -for lease in report_dataset: - print('| ' + format(lease['ip_address'], '<15') + ' | ' + \ - format(lease['hardware'], '<17') + ' | ' + \ - format(str((lease['ends'] - now) if lease['ends'] != 'never' else 'never'), '>20') + ' | ' + \ - lease['client-hostname']) - -print('+-----------------+-------------------+----------------------+-----------------') -print('| Total Active Leases: ' + str(len(report_dataset))) -print('| Report generated (UTC): ' + str(now)) -print('+------------------------------------------------------------------------------') diff --git a/src/containers/dgxie/kickstart.cfg b/src/containers/dgxie/kickstart.cfg deleted file mode 100644 index cdc149cc9..000000000 --- a/src/containers/dgxie/kickstart.cfg +++ /dev/null @@ -1,97 +0,0 @@ -#Accept the VMware End User License Agreement -vmaccepteula - -#clear paritions and install -clearpart --firstdisk --overwritevmfs -install --firstdisk --overwritevmfs - -#set the root password -rootpw d33pops! - -#Host Network Settings -network --bootproto=dhcp --addvmportgroup=1 - -reboot - -#Firstboot section 1 -%firstboot --interpreter=busybox - - -#Enable & start remote ESXi Shell (SSH) -vim-cmd hostsvc/enable_ssh -vim-cmd hostsvc/start_ssh - - -#Enable & start ESXi Shell (TSM) -vim-cmd hostsvc/enable_esx_shell -vim-cmd hostsvc/start_esx_shell - - -#Suppress Shell Warning -esxcli system settings advanced set -o /UserVars/SuppressShellWarning -i 1 -esxcli system settings advanced set -o /UserVars/ESXiShellTimeOut -i 1 - -#Disable ipv6 -esxcli network ip set --ipv6-enabled=0 - -# NTP Configuration -cat > /etc/ntp.conf << __NTP_CONFIG__ -restrict default kod nomodify notrap noquerynopeer -restrict 127.0.0.1 -server 0.us.pool.ntp.org -server 1.us.pool.ntp.org -server 2.us.pool.ntp.org -__NTP_CONFIG__ -/sbin/chkconfig ntpd on - -# enable High Performance -esxcli system settings advanced set --option=/Power/CpuPolicy --string-value="High Performance" - -# Disable CEIP -esxcli system settings advanced set -o /UserVars/HostClientCEIPOptIn -i 2 - -#Mount NFS -esxcli storage nfs41 add --hosts #VMW_NFS_IP# --share=/ --volume-name=nfsstore - -#Enable maintaince mode -esxcli system maintenanceMode set -e true - -#copy GRID driver -mkdir /vmfs/volumes/datastore1/vGPU_Driver -cp /vmfs/volumes/nfsstore/#VMW_GPU_VIB# /vmfs/volumes/datastore1/vGPU_Driver/ - -#install vGPU driver -esxcli software vib install -v /vmfs/volumes/datastore1/vGPU_Driver/#VMW_GPU_VIB# - -#enable vGPU mode -esxcli graphics host set --default-type=SharedPassthru - -#disable ECC -nvidia-smi -e 0 - -#Disable maintaince mode -esxcli system maintenanceMode set -e false - -#copy vmx files -mkdir /vmfs/volumes/datastore1/imported -cp /vmfs/volumes/nfsstore/*.vmx /vmfs/volumes/datastore1/imported/ - -#Create VMDK 001 -for disk in $(find /vmfs/volumes/datastore1/imported/ -type f -name \*.vmx -exec grep vmdk {} \; | awk '{print $NF}' | sed 's/\"//g') ; do - vmkfstools -c #VMW_VM_SIZE# /vmfs/volumes/datastore1/imported/${disk} -d thin -done - -#Register VMs to Host -find /vmfs/volumes/datastore1/imported/ -type f -name \*.vmx -exec vim-cmd solo/registervm {} \; - -#copy %first boot script logs to persisted datastore -cp /var/log/hostd.log "/vmfs/volumes/datastore1/firstboot-hostd.log" -cp /var/log/esxi_install.log "/vmfs/volumes/datastore1/firstboot-esxi_install.log" - -reboot - -#power on first VM -#vim-cmd vmsvc/power.on 1 - -#power on second VM -#vim-cmd vmsvc/power.on 2 diff --git a/src/containers/dgxie/mboot.efi b/src/containers/dgxie/mboot.efi deleted file mode 100644 index 52edcf5ea14980b982434d6e33d42e93774ba428..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 168632 zcmb@v3w%_?+4#Sk%|d|06ExAFpaB<66eOucvxb$d*}z%ZC~{G}wJJ(!@j}TiU=w)t$%*4I0to~E3bS|p38OK zvu;AIJX}(EO8LJ3mh~TZRrHi`W1=USiJFUh-L6a2h4bAmvn^@-B55p7 z#<#n#=y*o(C)E+0Hr=$oF%yAGN={Y(5tEX;qq9u=YFF4=XSQvPUSS%mJ*~;;MP{P( zdtGibXN1nuQ>D4>QMW6SxW;W-d)4ct!-;A6yljhDA1kX;cbh8FmqGX4uLa$1wTfb< zwMS|{E=k+_kEdzLno*EL=d$3X!Rd?)EqrAz*|n6`KrdR*Sp6Z$#Ri&*VTZqPyMj+i zOUgc+$+RYow$m9?GqK1wLyf%H?P9pwR+FQO!Gh1NgL!Fe{m^JF$d;R}Zkp$!T-0aU z(|j|`MA2UW3ZhT&Sks?ItCOntW|u4Vh9E$h*#h3AV2e|5Ef31PG1u+NBu$&9V}|-w zh1>PK3{WcNm1=-D6VrT?l?mZWA=Bzq%@ksq4P8myz+37#L5t6m%|yhvP^!$W?p61d zNLOMf%a^%o90i>!-_fb7e3Mg~HF6O;0EUwz3GWA=>t3AMrx#W9Lc>;v7^U<8vpeLQ zK|KdpBypqf8q<0&V(nB9Ey!kqu{(A*97pjqgk z`3q^JPVk{rjo<@)S#RkvRS)0fv~Z2OnPO1>Xui`l2&j>?4wwO^eUfPx=tFwA* z+is)n_xV&x)V!Gm^|k}*azNREE1!gV5~ZzGc@WWiDmhnR1g&*ZStI{AQo}=cL)J&i zC+*XteWyV|@vqBe!YQ<-AK#6(8X8KSz@KqAN$MO~}GBfMbpOT6dkRrSJ^R9XpueB}60%I`${Hdm) z*Y{^vLfB}069m*8L5qdJ@{U*;6^muZ+Gv}{U^NLzN8kjl^91jZufQR-0?>Dv@kPD@ zSFBJ9^%pvsoxW5kZ*Ihj`sPBARV)AqVXsn$G8x!qv|onI^lr$g%AB)thH1TNTGM>R zYSc)ew=X@6og&%5El{PTg3swSxD}w)y~1|-&33y$p~`LC^$?liMB@x; zd10e`4*7~1DwZ9oasg7Vu5++3T4AVkx@PG<=~K6qC{qhr@*0I?#E$sNVuK^rtLpVz zAY^%MO60pE)`p0`ecr_(YhP#?pI0f4ZgW25ZDdbA)pQKB}#_IYE zHAd#X4Gd@2hyCjpe4HK|TvlvauLiqh*h9VwK}Kzmp24)rq~Qv|!L%k61eX?@@!Jbr z(Sl&6E43AltM`CL>kBGYHO={OM#OWyuQ+Uv^Qr$l6-v_ckRa9A)*DP*ngLi<)El(k zNWZ9OORA}x`0mWuOj0uSSMrpiEZ_90CaKwBHZn;RkSLH(5o`34$2=^vX8sDDpRS<< z?i-coQYSg-KTFzh($7iSn-nE z(n1BM$Gb+7cCDI2`?mKlM?XBGKP)Uzm+&RM20ZU0M>Pn}-QHHx@{i5#GFfN8y3-5Z|MU2DcS zWE!h8rj_qA>&E(`S5VSqVL@6E*5b9g8KLL`A*_IF2&3l5m^j&d{JMI)WH=?BBsF-_KDILW%QKBFvH3(8+2 zF+A^MGz94iGk*9u<1QAQt7ZHh4201pa%RgN?na~Sah=MG7;TT}lxKp`c0Z}G^@h5E zVHMfoSpY#@^-qsbzlYq%xr0mBnL{?3@f}~7@wN3~|7PP(IEqtdr1~jkW>K$o8hOE% zuieIqB-PC!yUqAVO3*ir$CSF0Vy0U?KLY3xzcTKIQ@L7h>*qGw`jeU!zpaGi*D|H? z+lupywy#LC=j!=8N6+6G4)mVqM^e)qk~HJnfuF23{cE40u*(Rk^uV^H)oJ`>J+B}U zcE76zNW+L_373fkcMmDweM$4P5^V28kaY>gve#}gf10FfNipda-&$P3%fi z3eTx!Adop7#@zk2Ox+fFfwZ@&-}BBS2mR}ec%D=sFG^l(GGy(Dma${injW?%2FqxB zi+L2>W%*SEwq6mj zu3}58yk4-7SLCbmCjV!#{xbIC)29TrLK3l3rnNI-y|;#a-lghmHA{Ra-55?pMzf9P zN`2;hFW28=rw23ZQac^`QuDrY*bQ8zOt?y!Y<9I_t3MK8eBF#b%XZns{KVwUGpEv0 z%OUrI;hKlV4|yPUI9P0O%N<^~{wgaGs$SbMbtS{hIB$}b3>VnNJh0u1P+p+Mgaz4H zf3urmk+18D9fpP`VSS^O30N8Y^x@h(S4-g6{B_wAixabZWBr2hZ``p^OW>|QQ`by* zU)~G1P1G#p6}hB9TILj~5nI3Mb>W#1h@73~Y|EE?*`b-n$Va|aWEsM_E=Zg08ueZA9 z?ef189V|tqgz?yW>3-E)(|J}p{>{9g)_vz6vr@EJrwF@c$gUX=#9-#-#>DLdqXXiB zsnQvdjRH@ci&UCQGo zZ&7Ns^i++fs5R_bz6$egNQB0E`%ER`sO&Y-i$rw{tCwameDMRfue^h*3kJ!;SSeXn zJ6-p`5G|=+HugA%x$M6@uKJ~8i_Og1^e4RM@!sXloRjBrcZG`kxvuPJ30$mu(Z4#A zs)EAw4wY36m+bnpVD^^q2KHHM*|Upnvn0{j8%q4(E4ye6Rava%W}?-%grbR(0kU}E z1tH(uVC7~r9{7Y7!7C(816e~-s9(r|#xFv?v*mS$tP}?3X5d2-FYQKNsNN8=QknvU zI)JRHqot)KXl+pkFBU2YobQzXE9EI1T$!O1pZ`ZKMp|__RR%d#ZvS^xAg^3Q>gP_C zE!a1}YPM5Fu*nqy?zbCN*H9(o89jt9w=D9V4O%z)hDK+H*p#J%2?x}9|FeTRkaV!o z+Na8#rmuF&pHBII*)BzMqoh8%h^Cj--1`V5pBOe}F#1$tMt6y1tihMId9r5<6>ZE8K+pl3j=Xfsy)X8x7pa? zxbK`m8)`O<3$HiD8sJjr9xIwE{5etc7aCfFswb+|G2cUuTh9w@e{TH1@kZN!^Rg`D zi;gRb^{?KVdWzR^7aMK$JfXM457QTWA;yu6(b@`R_0t<5@wTK8&1|Ffpb(ZFI2YV# z>0(;Sp;*1!o>-EY@s;hp_fH;|J6YKQ8_FSVy(Nr>k&df@K?^Om8*pRQ!%Z4m>TC^d zdKzff0gKfvWc4p;YMxVLg;Qhizp2q@YE`{cqs^)D;{Q(#q1$7q;Tg7`kqFt|=Vd?= zHLp>Revb!AuEV*$Wqa1>PFioV9r@HN5r(^{RgdIVzS)U<*5|gwWG@>Qn$e-kwN?+w zal!=Qa4L44XsDfLq8@2uuZ%O>nu=6zZ}NYH)Tq*Iri1iIGk;;!! zyLdBJhkT316-6()qu;oK=*j6T$DI_NH13YrRNdwal)&C#w6>E-pP*Y=PV(EWM6_uy zoOxBX&!R`apn&cp8`zI@V%71CQ2K`Wf$}K3K-g-7h1Spkw!vwf&2rpSDyKi3^AmJS-V<19SB9M6i(J9|oMw#Hs zpMq@gnHCm?X?aJ|H1@CslwDcQ=Vx8?dbJC#%7QLXgTAlDxO%69<9cOt4}5x7o-1Jv zG?tf+mnIT5CJ(r1C}DmD^R!BQng<3eH-r-Pz3KojaGZmT5r8(eP$h|wK%Qi7=XI%Xziiu^_8q)n-i9V74*j3y9hMd{1sF1Y7u*`>kO6_I~PTG@v(h5ejAw)O&g<6SDckBb-&8cNvSP zjOi|edBz5@;)WIljr-Q6hd>X<1~V^#TE=KQ8>qMv(jHVjusHBL-KoWxhpd$KHVEl% z$ujXbYQmZwTDrXt<x9SM z9hcZOkKd0{uo3q07zzJpQJ}P=%g<<8xX%^qXS#32WP@=0k?9VM1}(Mgi%h0Ec|}$X zW)nlNsop)cdaGV`-QKSOBmW|(zy)=~F5$Jg+1i2}LMmc?mbE?w<;|`|SW@d)D~`2% zeEa~m+e5Jv8O8C8Voq27V!F%qi~-Q}cldH%pD%Co1>;%B+8*tnE{(wD3L@^^_?FsC zYjlh@w9ePpR5Kmc@YHYl&2=eIS7EtIiKzauql@7cPd zcRdeXwcFd7O+tDwMwiQvRUwLr_UPOUG`oYk;q+kBHn(G_HcfYtMZNUKcwozK02M7k z!FWTb23BwVhr4|{Je5i?%Pgv_oDExX1vTK~$p9NrK+Ru5)q z>HBrVmQrTk%jrSlziPXW&*=}$y3r@nTvUjD01$INg2rily{vU(Y^vxzOTk{8J@wKJ zNob^bGVNr*6R(kCKh3_(mX}{;U$CaM`u?1KsY*R98*8Fw)sCZWy)~7#4x=GqWby9$ zHT_7`JjwUavcUadtX(P)sZ3?W_LYeQze7DtrN-7wbRdG-KD<~W(lHnw@819*#`t)k zRz?k;@ise+n4zz+1M~Ta)4@!buu!u&Y=qu0VclZknXPVwF9Q;jSL<*3yRw_(eZUqu z2zlQX|K@n3)kc#ITkl3Zqk|e%?{z1^RA+7m>46}P%3<1tJ$f9ioz+RWK~o}DtiC|I zZ!ne@o;E2l&S!e?Ab?pCZ+&%gwNOkrvrb)z)h`_Xx;XZRSeqzNv;F-e!xWBpmN!;? zsQ%^vii~vig*5-$kX$)Pn{$xjQL6mYhZC3vu|Y;hnYpqpgILB1@9rUb z&}Fm}H6L!5rCr!#?Myw&PS>>ryd$%KT;aqiO2ipi*wcg@ zQOw#B${aH(v1l~Z;E?7>_S_?+*%`qIzt92vr%QxLzbV%6#80&_*4`5M41`@x@b1Y* z>!mD}>WwQEIcQc`r@~9V`Gl9S)xF?Yv%A&zpbQfK#O9wK8?lmXUWw;)c6k5(qivb= z=t0V<4-Uu*78Ilt=FIg%De}c^>or=RB(X+XtKO*F+dpf~^$bl9Q1|c0al}H~@vhr- z`{ejR4|b$P>GtCw9=qg)h?TFGewDUyM(iBZ5}S3R=GbjAz|)X)8BlD5STwN^B5hP+ zx=4vRF$eFbS$M)ms>I}tF|lSuwmLQQeNEOu0OVXhO2qL+V0QbkBPa;`90G}&GpU@e z&cfRXwC_nqK&n(8t&*r|CHqLVqYd0G4V>guPt@G2F*(xu5BVrF1-)ykc~rA%b;tfJ z!E{3~PrT!H|4Ami`lEfQ7N32ts%h8dv z_;r>`PHq`TlP!VcfQ3(dhmf4@-A7TOU7=;!+2o%Y5E7q1Mw7im&wA6_|0yqQ`5L+X zp9&h1uZl${nYn&-j&8S``+H}tzMe(Wo+aI0A#x1^#`T~yfsy?JXb$P;SUM%794M6@ z*yCQ6g@K>JA-yxeUu@c%^l}js5bj*I>vYH|ny9%#mtqBELnkJI$)Q7v9M)Ho#*T~``C9l~m#oa0 zZvw1TIw5<+A!e4Da;fV8*b+GBXS_tuwzkB#Z;Kzui=Hg0z3~GD(USOAM%0kkZ{i0A zN4@c{PL2&yk1l`pUZgZ~5BdOK-#C8MDlcVsfwmMc4udtK!W z*^U6NUXv>jks&kQqNTe*Q^I#^*i$_mg5p_QV3u#QebFi>G{uxF7G%Prp^2 zL;*N}1^cG}cjKpvZ_h`xFY5?S$7UFIToRWUt!xHDA7SeYqxC$})w>ZS#?$Y8EF$AB zF)twZgf)k++@pSovZA+@j@St%QgZy8{Ma^1Z=tm2Qt5dTKPE!_M`S(AR)0h^JOeQ) zeT7}0&&SXC_?H8i4`YG9xLkVHj$wO?jS*H%Y{y>pCq|(yd8^2mC-Nm{DE$bNNqVSm z-~`E*8G*K5A^TY$>GWdO+1#JXrs|!}L;jJxAeY2o6MrG!v`Qo7^fvU%i(}t z2>f8apKcOSXZQ z;Y8E`SKRPwX4XGi9#{DS((-87&D2)q>@Nk{TW?LuH``OvmNluMC9v*ZhA`%_ZY#s! zh*P^PRRBGz6M)4?byn|g+s#~moN%UH8`=NB^34&+Uoab_qKM%^tJCs6zzQ_&Vbf$G zZbs7A?ADPxn7O-T*|))gF-63TZ1B)9J|vC;tlq6I1tiLQs#xYnduywAS7)kw+3$j^ z%0~N^{6>2sDStpo!arZ?FBmNXss;Dxr7-lwwMD@n7|4Y-jpvUyX z-vaef3(V zB$#MVs)(78=)3{-t!O=MkRVhVsY5A`hV7~Oc0++Cc)JQO3%=`^gY$db~P|$k6ynF*KGXNpyj=5n4szXJ`bD;R^Vn5 z5j*f01={xvY)r&5R?T>fjNt;C{K=wk> zpse>K-li$rA zNl`UrT>NKw=a5S2o(dU_`~t0B{ashEI%Bg0WfM@F%gFv^F8gBL6S7yINcJ5%JBBs} z=vg6aWCb9X6ukl@SAjI~2$kg{TaK4cy5pAjFyFO<#_AM9LxcsG`Xv~)1Wvag+vtxG zEzTh&Js6s6d0|UnAbIGvrSh~&*8^opa!JuO(i+AsOoZj}?T4+>C&`E>J(l;6Bs6N4 z_aTzDH^s16-rwr<`#Sw|oqkKFZCzZeLIH!cdTgF+!79LayoHAFpL}ZD7=tI* zOk#?p3Nzt)R!$7)dg_n5j?s1uO{Z@+?V95#mw5sEar=&Zr?f7(N>-AJI+lw5DFgyA zts7PYrpkgO?$v}m z?3=rgA&=4iikcMZ9LB_YbT}Qg)~Y=v4jo@1brONWQs?&w5J2{lmNEL);NWp`DiFgC zS@lrH6*~?SM?aR7%V=8yzA5Y~*w8zrzFoRpgOR0*Jv#B!;0%^qbEIouCbNgXk1~ zTo;a>47iWqJi7l(9S;uO-hH$n|8hp>j_Fe`u>%*iF)Zz$c+~y_lxj<|f>&WeF(l?csYQZX?OgPw_#e?!#U;+LVfWsBbNnoxmyuO77r z`>$l(3PNPPNLEHpV%ULcz*07?rX;iwtN70oGcsoT2l>%+B_ihgNAS54#AC+;qfR=d zS5?dTRJ(y{nCYD8WwTDGBrslvu~)AGRYO)w;4JWj62?kl_LWw_y5>}YBs;EY&xVR; z5Imtq0TT!>qgVlwpR;in)`5hIhqjG}FiBv>`_3BV&)P|1#UEgaEeb3uh-}@@3 z?-d-;_n3o8vfX9s5um+QlJuZ(;rArj!2%sES!b3W&jU*Jl$Of7|IPVgxH=02r#KXF z`#6{D@@Fvex=v`yXYpF^^fF8?C>Tbo_7wMxR8@a}cJkpT0vvAq&*@ z*M)BrJ)!lu-J7!`&!`Qff%GuvkkZC%EwcNOP5fdi1lWjuJszFP4{%%#LoY?hyfCq7 zq{~DN-GpwGRYOm9)X)O0hW=h!a6~<|s*V=EpVLMk;){0p{D`E=v~fg?)+l$t51PNR&mVro9+`22T1TkW|5Qz)QyPbWg2#r4%d7iWHS zUR3na?Mqk|_;q;5Kax-6l1KE>=g9dt`sja1b*uDF9{)lgog-fzeRQENSadCqtUfA^ zW~-;KN;(AGkEo7D1v*pL2Oi@+UyKMx&h@hG~yXqB7{TT@Og(e@Ca zUpgKi>pv-)b|Y%rc7$0JLoMW^`>8w7(nN^54mjb=%RwBfOtLGCJeV)=!bGAJHX-ui zxaurxL}MDNMpK8zV|O!%%`}AWE5S1#B8`h2$-W8pi5bO^vRyib0+G5O7Z~vtvL&qh zmWVy4Fk)X>LM+F{iD|4dBX()*#>T$NJFKT7DIt6a}Ini@O|M}gf zqbq3>1*9F(f1Z^5EZwdmU+X{5lbIE=#4&413_v}DsD4ce+xxfMWuAvU$|IDRooV#% zGUCE_*B}fE0iQSro=4iAfDA3&~doX0)s3%rS zuBHVg*zsRUzSYPSuacxIdVy)L65&~#!3&(-xT(8ws8EK~tH&IG_S{&Ws~d03FE+he ztl*2)SdlIkN~oti?GBZP+K&oErH>Lluf47Y(xviD!zyJR9@~YA&@pn6DAJ{fKq+Xbn)~>tMB3Xf{^*qAhfP> zh6aMJG63XG0FCrwguBPvl3LM-;o%SXe~et;<<}f;H0l2-*MEc@jBs*k>c;>Vb>tC> zHLv^E>d4OoL{1&~o32UoqK213Rwh8=tEXh_^k$4dXGHd zME@t6!V5#30dzEl>olO6sXVl%a4Cttnu6%g4EMJ*g~Nr;wD5`0FHmzt`_0GDK1Hl)RT2ASpgaC+lM1c zi=FjBNIT1GR|!W@>1t5=nb2!8HUdm1(WZ`cRDZ(YE$g2Q^=s0p8#Ps@ro+Bk0{4KB zEB0NM*@gvzT1^{G{^~|&1W~Gkc^5JVF^VmDH9fU>t_|1pGkdT_vbQRqQn@jqe(FwJ^1p~kL#UfG(_il9~C%>njV0NGLz~R zQCy@<_-Y48;dmmrjGbXZdG*HHiRI5wEn_^e)@WLbPb8H1F3zV{gO@fW^0x-%tY17? z7}VBAw2X+==>(J2V)TROuNSGbz}(+sihv~+Ncg-gj<(&_7D9U7h6#3u-lbKn@V)EY zM(h2gma&b5t+&FP-j8@9$Ypg!Dn|U05LFAS(P}4%f{^N*p78TA$Vm&R-XbS)aEtT@ zeC(3+G%-1rMTUPODg=V*&T#vV!mxX%1nO-JS?_7N@nFPyAtce{`k0eCiN1#2ozw^+ zFZ!bRv8VYenudSfG~o4!b^W_2W*z2|uY~a!ZPhZM@pZ((^*yv1oF198r$UZDelgvg zv$G=XM)R^ZU1GYwX<1zEijIo-KY+8~pApl2scAQH>@~*!i(TrJGSNF?$4D#j&Wbet zn(b*1;Y~TG6fQxmGUsaXNjQzR%xEsW5|mJ_x}Ig}K*qvSvgVoT7+hq6ox38wl#@1a zsdCjHX$}9Mdw~d^hlM6tXR?+rILi?{8#9EMRmhpO_(D!zpD0mgF(ki*#V+jF@r7l? z7`C9pYsvMkJ^*<_H*kja}4)1?5aW-adwKDMZ7Huzy$Vk$R-&nIM0ApSIhu zql?rWxE}&f)3rcc5`|Qbt>*1x-f&~NGdL;SxNEetoYBg5-n7PQCJjV&9-!)%yVLhD zdgUT?bl?c&AU5kJ_h_6pI`PVfW7i`CQvwt5h zU!se)WK5KdczXpdKYU)w^!z}jUlI>LW*z(#GMQ!J{a>uq^GFqELZzP7MTIZK6$R)$ zYOxj=^$OB~(o7~*U9oZvB4iPawL{8H|L1b34rb=%CiH~Xdxcoof68QdhTuqLed6G; z2v^vqhB_5bP2a@ym4O;j47olSj}filLCtYibxYu*c~DQ3Bb^Pb83Gh)#%TsKsX(UR z;nqTwVHtIXz6cu4_d#r%&lL4Y&uQuhEGWIH>d}@%&=T^>Dw!3rOG88yHzh923okjC zVQjQK@1>1P^vtOQa2!KSy0&S~+ZDvJjF1}cx0>Xr|0TtG9K+U+3b>1)C!ARuwg!-0 z7F>F3dBiIH7<*m^{EaiA;J$vnY-*0%m(kK;*vQ@RGKW2v(-?xpKd1yrJFZ!grgK)y2W(x3-&$4%#JSV;~zlsfw z_`i&ngy-~DJX^sm;Zir<@2Pd{Rd|QcSqTfQNVjwTGR6tHEqE=lcBW5ha(|J&6T5S9 zSN(A8LVH-;_<1ro%~{FTm$F=A+tXg$cj;8vdfA)J}85`w5L&FNh1Q-2`f=;D@ANCT?co$?e@oScmqIt2|VdR1xH0&grRWS#;cR z_Z>9Yp8lCIdWWwKmqkCMn>*8B+IJOUgcksKRZRf(j-W?^kkC1GwYfjj34iTT2F zI#J}czqRH!)G@7;LtpkZk3?z3sn;}TxAw8p=baHZW}@Z-b@^#hwxJ;PJ=yUR0T0$n z=dK9$72sr5Y}G)0b57|dqs^b}Y3iW|8&jF0m$>%`{{&-uE7 z;8!}~ogig#+PXW`;*oBbjvSRWT7lwOy`g&JYR&%1xLhmxxIr1AV(mfi>^d?(>idvV z3jCms((w@1Tjr+i;Q{zIYigx~NbItA-35H3e5<+S1943> zRwpGM<^9!KC8eD@f{27YGNfIa5b<6YJ}dqkJjd$HA|Xa1C-u1XJ34M5>q|8X$m#QB z9E2){@(~Yrj%Ex#YMP`mCL`C}OWL0UsTw2LMAy}RTTr-6f`l`{OC{C-8!Z9@fZs*L z7Hh5U(9}!R)2A^~cFhEjW;GYXsq z>L-wl-nWt!>Hjik_R4RhW7@HO(iLok zi;2j{L_bthauD7E!XJQ;!~8=ZbcEF`&@l&XhO%Srnsp?rw{ik-4T<)T3si|yyHngM zHSwdht+aYr>gQE&z2aMj5Jr0{NL6oasf9&qZtq{1jFqBO%BFJy-%K0Jo7GPO%QzGA zraqI=f!r<5fiG7hL=)S@g&&J6Wx#Q^wk201$8k!%@onKWH%LCLx_{-C$jt4~3V9>Q zOFMGLHg!8Md?(ETGat_xbmjc~<2wwnqPm$>7H@~r6v&*Bz!84YW3 zpU4__#H*thcP-iVjI?cgH~)Z9!UM&9A*KgsR3i4Z>3^?zQ1E%FmWs2~WI@l+k!mVd z?shHVZTH~SC&u0}IMtABQd^zMG?AhK!vo%U?P{InE#Y%Rt_7P0W^RN_GiW8yW=DnNBx}t~}Gq9Ix zLn%)b1_}VOUcS%+y)TA{l_sSnyE^YO+A-pXD2PtO#S$Eh@eErmjmK}R_bgzBq9|*2 zo%)m=ic>sBYbRyn2TG&Ena{d3xHeabAiXvl5U5T&MNf5kdYx&DIs0b5@fAjeX!e2lqipN_Zb`7lJG; zdDZPAd?;djwD3V}9)%&!p8A$=+(VEX~UG#=`DMurrV zze+_QfpC0JK^Tl~5sYva^g2j=2c%@~!#KDZ!zzl%Jx|5+CQ26~);_aR(GmGpJMV0% zYdRGV4x=Ksu7&(N#NJTzckt%KT5eo+l{1}ZiprR%naiitaik**J%S40pA~T-!WqdY z00YCH{*i1j`@2HE8(nJPX?nbg7^`{uW1X+|kJMSGi9RhG(lLF8ZizINM9SenVG(fB z{YfmMVzA|QpDT82*cNwjElA|{uZyV>PTXE0TZek%ddSb7P!P5+MHsogcD10^t~_CD zLX{dkN%qLt;IKWvjE|Gm8RNACcM3rFZvpIQN$p!>UCMsk%eI}W0hM6a1Oys;0))g) zcHaC?t5x zIO4z==L@c+CroV@Z&B_KgEjZ9bI)V{vQB;9IwDRn3YE>$ullDkN-SVq=)TR&A!@Cq zZzl?>m-7umU$Rd4Rp?tAJbIxin$%`4yFrh!4#8hPr@PM}J2(<6D!h6fNykh)b*9$o zImiTuDfk}|7Y;2RQR(j#u7^sG#fBzc2JWWnjls!?J0_<92ZZwS3GQ8+{;aW;W`P+*Fa?(BmJV+wQ`Ty^s(n)klhLa>a=(jXs}T>mwTYhBwgiSeh{{+4zC7 z`JYxNTLK>W67#B=lABnl2%=}(kb`NXvLi*yogihjU4hVah`0B6%Pq8zVcZA_EbWig zWZd)NIr*?;wn)nLP|g{hL?}<$W-caOLRO*NU-$(5ok=vl%cRK_<{ z`J^3KPUWEWmjA28GlU`%rB9Oq^-DhUmfvLO7a#)<3?~YUF-Vw+9}tJp0-B-3RCmyS z$O-)Qn*P`3&*vtt1>VNC4R^#$f9JfpVhprY`nOoSxGF??mKb&!V5@i6jULZ)@#$>B z5#n>f@i7q<`%|VmDFN0U#a!={mr-+1qz|w0{h_0xq`Fcv53}e-l9?_hktYfEilVm> zNhp2A3SkbZoaA~26|je;=D;Opz$KjP-VSHuOE4^oMN-aXJ~%(@Y&wrDqv;O*GBB|)m#0;=8}DTj~wSrlrEBz*e=?)=QIB@ z2zH?y1iwo1?UFq88Le=&)rJ&uCu`v3^vw}_;(;csJzV)|*q>@XK4Nuc+D*^7;`#{r zUt6?2m64Ibnz0W9+YwWi)qz><9n=50apyJ}T@D^q?h~sg_p9CcGC7TylR4K?a%RN; zwsH3=aw6P+_y;7wP~GU~8!_ityB41-k>`oh)w~MIf}!c(G`}3z&=ZnJPq;}<;$#cm zicmE{J=th|1kdKYS0d0Rg?@FU!|2!U1JuyrV|KCnm_9~W=iy9}ipAWZviMZ zi`H^C2;^eG}$3;IMU5 zK}v?FY34e>J;@i2_{nI5?T8m^)QhGu;YA`p-0#jC(U6#SI!+RR3(%dWoo9M3D3~`8 zk~fWpPV{rL@|9pbg9QC$^isQYI?x&tV^8c z)?T5{_7(#qlMGcJAP$#g7}T2Fn7EoW?xIzNt&{cUz}9-?8b(I-vV402x2N^DkI@=p z`}XpPT;=%+x}rIt%k=M`*WdWhBzKK;#MjGmJ;09se`ckBOzekj=^dv3P%{w|Og=WZ zH|2s7_x{J-^ZJo*{1V;mfBF2gqz?8H&%@LyrhN`^UEEfB4zmjtN-V&$5CF!<-J<8c zl-)VZ#9VBYUmEdaq?}k1Bb9qh|Ch~$wprmf;S1B<(Ls1bjMnYcPG4s)9OYkF65DN0 zCL*uC+&0VXg@nPFLLcSf0X_EQp-@YCSavI|PQPYPE-(ii;=+IdubTdYVRq}Vb%5&` zxvcSmS0nDV^Mvm$xF4;xEMjfOemUnGcB+y(`@(!X&z?sF5}*dGw=ZN1{J5OM2*obL6%);z?lTubCd z9ZlRO*X)&<@ttMr%fV1|C5IB@9irx^-bVz5$q5n2Q6THHzQ!7{{jn+pQHoFn>xq(%7g&Qqc3=i0@utLZ#%nJu7IGxkvPe-`x zRQ7^iIRwISb?+z|kwYLgVG`BJu>WfYt*gnuws`r!rcXsJ=PIHmDRu+JOavkA z-FiQ}@+BOKp~VNMZonHPu~6L4%4={1E{RS)ap50PRG*|KK_wi8j}4HkwOW^fIf^$X zaJ&W4M#L2Qq5Epbm~tAVvK$4OzEAh!+ebkTuV(Q%M?rSTDfgW?oYKoB)Jdc>IcA@e z&U*MFXZnR<73+R##MLw8YaW#nSq8%|a~ASAqLEylrRhSXBPYcPCl<_MbN_qlR+f{5 zYW$XZa-t{VhkoyoiQuBpsd@r9su7LZXWI>^rGU1nuR z3^QNd&fM#WHeRYZ7&EEZdLd{$w%PO?S9e=sb96#tRR8K;eBA{h|0~T`1~VIim7j*J z4Z-;P{o7L>jt37684qn}4utIK1pTBYB!-@5Jk})l1_j%9dK!(#-VVE8jCihgHL?s( zxFAs|yDMT-S8&swiA%>03Ue3nhu9(SwD~ z8uqm9mhAP_yDvzvJy~lxB~uX{UZy839eP~)gG_hO{aSjn zoFW=(x?fB`f0WS#@orJ4(dyQ|MsYN9`J`}kYxC47!3N%6T}E)0br1(QK7~&}&h{=L zE9mbw?)*Dd^>M?8NuZNpQ(Qf;bwjiwey}`xBa7>Mv)~RhH}vKob586k>n-E4^&$Vk z`T3a_aS5R?zJ{5k-W?q*kGxn7M-M{oZz?;}XOaq5b_ZYE%X@klSL>Dd;)lv*y%rT} zkZXJM-RH!LSmte8@7R_;osgynL-enfM&G{F!&Nw3=!Xw0^P-xE?C%-BeK<<5cHm2>x6eJa(vA`L>#lkix4I&|bOC^eirrg_F94d>N`Yg3}wV zPKUgwMywa%#IQr07AC4$ago-EI%C0*wY`0N0keo<53wM6j6~(sKkZS&H^7`m=nYaL z=C`Ti0c;|1g{%KC0y-lwO8xF}}Jt_3b@7n%G+w($cr)pGV~n;YtURVxx=@TTlr-;8;yf^SbWI(Ve z2yun!BfJ`q75$cnIvLh^Me0#n%I>H#f1O1L0m5v7%QPLH=R%@=?R1YF9XijOl z4d9769;1~QBz-%VTzz-y9mQ?Qg_Ygju~G?>WO`KmK>t`-xA#=Z;KIH1N!{KP^v9M= z+UWKUlJw(`KP~|m@3sU!_w%~!YFD&SodH^1-W`(B^3}jtl^sZv<{TA&yS*PsKK~Mz zW&Az!26TB>lHpEYl}TFOZjx!P@m|5x?|q)X^G4Z$zw;z+m%mP({%So#o<25{?DqbG zT={2tZBoYSO|s7L0u()boh)5vo#BOGeCi7s7Z_sZO3hj&Z7*7x& zj}bo$7I* z4FH;Ma^JYKI6|1|B%7?Wq~4D~18IJPp)Yjy6DO8I?qi2D;=~V5YqHi& zmwi9*<8wXw82b!W&qmp%0~FJY9_s6OK2c%@4|bOr3cvGNGqOaWn0&(Pga2PN2K!zU&vGzV$pnAWCQIC%wILz<3@bwdUtwLKQy#a?H^OYfj z!P>iO5Ll}9@_BIBI!k^Bfb&{7MYQw!p96`R{(VpmT&Jpy>pk#aQ`;3iN8=%AmLbXd zD%2zN3#Nlh1mmdYjFD z^{CUh@tDL_xsj)2A8hjozKlidtt0AFNXBS&(T|9KUziINq9eth zFe7Z=f%K3P6J%8rkypg%1T!au{QIIssdvB~VJB?8lK%UJiRefTY(W+nv~9FvOT+(6 z)rk9gUq$MBf(;!nqvM}y#vSsNWqRO2t-{9%cbCqqvO`Sf*62XFLF3*ev`fIUGd&UR zTb?alm@7V2G1js~<_c+&yI1{_UFp4;mk%Xgv|iDL2k&O~!;>WQKI$jJi}lRql#y7xZeOBUN}jtDg1&yNJ8f zzCsjfr4tVT@o07yxBE0TC~h#}8>OudwQ`Mg(Ar|OiLh$MpL2e<<;bKK%@C(a zm4xUOs1F`-)}^#q5FLaehhwZyU}2puo(+?@um$;87A5rQhh$eL<-&#?TD#Uq44)C# zMd~nr{)q`OW2a=#n`p1hCDgXoX3f5;m1?OjY8fkVO{(!2y&ZNN6Q|^4-ICMD6_k`| zDN}!ha2>wZdjzaG{-`^%oy+!XsPyX`(AT&3?p3>8 zGThpQ5ZGFE6o`uF>WP`X=v#=tn5B|3b)2jtdYM30#uf~(cE-FzgcB_rV8Dtt17VDX zJL}8=VrMWt;uoqg2(fX zwieRlBBWBh@bQuJ=b@$#3h`%P`}+mKL}Wl=P=5aBm3fz^y(ax&(pYAnBDd-D3Wk&NWDYnY5Cwykq=(!qh!if z^k>==wo;)6!yjb7AoO--4?F#jh>B&%QVv@0U{~$ojB!#GF$k%*g!QmbR!xE> zey!c&*`GWrP;W!YEI0k>fGiWA{?1JHQTC#h7c0ELz+ zAPK!esyP^$HBG}b*)aKaGcIk*Aw}G#{^=%l8}(8X`4g^uUC6v3?A|P4bqA4iiYUEO zHq+Ig(nn#fEZVEMp8=;nNR}MgYbCKz(2y&?S|q_g*+-NY8#=^ud_36?H}!%rP~YY9 zvyidcmzQqRx0&iVFyZ8ShO}fwm`~`tvVBc455TPR{{#7P1a%#$KrNhQnZN<2wV(K(==aPva1I;tx1_Z#u=eT#2aLs2=}6%Yg96vMrFiz zjZsPXgi(3FuZ7}9rNi@$%13<<>i4I7kL%}iz9i51HwDiKsUUpU{{wex1MZhrAAE-q zD*fw!yIqNp)lPj8AjSW9vpI81 z@c9f}iC-JI3or!!%bRdxT!}DGZgF`Rq2R*AxKqu__tbSrf>e%nn7;~FzN>GjnfFJ}ObWXn9C)bg_|-yVDE~HVr67p9?CzQwoXi&fQ~Lwbooh zv2eEp=8zpFWF;_)$-pt_6%83>^x3@p0na$XR2qp2pICCiDKqT zT!Mry3WomO8jLU51t@M=Q)sqIG#ErU*_GuoZ2nbmRC7PaHWdgss*R zcyF}c#ptY-9Z1;N#iC!ev5axgv7DmbNabz}ht#0(ZaEr))Q>~qUF2@^!tUMar%mpF zSI<73t=K^!SaFqlf*eQUsFoEg;RrZLyVX-jSY7#NlCw=sVQ`jt#+@1+jy^n0MA`E6 z4V=x5UIop!-3d}b)~_sMI+AMV(Ijl1s&W{))D0~PvS<;}Gep*pW`z>sA;#gHeS1NL zmVU4<3ZXtodFcLR?k_5rh>C38ve;kKe`qt`{d#$ ziGAsZv*5&Zoix)e4dd?)%NPyW7~e54T_O4WW-mNY6GQX_$@niUy>dHJaVh|~XX&kU zoB1hIKgT#ITboRQsGQh0opJjyAGptoX*P3@gH39BuVRZ90oimm}vx z_2_O>zjGWEMUKg=!6Ro*I@n@b>ITXUB7q0%Jg(JZNXc{pPOU^kGghC?hr9WJ?>f_= z{zP8-5!x@ySwTCdCnn0hi_&c1FnwAutF0HO?KRZ9n0hP#VX>;Sm;OPAg)E%$(2t|m$TF-QfY#2=C482f z`zFY5%4z8)dt670>HHvF_O5e_39cLJYaV-HVpJtRVko;3l(Q$8d4p3%1S#^Wf4X2# z5Py_R%&W?jCx-1JyzSAVi1ktRRvJM=Og|;#(PuuUN~|=|F>9Zo2GlVSc^upEk;_7^ zlyfcqp6GGQ3j2qa<@XCM&F{aoupd9;( zB$_U6L|`2@W4B8WpOnZy7sO0|M|7WFN<{W-Q?DXYF>;&%${iomc6MCL)fOkuu~Cd5uw;aW^S%^*H`yH>2u)f&@H)0d|uNwK#jyXrHp^k)EirkweU&1lc#9zd@{5rtah1oNeqZjr01)?HXz}#uoMr5iiT{h7L9JIe7y%>Y@Zza%HUF>IZ$@zbm&?wIZMu_`T^iOEwI zt(bX(aC_o)h}04jVbQq}R{6R75cMQ}h2M|^_qW%!T;&u$5dqf*xLTjF&5f+9-)a_JB zlt4}j>)#sId#di6-ZM}Gqg84zqPu7(IzKuZu7a4qYEt8X0f!Wcd{@1 zs&2=!X~SGgx#%Rlhu>+?d~K?+0v;n1Rb?!na5`GzMMH>tX?3T>hg=avg|HiOCXdHEzlDsheTJAm?; zcMt6}49pcelR|dG`SL+xLH9d_a5F8Rbb>BpET4M3Mr3)zFlpCV-Y^tCOY{2>GI$#+ zbx)qIs)@MJOySQPl4&@DKO9FDKG;LOcR*U2`1W>K*v~C&D2%Ia=2nQ>pei7 zb)RF=;V?SCL&5jlQS+U9nNE&*XcKtgH+)g^Ud@*M4uk0ZHCc9P8-FR*3Z~gXHI~0% zEPq4)BFRhKkJaUEqKI2A4{ixuK+^S03GHKJhzA+~?$*};cU-F8V=IgwC?O7YDk->p z$>jhZuC48X5tK}geGNxSb}!K`wI#b|prXqm#&XP4g@_r&?0KU?j6hY0)L3OXKts!( z^ZkmV0GG4jvdBkDO(IFB=nECtITDZhE@4{?#`&4J_$$!oHudoyM_5H96P)onF~p)& ziEje_hfd+6j@@vpGHvioQyzn0NTYjZW?=8H0fMw`Uw zIgl?=Rv$ z$B4$MF%Ae#yL=F6KSKN5cqr|CfN{G4+rpr^ThsV2F^5&-mNTaH}zq zipq`>g|hE#x|RO+gJIbCxZ@rK_C?Q?+_Q`mGp|IX4_$rQUn;du!FlIi8T{T4gjVNp z@Tt4Xkd550{I@=r`|H7`#zr0LJE^cjF^sM(H*?_`+BI3qlgV~0DH4ME*T8BJZ8LW5=A50x!;q~1}F zoQ-L5;aSzkCPvv#4GU=SNp9d-14v(}a|^d|9gVlj?&9{rD30OgatImOY@2uY%w!#0 z;|&|9U2FC%oK&;6u~cP>mMpe%8VfnUOa8a1cZ89W)tJ>;Cp^#!K$6@4j8L1HX$T9l z`o>SQ7nSbpGbtR8htD^#1sDxJIlG{a^)br1MVt~5@zm@|D9)%8L5<9iApZ|re8+3i zY=&vY%vX!(cRB7DuSMfaf2c()Xrv&J#`p)mbaJ&os4@41$xgTr@btGab5ZB z=oNG7Y6=sl)-~N#W+hIFCT}i@Ca)?+SL&Em0{jg65%(PIG@O2JL<1V^{Wg(ovCRQd zcj5#97Oq23vt|=0O2w)N_wQ%7kdI>bm7g2nP)jJsrAk!2y5vK%}oNqp?L<6^nW+NT^ReCst%6FfMmWp3Jc#`H2WW5+~HR8KD*|;$*uc)nzkNX&S^a)_kqSL&egZ!5~i-E zGa9_Ft*)|D_sjBmyrP(Mww=NI(hck)E5b6|>jgM+n_RkbM6Co_sHUqncpqm(yEwm$ zwl;f!OJA8S%df0+4%ESA^_ILLOe8Lni^JMdCMJEXozbm*JtDcS*tx z0GAwP?|+Nu>p8aq0BA5T<@~_53hnW0;iY8#zsdjqLJLwj%t}-sf~jljE9u>c%+$a! z+@@_o$fYp&ol}`g|Cd~jng<_3TTyQhTtml_$GxBJmKrW2RZQxyv}0hh|DngVB06 z!^;=lF8;O%Vmh=iQJ

4kJrO_!hMo2v;`Flg|-03On4>-Xs#=KQab=f>o1Z{Q$92 zjkqV8#4G8uZkfUTYTAH|3rn0C_8_~KI_sV5DRrYvk_F4qb##@~yFY;xZwxBY*`{wb zdlwaR-BfA4aq`)zi}=5E9j+hbc1@4beTue<$8$3ST&~iQyiQ)Rl%-`4w}bZ7r^2Q; z+{4g<^IH}izeAU)L&)kK?x(j7cjycg*YC&uqkjC5s_LB^|6V^3cIk>av*jGRyzv%0 zd9lVUQRwQ9bLNXS$1d&EH|PZyIP9hwJi`g>Ycl;woQtNu&W;#+6T6)}_foP;KUu*$ z1wtyLE4A9k_rGru>|OYMSC=1}c9u)Y z)A+eJs2*oCI5OGXPn&NxTDExK7wfwz>)Z-W>lLqWN#3L2iqL<1M#sC@LNLNba(Zp> zhwZgkEGpLqAD@f#b)f$Gv~`e}BhEyWlIe4c7IGtfAugcsYR2N1nzo*kQ>AOUl?ScV z2Ol5TV6Clp#%0*Np0-Zi;xxKF{mP<+$6?^M^KPe9Cpop~k-M(o)8%TlL9W)yo!R7~4BmO-kQ)K+OrNq$#JW?d52Y;A0(sJ-gW3g_FIz33Q z97b-BSsl)u%JF!6@bQDl3)X-P_V)C6(Zc%x)=Y$Y=(5h~;>uBdUc5v-lYUZ{tY)nT zTuhsfAE1K`4o77clCy`%QMm1~oeMI562bdE*d#7_-Zm~aK#C5kK7tPJV`Ie31?Fh_ z;DvT%o$XZGmE)VbEN7xm1=v5Y!o9h! zykD2zHEPL0%ctSX7k!4@8tQ1vE9`ty!JbML(0;ki z4waxlt*FoAD7?JyaHO&|b6li#=}UO3;+&nk$ao)Wd=>^k$A%a{ee$v*PUmo>MQ+^0 z=Tf)SB6f9RwU0V&(aM(^f{*mvGQpeSiJdljpVOA<(FUvlOwEQrRImoN_++f zv|JR+^Y9{`sk4~t*dMwaMY^$wP5TqDym9h60Tu3Gf8S2;&U1fc4qhc@sh#DSb>cQV z+|$ftu5;YPSs<>yGF?}*@m3C#bkz-ZvIy~T9*-8;P1^?QI6$R4-Ff!Q+wH*FnYAN1 zq-QuVm(VYF=Ly6b9E7dBWtO?~pmv%z!JMqOxLMYKb_OEn;^hG4DlO1x$lSK1^8Fjx z0kM;%d+nF|TIS>p3XzP6&^O#5)L>SMXwzRW)U3B54H>;JcOANeD>%-}tQ1vohwT>Q zQt@M%$qe#^%q`j*>XZq>S~f!9ANa1pJ=5~~RI%Yw(iON}TvS)tqkW~xgy~Z2WvSAl25xmXIHjCwO&4W0 zWm&jAeOA%JKZ{aD(?fk~D`FXwTD44_x>CFJ+%+D7hO_IOm+SLBUsBuh4k&BjVmF79 zU&bPe$lq5~=Nxb6T~<;b2sAirxdKLiGeCxbVMK;}e+7L>D<~~<)!3UMW(Dn1(^{VL zS9=p_RpX?3)-SH6gKbwYN@=lLm3h0)q0+j*+D04Y=J(d3sIF($P;jXJlT|+x7@yCV zns!j07BQ-B5rJFAX%21#eMYY}S7M`AHRwu{*%>p!0H(1z`81HYTbEpnWn@~I6%eTl zqmI7}{@X_AJy{%XNZtUNKVQ;N^MxWjV%7yOj`I74+}UDdz9g0Zwqzmi7zDd`TuLm$ z$)=sWt)%gr^~usLb;)lq3?PY1peR|ig`G`zBC8slb@dJl5>{AP^_jXNSze#_H7=(< zeiPg9)!5{5bV3jHp4JWwf$k|p88`A0{i=1768%xdow);Ob_D=}MPmU|tuED=hc4LW zZrvc@Sdy{6X;r?=unRjGFIgNgwZrNPv6KK;JFBtH1RFM>_ReLS>;LQF?b7h#R*~Vo zH7{An&hgq?^OK`8CnpQp{Vr&_AZKxmKX-9xso0Ep<_wIby^Hfky~sS^onxrB-d>4HjpKn9KI6DuG!yM>VCsG$iigaSk(JE)xkVF;yV1r zJ}AZ&XjRV0n9Q5>kd?TuF65i#;w4e}3z@(Amf{s`Ye^ErQ1Wkri(z3a6*CPtc>l^N z5qx<%Z`ygqJ-S2+dnxWg-RGHr;`K`B)1F?u9>c*3#FjeiZc6icp_-Iw5-gWae$l2j8rSTRc6U+p#?Jh94p zmm^MMOM!W!t78n9x3^}*AT$EDo3-wkH7AxlGnuPfd63yB%x#cX>= zA8rOI#+Z3Ng1#{a!gM%IdlY>irbcOo%#8n+C3O|el=Pf$tT$4f5$6yb;Hk`xp&2G3 z(2x%qoJ0dzs;J7QR4b4$2ZmrkN2DN@GHSBbttJI8q)L>VD!IAY@9u{ID)ILuHYC4X zB=##>5v@~f4>?f|!a?tN~d9x+P zp{=1QJ=M6Tg6trF7zT9s!X?S#Pij*mZ!U(D{v!9zoj&DtN0XDFthqUu-nCs&p#tuA z&`Br483%yv1*ExVmRb|9`Oi@0I#88~ z;q=g>tp_C&1^A%I+^l0GK@XxL52DPKx>`KdEYfU{xupgcx|d+5h2wBWmmPGk1MdbE ziPL$9g&hmwi1c#w@CAr)nF99_U$0t;EXPrr7V0pA@222TFjHSa`M{vd|1W3a#Wti* zzrbmD*PsD@CYv1MO1CGZ*zRdY{0%NGAEhxu#~x&S=Vz&8fx!_mixu$Cj(dqq&7~MG zoy}$i_(zrp_{nB@SaJLt`O-95H0iQgmYED*mR3JZmd+7bdPZc~=!e}N!^>~G#Ji2e zEjmtZ;@5q_JXVg3ICGrti8bYOvPz!S1lXeZY`!%&|IK%J# z838c!h_qAB8vGbnxb80IzYpnjKT()5j&t3f;lvY36kPYy=ccU}a&Ys)##$q#n8{ZO z?HcMrB5*ql>odLI!PyLBS5(=@hvU16&zLhyCA3DCo5Bn1u(ubXYKq{NB>E8Tm$qhcVkia56bK8y^4l|o5&Fy8hlCx7at zsV|+uaBj__fS!-|GOWrTEOKu(L>O$=-V1gCCO{Pkxj)s4hX2pW~2M?D6r zQ+=pTF;MCCAr_H6{UVKEvsN~>T8_cNP-2Jsj1jYg&9b(t7v=QgOtT>0c4@X^_ica0 z5VujJsq{g=QZ~xHlBUulZmwVBBgEI|c>`WI$gbF+?rk>hLXW;FTwB{~?uyeQrxORc zCzX1_ut)bc(^g`>;IuU+Wl6U1A1`#!bj8LRFE`9cU})aDC5A^Ns-ww5!4Cu93|Y>1 zBVIk_CX!Bl#$ZnV=2(Dn$lPJrY$lJ%u61L+JJ*%K@N9SqaJ#nu@qj+2`yap8$C>_z zc#Rw4Lx)HTmVq)|A9RTwDPv&kl$h}mawGOHaH5+gBlV(d7(P$>lNu1&m(>E0FWeuS zdAyh?T_dEQkwJH=8Sca--Wr`SId1zH0yN3OP6D_xgtFf+da@R_SlNi(X0s05wuJ*A zlHNs_f}Sl4c2m~|mp)8)f-U+hXKJVe-rcReA#Cj?{gPYTTN|MI$e@E z3FMk1M)*he>MP6ico{H+HRQB-yCpJ~GO+#`a@1Y{YNQbFi0auXM5QkO*22lLtTY+&0y^$i-( zY)+l~=r^sLi|&~*>7LR_wJkL!qn)~qWaRjk@#c)CNUEToTiN$uKC44d?VSy3B@!dR z8U5wF?Pgd{gG2vACcmf3&fz&xIw)1vAFZnFRQ|wS z%J=bWDESc|*2SVN!*;k~z!gT_dnwLaNZG&!?pW}21 zm#NtC`NorF(^iE08D-{TusOiB2HJnwu3V3~>d0Vc0`Fmun#GyZF*!6%-)1xLMgg41 zFKgiri8j|M%izo3|WGie0fV}_!? zPc*neyht^k0h-xmfw0~8wq?tGoeUh05#4kDjl^k*({*xn2~d~|G5zkx8C0*4D49nH zpmokJk#o6>C9}0BS?IlcbFJCBFMe*DuFx(|a9uhN|eRN0q3xMuHP z+733qVgxPEJ3`|WvlmC-GVE!fy7`~|?yr`sG{+D3xrZ{U(jZ~+<V#Wa1JLq#410{cYqZx8JjIQc3%BslOUY?NIQ$ePGdLUA?Uoal`Ms&dpIByrH3QUMaT8{P15?Z^|CDUw+%Z5wH8!3Q>mR-bCd zi}P`Q(?gWfd$JhtP!sI0se~SFI^sS??uO>Wy4-21!#+iNwpuquGVVF#?4vC4>~NE! z2s~tcpXGQF)ulPCJ=E(emWCWS5ZpCX3wJ@HY0hEvyiDvM8_>_ z>u-F!OvrQE8?2pJ)fSEyt>_(}6=m-JHyHE2y40Oz?oNW9$hpmge35gVK4jxS_~Ol{ zrf-&T*#&tOzBt=mMj}^V{FMn`eBN;6V6(Q}{POjFR2xw==rJH4M$Z!PAVb4!TlyVxsD*n?3;ifD zZqP4mWKw|nhS#`~-ZN}4@pU5%_q#tf41F>7WkfyX-+uS|hLI;4vOGV7V4M||7kX(C zKW&Tq3sYNg=~tnM%yB;GKQxRgM8hxskz}yWkd6b1UiNFfpiz5!*jt-BPWh z00}7N60s^!BT$7dHFyYqPhy7VOSJ+1poZE|1pzbE5~|=W9U_A3K>RTW-)NIVQ zyG(KXQzfk?WU`2DX0!D8(aN%MILp=%S=

_6%ncB-vU_fN#}Oq)3Ie9=ZR545?*h zwoZyk%S+wsGYqjaJ_L;I6o-S&8_c4H_+v=TeH?;z+Locq4RSlUS%RS!kP4qC9dx%E ztt8lNL^4ufRQug1w59dv9=|ep{tadCo8cXK znC8cP(!^+%f6R}m$QBf7kM$#YBiXEEi}&SN!RF2M&98gEn_VgQfFC>Q5dgV#F3J-o z^bsBc^J&AXgt;K#R5Vjr;i%bp8EQa5pk8flKXdPufl)L-Z>!{Iw zcenZNcek709qwj+m5X;W{7{M8WkNH;Zdw8A{WTt zJuP0Qtw;AvxIxT$M5~JP8?L>2ywZ8fMEqD0%f*8aUCt)F7GLgh<~#@TuHhU9ZVy@; z_w38H8Z#Q{?RM&Bh8anpNraYj*R8MUt43ev=aj>_+1ywGZXW~oSSZ+&s)C+oZh`i@;u7Egptl^+)2{x zoJIs!SqWi}zakb^jqoA%Q{mpY4vaIpqFmILEtc#^`4R0IlD>a5zyH9F*uuhDL&R%ZHPBM}cHxBr?ZOcDv^fCuUic z5sIs%i`DPbARD;uR7nP^l{V{BPlP=LG8@}WhtO}_6NhtOMsBUO2BRs@v@B^?$gJ0c zp#AP-!zfJMAp@0;UpyPu5X$KFh*306vcxqt=^@=?Df``T4`+DOz{JXK>Y1Z0Ah=Gy z`}yH4G98)K-Oo~PkR(QYGcWP^ef$ZTG{-CK)WQrJZWqWFPw)z0CTT;3HpUux|b?IA1^i$a4&a;A1ZOr_CwWfg&(SNPxM2zZmA!#-O+w% zwwvRJ=DGXUs)0Mg*SR|t0wP1fTlk3HJt#~Bmrg`xh^Bt551i4nu^Vs0ro0TuN3b~T zzwaA%{9}!Qh1(-~{&9EWQ z83mH4Ieg1u6Is}qK}Z(jWaS3(xfpN!h~L14sq^ys)R$#~*EE`8npqrF{$lb!z2tc~ zXuX}9%YJ)vCopX*P(V~<|33uEW?~TSnBbqIAe(v%2bxvq4y!&MsH)O z`70x8+2KC=jt7UXf}6>s=@&E@N&L$4F)R^B1f|!YzLM=#Ta; zgwEr9$lnzbIN9OF=K8(V5vyLg#sRYnxM3a%|9TP;^H9Qu(F_CW?uI=aku4g&EAuG*^UJ_ouD-5L?LJ4_Os@9A@ zS{Wo{n4YMr!kx&HltJdos*&e<zd4`&~gSKMv@HCL_&y~_Tb2C{!fY~)dm|@R}=Xq?0!A;&2zAiDg>3JKJ3-0cVp|hrx$3?&%-ZDE6$TGoTuGx;gp#-KQQ*Y-})U#{DEc)-AB9vxw}1< z$PmftP49V!G@0oy>6m@4Hk)M7-hQ^TbgqspU1V7%zN9=O7^;t(L_zN#ptfZ>A(WZASP7SWAw^n6dZ`x~pu*gX&G8p9MrFTpW`-nw4RJwe+RL8FuBn9K}7| zo_>G0njT`tPd*#2BBa~Q?&#mLw6PV6KEOuRr~!a5>n5)73TY8Xb3DloyOC4V7;< z)Er*)XkF9c!drie%cWco=O^ENnhMEf=b;~&%GwJH z+Gladp{XOkeb$luSj%OQFn)L8kZy|1+Q7$N&K>{_nV==DdzP^0v zU~wY_vabF}vR*J^RJ?a5%*JEP9DmZli&vlqFDCdQ0_ANW_8rTw)!o4sZ?-AtvlgNxV^7@d zK3OaB5$#c1uh^ul+MHTw&Bc(Ja7`JL-Hge1P4>iElZlQMaGhdn0$0G0$)B}Z=8c{3 zRpx%y%lxP^GYcEd)pjw~3OkQ!;H_l8xkLa2n{Nl?*z`NX_&8ZW@XrNQHr#H|R26KV zt0a&xBI!L2HmQP7s?Rky>Aa!3pGB9XQ2N{FMsb0md=r!}4|L!yhmsbY zf79RO4`m^7{!M=~5{jYLV%LJDK*O?R8|Qpa?*s_$*R&yi={yf-(7&rFnqAdMpS$bO;4xLK)FW@ES`M^tEH*gmQ2(csv@5%@=Z%s*^>Dxd6r6=(3#3sO14sy zZCbVtlNeE=NTY11i=`@lj*6Sm+1dQ&>)dQ3s_WwOl*@!ZA@HiRSxr=psxG4HBGc}S zY)RgIdGg}vDrrKWj5$-nX%EE^f**Unmpk8}^upkiw_Xqnt<1Be{vyC z$lv2GF9MkzOI}@?T*wnx3(=c#%^%|&(Y&~v6kRzV<&f5sizVw(k#>?0>1-k=$8d!5r5H6|XX3ZUcw{bh|Je_mb5R2?4 z%lueqUo5gY&IMI#Z;Z>6jP?t;JOhci_GUbAAR2l#9>77#0CkRyM>sHchzyem%}PzN z;@p{W-ib%{MANHA#jTg4p*^wG{b&7a?7VNpBd(gnqlrUm=a_h;ldR;MOepRgASE7n zm5Sq`jq%W%(bSaVOhX&hPz~rA$EYl|(kx{od*T*8 z+sTw(J<4QGubym5uOlHAcw<&-YAEi!5(^!UhhC2b-Xr zv~E@^Tpf41qXGP>t&XOw1eiSNg}|gAqFF-yacByGn)m5;Kpn~#dM)m3nU#9%%tya{ z;eGMQwkV#1)L*MF>bw!v*hb#>;-C|HxmhWJeZmCe2vNQo54A@_@6dI^kP}@!K$Tus z4@8|UaqFFEq#ws^ajQ#Ge!1$Om3lO^?sHRqBf@hw#6!LD(3WU~v3--lW8`S|J&KT( zW(lnqDy=u8p?!>@MOP{1b#=fC(N#Z0R|(P8Jq)ENj%7I5hlCsvptFMnfSV8dGAUsj0XMToZ$woP7#tTsMON!s#^-4}kc@1NQr4mM}9WYV85b z#i@r*w61Pqb=uC{4(;X-L(Y7~Eq=P2!Sq86riddR=(aXrl$vs4+@aDe1u1^BAtC_% zAR(C%eV4)XX{m&PJpH-mNLXBfF$ir^L-EL4aa^)zhx4#f(%s;d(EfO!gG8Dn%z#dW z456q0m6qhHEgxz!!}>C#dD`+ji!l)}hz#{X2p}Hni$?n5);2Sspq@dEhW3A8TsMPC z;g`BpSp*7@G>W^z>B_pjg^OO%IhF`NQTV;>L`r*ikox>qf1O3^Rn0ov#`? zafx?HfwBJ66eV(EJ{oE&Qj3tu#mv=Tj(Yt;7)WHh*xUA)wR>oAy}9bOXXYzj9>kV- zC@>yc4Ofa<@5TeW^==@%4-y#SN{8dt`?2J-u<{#H8D%KqiBlWjc@TE~I(=iN?1W|O zueCj{D9zKIUUukdCofJaC9slZG#*-^79o+q+kBFj=KkjKC8p@#jI+PEwKIl2t|ErA zRctE#8|{oI^TceHXu5y2DL5Fns7UFBq)+hTNX;V@hmb8TBwA%FK+zfLe|hk)talN3 z*o&iJ{RFICF=t(Ng?NQ{MufG2*!Kg#-5Ygw5)8c$S5P5Q$GkNt$ zXDTEH3YA|rhhI{!6i)S@=kY{|ln+|w!UKe)6QehUr zRWX*n$XYS1P@->+Mq9lo{0IG9HEw1tqn2fwWxI@&>QLl zmT0;w7o$R1U@g(ju(&eGCj zLaPeN6L;1|)t}xxk{C>izlyk*UNxR<-X~a0_d=eXu~ca;_cZX<8rcIxv5-3}wY22d zOB*kdC_6Q>NBAg@TCH3hL|7EWebk@s0%&5LP6jnF5VK$-s|sVV7bfup72| ztoE43w=>;U2$<@#Sz#)8`!>=ebJRsRzc46YE3BqFg=ArU7^jXKi-5{|C%x8k;8sQy zfr$vQZsB!Wl)-a0f=~msBrSN*o~$I7)__1iP|Us%b6~tcx*)2mnhuk_T0(P_^9Y11O*RoIZf9%F=ocROr8Q`45kiZ`J zVCvrS?~d8p&O9h*bj;+!6ib38g~k&&4VWYuIK>-d0fh4|PzQh6XgG{`BbYm!#oQE& z4l&~=cnp>h(?Bu)tSy6+)dK>f?I8|Or+AR9R#EwPt&XRb=FIxnUz%rLA?F^$X8jY_ z#NZp{XcyDcHGr|M0ox2bs$PD5*C-a4*MKO*Y=C`q!tE5H)kHsLSlg_RbFqfZ3Q-dE#&Ebpu>LGJyH0Oo+)h)zN=82R}~Ow1P&3g)Ezv+ zE(RCsyeRe1^y<*@XMTvg{V%M|z(}P-Fvb{P7!gt}#5aM-;E27|4o?XXYON3wv3m-s z9tz0=uK|rp?2HUZ()-)jrZ&jz6=Tfe&WgwYa^k2VHW!nXW(83!VV;!+PRD2N-{E$C zE+d+fe=+K|KVxdeBpn_5P69jy8#;k1s zR!yWQZ%;(g^g;}^u|P@q-)X=l0$KXw7p3kE%~?45%P}V$&a%8A5)+V7Sf#{4QO8HJ zJui5_^MdZrt<#`W)kIqa+enZ%3WgGRUX6HCG*)j z=1q|};WMMZdMhj$Cb)W(GEFlaahMk>FL-6{t`?W;1mJ(n5Dh}f%2=~BgWHijItjb6 zwgP0n*A=0XkVHV#5+)JoOKM6P#R{6Um~aR>P+Fuj^~9NHu4=oRMYx#7%&rm z&VKPslu#{t1I~UFP=5NMebhN7@=8tc0gneFq{fWV3ptOuGr-orjG0F0^FosN_R9bm zwmRaeM?d+W?(_d8Me2@lX*7a7uphjlPe9&i5M#tHS9mde0;H#FbO9tGtbp)z($Piw zVlvGkxwgdu`Xu{UQ&1sPQ`Q($!l+tCq(?FYvy}JY7n-$R8!`mHF_4amE25P`ut952 zKS~RdY$q%djd|jyZaD5&p-w3P8`Ccs@wyDm8M)BtACo0gz9lqojRjn3&Enf=I-Xnz z=N^kNYcgf!RYVD@*aGQv8_p@jzUGg88@LtfN`bD*EHit74O-oxjQ|fy6O!7p0au#n z^4$#o6c(bq1<6lIf{_uq)u{H2G7%!LHeaY{i>IcYjZr7C9a(NS#^zY4FF*(@qq9dp zxe@a-$rt}o&e}Y!et;j%CnR*CkSZ-{US)ZlnCfzH1dkQ)h4G>ZY}w8xlBPtFw9PKV z%Mv3%K1wTVdE}AHZ|cyjP^Kr1?f=$M> zoVDmRi5=ctdX0J3mj@U?wKrxVaE%A9;>ZTRobAAdItNs#DNv)Uao#gCb$?lS)KAvO zLhS3X!AF!YC5%x++tKqFy*0!I)Oi~GX|407HYOjYwlVePGqTL|363WG znnFq*!Q46(%}&bOfF%X0gl!p!k0(?HkS-!eqG^{AZBNapg2-hXbY(6~J#<0yx#Qjg zO`OCd4y4vf%`nJjk!Z~F&fVrS%17SY@m*UdMpB{$()(_2)^$;Gq@>XWYxeFftsgP+$Nx!q-M`JM7g^Pc*PL?r~AEtW~){3KfFTA&v?K!dnl-tX9$s zO>qQE&jx{UrI%&B(@d8Nz%7GglWih(dZF=VmyEbxY>@1D6(g=mQsC{N&>qzM;z{X8 zSq5d_WU}mGO&SHjfbUs6C&PTP%pfPdV#wcHv;rG>Ng(RT8lv?;y-?UdZL)eIk|#|} z0P?KOd>5w9om3RJjP*3%h;SI6B^`HSH0_)iMMmKpYN>)A_=;OA;?CiVQ%g@Ny6Lg=1@whUR7xJdxx#~| z?jV~XYGU-}qBCNeP%UwZ41z4vs@jmApqEflfwRNcWykeoc&zs)@%%f z*xXqi4=C(Gti0p`VnXa((E|k6?uqn*@n`$Ti&$Mqmz2cB1;bj|iHPM*VZY~a49K|! z?j`PKV{U`7|F5Su?BSFDSUjc1Nn^Iv_LR&~p|z2R>A$FBikPtXG(3cqKsqn+lo88P0LSIa*pwFicL2oh{ z=x3#7oT-DJZ0iDk^<`ZTT`i#n4veh>=DI-!K0x5Zh?CknR)%=vMrEgk{;t4Bpc2MNV3wnY!JCP4WZ5j5%mk8D76Ta_%&%fc}Fbo-($TDF#g1ls3fE1@UNr%|?;egFYRIC{-rw6)zw2 zv z#Ua37Bqbhr5Ai)70`?W~Z3DRSdVm{Ch7WkT0UV+H!+;y^xs%DdS^x?3DQt7m&iI&j zWWZn?(9KqtvT)9X`I&n3q|(KmOEs%7O6?DAh_F@lH;Je$h#Sa3JgUP6A|(h1x>5R| z9wQWY3+8~r%+i>)kPQbXBza(J%u*@P#%33C5f#D0k-I^8AKZI`4iHE6)ipk99U{-J z(W3Im2FjyL6172#NcY95`%f&&?YP%D;&g=C0SpFi?bR1-JYs$(M~-MijB^qJL{5b8 zc6Q~U%mAV&V%%6}k#>|PMRo&N*n+?8b@~y{y9?oVRM9U>A6r>4&)qNrd~admmGCXw zHf7gEHUmm(%6UjeO#0NblMfC)_XBa}Sm>R|TB|$eu#Gh)@{UQL9C=4EYh!^oWwltN zVCYSN3Mj~~T7(U5W7+NnoC@n5>j0CfNP^ui%_8j$MD!sd+m%Pld@}jNvDDNHV$K$@ zgvcfCoB?Bqkx~j$*PNT9qYkPO5HCtSdtBzG*^iV?lsAU>oQXkt;$eebF4Ne{jI~^S@qj@~Ni98Q@8WN?OJqesfv7-_4E)%FpaM^8 zV+lpKfg7TlgtqDY@&dl%q6jMA=A*|*{`8MXsXAjl5U3LnoYw*eLtP;j!#UyW0)4FC z^CZ!I$=XZA9BA5RNB5c-a_M|$9vddtO^G2wP|@O&n6sb^_j`_wfL&W2Bi~Ii$JSks z#UqX=o43`TtL1p7ZXM_%jJbG#wl+J7X>}#6r0MMoox*tK?wX#3d4sd^>ucV<<>TDg zXguYT)E#G3n9EA|A9=67@?HFwx`8`%G?sgF9I$_Z+hGl<8B^-y=ix|w<-WS6BNyH} zj-vxBxkrC#F6mV?56Fl_15;LZ5K8k^O>tn&BK>a{&agz&b$#o%h^XkZF_t_(u~fOWfiG0M!)FCD0PjBQe+#&nIzQOT2(LXHDWL z0xj`E5+}CAi%6W%5)UM2mYBrR1X|)_NIbSBUQFVomiXA@%ugNG?&dzAjr9T|32ZzK%9T|+Y4@Sx9K+M-78c+O9FU;^Y0pY1iQ|s zfO0&|)!up%6_^`t+gx1%i1K!Ho%2_F@~)=0^KpVdr9QQA{?oNo_uMD=qw8wX+ccuj z{EAO6;Tl5j>BQp9hc_k4qW6^MSIxY zxOUa7TME1lRDgq$Odn_}AL>YNgo^>Xinh;O`pNbqmV8_o$8iaes{@mFxf96gwMReP z?Ydu@`r{DDfI-*XGfd06ac5t@&V7Y1A0!2JgM^z?I$qb#Yu$EDXCse3uE^jkeFR8X zCe3MY8z%sFn?@GSr#jQUt9w^-8_NgRcb=n*q~IS{ESz=MUD8rrZYz@hYliw*>d?uA zd~+92=;FMHJDqVh*I*3(rV38Dqi)XfbcdTURItU}q06RsuTafH7}FSSB!YAE_bj4C z|J=NyK6q~4cf6Q=TMCyp%ZgS#`>qbpbE~1mUN-%gT}yR_p06@{+35rKAJI{IzRK-o z?|kBf&+9}zUlsPUwcp(oGkfelO?i`Kq>;-T0U2Asw^ltJYq2Tj*O8 z&1w6a>}5J^f9-!f^8zWfrZe{+ul)Bf@=~({QGmAIzMQb{pe}es|S$x^fQ#~MoQ~Ergk#*M0yyNDq z&k1?HheOc$lY&piCKci=vbx@!k&jKX_^8T$4_S039=DU9 zFVl^r1^Arce$t)%|7A#s<%AR9B$ms`TP+D?wv*2*_N~lj-TdMNJgKZ)ET5#7o4aBd zLy4?Xxkr)BU5J;k+ve|D@!$xy?1t~ z)6*&%@amNA9@mh^dp8Zv8+B;x4UxlMY^4cxH$*lxSntCI8bVtd0__cv?z+$#F4XZn zNWGOYPZV`eX$Y;ft=Ad?yBnO9RPr%Xw1cWoQ%J?vlbMFEqT#yq8a!QYX~?Trqw5;- zit`&HZ`6g>HAL1mSiL+J?7^W@mXxTlx< zhT2|FN<(0ca1^3>y1&u1E&O;W-Y}xw*|ggrXJx{3q1sg+L5bJp_v<37g*GAIw7a_A z>hp$6?NV~6-OWk~tcM^P7?jW&+j`lwo5iz79~xTWd7kO7%FynQSNh|dS6ql|_BwBj zSJu(7&>Q@%*ZBH$rV;hqH1uIcUPHRML8xZz8F!6uw!f;-BPq3eo$2oyYNXxMhT5G* zyWMpnXw#5LXC>(4Z@mS1X2+P39cr68cGm^gYj{GNp~E_BuW8qeF}+lU!|l#D?LyDH zY4@oWS-Q%n-Te)DJSbxt(a635L#YLYgc>ZDVf5*%!J}($v`MK8tkFEu7&GL{P1_KM z8DlE-+g7udYoEUah81i+Ps=GncXd4*B{oVCoQs>WwsiG+;>+s^Rn!w#WjmVv@Oftu z?B+6<%2jX}F_f#|AD7iigrUytI_BR&k3P2q4*4O`+cvVdZG`W;Y$JX1?BT+EdvF%4 zZ>0P^`-Aegvp~ymVz%V(ckqSzLHYYGU;b`i;=V-k2>E-7k-zWC%HOkxg|DKUk@qEeBJY1Q&5tMW|INrJ+r0;kk^j!-8_^iKT;e`* zIX8o6vM#z6M&L&>&+{XY!}I+h!hC@rM26-~I*Ip^{Sne7`y-?Wh%ED}LHb8b=S4RY zK{dEWH$e*FAk_q^W}JeaB7mSSMF2s4Jdr#zBxq0t5Y_SLZ6e57>IbO?rwk^RYIrD1 zg1@wavplWfOtb`XK{NxZIj<#N4m6)=iQ}$* zW|ddX$C5LrwZyqVZz8A$&kH60Jyox$6Zi(ZOY-X)j=h?}b zv6lEbegt(+^MllRt{ z&9a`qr#-=<4lV&DBNeDyKA=ExIe2ANed>=rZ7XuSQ)0tfs85vuwf-#L;S0?NMq}?NE<;zdEd!c+MdYaX+;dHA+K$ zlp6IZa%R=25_bbmSWu&ephf|qgYFkh%ULyQEJb{f6dY2cYCo(-4TFRl<+Xjty~S%p zJ(ngWv|ZhM1m^p9G^ss;8BMC>XichsYCfzwd`pMIWQb9V5%6DTPk_z$VWAc2QV@t%)={BP;9X-*W~Vmx%oy4(9I`1266 z?5D~tW!F6V>TBxwBbXB|a-SlNJK!Pn+JePnV#$e1KA98zA(xR~(gRC_dH4WPb4>6O zoYo@~WCe>xMb@jIa*s*7jd0++BrncB znS%}%=)tkQn%eM>nnGg|#06ec#xDlC(m4_ritYtKT_3QmiCIoG=g;`AK**C_(3n}= z7eEO>^3Vkx!w^@am+jQaEIC-%GPh_E40w#4x*QsqND|uCL|&lc(X4GmJpJ!9 zjk^h)qln^s#E2@F&ozx@8~UQx5FDItsn<;dR5&;1R#vAxB2@d5YRh}L{sHQFfa>HS z&^MG_pMoQT4;JweLk&KT8tMWw)Wco#I%ZtDo8Fl>JIqsU_*;|m zj!^a3mucB~HESE}clE9vErm1-@HJAjj_;~y=oxXqyQ{=qqctjt zKev@;v0|;SS&pZLJbJi1^X}D}mWQvr!cJbx=xve{ZG;glX&UxFYG{BeNy*F`2*Wrb z9x*GF|0KKS#v;9yayv?Z@soHAT>w&KyQ`}*uiMT>L*?MG;EboS)L^^PXeRO_FJ&+q z9I;d9KY6^B#$$|Y%WH`$gWPbQ^tj;MepZ;nTL#M)*e%8d>0L#G zvnle~<)+9-CYS%?6zf1Et~A1BW{Dp`;oI}9nW9yd6hgR`$YD$|DmTs)l2Ff3aFo|k|4Z||6Aapp6M zKX&TJ#xaRAfEvn-5l?8%yxV){{G6QUmC{?Qkc_aspCA-9Yh>?Bydl72zP(-i_CBgV zKhvLL{VCL+!?+9Y9pI0b{}e)GKXEAH_d86)fAiBjVLN}u>d#@Eb@mSM$9UK4-Sjk> z_VvEPkEWCR2Lepht9!S61vuFkG7!2y)fe=FQ^F48md(9G8O@s7w4!Wz?{b{s^*+ZR zANtSXK~&Ixn%~|t^ru39eu&83`vd*?j{e-u9}oJUBfgUTr~G&k=Xw!(8LIb{J^bm` zpXc@GY5sW7&jk-aeK(yNDjVQ1EtsmL`*U$Jz?XUUVKiU3iRP#@^pLJjAtzqxu{qUPbtPO{F zlNrLaAAI!(PqBQzCRm3e}5td;JNLSwej zw1o2n_T4+HnKs7ZnqF1G6vsto`}*ZexK5K3T>20^izh;^f5E5Yt`#wS%uYt3iq^)-gFXBNorUV|Bzu`u0RAn>jJB6a&Q=)q z#FJu!?vq5IxvLvq51BD{@0`u&Qt)%X?-&2x@G=V3JH9Vu$8(Y2SR1Td8kWKttYq`W z{N8W883!vL3TreZxO6~He5@(HgB#ozuE&~&k!QsYmAjf zB8tfqoA>TO2jZ9(bAF4f_!Vz0b8kZZm%KC|p^aV$Y~Evc&VTHY+#EWBzg!l^06s3S z?ftU~v3-u{cmAKsA+d9(5Fzn-d9dXdN`yyOz=?SyP0CAHFPx_bOOX7+RwF!zRK}7h z9ve&*tXGRp!Au&;+?cKRTGji;@uuFzFs!3198n$Yk2K#kr)s3JRKd4ZXH7x>f92*R zjQAc}F6qb2nc z@n;pX5n|%$&7k}OXi%hOn>-}JAHBGt6si%&(6{?8XI&lVg< zK@?dtILDlF5*#Qs_Dua?fYH`Bvl;`s994>x4SAF^N&fhsxvU|t4$ih2>I9ww?;yPq zQ{PyJ6wR86@=TsOw`Zc2^L!yCBdvpLBdxtZQYF-To?6V-sy(>WF~2;~w7x_?P@$Ts zr!udN`eeabd<->pyw-}xy*BP2ZX-I_llh5~j+YZm4106bgL5A|Do3OhAB~^D%e)Dk zho+A}$XkmmYOh`{G!8C4%WfLXTQruxxeF%~(RgBJ8Xx(IGwtW$K>DXyZ#t&1Nc))c zaM)^3pV8JhPUeTEfr>=mGT{t){WN>oF)BpXyT(D0#mag;0zm!%i=!g%~Yki@W+N=~289&+@dzy^w0{>sMwP%n267M9v{b({JVCe3sec z6gOG9cpvz*8=4NBl_b*LU(L^CdSV$KBApD#80TrC~Z_>>RA}(W}_P zltyljgsh9aBw%q%x!=HB;rTUX@|Ov6qYk^dXPfqeEn5IvlfZ1>s;#nna+kI?j+=2! zBCkF7EPIe74ln#A%Q+*Pnm08#1IDF$9}nmy^;*OZw+6p?nQ2XjEIqu~s5dWmxL-Vl zxbmh$<}pS;ZdQ=E;%uq!oJr%x9$}DI)5&A1E@-*!c>C$>NJtniWO+E0pJ*m&@hJ?< zIa-6S-{kp-KDU|W%#5*}EZ!Jtbq{|AMgkXbJ0}{1f~S#Qb#-j!urTDGsxZ$S()}H7 zn=zWY7ZKxc1mh7I?rfil)i5L@B?;$$#5S(x@C zE*?vS>;>-aQ$6?*^th{8hZzULdKw7z?=IWoea3^wtq0{PCknmb=^#XRDBL|CI_dzMan%HiZuCz z`OV|K58k0wh^Ai0Wa16{@|^Z)33jZ@%Z$}(eRM9{ni#C=os|uGv-hg!r!xOP&0?ZQ zw;u-7VCA%Muu|S;f|YPM{<8PT-#Ou0{$7LYgJ9*C@E;ef zOa>cQcJ(#e7xFIhbh&{&PjJD6Cn9y|)xKe#k-36Jrrx<8;bm7NFLU+VdDD$;W1kzE zql(xtf>oXjqV?JmJOzYIcQY(g)b%uIU<59tyO{vrzvesA8myoP6{ZIjrUwo~)f)XdT2&bW*htA=e)qczZXi(Fy;(n# zrq{K@Djwv|r(~Kn%J}w$~n+7SVEnPzsNGh zV<_i<{4*q)n2NRRml(nldFqJ|ro?UQM6mPRDwrTjp9F_H!V4%teRM z)mK90O*>}`2i5M>EDw@Pd#(32lPj`Hn;X@3@|Y(cL`Z-^pt=?;_z7V~p`4sKgFon0 z0pR9fPP=>EIJs`$7i{^yA8TvOYTY@o9WX=|J9+L|s(>?U_H9LjKiW#h%$L;5TH47c z2|>e|SH-a-UgWKJ=2s8?s7IA)aRqt$w*-?Hf?=}Y?Famt27j=Xcww>}i!4vmQJ-Ds zi+*lcZcG+9s*7$FsRhJ0^X4-?Vqr~@|A)1A0gtLW z7yf6G88Sd%4;nBiNRX%mQ4=AWkWgom8L}e*qmhe>l@^=g!3)X^0TD?$q0P?jw6@x6 zPw$uWReO3)Zx$34bAe2tYC{mkO9fHr9)^201Q6%@y=%`TXfNM6|L33Q$?TcE*It)* zz3W}?ef3)JOyQ=`q}Zw(ASz0i%9$Q3YvB)BXYz)-iW-WN+D6|nI7IBu?rXUL%g8U( zb&94|GAHf{>UN${>9u?$rM72h9;uIPp4&L98>eJ@5MXCgHu9^vVbcHUO&P4uIw%=}P7 zwG%?&Ur{JU<9&XgH{`Z{&Yn{J(g5IeZ<}D zaE4u?gr2f_0*LqQTG?iHO0eQdsy56oNoT~yHsxkg^@?w?i6J_)Q256 zjTEHee;NT=5tUfzTB-V$^%^ z%JHwrl7Q?KTJazlYCZz}*&>|(wSIF}A%(BnVHLeyA9=Xy69;FFGPpeZM5OaB=S6$* zj6ltyt-perWKjsM%M+4W2zu_x8bHq)<2~z{8)=o+@}z}J1OyoN2*a|z3wKYiwu{YK zRjReT&Df#l*>sO=J>kVvn=MQZ;zRpiGp%W?8hyzvIK)Ju&<3txTXE^kw63pF!?i}t z=Q%F>Vg?x{WuwK9&>1*~)N+w6C>Lft+q#%uCGMXCiTgL%3yqjQ#EVaL^Kt;5=@~cC`$`^59 zv9>`}gg{7EdUKJGaRIt+{KX2%Lgu`SIhG}oKDdy}qioCbC`&D7Z4?_g(arGFTSXXR z?Uh+Rc?TRT;c7*oscdiqNSnaKw^f_W5<&J6o`qEc&%!E!XMww#oL(i9)2n21dX-F0 zuM+r|HabgTdzG+X%Gw*`U+@IuH$+bK1%=RqZ+)9}z=Wcv@G&l>*4Q+JGW-dt4Ue=5 zd{?Clb}CWFG_RV=V5bh=u3z7Wk%i-*M&$q0;ixa2z9XD%hIY7GZ`qMyOiPT{lfrLb z5Pml?x*q(xs=n!T($O$9mN|wtTI%q$idokl2V6>*A#8RL+G*qi>i}*G^zrbwGGIm6 zewC(MH*2Ni7B}@shZpn10DG}P%EMmZgSE$cAFX!+6ypR$nYV}>yJJov^?KT8_8NZds=IGQOcU`2e^fbW2@}oC3Eg>Yy9q$4s|+j zvUZ_b0``;DxIwvt>hqd(&gVuEBLdQ#BP<7?>C%+XIxB}mnC={&%VhXuw1q;~ zXg)o>t=1cMsUwe4N}3l7m>gzubk)VeKoz1Qkbz%}ppzIiI6v@XZyZ$+SSadiduRO~ zl`v2GtZ8fO%bfGW4L6Ql&4f2Mkgc)E7`K+<%EMywb(>L)={+N;7dg)FMafO@! zGf~Miql%MZ6UU=6CH3R|`Y97Uj?{PIca7XJ+2+BtmN%(SdJaEbJZS`s0w616D^|1o zLh7L}b!6iC_OP^;pUVS!r&|w*(4vSPmTT4DyFV?)tO4Lt)nFL;9Fk+j0+#`BLhDcU!I0B)R;kG4^t3CBzNCHkwWDQWDQ7>Fp_4A<+634yYH~3J|F_ zE*6TZfWjoF57d#hE(;0NdhQz_4d+zJGBQV$WjnzvpG+91q;QKJKmb~z7RcW?(tZ!o z2YalSY@k7yM1S*^?O4z_%i*}~c8CH~#<7JD`X1yjfVjS(ta{d|> zCbD4TFd`kj4a~3y3?Asws$4r7q56e1<|w94c)mkRMK=#Zgv96m;rtp=2SM@#6BKU` z+QKH$248G0ibcf{j#mpL#EnNFRm-9|8RDcX?Kgl;&IGBIzJ1r$k4W#A0PWcvj4s~! zd2`?xqwa^lDZuBP(BsKkIj?dR)PZwnYuF(dO3;4|7LC%qp>NWV>Pisa`lm{vLL0=n zA&W5DHX2%;6TeWl{+HMq<*zY5oS&kQ?$*tbsdRPJ+ih?mHAPm>#oF< zfuBE&bbTpt(ZJ6iM7rt|*_(yXCEUu86EHH2hdcyn;6r+p&Q`dL2ffBa*|CZo*@g)@ zP}<6l+_+E82;w|P*&BF4LJ%>%S%|5 zor5*zYMhkl^WMB-&^@4cfF`m2x6!7zrF+1d03V26%(^sHJ>FlUCIfW9 zuXqZzSaMDtmD@_e2V`~dXy%=qZ@d|!3b65UXvd?X3&5t=Qea;j<#1wtwr~L=xs;e{ zfA#Y93i~T3d^AqsE3({&XDR1G%WFL)>p$vF8fp_YGA*ZAuNBiEOTSo37PToh(Ix^) z>&v;00a*l!u|;o45ATJcw7xFONH%uNsBjl|Ne~Tb>4ZG!tTs+jLjVQ4w1`Venp1HO zg6ltO{b6$I#ShdN-FMQTIfliYveGiBI{E0c@Bx+nW<;A@AC}ZPP7yb_>Cp&wDnQDR8m_XQE$6H0aS`R;l#A$qh9Gvw0JUutrp(` zlrUJ*)u^p(Mmh(3z#O$NQEJ5yVVzG{w@Xdn$aJ+^C{Y@g$N#K@)vB1jRrU@QEk63O zx|5boH&p_=WM}-4qDJl&bj2p{pW0px7jqI-D-<< z@A{g|1ScBqvstG{e@jD@6c${NcNJbd)KqPJ4y+ePG|&aw^lW;ozy~dyWqqIO6W_5O zmsT)BU@T43T5!!bxVMVZAk|Y@i;AGYZtkaHC%R_DbI|_a43ZZRynnD`5tEM*|Q%`0KBlL4_rm?7tSm$8#2ip3}cwS?! zfmyB0#Rs7V)lBBH5-44T9$A&=SY1Sf9V%Fzh`9sVY!o%6ryg|crSS)c>7_?joL^J= z3E>~n;)kU{wJl6HXXoR;>FiEy@@sj2sczcufJwJc$t1EO+iGBvpUU9G#%S^KbT?V~ z)9F#-K|M+I&e6;D_QRL^oBo=lb4~l@ll0s)+Aw2IH1GbxhEZQ~qL!%@qCxc%Mf4t(4Itvvp@*rS1$*>Z|l+gvL{*k_r{C zpQ7m24=5OKp%$I^J8X7}f(W+yr0Qp7%xTBfxj|+O*Ob{lef@!xAH+zsqiEjwX8Xqbk7@{ii0)?w+EkQ{AR!`z-&>y$E;$i6$Jl1%O#WG(_ zsz{Ntbd*g=MT%QHaDb50ez)St%Lc`pOt^H>+I78nsax?lKZGZNUxZDU!^)3ZkCn?9 z@a9y($LMmwjPQA?gHkAc8#HjwZARZ$mk-MmkoXLiyU%8K5W`oWPn$bVrx1bOet|Zn zsxN_DdSq`V2FOf~dXE-cQb-03VE~f0ATf>7v~VcGjteE%l^`e#EE+z_(?EJkhN_Z) z@7&&jx+l@8JD~ zJ6ME=p$K@K=+UmUM=z#eMHIt$SxUstBRpMsP8=3 zv`<;K@dPsV#GOc2B_rW7#eOgQtcKTjAl=1Iwf7uzIK!JxeSeu7MZ2GPrS*S zxCo~vp#8mU$fiD=D04L^-N-%`Zi`<;QS@N>)y}Q)F2w>DAMPK-z5V5XH~lgzp_{b4 zH}$6XRF8@LEq#5(^&Cnkp;g(!%Vo~ZB@%eWBw~BvPjmfEd;6K_YG=2VK|;$2aHW$T%ElawteX&6~U zb;<4od+A7t8j$sg%%o&mT$XJUZNR)gug3gxuH?n|LN!7Pw0N#L4;#rIv3`D7BJd9u z>Y3=f>g zE$=PhLN)soz}v0MjaX5eP1vK^B{l{$Otx#7s7JH6>t>O!uge+olB@29vgsY+EM2xR zlq7A5R{*ucBofkGfR{;k67GVNXdePH=nNk8=%vSWt*Xmx$me)d=ZVB#Wou*mR$@WG z{4y`zpKCo2;>bjO5E79k&ta~~Ukz!Sm#aHZ+4D5dBVl=EbtMg6-FeoZd^xeB+L$AZ zgj-EkS{B9H^r=bnrYFrpWy+H7lF7R(HF-A;PTqCb$>hyr^6(0b7XPs?HF-4!Cv>BZ z`Mc#`%wP75zn{M+m_MRGQa=9pXVwsSUbHP>E?~%2h&0>kUSx@|3%?a)YrjJwsCj4p z>eCWxKqjJnla(m)OH3e@8nkz^(V3HbH*ru#7!6L$w1Y&j$kB<~XB@iRb9TaUS1X^0N zV&@`j@*JA+9cJ4PjREJ?8J~`8MSYSC6`PrKReE}b-^@yVrIt!h^%Yf~=VYK{qG%Xd zEodSOub`*e*;QkN34dDQ1>lVzsl~(kT9a$98j zn27SNiEn&nw5n6l;yc))m0%#kqxEOoSYSKG<;ix=R;cvO@XYAoMM@Yn_JM#ykP^gm z3Q14J6a2lTAU>F=xkOl)EKCCun#`j70w+dTn`IADDm#{(@uqUyfd_xjpA>bZ5)hCL zAf%HWEPS-}H=s~sN#Fcw*#Ot7w?j{fHmHFL6F(Z{@8!WI^5FCGfP@=qkpTt&fUl(} z_$@|Jl3tAxmYvdsrR|QWSYh4La>1&H>-G$!6zOj$CLBD3DjH16-$Jo z(Kc5z{yZ6faz15H!@-Rz_%gpDJ-r~R;Pr2>g2|LE`W$(zS4_q8gp|?I-E`Deucj~y zbpT9@RbWdjcpoY7B7v~Ih^C=l9#AEd*{K^lk~gEAkV+8k=;JPBeUfF3w80EpnX)#R z@QPGDwpw8lm09DYCcEcUCjJA`U!`jWwI$>m+t5cSKL>jXNYetOYDf`ZX>F+}*%1ar z+s4*cXqk5ZwHDEvN!vhN;1DhT;%V^)m zoKTL~u;AAg&QX3#G*f~v=-|tP*r41AtD8{YcpcBm2dd2&Iw@XRXr0t(#!aHydA7O$VZSe&1>Fn(Dly8Rd>u-10%c?wD%TVMYX<-*MvUzDV3S{O4y*3)4?Ig%3?$Yle?s2z*Ovl5)dYn2!km6sq`1W zfwonC9-gy??Yu>!QiTb-mcp{`Wu)>F9Z}=`6yoTEvTv;&z%#8-+2Ot;X1Uwc29EpM zyrRQ`p*QI<5gM2z_hzI>^7hx1End2!aL%$1AhIe&Q|7AW*6%?(BrJF@_Y)LDMYa3} zGmrEE9jJ}W?VnQ-Ud!_dENdNAUd<%^Vm|w)A*7Kr=H>;hmC9(tNV%Lkp zN5akN(S`XnrSGpmKYP{{cA1$hbu*JGBa$pyoN+Y8Y*?L`Iulw`NnLH_&0$z}cgHGP zaV8Q%g;Vq^?L7hs$4Nw_P2~<%bmb2dV(ZRnxl%4;c86j2*+rh2mLs(W88f zyPS{l*Yk04J0BC~SX1n^iI@x$=kKBm2+T+gOpekLgp3_bcF&_1OX9RKQGZ6kpLcy3m;*Eh{@l!YhN&rD`+N@g;H#vW}}kG2tO z#zAY|m3UXVuHfcw%#+caA&K7+ASJd*wa%Ijd1!?waut6jQ9ZtjP7&jI*u=xf$d6r!~mFHC>Q9@;-@Kpjl>U!su~~4 zbcC|H+=oLM4rk*-*$y0unmVRmfNuV>#)pR4Kc5-S&kGtK8j9l{BU5jqiE8Tj5G@*P z2|`PXRxn36FMyVn4>XFz|>o4gI;=4Q& zgV!EgebRabaGJSeMM@31RBQetgJ%_6f%F@9NoCQQKcY-3Ip4e1I$lFcZYE*~jc=uf zCVV#W#Kpf!D6M!UvFEd$itExgoG<$Tg+_9%kLz%*C0vJbE#x|sE3q?` z9 zIwlXA+9%6k5O=TEHi(PrauTI%*AYTW-_W&LZ#15Eg|)_#Yi*13SHzV8rJZX`;$<~* z1vhH}w}Hnu9y|vg_j3=d&6i;_Vjyw$pn}BgOh8kap+JFt$WLc^_vUloy@+>-UWGdm zGPFvVL0L7-nk-6OC39uv2sN@Uu$T2}@~*s#IrhgqNPNC==6O^HJq49QL}J~8C4q~)Vz{3A5sBR+D6mW#ZrUR43n!wOI@PDh zmPR{i7;9m%u%=6?7Pud=me@ViTAH{)7f;Kk4Yga072)GX%CJE$MLJ96_X_0SX|pm2 zfVL|SJ16z&Op~%$`nXc@J^6*~rvpsGFQnFU6x^NY+~FZq9LrMi5FFs44mHXa2Ny+Z zB>#MJN2T=J!n{c=P>WPgpB7u=?R`vZ*~cwm2$suA$2atV^$i=8T8k)G1!ePGJFZ8A`{x~a|SO;ywu?NHaK|k48kYE z9=urBe@UGKHuWtZh>W{8fgU_%Q(tvVP{zCp6T9<2Cu0_@n-0 z`S6B?{JJH7l0RCJEWfzn24Cwv`T4%avllcF`@Sgu(#A7o4aF2$lz*9Dz83(`$!J!# z{d-x%_~-6HrGweS|H56s!2jZ-;6V*rIG{^hL;KmZpVsewRG>7&TPax7*4Lhir_-{w z>B)Bg-L4^xHxUfzY%4wf;0Fn0QqCDT`JwIb)Ej=&{{2x=8)nhR+Y)%hqJPN@Px#$k zJUMpq`0t!zQ>cILQK$X8Xqyz2BpCkWHYnCsec84VbI83B9bNheGj(^Hmz@kvfWGYQ za)jHvb6TbS;N87)K}qnOuR_DdyL--(-Wq2Q7O|pUI%o8sJdUf4#ePE^pi>+b3ez%F zXHi>yq`ZnKIngbB$WK%~q3m<+9fhmx$MwOE89`ec>B!I{T^aSp4WA?q*2h36(Xh4b zQu-Rj>B7$r5sEP0`jOx|E57=Lv_xk;Aj)G1LM4i#=EL>o^x2_H>Q8n#lIYbVYhklq z5Cr9~5vNZWnc0+2jPbQh_mHd*kG$D<_Q6TTS=OL{Z#&c{|Zz>`~(;3jI!FNyC6YI)(bW7hBN1-kkIkX>c2co#+xuCm5>-}+WQ z^MD2euxXpKGwY4)3INymaAwjGK3s%01{#pwGjFnfEL)psSYp;9yHX(?%8~V#JWBH| z&!$HV&D+V6KnUm!DZ$EPJy57z1b_J_1+`T(?2Q~#YK*L){8?Aq6{n1Jp4G#wvF}N% zR$`>3$v4x5mdEA*bD^&FECGkn-!N9&D83C|K&~%&!|0Ff-P7k9m8S(_o!Z8%dS*zg zFB`%J6^W_&PQD(UV>l8)t7dKu4EV}Li+WB zZhc38wf38~dec>rcvl~APPR^L16YbZCq|h~FgZFP^`#B;dp03=+8mAb;w30cF}|bo zpXzIctoT|(`BUg`etj(SjHBMDJCDcX0#VoJd@XAx_*zy>lwt)fvnTN^rxhfo!woP7Y*sE9PezYyG+D9PoSl8Y**#Q zPAa{mWp=(%ha@_WCM&P$sw{T4l98iS_vM?i9x;reXdKS8ZrEi9ZHNNUAO4b|aC(zlo zF(Vzq6zw?8jP%x(=6KH6x+|sZ;VZ>b8gTM_Q6i2=cPF9z)E?&`NsP zdbjGK$;J4K{!+S+^`^5Pbi=}QOk5{Ns{UP9U%D$iM%xH0SnJX@F3(7gULvo9T$g>S zPxvUkA$_WcXBdfHXEFEQ!LNTweO+iL{2CxoYJ-f-d|Dn0ui(+v7Yk<@o%v&K3D594-~vfe{YBkl zM)2Kmf1OAC3L;}CuB2!vRkYs@?a*5GG71XxVsN7P9d_}D_$BGskr&-J%Nu5K85e(a zty6a@lhXrkJ2kLRw|Y*Gh_wu*XR8b^oxrMp(HsE@g7*{K8ONEnZ(`=bWw^SDnk;9p-LGE28i7Y558^J^ng<5ZB6} zbVlHo99{oqnHdHY(^@uiQxuq9hgeicl*uErxLhvhAd^DzWJjnR#3p%QZ3bA|18VzR z@~R;gA7OC1BU`-yrwC#t*|TT`nqGvI?hE#UD6>Y+}u_V(f(5#Pf2vC+XN@XhgY8i?MK`DcjV(~G(yO%CuOsTji@lSd5FU-|rOg;c{S4VF25hsG3T4)?N zqNorI7wNqAiL^&h90fyH4%`tH&aYvOxXJK~pL-{zdPDS`bun_fxCKZA1$CipP2pQY z(!&D5qk`O->TPX(wJOOT&P{E@0+%(J2LeL5ff&R+x0S;^rKWa2@n_{|tfUpYrW!4< zmnaHB{5(N?ep|$R{T#+js*!aSnj*DGtA9%0#e``$NJ2X9p^1ir3lB&0au|wOL@N;J zDRNPPSbcBHjq&H5pmpOIV9;p=;3#>&+K4mmxd@iRT+-YFvN1e)iNdx!agIdge#C0A ze}lf{yh7*AidX}#C&S*vg%EOUsr~$FqO>V}oBa#7N3y{TU*H4C-=PmF{X(^|z-Rqs zoD%jTWRCfo9$DjYgj;O>$u^eSu@ccL5d2WCtuzz$m8V}RvxfC&;Ibk&vMqHPX=TJR z1bzhKKUr@E0z|%5coMF9{8_syypmXfnF0#nSbSESIB~+y`IWGR<6nsUr%(pv5%?Zy z2)9YjLOi8DP?+rYi{)I$pf^k}=!ZO`bwotz4Q1jmD*}$x@DI`bfRs0eQ04?6$k>Cw z*c^}@abv_skSo|vwM9dZ7Yl)v`9{jmOL^~MrBEx7rK?Ok=hOR?v!3V&keN~e_WFzI z?Mp~diFmkpti=Su;85upQXPH7WtE`ORfMfmYgXDN3|Vl4XqKX@$o77OOiwIcr;A&e zmd>l_gYb~7<2fW3`9(4;_UKO{Cw^wRTN7ZYsCxDAY9UL9-=y*0<3CQx(3Ip+~@I_S76l=)Tn3kICCWB9E0L zC~8~)c;qO#6nK;^I+VIOS>AOLWw&nSxUQg+Zy9nVgsUr$K*3g%TKl!C|~{t-}=> z&Z5?u{oG7k-?-N43Ln)rgaK#}fUb45AMgzA(wFqF79CS**AfX)-sxE8_Qf00|!g@8t`sh<8*23xAToTqlQYFS>|fr=NZ}| z-mTFGKc*SM;9T6v5Nxg{Fg~(1Lr=b%i1Tw%=(rfD%IOA#N}{@!)mNB-dp}V{-POrg z5Z0yGM{ zM0yoD$K67f$A?9|>5Wb?52V8sEt%|b)}_6C`;2mojCdj@DOrAxbve6qwK0xfq$`$f z)7DZj>7m=wz}7a*QLe;f0S#u#FuFMe#6{5U)qLHxi? zp|S@riF}d~nmriT9Qh>k9IK%g=tu-`QO1(}9RFD5pqQIQhD}_Sx-CEK+fo2qyU}Cb z;4yD@1qg373#~HdLz2&>d}oDRah};cJ82VCN>K;fYru40&X|NjDa&24n&r;o8){RP zRYY{qe?kV$<#^cDdeb?=)b6;-S9-rITox$(vL{?D`aHHu)bh?A`99UYT;=Q`8ZkV^ zy1qwd-u=SuhQh=rWzvWo(y8e|YGMmi zjFn=pFowAAkegyBG;ap#^F3N~rcefnDZ^31K%Eku#RIl7xd$bL)N29&Ez$*K z%NDu>7csd2i?e6x929nB{1-f;JrGphHQh&Vlh%ZO(gA#ST$;BEZ>KMN%!X`j6m3#Z zz+9IX8o{10&MN;iG?y=8w6Pb#5xeSDbG3~@iP{VY`H~SR-5W0VYa3?e@iG4rXNP>x zw=b7D1;}Q=5(aS#t(ZpiuAzM+NbU!th zY82M8R2L8w*yduMa%d{Z+}l5Qy${u z3Q6NIwTF)@8~M2ECO)qIH*3ufdrF8wcGuH@{KdrvMe?EFFT@od-VzYI=kkm`B#=ZD z_ajiiJqsvE8cR|TQD%yi;l9bCFKMw;#3>>3`m*E@Yt1V{m$wWmB~dtP&3%Fk;1Iko z+7tqnKrcZn)BPqLkP6EtYmu0gl8plT{>OB9^?$0H^6KEa`XK$VnTM@`{M>SPcr>b{ z3!^ukX(VIIaxzSebBFIvoLBA%FX6|9p742c&l4Wb(T5J_!k$7CD_4zDf{d=AU`!2wx)15$75EpUY#2f&lneJBZ6O zcKC7rmRfPyasOOu60_kMp4z*I;?uCtaRQt0Gk9W%PYbXoj*T!Cd9ctnlJJdyCvk~Q z(PgdWzo-V%ywDMz3K%%{1|7jS=Vv%h5(|fqA9J=abRMEUSkf~f!X|f!AUQ==Wf{3d zu<{O!EpewTQSL4b%Vy%@+oJGZs@C=Bn!JBVogx3yG)|}HSlr! zulQIbLHc)mlaJ5K(>qJ8epZE0MOlJ>o0nRGif5*4KDzoAH?7`tN^MP-`ZiO7`^Hdy z3m*?_Q2b-0rIdV!DtaS zrP=b(<&d_*dc(_Qdc9Gb!L_8N7CC}2nBJJ|5RSBIb*XJyb>Ll;jms<-x47yOJKUXQ z?|vkwtRZi$7M=BpdE+?v@tu41%&HPGqS)@V{#v|nOYm+K7wU`jvW7zKD|3aAB;nP# zr)w2map6m0WN=j|*oFIoezI3z-+{+Lbe1!IDbJ+porhr%%kIz9qMF>P^5WR3HGh|$ zJ~%ScpP@AisN;X*1AiY(ayvfuI}eej@nk~|(rISHWb1ZYgJ6exo3nG^siQX~IUmuV z+=8q%l_V=t{v!RNt~ngO#{Et$ zc&zD@ll*w(i*Q;GI1gh7vj$uL3mCy9gl?f_nw@x1+c@3J3-B$8SIir8+V^J|M|K`j zJ^fv(>}0#yi3d%J9)RR%@^O1b!VXl{b{Lj_S9W};mj>niks(qh38W^4Y-pR9K z@$0TBZG&)>thkTV`Q!p6G9y}>G4n$JZ`9>AXhu&e(aDZ}XGdbv%nt!@!*K3N@0`7f zQQS#t4Tap1HQK%}BN~{JEb2Y^QE7j}g^@MhjIhU?HPxuhPh`a^m03!>bMQmjV`ZTX zRCcbEm5!Xh?@@=$MLapXSnx@(ER>(K0*?&#%uU1X)~sSyGLS45(P(USbZz#gJoN-k z5KpR&dmTX|>?vxCtQkklGKYh3_+s6K9GT|qN#r$_Nv7>cXI|om$O6*vTzVJ31N?dx zToS1T#n-udk2+So&S`cEO9Y8W>E=Q~73(8W*aMz}Vu-X;x~gpI@Q}Pg$()+fJcK*^ z)9n0-?5~NxzLZOEt*_e3enuL`1MaBxUD~#v+%xbb^hI-Rj&&Q6k_UXxF$x|7?X4}sLRHK(h{b*(wlM* z^2&%i1&ESgJnBA!rXP7)YkrJ-jn`u%Xq7&EK)V z;g`N8DfQi#Qz$X1amLl*!(HqnxZm#;pqvokL{o)F=;WiNhy*SrhVdnIL<)XDVoCpe z;;mj5x>DLNV$;X->a?gNi$Uw7FX@4F>*l!El{1H}3%F8SPgVg(Yn`*1AZjOH8&6*u zKEH7dX)s45+YI-Qs4W#^R@^%mQ(Wo0BsYQ~GsbuF6Sq{lS0~+b?yfHFYWRRH@K-$0 zHlrbx0~kjCfM1a_Gqh&m4{iU+ky0L<3!^5(wmf1rMZ&i>mO4U6gCowbC0^BiDGevC zFx}(1Qc<9E-`a^X1oM_%;EnKa)M!~u;Hr6 zsU<7jT^X5)JPtw0@M(|7pt#x-ZN9vI(gbH@Bs?6J63*(%aJG#6t#csTiKt6qSTqM| zTH6v+1#3t)zf1TC&04||#1wJ>KARI*j_c09h2Cl`yDGdZ+1@fo_*|9=8Yd(L_IJtK zkk&a#rD90iOV`wDd4n$^QX?dz|0ZmW&|^`>hE)sUETR*ip@_DDJUOUr@G1*qncHK{ zvwN^t9*hv_S)QUj3TNofm=u_97kEiMQ>D~scq(^B+NDB2k#T*WS2469k>Nl+?#gg9$Ov$Omy9TGNEA+uZ5#{8BAqPO2%UZ7OpKE(4|XQ1W0w9X?vW1{7q97zt-*5|2h9>4M5CgU348$ag=eJIn0 zFeui@LfP(>t1D_|JSY0SVM|OKIcOxKp)27SaAVzrua(Q+a+Fittix;|0D3a=Q;xO) zubsCba5H9vFA##ZwIz_xVZd#~!{BqA6@DS{1%G352=jM7vB?q8L9p=rYK1ne|HqKT zsf?q^D3O>d>Z4m4%VvaUNeQBf*Enh4ddP0FNSaKxrJ86g&ve9RKz9aegwxn#%~fT? z|1DJwr7AlgHya`@O(%E9g=PhtlemP7Uz!|YBxTm`;S03sIlQbKR}dLlbQm_}tiSZn zYj02mNvMqelY*D<1+_pd)wSyA^jXMWF6#|75Sx=y-Q0?QrE8$Gz$5WrzUISWEEu%A zJl*b><;xn9g@&ZRb7Udn0tzhDpfH03qT!@o(5dL=9(4#;Gjd`fuc=~?k3M|KiQoe? z^%X`S#Iej?`SNhCPup;qpuWhVjL2Dx{UqWQGevH_Ulc%z{WeIn&25P?;q#Lj;RT|u zEf54jWrVv3T26uJQN;A3US z2n3B!;e*wL>2i?U&`UqTgb^xu&_$m0Q9_c3vkkRok7>#FCXRLLr8`0+e68gpeJ$nU zM3p!q77tb%_lKq&s}d&$?|r7MU66xlT^cVlje(KLKuKFD9w=z*lvMf7(|EOphWSda zll?rFI4(Dh{!WPBCr(1xO(#lf+{#vDw&Y~#NoU$gGGzfdM?OdV4G`Oa6j=h^uxE>| z4FOeYnf%)3dBo|HnJyFDIen?N{#$&R6P<@?hzu`%`Vx*`?a?-S1JN6x7wiX<;E=P} zfSiTwX3P49MGh9KL=mqi+|P*-oW5>vJbh0nr}6Yde(mwC;NdP1;BYkO=ETUNUbe)0 zn3!Zo&60zJ!unf-j`F*;X2JM%SC=o;n#q3TSeH|NX=q`2t+xJJdK1~^jPIo8@+G00 z%kQ#jdwe_(%2$Mzts7N-f9R6(<#r@qe2Cl*Yt1i7`BhqT2iN#T!F?_L`A*2* zgivPn$#((_y`cu2xqOS@{csuPXxdBT$cE*W=a7W06R4buRxPp?kCYQ2LD5fFVsVmI2Q zLI>FgsW4oq%CnE8%3IG+jsm&P!Fd_m8`+l^=~!qz$um@R6C84G49#3Q_}W??N&BP@ z;lW^i()1U2me?nv0LFR5B$n9_Oew52Kq2}q3Hwrmx6uFrpR&X-eyDBE@miPgx=!G^ zFM|fcFDsFP!{CVABIRrvcdU;gbk%77Dx@@C5z-px21V;c6hMoCN*42s;`1PB=c;~j znBL7mn%i_k)HHU#NPrf)PBLHCO5)4Kl1F*DV*daC?3rX} zRo-3TkhrLPB*`O?%g*hl_wTy-$J65fu-YBjo=n^xh%R-`S#ns6j9;K6IE#=N`as65 z^1n*bl=tj(6EbM2?NjxiGyFTKmpaK4%Zc+BqtDB7w;NY)vQ`a{4`1@R5r}!hc zC;nP~MJT)cN?aB2w*I{{F@$RnMb*jnqiloWnT@4|;i;`RIoUUBmb6v&1>BCj(2&T! zJ!&IesI5OD!;G|N_@9#jC$fX>MQDxd|rVfyAQc@vpe2(A1z>fftAdFJF>5_C%S z^0!Xif{`qGJJ4DLi9AxXWPerbU6(rieOso7Ci3axaI?dO#1A%pkQeg!BM0_KvB=@; z0;R7!JWM!%)w=PyM5y`g@B*3)d8fzZ@2s=E-SCXALAMPeY8B&dYT`%DLH$dZDFMVn1^<&h zV@Oyy1{9l#!)!lz7b|9?Ch8g)%GEZzGzy6a;0#!f&;j7;l_KfL?ZNy(%DWoW$En7; zi8c{`Q5!u-faJ-dx2p?cHKk`7E}9daJ`PQ(ie@}%Jw`Ev18VO~>_w_0bNaC~e&Zrg z6CK@mBht9myiMEO9;$3*Zw$=@$^1rpWuGe}FO(H|Z%;tmyp8{kn&|j$N>?u-mxXJ% zJliMFI)bI`0|R`<9-#l0qHQdj==dji0S*iY$&sPDK&wj&Hn%atw$SHlsK)51P&I*2 zd-nuc6_glR6TS2cyv5J%>LqcSnyN5P73h|wQMx_!Yw9?^phr|RG8{Em)=cJH#|___ zr6@6wS36Pc_V`Dzmz0E8!)@s5qF#kndg%$yiZj};ab3V-y{eXuw#S($tcGoV)77XK zbtm3Fsb=ESd6n|atIRC(^Dgs7!Y0>w0_L5r=;M=D{o>!=FsrhsRe2^=xuRbmJAY%$ z3YhbArd4HQnKwT_&e;PiFl~XWy?bnCm1|)UPlYZv z1DC%jM#iFm`AGJ}9n)@fRYkvEcI4F;+x^DQ0Nytqe^IC3*&Q(6449wiMAB}}kMVe7 zdw}GT*+uQB1!hIRp8Ff^;g14FTl=57XQ0xmh-vX>9=F0L^51twKjmp*k$8T&279B#<&u7s2VKDkk@kGxv)qWfXXx?A6 zm9~Qu_wyy_{3rmqkTz+NfBW5{PYM+6o!G-izw=O4;VYmP3cB%HVrUO6St(yK!lhUW`RD1O5s_rv8 z|N8gsW*vCty@~&<-Sc11x~}sdwJUYA7E%x&C?K9jk4_Va?0(sa>YR$1Qi7hc*Na2_n` zp~prK{q@odSz%p|e1Edv`8vI&H99Blf`4>OFqswf>(Qq#|LH;MZ|@!#C^|uZi#q(y zUGV%K;4s?n+zL`%v%c`?YXWe=Z>r-dlz~eSNmvtwHH~bwZ_bQQ)}%@ z^MB1S4hEw?$o%D`$N!Ta(Sb_?mlS*$Fb)@WoZOXKeJ)ydb`0GWj6Qx|=}#W}xM*v@ zxu2p}JdvDIX>_)~a^cCHG7rvOLlZ?i1J3tlmi{!xfATeFLONj_8oHYwbish8><*ZV zK=AHS&fP;H_dmGk+b?H-6SD7E`{B`K(OUs$FYOd`-g^6&{KlK@+lTv4?%)M4zAQs` zKw^s8PqM60HFg*lB=MJ<-QU{dAOPZot`9$%C&vQg@QRB(4C_R3qX@y4ozjdsMU zaD)@t2&XDrtBq6GF41e>gpoO#(^Ey+jpUd`AdicR*L9weH2oxZ~oS5VM(7N}q@GL7Z2JW$*z=As~`tYUrKJO23 zXcV^t1y-d0`jr!5=Qc@|;?+2VS8zS!jq1o&pY;XaO7zf!!$Ty2UHg6{gX>W+PF*<2 z=*5Tdj%|9>qxa?3Nxm}UZ}geoibKc4W5#v`#^g3`5{pDZN91%?!^JSsFC3zCwo~Al zzGQM!2~UyzjiZU7o32ye3p%VPD1^1lE@xEMnw2?5LPziZgx|#m=u2uWBgQJ zNFzh_Q50gY74^}h#DCQ%{}g^(ag_dBRPeh}+eOw*Mh(qj90=HVdVZg&vKY&ZJwz=7 zKWg|;eUaa z94C*fnm0vMkN2!^3uwW;mWS7h`lQEF$!R!VG#Tk;xNH5DTdLB5e(jPwQlrYCEU!sI z(SaY|;YU&LC*9AA5?c%O3_SRul(ONq$GV>%D4hOV6~-GR@Af;O7s@V*w+7`izaIm^k0kYy85*x3{|X}@rkuQMogTgEjEL>weKyEb%S`zuNHn>G z4x#7OzVQ=*LDW7b^4Yw|vxjkV)D2ta2WT=lJvtkWhiep0yI=wF*r)rs8$P5&01n95 zZ{0`35*q{n#G0j$^Qg6uugJGfNzlmBJZfF3o^7q(@i8g4ZR8ir(m)w`U-^6*QYQQ? zPS-U>$HXb79~FXhIlb-(uynss1qA^Q4`G`Q!POd_`V#&z%oJm>eCE#8X25NH1PuM`Hj=shIdfd$J&qP zn$1m)Y@WNG^S2-LPMhUI$J;xt`FjrJ|=+pxtff39NM3kq@y%r7+AV17Y?IbJ2J#duvj)DG;eadvUaB@*rgj>6Up zFv^|ETv=(38^T0B8)PDh(|xudk)5iMIF0PF8~K(@4~=Y-_S9UyV93NIen*w# zs64VSGc<|4w&hgdnrx|biBy_w1D)ixifQGB9MQ3d0C`(r?lgMicOwzUP2}YGSNTwK z@iaZM&gTeC<)S-6mx_0my|TdA@p|cua4v4HS59Y(5M+F!4KcY>73l0VHQHD7lCd=u zmiDZeir5%Al`*T;H6%E88@x=gq&w_F*`)<>Vk{s@fKmhmwh^)NCqt8zNhpFORkG}{ zgx23#La+_}m=sx4?g-B^XWN~LRR&nr99iU(%g((t7 z)ktpm4E2RB@Ru$^xsDIf+FB0(u13eJOS3_wGN>8P*v?fDz5kB*T)Ln@tolJ(F4;<& z#kZhwMv=DuKlu_lwOm{OGrrh|Z_7kz%}?@MEd`7eIa%A-o$y&qOMMMs-Z9+tXccWa#1gz*sM`1KAwBxgJW->q=>kjQp}XH zNvtj5u$fffV^wPAtl@idc7k&+#gxzfr)a<4HkFP8qf7$ zs;mux7Cx!0`olBi@# zszZraNQ_Ac+IjRXDK^t!GSEUU$XET5Ego@Z#Z(&kSX=pKa-`;NuR{x|ip)S_~MZMEpe=v{{KGVlLmH z&3HP*d^*#c?893zG)WTT9ei#A&+s=Tv99M*iF#oWn6yqF7Y+mG8ut%>4pNlh+G$vk zy;`K72&^AUb?t{)W`|rx96ymryGQjn)0kZnIi)qY%Vw9Xl$XCPFQ4&}pIm5x3}olt zq9m!aE5y(L%@TMoF#oMNMD<`))_R>hyeIb3(*5Ai1fr@|w#qL2FcVwT_@b*=X%?MoqWL&aot;tiM zR$%tE+{5&@ANI(XJ1jRw8tE^vzLt)vUM420F8cp<7bmPFnX%n8X4a)1)udlFSqA?% z&?<&k)z%`l7h-z7L!P!DCX1{^3XFU5@8sU+t%v&uI&$4-I`S-d^RHo*?#O6Zwp|8B;5yIc7>J+S~pBmV43nSdXsoVS~iv~k_pK)mgcFZeF`Y} z*!_!nSet4(fg`33R1K=Aa6RExv%(CO(kbbW?OtiFSmZD5GmiVUS+BC~EoX-W5qd+{ z>gGcd-5a{ntbjgjLvigZIA%VWABeh~#)*Ose<>Wa91gUDE!;}}1)+-~s;hJul(|s4 zYHX*FylcIO*eWFiGzGI4k?E-5RlFi1YZifm7vV-2t6ao_;NqEK_mIFTgBuE7r$?|> z+N5*-R&uOIN#ee~7p5*o)*tgh^2{P9nU#ymDhgK&!^hC5TZFU95OLoc8Yn%!!ew+P zI#l1LsJ;=Glj@u3(rZ))eW?zL0;fiLsX7R;Y732tKScX!m~vRKPjL=|Jub6*>^-N) z`oHXP$zYFt|DSvOjgM@Q{=*)3!>OFxW3}M{+;X~V*5&qfWrV8FLCEh>{GTn4#2X4| zqj74^0X`fA*I}{z*pQ{M9ahROR2&p*1SrWi&H^rbo z$Dku?avb3)Il4=T+OBh2-pohH_1!w2XVvQ6~BUhi}h(>Br*vp zvbBOL{@G9GPrRqWTQAW>TL5eBRx!~8}!)TIW+ZB2fSa`Lbm-c(DX zMshLx3TpDRYw#-ZvXbXGw?cK|bD@xGQl1v4{eRAWj9^j(={WNzvTp)Ia$~|A*3=<> z)ySJ{@OXpb8PtAwBwPN-rqdVmz<8{gIWQtE@@Y2m*+4Gzi`BNDEiiD=Xe{z|9>^a+ z2+C;x(8v!{-Nwo77D0&=xTU<^$$6jRk@o4%U5PiKbF`glJSG+5_`dZ{cJqy{3fZy_ zXGZpA)fU|)My1dnST&S`JYE* zN%}XzJtR%lr_vKzM+iN+sHGjL6K8uCrlBBRZ@f^aVDnW9AmOypNJjY9VCl;%>{?p}YF+y& z*#8xeio|X>cq*hek$g5X79&m?LExSdc`uV$$i!4Z_&$u@kaV(MA~(c_TdE_^Wb-W z?A%^@VkIX`yMOaNY8fO0365xpoFex0-{3OsS&2U6G@chEt2df$puK~XAv8m4K1-IK z`pEN7!o3Pd`7o7f&Ex2pjYrYP1Vv1>Fj7c0fK3Elrl^++V>$<&YH*P9S>qwXHcr)G z(()<9@>75OMG;O+g?#2?f==M(!>V@3AWfsp-=Xrd3Gp){f@*s78HM3iH^lfHL>(dZ z-AjFn3b7)Ee3G1sJN(f3;g5XcUin}XKJ&8nS1wf>2r>uWxa24Jhx>?ZA=YCo5s|%8 z@)+V*&}{IeMX&+UizSn-t(W6!W0AC4ApR6TSpt4_&W@}B`x)bK_;(^I;Q^plLr$Zw z^+9JV$o5bMOzZ1}R;1R%c2^oT{$9FDwBr^Ndg@qsc{JN=w^0?%xe7P;Jp;A3>NVpr85w9O zee|^)ccy>ythvotvnZSmu;&2m_c_1+-$Mvl>S zbN0gnbFq8?Mm9b2wJvwIdbN$K1T0;HR7sL2{Zp#c-q)ePk7F4~<@hZ4x3{zl_zw#B z@mP2Z@IR`+KWYH{?StTdm$i59I2HKS7V$@WbZJ^89Tf1ZA=~7uufxZxEhVN?TTj*~ zM`WL7tXXW--NpE4GycB=>I0u?EG@RH_WmhQ7XwsktdSZ)cnPhZOhqi~+=33Xa&c+T z%CdrI->(e*lan01Rvd$g(!(gCww zc~Jl}Z$yk|m-cH_u>#sDs9aRk>ua4(>>PpQxT;@(6A~KRL&3g=s`Aw;43sPVy~B~Ne+|Uf*N?1GgVN%8b5P{(rdgFCZ7-FVt`qi#k>C$ z1KyCaUqNV$!3ii;lXYk1WW{7pLG~h1qqq%=q*u@tS(EDsWm!Lqi@MM{AGtBjjjHmC z_!=d-Ti83gfU2XT0-drcjlY zSwh`ip#p)q|B3~9knU=;A=lb#DJQ;B^yOyCDdk_hod(T&^2-C}T~hjfPwXhqEx+_S%? z6r>5aI-KaX+vT+3!FY?0?CSDVbtqWFtj^m@(87bfoF05QbeXv#fWo){Tk#6r+J|M* ztP7ZHd^Dl!1#jriqt+|jlvrBLtn~Q_f=>+bzaYwR6wBh`p-y|SQ?67SXKn3ko%P>D zWHW%y@O>P;RuWkY8V+q*tHitT?Wr%JN$7)>34-vbKI>!kN@#j4sOI>2QSc_Fr-ZvL za099+M$eeFfnR!AFfX)OF{X);v7kEjHsO9=dX_%v7-v{zdKKfscvXA&0x>M~Rd_Sx zoaTQkj0JU>*#X6FDr1x~c@qfcwGJQ%$zgz7wDp1mk+mLh{Em|Fu*jN)S=v{B&pr5t zIzfT?X5~WgQ@#`3-X7U&%a=>%JYs@OJ3 zAC*1GiCe#@a%zM2BK>~a4^+wb~SNO^rGdx+0`?oOT z@_-YYCetZF*Tfim-V!7IF9;UOJj&_?17*FWkRnpJR)fne}tAp-A@0Zmdv zWAd72&av%Tc5xvsI}fDG;`|#PtsY>&mYaHf0|ki5*tDE0&LxlyYYb^Gl_4VKzA=Q~ zHAWdB&Pd$fQEKzjo(HehqtOq9=i#ucDI9X8xAzlJ*_=|VBb=MT<$r%0<=c8;ms3v& zjTn4Kal4^041j6V73+di%Fcb(dQv=cc$Qvm+*!gLsP)PmyeN3Cj5N5Sr95+#Ic(Yc zG6j;DlFWTV27Z4w)|fU;;iKP~cD}Aq1;aVy4`jA*i@(9^N(_&D(R(I6Pj=EtYph-H zIw__vQN}Gf@{Q2OUNTTS*Bct zPv(V>6}FXjtQ@Ir%$|kuwmuoCDM=jho0vI*7yF-QLq#|%afG}@B!iCDx~h%yx*WbPpVL_%^9dXDk8k^9%E}jC4Ff2#1ND*Bq#Vy8rkEJ@ zBOLpyoKR{AuO-6)&#_ux96FDrOH)S%ogV}X4y}&I{k$uC%X$9D8K<`X8h#48PvI)D zpo=74^ft+savM~ENad{=WX-*61c`Ao$x<9N_?oJETzG%sw$kk@FNM9&pIa}Lj;_!) z-gDQ&K=fW`z2C^byMC6zPyD|VZ=mlDav0nurRGRF33E7&V&MljG7{`z(b4DC@=Xh3 z8e=ErcgMh!l5?JT6A*|Rk&s}^{4ZgcBERq33TII>zg@aptK4n8X^hLSHZR<{FOqa> zj|cKEZ907m!&^CuXrxOWp<%XdxlDM{WQG*Dtm!o2AN;et~wHTjprC6YqJoLa4Id?gV5f-_k9@q?EJI5n4( z^g?8#E;yw-w*_0Lat0jymy{fta2Fl+|8cR=?pNkJY=8x){5vIB8=p$j7^ZITJb=H6 ze<#ULLitkz#$EY^LF2=~6?b7XBRNnoIz5{=E(iE?qtpKoY`MzcGFM=KUq<*XbQ=Se z1dQYBcZ6I!;)#Bgsl%gpCnxTT-ZH|0H-4|Nm4LGB`ee;LCE+oklGc2faSC@vMiH0% z&nk1}YhTR<4#K)j|G(_Lc_7qZ_c%Udrx+9|swqoRM6#A;WGzdgL@8s8br_62SwdN& z)H`XV6)mzview2%WeaH`C6P+DLg9DrYbJT#Pw(gRd_Uhmf8VL&-t)Tm+&y1V)P}D_W&X`69`+Q&q?iZOnIKbVyx8^(d~JUo3c-|x?uC+pxrG9- zV3r`K9}kJc;-Ycl>O`a_Br2jotY0EdKYkrl2R{8+KUq9PgzFkiWxT{Oy$}7 z&>=#wE=1A2=PQg8IgUY7WHv+HSZ_Rt0d+WRCU~|44k=O`yes&1EG+2P2n1UbAxE%w z=LN#fp`t8krIpxurv^D+Jo-=+Klv%hIl(N_p1-2P^apNeaO*sy;AzMeq1S&m*D;#JVGweVLYsa*ELE2aMmcQ94?$GbQ^*{CzZk<6I)x-K(Z-fVxOL{_4wFhOf36k1N2Kur8F`aUH7C8P@;?DhkA;Q&G|GhUCrSRXSF znjmbtf$~RI=b#HDzYigi#I-?%_&x+%mP{hYZ^H8c_!9I1EJ9=-a?rN~VV0%X*MwS2 z419a$>!!r{0)*)ASOsi^&s@P^K_DE+1e{O^pVFv?!qvn5J%gYMrf+D*H$3ATnel;; zS@FKj_`c8hP(5L$GCt#*nDPCX@%^0ffjn3gAWD{Rn(2eTedx`)Y1|_cESDQcz5NQ4 z(3v_k7hM1fnL)JvPgu)hO6>dKFHu3e0a1Y&3FkMfp>yIoPCzyD}tQiY7-16&EOz5sNLAe!m8c+aaG7}*aDvgg&VWr95Det%xaRb z$rbiF7$n%~!{tY^tp*7p8;s*P;H4of=Y2$DBRaeZUw%eAd*FN<40WW0O+m)8!>BS6 zp<{(G-z}tstvj0ELr5rAM@b2Gx}@eeiDuIfX|+j$oiUbC(EL7e6Hioyph+{#MtfnO8e0B0K9vYIZBf>Ywn?@jSv(Ebc;{FD6%gCC3*)3wV6(J`smE=U_=5{M9^ZY(BE zn!A}y0=LkNjiCVplj_(w3rw0Mu1<p>Y10gOX`SvWWbgA%@7AAJBk4b>)D^jgv|e)TfL z*wsW6xYSJJz5mz#5RyheV=ELGhE9Fi*GvkA&!1yYKQYjNO4JUkQ7iEbQv=!O@GX@1 z)pK-XSHrH=a*{@!KkUnfJvaSwa4hXt{tU@~N)1Z~3TP@jNP-uL0n)nhve4tgC*I58 z%|tJ|ZmjF1ZoJzRxKrRxgFD^d0zp0X~)l+Vy9HJ=FTKK5PLnorI0~UHuJ^4J5(oKf0>!u3w0MI0GE}H{7T@Bc_K$- z-(%<@u=BTkh!&gGE`kKPAi@3{#&EkL4};Hi{ah)+eh-L=c7_>=mG{A|qYulw*kvD? z$(A!al~}kyjR0*NJ{fGX>?@*x81`p!ZQTJq;g)~3Wk6!s58k|^K_%`HT(hY$6zB~%kDA4m z`}1QR9U61K!uQi*G>ZFb0=5Vo#RuyFd4?*T2}|`sXc~io%BF;SY^1wmDesure&Pu{kUG#>$3_Kuo7;T3j8>J@fIq8lr#Lb^r+3PYo8N-pjJFxCkpt8OtR>W^KoPzg$4xC|)$=N70JKmx}J(4hjXYJv^<8seeNE}gx)w_rthY#Z1i(P)SY z4~iPohnrZCml!n$H{jd^FPuJsZx~`8xWqaSJbjNa%#sit2%d1^s4-(g%t`Q8b0%;A zH~pug#+yEyTj4urLN8`Y4vGA0dW^ znX5Vk_Zse;n(o6xlxogTAWD4SC>Solk~j*is6Lz$^Wf*-a+5n|94@YuID`+DH56C`^R z#ghM(?0J-|*j)y23#xN~=_&Riu-6CH7W*lXPc^3zM8WsULw2zK*0&MTtLBtL&-&Cb zzcl7wh4~j@{-xkg5HSZoq`3;xghnc&!$jBp$^yQiFc&hpPlT-dHUV!DTFp=xwBI{X z%Y5hbbZZPr*}T3TIasjAWH? z2~!m==4D)GINVLg|JWQAQse~YXmR7&aHZ(wConQu0@-Q8F%x(pwy3WdMtg|&?FeM) z2zTzo!}7i>@GwGaEtV&|0#J#8RDRWReOT@UMlVB(y*hqyKZuqP77u-QBY0Lk35eH^ zh=XQ5TYIA zg#)pjprr$BqkU-ny{Vg+0sTN8p%sEsf#dZ^kaS=YUBgrY2;2#G36g52^9&&0dcN_NBwZ^ugVpO*L(jy zGBnHluE6!Nnkc9BdOUvGaPx#NBqai0mBzvOe!Snoe%Nb-t&*tdLt2m{rt{uAkRvQs ziuKl-FpDDr<*4So0V#oOE`WpcSMJ@uV+vZYD6DiZhK*AzybzIACcA{-G~yE*ceVe0rgkdU>%|L5XFVJSx;g>qMW^Ff|)EY#8Oac0UD4K##|Z1ht!s# zsc@po+bQH|HG|W;=`_YMs*Dd20o*^(dlX{#)kAI(T2xjR-TSdDYy(hif4GrGak{NmriL3{xrC%n29T?vuwDKTT{K7J6 zFk=XuoH;kd?}tG{-- z|1%sbTiKrp%L+eD5~QTiX)7XVF7gxng=J1C9a;~ENzE>pIVaGhV1@+qV&lke*eV8h z=3@&aN9%E0Sl&~Z7f7++7Gt;^2|lX~2uyC-U~yJz0TdiPgSw9EMUf)m#2Ng=0MM!? z2W)7^3r??=A`r_6cfT$rEH8^|MQ_Z?2x|f7At=TTITs1ylAb4(CA~@NO=^g1m~I*A zpG1k^a7z<-a2}sD%y8`fHc#9oj0BfBDmRj!KPIKh*tJ%~zS$_ij#oZH64vDKts@Z> zGs+K^+j<}Qu6W3M2}v+^N!h^-(PRgMW0lLtZ>@F}(vJRA#Y2j2B=?zY3mhGgNxRe% zy0fLFWm`*&mgW2l>LJA;fm7dpW)eHTeQ3KD)*^h^J<@f^`mphE_4(j)Ppc$KJ+50z z>`ZbnD1TV&@4T;*b~JR|nC9unxOqc2uU_3>a45OzFHbr> z!%ekgTj+HAlyaHc%_&W(nEZDievS_=)Yz4@!tAB$F`ulU4Pu{ff1Yv`+&RsN*zI^3 zH+{V19`sIB@nwB0Pp@8MbMyXE>3M2b#q!MdR&?=*=y0yM1TXRo@~DR{x4VU(N**cx zsQS?1md$9(xypszIe|f)w>76uJMeu?A3N>dQ`>L%?4!l_QPp9>(o-uedd%ds-Y%#W ztqZ7ITGUkEuc>3JS86_gY2M@f6FH{)rQ<83qAzY2*=HH7P)Mr`ulM?~r+@48yP7w9 z_ym>&9926rJT9S~scma6L=@*gO!2<*Df+#EM2(%!LeWvv^qTmD?EcoIl9iDRk3GhW z*oTih)qO*~$R#(H_T9;Mc)M=5rj6b+$G}ll+*YaM*QYn8zB)P0mGj9pY^Byi`H=JK z6AF`innSLd?^kslQG6$y&+RgqNo4ELc(+EQ?VZo{BcjIUT`p2fvwRhQoUIoPT&mL{ z?|iA#ts(U1cAk?KM~Cn0B%Iz{CvNWgX=3cfgwasFT4H?W(Mmt(P@A5(C-iHR)BvsQ zeWB42Jt1lgTl(A9u^P)AZKrJV(rouGkF?%+V00q>knUigK;7%r)Sb1IZ8!tHEko;W z)_#?7vUoUJo$vZqG@;>v%>msHdt%c2M<&v(cgoTi-^m%eAXir`Ozyt4v?kFb?Nzk@ z-oZnZGkSU{bItjs9fs1lN7=^KrBOJ=zO*0qi$ASfsw(a9Goj#B)*gw@#DkB>!a;fe`CR| z58iI|$1CsjD=erfmKa^UyUn*i@59nb9}SM}W0QOhy9drFWbcvn-7(fq|_32&Xb%r62m){S^?S7VEFj;Ymoo!ET%^s`i6%+Ckm6Pj# zG)|MJqqx(wTz3n$j_fIZGrT4IUh9zHI?W^OD~fx>tyXtibXR^qk(+T&o3byx^WX&O z&=2pDo0Hc49@7FFrU?Q9CG8P=hf2Rs;+G_Se@hvC5_I%fV5G!4y&qi@`!~3TKba^Q z47lfPP(H>VUJ=6g_U7kNyu@MYFKnSAM2U>AT-hJgZnh|Ys!z~aSL^x53iYSW-*o?^Rt zFK|_rk$2kb6DOo7OZlHSzqfp~+kU#a$*EEBi@Bn|+f$cjhf$pOtHO&9OLn^^1cg^; zXK0+O8Y{do?%;CJv%*$K%;WWs^{OU&m)Ff(`=Gi%@%`b_GPTqyi;2mA$UB)gdYrhX zZoJM-tf-cX4aX0CtLsI!%_1(oW!Po)%&r#udm+7{|NR|h{exw$g|}r+)u~ zZ|VOxdCsO;p1raybxpk96Y79uvc8(nn)PO%>VK3Kl)Q|Ky!^7DPrbl+cxT0SjfD-I zg@Zzawx=x?-n_J5=crhu=Zluz64&(D*e|AD<$hjD_cp-qj1oSS*KJ-Jqa^kA)^tnd zzPa1zaeIgS>`Q%fQ$*KK<#t4<2-iqwaJ@SHmyBDz-{9%;rcHzA%H2+Pzmv95KVVck zP|$KHq55n&fqLe8;N$airTp1e+0s|{$wzfx@wyuAx9Qmr(!Ez%IlBt1grZ)=jt5-p zbpEo{Qae>}?JarFYNbq_Wj|6Mu3*pU78oe43AeppaX3?Jc+Q(@=?uPOM#&j1^!@B_ zEGdt;yuAeYTx4bE|6J2#7{hmag-or?F7;*N3ZLa3*!}ceZPuB8{Omk_b*m1_%>-)( zA2l;Ca6>Vldq7Y2XCsB$ipcdU`{&23_C4yvK4^Nr_x=UEqF%{n=S7*xC&n7Q6E;^U zmRfUNw4~LDZ5?P`*n6~5FpIiPitA*n9MQ)nMnzzxsIhphce=aa!-l3J(eg_Rt*((Q zd1z`&PWC5tng$xL$TU9PRZx<%j$&6^l|NTPqb{EGZqwE0=8SigK#NCbc3v#XYaOO5 zxcZ3Eu%E-41smy<%R#=^zMh{S^f3C9hSMS*wwgmLmrx`prc00X-P_;MPup@y zB)B^7`^E9nRaU<5hJt*m&yZz4KGk0Fh`XX}SxwP_#wGo4p0w_J@iC~;>r1%BzR|n% z1-@5ZzH(hjeR!d2AB8WQf4r0+fNg%n%C%cdlu|4+H`24J`I^l$aF@A zFl5jHi@iyO zaDEmpz;X#J7rF!Vi-41Gm!EO!CRMdlv-A{MgTMJyLK z2cTbs1j}8@a)~UL>HP2iB6%S|@*g#gJ50g@L2BrgO=UI=(_;W$(VBrha8BroJg z@q;9#>6xPrI^m@9@8!}1~EVA2Zt3gQ-Ft{6^yRtZokpa9s9>7V@<&-Sms zxidf3b202E_=g`e!b;{R&W2TC2QH zMO95*gX~CgqB{SbiQnNQy^Z=F9j|Yy$W8^*YYN|z3Id%(?Z^Rp}Aq)@OR z?-q13?It<2`1-h9XzXU+VDyY*^JENvlScvH<%!G!6+fjK*ZS}3 zG^poqePyfFc%+{b$bqVOW$qbk42>kp|N*5Y_bAj~(PZ zv!|8?ntr&Fi*FT+pOG$iZfZOJT+8!81$r(x9c@n&GtSrM6+ z98sdL7{YTX@1oZ`{iW^h$)9QG9x|t1p!@7J@39rxIKJF--23#85xzEEa%gkLgM z!R2OetcOOXfh5mr`)4o8h5+a^0dR=aTd$y0q$B*R96`=RRgPnp1p7lx_JM8P08`>+e>r zoO3VlsNJnq9PatMGN#)qlinqPN+ra`&o>7tTGM)O3`;wB`Nnfmfl+I$zFt*3^7J@9nd} zvHaJrUQsWWJ`pXA+$nSCTUTHXJ<#a3SdSe~thYO1`q6qW=UsE~LeuA{huyLr$C{@- z$OiQ^(V(|{>0g@o+pMEF%q#NQHOMk98Q(=68qAE`OkM_hX>W3G%zkCp+U`83Q}^ro z=W15jC0SMHiaWyV^NjkY3lt@W?%j*8xcf-)SiaSji?^gWRU7Ue<7#^%YtvzU-=$qr zA;dYqXcPGXo4Bj}U#sYHYNc~-ollxuDc;FxmpH&C|Nq+m|7-vMul@i3yZw`t#24;& zR9#zXCBKTJisoc661eX`hTZm>6p{r`eEUO_;)=%j-hqj__K|mUEefuig=H5c$w!^5 zmqsN5bhkx=>Pt@WNYN>NH7GE5l=w+GE!fD$QGG5 zH@-CJ+>(zHPNU`V9=IlDT>DYPszKH;?1yV0_3gJ{oi(?OrBwaIc^9k533CjUSZ&Tv zG~VjnK@CVk?)h-PVsa`pxuHcXz3%aZYJ5~Nck-zq>ylgzYDVVf)Ts4*S*gow zCkl9D>#_wc><9J;&_)cO=(Kwt5~%RoC~P;(CQ15O?oBI?AO$`*YEwPI)gzx&Re-x3 z9W7q6D4yiAv)IlxwKs5Mdppf;iG{q#o-oz9XXM3Kyq3Ue3*%DnSSdxNA92i1XtXLg zDq1!2+Gb?n!Cx5_11%}--}*|pvojMVZhLenosk=mi}@Vv@#wm-cQakweoR5mu&Yb8 z!#SP1Dx6~7YrUL0Z1#RQkL^-LmbgoDw4zLUnru;aRrh#W$E4=t;nuyAz4av?SrL1ujh z@OV5iy6w#-;gp+mBohlCdZ(X_BNay*+Do!?)6^F|&=FCJ5fE6Zmulg{o@E_YHW9on zvd+m%`lkH&>TaL@O?R_Tk z*l}KbqWsq4ly%9y#Y-F7)44tkhwZ*mK{fJ9Hm0pfxAN~*71qq+=9P4ZcT%gUIzrvS z87Y^IOAb4WCtq427k}?fN!`%-#D;#yj>$5~fu8nzT_y8>He~SYM8(U5Pl^^TG`%X$7L*u`idj;O> zSv0$CsUh2|OkVE(iaM^m&_s?HQ47u@%XtEa*EbScV@~jQ4Q!k1G`@(}>ezE0KKXNe zT1LC(cwdx}Fskidyd?0#QrX20ORmpZBYv9vc~M2@HL=kGw}lUf;C)`|_jy-ja)pvj z8G`feHi$?nPR$QVDiSs84q5QV&`iB;^}cVg3l6L7k%GGDZk~o;zy6>wXg9)%17RNNM#((lAcl;SW&xEUdGhS zQF8B!^ySM>_7FverIxL{6(qN`IH~-~S3>$&;3^Bt0N=p{eg>yo{DWS_g;^AC4j1L&-?z-MKJ=1bMDVdC zdO?rUIYK5MRqxGFD4`wH5A*DpP^b4Dned_=DDc?%DbSs5tCFjV`mh^M_)SM05l@Qq zwN+%x@;--WZC9x8?m0Q#Wnb=`==R2iSl4E};Jk>5`BJOR!T!mceo{It>Ubs0kKVR3 zz5F`WEcMg_!<%2ZjJ}!c8k9@z*VkYBgv8iE(3R^k(OY~dVdECpL2DN_b3rRyWBp0Et)r3)71F$9PPwZ7T31v)q;yA#6zC{(0g!h+&_EQm~kI& zI6SIiNJcCAtB{T1;9n%(k&CP9MkkIZet$A;@y)?LsNF{G?sI{-pD#4tX}>HP_321R z%kOMjHv0PgX0=z(=X$gaIA^sMJ|B1%nJ3@Eamlek`DuEiyLV5s{UWI*!KvLwTZ4+&ExGh6+bo{ue`9ox%_yqP+8_pv%0jZ!=S0!hl4_M;IcXW#~VElJXzFS zRxf{t?Wu0c&E&wJo<|h)RvlfqvhUa#?JFtQtes8_eO!LLD&oywV>Kmd^*6%KrOZ=5 zb3SRH zwa}pc?9IZmH-)d$Z%9|)xGoksn6ai``AWbx|4SsD+m|26j-?N8)I5K6GW6ov6Qvg# z7!L|QEabXXnxR`5_hkR=xi6pOtTJpC&lrGj?6?VcFi!8yPh& z8+m;FW#~_zc!%FOE+Qr*|Ks*0opTLh2UqmgFVZ{AxNx0Vol|QR!lQk2ZTilKNn!;q z>-idR2lL$2x2&1F=hKO|Uu|O!o_Md-c<=!A)*H>&b{2>0byg||9uVyQ_%3kAR{rOo zbzSdUj-4M{_4Ps831d0M<#rY}=iYv!n}i?Z8Liouq1z%>_S4h6Ax&k~ZOOzcv6Qs* zaJdx=jVAAg5=m@F5040b?={ZU+A;t14xLQN!Xt~7n!lfqbFUAWPsn_9y^A}%G}@t( ztI0ck_w5A|J+?(zBO5wJOk+M&G`@)la}!uox7Wisu<`mQkJ{u#E#`HQ&Pi|uAB?g*^Zgh$=&+FE1x=#PQ?`|7`iMtZO!Jr;#0BlSD%hG zrnQO@<1e>;Frb`C*YOyyiE3Wps#@+#h!3t)v+vH{(zQ!>+S*(?wf7q%HX&PdKuvT^ zcR$D371||IOZ%3Qg0`K!+4d&kQO+*q^@X;dcQ?M@mJ_zcsAb`&Y1*>&j=Y8kUwr6W z`*43!lJ~Q7pI7FNTOQf7y?ei=(hkii-oqI^Udl^($`9~+UePSL)TMi`{J>AnHL@lP zHkps!91d8%Hbv-Md{F7C)Wz*5h?{Fif)uii%gXSRO_!ZX?h?CGz6Wc_S)52ovmY9i ze9yVi!^|P9Z8WLT@k4)M$DXZ8ycKGF1tIN*IMGT{1R7C?y`e3~eu|EiZX=42VXW_;a} zW26_kGFJCnPxbdzkLMpcZ!pJE;OjzeH`DW_mZvwoR9z9Ra`I@(mMH~=lXt9Tes=Vp z6ZWw`6e!xY?x})m)-{uOA%h+rc4A~yaX|8$ck{mO+Bs?4XOgPDZEzXaewsd6``Oi} z3DxykYGLZ6@K$@7O0WF5fY8=2j~>ul7u)jXp6}{jGJiNSICFEGTiv`ofxv6K&R5*g zs1CzNOl6&Ue^Nwhj%vE9m!(8fhN$y_3ZEC+t2^kcP3|=R^)-@%_jpfH?p_oAi%S%8 z#f`O_Hp<(2{iyu*vhPZ=i{;X+oW~(g=lcCczT-T|#o2+{e;~JF$`=9h>XW>dceyPZpbSbH0(TivMJYRSBk31`|jB1XlnMX_A?&OhKHt9GX5>KuDG#|gqg2F{> zpCD*a5_$MX*Lx8knz{+@ZgC)WUvTm9-N4@6xS8XbQ4@d6Cp*zj^iX!;Jv=N& zBxq#E7^>p)AxE4!jqCVlas1`I`*}vnM+pyXs5$M`!ngajB$O6?{COuf)baJ=PLUV# zCWo3shkraXrJCiDcW4$^KD}6i^LtjT_;yq0Ws=tO<87B3A9yymj{Ml@*5|&?tNrH< zmsrCW4n+<69P$ih_Qmh7@|o^=%^zC3h0|xl3O3T!(m8^HwR4FFY+MVZiS(HKJI>E| zpOU-Nc6W>m$hOy}<}{=(uW8G?Wt*?HK;agDWR%DI>yoB^vdPGDr9?r}feNp6=Qq?sl2o0`+r_@lK!R!)347$9*pM zE8&>_L7t0EOYlRkZd?^p-yLo2Bd9KQ3u&r8ULIT@rk;rh#5J#aBlsf3&gJ!{$K!W8 z=}Dyn;|p$=Zgj{=YH4k~*n7V5SeW|riU$^*Lt7h*T?Vg|*tqBvaJ@FpQ{DWWf7Oed zeB!36?3=!kI6@pc9R?~hT{^cZc_j@Px|Igkk@F9gI7cm#rZ=T;b$zS)cy8eF;yFg_ z%h^QlY~z$!Q+0QJY2Lj#D_7mJJGv|1oui^{I-{VWUTJmvxrCh^*^5s*Rpfl449WPr zUp&L+d5rH2FIQH-fU1r!w@VBs&Su{6`|XTC&gK^tdObjp75RNK?IEU_HK;?@2ceC68>PoZY}2Zr;_x66&6vhN{EW_S|`&y{&ylJgADra>L&* zC{+~3NhJFp;Z9$-Ag$zwKy>0=>8y_64S56VjN#yE_g>>tQkD3=9UXETz6M)x_Bn>> zS6Ij&d}%FF{g+5QK2e}p_ljh1)P42#3ib3V{$<4@rhF+Ghvy}xJl*ve$7|50Wb8HJ zm=M@y_2_P~z4gU(nn-M-PHS?Cz=@8w)GalSvxGi%O*D;8)E)D6Y}@4#+_PBMx}d|> zB05b}KzIj1L~@~&y7vn$Nz!TRq(??5ECATl@k*OFs5{}4voDIpr@d9qc)U!mWYXb= zRM*X0It^dagrnpS@aEBKjis(NSc!c65oRd+mKx}~Mkn~2s+95V#k}Hv9Kv!co2^QQ zwi+kq2U9z|9l{2bKJ*MvO-)wZZ)r#leO#BW6&0^KaVnX+IJd+)C^Lhqu|6tw^YYXz zKHh?fTEXnP*gXUG7KS4<0nc`wCw>(IhlcHhH-02ZvX#@k%by35B2K8b870a0a9zd~ zRF#NFNBfZC7rELM@7x&Jn`%dEZx@lbSTa{NY|jdD`7_!$iPv{hal+|QN>&Nkjz^9b zST(*LsSe*eymyFJGnO^%cJCE zp50_#8n5=XX!Y>b5>h!T+#-WZR1;Usa~}-RdQr^zCgy=t0F0{-dp3ue_ANnwK+PTd>U-^+@#B5!Ip`z?R(qgk2^k= zcqX41KW|-1@z$lqy~$kZ?G3xbhCdlmD{jz?lfC?{(${DTtM*Fra%b^KS-T7AP*eg&kLu0jn{dsd`xQ67qx$Ug}WMi z^fNCu!%|yr-pp5Abrt;_iJ^I%7NRi%^DK)9jq4BdpNMIlyKSJ0chR^L&+}teeCOo( z=Ik=kl90LRy|}y9Xz7K(B}*I@%Zjg=bA8cg@@cVaofQk+3Py!hh91uQo;od*D|1DV zVVW$m!EXNiDMd-qqNI=oA>C%`W`=Jx4zF%oUwbr3bI#Zu9sMsGv=2nj*-)mrRf}y4 zRdx4*9cpLx(N$hls;(2u!YkR_k5)Y1lE1duhci|TC{4-8@2r({ zG&5bEzG5%2=j8HbQo|y0LAO?}y4$f-_NRm4N^L$Lc^`?9)%gLH3ZoBCtx>yh!~0ZX zna_?7-x*76vwdGFpA0-EEfHX8vC413ps#<+X@ju1S3%*M3oZ8X^N5DlJ1z^3@VgYG zx8zs|NBX0^)sH4=B?>v7VfqK@>JuGa6G!?y3J%cR13&F_RocqtHmt7Vcr%=b;whp- zUUkjcp|9MMdZq1|lheI-&db^Fy1a2qG;XUSnuwfVu-R&<`KDz5V2cjQPjd<0I#avb zN6k`SUp9PjD%FVV>rDe)^Kbh5rOHW9*6QmLb};lzdgL}H99nEW=(=T_GMkHye~I?? z+pWA-V`Yc7YF;hg5<35_WocN8-CK>`okXinTj$RQc3e7`Yu}%|ZkOy2_T6*?yFEA7 zcy_6XN4)v&bNg*>{g-!)%k4cI@^^ihYWH|w&0KJB*Nf z*SziS`yAWce@!+1quZ`~lkL1TT#K+G)&>P#8?LUm` zGajml3jZ3dB{OJfBQ(NG`fIdq)y40L$0xp7Oh0K4vUhlXSIy?d&p3gX+V37gi&tYf@pe{c+{-4L{18_g^Rz>OEd(b~E$Q;i|NUwR@{;=Liix*5~$na=>$A zeOdFOr)+oRlW(Tz9`XDccyyJX!m+-U3sbIWpE==VeeL-2k3)aGiKt2|sTn&LcBB4` z`n;606T#(})wjn)}c=yCrXO5@W zE_sb-_XmaEloj55gMMB3`VH7NHW(>(MY(^?CI4*!mv8Hk(#K>UoY&kqd@*$L>V?u1 zXA2%M8g6kd{7|TyQF?p-leoMmFX!eHa@XgY5FB$7sN0I$mb=~+G2T;T^^NOJ@)@O) zj`&;mB;G{bx7+=qGi-@D=hKNJBVvovqjzQ#HT%TV{E^^)p~#8Aoak(#@Cu}Zq*;QunSaPsq?X0@W9p`f$wxb zKj$B_yzlyT)!2FC6KN0ZmMhA=J!fMPZbJW7Gs<(UMK@!g=g%@Rm9z%)M9JH$(o)2# zR>*~?-<>pC$VMWD3XU8;nrYno{q+1DT9TPMJCqh5DU3V+y?K5>z5De?nS|1C?k=uM zhv?nu-c3Cc3vQ2O71^4KbZ%&@_z>e37V&0p-5P<)Kx2Mtu4@gMqeiy9DBV<_ zo;v!odHTo~=PhB|U#N)-+*x5($f>*XQqDlN#N616kuP%=X}Gk7uWAZPKIXM_WWCIH z_NY@woRin+#L6Dq@%fg%gZ11oZlxt=nVnY#>+en-?fhh@P!wl7<|~~p_6{5IjUy7T-9|!a6G}c+g`11SJ#$obL(l{Z@sC~*$J@>QMCb4 zj{Ul0CE6>_F6&z=b#hw}DdA1qP32uVk3QQLuHW{)aktTyu$<{n3tJr5FQdIUXvq6; zZQlp)q@?{TKc9Pc#Bw}$fA{u1nmd#_&Q@re#Fx2~oG*V**)3rqrx9$QmU7~~ zffV?&mnT zVFBPizWn)(TVARnJiJzmea%l@ekwbCd+z}uLQR+7QNi{na@R{U)fWmGMqCb%d|vJB z?+}6)^sUdHdc>$bZF6aG`G8o8(c}${ECZq4j|0rUa6KK2qrB3WNpu<`^^UIzwj>=} zvd&5-_eKAy*8BP6)qQge&L3L%RlxDQsT=o()0U;tD^y>m96hO`pfI&X#`@05bG;ou z583+&uj>*G%u-c&Dim*Wja{e5pg1a$`0h<|z|LLY=9%=_P7ZF(iY2G|I#sY2-xX|_B1Yt|r0yvF3k5dRI&WRTk;^f3Fj87 z5jGr_;RLy1w;_)M01@zXaDi!_2moEc-#|F*TIAUQ06Q#skKpEU!_GY(*tN%_4tN7j zh#Pht^0)%51Uv^^NPs66z!>naKo7ehd8`5E<3xDg0sSiQ`vI(j@P|Op4}K>ADZn$q zMe(Bn^Z|!$;@l{`T>!9ylQ$JNpW6!jp#U2Ie-7n=9iTikfYpE(fjbxA5P&J*V?YnP zF?qHE2t)oCA%98mgIUP47Q!Du{;?l6sHSCna8aR{r~_~2!9Ou z!!Ax9*w)4)1Ne1tk-QH87y|wY=wVkV&u)MvfS-Z)W1LADiHpNrGF~m z(tuwD7s)FIzyR=%K#%lq53m^UQ!M@O16&L67cBjI0xl2uU2su-N(3+id;;i^{!yKo z2j#D1>7M~O=z+JIrGF$-Nx-jwi{eKCYy`Xq=#l{tf6+d29d{%+h}#;A#;5l%;Ab=6zUw|ITa}NNt zTi$al{f7gt19%5Z|8&6D0A30%s!xXjEC8Qk>HnYFe>tR&^zQ&bg!t#d#Q{VB=mOpa z;Yj~G0K@=KX6Zi&aCN{NS^9Scd?nzy;39p+0vH262=qw*|J43#S^9T^I8qQl3tSXG z8bBZLKA=bG?E;Vh{3J{Np@44yyq%?g8sMt|zXR@EfI|SLfR6(`(&s<5|0BDi>fNB|Px?;srUV+*hl@MC{#|IIA@yFs`tgy(~c_>BWF0el$f zkv{*a{nxSdPlY(r5dRvuNM11j27vbiJ<`8Dz+%8pv-H0Ya4o=Jvh?o>xIEy+;PL|` z0+<2*1L%?d|Ec}gu=G!XIFb-Q16&k83ScAPAAla|ez|r0(Ll0zy@at;7l?!`CmUl zpg{kGFgjs4Log4{55U+B|3u;60{92})^Q8r-y--Y4*w+J-(vWu3;*ztCOf3hiRH%+ zsVu}T!ztlNI5I90R}TBwBk_1PJUgBP&xPl~&&3Pk#qdPDB5bU8Ks@})|CfvS_{EDb zlmzkfi>F_FA>IUWX1GT{&&-<(a_52a@gHApy#Yy5;;gsR@!A6`Fj)G(0 zqHq^+mAH3sA^_!%!x7!QoM~h~Dp5&ct%8ygaUU=Y3Nene-wzn7B& z(UeYha`SQ_Qs`dJZZ7`5WIs2$7tz_5?n#tptdWLqVqgw#Bi*0ogc15u$xf{3L}xb| z72^Fj$^7pVz_{3m@uvxy@c}Qhd8PXL(tU}6u80+HvM&Rb=HFvsoHByF{K!F?e0;E( z--GJwMWtzNFf=hW*a9DN)BgRC?(L_w!D5TGiG{h<9xLmuCgw(1s3+aYpGIZ;rcok- zq=~;$SgW9*@GlR%Fe*PcKN?kQgSCmZsR8!D+n4I>7NoU7-(;)7Mr(_$w%AiII?|Y% zv(^ShNJoi9>Pe;h12a|z8!gQBt+0oGWE=`)MfD^4xl&mL15x<7D$FYBtU}E$6A+TT zy{TSKL}r0mVMIUr?8GQ^Pfs$$p}BccnTh`IWC6LEx-tBSbY~(f0V0D6N`w0CM05-$ zE(duoC$98!^`%mYuEE}}R4;}sGsAxqH%<2y{isCw32uJ|GxF-8jQGJ0Qmah|WrJFN~>*k9Io(TxFR_hY33;{Kg|peq%qkuLw%Dp3^&G8&zN1ja1hpXv>; zW#UXUXJvtOskEBtL}#^oUkW5nCz6?JN7?9ESXi4FYHff%Vh`#guzCiGK%GIo!GEX= z|5C6SRuq4!><`r!lKoXvXQ_?s?}ycYP#i|H8oE8Azn7aI1JxA<6$IhLP+$pbwlG=o zwN!pd!^hvv*8|$7xje<=n_GDpfYehiOh76=2*GT=+BGJ z%0m%2g#Lg*l($5pMv}sGHoL7r=3aDPzyF>ghQGHr5Maz~GcYtEZuR#$VDs4^m+{wxRT7mn4UaLiVL`&EZ< zEXBeDrnJEWWyBen8lqmvV`eP#C40GGtzxHQkTfkw6ZA&dJp2-3Z&XaEGJ8Nm!cswZv(sE_jbtwhm4-@YwQ(lj z8CruND4v_AC)LRf`Wsp>lWHa&2w;YuWNu5eur@Z>O5AK0YjUaz6$f1KIr}5pZ=#5 znGFU}EBqSTp|7F3L097oat$EU+?^{okhY z2d1#@*w2;hMO0!{xBs4%|8v^EG4S7I_4ioRAP_bM89;v!^nZ@b6eAy2I}yGA8fkx) zuA!b<)BVV_*_wn(^k1uz480Q>I#z}^b%rxcvHusT{&fKQfLr)}7M+_E_8C z!!d<{+w6!lbHo`r;ta{C_)vxEz7T`iTG8C$_vnlUNvfx}UobR$O)^DutERW6l_o>e zPjfp}-`H}SA#@*JUjCj`U+8zqzAoT{J_EWEd1(sD6ldnfz@{623&U+=^WS=4PqH^3&InD=NhwjA1=A$U;X3c7zM+Q~wmoG)Jgj$gXqZ!(k-Mj*r=&qmhQ;=Hm6Y zK3_3wSH;iqD9}J{0*uuaIdUN&n!-V3*9JYKt1CjC1 z_+_r4INQ8nYJmoP%;v=EB$69u$fLAp*o5&J68FW@^Y_LC#T)>ztTtnFR#a{M=yW2( z6O;yto5AFT$(Jik&C#eZdxrnd;nM!{nEgY9Ap}X~Pf5X8=|^_+V!*J>GEAW&zYu|5$zUfsArcVjG-L(CG<4Qr3PMey`@6XQ>UPn54|PSqIcM2PSOK7A|CR)roiMGT z05>Np9W_`g6p>{MQe3GNtadX4S*-#aA`CEN5sXVrrhiHX6di>9|3iE~7y_}pW`()1 z(qiU?Sj7Sj>An=C=NS^jo|!+ZXK?g)2LAuvbNpM=M7868=u7?>{Ku~3Ut`Wx%0Kla zFvch_yA|j+{WOU%i3=jG{N1}CT$YHf{QbGnvhw0JHVa6!X7qGXIv= zKeiZF%ztPw*q}7K$`h$U6tJ*hVnYsc^Yr&5{w_3)>g57!c)wNRIb$u%bUy#u09kp@ zw7*$HAZih)di>AX>A?c82N9MXiDW;bDHu)$0L(5J&8=aIP25gG8bTcw*%A5}qP2w? z4(9rPs_TAtzp%s!?Gl#2&|(!@MS|4Kx0#y4Iv5P2Ud(5hMwkm85E&u`sjK4tjgr-I zD`CUcuh9w@LiMG~Gi5X@8{Drc(X5%ofl?D$x?6BQd!sJ>V~ zH&cum;@LG|r!?iiUTr|F6_WfvA^9Wo&i>fwZwRuaF)fteMPLvy3kDN1))Si`Lkl7@ zD-LVY3KkEY7Jv;7GipMt&F)iC3|NPB^QHLH$iB0r1o0qcPdY{~?G3FJ#%ETwkcD2z z6IO-DzHUsbQX1@An8L$KFUpNfWRRVyP^SXieCb}usK$7-_9X)mT4J7+%`XEU)km}v zhzObO02naw0%M5=6#tm!)>$^zOzXkMLrjm%QEUsE=rHmBHs@JIcY+mP7qA^+2tvi& zNTwlCVcq>Kg|M{1oUTLXAdju-(H-4k$(e{MC=>^4uBh?;feRSGevO#FQ~f0z2t-R* z(80612+%B;pUgJ)Yrx%x79##bxFMK&|0$f|PeDBum`=Z}H?p4}8HPSES;2n9ENGV1 zhgvdgC{@H(NoHn542bVZCPF99Kz#yCSN|<)I2eUcR%`_v=JD|I>Q`E5MF(cNVEmhd z^*bw1SfBi#nZnsXWrLNyf8}7M3z!Ac{_CM*^afR+^!;UVTUhUrJgC92x&q>54&%RG zFM0br(qKvmp%A7?lt;tU36EVD}cvl0UtxH)5MsxuV7lCqj9 z8)jKRCbPr@GD8bbxZjb`Vwk_zOaW$?4!~w2d#H?{b&5DeMT&~53R!KP3dLDfP0iU^ zSxIFrRZT;Ys^+Ys?C40LsHv`1(@=F(Qzbi-71cGA9oLd6>g#|&!-=BotfZ!*rs_mh zQB+yyZiIBGajl$;cu$&TuZ>Pm`^l{^_G?c;Wa8_1S zS9f$)rl?ZZs{DWLeFs!j%d+SI1Cm8h1Qi@n1d*`!V1h&d}N=Jc!g^dO?gbMJrmtoz%kErc}xmvLr$tDI@rVlp|y@NhVV!Wb;`ZzKA2^a+w0LOe7#hJeH8jhj*VT;4sA! zAxBC|*b<(MD31nh{h!6<~u|OuJhCs~Yi&$)?L;z$`0iP7`g-jMakSP>$_%aCz|M=WwV$fo`@rYMzAFk zCUl+6fkEL(g#w|BFJQ_zVpv8Z0rXTr@JJq$lnVJglFwy{NO%TJFbsl3Amfoz0Zcqg zOi0)qi3IYYUa43>%3$ya7$&|@DkM2viHHN!C=-YzEG{7s!n%?01Y$lRVTxHym`NcJ zk!+bvDCEjS908vN4}-yyNZ4%f?ePebBj)g>B0gV6NJ$t^p-3j-5(6nH&*M#^y;_92umMERL9z!dhX;M0}ZmBVnO6&LUW7oFoK`;1O(|m@8q5m>f2& zI+>6Z!p(3TnN%VXa@l-BB!OperD)E0907~PVe$AA}Smp2nNM!J0i=m4$K8GiQd60@&@KOs&cprHjrVLgv2c}Ub;0ag~Qp)7R z>f;M}GCrF_h}f|FgupKN943z>nLIvIEaQ+o7#TJpBVf*9poG9LI6RgFRy|(;oQ1<> zO2lG53>c3qhQW}@V6{u&_2*0Z1Xsv}b%Itm9~PQ~1zZBMfLlljc&vcU=ki!=IDO25 zCzC7|ODtmXp$QT;lL;%02doG<7rZrG5eyNVBNNCt1k48vDO1RYmypS2Gr2-i1j|H9 zASMAT1_qrAC1g;aRLUYn5&_~H1Uy>IM=O{lg}_F{Fn@e8TMXThaAC!WSYi$nI1eln z8H|Zk1~>aL3FrWqDP@y9*bKRRu^0%XGBHm?NTKawcxS+WD&tpDl*A@kCr1ypF(>I0T;!4d<|!GAS@sA<5>l z_@so#BUnrUk0T=lB$|I7n=gT$GGTEFgaQ~lSdtfY0E7aP{_LF`H#UlF}?I!}T?nbmPE^M;Y*%Y?Dm}Jm|R? zq6(0dsc;KoSrJ4!u(8Kb!3<5ZCBqTRz&}dSAdN{G5z-K$IGMp>+KIqWr5H|ZmucP< zh43b2Okgp%>|c>$n?j35HA%(J5kD&{%~aoJ*$Urjlf|sLMg?LuR@`XK3Q*J5+-&|+ zo)}!|@s5@fTcpqeq)}=j_Q6svB$$YwN`WrcT%Gz=K2~?UzK`^un*P9w@tq1Qj|!{b z&r^v7VrwwMQA%zY_=71~-5rdAzce{rPT>r6H9T8+|Clsf)q@)MPiN%wZ^$E1Bj7u2nGKQ+fgj$zc#{ zhP?cgG)(hW=v_!vOn|_VINI>&2=Ie^pDtwrnja*WQ{G~j0PqNjw4*8Fyst$_42_@ z_wVno{0M#%MU;Y~Q^@BV2?0@IDB=U_9$?)A&q7#yG6NjTNMDPNqPhCI;=0_y5RZbC zP&a58K_2O*QICM0C4k;X^P^(QART5{=~kA2>j`F~Ng{Xm?oUmKFjgE&eV{B7Xo?gT zd#b0>BxsO=FJh+$r1IE*6LI>CewelG2w zNMRX(CgNy0QNox2>a_&k%Abj3!uYX@XFxF6@9P65I2O4cqQwkF<9@CUHEJvsDEQl! zQDjtyete>W!CC&`VOQ#2Arr%5@UUtBv^tUg35yHOHY_a4H;3jzo2Ce!{H5#uSL96# zLFp`SMid=o54((53Okd8fe5V?*2yl84DhB?y8jOu5Q||c?h863*rj_>##~2K02D!m z*_YaCZNWSi8>`4bw&e&%*wEalfHc?Wi0D{AwG|bN2!p*c5xL`);VX;~;N!?T1B)5x z>;{V%vVC`YChg7x! zuIi@6#tGe{#Iea#NFUAD&l$Dnv?GKW!;_?9AIhWwQA9KkDFjf0b;I2i88~P?)95O? z3L3WoG({pcNbatHEBXup)HLYv2ypj8#Xmkq;RN<_``#8cpk@D-RykHGDs-Ub{Fau} z)CP4L41^u|8)_J1vWZ+Dk;C}@p56}4fU=i5R1psMuZq9~DU0Z$A}~9DQ3TDOvWOlk z0+aU_MbPx3C1i#Qo_2&A2X2Fi?Da??cvts;}JqJ3tV*W=$)WaW1ai0;f zYD;41TMJ@EZxOLt3u4r(ABk~45iz|5G3w8c#LX8l_!|A3{rsHmeEj@q!{F=lVATra z?T{-aI%)_AKvZN07!wtrhX9R66Ag%@vB{W-`V6_Pa4r&aBY<;)Xr=rBxtcf^#E*E` z;}w7XAwwHyM8}2*qrxUqasySfce4bVH&E*Qf>IbU3_=Q&^bCa1yoV`gUJGU>61IDY z*g(4%&D{=R%1B43l42duYfw!fdSpmKqzI%wKMG+$oC?B*!KM9*p^1q`#&Km`J`~oh zt~NiAxx$`9!!%;_c^G`@fTEtbgjlMarQ%FYlAaGU>8&jzX$rnpeQ^LMNdqD zLtWxPLZX5xK$fN3DoZ0)Lk;0j2tWywg+T`>_CuvXAdE6`7@cB4&~+-Xo92&V>Y*Zg zT#+Q|dj)RbKn5u?AuJA=rcu*>sK^-?iHVaYNTQ>XBWXTOpR^WspSkFjtwhBX5lJB8 zGAzm!WC^fAN5#|JC>%Mgl?kptD0v{xfyX!@2JG}8Wx{rh+(-&v=FfB}s)e%mw;=pp zgCp!#woqdeB}$<5T@<+oU7G+BHi#7=F-cN|=!Vj1abb}$ASzO_9F>6nG(+QM@iEO% z5W!G0X|Uj=@$e2QPkC5^Ljzpn!%zfrBH9=w(coJK!5q26koN;c4TEq8ff*>s6Lkd^ zA{C&lO``-;)B;DSMkI`bn4C!Jvp7tZJHl81eA{y#np+v3%DufJTg3%F|?~&TrwXB%b zmQ~Tb{a~V_AmIMjZ36c#8b0KfS33HVdeFQhBo0y1M0hFSlv0>DzPBPE7Y!J40V0(S zH>SChO$g2%q#f8rM@1z6d68xs6dHAS2jg%X&^qw(@?=vgI~8lB$e>(k=;3G+L@q;L zf4`>Dv<6Aj3cT7>L^L%*ib6eT!zfL`$}-MM(Row~E29kLkrXHYzG8?t{_eJ?3LrB= zvzMOImV}^KgU2%-xSEm=Q{#g=^UE=BRsp=6|7KbLSnOLu?0^~v2z8IAjR4ghu|yQ~ zt5~V1vnZfK ziD2i5{2tBMtO`c72*nKoTc<)ro7biAbEDmXIx2;>2I?#l%&F3}+?xRo&}bAZfXqVJ zPzt+&LPJx;#{!!}fzOJlLI{<_spw^bO-K13b_uk3K|63h#ly70<`oga0Go|H2tA3? zSa%6T0Qj?*1kGchhqFtdzdOy#pGDYt_%ZDWg67ZW)0}BB@GpWUg)^BR-agPTXgId8 zIZ=!R@kV4uR2(Wn=4L96gbGcA;2SB79&mF|reP?h=me)NBP0P^DUeA99o2vx*TeaT zy1%axYf$nddccN>Or>bU!Qq~ikC>)BH-VNaW&1#mbt!N`36xM6>=P-|0fVv!D#>vj zR7fX^y`m{AjTJK@!yv>IMz5&?5mN1`J+&z*4k9mMbQL)Yhj$zeSs5XyI2kP7D1-?e zPk^37&@7l4@l82Y5jc1W?V}hscn|EN zWp-2>nutKA2iY>h0k?_joAQ~;#{BreA4|ZN24Zl_5`U74IxK_m7{pbfjBiZ=qF6#2RjIaiH4V?l7Qk|KN>!%W5s{^Cq2=j(O$*SbNTRrA6)70y7f6Y(eN++ zhqA((cV{18biPfo9<4rGt>+v>9}n~~G%ui)Q146g0y$*Ik)kE%+sx}nlOGFWyc0Xo2UNr&WFX`KYeg#>~#KEQRlP zvr~M(TL~#QaoyCxt6;CLR(JEY<82cs8r-^cvCWG9iu?;#Jg%jE1$vGGUlcV$t6wCv zN1$lW!WOv2$RSGuCExSM4`rX$*$%k3g1$?O`~@w_JJ9TD=ocU7rhU^@rG{f*TLo-B zHt4g4-tY4b{vy90@Au_R{-S*6Kgn0N-?~LU+$KQ%DD#OH`8d0&|AN2BSJJnH%Fzlm zQhW-=P>i7r!*B~s3D-2BpCpW{DNyP=5Anx;z<+QACUwK(ZlI_%6XVKp$J6m2eBwV< zp%FNa*S&HyW?L)Y6h+Fs7WmQ@xU!rQ{=4-l%PHYsl~>X!p}no5y<80YVmJ`Pkr;+x zn22E}h6^!Vi(v(ZdoetR;SCJ$W7vq{R}6LS6zwp$ zhO01Khv7yH_h5Jy!z&m*!tgzYD)x$YX=B(9!)_QV-z#g5g3bvK?2S$ej(i%!gBVt0 z7=^#mb3MkhF`S3tGz=3le2d{-3_~!4+hM4mPi)2W;TAXQ=K{_@hT$Fz%P|ba&tHx4 zMHtS+Fa^U14D&HP!%WdH5l)BOfT*9-`1uTscgFb!N;(Wb;(A0FKEkjT!?PG3#&8>k zd|bXehRHZxx&P(h=fkZ~)K5E1r-Pvih7GvfC=736cnQN381BQc9K#hDPQfr9!&*`? z?#gv=2jg#1GdhbF+}Y{blowotdGFBJBFxzh|aeK-6)L5VTjfPqDyN*Hyh)57@~1Nbemex?Z)_J z4AFW(bPrn4y;S1O)&rvJfN^C%x?#Lev-N=JyfChuk3Z{&AEr~z??{Y?V~F~TekWmE zx!$H>d?|+LIf$;X1>FXWAHWdx7tvj6L05eV8I&`M@@g3f$8TaVHvWK8Ax`6!`IGV{?uv3_F;vncp3wX!375Z%>0J6N%C&sHvK&_bnyT20%U{RkqCI|Jp3ziN z@!=INKWc!Y+$@nPZ;oLi z-gkRooQps=8!;C%sYo#r}&btdZ=>#Wxi>-g(nJe*W%S!i6~8H{{us`lXgP^xZu zej!zJef~qLUPX-o1;J2j&ANx{dHpAc9FcOu&A=Ax@d3F(W0|OHAVGB4~rU#-WSn|HH!_3 zO^O-C*2P3Izu2jGKyg5EXmLz&QgLc=W^q<=cCoy;u(-0gx_EE#(c+rohs6!W?~7?A znk5D$CMAp#>k^`bU*c3Upd_Fqv?Qh^sU)=|vm~n|yF^}6SW;P1U9z|2Xh}`U!;*%Q z_a(Ga%~FF>lTt>hbtzHGFLf#%Q0h|}TpC)MRGL|uRVpv7EUhj*T3SE|-^AmhUY; zT3%EBu)Lw1R$)+KQej=euW+jHsR*tJtw^fKtjMa6S5#J1R~)UVsi?1LsQ9zD0py%6 zkf*9jBhtGp(a^#%Ppwqt>0L4)DP2`Xg=|geX=o`U;Z)RV#3&8zo*HmLC-8C=x_k)X zNwja4(wfkMm?k+rS2Yzv^IDXpGtBRFZk0brkax+p+A(*oevPc=;JJ0n*S{Y5(SDA8 z)goVcx{f`Ou5L`Et9_NLsj8@|8o^D>G)rRRh1jS&?;I#8(-Kvy4A9bnCZ(m3I)s*n z8iW}csSfiajfge~(KON?A&f&;PQ^z@kp_f5N^E7+%2z6ZI3)>bPIN*k+D7fX!o;!B zanZ7PhHG>zIxCA@0Ax?12g+A7viN1b!cXk#%z$e@Ny5Cn9tr2Q;9L#?2UCb3K+#z| zB}$}a{4H%H2zx~vt-iNW8I?-;vE8UZ2nH2W2thcpM7oMa^YMczRZCZC4NE{pTQwbG z6Ia@tU0na|h~xYd_bc{TbvEf4B42O@)MCae1{otoP z^-=t^fl`kpa6RM8Xvf3_d)Hd?FF1@l_VnET`c!ir4|W~FKd>mA$#4V(1sefnpF z7+KST!`1yXGl*HXgO2zAHlbW={^SvBvopVNU)>2U>bK&~f}6#Q51+mrL_a(@yW--e zlV1LtU;NSa=B#j4SidTH=_*rTL?#lQVRX$b=_Z76(<1j#RngFdS3q68m6{r1h6wcO z#`LxwUbA?8PU;alXu5Qa=w9AFzHn@e ziBNT;o=m1dK$1OK0v?>t5;EatAg;)c4d%tfuJo8yb1r$4QI{;TXLYeZ(D?WmxQsS2G0{F!xz_E$ z9pw-c9u^%F=OB)aaDW8_s~;APt20{BW6>Wwf@{a**+VuFtn4%dg8CD_gpU#@R5RTE zvVI6Fiu?T&;t5^UY$FwwZ*)~SBKzYz4r^3}CiA(Gl(f5HT~D(K+vgWd5-z&BXzmTu zgx5BidQpqS+lPkJvL?HaHkf%P`3^m3*|=n{N-MuX6FN-O?s0bY$&H(r&A(=5b$Pcn zSy@4D@wB{fYl(VzXujGb;pI+z)~dmuHtiFt=5K2Mre{LqWxu7}<04-_dT@E>y+y1C zp^@)w#~o20@T70FmQh{UOdn_8)!W@vn6=#H&mH@miqPF~yv=!yoDoHLd^|^#=SUvV z>W;2Dk!Ctr#UX6G)%9~HC-0qeX6-_i!8V$0zwDPAY+c`tJ^WVCyN8Cm8zxzo@2pze zn(<~|mZsHfn+oq23xiGkocfNRyP?gI1EbeZdA#&s-Ifn(kKA2}bPWUGjPDhkQCp~a z;ULA^=`99}NL%o?Fq__l04-NFBj$Hz(~My<{6oXw(nBA(GAWt523Z=fxQxmd0@5f3 zNfL0Z867KTbAUlIxe64HjdFtiS1|Fv;*ABG5uQifwZ;wB&EGpCMI%AiRmERYZ=BNV z*}};>RdIWCwGR(+7s?MA+bzBDrlg`FRJ z77XQ&e)DPjwItPR-=1oBTzYA0wW-mgEqf<9O}f?D+UnTH7sJPNx<6&{l)VjZRmp;T zwN;|oUVHd4w?n(NJ;ut{-RKO#1!A}3*VZO=r`!KCC_X~PWMTz88NBNc;+wJmuXWSZB^;I=5S67T-YMG zl-rr&o@TA-5DKl5L9}k-7wUu>u#F!Wqr1%4_12VtHhV4)S<`ixX764W;|f7`p%~*Z z!jBj#_m+EQ491KL7~Ic{k>ZLFSR;Yki z_?y?w=citJX42@;);QduPBy2q>%Ak>YMwpb$4W07KG?+WW$2Ptr_MCQHk9wUkug2n zwu>jLXG7~j4WTnOFC4~9$QhSDu1}t2oBU^GSLo#@YMcV))6OmN6fLmm5N|fe(L>*D z-QCKSY|e&{Tl@-b)1Kz5k3YJ)di%qUZnC=`{`&KVr)*qnoL!Z1HneY6*Ub)N=KD;a zdw6d8Sm&HZd%gPQ=OYsCZVB7mCF5hOb8l~ZovSUgP4qLjWb%$g_{J{DsC>y0jx&8z z)jQ2+_&TQD*W8&OTr#XohUKIm4_f)2$o6l$&SR;i!_n?{ofmf)rP}Y_(mT^f_UYSi zhlg?Ou-ldv8?H`zpr&Dy_s%@4{{fobWz(5=^f^`{wejuNy*O(nmk08K-}4t&*C)li z-|m?)WAWSia#i=|(*0ZVri2+Mv|70G&AOd|Qv=BKcC|!$yUQS}6KTu;H*nEE71^6< z5%RQy1ia>0h}Tjhb(_R1&?-PK|E^=`koq7Qe8<}n4&MvXNeepT$+xrLh7NbSbE;ct zH^Y=c>m{~J3wZGkCs;i{^&IAHJdX%yR$P}vLQ#iQ6WVYHibZIVV`jks{hWh+TwHbc+Xz-H z9B^(-|1Wjh1V&$mYvlBARa_UC>RIk&M4Eo_NjR|ihUF7ibIpjM1H?{~#fG=PEHvFd z?1k4MQNfZm!FEH2(~d?Pb@r3au)1yi(EEw!jk~RTwF;haVe_T8yZYs%8x11TjrunY zxtf{^nQp{|G-lJhAr;f8DFMIe|#Y|Um)X-6`4Fk|J(^YI?Y^`C}=?3c@S3NCI zTTSH`I%ezfyFOQMi(UTkapk9lHsK-drmK0LinvC;>h7;{`$vDq(M?4!`12}(SK`$W(UC2#`< zU!gyLNB?h51T5I!E87nu+TJG4g`6*=mv6ad;rP&|2j_W?b5TdXlYJcD+wE$jmvo5U zMm;>hb@5$sH#2G1M~7G!vclWxpNkpzc;j7*af|OoiVg3d3B5Qul$)5DbSiL5|HC_v z4icm^hEF4VjW21`tPAh7wcGQ~v&YVOI8o>+kW~!Hi=FGLm#}JwzEiP|(LLL<2?a;B zC(L+xquryv5BGFGb!(ijiRz-nFRj~*nfY>y=9oM`OW)l~q^`8aUA>EYZ5kXw_YJeL zTYH+HV$SQ=Yev=vbFumGoSNVhduq-Fx&09{%iQmyRZ%IqE?j%{AEm^yyzX136&x&h zxijJDu{jMUZJh6&Ia6cO&x=Lxx~ij6pzH`E)xMcrc(n51yPS5a`l9wRy{7nb7X&Sy zpz?!wcS+o9)GRXVwdF@5M}K;J6}BM=f3Uz zo(I-ve8%?V+<$ET>GfEHw^kLq7|G{@B6hoK4zaHLD5=XXs2kbc>p<8*tKNh9RLb{y zE}Gnbb@%i>JMGgZAKskK)UhlYT|ch=+U;ADiY-JZ)VFFlM!D~PwQu>9N}aQ|<>|CG zTT84;$}M!KX|c{GsPxvZJW*`naF_12zC7!VB>w%$;Soan-Cahkoz{NTj@Sid$@Te+ zC-y3hozH*kp}Vs7efQfZ!e{yn%MIIncW*#gzP0^in)KDKfd@YBVI5xX(YK>)=BeT1 zN_syD**w|T7c}AVz~KUwny^-y4aL+{TMQPFmif1EH8AlhxLS|zTn%Zt3|DlGEkpsM zBOCBN0ih5$56Uo76etcyz&%CiekU%6Qh+%Ml#S5;GXMQ6PE};4w==QLtX_<#pN*Ei z`Ir*6I6U6p;8KA$Eq(f@XMcnawjF!uz&+vq30wQhdX8*p=VwJHJ3Dvmu*B@NmeIUF zghb#5!maho;LK_MuDVyes`;85U2ZHYy&Ol3=$@YEu6OqF8;`V8d%6WP1A_a%3h>Ba zPBV`ehAbGH=D*?0BF}~U4z3%0PG}TmeYnGijMN=Bg`fNHvoh-zVi?<@%dCpQl9@iD z4Wqv#c6MwSkx;s8mW!9u&~vRw(wGNs{6pH zZW9JgDk`YHpp4R*@vdL8rvHded+F`)6Z0SDjuM5Qb6lJ?^2+HICI|ngOb=EqtXWv9 znCXGNkM*BodVf-PJdq|xe2#%keQ8UGw1q_4{H6i5R|6%6a8&X?RTX3MPg$(b;DHX3 zXmOl_tB;=p{!SSC(D+Egsj0Xs!8T_y%xInv{2l_*egthS%?E#;ZB5&dcc%z46JqlW)k~XVh;Oef>Bh z(v;Y^|B&q1xSMYtG$idhQn<$Q&WD}P!u>~R_8gk-X838F-<*?;!G^OYoOE(>UU_{~ z`){K@jA_rw?lydI{0kR@qZNBn4yU~0O%CeU>DBV&K?{6o75vui-#8?vNv{@d4!?Wk z^xNYTY0-lTw{8}F&OB2qSUNwj$IdNZ=(l3lG6z|G8CK!&b<4eZtR14UE3S=jdi3)2 za#_}#Z{tKMHx3QbSEVw&5e$C>> z!v)$)GxB$>=}=C0nOeO$b)^NLvDxyTSCt|!=n?6(oU?`Di!Q_diX?5`li@ZW{uU;Rm@XAb)0GAFQ^(>dsJ`O@`*Fz zdTZX#wtGCco@H;?F>N7srtjKF4BQETS zXAZt8p7dtN*iF2C;9XPM9VD>>&64Z9OoH)7wpI%4mallQ$o>FpWq{b;qh zDc@n|nE-YB7kY9ZPVEaVt4^0TPT`nqZm;EDyD)ZsU3}(1!<=+AFnpSnhDw+OIrwNDX56Q5PeD@fn@&Ww@dvQqA9F_zZ^G(w49xv~dN9=Yy^Y-qTK2_(O+Y zirnj>K$OdER#ug?Cb~D7F(}Pdu1&x<-qKXnjPC^JC_lH#`N5_D?WM%M9%Ej4iO$to zPuZh$#roU6;Yl3(Scn>!wIRm!kkjFPpAFB?x41p&U_a}95veyOtu_d%X_PJR8U0*6 zeEw|j4vY1AMjm!`HCwb`Rjk$RJ3FQodoH|nX6f9k{S1lmv5OycWHZh<9=tOAOjIO$vr{k-wUn=k9^6TWn`VH#R6M?62f{&2m6!E3LRj#JJJmb`A0xgo=< zn#<`orE{l0%DOz>vT4j{UeHR*)amEHbsKx^>gU-u`*II3RE=j}>gz*fOn;yHw(8od zkO51NY`f@kWRRP=*>>*4UGLk5FW!AB>+$}Y0R?=pvmQ(moRRBHw;Sn}y~ZhLc9 z)s33RXNPQByJmi)nvZ>cVhOR!sw8*cyr<>+x{$_;ymiS}x>fhPo|Kt-^o@Jb&X&pW zt4et;8oZ^w@zXnBTQ5p_kaXeLU1q*ay}M@3s}LvmzPmoYdU&Tth&;>x?&8f)r=*+c zEeV*k<_l@_>g=Iwm!H=S${g!>=+ql;vQI&6O^H1>XqEib(rI%pYt^Lq6lpqa66uC3;kEeLi1s(fX{hZ0>(W+uDc34=FfH00&-Fpt!T`S( z(@4brdh5{DhTFNr;D~Tkn2+VsPxBzlh>C3p5#WQ;b>8F;Du$;<2Et>)ET-j;0gYe-SiN{LX^m&M)N$iZ=wJLAFX|k^u9#(MA^hNXU5);1xu$$~(w4%EN3T_%%v*DK zTszl=w{vPAKkILB;V7}U$S&GBC^6yvV8?MGan~Q1>i3^r@o~e`E#8jH%hm)wtQBZ< zH%gD)am6^my>@T!q>G~)>HUcXU&rfxy0CR>rgxg^=~0_?cAb3mNX4+@%g{7laNB1i z!vg~47Bg5G1Es^))SfbS@4SM2Rh{NSZy&VZ>Z1XiPu9^LKP6#IM3wvjywds5@+8SL<)`_&ZQ!~r&|H!kkdrEja+36q}kR33j*H@|--v%~Zu zn>v(E->+*f(wx8XL(sG!i~AqU4xS~}2>L8&m-OZ~qoc2OWLqJ*#IOH1hH3O-`tm_l zT3+d8*N9h>yLYiM8nrKZ1^-}=I|f#>)N|^c$L+}N^>kYL#^N%Efa}k8s^!d%s;vtj z_{Wuw4-RVfH`2cE&tjZ9Se5wl^wF(459S=`{l}`SucjZ}`Pr#`!yhTFwXeROxL~W+ z60KJDNpID<+V*|^XvNyd!TM?%4F}4@O!ZS&%lbb2vh_=2WbT8csczG5mknJx^yK}Q zXIdLf9@zRyC)Gm<^N#s?Xp!mqG9q1Hg!a>v|HjLp`Xjq&zIBcPR`A{1qf4S#kl(n9 z{%vPav*mzYHoY&aM$TKXe&Jn*oYu8wr}QgcZz+i;->tZOL9RlMi3)0;J!AC5V}&(e z&VHc1Aqa=yqs}YBT?iNp~vUtAniaP(o1>TXMt=yiuSn?KyCJr@#5&y+BuJd>~9J8WXWPqZ~@J+d}6=A1Nc<5)Y)F z)H8k3YtV6z=zY;O2ksu7VQesWWc03Uwjo7^yiZ>$XHVK4f9E{&^0tg+`ZfJU40qEZ zhPw<_Y+pJ2*=x_+52x4LOt<`;ldc;;r0e=O4Ljws(RG6~ryq73T@sS)kjn;z+bNX? zBJET&*#x{bX}joY+ljQTu)45TFK4Q%5?z{=p|>?Sb8OSqTP)tA{(^<;w?5FD=vkJe z+E(^#%{S|*AGN=QM?I{%?Bo<%t2+FwOM44e02UuBCy%6;^#qjrkDoA@)>)0 zc{4mBqD8_8v>7mbz!2vdFClvnR@Am()I8H4u4F@(RDju+`VKMpQbrG^JY91W*cP*4 ze@j*DZH8^SY#PQf-_fDUd)bRtUu`z(%xb>Lfs{`8HfxTHJ823*k!bHSGHBM6s0ns) zLZk>N8q&F07E)f|PKHS8+c3Z-g4%&yA{^Mns2>zD!vlAf68#&8;4l3a`CImTy6fMr zjIdmjo*3WS>tJ%u123Lm5<5EM_>O$$CcAw$#o|wI!oOAb2%Oob-?fRo`1|MeNb7o$ zrN=0^+m@8f`C1rp&#+(3s_dm}rthkIk+HYK?h_a0ZuWg(S!<^{KDOJt(mL_MaKU|c zqsuY#C$?|edM;0zeCn(&y+57yJO#>�HQzPro_R?!N(T3IO+nlmNkmuI|F0me^qOIv#s2l z)ccLoS61E_ce!oRz80?;wRzjG zYv?sCaeAiA%JAb`JFk>Ie(ML7TwHA7=N_urI!!SK3$!@Sa-(ZHMJuXwE~=u_iJxx8nA8U zxUME2t8By5R%{#FqhPtt&Iu1o(jq>;*m`lEuv_TRRW1&7D|2o4jjpq?n6zL@z{5*j zUE}%~>Z`YJJ2KkFB0X+sN@T^k*@DGZ5%PtLmm3Z~c5r;hIWrR9wOJN;B2|6k34-5O z=6WfRc2!+(XW(R?FYCHAF4yhSUZwZ9{;O=?+FcC)JZvHid8XyKz%O^p+UYy2xS;`G zj{LT@!7HJ$5O`Btkh?!Cc#{rw)wa92~6!!O@f#!NPqz?Z z7lqq*)@ZD4=X3Lv@g(0~s#i=qe5q(i8dX2|kjc$n-OOuCtBV5Ys;!7?m4;)!lDiw| z0~#A*-uD_aeCip;PkK8W4?SD9dfUZo1yv?SVWlwV7EoX72oLDhyhr8EJ$0rR^@_kQj*p>y(qdnrpfJMHvOs#MNP`8G5)b^XGN8{PU#=FIRm7Ple? ziz{!w1|!)5;0tq+P#^ktyn?E~l7qg>f%3F(zqNUFYLX?j)XCP=`3NqPi)A_#II4{J zp_5=By8D)~kuJ0rFEou??w~!@bS~2?ce1 z^*?E)cc-5JNi)vkk&Jf4Os(Ucu?BP2J6v{1E%hJ_dlpq|8NayxQxPSJs5A*;{sQp4&JBm4VxaOH)p#ys^zBG_`%*^s2p&4->n0iq`6I zx!s{k&HJW-R%wCO6Q+DiUzmD5CcgKwV+ogbKhXL3Y1_F$33+vPKDGTezq*iqnRdu= znvdgM6>U8mm;RByDcg0Up9~tpU3F>{UsrdxidPTw!IyTc9wn~0-g|H~$f@s2&O6ID z0^XO6O8eELtKY)zzkYDHId4I6QC`g!&ku)-eqp~d`i1?< z=m+~1+B?yAbdm5?StOT0VE`n2c?^6vmID1xKQ3M62akf{xQ`w>?xRT^_c+8L3JIz{`q^9ZQ@A)h_`qjVx z;L(e|y&y98^og+4eqk8hf6I0I`+2qF8&dD(b+Pz3fHQlntjC#c9piFmeeL0Us^Hee zD{mXNw4Y|`I)Bd6UX5NMg9mIPjaD7@oPT$!Tdt3Ge!|AJ$11n!u6?re$dTD>yA_s? z(z0{=)Y=rxKbP|MUYoZ4G-kH#OrO>1nR<%jwg)%bOjEDyomIW|qKUfh82_Q0>V_1Q z(Hncu)|uWvb9ByP!;FhB{W_H#yL&S?vZ8*~l0FHCyO<=uwXJ-+{d(cX(bbzKI;Gq{ zs9G6hlmgmpURni_R@!Vs98|NN9^1;%+AvY$#^ICO)alMUju4a4KGu~!frw9S`zwN+ zq4^Ma^dDU-=~}d=MfsODt(X^^><=C+m(`dstUBq=)$?AnXyb#%drP7twA_=8tr!PZ zFYZ@)ZSRAv?hDrs9=P6h_F|pDBXuTHa$17lUd~Qqot}$UUi#$gD-YD{!^?l8GW31F zU7|@G%cqBTZ0@o;xBckUS$dy4dd~2hD&C}XY2YHgu(z?Sr?t5%;OzHww~>_bLH;b1%bNSQ`k1$y$e#?ZM)#n^_0$m!N(TA zcRd$9V}M|L-3#M%qZ3}*{nYp6?3Scd`Cp7%I(hoKiHDwF80tBsPvXv-P8|n|f@9um z>fB&jKRGvYgr+^MaPP=2oblFN*IvsWMdX1^%NcB1aEZPO5%eFP2rySQO$6}x|E{@e z@AVBm!nFIGu5R>PelX@eDyqJdvj+n_;bhP4U zX&QYET0Q06R<-pN*2f;-VsGcJ>3GH0G|%mmZS+{xzV*X5t#r)_(7x3E%-P%NmoF@w zx?z?>SwV30@O;{%iMhAC4lU3Ok!-xQPqt>`=l#RV?Q_G=+z#!up^rS$y{E=d)41af z8`59QxI1L52{FsYWn75xYp><67rh+a#^p&tw0fhceXIM^%fi168UAW{=jge2kIq!> zon^x6*|Ydccu9r%T92J>3nzcjQJv+rvMRrH-^HDRS@ZUG3lH1J3^=RZcX7AOk)6Lh z?bIW!JoDrAHdn9wSIkwiw>w7+Gfwp2w6h*HN$~obb+z$+)wZm6rkS5>oqWS|#tQQj zKZT@PX7u>cT*YDkW3E!@i2si?X% z3S_#U%~f}uk9gX=&hK@6RZ!9!+Wlb3_(lH*%~dWQxu=(G-`@N4)e8r17`}2GD>j+9 zwyRF@;LFxJPqJg}Q;xhA2Hsoeel2TXU0inG(c8@@^|LznXznPn?OoGG*8S0U&hMCA z+&!zZ!f)yOSiC!S)j(|aGzZ69{XbiT9@qc? diff --git a/src/containers/dgxie/nginx.conf b/src/containers/dgxie/nginx.conf deleted file mode 100644 index 82affda43..000000000 --- a/src/containers/dgxie/nginx.conf +++ /dev/null @@ -1,43 +0,0 @@ -#user www; -worker_processes auto; -#daemon on; - -error_log /var/log/nginx/error.log warn; -pid /var/run/nginx.pid; - -events { - worker_connections 1024; -} - -http { - include /etc/nginx/mime.types; - default_type application/octet-stream; - sendfile on; - access_log /var/log/nginx/access.log; - keepalive_timeout 3000; - server { - listen #HTTP_PORT#; - root /www; - index index.html index.htm; - autoindex on; - server_name localhost; - client_max_body_size 32m; - error_page 500 502 503 504 /50x.html; - location /50x.html { - root /var/lib/nginx/html; - } - location /hosts { - proxy_pass http://127.0.0.1:5000; - } - location /log { - proxy_pass http://127.0.0.1:5000; - } - location /install { - proxy_pass http://127.0.0.1:5000; - proxy_set_header X-Real-IP $remote_addr; - } - location /v1 { - proxy_pass http://127.0.0.1:9090; - } - } -} diff --git a/src/containers/dgxie/start b/src/containers/dgxie/start deleted file mode 100755 index 27d0d229f..000000000 --- a/src/containers/dgxie/start +++ /dev/null @@ -1,127 +0,0 @@ -#!/bin/sh - -### CONFIG -MNT=${MNT:-/data} -ISO=${ISO:-/iso} -VISO=${VISO:-/vmware_iso} -PRESEED=${ISO}/preseed/pxe.template.seed -DGX_INT=${DGX_INT:-enp1s0f0} -DGX_DISK=${DGX_DISK:-sda} -DGX_KBD=${DGX_KBD:-us} -DGX_KERN_EXTRA=${DGX_KERN_EXTRA:-} -DGX_EXTRA_PACKAGES=${DGX_EXTRA_PACKAGES:-} -DGX_HTTPS_PROXY=${DGX_HTTPS_PROXY:-} -HTTP_INT=${HTTP_INT:-eth1} -HTTP_PORT=${HTTP_PORT:-13370} -DHCP_INT=${DHCP_INT:-eth1} -NAT_INT_PUB=${NAT_INT_PUB:-eth0} -NAT_INT_PRV=${NAT_INT_PRV:-eth1} -NETWORK=${NETWORK:-192.168.1.0} -NETMASK=${NETMASK:-255.255.255.0} -GATEWAY=${GATEWAY:-192.168.1.1} -DNS1=${DNS1:-8.8.8.8} -DNS2=${DNS2:-8.8.4.4} -DHCP_START=${DHCP_START:-192.168.1.100} -DHCP_END=${DHCP_END:-192.168.1.199} -LEASETIME=${LEASETIME:-7200} -DOMAIN=${DOMAIN:-local} -NTP=${NTP:-pool.ntp.org} -VMW_NFS_IP=${VMW_NFS_IP:-192.168.1.1} -VMW_GPU_VIB=${VMW_GPU_VIB:-NVIDIA-VMware_ESXi_6.7_Host_Driver-418.66-1OEM.670.0.0.8169922.x86_64.vib} -VMW_VM_SIZE=${VMW_VM_SIZE:-100g} -### - -HTTP_IP=$(ip -4 addr show ${HTTP_INT} | grep -oP '(?<=inet\s)\d+(\.\d+){3}') - -mkdir -p "${ISO}" -mkdir -p "${VISO}" - -# DGX installer ISO -dgx_iso="$(find ${MNT} -name DGXServer\*.iso)" -if [ "z${dgx_iso}" != "z" ] ; then - mount -o loop -t iso9660 "$dgx_iso" "${ISO}" - cp ${PRESEED} /www/dgx.seed - PRESEED=/www/dgx.seed - ln -sf ${ISO} /www/iso - - # Modify preseed - sed -i "s//${HTTP_IP}/g" ${PRESEED} - sed -i "s/^d-i live-installer.*/d-i live-installer\/net-image string ftp:\/\/${HTTP_IP}\/install\/filesystem.squashfs/g" ${PRESEED} - sed -i "s/^d-i mirror\/ftp\/hostname.*/d-i mirror\/ftp\/hostname string ${HTTP_IP}/g" ${PRESEED} - sed -i "s/^d-i apt-setup\/local0\/repository.*/d-i apt-setup\/local0\/repository string deb ftp:\/\/${HTTP_IP}\/ bionic main multiverse restricted universe/g" ${PRESEED} - sed -i "s/^d-i mirror\/ftp\/directory.*/d-i mirror\/ftp\/directory string \//g" ${PRESEED} - sed -i "s/^d-i apt-setup\/security_host.*/d-i apt-setup\/security_host string ${HTTP_IP}/g" ${PRESEED} - - # Try to get install process to fix BIOS boot order on pre-3.18 BMC firmwares; sets DGX to boot from first disk - # Turn off chassis identify light - sed -i "s/in-target sh -c \"logger preseed\/late_command: end\"\;/modprobe -v ipmi_devintf\; \\\\\n\\t in-target sh -c \"sudo ipmitool raw 0x00 0x08 0x05 0xe0 0x08 0x00 0x00 0x00\"\; \\\\\n\\t in-target sh -c \"sudo ipmitool chassis identify 0\"\; \\\\\n\\t in-target sh -c \"curl -X POST -d action=end ${HTTP_IP}:${HTTP_PORT}\/install\"\; \\\\\n\\tin-target sh -c \"logger preseed\/late_command: end\"\;/g" ${PRESEED} - sed -i "s~^d-i mirror/https/proxy.*~d-i mirror/https/proxy string ${DGX_HTTPS_PROXY}~g" ${PRESEED} - sed -i "s~^d-i pkgsel/include string ~d-i pkgsel/include string ${DGX_EXTRA_PACKAGES} ~g" ${PRESEED} - - # Add early command to log install start - #sed -i "s/^# Use default/d-i preseed\/early_command string wget --post-data action=start ${HTTP_IP}\/install\\n\\n# Use default/g" ${PRESEED} - #sed -i "s/early_command string/early_command string wget --post-data action=start 192.168.1.1\/install \;/g" ${PRESEED} - - # Useful for Xen VM - sed -i "s/\/dev\/sda/\/dev\/${DGX_DISK}/g" ${PRESEED} - sed -i "s/enp1s0f0/${DGX_INT}/g" ${PRESEED} - - # FTP - sed -i "s~/srv/ftp~${ISO}~g" /etc/passwd -fi - -# Mount VMware ESXi installer -vmware_iso="$(find ${MNT} -name VMware-VMvisor-Installer\*.iso)" -if [ "z${vmware_iso}" != "z" ] ; then - mount -o loop -t iso9660 "$vmware_iso" "${VISO}" - ln -sf ${VISO} /www/viso - cp "${VISO}/boot.cfg" /www/vmware/boot.cfg - sed -i "s~^kernel=~kernel=http://${HTTP_IP}:${HTTP_PORT}/viso~g" /www/vmware/boot.cfg - sed -i "s~^kernelopt=.*~kernelopt=ks=http://${HTTP_IP}:${HTTP_PORT}/vmware/kickstart.cfg~g" /www/vmware/boot.cfg - sed -i "s~^modules=~modules=http://${HTTP_IP}:${HTTP_PORT}/viso~g" /www/vmware/boot.cfg - sed -i "s~--- /~--- http://${HTTP_IP}:${HTTP_PORT}/viso/~g" /www/vmware/boot.cfg - sed -i "s/\#VMW_NFS_IP\#/${VMW_NFS_IP}/g" /www/vmware/kickstart.cfg - sed -i "s/\#VMW_GPU_VIB\#/${VMW_GPU_VIB}/g" /www/vmware/kickstart.cfg - sed -i "s/\#VMW_VM_SIZE\#/${VMW_VM_SIZE}/g" /www/vmware/kickstart.cfg -fi - -# DHCP Server -sed -i "s/\#DHCP_INT\#/${DHCP_INT}/g" /etc/dnsmasq.conf -sed -i "s/\#LEASETIME\#/${LEASETIME}/g" /etc/dnsmasq.conf -sed -i "s/\#DOMAIN\#/${DOMAIN}/g" /etc/dnsmasq.conf -sed -i "s/\#NETWORK\#/${NETWORK}/g" /etc/dnsmasq.conf -sed -i "s/\#NETMASK\#/${NETMASK}/g" /etc/dnsmasq.conf -sed -i "s/\#GATEWAY\#/${GATEWAY}/g" /etc/dnsmasq.conf -sed -i "s/\#DNS1\#/${DNS1}/g" /etc/dnsmasq.conf -sed -i "s/\#DNS2\#/${DNS2}/g" /etc/dnsmasq.conf -sed -i "s/\#DHCP_START\#/${DHCP_START}/g" /etc/dnsmasq.conf -sed -i "s/\#DHCP_END\#/${DHCP_END}/g" /etc/dnsmasq.conf -if [ ! -z "${NTP}" ]; then - sed -i "s/\#NTP\#/${NTP}/g" /etc/dnsmasq.conf - sed -i "s/\#dhcp-option=42,/dhcp-option=42,/g" /etc/dnsmasq.conf -fi - -# HTTP Server -sed -i "s/\#HTTP_PORT\#/${HTTP_PORT}/g" /etc/nginx/nginx.conf - -# Set up NAT -if [ "${NAT_ENABLE}" -ne 0 ]; then - # NAT - /sbin/iptables -t nat -A POSTROUTING -o ${NAT_INT_PUB} -j MASQUERADE - /sbin/iptables -A FORWARD -i ${NAT_INT_PUB} -o ${NAT_INT_PRV} -m state --state RELATED,ESTABLISHED -j ACCEPT - /sbin/iptables -A FORWARD -i ${NAT_INT_PRV} -o ${NAT_INT_PUB} -j ACCEPT - sysctl -w net.ipv4.ip_forward=1 -fi - -# Run some servers -nginx & -/usr/sbin/vsftpd /etc/vsftpd.conf & -/usr/local/bin/rest_api.py >/dev/null 2>&1 & -python /api.py & -/usr/bin/pixiecore api http://127.0.0.1:${HTTP_PORT} --dhcp-no-bind --debug --port 81 |& tee -a /var/log/pixiecore.log & -if [ ${DHCP_ENABLE} -ne 0 ] ; then - dnsmasq - tail -f /var/log/dnsmasq.log /var/log/pixiecore.log -else - tail -f /var/log/nginx/* /var/log/pixiecore.log -fi diff --git a/src/containers/dgxie/vsftpd.conf b/src/containers/dgxie/vsftpd.conf deleted file mode 100644 index 54ed0efee..000000000 --- a/src/containers/dgxie/vsftpd.conf +++ /dev/null @@ -1,123 +0,0 @@ -# Example config file /etc/vsftpd.conf -# -# The default compiled in settings are fairly paranoid. This sample file -# loosens things up a bit, to make the ftp daemon more usable. -# Please see vsftpd.conf.5 for all compiled in defaults. -# -# READ THIS: This example file is NOT an exhaustive list of vsftpd options. -# Please read the vsftpd.conf.5 manual page to get a full idea of vsftpd's -# capabilities. -# -# Allow anonymous FTP? (Beware - allowed by default if you comment this out). -anonymous_enable=YES -# -# Uncomment this to allow local users to log in. -#local_enable=YES -# -# Uncomment this to enable any form of FTP write command. -#write_enable=YES -# -# Default umask for local users is 077. You may wish to change this to 022, -# if your users expect that (022 is used by most other ftpd's) -#local_umask=022 -# -# Uncomment this to allow the anonymous FTP user to upload files. This only -# has an effect if the above global write enable is activated. Also, you will -# obviously need to create a directory writable by the FTP user. -#anon_upload_enable=YES -# -# Uncomment this if you want the anonymous FTP user to be able to create -# new directories. -#anon_mkdir_write_enable=YES -# -# Activate directory messages - messages given to remote users when they -# go into a certain directory. -dirmessage_enable=YES -# -# Activate logging of uploads/downloads. -xferlog_enable=YES -# -# Make sure PORT transfer connections originate from port 20 (ftp-data). -connect_from_port_20=YES -# -# If you want, you can arrange for uploaded anonymous files to be owned by -# a different user. Note! Using "root" for uploaded files is not -# recommended! -#chown_uploads=YES -#chown_username=whoever -# -# You may override where the log file goes if you like. The default is shown -# below. -#xferlog_file=/var/log/vsftpd.log -# -# If you want, you can have your log file in standard ftpd xferlog format. -# Note that the default log file location is /var/log/xferlog in this case. -#xferlog_std_format=YES -# -# You may change the default value for timing out an idle session. -#idle_session_timeout=600 -# -# You may change the default value for timing out a data connection. -#data_connection_timeout=120 -# -# It is recommended that you define on your system a unique user which the -# ftp server can use as a totally isolated and unprivileged user. -#nopriv_user=ftpsecure -# -# Enable this and the server will recognise asynchronous ABOR requests. Not -# recommended for security (the code is non-trivial). Not enabling it, -# however, may confuse older FTP clients. -#async_abor_enable=YES -# -# By default the server will pretend to allow ASCII mode but in fact ignore -# the request. Turn on the below options to have the server actually do ASCII -# mangling on files when in ASCII mode. -# Beware that on some FTP servers, ASCII support allows a denial of service -# attack (DoS) via the command "SIZE /big/file" in ASCII mode. vsftpd -# predicted this attack and has always been safe, reporting the size of the -# raw file. -# ASCII mangling is a horrible feature of the protocol. -#ascii_upload_enable=YES -#ascii_download_enable=YES -# -# You may fully customise the login banner string: -#ftpd_banner=Welcome to blah FTP service. -# -# You may specify a file of disallowed anonymous e-mail addresses. Apparently -# useful for combatting certain DoS attacks. -#deny_email_enable=YES -# (default follows) -#banned_email_file=/etc/vsftpd.banned_emails -# -# You may specify an explicit list of local users to chroot() to their home -# directory. If chroot_local_user is YES, then this list becomes a list of -# users to NOT chroot(). -# (Warning! chroot'ing can be very dangerous. If using chroot, make sure that -# the user does not have write access to the top level directory within the -# chroot) -#chroot_local_user=YES -#chroot_list_enable=YES -# (default follows) -#chroot_list_file=/etc/vsftpd.chroot_list -# -# You may activate the "-R" option to the builtin ls. This is disabled by -# default to avoid remote users being able to cause excessive I/O on large -# sites. However, some broken FTP clients such as "ncftp" and "mirror" assume -# the presence of the "-R" option, so there is a strong case for enabling it. -#ls_recurse_enable=YES -# -# When "listen" directive is enabled, vsftpd runs in standalone mode and -# listens on IPv4 sockets. This directive cannot be used in conjunction -# with the listen_ipv6 directive. -listen=YES -# -# This directive enables listening on IPv6 sockets. To listen on IPv4 and IPv6 -# sockets, you must run two copies of vsftpd with two configuration files. -# Make sure, that one of the listen options is commented !! -#listen_ipv6=YES -local_enable=YES -passwd_chroot_enable=yes -seccomp_sandbox=NO -pasv_enable=Yes -pasv_max_port=10100 -pasv_min_port=10090 diff --git a/src/containers/pixiecore/Dockerfile b/src/containers/pixiecore/Dockerfile deleted file mode 100644 index d609ed998..000000000 --- a/src/containers/pixiecore/Dockerfile +++ /dev/null @@ -1,26 +0,0 @@ -FROM golang:1.10.2 - -ENV COMMIT_HASH 03425a0d6ae36852d5ea7b446571bbcd3829d717 -ENV CUSTOM_FORK_AUTHOR deepops -RUN apt-get update -RUN apt-get install -qy --no-install-recommends wget git -RUN [ -d ${GOPATH}/bin ] || mkdir ${GOPATH}/bin -RUN go get -u github.com/golang/dep/cmd/dep -RUN mkdir -p ${GOPATH}/src/go.universe.tf -WORKDIR /go/src/go.universe.tf -RUN git clone https://github.com/google/netboot.git -WORKDIR /go/src/go.universe.tf/netboot -RUN git remote add ${CUSTOM_FORK_AUTHOR} https://github.com/${CUSTOM_FORK_AUTHOR}/netboot.git && git fetch ${CUSTOM_FORK_AUTHOR} && git checkout ${COMMIT_HASH} -RUN dep ensure -RUN ls -al ./vendor -WORKDIR /go/src -RUN CGO_ENABLED=0 GOOS=linux go build -o /bin/pixiecore -ldflags "-w -s -v -extldflags -static" go.universe.tf/netboot/cmd/pixiecore - -FROM alpine:3.6 -MAINTAINER Douglas Holt - -RUN apk add --no-cache ca-certificates -COPY --from=0 /bin/pixiecore /usr/bin/pixiecore -RUN chmod +x /usr/bin/pixiecore - -ENTRYPOINT ["/usr/bin/pixiecore"] diff --git a/src/containers/pxe/dhcp/Dockerfile b/src/containers/pxe/dhcp/Dockerfile deleted file mode 100644 index 3555906ab..000000000 --- a/src/containers/pxe/dhcp/Dockerfile +++ /dev/null @@ -1,11 +0,0 @@ -FROM ubuntu:16.04 - -MAINTAINER Douglas Holt - -RUN apt-get update && \ - apt-get -y install dnsmasq - -VOLUME /etc/dnsmasq.d - -#ENTRYPOINT ["dnsmasq"] -CMD ["dnsmasq", "-d"] diff --git a/src/containers/pxe/dhcp/dnsmasq.conf b/src/containers/pxe/dhcp/dnsmasq.conf deleted file mode 100644 index cd6f8d9e5..000000000 --- a/src/containers/pxe/dhcp/dnsmasq.conf +++ /dev/null @@ -1,18 +0,0 @@ -domain-needed -bogus-priv -strict-order -no-resolv -no-poll -expand-hosts -cache-size=2048 -bind-interfaces - -server=8.8.8.8 -server=8.8.4.4 -domain=localdomain - -log-queries -log-dhcp -log-facility=/var/log/dnsmasq.log - -conf-dir=/etc/dnsmasq.d,*.conf diff --git a/src/containers/pxe/docker-compose.yml b/src/containers/pxe/docker-compose.yml deleted file mode 100644 index c03428223..000000000 --- a/src/containers/pxe/docker-compose.yml +++ /dev/null @@ -1,28 +0,0 @@ -version: '3.2' -services: - dhcp: - build: dhcp - network_mode: "host" - privileged: true - volumes: - - type: bind - source: ./dhcp/dnsmasq.conf - target: /etc/dnsmasq.conf - pxe-coreos: - network_mode: "host" - image: danderson/pixiecore - command: ["boot", - "https://alpha.release.core-os.net/amd64-usr/current/coreos_production_pxe.vmlinuz", - "https://alpha.release.core-os.net/amd64-usr/current/coreos_production_pxe_image.cpio.gz", - "--cmdline", - "coreos.autologin", - "--dhcp-no-bind"] - pxe: - network_mode: "host" - image: deepops/pixiecore - command: ["boot", - "http://archive.ubuntu.com/ubuntu/dists/bionic-updates/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/linux", - "http://archive.ubuntu.com/ubuntu/dists/bionic-updates/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/initrd.gz", - "--cmdline", - "auto=true priority=critical preseed/url=https://bit.ly/nvdeepseed", - "--dhcp-no-bind"] diff --git a/src/containers/pxe/preseed b/src/containers/pxe/preseed deleted file mode 100644 index 67a256a11..000000000 --- a/src/containers/pxe/preseed +++ /dev/null @@ -1,491 +0,0 @@ -#### Contents of the preconfiguration file (for stretch) -### Localization -# Preseeding only locale sets language, country and locale. -d-i debian-installer/locale string en_US - -# The values can also be preseeded individually for greater flexibility. -#d-i debian-installer/language string en -#d-i debian-installer/country string NL -#d-i debian-installer/locale string en_US.UTF-8 -# Optionally specify additional locales to be generated. -#d-i localechooser/supported-locales multiselect en_US.UTF-8, nl_NL.UTF-8 - -# Keyboard selection. -# Disable automatic (interactive) keymap detection. -d-i console-setup/ask_detect boolean false -d-i keyboard-configuration/xkb-keymap select us -# To select a variant of the selected layout: -#d-i keyboard-configuration/xkb-keymap select us(dvorak) -# d-i keyboard-configuration/toggle select No toggling - -### Network configuration -# Disable network configuration entirely. This is useful for cdrom -# installations on non-networked devices where the network questions, -# warning and long timeouts are a nuisance. -#d-i netcfg/enable boolean false - -# netcfg will choose an interface that has link if possible. This makes it -# skip displaying a list if there is more than one interface. -d-i netcfg/choose_interface select auto - -# To set a different link detection timeout (default is 3 seconds). -# Values are interpreted as seconds. -#d-i netcfg/link_wait_timeout string 10 - -# If you have a slow dhcp server and the installer times out waiting for -# it, this might be useful. -#d-i netcfg/dhcp_timeout string 60 -#d-i netcfg/dhcpv6_timeout string 60 - -# If you prefer to configure the network manually, uncomment this line and -# the static network configuration below. -#d-i netcfg/disable_autoconfig boolean true - -# If you want the preconfiguration file to work on systems both with and -# without a dhcp server, uncomment these lines and the static network -# configuration below. -#d-i netcfg/dhcp_failed note -#d-i netcfg/dhcp_options select Configure network manually - -# Static network configuration. -# -# IPv4 example -#d-i netcfg/get_ipaddress string 192.168.1.42 -#d-i netcfg/get_netmask string 255.255.255.0 -#d-i netcfg/get_gateway string 192.168.1.1 -#d-i netcfg/get_nameservers string 192.168.1.1 -#d-i netcfg/confirm_static boolean true -# -# IPv6 example -#d-i netcfg/get_ipaddress string fc00::2 -#d-i netcfg/get_netmask string ffff:ffff:ffff:ffff:: -#d-i netcfg/get_gateway string fc00::1 -#d-i netcfg/get_nameservers string fc00::1 -#d-i netcfg/confirm_static boolean true - -# Any hostname and domain names assigned from dhcp take precedence over -# values set here. However, setting the values still prevents the questions -# from being shown, even if values come from dhcp. -d-i netcfg/get_hostname string unassigned-hostname -d-i netcfg/get_domain string unassigned-domain - -# If you want to force a hostname, regardless of what either the DHCP -# server returns or what the reverse DNS entry for the IP is, uncomment -# and adjust the following line. -#d-i netcfg/hostname string ubuntu - -# Disable that annoying WEP key dialog. -d-i netcfg/wireless_wep string -# The wacky dhcp hostname that some ISPs use as a password of sorts. -#d-i netcfg/dhcp_hostname string radish - -# If non-free firmware is needed for the network or other hardware, you can -# configure the installer to always try to load it, without prompting. Or -# change to false to disable asking. -#d-i hw-detect/load_firmware boolean true - -### Network console -# Use the following settings if you wish to make use of the network-console -# component for remote installation over SSH. This only makes sense if you -# intend to perform the remainder of the installation manually. -#d-i anna/choose_modules string network-console -#d-i network-console/authorized_keys_url string http://10.0.0.1/openssh-key -#d-i network-console/password password r00tme -#d-i network-console/password-again password r00tme -# Use this instead if you prefer to use key-based authentication -#d-i network-console/authorized_keys_url http://host/authorized_keys - -### Mirror settings -# If you select ftp, the mirror/country string does not need to be set. -#d-i mirror/protocol string ftp -d-i mirror/country string manual -d-i mirror/http/hostname string archive.ubuntu.com -d-i mirror/http/directory string /ubuntu -d-i mirror/http/proxy string - -# Alternatively: by default, the installer uses CC.archive.ubuntu.com where -# CC is the ISO-3166-2 code for the selected country. You can preseed this -# so that it does so without asking. -#d-i mirror/http/mirror select CC.archive.ubuntu.com - -# Suite to install. -#d-i mirror/suite string stretch -# Suite to use for loading installer components (optional). -#d-i mirror/udeb/suite string stretch -# Components to use for loading installer components (optional). -#d-i mirror/udeb/components multiselect main, restricted - -### Account setup -# Skip creation of a root account (normal user account will be able to -# use sudo). The default is false; preseed this to true if you want to set -# a root password. -#d-i passwd/root-login boolean false -# Alternatively, to skip creation of a normal user account. -#d-i passwd/make-user boolean false - -# Root password, either in clear text -#d-i passwd/root-password password r00tme -#d-i passwd/root-password-again password r00tme -# or encrypted using a crypt(3) hash. -#d-i passwd/root-password-crypted password [crypt(3) hash] - -# To create a normal user account. -d-i passwd/user-fullname string Ubuntu User -d-i passwd/username string ubuntu -# Normal user's password, either in clear text -d-i passwd/user-password password deepops -d-i passwd/user-password-again password deepops -# or encrypted using a crypt(3) hash. -#d-i passwd/user-password-crypted password [crypt(3) hash] -# Create the first user with the specified UID instead of the default. -#d-i passwd/user-uid string 1010 -# The installer will warn about weak passwords. If you are sure you know -# what you're doing and want to override it, uncomment this. -d-i user-setup/allow-password-weak boolean true - -# The user account will be added to some standard initial groups. To -# override that, use this. -#d-i passwd/user-default-groups string audio cdrom video - -# Set to true if you want to encrypt the first user's home directory. -d-i user-setup/encrypt-home boolean false - -### Clock and time zone setup -# Controls whether or not the hardware clock is set to UTC. -d-i clock-setup/utc boolean true - -# You may set this to any valid setting for $TZ; see the contents of -# /usr/share/zoneinfo/ for valid values. -d-i time/zone string US/Pacific - -# Controls whether to use NTP to set the clock during the install -d-i clock-setup/ntp boolean true -# NTP server to use. The default is almost always fine here. -#d-i clock-setup/ntp-server string ntp.example.com - -### i386 specific disk storage -# Activate DASD disks -#d-i s390-dasd/dasd string 0.0.0200,0.0.0300,0.0.0400 - -# DASD configuration; by default dasdfmt (low-level format) if needed -#d-i s390-dasd/auto-format boolean true -#d-i s390-dasd/force-format boolean true - -# zFCP activation and configuration -# d-i s390-zfcp/zfcp string 0.0.1b34:0x400870075678a1b2:0x201480c800000000, \ -# 0.0.1b34:0x400870075679a1b2:0x201480c800000000 - -### Partitioning -## Partitioning example -# If the system has free space you can choose to only partition that space. -# This is only honoured if partman-auto/method (below) is not set. -# Alternatives: custom, some_device, some_device_crypto, some_device_lvm. -#d-i partman-auto/init_automatically_partition select biggest_free - -# Alternatively, you may specify a disk to partition. If the system has only -# one disk the installer will default to using that, but otherwise the device -# name must be given in traditional, non-devfs format (so e.g. /dev/sda -# and not e.g. /dev/discs/disc0/disc). -# For example, to use the first SCSI/SATA hard disk: -#d-i partman-auto/disk string /dev/sda -# In addition, you'll need to specify the method to use. -# The presently available methods are: -# - regular: use the usual partition types for your architecture -# - lvm: use LVM to partition the disk -# - crypto: use LVM within an encrypted partition -d-i partman-auto/method string lvm - -# If one of the disks that are going to be automatically partitioned -# contains an old LVM configuration, the user will normally receive a -# warning. This can be preseeded away... -d-i partman-lvm/device_remove_lvm boolean true -# The same applies to pre-existing software RAID array: -d-i partman-md/device_remove_md boolean true -# And the same goes for the confirmation to write the lvm partitions. -d-i partman-lvm/confirm boolean true -d-i partman-lvm/confirm_nooverwrite boolean true - -# For LVM partitioning, you can select how much of the volume group to use -# for logical volumes. -d-i partman-auto-lvm/guided_size string max -#d-i partman-auto-lvm/guided_size string 10GB -#d-i partman-auto-lvm/guided_size string 50% - -# You can choose one of the three predefined partitioning recipes: -# - atomic: all files in one partition -# - home: separate /home partition -# - multi: separate /home, /var, and /tmp partitions -d-i partman-auto/choose_recipe select atomic - -# Or provide a recipe of your own... -# If you have a way to get a recipe file into the d-i environment, you can -# just point at it. -#d-i partman-auto/expert_recipe_file string /hd-media/recipe - -# If not, you can put an entire recipe into the preconfiguration file in one -# (logical) line. This example creates a small /boot partition, suitable -# swap, and uses the rest of the space for the root partition: -#d-i partman-auto/expert_recipe string \ -# boot-root :: \ -# 40 50 100 ext3 \ -# $primary{ } $bootable{ } \ -# method{ format } format{ } \ -# use_filesystem{ } filesystem{ ext3 } \ -# mountpoint{ /boot } \ -# . \ -# 500 10000 1000000000 ext3 \ -# method{ format } format{ } \ -# use_filesystem{ } filesystem{ ext3 } \ -# mountpoint{ / } \ -# . \ -# 64 512 300% linux-swap \ -# method{ swap } format{ } \ -# . - -# If you just want to change the default filesystem from ext3 to something -# else, you can do that without providing a full recipe. -#d-i partman/default_filesystem string ext4 - -# The full recipe format is documented in the file partman-auto-recipe.txt -# included in the 'debian-installer' package or available from D-I source -# repository. This also documents how to specify settings such as file -# system labels, volume group names and which physical devices to include -# in a volume group. - -# This makes partman automatically partition without confirmation, provided -# that you told it what to do using one of the methods above. -d-i partman-partitioning/confirm_write_new_label boolean true -d-i partman/choose_partition select finish -d-i partman/confirm boolean true -d-i partman/confirm_nooverwrite boolean true - -## Partitioning using RAID -# The method should be set to "raid". -#d-i partman-auto/method string raid -# Specify the disks to be partitioned. They will all get the same layout, -# so this will only work if the disks are the same size. -#d-i partman-auto/disk string /dev/sda /dev/sdb - -# Next you need to specify the physical partitions that will be used. -#d-i partman-auto/expert_recipe string \ -# multiraid :: \ -# 1000 5000 4000 raid \ -# $primary{ } method{ raid } \ -# . \ -# 64 512 300% raid \ -# method{ raid } \ -# . \ -# 500 10000 1000000000 raid \ -# method{ raid } \ -# . - -# Last you need to specify how the previously defined partitions will be -# used in the RAID setup. Remember to use the correct partition numbers -# for logical partitions. RAID levels 0, 1, 5, 6 and 10 are supported; -# devices are separated using "#". -# Parameters are: -# \ -# - -#d-i partman-auto-raid/recipe string \ -# 1 2 0 ext3 / \ -# /dev/sda1#/dev/sdb1 \ -# . \ -# 1 2 0 swap - \ -# /dev/sda5#/dev/sdb5 \ -# . \ -# 0 2 0 ext3 /home \ -# /dev/sda6#/dev/sdb6 \ -# . - -# For additional information see the file partman-auto-raid-recipe.txt -# included in the 'debian-installer' package or available from D-I source -# repository. - -# This makes partman automatically partition without confirmation. -d-i partman-md/confirm boolean true -d-i partman-partitioning/confirm_write_new_label boolean true -d-i partman/choose_partition select finish -d-i partman/confirm boolean true -d-i partman/confirm_nooverwrite boolean true - -## Controlling how partitions are mounted -# The default is to mount by UUID, but you can also choose "traditional" to -# use traditional device names, or "label" to try filesystem labels before -# falling back to UUIDs. -#d-i partman/mount_style select uuid - -### Base system installation -# Configure a path to the preconfigured base filesystem. This can be used to -# specify a path for the installer to retrieve the filesystem image that will -# be deployed to disk and used as a base system for the installation. -#d-i live-installer/net-image string /install/filesystem.squashfs - -# Configure APT to not install recommended packages by default. Use of this -# option can result in an incomplete system and should only be used by very -# experienced users. -#d-i base-installer/install-recommends boolean false - -# The kernel image (meta) package to be installed; "none" can be used if no -# kernel is to be installed. -#d-i base-installer/kernel/image string linux-generic - -### Apt setup -# You can choose to install restricted and universe software, or to install -# software from the backports repository. -#d-i apt-setup/restricted boolean true -#d-i apt-setup/universe boolean true -#d-i apt-setup/backports boolean true -# Uncomment this if you don't want to use a network mirror. -#d-i apt-setup/use_mirror boolean false -# Select which update services to use; define the mirrors to be used. -# Values shown below are the normal defaults. -#d-i apt-setup/services-select multiselect security -#d-i apt-setup/security_host string security.ubuntu.com -#d-i apt-setup/security_path string /ubuntu - -# Additional repositories, local[0-9] available -#d-i apt-setup/local0/repository string \ -# http://local.server/ubuntu stretch main -#d-i apt-setup/local0/comment string local server -# Enable deb-src lines -#d-i apt-setup/local0/source boolean true -# URL to the public key of the local repository; you must provide a key or -# apt will complain about the unauthenticated repository and so the -# sources.list line will be left commented out -#d-i apt-setup/local0/key string http://local.server/key - -# By default the installer requires that repositories be authenticated -# using a known gpg key. This setting can be used to disable that -# authentication. Warning: Insecure, not recommended. -#d-i debian-installer/allow_unauthenticated boolean true - -# Uncomment this to add multiarch configuration for i386 -#d-i apt-setup/multiarch string i386 - - -### Package selection -tasksel tasksel/first multiselect server -#tasksel tasksel/first multiselect ubuntu-desktop -#tasksel tasksel/first multiselect lamp-server, print-server -#tasksel tasksel/first multiselect kubuntu-desktop - -# Individual additional packages to install -d-i pkgsel/include string openssh-server -# Whether to upgrade packages after debootstrap. -# Allowed values: none, safe-upgrade, full-upgrade -d-i pkgsel/upgrade select none - -# Language pack selection -#d-i pkgsel/language-packs multiselect de, en, zh - -# Policy for applying updates. May be "none" (no automatic updates), -# "unattended-upgrades" (install security updates automatically), or -# "landscape" (manage system with Landscape). -d-i pkgsel/update-policy select none - -# Some versions of the installer can report back on what software you have -# installed, and what software you use. The default is not to report back, -# but sending reports helps the project determine what software is most -# popular and include it on CDs. -popularity-contest popularity-contest/participate boolean false - -# By default, the system's locate database will be updated after the -# installer has finished installing most packages. This may take a while, so -# if you don't want it, you can set this to "false" to turn it off. -d-i pkgsel/updatedb boolean false - -### Boot loader installation -# Grub is the default boot loader (for x86). If you want lilo installed -# instead, uncomment this: -#d-i grub-installer/skip boolean true -# To also skip installing lilo, and install no bootloader, uncomment this -# too: -#d-i lilo-installer/skip boolean true - - -# This is fairly safe to set, it makes grub install automatically to the MBR -# if no other operating system is detected on the machine. -d-i grub-installer/only_debian boolean true - -# This one makes grub-installer install to the MBR if it also finds some other -# OS, which is less safe as it might not be able to boot that other OS. -d-i grub-installer/with_other_os boolean true - -# Due notably to potential USB sticks, the location of the MBR can not be -# determined safely in general, so this needs to be specified: -#d-i grub-installer/bootdev string /dev/sda -# To install to the first device (assuming it is not a USB stick): -#d-i grub-installer/bootdev string default - -# Alternatively, if you want to install to a location other than the mbr, -# uncomment and edit these lines: -#d-i grub-installer/only_debian boolean false -#d-i grub-installer/with_other_os boolean false -#d-i grub-installer/bootdev string (hd0,1) -# To install grub to multiple disks: -#d-i grub-installer/bootdev string (hd0,1) (hd1,1) (hd2,1) - -# Optional password for grub, either in clear text -#d-i grub-installer/password password r00tme -#d-i grub-installer/password-again password r00tme -# or encrypted using an MD5 hash, see grub-md5-crypt(8). -#d-i grub-installer/password-crypted password [MD5 hash] - -# Use the following option to add additional boot parameters for the -# installed system (if supported by the bootloader installer). -# Note: options passed to the installer will be added automatically. -d-i debian-installer/add-kernel-opts string nouveau.modeset=0 rd.driver.blacklist=nouveau - -### Finishing up the installation -# During installations from serial console, the regular virtual consoles -# (VT1-VT6) are normally disabled in /etc/inittab. Uncomment the next -# line to prevent this. -#d-i finish-install/keep-consoles boolean true - -# Avoid that last message about the install being complete. -d-i finish-install/reboot_in_progress note - -# This will prevent the installer from ejecting the CD during the reboot, -# which is useful in some situations. -#d-i cdrom-detect/eject boolean false - -# This is how to make the installer shutdown when finished, but not -# reboot into the installed system. -#d-i debian-installer/exit/halt boolean true -# This will power off the machine instead of just halting it. -#d-i debian-installer/exit/poweroff boolean true - -### Preseeding other packages -# Depending on what software you choose to install, or if things go wrong -# during the installation process, it's possible that other questions may -# be asked. You can preseed those too, of course. To get a list of every -# possible question that could be asked during an install, do an -# installation, and then run these commands: -# debconf-get-selections --installer > file -# debconf-get-selections >> file - - -#### Advanced options -### Running custom commands during the installation -## i386 Preseed Example -# d-i preseeding is inherently not secure. Nothing in the installer checks -# for attempts at buffer overflows or other exploits of the values of a -# preconfiguration file like this one. Only use preconfiguration files from -# trusted locations! To drive that home, and because it's generally useful, -# here's a way to run any shell command you'd like inside the installer, -# automatically. - -# This first command is run as early as possible, just after -# preseeding is read. -#d-i preseed/early_command string anna-install some-udeb -# This command is run immediately before the partitioner starts. It may be -# useful to apply dynamic partitioner preseeding that depends on the state -# of the disks (which may not be visible when preseed/early_command runs). -#d-i partman/early_command \ -# string debconf-set partman-auto/disk "$(list-devices disk | head -n1)" -# This command is run just before the install finishes, but when there is -# still a usable /target directory. You can chroot to /target and use it -# directly, or use the apt-install and in-target commands to easily install -# packages and run commands in the target system. -#d-i preseed/late_command string apt-install zsh; in-target chsh -s /bin/zsh diff --git a/workloads/examples/k8s/services/nfs-dgx-iso.yml b/workloads/examples/k8s/services/nfs-dgx-iso.yml deleted file mode 100644 index 367fa37cb..000000000 --- a/workloads/examples/k8s/services/nfs-dgx-iso.yml +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: v1 -kind: PersistentVolume -metadata: - name: nfs-dgx-iso -spec: - capacity: - storage: 5Gi - accessModes: - - ReadOnlyMany - nfs: - server: # - path: "/path/to/iso/DGXServer-3.1.2.170902_f8777e" ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: nfs-dgx-iso -spec: - accessModes: - - ReadOnlyMany - storageClassName: "" - resources: - requests: - storage: 5Gi diff --git a/workloads/examples/k8s/services/pxe.yml b/workloads/examples/k8s/services/pxe.yml deleted file mode 100644 index f4a607324..000000000 --- a/workloads/examples/k8s/services/pxe.yml +++ /dev/null @@ -1,27 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: pxe-server -spec: - hostNetwork: true - containers: - - name: pxe-server - image: deepops/provision/pxe # change me - volumeMounts: - - name: config-volume - mountPath: /data - - name: nfs - mountPath: "/iso" - imagePullSecrets: - - name: secret # change me - volumes: - - name: config-volume - configMap: - name: pxe - items: - - key: machines.json - path: machines.json - - name: nfs - persistentVolumeClaim: - claimName: nfs-dgx-iso - restartPolicy: Never diff --git a/workloads/services/k8s/dgxie/.helmignore b/workloads/services/k8s/dgxie/.helmignore deleted file mode 100644 index f0c131944..000000000 --- a/workloads/services/k8s/dgxie/.helmignore +++ /dev/null @@ -1,21 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*~ -# Various IDEs -.project -.idea/ -*.tmproj diff --git a/workloads/services/k8s/dgxie/Chart.yaml b/workloads/services/k8s/dgxie/Chart.yaml deleted file mode 100644 index 867116268..000000000 --- a/workloads/services/k8s/dgxie/Chart.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: v1 -appVersion: "1.0" -description: A Helm chart for Kubernetes -name: dgxie -version: 0.1.2 diff --git a/workloads/services/k8s/dgxie/templates/NOTES.txt b/workloads/services/k8s/dgxie/templates/NOTES.txt deleted file mode 100644 index e69de29bb..000000000 diff --git a/workloads/services/k8s/dgxie/templates/_helpers.tpl b/workloads/services/k8s/dgxie/templates/_helpers.tpl deleted file mode 100644 index 2474f1dc3..000000000 --- a/workloads/services/k8s/dgxie/templates/_helpers.tpl +++ /dev/null @@ -1,32 +0,0 @@ -{{/* vim: set filetype=mustache: */}} -{{/* -Expand the name of the chart. -*/}} -{{- define "dgxie.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "dgxie.fullname" -}} -{{- if .Values.fullnameOverride -}} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- if contains $name .Release.Name -}} -{{- .Release.Name | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} -{{- end -}} -{{- end -}} -{{- end -}} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "dgxie.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} -{{- end -}} diff --git a/workloads/services/k8s/dgxie/templates/deployment.yaml b/workloads/services/k8s/dgxie/templates/deployment.yaml deleted file mode 100644 index 6c4a30e8a..000000000 --- a/workloads/services/k8s/dgxie/templates/deployment.yaml +++ /dev/null @@ -1,139 +0,0 @@ -apiVersion: apps/v1beta2 -kind: Deployment -metadata: - name: {{ template "dgxie.fullname" . }} - labels: - app: {{ template "dgxie.name" . }} - chart: {{ template "dgxie.chart" . }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} -spec: - replicas: {{ .Values.replicaCount }} - selector: - matchLabels: - app: {{ template "dgxie.name" . }} - release: {{ .Release.Name }} - template: - metadata: - labels: - app: {{ template "dgxie.name" . }} - release: {{ .Release.Name }} - spec: - hostNetwork: true - containers: - - name: {{ .Chart.Name }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - securityContext: - privileged: true - volumeMounts: - - name: dhcp-config-volume - mountPath: "/etc/dnsmasq.d" - - name: pxe-machines-vol - mountPath: "/etc/machines/" - - name: dhcp-leases - mountPath: "/var/lib/misc/" - - name: cephfs - mountPath: "{{ .Values.mntPath }}" - env: - # Path in container to storage containing ISO image - - name: MNT - value: "{{ .Values.mntPath }}" - # Path in container to mounted DGX ISO - - name: ISO - value: "{{ .Values.isoPath }}" - # Default DGX boot mode - # Options: - # "DGX": boot DGX to install media - # "local": boot DGX to local disk - - name: DEFAULT - value: "{{ .Values.bootMode }}" - # DGX network interface to use during install - - name: INT - value: "{{ .Values.dgxNetInt }}" - # DGX disk to use during install - - name: DISK - value: "{{ .Values.dgxDisk }}" - # DGX Keyboard layout to use - - name: KBD - value: "{{ .Values.dgxKbd }}" - # Extra kernel parameters to pass during DGX OS install - # i.e. "rebuild-raid" - - name: KERN_EXTRA - value: "{{ .Values.dgxKernExtra }}" - # Network interface of public network on management servers - - name: HOST_INT_PUB - value: "{{ .Values.mgmtIntPub }}" - # Network interface of private network where DGX are connected on management servers - - name: HOST_INT_PRV - value: "{{ .Values.mgmtIntPrv }}" - # Network domain - - name: DOMAIN - value: "{{ .Values.netDomain }}" - # IP address of private network interface on management server - - name: IP - value: "{{ .Values.netPrvIp }}" - # Private Network - - name: NETWORK - value: "{{ .Values.netPrvNet }}" - # Private network netmask - - name: NETMASK - value: "{{ .Values.netPrvNetmask }}" - # Private network gateway - - name: GATEWAY - value: "{{ .Values.netPrvGateway }}" - # DNS nameservers - - name: DNS1 - value: "{{ .Values.netPrvDns1 }}" - - name: DNS2 - value: "{{ .Values.netPrvDns2 }}" - # DHCP dynamic address range - - name: DHCP_START - value: "{{ .Values.netPrvDhcpStart }}" - - name: DHCP_END - value: "{{ .Values.netPrvDhcpEnd }}" - # DHCP lease time - - name: LEASETIME - value: "{{ .Values.netPrvDhcpLease }}" - # HTTP port for HTTP server - - name: HTTP_PORT - value: "{{ .Values.httpPort }}" - - name: HTTPS_PROXY - value: "{{ .Values.httpsProxy }}" - - name: EXTRA_PACKAGES - value: "{{ .Values.extraPackages }}" - - name: NTP - value: "{{ .Values.ntp }}" - resources: -{{ toYaml .Values.resources | indent 12 }} - {{- with .Values.nodeSelector }} - nodeSelector: -{{ toYaml . | indent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: -{{ toYaml . | indent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: -{{ toYaml . | indent 8 }} - {{- end }} - volumes: - - name: dhcp-config-volume - configMap: - name: dhcpd - - name: pxe-machines-vol - configMap: - name: pxe-machines - - name: cephfs - flexVolume: - driver: ceph.rook.io/rook - fsType: ceph - options: - fsName: cephfs - clusterNamespace: rook-ceph - path: /iso - - name: dhcp-leases - hostPath: - path: /var/lib/dhcp - type: DirectoryOrCreate diff --git a/workloads/services/k8s/dgxie/values.yaml b/workloads/services/k8s/dgxie/values.yaml deleted file mode 100644 index bacfcb832..000000000 --- a/workloads/services/k8s/dgxie/values.yaml +++ /dev/null @@ -1,56 +0,0 @@ -# Default values for dgxie. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -# DGXie config -bootMode: DGX -mntPath: /data -isoPath: /iso -dgxNetInt: enp1s0f0 -dgxDisk: sda -dgxKbd: us -dgxKernExtra: '' -mgmtIntPub: eth0 -mgmtIntPrv: eth1 -netDomain: local -netPrvIp: 192.168.1.1 -netPrvNet: 192.168.1.0 -netPrvNetmask: 255.255.255.0 -netPrvGateway: 192.168.1.1 -netPrvDns1: 192.168.1.1 -netPrvDns2: 8.8.8.8 -netPrvDhcpStart: 192.168.1.100 -netPrvDhcpEnd: 192.168.1.199 -netPrvDhcpLease: 7200 -httpPort: 13370 -httpsProxy: '' -extraPackages: '' - -# Deployment config -replicaCount: 1 - -ntp: '' - -image: - repository: deepops/dgxie - tag: latest - pullPolicy: IfNotPresent - -resources: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - -nodeSelector: - node-role.kubernetes.io/control-plane: "" - -tolerations: [] - -affinity: {} diff --git a/workloads/services/k8s/iso-loader.yml b/workloads/services/k8s/iso-loader.yml deleted file mode 100644 index 33558d2dd..000000000 --- a/workloads/services/k8s/iso-loader.yml +++ /dev/null @@ -1,32 +0,0 @@ -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: iso-loader -spec: - replicas: 1 - template: - metadata: - labels: - app: iso-loader - spec: - containers: - - name: iso-loader - image: ubuntu:16.04 - # /bin/bash -c "trap : TERM INT; sleep infinity & wait" - command: ["/bin/bash", "-c", "--"] - args: ["trap : TERM INT; mkdir -p /data/iso; sleep infinity & wait"] - resources: - limits: - cpu: 100m - memory: 100Mi - volumeMounts: - - name: cephfs - mountPath: /data - volumes: - - name: cephfs - flexVolume: - driver: ceph.rook.io/rook - fsType: ceph - options: - fsName: cephfs - clusterNamespace: rook-ceph From 0df3a981eb9334ed0faf3573758200c85f77cbc4 Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Thu, 28 May 2026 11:51:00 -0600 Subject: [PATCH 19/27] docs: clarify legacy CI and virtual lab status --- README.md | 2 +- docs/deepops/testing.md | 21 ++++++++++++--------- virtual/README.md | 15 +++++++++------ virtual/vagrant_startup.sh | 4 ++-- workloads/jenkins/README.md | 12 ++++++++---- 5 files changed, 32 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 68df95a20..2a5a0b7ee 100644 --- a/README.md +++ b/README.md @@ -83,7 +83,7 @@ The `nvidia-dgx` role can install NVIDIA DGX platform software on supported DGX ### Virtual -To try DeepOps before deploying it on an actual cluster, a virtualized version of DeepOps may be deployed on a single node using Vagrant. This can be used for testing, adding new features, or configuring DeepOps to meet deployment-specific needs. +To try DeepOps before deploying it on an actual cluster, a virtualized version of DeepOps may be deployed on a single node using Vagrant. This path is useful for learning and local experimentation, but it is a legacy/community-supported lab path and should not be treated as release-grade validation for current GPU clusters. Consult the [Virtual DeepOps Deployment Guide](virtual/README.md) to build a GPU-enabled virtual cluster with DeepOps. diff --git a/docs/deepops/testing.md b/docs/deepops/testing.md index c42b116f6..c4fc6c3b6 100644 --- a/docs/deepops/testing.md +++ b/docs/deepops/testing.md @@ -27,25 +27,28 @@ This can be useful for excluding specific roles that have known issues or are st ## DeepOps end-to-end testing -The DeepOps project leverages a private Jenkins server to run continuous integration tests. Testing is done using the [virtual](../../virtual) deployment mechanism. Several Vagrant VMs are created, the cluster is deployed, tests are executed, and then the VMs are destroyed. +Public DeepOps pull requests are validated with GitHub Actions for setup, linting, CodeQL, and selected Molecule role tests. Those checks catch many packaging and role-regression issues, but they do not replace deployment validation on real GPU systems. -The goal of the DeepOps CI is to prevent bugs from being introduced into the code base and to identify when changes in 3rd party platforms have occurred or impacted the DeepOps deployment mechanisms. In general, K8s and Slurm deployment issues are detected and resolved with urgency. Many components of DeepOps are 3rd party open source tools that may silently fail or suddenly change without notice. The team will make a best-effort to resolve these issues and include regression tests, however there may be times where a fix is unavailable. Historically, this has been an issue with Rook-Ceph and Kubeflow, and those GitHub communities are best equipped to help with resolutions. +DeepOps also retains a legacy Jenkins/Vagrant test harness in the [jenkins](../../workloads/jenkins) and [virtual](../../virtual) directories. Treat those files as community-supported reference material unless maintainers explicitly say a Jenkins job is still authoritative. New release validation should record the exact environment, operating system, GPU stack, and workload checks used for the pull request or release. -### Testing Method +The goal of DeepOps validation is to prevent bugs from being introduced into the code base and to identify when changes in third-party platforms have affected the DeepOps deployment mechanisms. In general, Kubernetes and Slurm deployment issues are detected and resolved with urgency. Many components of DeepOps are third-party open source tools that may silently fail or change without notice. The team will make a best-effort to resolve these issues and include regression tests, however there may be times where a fix is unavailable. Historically, this has been an issue with Rook-Ceph and Kubeflow, and those GitHub communities are best equipped to help with resolutions. -DeepOps CI contains two types of automated tests: +### Testing Method -- Nightly tests. These are more exhaustive and run on a nightly basis against the `master` branch. +DeepOps currently uses these testing layers: -- PR tests. These are faster and are executed against every open PR when commits are made to `master`. They are also when a commit is made to any DeepOps branch (`release-20.12`, `master`, etc.). Results are integrated into GitHub. +- GitHub Actions for public pull request checks, including setup, linting, CodeQL, and selected Molecule role tests. +- Focused local validation for changed playbooks, roles, scripts, and documentation before opening or updating a pull request. +- GPU-backed deployment validation for changes that affect Slurm, Kubernetes, drivers, container runtimes, DGX platform software, or workload examples. +- Legacy Jenkins/Vagrant jobs as reference material for operators who still run that harness. -In addition to the automated tests, we also provide developers the a method to manually kick off a test run against one or more deployment configurations in parallel from the below testing matrix through the [Jenkins-matrix](../../workloads/jenkins/Jenkinsfile-matrix) Jenkinsfile. +If a change requires GPU-backed validation, document the validation environment and results in the pull request. If that validation cannot be run, state the gap explicitly instead of relying on the legacy Jenkins matrix. ### Tests -A short description of the nightly testing is outlined below. The full suit of tests can be reviewed in the [jenkins](../../workloads/jenkins) directory. Additional details can be found [here](../../workloads/jenkins/README.md). +A short description of the historical Jenkins test matrix is outlined below. The full suite of legacy jobs can be reviewed in the [jenkins](../../workloads/jenkins) directory. These rows are not a promise of current public CI coverage; check the pull request's GitHub Actions and validation notes for current status. -**Testing Matrix** +**Legacy Jenkins Testing Matrix** | Test | [PR](../../workloads/jenkins/Jenkinsfile) | [Nightly](../../workloads/jenkins/Jenkinsfile-nightly) | [Nightly Multi-node](../../workloads/jenkins/Jenkinsfile-multi-nightly) | Comments | | --------------------------------------------------- | ----------------------------------------- | ------------------------------------------------------ | ----------------------------------------------------------------------- | ------------------------------------ | diff --git a/virtual/README.md b/virtual/README.md index 72d4156b6..19db58a21 100644 --- a/virtual/README.md +++ b/virtual/README.md @@ -1,9 +1,11 @@ # DeepOps Virtual -Set up a virtual cluster with DeepOps. Useful for... +Set up a virtual cluster with DeepOps. This is a legacy/community-supported lab +path for learning and local experimentation; it is not the release validation +path for current GPU clusters. Useful for... 1. Learning how to deploy DeepOps on limited hardware -2. Testing new features in DeepOps +2. Testing small changes in a local lab before validating on real systems 3. Tailoring DeepOps in a local environment before deploying it to the production cluster ## Requirements @@ -26,6 +28,8 @@ If deploying kubeflow or another resource-intensive application in this environm Running DeepOps virtually assumes that the host machine's OS is an approved OS. If this is not the case, the scripts used in the steps below may be modified to work with a different OS. +The Vagrantfiles currently cover Ubuntu 18.04, Ubuntu 20.04, Ubuntu 22.04, CentOS 7, and CentOS 8. The startup script defaults to Ubuntu 20.04. Newer operating systems should be validated on real target systems unless the Vagrantfiles have been refreshed and tested for that release. + Also, using VMs and optionally GPU passthrough assumes that the host machine has been configured to enable virtualization in the BIOS. For instructions on how to accomplish this, refer to the sections at the bottom of this README: [Enabling virtualization and GPU passthrough](#enabling-virtualization-and-gpu-passthrough). ## Start the Virtual Cluster @@ -41,7 +45,7 @@ Also, using VMs and optionally GPU passthrough assumes that the host machine has 2. In the virtual directory, startup vagrant. This will start 3 VMs by default. ```sh - # NOTE: The default VM OS is Ubuntu. If you wish the VMs to spawn CentOS, + # NOTE: The default VM OS is Ubuntu 20.04. If you wish the VMs to spawn CentOS, # configure the DEEPOPS_VAGRANT_FILE variable accordingly... # export DEEPOPS_VAGRANT_FILE=$(pwd)/Vagrantfile-centos # NOTE: virtual-gpu01 requires GPU passthrough, by default it is not enabled @@ -135,7 +139,7 @@ $ lspci -nnk | grep NVIDIA In this example, the GPU at `08:00.0` is chosen. -In the `Vagrantfile` there is a "magic string" `#BUS-GPU01` that is utilized in Jenkins automation. This can be updated manually. +In the `Vagrantfile` there is a "magic string" `#BUS-GPU01` that was used by the legacy Jenkins automation. This can be updated manually. Uncomment the `#BUS-GPU01 v.pci` configuration and update it with a mapping to the bus discovered with `lspci`... @@ -159,7 +163,7 @@ The default Vagrantfiles create VMs that are very minimal in terms of resources ### Specify the cluster Operating System -By default, all virtual nodes will deploy with Ubuntu 18.04. This can be changed by overriding the environment variables `DEEPOPS_VAGRANT_OS` and `DEEPOPS_OS_VERSION`. Supported OS versions are Ubuntu 18.04, Ubuntu 20.04, CentOS 7, and CentOS 8. +By default, all virtual nodes deploy with Ubuntu 20.04. This can be changed by overriding the environment variables `DEEPOPS_VAGRANT_OS` and `DEEPOPS_OS_VERSION`. Available Vagrantfiles include Ubuntu 18.04, Ubuntu 20.04, Ubuntu 22.04, CentOS 7, and CentOS 8. ```sh export DEEPOPS_VAGRANT_OS=centos @@ -355,4 +359,3 @@ $ lspci -nnk -d 10de:1db1 ``` - diff --git a/virtual/vagrant_startup.sh b/virtual/vagrant_startup.sh index f7f9bf1b2..6dd3aaf38 100755 --- a/virtual/vagrant_startup.sh +++ b/virtual/vagrant_startup.sh @@ -5,8 +5,8 @@ set -xe # Get absolute path for script, and convenience vars for virtual and root VIRT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -# The default Vagrant Operating System is Ubuntu 18.04 -# To override thise, change these variables to a supported OS +# The default Vagrant operating system is Ubuntu 20.04. +# To override this, change these variables to a supported OS. DEEPOPS_VAGRANT_OS=${DEEPOPS_VAGRANT_OS:-ubuntu} DEEPOPS_OS_VERSION=${DEEPOPS_OS_VERSION:-20.04} diff --git a/workloads/jenkins/README.md b/workloads/jenkins/README.md index 9f8ea1abb..c97f719f4 100644 --- a/workloads/jenkins/README.md +++ b/workloads/jenkins/README.md @@ -1,8 +1,13 @@ # Jenkins Files -We have several Jenkinsfiles. There is one that is meant to be a lightweight verification that quickly runs against all PRs. +This directory contains the legacy Jenkins/Vagrant test harness. Current public +pull request checks run through GitHub Actions; do not assume these Jenkins jobs +are authoritative unless maintainers explicitly enable and reference them for a +specific validation run. -In addition to that we have several which are meant to run nightly and be more robust checks on functionality to check if dependencies have broken. +We have several Jenkinsfiles. There is one that was meant to be a lightweight verification that quickly runs against all PRs. + +In addition to that we have several which were meant to run nightly and be more robust checks on functionality to check if dependencies have broken. ## Configuration @@ -14,7 +19,7 @@ lock(resource: null, label: 'gpu', quantity: 1, variable: 'GPUDATA') ## Jenkinsfile -This is the original Jenkinsfile that runs every time a PR is created. It does a quick test to verify: +This is the original Jenkinsfile that ran every time a PR was created. It does a quick test to verify: * K8S deploys * Slurm Deploys @@ -36,4 +41,3 @@ This does everything `Jenkinsfile-nightly` does in addition to: * Deploys 3 management nodes * Deploys 2 GPU nodes * Runs a multi-node GPU Verification - From d59fc62d83bc675f19f2ff4da0b5ca24e8449747 Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Thu, 28 May 2026 11:10:55 -0600 Subject: [PATCH 20/27] Refresh Container Toolkit airgap guidance --- docs/airgap/mirror-http-files.md | 5 ++--- docs/airgap/ngc-ready.md | 20 +++++++------------- 2 files changed, 9 insertions(+), 16 deletions(-) diff --git a/docs/airgap/mirror-http-files.md b/docs/airgap/mirror-http-files.md index 9cfe2191b..098e8ec95 100644 --- a/docs/airgap/mirror-http-files.md +++ b/docs/airgap/mirror-http-files.md @@ -34,8 +34,7 @@ Then, for any files you need to make available for download, simply copy these f To configure DeepOps to make use of this server, you will need to configure the specific variables for the download URL of each file. The best place to find these variables is the `defaults/main.yml` file of each role you plan to make use of. -For example, in the [nvidia-docker role](https://github.com/NVIDIA/ansible-role-nvidia-docker), the wrapper script is downloaded from the [nvidia-docker Github repositories](https://raw.githubusercontent.com/NVIDIA/nvidia-docker/master/nvidia-docker). - -To mirror this file offline, you would download the script and place it on your offline repo server, then configure the `nvidia_docker_wrapper_url` variable with the alternate URL. +Current NVIDIA Container Toolkit installs use package repositories and `nvidia-ctk` runtime configuration rather than a standalone wrapper script. +Mirror those package repositories with the Apt or RPM mirror workflows, then point the `nvidia_container_toolkit_*` repository variables at your local mirrors. Documentation on specific offline workflows (such as the [NGC ready offline doc](./ngc-ready.md)) may list specific files that need to be downloaded for those workflows. diff --git a/docs/airgap/ngc-ready.md b/docs/airgap/ngc-ready.md index 35783858c..0efbe0483 100644 --- a/docs/airgap/ngc-ready.md +++ b/docs/airgap/ngc-ready.md @@ -28,9 +28,8 @@ The following Apt repositories will need to be mirrored in the offline environme For instructions on mirroring these repositories, see the [doc on Apt mirrors](./mirror-apt-repos.md). -The following files will need to be downloaded and made available in an HTTP mirror: +The following files may need to be downloaded and made available in an HTTP mirror: -- nvidia-docker wrapper (found [here](https://raw.githubusercontent.com/NVIDIA/nvidia-docker/master/nvidia-docker)) - DCGM package (optional) For instructions on setting up an HTTP mirror, see the [doc on HTTP mirrors](./mirror-http-files.md). @@ -53,10 +52,9 @@ The following RPM repositories will need to be mirrored in the offline environme For instructions on mirroring these repositories, see the [doc on RPM mirrors](./mirror-rpm-repos.md). -The following files will need to be downloaded and made available in an HTTP mirror: +The following files may need to be downloaded and made available in an HTTP mirror: - EPEL package (found [here](https://fedoraproject.org/wiki/EPEL)) -- nvidia-docker wrapper (found [here](https://raw.githubusercontent.com/NVIDIA/nvidia-docker/master/nvidia-docker)) - DCGM package (optional) For instructions on setting up an HTTP mirror, see the [doc on HTTP mirrors](./mirror-http-files.md). @@ -134,8 +132,8 @@ To configure alternate URLs for these repositories, set the following variables docker_ubuntu_repo_base_url: "http:///" docker_ubuntu_repo_gpgkey: "http:///" -nvidia_docker_repo_base_url: "http:///" -nvidia_docker_repo_gpg_url: "http:///" +nvidia_container_toolkit_repo_base_url: "http:///" +nvidia_container_toolkit_repo_gpg_url: "http:///" ``` **Enterprise Linux** @@ -144,17 +142,13 @@ nvidia_docker_repo_gpg_url: "http:////" docker_rh_repo_gpgkey: "http:///" -nvidia_docker_repo_base_url: "http:///" -nvidia_docker_repo_gpg_url: "http:///" +nvidia_container_toolkit_rpm_repo_url: "http:///" ``` ### Configure DeepOps to use your mirrors for HTTP downloads -In all cases, you will need to provide a URL to download the nvidia-docker wrapper: - -```bash -nvidia_docker_wrapper_url: "http:////nvidia-docker" -``` +Current NVIDIA Container Toolkit installs do not need a standalone `nvidia-docker` wrapper file. +Use this section only for other direct HTTP downloads required by the roles you enable. If installing on Enterprise Linux, you will need to provide a URL for the EPEL package. For example, From 972c7a802929a927221c26d42f973b019e2e35ec Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Thu, 28 May 2026 17:38:50 -0600 Subject: [PATCH 21/27] Clarify legacy OS support matrix --- README.md | 20 ++++++++-------- docs/deepops/testing.md | 29 ++++++++---------------- docs/ngc-ready/README.md | 9 ++++---- docs/slurm-cluster/slurm-single-node.md | 2 +- playbooks/bootstrap/bootstrap-python.yml | 4 ++-- roles/singularity_wrapper/tasks/main.yml | 2 +- virtual/README.md | 10 +++----- 7 files changed, 32 insertions(+), 44 deletions(-) diff --git a/README.md b/README.md index c2dde8826..fd3f8ea84 100644 --- a/README.md +++ b/README.md @@ -36,20 +36,22 @@ It is recommended to use the latest release branch for stable code (linked above ### Provisioning System -The provisioning system is used to orchestrate the running of all playbooks and one will be needed when instantiating Kubernetes or Slurm clusters. Supported operating systems which are tested and supported include: +The provisioning system is used to orchestrate the running of all playbooks and one will be needed when instantiating Kubernetes or Slurm clusters. Current release validation focuses on: -- NVIDIA DGX OS 4, 5, 6, 7 -- Ubuntu 18.04 LTS, 20.04, 22.04 LTS, 24.04 LTS -- CentOS 7, 8 +- Ubuntu 22.04 LTS and 24.04 LTS +- NVIDIA DGX OS 6 and 7 + +DeepOps still retains legacy/community-maintained paths for older environments such as DGX OS 4/5, Ubuntu 18.04/20.04, and CentOS 7/8. Treat those paths as compatibility references unless your site validates them for the release you deploy. ### Cluster System -The cluster nodes will follow the requirements described by Slurm or Kubernetes. You may also use a cluster node as a provisioning system but it is not required. +The cluster nodes will follow the requirements described by Slurm or Kubernetes. You may also use a cluster node as a provisioning system but it is not required. Current release validation focuses on: + +- Ubuntu 22.04 LTS and 24.04 LTS for generic Kubernetes and Slurm deployments +- NVIDIA DGX OS 6 and 7 for DGX systems +- Red Hat Enterprise Linux / Rocky Linux 8 and 9 for DGX platform software installation through the `nvidia-dgx` role -- NVIDIA DGX OS 4, 5, 6, 7 -- Ubuntu 18.04 LTS, 20.04, 22.04 LTS, 24.04 LTS -- CentOS 7, 8 -- Red Hat Enterprise Linux / Rocky Linux 8 and 9 for the DGX software stack through the `nvidia-dgx` role +DeepOps still retains legacy/community-maintained paths for older environments such as DGX OS 4/5, Ubuntu 18.04/20.04, CentOS 7/8, and the historical DGX EL7 stack. Treat those paths as compatibility references unless your site validates them for the release you deploy. You may also install a supported operating system on all servers via a 3rd-party solution such as [MAAS](https://maas.io/) or [Foreman](https://www.theforeman.org/), or via an existing site-standard automated installer. For new Ubuntu 24.04 or DGX OS 7 deployments, prefer Ubuntu autoinstall/cloud-init or MAAS and then apply DeepOps roles after the OS is present. diff --git a/docs/deepops/testing.md b/docs/deepops/testing.md index c4fc6c3b6..95125287c 100644 --- a/docs/deepops/testing.md +++ b/docs/deepops/testing.md @@ -48,16 +48,16 @@ If a change requires GPU-backed validation, document the validation environment A short description of the historical Jenkins test matrix is outlined below. The full suite of legacy jobs can be reviewed in the [jenkins](../../workloads/jenkins) directory. These rows are not a promise of current public CI coverage; check the pull request's GitHub Actions and validation notes for current status. -**Legacy Jenkins Testing Matrix** +**Validation Matrix** | Test | [PR](../../workloads/jenkins/Jenkinsfile) | [Nightly](../../workloads/jenkins/Jenkinsfile-nightly) | [Nightly Multi-node](../../workloads/jenkins/Jenkinsfile-multi-nightly) | Comments | | --------------------------------------------------- | ----------------------------------------- | ------------------------------------------------------ | ----------------------------------------------------------------------- | ------------------------------------ | -| Ubuntu 18.04 | x | x | x | | -| Ubuntu 20.04 | | x | x | | +| Ubuntu 18.04 | x | x | x | Legacy Jenkins/Vagrant reference only | +| Ubuntu 20.04 | | x | x | Legacy Jenkins/Vagrant reference only | | Ubuntu 22.04 | | | | setup.sh and Molecule GitHub Actions | | Ubuntu 24.04 | | | | setup.sh and Molecule GitHub Actions | -| CentOS 7 | | x | x | | -| CentOS | | | x | | +| CentOS 7 | | x | x | Legacy Jenkins/Vagrant reference only | +| CentOS 8 | | | x | Legacy Jenkins/Vagrant reference only | | DGX OS | | | | Syntax-checked only; full validation requires DGX hardware | | RHEL | | | | DGX software-stack role syntax-checked only; full validation requires DGX hardware and subscriptions | | 1 mgmt node | x | x | | | @@ -123,30 +123,19 @@ molecule init scenario -r --driver-name docker ``` 4. In the file `molecule/default/molecule.yml`, define the list of platforms to be tested. - DeepOps currently supports operating systems based on Ubuntu 18.04, Ubuntu 20.04, Ubuntu 22.04, Ubuntu 24.04, EL7, and EL8. - The DGX software stack role also supports Red Hat Enterprise Linux / Rocky Linux 8 and 9 for DGX platform software installation. - To test these stacks, the following `platforms` stanza can be used. + DeepOps currently uses Ubuntu 22.04 and Ubuntu 24.04 for setup and Molecule GitHub Actions. + Add Red Hat family images only for roles that explicitly support them, and validate the image choice for that role. + Keep Ubuntu 18.04, Ubuntu 20.04, CentOS 7, and CentOS 8 scenarios in separately named legacy test scenarios when maintaining older compatibility paths. + To test the current Ubuntu stacks, the following `platforms` stanza can be used. ```yaml platforms: - - name: ubuntu-1804 - image: geerlingguy/docker-ubuntu1804-ansible - pre_build_image: true - - name: ubuntu-2004 - image: geerlingguy/docker-ubuntu2004-ansible - pre_build_image: true - name: ubuntu-2204 image: geerlingguy/docker-ubuntu2204-ansible pre_build_image: true - name: ubuntu-2404 image: geerlingguy/docker-ubuntu2404-ansible pre_build_image: true - - name: centos-7 - image: geerlingguy/docker-centos7-ansible - pre_build_image: true - - name: centos-8 - image: geerlingguy/docker-centos8-ansible - pre_build_image: true ``` 5. If you haven't already, define your role's metadata in the file `meta/main.yml`. diff --git a/docs/ngc-ready/README.md b/docs/ngc-ready/README.md index 04b0e2e61..e7618d07b 100644 --- a/docs/ngc-ready/README.md +++ b/docs/ngc-ready/README.md @@ -14,10 +14,11 @@ These instructions assume the following: - You have a NGC-Ready server. To determine if your server is NGC-Ready, please review the list of validated servers at the NGC-Ready Server documentation page - https://docs.nvidia.com/certification-programs/ngc-ready-systems/index.html - Your NGC-Ready Server has a compatible Linux distribution installed: - - Ubuntu Server 20.04 LTS - Ubuntu Server 22.04 LTS - Ubuntu Server 24.04 LTS - - CentOS 7 + - Red Hat Enterprise Linux / Rocky Linux 8 or 9 when the referenced roles are validated for your server + +Legacy Ubuntu 20.04 and CentOS 7 environments may still work for existing deployments, but they are not current release validation targets. ## Setup @@ -41,7 +42,7 @@ This process will install the latest NVIDIA GPU Drivers, and Docker with the NVI # : IP of NGC-Ready server, or localhost. The trailing comma is required # If SSH requires a password, add: -k # If sudo requires a password, add: -K -ansible-playbook -u -i , playbooks/ngc-ready.yml +ansible-playbook -u -i , playbooks/ngc-ready-server.yml ``` ## Testing @@ -55,5 +56,5 @@ This process will test the functionality of the NGC-Ready server by running a fu # : IP of NGC-Ready server, or localhost. The trailing comma is required # If SSH requires a password, add: -k # If sudo requires a password, add: -K -ansible-playbook -u -i , playbooks/ngc-ready.yml --tags test +ansible-playbook -u -i , playbooks/ngc-ready-server.yml --tags test ``` diff --git a/docs/slurm-cluster/slurm-single-node.md b/docs/slurm-cluster/slurm-single-node.md index 18174b52d..fda8d8dde 100644 --- a/docs/slurm-cluster/slurm-single-node.md +++ b/docs/slurm-cluster/slurm-single-node.md @@ -15,7 +15,7 @@ Single Node Slurm Deployment Guide ## Introduction -The general requirements and procedure for Slurm setup via deepops is documented in the [README.md](README.md) for the slurm-cluster. The instructions below outline the steps to deviate from the general setup to enable single node DeepOps Slurm setup. The machine on which Slurm is being deployed should be up to date in a stable state with GPU drivers already installed and functional. The supported operating systems are Ubuntu 18.04, 20.04, 22.04, and 24.04; CentOS 7 and 8; and RHEL 7 and 8, with RHEL 8 preferred among the RHEL paths. +The general requirements and procedure for Slurm setup via deepops is documented in the [README.md](README.md) for the slurm-cluster. The instructions below outline the steps to deviate from the general setup to enable single node DeepOps Slurm setup. The machine on which Slurm is being deployed should be up to date in a stable state with GPU drivers already installed and functional. Current release validation should target Ubuntu 22.04 or Ubuntu 24.04. Older Ubuntu, CentOS, and RHEL paths are historical compatibility references and should be validated locally before use. ## Deployment Procedure diff --git a/playbooks/bootstrap/bootstrap-python.yml b/playbooks/bootstrap/bootstrap-python.yml index 1885f8baa..241381de8 100644 --- a/playbooks/bootstrap/bootstrap-python.yml +++ b/playbooks/bootstrap/bootstrap-python.yml @@ -35,13 +35,13 @@ state: present when: ansible_python.version.major == 3 - - name: install epel on EL7 + - name: legacy EL7 - install epel package: name: epel-release state: present when: (ansible_python.version.major == 2) and (ansible_os_family == "RedHat") and (ansible_distribution_major_version == "7") - - name: install python 2 libraries on EL7 + - name: legacy EL7 - install python 2 libraries package: name: - python-setuptools diff --git a/roles/singularity_wrapper/tasks/main.yml b/roles/singularity_wrapper/tasks/main.yml index 82336c7db..f4e8890f3 100644 --- a/roles/singularity_wrapper/tasks/main.yml +++ b/roles/singularity_wrapper/tasks/main.yml @@ -1,5 +1,5 @@ --- -- name: centos 8 - ensure powertools installed +- name: legacy CentOS 8 - ensure powertools installed block: - name: ensure prereq packages installed yum: diff --git a/virtual/README.md b/virtual/README.md index 19db58a21..b1f3809a0 100644 --- a/virtual/README.md +++ b/virtual/README.md @@ -23,12 +23,9 @@ If deploying kubeflow or another resource-intensive application in this environm ### Operating System Requirements -* Ubuntu 18.04 (or greater) -* CentOS 7.6 (or greater) +Running DeepOps virtually assumes that the host machine's OS is suitable for Vagrant, libvirt, and any optional GPU passthrough configuration. Ubuntu 22.04 LTS is the preferred host path for this legacy lab workflow. -Running DeepOps virtually assumes that the host machine's OS is an approved OS. If this is not the case, the scripts used in the steps below may be modified to work with a different OS. - -The Vagrantfiles currently cover Ubuntu 18.04, Ubuntu 20.04, Ubuntu 22.04, CentOS 7, and CentOS 8. The startup script defaults to Ubuntu 20.04. Newer operating systems should be validated on real target systems unless the Vagrantfiles have been refreshed and tested for that release. +The Vagrantfiles currently cover Ubuntu 18.04, Ubuntu 20.04, Ubuntu 22.04, CentOS 7, and CentOS 8. The startup script defaults to Ubuntu 20.04. Treat these Vagrantfiles as legacy/community-supported lab references; validate current release work on real target systems unless the Vagrantfiles have been refreshed and tested for that release. Also, using VMs and optionally GPU passthrough assumes that the host machine has been configured to enable virtualization in the BIOS. For instructions on how to accomplish this, refer to the sections at the bottom of this README: [Enabling virtualization and GPU passthrough](#enabling-virtualization-and-gpu-passthrough). @@ -163,7 +160,7 @@ The default Vagrantfiles create VMs that are very minimal in terms of resources ### Specify the cluster Operating System -By default, all virtual nodes deploy with Ubuntu 20.04. This can be changed by overriding the environment variables `DEEPOPS_VAGRANT_OS` and `DEEPOPS_OS_VERSION`. Available Vagrantfiles include Ubuntu 18.04, Ubuntu 20.04, Ubuntu 22.04, CentOS 7, and CentOS 8. +By default, all virtual nodes deploy with Ubuntu 20.04. This can be changed by overriding the environment variables `DEEPOPS_VAGRANT_OS` and `DEEPOPS_OS_VERSION`. Available Vagrantfiles include Ubuntu 18.04, Ubuntu 20.04, Ubuntu 22.04, CentOS 7, and CentOS 8. These operating system choices are maintained as legacy lab fixtures, not as release validation targets. ```sh export DEEPOPS_VAGRANT_OS=centos @@ -358,4 +355,3 @@ $ lspci -nnk -d 10de:1db1 Kernel modules: nvidiafb, nouveau, nvidia_drm, nvidia_vgpu_vfio, nvidia ``` - From 9db02ec86c8719b713ae8058e7a31c0d7aa68091 Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Thu, 28 May 2026 17:57:48 -0600 Subject: [PATCH 22/27] Refresh RAPIDS and RoCE examples --- config.example/helm/rapids-dask.yml | 14 +++++------ docs/container/docker-rootless.md | 2 +- docs/k8s-cluster/roce_backend.md | 14 +++++++---- roles/roce_backend/README.md | 16 ++++++------ roles/roce_backend/tasks/mofed-install.yaml | 2 +- roles/roce_backend/vars/main.yml | 22 ++++++---------- workloads/examples/k8s/dask-rapids/README.md | 7 +++--- .../k8s/dask-rapids/docker/Dockerfile | 25 ++++++------------- .../k8s/dask-rapids/docker/prepare.sh | 8 +++--- 9 files changed, 48 insertions(+), 62 deletions(-) diff --git a/config.example/helm/rapids-dask.yml b/config.example/helm/rapids-dask.yml index 246bd3d14..21788310b 100644 --- a/config.example/helm/rapids-dask.yml +++ b/config.example/helm/rapids-dask.yml @@ -4,10 +4,8 @@ # Specify the resources used for each worker as well as the number of workers. worker: image: - # repository: nvcr.io/nvidia/rapidsai/rapidsai - # repository: dask-rapids - repository: supertetelman/k8s-rapids-dask - tag: cuda9.2-runtime-ubuntu16.04 + repository: nvcr.io/nvidia/rapidsai/notebooks + tag: 26.04-cuda12-py3.13 env: replicas: 1 resources: @@ -18,15 +16,15 @@ worker: scheduler: image: - repository: supertetelman/k8s-rapids-dask - tag: cuda9.2-runtime-ubuntu16.04 + repository: nvcr.io/nvidia/rapidsai/notebooks + tag: 26.04-cuda12-py3.13 # By default we should be doing all Dask works on workers using calls to distributed.Client() # If you would like to run/test your GPU code without using workers you may comment the resources section jupyter: image: - repository: supertetelman/k8s-rapids-dask - tag: cuda9.2-runtime-ubuntu16.04 + repository: nvcr.io/nvidia/rapidsai/notebooks + tag: 26.04-cuda12-py3.13 resources: requests: nvidia.com/gpu: 0 diff --git a/docs/container/docker-rootless.md b/docs/container/docker-rootless.md index 9bfe3dd13..c40cd8098 100644 --- a/docs/container/docker-rootless.md +++ b/docs/container/docker-rootless.md @@ -77,7 +77,7 @@ module load rootless-docker start_rootless_docker.sh # specify --quiet option to hide rootles docker messages -docker run --gpus all -it --rm nvcr.io/nvidia/cuda:11.0-base-ubuntu18.04 +docker run --gpus all -it --rm nvcr.io/nvidia/cuda:13.0.2-base-ubuntu24.04 root@445bf5cca686:/# echo NGPUS: $(nvidia-smi -L | wc -l) NGPUS: 1 diff --git a/docs/k8s-cluster/roce_backend.md b/docs/k8s-cluster/roce_backend.md index aa5c9dba6..0bee96738 100644 --- a/docs/k8s-cluster/roce_backend.md +++ b/docs/k8s-cluster/roce_backend.md @@ -80,11 +80,15 @@ dev_id: 101c num_vf: 8 -5. Mellanox Ofed place and image name - mofed_site_place, mofed_file_name. - -mofed_site_place: "MLNX_OFED-4.6-1.0.1.1" - -mofed_file_name: "MLNX_OFED_LINUX-4.6-1.0.1.1-ubuntu18.04-x86_64.iso" +5. NVIDIA OFED version, site place and image name - mofed_version, mofed_site_place, mofed_file_name. + +For new Kubernetes RDMA/RoCE deployments, prefer NVIDIA Network Operator coverage with DOCA-OFED driver management. This DeepOps role remains a legacy direct-host install path for environments that still need it, so verify the OFED package against your exact operating system and kernel before using it in production. + +mofed_version: "24.10-4.1.4.0" + +mofed_site_place: "MLNX_OFED-24.10-4.1.4.0" + +mofed_file_name: "MLNX_OFED_LINUX-24.10-4.1.4.0-ubuntu24.04-x86_64.iso" ## Dependencies diff --git a/roles/roce_backend/README.md b/roles/roce_backend/README.md index 08aad44e4..e3d72a807 100755 --- a/roles/roce_backend/README.md +++ b/roles/roce_backend/README.md @@ -71,13 +71,15 @@ dev_id: "101c" num_vf: 8 -5. Mellanox Ofed version, site place and image name - mofed_version, mofed_site_place, mofed_file_name. -``` -#Mellanox OFED parameters -mofed_version: "4.7-3.2.9.0" -mofed_site_place: "MLNX_OFED-4.7-3.2.9.0" -mofed_file_name: "MLNX_OFED_LINUX-4.7-3.2.9.0-ubuntu18.04-x86_64.iso" -``` +5. NVIDIA OFED version, site place and image name - mofed_version, mofed_site_place, mofed_file_name. + +For new Kubernetes RDMA/RoCE deployments, prefer NVIDIA Network Operator coverage with DOCA-OFED driver management. This DeepOps role remains a legacy direct-host install path for environments that still need it, so verify the OFED package against your exact operating system and kernel before using it in production. +``` +# NVIDIA OFED parameters +mofed_version: "24.10-4.1.4.0" +mofed_site_place: "MLNX_OFED-24.10-4.1.4.0" +mofed_file_name: "MLNX_OFED_LINUX-24.10-4.1.4.0-ubuntu24.04-x86_64.iso" +``` Dependencies diff --git a/roles/roce_backend/tasks/mofed-install.yaml b/roles/roce_backend/tasks/mofed-install.yaml index 491c060ef..12ce4e82f 100755 --- a/roles/roce_backend/tasks/mofed-install.yaml +++ b/roles/roce_backend/tasks/mofed-install.yaml @@ -22,7 +22,7 @@ shell: | cd /tmp rm -f /tmp/{{ mofed_file_name }} - wget http://content.mellanox.com/ofed/{{ mofed_site_place }}/{{ mofed_file_name }} + wget https://content.mellanox.com/ofed/{{ mofed_site_place }}/{{ mofed_file_name }} mkdir -p /mnt/iso mount -o loop /tmp/{{ mofed_file_name }} /mnt/iso /mnt/iso/mlnxofedinstall --all -q diff --git a/roles/roce_backend/vars/main.yml b/roles/roce_backend/vars/main.yml index e3eeea120..0353a4e08 100755 --- a/roles/roce_backend/vars/main.yml +++ b/roles/roce_backend/vars/main.yml @@ -31,18 +31,12 @@ vendor: "15b3" dev_id: "101c" num_vf: 8 -#Mellanox OFED parameters -mofed_version: "4.7-3.2.9.0" -mofed_site_place: "MLNX_OFED-4.7-3.2.9.0" -mofed_file_name: "MLNX_OFED_LINUX-4.7-3.2.9.0-ubuntu18.04-x86_64.iso" - -# before K8s 1.16 -multus_ds: "https://raw.githubusercontent.com/intel/multus-cni/master/images/multus-daemonset-pre-1.16.yml" -sriov_dp_ds: "https://raw.githubusercontent.com/intel/sriov-network-device-plugin/master/deployments/k8s-v1.10-v1.15/sriovdp-daemonset.yaml" -sriov_cni_ds: "https://raw.githubusercontent.com/intel/sriov-cni/master/images/k8s-v1.10-v1.15/sriov-cni-daemonset.yaml" - -# from K8s 1.16 -# multus_ds: "https://raw.githubusercontent.com/intel/multus-cni/master/images/multus-daemonset.yml" -# sriov_dp_ds: "https://raw.githubusercontent.com/intel/sriov-network-device-plugin/master/deployments/k8s-v1.16/sriovdp-daemonset.yaml" -# sriov_cni_ds: "https://raw.githubusercontent.com/intel/sriov-cni/master/images/k8s-v1.16/sriov-cni-daemonset.yaml" +# NVIDIA OFED parameters. Prefer NVIDIA Network Operator with DOCA-OFED for +# new Kubernetes RDMA/RoCE deployments; this role is a legacy direct-host path. +mofed_version: "24.10-4.1.4.0" +mofed_site_place: "MLNX_OFED-24.10-4.1.4.0" +mofed_file_name: "MLNX_OFED_LINUX-24.10-4.1.4.0-ubuntu24.04-x86_64.iso" +multus_ds: "https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/master/deployments/multus-daemonset.yml" +sriov_dp_ds: "https://raw.githubusercontent.com/k8snetworkplumbingwg/sriov-network-device-plugin/master/deployments/sriovdp-daemonset.yaml" +sriov_cni_ds: "https://raw.githubusercontent.com/k8snetworkplumbingwg/sriov-cni/master/images/sriov-cni-daemonset.yaml" diff --git a/workloads/examples/k8s/dask-rapids/README.md b/workloads/examples/k8s/dask-rapids/README.md index da96e6bf6..590d4f46e 100644 --- a/workloads/examples/k8s/dask-rapids/README.md +++ b/workloads/examples/k8s/dask-rapids/README.md @@ -34,6 +34,8 @@ DeepOps includes support for running a local registry, but configuration of that In my workflow below, I am pushing the image to [Docker Hub](https://hub.docker.com). If you haven't used Docker Hub before, the [quickstart documentation](https://docs.docker.com/docker-hub/) provides a good tutorial. +The included Dockerfile extends the current NGC RAPIDS notebooks image, `nvcr.io/nvidia/rapidsai/notebooks:26.04-cuda12-py3.13`, which provides RAPIDS, Dask CUDA, and JupyterLab on Ubuntu 24.04. If you move to a newer RAPIDS release, update both the Dockerfile and Helm values together. + ### Editing the deployment scripts We'll deploy the RAPIDS/Dask container in one step using the `deploy.sh` script. @@ -140,9 +142,8 @@ Feel free to adjust the number of CPU cores or GPUs used and the parameters for ## Experimenting further -The base container we used for this benchmark contains more examples using RAPIDS in the `cuml/` directory, -as well as an end-to-end workflow example based on a Fannie Mae mortgage dataset in the `mortgage/` directory. -Both directories can be accessed easily via JupyterLab. +The base container used for this benchmark includes RAPIDS libraries, Dask CUDA, and JupyterLab. +For more examples and current image tags, use the RAPIDS documentation and NGC container catalog. You can also experiment with the custom container by making changes to the `Dockerfile` used to create it, in `examples/k8s/dask-rapids/docker`. diff --git a/workloads/examples/k8s/dask-rapids/docker/Dockerfile b/workloads/examples/k8s/dask-rapids/docker/Dockerfile index b84ea955e..358fce704 100644 --- a/workloads/examples/k8s/dask-rapids/docker/Dockerfile +++ b/workloads/examples/k8s/dask-rapids/docker/Dockerfile @@ -1,24 +1,13 @@ -# Base our new image on the CUDA 9.2 RAPIDS image from upstream -FROM nvcr.io/nvidia/rapidsai/rapidsai:cuda9.2-runtime-ubuntu16.04 +# Base the example on the current RAPIDS notebooks image from NGC. +FROM nvcr.io/nvidia/rapidsai/notebooks:26.04-cuda12-py3.13 + +USER root -# Fix font-manager package RUN apt-get update && \ - apt-get install -y --fix-missing font-manager && \ + apt-get install -y --no-install-recommends font-manager && \ rm -rf /var/lib/apt/lists/* -# The name of the Anaconda Python environment we'll use (from upstream) -ENV CONDA_ENV rapids - -# Install additional Python packages into the environment -# (If you want to install more packages, add them here!) -RUN source activate $CONDA_ENV && \ - conda install -y unzip python-graphviz && \ - pip install ipyvolume dask-kubernetes matplotlib cupy-cuda92 +USER rapids # Copy the parallel sum notebook in -COPY ParallelSum.ipynb /rapids/notebooks/ParallelSum.ipynb - -# Set up image to be run -COPY prepare.sh /usr/bin/prepare.sh -CMD ["jupyter", "lab", "--ip=0.0.0.0", "--allow-root", "--no-browser", "--NotebookApp.token='dask'"] -ENTRYPOINT ["tini", "--", "/usr/bin/prepare.sh"] +COPY --chown=rapids:conda ParallelSum.ipynb /home/rapids/notebooks/ParallelSum.ipynb diff --git a/workloads/examples/k8s/dask-rapids/docker/prepare.sh b/workloads/examples/k8s/dask-rapids/docker/prepare.sh index 8dd3df26c..7b6251c59 100755 --- a/workloads/examples/k8s/dask-rapids/docker/prepare.sh +++ b/workloads/examples/k8s/dask-rapids/docker/prepare.sh @@ -27,11 +27,9 @@ if [ "$EXTRA_PIP_PACKAGES" ]; then fi # Activate the specified or default conda environment -#conda_env="rapids" -#if [ "$CONDA_ENV" ]; then -# conda_env="$CONDA_ENV" -#fi -source activate $CONDA_ENV +CONDA_ENV="${CONDA_ENV:-base}" +source /opt/conda/etc/profile.d/conda.sh +conda activate "$CONDA_ENV" # Run pre-commands if [ "$PRE_RUN_HOOK" ]; then From ce10816bfa8e475b59f9aa70bb2479460341d807 Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Thu, 28 May 2026 18:15:24 -0600 Subject: [PATCH 23/27] Refresh workload and registry examples --- config.example/group_vars/all.yml | 4 ++-- docs/airgap/ngc-ready.md | 12 +++++----- docs/container/nginx-docker-cache.md | 4 ++-- docs/k8s-cluster/kubernetes-usage.md | 10 ++++---- docs/slurm-cluster/README.md | 8 +++---- docs/slurm-cluster/slurm-perf-cluster.md | 6 ++--- docs/slurm-cluster/slurm-single-node.md | 24 ++++++++++--------- playbooks/slurm-cluster/slurm-validation.yml | 2 +- .../defaults/main.yml | 2 +- .../k8s/dask-rapids/docker/Dockerfile | 5 +++- workloads/examples/k8s/pytorch-job.yml | 4 ++-- 11 files changed, 43 insertions(+), 38 deletions(-) diff --git a/config.example/group_vars/all.yml b/config.example/group_vars/all.yml index d98664bdc..60ef0104c 100644 --- a/config.example/group_vars/all.yml +++ b/config.example/group_vars/all.yml @@ -316,5 +316,5 @@ standalone_container_registry_port: "5000" # Configuration for NGC-Ready playbook # ################################################################################ ngc_ready_cuda_container: "nvcr.io/nvidia/cuda:12.4.1-base-ubuntu22.04" -ngc_ready_pytorch: "nvcr.io/nvidia/pytorch:24.04-py3" -ngc_ready_tensorflow: "nvcr.io/nvidia/tensorflow:24.04-tf2-py3" +ngc_ready_pytorch: "nvcr.io/nvidia/pytorch:26.04-py3" +ngc_ready_tensorflow: "nvcr.io/nvidia/tensorflow:25.02-tf2-py3" diff --git a/docs/airgap/ngc-ready.md b/docs/airgap/ngc-ready.md index 0efbe0483..d93757c56 100644 --- a/docs/airgap/ngc-ready.md +++ b/docs/airgap/ngc-ready.md @@ -37,8 +37,8 @@ For instructions on setting up an HTTP mirror, see the [doc on HTTP mirrors](./m Container images are only needed if you want to run the tests built into the playbook: - nvcr.io/nvidia/cuda:12.4.1-base-ubuntu22.04 -- nvcr.io/nvidia/pytorch:24.04-py3 -- nvcr.io/nvidia/tensorflow:24.04-tf2-py3 +- nvcr.io/nvidia/pytorch:26.04-py3 +- nvcr.io/nvidia/tensorflow:25.02-tf2-py3 For instructions on setting up a Docker registry mirror, see the [doc on Docker mirrors](./mirror-docker-images.md). @@ -62,8 +62,8 @@ For instructions on setting up an HTTP mirror, see the [doc on HTTP mirrors](./m Container images (how to mirror) are only needed if you want to run the tests built into the playbook: - nvcr.io/nvidia/cuda:12.4.1-base-ubuntu22.04 -- nvcr.io/nvidia/pytorch:24.04-py3 -- nvcr.io/nvidia/tensorflow:24.04-tf2-py3 +- nvcr.io/nvidia/pytorch:26.04-py3 +- nvcr.io/nvidia/tensorflow:25.02-tf2-py3 For instructions on setting up a Docker registry mirror, see the [doc on Docker mirrors](./mirror-docker-images.md). @@ -177,8 +177,8 @@ If running the container tests as part of the NGC-Ready playbook, set the follow ```bash ngc_ready_cuda_container: "/nvidia/cuda:12.4.1-base-ubuntu22.04" -ngc_ready_pytorch: "/nvidia/pytorch:24.04-py3" -ngc_ready_tensorflow: "/nvidia/tensorflow:24.04-tf2-py3" +ngc_ready_pytorch: "/nvidia/pytorch:26.04-py3" +ngc_ready_tensorflow: "/nvidia/tensorflow:25.02-tf2-py3" ``` ## Running the NGC-Ready playbook diff --git a/docs/container/nginx-docker-cache.md b/docs/container/nginx-docker-cache.md index 178172537..34366ad32 100644 --- a/docs/container/nginx-docker-cache.md +++ b/docs/container/nginx-docker-cache.md @@ -42,8 +42,8 @@ The following variables are the most common configuration you may want to adjust | Variable | Default value | Description | | ------------------------------------------ | ---------------------------------------- | ----------------------------------------------------------------------------- | -| `nginx_docker_cache_image` | `"rpardini/docker-registry-proxy:0.6.1"` | Container image used to deploy the proxy | -| `nginx_docker_cache_registry_string` | `"quay.io k8s.gcr.io gcr.io nvcr.io"` | Space-separated list of registries to proxy | +| `nginx_docker_cache_image` | `"rpardini/docker-registry-proxy:0.6.5"` | Container image used to deploy the proxy | +| `nginx_docker_cache_registry_string` | `"registry.k8s.io quay.io k8s.gcr.io gcr.io nvcr.io"` | Space-separated list of registries to proxy; `k8s.gcr.io` is retained for older clusters while current Kubernetes images use `registry.k8s.io` | | `nginx_docker_cache_manifests` | `"false"` | Flag to determine whether to cache image manifests | | `nginx_docker_cache_manifest_default_time` | "1h" | If manifests are cached, time to cache them | | `nginx_docker_cache_hostgroup` | `"cache"` | Ansible inventory host group where proxy is deployed | diff --git a/docs/k8s-cluster/kubernetes-usage.md b/docs/k8s-cluster/kubernetes-usage.md index 106f4ecc2..2511509f6 100644 --- a/docs/k8s-cluster/kubernetes-usage.md +++ b/docs/k8s-cluster/kubernetes-usage.md @@ -10,7 +10,7 @@ Kubernetes Usage Guide ## Introduction -Most of the following examples can be configured and executed through the Kubernetes Dashboard. For a basic run-through on how to leverage the Kubernetes Dashboard, please see the [official documentation](https://kubernetes.io/docs/tasks/access-application-cluster/web-ui-dashboard/). The following examples `kubectl` on the master node instead. +Most of the following examples can be configured and executed through the Kubernetes Dashboard. For a basic run-through on how to leverage the Kubernetes Dashboard, please see the [official documentation](https://kubernetes.io/docs/tasks/access-application-cluster/web-ui-dashboard/). The following examples use `kubectl` on the master node instead. ## Simple Commands @@ -63,12 +63,12 @@ kubectl get pods --all-namespaces 4. Delete the job (and the corresponding pod). ```bash - kubectl delete job cuda-job + kubectl delete job pytorch-job ``` ## Using NGC Containers with Kubernetes and Launching Jobs -[NVIDIA GPU Cloud (NGC)](https://docs.nvidia.com/ngc/ngc-introduction) manages a catalog of fully integrated and optimized DL framework containers that take full advantage of NVIDIA GPUs in both single and multi-GPU configurations. They include NVIDIA CUDA® Toolkit, DIGITS workflow, and the following DL frameworks: NVCaffe, Caffe2, Microsoft Cognitive Toolkit (CNTK), MXNet, PyTorch, TensorFlow, Theano, and Torch. These framework containers are delivered ready-to-run, including all necessary dependencies such as the CUDA runtime and NVIDIA libraries. +[NVIDIA GPU Cloud (NGC)](https://docs.nvidia.com/ngc/ngc-introduction) manages a catalog of optimized GPU containers for CUDA, PyTorch, TensorFlow, Triton Inference Server, RAPIDS, and other NVIDIA software. Use the NGC catalog and the NVIDIA framework container release notes to choose the current image for your workload. To access the NGC container registry via Kubernetes, add a secret which will be employed when Kubernetes asks NGC to pull container images from it. @@ -105,9 +105,9 @@ To access the NGC container registry via Kubernetes, add a secret which will be - name: nvcr.dgxkey containers: - name: pytorch-container - image: nvcr.io/nvidia/pytorch:19.02-py3 + image: nvcr.io/nvidia/pytorch:26.04-py3 command: ["/bin/sh"] - args: ["-c", "python /workspace/examples/upstream/mnist/main.py"] + args: ["-c", "python -c 'import torch; print(\"cuda_available=\", torch.cuda.is_available()); print(\"device_count=\", torch.cuda.device_count())'"] resources: limits: nvidia.com/gpu: 1 diff --git a/docs/slurm-cluster/README.md b/docs/slurm-cluster/README.md index e5842cd4f..5d60986c4 100644 --- a/docs/slurm-cluster/README.md +++ b/docs/slurm-cluster/README.md @@ -87,7 +87,7 @@ default parameters that can be overriden: ```bash # String; Container for nccl performance/validation tests. Either docker # tag or can be path to sqsh file. - base_container: "nvcr.io/nvidia/tensorflow:21.09-tf2-py3" + base_container: "nvcr.io/nvidia/pytorch:26.04-py3" # String; Container to be created or one that might exist with nccl tests. # If `compile_nccl_tests` is True, it must be a sqsh file. @@ -166,17 +166,17 @@ NOTE: This will use Pyxis to download a container. ```bash ansible-playbook -l slurm-cluster playbooks/slurm-cluster/slurm-validation.yml \ - -e '{base_container: nvcr.io/nvidia/pytorch:21.09-py3}' \ + -e '{base_container: nvcr.io/nvidia/pytorch:26.04-py3}' \ -e '{nccl_tests_container: "${HOME}/enroot_images/nccl_tests_torch_val.sqsh"}' \ -e '{num_nodes: 2}' \ -e '{srun_exports: "NCCL_DEBUG=INFO,OMPI_MCA_pml=^ucx,OMPI_MCA_coll=^hcoll"}' \ -e '{cleanup: True}' ``` -3. Example to run on 1 node using existing NCCL container from a docker repo. +3. Example to run on 1 node using an existing NCCL test container from a site registry. ```bash ansible-playbook -l slurm-cluster playbooks/slurm-cluster/slurm-validation.yml \ - -e '{nccl_tests_container: deepops/nccl-tests-tf20.06-ubuntu18.04:latest}' \ + -e '{nccl_tests_container: registry.example.com/hpc/nccl-tests:latest}' \ -e '{compile_nccl_tests: False}' \ -e '{num_nodes: 1}' ``` diff --git a/docs/slurm-cluster/slurm-perf-cluster.md b/docs/slurm-cluster/slurm-perf-cluster.md index 6b252f0d8..7aa92aec0 100644 --- a/docs/slurm-cluster/slurm-perf-cluster.md +++ b/docs/slurm-cluster/slurm-perf-cluster.md @@ -254,7 +254,7 @@ If errors are noticed when running `sinfo -R`, it's also helpful to search the l sudo journalctl -e | grep slurm ``` -To re-run the test manually, from the slurm login node... +To re-run the test manually, from the slurm login node. Replace `registry.example.com/hpc/nccl-tests:latest` with your site's current NCCL tests image or a `.sqsh` image built by `playbooks/slurm-cluster/slurm-validation.yml`. ```bash # on the slurm login node @@ -269,7 +269,7 @@ scancel sudo scontrol update nodename= state=idle # run the test again -srun -N --mpi=pmix --exclusive --container-image=deepops/nccl-tests-tf20.06-ubuntu18.04 --ntasks-per-node=8 -G all_reduce_perf -b 1M -e 4G -f 2 -g +srun -N --mpi=pmix --exclusive --container-image=registry.example.com/hpc/nccl-tests:latest --ntasks-per-node=8 -G all_reduce_perf -b 1M -e 4G -f 2 -g ``` ### Performance validation test results are suboptimal @@ -289,7 +289,7 @@ Try running the test from the slurm login node, but with debug output enabled... ```bash # from the slurm login node -$ NCCL_DEBUG=INFO srun -N --mpi=pmix --exclusive --container-image=deepops/nccl-tests-tf20.06-ubuntu18.04 --ntasks-per-node=8 -G all_reduce_perf -b 1M -e 4G -f 2 -g +$ NCCL_DEBUG=INFO srun -N --mpi=pmix --exclusive --container-image=registry.example.com/hpc/nccl-tests:latest --ntasks-per-node=8 -G all_reduce_perf -b 1M -e 4G -f 2 -g # examine the output, looking for any mention of `GDRDMA` # for example: `NET/IB/0/GDRDMA` diff --git a/docs/slurm-cluster/slurm-single-node.md b/docs/slurm-cluster/slurm-single-node.md index fda8d8dde..94518be98 100644 --- a/docs/slurm-cluster/slurm-single-node.md +++ b/docs/slurm-cluster/slurm-single-node.md @@ -368,11 +368,11 @@ compute-session:start_rootless_docker.sh ``` An option “--quiet” can be passed to the “start_rootless_docker.sh” script to -hide rootless docker messages. Pull/run a docker image: +hide rootless docker messages. Pull/run a site-maintained NCCL tests image: ```bash compute-session:docker run --gpus=all --rm -it \ - deepops/nccl-tests-tf20.06-ubuntu18.04:latest \ + registry.example.com/hpc/nccl-tests:latest \ mpirun --allow-run-as-root -np 2 all_reduce_perf -b 1M -e 4G -f 2 -g 1 ``` @@ -386,7 +386,7 @@ module load rootless-docker start_rootless_docker.sh --quiet -docker run --gpus=all --rm -t deepops/nccl-tests-tf20.06-ubuntu18.04:latest \ +docker run --gpus=all --rm -t registry.example.com/hpc/nccl-tests:latest \ mpirun --allow-run-as-root -np 2 all_reduce_perf -b 1M -e 4G -f 2 -g 1 stop_rootless_docker.sh @@ -403,7 +403,7 @@ starting the container and checking the number of GPUs and CPUs available. ```bash compute-session:docker run --gpus=all --rm -it \ - deepops/nccl-tests-tf20.06-ubuntu18.04:latest \ + registry.example.com/hpc/nccl-tests:latest \ bash -c 'echo NGPUS: $(nvidia-smi -L | wc -l) NCPUS: $(nproc)' NGPUS: 2 NCPUS: 2 ``` @@ -416,7 +416,7 @@ already does not have permission to outside of the container. ```bash compute-session:docker run --gpus=all --rm -it -v ${PWD}:${PWD} --workdir=${PWD} \ - deepops/nccl-tests-tf20.06-ubuntu18.04:latest bash -c 'touch somefile-in-container' + registry.example.com/hpc/nccl-tests:latest bash -c 'touch somefile-in-container' ``` Then outside of the container. @@ -434,7 +434,7 @@ outside of the container. ```bash compute-session:docker run --gpus=all --rm -it -v /etc/slurm:/slurm --workdir=${PWD} \ - deepops/nccl-tests-tf20.06-ubuntu18.04:latest bash -c 'cat /slurm/slurmdbd.conf' + registry.example.com/hpc/nccl-tests:latest bash -c 'cat /slurm/slurmdbd.conf' cat: /slurm/slurmdbd.conf: Permission denied ``` @@ -464,13 +464,15 @@ Singularity and enroot could also be deployed via DeepOps. These would be useful for multi-node jobs if running on more than one DGX system. Enroot with pyxis can be tested by running: +The examples below use `registry.example.com/hpc/nccl-tests:latest` as a placeholder for a site-maintained NCCL tests image. + ```bash login-session:srun --mpi=pmi2 --ntasks=2 --gpus-per-task=1 \ - --container-image=deepops/nccl-tests-tf20.06-ubuntu18.04:latest \ + --container-image=registry.example.com/hpc/nccl-tests:latest \ all_reduce_perf -b 1M -e 4G -f 2 -g 1 ``` -The pyxis+enroot is invoked via option “ --container-image=deepops/nccl-tests-tf20.06-ubuntu18.04:latest” +The pyxis+enroot is invoked via option “ --container-image=registry.example.com/hpc/nccl-tests:latest” to run the “all_reduce_perf” nccl test. Refer to enroot and pyxis documentation for further details. @@ -490,7 +492,7 @@ Then invoke as: ```bash login-session:srun --ntasks=2 --gpus-per-task=1 --no-container-remap-root \ - --container-image=deepops/nccl-tests-tf20.06-ubuntu18.04:latest --container-workdir=${PWD} \ + --container-image=registry.example.com/hpc/nccl-tests:latest --container-workdir=${PWD} \ test-allreduce.sh ``` @@ -507,7 +509,7 @@ Singularity could be used in a similar fashion to enroot. Don’t forget the ```bash login-session:srun --mpi=pmi2 --ntasks=2 --gpus-per-task=1 \ - singularity exec --nv docker://deepops/nccl-tests-tf20.06-ubuntu18.04:latest \ + singularity exec --nv docker://registry.example.com/hpc/nccl-tests:latest \ all_reduce_perf -b 1M -e 4G -f 2 -g 1 ``` @@ -516,7 +518,7 @@ with enroot): ```bash login-session:srun --ntasks=2 --gpus-per-task=1 \ - singularity exec --nv docker://deepops/nccl-tests-tf20.06-ubuntu18.04:latest \ + singularity exec --nv docker://registry.example.com/hpc/nccl-tests:latest \ ${PWD}/test_allreduce.sh ``` diff --git a/playbooks/slurm-cluster/slurm-validation.yml b/playbooks/slurm-cluster/slurm-validation.yml index 0cc03417d..5e575941c 100644 --- a/playbooks/slurm-cluster/slurm-validation.yml +++ b/playbooks/slurm-cluster/slurm-validation.yml @@ -11,7 +11,7 @@ vars: # String; Container for nccl performance/validation tests. Either docker # repo or can be path to sqsh file. - base_container: "nvcr.io/nvidia/tensorflow:21.09-tf2-py3" + base_container: "nvcr.io/nvidia/pytorch:26.04-py3" # String; Container to be created or one that might exist with nccl tests. # If `compile_nccl_tests` is True, it must be a sqsh file. nccl_tests_container: "${HOME}/enroot_images/nccl_tests_slurm_val.sqsh" diff --git a/roles/nginx-docker-registry-cache/defaults/main.yml b/roles/nginx-docker-registry-cache/defaults/main.yml index 6824bc7d5..cea6bada1 100644 --- a/roles/nginx-docker-registry-cache/defaults/main.yml +++ b/roles/nginx-docker-registry-cache/defaults/main.yml @@ -5,7 +5,7 @@ nginx_docker_cache_image: "rpardini/docker-registry-proxy:0.6.5" nginx_docker_cache_mirror_path: "/opt/deepops/nginx-docker-cache/mirror" nginx_docker_cache_ca_path: "/opt/deepops/nginx-docker-cache/ca" -nginx_docker_cache_registry_string: "quay.io k8s.gcr.io gcr.io nvcr.io" +nginx_docker_cache_registry_string: "registry.k8s.io quay.io k8s.gcr.io gcr.io nvcr.io" nginx_docker_cache_manifests: "false" nginx_docker_cache_manifest_default_time: "1h" diff --git a/workloads/examples/k8s/dask-rapids/docker/Dockerfile b/workloads/examples/k8s/dask-rapids/docker/Dockerfile index 358fce704..6db4abe1e 100644 --- a/workloads/examples/k8s/dask-rapids/docker/Dockerfile +++ b/workloads/examples/k8s/dask-rapids/docker/Dockerfile @@ -5,9 +5,12 @@ USER root RUN apt-get update && \ apt-get install -y --no-install-recommends font-manager && \ + mkdir -p /opt/rapids/notebooks && \ + chown -R rapids:conda /opt/rapids && \ rm -rf /var/lib/apt/lists/* USER rapids +WORKDIR /opt/rapids/notebooks # Copy the parallel sum notebook in -COPY --chown=rapids:conda ParallelSum.ipynb /home/rapids/notebooks/ParallelSum.ipynb +COPY --chown=rapids:conda ParallelSum.ipynb ./ParallelSum.ipynb diff --git a/workloads/examples/k8s/pytorch-job.yml b/workloads/examples/k8s/pytorch-job.yml index 9f20d8728..69a38d749 100644 --- a/workloads/examples/k8s/pytorch-job.yml +++ b/workloads/examples/k8s/pytorch-job.yml @@ -8,9 +8,9 @@ spec: spec: containers: - name: pytorch-container - image: nvcr.io/nvidia/pytorch:19.02-py3 + image: nvcr.io/nvidia/pytorch:26.04-py3 command: ["/bin/sh"] - args: ["-c", "python /workspace/examples/upstream/mnist/main.py"] + args: ["-c", "python -c 'import torch; print(\"cuda_available=\", torch.cuda.is_available()); print(\"device_count=\", torch.cuda.device_count())'"] resources: limits: nvidia.com/gpu: 1 From 2e08909a69b77e1e472bbb777ec0e532b453905f Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Tue, 30 Jun 2026 20:42:33 -0600 Subject: [PATCH 24/27] fix: handle non-dict parsed JSON in firmware version manifest parser parse_versioning kept the last json.loads-parseable line and then indexed it with a string key. When that line parsed to a non-dict (e.g. a JSON array), the index raised an uncaught TypeError because only KeyError was handled, aborting the task. Guard that the parsed value is a dict before indexing and fall through to the existing 'no JSON could be loaded' path otherwise. Adds a regression test covering the non-dict input case. --- .../files/parse_manifest.py | 7 +++- .../tests/test_parse_manifest.py | 34 +++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 roles/nvidia-dgx-firmware/tests/test_parse_manifest.py diff --git a/roles/nvidia-dgx-firmware/files/parse_manifest.py b/roles/nvidia-dgx-firmware/files/parse_manifest.py index 0c361cfc4..d0fb1506a 100644 --- a/roles/nvidia-dgx-firmware/files/parse_manifest.py +++ b/roles/nvidia-dgx-firmware/files/parse_manifest.py @@ -3,7 +3,7 @@ import json from collections import OrderedDict -from sys import argv +from sys import argv, exit def return_json(payload): @@ -173,6 +173,11 @@ def return_json(payload): manifest_json = json.loads(line) except ValueError: pass + + if not isinstance(manifest_json, dict): + print('No JSON could be loaded, is the container already running?') + exit(1) + try: if manifest_json['ErrorWritingVersioning']: print('No JSON could be loaded, is the container already running?') diff --git a/roles/nvidia-dgx-firmware/tests/test_parse_manifest.py b/roles/nvidia-dgx-firmware/tests/test_parse_manifest.py new file mode 100644 index 000000000..9135d0978 --- /dev/null +++ b/roles/nvidia-dgx-firmware/tests/test_parse_manifest.py @@ -0,0 +1,34 @@ +import subprocess +import sys +import tempfile +import unittest + +from pathlib import Path + + +SCRIPT = Path(__file__).resolve().parents[1] / 'files' / 'parse_manifest.py' + + +class ParseManifestTest(unittest.TestCase): + def test_parse_versioning_rejects_non_dict_json(self): + with tempfile.TemporaryDirectory() as temp_dir: + manifest = Path(temp_dir) / 'versioning.json' + manifest.write_text('[1,2,3]\n') + + result = subprocess.run( + [sys.executable, str(SCRIPT), 'parse_versioning', str(manifest)], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + + self.assertEqual(result.returncode, 1) + self.assertEqual( + result.stdout, + 'No JSON could be loaded, is the container already running?\n', + ) + self.assertEqual(result.stderr, '') + + +if __name__ == '__main__': + unittest.main() From 01b332f69101702caef80dbb2b1910c484c3ec67 Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Thu, 2 Jul 2026 06:44:06 -0600 Subject: [PATCH 25/27] chore(release): bump component versions for 26.07 --- roles/alertmanager/defaults/main.yml | 2 +- roles/grafana/defaults/main.yml | 2 +- roles/nvidia-gpu-operator/defaults/main.yml | 2 +- roles/nvidia-k8s-gpu-device-plugin/defaults/main.yml | 2 +- roles/nvidia-k8s-gpu-feature-discovery/defaults/main.yml | 2 +- roles/nvidia-mig-manager/defaults/main.yml | 4 ++-- roles/nvidia-network-operator/vars/main.yaml | 2 +- roles/prometheus/defaults/main.yml | 2 +- roles/slurm/defaults/main.yml | 2 +- roles/spack/defaults/main.yml | 2 +- scripts/k8s/deploy_monitoring.sh | 2 +- 11 files changed, 12 insertions(+), 12 deletions(-) diff --git a/roles/alertmanager/defaults/main.yml b/roles/alertmanager/defaults/main.yml index 8bfdbc49d..7862048f9 100644 --- a/roles/alertmanager/defaults/main.yml +++ b/roles/alertmanager/defaults/main.yml @@ -1,6 +1,6 @@ alertmanager_config_dir: /etc/alertmanager alertmanager_config_src: templates/alertmanager.yml.j2 -alertmanager_container: "prom/alertmanager:v0.32.1" +alertmanager_container: "prom/alertmanager:v0.33.0" alertmanager_svc_name: "docker.alertmanager.service" alertmanager_docker_volume_name: "deepops_alertmanager_metrics" alertmanager_state: started diff --git a/roles/grafana/defaults/main.yml b/roles/grafana/defaults/main.yml index 920a37943..0299dae4c 100644 --- a/roles/grafana/defaults/main.yml +++ b/roles/grafana/defaults/main.yml @@ -2,7 +2,7 @@ grafana_config_dir: /etc/grafana grafana_config_template: templates/grafana.ini.j2 grafana_data_dir: /var/lib/grafana grafana_user_id: 472 -grafana_container: "grafana/grafana:13.0.1" +grafana_container: "grafana/grafana:13.1.0" grafana_svc_name: "docker.grafana.service" grafana_state: started grafana_enabled: yes diff --git a/roles/nvidia-gpu-operator/defaults/main.yml b/roles/nvidia-gpu-operator/defaults/main.yml index ebecc5d60..fb9dc7144 100644 --- a/roles/nvidia-gpu-operator/defaults/main.yml +++ b/roles/nvidia-gpu-operator/defaults/main.yml @@ -12,7 +12,7 @@ gpu_operator_nvaie_helm_repo: "https://helm.ngc.nvidia.com/nvaie" gpu_operator_nvaie_chart_name: "nvaie/gpu-operator" # NVAIE GPU Operator may require different version, check NGC enterprise collection. -gpu_operator_chart_version: "v26.3.1" +gpu_operator_chart_version: "v26.3.3" k8s_gpu_mig_strategy: "mixed" diff --git a/roles/nvidia-k8s-gpu-device-plugin/defaults/main.yml b/roles/nvidia-k8s-gpu-device-plugin/defaults/main.yml index 021d9a3f2..6b4306af7 100644 --- a/roles/nvidia-k8s-gpu-device-plugin/defaults/main.yml +++ b/roles/nvidia-k8s-gpu-device-plugin/defaults/main.yml @@ -2,6 +2,6 @@ k8s_gpu_plugin_helm_repo: "https://nvidia.github.io/k8s-device-plugin" k8s_gpu_plugin_chart_name: "nvdp/nvidia-device-plugin" k8s_gpu_plugin_release_name: "nvidia-device-plugin" -k8s_gpu_plugin_chart_version: "0.19.1" +k8s_gpu_plugin_chart_version: "0.19.3" k8s_gpu_plugin_init_error: "false" k8s_gpu_mig_strategy: "mixed" diff --git a/roles/nvidia-k8s-gpu-feature-discovery/defaults/main.yml b/roles/nvidia-k8s-gpu-feature-discovery/defaults/main.yml index 77111b3ff..ec7bbd0bf 100644 --- a/roles/nvidia-k8s-gpu-feature-discovery/defaults/main.yml +++ b/roles/nvidia-k8s-gpu-feature-discovery/defaults/main.yml @@ -2,5 +2,5 @@ k8s_gpu_feature_discovery_helm_repo: "https://nvidia.github.io/k8s-device-plugin" k8s_gpu_feature_discovery_chart_name: "nvgfd/gpu-feature-discovery" k8s_gpu_feature_discovery_release_name: "gpu-feature-discovery" -k8s_gpu_feature_discovery_chart_version: "0.19.1" +k8s_gpu_feature_discovery_chart_version: "0.19.3" k8s_gpu_mig_strategy: "mixed" diff --git a/roles/nvidia-mig-manager/defaults/main.yml b/roles/nvidia-mig-manager/defaults/main.yml index 80d0c60f4..fde9e0f02 100644 --- a/roles/nvidia-mig-manager/defaults/main.yml +++ b/roles/nvidia-mig-manager/defaults/main.yml @@ -1,3 +1,3 @@ --- -mig_manager_url_deb: https://github.com/NVIDIA/mig-parted/releases/download/v0.14.1/nvidia-mig-manager_0.14.1-1_amd64.deb -mig_manager_url_rpm: https://github.com/NVIDIA/mig-parted/releases/download/v0.14.1/nvidia-mig-manager-0.14.1-1.x86_64.rpm +mig_manager_url_deb: https://github.com/NVIDIA/mig-parted/releases/download/v0.14.2/nvidia-mig-manager_0.14.2-1_amd64.deb +mig_manager_url_rpm: https://github.com/NVIDIA/mig-parted/releases/download/v0.14.2/nvidia-mig-manager-0.14.2-1.x86_64.rpm diff --git a/roles/nvidia-network-operator/vars/main.yaml b/roles/nvidia-network-operator/vars/main.yaml index 7be8ee253..f7976df43 100644 --- a/roles/nvidia-network-operator/vars/main.yaml +++ b/roles/nvidia-network-operator/vars/main.yaml @@ -6,7 +6,7 @@ # if_name must match k8s network annotation name # -nvidia_network_operator_version: "26.1.1" +nvidia_network_operator_version: "26.4.0" nvidia_network_operator_image_tag: "network-operator-v{{ nvidia_network_operator_version }}" nvidia_network_operator_namespace: "network-operator" nvidia_network_operator_ipam_type: "nv-ipam" diff --git a/roles/prometheus/defaults/main.yml b/roles/prometheus/defaults/main.yml index 7349da1b3..fe354a7b8 100644 --- a/roles/prometheus/defaults/main.yml +++ b/roles/prometheus/defaults/main.yml @@ -1,7 +1,7 @@ prometheus_config_dir: /etc/prometheus prometheus_config_src: templates/prometheus.yml.j2 prometheus_alert_rules_src: templates/alert_rules.yml.j2 -prometheus_container: "prom/prometheus:v3.11.3" +prometheus_container: "prom/prometheus:v3.13.0" prometheus_svc_name: "docker.prometheus.service" prometheus_docker_volume_name: "deepops_prometheus_metrics" prometheus_state: started diff --git a/roles/slurm/defaults/main.yml b/roles/slurm/defaults/main.yml index 735ae16ee..ac0435c0e 100644 --- a/roles/slurm/defaults/main.yml +++ b/roles/slurm/defaults/main.yml @@ -7,7 +7,7 @@ hwloc_build_dir: /opt/deepops/build/hwloc pmix_build_dir: /opt/deepops/build/pmix slurm_workflow_build: yes -slurm_version: "25.11.6" +slurm_version: "26.05.1" slurm_src_url: "https://download.schedmd.com/slurm/slurm-{{ slurm_version }}.tar.bz2" slurm_build_make_clean: no slurm_build_dir_cleanup: no diff --git a/roles/spack/defaults/main.yml b/roles/spack/defaults/main.yml index 76e356c63..63192c63b 100644 --- a/roles/spack/defaults/main.yml +++ b/roles/spack/defaults/main.yml @@ -1,7 +1,7 @@ --- spack_repo: "https://github.com/spack/spack.git" spack_install_dir: "/sw/spack" -spack_version: "v1.1.1" +spack_version: "v1.2.0" spack_user: "root" spack_group: "root" diff --git a/scripts/k8s/deploy_monitoring.sh b/scripts/k8s/deploy_monitoring.sh index 05a5851a7..51bdd7569 100755 --- a/scripts/k8s/deploy_monitoring.sh +++ b/scripts/k8s/deploy_monitoring.sh @@ -24,7 +24,7 @@ if [ ! -d "${DEEPOPS_CONFIG_DIR}" ]; then fi HELM_CHARTS_REPO_PROMETHEUS="${HELM_CHARTS_REPO_PROMETHEUS:-https://prometheus-community.github.io/helm-charts}" -HELM_PROMETHEUS_CHART_VERSION="${HELM_PROMETHEUS_CHART_VERSION:-85.0.3}" +HELM_PROMETHEUS_CHART_VERSION="${HELM_PROMETHEUS_CHART_VERSION:-87.5.1}" ingress_name="ingress-nginx" PROMETHEUS_YAML_CONFIG="${PROMETHEUS_YAML_CONFIG:-${DEEPOPS_CONFIG_DIR}/helm/monitoring.yml}" From 3dfe40ec62844ae747d0a13767b946cc2af55983 Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Thu, 2 Jul 2026 06:44:06 -0600 Subject: [PATCH 26/27] fix: derive DEEPOPS_VERSION for script debug output scripts/common.sh printed an always-empty DEEPOPS_VERSION unless a user happened to set it in config/env.sh. Default it to the checkout's git tag description (git describe --tags --always) with an unknown fallback, keep the env.sh override, and document the variable in config.example/env.sh. --- config.example/env.sh | 4 ++++ scripts/common.sh | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/config.example/env.sh b/config.example/env.sh index 288e585db..02ce5b85e 100644 --- a/config.example/env.sh +++ b/config.example/env.sh @@ -4,3 +4,7 @@ # See deepops/scripts/common.sh for implementation details DEEPOPS_EXAMPLE_VAR="" + +# DeepOps version reported in script debug output. +# Defaults to the checkout's git tag description; uncomment to override. +# DEEPOPS_VERSION="" diff --git a/scripts/common.sh b/scripts/common.sh index 024f0b690..a08174d38 100644 --- a/scripts/common.sh +++ b/scripts/common.sh @@ -19,5 +19,11 @@ else source ${DEEPOPS_CONFIG_DIR}/env.sh fi +# Determine the DeepOps version for debug output. +# Can be overridden in config/env.sh; defaults to the checkout's git tag description. +if [ -z "${DEEPOPS_VERSION}" ]; then + DEEPOPS_VERSION="$(git -C "${ROOT_DIR}" describe --tags --always 2>/dev/null || echo unknown)" +fi + # Print out base debug echo "Starting '${0}'; DeepOps version '${DEEPOPS_VERSION}'" From e806b5c7ee423d31f193a15cb015f472368b3c54 Mon Sep 17 00:00:00 2001 From: Doug Holt Date: Thu, 28 May 2026 21:15:24 -0600 Subject: [PATCH 27/27] Clarify Kubernetes storage helper ownership --- config.example/group_vars/k8s_cluster.yml | 4 +- config.example/group_vars/netapp-trident.yml | 62 ++++++++-------- docs/deepops/proxy-k8s-cluster.md | 2 +- docs/k8s-cluster/README.md | 27 ++++--- docs/k8s-cluster/kubeflow.md | 4 +- .../k8s-cluster/nfs-client-provisioner.yml | 18 +++-- roles/netapp-trident/README.md | 4 +- roles/netapp-trident/defaults/main.yml | 58 ++++++++------- roles/netapp-trident/tasks/main.yml | 71 +++++++++---------- .../nfs-client-provisioner/defaults/main.yml | 2 +- scripts/k8s/deploy_kubeflow.sh | 2 +- scripts/k8s/deploy_monitoring.sh | 4 +- scripts/k8s/deploy_rook.sh | 10 ++- 13 files changed, 150 insertions(+), 118 deletions(-) diff --git a/config.example/group_vars/k8s_cluster.yml b/config.example/group_vars/k8s_cluster.yml index 81b5913ab..237829576 100644 --- a/config.example/group_vars/k8s_cluster.yml +++ b/config.example/group_vars/k8s_cluster.yml @@ -46,8 +46,10 @@ dashboard_metrics_scraper_repo: "kubernetesui/metrics-scraper" # Ensure hosts file generation only runs across k8s cluster hosts_add_ansible_managed_hosts_groups: ["k8s_cluster"] -# NFS Client Provisioner +# NFS Client Provisioner # Playbook: nfs-client-provisioner.yml +# This convenience default creates a non-HA NFS export on the first control-plane +# node. For production clusters, point this at site-owned storage or disable it. k8s_nfs_client_provisioner: true k8s_deploy_nfs_server: true k8s_nfs_mkdir: true # Set to false if an export dir is already configured with proper permissions diff --git a/config.example/group_vars/netapp-trident.yml b/config.example/group_vars/netapp-trident.yml index 33b9aa9c4..d8a27ebf1 100644 --- a/config.example/group_vars/netapp-trident.yml +++ b/config.example/group_vars/netapp-trident.yml @@ -1,29 +1,36 @@ --- # vars file for netapp-trident playbook -# URL of the Trident installer package that you wish to download and use -trident_version: "22.01.0" +# Version of the Trident installer package to download when tridentctl is needed +trident_version: "26.02.1" trident_installer_url: "https://github.com/NetApp/trident/releases/download/v{{ trident_version }}/trident-installer-{{ trident_version }}.tar.gz" +# Official Trident Helm chart settings +trident_helm_repo_name: netapp-trident +trident_helm_repo_url: https://netapp.github.io/trident-helm-chart +trident_helm_release_name: trident-operator +trident_helm_chart_name: "{{ trident_helm_repo_name }}/trident-operator" +trident_helm_chart_version: "100.2602.1" + # Namespace to install Trident in trident_namespace: deepops-trident # Denotes whether or not to create new backends after deploying trident -# For more info, refer to: https://netapp-trident.readthedocs.io/en/stable-v20.04/kubernetes/operator-install.html#creating-a-trident-backend -create_backends: true +# For more info, refer to: https://docs.netapp.com/us-en/trident/trident-use/ontap-nas-examples.html +create_backends: false +trident_hide_backend_details: true # List of backends to create -# For more info on parameter values, refer to: https://netapp-trident.readthedocs.io/en/stable-v20.04/kubernetes/operations/tasks/backends/ontap.html -# Note: Parameters other than those listed below are not avaible when creating a backend via DeepOps +# For more info on parameter values, refer to: https://docs.netapp.com/us-en/trident/trident-use/ontap-nas-examples.html +# Note: Parameters other than those listed below are not available when creating a backend via DeepOps # If you wish to use other parameter values, you must create your backend manually. backends_to_create: - backendName: ontap-flexvol storageDriverName: ontap-nas # only 'ontap-nas' and 'ontap-nas-flexgroup' are supported when creating a backend via DeepOps - managementLIF: 10.61.188.40 - dataLIF: 192.168.200.41 - svm: ai221_data - username: admin - password: NetApp!23 + managementLIF: ontap-mgmt.example.com + dataLIF: ontap-nfs.example.com + svm: svm_nfs + username: vsadmin storagePrefix: trident limitAggregateUsage: "" limitVolumeSize: "" @@ -41,11 +48,10 @@ backends_to_create: tieringPolicy: none - backendName: ontap-flexgroup storageDriverName: ontap-nas-flexgroup # only 'ontap-nas' and 'ontap-nas-flexgroup' are supported when creating a backend via DeepOps - managementLIF: 10.61.188.40 - dataLIF: 192.168.200.41 - svm: ai221_data - username: admin - password: NetApp!23 + managementLIF: ontap-mgmt.example.com + dataLIF: ontap-nfs.example.com + svm: svm_nfs + username: vsadmin storagePrefix: trident limitAggregateUsage: "" limitVolumeSize: "" @@ -64,12 +70,12 @@ backends_to_create: # Add additional backends as needed # Denotes whether or not to create new StorageClasses for your NetApp storage -# For more info, refer to: https://netapp-trident.readthedocs.io/en/stable-v20.04/kubernetes/operator-install.html#creating-a-storage-class -create_StorageClasses: true +# For more info, refer to: https://docs.netapp.com/us-en/trident/trident-use/create-stor-class.html +create_StorageClasses: false # List of StorageClasses to create # Note: Each item in the list should be an actual K8s StorageClass definition in yaml format -# For more info on StorageClass definitions, refer to https://netapp-trident.readthedocs.io/en/stable-v20.04/kubernetes/concepts/objects.html#kubernetes-storageclass-objects. +# For more info on StorageClass definitions, refer to https://docs.netapp.com/us-en/trident/trident-use/create-stor-class.html storageClasses_to_create: - apiVersion: storage.k8s.io/v1 kind: StorageClass @@ -90,30 +96,30 @@ storageClasses_to_create: # Add additional StorageClasses as needed # Denotes whether or not to enable volume snapshots in your Kubernetes cluster -# For more info, refer to: https://netapp-trident.readthedocs.io/en/stable-v21.01/kubernetes/operations/tasks/volumes/snapshots.html -enable_volume_snapshots: true +# For more info, refer to: https://docs.netapp.com/us-en/trident/trident-use/vol-snapshots.html +enable_volume_snapshots: false # List of volume snapshot CRDs to create volume_snapshot_CRD_manifest_URLs: - - https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/release-3.0/client/config/crd/snapshot.storage.k8s.io_volumesnapshotclasses.yaml - - https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/release-3.0/client/config/crd/snapshot.storage.k8s.io_volumesnapshotcontents.yaml - - https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/release-3.0/client/config/crd/snapshot.storage.k8s.io_volumesnapshots.yaml + - https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/release-8.3/client/config/crd/snapshot.storage.k8s.io_volumesnapshotclasses.yaml + - https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/release-8.3/client/config/crd/snapshot.storage.k8s.io_volumesnapshotcontents.yaml + - https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/release-8.3/client/config/crd/snapshot.storage.k8s.io_volumesnapshots.yaml # List of snapshot controller manifest URLs snapshot_controller_manifest_URLs: - - https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/release-3.0/deploy/kubernetes/snapshot-controller/rbac-snapshot-controller.yaml - - https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/release-3.0/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml + - https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/release-8.3/deploy/kubernetes/snapshot-controller/rbac-snapshot-controller.yaml + - https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/release-8.3/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml # List of volume snapshot classes to create volume_snapshot_classes_to_create: - - apiVersion: snapshot.storage.k8s.io/v1beta1 + - apiVersion: snapshot.storage.k8s.io/v1 kind: VolumeSnapshotClass metadata: name: csi-snapclass driver: csi.trident.netapp.io deletionPolicy: Delete -# Denotes whether or not to copy tridenctl binary to localhost +# Denotes whether or not to copy tridentctl binary to localhost copy_tridentctl_to_localhost: true # Directory that tridentctl will be copied to on localhost tridentctl_copy_to_directory: ../../ # will be copied to 'deepops/' directory diff --git a/docs/deepops/proxy-k8s-cluster.md b/docs/deepops/proxy-k8s-cluster.md index 165a64e5c..4e478748f 100644 --- a/docs/deepops/proxy-k8s-cluster.md +++ b/docs/deepops/proxy-k8s-cluster.md @@ -37,4 +37,4 @@ Before executing the scripts to continue setting up and installing services you 1. Run the `scripts/deepops/proxy.sh` script to setup environment variables. Then continue running all the scripts necessary for your environment. 2. Rather than setup the variables for your current shell, you can run each script to use the proxy without impacting your current env. For example: - `. scripts/deepops/proxy.sh && scripts/k8s/deploy_rook.sh` + `. scripts/deepops/proxy.sh && DEEPOPS_ENABLE_DEPRECATED_ROOK=true scripts/k8s/deploy_rook.sh` diff --git a/docs/k8s-cluster/README.md b/docs/k8s-cluster/README.md index 3230f24a5..c9d978efc 100644 --- a/docs/k8s-cluster/README.md +++ b/docs/k8s-cluster/README.md @@ -11,7 +11,7 @@ Instructions for deploying a GPU cluster with Kubernetes - [Persistent Storage](#persistent-storage) - [NFS Client Provisioner](#nfs-client-provisioner) - [Ceph Cluster (deprecated)](#ceph-cluster-deprecated) - - [NetApp Astra Trident](#netapp-astra-trident) + - [NetApp Trident](#netapp-trident) - [Monitoring](#monitoring) - [Logging](#logging) - [Centralized syslog](#centralized-syslog) @@ -124,9 +124,9 @@ Run the following script to create an administrative user and print out the dash #### NFS Client Provisioner -The default behavior of DeepOps is to setup an NFS server on the first `kube_control_plane` node. This temporary NFS server is used by the `nfs-client-provisioner` which is installed as the default StorageClass of a standard DeepOps deployment. +The default DeepOps Kubernetes storage path is the `nfs-subdir-external-provisioner` chart, backed by an NFS export on the first `kube_control_plane` node. This is a convenience default for examples, smoke tests, and small internal clusters. It is not a highly available production storage design. -To use an existing nfs server server update the `k8s_nfs_server` and `k8s_nfs_export_path` variables in `config/group_vars/k8s-cluster.yml` and set the `k8s_deploy_nfs_server` to false in `config/group_vars/k8s-cluster.yml`. Additionally, the `k8s_nfs_mkdir` variable can be set to `false` if the export directory is already configured on the server. +To use an existing NFS server, update the `k8s_nfs_server` and `k8s_nfs_export_path` variables in `config/group_vars/k8s-cluster.yml` and set `k8s_deploy_nfs_server` to `false`. Additionally, set `k8s_nfs_mkdir` to `false` if the export directory is already configured on the server. To manually install or re-install the `nfs-client-provisioner` run: @@ -134,13 +134,16 @@ To manually install or re-install the `nfs-client-provisioner` run: ansible-playbook playbooks/k8s-cluster/nfs-client-provisioner.yml ``` -To skip this installation set `k8s_nfs_client_provisioner` to `false`. +To skip this installation, set `k8s_nfs_client_provisioner` to `false`. #### Ceph Cluster (deprecated) -For a non-nfs based alternative, deploy a Ceph cluster running on Kubernetes for services that require persistent storage (such as Kubeflow): +The Rook/Ceph helper is deprecated and community-supported. It predates the current Rook Helm chart layout and should not be used as a new production storage path without a site-owned Rook/Ceph design and validation plan. + +For legacy environments that still use this helper, deploy a Ceph cluster running on Kubernetes with: ```bash +export DEEPOPS_ENABLE_DEPRECATED_ROOK=true ./scripts/k8s/deploy_rook.sh ``` @@ -150,9 +153,9 @@ Poll the Ceph status by running (this script will return when Ceph initializatio ./scripts/k8s/deploy_rook.sh -w ``` -#### NetApp Astra Trident +#### NetApp Trident -Deploy NetApp Astra Trident for services that require persistent storage (such as Kubeflow). Note that you must have a supported NetApp storage system/instance/service in order to use Astra Trident to provision persistent storage. +The Trident role is optional and community-supported in DeepOps. Use it only when the site already has a supported NetApp storage system or service and a storage owner who can validate the backend, StorageClass, and snapshot policy against current NetApp documentation. 1. Set configuration parameters. @@ -160,7 +163,9 @@ Deploy NetApp Astra Trident for services that require persistent storage (such a vi config/group_vars/netapp-trident.yml ``` -2. Deploy Astra Trident using Ansible. + By default, the example configuration installs Trident but does not create a backend, StorageClass, or snapshot controller. Set `create_backends`, `create_StorageClasses`, or `enable_volume_snapshots` to `true` only after replacing the example backend values with site-specific storage details and auth values. + +2. Deploy Trident using Ansible. ```bash # NOTE: If SSH requires a password, add: `-k` @@ -169,7 +174,7 @@ Deploy NetApp Astra Trident for services that require persistent storage (such a ansible-playbook -l k8s_cluster playbooks/k8s-cluster/netapp-trident.yml ``` -3. Verify that Astra Trident is running. +3. Verify that Trident is running. ```bash ./tridentctl -n deepops-trident version @@ -181,11 +186,11 @@ Deploy NetApp Astra Trident for services that require persistent storage (such a +----------------+----------------+ | SERVER VERSION | CLIENT VERSION | +----------------+----------------+ - | 22.01.0 | 22.01.0 | + | 26.02.1 | 26.02.1 | +----------------+----------------+ ``` -For more information on Astra Trident, please refer to the [official documentation](https://docs.netapp.com/us-en/trident/index.html). +For more information on Trident, refer to the [official documentation](https://docs.netapp.com/us-en/trident/index.html). ### Monitoring diff --git a/docs/k8s-cluster/kubeflow.md b/docs/k8s-cluster/kubeflow.md index 59f4d6797..f5bfc1e80 100644 --- a/docs/k8s-cluster/kubeflow.md +++ b/docs/k8s-cluster/kubeflow.md @@ -29,7 +29,7 @@ As part of the Kubeflow installation, the MPI Operator will also be installed. T Deploy Kubernetes by following the [DeepOps Kubernetes Deployment Guide](README.md) -Kubeflow requires a DefaultStorageClass to be defined. By default DeepOps installs the `nfs-client-provisioner` using the [nfs-client-provisioner.yml playbook](../../playbooks/k8s-cluster/nfs-client-provisioner.yml). This playbook can re run manually. As an NFS alternative [Ceph](../../scripts/k8s/deploy_rook.sh), [Trident](../../playbooks/k8s-cluster/netapp-trident.yml) or an alternative StorageClass can be used. +Kubeflow requires a DefaultStorageClass to be defined. By default DeepOps installs the `nfs-client-provisioner` using the [nfs-client-provisioner.yml playbook](../../playbooks/k8s-cluster/nfs-client-provisioner.yml). This playbook can be rerun manually. For production or HA storage, use a site-owned StorageClass. The legacy [Ceph](../../scripts/k8s/deploy_rook.sh) helper is deprecated, and [Trident](../../playbooks/k8s-cluster/netapp-trident.yml) is optional and community-supported unless a site storage owner validates it. Deploy Kubeflow: @@ -125,6 +125,6 @@ To correct this issue: 1. Uninstall Rook/Ceph: `./scripts/k8s/deploy_rook.sh -d` 2. Uninstall Kubeflow: `./scripts/k8s/deploy_kubeflow.sh -d` -3. Re-install Rook/ceph: `./scripts/k8s/deploy_rook.sh` +3. Re-install Rook/Ceph: `DEEPOPS_ENABLE_DEPRECATED_ROOK=true ./scripts/k8s/deploy_rook.sh` 4. Poll for Ceph to initialize (wait for this script to exit): `./scripts/k8s/deploy_rook.sh -w` 5. Re-install Kubeflow: `./scripts/k8s/deploy_kubeflow.sh` diff --git a/playbooks/k8s-cluster/nfs-client-provisioner.yml b/playbooks/k8s-cluster/nfs-client-provisioner.yml index 200f27f99..d40741e58 100644 --- a/playbooks/k8s-cluster/nfs-client-provisioner.yml +++ b/playbooks/k8s-cluster/nfs-client-provisioner.yml @@ -1,14 +1,14 @@ --- # This playbook is used to setup the nfs-client-provisioner # If the cluster already has an nfs server setup run with --skip-tags="nfs_server,nfs_mkdir" -# Otherwhise an NFS server will be configured on kube_control_plane[0] -# After settting up an NFS server, nfs software is installed on all nodes -# Lastly, the nfs-client-provisioner is helm installed using the role +# Otherwise an NFS server will be configured on kube_control_plane[0] +# After setting up an NFS server, NFS software is installed on all nodes. +# Lastly, the nfs-client-provisioner chart is installed using the role. # -# The default NFS configuration of this playbook is not meant for production systems -# For production deployments please configur NFS with storage-partner guidance +# The default NFS configuration of this playbook is not meant for production systems. +# For production deployments, configure storage with storage-partner guidance. # -# Install the required NFS software on all nodes and then helm install the nfs-client +# Install the required NFS software on all nodes and then helm install the NFS client. - hosts: kube_control_plane[0] become: yes tasks: @@ -17,13 +17,17 @@ path: "{{ k8s_nfs_export_path }}" state: directory mode: "u=rwx,g=rwx,o=rwx" - when: k8s_nfs_mkdir and k8s_deploy_nfs_server + when: k8s_nfs_mkdir and k8s_deploy_nfs_server + tags: + - nfs_mkdir - name: Deploy NFS server master include_role: name: nfs vars: nfs_is_server: yes when: k8s_deploy_nfs_server + tags: + - nfs_server - hosts: "k8s_cluster" become: yes diff --git a/roles/netapp-trident/README.md b/roles/netapp-trident/README.md index 7911d097b..ad20f9b69 100644 --- a/roles/netapp-trident/README.md +++ b/roles/netapp-trident/README.md @@ -3,6 +3,8 @@ netapp-trident Ansible role that can be used to deploy NetApp Trident within a Kubernetes cluster. +This role is optional and community-supported in DeepOps. Use it only when a site storage owner can validate the NetApp backend, StorageClass, and snapshot policy against current NetApp documentation. + Requirements ------------ @@ -15,7 +17,7 @@ Requirements Role Variables -------------- -See defaults/main.yml, vars/main.yml +See defaults/main.yml and `config.example/group_vars/netapp-trident.yml`. Dependencies ------------ diff --git a/roles/netapp-trident/defaults/main.yml b/roles/netapp-trident/defaults/main.yml index 688c4d6cd..ec880b251 100644 --- a/roles/netapp-trident/defaults/main.yml +++ b/roles/netapp-trident/defaults/main.yml @@ -1,29 +1,36 @@ --- # defaults file for netapp-trident role -# URL of the Trident installer package that you wish to download and use -trident_version: "21.01.2" +# Version of the Trident installer package to download when tridentctl is needed +trident_version: "26.02.1" trident_installer_url: "https://github.com/NetApp/trident/releases/download/v{{ trident_version }}/trident-installer-{{ trident_version }}.tar.gz" +# Official Trident Helm chart settings +trident_helm_repo_name: netapp-trident +trident_helm_repo_url: https://netapp.github.io/trident-helm-chart +trident_helm_release_name: trident-operator +trident_helm_chart_name: "{{ trident_helm_repo_name }}/trident-operator" +trident_helm_chart_version: "100.2602.1" + # Namespace to install Trident in trident_namespace: trident # Denotes whether or not to create new backends after deploying trident -# For more info, refer to: https://netapp-trident.readthedocs.io/en/stable-v20.04/kubernetes/operator-install.html#creating-a-trident-backend +# For more info, refer to: https://docs.netapp.com/us-en/trident/trident-use/ontap-nas-examples.html create_backends: false +trident_hide_backend_details: true # List of backends to create -# For more info on parameter values, refer to: https://netapp-trident.readthedocs.io/en/stable-v20.04/kubernetes/operations/tasks/backends/ontap.html -# Note: Parameters other than those listed below are not avaible when creating a backend via DeepOps +# For more info on parameter values, refer to: https://docs.netapp.com/us-en/trident/trident-use/ontap-nas-examples.html +# Note: Parameters other than those listed below are not available when creating a backend via DeepOps # If you wish to use other parameter values, you must create your backend manually. backends_to_create: - backendName: ontap-flexvol storageDriverName: ontap-nas # only 'ontap-nas' and 'ontap-nas-flexgroup' are supported when creating a backend via DeepOps - managementLIF: 10.61.188.40 - dataLIF: 192.168.200.41 - svm: ai221_data - username: admin - password: NetApp!23 + managementLIF: ontap-mgmt.example.com + dataLIF: ontap-nfs.example.com + svm: svm_nfs + username: vsadmin storagePrefix: trident limitAggregateUsage: "" limitVolumeSize: "" @@ -41,11 +48,10 @@ backends_to_create: tieringPolicy: none - backendName: ontap-flexgroup storageDriverName: ontap-nas-flexgroup # only 'ontap-nas' and 'ontap-nas-flexgroup' are supported when creating a backend via DeepOps - managementLIF: 10.61.188.40 - dataLIF: 192.168.200.41 - svm: ai221_data - username: admin - password: NetApp!23 + managementLIF: ontap-mgmt.example.com + dataLIF: ontap-nfs.example.com + svm: svm_nfs + username: vsadmin storagePrefix: trident limitAggregateUsage: "" limitVolumeSize: "" @@ -64,12 +70,12 @@ backends_to_create: # Add additional backends as needed # Denotes whether or not to create new StorageClasses for your NetApp storage -# For more info, refer to: https://netapp-trident.readthedocs.io/en/stable-v20.04/kubernetes/operator-install.html#creating-a-storage-class +# For more info, refer to: https://docs.netapp.com/us-en/trident/trident-use/create-stor-class.html create_StorageClasses: false # List of StorageClasses to create # Note: Each item in the list should be an actual K8s StorageClass definition in yaml format -# For more info on StorageClass definitions, refer to https://netapp-trident.readthedocs.io/en/stable-v20.04/kubernetes/concepts/objects.html#kubernetes-storageclass-objects. +# For more info on StorageClass definitions, refer to https://docs.netapp.com/us-en/trident/trident-use/create-stor-class.html storageClasses_to_create: - apiVersion: storage.k8s.io/v1 kind: StorageClass @@ -90,30 +96,30 @@ storageClasses_to_create: # Add additional StorageClasses as needed # Denotes whether or not to enable volume snapshots in your Kubernetes cluster -# For more info, refer to: https://netapp-trident.readthedocs.io/en/stable-v21.01/kubernetes/operations/tasks/volumes/snapshots.html -enable_volume_snapshots: true +# For more info, refer to: https://docs.netapp.com/us-en/trident/trident-use/vol-snapshots.html +enable_volume_snapshots: false # List of volume snapshot CRDs to create volume_snapshot_CRD_manifest_URLs: - - https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/release-3.0/client/config/crd/snapshot.storage.k8s.io_volumesnapshotclasses.yaml - - https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/release-3.0/client/config/crd/snapshot.storage.k8s.io_volumesnapshotcontents.yaml - - https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/release-3.0/client/config/crd/snapshot.storage.k8s.io_volumesnapshots.yaml + - https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/release-8.3/client/config/crd/snapshot.storage.k8s.io_volumesnapshotclasses.yaml + - https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/release-8.3/client/config/crd/snapshot.storage.k8s.io_volumesnapshotcontents.yaml + - https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/release-8.3/client/config/crd/snapshot.storage.k8s.io_volumesnapshots.yaml # List of snapshot controller manifest URLs snapshot_controller_manifest_URLs: - - https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/release-3.0/deploy/kubernetes/snapshot-controller/rbac-snapshot-controller.yaml - - https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/release-3.0/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml + - https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/release-8.3/deploy/kubernetes/snapshot-controller/rbac-snapshot-controller.yaml + - https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/release-8.3/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml # List of volume snapshot classes to create volume_snapshot_classes_to_create: - - apiVersion: snapshot.storage.k8s.io/v1beta1 + - apiVersion: snapshot.storage.k8s.io/v1 kind: VolumeSnapshotClass metadata: name: csi-snapclass driver: csi.trident.netapp.io deletionPolicy: Delete -# Denotes whether or not to copy tridenctl binary to localhost +# Denotes whether or not to copy tridentctl binary to localhost copy_tridentctl_to_localhost: true # Directory that tridentctl will be copied to on localhost tridentctl_copy_to_directory: ./ diff --git a/roles/netapp-trident/tasks/main.yml b/roles/netapp-trident/tasks/main.yml index 71788b7c4..d8840271f 100644 --- a/roles/netapp-trident/tasks/main.yml +++ b/roles/netapp-trident/tasks/main.yml @@ -1,32 +1,37 @@ --- # tasks file for netapp-trident role -- name: Download and extract Trident installer - unarchive: - src: '{{ trident_installer_url }}' - dest: '{{ deepops_dir }}' - remote_src: yes - -- name: Copy namespace config file to control node - template: - src: namespace.j2 - dest: '{{ deepops_dir }}/trident-installer/namespace.yaml' - mode: "0644" +- name: Add Trident Helm repo + command: '/usr/local/bin/helm repo add --force-update {{ trident_helm_repo_name }} {{ trident_helm_repo_url }}' run_once: true + changed_when: false -- name: Create Trident namespace - kube: - state: present - resource: Namespace - name: trident - filename: '{{ deepops_dir }}/trident-installer/namespace.yaml' +- name: Update Helm repos + command: /usr/local/bin/helm repo update run_once: true + changed_when: false - name: Install Trident using Helm - command: '/usr/local/bin/helm -n {{ trident_namespace }} install trident {{ deepops_dir }}/trident-installer/helm/trident-operator-{{ trident_version }}.tgz' + command: '/usr/local/bin/helm upgrade --install {{ trident_helm_release_name }} {{ trident_helm_chart_name }} --namespace {{ trident_namespace }} --create-namespace --version {{ trident_helm_chart_version }} --wait' run_once: true changed_when: false +- name: Create Trident working directory + file: + path: '{{ deepops_dir }}/trident-installer' + state: directory + mode: "0755" + when: create_backends or create_StorageClasses or enable_volume_snapshots or copy_tridentctl_to_localhost + run_once: true + +- name: Download and extract Trident installer + unarchive: + src: '{{ trident_installer_url }}' + dest: '{{ deepops_dir }}' + remote_src: yes + when: create_backends or copy_tridentctl_to_localhost + run_once: true + - name: Copy backend config file(s) to control node template: src: backend.j2 @@ -35,6 +40,7 @@ with_items: '{{ backends_to_create }}' when: create_backends run_once: true + no_log: "{{ trident_hide_backend_details | bool }}" - name: Create Trident backend(s) command: '{{ deepops_dir }}/trident-installer/tridentctl -n {{ trident_namespace }} create backend -f {{ deepops_dir }}/trident-installer/backend-{{ item.backendName }}.json' @@ -45,10 +51,11 @@ until: result.rc == 0 when: create_backends run_once: true + no_log: "{{ trident_hide_backend_details | bool }}" - name: Copy StorageClass config file(s) to control node copy: - content: '{{ item }}' + content: '{{ item | to_nice_yaml }}' dest: '{{ deepops_dir }}/trident-installer/sc-{{ item.metadata.name }}.yaml' mode: "0644" with_items: '{{ storageClasses_to_create }}' @@ -56,34 +63,29 @@ run_once: true - name: Create StorageClasses - kube: - state: present - resource: StorageClass - name: '{{ item.metadata.name }}' - filename: '{{ deepops_dir }}/trident-installer/sc-{{ item.metadata.name }}.yaml' + command: 'kubectl apply -f {{ deepops_dir }}/trident-installer/sc-{{ item.metadata.name }}.yaml' with_items: '{{ storageClasses_to_create }}' when: create_StorageClasses run_once: true + changed_when: false - name: Create volume snapshot CRDs - kube: - state: present - filename: '{{ item }}' + command: 'kubectl apply -f {{ item }}' with_items: '{{ volume_snapshot_CRD_manifest_URLs }}' when: enable_volume_snapshots run_once: true + changed_when: false - name: Create snapshot controller - kube: - state: present - filename: '{{ item }}' + command: 'kubectl apply -f {{ item }}' with_items: '{{ snapshot_controller_manifest_URLs }}' when: enable_volume_snapshots run_once: true + changed_when: false - name: Copy volume snapshot class config file(s) to control node copy: - content: '{{ item }}' + content: '{{ item | to_nice_yaml }}' dest: '{{ deepops_dir }}/trident-installer/snapclass-{{ item.metadata.name }}.yaml' mode: "0644" with_items: '{{ volume_snapshot_classes_to_create }}' @@ -91,14 +93,11 @@ run_once: true - name: Create volume snapshot classes - kube: - state: present - resource: VolumeSnapshotClass - name: '{{ item.metadata.name }}' - filename: '{{ deepops_dir }}/trident-installer/snapclass-{{ item.metadata.name }}.yaml' + command: 'kubectl apply -f {{ deepops_dir }}/trident-installer/snapclass-{{ item.metadata.name }}.yaml' with_items: '{{ volume_snapshot_classes_to_create }}' when: enable_volume_snapshots run_once: true + changed_when: false - name: Fetch tridentctl from control node fetch: diff --git a/roles/nfs-client-provisioner/defaults/main.yml b/roles/nfs-client-provisioner/defaults/main.yml index 78a467a77..16fdc6ecb 100644 --- a/roles/nfs-client-provisioner/defaults/main.yml +++ b/roles/nfs-client-provisioner/defaults/main.yml @@ -6,4 +6,4 @@ k8s_nfs_client_repo_name: "nfs-subdir-external-provisioner" k8s_nfs_client_helm_repo: "https://kubernetes-sigs.github.io/nfs-subdir-external-provisioner" k8s_nfs_client_chart_name: "{{ k8s_nfs_client_repo_name }}/nfs-subdir-external-provisioner" k8s_nfs_client_release_name: "nfs-subdir-external-provisioner" -k8s_nfs_client_chart_version: "4.0.13" +k8s_nfs_client_chart_version: "4.0.18" diff --git a/scripts/k8s/deploy_kubeflow.sh b/scripts/k8s/deploy_kubeflow.sh index 69eea2187..16acd06bc 100755 --- a/scripts/k8s/deploy_kubeflow.sh +++ b/scripts/k8s/deploy_kubeflow.sh @@ -106,7 +106,7 @@ function install_dependencies() { if [ $? -ne 0 ] ; then echo "No storageclass found" echo "To setup the nfs-client-provisioner (preferred), run: ansible-playbook playbooks/k8s-cluster/nfs-client-provisioner.yml" - echo "To provision Ceph storage, run: ./scripts/k8s/deploy_rook.sh" + echo "The Rook/Ceph helper is deprecated; use a site-owned StorageClass or see docs/k8s-cluster/README.md before using it." exit 1 fi diff --git a/scripts/k8s/deploy_monitoring.sh b/scripts/k8s/deploy_monitoring.sh index 51bdd7569..abc67aa13 100755 --- a/scripts/k8s/deploy_monitoring.sh +++ b/scripts/k8s/deploy_monitoring.sh @@ -235,7 +235,7 @@ function install_dependencies() { # Install/initialize Helm if needed ./scripts/k8s/install_helm.sh - # StorageClasse (for volumes and MySQL DB) + # StorageClass (for volumes and MySQL DB) kubectl get storageclass 2>&1 | grep "(default)" >/dev/null 2>&1 if [ $? -ne 0 ] ; then echo "No storageclass found" @@ -247,7 +247,7 @@ function install_dependencies() { else echo "To continue without persistent storage, run '${0} -x'" echo "To setup the nfs-client-provisioner (preferred), run: ansible-playbook playbooks/k8s-cluster/nfs-client-provisioner.yml" - echo "To provision Ceph storage, run: ./scripts/k8s/deploy_rook.sh" + echo "The Rook/Ceph helper is deprecated; use a site-owned StorageClass or see docs/k8s-cluster/README.md before using it." exit 1 fi fi diff --git a/scripts/k8s/deploy_rook.sh b/scripts/k8s/deploy_rook.sh index a8fffd923..0a9ad1c6d 100755 --- a/scripts/k8s/deploy_rook.sh +++ b/scripts/k8s/deploy_rook.sh @@ -39,10 +39,12 @@ function help_me() { echo "Usage:" echo "-h This message." echo "-p Print out the connection info for Rook-Ceph." - echo "-d Delete Rook from your system (this delete any created volumes)." + echo "-d Delete Rook from your system (this deletes any created volumes)." echo "-w Poll for rook-ceph to reach a healthy and initialized state." echo "-u Create a new dashboard user (default username: 'admin' password: 'deepops', set with env variables DEEPOPS_ROOK_USER/DEEPOPS_ROOK_PASS)." echo "-x Install Rook-Ceph, but do not set it as the Default StorageClass." + echo + echo "This helper is deprecated. New installs require DEEPOPS_ENABLE_DEPRECATED_ROOK=true." } @@ -141,6 +143,12 @@ function get_opts() { function install_rook() { + if [ "${DEEPOPS_ENABLE_DEPRECATED_ROOK}" != "true" ]; then + echo "DEPRECATION NOTICE: deploy_rook.sh is deprecated and community-supported." + echo "Set DEEPOPS_ENABLE_DEPRECATED_ROOK=true only for legacy environments that still need this helper." + exit 1 + fi + # Install Helm if it is not already installed ${SCRIPT_DIR}/install_helm.sh