diff --git a/.claude/settings.local.json b/.claude/settings.local.json new file mode 100644 index 0000000..a8b97a3 --- /dev/null +++ b/.claude/settings.local.json @@ -0,0 +1,14 @@ +{ + "permissions": { + "allow": [ + "Bash(git -C /home/spoulson/src/frostd log --oneline)", + "Bash(git *)", + "Bash(go version *)", + "Bash(go get *)", + "Bash(chmod +x *)", + "Bash(go build *)", + "Bash(go test *)", + "Bash(go mod *)" + ] + } +} diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml new file mode 100644 index 0000000..79e0cde --- /dev/null +++ b/.github/workflows/verify.yml @@ -0,0 +1,35 @@ +name: Verify + +on: + push: + branches: [main] + pull_request: + branches: [main] + +env: + GOLANGCI_LINT_VERSION: v2.11.4 + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + - uses: golangci/golangci-lint-action@v9 + with: + version: ${{ env.GOLANGCI_LINT_VERSION }} + + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + - run: go test -race ./... diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9123135 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +/frostd +/bin +frostd*.deb diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..646718f --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,206 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with +code in this repository. + +## Project + +`frostd` is a daemon that dynamically regulates fan speeds on Dell PowerEdge +servers based on hardware temperature metrics. + +`frostd` supports Dell PowerEdge 12th generation servers, such as R720. + +`frostd` identifies supported devices and actively monitors metrics and adjusts +chassis fan speed using IPMI. + +## Sensor Metrics + +Devices are grouped by sensor type. Each sensor type has zero or more sensor +IDs. + +### CPU Sensor Metrics + +`frostd` uses IPMI to read CPU temperatures of each package. Each package is +listed as a sensor ID. + +### NVIDIA Tesla P40 GPU Sensor Metrics + +`frostd` supports monitoring of an optional NVIDIA Tesla P40 GPU. frostd uses +`nvidia-smi` command to read GPU temperature. + +#### System Prerequisites + +- Compatible NVIDIA driver installed +- Compatible Cuda driver installed + - This installs the `nvidia-smi` command + +## Fan Speed Control + +frostd computes a suggested fan speed in range [0,100] based on configurable +ideal and maximum device temperatures and an easing formula. + +### Suggestion Algorithm + +For each sensor type, read the temperature and compute aggregate temperature +metric from average of last _n_ samples. + +The default easing type is "parabolic". Compute the suggested fan speed for +each device with the following pseudocode formula: + +``` +suggested_speed = min( + max_speed, + (actual_temp - ideal_temp)^2 * (max_temp/((max_temp - ideal_temp)^2)) +) +``` + +Design assumptions: +- Temperature unit is Celcius. +- `max_speed` = 100. +- `max_temp` is the upper bound temperature at which fans will be run at +highest speed. +- `ideal_temp` is the lower bound temperature at which fans will be run at +lowest speed. +- `max_temp` and `ideal_temp` are configured per device. +- `actual_temp` is the sensor aggregate temperature metric. +- Set fan speed to the max `suggested_speed` computed from all devices. This +ensures adequate airflow for the device that needs it most. + +## Configuration + +frostd reads a YAML config file in the default path `/etc/frostd.yaml` or +overridden by specifying CLI argument `-c `. + +The YAML config specifies which sensor types to monitor: CPU and/or NVIDIA +Tesla P40 GPU. Each sensor type includes settings for: +- ideal temperature in Celcius (Default 40) +- maximum temperature in Celcius (Default 75) +- sample size (Default 3) +- sample time interval (Default 15s) + +All instances of a sensor type will be discovered (e.g. multiple CPU packages, +multiple GPUs). + +Log file output file is specified (Default `/var/log/frostd/frostd.log`). + +Dry run mode is enabled with `dry_run: true` (Default `false`). When enabled, +frostd monitors temperature metrics and logs suggested fan speeds but does not +issue any fan speed changes via IPMI. Useful for observing behavior without +affecting hardware. + +## Technology Stack + +`frostd` is written in Golang and compiled to standalone executable file to be +run on Debian or Ubuntu Linux on x86-64 architecture. The executable runs as a +systemd service. + +## Developer Prerequisites + +- Debian or Ubuntu Linux OS on x86-64 architecture +- Golang 1.26+ installed +- `make` installed + +## Software Design + +### Technology Stack + +- `frostd` is a systemd service written in latest Golang. +- It must be run on a Dell PowerEdge 12th generation server. + +### Tooling + +- Use `slog` package for logging. +- Use module `github.com/bougou/go-ipmi` for IPMI support. Do not depend on +the commonly used `ipmitool` command. + +### Service Lifecycle + +Service starts up by parsing and validating the configuration file. + +Then, the service runs in a loop. On each iteration: +- Polls temperature metrics +- Compute suggested fan speed +- Set chassis fan speed +- Sleep until next interval + - Sleep to incremental times, not static durations. This ensures loops occur + on steady cadence even if one iteration takes a bit longer to process. + +The service immediately and gracefully stops when requested with command: +`systemctl stop frostd`. + +### Tests + +Tests are required to confirm requirements and prevent regressions during code +maintenance. + +Use module `github.com/stretchr/testify` for `assert` and `require` packages +for simplifying assertions and validation logic. + +End-to-end tests are required for functionality: +- Configuration parsing + - YAML file structure + - Validation for each field with happy path, numeric range errors, and likely special case checks +- Computing aggregate temperature metrics +- Computing suggested fan speeds from mock temperature samples +- Reading actual CPU temperature metrics +- Reading actual NVIDIA GPU temperature metrics using `nvidia-smi` + +### Error Handling + +#### Startup + +On startup, validate configuration. If configuration file cannot be parsed or +contains validation errors, log a descriptive error message and exit. + +#### Runtime + +If there are errors reading temperature metrics, log a descriptive error and +retry on next iteration. + +Log all sensor temperature metrics, computed aggregate metrics, and suggested +fan speed. + +If there are errors setting fan speed, log a descriptive error. Retry on the +next iteration when fan speed is suggested again. + +## Launch From Source + +The repository will provide a `make run` target that will: +- Call `go run` to launch the service using the configuration file `dev.yaml`. +- `dev.yaml` is a copy of the default configuration file used for installing +`/etc/frostd.yaml`. +- Log output goes to stdout instead of a log file. + +## Software Package Installation + +The repository will provide a `make package` target that will: +- Build the executable +- Build a Debian .deb package that can be installed with `apt install` on +Debian and Ubuntu Linux OS + +The package includes the compiled executable, default configuration for +`/etc/frostd.yaml`, and systemd service configuration. + +The package installation script steps: +- Copy executable to `/usr/local/bin` +- Copy default config file to `/etc/frostd.yaml` +- Add systemd configuration for `frostd` service +- Start `frostd` service + +## Code Style + +### Makefile + +Use `.PHONY` directives to indicate a target is a descriptive script name and +not a literal filename. Place the directive on the line before its definition, +not as a single directive for all targets. This helps prevent forgetting to +update the list. + +### Golang + +Write code for latest Golang version. Follow Golang's current standards of code +style and best practices as outlined in Effective Go by Google. + +## License + +License: Apache 2.0 diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..e1f07b7 --- /dev/null +++ b/Makefile @@ -0,0 +1,50 @@ +BINARY = frostd +GOLANGCI_LINT = ./bin/golangci-lint +GOLANGCI_LINT_VERSION = v2.11.4 + +.PHONY: build +build: + go build -o $(BINARY) . + +.PHONY: run +run: + go run . -c dev.yaml + +.PHONY: test +test: + go test -race ./... + +$(GOLANGCI_LINT): + curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b ./bin $(GOLANGCI_LINT_VERSION) + +.PHONY: lint +lint: $(GOLANGCI_LINT) + $(GOLANGCI_LINT) run + +.PHONY: install +install: build + test -f /etc/frostd.yaml || sudo cp frostd.yaml /etc/frostd.yaml + sudo cp packaging/frostd.logrotate /etc/logrotate.d/frostd + sudo sed "s|ExecStart=.*|ExecStart=$(CURDIR)/$(BINARY)|" \ + packaging/frostd.service \ + | sudo tee /etc/systemd/system/frostd.service > /dev/null + sudo systemctl daemon-reload + sudo systemctl enable frostd + sudo systemctl start frostd + +.PHONY: uninstall +uninstall: + sudo systemctl stop frostd || true + sudo systemctl disable frostd || true + sudo rm -f /etc/systemd/system/frostd.service + sudo systemctl daemon-reload + +.PHONY: uninstall_clean +uninstall_clean: uninstall + LOG_FILE=$$(awk '/^log_file:/{print $$2}' /etc/frostd.yaml 2>/dev/null); \ + sudo rm -f /etc/frostd.yaml; \ + if [ -n "$$LOG_FILE" ]; then sudo rm -f "$$LOG_FILE"; fi + +.PHONY: package +package: build + packaging/build-deb.sh diff --git a/README.md b/README.md index fd0467a..aabba6d 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,190 @@ # frostd -Custom Dell PowerEdge chassis fan speed regulator based on hardware temperature metrics + +`frostd` is a daemon that dynamically regulates chassis fan speeds on Dell +PowerEdge 12th generation servers (e.g. R720) based on real-time temperature +metrics. It reads CPU temperatures via IPMI and optionally GPU temperatures via +`nvidia-smi`, then applies a parabolic easing curve to derive a fan speed +percentage that keeps hardware cool without running fans harder than necessary. + +Configure a temperature range between ideal (lowest fan speed) to maximum +(highest fan speed). `frostd` adjusts fan speed to safely land somewhere in this +range. + +## Why not use the server's built-in automatic fan mode? + +The Baseboard Management Controller (BMC) can dynamically adjust fan speed to +keep CPU temps cool, but it doesn't know how to regulate GPU temps. + +Additionally, the BMC settings are non-adjustable. With `frostd` it is possible +to set much lower fan speeds than usual and still keep all components cool. +This allows for a measurable power reduction at idle. + +## Features + +- Monitors CPU package temperatures via IPMI natively (no dependency on +`ipmitool`) +- Optional NVIDIA GPU temperature monitoring via `nvidia-smi` +- Parabolic fan speed curve between configurable ideal and maximum temperatures +- Independent polling intervals per sensor type +- Rolling sample window to smooth transient temperature spikes +- Dry-run mode for observing behaviour without changing fan speeds +- Structured logging (text or JSON) to file or stdout +- Optional Prometheus metrics endpoint +- Runs as a systemd service + +## Requirements + +- Dell PowerEdge 12th generation server (e.g. R720) +- Debian or Ubuntu Linux, x86-64 +- Go 1.26+ (build only) +- `make` (build only) +- Compatible NVIDIA driver and CUDA toolkit (GPU monitoring only) + +## Dev Tooling +### Build + +```sh +make build +``` + +The compiled executable is written to `./frostd`. + +### Run from source + +```sh +make run +``` + +Launches the service using `dev.yaml` as the configuration file, with log +output to stdout. + +### Local install + +```sh +make install +``` + +This will: + +1. Build the binary +2. Copy `frostd.yaml` to `/etc/frostd.yaml` (skipped if the file already exists) +3. Install the logrotate configuration to `/etc/logrotate.d/frostd` +4. Install and enable the `frostd` systemd service + +### Local uninstall + +```sh +make uninstall +``` + +This reverts an install from `make install`. It stops and removes the systemd +service. The configuration file and logs are preserved. + +```sh +make uninstall_clean +``` + +Same as `uninstall`, but also removes `/etc/frostd.yaml` and the log file +specified within it. + +## Package install +Build and install a Debian .deb package. + +```sh +make package +``` + +Produces a `frostd__amd64.deb` file that can be installed with: + +```sh +sudo apt install ./frostd__amd64.deb +``` + +## Configuration + +The default configuration file path is `/etc/frostd.yaml`. Override with: + +```sh +frostd -c /path/to/config.yaml +``` + +### Top-level options + +| Field | Type | Default | Description | +|--------------------|----------|------------|-------------| +| `log_file` | string | *(stdout)* | Path to log file. Leave empty to log to stdout. | +| `log_format` | string | `text` | Log format: `text` or `json`. | +| `dry_run` | bool | `false` | Log suggested fan speeds without applying them. | +| `fan_log_interval` | duration | `15s` | How often to log current fan speeds. | + +### Prometheus options (`prometheus`) + +Optional. Omit the section (or leave `listen_addr` empty) to disable. + +| Field | Type | Default | Description | +|---------------|--------|---------|-------------| +| `listen_addr` | string | *(none)*| `host:port` to bind the metrics HTTP server. When set, `GET /metrics` exposes Prometheus metrics. | + +#### Exported metrics + +| Metric | Labels | Description | +|----------------------------------------------|-----------------|-------------| +| `frostd_sensor_temperature` | `sensor`, `id` | Current temperature reading per sensor ID, in Celsius. | +| `frostd_sensor_aggregate_temperature` | `sensor` | Rolling aggregate temperature used for fan speed calculation, in Celsius. | +| `frostd_suggested_fan_speed_percent` | `sensor` | Fan speed percentage suggested by this sensor type. | +| `frostd_actual_fan_rpm` | `fan` | Actual fan speed in RPM (for fans that report RPM). | +| `frostd_actual_fan_speed_percent` | `fan` | Actual fan speed as a percentage (for fans that report percent). | +| `frostd_system_fan_speed_percent` | *(none)* | Commanded system fan speed — the highest suggestion across all sensor types. | + +### Sensor options (`cpu` / `gpu`) + +Both `cpu` and `gpu` sections are optional, but at least one must be enabled. + +| Field | Type | Default | Description | +|-------------------|----------|---------|-------------| +| `ideal_temp` | float | `40` | Temperature (°C) at which fans run at minimum speed. | +| `max_temp` | float | `75` | Temperature (°C) at which fans run at full speed. | +| `sample_size` | int | `3` | Number of polling samples to retain in the rolling average. | +| `sample_interval` | duration | `15s` | How often to poll temperature for this sensor type. | + +### Fan speed algorithm + +For each sensor type, `frostd` polls temperature, takes the maximum reading +across all sensor IDs, and maintains a rolling average of the last +`sample_size` maximums. The suggested fan speed is: + +``` +suggested = min(100, (actual - ideal)² × 100 / (max - ideal)²) +``` + +The system fan speed is set to the highest suggestion across all active sensor +types. Fan speed is only updated when at least one sensor has reported since +the last update, and only when the suggested speed has changed. + +### Example configuration + +```yaml +log_file: /var/log/frostd/frostd.log +log_format: json +dry_run: false +fan_log_interval: 15s + +prometheus: + listen_addr: ":9100" + +cpu: + ideal_temp: 40 + max_temp: 75 + sample_size: 3 + sample_interval: 15s + +gpu: + ideal_temp: 45 + max_temp: 80 + sample_size: 3 + sample_interval: 10s +``` + +## License + +Apache 2.0 — see [LICENSE](LICENSE). diff --git a/config.go b/config.go new file mode 100644 index 0000000..84755e5 --- /dev/null +++ b/config.go @@ -0,0 +1,118 @@ +package main + +import ( + "fmt" + "os" + "time" + + "gopkg.in/yaml.v3" +) + +type Config struct { + LogFile string `yaml:"log_file"` + LogFormat string `yaml:"log_format"` + DryRun bool `yaml:"dry_run"` + FanLogInterval time.Duration `yaml:"fan_log_interval"` + Prometheus *PrometheusConfig `yaml:"prometheus"` + CPU *SensorConfig `yaml:"cpu"` + GPU *SensorConfig `yaml:"gpu"` +} + +type PrometheusConfig struct { + ListenAddr string `yaml:"listen_addr"` +} + +type SensorConfig struct { + IdealTemp float64 `yaml:"ideal_temp"` + MaxTemp float64 `yaml:"max_temp"` + SampleSize int `yaml:"sample_size"` + SampleInterval time.Duration `yaml:"sample_interval"` +} + +func loadConfig(path string) (*Config, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("reading config file: %w", err) + } + var cfg Config + if err := yaml.Unmarshal(data, &cfg); err != nil { + return nil, fmt.Errorf("parsing config file: %w", err) + } + applyDefaults(&cfg) + if err := validateConfig(&cfg); err != nil { + return nil, err + } + return &cfg, nil +} + +func applyDefaults(cfg *Config) { + if cfg.LogFormat == "" { + cfg.LogFormat = "text" + } + if cfg.FanLogInterval == 0 { + cfg.FanLogInterval = 15 * time.Second + } + if cfg.CPU != nil { + applySensorDefaults(cfg.CPU) + } + if cfg.GPU != nil { + applySensorDefaults(cfg.GPU) + } +} + +func applySensorDefaults(d *SensorConfig) { + if d.IdealTemp == 0 { + d.IdealTemp = 40 + } + if d.MaxTemp == 0 { + d.MaxTemp = 75 + } + if d.SampleSize == 0 { + d.SampleSize = 3 + } + if d.SampleInterval == 0 { + d.SampleInterval = 15 * time.Second + } +} + +func validateConfig(cfg *Config) error { + if cfg.CPU == nil && cfg.GPU == nil { + return fmt.Errorf("config must enable at least one sensor type (cpu or gpu)") + } + if cfg.LogFormat != "text" && cfg.LogFormat != "json" { + return fmt.Errorf("log_format must be \"text\" or \"json\", got %q", cfg.LogFormat) + } + if cfg.FanLogInterval <= 0 { + return fmt.Errorf("fan_log_interval must be positive, got %s", cfg.FanLogInterval) + } + if cfg.CPU != nil { + if err := validateSensorConfig("cpu", cfg.CPU); err != nil { + return err + } + } + if cfg.GPU != nil { + if err := validateSensorConfig("gpu", cfg.GPU); err != nil { + return err + } + } + return nil +} + +func validateSensorConfig(name string, d *SensorConfig) error { + if d.IdealTemp <= 0 { + return fmt.Errorf("%s: ideal_temp must be positive, got %.1f", name, d.IdealTemp) + } + if d.MaxTemp <= 0 { + return fmt.Errorf("%s: max_temp must be positive, got %.1f", name, d.MaxTemp) + } + if d.IdealTemp >= d.MaxTemp { + return fmt.Errorf("%s: ideal_temp (%.1f) must be less than max_temp (%.1f)", name, d.IdealTemp, d.MaxTemp) + } + if d.SampleSize < 1 { + return fmt.Errorf("%s: sample_size must be at least 1, got %d", name, d.SampleSize) + } + if d.SampleInterval <= 0 { + return fmt.Errorf("%s: sample_interval must be positive, got %s", name, d.SampleInterval) + } + return nil +} diff --git a/config_test.go b/config_test.go new file mode 100644 index 0000000..3283d24 --- /dev/null +++ b/config_test.go @@ -0,0 +1,186 @@ +package main + +import ( + "os" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func writeTempConfig(t *testing.T, content string) string { + t.Helper() + f, err := os.CreateTemp(t.TempDir(), "frostd-*.yaml") + require.NoError(t, err) + _, err = f.WriteString(content) + require.NoError(t, err) + err = f.Close() + require.NoError(t, err) + return f.Name() +} + +func TestLoadConfig_HappyPath(t *testing.T) { + path := writeTempConfig(t, ` +log_file: /var/log/frostd/frostd.log +cpu: + ideal_temp: 45 + max_temp: 80 + sample_size: 5 + sample_interval: 10s +gpu: + ideal_temp: 50 + max_temp: 85 + sample_size: 4 + sample_interval: 20s +`) + cfg, err := loadConfig(path) + require.NoError(t, err) + assert.Equal(t, "/var/log/frostd/frostd.log", cfg.LogFile) + assert.False(t, cfg.DryRun) + assert.Equal(t, 45.0, cfg.CPU.IdealTemp) + assert.Equal(t, 80.0, cfg.CPU.MaxTemp) + assert.Equal(t, 5, cfg.CPU.SampleSize) + assert.Equal(t, 10*time.Second, cfg.CPU.SampleInterval) + assert.Equal(t, 50.0, cfg.GPU.IdealTemp) +} + +func TestLoadConfig_DryRunEnabled(t *testing.T) { + path := writeTempConfig(t, ` +dry_run: true +cpu: + ideal_temp: 40 + max_temp: 75 +`) + cfg, err := loadConfig(path) + require.NoError(t, err) + assert.True(t, cfg.DryRun) +} + +func TestLoadConfig_DryRunDefault(t *testing.T) { + path := writeTempConfig(t, `cpu: {}`) + cfg, err := loadConfig(path) + require.NoError(t, err) + assert.False(t, cfg.DryRun) +} + +func TestLoadConfig_Defaults(t *testing.T) { + path := writeTempConfig(t, `cpu: {}`) + cfg, err := loadConfig(path) + require.NoError(t, err) + assert.Equal(t, "text", cfg.LogFormat) + assert.Equal(t, 40.0, cfg.CPU.IdealTemp) + assert.Equal(t, 75.0, cfg.CPU.MaxTemp) + assert.Equal(t, 3, cfg.CPU.SampleSize) + assert.Equal(t, 15*time.Second, cfg.CPU.SampleInterval) +} + +func TestLoadConfig_LogFormatJSON(t *testing.T) { + path := writeTempConfig(t, ` +log_format: json +cpu: {} +`) + cfg, err := loadConfig(path) + require.NoError(t, err) + assert.Equal(t, "json", cfg.LogFormat) +} + +func TestLoadConfig_LogFormatInvalid(t *testing.T) { + path := writeTempConfig(t, ` +log_format: xml +cpu: {} +`) + _, err := loadConfig(path) + assert.ErrorContains(t, err, "log_format") +} + +func TestLoadConfig_NoDevices(t *testing.T) { + path := writeTempConfig(t, `log_file: /tmp/test.log`) + _, err := loadConfig(path) + assert.ErrorContains(t, err, "at least one sensor type") +} + +func TestLoadConfig_IdealTempEqualMaxTemp(t *testing.T) { + path := writeTempConfig(t, ` +cpu: + ideal_temp: 75 + max_temp: 75 +`) + _, err := loadConfig(path) + assert.Error(t, err) +} + +func TestLoadConfig_IdealTempGreaterThanMaxTemp(t *testing.T) { + path := writeTempConfig(t, ` +cpu: + ideal_temp: 80 + max_temp: 75 +`) + _, err := loadConfig(path) + assert.Error(t, err) +} + +func TestLoadConfig_SampleSizeNegative(t *testing.T) { + path := writeTempConfig(t, ` +cpu: + ideal_temp: 40 + max_temp: 75 + sample_size: -1 + sample_interval: 15s +`) + _, err := loadConfig(path) + assert.ErrorContains(t, err, "sample_size") +} + +func TestLoadConfig_NegativeSampleInterval(t *testing.T) { + path := writeTempConfig(t, ` +cpu: + ideal_temp: 40 + max_temp: 75 + sample_size: 3 + sample_interval: -5s +`) + _, err := loadConfig(path) + assert.ErrorContains(t, err, "sample_interval") +} + +func TestLoadConfig_InvalidYAML(t *testing.T) { + path := writeTempConfig(t, `:::invalid yaml:::`) + _, err := loadConfig(path) + assert.Error(t, err) +} + +func TestLoadConfig_MissingFile(t *testing.T) { + _, err := loadConfig("/nonexistent/path/frostd.yaml") + assert.Error(t, err) +} + +func TestLoadConfig_PrometheusWithListenAddr(t *testing.T) { + path := writeTempConfig(t, ` +cpu: {} +prometheus: + listen_addr: ":9100" +`) + cfg, err := loadConfig(path) + require.NoError(t, err) + require.NotNil(t, cfg.Prometheus) + assert.Equal(t, ":9100", cfg.Prometheus.ListenAddr) +} + +func TestLoadConfig_PrometheusAbsent(t *testing.T) { + path := writeTempConfig(t, `cpu: {}`) + cfg, err := loadConfig(path) + require.NoError(t, err) + assert.Nil(t, cfg.Prometheus) +} + +func TestLoadConfig_PrometheusEmptyListenAddr(t *testing.T) { + path := writeTempConfig(t, ` +cpu: {} +prometheus: {} +`) + cfg, err := loadConfig(path) + require.NoError(t, err) + require.NotNil(t, cfg.Prometheus) + assert.Equal(t, "", cfg.Prometheus.ListenAddr) +} diff --git a/cpu.go b/cpu.go new file mode 100644 index 0000000..939346f --- /dev/null +++ b/cpu.go @@ -0,0 +1,53 @@ +package main + +import ( + "context" + "fmt" + + ipmi "github.com/bougou/go-ipmi" +) + +type CPUReader struct { + newClient func() (ipmiClient, error) +} + +// ReadTemperatures returns CPU package temperatures via IPMI, keyed by sensor name. +// It reads temperature sensors with entity ID 0x03 (processor). +func (r *CPUReader) ReadTemperatures() (map[string]float64, error) { + ctx := context.Background() + client, err := r.newClient() + if err != nil { + return nil, fmt.Errorf("creating IPMI client: %w", err) + } + if err := client.Connect(ctx); err != nil { + return nil, fmt.Errorf("connecting to IPMI: %w", err) + } + defer func() { + _ = client.Close(ctx) + }() + + sensors, err := client.GetSensors(ctx, + ipmi.SensorFilterOptionIsSensorType(ipmi.SensorTypeTemperature)) + if err != nil { + return nil, fmt.Errorf("getting IPMI temperature sensors: %w", err) + } + + temps := map[string]float64{} + for _, s := range sensors { + if uint8(s.EntityID) != 0x03 { + continue + } + key := s.Name + for i := 2; ; i++ { + if _, exists := temps[key]; !exists { + break + } + key = fmt.Sprintf("%s_%d", s.Name, i) + } + temps[key] = s.Value + } + if len(temps) == 0 { + return nil, fmt.Errorf("no CPU temperature sensors found") + } + return temps, nil +} diff --git a/cpu_test.go b/cpu_test.go new file mode 100644 index 0000000..3580bc7 --- /dev/null +++ b/cpu_test.go @@ -0,0 +1,91 @@ +package main + +import ( + "testing" + + ipmi "github.com/bougou/go-ipmi" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func makeProcessorTempSensor(name string, value float64, entityInstance uint8) *ipmi.Sensor { + return &ipmi.Sensor{ + Name: name, + SensorType: ipmi.SensorTypeTemperature, + EntityID: ipmi.EntityID(0x03), + EntityInstance: ipmi.EntityInstance(entityInstance), + Value: value, + } +} + +func makeInletTempSensor(value float64) *ipmi.Sensor { + return &ipmi.Sensor{ + SensorType: ipmi.SensorTypeTemperature, + EntityID: ipmi.EntityID(0x07), // system board + Value: value, + } +} + +func TestCPUReader_ReturnsProcessorTemps(t *testing.T) { + mock := &mockIPMIClient{ + sensors: []*ipmi.Sensor{ + makeProcessorTempSensor("CPU1 Temp", 45, 1), + makeProcessorTempSensor("CPU2 Temp", 50, 2), + makeInletTempSensor(23), + }, + } + reader := &CPUReader{newClient: mockIPMIFactory(mock)} + temps, err := reader.ReadTemperatures() + require.NoError(t, err) + assert.Equal(t, map[string]float64{"CPU1 Temp": 45, "CPU2 Temp": 50}, temps) +} + +func TestCPUReader_DeduplicatesSensorNames(t *testing.T) { + mock := &mockIPMIClient{ + sensors: []*ipmi.Sensor{ + makeProcessorTempSensor("Temp", 45, 1), + makeProcessorTempSensor("Temp", 50, 2), + makeProcessorTempSensor("Temp", 55, 3), + }, + } + reader := &CPUReader{newClient: mockIPMIFactory(mock)} + temps, err := reader.ReadTemperatures() + require.NoError(t, err) + assert.Equal(t, map[string]float64{"Temp": 45, "Temp_2": 50, "Temp_3": 55}, temps) +} + +func TestCPUReader_IgnoresNonProcessorSensors(t *testing.T) { + mock := &mockIPMIClient{ + sensors: []*ipmi.Sensor{ + makeInletTempSensor(23), + }, + } + reader := &CPUReader{newClient: mockIPMIFactory(mock)} + _, err := reader.ReadTemperatures() + assert.ErrorContains(t, err, "no CPU temperature sensors found") +} + +func TestCPUReader_EmptySensorList(t *testing.T) { + mock := &mockIPMIClient{sensors: []*ipmi.Sensor{}} + reader := &CPUReader{newClient: mockIPMIFactory(mock)} + _, err := reader.ReadTemperatures() + assert.Error(t, err) +} + +func TestCPUReader_ClientConnectError(t *testing.T) { + reader := &CPUReader{newClient: mockIPMIFactoryErr("no IPMI device")} + _, err := reader.ReadTemperatures() + assert.Error(t, err) +} + +func TestCPUReader_GetSensorsError(t *testing.T) { + mock := &mockIPMIClient{rawErr: errIPMI("ipmitool failed")} + reader := &CPUReader{newClient: mockIPMIFactory(mock)} + _, err := reader.ReadTemperatures() + assert.Error(t, err) +} + +// errIPMI is a simple error helper for tests. +type errIPMI string + +func (e errIPMI) Error() string { return string(e) } diff --git a/dev.yaml b/dev.yaml new file mode 100644 index 0000000..fe59b7b --- /dev/null +++ b/dev.yaml @@ -0,0 +1,18 @@ +log_file: "" +# dry_run: true + +# prometheus: +# listen_addr: ":9100" + +cpu: + ideal_temp: 40 + max_temp: 80 + sample_size: 3 + sample_interval: 15s + +gpu: + # Constant LLM loads maintain ~75-80C and then quickly falls to 60C + ideal_temp: 40 + max_temp: 80 + sample_size: 3 + sample_interval: 10s diff --git a/executor.go b/executor.go new file mode 100644 index 0000000..db773c4 --- /dev/null +++ b/executor.go @@ -0,0 +1,13 @@ +package main + +import "os/exec" + +type CommandRunner interface { + Run(name string, args ...string) ([]byte, error) +} + +type RealRunner struct{} + +func (r RealRunner) Run(name string, args ...string) ([]byte, error) { + return exec.Command(name, args...).Output() +} diff --git a/fan.go b/fan.go new file mode 100644 index 0000000..965215e --- /dev/null +++ b/fan.go @@ -0,0 +1,106 @@ +package main + +import ( + "context" + "fmt" + "math" + + ipmi "github.com/bougou/go-ipmi" +) + +// SuggestSpeed computes fan speed [0,100] using a parabolic easing curve. +func SuggestSpeed(actualTemp, idealTemp, maxTemp float64) int { + if actualTemp <= idealTemp { + return 0 + } + deltaTemp1 := actualTemp - idealTemp + deltaTemp2 := maxTemp - idealTemp + speed := (deltaTemp1 * deltaTemp1) * (100 / (deltaTemp2 * deltaTemp2)) + if speed > 100 { + return 100 + } + return int(math.Round(speed)) +} + +type FanReading struct { + Name string + RPM *float64 + Percent *float64 +} + +type FanController interface { + ReadFanSpeeds(ctx context.Context) ([]FanReading, error) + SetSpeed(ctx context.Context, percent int) error +} + +type IPMIFanController struct { + newClient func() (ipmiClient, error) +} + +// ReadFanSpeeds returns current readings from all IPMI fan sensors. +// Each reading includes RPM if the sensor reports in RPM, or percent if the sensor reports as a percentage. +func (c *IPMIFanController) ReadFanSpeeds(ctx context.Context) ([]FanReading, error) { + client, err := c.newClient() + if err != nil { + return nil, fmt.Errorf("creating IPMI client: %w", err) + } + if err := client.Connect(ctx); err != nil { + return nil, fmt.Errorf("connecting to IPMI: %w", err) + } + defer func() { + _ = client.Close(ctx) + }() + + sensors, err := client.GetSensors(ctx, + ipmi.SensorFilterOptionIsSensorType(ipmi.SensorTypeFan)) + if err != nil { + return nil, fmt.Errorf("getting IPMI fan sensors: %w", err) + } + + var readings []FanReading + for _, s := range sensors { + if !s.HasAnalogReading { + continue + } + r := FanReading{Name: s.Name} + if s.SensorUnit.Percentage { + v := s.Value + r.Percent = &v + } else if s.SensorUnit.BaseUnit == ipmi.SensorUnitType_RPM { + v := s.Value + r.RPM = &v + } + readings = append(readings, r) + } + return readings, nil +} + +// SetSpeed sets the chassis fan speed to the given percentage [0,100]. +// It uses Dell OEM IPMI raw commands to enable manual fan control and set the speed. +func (c *IPMIFanController) SetSpeed(ctx context.Context, percent int) error { + if percent < 0 || percent > 100 { + return fmt.Errorf("fan speed %d out of range [0,100]", percent) + } + + client, err := c.newClient() + if err != nil { + return fmt.Errorf("creating IPMI client: %w", err) + } + if err := client.Connect(ctx); err != nil { + return fmt.Errorf("connecting to IPMI: %w", err) + } + defer func() { + _ = client.Close(ctx) + }() + + // Dell OEM: enable manual fan control + if _, err := client.RawCommand(ctx, ipmi.NetFn(0x30), 0x30, []byte{0x01, 0x00}, ""); err != nil { + return fmt.Errorf("enabling manual fan control: %w", err) + } + + // Dell OEM: set fan speed + if _, err := client.RawCommand(ctx, ipmi.NetFn(0x30), 0x30, []byte{0x02, 0xff, byte(percent)}, ""); err != nil { + return fmt.Errorf("setting fan speed to %d%%: %w", percent, err) + } + return nil +} diff --git a/fan_test.go b/fan_test.go new file mode 100644 index 0000000..e6b255d --- /dev/null +++ b/fan_test.go @@ -0,0 +1,127 @@ +package main + +import ( + "testing" + + ipmi "github.com/bougou/go-ipmi" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestSuggestSpeed_AtIdealTemp(t *testing.T) { + assert.Equal(t, 0, SuggestSpeed(40, 40, 75)) +} + +func TestSuggestSpeed_BelowIdealTemp(t *testing.T) { + assert.Equal(t, 0, SuggestSpeed(35, 40, 75)) +} + +func TestSuggestSpeed_AtMaxTemp(t *testing.T) { + assert.Equal(t, 100, SuggestSpeed(75, 40, 75)) +} + +func TestSuggestSpeed_AboveMaxTemp(t *testing.T) { + assert.Equal(t, 100, SuggestSpeed(90, 40, 75)) +} + +func TestSuggestSpeed_Midpoint(t *testing.T) { + // ideal=40, max=75, actual=57.5 → (17.5)^2 * (100/1225) = 25 + assert.Equal(t, 25, SuggestSpeed(57.5, 40, 75)) +} + +func TestSuggestSpeed_ThreeQuarterTemp(t *testing.T) { + // actual = 40 + 0.75*35 = 66.25 → (26.25)^2 * (100/1225) ≈ 56.25 → rounds to 56 + assert.Equal(t, 56, SuggestSpeed(66.25, 40, 75)) +} + +func TestIPMIFanController_SetSpeed(t *testing.T) { + mock := &mockIPMIClient{} + ctrl := &IPMIFanController{newClient: mockIPMIFactory(mock)} + require.NoError(t, ctrl.SetSpeed(t.Context(), 50)) +} + +func TestIPMIFanController_SetSpeedOutOfRange(t *testing.T) { + mock := &mockIPMIClient{} + ctrl := &IPMIFanController{newClient: mockIPMIFactory(mock)} + assert.Error(t, ctrl.SetSpeed(t.Context(), 101)) + assert.Error(t, ctrl.SetSpeed(t.Context(), -1)) +} + +func TestIPMIFanController_SetSpeedIPMIError(t *testing.T) { + mock := &mockIPMIClient{rawErr: errIPMI("raw command failed")} + ctrl := &IPMIFanController{newClient: mockIPMIFactory(mock)} + assert.Error(t, ctrl.SetSpeed(t.Context(), 50)) +} + +func TestIPMIFanController_SetSpeedConnectError(t *testing.T) { + ctrl := &IPMIFanController{newClient: mockIPMIFactoryErr("no IPMI device")} + assert.Error(t, ctrl.SetSpeed(t.Context(), 50)) +} + +func ptr[T any](v T) *T { return &v } + +func TestIPMIFanController_ReadFanSpeeds_RPM(t *testing.T) { + mock := &mockIPMIClient{ + sensors: []*ipmi.Sensor{ + { + Name: "Fan1", + SensorType: ipmi.SensorTypeFan, + HasAnalogReading: true, + Value: 3600, + SensorUnit: ipmi.SensorUnit{BaseUnit: ipmi.SensorUnitType_RPM}, + }, + }, + } + ctrl := &IPMIFanController{newClient: mockIPMIFactory(mock)} + readings, err := ctrl.ReadFanSpeeds(t.Context()) + require.NoError(t, err) + require.Len(t, readings, 1) + assert.Equal(t, "Fan1", readings[0].Name) + assert.Equal(t, ptr(3600.0), readings[0].RPM) + assert.Nil(t, readings[0].Percent) +} + +func TestIPMIFanController_ReadFanSpeeds_Percent(t *testing.T) { + mock := &mockIPMIClient{ + sensors: []*ipmi.Sensor{ + { + Name: "Fan1", + SensorType: ipmi.SensorTypeFan, + HasAnalogReading: true, + Value: 50, + SensorUnit: ipmi.SensorUnit{Percentage: true}, + }, + }, + } + ctrl := &IPMIFanController{newClient: mockIPMIFactory(mock)} + readings, err := ctrl.ReadFanSpeeds(t.Context()) + require.NoError(t, err) + require.Len(t, readings, 1) + assert.Equal(t, ptr(50.0), readings[0].Percent) + assert.Nil(t, readings[0].RPM) +} + +func TestIPMIFanController_ReadFanSpeeds_SkipsDiscrete(t *testing.T) { + mock := &mockIPMIClient{ + sensors: []*ipmi.Sensor{ + {Name: "Fan1", SensorType: ipmi.SensorTypeFan, HasAnalogReading: false}, + }, + } + ctrl := &IPMIFanController{newClient: mockIPMIFactory(mock)} + readings, err := ctrl.ReadFanSpeeds(t.Context()) + require.NoError(t, err) + assert.Empty(t, readings) +} + +func TestIPMIFanController_ReadFanSpeeds_IPMIError(t *testing.T) { + mock := &mockIPMIClient{rawErr: errIPMI("sensor read failed")} + ctrl := &IPMIFanController{newClient: mockIPMIFactory(mock)} + _, err := ctrl.ReadFanSpeeds(t.Context()) + assert.Error(t, err) +} + +func TestIPMIFanController_ReadFanSpeeds_ConnectError(t *testing.T) { + ctrl := &IPMIFanController{newClient: mockIPMIFactoryErr("no IPMI device")} + _, err := ctrl.ReadFanSpeeds(t.Context()) + assert.Error(t, err) +} diff --git a/frostd.yaml b/frostd.yaml new file mode 100644 index 0000000..8dcb2a3 --- /dev/null +++ b/frostd.yaml @@ -0,0 +1,18 @@ +log_file: /var/log/frostd/frostd.log +log_format: json +dry_run: false + +# prometheus: +# listen_addr: ":9100" + +cpu: + ideal_temp: 40 + max_temp: 75 + sample_size: 3 + sample_interval: 15s + +gpu: + ideal_temp: 40 + max_temp: 75 + sample_size: 3 + sample_interval: 15s diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..5b471b5 --- /dev/null +++ b/go.mod @@ -0,0 +1,38 @@ +module frostd + +go 1.26 + +require ( + github.com/bougou/go-ipmi v0.8.3 + github.com/prometheus/client_golang v1.23.2 + github.com/stretchr/testify v1.11.1 + gopkg.in/yaml.v3 v3.0.1 +) + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/fatih/color v1.15.0 // indirect + github.com/google/uuid v1.1.2 // indirect + github.com/kr/pretty v0.3.1 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect + github.com/mattn/go-runewidth v0.0.16 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/olekukonko/errors v1.1.0 // indirect + github.com/olekukonko/ll v0.0.9 // indirect + github.com/olekukonko/tablewriter v1.0.9 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.66.1 // indirect + github.com/prometheus/procfs v0.16.1 // indirect + github.com/rivo/uniseg v0.2.0 // indirect + github.com/rogpeppe/go-internal v1.10.0 // indirect + go.yaml.in/yaml/v2 v2.4.2 // indirect + golang.org/x/net v0.43.0 // indirect + golang.org/x/sys v0.35.0 // indirect + golang.org/x/term v0.34.0 // indirect + google.golang.org/protobuf v1.36.8 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..674367f --- /dev/null +++ b/go.sum @@ -0,0 +1,75 @@ +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bougou/go-ipmi v0.8.3 h1:jegrU+5lx5Moe/MiggLFIHCwDoVPd/BZuIOKYZS3BUY= +github.com/bougou/go-ipmi v0.8.3/go.mod h1:HWli0nfKgnBtD/3ViiDaqp6wHZogZrg5A5ctMMDUYS0= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= +github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= +github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/olekukonko/errors v1.1.0 h1:RNuGIh15QdDenh+hNvKrJkmxxjV4hcS50Db478Ou5sM= +github.com/olekukonko/errors v1.1.0/go.mod h1:ppzxA5jBKcO1vIpCXQ9ZqgDh8iwODz6OXIGKU8r5m4Y= +github.com/olekukonko/ll v0.0.9 h1:Y+1YqDfVkqMWuEQMclsF9HUR5+a82+dxJuL1HHSRpxI= +github.com/olekukonko/ll v0.0.9/go.mod h1:En+sEW0JNETl26+K8eZ6/W4UQ7CYSrrgg/EdIYT2H8g= +github.com/olekukonko/tablewriter v1.0.9 h1:XGwRsYLC2bY7bNd93Dk51bcPZksWZmLYuaTHR0FqfL8= +github.com/olekukonko/tablewriter v1.0.9/go.mod h1:5c+EBPeSqvXnLLgkm9isDdzR3wjfBkHR9Nhfp3NWrzo= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs= +github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= +github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= +github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= +github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= +go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= +golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= +golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= +golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/term v0.34.0 h1:O/2T7POpk0ZZ7MAzMeWFSg6S5IpWd/RXDlM9hgM3DR4= +golang.org/x/term v0.34.0/go.mod h1:5jC53AEywhIVebHgPVeg0mj8OD3VO9OzclacVrqpaAw= +google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= +google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/gpu.go b/gpu.go new file mode 100644 index 0000000..76c34dd --- /dev/null +++ b/gpu.go @@ -0,0 +1,47 @@ +package main + +import ( + "fmt" + "strconv" + "strings" +) + +type GPUReader struct { + runner CommandRunner +} + +// ReadTemperatures returns GPU temperatures via nvidia-smi, keyed by "gpu0", "gpu1", etc. +func (r *GPUReader) ReadTemperatures() (map[string]float64, error) { + out, err := r.runner.Run("nvidia-smi", "--query-gpu=temperature.gpu", "--format=csv,noheader") + if err != nil { + return nil, fmt.Errorf("nvidia-smi: %w", err) + } + return parseGPUTemps(string(out)) +} + +// parseGPUTemps parses nvidia-smi CSV output (one temperature per line). +func parseGPUTemps(output string) (map[string]float64, error) { + temps := map[string]float64{} + for _, line := range strings.Split(strings.TrimSpace(output), "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + temp, err := strconv.ParseFloat(line, 64) + if err != nil { + return nil, fmt.Errorf("parsing GPU temperature %q: %w", line, err) + } + key := "Temp" + for i := 2; ; i++ { + if _, exists := temps[key]; !exists { + break + } + key = fmt.Sprintf("Temp_%d", i) + } + temps[key] = temp + } + if len(temps) == 0 { + return nil, fmt.Errorf("no GPU temperatures found in nvidia-smi output") + } + return temps, nil +} diff --git a/gpu_test.go b/gpu_test.go new file mode 100644 index 0000000..d0113e1 --- /dev/null +++ b/gpu_test.go @@ -0,0 +1,46 @@ +package main + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGPUReader_ParsesSingleGPU(t *testing.T) { + temps, err := parseGPUTemps("72\n") + require.NoError(t, err) + assert.Equal(t, map[string]float64{"Temp": 72}, temps) +} + +func TestGPUReader_ParsesMultipleGPUs(t *testing.T) { + temps, err := parseGPUTemps("68\n74\n") + require.NoError(t, err) + assert.Equal(t, map[string]float64{"Temp": 68, "Temp_2": 74}, temps) +} + +func TestGPUReader_EmptyOutput(t *testing.T) { + _, err := parseGPUTemps("") + assert.Error(t, err) +} + +func TestGPUReader_InvalidLine(t *testing.T) { + _, err := parseGPUTemps("not a number\n") + assert.Error(t, err) +} + +func TestGPUReader_ReadTemperatures_CallsNvidiaSmi(t *testing.T) { + r := &mockRunner{output: []byte("72\n")} + reader := &GPUReader{runner: r} + temps, err := reader.ReadTemperatures() + require.NoError(t, err) + assert.Equal(t, map[string]float64{"Temp": 72}, temps) +} + +func TestGPUReader_ReadTemperatures_CommandError(t *testing.T) { + r := &mockRunner{err: errors.New("nvidia-smi not found")} + reader := &GPUReader{runner: r} + _, err := reader.ReadTemperatures() + assert.Error(t, err) +} diff --git a/ipmi_client.go b/ipmi_client.go new file mode 100644 index 0000000..422809c --- /dev/null +++ b/ipmi_client.go @@ -0,0 +1,18 @@ +package main + +import ( + "context" + + ipmi "github.com/bougou/go-ipmi" +) + +type ipmiClient interface { + Connect(ctx context.Context) error + Close(ctx context.Context) error + GetSensors(ctx context.Context, opts ...ipmi.SensorFilterOption) ([]*ipmi.Sensor, error) + RawCommand(ctx context.Context, netFn ipmi.NetFn, cmd uint8, data []byte, name string) (*ipmi.CommandRawResponse, error) +} + +func newRealIPMIClient() (ipmiClient, error) { + return ipmi.NewOpenClient() +} diff --git a/main.go b/main.go new file mode 100644 index 0000000..c4114a4 --- /dev/null +++ b/main.go @@ -0,0 +1,102 @@ +package main + +import ( + "context" + "flag" + "fmt" + "io" + "log/slog" + "os" + "os/signal" + "path/filepath" + "syscall" +) + +const version = "0.1.0" + +type args struct { + configPath string + showVersion bool +} + +func parseArgs() args { + configPath := flag.String("c", "/etc/frostd.yaml", "path to config file") + showVersion := flag.Bool("version", false, "print version and exit") + flag.Parse() + return args{configPath: *configPath, showVersion: *showVersion} +} + +func main() { + a := parseArgs() + + if a.showVersion { + fmt.Println(version) + return + } + + cfg, err := loadConfig(a.configPath) + if err != nil { + fmt.Fprintf(os.Stderr, "frostd: configuration error: %v\n", err) + os.Exit(1) + } + + logger, closer, err := setupLogger(cfg.LogFile, cfg.LogFormat) + if err != nil { + fmt.Fprintf(os.Stderr, "frostd: failed to open log file: %v\n", err) + os.Exit(1) + } + defer closer() + + fanCtrl := &IPMIFanController{newClient: newRealIPMIClient} + + var monitors []*SensorMonitor + if cfg.CPU != nil { + monitors = append(monitors, newSensorMonitor("cpu", cfg.CPU, &CPUReader{newClient: newRealIPMIClient})) + } + if cfg.GPU != nil { + monitors = append(monitors, newSensorMonitor("gpu", cfg.GPU, &GPUReader{runner: RealRunner{}})) + } + + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT) + defer stop() + + svc := &Service{ + cfg: cfg, + fanCtrl: fanCtrl, + monitors: monitors, + logger: logger, + prom: newPrometheusMetrics(), + } + if cfg.Prometheus != nil && cfg.Prometheus.ListenAddr != "" { + startPrometheusServer(ctx, cfg.Prometheus.ListenAddr, svc.prom, logger) + } + + logger.Info("frostd started", "config", a.configPath, "dry_run", cfg.DryRun) + svc.run(ctx) + logger.Info("frostd stopping") +} + +func setupLogger(logFile, logFormat string) (*slog.Logger, func(), error) { + var w io.Writer = os.Stdout + closer := func() {} + + if logFile != "" { + if err := os.MkdirAll(filepath.Dir(logFile), 0o755); err != nil { + return nil, nil, err + } + f, err := os.OpenFile(logFile, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o644) + if err != nil { + return nil, nil, err + } + w = f + closer = func() { _ = f.Close() } + } + + var handler slog.Handler + if logFormat == "json" { + handler = slog.NewJSONHandler(w, nil) + } else { + handler = slog.NewTextHandler(w, nil) + } + return slog.New(handler), closer, nil +} diff --git a/metrics.go b/metrics.go new file mode 100644 index 0000000..7c1e8ed --- /dev/null +++ b/metrics.go @@ -0,0 +1,58 @@ +package main + +import "fmt" + +type TempReader interface { + ReadTemperatures() (map[string]float64, error) +} + +type SensorMonitor struct { + name string + cfg *SensorConfig + reader TempReader + samples map[string][]float64 +} + +func newSensorMonitor(name string, cfg *SensorConfig, reader TempReader) *SensorMonitor { + return &SensorMonitor{ + name: name, + cfg: cfg, + reader: reader, + samples: map[string][]float64{}, + } +} + +// Poll reads current temperatures and updates per-sensor-ID rolling sample buffers. +// Returns the latest readings keyed by sensor ID, the rolling aggregate per sensor +// ID, and any error. +func (m *SensorMonitor) Poll() (map[string]float64, map[string]float64, error) { + temps, err := m.reader.ReadTemperatures() + if err != nil { + return nil, nil, fmt.Errorf("%s: reading temperatures: %w", m.name, err) + } + if len(temps) == 0 { + return nil, nil, fmt.Errorf("%s: no temperature readings returned", m.name) + } + + for id, t := range temps { + m.samples[id] = append(m.samples[id], t) + if len(m.samples[id]) > m.cfg.SampleSize { + m.samples[id] = m.samples[id][len(m.samples[id])-m.cfg.SampleSize:] + } + } + + return temps, m.Aggregates(), nil +} + +// Aggregates returns the rolling average for each sensor ID. +func (m *SensorMonitor) Aggregates() map[string]float64 { + result := make(map[string]float64, len(m.samples)) + for id, samples := range m.samples { + var sum float64 + for _, s := range samples { + sum += s + } + result[id] = sum / float64(len(samples)) + } + return result +} diff --git a/metrics_test.go b/metrics_test.go new file mode 100644 index 0000000..a2c0c3c --- /dev/null +++ b/metrics_test.go @@ -0,0 +1,89 @@ +package main + +import ( + "errors" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type staticReader struct { + temps map[string]float64 + err error +} + +func (r *staticReader) ReadTemperatures() (map[string]float64, error) { + return r.temps, r.err +} + +func testSensorConfig() *SensorConfig { + return &SensorConfig{ + IdealTemp: 40, + MaxTemp: 75, + SampleSize: 3, + SampleInterval: 15 * time.Second, + } +} + +func TestSensorMonitor_AccumulatesSamples(t *testing.T) { + reader := &staticReader{temps: map[string]float64{"s0": 50, "s1": 60}} + m := newSensorMonitor("cpu", testSensorConfig(), reader) + temps, aggs, err := m.Poll() + require.NoError(t, err) + assert.Equal(t, map[string]float64{"s0": 50, "s1": 60}, temps) + assert.Len(t, m.samples, 2) + assert.Equal(t, map[string]float64{"s0": 50.0, "s1": 60.0}, aggs) +} + +func TestSensorMonitor_CapsSamplesAtSampleSize(t *testing.T) { + cfg := testSensorConfig() + cfg.SampleSize = 3 + reader := &staticReader{temps: map[string]float64{"s0": 0}} + m := newSensorMonitor("cpu", cfg, reader) + + for i := 0; i < 5; i++ { + reader.temps = map[string]float64{"s0": float64(i * 10)} + _, _, err := m.Poll() + require.NoError(t, err) + } + assert.Len(t, m.samples["s0"], 3) + // last 3 polls: 20, 30, 40 + assert.Equal(t, (20.0+30.0+40.0)/3, m.Aggregates()["s0"]) +} + +func TestSensorMonitor_AggregatesBeforeAnyPolls(t *testing.T) { + m := newSensorMonitor("cpu", testSensorConfig(), &staticReader{temps: map[string]float64{"s0": 50}}) + assert.Empty(t, m.Aggregates()) +} + +func TestSensorMonitor_ReaderError(t *testing.T) { + reader := &staticReader{err: errors.New("ipmitool failed")} + m := newSensorMonitor("cpu", testSensorConfig(), reader) + _, _, err := m.Poll() + assert.Error(t, err) +} + +func TestSensorMonitor_EmptyReadings(t *testing.T) { + reader := &staticReader{temps: map[string]float64{}} + m := newSensorMonitor("cpu", testSensorConfig(), reader) + _, _, err := m.Poll() + assert.Error(t, err) +} + +func TestSensorMonitor_MultipleSensorIDsHaveIndependentAggregates(t *testing.T) { + cfg := testSensorConfig() + cfg.SampleSize = 2 + reader := &staticReader{temps: map[string]float64{"s0": 50, "s1": 70}} + m := newSensorMonitor("cpu", cfg, reader) + + _, _, err := m.Poll() + require.NoError(t, err) + reader.temps = map[string]float64{"s0": 60, "s1": 80} + _, aggs, err := m.Poll() + require.NoError(t, err) + + assert.Equal(t, (50.0+60.0)/2, aggs["s0"]) + assert.Equal(t, (70.0+80.0)/2, aggs["s1"]) +} diff --git a/packaging/build-deb.sh b/packaging/build-deb.sh new file mode 100755 index 0000000..2ec24aa --- /dev/null +++ b/packaging/build-deb.sh @@ -0,0 +1,29 @@ +#!/bin/sh +set -e + +BINARY=frostd +VERSION=$(git describe --tags --always --dirty 2>/dev/null || echo "0.0.0") +PKG=pkg + +rm -rf "$PKG" +mkdir -p "$PKG/DEBIAN" +mkdir -p "$PKG/usr/local/bin" +mkdir -p "$PKG/etc/logrotate.d" +mkdir -p "$PKG/lib/systemd/system" + +cp packaging/debian/control "$PKG/DEBIAN/control" +sed -i "s/^Version:.*/Version: $VERSION/" "$PKG/DEBIAN/control" + +cp packaging/debian/postinst "$PKG/DEBIAN/postinst" +cp packaging/debian/prerm "$PKG/DEBIAN/prerm" +chmod 755 "$PKG/DEBIAN/postinst" "$PKG/DEBIAN/prerm" + +cp "$BINARY" "$PKG/usr/local/bin/$BINARY" +cp frostd.yaml "$PKG/etc/frostd.yaml" +cp packaging/frostd.service "$PKG/lib/systemd/system/frostd.service" +cp packaging/frostd.logrotate "$PKG/etc/logrotate.d/frostd" + +dpkg-deb --build "$PKG" "${BINARY}_${VERSION}_amd64.deb" +rm -rf "$PKG" + +echo "Built: ${BINARY}_${VERSION}_amd64.deb" diff --git a/packaging/debian/control b/packaging/debian/control new file mode 100644 index 0000000..133067e --- /dev/null +++ b/packaging/debian/control @@ -0,0 +1,9 @@ +Package: frostd +Version: 1.0.0 +Section: utils +Priority: optional +Architecture: amd64 +Maintainer: Shawn Poulson +Description: Dell PowerEdge fan speed regulator + frostd dynamically regulates fan speeds on Dell PowerEdge 12th generation + servers based on CPU and GPU temperature metrics read via IPMI and nvidia-smi. diff --git a/packaging/debian/postinst b/packaging/debian/postinst new file mode 100644 index 0000000..9adc730 --- /dev/null +++ b/packaging/debian/postinst @@ -0,0 +1,5 @@ +#!/bin/sh +set -e +systemctl daemon-reload +systemctl enable frostd +systemctl start frostd diff --git a/packaging/debian/prerm b/packaging/debian/prerm new file mode 100644 index 0000000..8f82868 --- /dev/null +++ b/packaging/debian/prerm @@ -0,0 +1,4 @@ +#!/bin/sh +set -e +systemctl stop frostd || true +systemctl disable frostd || true diff --git a/packaging/frostd.logrotate b/packaging/frostd.logrotate new file mode 100644 index 0000000..89ab925 --- /dev/null +++ b/packaging/frostd.logrotate @@ -0,0 +1,9 @@ +/var/log/frostd/frostd.log { + daily + rotate 7 + compress + delaycompress + missingok + notifempty + copytruncate +} diff --git a/packaging/frostd.service b/packaging/frostd.service new file mode 100644 index 0000000..ba053a9 --- /dev/null +++ b/packaging/frostd.service @@ -0,0 +1,12 @@ +[Unit] +Description=frostd - Dell PowerEdge fan speed regulator +After=network.target + +[Service] +Type=simple +ExecStart=/usr/local/bin/frostd +Restart=on-failure +RestartSec=5s + +[Install] +WantedBy=multi-user.target diff --git a/prometheus.go b/prometheus.go new file mode 100644 index 0000000..f28570f --- /dev/null +++ b/prometheus.go @@ -0,0 +1,80 @@ +package main + +import ( + "context" + "log/slog" + "net/http" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +type PrometheusMetrics struct { + sensorTemp *prometheus.GaugeVec + aggregateTemp *prometheus.GaugeVec + suggestedSpeed *prometheus.GaugeVec + fanSpeedRPM *prometheus.GaugeVec + fanSpeedPercent *prometheus.GaugeVec + systemSpeed prometheus.Gauge + registry *prometheus.Registry +} + +func newPrometheusMetrics() *PrometheusMetrics { + reg := prometheus.NewRegistry() + m := &PrometheusMetrics{ + sensorTemp: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "frostd_sensor_temperature", + Help: "Current temperature reading per sensor, in Celsius.", + }, []string{"sensor", "id"}), + aggregateTemp: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "frostd_sensor_aggregate_temperature", + Help: "Rolling aggregate temperature per sensor type, in Celsius.", + }, []string{"sensor", "id"}), + suggestedSpeed: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "frostd_suggested_fan_speed_percent", + Help: "Suggested fan speed percentage per sensor type.", + }, []string{"sensor"}), + fanSpeedRPM: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "frostd_actual_fan_rpm", + Help: "Actual fan speed in RPM.", + }, []string{"fan"}), + fanSpeedPercent: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "frostd_actual_fan_speed_percent", + Help: "Actual fan speed as percentage.", + }, []string{"fan"}), + systemSpeed: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "frostd_system_fan_speed_percent", + Help: "Commanded system fan speed percentage.", + }), + registry: reg, + } + reg.MustRegister( + m.sensorTemp, + m.aggregateTemp, + m.suggestedSpeed, + m.fanSpeedRPM, + m.fanSpeedPercent, + m.systemSpeed, + ) + return m +} + +func startPrometheusServer(ctx context.Context, addr string, m *PrometheusMetrics, logger *slog.Logger) { + mux := http.NewServeMux() + mux.Handle("/metrics", promhttp.HandlerFor(m.registry, promhttp.HandlerOpts{})) + srv := &http.Server{Addr: addr, Handler: mux} + + go func() { + <-ctx.Done() + if err := srv.Shutdown(context.Background()); err != nil { + logger.Error("prometheus server shutdown error", "error", err) + } + }() + + go func() { + logger.Info("prometheus metrics server listening", "addr", addr) + if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed { + logger.Error("prometheus metrics server error", "error", err) + } + }() +} diff --git a/service.go b/service.go new file mode 100644 index 0000000..21a620f --- /dev/null +++ b/service.go @@ -0,0 +1,144 @@ +package main + +import ( + "context" + "log/slog" + "time" +) + +type speedUpdate struct { + sensor string + speed int +} + +type Service struct { + cfg *Config + fanCtrl FanController + monitors []*SensorMonitor + logger *slog.Logger + prom *PrometheusMetrics +} + +func (s *Service) run(ctx context.Context) { + prevFanReadings := map[string]FanReading{} + s.logFanSpeeds(ctx, prevFanReadings) + + speedCh := make(chan speedUpdate, len(s.monitors)) + latestSpeeds := make(map[string]int, len(s.monitors)) + for _, m := range s.monitors { + latestSpeeds[m.name] = 0 + go s.runSensor(ctx, m, speedCh) + } + + fanLogTicker := time.NewTicker(s.cfg.FanLogInterval) + defer fanLogTicker.Stop() + pendingUpdate := false + lastSpeed := -1 + + for { + select { + case <-ctx.Done(): + return + case <-fanLogTicker.C: + s.logFanSpeeds(ctx, prevFanReadings) + case update := <-speedCh: + latestSpeeds[update.sensor] = update.speed + pendingUpdate = true + } + + if !pendingUpdate { + continue + } + + pendingUpdate = false + maxSpeed := 0 + for _, s := range latestSpeeds { + if s > maxSpeed { + maxSpeed = s + } + } + if maxSpeed == lastSpeed { + continue + } + lastSpeed = maxSpeed + + s.prom.systemSpeed.Set(float64(maxSpeed)) + + if s.cfg.DryRun { + s.logger.Info("dry run: skipping fan speed change", "system_percent", maxSpeed) + continue + } + + s.logger.Info("setting fan speed", "system_percent", maxSpeed) + if err := s.fanCtrl.SetSpeed(ctx, maxSpeed); err != nil { + s.logger.Error("failed to set fan speed", "error", err) + } + } +} + +func (s *Service) runSensor(ctx context.Context, m *SensorMonitor, ch chan<- speedUpdate) { + nextTick := time.Now() + for { + nextTick = nextTick.Add(m.cfg.SampleInterval) + + temps, aggs, err := m.Poll() + if err != nil { + s.logger.Error("sensor temperature poll failed", "sensor", m.name, "error", err) + } else { + s.logger.Info("sensor temperature", "sensor", m.name, "latest_temps", temps, "aggregate_temps", aggs) + var maxAgg float64 + for _, a := range aggs { + if a > maxAgg { + maxAgg = a + } + } + speed := SuggestSpeed(maxAgg, m.cfg.IdealTemp, m.cfg.MaxTemp) + s.logger.Info("sensor suggested fan speed", "sensor", m.name, "suggest_percent", speed) + for id, t := range temps { + s.prom.sensorTemp.WithLabelValues(m.name, id).Set(t) + } + for id, a := range aggs { + s.prom.aggregateTemp.WithLabelValues(m.name, id).Set(a) + } + s.prom.suggestedSpeed.WithLabelValues(m.name).Set(float64(speed)) + select { + case ch <- speedUpdate{sensor: m.name, speed: speed}: + case <-ctx.Done(): + return + } + } + + delay := time.Until(nextTick) + if delay < 0 { + delay = 0 + } + select { + case <-ctx.Done(): + return + case <-time.After(delay): + } + } +} + +func (s *Service) logFanSpeeds(ctx context.Context, prev map[string]FanReading) { + readings, err := s.fanCtrl.ReadFanSpeeds(ctx) + if err != nil { + s.logger.Error("failed to read current fan speeds", "error", err) + return + } + for _, r := range readings { + args := []any{"fan", r.Name} + if r.RPM != nil { + args = append(args, "rpm", int(*r.RPM)) + if p, ok := prev[r.Name]; ok && p.RPM != nil { + args = append(args, "delta", int(*r.RPM-*p.RPM)) + } + s.prom.fanSpeedRPM.WithLabelValues(r.Name).Set(*r.RPM) + } + if r.Percent != nil { + s.prom.fanSpeedPercent.WithLabelValues(r.Name).Set(*r.Percent) + } + s.logger.Info("current fan speed", args...) + prev[r.Name] = r + } +} diff --git a/set_auto.sh b/set_auto.sh new file mode 100755 index 0000000..fe5d881 --- /dev/null +++ b/set_auto.sh @@ -0,0 +1,5 @@ +#!/bin/sh +# Set automatic fan control mode. +# Run as root. + +ipmitool -I open raw 0x30 0x30 0x01 0x01 diff --git a/testhelpers_test.go b/testhelpers_test.go new file mode 100644 index 0000000..330a82a --- /dev/null +++ b/testhelpers_test.go @@ -0,0 +1,50 @@ +package main + +import ( + "context" + "errors" + + ipmi "github.com/bougou/go-ipmi" +) + +// mockIPMIClient implements ipmiClient for tests. +type mockIPMIClient struct { + sensors []*ipmi.Sensor + rawErr error + connectErr error +} + +func (m *mockIPMIClient) Connect(_ context.Context) error { return m.connectErr } +func (m *mockIPMIClient) Close(_ context.Context) error { return nil } + +func (m *mockIPMIClient) GetSensors(_ context.Context, _ ...ipmi.SensorFilterOption) ([]*ipmi.Sensor, error) { + if m.rawErr != nil { + return nil, m.rawErr + } + return m.sensors, nil +} + +func (m *mockIPMIClient) RawCommand(_ context.Context, _ ipmi.NetFn, _ uint8, _ []byte, _ string) (*ipmi.CommandRawResponse, error) { + if m.rawErr != nil { + return nil, m.rawErr + } + return &ipmi.CommandRawResponse{}, nil +} + +func mockIPMIFactory(mock *mockIPMIClient) func() (ipmiClient, error) { + return func() (ipmiClient, error) { return mock, nil } +} + +func mockIPMIFactoryErr(err string) func() (ipmiClient, error) { + return func() (ipmiClient, error) { return nil, errors.New(err) } +} + +// mockRunner implements CommandRunner for tests. +type mockRunner struct { + output []byte + err error +} + +func (m *mockRunner) Run(_ string, _ ...string) ([]byte, error) { + return m.output, m.err +}