feat(observability): improve observability endpoints and docs

core/observability.go

@@ -144,9 +144,16 @@ func (n *Nylon) handleDiscovery(w http.ResponseWriter, _ *http.Request) {
 			targets = append(targets, net.JoinHostPort(addr.String(), port))
 		}
 		if len(targets) != 0 {
+			nodeType := "router"
+			if n.CentralCfg.IsClient(node.Id) {
+				nodeType = "passive"
+			}
 			groups = append(groups, discoveryGroup{
 				Targets: targets,
-				Labels:  map[string]string{"nylon_node": string(node.Id)},
+				Labels: map[string]string{
+					"nylon_node":      string(node.Id),
+					"nylon_node_type": nodeType,
+				},
 			})
 		}
 	}
@@ -159,7 +166,7 @@ func writePrometheusMetrics(w io.Writer, status *protocol.StatusResponse) {
 	node := status.GetNode()
 	stats := node.GetStats()
 	metrics.metric("nylon_up", "Whether the nylon daemon is ready.", "gauge", nil, 1)
-	metrics.metric("nylon_config_timestamp_seconds", "Unix timestamp of the active central configuration.", "gauge", nil, float64(node.ConfigTimestamp)/float64(time.Second))
+	metrics.metric("nylon_config_timestamp_seconds", "Unix timestamp of the active central configuration.", "gauge", nil, float64(node.ConfigTimestamp/int64(time.Second)))
 	metrics.metric("nylon_neighbours", "Number of configured neighbours.", "gauge", nil, float64(stats.NeighbourCount))
 	metrics.metric("nylon_active_endpoints", "Number of active peer endpoints.", "gauge", nil, float64(stats.ActiveEndpointCount))
 	metrics.metric("nylon_selected_routes", "Number of selected Babel routes.", "gauge", nil, float64(stats.SelectedRouteCount))
@@ -174,7 +181,7 @@ func writePrometheusMetrics(w io.Writer, status *protocol.StatusResponse) {
 		metrics.metric("nylon_wireguard_peer_receive_bytes_total", "WireGuard bytes received from a peer.", "counter", labels, float64(wg.RxBytes))
 		handshake := float64(0)
 		if wg.LatestHandshakeUnix > 0 {
-			handshake = float64(wg.LatestHandshakeUnix) / float64(time.Second)
+			handshake = float64(wg.LatestHandshakeUnix / int64(time.Second))
 		}
 		metrics.metric("nylon_wireguard_peer_latest_handshake_seconds", "Unix time of the latest WireGuard handshake.", "gauge", labels, handshake)
 		for _, endpoint := range neigh.Endpoints {
@@ -225,5 +232,5 @@ func (m *metricWriter) metric(name, help, metricType string, labels map[string]s
 		}
 		_, _ = io.WriteString(m.w, "}")
 	}
-	_, _ = fmt.Fprintf(m.w, " %g\n", value)
+	_, _ = fmt.Fprintf(m.w, " %s\n", strconv.FormatFloat(value, 'f', -1, 64))
 }

core/observability_test.go

@@ -8,6 +8,7 @@ import (
 	"net/netip"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/encodeous/nylon/protocol"
 	"github.com/encodeous/nylon/state"
@@ -32,31 +33,54 @@ func TestObservabilityHealth(t *testing.T) {
 func TestObservabilityDiscovery(t *testing.T) {
 	n := &Nylon{ConfigState: state.ConfigState{
 		LocalCfg: state.LocalCfg{ObservabilityAddr: "0.0.0.0:9090"},
-		CentralCfg: state.CentralCfg{Routers: []state.RouterCfg{{
-			NodeCfg: state.NodeCfg{
-				Id:        "alice",
-				Addresses: []netip.Addr{netip.MustParseAddr("10.0.0.1"), netip.MustParseAddr("fd00::1")},
-			},
-		}}},
+		CentralCfg: state.CentralCfg{
+			Routers: []state.RouterCfg{{
+				NodeCfg: state.NodeCfg{
+					Id:        "alice",
+					Addresses: []netip.Addr{netip.MustParseAddr("10.0.0.1"), netip.MustParseAddr("fd00::1")},
+				},
+			}},
+			Clients: []state.ClientCfg{{
+				NodeCfg: state.NodeCfg{
+					Id:        "phone",
+					Addresses: []netip.Addr{netip.MustParseAddr("10.0.0.2")},
+				},
+			}},
+		},
 	}}
 	rec := httptest.NewRecorder()
 	n.handleDiscovery(rec, httptest.NewRequest(http.MethodGet, "/discovery", nil))
 	require.Equal(t, http.StatusOK, rec.Code)
-	require.JSONEq(t, `[{"targets":["10.0.0.1:9090","[fd00::1]:9090"],"labels":{"nylon_node":"alice"}}]`, rec.Body.String())
+	require.JSONEq(t, `[
+		{"targets":["10.0.0.1:9090","[fd00::1]:9090"],"labels":{"nylon_node":"alice","nylon_node_type":"router"}},
+		{"targets":["10.0.0.2:9090"],"labels":{"nylon_node":"phone","nylon_node_type":"passive"}}
+	]`, rec.Body.String())
 }
 
 func TestPrometheusMetrics(t *testing.T) {
 	status := &protocol.StatusResponse{
-		Node: &protocol.NodeStatus{Stats: &protocol.NodeStats{TxBytes: 12}},
+		Node: &protocol.NodeStatus{
+			ConfigTimestamp: 1731117600 * int64(time.Second),
+			Stats:           &protocol.NodeStats{TxBytes: 12},
+		},
 		Neighbours: []*protocol.NeighbourInfo{
-			{PeerId: "bob", Wireguard: &protocol.WireGuardPeerStats{TxBytes: 7}},
+			{
+				PeerId: "bob",
+				Wireguard: &protocol.WireGuardPeerStats{
+					TxBytes:             7,
+					LatestHandshakeUnix: 1786226710556904600,
+				},
+			},
 			{PeerId: "eve", Wireguard: &protocol.WireGuardPeerStats{TxBytes: 5}},
 		},
 	}
 	var buf bytes.Buffer
 	writePrometheusMetrics(&buf, status)
 	output := buf.String()
 	require.Contains(t, output, "# TYPE nylon_wireguard_transmit_bytes_total counter")
+	require.Contains(t, output, "nylon_config_timestamp_seconds 1731117600\n")
 	require.Contains(t, output, `nylon_wireguard_peer_transmit_bytes_total{peer="bob"} 7`)
+	require.Contains(t, output, `nylon_wireguard_peer_latest_handshake_seconds{peer="bob"} 1786226710`)
+	require.NotContains(t, output, `nylon_wireguard_peer_latest_handshake_seconds{peer="bob"} 1.786`)
 	require.Equal(t, 1, strings.Count(output, "# HELP nylon_wireguard_peer_transmit_bytes_total "))
 }

docs/guides/getting-started.mdx

@@ -107,5 +107,6 @@ The Linux and macOS versions are well tested, but the Windows TUN interface has
 - Learn how to connect [Passive Nodes](/guides/wg-clients) to support edge platforms like iOS.
 - Discover how to use [Config Distribution](/guides/config-distribution) to manage your network configuration with ease.
 - Setup nylon without a static public IP using [Dynamic DNS & Port Forwarding](/guides/port-forward).
+- Monitor your nodes with [Prometheus metrics and health checks](/guides/observability).
 {/* TODO: Advanced Routing guide (Anycast, Prefix Healthchecks) */}
 {/* TODO: Monitoring and Debugging guide */}

docs/guides/observability.mdx

@@ -0,0 +1,101 @@
+---
+title: Observability
+description: Monitor nylon with Prometheus metrics, health checks, and service discovery.
+sidebar:
+  order: 5
+---
+
+Nylon can expose an HTTP server for Prometheus metrics, health checks, and service discovery.
+
+## Enable the server
+
+Set `observability_addr` in each node's `node.yaml`:
+
+```yaml title="node.yaml"
+observability_addr: "0.0.0.0:9090"
+```
+
+Restart nylon after changing the node configuration. You can then check the local node:
+
+```bash
+curl http://127.0.0.1:9090/healthz
+curl http://127.0.0.1:9090/readyz
+curl http://127.0.0.1:9090/metrics
+```
+
+:::caution
+The observability server does not provide authentication or TLS. Bind it to a trusted interface, or protect it with a firewall or authenticated reverse proxy. Avoid exposing it directly to the internet.
+:::
+
+## Endpoints
+
+| Endpoint | Purpose |
+| --- | --- |
+| `/healthz` | Returns `200 OK` while the daemon is running and `503 Service Unavailable` while it is shutting down. |
+| `/readyz` | Returns `200 OK` when nylon's internal event loop responds, or `503 Service Unavailable` if it cannot respond within one second. It does not check whether every peer is reachable. |
+| `/metrics` | Returns metrics in the Prometheus text exposition format. Returns `503 Service Unavailable` if nylon cannot collect a status snapshot within one second. |
+| `/discovery` | Returns Prometheus HTTP service-discovery targets for the addresses in `central.yaml`. |
+
+## Configure Prometheus
+
+For a single node, add a static scrape target:
+
+```yaml title="prometheus.yml"
+scrape_configs:
+  - job_name: nylon
+    static_configs:
+      - targets:
+          - "10.0.0.1:9090"
+```
+
+For a mesh, Prometheus can discover the node addresses from any nylon observability server:
+
+```yaml title="prometheus.yml"
+scrape_configs:
+  - job_name: nylon
+    http_sd_configs:
+      - url: "http://10.0.0.1:9090/discovery"
+        refresh_interval: 30s
+    relabel_configs:
+      - source_labels: [nylon_node_type]
+        regex: passive
+        action: drop
+```
+
+The discovery response uses every node address from `central.yaml`. Each target has a `nylon_node` label containing the node ID and a `nylon_node_type` label set to either `router` or `passive`. The example drops passive targets because they do not run the nylon daemon.
+
+Configure the same observability port on every router, and make sure Prometheus can reach their nylon addresses on that port.
+
+## Metrics
+
+All metric names start with `nylon_`.
+
+| Metric | Type | Labels | Description |
+| --- | --- | --- | --- |
+| `nylon_up` | Gauge | — | `1` when nylon can produce a metrics snapshot. |
+| `nylon_config_timestamp_seconds` | Gauge | — | Unix timestamp of the active central configuration. |
+| `nylon_neighbours` | Gauge | — | Number of configured neighbours. |
+| `nylon_active_endpoints` | Gauge | — | Number of active peer endpoints. |
+| `nylon_selected_routes` | Gauge | — | Number of selected Babel routes. |
+| `nylon_advertised_prefixes` | Gauge | — | Number of locally advertised prefixes. |
+| `nylon_wireguard_transmit_bytes_total` | Counter | — | WireGuard bytes transmitted by this node. |
+| `nylon_wireguard_receive_bytes_total` | Counter | — | WireGuard bytes received by this node. |
+| `nylon_wireguard_peer_transmit_bytes_total` | Counter | `peer` | WireGuard bytes transmitted to a peer. |
+| `nylon_wireguard_peer_receive_bytes_total` | Counter | `peer` | WireGuard bytes received from a peer. |
+| `nylon_wireguard_peer_latest_handshake_seconds` | Gauge | `peer` | Unix time of the peer's latest WireGuard handshake, or `0` before the first handshake. |
+| `nylon_endpoint_active` | Gauge | `peer`, `endpoint` | `1` when the endpoint is active, otherwise `0`. |
+| `nylon_endpoint_metric` | Gauge | `peer`, `endpoint` | Current Babel endpoint metric. |
+| `nylon_endpoint_rtt_seconds` | Gauge | `peer`, `endpoint` | Filtered endpoint round-trip time in seconds. |
+| `nylon_route_metric` | Gauge | `prefix`, `router`, `next_hop` | Metric of a selected Babel route. |
+
+For example, alert when a node cannot be scraped or when a peer has not completed a handshake in the last five minutes:
+
+```text
+up{job="nylon"} == 0
+```
+
+```text
+time() - nylon_wireguard_peer_latest_handshake_seconds > 300
+```
+
+The handshake metric is `0` before the first successful handshake, so this expression also selects peers that have never connected. When using it in an alerting rule, add `for: 5m` to avoid firing while nylon starts and establishes its initial peer connections.