From bff743d1e546fc094bfde15eebdfdc3cdc62d180 Mon Sep 17 00:00:00 2001 From: Adam Chen Date: Sat, 8 Aug 2026 22:11:10 +0000 Subject: [PATCH 1/2] feat(observability): truncate time to seconds to prevent SI notation --- core/observability.go | 6 +++--- core/observability_test.go | 17 +++++++++++++++-- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/core/observability.go b/core/observability.go index 619130f..25bd21d 100644 --- a/core/observability.go +++ b/core/observability.go @@ -159,7 +159,7 @@ func writePrometheusMetrics(w io.Writer, status *protocol.StatusResponse) { node := status.GetNode() stats := node.GetStats() metrics.metric("nylon_up", "Whether the nylon daemon is ready.", "gauge", nil, 1) - metrics.metric("nylon_config_timestamp_seconds", "Unix timestamp of the active central configuration.", "gauge", nil, float64(node.ConfigTimestamp)/float64(time.Second)) + metrics.metric("nylon_config_timestamp_seconds", "Unix timestamp of the active central configuration.", "gauge", nil, float64(node.ConfigTimestamp/int64(time.Second))) metrics.metric("nylon_neighbours", "Number of configured neighbours.", "gauge", nil, float64(stats.NeighbourCount)) metrics.metric("nylon_active_endpoints", "Number of active peer endpoints.", "gauge", nil, float64(stats.ActiveEndpointCount)) metrics.metric("nylon_selected_routes", "Number of selected Babel routes.", "gauge", nil, float64(stats.SelectedRouteCount)) @@ -174,7 +174,7 @@ func writePrometheusMetrics(w io.Writer, status *protocol.StatusResponse) { metrics.metric("nylon_wireguard_peer_receive_bytes_total", "WireGuard bytes received from a peer.", "counter", labels, float64(wg.RxBytes)) handshake := float64(0) if wg.LatestHandshakeUnix > 0 { - handshake = float64(wg.LatestHandshakeUnix) / float64(time.Second) + handshake = float64(wg.LatestHandshakeUnix / int64(time.Second)) } metrics.metric("nylon_wireguard_peer_latest_handshake_seconds", "Unix time of the latest WireGuard handshake.", "gauge", labels, handshake) for _, endpoint := range neigh.Endpoints { @@ -225,5 +225,5 @@ func (m *metricWriter) metric(name, help, metricType string, labels map[string]s } _, _ = io.WriteString(m.w, "}") } - _, _ = fmt.Fprintf(m.w, " %g\n", value) + _, _ = fmt.Fprintf(m.w, " %s\n", strconv.FormatFloat(value, 'f', -1, 64)) } diff --git a/core/observability_test.go b/core/observability_test.go index 66fb37c..fe97852 100644 --- a/core/observability_test.go +++ b/core/observability_test.go @@ -8,6 +8,7 @@ import ( "net/netip" "strings" "testing" + "time" "github.com/encodeous/nylon/protocol" "github.com/encodeous/nylon/state" @@ -47,9 +48,18 @@ func TestObservabilityDiscovery(t *testing.T) { func TestPrometheusMetrics(t *testing.T) { status := &protocol.StatusResponse{ - Node: &protocol.NodeStatus{Stats: &protocol.NodeStats{TxBytes: 12}}, + Node: &protocol.NodeStatus{ + ConfigTimestamp: 1731117600 * int64(time.Second), + Stats: &protocol.NodeStats{TxBytes: 12}, + }, Neighbours: []*protocol.NeighbourInfo{ - {PeerId: "bob", Wireguard: &protocol.WireGuardPeerStats{TxBytes: 7}}, + { + PeerId: "bob", + Wireguard: &protocol.WireGuardPeerStats{ + TxBytes: 7, + LatestHandshakeUnix: 1786226710556904600, + }, + }, {PeerId: "eve", Wireguard: &protocol.WireGuardPeerStats{TxBytes: 5}}, }, } @@ -57,6 +67,9 @@ func TestPrometheusMetrics(t *testing.T) { writePrometheusMetrics(&buf, status) output := buf.String() require.Contains(t, output, "# TYPE nylon_wireguard_transmit_bytes_total counter") + require.Contains(t, output, "nylon_config_timestamp_seconds 1731117600\n") require.Contains(t, output, `nylon_wireguard_peer_transmit_bytes_total{peer="bob"} 7`) + require.Contains(t, output, `nylon_wireguard_peer_latest_handshake_seconds{peer="bob"} 1786226710`) + require.NotContains(t, output, `nylon_wireguard_peer_latest_handshake_seconds{peer="bob"} 1.786`) require.Equal(t, 1, strings.Count(output, "# HELP nylon_wireguard_peer_transmit_bytes_total ")) } From 6cfa215bf81a7a370219e68b96877fd1ca6f748c Mon Sep 17 00:00:00 2001 From: Adam Chen Date: Sat, 8 Aug 2026 22:36:08 +0000 Subject: [PATCH 2/2] feat(observability): update docs and introduce node type --- core/observability.go | 9 ++- core/observability_test.go | 25 +++++--- docs/guides/getting-started.mdx | 1 + docs/guides/observability.mdx | 101 ++++++++++++++++++++++++++++++++ 4 files changed, 128 insertions(+), 8 deletions(-) create mode 100644 docs/guides/observability.mdx diff --git a/core/observability.go b/core/observability.go index 25bd21d..63971df 100644 --- a/core/observability.go +++ b/core/observability.go @@ -144,9 +144,16 @@ func (n *Nylon) handleDiscovery(w http.ResponseWriter, _ *http.Request) { targets = append(targets, net.JoinHostPort(addr.String(), port)) } if len(targets) != 0 { + nodeType := "router" + if n.CentralCfg.IsClient(node.Id) { + nodeType = "passive" + } groups = append(groups, discoveryGroup{ Targets: targets, - Labels: map[string]string{"nylon_node": string(node.Id)}, + Labels: map[string]string{ + "nylon_node": string(node.Id), + "nylon_node_type": nodeType, + }, }) } } diff --git a/core/observability_test.go b/core/observability_test.go index fe97852..9e017e9 100644 --- a/core/observability_test.go +++ b/core/observability_test.go @@ -33,17 +33,28 @@ func TestObservabilityHealth(t *testing.T) { func TestObservabilityDiscovery(t *testing.T) { n := &Nylon{ConfigState: state.ConfigState{ LocalCfg: state.LocalCfg{ObservabilityAddr: "0.0.0.0:9090"}, - CentralCfg: state.CentralCfg{Routers: []state.RouterCfg{{ - NodeCfg: state.NodeCfg{ - Id: "alice", - Addresses: []netip.Addr{netip.MustParseAddr("10.0.0.1"), netip.MustParseAddr("fd00::1")}, - }, - }}}, + CentralCfg: state.CentralCfg{ + Routers: []state.RouterCfg{{ + NodeCfg: state.NodeCfg{ + Id: "alice", + Addresses: []netip.Addr{netip.MustParseAddr("10.0.0.1"), netip.MustParseAddr("fd00::1")}, + }, + }}, + Clients: []state.ClientCfg{{ + NodeCfg: state.NodeCfg{ + Id: "phone", + Addresses: []netip.Addr{netip.MustParseAddr("10.0.0.2")}, + }, + }}, + }, }} rec := httptest.NewRecorder() n.handleDiscovery(rec, httptest.NewRequest(http.MethodGet, "/discovery", nil)) require.Equal(t, http.StatusOK, rec.Code) - require.JSONEq(t, `[{"targets":["10.0.0.1:9090","[fd00::1]:9090"],"labels":{"nylon_node":"alice"}}]`, rec.Body.String()) + require.JSONEq(t, `[ + {"targets":["10.0.0.1:9090","[fd00::1]:9090"],"labels":{"nylon_node":"alice","nylon_node_type":"router"}}, + {"targets":["10.0.0.2:9090"],"labels":{"nylon_node":"phone","nylon_node_type":"passive"}} + ]`, rec.Body.String()) } func TestPrometheusMetrics(t *testing.T) { diff --git a/docs/guides/getting-started.mdx b/docs/guides/getting-started.mdx index 52eb1f0..21d80b2 100644 --- a/docs/guides/getting-started.mdx +++ b/docs/guides/getting-started.mdx @@ -107,5 +107,6 @@ The Linux and macOS versions are well tested, but the Windows TUN interface has - Learn how to connect [Passive Nodes](/guides/wg-clients) to support edge platforms like iOS. - Discover how to use [Config Distribution](/guides/config-distribution) to manage your network configuration with ease. - Setup nylon without a static public IP using [Dynamic DNS & Port Forwarding](/guides/port-forward). +- Monitor your nodes with [Prometheus metrics and health checks](/guides/observability). {/* TODO: Advanced Routing guide (Anycast, Prefix Healthchecks) */} {/* TODO: Monitoring and Debugging guide */} \ No newline at end of file diff --git a/docs/guides/observability.mdx b/docs/guides/observability.mdx new file mode 100644 index 0000000..aa94ad2 --- /dev/null +++ b/docs/guides/observability.mdx @@ -0,0 +1,101 @@ +--- +title: Observability +description: Monitor nylon with Prometheus metrics, health checks, and service discovery. +sidebar: + order: 5 +--- + +Nylon can expose an HTTP server for Prometheus metrics, health checks, and service discovery. + +## Enable the server + +Set `observability_addr` in each node's `node.yaml`: + +```yaml title="node.yaml" +observability_addr: "0.0.0.0:9090" +``` + +Restart nylon after changing the node configuration. You can then check the local node: + +```bash +curl http://127.0.0.1:9090/healthz +curl http://127.0.0.1:9090/readyz +curl http://127.0.0.1:9090/metrics +``` + +:::caution +The observability server does not provide authentication or TLS. Bind it to a trusted interface, or protect it with a firewall or authenticated reverse proxy. Avoid exposing it directly to the internet. +::: + +## Endpoints + +| Endpoint | Purpose | +| --- | --- | +| `/healthz` | Returns `200 OK` while the daemon is running and `503 Service Unavailable` while it is shutting down. | +| `/readyz` | Returns `200 OK` when nylon's internal event loop responds, or `503 Service Unavailable` if it cannot respond within one second. It does not check whether every peer is reachable. | +| `/metrics` | Returns metrics in the Prometheus text exposition format. Returns `503 Service Unavailable` if nylon cannot collect a status snapshot within one second. | +| `/discovery` | Returns Prometheus HTTP service-discovery targets for the addresses in `central.yaml`. | + +## Configure Prometheus + +For a single node, add a static scrape target: + +```yaml title="prometheus.yml" +scrape_configs: + - job_name: nylon + static_configs: + - targets: + - "10.0.0.1:9090" +``` + +For a mesh, Prometheus can discover the node addresses from any nylon observability server: + +```yaml title="prometheus.yml" +scrape_configs: + - job_name: nylon + http_sd_configs: + - url: "http://10.0.0.1:9090/discovery" + refresh_interval: 30s + relabel_configs: + - source_labels: [nylon_node_type] + regex: passive + action: drop +``` + +The discovery response uses every node address from `central.yaml`. Each target has a `nylon_node` label containing the node ID and a `nylon_node_type` label set to either `router` or `passive`. The example drops passive targets because they do not run the nylon daemon. + +Configure the same observability port on every router, and make sure Prometheus can reach their nylon addresses on that port. + +## Metrics + +All metric names start with `nylon_`. + +| Metric | Type | Labels | Description | +| --- | --- | --- | --- | +| `nylon_up` | Gauge | — | `1` when nylon can produce a metrics snapshot. | +| `nylon_config_timestamp_seconds` | Gauge | — | Unix timestamp of the active central configuration. | +| `nylon_neighbours` | Gauge | — | Number of configured neighbours. | +| `nylon_active_endpoints` | Gauge | — | Number of active peer endpoints. | +| `nylon_selected_routes` | Gauge | — | Number of selected Babel routes. | +| `nylon_advertised_prefixes` | Gauge | — | Number of locally advertised prefixes. | +| `nylon_wireguard_transmit_bytes_total` | Counter | — | WireGuard bytes transmitted by this node. | +| `nylon_wireguard_receive_bytes_total` | Counter | — | WireGuard bytes received by this node. | +| `nylon_wireguard_peer_transmit_bytes_total` | Counter | `peer` | WireGuard bytes transmitted to a peer. | +| `nylon_wireguard_peer_receive_bytes_total` | Counter | `peer` | WireGuard bytes received from a peer. | +| `nylon_wireguard_peer_latest_handshake_seconds` | Gauge | `peer` | Unix time of the peer's latest WireGuard handshake, or `0` before the first handshake. | +| `nylon_endpoint_active` | Gauge | `peer`, `endpoint` | `1` when the endpoint is active, otherwise `0`. | +| `nylon_endpoint_metric` | Gauge | `peer`, `endpoint` | Current Babel endpoint metric. | +| `nylon_endpoint_rtt_seconds` | Gauge | `peer`, `endpoint` | Filtered endpoint round-trip time in seconds. | +| `nylon_route_metric` | Gauge | `prefix`, `router`, `next_hop` | Metric of a selected Babel route. | + +For example, alert when a node cannot be scraped or when a peer has not completed a handshake in the last five minutes: + +```text +up{job="nylon"} == 0 +``` + +```text +time() - nylon_wireguard_peer_latest_handshake_seconds > 300 +``` + +The handshake metric is `0` before the first successful handshake, so this expression also selects peers that have never connected. When using it in an alerting rule, add `for: 5m` to avoid firing while nylon starts and establishes its initial peer connections.