From 68208743ef7d5875ef29ef5a3c4103205c2541d3 Mon Sep 17 00:00:00 2001 From: Eric Wendland Date: Tue, 28 Jul 2026 22:32:41 +0200 Subject: [PATCH] core(routing): reconcile against system route state --- core/nylon.go | 5 +- core/nylon_wireguard.go | 86 +++++++++++++++----------- core/sys_darwin.go | 122 +++++++++++++++++++++++++++++++++---- core/sys_linux.go | 80 +++++++++++++++++++++--- core/sys_utils.go | 9 ++- core/sys_windows.go | 112 +++++++++++++++++++++++++++++----- core/system_routes.go | 27 ++++++++ core/system_routes_test.go | 112 ++++++++++++++++++++++++++++++++++ 8 files changed, 477 insertions(+), 76 deletions(-) create mode 100644 core/system_routes.go create mode 100644 core/system_routes_test.go diff --git a/core/nylon.go b/core/nylon.go index 61553d23..88c378b2 100644 --- a/core/nylon.go +++ b/core/nylon.go @@ -62,6 +62,7 @@ type Nylon struct { wgUapi net.Listener Interface string Device *device.Device + SystemRoutes SystemRoutes observability *observabilityServer // only used for debugging & tests @@ -74,9 +75,7 @@ type Nylon struct { } type AppliedSystemState struct { - Routes []netip.Prefix - Aliases []netip.Addr - Peers map[state.NodeId]state.NyPublicKey + Peers map[state.NodeId]state.NyPublicKey } func NewNylon(ccfg state.CentralCfg, ncfg state.LocalCfg, logLevel slog.Level, configPath string, aux map[string]any, opts state.NylonOptions, tunables *state.RouterTunables) (*Nylon, error) { diff --git a/core/nylon_wireguard.go b/core/nylon_wireguard.go index d83062fa..4dcbe411 100644 --- a/core/nylon_wireguard.go +++ b/core/nylon_wireguard.go @@ -7,7 +7,6 @@ import ( "errors" "fmt" "net/netip" - "runtime" "slices" "github.com/encodeous/nylon/polyamide/conn" @@ -29,6 +28,9 @@ func (n *Nylon) initWireGuard() error { n.Device = dev n.Tun = tdev n.Interface = itfName + if !n.NoTun && n.SystemRoutes == nil { + n.SystemRoutes = NewSystemRoutes(n.Log, n.Tun) + } n.InstallTC() n.Log.Info("installed nylon traffic control filter for polysock") @@ -68,19 +70,13 @@ listen_port=%d } if !n.NoNetConfigure && !n.NoTun { - for _, addr := range n.GetRouter(n.LocalCfg.Id).Addresses { - err := ConfigureAlias(n.Log, itfName, addr) - if err != nil { - n.Log.Error("failed to configure alias", "err", err) - } else if !slices.Contains(n.AppliedSystem.Aliases, addr) { - n.AppliedSystem.Aliases = append(n.AppliedSystem.Aliases, addr) - } - } - err = InitInterface(n.Log, itfName) if err != nil { return err } + if err := n.SyncSystemState(); err != nil { + n.Log.Warn("initial system networking reconciliation incomplete; will retry", "err", err) + } } // run post-up commands @@ -103,17 +99,27 @@ listen_port=%d } func (n *Nylon) cleanupWireGuard() error { - // remove routes - for _, route := range n.AppliedSystem.Routes { - err := RemoveRoute(n.Log, n.Tun, n.Interface, route) - if err != nil { - n.Log.Error("failed to remove route", "err", err) + if !n.NoNetConfigure && !n.NoTun && n.SystemRoutes != nil { + if routes, err := n.SystemRoutes.InterfaceRoutes(n.Interface); err != nil { + n.Log.Error("failed to read routes during cleanup", "err", err) + } else { + for _, route := range routes { + if err := n.SystemRoutes.DeleteRoute(n.Interface, route); err != nil { + n.Log.Error("failed to remove route", "err", err) + } + } } - } - for _, addr := range n.AppliedSystem.Aliases { - err := RemoveAlias(n.Log, n.Interface, addr) - if err != nil { - n.Log.Error("failed to remove alias", "err", err) + if addresses, err := n.SystemRoutes.InterfaceAddresses(n.Interface); err != nil { + n.Log.Error("failed to read addresses during cleanup", "err", err) + } else { + for _, address := range addresses { + if address.Addr().IsLinkLocalUnicast() { + continue + } + if err := n.SystemRoutes.DeleteAddress(n.Interface, address.Addr()); err != nil { + n.Log.Error("failed to remove alias", "err", err) + } + } } } // run pre-down commands @@ -243,47 +249,54 @@ func (n *Nylon) SyncSystemState() error { func (n *Nylon) syncAliases() error { desired := n.GetRouter(n.LocalCfg.Id).Addresses - applied := slices.Clone(n.AppliedSystem.Aliases) + actualPrefixes, err := n.SystemRoutes.InterfaceAddresses(n.Interface) + if err != nil { + return fmt.Errorf("read interface addresses: %w", err) + } + actual := make([]netip.Addr, 0, len(actualPrefixes)) + for _, prefix := range actualPrefixes { + actual = append(actual, prefix.Addr()) + } var syncErr error // we must first add the new alias before removing the old ones, else the system might flush our routes for _, newEntry := range desired { - if !slices.Contains(applied, newEntry) { + if !slices.Contains(actual, newEntry) { n.Log.Debug("installing alias", "addr", newEntry.String()) - err := ConfigureAlias(n.Log, n.Interface, newEntry) + err := n.SystemRoutes.AddAddress(n.Interface, newEntry) if err != nil { n.Log.Error("failed to configure alias", "err", err) syncErr = errors.Join(syncErr, fmt.Errorf("install alias %s: %w", newEntry, err)) continue } - applied = append(applied, newEntry) + actual = append(actual, newEntry) } } - hadAliases := len(applied) != 0 - for _, oldEntry := range slices.Clone(applied) { + for _, oldEntry := range slices.Clone(actual) { + if oldEntry.IsLinkLocalUnicast() { + continue + } if !slices.Contains(desired, oldEntry) { n.Log.Debug("removing old alias", "addr", oldEntry.String()) - err := RemoveAlias(n.Log, n.Interface, oldEntry) + err := n.SystemRoutes.DeleteAddress(n.Interface, oldEntry) if err != nil { n.Log.Error("failed to remove alias", "err", err) syncErr = errors.Join(syncErr, fmt.Errorf("remove alias %s: %w", oldEntry, err)) continue } - applied = slices.DeleteFunc(applied, func(addr netip.Addr) bool { + actual = slices.DeleteFunc(actual, func(addr netip.Addr) bool { return addr == oldEntry }) } } - // special case for linux: if all aliases are removed, the kernel will also flush the routes - if hadAliases && len(applied) == 0 && runtime.GOOS == "linux" { - n.AppliedSystem.Routes = nil - } - n.AppliedSystem.Aliases = applied return syncErr } func (n *Nylon) syncSystemRoutes() error { newEntries := n.ComputeSysRouteTable() - applied := slices.Clone(n.AppliedSystem.Routes) + applied, err := n.SystemRoutes.InterfaceRoutes(n.Interface) + if err != nil { + return fmt.Errorf("read interface routes: %w", err) + } var syncErr error // Install new routes before removing old ones so a partial reconciliation // preserves as much connectivity as possible. @@ -291,7 +304,7 @@ func (n *Nylon) syncSystemRoutes() error { if !slices.Contains(applied, newEntry) { // install route n.Log.Debug("installing new route", "prefix", newEntry.String()) - err := ConfigureRoute(n.Log, n.Tun, n.Interface, newEntry) + err := n.SystemRoutes.AddRoute(n.Interface, newEntry) if err != nil { n.Log.Error("failed to configure route", "err", err) syncErr = errors.Join(syncErr, fmt.Errorf("install route %s: %w", newEntry, err)) @@ -304,7 +317,7 @@ func (n *Nylon) syncSystemRoutes() error { if !slices.Contains(newEntries, oldEntry) { // uninstall route n.Log.Debug("removing old route", "prefix", oldEntry.String()) - err := RemoveRoute(n.Log, n.Tun, n.Interface, oldEntry) + err := n.SystemRoutes.DeleteRoute(n.Interface, oldEntry) if err != nil { n.Log.Error("failed to remove route", "err", err) syncErr = errors.Join(syncErr, fmt.Errorf("remove route %s: %w", oldEntry, err)) @@ -315,6 +328,5 @@ func (n *Nylon) syncSystemRoutes() error { }) } } - n.AppliedSystem.Routes = applied return syncErr } diff --git a/core/sys_darwin.go b/core/sys_darwin.go index e9580568..15cc7ddf 100644 --- a/core/sys_darwin.go +++ b/core/sys_darwin.go @@ -1,9 +1,12 @@ package core import ( + "bufio" "log/slog" "net" "net/netip" + "strconv" + "strings" "github.com/encodeous/nylon/polyamide/ipc" "github.com/encodeous/nylon/polyamide/tun" @@ -23,19 +26,62 @@ func InitInterface(logger *slog.Logger, ifName string) error { return nil } -func ConfigureAlias(logger *slog.Logger, ifName string, addr netip.Addr) error { +func NewSystemRoutes(logger *slog.Logger, dev tun.Device) SystemRoutes { + return &commandSystemRoutes{logger: logger, dev: dev} +} + +func (s *commandSystemRoutes) InterfaceAddresses(ifName string) ([]netip.Prefix, error) { + out, err := ExecOutput(s.logger, "/sbin/ifconfig", ifName) + if err != nil { + return nil, err + } + var prefixes []netip.Prefix + scanner := bufio.NewScanner(strings.NewReader(string(out))) + for scanner.Scan() { + fields := strings.Fields(scanner.Text()) + if len(fields) < 2 || (fields[0] != "inet" && fields[0] != "inet6") { + continue + } + addrText := strings.Split(fields[1], "%")[0] + addr, err := netip.ParseAddr(addrText) + if err != nil { + return nil, err + } + bits := addr.BitLen() + for i, field := range fields { + if field != "netmask" || i+1 >= len(fields) { + continue + } + mask := strings.TrimPrefix(fields[i+1], "0x") + value, err := strconv.ParseUint(mask, 16, 64) + if err == nil && addr.Is4() { + bits = 0 + for value != 0 { + bits += int(value & 1) + value >>= 1 + } + } else if err == nil && addr.Is6() { + bits = len(strings.TrimRight(mask, "0")) * 4 + } + } + prefixes = append(prefixes, netip.PrefixFrom(addr, bits)) + } + return prefixes, scanner.Err() +} + +func (s *commandSystemRoutes) AddAddress(ifName string, addr netip.Addr) error { if addr.Is4() { - return Exec(logger, "/sbin/ifconfig", ifName, "alias", addr.String(), "255.255.255.255") + return Exec(s.logger, "/sbin/ifconfig", ifName, "alias", addr.String(), "255.255.255.255") } else { - return Exec(logger, "/sbin/ifconfig", ifName, "inet6", addr.String(), "alias") + return Exec(s.logger, "/sbin/ifconfig", ifName, "inet6", addr.String(), "alias") } } -func RemoveAlias(logger *slog.Logger, ifName string, addr netip.Addr) error { +func (s *commandSystemRoutes) DeleteAddress(ifName string, addr netip.Addr) error { if addr.Is4() { - return Exec(logger, "/sbin/ifconfig", ifName, "-alias", addr.String()) + return Exec(s.logger, "/sbin/ifconfig", ifName, "-alias", addr.String()) } else { - return Exec(logger, "/sbin/ifconfig", ifName, "inet6", addr.String(), "-alias") + return Exec(s.logger, "/sbin/ifconfig", ifName, "inet6", addr.String(), "-alias") } } @@ -60,22 +106,74 @@ func PrefixToMaskString(p netip.Prefix) string { return net.IP(mask).String() } -func ConfigureRoute(logger *slog.Logger, dev tun.Device, itfName string, route netip.Prefix) error { +func (s *commandSystemRoutes) InterfaceRoutes(ifName string) ([]netip.Prefix, error) { + var prefixes []netip.Prefix + for _, family := range []string{"inet", "inet6"} { + out, err := ExecOutput(s.logger, "/usr/sbin/netstat", "-rn", "-f", family) + if err != nil { + return nil, err + } + scanner := bufio.NewScanner(strings.NewReader(string(out))) + for scanner.Scan() { + fields := strings.Fields(scanner.Text()) + if len(fields) < 4 || fields[len(fields)-1] != ifName || + !strings.Contains(fields[2], "S") { + continue + } + prefix, ok := parseDarwinRoutePrefix(fields[0], family) + if ok { + prefixes = append(prefixes, prefix) + } + } + if err := scanner.Err(); err != nil { + return nil, err + } + } + return prefixes, nil +} + +func parseDarwinRoutePrefix(destination, family string) (netip.Prefix, bool) { + destination = strings.Split(destination, "%")[0] + if destination == "default" { + return netip.Prefix{}, false + } + if prefix, err := netip.ParsePrefix(destination); err == nil { + return prefix.Masked(), true + } + if addr, err := netip.ParseAddr(destination); err == nil { + return netip.PrefixFrom(addr, addr.BitLen()), true + } + if family == "inet" { + parts := strings.Split(destination, ".") + if len(parts) > 0 && len(parts) < 4 { + bits := len(parts) * 8 + for len(parts) < 4 { + parts = append(parts, "0") + } + if addr, err := netip.ParseAddr(strings.Join(parts, ".")); err == nil { + return netip.PrefixFrom(addr, bits), true + } + } + } + return netip.Prefix{}, false +} + +func (s *commandSystemRoutes) AddRoute(itfName string, route netip.Prefix) error { if route.Addr().Is6() { - return Exec(logger, "/sbin/route", "-n", "add", "-inet6", route.String(), "-interface", itfName) + return Exec(s.logger, "/sbin/route", "-n", "add", "-inet6", route.String(), "-interface", itfName) } else { addr := route.Addr() netmask := PrefixToMaskString(route) - return Exec(logger, "/sbin/route", "-n", "add", "-net", addr.String(), "-netmask", netmask, "-interface", itfName) + return Exec(s.logger, "/sbin/route", "-n", "add", "-net", addr.String(), "-netmask", netmask, "-interface", itfName) } } -func RemoveRoute(logger *slog.Logger, dev tun.Device, itfName string, route netip.Prefix) error { +func (s *commandSystemRoutes) DeleteRoute(itfName string, route netip.Prefix) error { if route.Addr().Is6() { - return Exec(logger, "/sbin/route", "-n", "delete", "-inet6", route.String(), "-interface", itfName) + return Exec(s.logger, "/sbin/route", "-n", "delete", "-inet6", route.String(), "-interface", itfName) } else { addr := route.Addr() netmask := PrefixToMaskString(route) - return Exec(logger, "/sbin/route", "-n", "delete", "-net", addr.String(), "-netmask", netmask, "-interface", itfName) + return Exec(s.logger, "/sbin/route", "-n", "delete", "-net", addr.String(), "-netmask", netmask, "-interface", itfName) } } diff --git a/core/sys_linux.go b/core/sys_linux.go index 0fe998a3..52101fce 100644 --- a/core/sys_linux.go +++ b/core/sys_linux.go @@ -1,6 +1,7 @@ package core import ( + "encoding/json" "log/slog" "net" "net/netip" @@ -28,18 +29,81 @@ func InitInterface(logger *slog.Logger, ifName string) error { return nil } -func ConfigureAlias(logger *slog.Logger, ifName string, addr netip.Addr) error { - return Exec(logger, "ip", "addr", "add", state.AddrToPrefix(addr).String(), "dev", ifName) +func NewSystemRoutes(logger *slog.Logger, dev tun.Device) SystemRoutes { + return &commandSystemRoutes{logger: logger, dev: dev} } -func RemoveAlias(logger *slog.Logger, ifName string, addr netip.Addr) error { - return Exec(logger, "ip", "addr", "del", state.AddrToPrefix(addr).String(), "dev", ifName) +func (s *commandSystemRoutes) InterfaceAddresses(ifName string) ([]netip.Prefix, error) { + out, err := ExecOutput(s.logger, "ip", "-json", "address", "show", "dev", ifName) + if err != nil { + return nil, err + } + var links []struct { + AddrInfo []struct { + Family string `json:"family"` + Local string `json:"local"` + PrefixLen int `json:"prefixlen"` + } `json:"addr_info"` + } + if err := json.Unmarshal(out, &links); err != nil { + return nil, err + } + var prefixes []netip.Prefix + for _, link := range links { + for _, address := range link.AddrInfo { + if address.Family != "inet" && address.Family != "inet6" { + continue + } + addr, err := netip.ParseAddr(address.Local) + if err != nil { + return nil, err + } + prefixes = append(prefixes, netip.PrefixFrom(addr, address.PrefixLen)) + } + } + return prefixes, nil +} + +func (s *commandSystemRoutes) AddAddress(ifName string, addr netip.Addr) error { + return Exec(s.logger, "ip", "addr", "add", state.AddrToPrefix(addr).String(), "dev", ifName) +} + +func (s *commandSystemRoutes) DeleteAddress(ifName string, addr netip.Addr) error { + return Exec(s.logger, "ip", "addr", "del", state.AddrToPrefix(addr).String(), "dev", ifName) +} + +func (s *commandSystemRoutes) InterfaceRoutes(ifName string) ([]netip.Prefix, error) { + out, err := ExecOutput(s.logger, "ip", "-json", "route", "show", "dev", ifName) + if err != nil { + return nil, err + } + var routes []struct { + Destination string `json:"dst"` + Protocol string `json:"protocol"` + } + if err := json.Unmarshal(out, &routes); err != nil { + return nil, err + } + prefixes := make([]netip.Prefix, 0, len(routes)) + for _, route := range routes { + // Kernel routes are owned by interface addresses, not Nylon's route + // reconciler. Deleting one can leave the address present but unusable. + if route.Protocol == "kernel" || route.Destination == "" || route.Destination == "default" { + continue + } + prefix, err := netip.ParsePrefix(route.Destination) + if err != nil { + return nil, err + } + prefixes = append(prefixes, prefix.Masked()) + } + return prefixes, nil } -func ConfigureRoute(logger *slog.Logger, dev tun.Device, itfName string, route netip.Prefix) error { - return Exec(logger, "ip", "route", "add", route.String(), "dev", itfName) +func (s *commandSystemRoutes) AddRoute(ifName string, route netip.Prefix) error { + return Exec(s.logger, "ip", "route", "add", route.String(), "dev", ifName) } -func RemoveRoute(logger *slog.Logger, dev tun.Device, itfName string, route netip.Prefix) error { - return Exec(logger, "ip", "route", "del", route.String(), "dev", itfName) +func (s *commandSystemRoutes) DeleteRoute(ifName string, route netip.Prefix) error { + return Exec(s.logger, "ip", "route", "del", route.String(), "dev", ifName) } diff --git a/core/sys_utils.go b/core/sys_utils.go index d216c6c0..b22d105c 100644 --- a/core/sys_utils.go +++ b/core/sys_utils.go @@ -13,10 +13,15 @@ func ExecSplit(logger *slog.Logger, command string) error { } func Exec(logger *slog.Logger, name string, arg ...string) error { + _, err := ExecOutput(logger, name, arg...) + return err +} + +func ExecOutput(logger *slog.Logger, name string, arg ...string) ([]byte, error) { out, err := exec.Command(name, arg...).CombinedOutput() logger.Debug("exec command", "cmd", name, "arg", arg, "out", string(out)) if err != nil { - return fmt.Errorf("error executing command: %s %s. %w. Output: %s", name, arg, err, out) + return nil, fmt.Errorf("error executing command: %s %s. %w. Output: %s", name, arg, err, out) } - return nil + return out, nil } diff --git a/core/sys_windows.go b/core/sys_windows.go index 81f3601f..37775e2d 100644 --- a/core/sys_windows.go +++ b/core/sys_windows.go @@ -1,6 +1,8 @@ package core import ( + "bytes" + "encoding/json" "log/slog" "net" "net/netip" @@ -28,22 +30,104 @@ func InitInterface(logger *slog.Logger, ifName string) error { return nil } -func ConfigureAlias(logger *slog.Logger, ifName string, addr netip.Addr) error { +func NewSystemRoutes(logger *slog.Logger, dev tun.Device) SystemRoutes { + return &commandSystemRoutes{logger: logger, dev: dev} +} + +type windowsAddress struct { + IPAddress string `json:"IPAddress"` + PrefixLength int `json:"PrefixLength"` +} + +type windowsRoute struct { + DestinationPrefix string `json:"DestinationPrefix"` + Protocol string `json:"Protocol"` +} + +func decodePowerShellObjects[T any](out []byte) ([]T, error) { + out = bytes.TrimSpace(out) + if len(out) == 0 { + return nil, nil + } + var values []T + if out[0] == '[' { + return values, json.Unmarshal(out, &values) + } + var value T + if err := json.Unmarshal(out, &value); err != nil { + return nil, err + } + return []T{value}, nil +} + +func powerShellQuote(value string) string { + return "'" + strings.ReplaceAll(value, "'", "''") + "'" +} + +func (s *commandSystemRoutes) InterfaceAddresses(ifName string) ([]netip.Prefix, error) { + script := "Get-NetIPAddress -InterfaceAlias " + powerShellQuote(ifName) + + " | Select-Object IPAddress,PrefixLength | ConvertTo-Json -Compress" + out, err := ExecOutput(s.logger, "powershell", "-NoProfile", "-NonInteractive", "-Command", script) + if err != nil { + return nil, err + } + addresses, err := decodePowerShellObjects[windowsAddress](out) + if err != nil { + return nil, err + } + prefixes := make([]netip.Prefix, 0, len(addresses)) + for _, address := range addresses { + addr, err := netip.ParseAddr(strings.Split(address.IPAddress, "%")[0]) + if err != nil { + return nil, err + } + prefixes = append(prefixes, netip.PrefixFrom(addr, address.PrefixLength)) + } + return prefixes, nil +} + +func (s *commandSystemRoutes) AddAddress(ifName string, addr netip.Addr) error { if addr.Is6() { - return Exec(logger, "netsh", "interface", "ipv6", "add", "address", ifName, addr.String()) + return Exec(s.logger, "netsh", "interface", "ipv6", "add", "address", ifName, addr.String()) } - return Exec(logger, "netsh", "interface", "ip", "add", "address", ifName, addr.String()) + return Exec(s.logger, "netsh", "interface", "ip", "add", "address", ifName, addr.String()) } -func RemoveAlias(logger *slog.Logger, ifName string, addr netip.Addr) error { +func (s *commandSystemRoutes) DeleteAddress(ifName string, addr netip.Addr) error { if addr.Is6() { - return Exec(logger, "netsh", "interface", "ipv6", "delete", "address", ifName, addr.String()) + return Exec(s.logger, "netsh", "interface", "ipv6", "delete", "address", ifName, addr.String()) + } + return Exec(s.logger, "netsh", "interface", "ip", "delete", "address", ifName, addr.String()) +} + +func (s *commandSystemRoutes) InterfaceRoutes(ifName string) ([]netip.Prefix, error) { + script := "Get-NetRoute -InterfaceAlias " + powerShellQuote(ifName) + + " | Select-Object DestinationPrefix,@{Name='Protocol';Expression={$_.Protocol.ToString()}}" + + " | ConvertTo-Json -Compress" + out, err := ExecOutput(s.logger, "powershell", "-NoProfile", "-NonInteractive", "-Command", script) + if err != nil { + return nil, err + } + routes, err := decodePowerShellObjects[windowsRoute](out) + if err != nil { + return nil, err + } + prefixes := make([]netip.Prefix, 0, len(routes)) + for _, route := range routes { + if route.Protocol == "Local" { + continue + } + prefix, err := netip.ParsePrefix(route.DestinationPrefix) + if err != nil { + return nil, err + } + prefixes = append(prefixes, prefix.Masked()) } - return Exec(logger, "netsh", "interface", "ip", "delete", "address", ifName, addr.String()) + return prefixes, nil } -func ConfigureRoute(logger *slog.Logger, dev tun.Device, itfName string, route netip.Prefix) error { - ifId := wintypes.LUID((dev.(*tun.NativeTun)).LUID()) +func (s *commandSystemRoutes) AddRoute(itfName string, route netip.Prefix) error { + ifId := wintypes.LUID((s.dev.(*tun.NativeTun)).LUID()) itf, err := ifId.Interface() if err != nil { return err @@ -51,17 +135,17 @@ func ConfigureRoute(logger *slog.Logger, dev tun.Device, itfName string, route n ifIndex := strconv.FormatUint(uint64(itf.InterfaceIndex), 10) if route.Addr().Is6() { - return Exec(logger, "route", "add", route.String(), "::", "IF", ifIndex) + return Exec(s.logger, "route", "add", route.String(), "::", "IF", ifIndex) } else { addr := route.Addr() _, mask, _ := net.ParseCIDR(route.String()) maskStr := net.IP(mask.Mask).String() - return Exec(logger, "route", "add", addr.String(), "mask", maskStr, "0.0.0.0", "IF", ifIndex) + return Exec(s.logger, "route", "add", addr.String(), "mask", maskStr, "0.0.0.0", "IF", ifIndex) } } -func RemoveRoute(logger *slog.Logger, dev tun.Device, itfName string, route netip.Prefix) error { - ifId := wintypes.LUID((dev.(*tun.NativeTun)).LUID()) +func (s *commandSystemRoutes) DeleteRoute(itfName string, route netip.Prefix) error { + ifId := wintypes.LUID((s.dev.(*tun.NativeTun)).LUID()) itf, err := ifId.Interface() if err != nil { return err @@ -69,11 +153,11 @@ func RemoveRoute(logger *slog.Logger, dev tun.Device, itfName string, route neti ifIndex := strconv.FormatUint(uint64(itf.InterfaceIndex), 10) if route.Addr().Is6() { - return Exec(logger, "route", "delete", route.String(), "::", "IF", ifIndex) + return Exec(s.logger, "route", "delete", route.String(), "::", "IF", ifIndex) } else { addr := route.Addr() _, mask, _ := net.ParseCIDR(route.String()) maskStr := net.IP(mask.Mask).String() - return Exec(logger, "route", "delete", addr.String(), "mask", maskStr, "0.0.0.0", "IF", ifIndex) + return Exec(s.logger, "route", "delete", addr.String(), "mask", maskStr, "0.0.0.0", "IF", ifIndex) } } diff --git a/core/system_routes.go b/core/system_routes.go new file mode 100644 index 00000000..56dc8558 --- /dev/null +++ b/core/system_routes.go @@ -0,0 +1,27 @@ +package core + +import ( + "log/slog" + "net/netip" + + "github.com/encodeous/nylon/polyamide/tun" +) + +// SystemRoutes provides an authoritative view of addresses and routes attached +// to an interface. Implementations must read the operating system on every +// Interface* call; callers use that state to recover from external changes and +// partially failed commands. +type SystemRoutes interface { + InterfaceAddresses(ifName string) ([]netip.Prefix, error) + AddAddress(ifName string, addr netip.Addr) error + DeleteAddress(ifName string, addr netip.Addr) error + + InterfaceRoutes(ifName string) ([]netip.Prefix, error) + AddRoute(ifName string, prefix netip.Prefix) error + DeleteRoute(ifName string, prefix netip.Prefix) error +} + +type commandSystemRoutes struct { + logger *slog.Logger + dev tun.Device +} diff --git a/core/system_routes_test.go b/core/system_routes_test.go new file mode 100644 index 00000000..7270d868 --- /dev/null +++ b/core/system_routes_test.go @@ -0,0 +1,112 @@ +package core + +import ( + "errors" + "io" + "log/slog" + "net/netip" + "slices" + "testing" + + "github.com/encodeous/nylon/state" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type fakeSystemRoutes struct { + addresses []netip.Prefix + routes []netip.Prefix + addrsRead error + routesRead error +} + +func (f *fakeSystemRoutes) InterfaceAddresses(string) ([]netip.Prefix, error) { + return slices.Clone(f.addresses), f.addrsRead +} + +func (f *fakeSystemRoutes) AddAddress(_ string, addr netip.Addr) error { + f.addresses = append(f.addresses, state.AddrToPrefix(addr)) + return nil +} + +func (f *fakeSystemRoutes) DeleteAddress(_ string, addr netip.Addr) error { + f.addresses = slices.DeleteFunc(f.addresses, func(prefix netip.Prefix) bool { + return prefix.Addr() == addr + }) + return nil +} + +func (f *fakeSystemRoutes) InterfaceRoutes(string) ([]netip.Prefix, error) { + return slices.Clone(f.routes), f.routesRead +} + +func (f *fakeSystemRoutes) AddRoute(_ string, prefix netip.Prefix) error { + f.routes = append(f.routes, prefix) + return nil +} + +func (f *fakeSystemRoutes) DeleteRoute(_ string, prefix netip.Prefix) error { + f.routes = slices.Delete(f.routes, slices.Index(f.routes, prefix), slices.Index(f.routes, prefix)+1) + return nil +} + +func TestSyncSystemStateReconcilesAgainstLiveState(t *testing.T) { + desiredAddress := netip.MustParseAddr("10.0.0.1") + desiredRoute := netip.MustParsePrefix("10.1.0.0/16") + externalAddress := netip.MustParsePrefix("10.0.0.2/32") + externalRoute := netip.MustParsePrefix("10.2.0.0/16") + system := &fakeSystemRoutes{ + addresses: []netip.Prefix{externalAddress}, + routes: []netip.Prefix{externalRoute}, + } + n := &Nylon{ + ConfigState: state.ConfigState{ + CentralCfg: state.CentralCfg{Routers: []state.RouterCfg{{ + NodeCfg: state.NodeCfg{Id: "a", Addresses: []netip.Addr{desiredAddress}}, + }}}, + LocalCfg: state.LocalCfg{Id: "a"}, + }, + RouterState: &state.RouterState{ + Routes: map[netip.Prefix]state.SelRoute{desiredRoute: {Nh: "b"}}, + }, + Interface: "nylon0", + SystemRoutes: system, + Log: slog.New(slog.NewTextHandler(io.Discard, nil)), + } + + require.NoError(t, n.SyncSystemState()) + assert.Equal(t, []netip.Prefix{state.AddrToPrefix(desiredAddress)}, system.addresses) + assert.Equal(t, []netip.Prefix{desiredRoute}, system.routes) + + // Simulate another process removing Nylon's state between reconciliations. + system.addresses = nil + system.routes = nil + require.NoError(t, n.SyncSystemState()) + assert.Equal(t, []netip.Prefix{state.AddrToPrefix(desiredAddress)}, system.addresses) + assert.Equal(t, []netip.Prefix{desiredRoute}, system.routes) +} + +func TestSyncSystemStateDoesNotMutateAfterReadFailure(t *testing.T) { + readErr := errors.New("route query failed") + system := &fakeSystemRoutes{routesRead: readErr} + n := &Nylon{ + ConfigState: state.ConfigState{ + CentralCfg: state.CentralCfg{Routers: []state.RouterCfg{{ + NodeCfg: state.NodeCfg{Id: "a"}, + }}}, + LocalCfg: state.LocalCfg{Id: "a"}, + }, + RouterState: &state.RouterState{ + Routes: map[netip.Prefix]state.SelRoute{ + netip.MustParsePrefix("10.1.0.0/16"): {Nh: "b"}, + }, + }, + Interface: "nylon0", + SystemRoutes: system, + Log: slog.New(slog.NewTextHandler(io.Discard, nil)), + } + + err := n.SyncSystemState() + require.ErrorIs(t, err, readErr) + assert.Empty(t, system.routes) +}