Skip to content

core(routing): reconcile against system route state #148

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
tionis wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

core(routing): reconcile against system route state #148

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 2 additions & 3 deletions core/nylon.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@ type Nylon struct {
wgUapi net.Listener
Interface string
Device *device.Device
SystemRoutes SystemRoutes
observability *observabilityServer

// only used for debugging & tests
Expand All @@ -74,9 +75,7 @@ type Nylon struct {
}

type AppliedSystemState struct {
Routes []netip.Prefix
Aliases []netip.Addr
Peers map[state.NodeId]state.NyPublicKey
Peers map[state.NodeId]state.NyPublicKey
}

func NewNylon(ccfg state.CentralCfg, ncfg state.LocalCfg, logLevel slog.Level, configPath string, aux map[string]any, opts state.NylonOptions, tunables *state.RouterTunables) (*Nylon, error) {
Expand Down
86 changes: 49 additions & 37 deletions core/nylon_wireguard.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ import (
"errors"
"fmt"
"net/netip"
"runtime"
"slices"

"github.com/encodeous/nylon/polyamide/conn"
Expand All @@ -29,6 +28,9 @@ func (n *Nylon) initWireGuard() error {
n.Device = dev
n.Tun = tdev
n.Interface = itfName
if !n.NoTun && n.SystemRoutes == nil {
n.SystemRoutes = NewSystemRoutes(n.Log, n.Tun)
}

n.InstallTC()
n.Log.Info("installed nylon traffic control filter for polysock")
Expand Down Expand Up @@ -68,19 +70,13 @@ listen_port=%d
}

if !n.NoNetConfigure && !n.NoTun {
for _, addr := range n.GetRouter(n.LocalCfg.Id).Addresses {
err := ConfigureAlias(n.Log, itfName, addr)
if err != nil {
n.Log.Error("failed to configure alias", "err", err)
} else if !slices.Contains(n.AppliedSystem.Aliases, addr) {
n.AppliedSystem.Aliases = append(n.AppliedSystem.Aliases, addr)
}
}

err = InitInterface(n.Log, itfName)
if err != nil {
return err
}
if err := n.SyncSystemState(); err != nil {
n.Log.Warn("initial system networking reconciliation incomplete; will retry", "err", err)
}
}

// run post-up commands
Expand All @@ -103,17 +99,27 @@ listen_port=%d
}

func (n *Nylon) cleanupWireGuard() error {
// remove routes
for _, route := range n.AppliedSystem.Routes {
err := RemoveRoute(n.Log, n.Tun, n.Interface, route)
if err != nil {
n.Log.Error("failed to remove route", "err", err)
if !n.NoNetConfigure && !n.NoTun && n.SystemRoutes != nil {
if routes, err := n.SystemRoutes.InterfaceRoutes(n.Interface); err != nil {
n.Log.Error("failed to read routes during cleanup", "err", err)
} else {
for _, route := range routes {
if err := n.SystemRoutes.DeleteRoute(n.Interface, route); err != nil {
n.Log.Error("failed to remove route", "err", err)
}
}
}
}
for _, addr := range n.AppliedSystem.Aliases {
err := RemoveAlias(n.Log, n.Interface, addr)
if err != nil {
n.Log.Error("failed to remove alias", "err", err)
if addresses, err := n.SystemRoutes.InterfaceAddresses(n.Interface); err != nil {
n.Log.Error("failed to read addresses during cleanup", "err", err)
} else {
for _, address := range addresses {
if address.Addr().IsLinkLocalUnicast() {
continue
}
if err := n.SystemRoutes.DeleteAddress(n.Interface, address.Addr()); err != nil {
n.Log.Error("failed to remove alias", "err", err)
}
}
}
}
// run pre-down commands
Expand Down Expand Up @@ -243,55 +249,62 @@ func (n *Nylon) SyncSystemState() error {

func (n *Nylon) syncAliases() error {
desired := n.GetRouter(n.LocalCfg.Id).Addresses
applied := slices.Clone(n.AppliedSystem.Aliases)
actualPrefixes, err := n.SystemRoutes.InterfaceAddresses(n.Interface)
if err != nil {
return fmt.Errorf("read interface addresses: %w", err)
}
actual := make([]netip.Addr, 0, len(actualPrefixes))
for _, prefix := range actualPrefixes {
actual = append(actual, prefix.Addr())
}
var syncErr error
// we must first add the new alias before removing the old ones, else the system might flush our routes
for _, newEntry := range desired {
if !slices.Contains(applied, newEntry) {
if !slices.Contains(actual, newEntry) {
n.Log.Debug("installing alias", "addr", newEntry.String())
err := ConfigureAlias(n.Log, n.Interface, newEntry)
err := n.SystemRoutes.AddAddress(n.Interface, newEntry)
if err != nil {
n.Log.Error("failed to configure alias", "err", err)
syncErr = errors.Join(syncErr, fmt.Errorf("install alias %s: %w", newEntry, err))
continue
}
applied = append(applied, newEntry)
actual = append(actual, newEntry)
}
}
hadAliases := len(applied) != 0
for _, oldEntry := range slices.Clone(applied) {
for _, oldEntry := range slices.Clone(actual) {
if oldEntry.IsLinkLocalUnicast() {
continue
}
if !slices.Contains(desired, oldEntry) {
n.Log.Debug("removing old alias", "addr", oldEntry.String())
err := RemoveAlias(n.Log, n.Interface, oldEntry)
err := n.SystemRoutes.DeleteAddress(n.Interface, oldEntry)
if err != nil {
n.Log.Error("failed to remove alias", "err", err)
syncErr = errors.Join(syncErr, fmt.Errorf("remove alias %s: %w", oldEntry, err))
continue
}
applied = slices.DeleteFunc(applied, func(addr netip.Addr) bool {
actual = slices.DeleteFunc(actual, func(addr netip.Addr) bool {
return addr == oldEntry
})
}
}
// special case for linux: if all aliases are removed, the kernel will also flush the routes
if hadAliases && len(applied) == 0 && runtime.GOOS == "linux" {
n.AppliedSystem.Routes = nil
}
n.AppliedSystem.Aliases = applied
return syncErr
}

func (n *Nylon) syncSystemRoutes() error {
newEntries := n.ComputeSysRouteTable()
applied := slices.Clone(n.AppliedSystem.Routes)
applied, err := n.SystemRoutes.InterfaceRoutes(n.Interface)
if err != nil {
return fmt.Errorf("read interface routes: %w", err)
}
var syncErr error
// Install new routes before removing old ones so a partial reconciliation
// preserves as much connectivity as possible.
for _, newEntry := range newEntries {
if !slices.Contains(applied, newEntry) {
// install route
n.Log.Debug("installing new route", "prefix", newEntry.String())
err := ConfigureRoute(n.Log, n.Tun, n.Interface, newEntry)
err := n.SystemRoutes.AddRoute(n.Interface, newEntry)
if err != nil {
n.Log.Error("failed to configure route", "err", err)
syncErr = errors.Join(syncErr, fmt.Errorf("install route %s: %w", newEntry, err))
Expand All @@ -304,7 +317,7 @@ func (n *Nylon) syncSystemRoutes() error {
if !slices.Contains(newEntries, oldEntry) {
// uninstall route
n.Log.Debug("removing old route", "prefix", oldEntry.String())
err := RemoveRoute(n.Log, n.Tun, n.Interface, oldEntry)
err := n.SystemRoutes.DeleteRoute(n.Interface, oldEntry)
if err != nil {
n.Log.Error("failed to remove route", "err", err)
syncErr = errors.Join(syncErr, fmt.Errorf("remove route %s: %w", oldEntry, err))
Expand All @@ -315,6 +328,5 @@ func (n *Nylon) syncSystemRoutes() error {
})
}
}
n.AppliedSystem.Routes = applied
return syncErr
}
122 changes: 110 additions & 12 deletions core/sys_darwin.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,12 @@
package core

import (
"bufio"
"log/slog"
"net"
"net/netip"
"strconv"
"strings"

"github.com/encodeous/nylon/polyamide/ipc"
"github.com/encodeous/nylon/polyamide/tun"
Expand All @@ -23,19 +26,62 @@ func InitInterface(logger *slog.Logger, ifName string) error {
return nil
}

func ConfigureAlias(logger *slog.Logger, ifName string, addr netip.Addr) error {
func NewSystemRoutes(logger *slog.Logger, dev tun.Device) SystemRoutes {
return &commandSystemRoutes{logger: logger, dev: dev}
}

func (s *commandSystemRoutes) InterfaceAddresses(ifName string) ([]netip.Prefix, error) {
out, err := ExecOutput(s.logger, "/sbin/ifconfig", ifName)
if err != nil {
return nil, err
}
var prefixes []netip.Prefix
scanner := bufio.NewScanner(strings.NewReader(string(out)))
for scanner.Scan() {
fields := strings.Fields(scanner.Text())
if len(fields) < 2 || (fields[0] != "inet" && fields[0] != "inet6") {
continue
}
addrText := strings.Split(fields[1], "%")[0]
addr, err := netip.ParseAddr(addrText)
if err != nil {
return nil, err
}
bits := addr.BitLen()
for i, field := range fields {
if field != "netmask" || i+1 >= len(fields) {
continue
}
mask := strings.TrimPrefix(fields[i+1], "0x")
value, err := strconv.ParseUint(mask, 16, 64)
if err == nil && addr.Is4() {
bits = 0
for value != 0 {
bits += int(value & 1)
value >>= 1
}
} else if err == nil && addr.Is6() {
bits = len(strings.TrimRight(mask, "0")) * 4
}
}
prefixes = append(prefixes, netip.PrefixFrom(addr, bits))
}
return prefixes, scanner.Err()
}

func (s *commandSystemRoutes) AddAddress(ifName string, addr netip.Addr) error {
if addr.Is4() {
return Exec(logger, "/sbin/ifconfig", ifName, "alias", addr.String(), "255.255.255.255")
return Exec(s.logger, "/sbin/ifconfig", ifName, "alias", addr.String(), "255.255.255.255")
} else {
return Exec(logger, "/sbin/ifconfig", ifName, "inet6", addr.String(), "alias")
return Exec(s.logger, "/sbin/ifconfig", ifName, "inet6", addr.String(), "alias")
}
}

func RemoveAlias(logger *slog.Logger, ifName string, addr netip.Addr) error {
func (s *commandSystemRoutes) DeleteAddress(ifName string, addr netip.Addr) error {
if addr.Is4() {
return Exec(logger, "/sbin/ifconfig", ifName, "-alias", addr.String())
return Exec(s.logger, "/sbin/ifconfig", ifName, "-alias", addr.String())
} else {
return Exec(logger, "/sbin/ifconfig", ifName, "inet6", addr.String(), "-alias")
return Exec(s.logger, "/sbin/ifconfig", ifName, "inet6", addr.String(), "-alias")
}
}

Expand All @@ -60,22 +106,74 @@ func PrefixToMaskString(p netip.Prefix) string {
return net.IP(mask).String()
}

func ConfigureRoute(logger *slog.Logger, dev tun.Device, itfName string, route netip.Prefix) error {
func (s *commandSystemRoutes) InterfaceRoutes(ifName string) ([]netip.Prefix, error) {
var prefixes []netip.Prefix
for _, family := range []string{"inet", "inet6"} {
out, err := ExecOutput(s.logger, "/usr/sbin/netstat", "-rn", "-f", family)
if err != nil {
return nil, err
}
scanner := bufio.NewScanner(strings.NewReader(string(out)))
for scanner.Scan() {
fields := strings.Fields(scanner.Text())
if len(fields) < 4 || fields[len(fields)-1] != ifName ||
!strings.Contains(fields[2], "S") {
continue
}
prefix, ok := parseDarwinRoutePrefix(fields[0], family)
if ok {
prefixes = append(prefixes, prefix)
}
}
if err := scanner.Err(); err != nil {
return nil, err
}
}
return prefixes, nil
}

func parseDarwinRoutePrefix(destination, family string) (netip.Prefix, bool) {
destination = strings.Split(destination, "%")[0]
if destination == "default" {
return netip.Prefix{}, false
}
if prefix, err := netip.ParsePrefix(destination); err == nil {
return prefix.Masked(), true
}
if addr, err := netip.ParseAddr(destination); err == nil {
return netip.PrefixFrom(addr, addr.BitLen()), true
}
if family == "inet" {
parts := strings.Split(destination, ".")
if len(parts) > 0 && len(parts) < 4 {
bits := len(parts) * 8
for len(parts) < 4 {
parts = append(parts, "0")
}
if addr, err := netip.ParseAddr(strings.Join(parts, ".")); err == nil {
return netip.PrefixFrom(addr, bits), true
}
}
}
return netip.Prefix{}, false
}

func (s *commandSystemRoutes) AddRoute(itfName string, route netip.Prefix) error {
if route.Addr().Is6() {
return Exec(logger, "/sbin/route", "-n", "add", "-inet6", route.String(), "-interface", itfName)
return Exec(s.logger, "/sbin/route", "-n", "add", "-inet6", route.String(), "-interface", itfName)
} else {
addr := route.Addr()
netmask := PrefixToMaskString(route)
return Exec(logger, "/sbin/route", "-n", "add", "-net", addr.String(), "-netmask", netmask, "-interface", itfName)
return Exec(s.logger, "/sbin/route", "-n", "add", "-net", addr.String(), "-netmask", netmask, "-interface", itfName)
}
}

func RemoveRoute(logger *slog.Logger, dev tun.Device, itfName string, route netip.Prefix) error {
func (s *commandSystemRoutes) DeleteRoute(itfName string, route netip.Prefix) error {
if route.Addr().Is6() {
return Exec(logger, "/sbin/route", "-n", "delete", "-inet6", route.String(), "-interface", itfName)
return Exec(s.logger, "/sbin/route", "-n", "delete", "-inet6", route.String(), "-interface", itfName)
} else {
addr := route.Addr()
netmask := PrefixToMaskString(route)
return Exec(logger, "/sbin/route", "-n", "delete", "-net", addr.String(), "-netmask", netmask, "-interface", itfName)
return Exec(s.logger, "/sbin/route", "-n", "delete", "-net", addr.String(), "-netmask", netmask, "-interface", itfName)
}
}
Loading
Loading