feat(core): multi-link routing

core/ipc_handler.go

@@ -223,14 +223,20 @@ func buildEndpoints(neigh *state.Neighbour) []*protocol.EndpointInfo {
 		if ap, err := nep.DynEP.Get(); err == nil {
 			resolved = new(ap.String())
 		}
+		bindSource := ""
+		if nep.Bind.Source.IsValid() {
+			bindSource = nep.Bind.Source.String()
+		}
 		eps = append(eps, &protocol.EndpointInfo{
-			Address:         nep.DynEP.Value,
-			Resolved:        resolved,
-			Active:          ep.IsActive(),
-			RemoteInit:      ep.IsRemote(),
-			Metric:          ep.Metric(),
-			FilteredRttNs:   int64(nep.FilteredPing()),
-			StabilizedRttNs: int64(nep.StabilizedPing()),
+			Address:            nep.DynEP.Value,
+			Resolved:           resolved,
+			Active:             ep.IsActive(),
+			RemoteInit:         ep.IsRemote(),
+			Metric:             ep.Metric(),
+			FilteredRttNs:      int64(nep.FilteredPing()),
+			StabilizedRttNs:    int64(nep.StabilizedPing()),
+			LocalBindInterface: nep.Bind.Interface,
+			LocalBindSource:    bindSource,
 		})
 	}
 	slices.SortFunc(eps, func(a, b *protocol.EndpointInfo) int {

core/nylon_apply.go

@@ -70,7 +70,7 @@ func (n *Nylon) reconcileRouterState(next *state.CentralCfg) error {
 			continue
 		}
 		// configure existing neighbours
-		reconcileConfiguredEndpoints(neigh, cfg.Endpoints, &n.RouterTunables)
+		reconcileConfiguredEndpoints(neigh, configuredEndpoints(n.LocalCfg.Binds, cfg.Endpoints, &n.RouterTunables))
 		neighs = append(neighs, neigh)
 		delete(desired, neigh.Id)
 	}
@@ -88,9 +88,7 @@ func (n *Nylon) reconcileRouterState(next *state.CentralCfg) error {
 			Routes: make(map[netip.Prefix]state.NeighRoute),
 			Eps:    make([]state.Endpoint, 0, len(cfg.Endpoints)),
 		}
-		for _, ep := range cfg.Endpoints {
-			stNeigh.Eps = append(stNeigh.Eps, state.NewEndpoint(ep, false, nil, &n.RouterTunables))
-		}
+		stNeigh.Eps = append(stNeigh.Eps, configuredEndpoints(n.LocalCfg.Binds, cfg.Endpoints, &n.RouterTunables)...)
 		neighs = append(neighs, stNeigh)
 	}
 	n.RouterState.Neighbours = neighs
@@ -107,31 +105,83 @@ func (n *Nylon) reconcileRouterState(next *state.CentralCfg) error {
 	return nil
 }
 
-func reconcileConfiguredEndpoints(neigh *state.Neighbour, desired []*state.DynamicEndpoint, t *state.RouterTunables) {
-	desiredByValue := make(map[string]*state.DynamicEndpoint, len(desired))
+func configuredEndpoints(binds []state.LocalBind, endpoints []*state.DynamicEndpoint, t *state.RouterTunables) []state.Endpoint {
+	if len(binds) == 0 {
+		eps := make([]state.Endpoint, 0, len(endpoints))
+		for _, ep := range endpoints {
+			eps = append(eps, state.NewEndpoint(ep, false, nil, t))
+		}
+		return eps
+	}
+
+	eps := make([]state.Endpoint, 0, len(endpoints)*len(binds))
+	for _, ep := range endpoints {
+		for _, bind := range binds {
+			if !bindMatchesEndpoint(bind, ep) {
+				continue
+			}
+			nep := state.NewEndpoint(ep, false, nil, t)
+			nep.Bind = bind
+			eps = append(eps, nep)
+		}
+	}
+	return eps
+}
+
+func bindMatchesEndpoint(bind state.LocalBind, ep *state.DynamicEndpoint) bool {
+	if !bind.Source.IsValid() {
+		return true
+	}
+	host, _, err := ep.Parse()
+	if err != nil {
+		return false
+	}
+	addr, err := netip.ParseAddr(host)
+	if err != nil {
+		return true
+	}
+	return state.SameIPFamily(bind.Source, addr)
+}
+
+type endpointIdentity struct {
+	value string
+	bind  state.LocalBind
+}
+
+func endpointKey(ep *state.NylonEndpoint) endpointIdentity {
+	return endpointIdentity{
+		value: ep.DynEP.Value,
+		bind:  ep.Bind,
+	}
+}
+
+func reconcileConfiguredEndpoints(neigh *state.Neighbour, desired []state.Endpoint) {
+	desiredByKey := make(map[endpointIdentity]state.Endpoint, len(desired))
 	for _, ep := range desired {
-		desiredByValue[ep.Value] = ep
+		desiredByKey[endpointKey(ep.AsNylonEndpoint())] = ep
 	}
 
 	eps := make([]state.Endpoint, 0, len(neigh.Eps)+len(desired))
-	seen := make(map[string]struct{}, len(desired))
+	seen := make(map[endpointIdentity]struct{}, len(desired))
 	for _, ep := range neigh.Eps {
 		nep := ep.AsNylonEndpoint()
 		if ep.IsRemote() {
 			eps = append(eps, ep)
 			continue
 		}
 		// only keep if desired
-		if desiredEp, ok := desiredByValue[nep.DynEP.Value]; ok {
+		key := endpointKey(nep)
+		if _, ok := desiredByKey[key]; ok {
 			eps = append(eps, ep)
-			seen[desiredEp.Value] = struct{}{}
+			seen[key] = struct{}{}
 		}
 	}
 	for _, ep := range desired {
-		if _, ok := seen[ep.Value]; ok {
+		key := endpointKey(ep.AsNylonEndpoint())
+		if _, ok := seen[key]; ok {
 			continue
 		}
-		eps = append(eps, state.NewEndpoint(ep, false, nil, t))
+		eps = append(eps, ep)
 	}
 	neigh.Eps = eps
 }

core/nylon_apply_test.go

@@ -134,3 +134,32 @@ func testCentralConfig(id state.NodeId, prefixes ...state.PrefixHealthWrapper) s
 		},
 	}
 }
+
+func TestConfiguredEndpointsExpandsBindsAcrossMatchingEndpointFamilies(t *testing.T) {
+	tunables := state.DefaultRouterTunables()
+	eps := configuredEndpoints([]state.LocalBind{
+		{Source: netip.MustParseAddr("192.0.2.10")},
+		{Source: netip.MustParseAddr("2001:db8::10")},
+	}, []*state.DynamicEndpoint{
+		state.NewDynamicEndpoint("198.51.100.10:57175"),
+		state.NewDynamicEndpoint("[2001:db8::20]:57175"),
+	}, &tunables)
+
+	assert.Len(t, eps, 2)
+	assert.Equal(t, netip.MustParseAddr("192.0.2.10"), eps[0].AsNylonEndpoint().Bind.Source)
+	assert.Equal(t, "198.51.100.10:57175", eps[0].AsNylonEndpoint().DynEP.Value)
+	assert.Equal(t, netip.MustParseAddr("2001:db8::10"), eps[1].AsNylonEndpoint().Bind.Source)
+	assert.Equal(t, "[2001:db8::20]:57175", eps[1].AsNylonEndpoint().DynEP.Value)
+}
+
+func TestConfiguredEndpointsUsesDefaultEndpointWhenNoBindsConfigured(t *testing.T) {
+	tunables := state.DefaultRouterTunables()
+	eps := configuredEndpoints(nil, []*state.DynamicEndpoint{
+		state.NewDynamicEndpoint("198.51.100.10:57175"),
+		state.NewDynamicEndpoint("[2001:db8::20]:57175"),
+	}, &tunables)
+
+	assert.Len(t, eps, 2)
+	assert.False(t, eps[0].AsNylonEndpoint().Bind.Source.IsValid())
+	assert.False(t, eps[1].AsNylonEndpoint().Bind.Source.IsValid())
+}

core/nylon_endpoints.go

@@ -17,6 +17,7 @@ import (
 type EpPing struct {
 	TimeSent time.Time
 	Peer     state.NodeId
+	Endpoint *state.NylonEndpoint
 	Complete func(protocol.EndpointProbeStatus, time.Duration)
 }
 
@@ -92,6 +93,7 @@ func (n *Nylon) sendEndpointProbe(node state.NodeId, ep *state.NylonEndpoint, ti
 	n.PingBuf.Set(token, EpPing{
 		TimeSent: sentAt,
 		Peer:     node,
+		Endpoint: ep,
 		Complete: func(status protocol.EndpointProbeStatus, latency time.Duration) {
 			if timeoutTimer != nil {
 				timeoutTimer.Stop()
@@ -197,32 +199,24 @@ func handleProbePong(n *Nylon, node state.NodeId, token uint64, ep conn.Endpoint
 		n.Log.Warn("probe came back from wrong peer", "expected", health.Peer, "actual", node)
 		return
 	}
-	receivedAt := time.Now()
-	latency := receivedAt.Sub(health.TimeSent)
+	latency := time.Since(health.TimeSent)
 	health.Complete(protocol.EndpointProbeStatus_ENDPOINT_PROBE_REPLIED, latency)
 
-	// check if link exists
-	for _, neigh := range n.RouterState.Neighbours {
-		for _, dep := range neigh.Eps {
-			dpLink := dep.AsNylonEndpoint()
-			ap, err := dpLink.DynEP.Get()
-			if err == nil && ap == ep.DstIPPort() && neigh.Id == node {
-				// we have a link
-				if n.DBG_log_probe {
-					n.Log.Debug("probe back", "peer", node, "ping", latency)
-				}
-				dpLink.Renew()
-				dpLink.UpdatePing(latency)
+	dpLink := health.Endpoint
+	if dpLink == nil {
+		n.Log.Warn("probe came back without a recorded endpoint", "from", ep.DstToString(), "node", node)
+		return
+	}
+	if n.DBG_log_probe {
+		n.Log.Debug("probe back", "peer", node, "ping", latency)
+	}
+	dpLink.Renew()
+	dpLink.UpdatePing(latency)
 
-				// update wireguard endpoint
-				dpLink.WgEndpoint = ep
+	// Record the reply's current WireGuard endpoint on the endpoint that sent the probe.
+	dpLink.WgEndpoint = ep
 
-				ComputeRoutes(n.RouterState, n)
-				return
-			}
-		}
-	}
-	n.Log.Warn("probe came back and couldn't find link", "from", ep.DstToString(), "node", node)
+	ComputeRoutes(n.RouterState, n)
 }
 
 func (n *Nylon) probeLinks(active bool) error {

docs/reference/config.mdx

@@ -22,6 +22,14 @@ log_path: "" # write logs to this file (empty = stderr only)
 interface_name: "" # override the interface name (default: "nylon", or utunX on macOS)
 dns_resolvers: [] # DNS servers for nylon's own lookups, e.g. ["1.1.1.1:53"]
 
+# Local endpoint selectors. Nylon probes each peer endpoint through matching
+# binds and uses the best active link for the routing edge.
+binds:
+  - interface: eth0
+    source: 192.0.2.10
+  - interface: eth0
+    source: 2001:db8::10
+
 # Bootstrap: fetch central.yaml from a remote bundle on first start
 dist:
   url: https://static.example.com/network1.nybundle

docs/reference/endpoint-local-binds.mdx

@@ -0,0 +1,77 @@
+# Endpoint Local Binds
+
+Nylon keeps routing decisions separate from transport endpoint selection.
+Routing chooses which node to visit next. Endpoint selection chooses which
+underlay address, source address, and local interface are used for the UDP
+packet sent to that next node.
+
+## Model
+
+At the routing level, a neighbour is represented once. Multiple underlay
+endpoints for the same neighbour are link-level candidates for the same routing
+edge, and Nylon surfaces the best active endpoint metric to the router.
+
+At the link level, a Nylon endpoint may carry an optional local bind selector:
+
+- source address: the local address to use for the outgoing UDP packet
+- interface index: the local interface to constrain the outgoing UDP packet to
+
+The source address and endpoint address must be in the same IP family when a
+source address is configured. If no local bind selector is configured, endpoint
+traffic uses the kernel's normal source address and route selection.
+
+Users configure local bind selectors in `node.yaml`:
+
+```yaml
+binds:
+  - interface: eth0
+    source: 192.0.2.10
+  - interface: eth0
+    source: 2001:db8::10
+```
+
+For each peer endpoint, Nylon creates link-level candidates for the configured
+binds whose source address family matches the remote endpoint. Interface-only
+binds are applied to all endpoint families. If `binds` is empty, Nylon keeps the
+default behaviour and creates one unbound candidate per endpoint.
+
+This design does not create one socket per local bind. A Nylon device owns one
+IPv4 UDP socket and one IPv6 UDP socket when both address families are
+available. Local bind selectors are attached to endpoint sends.
+
+## Linux Transport
+
+On Linux, endpoint-local binds are encoded as per-message ancillary data:
+
+- IPv4 uses `IP_PKTINFO` with `in_pktinfo.ipi_spec_dst` and
+  `in_pktinfo.ipi_ifindex`.
+- IPv6 uses `IPV6_PKTINFO` with `in6_pktinfo.ipi6_addr` and
+  `in6_pktinfo.ipi6_ifindex`.
+
+When both a source address and interface index are provided, Nylon sends both in
+the packet info control message. The source address remains the requested source
+address; the interface index constrains output interface selection. When only an
+interface is provided, the packet info source address is left unspecified and
+the kernel may select an address for that interface.
+
+This keeps probe packets and other control traffic precise without changing the
+socket-level bind. It also allows different endpoints for the same peer to use
+different local bind selectors while still sharing the device sockets.
+
+## Non-Linux Platforms
+
+The endpoint-local bind mechanism is currently implemented only for Linux.
+Other platforms keep the default socket behaviour. A cross-platform user-facing
+configuration should only expose selectors that the current platform can honor,
+or should fail validation before the device starts.
+
+## Routing Behaviour
+
+Endpoint-local binds affect only the link-level endpoint used to send packets.
+They do not create additional routing-level neighbours or independent routed
+edges. Nylon still computes routes over nodes and uses the best active endpoint
+metric for each neighbour.
+
+Bandwidth aggregation, ECMP, and max-flow style forwarding are outside this
+model. Supporting those would require different routing semantics rather than
+additional endpoint-local bind selectors.

polyamide/conn/bind_std.go

@@ -374,12 +374,12 @@ func (s *StdNetBind) Send(bufs [][]byte, endpoint Endpoint) error {
 	defer s.udpAddrPool.Put(ua)
 	if is6 {
 		as16 := endpoint.DstIP().As16()
-		copy(ua.IP, as16[:])
 		ua.IP = ua.IP[:16]
+		copy(ua.IP, as16[:])
 	} else {
 		as4 := endpoint.DstIP().As4()
-		copy(ua.IP, as4[:])
 		ua.IP = ua.IP[:4]
+		copy(ua.IP, as4[:])
 	}
 	ua.Port = int(endpoint.(*StdNetEndpoint).Port())
 	var (
@@ -526,6 +526,7 @@ func splitCoalescedMessages(msgs []ipv6.Message, firstMsgAt int, getGSO getGSOFu
 		if err != nil {
 			return n, err
 		}
+		control := append([]byte(nil), msg.OOB[:msg.NN]...)
 		if gsoSize > 0 {
 			numToSplit = (msg.N + gsoSize - 1) / gsoSize
 			end = gsoSize
@@ -536,6 +537,8 @@ func splitCoalescedMessages(msgs []ipv6.Message, firstMsgAt int, getGSO getGSOFu
 			}
 			copied := copy(msgs[n].Buffers[0], msg.Buffers[0][start:end])
 			msgs[n].N = copied
+			msgs[n].OOB = append(msgs[n].OOB[:0], control...)
+			msgs[n].NN = len(control)
 			msgs[n].Addr = msg.Addr
 			start = end
 			end += gsoSize

polyamide/conn/bind_std_test.go

@@ -248,3 +248,27 @@ func Test_splitCoalescedMessages(t *testing.T) {
 		})
 	}
 }
+
+func Test_splitCoalescedMessagesPreservesControl(t *testing.T) {
+	msgs := []ipv6.Message{
+		{Buffers: [][]byte{make([]byte, 1<<16-1)}, OOB: make([]byte, 0, 2)},
+		{Buffers: [][]byte{make([]byte, 1<<16-1)}, OOB: make([]byte, 0, 2)},
+		{Buffers: [][]byte{make([]byte, 1<<16-1)}, N: 2, NN: 2, OOB: []byte{1, 0}},
+		{Buffers: [][]byte{make([]byte, 1<<16-1)}, OOB: make([]byte, 0, 2)},
+	}
+	got, err := splitCoalescedMessages(msgs, 2, mockGetGSOSize)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	if got != 2 {
+		t.Fatalf("got to eval: %d want: %d", got, 2)
+	}
+	for i := 0; i < got; i++ {
+		if msgs[i].NN != 2 {
+			t.Fatalf("msg[%d].NN: %d want: 2", i, msgs[i].NN)
+		}
+		if gotGSO, err := mockGetGSOSize(msgs[i].OOB[:msgs[i].NN]); err != nil || gotGSO != 1 {
+			t.Fatalf("msg[%d] gso: %d err: %v", i, gotGSO, err)
+		}
+	}
+}

polyamide/conn/sticky_default.go

@@ -21,6 +21,8 @@ func (e *StdNetEndpoint) SrcToString() string {
 	return ""
 }
 
+func (e *StdNetEndpoint) SetSrc(addr netip.Addr, ifidx int32) {}
+
 // TODO: macOS, FreeBSD and other BSDs likely do support the sticky sockets
 // {get,set}srcControl feature set, but use alternatively named flags and need
 // ports and require testing.