headscale/hscontrol/servertest/routes_test.go

package servertest_test

import (
	"context"
	"net/netip"
	"slices"
	"testing"
	"time"

	"github.com/juanfont/headscale/hscontrol/servertest"
	"github.com/juanfont/headscale/hscontrol/types"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"tailscale.com/tailcfg"
	"tailscale.com/types/netmap"
)

// TestRoutes verifies that route advertisements and approvals
// propagate correctly through the control plane to all peers.
//
//nolint:gocyclo // table-driven test driver with many independent subtests
func TestRoutes(t *testing.T) {
	t.Parallel()

	t.Run("node_addresses_in_allowed_ips", func(t *testing.T) {
		t.Parallel()
		h := servertest.NewHarness(t, 2)

		// Each peer's AllowedIPs should contain the peer's addresses.
		for _, c := range h.Clients() {
			nm := c.Netmap()
			require.NotNil(t, nm)

			for _, peer := range nm.Peers {
				addrs := make(map[netip.Prefix]bool)
				for i := range peer.Addresses().Len() {
					addrs[peer.Addresses().At(i)] = true
				}

				for i := range peer.AllowedIPs().Len() {
					aip := peer.AllowedIPs().At(i)
					if addrs[aip] {
						delete(addrs, aip)
					}
				}

				assert.Empty(t, addrs,
					"client %s: peer %d AllowedIPs should contain all of Addresses",
					c.Name, peer.ID())
			}
		}
	})

	t.Run("advertised_routes_in_hostinfo", func(t *testing.T) {
		t.Parallel()

		srv := servertest.NewServer(t)
		user := srv.CreateUser(t, "advroute-user")

		routePrefix := netip.MustParsePrefix("192.168.1.0/24")

		c1 := servertest.NewClient(t, srv, "advroute-node1",
			servertest.WithUser(user))
		c2 := servertest.NewClient(t, srv, "advroute-node2",
			servertest.WithUser(user))

		c1.WaitForPeers(t, 1, 10*time.Second)

		// Update hostinfo with advertised routes.
		c1.Direct().SetHostinfo(&tailcfg.Hostinfo{
			BackendLogID: "servertest-advroute-node1",
			Hostname:     "advroute-node1",
			RoutableIPs:  []netip.Prefix{routePrefix},
		})

		// Send a non-streaming update to push the new [tailcfg.Hostinfo].
		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
		defer cancel()

		_ = c1.Direct().SendUpdate(ctx)

		// The observer should eventually see the advertised routes
		// in the peer's [tailcfg.Hostinfo].
		c2.WaitForCondition(t, "advertised route in hostinfo",
			15*time.Second,
			func(nm *netmap.NetworkMap) bool {
				for _, p := range nm.Peers {
					hi := p.Hostinfo()
					if hi.Valid() && hi.Hostname() == "advroute-node1" {
						for i := range hi.RoutableIPs().Len() {
							if hi.RoutableIPs().At(i) == routePrefix {
								return true
							}
						}
					}
				}

				return false
			})
	})

	t.Run("route_advertise_and_approve", func(t *testing.T) {
		t.Parallel()

		srv := servertest.NewServer(t)
		user := srv.CreateUser(t, "fullrt-user")

		route := netip.MustParsePrefix("10.0.0.0/24")

		c1 := servertest.NewClient(t, srv, "fullrt-advertiser",
			servertest.WithUser(user))
		c2 := servertest.NewClient(t, srv, "fullrt-observer",
			servertest.WithUser(user))

		c1.WaitForPeers(t, 1, 10*time.Second)
		c2.WaitForPeers(t, 1, 10*time.Second)

		// Step 1: Advertise the route by updating [tailcfg.Hostinfo].
		c1.Direct().SetHostinfo(&tailcfg.Hostinfo{
			BackendLogID: "servertest-fullrt-advertiser",
			Hostname:     "fullrt-advertiser",
			RoutableIPs:  []netip.Prefix{route},
		})

		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
		defer cancel()

		_ = c1.Direct().SendUpdate(ctx)

		// Wait for the server to process the [tailcfg.Hostinfo] update
		// by waiting for observer to see the advertised route.
		c2.WaitForCondition(t, "hostinfo update propagated",
			10*time.Second,
			func(nm *netmap.NetworkMap) bool {
				for _, p := range nm.Peers {
					hi := p.Hostinfo()
					if hi.Valid() && hi.Hostname() == "fullrt-advertiser" {
						return hi.RoutableIPs().Len() > 0
					}
				}

				return false
			})

		// Step 2: Approve the route on the server.
		nodeID := findNodeID(t, srv, "fullrt-advertiser")

		_, routeChange, err := srv.State().SetApprovedRoutes(
			nodeID, []netip.Prefix{route})
		require.NoError(t, err)
		srv.App.Change(routeChange)

		// Step 3: Observer should see the route in AllowedIPs.
		c2.WaitForCondition(t, "approved route in AllowedIPs",
			15*time.Second,
			func(nm *netmap.NetworkMap) bool {
				for _, p := range nm.Peers {
					hi := p.Hostinfo()
					if hi.Valid() && hi.Hostname() == "fullrt-advertiser" {
						for i := range p.AllowedIPs().Len() {
							if p.AllowedIPs().At(i) == route {
								return true
							}
						}
					}
				}

				return false
			})
	})

	t.Run("allowed_ips_superset_of_addresses", func(t *testing.T) {
		t.Parallel()
		h := servertest.NewHarness(t, 3)

		for _, c := range h.Clients() {
			nm := c.Netmap()
			require.NotNil(t, nm)

			for _, peer := range nm.Peers {
				allowedSet := make(map[netip.Prefix]bool)
				for i := range peer.AllowedIPs().Len() {
					allowedSet[peer.AllowedIPs().At(i)] = true
				}

				for i := range peer.Addresses().Len() {
					addr := peer.Addresses().At(i)
					assert.True(t, allowedSet[addr],
						"client %s: peer %d Address %v should be in AllowedIPs",
						c.Name, peer.ID(), addr)
				}
			}
		}
	})

	t.Run("addresses_are_in_cgnat_range", func(t *testing.T) {
		t.Parallel()
		h := servertest.NewHarness(t, 2)

		cgnat := netip.MustParsePrefix("100.64.0.0/10")
		ula := netip.MustParsePrefix("fd7a:115c:a1e0::/48")

		for _, c := range h.Clients() {
			nm := c.Netmap()
			require.NotNil(t, nm)
			require.True(t, nm.SelfNode.Valid())

			for i := range nm.SelfNode.Addresses().Len() {
				addr := nm.SelfNode.Addresses().At(i)
				inCGNAT := cgnat.Contains(addr.Addr())
				inULA := ula.Contains(addr.Addr())
				assert.True(t, inCGNAT || inULA,
					"client %s: address %v should be in CGNAT or ULA range",
					c.Name, addr)
			}
		}
	})

	// Reproduces the HA secondary recovery race:
	// HA tracking loses the secondary subnet router after all routers serving
	// the route have been offline simultaneously and one of them returns.
	//
	// Two assertions split the failure surface:
	//   R1 — server-side primary route state restores after reconnect.
	//   R2 — observer's [netmap.NetworkMap] shows the reconnected router online with
	//        the route in its primary set.
	// If R1 fails the bug is in [state.State.Connect] / primaryRoutes; if R1 passes
	// and R2 fails the bug is in change broadcast / [mapper.Batcher].
	//
	// Caveat: [TestClient.Reconnect] re-registers via [controlclient.Direct.TryLogin] in addition
	// to starting a new poll session. Production reconnects after a brief
	// network outage may bypass re-registration. If this test passes on
	// main, fall back to the integration variant noted in the plan
	// (TestHASubnetRouterFailover with all routers offline simultaneously).
	t.Run("ha_secondary_recovers_after_all_offline", func(t *testing.T) {
		t.Parallel()

		srv := servertest.NewServer(t)
		user := srv.CreateUser(t, "ha3203-user")

		route := netip.MustParsePrefix("10.0.0.0/24")

		r1 := servertest.NewClient(t, srv, "ha3203-router1",
			servertest.WithUser(user))
		r2 := servertest.NewClient(t, srv, "ha3203-router2",
			servertest.WithUser(user))
		obs := servertest.NewClient(t, srv, "ha3203-observer",
			servertest.WithUser(user))

		obs.WaitForPeers(t, 2, 10*time.Second)

		// Both routers advertise the same route via their [tailcfg.Hostinfo].
		advertise := func(c *servertest.TestClient, name string) {
			t.Helper()
			c.Direct().SetHostinfo(&tailcfg.Hostinfo{
				BackendLogID: "servertest-" + name,
				Hostname:     name,
				RoutableIPs:  []netip.Prefix{route},
			})

			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
			defer cancel()

			_ = c.Direct().SendUpdate(ctx)
		}
		advertise(r1, "ha3203-router1")
		advertise(r2, "ha3203-router2")

		// Approve the route on both routers explicitly. Auto-approvers
		// would also work but introduce a policy dependency the harness
		// does not currently set up here.
		approve := func(name string) {
			t.Helper()
			id := findNodeID(t, srv, name)

			_, ch, err := srv.State().SetApprovedRoutes(id, []netip.Prefix{route})
			require.NoError(t, err)
			srv.App.Change(ch)
		}
		approve("ha3203-router1")
		approve("ha3203-router2")

		// Sanity: r1 starts as primary (lower NodeID by registration order).
		r1ID := findNodeID(t, srv, "ha3203-router1")
		r2ID := findNodeID(t, srv, "ha3203-router2")

		hasRoute := func(id types.NodeID) bool {
			return slices.Contains(srv.State().GetNodePrimaryRoutes(id), route)
		}

		assert.Eventually(t, func() bool { return hasRoute(r1ID) },
			10*time.Second, 100*time.Millisecond,
			"r1 should be primary initially")

		// 1. Take r1 offline. After the 10s grace period, r2 should take over.
		r1.Disconnect(t)
		assert.Eventually(t, func() bool { return hasRoute(r2ID) && !hasRoute(r1ID) },
			20*time.Second, 200*time.Millisecond,
			"r2 should take over as primary after r1 offline")

		// 2. Take r2 offline. With both routers gone, no primary should remain.
		r2.Disconnect(t)
		assert.Eventually(t, func() bool { return !hasRoute(r1ID) && !hasRoute(r2ID) },
			20*time.Second, 200*time.Millisecond,
			"no primary should be assigned while both routers are offline")

		// 3. Reconnect r2 (cable plugged back in).
		r2.Reconnect(t)

		// [tailcfg.Hostinfo] is part of the [controlclient.Direct] state; the [TestClient.Reconnect]
		// helper re-registers via [controlclient.Direct.TryLogin] which carries the same [tailcfg.Hostinfo]
		// that was set above. Push it again to be sure the announced route
		// is registered in the new session.
		advertise(r2, "ha3203-router2")

		// R1: server-side state must restore r2 as primary.
		assert.Eventually(t, func() bool { return hasRoute(r2ID) },
			15*time.Second, 200*time.Millisecond,
			"R1: r2 should be re-registered as primary after reconnect — issue #3203")

		// R2: observer must see r2 online with the route in its primary set.
		obs.WaitForCondition(t, "R2: observer sees r2 online with primary route",
			15*time.Second,
			func(nm *netmap.NetworkMap) bool {
				for _, p := range nm.Peers {
					hi := p.Hostinfo()
					if !hi.Valid() || hi.Hostname() != "ha3203-router2" {
						continue
					}

					online, known := p.Online().GetOk()
					if !known || !online {
						return false
					}

					for i := range p.PrimaryRoutes().Len() {
						if p.PrimaryRoutes().At(i) == route {
							return true
						}
					}
				}

				return false
			})
	})
}

// [findNodeID] is defined in issues_test.go.