bastion 0.53.2

client-go/rotate.go

@@ -0,0 +1,535 @@
+package main
+
+// Endpoint rotation: host-directed, with a candidate list and a probe.
+//
+// The platform's registry (bastion_peers + the pod-Hub identity) says where this
+// bastion should dial. The host answers the heartbeat with:
+//
+//	{"rotate_endpoint": {"endpoint": "host:port",            // the v4 port entry (always served)
+//	                     "endpoints": ["[v6addr]:51820", "host:port"],  // preferred first
+//	                     "endpoint_pubkey": "base64"}}          // the pod-Hub's key, never a relay's
+//
+// Trust: the directive rides the authenticated mTLS heartbeat RESPONSE from
+// the bastion's OWN host — the same lane self-update (update.go) and
+// containment (contain.go) already trust. This client only ever rotates to what
+// its own host tells it, and only ever pins the pod-Hub's pubkey (relay CHOICE
+// is a performance decision, not a trust decision — the relay holds no key).
+//
+// The relay uses an address-per-pairing model on its own IPv6 /64. An
+// IPv4-only network cannot reach that entry at all, and a
+// blind swap to an unreachable endpoint would strand the tunnel — the heartbeat
+// that carries the correction rides the tunnel that just died. So a candidate is
+// PROBED before it is committed: the peer is swapped, and the swap is kept only if
+// a real WireGuard handshake completes within rotateProbeTimeout. That measures the
+// path that will actually carry traffic and proves the relay forwards; a cheaper
+// echo would prove neither. A candidate that fails is remembered and not retried
+// for rotateRetryAfter, so the host repeating the directive every beat costs
+// nothing. Failover works the same way with no platform round trip: if the live
+// session's last handshake is older than WireGuard's own RejectAfterTime the path
+// is dead by the protocol's rule, and the next candidate in the list is probed.
+//
+// The swap is IN-PROCESS (device.IpcSet with replace_peers), never a process
+// exit: a desktop bastion often runs unsupervised (no systemd), so an
+// exit-to-restart rotate would silently drop the tunnel until a human
+// notices. Worker lanes (egress/mgmt/harness) ride WebSockets over the
+// netstack; their keepalive/reconnect loops absorb the peer swap.
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"golang.zx2c4.com/wireguard/device"
+)
+
+const (
+	// rotateProbeTimeout: wireguard-go retransmits an unanswered initiation every
+	// RekeyTimeout (5 s, device/constants.go). Two attempts plus the round trip is the
+	// protocol's own "this path does not answer".
+	rotateProbeTimeout = 12 * time.Second
+	// A candidate that failed its probe is skipped for rotateRetryFirst, then for
+	// rotateRetryAfter doubling on each further failure up to rotateRetryMax. The host
+	// re-sends the directive on every 60 s beat while we are not on the preferred
+	// entry, and every probe costs the tunnel up to rotateProbeTimeout — so for a
+	// bastion on an IPv4-only network (which can never reach the v6 entry) the retry
+	// has to become rare, not merely hourly.
+	//
+	// rotateRetryFirst exists because the FIRST failure is the one most likely to be
+	// transient, and going straight to an hour made a single unlucky probe expensive.
+	// Measured while pairing 20 machines at once: every v6 probe that failed
+	// during the enrolment burst succeeded on its next attempt once the fleet was idle —
+	// one host committed on its first probe after a plain service restart with nothing
+	// else changed. Under the old schedule that bastion would have sat on its fallback
+	// entry for up to SEVEN DAYS after a miss a two-minute retry would have cleared.
+	// The ceiling drops from a week to a DAY, and the number is set by what a probe
+	// costs rather than by taste: probing swaps the LIVE peer, so each attempt on a
+	// candidate that cannot answer is up to 12 s with the tunnel pointed at nothing.
+	// At a 6 h cap that is ~48 s a day, forever, for every bastion on an IPv4-only
+	// network probing a v6 entry it will never reach — the steady-state cost the
+	// original 7-day ceiling existed to avoid. 24 h keeps it at ~12 s a day while
+	// still picking up a path that comes back within the day instead of the
+	// following week.
+	rotateRetryFirst = 2 * time.Minute
+	rotateRetryAfter = time.Hour
+	rotateRetryMax   = 24 * time.Hour
+	// deadPathAfter: RejectAfterTime (180 s) is when WireGuard itself gives up on a
+	// session — but only the INITIATOR rekeys proactively, so if the pod-Hub started
+	// the current session an idle bastion can legitimately sit at 180 s until its next
+	// keepalive (25 s) triggers a new initiation (RekeyTimeout 5 s). Margin on top of
+	// that, or a live path is declared dead once an hour.
+	deadPathAfter = 240 * time.Second
+	// routeCheckTimeout bounds the dial in endpointRoutable. Every candidate is a
+	// literal today — the host builds both entries from addresses, and wireguard-go's
+	// UAPI parses an endpoint with netip.ParseAddrPort, which rejects a name outright
+	// (conn/bind_std.go) — so the dial is a pure routing-table lookup, microseconds,
+	// and this timeout never engages. It is here for the case where a name reaches
+	// this path anyway: bounding it means one skipped beat rather than a stalled
+	// heartbeat, and such a candidate could not be committed regardless, because
+	// swapPeer would fail on the same parse. Measured on a v4-only box: ~50µs for a
+	// literal, ~25ms for a name that does not resolve.
+	routeCheckTimeout = 2 * time.Second
+)
+
+// errNoLocalPath marks a candidate this machine's own routing table cannot reach.
+// It is deliberately distinct from a probe failure: a probe failure means "this
+// path did not answer", which may be transient and earns the backoff in
+// retryAfter; errNoLocalPath means "this machine has no way to even try", which is
+// a property of the local network and costs nothing to re-test.
+var errNoLocalPath = errors.New("no path to it from this machine")
+
+// dialForRouteCheck is net.DialTimeout, indirected so a test can answer without a
+// network. Nothing else in this package may replace it.
+var dialForRouteCheck = net.DialTimeout
+
+// The live-device steps of a probe (swap the peer, wait for its handshake, read the
+// last one), indirected so a test can drive a probe through to a commit or a
+// failover without a WireGuard peer. Nothing else in this package may replace them.
+var (
+	livePeerSwap      = swapPeer
+	liveHandshake     = handshakeWithin
+	liveLastHandshake = func(dev *device.Device) time.Time { return lastHandshake(dev) }
+)
+
+// endpointRoutable asks THIS MACHINE's kernel whether it can select a route and a
+// source address for endpoint. Connecting a UDP socket sends no packet — it is a
+// route lookup and a bind — so the answer is free, and it is the kernel's answer
+// rather than an inference from the address's shape.
+//
+// It exists because probing is not free. probeAndCommit swaps the LIVE peer and
+// waits up to rotateProbeTimeout for a handshake, so a candidate on an address
+// family this machine does not have costs a real tunnel outage to rule out, and
+// then costs it again on every retry. Measured on a real desktop:
+// three pairings, three `no handshake within 12s` lines against the relay's
+// per-pairing IPv6 entries, and no IPv6 path on that network at all — so each
+// restart spent up to 36s with live tunnels pointed at addresses the kernel would
+// have rejected instantly. The relay itself was healthy throughout (ten other
+// bastions were committed on those same v6 entries at the time), which is exactly
+// why the bastion cannot infer this from the candidate failing: the endpoint is
+// fine, the local network is not.
+//
+// This is a PRE-FILTER, never a verdict. A route existing does not mean packets
+// arrive, so a candidate that passes still gets the full handshake probe. Only the
+// case where the kernel says it has nowhere to send is skipped, and it is skipped
+// WITHOUT being marked failed, so a machine that gains IPv6 picks the preferred
+// entry up on the next beat instead of waiting out a backoff.
+func endpointRoutable(endpoint string) error {
+	c, err := dialForRouteCheck("udp", endpoint, routeCheckTimeout)
+	if err != nil {
+		return fmt.Errorf("%w (%v)", errNoLocalPath, err)
+	}
+	return c.Close()
+}
+
+// probeFailure is the memory of one probe target that did not answer: when, and
+// how many times in a row — the retry interval doubles with the count.
+type probeFailure struct {
+	at    time.Time
+	count int
+}
+
+// probeTarget is what a probe actually tests: an entry AND the Hub key pinned for
+// it. A host can move its machines to a new Hub key on the same entries, so one
+// entry is probed under two keys, and the two answers are independent. Remembering
+// failures per entry alone let a dead spell under the old key hold up the move to
+// the new one for up to rotateRetryMax, and let a probe of a key the Hub did not
+// answer yet keep the machine off its preferred entry under the key it runs.
+func probeTarget(endpoint, pubkey string) string {
+	return endpoint + " " + pubkey
+}
+
+// rotateState remembers probe failures per probe target (probeTarget). The entry is
+// per pairing (each pairing has its own v6 address), so every pairing keeps its own
+// memory.
+type rotateState struct {
+	mu     sync.Mutex
+	failed map[string]probeFailure
+	// noPath remembers which endpoints have already been reported as unreachable
+	// from this machine, so a PERMANENT local condition (an IPv4-only network
+	// against a v6 entry) is stated once rather than on every 60s beat. It is not
+	// a backoff: the check itself still runs every beat, because it is free and
+	// because a machine that gains IPv6 should move the same minute.
+	noPath map[string]bool
+	// mu also guards Config.savePending: whether a running copy's committed move
+	// has not reached disk (setUnsaved). That state is the run's own, not the
+	// host's: a run started again for the host loads the file and runs on what
+	// it holds, so nothing the run before it left unsaved is pending for it.
+}
+
+var rotation = rotateState{failed: map[string]probeFailure{}}
+
+// retryAfter is the skip interval for a candidate that has failed `count` times in a
+// row: the first failure gets the short grace, every one after that starts at
+// rotateRetryAfter and doubles, capped at rotateRetryMax.
+func retryAfter(count int) time.Duration {
+	if count <= 1 {
+		return rotateRetryFirst
+	}
+	d := rotateRetryAfter
+	for i := 2; i < count && d < rotateRetryMax; i++ {
+		d *= 2
+	}
+	if d > rotateRetryMax {
+		d = rotateRetryMax
+	}
+	return d
+}
+
+func (s *rotateState) failedRecently(target string) bool {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	f, ok := s.failed[target]
+	return ok && time.Since(f.at) < retryAfter(f.count)
+}
+
+func (s *rotateState) markFailed(target string) {
+	s.mu.Lock()
+	f := s.failed[target]
+	s.failed[target] = probeFailure{at: time.Now(), count: f.count + 1}
+	s.mu.Unlock()
+}
+
+func (s *rotateState) clear(target string) {
+	s.mu.Lock()
+	delete(s.failed, target)
+	s.mu.Unlock()
+}
+
+// notePathless records that ep is unreachable from here and reports whether this
+// is the FIRST time — which is the only time it is worth a log line. The map is
+// created here rather than in a constructor because rotateState is also built
+// literally (tests, and the package var below) with only `failed` set.
+func (s *rotateState) notePathless(ep string) bool {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if s.noPath == nil {
+		s.noPath = map[string]bool{}
+	}
+	if s.noPath[ep] {
+		return false
+	}
+	s.noPath[ep] = true
+	return true
+}
+
+// clearPathless forgets that ep was ever unreachable, so if the local network
+// loses the path again it is reported again instead of staying silent.
+func (s *rotateState) clearPathless(ep string) {
+	s.mu.Lock()
+	delete(s.noPath, ep)
+	s.mu.Unlock()
+}
+
+// setUnsaved records whether the running copy cfg runs on a committed move its
+// file does not hold yet. The heartbeat reports it (endpoint_save_pending).
+func (s *rotateState) setUnsaved(cfg *Config, pending bool) {
+	s.mu.Lock()
+	cfg.savePending = pending
+	s.mu.Unlock()
+}
+
+func (s *rotateState) isUnsaved(cfg *Config) bool {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return cfg.savePending
+}
+
+// persistEndpoints writes the entry and Hub key this pairing now runs on. A write
+// that fails is retried on every beat (retryEndpointSave) until one lands: until
+// then a restart reads the OLD entry and key from disk, and once the host retires
+// that key the machine is stranded, because the heartbeat that would correct it
+// rides the tunnel that no longer comes up. A pairing whose file is gone was
+// unpaired, one whose file holds another WireGuard key was paired again, and a
+// stopped run's copy writes nothing (errRunStopped): in each case there is
+// nothing of this copy's to save and nothing to retry. The pending state is
+// this copy's own, so none of this touches a run started again for the host.
+func persistEndpoints(cfg *Config) error {
+	err := cfg.saveEndpoints()
+	rotation.setUnsaved(cfg, err != nil && !errors.Is(err, errPairingGone) &&
+		!errors.Is(err, errPairingReplaced) && !errors.Is(err, errRunStopped))
+	return err
+}
+
+// retryEndpointSave saves a committed move whose earlier save failed. Called on
+// every beat; a no-op unless a save is pending.
+func retryEndpointSave(cfg *Config) {
+	if !rotation.isUnsaved(cfg) {
+		return
+	}
+	if err := persistEndpoints(cfg); err == nil {
+		fmt.Printf("rotate: endpoint %s and its Hub key saved on a later beat\n", cfg.Endpoint)
+	}
+}
+
+// candidateList normalises a host-supplied candidate list: trimmed, de-duplicated,
+// order kept, and the single "endpoint" appended if the list omits it (an older
+// host sends no list at all — then the list is just that one entry).
+func candidateList(endpoints []string, endpoint string) []string {
+	var out []string
+	seen := map[string]bool{}
+	for _, e := range append(append([]string{}, endpoints...), endpoint) {
+		e = strings.TrimSpace(e)
+		if e == "" || seen[e] {
+			continue
+		}
+		seen[e] = true
+		out = append(out, e)
+	}
+	return out
+}
+
+// errPairingReplaced is saveEndpoints' answer when the file on disk is a newer
+// pairing with the same host: `pair --replace`, or an unpair and a new pair,
+// wrote it with its own WireGuard key, while this running copy still holds the
+// one it replaced. The entry and Hub key this copy runs on were pinned for that
+// old key, and the new pairing would come up on them at its next start.
+var errPairingReplaced = errors.New("pairing file holds a newer pairing with this host — not writing the old one's entry and Hub key into it")
+
+// saveEndpoints persists ONLY the endpoint fields of a running pairing. The
+// heartbeat goroutine's Config was loaded at start; writing the whole struct
+// would put its stale State back over a pending/revoked flip the mgmt lane or
+// claim-anchor --rotate made since. It writes nothing into a file that holds a
+// different WireGuard key (errPairingReplaced): only `pair` sets that key.
+func (c *Config) saveEndpoints() error {
+	return c.updateOnDiskIf(func(d *Config) error {
+		if d.WGPrivateKey != c.WGPrivateKey {
+			return errPairingReplaced
+		}
+		d.Endpoint, d.EndpointPubKey, d.Endpoints = c.Endpoint, c.EndpointPubKey, c.Endpoints
+		return nil
+	})
+}
+
+// parseRotateTarget extracts the rotate directive: the ordered candidates and the
+// pod-Hub pubkey, or an empty list.
+func parseRotateTarget(heartbeatResponse []byte) (candidates []string, pubkey string) {
+	var hb struct {
+		Rotate struct {
+			Endpoint       string   `json:"endpoint"`
+			Endpoints      []string `json:"endpoints"`
+			EndpointPubKey string   `json:"endpoint_pubkey"`
+		} `json:"rotate_endpoint"`
+	}
+	if json.Unmarshal(heartbeatResponse, &hb) != nil {
+		return nil, ""
+	}
+	return candidateList(hb.Rotate.Endpoints, hb.Rotate.Endpoint), strings.TrimSpace(hb.Rotate.EndpointPubKey)
+}
+
+// maybeRotateEndpoint applies a pending endpoint rotation: walk the candidates in
+// preference order down to the entry we are on, probe each one we have not recently
+// proved dead, and commit the first that handshakes. No-op when we are already on the
+// preferred entry. Failure leaves BOTH the device and the config on the old endpoint
+// (consistent), logs, and the host will ask again on a later beat.
+func maybeRotateEndpoint(heartbeatResponse []byte, cfg *Config, dev *device.Device) {
+	candidates, pubkey := parseRotateTarget(heartbeatResponse)
+	if len(candidates) == 0 || pubkey == "" {
+		return
+	}
+	if candidates[0] == cfg.Endpoint && pubkey == cfg.EndpointPubKey {
+		// Already on the preferred entry; keep the list current for failover.
+		if !sameList(cfg.Endpoints, candidates) {
+			cfg.Endpoints = candidates
+			_ = persistEndpoints(cfg)
+		}
+		return
+	}
+	for _, cand := range candidates {
+		if cand == cfg.Endpoint && pubkey == cfg.EndpointPubKey {
+			// Every entry above the current one failed its probe (or was skipped as
+			// recently failed): we are on the best reachable entry.
+			if !sameList(cfg.Endpoints, candidates) {
+				cfg.Endpoints = candidates
+				_ = persistEndpoints(cfg)
+			}
+			return
+		}
+		if rotation.failedRecently(probeTarget(cand, pubkey)) {
+			continue
+		}
+		if err := probeAndCommit(cfg, dev, cand, pubkey, candidates); err != nil {
+			// A candidate this machine cannot route to is a standing fact about the
+			// local network, not an event: say it once and stay quiet, or an
+			// IPv4-only box prints the same line every minute forever.
+			if errors.Is(err, errNoLocalPath) {
+				if rotation.notePathless(cand) {
+					fmt.Printf("rotate: candidate %s: %v — skipping it until that changes (staying on %s)\n", cand, err, cfg.Endpoint)
+				}
+				continue
+			}
+			fmt.Printf("rotate: candidate %s: %v (staying on %s)\n", cand, err, cfg.Endpoint)
+			continue
+		}
+		return
+	}
+}
+
+// maybeFailover is the platform-free half: called on every beat, it moves to the next
+// reachable candidate when the live session's path is dead by WireGuard's own rule.
+// It never runs while the tunnel has a handshake younger than deadPathAfter.
+//
+// It is also where a committed move whose save failed is saved: this runs on every
+// beat whether or not the host answers, and a save needs only the disk.
+func maybeFailover(cfg *Config, dev *device.Device, startedAt time.Time) {
+	retryEndpointSave(cfg)
+	if len(cfg.Endpoints) < 2 {
+		return
+	}
+	lh := liveLastHandshake(dev)
+	if lh.IsZero() {
+		// Never handshaked since this process started: give the boot endpoint the
+		// same grace before declaring it dead.
+		if time.Since(startedAt) < deadPathAfter {
+			return
+		}
+	} else if time.Since(lh) < deadPathAfter {
+		return
+	}
+	fmt.Printf("rotate: no handshake on %s for %s — path dead, trying the other candidates\n",
+		cfg.Endpoint, deadPathAfter)
+	dead := cfg.Endpoint
+	rotation.markFailed(probeTarget(dead, cfg.EndpointPubKey))
+	for _, cand := range cfg.Endpoints {
+		if cand == dead {
+			continue
+		}
+		// A recent probe failure does not exempt a candidate here: the alternative is
+		// a tunnel that stays dead.
+		if err := probeAndCommit(cfg, dev, cand, cfg.EndpointPubKey, cfg.Endpoints); err != nil {
+			fmt.Printf("rotate: failover candidate %s: %v\n", cand, err)
+			continue
+		}
+		return
+	}
+	// Nothing answered; restore the original so a path that comes back is used.
+	_ = livePeerSwap(cfg, dev, dead, cfg.EndpointPubKey)
+}
+
+// probeAndCommit swaps the live peer to endpoint, waits for a handshake to prove the
+// path, and either persists the move or swaps back.
+func probeAndCommit(cfg *Config, dev *device.Device, endpoint, pubkey string, list []string) error {
+	// BEFORE the live peer moves: does this machine have anywhere to send? The
+	// check is here rather than in the two callers so neither can forget it, and
+	// so the live tunnel is never spent proving something the routing table
+	// already knows. A candidate refused here is NOT marked failed (see
+	// errNoLocalPath).
+	if err := endpointRoutable(endpoint); err != nil {
+		return err
+	}
+	rotation.clearPathless(endpoint)
+	t0 := time.Now()
+	target := probeTarget(endpoint, pubkey)
+	if err := livePeerSwap(cfg, dev, endpoint, pubkey); err != nil {
+		rotation.markFailed(target)
+		return fmt.Errorf("live peer swap failed: %w", err)
+	}
+	if !liveHandshake(dev, t0, rotateProbeTimeout) {
+		rotation.markFailed(target)
+		if err := livePeerSwap(cfg, dev, cfg.Endpoint, cfg.EndpointPubKey); err != nil {
+			return fmt.Errorf("no handshake within %s AND swap back failed: %w", rotateProbeTimeout, err)
+		}
+		return fmt.Errorf("no handshake within %s", rotateProbeTimeout)
+	}
+	old := cfg.Endpoint
+	cfg.Endpoint = endpoint
+	cfg.EndpointPubKey = pubkey
+	cfg.Endpoints = list
+	rotation.clear(target)
+	if err := persistEndpoints(cfg); err != nil {
+		again := ""
+		if rotation.isUnsaved(cfg) {
+			again = " (retried every beat)"
+		}
+		fmt.Printf("rotate: swapped live to %s but config save failed: %v%s\n", endpoint, err, again)
+		return nil
+	}
+	fmt.Printf("rotate: endpoint %s -> %s applied (handshake-confirmed, config saved)\n", old, endpoint)
+	return nil
+}
+
+// swapPeer replaces the device's single peer in one UAPI transaction — same shape as
+// the boot-time config in run.go. AllowedIPs stays the host's tunnel alias ONLY.
+func swapPeer(cfg *Config, dev *device.Device, endpoint, pubkey string) error {
+	privHex, err := b64ToHex(cfg.WGPrivateKey)
+	if err != nil {
+		return fmt.Errorf("bad private key in config: %w", err)
+	}
+	pubHex, err := b64ToHex(pubkey)
+	if err != nil {
+		return fmt.Errorf("bad endpoint pubkey: %w", err)
+	}
+	hostCIDR, err := hostTunnelAllowedIP(cfg.HostTunnelIP)
+	if err != nil {
+		return err
+	}
+	uapi := fmt.Sprintf("private_key=%s\nreplace_peers=true\npublic_key=%s\nendpoint=%s\npersistent_keepalive_interval=25\nallowed_ip=%s\n",
+		privHex, pubHex, endpoint, hostCIDR)
+	return dev.IpcSet(uapi)
+}
+
+// handshakeWithin polls the device until the peer reports a handshake newer than t0.
+func handshakeWithin(dev *device.Device, t0 time.Time, d time.Duration) bool {
+	deadline := time.Now().Add(d)
+	for time.Now().Before(deadline) {
+		if lh := lastHandshake(dev); !lh.IsZero() && !lh.Before(t0.Truncate(time.Second)) {
+			return true
+		}
+		time.Sleep(250 * time.Millisecond)
+	}
+	return false
+}
+
+// lastHandshake reads the (single) peer's last_handshake_time_sec from the UAPI dump;
+// zero when there is no peer or it never handshaked.
+func lastHandshake(dev interface{ IpcGet() (string, error) }) time.Time {
+	dump, err := dev.IpcGet()
+	if err != nil {
+		return time.Time{}
+	}
+	for _, line := range strings.Split(dump, "\n") {
+		if v, ok := strings.CutPrefix(line, "last_handshake_time_sec="); ok {
+			secs, err := strconv.ParseInt(strings.TrimSpace(v), 10, 64)
+			if err != nil || secs == 0 {
+				return time.Time{}
+			}
+			return time.Unix(secs, 0)
+		}
+	}
+	return time.Time{}
+}
+
+func sameList(a, b []string) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for i := range a {
+		if a[i] != b[i] {
+			return false
+		}
+	}
+	return true
+}