bastion 0.55.3

client-go/egress.go

@@ -3,6 +3,7 @@ package main
 import (
 	"bufio"
 	"context"
+	"errors"
 	"io"
 	"net"
 	"strings"
@@ -63,12 +64,12 @@ func handleEgressConn(client net.Conn, dialOut egressDialer, self string) {
 	defer cancel()
 
 	if strings.EqualFold(method, "CONNECT") {
-		dialAddr, why := egressVettedAddr(ctx, ensurePort(target, "443"))
+		dialAddrs, why := egressVettedAddrs(ctx, ensurePort(target, "443"))
 		if why != "" {
 			_, _ = client.Write([]byte("HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\nX-Privatae-Refused: " + why + "\r\n\r\n"))
 			return
 		}
-		upstream, err := dialOut(ctx, "tcp", dialAddr)
+		upstream, err := dialEach(ctx, dialOut, dialAddrs)
 		if err != nil {
 			_, _ = client.Write([]byte("HTTP/1.1 502 Bad Gateway\r\nContent-Length: 0\r\n\r\n"))
 			return
@@ -94,12 +95,12 @@ func handleEgressConn(client net.Conn, dialOut egressDialer, self string) {
 	if host == "" {
 		return
 	}
-	dialAddr, why := egressVettedAddr(ctx, ensurePort(host, "80"))
+	dialAddrs, why := egressVettedAddrs(ctx, ensurePort(host, "80"))
 	if why != "" {
 		_, _ = client.Write([]byte("HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\nX-Privatae-Refused: " + why + "\r\n\r\n"))
 		return
 	}
-	upstream, err := dialOut(ctx, "tcp", dialAddr)
+	upstream, err := dialEach(ctx, dialOut, dialAddrs)
 	if err != nil {
 		_, _ = client.Write([]byte("HTTP/1.1 502 Bad Gateway\r\nContent-Length: 0\r\n\r\n"))
 		return
@@ -145,47 +146,65 @@ func splice(clientReader io.Reader, client, upstream net.Conn) {
 	_ = client.Close()
 }
 
-// egressInternalRefusal: on an ANCHORED box the egress lane reaches the public
-// internet and nothing else. Loopback, link-local and private addresses — this
-// machine's own services, the LAN it sits on, the mesh — are refused before
-// any dial, whether named by address or by a name that resolves to one. The
-// lane is USE of the machine's connection (writes.go), and that holds only
+// egressInternalRefusal is the refusal egressVettedAddrs gives, or "".
+//
+// The lane is USE of the machine's connection (writes.go), and that holds only
 // while it cannot be turned inward: `POST http://127.0.0.1:11434/api/pull`
-// down this lane would reach ollama around the forward's filter, and anything
-// on the LAN around every other gate. So is any address the machine takes as
-// its own (ownaddr.go): a public one on its NIC, or a range a `local` route
-// delivers here. A local box keeps the lane it had.
-// Resolution failures refuse too — a name this box cannot resolve is not one
+// down this lane would reach ollama around the forward's filter and around the
+// `models` switch, and this machine's other services around theirs. So on
+// every box the lane refuses this machine: `localhost` names, loopback and
+// unspecified addresses, link-local ones (the cloud metadata address
+// included), and any address the machine takes as its own (ownaddr.go), such
+// as a public one on its NIC or a range a `local` route delivers here.
+//
+// What differs is the network around it. On an ANCHORED box the lane reaches
+// the public internet and nothing else: private, CGNAT and reserved addresses
+// (the LAN, the mesh) are refused too. On a LOCAL box the LAN stays reachable:
+// the host there is the one the person paired, and their own network is
+// theirs to reach.
+//
+// Resolution failures refuse too: a name this box cannot resolve is not one
 // it can vouch for.
 //
-// THE ADDRESS THE CHECK VETTED IS THE ADDRESS DIALLED. egressVettedAddr
-// returns "ip:port" on an anchored box, and the dial takes that, never the
-// name: a name resolved once for the check and again for the dial is a
-// rebinding hole (a public answer for the check, 127.0.0.1 for the dial).
+// THE ADDRESS THE CHECK VETTED IS THE ADDRESS DIALLED. egressVettedAddrs
+// returns "ip:port" forms, and the dial takes those, never the name: a name
+// resolved once for the check and again for the dial is a rebinding hole (a
+// public answer for the check, 127.0.0.1 for the dial).
 func egressInternalRefusal(ctx context.Context, hostport string) string {
-	_, why := egressVettedAddr(ctx, hostport)
+	_, why := egressVettedAddrs(ctx, hostport)
 	return why
 }
 
-// egressResolve is how the anchored lane resolves a name. A seam so a test can
-// answer for a name without DNS.
+// egressResolve is how the lane resolves a name. A seam so a test can answer
+// for a name without DNS.
 var egressResolve = net.DefaultResolver.LookupIPAddr
 
-// egressVettedAddr: (the address to dial, "") or ("", the refusal). On a
-// local box the input is returned as it came (the OS resolves at the dial,
-// as before); on an anchored box every address the name resolves to is
-// checked and the FIRST one is what the dial gets, as "ip:port".
+// egressVettedAddr: (the first address to dial, "") or ("", the refusal).
 func egressVettedAddr(ctx context.Context, hostport string) (string, string) {
-	if currentAnchor() == nil {
-		return hostport, ""
+	all, why := egressVettedAddrs(ctx, hostport)
+	if why != "" {
+		return "", why
 	}
+	return all[0], ""
+}
+
+// egressVettedAddrs: every address the name resolved to, as "ip:port" in the
+// resolver's order, or (nil, the refusal). Any one refused address refuses the
+// name. The dial tries them in order (dialEach), so a dual-stack site still
+// answers from a machine whose IPv6 goes nowhere, as it did when the OS dialled
+// the name.
+func egressVettedAddrs(ctx context.Context, hostport string) ([]string, string) {
+	anchored := currentAnchor() != nil
 	host, port, err := net.SplitHostPort(hostport)
 	if err != nil {
 		host, port = hostport, ""
 	}
 	host = strings.Trim(strings.TrimSpace(host), "[]")
 	if strings.EqualFold(host, "localhost") || strings.HasSuffix(strings.ToLower(host), ".localhost") {
-		return "", "refused: egress to " + host + " — this lane reaches the public internet only on an anchored machine"
+		if anchored {
+			return nil, "refused: egress to " + host + " — this lane reaches the public internet only on an anchored machine"
+		}
+		return nil, "refused: egress to " + host + " — this lane never reaches this machine"
 	}
 	var addrs []net.IP
 	if ip := net.ParseIP(host); ip != nil {
@@ -193,50 +212,150 @@ func egressVettedAddr(ctx context.Context, hostport string) (string, string) {
 	} else {
 		resolved, err := egressResolve(ctx, host)
 		if err != nil || len(resolved) == 0 {
-			return "", "refused: egress to " + host + " — could not resolve it on this machine"
+			return nil, "refused: egress to " + host + " — could not resolve it on this machine"
 		}
 		for _, r := range resolved {
 			addrs = append(addrs, r.IP)
 		}
 	}
 	for _, ip := range addrs {
-		if internalIP(ip) {
-			return "", "refused: egress to " + host + " (" + ip.String() + ") — this lane reaches the public " +
+		if anchored && internalIP(ip) {
+			return nil, "refused: egress to " + host + " (" + ip.String() + ") — this lane reaches the public " +
 				"internet only on an anchored machine, never this machine, its network or the mesh"
 		}
+		if !anchored && thisMachineOrLinkIP(ip) {
+			return nil, "refused: egress to " + host + " (" + ip.String() + ") — this lane never reaches " +
+				"this machine or a link-local address"
+		}
 	}
-	// This machine is not only loopback: a public address on its NIC, or a
-	// range the kernel delivers locally through a `local` route, reaches its
-	// 0.0.0.0 services too (ownaddr.go). This lane has no allow list, so the
-	// refusal has no exception; a local box returned above and is unchanged.
+	// This machine is not only loopback: a public address on its NIC, a LAN
+	// address on it, or a range the kernel delivers locally through a `local`
+	// route, reaches its 0.0.0.0 services too, and a container's or VM's
+	// address on a network made here reaches a service on it (ownaddr.go). This
+	// lane has no allow list, so the refusal has no exception on either kind of
+	// box.
 	own, err := ownDestination(addrs)
 	if err != nil {
-		return "", "refused: egress to " + host + " — this machine could not tell whether that is itself, so it cannot be vetted"
+		return nil, "refused: egress to " + host + " — this machine could not tell whether that is itself, so it cannot be vetted"
 	}
 	if own {
-		return "", "refused: egress to " + host + " — that is this machine; this lane reaches the public " +
-			"internet only on an anchored machine, never this machine, its network or the mesh"
+		if anchored {
+			return nil, "refused: egress to " + host + " — that is this machine; this lane reaches the public " +
+				"internet only on an anchored machine, never this machine, its network or the mesh"
+		}
+		return nil, "refused: egress to " + host + " — that is this machine; this lane never reaches this machine"
 	}
-	return net.JoinHostPort(addrs[0].String(), port), ""
+	out := make([]string, 0, len(addrs))
+	for _, ip := range addrs {
+		out = append(out, net.JoinHostPort(ip.String(), port))
+	}
+	return out, ""
 }
 
-// internalIP: loopback, link-local, private (RFC 1918, ULA), CGNAT, unspecified.
-func internalIP(ip net.IP) bool {
-	if ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() ||
-		ip.IsPrivate() || ip.IsUnspecified() || ip.IsMulticast() {
-		return true
+// dialEachStagger is how long an attempt has to answer before the next
+// address is tried beside it: the connection attempt delay of Happy Eyeballs
+// (RFC 8305 recommends 250 ms), which the OS dialler used when it was given
+// the name.
+var dialEachStagger = 300 * time.Millisecond
+
+// dialEach dials vetted addresses, in order, and returns the first that
+// answers. The next address starts as soon as an attempt fails, or when the
+// attempts in flight have not answered within dialEachStagger, so an address
+// whose packets go nowhere costs a fraction of a second, not the dial budget.
+// Only these addresses are ever dialled. The attempts still in flight when one
+// answers are cancelled, and any that connects anyway is closed.
+func dialEach(ctx context.Context, dialOut egressDialer, addrs []string) (net.Conn, error) {
+	if len(addrs) == 0 {
+		return nil, errors.New("no address")
+	}
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+	type attempt struct {
+		c   net.Conn
+		err error
+	}
+	results := make(chan attempt, len(addrs))
+	next, inFlight := 0, 0
+	start := func() {
+		a := addrs[next]
+		next++
+		inFlight++
+		go func() {
+			c, err := dialOut(ctx, "tcp", a)
+			results <- attempt{c, err}
+		}()
+	}
+	stagger := time.NewTimer(dialEachStagger)
+	defer stagger.Stop()
+	start()
+	var err error
+	for inFlight > 0 {
+		var tick <-chan time.Time
+		if next < len(addrs) {
+			tick = stagger.C
+		}
+		select {
+		case r := <-results:
+			inFlight--
+			if r.err == nil {
+				go func(n int) {
+					for ; n > 0; n-- {
+						if late := <-results; late.c != nil {
+							_ = late.c.Close()
+						}
+					}
+				}(inFlight)
+				return r.c, nil
+			}
+			err = r.err
+			if next < len(addrs) {
+				start()
+				stagger.Reset(dialEachStagger)
+			}
+		case <-tick:
+			start()
+			stagger.Reset(dialEachStagger)
+		}
+	}
+	return nil, err
+}
+
+// ipClass is what an address reaches, read by every lane that dials for
+// someone else. The address rules live here once: internalIP is "anything but
+// public", and the local egress lane refuses the classes that are this machine
+// or its link.
+type ipClass int
+
+const (
+	ipPublic      ipClass = iota // the public internet
+	ipThisMachine                // loopback, unspecified, 0.0.0.0/8
+	ipLink                       // link-local unicast and multicast (the cloud metadata address)
+	ipNetwork                    // private (RFC 1918, ULA), CGNAT, multicast, benchmarking, reserved
+	ipUnreadable                 // a translated form whose IPv4 address this box cannot read
+	ipObsolete                   // IPv4-compatible ::/96, IPv4-translated ::ffff:0:0/96, site-local fec0::/10
+)
+
+func classifyIP(ip net.IP) ipClass {
+	switch {
+	case ip.IsLoopback(), ip.IsUnspecified():
+		return ipThisMachine
+	case ip.IsLinkLocalUnicast(), ip.IsLinkLocalMulticast():
+		return ipLink
+	case ip.IsPrivate(), ip.IsMulticast():
+		return ipNetwork
 	}
 	if v4 := ip.To4(); v4 != nil {
-		// 100.64.0.0/10 (carrier NAT) is as internal as 10/8 for this purpose.
-		if v4[0] == 100 && v4[1]&0xc0 == 64 {
-			return true
+		// 0.0.0.0/8 is "this host on this network".
+		if v4[0] == 0 {
+			return ipThisMachine
 		}
-		// 0.0.0.0/8 ("this network"), 198.18.0.0/15 (benchmarking) and
-		// 240.0.0.0/4 (reserved) are never a public host.
-		if v4[0] == 0 || (v4[0] == 198 && v4[1]&0xfe == 18) || v4[0] >= 240 {
-			return true
+		// 100.64.0.0/10 (carrier NAT) is as internal as 10/8 for this purpose;
+		// 198.18.0.0/15 (benchmarking) and 240.0.0.0/4 (reserved) are never a
+		// public host.
+		if (v4[0] == 100 && v4[1]&0xc0 == 64) || (v4[0] == 198 && v4[1]&0xfe == 18) || v4[0] >= 240 {
+			return ipNetwork
 		}
-		return false
+		return ipPublic
 	}
 	// IPv6 forms that carry an IPv4 address inside them reach that address, so
 	// they are judged by it: NAT64 64:ff9b::/96 (the last four bytes — how an
@@ -245,15 +364,51 @@ func internalIP(ip net.IP) bool {
 	if len(ip) == net.IPv6len {
 		if ip[0] == 0x00 && ip[1] == 0x64 && ip[2] == 0xff && ip[3] == 0x9b {
 			if ip[4]|ip[5]|ip[6]|ip[7]|ip[8]|ip[9]|ip[10]|ip[11] == 0 {
-				return internalIP(net.IPv4(ip[12], ip[13], ip[14], ip[15]))
+				return classifyIP(net.IPv4(ip[12], ip[13], ip[14], ip[15]))
 			}
 			// The rest of 64:ff9b::/32 (local-use translators, RFC 8215) maps
-			// to IPv4 in ways this box cannot read: refused.
-			return true
+			// to IPv4 in ways this box cannot read.
+			return ipUnreadable
 		}
 		if ip[0] == 0x20 && ip[1] == 0x02 {
-			return internalIP(net.IPv4(ip[2], ip[3], ip[4], ip[5]))
+			return classifyIP(net.IPv4(ip[2], ip[3], ip[4], ip[5]))
 		}
+		// Deprecated forms that are no destination of their own: IPv4-compatible
+		// ::a.b.c.d (RFC 4291 2.5.5.1), IPv4-translated ::ffff:0:a.b.c.d (SIIT)
+		// and site-local fec0::/10 (RFC 3879). Where each is delivered depends on
+		// the platform, so no lane dials them. (:: and ::1 are matched above.)
+		if isZeros(ip[0:12]) ||
+			(isZeros(ip[0:8]) && ip[8] == 0xff && ip[9] == 0xff && ip[10] == 0 && ip[11] == 0) ||
+			(ip[0] == 0xfe && ip[1]&0xc0 == 0xc0) {
+			return ipObsolete
+		}
+	}
+	return ipPublic
+}
+
+func isZeros(b []byte) bool {
+	for _, c := range b {
+		if c != 0 {
+			return false
+		}
+	}
+	return true
+}
+
+// internalIP: loopback, link-local, private (RFC 1918, ULA), CGNAT,
+// unspecified, multicast and reserved, also inside a NAT64 or 6to4 address,
+// and the deprecated IPv4-compatible, IPv4-translated and site-local forms.
+func internalIP(ip net.IP) bool {
+	return classifyIP(ip) != ipPublic
+}
+
+// thisMachineOrLinkIP: what the local egress lane refuses by address class.
+// An address it cannot read is refused, since it could be either, and so is a
+// deprecated form, which is no destination of its own.
+func thisMachineOrLinkIP(ip net.IP) bool {
+	switch classifyIP(ip) {
+	case ipThisMachine, ipLink, ipUnreadable, ipObsolete:
+		return true
 	}
 	return false
 }