privatae-bastion/client-go/ownaddr_netns_linux_test.go

//go:build linux

package main

import (
	"bufio"
	"context"
	"errors"
	"net"
	"os"
	"os/exec"
	"strings"
	"sync/atomic"
	"syscall"
	"testing"
	"time"

	"golang.org/x/sys/unix"
)

const ownAddrNetnsEnv = "PRIVATAE_TEST_OWNADDR_NETNS"

// THE KERNEL'S OWN ANSWER, IN A REAL NETWORK NAMESPACE. A `local` route
// (AnyIP) makes the kernel deliver a whole range to this machine although no
// interface holds those addresses, so a service on 0.0.0.0 answers on every
// one of them. This runs the test binary again in a private user and network
// namespace (as `unshare -rn` does), adds `local 192.0.2.0/24` and
// `local 2001:db8:a::/64` on lo, a broadcast, two anycast and a multicast
// route, proves a 0.0.0.0 listener answers on the local ranges, and then asks
// every lane that dials for someone else: a coder run's proxy and the anchored
// egress lane (each vetted, and each end to end over its own connection with a
// real dialer, answering 403), and the backup lanes' URL check and dial wall.
// Each address, the IPv4-mapped and NAT64 forms of each IPv4 one, and a name
// for each are refused, each lane's reason read so that a name the lane could
// not resolve (refused for that) does not count; nothing reaches the
// listener, and an address or name with a unicast route or no route passes.
// The lookup itself is checked against the kernel's local, broadcast,
// anycast, multicast, unicast and no-route answers, so the unicast answer is
// read even on a machine with no default route. Skipped where a private
// namespace cannot be made.
//
// MUTATION: set ownRouteLookup to nil (the interface check alone).
// RESULT: red — the AnyIP destinations are vetted and reach the listener.
// MUTATION: in routeKindOfType, test `!= unix.RTN_LOCAL`.
// RESULT: red — the broadcast and anycast destinations pass every lane.
func TestAnyIPDestinationsAreRefusedInARealNamespace(t *testing.T) {
	if os.Getenv(ownAddrNetnsEnv) != "1" {
		if _, err := exec.LookPath("ip"); err != nil {
			if _, err := os.Stat("/usr/sbin/ip"); err != nil {
				t.Skip("needs ip(8)")
			}
		}
		cmd := exec.Command(os.Args[0], "-test.run=^TestAnyIPDestinationsAreRefusedInARealNamespace$", "-test.v", "-test.count=1")
		cmd.Env = append(os.Environ(), ownAddrNetnsEnv+"=1")
		cmd.SysProcAttr = &syscall.SysProcAttr{
			Cloneflags:  syscall.CLONE_NEWUSER | syscall.CLONE_NEWNET,
			UidMappings: []syscall.SysProcIDMap{{ContainerID: 0, HostID: os.Getuid(), Size: 1}},
			GidMappings: []syscall.SysProcIDMap{{ContainerID: 0, HostID: os.Getgid(), Size: 1}},
		}
		out, err := cmd.CombinedOutput()
		if cmd.ProcessState == nil {
			t.Skipf("no private user and network namespace here: %v", err)
		}
		t.Logf("in a private network namespace:\n%s", out)
		if strings.Contains(string(out), "--- SKIP") {
			t.Skip("the namespaced run could not set up its routes")
		}
		if err != nil {
			t.Fatalf("the namespaced run failed: %v", err)
		}
		if !strings.Contains(string(out), "--- PASS: TestAnyIPDestinationsAreRefusedInARealNamespace") {
			t.Fatal("the namespaced run did not run")
		}
		return
	}

	ipBin, err := exec.LookPath("ip")
	if err != nil {
		ipBin = "/usr/sbin/ip"
	}
	for _, args := range [][]string{
		{"link", "set", "lo", "up"},
		{"route", "add", "local", "192.0.2.0/24", "dev", "lo"},
		{"-6", "route", "add", "local", "2001:db8:a::/64", "dev", "lo"},
		// Plain unicast routes, so the kernel's unicast answer is read here
		// whatever routes the machine running the tests has (a box with no
		// default route answers every public address with no route at all).
		{"route", "add", "198.51.100.0/24", "dev", "lo"},
		{"-6", "route", "add", "2001:db8:b::/64", "dev", "lo"},
		// The other route types the kernel answers with, none of them unicast:
		// each is this machine, so a lookup that took only `local` as this
		// machine would let them through.
		{"route", "add", "broadcast", "198.51.101.255", "dev", "lo"},
		{"route", "add", "anycast", "198.51.102.0/24", "dev", "lo"},
		{"-6", "route", "add", "anycast", "2001:db8:d::/64", "dev", "lo"},
		{"route", "add", "multicast", "233.252.0.0/24", "dev", "lo"},
	} {
		if out, err := exec.Command(ipBin, args...).CombinedOutput(); err != nil {
			t.Skipf("ip %s: %v %s", strings.Join(args, " "), err, out)
		}
	}
	// The real kernel's three answers, read by the real rtnetlink lookup.
	for addr, want := range map[string]routeKind{
		"192.0.2.55": routeThisMachine, "2001:db8:a::55": routeThisMachine, "::ffff:192.0.2.55": routeThisMachine,
		"198.51.100.9": routeLeaves, "2001:db8:b::9": routeLeaves,
		"203.0.113.9": routeNowhere, "2001:db8:c::9": routeNowhere,
		"198.51.101.255": routeThisMachine, // broadcast
		"198.51.102.5":   routeThisMachine, // anycast
		"2001:db8:d::5":  routeThisMachine, // anycast
		"233.252.0.5":    routeThisMachine, // multicast
	} {
		if got, err := ownRouteLookup(net.ParseIP(addr)); err != nil || got != want {
			t.Errorf("route lookup %s: got kind %d err %v, want kind %d", addr, got, err, want)
		}
	}
	ln, err := net.Listen("tcp", ":0") // 0.0.0.0 and [::]
	if err != nil {
		t.Fatal(err)
	}
	defer ln.Close()
	var hits atomic.Int64
	go func() {
		for {
			c, err := ln.Accept()
			if err != nil {
				return
			}
			hits.Add(1)
			c.Close()
		}
	}()
	_, port, _ := net.SplitHostPort(ln.Addr().String())
	// The hazard is real here: the listener answers on both AnyIP ranges.
	for _, a := range []string{"192.0.2.55", "2001:db8:a::55"} {
		c, err := net.DialTimeout("tcp", net.JoinHostPort(a, port), 2*time.Second)
		if err != nil {
			t.Fatalf("the 0.0.0.0 listener should answer on %s: %v", a, err)
		}
		c.Close()
	}
	for deadline := time.Now().Add(2 * time.Second); hits.Load() < 2 && time.Now().Before(deadline); {
		time.Sleep(10 * time.Millisecond)
	}
	if hits.Load() != 2 {
		t.Fatalf("the listener saw %d of the 2 direct connections", hits.Load())
	}
	ownAddrReset()

	names := map[string][]string{"anyip.example": {"192.0.2.55"}, "anyip6.example": {"2001:db8:a::55"},
		"bcast.example": {"198.51.101.255"}, "anycast.example": {"198.51.102.5"}, "anycast6.example": {"2001:db8:d::5"},
		"unicast.example": {"198.51.100.9"}, "unicast6.example": {"2001:db8:b::9"},
		"nowhere.example": {"203.0.113.9"}, "nowhere6.example": {"2001:db8:c::9"}}
	stubResolve(t, names)
	stubEgressResolve(t, names)
	ctx := context.Background()
	targets := []string{
		net.JoinHostPort("192.0.2.55", port), net.JoinHostPort("2001:db8:a::55", port),
		net.JoinHostPort("::ffff:192.0.2.55", port), net.JoinHostPort("64:ff9b::c000:237", port),
		"anyip.example:" + port, "anyip6.example:" + port,
		// Broadcast and anycast routes: this machine too, by address and by name.
		net.JoinHostPort("198.51.101.255", port), net.JoinHostPort("198.51.102.5", port),
		net.JoinHostPort("2001:db8:d::5", port), net.JoinHostPort("::ffff:198.51.102.5", port),
		"bcast.example:" + port, "anycast6.example:" + port,
		// The IPv4-mapped and NAT64 forms of the broadcast and anycast addresses.
		net.JoinHostPort("::ffff:198.51.101.255", port), net.JoinHostPort("64:ff9b::c633:65ff", port),
		net.JoinHostPort("64:ff9b::c633:6605", port), "anycast.example:" + port,
	}
	// A unicast route and no route at all are both not this machine, by address
	// and by name: each passes, and the address to dial is the one vetted.
	passes := map[string]string{
		"198.51.100.9:443": "198.51.100.9:443", "203.0.113.9:443": "203.0.113.9:443",
		"[2001:db8:b::9]:443": "[2001:db8:b::9]:443", "[2001:db8:c::9]:443": "[2001:db8:c::9]:443",
		"unicast.example:443": "198.51.100.9:443", "unicast6.example:443": "[2001:db8:b::9]:443",
		"nowhere.example:443": "203.0.113.9:443", "nowhere6.example:443": "[2001:db8:c::9]:443",
	}
	// Each refusal must be the this-machine refusal. A name the lane's resolver
	// could not answer would be refused too ("does not resolve"), so a refusal
	// alone would not show the name was resolved and its address vetted.
	for _, hp := range targets {
		if got, why := coderVettedAddr(ctx, hp, nil); !strings.Contains(why, "never this machine") {
			t.Errorf("coder run: %s is this machine and must be refused as such, got dial %q, %q", hp, got, why)
		}
	}
	for hp, want := range passes {
		if got, why := coderVettedAddr(ctx, hp, nil); why != "" || got != want {
			t.Errorf("coder run: %s leaves this machine or goes nowhere, so it passes as %s: %q %q", hp, want, got, why)
		}
	}
	// The same requests, sent to each lane's socket with its real dialer.
	requests := []string{
		"CONNECT " + targets[0] + " HTTP/1.1\r\n\r\n",
		"CONNECT " + targets[1] + " HTTP/1.1\r\n\r\n",
		"CONNECT " + targets[2] + " HTTP/1.1\r\n\r\n",
		"CONNECT " + targets[4] + " HTTP/1.1\r\n\r\n",
		"CONNECT " + targets[5] + " HTTP/1.1\r\n\r\n",
		"CONNECT " + targets[6] + " HTTP/1.1\r\n\r\n",
		"CONNECT " + targets[7] + " HTTP/1.1\r\n\r\n",
		"CONNECT " + targets[10] + " HTTP/1.1\r\n\r\n",
		"GET http://" + targets[0] + "/ HTTP/1.1\r\nHost: 192.0.2.55\r\n\r\n",
		"GET http://" + targets[4] + "/ HTTP/1.1\r\nHost: anyip.example\r\n\r\n",
	}
	ask := func(c net.Conn, req string) string {
		_ = c.SetDeadline(time.Now().Add(5 * time.Second))
		_, _ = c.Write([]byte(req))
		line, _ := bufio.NewReader(c).ReadString('\n')
		c.Close()
		return line
	}

	// A coder run's proxy end to end.
	t.Setenv("XDG_CONFIG_HOME", t.TempDir())
	oldChown := coderChown
	t.Cleanup(func() { coderChown = oldChown })
	coderChown = func(string, int, int) error { return nil }
	dir := t.TempDir()
	if err := coderListenEgress("run-anyip", dir, os.Getuid()); err != nil {
		t.Fatal(err)
	}
	defer coderCloseEgress("run-anyip")
	for _, req := range requests {
		c, err := net.Dial("unix", dir+"/"+coderEgressSock)
		if err != nil {
			t.Fatal(err)
		}
		if line := ask(c, req); !strings.HasPrefix(line, "HTTP/1.1 403") {
			t.Errorf("the proxy must refuse %q, answered %q", req, line)
		}
	}

	// The anchored egress lane, in the same namespace: the vetting, then the
	// lane end to end as the anchor host, with a real dialer.
	setupBox(t)
	for _, hp := range targets {
		if addr, why := egressVettedAddr(ctx, hp); !strings.Contains(why, "that is this machine") {
			t.Errorf("anchored egress: %s is this machine and must be refused as such, got %q, %q", hp, addr, why)
		}
	}
	for hp, want := range passes {
		if addr, why := egressVettedAddr(ctx, hp); why != "" || addr != want {
			t.Errorf("anchored egress: %s leaves this machine or goes nowhere, so it passes as %s: %q %q", hp, want, addr, why)
		}
	}
	dialer := (&net.Dialer{Timeout: 2 * time.Second}).DialContext
	for _, req := range requests {
		srv, cli := net.Pipe()
		go handleEgressConn(srv, dialer, tAnchor)
		if line := ask(cli, req); !strings.HasPrefix(line, "HTTP/1.1 403") {
			t.Errorf("the anchored lane must refuse %q, answered %q", req, line)
		}
	}

	// The backup lanes' wall: refused at the URL check and at the dial.
	stubBackupLookup(t, names)
	for _, hp := range targets {
		if err := validateBackupURL("https://" + hp + "/b.tar"); err == nil || !strings.Contains(err.Error(), "on this machine or this network") {
			t.Errorf("backup: %s is this machine and must be refused as such: %v", hp, err)
		}
	}
	for hp := range passes {
		if err := validateBackupURL("https://" + hp + "/b.tar"); err != nil {
			t.Errorf("backup: %s leaves this machine or goes nowhere, so it passes: %v", hp, err)
		}
	}
	for _, a := range []string{"192.0.2.55", "2001:db8:a::55", "198.51.101.255", "198.51.102.5", "2001:db8:d::5"} {
		if backupDialAllowed(net.ParseIP(a)) {
			t.Errorf("backup: the dial to %s is this machine and must be refused", a)
		}
	}
	for _, a := range []string{"198.51.100.9", "203.0.113.9", "2001:db8:b::9"} {
		if !backupDialAllowed(net.ParseIP(a)) {
			t.Errorf("backup: %s leaves this machine or goes nowhere, so it passes", a)
		}
	}

	time.Sleep(100 * time.Millisecond)
	if n := hits.Load(); n != 2 {
		t.Fatalf("nothing the lanes were asked for may reach the listener: it saw %d connections after the 2 direct ones", n-2)
	}
}

// stubEgressResolve answers the anchored lane's name lookups from m.
func stubEgressResolve(t *testing.T, m map[string][]string) {
	t.Helper()
	old := egressResolve
	t.Cleanup(func() { egressResolve = old })
	egressResolve = func(_ context.Context, host string) ([]net.IPAddr, error) {
		ips, ok := m[strings.ToLower(host)]
		if !ok {
			return nil, errors.New("no such host")
		}
		var out []net.IPAddr
		for _, s := range ips {
			out = append(out, net.IPAddr{IP: net.ParseIP(s)})
		}
		return out, nil
	}
}

// stubBackupLookup answers the backup lanes' name lookups from m.
func stubBackupLookup(t *testing.T, m map[string][]string) {
	t.Helper()
	old := backupLookupIP
	t.Cleanup(func() { backupLookupIP = old })
	backupLookupIP = func(host string) ([]net.IP, error) {
		if ip := net.ParseIP(host); ip != nil {
			return []net.IP{ip}, nil
		}
		ips, ok := m[strings.ToLower(host)]
		if !ok {
			return nil, errors.New("no such host")
		}
		var out []net.IP
		for _, s := range ips {
			out = append(out, net.ParseIP(s))
		}
		return out, nil
	}
}

// Only a unicast route leaves this machine. Every other route type the kernel
// can answer with is this machine: a lookup that took only `local` as this
// machine would pass broadcast, anycast and multicast destinations.
//
// MUTATION: in routeKindOfType, test `!= unix.RTN_LOCAL` instead of
// `== unix.RTN_UNICAST`.
// RESULT: red — broadcast, anycast and multicast read as leaving.
func TestOnlyAUnicastRouteLeavesThisMachine(t *testing.T) {
	if got := routeKindOfType(unix.RTN_UNICAST); got != routeLeaves {
		t.Fatalf("a unicast route leaves the machine: got kind %d", got)
	}
	for name, typ := range map[string]uint8{
		"local": unix.RTN_LOCAL, "broadcast": unix.RTN_BROADCAST, "anycast": unix.RTN_ANYCAST,
		"multicast": unix.RTN_MULTICAST, "unspec": unix.RTN_UNSPEC, "nat": unix.RTN_NAT, "xresolve": unix.RTN_XRESOLVE,
	} {
		if got := routeKindOfType(typ); got != routeThisMachine {
			t.Errorf("a %s route is this machine: got kind %d", name, got)
		}
	}
}