Privatae
326 lines
16 KiB
Go
326 lines
16 KiB
Go
//go:build linux
|
|
|
|
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/binary"
|
|
"errors"
|
|
"net"
|
|
"os"
|
|
"os/exec"
|
|
"strings"
|
|
"syscall"
|
|
"testing"
|
|
|
|
"golang.org/x/sys/unix"
|
|
)
|
|
|
|
// The rule alone, over link tables written out here: which links keep a
|
|
// packet on this machine.
|
|
func TestInsideMachineReadsTheLinkAndABridgesPorts(t *testing.T) {
|
|
links := map[uint32]linkInfo{
|
|
1: {kind: ""}, // lo
|
|
2: {kind: ""}, // a network card
|
|
3: {kind: "bridge"}, // docker0
|
|
4: {kind: "veth", master: 3}, // a container's link on docker0
|
|
5: {kind: "veth"}, // a pod's link, routed to directly (Calico)
|
|
6: {kind: "bridge"}, // virbr0
|
|
7: {kind: "dummy", master: 6}, // libvirt's placeholder port
|
|
8: {kind: "tun", master: 6}, // a VM's tap
|
|
9: {kind: "bridge"}, // br0 on a server: the LAN
|
|
10: {kind: "", master: 9}, // its network card
|
|
11: {kind: "veth", master: 9}, // and a container bridged onto the LAN
|
|
12: {kind: "bridge"}, // a Docker network with no container yet
|
|
13: {kind: "wireguard"}, // a VPN
|
|
14: {kind: "vxlan"}, // a tunnel to other machines (flannel.1)
|
|
15: {kind: "bridge"}, // a bridge over a VLAN of the card
|
|
16: {kind: "vlan", master: 15}, //
|
|
17: {kind: "bridge"}, // a bridge with a bridge as its port
|
|
18: {kind: "bridge", master: 17}, //
|
|
19: {kind: "tun"}, // a VPN's tun
|
|
20: {kind: "macvlan"}, // a macvlan over the card
|
|
}
|
|
for oif, want := range map[uint32]bool{
|
|
1: false, 2: false, 3: true, 4: true, 5: true, 6: true, 9: false, 12: true,
|
|
13: false, 14: false, 15: false, 17: false, 19: false, 20: false,
|
|
99: false, // not in the table: linkInsideMachine refuses it before asking
|
|
} {
|
|
if got := insideMachine(links, oif); got != want {
|
|
t.Errorf("link %d (%q): inside this machine = %v, want %v", oif, links[oif].kind, got, want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// A link dump's answers, written out here: the kind and master of each link,
|
|
// NLMSG_DONE ends it, and a dump that ended on an error (the status NLMSG_DONE
|
|
// carries) or was interrupted is not taken as the whole table.
|
|
//
|
|
// MUTATION: return done at NLMSG_DONE without reading its status.
|
|
// RESULT: red — a dump that stopped on EMSGSIZE, missing its later links, is
|
|
// taken as the table.
|
|
func TestReadLinkDumpReadsKindsMastersAndTheDumpsStatus(t *testing.T) {
|
|
ne := binary.NativeEndian
|
|
attr := func(typ uint16, val []byte) []byte {
|
|
b := make([]byte, (unix.SizeofRtAttr+len(val)+3)&^3)
|
|
ne.PutUint16(b[0:], uint16(unix.SizeofRtAttr+len(val)))
|
|
ne.PutUint16(b[2:], typ)
|
|
copy(b[unix.SizeofRtAttr:], val)
|
|
return b
|
|
}
|
|
u32 := func(v uint32) []byte { b := make([]byte, 4); ne.PutUint32(b, v); return b }
|
|
link := func(seq, idx, master uint32, kind string, flags uint16) syscall.NetlinkMessage {
|
|
d := make([]byte, unix.SizeofIfInfomsg)
|
|
ne.PutUint32(d[4:], idx)
|
|
if kind != "" {
|
|
d = append(d, attr(unix.IFLA_LINKINFO, attr(unix.IFLA_INFO_KIND, append([]byte(kind), 0)))...)
|
|
}
|
|
if master != 0 {
|
|
d = append(d, attr(unix.IFLA_MASTER, u32(master))...)
|
|
}
|
|
return syscall.NetlinkMessage{Header: syscall.NlMsghdr{Type: unix.RTM_NEWLINK, Flags: flags, Seq: seq}, Data: d}
|
|
}
|
|
doneWith := func(seq uint32, errno syscall.Errno) syscall.NetlinkMessage {
|
|
return syscall.NetlinkMessage{Header: syscall.NlMsghdr{Type: unix.NLMSG_DONE, Seq: seq}, Data: u32(uint32(-int32(errno)))}
|
|
}
|
|
|
|
links := map[uint32]linkInfo{}
|
|
done, intr, err := readLinkDump([]syscall.NetlinkMessage{
|
|
link(7, 3, 0, "bridge", 0), link(7, 4, 3, "veth", 0), link(7, 2, 0, "", 0),
|
|
link(8, 9, 0, "bridge", 0), // another request's answer
|
|
doneWith(7, 0),
|
|
}, 7, links)
|
|
if err != nil || !done || intr {
|
|
t.Fatalf("a whole dump: done %v interrupted %v err %v", done, intr, err)
|
|
}
|
|
want := map[uint32]linkInfo{3: {kind: "bridge"}, 4: {kind: "veth", master: 3}, 2: {}}
|
|
if len(links) != len(want) {
|
|
t.Fatalf("links %v, want %v", links, want)
|
|
}
|
|
for i, l := range want {
|
|
if links[i] != l {
|
|
t.Errorf("link %d: %+v, want %+v", i, links[i], l)
|
|
}
|
|
}
|
|
|
|
if done, _, err := readLinkDump([]syscall.NetlinkMessage{link(7, 3, 0, "bridge", 0), doneWith(7, unix.EMSGSIZE)},
|
|
7, map[uint32]linkInfo{}); err == nil || done {
|
|
t.Errorf("a dump that ended on EMSGSIZE is an error, got done %v err %v", done, err)
|
|
}
|
|
if _, intr, err := readLinkDump([]syscall.NetlinkMessage{link(7, 3, 0, "bridge", unix.NLM_F_MULTI|unix.NLM_F_DUMP_INTR)},
|
|
7, map[uint32]linkInfo{}); err != nil || !intr {
|
|
t.Errorf("an answer marked interrupted marks the dump, got %v %v", intr, err)
|
|
}
|
|
if done, intr, err := readLinkDump([]syscall.NetlinkMessage{link(7, 3, 0, "bridge", 0)}, 7, map[uint32]linkInfo{}); done || intr || err != nil {
|
|
t.Errorf("a batch without NLMSG_DONE is not the end: %v %v %v", done, intr, err)
|
|
}
|
|
errMsg := doneWith(7, unix.EPERM)
|
|
errMsg.Header.Type = unix.NLMSG_ERROR
|
|
if _, _, err := readLinkDump([]syscall.NetlinkMessage{errMsg}, 7, map[uint32]linkInfo{}); !errors.Is(err, unix.EPERM) {
|
|
t.Errorf("an NLMSG_ERROR in the dump is its error, got %v", err)
|
|
}
|
|
}
|
|
|
|
const insideNetnsEnv = "PRIVATAE_TEST_INSIDE_NETNS"
|
|
|
|
// THE KERNEL'S OWN ANSWER FOR NETWORKS INSIDE THIS MACHINE. The local egress
|
|
// lane keeps the person's LAN, but a container or VM on a network made on
|
|
// this machine is a service on this machine, reached around its switches the
|
|
// same way loopback is. In a private user and network namespace this builds
|
|
// the networks Docker, libvirt and Calico make (a bridge whose only port is a
|
|
// veth, a bridge whose ports are a dummy and nothing else, a bridge with no
|
|
// port yet, a veth routed to directly, and the same with an IPv6 range and a
|
|
// public range), and links that leave the machine: a bridge over a VLAN,
|
|
// which stands for a bridge over the network card; a route straight over a
|
|
// dummy, which stands for the card itself; veths that carry this machine's
|
|
// main-table default routes, as a container's own veth does; and routes
|
|
// through a gateway. It asks the real rtnetlink lookup, the local egress lane
|
|
// on an unanchored box, a coder run's proxy and the backup lanes' wall.
|
|
//
|
|
// MUTATION: return routeLeaves for every unicast route (drop the link check).
|
|
// RESULT: red — every inside address passes the local lane, the public one
|
|
// passes the coder lane.
|
|
// MUTATION: count "dummy" as an outward port.
|
|
// RESULT: red — the libvirt-style bridge is the LAN.
|
|
// MUTATION: drop the main-table default-route links from the judgement.
|
|
// RESULT: red — from a machine that is itself a container, the network on its
|
|
// own veths (IPv4, a multipath default's second veth, IPv6) is this machine.
|
|
// MUTATION: skip a default route's RTA_MULTIPATH nexthops.
|
|
// RESULT: red — both veths of the multipath default are inside.
|
|
// MUTATION: count a default route in any table, not only main.
|
|
// RESULT: red — the veth whose default is in table 100 leaves.
|
|
// MUTATION: judge the out link of a route through a gateway too.
|
|
// RESULT: red — from a machine that is itself a container, the internet
|
|
// (IPv4 and IPv6) and a LAN behind a router VM are this machine, and every
|
|
// lane refuses them.
|
|
func TestNetworksInsideThisMachineAreThisMachine(t *testing.T) {
|
|
if os.Getenv(insideNetnsEnv) != "1" {
|
|
if _, err := exec.LookPath("ip"); err != nil {
|
|
if _, err := os.Stat("/usr/sbin/ip"); err != nil {
|
|
t.Skip("needs ip(8)")
|
|
}
|
|
}
|
|
cmd := exec.Command(os.Args[0], "-test.run=^TestNetworksInsideThisMachineAreThisMachine$", "-test.v", "-test.count=1")
|
|
cmd.Env = append(os.Environ(), insideNetnsEnv+"=1")
|
|
cmd.SysProcAttr = &syscall.SysProcAttr{
|
|
Cloneflags: syscall.CLONE_NEWUSER | syscall.CLONE_NEWNET,
|
|
UidMappings: []syscall.SysProcIDMap{{ContainerID: 0, HostID: os.Getuid(), Size: 1}},
|
|
GidMappings: []syscall.SysProcIDMap{{ContainerID: 0, HostID: os.Getgid(), Size: 1}},
|
|
}
|
|
out, err := cmd.CombinedOutput()
|
|
if cmd.ProcessState == nil {
|
|
t.Skipf("no private user and network namespace here: %v", err)
|
|
}
|
|
t.Logf("in a private network namespace:\n%s", out)
|
|
if strings.Contains(string(out), "--- SKIP") {
|
|
t.Skip("the namespaced run could not build its networks")
|
|
}
|
|
if err != nil {
|
|
t.Fatalf("the namespaced run failed: %v", err)
|
|
}
|
|
if !strings.Contains(string(out), "--- PASS: TestNetworksInsideThisMachineAreThisMachine") {
|
|
t.Fatal("the namespaced run did not run")
|
|
}
|
|
return
|
|
}
|
|
|
|
ipBin, err := exec.LookPath("ip")
|
|
if err != nil {
|
|
ipBin = "/usr/sbin/ip"
|
|
}
|
|
for _, args := range [][]string{
|
|
{"link", "set", "lo", "up"},
|
|
// Docker: a bridge whose only port is a container's veth, with an IPv6
|
|
// range too, and a second network on a public range.
|
|
{"link", "add", "br-ctr", "type", "bridge"},
|
|
{"link", "add", "vh0", "type", "veth", "peer", "name", "vc0"},
|
|
{"link", "set", "vh0", "master", "br-ctr"},
|
|
{"addr", "add", "172.31.5.1/24", "dev", "br-ctr"},
|
|
{"-6", "addr", "add", "2001:db8:5::1/64", "dev", "br-ctr", "nodad"},
|
|
{"link", "add", "br-pub", "type", "bridge"},
|
|
{"link", "add", "vh1", "type", "veth", "peer", "name", "vc1"},
|
|
{"link", "set", "vh1", "master", "br-pub"},
|
|
{"addr", "add", "203.0.113.1/24", "dev", "br-pub"},
|
|
// libvirt: a bridge whose port is a dummy.
|
|
{"link", "add", "br-vm", "type", "bridge"},
|
|
{"link", "add", "dvm0", "type", "dummy"},
|
|
{"link", "set", "dvm0", "master", "br-vm"},
|
|
{"addr", "add", "192.168.122.1/24", "dev", "br-vm"},
|
|
// A Docker network no container has joined yet.
|
|
{"link", "add", "br-empty", "type", "bridge"},
|
|
{"addr", "add", "172.29.0.1/24", "dev", "br-empty"},
|
|
// Calico: a pod's veth, routed to directly (the route is added once
|
|
// the link is up, below).
|
|
{"link", "add", "cali0", "type", "veth", "peer", "name", "cali0p"},
|
|
// The LAN: a bridge over a VLAN of a card, and the card itself.
|
|
{"link", "add", "dlan0", "type", "dummy"},
|
|
{"link", "add", "link", "dlan0", "name", "dlan0.7", "type", "vlan", "id", "7"},
|
|
{"link", "add", "br-lan", "type", "bridge"},
|
|
{"link", "set", "dlan0.7", "master", "br-lan"},
|
|
{"addr", "add", "10.77.0.1/24", "dev", "br-lan"},
|
|
// The way out through a veth: this machine is itself a container, and
|
|
// its default routes go through its veths to a gateway, the machine it
|
|
// runs on: IPv4 over two (a multipath default), IPv6 over a third. A
|
|
// veth that carries a main-table default leads out of this machine, so
|
|
// the network on it is the LAN, not inside.
|
|
{"link", "add", "ctr0", "type", "veth", "peer", "name", "ctr0p"},
|
|
{"addr", "add", "10.88.0.2/24", "dev", "ctr0"},
|
|
{"link", "add", "ctr1", "type", "veth", "peer", "name", "ctr1p"},
|
|
{"addr", "add", "10.89.0.2/24", "dev", "ctr1"},
|
|
{"link", "add", "ctr6", "type", "veth", "peer", "name", "ctr6p"},
|
|
{"-6", "addr", "add", "2001:db8:88::2/64", "dev", "ctr6", "nodad"},
|
|
// A veth with a default route in another table only: still inside.
|
|
{"link", "add", "pbr0", "type", "veth", "peer", "name", "pbr0p"},
|
|
{"addr", "add", "10.91.0.2/24", "dev", "pbr0"},
|
|
// A range routed through a veth with this machine's own address on it
|
|
// as the gateway: the kernel answers it as on the link, so it is
|
|
// inside, whatever the gateway.
|
|
{"link", "add", "cilh", "type", "veth", "peer", "name", "ciln"},
|
|
{"addr", "add", "10.66.0.191/32", "dev", "cilh"},
|
|
} {
|
|
if out, err := exec.Command(ipBin, args...).CombinedOutput(); err != nil {
|
|
t.Skipf("ip %s: %v %s", strings.Join(args, " "), err, out)
|
|
}
|
|
}
|
|
for _, l := range []string{"br-ctr", "vh0", "vc0", "br-pub", "vh1", "vc1", "br-vm", "dvm0", "br-empty",
|
|
"cali0", "cali0p", "dlan0", "dlan0.7", "br-lan", "ctr0", "ctr0p", "ctr1", "ctr1p", "ctr6", "ctr6p",
|
|
"pbr0", "pbr0p", "cilh", "ciln"} {
|
|
if out, err := exec.Command(ipBin, "link", "set", l, "up").CombinedOutput(); err != nil {
|
|
t.Skipf("ip link set %s up: %v %s", l, err, out)
|
|
}
|
|
}
|
|
// The card: a link with no kind cannot be made here, so a route over the
|
|
// VLAN's lower link stands for it. A dummy as the route's own link leaves:
|
|
// only a bridge's ports are read for dummies.
|
|
for _, args := range [][]string{
|
|
{"route", "add", "172.30.9.0/24", "dev", "cali0"},
|
|
{"route", "add", "192.168.50.0/24", "dev", "dlan0"},
|
|
// Through a gateway on an inside link, the packet goes where that
|
|
// gateway sends it: the internet from a container, a LAN behind a
|
|
// router VM on a bridge of taps.
|
|
{"route", "add", "default", "nexthop", "via", "10.88.0.1", "dev", "ctr0", "nexthop", "via", "10.89.0.1", "dev", "ctr1"},
|
|
{"-6", "route", "add", "default", "via", "2001:db8:88::1", "dev", "ctr6"},
|
|
{"route", "add", "default", "via", "10.91.0.1", "dev", "pbr0", "table", "100"},
|
|
{"route", "add", "192.168.60.0/24", "via", "192.168.122.254", "dev", "br-vm"},
|
|
{"route", "add", "10.66.0.0/24", "via", "10.66.0.191", "dev", "cilh"},
|
|
} {
|
|
if out, err := exec.Command(ipBin, args...).CombinedOutput(); err != nil {
|
|
t.Skipf("ip %s: %v %s", strings.Join(args, " "), err, out)
|
|
}
|
|
}
|
|
ownAddrReset()
|
|
|
|
inside := []string{"172.31.5.9", "2001:db8:5::9", "203.0.113.9", "192.168.122.9", "172.29.0.9", "172.30.9.9",
|
|
"10.66.0.9", "10.91.0.9"}
|
|
leaves := []string{"10.77.0.9", "192.168.50.9", "198.51.100.9", "2001:db8:ff::9", "192.168.60.9",
|
|
"10.88.0.9", "10.89.0.9", "2001:db8:88::9"}
|
|
for _, a := range inside {
|
|
if got, err := ownRouteLookup(net.ParseIP(a)); err != nil || got != routeThisMachine {
|
|
t.Errorf("route lookup %s: got kind %d err %v, want this machine", a, got, err)
|
|
}
|
|
}
|
|
for _, a := range leaves {
|
|
if got, err := ownRouteLookup(net.ParseIP(a)); err != nil || got != routeLeaves {
|
|
t.Errorf("route lookup %s: got kind %d err %v, want leaves", a, got, err)
|
|
}
|
|
}
|
|
|
|
// The local egress lane on an unanchored box: an inside address, by address
|
|
// and by name, is refused as this machine; the LAN passes.
|
|
t.Setenv("XDG_CONFIG_HOME", t.TempDir())
|
|
if currentAnchor() != nil || boxMode() != modeLocal {
|
|
t.Fatal("the namespaced run must be a local, unanchored box")
|
|
}
|
|
names := map[string][]string{"db.example.test": {"172.31.5.9"}, "vm.example.test": {"192.168.122.9"},
|
|
"nas.example.test": {"10.77.0.9"}}
|
|
stubEgressResolve(t, names)
|
|
ctx := context.Background()
|
|
for _, hp := range []string{"172.31.5.9:5432", "[2001:db8:5::9]:5432", "203.0.113.9:443", "192.168.122.9:22",
|
|
"172.29.0.9:80", "172.30.9.9:8080", "10.66.0.9:80", "10.91.0.9:80", "db.example.test:5432", "vm.example.test:22"} {
|
|
if got, why := egressVettedAddrs(ctx, hp); !strings.Contains(why, "that is this machine") {
|
|
t.Errorf("local egress: %s is on a network inside this machine and must be refused as this machine, got %v %q", hp, got, why)
|
|
}
|
|
}
|
|
for hp, want := range map[string]string{"10.77.0.9:445": "10.77.0.9:445", "192.168.50.9:631": "192.168.50.9:631",
|
|
"nas.example.test:445": "10.77.0.9:445", "198.51.100.9:443": "198.51.100.9:443",
|
|
"[2001:db8:ff::9]:443": "[2001:db8:ff::9]:443", "192.168.60.9:445": "192.168.60.9:445",
|
|
"10.88.0.9:80": "10.88.0.9:80", "10.89.0.9:80": "10.89.0.9:80", "[2001:db8:88::9]:80": "[2001:db8:88::9]:80"} {
|
|
if got, why := egressVettedAddrs(ctx, hp); why != "" || len(got) != 1 || got[0] != want {
|
|
t.Errorf("local egress: %s is the LAN and passes as %s: %v %q", hp, want, got, why)
|
|
}
|
|
}
|
|
// A coder run's proxy refuses the public range on an inside network too
|
|
// (its private ranges it refuses by class already).
|
|
if got, why := coderVettedAddr(ctx, "203.0.113.9:443", nil); !strings.Contains(why, "never this machine") {
|
|
t.Errorf("coder run: 203.0.113.9 is on a network inside this machine, got dial %q, %q", got, why)
|
|
}
|
|
// From a machine that is itself a container, the internet is through its
|
|
// veth: a coder run's proxy and the backup lanes still reach it.
|
|
if got, why := coderVettedAddr(ctx, "198.51.100.9:443", nil); why != "" || got != "198.51.100.9:443" {
|
|
t.Errorf("coder run: 198.51.100.9 is through the gateway and passes, got dial %q, %q", got, why)
|
|
}
|
|
if !backupAddrAllowed(net.ParseIP("2001:db8:ff::9")) || !backupAddrAllowed(net.ParseIP("198.51.100.9")) {
|
|
t.Error("backup: an address through the gateway leaves this machine and is allowed")
|
|
}
|
|
}
|