bastion 0.55.3

client-go/ops_taskcap_linux_test.go

@@ -0,0 +1,327 @@
+package main
+
+import (
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"golang.org/x/sys/unix"
+)
+
+// opTaskCapRun runs one dry run of the op fixture with the given run body
+// (@ROOT@ in it is the op's effects folder), at the given privilege, and
+// returns its report and its report's JSON (what the host receives). A
+// timeLimit above 0 replaces the fixture's.
+func opTaskCapRun(t *testing.T, privilege, body string, timeLimit int) (*opReport, string, error) {
+	t.Helper()
+	n := time.Now().UnixNano()
+	root := filepath.Join(opTestRoot, fmt.Sprintf("tc%d", n))
+	opDir := filepath.Join(opTestRoot, fmt.Sprintf("tcop%d", n))
+	for _, p := range []string{root, opDir} {
+		p := p
+		t.Cleanup(func() { _ = os.RemoveAll(p) })
+	}
+	_ = os.MkdirAll(root, 0o777)
+	_ = os.Chmod(root, 0o777)
+	_ = os.MkdirAll(opDir, 0o755)
+	for _, f := range opFixture(root, strings.ReplaceAll(body, "@ROOT@", root)) {
+		raw, _ := base64.StdEncoding.DecodeString(f.B64)
+		_ = os.WriteFile(filepath.Join(opDir, f.Path), raw, os.FileMode(f.Mode))
+	}
+	man := strings.Replace(fmt.Sprintf(opTestManifest, root), "privilege: root", "privilege: "+privilege, 1)
+	if timeLimit > 0 {
+		man = strings.Replace(man, "time_limit: 30", fmt.Sprintf("time_limit: %d", timeLimit), 1)
+	}
+	m, err := parseOpManifest([]byte(man))
+	if err != nil {
+		t.Fatal(err)
+	}
+	rep, err := opRunSandboxed(opDir, m, opModeDryRun, `{"name":"x","count":1,"kind":"a"}`)
+	if rep == nil {
+		return nil, "", err
+	}
+	t.Cleanup(rep.cleanup)
+	raw, _ := json.Marshal(rep)
+	return rep, string(raw), err
+}
+
+// opFakeSystemdRun puts a stand-in for systemd-run first on PATH. With a
+// cgroup it does what `systemd-run --scope -p TasksMax=N` does for the
+// process it runs: a cgroup of its own whose pids.max is N, then the command.
+// The cgroup is a fresh one at the top of the hierarchy, removed after the
+// test; no unit is started. Without one it fails the way systemd-run does
+// with no bus to reach. It records the arguments it was given.
+func opFakeSystemdRun(t *testing.T, cgroup string) (argsFile string) {
+	t.Helper()
+	dir := t.TempDir()
+	argsFile = filepath.Join(dir, "args")
+	body := "echo 'Failed to connect to bus: No such file or directory' >&2; exit 1\n"
+	if cgroup != "" {
+		body = `max=
+while [ $# -gt 0 ]; do
+	case "$1" in
+	--) shift; break ;;
+	-p) shift; case "$1" in TasksMax=*) max=${1#TasksMax=} ;; esac ;;
+	esac
+	shift
+done
+mkdir -p ` + cgroup + `
+[ -z "$max" ] || echo "$max" > ` + cgroup + `/pids.max
+echo $$ > ` + cgroup + `/cgroup.procs
+exec "$@"
+`
+		t.Cleanup(func() {
+			// The cgroup empties once the sandbox's pid 1 is gone, which takes
+			// every task the op left behind with it.
+			for i := 0; i < 100; i++ {
+				if err := os.Remove(cgroup); err == nil || os.IsNotExist(err) {
+					return
+				}
+				time.Sleep(100 * time.Millisecond)
+			}
+			t.Errorf("the test cgroup %s did not empty", cgroup)
+		})
+	}
+	script := "#!/bin/sh\nprintf '%s\\n' \"$@\" > " + argsFile + "\n" + body
+	if err := os.WriteFile(filepath.Join(dir, "systemd-run"), []byte(script), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	t.Setenv("PATH", dir+":"+os.Getenv("PATH"))
+	return argsFile
+}
+
+// A root op's tasks are capped as a user op's are. RLIMIT_NPROC does not
+// hold root (the kernel exempts it), so a root op runs in a scope of its own
+// whose pids.max counts every task in it, root's included: a loop starting
+// 100 background tasks is stopped at the cap, and the report says a cap held.
+func TestARootOpIsStoppedAtItsTaskCap(t *testing.T) {
+	requireSandbox(t)
+	if ctl, err := os.ReadFile("/sys/fs/cgroup/cgroup.subtree_control"); err != nil || !strings.Contains(" "+string(ctl), " pids") {
+		t.Skip("no pids controller at the top of the unified cgroup hierarchy here")
+	}
+	if _, err := os.Stat("/run/systemd/system"); err != nil {
+		t.Skip("the bastion asks for a scope only where systemd is init")
+	}
+	cgroup := fmt.Sprintf("/sys/fs/cgroup/privatae-optest-%d", os.Getpid())
+	argsFile := opFakeSystemdRun(t, cgroup)
+
+	rep, js, err := opTaskCapRun(t, "root", `i=0; while [ $i -lt 100 ]; do echo "STARTING=$i"; sleep 5 & i=$((i+1)); done; echo DONE; true`, 0)
+	if rep == nil {
+		t.Fatalf("the op did not run: %v", err)
+	}
+	if err == nil || !strings.Contains(strings.ToLower(err.Error()), "fork") || strings.Contains(rep.Output, "DONE") {
+		t.Fatalf("a root op started 100 background tasks with nothing stopping it: %v, %q", err, rep.Output)
+	}
+	last := ""
+	for _, line := range strings.Split(rep.Output, "\n") {
+		if strings.HasPrefix(line, "STARTING=") {
+			last = strings.TrimPrefix(line, "STARTING=")
+		}
+	}
+	if n, _ := strconv.Atoi(last); n < 50 {
+		t.Errorf("the cap stopped the op after %s of its own tasks: the sandbox's own processes take too much of it", last)
+	}
+	t.Logf("the root op's fork loop was stopped at task %s", last)
+	if !strings.Contains(js, `"tasks_capped":true`) || strings.Contains(js, `"tasks_note"`) {
+		t.Errorf("the report does not say the cap held: %s", js)
+	}
+
+	raw, _ := os.ReadFile(argsFile)
+	args := strings.Split(strings.TrimSpace(string(raw)), "\n")
+	want := []string{"--scope", "--slice=" + opActionSlice, "-p\nTasksMax="}
+	for _, w := range want {
+		if !strings.Contains(string(raw), w) {
+			t.Errorf("systemd-run was not given %q: %q", w, args)
+		}
+	}
+	unit := ""
+	for _, a := range args {
+		if strings.HasPrefix(a, "--unit=") {
+			unit = strings.TrimPrefix(a, "--unit=")
+		}
+	}
+	if !strings.HasPrefix(unit, opScopeUnitPrefix) || strings.HasPrefix(unit, opActionUnitPrefix) {
+		t.Errorf("the scope's unit is %q: it must be a files op's, never one the action ops' sweep takes for its own", unit)
+	}
+
+	// A root op that starts a few tasks finishes as before.
+	rep, js, err = opTaskCapRun(t, "root", `for i in 1 2 3 4 5; do sleep 0.1 & done; wait; echo RAN-AS=$(id -u); true`, 0)
+	if err != nil {
+		t.Fatalf("a root op starting five tasks: %v", err)
+	}
+	if !strings.Contains(rep.Output, "RAN-AS=0") || !strings.Contains(js, `"tasks_capped":true`) {
+		t.Errorf("a capped root op: %q %s", rep.Output, js)
+	}
+
+	// The time limit still ends a run in its scope, with everything it
+	// started (the cgroup must empty for the cleanup to remove it).
+	rep, _, err = opTaskCapRun(t, "root", `sleep 30 & sleep 30; true`, 1)
+	if err == nil || rep == nil || !rep.TimedOut {
+		t.Fatalf("a root op past its time limit in its scope: %v", err)
+	}
+}
+
+// opTaskCapRoom runs a root op's fork loop in its scope and returns how many
+// tasks the op held when its next fork was refused.
+func opTaskCapRoom(t *testing.T) int {
+	t.Helper()
+	rep, _, err := opTaskCapRun(t, "root", `i=0; while [ $i -lt 150 ]; do echo "STARTING=$i"; sleep 5 & i=$((i+1)); done; echo DONE; true`, 0)
+	if rep == nil || err == nil || strings.Contains(rep.Output, "DONE") {
+		t.Fatalf("the fork loop was not stopped by the scope's cap: %v", err)
+	}
+	last := -1
+	for _, line := range strings.Split(rep.Output, "\n") {
+		if v, ok := strings.CutPrefix(line, "STARTING="); ok {
+			last, _ = strconv.Atoi(v)
+		}
+	}
+	// STARTING=n is printed before the fork of task n: the shell and n tasks
+	// were running when that fork was refused.
+	return last + 1
+}
+
+// A root op has the same room under its scope's cap on a machine of any
+// size. The two bastion processes in the scope are Go programs, and the Go
+// runtime starts threads by the CPUs it may use: left to that, with the
+// runtime set for 64 CPUs they held 44 of the scope's 96 tasks, leaving the
+// op fewer than a user op's 64, and set for 256 the sandbox's pid 1 could not
+// start at all. Here the bastion's CPUs are narrowed to one, then given back:
+// the op must have as much room either way.
+func TestARootOpsTaskRoomDoesNotShrinkWithTheMachinesCPUs(t *testing.T) {
+	requireSandbox(t)
+	if ctl, err := os.ReadFile("/sys/fs/cgroup/cgroup.subtree_control"); err != nil || !strings.Contains(" "+string(ctl), " pids") {
+		t.Skip("no pids controller at the top of the unified cgroup hierarchy here")
+	}
+	if _, err := os.Stat("/run/systemd/system"); err != nil {
+		t.Skip("the bastion asks for a scope only where systemd is init")
+	}
+	var all unix.CPUSet
+	if err := unix.SchedGetaffinity(0, &all); err != nil || all.Count() < 4 {
+		t.Skip("needs at least 4 CPUs to show a difference")
+	}
+	opFakeSystemdRun(t, fmt.Sprintf("/sys/fs/cgroup/privatae-optest-cpus-%d", os.Getpid()))
+
+	// The sandbox is started from this goroutine's thread (opRunSandboxCmd
+	// holds it), and every process under it takes that thread's CPUs.
+	runtime.LockOSThread()
+	defer runtime.UnlockOSThread()
+	defer func() { _ = unix.SchedSetaffinity(0, &all) }()
+	var one unix.CPUSet
+	for c := 0; ; c++ {
+		if all.IsSet(c) {
+			one.Set(c)
+			break
+		}
+	}
+	if err := unix.SchedSetaffinity(0, &one); err != nil {
+		t.Fatal(err)
+	}
+	narrow := opTaskCapRoom(t)
+	if err := unix.SchedSetaffinity(0, &all); err != nil {
+		t.Fatal(err)
+	}
+	wide := opTaskCapRoom(t)
+	t.Logf("a root op held %d tasks with 1 CPU and %d with %d CPUs", narrow, wide, all.Count())
+	if narrow < opSandboxTasksMax || wide < opSandboxTasksMax {
+		t.Errorf("a root op held fewer tasks than a user op's %d: %d with 1 CPU, %d with %d", opSandboxTasksMax, narrow, wide, all.Count())
+	}
+	if wide < narrow-3 {
+		t.Errorf("a root op's room shrank from %d tasks to %d with %d CPUs: the bastion's own processes in its scope took it", narrow, wide, all.Count())
+	}
+}
+
+// Where no scope can be made, a root op runs as before and its report says
+// its tasks were not capped, and why; it never claims a cap. A user op's
+// tasks are held by RLIMIT_NPROC with no scope at all.
+func TestARootOpWithNoScopeSaysItsTasksWereNotCapped(t *testing.T) {
+	requireSandbox(t)
+	body := `echo RAN-AS=$(id -u); true`
+
+	// systemd-run is there and fails: the op still runs, once.
+	opFakeSystemdRun(t, "")
+	rep, js, err := opTaskCapRun(t, "root", `echo RUN >> @ROOT@/runs; echo "RUNS=$(wc -l < @ROOT@/runs)"; `+body, 0)
+	if err != nil {
+		t.Fatalf("a root op where systemd-run fails: %v", err)
+	}
+	if !strings.Contains(rep.Output, "RAN-AS=0") || !strings.Contains(rep.Output, "RUNS=1") {
+		t.Errorf("the op did not run as root, once: %q", rep.Output)
+	}
+	if strings.Contains(js, `"tasks_capped":true`) || !strings.Contains(js, `"tasks_note":"`) || !strings.Contains(js, "not capped") || !strings.Contains(js, "Failed to connect to bus") {
+		t.Errorf("the report must say the root op's tasks were not capped and why: %s", js)
+	}
+
+	// No systemd-run at all.
+	empty := t.TempDir()
+	t.Setenv("PATH", empty)
+	_, js, err = opTaskCapRun(t, "root", body, 0)
+	if err != nil {
+		t.Fatalf("a root op with no systemd-run: %v", err)
+	}
+	if strings.Contains(js, `"tasks_capped":true`) || !strings.Contains(js, "not capped") || !strings.Contains(js, "systemd-run is not installed") {
+		t.Errorf("the report must say the root op's tasks were not capped and why: %s", js)
+	}
+
+	// A user op needs no scope: its cap is the rlimit, and the report says so.
+	if _, _, err := opUnprivilegedIdentity(); err == nil {
+		_, js, err = opTaskCapRun(t, "user", body, 0)
+		if err != nil {
+			t.Fatalf("a user op: %v", err)
+		}
+		if !strings.Contains(js, `"tasks_capped":true`) || strings.Contains(js, `"tasks_note"`) {
+			t.Errorf("a user op's report: %s", js)
+		}
+	}
+}
+
+// An action op's program needs no scope of its own: it runs as a uid that no
+// other run, account or subordinate range has (opNextUID), so RLIMIT_NPROC
+// counts its own tasks only, and its unit's TasksMax holds every task in the
+// unit, root's included. systemd-run is a stand-in that records what it was
+// given and the spec it was handed; no unit starts.
+func TestAnActionOpsTasksAreItsOwnAndItsUnitIsCapped(t *testing.T) {
+	// The run mounts its own tmpfs before it reaches systemd-run, which takes
+	// what the sandbox takes: root that may mount and make namespaces.
+	requireSandbox(t)
+	dir := t.TempDir()
+	rec := filepath.Join(dir, "rec")
+	script := "#!/bin/sh\nprintf '%s\\n' \"$@\" >> " + rec + "\nfor a; do last=$a; done\ncat \"$last\" >> " + rec + "\necho >> " + rec + "\n"
+	if err := os.WriteFile(filepath.Join(dir, "systemd-run"), []byte(script), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	t.Setenv("PATH", dir+":"+os.Getenv("PATH"))
+	m, err := parseOpManifest([]byte(opActionManifest))
+	if err != nil {
+		t.Fatal(err)
+	}
+	opDir := t.TempDir()
+	uids := map[uint32]bool{}
+	for i := 0; i < 2; i++ {
+		_ = os.Remove(rec)
+		if _, _, err := opRunActionProgram(opDir, m, opModeDryRun, `{}`, nil, fmt.Sprintf("tc%d", i)); err != nil {
+			t.Fatalf("the stand-in unit: %v", err)
+		}
+		raw, err := os.ReadFile(rec)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !strings.Contains(string(raw), "-p\nTasksMax=512\n") || !strings.Contains(string(raw), "--slice="+opActionSlice+"\n") {
+			t.Errorf("the action op's unit is not held to a task cap: %q", raw)
+		}
+		var spec opActionSpec
+		js := string(raw)[strings.Index(string(raw), "{"):]
+		if err := json.Unmarshal([]byte(strings.TrimSpace(js)), &spec); err != nil {
+			t.Fatalf("the spec the unit was handed: %v %q", err, js)
+		}
+		if spec.UID < opUIDBase || spec.GID != spec.UID || uids[spec.UID] {
+			t.Errorf("run %d's program runs as uid %d: not a fresh one of the ops' range", i, spec.UID)
+		}
+		uids[spec.UID] = true
+	}
+}