Privatae
328 lines
13 KiB
Go
328 lines
13 KiB
Go
package main
|
|
|
|
import (
|
|
"encoding/base64"
|
|
"encoding/json"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"runtime"
|
|
"strconv"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"golang.org/x/sys/unix"
|
|
)
|
|
|
|
// opTaskCapRun runs one dry run of the op fixture with the given run body
|
|
// (@ROOT@ in it is the op's effects folder), at the given privilege, and
|
|
// returns its report and its report's JSON (what the host receives). A
|
|
// timeLimit above 0 replaces the fixture's.
|
|
func opTaskCapRun(t *testing.T, privilege, body string, timeLimit int) (*opReport, string, error) {
|
|
t.Helper()
|
|
n := time.Now().UnixNano()
|
|
root := filepath.Join(opTestRoot, fmt.Sprintf("tc%d", n))
|
|
opDir := filepath.Join(opTestRoot, fmt.Sprintf("tcop%d", n))
|
|
for _, p := range []string{root, opDir} {
|
|
p := p
|
|
t.Cleanup(func() { _ = os.RemoveAll(p) })
|
|
}
|
|
_ = os.MkdirAll(root, 0o777)
|
|
_ = os.Chmod(root, 0o777)
|
|
_ = os.MkdirAll(opDir, 0o755)
|
|
for _, f := range opFixture(root, strings.ReplaceAll(body, "@ROOT@", root)) {
|
|
raw, _ := base64.StdEncoding.DecodeString(f.B64)
|
|
_ = os.WriteFile(filepath.Join(opDir, f.Path), raw, os.FileMode(f.Mode))
|
|
}
|
|
man := strings.Replace(fmt.Sprintf(opTestManifest, root), "privilege: root", "privilege: "+privilege, 1)
|
|
if timeLimit > 0 {
|
|
man = strings.Replace(man, "time_limit: 30", fmt.Sprintf("time_limit: %d", timeLimit), 1)
|
|
}
|
|
m, err := parseOpManifest([]byte(man))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
rep, err := opRunSandboxed(opDir, m, opModeDryRun, `{"name":"x","count":1,"kind":"a"}`)
|
|
if rep == nil {
|
|
return nil, "", err
|
|
}
|
|
t.Cleanup(rep.cleanup)
|
|
raw, _ := json.Marshal(rep)
|
|
return rep, string(raw), err
|
|
}
|
|
|
|
// opFakeSystemdRun puts a stand-in for systemd-run first on PATH. With a
|
|
// cgroup it does what `systemd-run --scope -p TasksMax=N` does for the
|
|
// process it runs: a cgroup of its own whose pids.max is N, then the command.
|
|
// The cgroup is a fresh one at the top of the hierarchy, removed after the
|
|
// test; no unit is started. Without one it fails the way systemd-run does
|
|
// with no bus to reach. It records the arguments it was given.
|
|
func opFakeSystemdRun(t *testing.T, cgroup string) (argsFile string) {
|
|
t.Helper()
|
|
dir := t.TempDir()
|
|
argsFile = filepath.Join(dir, "args")
|
|
body := "echo 'Failed to connect to bus: No such file or directory' >&2; exit 1\n"
|
|
if cgroup != "" {
|
|
body = `max=
|
|
while [ $# -gt 0 ]; do
|
|
case "$1" in
|
|
--) shift; break ;;
|
|
-p) shift; case "$1" in TasksMax=*) max=${1#TasksMax=} ;; esac ;;
|
|
esac
|
|
shift
|
|
done
|
|
mkdir -p ` + cgroup + `
|
|
[ -z "$max" ] || echo "$max" > ` + cgroup + `/pids.max
|
|
echo $$ > ` + cgroup + `/cgroup.procs
|
|
exec "$@"
|
|
`
|
|
t.Cleanup(func() {
|
|
// The cgroup empties once the sandbox's pid 1 is gone, which takes
|
|
// every task the op left behind with it.
|
|
for i := 0; i < 100; i++ {
|
|
if err := os.Remove(cgroup); err == nil || os.IsNotExist(err) {
|
|
return
|
|
}
|
|
time.Sleep(100 * time.Millisecond)
|
|
}
|
|
t.Errorf("the test cgroup %s did not empty", cgroup)
|
|
})
|
|
}
|
|
script := "#!/bin/sh\nprintf '%s\\n' \"$@\" > " + argsFile + "\n" + body
|
|
if err := os.WriteFile(filepath.Join(dir, "systemd-run"), []byte(script), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Setenv("PATH", dir+":"+os.Getenv("PATH"))
|
|
return argsFile
|
|
}
|
|
|
|
// A root op's tasks are capped as a user op's are. RLIMIT_NPROC does not
|
|
// hold root (the kernel exempts it), so a root op runs in a scope of its own
|
|
// whose pids.max counts every task in it, root's included: a loop starting
|
|
// 100 background tasks is stopped at the cap, and the report says a cap held.
|
|
func TestARootOpIsStoppedAtItsTaskCap(t *testing.T) {
|
|
requireSandbox(t)
|
|
if ctl, err := os.ReadFile("/sys/fs/cgroup/cgroup.subtree_control"); err != nil || !strings.Contains(" "+string(ctl), " pids") {
|
|
t.Skip("no pids controller at the top of the unified cgroup hierarchy here")
|
|
}
|
|
if _, err := os.Stat("/run/systemd/system"); err != nil {
|
|
t.Skip("the bastion asks for a scope only where systemd is init")
|
|
}
|
|
cgroup := fmt.Sprintf("/sys/fs/cgroup/privatae-optest-%d", os.Getpid())
|
|
argsFile := opFakeSystemdRun(t, cgroup)
|
|
|
|
rep, js, err := opTaskCapRun(t, "root", `i=0; while [ $i -lt 100 ]; do echo "STARTING=$i"; sleep 5 & i=$((i+1)); done; echo DONE; true`, 0)
|
|
if rep == nil {
|
|
t.Fatalf("the op did not run: %v", err)
|
|
}
|
|
if err == nil || !strings.Contains(strings.ToLower(err.Error()), "fork") || strings.Contains(rep.Output, "DONE") {
|
|
t.Fatalf("a root op started 100 background tasks with nothing stopping it: %v, %q", err, rep.Output)
|
|
}
|
|
last := ""
|
|
for _, line := range strings.Split(rep.Output, "\n") {
|
|
if strings.HasPrefix(line, "STARTING=") {
|
|
last = strings.TrimPrefix(line, "STARTING=")
|
|
}
|
|
}
|
|
if n, _ := strconv.Atoi(last); n < 50 {
|
|
t.Errorf("the cap stopped the op after %s of its own tasks: the sandbox's own processes take too much of it", last)
|
|
}
|
|
t.Logf("the root op's fork loop was stopped at task %s", last)
|
|
if !strings.Contains(js, `"tasks_capped":true`) || strings.Contains(js, `"tasks_note"`) {
|
|
t.Errorf("the report does not say the cap held: %s", js)
|
|
}
|
|
|
|
raw, _ := os.ReadFile(argsFile)
|
|
args := strings.Split(strings.TrimSpace(string(raw)), "\n")
|
|
want := []string{"--scope", "--slice=" + opActionSlice, "-p\nTasksMax="}
|
|
for _, w := range want {
|
|
if !strings.Contains(string(raw), w) {
|
|
t.Errorf("systemd-run was not given %q: %q", w, args)
|
|
}
|
|
}
|
|
unit := ""
|
|
for _, a := range args {
|
|
if strings.HasPrefix(a, "--unit=") {
|
|
unit = strings.TrimPrefix(a, "--unit=")
|
|
}
|
|
}
|
|
if !strings.HasPrefix(unit, opScopeUnitPrefix) || strings.HasPrefix(unit, opActionUnitPrefix) {
|
|
t.Errorf("the scope's unit is %q: it must be a files op's, never one the action ops' sweep takes for its own", unit)
|
|
}
|
|
|
|
// A root op that starts a few tasks finishes as before.
|
|
rep, js, err = opTaskCapRun(t, "root", `for i in 1 2 3 4 5; do sleep 0.1 & done; wait; echo RAN-AS=$(id -u); true`, 0)
|
|
if err != nil {
|
|
t.Fatalf("a root op starting five tasks: %v", err)
|
|
}
|
|
if !strings.Contains(rep.Output, "RAN-AS=0") || !strings.Contains(js, `"tasks_capped":true`) {
|
|
t.Errorf("a capped root op: %q %s", rep.Output, js)
|
|
}
|
|
|
|
// The time limit still ends a run in its scope, with everything it
|
|
// started (the cgroup must empty for the cleanup to remove it).
|
|
rep, _, err = opTaskCapRun(t, "root", `sleep 30 & sleep 30; true`, 1)
|
|
if err == nil || rep == nil || !rep.TimedOut {
|
|
t.Fatalf("a root op past its time limit in its scope: %v", err)
|
|
}
|
|
}
|
|
|
|
// opTaskCapRoom runs a root op's fork loop in its scope and returns how many
|
|
// tasks the op held when its next fork was refused.
|
|
func opTaskCapRoom(t *testing.T) int {
|
|
t.Helper()
|
|
rep, _, err := opTaskCapRun(t, "root", `i=0; while [ $i -lt 150 ]; do echo "STARTING=$i"; sleep 5 & i=$((i+1)); done; echo DONE; true`, 0)
|
|
if rep == nil || err == nil || strings.Contains(rep.Output, "DONE") {
|
|
t.Fatalf("the fork loop was not stopped by the scope's cap: %v", err)
|
|
}
|
|
last := -1
|
|
for _, line := range strings.Split(rep.Output, "\n") {
|
|
if v, ok := strings.CutPrefix(line, "STARTING="); ok {
|
|
last, _ = strconv.Atoi(v)
|
|
}
|
|
}
|
|
// STARTING=n is printed before the fork of task n: the shell and n tasks
|
|
// were running when that fork was refused.
|
|
return last + 1
|
|
}
|
|
|
|
// A root op has the same room under its scope's cap on a machine of any
|
|
// size. The two bastion processes in the scope are Go programs, and the Go
|
|
// runtime starts threads by the CPUs it may use: left to that, with the
|
|
// runtime set for 64 CPUs they held 44 of the scope's 96 tasks, leaving the
|
|
// op fewer than a user op's 64, and set for 256 the sandbox's pid 1 could not
|
|
// start at all. Here the bastion's CPUs are narrowed to one, then given back:
|
|
// the op must have as much room either way.
|
|
func TestARootOpsTaskRoomDoesNotShrinkWithTheMachinesCPUs(t *testing.T) {
|
|
requireSandbox(t)
|
|
if ctl, err := os.ReadFile("/sys/fs/cgroup/cgroup.subtree_control"); err != nil || !strings.Contains(" "+string(ctl), " pids") {
|
|
t.Skip("no pids controller at the top of the unified cgroup hierarchy here")
|
|
}
|
|
if _, err := os.Stat("/run/systemd/system"); err != nil {
|
|
t.Skip("the bastion asks for a scope only where systemd is init")
|
|
}
|
|
var all unix.CPUSet
|
|
if err := unix.SchedGetaffinity(0, &all); err != nil || all.Count() < 4 {
|
|
t.Skip("needs at least 4 CPUs to show a difference")
|
|
}
|
|
opFakeSystemdRun(t, fmt.Sprintf("/sys/fs/cgroup/privatae-optest-cpus-%d", os.Getpid()))
|
|
|
|
// The sandbox is started from this goroutine's thread (opRunSandboxCmd
|
|
// holds it), and every process under it takes that thread's CPUs.
|
|
runtime.LockOSThread()
|
|
defer runtime.UnlockOSThread()
|
|
defer func() { _ = unix.SchedSetaffinity(0, &all) }()
|
|
var one unix.CPUSet
|
|
for c := 0; ; c++ {
|
|
if all.IsSet(c) {
|
|
one.Set(c)
|
|
break
|
|
}
|
|
}
|
|
if err := unix.SchedSetaffinity(0, &one); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
narrow := opTaskCapRoom(t)
|
|
if err := unix.SchedSetaffinity(0, &all); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
wide := opTaskCapRoom(t)
|
|
t.Logf("a root op held %d tasks with 1 CPU and %d with %d CPUs", narrow, wide, all.Count())
|
|
if narrow < opSandboxTasksMax || wide < opSandboxTasksMax {
|
|
t.Errorf("a root op held fewer tasks than a user op's %d: %d with 1 CPU, %d with %d", opSandboxTasksMax, narrow, wide, all.Count())
|
|
}
|
|
if wide < narrow-3 {
|
|
t.Errorf("a root op's room shrank from %d tasks to %d with %d CPUs: the bastion's own processes in its scope took it", narrow, wide, all.Count())
|
|
}
|
|
}
|
|
|
|
// Where no scope can be made, a root op runs as before and its report says
|
|
// its tasks were not capped, and why; it never claims a cap. A user op's
|
|
// tasks are held by RLIMIT_NPROC with no scope at all.
|
|
func TestARootOpWithNoScopeSaysItsTasksWereNotCapped(t *testing.T) {
|
|
requireSandbox(t)
|
|
body := `echo RAN-AS=$(id -u); true`
|
|
|
|
// systemd-run is there and fails: the op still runs, once.
|
|
opFakeSystemdRun(t, "")
|
|
rep, js, err := opTaskCapRun(t, "root", `echo RUN >> @ROOT@/runs; echo "RUNS=$(wc -l < @ROOT@/runs)"; `+body, 0)
|
|
if err != nil {
|
|
t.Fatalf("a root op where systemd-run fails: %v", err)
|
|
}
|
|
if !strings.Contains(rep.Output, "RAN-AS=0") || !strings.Contains(rep.Output, "RUNS=1") {
|
|
t.Errorf("the op did not run as root, once: %q", rep.Output)
|
|
}
|
|
if strings.Contains(js, `"tasks_capped":true`) || !strings.Contains(js, `"tasks_note":"`) || !strings.Contains(js, "not capped") || !strings.Contains(js, "Failed to connect to bus") {
|
|
t.Errorf("the report must say the root op's tasks were not capped and why: %s", js)
|
|
}
|
|
|
|
// No systemd-run at all.
|
|
empty := t.TempDir()
|
|
t.Setenv("PATH", empty)
|
|
_, js, err = opTaskCapRun(t, "root", body, 0)
|
|
if err != nil {
|
|
t.Fatalf("a root op with no systemd-run: %v", err)
|
|
}
|
|
if strings.Contains(js, `"tasks_capped":true`) || !strings.Contains(js, "not capped") || !strings.Contains(js, "systemd-run is not installed") {
|
|
t.Errorf("the report must say the root op's tasks were not capped and why: %s", js)
|
|
}
|
|
|
|
// A user op needs no scope: its cap is the rlimit, and the report says so.
|
|
if _, _, err := opUnprivilegedIdentity(); err == nil {
|
|
_, js, err = opTaskCapRun(t, "user", body, 0)
|
|
if err != nil {
|
|
t.Fatalf("a user op: %v", err)
|
|
}
|
|
if !strings.Contains(js, `"tasks_capped":true`) || strings.Contains(js, `"tasks_note"`) {
|
|
t.Errorf("a user op's report: %s", js)
|
|
}
|
|
}
|
|
}
|
|
|
|
// An action op's program needs no scope of its own: it runs as a uid that no
|
|
// other run, account or subordinate range has (opNextUID), so RLIMIT_NPROC
|
|
// counts its own tasks only, and its unit's TasksMax holds every task in the
|
|
// unit, root's included. systemd-run is a stand-in that records what it was
|
|
// given and the spec it was handed; no unit starts.
|
|
func TestAnActionOpsTasksAreItsOwnAndItsUnitIsCapped(t *testing.T) {
|
|
// The run mounts its own tmpfs before it reaches systemd-run, which takes
|
|
// what the sandbox takes: root that may mount and make namespaces.
|
|
requireSandbox(t)
|
|
dir := t.TempDir()
|
|
rec := filepath.Join(dir, "rec")
|
|
script := "#!/bin/sh\nprintf '%s\\n' \"$@\" >> " + rec + "\nfor a; do last=$a; done\ncat \"$last\" >> " + rec + "\necho >> " + rec + "\n"
|
|
if err := os.WriteFile(filepath.Join(dir, "systemd-run"), []byte(script), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Setenv("PATH", dir+":"+os.Getenv("PATH"))
|
|
m, err := parseOpManifest([]byte(opActionManifest))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
opDir := t.TempDir()
|
|
uids := map[uint32]bool{}
|
|
for i := 0; i < 2; i++ {
|
|
_ = os.Remove(rec)
|
|
if _, _, err := opRunActionProgram(opDir, m, opModeDryRun, `{}`, nil, fmt.Sprintf("tc%d", i)); err != nil {
|
|
t.Fatalf("the stand-in unit: %v", err)
|
|
}
|
|
raw, err := os.ReadFile(rec)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !strings.Contains(string(raw), "-p\nTasksMax=512\n") || !strings.Contains(string(raw), "--slice="+opActionSlice+"\n") {
|
|
t.Errorf("the action op's unit is not held to a task cap: %q", raw)
|
|
}
|
|
var spec opActionSpec
|
|
js := string(raw)[strings.Index(string(raw), "{"):]
|
|
if err := json.Unmarshal([]byte(strings.TrimSpace(js)), &spec); err != nil {
|
|
t.Fatalf("the spec the unit was handed: %v %q", err, js)
|
|
}
|
|
if spec.UID < opUIDBase || spec.GID != spec.UID || uids[spec.UID] {
|
|
t.Errorf("run %d's program runs as uid %d: not a fresh one of the ops' range", i, spec.UID)
|
|
}
|
|
uids[spec.UID] = true
|
|
}
|
|
}
|