bastion 0.53.2

client-go/ops_action_linux_test.go

@@ -0,0 +1,836 @@
+package main
+
+// The action-op gate, run for real (ops_action_linux.go): a program in a
+// systemd transient unit, as a fresh uid, held to its destinations, seeing
+// only the closed view, leaving nothing behind. These need root and systemd,
+// and start real units on the machine; they skip anywhere else.
+//
+// Run them in a private PID namespace but NOT a private mount namespace: the
+// unit is started by the machine's systemd, in the machine's mounts, and must
+// see the run's tmpfs the test mounted:
+//
+//	GOTMPDIR=/srv/work/tmp-gotest TMPDIR=/srv/work/tmp-gotest unshare --pid --fork go test -run Action .
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"net"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"strconv"
+	"strings"
+	"sync"
+	"syscall"
+	"testing"
+	"time"
+
+	"golang.org/x/sys/unix"
+)
+
+func requireActionSandbox(t *testing.T) {
+	t.Helper()
+	if ok, why := opActionAvailable(); !ok {
+		t.Skip("no action sandbox here: " + why)
+	}
+	if err := os.MkdirAll(opTestRoot, 0o755); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// actionOp writes an action op (a manifest with extra lines, a shell program)
+// to a fresh directory and parses it.
+func actionOp(t *testing.T, extra, program string) (string, *opManifest) {
+	t.Helper()
+	man := `name: optest-action
+version: 1.0.0
+summary: An action op under test.
+kind: action
+target: machine
+privilege: user
+interpreter: /bin/sh
+time_limit: 30
+params:
+  name: {type: string, max_len: 32}
+` + extra + `modes: [dry_run, apply, verify]
+undo: none
+`
+	m, err := parseOpManifest([]byte(man))
+	if err != nil {
+		t.Fatalf("manifest: %v\n%s", err, man)
+	}
+	dir := filepath.Join(opTestRoot, fmt.Sprintf("aop%d", time.Now().UnixNano()))
+	if err := os.MkdirAll(dir, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { _ = os.RemoveAll(dir) })
+	_ = os.WriteFile(filepath.Join(dir, "op.yaml"), []byte(man), 0o644)
+	_ = os.WriteFile(filepath.Join(dir, "run"), []byte("#!/bin/sh\n"+program+"\n"), 0o755)
+	return dir, m
+}
+
+func actionTerms(m *opManifest) *opTerms {
+	return &opTerms{Kind: opKindRun, Op: m.Name, Version: m.Version, SHA256: strings.Repeat("a", 64),
+		Privilege: m.Privilege, Params: `{"name":"x"}`, Nonce: fmt.Sprintf("n-%d-0123456789", time.Now().UnixNano()),
+		Expires: time.Now().Add(time.Hour).UTC().Format(time.RFC3339), Uses: 1}
+}
+
+func runAction(t *testing.T, dir string, m *opManifest, tm *opTerms, mode string, blobs []string) (*opReport, error) {
+	t.Helper()
+	id, _ := newID()
+	actionRunsMu.Lock()
+	actionRuns[t.Name()] = append(actionRuns[t.Name()], id)
+	actionRunsMu.Unlock()
+	t.Cleanup(func() {
+		actionRunsMu.Lock()
+		delete(actionRuns, t.Name())
+		actionRunsMu.Unlock()
+	})
+	return opRunAction(dir, m, tm, mode, id, blobs, nil)
+}
+
+var (
+	actionRunsMu sync.Mutex
+	actionRuns   = map[string][]string{} // test name -> the run ids it started
+)
+
+// leftUnits is the op units still on the machine from the runs this test
+// started. The machine's list is not asked alone: another test run on the
+// same machine has units of its own, and those are not this test's leftovers.
+func leftUnits(t *testing.T) []string {
+	actionRunsMu.Lock()
+	ids := append([]string(nil), actionRuns[t.Name()]...)
+	actionRunsMu.Unlock()
+	var out []string
+	for _, u := range opActionUnits() {
+		for _, id := range ids {
+			if strings.HasPrefix(u, opActionUnitPrefix+truncate(id, 16)+"-") {
+				out = append(out, u)
+			}
+		}
+	}
+	return out
+}
+
+// localAddr is an address of this machine that is not loopback, so a test
+// can listen on it and name it as a destination.
+func localAddr(t *testing.T) net.IP {
+	addrs, _ := net.InterfaceAddrs()
+	for _, a := range addrs {
+		if n, ok := a.(*net.IPNet); ok && n.IP.To4() != nil && !n.IP.IsLoopback() && n.IP.IsGlobalUnicast() {
+			return n.IP
+		}
+	}
+	t.Skip("no non-loopback IPv4 address here")
+	return nil
+}
+
+func listen(t *testing.T, addr string) (net.Listener, int) {
+	t.Helper()
+	ln, err := net.Listen("tcp", addr)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { _ = ln.Close() })
+	go func() {
+		for {
+			c, err := ln.Accept()
+			if err != nil {
+				return
+			}
+			_, _ = c.Write([]byte("hello\n"))
+			_ = c.Close()
+		}
+	}()
+	return ln, ln.Addr().(*net.TCPAddr).Port
+}
+
+// A destination the op did not declare is out of reach: another port on the
+// declared address (Landlock), another address (the unit's allowlist), and
+// this machine's loopback.
+func TestActionProgramReachesOnlyItsDestinations(t *testing.T) {
+	requireActionSandbox(t)
+	setupBox(t)
+	ip := localAddr(t)
+	_, declared := listen(t, net.JoinHostPort(ip.String(), "0"))
+	_, otherPort := listen(t, net.JoinHostPort(ip.String(), "0"))
+	_, loop := listen(t, "127.0.0.1:0")
+	// The declared port on another address: only the unit's address
+	// allowlist stands between the program and it (Landlock allows the port).
+	listen(t, net.JoinHostPort("127.0.0.1", strconv.Itoa(declared)))
+	// This machine's own address is refused as a destination outside tests;
+	// here it stands in for a remote one, reached by a name.
+	prevOwn, prevRes := opActionOwnAddrs, opActionResolve
+	opActionOwnAddrs = func() []net.IP { return nil }
+	opActionResolve = func(_ context.Context, h string) ([]net.IP, error) {
+		switch h {
+		case "dest.optest.invalid":
+			return []net.IP{ip}, nil
+		case "meta.optest.invalid":
+			return []net.IP{net.ParseIP("169.254.169.254")}, nil
+		}
+		return nil, fmt.Errorf("no")
+	}
+	t.Cleanup(func() { opActionOwnAddrs, opActionResolve = prevOwn, prevRes })
+	probe := func(host string, port int) string {
+		return fmt.Sprintf(`python3 -c 'import socket,sys
+try:
+    s=socket.create_connection((%q,%d),timeout=3); print("REACHED %s:%d", s.recv(6).decode().strip())
+except Exception as e:
+    print("BLOCKED %s:%d", type(e).__name__)'`, host, port, host, port, host, port)
+	}
+	prog := strings.Join([]string{
+		probe("dest.optest.invalid", declared),
+		probe(ip.String(), otherPort),
+		probe("127.0.0.1", loop),
+		probe("127.0.0.1", declared),
+		probe("1.1.1.1", 443),
+	}, "\n")
+	dir, m := actionOp(t, fmt.Sprintf("destinations: [\"dest.optest.invalid:%d\"]\n", declared), prog)
+	rep, err := runAction(t, dir, m, actionTerms(m), opModeApply, nil)
+	if err != nil {
+		t.Fatalf("%v\n%s", err, repOut(rep))
+	}
+	for _, want := range []string{
+		fmt.Sprintf("REACHED dest.optest.invalid:%d hello", declared),
+		fmt.Sprintf("BLOCKED %s:%d", ip, otherPort),
+		fmt.Sprintf("BLOCKED 127.0.0.1:%d", loop),
+		fmt.Sprintf("BLOCKED 127.0.0.1:%d", declared),
+		"BLOCKED 1.1.1.1:443",
+	} {
+		if !strings.Contains(rep.Output, want) {
+			t.Errorf("want %q in:\n%s", want, rep.Output)
+		}
+	}
+	// A dry run, which nobody approved, reaches nothing at all, its declared
+	// destination included.
+	rep, err = runAction(t, dir, m, actionTerms(m), opModeDryRun, nil)
+	if err != nil {
+		t.Fatalf("%v\n%s", err, repOut(rep))
+	}
+	if strings.Contains(rep.Output, "REACHED") || !strings.Contains(rep.Output, fmt.Sprintf("BLOCKED dest.optest.invalid:%d", declared)) {
+		t.Fatalf("a dry run has no network:\n%s", rep.Output)
+	}
+	// A name that resolves to the metadata service is refused at run start.
+	if _, err := opResolveDests(context.Background(), []string{"meta.optest.invalid:80"}); err == nil || !strings.Contains(err.Error(), "link-local") {
+		t.Fatalf("a destination that resolves to the metadata service: %v", err)
+	}
+}
+
+func repOut(r *opReport) string {
+	if r == nil {
+		return "(no report)"
+	}
+	return r.Output
+}
+
+// The program is a uid no one else is, with no capability, and it sees only
+// the closed view: not /root, not the docker socket, not /etc/shadow, not an
+// abstract socket outside it, and it writes nowhere but /tmp.
+func TestActionProgramSeesNothingItWasNotGiven(t *testing.T) {
+	requireActionSandbox(t)
+	setupBox(t)
+	abs := fmt.Sprintf("privatae-optest-%d", time.Now().UnixNano())
+	aln, err := net.Listen("unix", "@"+abs)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer aln.Close()
+	go func() {
+		for {
+			c, err := aln.Accept()
+			if err != nil {
+				return
+			}
+			_ = c.Close()
+		}
+	}()
+	readable := filepath.Join(opTestRoot, fmt.Sprintf("r%d.txt", time.Now().UnixNano()))
+	_ = os.WriteFile(readable, []byte("DECLARED-READ\n"), 0o644)
+	t.Cleanup(func() { _ = os.RemoveAll(readable) })
+	// A folder anyone may write, declared as a read: only the read-only
+	// machine keeps the program from writing into it.
+	open := filepath.Join(opTestRoot, fmt.Sprintf("w%d", time.Now().UnixNano()))
+	_ = os.Mkdir(open, 0o777)
+	_ = os.Chmod(open, 0o777)
+	t.Cleanup(func() { _ = os.RemoveAll(open) })
+	prog := `echo "UID=$(id -u)"
+grep -E '^(CapEff|CapBnd|NoNewPrivs)' /proc/self/status
+echo "ROOT=[$(ls -A /root 2>&1 | tr '\n' ' ')]"
+echo "SHADOW=[$(cat /etc/shadow 2>&1 | head -c 40)]"
+for s in /var/run/docker.sock /run/docker.sock /run/containerd/containerd.sock /run/systemd/private; do
+  [ -e "$s" ] && echo "SEES $s" || echo "NO $s"
+done
+python3 -c 'import socket
+s=socket.socket(socket.AF_UNIX)
+try:
+    s.connect("\0` + abs + `"); print("ABSTRACT REACHED")
+except Exception as e:
+    print("ABSTRACT BLOCKED", type(e).__name__)'
+cat ` + readable + `
+touch /usr/local/optest-probe 2>/dev/null && echo "WROTE /usr/local" || echo "RO /usr/local"
+touch ` + opTestRoot + `/optest-probe 2>/dev/null && echo "WROTE optest" || echo "RO optest"
+touch ` + open + `/probe 2>/dev/null && echo "WROTE open" || echo "RO open"
+touch /tmp/ok && echo "WROTE /tmp"
+echo x > /dev/null && echo "DEVNULL-OK"
+echo "HOSTNAME=$(hostname)"
+echo "CGROUP=$(cat /proc/self/cgroup)"`
+	dir, m := actionOp(t, "reads: ["+strconv.Quote(readable)+", "+strconv.Quote(open+"/")+"]\n", prog)
+	rep, err := runAction(t, dir, m, actionTerms(m), opModeApply, nil)
+	if err != nil {
+		t.Fatalf("%v\n%s", err, repOut(rep))
+	}
+	out := rep.Output
+	var uid uint64
+	for _, l := range strings.Split(out, "\n") {
+		if strings.HasPrefix(l, "UID=") {
+			uid, _ = strconv.ParseUint(strings.TrimPrefix(l, "UID="), 10, 64)
+		}
+	}
+	if uid < opUIDBase || uid >= opUIDBase+opUIDSpan {
+		t.Errorf("the program runs as a uid set aside for ops, got %d", uid)
+	}
+	for _, want := range []string{"CapEff:\t0000000000000000", "CapBnd:\t0000000000000000", "NoNewPrivs:\t1",
+		"ROOT=[]", "SHADOW=[]", "NO /var/run/docker.sock", "NO /run/docker.sock", "NO /run/systemd/private",
+		"CGROUP=0::/" + opActionSlice + "/" + opActionUnitPrefix, "ABSTRACT BLOCKED", "DECLARED-READ", "RO /usr/local", "RO optest", "RO open", "WROTE /tmp", "DEVNULL-OK", "HOSTNAME=op-sandbox"} {
+		if !strings.Contains(out, want) {
+			t.Errorf("want %q in:\n%s", want, out)
+		}
+	}
+	if _, err := os.Stat("/usr/local/optest-probe"); err == nil {
+		_ = os.Remove("/usr/local/optest-probe")
+		t.Error("the program wrote to the machine")
+	}
+}
+
+// A credential reaches the program, never its output, and nothing of the run
+// is left afterwards: not the value on disk, not the tmpfs it sat on, not the
+// work folder, not the unit, not a process the program left running.
+func TestActionCredentialIsHandedAndNothingIsLeft(t *testing.T) {
+	requireActionSandbox(t)
+	setupBox(t)
+	secret := fmt.Sprintf("optest-secret-%d", time.Now().UnixNano())
+	// Every process of the program carries this mark, so a leftover is told
+	// apart from a process of another test run that was given the same uid.
+	mark := fmt.Sprintf("OPTEST_MARK=optest-mark-%d", time.Now().UnixNano())
+	prog := `export ` + mark + `
+echo "TOKEN=$PRIVATAE_CRED_TOKEN"
+echo "$PRIVATAE_CRED_TOKEN" > /tmp/leak
+echo "UID=$(id -u)"
+setsid sleep 300 >/dev/null 2>&1 < /dev/null &
+echo started`
+	dir, m := actionOp(t, "credentials: [token]\n", prog)
+	tm := actionTerms(m)
+	pub, _, err := opEncKey()
+	if err != nil {
+		t.Fatal(err)
+	}
+	blob, _ := opSealCredFor(pub, opSealedCred{Name: "token", Value: secret, Nonce: tm.Nonce, Expires: tm.Expires})
+	// A dry run is handed nothing.
+	if _, err := runAction(t, dir, m, tm, opModeDryRun, []string{blob}); err == nil {
+		t.Fatal("a dry run handed a credential is refused")
+	}
+	rep, err := runAction(t, dir, m, tm, opModeApply, []string{blob})
+	if err != nil {
+		t.Fatalf("%v\n%s", err, repOut(rep))
+	}
+	if strings.Contains(rep.Output, secret) || !strings.Contains(rep.Output, "TOKEN=***") {
+		t.Fatalf("the program had it and its output does not: %s", rep.Output)
+	}
+	var uid string
+	for _, l := range strings.Split(rep.Output, "\n") {
+		if strings.HasPrefix(l, "UID=") {
+			uid = strings.TrimPrefix(l, "UID=")
+		}
+	}
+	root, _ := opRunsRoot()
+	entries, _ := os.ReadDir(root)
+	if len(entries) != 0 {
+		t.Errorf("the run left %d entries in %s", len(entries), root)
+	}
+	mi, _ := os.ReadFile("/proc/self/mountinfo")
+	if strings.Contains(string(mi), root) {
+		t.Error("the run's tmpfs is still mounted")
+	}
+	d, _ := configDir()
+	_ = filepath.Walk(d, func(p string, info os.FileInfo, err error) error {
+		if err == nil && info.Mode().IsRegular() {
+			if b, _ := os.ReadFile(p); strings.Contains(string(b), secret) {
+				t.Errorf("the credential is on disk at %s", p)
+			}
+		}
+		return nil
+	})
+	// The machine's systemd starts the unit outside this test's pid
+	// namespace, so the check reads every process it can see: any with the
+	// run's uid and this run's mark.
+	time.Sleep(500 * time.Millisecond)
+	procs, _ := filepath.Glob("/proc/[0-9]*/status")
+	for _, p := range procs {
+		b, _ := os.ReadFile(p)
+		for _, l := range strings.Split(string(b), "\n") {
+			if f := strings.Fields(l); len(f) >= 2 && f[0] == "Uid:" && f[1] == uid {
+				env, _ := os.ReadFile(filepath.Join(filepath.Dir(p), "environ"))
+				if strings.Contains(string(env), mark) {
+					t.Errorf("a process of the run's uid %s is still running: %s", uid, p)
+				}
+			}
+		}
+	}
+	if u := leftUnits(t); len(u) != 0 {
+		t.Errorf("op units remain: %v", u)
+	}
+}
+
+// An apply whose machine moved since the dry run lands nothing, and runs
+// nothing: the program is never started.
+func TestActionApplyOfAnotherChangeListRunsNothing(t *testing.T) {
+	requireActionSandbox(t)
+	setupBox(t)
+	marker := filepath.Join(opTestRoot, fmt.Sprintf("ran%d", time.Now().UnixNano()))
+	dir, m := actionOp(t, "", "echo ran")
+	tm := actionTerms(m)
+	tm.Changes = strings.Repeat("0", 64)
+	d, _ := configDir()
+	before, _ := os.ReadFile(filepath.Join(d, opUIDFile))
+	rep, err := runAction(t, dir, m, tm, opModeApply, nil)
+	if err == nil || !strings.Contains(err.Error(), "refused before anything ran") || rep != nil {
+		t.Fatalf("refused, with no run: %v %+v", err, rep)
+	}
+	after, _ := os.ReadFile(filepath.Join(d, opUIDFile))
+	if string(before) != string(after) {
+		t.Fatal("a uid was spent, so a program started")
+	}
+	_ = marker
+}
+
+// What the program reports it pushed becomes a receipt signed here; a result
+// file that is a link is refused, never followed as root.
+func TestActionResultBecomesAReceiptAndALinkIsRefused(t *testing.T) {
+	requireActionSandbox(t)
+	setupBox(t)
+	digest := "sha256:" + strings.Repeat("7", 64)
+	res := `{"images":[{"image":"forge.example.com/acme/web","digest":"` + digest +
+		`","source":"https://forge.example.com/acme/app.git","commit":"` + strings.Repeat("6", 40) + `"}]}`
+	dir, m := actionOp(t, "", "[ \"$PRIVATAE_OP_MODE\" = apply ] && printf '%s' '"+res+"' > \"$PRIVATAE_OP_RESULT\"; true")
+	rep, err := runAction(t, dir, m, actionTerms(m), opModeApply, nil)
+	if err != nil {
+		t.Fatalf("%v\n%s", err, repOut(rep))
+	}
+	if len(rep.Receipts) != 1 {
+		t.Fatalf("one receipt: %+v", rep.Receipts)
+	}
+	if r, err := opReceiptFor(digest); err != nil || r == nil || r.Commit != strings.Repeat("6", 40) || r.Op != "optest-action" {
+		t.Fatalf("held and verified: %+v %v", r, err)
+	}
+	dir2, m2 := actionOp(t, "", "[ \"$PRIVATAE_OP_MODE\" = apply ] && ln -s /etc/passwd \"$PRIVATAE_OP_RESULT\"; true")
+	if _, err := runAction(t, dir2, m2, actionTerms(m2), opModeApply, nil); err == nil || !strings.Contains(err.Error(), "not a plain file") {
+		t.Fatalf("a link is refused: %v", err)
+	}
+}
+
+// A bastion that died mid-run leaves a mounted run dir and a work folder;
+// the next start removes both.
+func TestStartupSweepRemovesWhatADeadRunLeft(t *testing.T) {
+	requireActionSandbox(t)
+	setupBox(t)
+	root, _ := opRunsRoot()
+	run := filepath.Join(root, "run-dead")
+	_ = os.Mkdir(run, 0o700)
+	if err := unix.Mount("tmpfs", run, "tmpfs", 0, "size=1m"); err != nil {
+		t.Fatal(err)
+	}
+	_ = os.WriteFile(filepath.Join(run, "cred-0"), []byte("x"), 0o400)
+	_ = os.Mkdir(run+"-work", 0o700)
+	opStartupSweep()
+	if entries, _ := os.ReadDir(root); len(entries) != 0 {
+		t.Fatalf("left: %v", entries)
+	}
+	var st syscall.Statfs_t
+	if err := syscall.Statfs(run, &st); err == nil {
+		t.Fatal("the dead run's tmpfs is still there")
+	}
+}
+
+// The builder's socket reaches the program in an apply, through the socket's
+// group, and never in a dry run.
+func TestActionBuilderSocketOnlyInApply(t *testing.T) {
+	requireActionSandbox(t)
+	setupBox(t)
+	sock := filepath.Join(t.TempDir(), "buildkitd.sock")
+	ln, err := net.Listen("unix", sock)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer ln.Close()
+	go func() {
+		for {
+			c, err := ln.Accept()
+			if err != nil {
+				return
+			}
+			_, _ = c.Write([]byte("builder\n"))
+			_ = c.Close()
+		}
+	}()
+	// root:nogroup 0660: only the socket's group reaches it.
+	_ = os.Chown(sock, 0, 65534)
+	_ = os.Chmod(sock, 0o660)
+	prev := opBuilderSocket
+	opBuilderSocket = sock
+	t.Cleanup(func() { opBuilderSocket = prev })
+	prog := `echo "HOST=$BUILDKIT_HOST"
+python3 -c 'import socket
+s=socket.socket(socket.AF_UNIX)
+try:
+    s.connect("/run/privatae-builder/buildkitd.sock"); print("BUILDER", s.recv(8).decode().strip())
+except Exception as e:
+    print("NO BUILDER", type(e).__name__)'`
+	dir, m := actionOp(t, "builder: true\n", prog)
+	rep, err := runAction(t, dir, m, actionTerms(m), opModeApply, nil)
+	if err != nil {
+		t.Fatalf("%v\n%s", err, repOut(rep))
+	}
+	if !strings.Contains(rep.Output, "HOST=unix:///run/privatae-builder/buildkitd.sock") || !strings.Contains(rep.Output, "BUILDER builder") {
+		t.Fatalf("the apply reaches the builder: %s", rep.Output)
+	}
+	rep, err = runAction(t, dir, m, actionTerms(m), opModeDryRun, nil)
+	if err != nil {
+		t.Fatalf("%v\n%s", err, repOut(rep))
+	}
+	if !strings.Contains(rep.Output, "HOST=\n") || !strings.Contains(rep.Output, "NO BUILDER") {
+		t.Fatalf("a dry run never does: %s", rep.Output)
+	}
+	// A socket in root's group would hand the program that group: refused.
+	_ = os.Chown(sock, 0, 0)
+	if _, err := runAction(t, dir, m, actionTerms(m), opModeApply, nil); err == nil || !strings.Contains(err.Error(), "privileged group") {
+		t.Fatalf("a builder socket in root's group: %v", err)
+	}
+	// A machine with no builder refuses an apply that builds.
+	opBuilderSocket = filepath.Join(t.TempDir(), "absent.sock")
+	if _, err := runAction(t, dir, m, actionTerms(m), opModeApply, nil); err == nil || !strings.Contains(err.Error(), "no builder") {
+		t.Fatalf("no builder: %v", err)
+	}
+}
+
+// Through a real unit: a credential a verb names never reaches the program,
+// and the program's own does (scrubbed from what it prints).
+func TestActionVerbCredentialNeverReachesTheRealProgram(t *testing.T) {
+	requireActionSandbox(t)
+	setupBox(t)
+	useFileRepos(t)
+	origin, tree, _, c2 := twoCommitRepo(t, func(string) {}, func(o string) {
+		_ = os.WriteFile(filepath.Join(o, "a.txt"), []byte("two\n"), 0o644)
+	})
+	extra := fmt.Sprintf(`params:
+  commit: {type: string, max_len: 64}
+credentials: [forge-read, registry]
+verbs:
+  - {verb: tree.checkout, path: %q, repo: %q, commit: "{{commit}}", credential: forge-read}
+`, tree, "file://"+origin)
+	prog := `echo "FORGE=[$PRIVATAE_CRED_FORGE_READ]"; echo "REG=[$PRIVATAE_CRED_REGISTRY]"`
+	man := strings.Replace(strings.Replace(`name: optest-action
+version: 1.0.0
+summary: An action op under test.
+kind: action
+target: machine
+privilege: root
+interpreter: /bin/sh
+time_limit: 30
+EXTRA
+modes: [dry_run, apply, verify]
+undo: none
+`, "EXTRA\n", extra, 1), "\n\n", "\n", -1)
+	m, err := parseOpManifest([]byte(man))
+	if err != nil {
+		t.Fatalf("%v\n%s", err, man)
+	}
+	dir := filepath.Join(opTestRoot, fmt.Sprintf("aopv%d", time.Now().UnixNano()))
+	_ = os.MkdirAll(dir, 0o755)
+	t.Cleanup(func() { _ = os.RemoveAll(dir) })
+	_ = os.WriteFile(filepath.Join(dir, "op.yaml"), []byte(man), 0o644)
+	_ = os.WriteFile(filepath.Join(dir, "run"), []byte("#!/bin/sh\n"+prog+"\n"), 0o755)
+	tm := actionTerms(m)
+	tm.Params = fmt.Sprintf(`{"commit":%q}`, c2)
+	pub, _, _ := opEncKey()
+	var blobs []string
+	for name, val := range map[string]string{"forge-read": "forge-secret-value", "registry": "registry-secret-value"} {
+		b, _ := opSealCredFor(pub, opSealedCred{Name: name, Value: val, Nonce: tm.Nonce, Expires: tm.Expires})
+		blobs = append(blobs, b)
+	}
+	rep, err := runAction(t, dir, m, tm, opModeApply, blobs)
+	if err != nil {
+		t.Fatalf("%v\n%s", err, repOut(rep))
+	}
+	if !strings.Contains(rep.Output, "FORGE=[]") || !strings.Contains(rep.Output, "REG=[***]") {
+		t.Fatalf("the verb's credential is withheld and the program's handed:\n%s", rep.Output)
+	}
+	if gitT(t, tree, "rev-parse", "HEAD") != c2 {
+		t.Fatal("the verb ran with its credential")
+	}
+}
+
+// allocProg holds `mib` MiB, touching every page, in 50 MiB steps.
+func allocProg(mib int) string {
+	return fmt.Sprintf(`python3 -c '
+import time
+print("ADJ", open("/proc/self/oom_score_adj").read().strip(), flush=True)
+held = []
+for i in range(%d // 50):
+    b = bytearray(50 << 20)
+    b[::4096] = b"\x01" * len(b[::4096])
+    held.append(b)
+    time.sleep(0.05)
+print("HELD", len(held) * 50, flush=True)
+rss = [l for l in open("/proc/self/status") if l.startswith("VmRSS:")][0].split()[1]
+print("RSS", int(rss) // 1024, flush=True)
+'`, mib)
+}
+
+// No ceiling: a run holds what it needs while the machine has it to spare.
+// When the machine runs down to its floor, the op is stopped first and says
+// why; a machine already under the floor starts no op; and the kernel, if it
+// must kill, prefers the op.
+func TestActionMemoryIsNotCappedAndTheOpYieldsFirst(t *testing.T) {
+	requireActionSandbox(t)
+	setupBox(t)
+	_, avail, ok := opMemInfo()
+	if !ok || avail < 10<<30 {
+		t.Skip("needs 10 GiB free to show a run is not capped and yields")
+	}
+	dir, m := actionOp(t, "", allocProg(1500))
+	rep, err := runAction(t, dir, m, actionTerms(m), opModeApply, nil)
+	if err != nil {
+		t.Fatalf("a run with memory to spare is not limited: %v\n%s", err, repOut(rep))
+	}
+	if !strings.Contains(rep.Output, "HELD 1500") || !strings.Contains(rep.Output, "ADJ 1000") {
+		t.Fatalf("it held 1500 MiB, and the kernel would kill it first:\n%s", rep.Output)
+	}
+	// In memory, not pushed to swap by a limit on the unit.
+	var rss int
+	for _, l := range strings.Split(rep.Output, "\n") {
+		if strings.HasPrefix(l, "RSS ") {
+			rss, _ = strconv.Atoi(strings.TrimPrefix(l, "RSS "))
+		}
+	}
+	if rss < 1400 {
+		t.Fatalf("it held its 1500 MiB in memory, got %d MiB resident:\n%s", rss, rep.Output)
+	}
+	// The machine runs short: the floor is put 300 MiB under what is free now.
+	prev := opMemoryFloor
+	t.Cleanup(func() { opMemoryFloor = prev })
+	_, avail, _ = opMemInfo()
+	opMemoryFloor = func() uint64 { return avail - 300<<20 }
+	bigDir, big := actionOp(t, "", allocProg(8000))
+	rep, err = runAction(t, bigDir, big, actionTerms(big), opModeApply, nil)
+	if err == nil || !strings.Contains(err.Error(), "stopped first") || strings.Contains(repOut(rep), "HELD") {
+		t.Fatalf("the op yields when the machine runs short: %v\n%s", err, repOut(rep))
+	}
+	if !strings.Contains(err.Error(), "at its peak") || strings.Contains(err.Error(), "used 0 MiB") {
+		t.Fatalf("the refusal says what the op had used: %v", err)
+	}
+	// Already under the floor: nothing starts, no uid is spent.
+	_, avail, _ = opMemInfo()
+	opMemoryFloor = func() uint64 { return avail + 1<<30 }
+	d, _ := configDir()
+	before, _ := os.ReadFile(filepath.Join(d, opUIDFile))
+	if _, err := runAction(t, dir, m, actionTerms(m), opModeDryRun, nil); err == nil || !strings.Contains(err.Error(), "did not start") {
+		t.Fatalf("a machine under its floor starts no op: %v", err)
+	}
+	if after, _ := os.ReadFile(filepath.Join(d, opUIDFile)); string(before) != string(after) {
+		t.Fatal("a uid was spent")
+	}
+	if u := leftUnits(t); len(u) != 0 {
+		t.Fatalf("op units remain: %v", u)
+	}
+}
+
+func TestMemoryFloorSitsAboveTheKubeletsEviction(t *testing.T) {
+	total, _, _ := opMemInfo()
+	gib := uint64(1 << 30)
+	for _, c := range []struct {
+		what       string
+		files      map[string]string // relative path -> content
+		want       uint64
+		unreadable bool
+	}{
+		{"the config's setting, not its comment", map[string]string{"config.yaml": "#   eviction-hard   9Gi = a comment\nkubelet-arg:\n  - \"eviction-hard=memory.available<3Gi,nodefs.available<10%\"\n"}, 3 * gib, false},
+		{"a percentage of the machine", map[string]string{"config.yaml.d/10-evict.yaml": "kubelet-arg:\n  - eviction-hard=memory.available<10%\n"}, total / 10, false},
+		{"a decimal", map[string]string{"config.yaml": "kubelet-arg: [\"eviction-hard=memory.available<1.5Gi\"]\n"}, 3 * gib / 2, false},
+		{"a decimal unit", map[string]string{"config.yaml": "kubelet-arg: [\"eviction-hard=memory.available<2G\"]\n"}, 2e9, false},
+		{"soft eviction counts too", map[string]string{"config.yaml": "kubelet-arg:\n  - eviction-hard=memory.available<512Mi\n  - eviction-soft=memory.available<4Gi\n"}, 4 * gib, false},
+		{"a flag on the k3s unit", map[string]string{"k3s.service": "ExecStart=/usr/local/bin/k3s server --kubelet-arg=eviction-hard=memory.available<2Gi\n"}, 2 * gib, false},
+		{"a kubelet config file's form", map[string]string{"config.yaml.d/kubelet.yaml": "evictionHard:\n  memory.available: \"3Gi\"\n"}, 3 * gib, false},
+		{"a value that cannot be read", map[string]string{"config.yaml": "kubelet-arg: [\"eviction-hard=memory.available<lots\"]\n"}, 0, true},
+		{"the kubelet config k3s writes", map[string]string{"kubelet.conf.d/00-k3s-defaults.conf": "evictionHard:\n  imagefs.available: 5%\n  memory.available: 2Gi\n"}, 2 * gib, false},
+		{"a kubelet config file a flag names", map[string]string{"config.yaml": "kubelet-arg:\n  - \"config={DIR}/kubelet.yaml\"\n", "kubelet.yaml": "evictionHard:\n  memory.available: \"4Gi\"\n"}, 4 * gib, false},
+		{"a kubelet config folder a flag names", map[string]string{"k3s.service": "ExecStart=/usr/local/bin/k3s agent --kubelet-arg=config-dir={DIR}/kd\n", "kd/10.conf": "evictionSoft:\n  memory.available: 5Gi\n"}, 5 * gib, false},
+		{"an empty kubelet config folder", map[string]string{"config.yaml": "kubelet-arg:\n  - config-dir={DIR}/empty\n", "empty/.keep": ""}, 0, false},
+		{"a kubelet config file that is not there", map[string]string{"config.yaml": "kubelet-arg:\n  - config={DIR}/missing.yaml\n"}, 0, true},
+		{"a flag split over two lines of the unit", map[string]string{"k3s.service": "ExecStart=/usr/local/bin/k3s \\\n    server \\\n    '--kubelet-arg' \\\n    'config={DIR}/kubelet.yaml' \\\n", "kubelet.yaml": "evictionHard:\n  memory.available: 4Gi\n"}, 4 * gib, false},
+		{"a split flag naming a file that is not there", map[string]string{"k3s.service": "ExecStart=/usr/local/bin/k3s \\\n    '--kubelet-arg' \\\n    'config={DIR}/missing.yaml' \\\n"}, 0, true},
+		{"single quotes", map[string]string{"kubelet.conf.d/50.conf": "evictionSoft:\n  'memory.available': '3Gi'\n"}, 3 * gib, false},
+	} {
+		dir := t.TempDir()
+		for rel, body := range c.files {
+			_ = os.MkdirAll(filepath.Dir(filepath.Join(dir, rel)), 0o755)
+			_ = os.WriteFile(filepath.Join(dir, rel), []byte(strings.ReplaceAll(body, "{DIR}", dir)), 0o644)
+		}
+		prevC, prevU := opK3sConfigGlobs, opK3sUnitGlobs
+		opK3sConfigGlobs = []string{filepath.Join(dir, "config.yaml"), filepath.Join(dir, "config.yaml.d/*.yaml"), filepath.Join(dir, "kubelet.conf.d/*.conf")}
+		opK3sUnitGlobs = []string{filepath.Join(dir, "k3s*.service")}
+		got, unreadable := opKubeletEviction()
+		floor := opMemoryFloorForMachine()
+		opK3sConfigGlobs, opK3sUnitGlobs = prevC, prevU
+		if got != c.want || unreadable != c.unreadable {
+			t.Errorf("%s: got %d unreadable=%v, want %d unreadable=%v", c.what, got, unreadable, c.want, c.unreadable)
+		}
+		if c.want > 0 && floor < c.want+opMemoryKubeletPad {
+			t.Errorf("%s: the floor %d is not above the kubelet's line %d", c.what, floor, c.want)
+		}
+		if c.unreadable && floor < total/10+opMemoryKubeletPad {
+			t.Errorf("%s: an unreadable setting makes the floor cautious, got %d", c.what, floor)
+		}
+		opK3sConfigGlobs = []string{filepath.Join(dir, "config.yaml"), filepath.Join(dir, "config.yaml.d/*.yaml"), filepath.Join(dir, "kubelet.conf.d/*.conf")}
+		opK3sUnitGlobs = []string{filepath.Join(dir, "k3s*.service")}
+		note := opFloorNote()
+		opK3sConfigGlobs, opK3sUnitGlobs = prevC, prevU
+		if (note != "") != c.unreadable {
+			t.Errorf("%s: a stop says when the floor was raised for an unreadable setting, and only then: %q", c.what, note)
+		}
+	}
+	// The product reads where k3s v1.35 writes the kubelet's own config.
+	found := false
+	for _, g := range opK3sConfigGlobs {
+		found = found || g == "/var/lib/rancher/k3s/agent/etc/kubelet.conf.d/*.conf"
+	}
+	if !found {
+		t.Fatal("the kubelet config k3s writes is not read")
+	}
+	// No kubelet: the largest of 1 GiB and 5%.
+	prevC, prevU := opK3sConfigGlobs, opK3sUnitGlobs
+	opK3sConfigGlobs, opK3sUnitGlobs = []string{filepath.Join(t.TempDir(), "absent")}, nil
+	defer func() { opK3sConfigGlobs, opK3sUnitGlobs = prevC, prevU }()
+	want := gib
+	if total/20 > want {
+		want = total / 20
+	}
+	if got := opMemoryFloorForMachine(); got != want {
+		t.Fatalf("with no kubelet: %d, want %d", got, want)
+	}
+}
+
+// A kill that does not land (no such unit, no cgroup yet) is never reported
+// as the op stopped.
+func TestTheMemoryWatchReportsOnlyAKillThatLanded(t *testing.T) {
+	stop := opMemoryWatch("privatae-op-nonexistent-optest", 1<<62)
+	time.Sleep(4 * opMemoryWatchPeriod)
+	v := stop()
+	if v.Stopped {
+		t.Fatalf("a kill that found nothing is not a stop: %+v", v)
+	}
+}
+
+// The second signal: the machine's own services waiting on memory. The op's
+// own stalls (its slice) never count; another cgroup's do, and stop it.
+func TestActionYieldsWhenTheServicesWaitOnMemoryNotForItsOwnStalls(t *testing.T) {
+	requireActionSandbox(t)
+	setupBox(t)
+	fake := t.TempDir()
+	write := func(cg string, some float64) {
+		_ = os.MkdirAll(filepath.Join(fake, cg), 0o755)
+		_ = os.WriteFile(filepath.Join(fake, cg, "memory.pressure"),
+			[]byte(fmt.Sprintf("some avg10=%.2f avg60=0.00 avg300=0.00 total=1\nfull avg10=0.00 avg60=0.00 avg300=0.00 total=1\n", some)), 0o644)
+	}
+	write("system.slice", 0.10)
+	write("kubepods.slice", 40) // a pod reclaiming against its own limit, already
+	write(opActionSlice, 0.10)
+	prev := opCgroupRoot
+	opCgroupRoot = fake
+	t.Cleanup(func() { opCgroupRoot = prev })
+	// The op's own slice rises to 90 during the run: its own stalls.
+	dir, m := actionOp(t, "", "sleep 3; echo DONE")
+	go func() { time.Sleep(time.Second); write(opActionSlice, 90) }()
+	rep, err := runAction(t, dir, m, actionTerms(m), opModeApply, nil)
+	if err != nil || !strings.Contains(rep.Output, "DONE") {
+		t.Fatalf("neither the op's own stalls nor a group already stalling stop it: %v\n%s", err, repOut(rep))
+	}
+	// The services start waiting one second into a longer run.
+	dir2, m2 := actionOp(t, "", "sleep 20; echo DONE")
+	go func() { time.Sleep(time.Second); write("system.slice", 12.5) }()
+	start := time.Now()
+	rep, err = runAction(t, dir2, m2, actionTerms(m2), opModeApply, nil)
+	if err == nil || !strings.Contains(err.Error(), "system.slice began waiting on memory 12.4% more of the time than it had during the run") || strings.Contains(repOut(rep), "DONE") {
+		t.Fatalf("the services waiting stops the op: %v\n%s", err, repOut(rep))
+	}
+	if time.Since(start) > 10*time.Second {
+		t.Fatal("it was not stopped promptly")
+	}
+	// A group that keeps stalling at a level does not stop the next run: only
+	// a rise during the run does.
+	rep, err = runAction(t, dir, m, actionTerms(m), opModeDryRun, nil)
+	if err != nil || !strings.Contains(rep.Output, "DONE") {
+		t.Fatalf("a steady level is not a rise: %v\n%s", err, repOut(rep))
+	}
+	// A group that spiked as the run began and fell back does not hide a
+	// rise the run causes later: the rise is from its lowest reading.
+	write("system.slice", 30)
+	dir3, m3 := actionOp(t, "", "sleep 20; echo DONE")
+	go func() {
+		time.Sleep(time.Second)
+		write("system.slice", 0)
+		time.Sleep(time.Second)
+		write("system.slice", 12)
+	}()
+	rep, err = runAction(t, dir3, m3, actionTerms(m3), opModeApply, nil)
+	if err == nil || !strings.Contains(err.Error(), "system.slice began waiting on memory 12.0% more") || strings.Contains(repOut(rep), "DONE") {
+		t.Fatalf("a rise after a spike at the start: %v\n%s", err, repOut(rep))
+	}
+}
+
+// The watch keeps trying: the machine is short before the op's unit exists
+// (the first kill finds nothing), and the unit that starts a second later is
+// still stopped.
+func TestTheMemoryWatchKeepsTryingUntilTheKillLands(t *testing.T) {
+	requireActionSandbox(t)
+	unit := fmt.Sprintf("%swatchtest-%d", opActionUnitPrefix, time.Now().UnixNano())
+	stop := opMemoryWatch(unit, 1<<62)
+	defer func() { opKillUnit(unit) }()
+	time.Sleep(time.Second)
+	// Started the way a run starts its unit (--wait, so it holds until the
+	// unit ends), in the background. Without --quiet, systemd-run reports how
+	// the unit ended: that is the proof it ran and was killed. Polling for its
+	// cgroup is not: the watch kills within one period of the unit starting,
+	// so the cgroup can come and go between two polls.
+	var report bytes.Buffer
+	cmd := exec.Command("systemd-run", "--wait", "--collect", "--unit="+unit, "--slice="+opActionSlice,
+		"-p", "RuntimeMaxSec=60", "/bin/sleep", "30")
+	cmd.Env = []string{"PATH=/usr/sbin:/usr/bin:/sbin:/bin"}
+	cmd.Stdout, cmd.Stderr = &report, &report
+	if err := cmd.Start(); err != nil {
+		t.Fatalf("start the unit: %v", err)
+	}
+	ended := make(chan struct{})
+	go func() { _ = cmd.Wait(); close(ended) }()
+	select {
+	case <-ended:
+	case <-time.After(10 * time.Second):
+		t.Fatal("the unit was not stopped: its 30-second sleep is still running")
+	}
+	v := stop()
+	gone := false
+	for i := 0; i < 100 && !gone; i++ {
+		if _, err := os.Stat(opUnitCgroupDir(unit)); err != nil {
+			gone = true
+		} else {
+			time.Sleep(50 * time.Millisecond)
+		}
+	}
+	out := report.String()
+	killed := strings.Contains(out, "Finished with result: signal") && regexp.MustCompile(`code=killed.*KILL`).MatchString(out)
+	if !killed || !gone || !v.Stopped {
+		t.Fatalf("the unit that came after the first try was stopped: killed=%v gone=%v %+v\n%s", killed, gone, v, out)
+	}
+}