privatae-bastion/client-go/ops_plan_linux_test.go

package main

import (
	"context"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"testing"
	"time"
)

// holdActionOp puts an action op in the pairing's store, as op_sync would.
func holdActionOp(t *testing.T, dcn, man, program string) (string, *opManifest) {
	t.Helper()
	m, err := parseOpManifest([]byte(man))
	if err != nil {
		t.Fatalf("%v\n%s", err, man)
	}
	files := []opFile{
		{Path: "op.yaml", Mode: 0o644, B64: base64.StdEncoding.EncodeToString([]byte(man))},
		{Path: "run", Mode: 0o755, B64: base64.StdEncoding.EncodeToString([]byte("#!/bin/sh\n" + program + "\n"))},
	}
	sha, err := opDirDigest(files)
	if err != nil {
		t.Fatal(err)
	}
	if err := opInstall(dcn, sha, files); err != nil {
		t.Fatal(err)
	}
	return sha, m
}

// A whole ship, for real, on this machine: a build whose receipt carries the
// digest, a root roll that takes it from that receipt (adopting the live image
// as the first roll), and a check that fails; the failed check's receipt
// undoes the roll, to the image that was live before it.
func TestAShipPlanRunsEachLineOnceAndUndoesOnAFailedCheck(t *testing.T) {
	requireActionSandbox(t)
	dcn := planBox(t)
	useAncestry(t, func(_, _, _ string, _ []string) error { return nil })
	commit := strings.Repeat("7", 40)
	digest := "sha256:" + strings.Repeat("5", 64)
	old := tImage + "@sha256:" + strings.Repeat("4", 64)
	f := &fakeKube{containers: map[string]string{"bridge": old}}
	useFakeKube(t, f)

	head := `version: 1.0.0
summary: A step of the ship under test.
kind: action
target: machine
interpreter: /bin/sh
time_limit: 30
`
	res := fmt.Sprintf(`{"images":[{"image":"%s","digest":"%s","source":"%s","commit":"%s"}]}`, tImage, digest, tRepo, commit)
	buildSHA, _ := holdActionOp(t, dcn, "name: bridge-build\n"+head+"privilege: user\nparams:\n  commit: {type: string, max_len: 40}\nmodes: [dry_run, apply, verify]\nundo: none\n",
		`[ "$PRIVATAE_OP_MODE" = apply ] && printf '%s' '`+res+`' > "$PRIVATAE_OP_RESULT"; true`)
	rollSHA, rollM := holdActionOp(t, dcn, "name: bridge-roll\n"+head+"privilege: root\nparams:\n  digest: {type: string, max_len: 71}\n"+
		"verbs:\n  - {verb: k8s.set_image, namespace: ds, kind: deployment, name: bridge, container: bridge, image: "+tImage+", digest: \"{{digest}}\", repo: \""+tRepo+"\"}\n"+
		"modes: [dry_run, apply, verify]\nundo: none\n", "true")
	// The check: its apply succeeds, its verify (the probe) fails — a failed
	// CHECK, which is what may start an undo of an earlier line.
	checkSHA, _ := holdActionOp(t, dcn, "name: bridge-check\n"+head+"privilege: user\nparams: {}\nmodes: [dry_run, apply, verify]\nundo: none\n",
		`[ "$PRIVATAE_OP_MODE" = verify ] && { echo "the bridge does not answer"; exit 1; }; true`)

	hn := planHost()
	rollLine := shipPlanLine{N: 2, Step: "roll", Machine: hn, Op: "bridge-roll", OpSHA256: rollSHA, Params: `{}`,
		Inputs: map[string]string{"digest": "build.digest"}, Undo: true, AdoptLive: true}
	rollLine.Verbs = planVerbsForShow(rollM, &rollLine)
	draft := map[string]any{"kind": shipPlanKind, "plan": map[string]any{
		"pipeline": "bridge", "commit": commit, "expires": time.Now().Add(time.Hour).UTC().Format(time.RFC3339),
		"lines": []any{
			shipPlanLine{N: 1, Step: "build", Machine: hn, Op: "bridge-build", OpSHA256: buildSHA, Params: fmt.Sprintf(`{"commit":"%s"}`, commit)},
			rollLine,
			shipPlanLine{N: 3, Step: "check", Machine: hn, Op: "bridge-check", OpSHA256: checkSHA, Params: `{}`},
		}}}
	// Through JSON, as the pod sends it.
	raw, _ := json.Marshal(draft)
	var details map[string]any
	_ = json.Unmarshal(raw, &details)
	b64, p := signPlan(t, dcn, details, planCode(t))
	if !p.Root || !p.CodeGiven {
		t.Fatal("a plan with a roll is root, approved with the code")
	}
	encode := func(rs ...any) string {
		b, _ := json.Marshal(rs)
		return base64.StdEncoding.EncodeToString(b)
	}
	receiptOf := func(out map[string]any) opSignedReceipt {
		return out["receipt"].(opSignedReceipt)
	}

	// 1. The build: its receipt names the image it pushed, at the plan's commit.
	out1, err := planRunLine(dcn, b64, "1", "", nil)
	if err != nil || out1["ok"] != true {
		t.Fatalf("the build ran: %v %v", out1["error"], err)
	}
	r1 := receiptOf(out1)
	// The roll cannot run before the build's receipt is handed to it...
	if _, err := planRunLine(dcn, b64, "2", "", nil); err == nil || !strings.Contains(err.Error(), "runs only after line 1") {
		t.Fatalf("the roll before its build: %v", err)
	}
	// 2. ...and with it, rolls exactly the digest the build reported.
	out2, err := planRunLine(dcn, b64, "2", encode(r1), nil)
	if err != nil || out2["ok"] != true {
		t.Fatalf("the roll ran: %v %v", out2["error"], err)
	}
	if f.containers["bridge"] != tImage+"@"+digest {
		t.Fatalf("the workload runs the built digest: %s", f.containers["bridge"])
	}
	if _, _, c, ok := opRolledCommit(digest); !ok || c != commit {
		t.Fatal("the roll is recorded, so the next plan moves forward from it")
	}
	// A replay of the same line is refused: it succeeded, so no attempt is left
	// to it, whatever the plan allowed.
	if _, err := planRunLine(dcn, b64, "2", encode(r1), nil); err == nil || !strings.Contains(err.Error(), "already succeeded") {
		t.Fatalf("a replayed line: %v", err)
	}
	// 3. The check (after the build and the roll, their receipts with it)
	// fails, and says so in its signed receipt.
	r2 := receiptOf(out2)
	out3, err := planRunLine(dcn, b64, "3", encode(r1, r2), nil)
	if err != nil || out3["ok"] != true || !strings.Contains(fmt.Sprint(out3["verify_error"]), "verify exited 1") {
		t.Fatalf("the check ran and its verify failed: %v %v", out3, err)
	}
	// The pod that lost an answer reads it back, and nothing is spent.
	if path, _ := planReceiptsPath(p.Nonce, 3); path != "" {
		raw, _ := os.ReadFile(path)
		var kept []opSignedReceipt
		_ = json.Unmarshal(raw, &kept)
		if len(kept) != 1 {
			t.Fatalf("line 3's receipt is kept for op_plan_status: %d", len(kept))
		}
	}
	failed, _ := json.Marshal(receiptOf(out3))
	// The failed check undoes the roll, to the image live before it.
	undo, err := planUndoLine(b64, "2", base64.StdEncoding.EncodeToString(failed))
	if err != nil || undo["ok"] != true {
		t.Fatalf("the undo ran: %v %v", undo, err)
	}
	if f.containers["bridge"] != old {
		t.Fatalf("the workload is back on %s, got %s", old, f.containers["bridge"])
	}
	path, _ := planRunPath(p.Nonce, 2)
	if _, err := os.Stat(path); err != nil {
		t.Fatal("the roll's record is kept for its undo")
	}
	// A plan whose roll names a digest no step of it built (bound in its
	// params, not taken from a receipt) rolls nothing.
	foreign := "sha256:" + strings.Repeat("6", 64)
	stray := shipPlanLine{N: 1, Step: "roll", Machine: hn, Op: "bridge-roll", OpSHA256: rollSHA,
		Params: fmt.Sprintf(`{"digest":"%s"}`, foreign), AdoptLive: true}
	stray.Verbs = planVerbsForShow(rollM, &stray)
	raw, _ = json.Marshal(map[string]any{"kind": shipPlanKind, "plan": map[string]any{
		"pipeline": "bridge", "commit": commit, "expires": time.Now().Add(time.Hour).UTC().Format(time.RFC3339),
		"lines": []any{stray}}})
	details = nil
	_ = json.Unmarshal(raw, &details)
	b64b, _ := signPlan(t, dcn, details, planCodeAt(1)) // the first code is spent
	outS, err := planRunLine(dcn, b64b, "1", "", nil)
	if err != nil || outS["ok"] != false || !strings.Contains(fmt.Sprint(outS["error"]), "no step of this plan built") {
		t.Fatalf("a digest no step of the plan built: %v %v", outS["error"], err)
	}
	if f.containers["bridge"] != old {
		t.Fatalf("nothing was rolled: %s", f.containers["bridge"])
	}
}

// An undo takes back only what the line left: a workload or tree that moved
// since is someone else's change, and is left alone.
func TestAnUndoLeavesWhatMovedSinceAlone(t *testing.T) {
	setupBox(t)
	was, set, other := tImage+"@sha256:"+strings.Repeat("1", 64), tImage+"@sha256:"+strings.Repeat("2", 64), tImage+"@sha256:"+strings.Repeat("3", 64)
	f := &fakeKube{containers: map[string]string{"bridge": other}}
	useFakeKube(t, f)
	v := setImageLine("sha256:" + strings.Repeat("2", 64))
	if _, err := planUndoVerb(context.Background(), v, was, set); err == nil || !strings.Contains(err.Error(), "moved since") {
		t.Fatalf("a workload that moved since: %v", err)
	}
	if len(f.sets()) != 0 {
		t.Fatal("it was set anyway")
	}
	f.containers["bridge"] = was
	if note, err := planUndoVerb(context.Background(), v, was, set); err != nil || !strings.Contains(note, "already") {
		t.Fatalf("already back where it started: %q %v", note, err)
	}
	f.containers["bridge"] = set
	if _, err := planUndoVerb(context.Background(), v, was, set); err != nil || f.containers["bridge"] != was {
		t.Fatalf("what the line left is taken back: %v %s", err, f.containers["bridge"])
	}
}

// An undo puts the state back first and only then restarts the unit, so the
// unit starts on what was put back, never on the failed code.
func TestAnUndoRestartsTheUnitOnlyAfterPuttingTheStateBack(t *testing.T) {
	dcn := planBox(t)
	was, set := tImage+"@sha256:"+strings.Repeat("1", 64), tImage+"@sha256:"+strings.Repeat("2", 64)
	var order []string
	f := &fakeKube{containers: map[string]string{"bridge": set}}
	prevK := opKubectl
	opKubectl = func(ctx context.Context, args ...string) (string, error) {
		if len(args) >= 3 && args[2] == "set" {
			order = append(order, "set image")
		}
		return f.run(ctx, args...)
	}
	prevS := opSystemctl
	opSystemctl = func(_ context.Context, args ...string) (string, error) {
		if args[0] == "show" {
			return "LoadState=loaded\nActiveState=active\nSubState=running\nMainPID=1\n", nil
		}
		order = append(order, "systemctl "+args[0])
		return "", nil
	}
	t.Cleanup(func() { opKubectl, opSystemctl = prevK, prevS })
	d := planDraft(false)
	lines := d["plan"].(map[string]any)["lines"].([]any)
	lines = append(lines, map[string]any{"n": 3, "step": "check", "machine": planHost(), "op": "bridge-check",
		"op_sha256": strings.Repeat("e", 64), "params": `{}`})
	d["plan"].(map[string]any)["lines"] = lines
	b64, p := signPlan(t, dcn, d, "")
	raw, _ := base64.StdEncoding.DecodeString(b64)
	var e shipPlanEnvelope
	_ = json.Unmarshal(raw, &e)
	_, sha, _ := planOpen(e)
	// Line 2 set the image, then restarted the unit.
	rec := planRunRecord{Plan: p.Nonce, N: 2, Attempt: 1, State: planStatePassed,
		Verbs: []opVerbLine{setImageLine("sha256:" + strings.Repeat("2", 64)), {Verb: "unit.restart", Args: map[string]string{"unit": "app.service"}}},
		Froms: []string{was, ""}, Tos: []string{set, ""}}
	path, _ := planRunPath(p.Nonce, 2)
	body, _ := json.Marshal(rec)
	_ = os.WriteFile(path, body, 0o600)
	failed, _ := lineReceiptSign(lineReceipt{Plan: p.Nonce, PlanSHA256: sha, Pipeline: p.Pipeline, Commit: p.Commit,
		N: 3, Step: "check", Machine: planHost(), OK: true, VerifyError: "verify exited 1", Attempt: 1})
	fb, _ := json.Marshal(failed)
	out, err := planUndoLine(b64, "2", base64.StdEncoding.EncodeToString(fb))
	if err != nil || out["ok"] != true {
		t.Fatalf("the undo ran: %v %v", out, err)
	}
	if strings.Join(order, ", ") != "set image, systemctl restart" {
		t.Fatalf("the image goes back before the unit restarts: %v", order)
	}
	if f.containers["bridge"] != was {
		t.Fatal("the image is back")
	}
}

// Moving a tree back never overwrites a file the machine keeps.
func TestAnUndoOfATreeNeverOverwritesAnIgnoredFile(t *testing.T) {
	if os.Geteuid() != 0 {
		t.Skip("needs root for the tree's ownership checks")
	}
	// A name is matched as itself, never as a pathspec: ":(literal)k.env"
	// read as one names k.env, and would miss the file.
	for _, name := range []string{".env", "[ab].env", ":(literal)k.env"} {
		t.Run(name, func(t *testing.T) {
			setupBox(t)
			useFileRepos(t)
			// c1 tracks the file; c2 stops tracking it (it is ignored from then on).
			origin, tree, c1, c2 := twoCommitRepo(t, func(o string) {
				_ = os.WriteFile(filepath.Join(o, name), []byte("FROM-REPO\n"), 0o644)
				_ = os.WriteFile(filepath.Join(o, ".gitignore"), []byte("*.env\n"), 0o644)
				gitT(t, o, "--literal-pathspecs", "add", "-f", "--", name)
			}, func(o string) {
				gitT(t, o, "--literal-pathspecs", "rm", "-q", "--cached", "--", name)
				_ = os.WriteFile(filepath.Join(o, "a.txt"), []byte("two\n"), 0o644)
			})
			line := opVerbLine{Verb: "tree.checkout", Args: map[string]string{"path": tree, "repo": "file://" + origin, "commit": c2}}
			if _, err := opTreeDo(context.Background(), line, nil, ""); err != nil {
				t.Fatal(err)
			}
			_ = os.WriteFile(filepath.Join(tree, name), []byte("LOCAL-SECRET\n"), 0o600)
			if _, err := planUndoVerb(context.Background(), line, c1, c2); err == nil || !strings.Contains(err.Error(), "a file this machine keeps") {
				t.Fatalf("an undo over an ignored file: %v", err)
			}
			if b, _ := os.ReadFile(filepath.Join(tree, name)); string(b) != "LOCAL-SECRET\n" {
				t.Fatalf("the kept file was overwritten: %q", b)
			}
		})
	}
}

// shipRig is a three-line ship on this machine: build (its receipt carries
// the digest), roll (root, sets the image, undoable, two attempts), check.
// rollProgram and checkProgram are the roll's and the check's programs.
func shipRig(t *testing.T, buildProgram, rollProgram, checkProgram string, checkAttempts int) (dcn, b64 string, f *fakeKube, old, built string) {
	t.Helper()
	requireActionSandbox(t)
	dcn = planBox(t)
	useAncestry(t, func(_, _, _ string, _ []string) error { return nil })
	commit := strings.Repeat("7", 40)
	digest := "sha256:" + strings.Repeat("5", 64)
	old = tImage + "@sha256:" + strings.Repeat("4", 64)
	built = tImage + "@" + digest
	f = &fakeKube{containers: map[string]string{"bridge": old}}
	useFakeKube(t, f)
	head := "version: 1.0.0\nsummary: A step of the ship under test.\nkind: action\ntarget: machine\ninterpreter: /bin/sh\ntime_limit: 30\n"
	res := fmt.Sprintf(`{"images":[{"image":"%s","digest":"%s","source":"%s","commit":"%s"}]}`, tImage, digest, tRepo, commit)
	if buildProgram == "" {
		buildProgram = `[ "$PRIVATAE_OP_MODE" = apply ] && printf '%s' '` + res + `' > "$PRIVATAE_OP_RESULT"; true`
	} else {
		buildProgram = strings.ReplaceAll(buildProgram, "RESULT", res)
	}
	buildSHA, _ := holdActionOp(t, dcn, "name: bridge-build\n"+head+"privilege: user\nparams:\n  commit: {type: string, max_len: 40}\nmodes: [dry_run, apply, verify]\nundo: none\n", buildProgram)
	rollSHA, rollM := holdActionOp(t, dcn, "name: bridge-roll\n"+head+"privilege: root\nparams:\n  digest: {type: string, max_len: 71}\n"+
		"verbs:\n  - {verb: k8s.set_image, namespace: ds, kind: deployment, name: bridge, container: bridge, image: "+tImage+", digest: \"{{digest}}\", repo: \""+tRepo+"\"}\n"+
		"modes: [dry_run, apply, verify]\nundo: none\n", rollProgram)
	checkSHA, _ := holdActionOp(t, dcn, "name: bridge-check\n"+head+"privilege: user\nparams: {}\nmodes: [dry_run, apply, verify]\nundo: none\n", checkProgram)
	hn := planHost()
	rollLine := shipPlanLine{N: 2, Step: "roll", Machine: hn, Op: "bridge-roll", OpSHA256: rollSHA, Params: `{}`,
		Inputs: map[string]string{"digest": "build.digest"}, Undo: true, AdoptLive: true, Attempts: 2}
	rollLine.Verbs = planVerbsForShow(rollM, &rollLine)
	draft := map[string]any{"kind": shipPlanKind, "plan": map[string]any{
		"pipeline": "bridge", "commit": commit, "expires": time.Now().Add(time.Hour).UTC().Format(time.RFC3339),
		"lines": []any{
			shipPlanLine{N: 1, Step: "build", Machine: hn, Op: "bridge-build", OpSHA256: buildSHA, Params: fmt.Sprintf(`{"commit":"%s"}`, commit), Attempts: 2},
			rollLine,
			shipPlanLine{N: 3, Step: "check", Machine: hn, Op: "bridge-check", OpSHA256: checkSHA, Params: `{}`, Attempts: checkAttempts},
		}}}
	raw, _ := json.Marshal(draft)
	var details map[string]any
	_ = json.Unmarshal(raw, &details)
	b64, _ = signPlan(t, dcn, details, planCode(t))
	return dcn, b64, f, old, built
}

func rigReceipts(outs ...map[string]any) string {
	var rs []opSignedReceipt
	for _, o := range outs {
		rs = append(rs, o["receipt"].(opSignedReceipt))
	}
	b, _ := json.Marshal(rs)
	return base64.StdEncoding.EncodeToString(b)
}

func rigReceipt(o map[string]any) string {
	b, _ := json.Marshal(o["receipt"].(opSignedReceipt))
	return base64.StdEncoding.EncodeToString(b)
}

// failKubeAt makes the listed kubectl calls of a kind (set, rollout) fail,
// counted from 1.
func failKubeAt(t *testing.T, kind string, calls ...int) {
	t.Helper()
	count := 0
	prev := opKubectl
	opKubectl = func(ctx context.Context, args ...string) (string, error) {
		if len(args) >= 3 && args[2] == kind {
			count++
			for _, n := range calls {
				if count == n {
					return "boom", fmt.Errorf("exit 1")
				}
			}
		}
		return prev(ctx, args...)
	}
	t.Cleanup(func() { opKubectl = prev })
}

// Two calls for the same line at once: the second is refused while the first
// runs, and once the first passed nothing runs the line again.
func TestALineRunsOneAttemptAtATime(t *testing.T) {
	dcn, b64, _, _, _ := shipRig(t, `[ "$PRIVATAE_OP_MODE" = apply ] && { sleep 3; printf '%s' 'RESULT' > "$PRIVATAE_OP_RESULT"; }; true`, "true", "true", 1)
	type res struct {
		out map[string]any
		err error
	}
	first := make(chan res, 1)
	go func() {
		out, err := planRunLine(dcn, b64, "1", "", nil)
		first <- res{out, err}
	}()
	time.Sleep(700 * time.Millisecond)
	if _, err := planRunLine(dcn, b64, "1", "", nil); err == nil || !strings.Contains(err.Error(), "attempt 1 of line 1 is running here") {
		t.Fatalf("a second attempt while the first runs: %v", err)
	}
	r := <-first
	if r.err != nil || r.out["ok"] != true {
		t.Fatalf("the first attempt: %v %v", r.out, r.err)
	}
	if _, err := planRunLine(dcn, b64, "1", "", nil); err == nil || !strings.Contains(err.Error(), "already succeeded") {
		t.Fatalf("an attempt after one passed: %v", err)
	}
}

// A roll whose first attempt set the image and failed, and whose retry
// passed; the check then fails: the undo goes back to the image before the
// FIRST attempt, not to where the failed attempt left it — and the line, once
// undone, never runs again.
func TestAnUndoAfterARetryGoesBackToBeforeTheFirstAttempt(t *testing.T) {
	dcn, b64, f, old, built := shipRig(t, "", "true", `[ "$PRIVATAE_OP_MODE" = verify ] && { echo "the bridge does not answer"; exit 1; }; true`, 1)
	failKubeAt(t, "rollout", 1)
	b1, err := planRunLine(dcn, b64, "1", "", nil)
	if err != nil || b1["ok"] != true {
		t.Fatalf("build: %v %v", b1, err)
	}
	a1, err := planRunLine(dcn, b64, "2", rigReceipts(b1), nil)
	if err != nil || a1["ok"] != false || f.containers["bridge"] != built {
		t.Fatalf("roll attempt 1 sets the image and fails its rollout: %v %v %s", a1, err, f.containers["bridge"])
	}
	a2, err := planRunLine(dcn, b64, "2", rigReceipts(b1), nil)
	if err != nil || a2["ok"] != true {
		t.Fatalf("roll attempt 2: %v %v", a2, err)
	}
	c, err := planRunLine(dcn, b64, "3", rigReceipts(b1, a2), nil)
	if err != nil || c["verify_error"] == "" {
		t.Fatalf("the check fails: %v %v", c, err)
	}
	u, err := planUndoLine(b64, "2", rigReceipt(c))
	if err != nil || u["ok"] != true {
		t.Fatalf("the undo: %v %v", u, err)
	}
	if f.containers["bridge"] != old {
		t.Fatalf("the undo went back to %s, not to %s from before the first attempt", f.containers["bridge"], old)
	}
	if _, err := planRunLine(dcn, b64, "2", rigReceipts(b1), nil); err == nil || !strings.Contains(err.Error(), "was undone") {
		t.Fatalf("a line run after its undo: %v", err)
	}
}

// Only the latest attempt's failure undoes the line itself, and never once an
// attempt passed.
func TestOnlyTheLatestAttemptsFailureUndoesTheLine(t *testing.T) {
	t.Run("an attempt passed after it", func(t *testing.T) {
		dcn, b64, f, _, built := shipRig(t, "", "true", "true", 1)
		failKubeAt(t, "set", 1)
		b1, _ := planRunLine(dcn, b64, "1", "", nil)
		a1, err := planRunLine(dcn, b64, "2", rigReceipts(b1), nil)
		if err != nil || a1["ok"] != false {
			t.Fatalf("roll attempt 1 fails: %v %v", a1, err)
		}
		a2, err := planRunLine(dcn, b64, "2", rigReceipts(b1), nil)
		if err != nil || a2["ok"] != true {
			t.Fatalf("roll attempt 2 passes: %v %v", a2, err)
		}
		if _, err := planUndoLine(b64, "2", rigReceipt(a1)); err == nil || !strings.Contains(err.Error(), "passed here") {
			t.Fatalf("attempt 1's failure after attempt 2 passed: %v", err)
		}
		if f.containers["bridge"] != built {
			t.Fatal("the shipped image stays")
		}
	})
	t.Run("both attempts failed", func(t *testing.T) {
		dcn, b64, f, old, built := shipRig(t, "", "true", "true", 1)
		failKubeAt(t, "rollout", 1, 2)
		b1, _ := planRunLine(dcn, b64, "1", "", nil)
		a1, _ := planRunLine(dcn, b64, "2", rigReceipts(b1), nil)
		a2, err := planRunLine(dcn, b64, "2", rigReceipts(b1), nil)
		if err != nil || a1["ok"] != false || a2["ok"] != false || f.containers["bridge"] != built {
			t.Fatalf("both attempts fail with the image set: %v %v %v", a1, a2, err)
		}
		if _, err := planRunLine(dcn, b64, "2", rigReceipts(b1), nil); err == nil || !strings.Contains(err.Error(), "all it may") {
			t.Fatalf("a third attempt of two: %v", err)
		}
		if _, err := planUndoLine(b64, "2", rigReceipt(a1)); err == nil || !strings.Contains(err.Error(), "latest attempt here is 2") {
			t.Fatalf("an earlier attempt's failure: %v", err)
		}
		u, err := planUndoLine(b64, "2", rigReceipt(a2))
		if err != nil || u["ok"] != true || f.containers["bridge"] != old {
			t.Fatalf("the latest attempt's failure undoes to before the first: %v %v %s", u, err, f.containers["bridge"])
		}
	})
}

// A later line's failed check undoes an earlier line only on its last
// allowed attempt: a retry of the check may still pass.
func TestALaterLinesCheckUndoesOnlyOnItsLastAttempt(t *testing.T) {
	dcn, b64, f, old, _ := shipRig(t, "", "true", `[ "$PRIVATAE_OP_MODE" = verify ] && exit 1; true`, 2)
	b1, _ := planRunLine(dcn, b64, "1", "", nil)
	r1, _ := planRunLine(dcn, b64, "2", rigReceipts(b1), nil)
	c1, err := planRunLine(dcn, b64, "3", rigReceipts(b1, r1), nil)
	if err != nil || c1["verify_error"] == "" {
		t.Fatalf("check attempt 1 fails: %v %v", c1, err)
	}
	if _, err := planUndoLine(b64, "2", rigReceipt(c1)); err == nil || !strings.Contains(err.Error(), "only its last attempt") {
		t.Fatalf("attempt 1 of 2 of the check: %v", err)
	}
	c2, err := planRunLine(dcn, b64, "3", rigReceipts(b1, r1), nil)
	if err != nil || c2["verify_error"] == "" {
		t.Fatalf("check attempt 2 fails: %v %v", c2, err)
	}
	if u, err := planUndoLine(b64, "2", rigReceipt(c2)); err != nil || u["ok"] != true || f.containers["bridge"] != old {
		t.Fatalf("the last attempt's failed check undoes the roll: %v %v", u, err)
	}
}

// A verb the machine cannot record for its undo does not run.
func TestAVerbThatCannotBeRecordedDoesNotRun(t *testing.T) {
	dcn, b64, f, old, _ := shipRig(t, "", `[ "$PRIVATAE_OP_MODE" = apply ] && sleep 2; true`, "true", 1)
	b1, _ := planRunLine(dcn, b64, "1", "", nil)
	p, _, _, _ := planDecodeArgs([]string{b64, "2"})
	go func() {
		// While the roll's program runs, the record stops being this attempt's.
		time.Sleep(700 * time.Millisecond)
		planLineMu.Lock()
		defer planLineMu.Unlock()
		rec, _ := planLoadRun(p.Nonce, 2)
		rec.Attempt = 9
		_ = planSaveRun(rec)
	}()
	r, err := planRunLine(dcn, b64, "2", rigReceipts(b1), nil)
	if err != nil || r["ok"] != false || !strings.Contains(fmt.Sprint(r["error"]), "could not be recorded") {
		t.Fatalf("the roll: %v %v", r, err)
	}
	if f.containers["bridge"] != old {
		t.Fatal("the image was set with no record of it")
	}
}

// An attempt the bastion died in: settled at startup from its kept receipt,
// or, with none, as failed with a signed receipt saying so; the next attempt
// may then run, and the pod reads the settled receipt.
func TestAnAttemptTheBastionDiedInIsSettled(t *testing.T) {
	dcn := planBox(t)
	b64, p := signPlan(t, dcn, planDraft(false), "")
	raw, _ := base64.StdEncoding.DecodeString(b64)
	var e shipPlanEnvelope
	_ = json.Unmarshal(raw, &e)
	_, sha, _ := planOpen(e)
	l := &p.Lines[1]
	l.Attempts = 3
	tmpl := lineReceipt{Plan: p.Nonce, PlanSHA256: sha, Pipeline: p.Pipeline, Commit: p.Commit, N: 2, Step: l.Step, Machine: planHost()}
	k, err := planBeginAttempt(p, l, tmpl)
	if err != nil || k != 1 {
		t.Fatalf("attempt 1: %d %v", k, err)
	}
	if _, err := planBeginAttempt(p, l, tmpl); err == nil || !strings.Contains(err.Error(), "is running here") {
		t.Fatalf("a second attempt while this process runs the first: %v", err)
	}
	died := func() {
		planLineMu.Lock()
		defer planLineMu.Unlock()
		rec, _ := planLoadRun(p.Nonce, 2)
		rec.Proc = "gone:1:1"
		_ = planSaveRun(rec)
	}
	died()
	planSettleStopped()
	got := planKeptReceipt(p.Nonce, 2, 1)
	if got == nil || !got.failed() || !strings.Contains(got.Error, "stopped when the bastion") {
		t.Fatalf("the stopped attempt's receipt: %+v", got)
	}
	if k, err := planBeginAttempt(p, l, tmpl); err != nil || k != 2 {
		t.Fatalf("attempt 2 after a settled attempt 1: %d %v", k, err)
	}
	// Attempt 2 passed and its receipt was kept, and the process died before
	// it recorded the end: settled as passed, from the receipt.
	pass := tmpl
	pass.Attempt, pass.OK = 2, true
	s, _ := lineReceiptSign(pass)
	if err := planKeepReceipt(p.Nonce, 2, s); err != nil {
		t.Fatal(err)
	}
	died()
	if _, err := planBeginAttempt(p, l, tmpl); err == nil || !strings.Contains(err.Error(), "already succeeded") {
		t.Fatalf("an attempt after a passed one the bastion died before recording: %v", err)
	}
}

// The undo's rules against the line's record, each on its own.
func TestAnUndoMatchesTheReceiptToTheLinesRecord(t *testing.T) {
	cases := []struct {
		name  string
		rec   planRunRecord
		fr    lineReceipt
		later bool
		want  string
	}{
		{"while an attempt runs", planRunRecord{N: 2, Attempt: 1, State: planStateRunning}, lineReceipt{N: 3, Attempt: 1, OK: true, VerifyError: "x"}, true, "running here"},
		{"twice", planRunRecord{N: 2, Attempt: 1, State: planStateUndone}, lineReceipt{N: 2, Attempt: 1}, false, "undone here already"},
		{"a later failure of a line that did not pass", planRunRecord{N: 2, Attempt: 2, State: planStateFailed}, lineReceipt{N: 3, Attempt: 1, OK: true, VerifyError: "x"}, true, "did not pass here"},
		{"its own failure once an attempt passed", planRunRecord{N: 2, Attempt: 2, State: planStatePassed}, lineReceipt{N: 2, Attempt: 2}, false, "passed here"},
		{"an earlier attempt's failure", planRunRecord{N: 2, Attempt: 2, State: planStateFailed}, lineReceipt{N: 2, Attempt: 1}, false, "latest attempt here is 2"},
	}
	for _, c := range cases {
		if err := planUndoAllowed(&c.rec, &c.fr, c.later); err == nil || !strings.Contains(err.Error(), c.want) {
			t.Errorf("%s: %v", c.name, err)
		}
	}
	if err := planUndoAllowed(&planRunRecord{N: 2, Attempt: 2, State: planStateFailed}, &lineReceipt{N: 2, Attempt: 2}, false); err != nil {
		t.Errorf("the latest attempt's failure: %v", err)
	}
	if err := planUndoAllowed(&planRunRecord{N: 2, Attempt: 1, State: planStatePassed}, &lineReceipt{N: 3, Attempt: 1, OK: true, VerifyError: "x"}, true); err != nil {
		t.Errorf("a later failure of a line that passed: %v", err)
	}
}

// An attempt the spent ledger and the record disagree on does not run, and a
// line's record outlives its plan only by planKeepAfterExpiry.
func TestALinesRecordAndLedgerAgreeAndAreLetGo(t *testing.T) {
	dcn := planBox(t)
	_, p := signPlan(t, dcn, planDraft(false), "")
	l := &p.Lines[1]
	l.Attempts = 2
	if _, err := planSpendAttempt(p, l); err != nil {
		t.Fatal(err)
	}
	if _, err := planBeginAttempt(p, l, lineReceipt{}); err == nil || !strings.Contains(err.Error(), "out of order") {
		t.Fatalf("an attempt spent with no record of it: %v", err)
	}
	keep := func(nonce, exp string) (string, string) {
		rec := &planRunRecord{Plan: nonce, N: 1, Attempt: 1, State: planStatePassed, Expires: exp}
		if err := planSaveRun(rec); err != nil {
			t.Fatal(err)
		}
		_ = planKeepReceipt(nonce, 1, opSignedReceipt{})
		a, _ := planRunPath(nonce, 1)
		b, _ := planReceiptsPath(nonce, 1)
		return a, b
	}
	oldRec, oldRcpt := keep("oldplan", time.Now().Add(-planKeepAfterExpiry-time.Hour).UTC().Format(time.RFC3339))
	newRec, newRcpt := keep("newplan", time.Now().Add(-time.Hour).UTC().Format(time.RFC3339))
	planSettleStopped()
	for _, f := range []string{oldRec, oldRcpt} {
		if _, err := os.Stat(f); err == nil {
			t.Errorf("%s outlived its plan", f)
		}
	}
	for _, f := range []string{newRec, newRcpt} {
		if _, err := os.Stat(f); err != nil {
			t.Errorf("%s was let go early", f)
		}
	}
}

// A retry whose target something else moved since the first attempt does not
// run: an undo to before that attempt would take the other change back.
func TestARetryLeavesATargetSomethingElseMovedAlone(t *testing.T) {
	dcn, b64, f, _, _ := shipRig(t, "", "true", "true", 1)
	failKubeAt(t, "set", 1)
	b1, _ := planRunLine(dcn, b64, "1", "", nil)
	a1, err := planRunLine(dcn, b64, "2", rigReceipts(b1), nil)
	if err != nil || a1["ok"] != false {
		t.Fatalf("roll attempt 1 fails before it sets anything: %v %v", a1, err)
	}
	// Another pipeline rolls the workload (a security fix, say).
	fix := tImage + "@sha256:" + strings.Repeat("3", 64)
	f.containers["bridge"] = fix
	a2, err := planRunLine(dcn, b64, "2", rigReceipts(b1), nil)
	if err != nil || a2["ok"] != false || !strings.Contains(fmt.Sprint(a2["error"]), "something else moved it since") {
		t.Fatalf("the retry over a moved target: %v %v", a2, err)
	}
	if f.containers["bridge"] != fix {
		t.Fatalf("the retry rolled over the other change: %s", f.containers["bridge"])
	}
	u, err := planUndoLine(b64, "2", rigReceipt(a2))
	if err != nil {
		t.Fatalf("the undo on the retry's failure: %v", err)
	}
	if u["ok"] != false || f.containers["bridge"] != fix {
		t.Fatalf("the undo took the other change back: %v %s", u, f.containers["bridge"])
	}
}

// Only a process known to be gone settles its attempt: a /proc that cannot
// be read is no answer.
func TestOnlyAProcessKnownToBeGoneIsSettled(t *testing.T) {
	if planProc == "" || planProcGone(planProc) {
		t.Fatalf("this process reads as gone: %q", planProc)
	}
	parts := strings.Split(planProc, ":")
	if !planProcGone(parts[0] + ":" + parts[1] + ":1") {
		t.Error("another process under this pid is not this one")
	}
	if !planProcGone(parts[0] + ":999999999:1") {
		t.Error("a pid that is not there")
	}
	prev := planProcRoot
	t.Cleanup(func() { planProcRoot = prev })
	planProcRoot = t.TempDir() // no boot id, no processes: nothing reads
	if planProcGone(planProc) || planProcGone(parts[0]+":999999999:1") {
		t.Error("a /proc that does not read settled an attempt")
	}
}

// A target something else moves while the line's program runs is left
// alone: the verb refuses the state it finds, so no undo can take the other
// change back.
func TestATargetMovedWhileTheProgramRunsIsLeftAlone(t *testing.T) {
	dcn, b64, f, _, _ := shipRig(t, "", `[ "$PRIVATAE_OP_MODE" = apply ] && sleep 3; true`, `[ "$PRIVATAE_OP_MODE" = verify ] && exit 1; true`, 1)
	b1, _ := planRunLine(dcn, b64, "1", "", nil)
	fix := tImage + "@sha256:" + strings.Repeat("3", 64)
	var mu sync.Mutex
	prev := opKubectl
	opKubectl = func(ctx context.Context, args ...string) (string, error) {
		mu.Lock()
		defer mu.Unlock()
		return prev(ctx, args...)
	}
	t.Cleanup(func() { opKubectl = prev })
	go func() {
		time.Sleep(1500 * time.Millisecond)
		mu.Lock()
		f.containers["bridge"] = fix // another pipeline rolls it
		mu.Unlock()
	}()
	r, err := planRunLine(dcn, b64, "2", rigReceipts(b1), nil)
	if err != nil || r["ok"] != false || !strings.Contains(fmt.Sprint(r["error"]), "something else moved it since") {
		t.Fatalf("the roll over a target moved while its program ran: %v %v", r, err)
	}
	if f.containers["bridge"] != fix {
		t.Fatalf("the roll set over the other change: %s", f.containers["bridge"])
	}
	// Its own failure is the latest attempt's: the undo finds nothing of
	// this line on the workload, and leaves the other change in place.
	if _, err := planUndoLine(b64, "2", rigReceipt(r)); err != nil {
		t.Fatalf("the undo: %v", err)
	}
	if f.containers["bridge"] != fix {
		t.Fatalf("the undo took the other change back: %s", f.containers["bridge"])
	}
}

// A verb refuses a target in any other state than the one it was approved
// and recorded to move from.
func TestAVerbMovesOnlyFromTheStateItWasApprovedToMoveFrom(t *testing.T) {
	if err := opMovedSince("a", "a"); err != nil {
		t.Errorf("the approved state: %v", err)
	}
	if err := opMovedSince("b", ""); err != nil {
		t.Errorf("a verb with no state (a unit): %v", err)
	}
	if err := opMovedSince("b", "a"); err == nil {
		t.Error("another state")
	}
	setupBox(t)
	live, next := "sha256:"+strings.Repeat("1", 64), "sha256:"+strings.Repeat("2", 64)
	issueReceipt(t, live, strings.Repeat("a", 40))
	issueReceipt(t, next, strings.Repeat("b", 40))
	if err := opRecordRoll(live, opRollRec{Image: tImage, Source: tRepo, Commit: strings.Repeat("a", 40)}); err != nil {
		t.Fatal(err)
	}
	useAncestry(t, func(_, _, _ string, _ []string) error { return nil })
	f := &fakeKube{containers: map[string]string{"bridge": tImage + "@" + live}}
	useFakeKube(t, f)
	if _, err := opImageDo(context.Background(), setImageLine(next), nil, nil, tImage+"@sha256:"+strings.Repeat("9", 64)); err == nil || !strings.Contains(err.Error(), "something else moved it since") || len(f.sets()) != 0 {
		t.Fatalf("an image in another state than approved: %v %v", err, f.sets())
	}
	// Something else rolls it while the ancestry fetch runs: read again
	// right before the write, and nothing is set.
	fix := tImage + "@sha256:" + strings.Repeat("3", 64)
	useAncestry(t, func(_, _, _ string, _ []string) error { f.containers["bridge"] = fix; return nil })
	if _, err := opImageDo(context.Background(), setImageLine(next), nil, nil, tImage+"@"+live); err == nil || !strings.Contains(err.Error(), "something else moved it since") || len(f.sets()) != 0 || f.containers["bridge"] != fix {
		t.Fatalf("an image rolled while the ancestry was fetched: %v %v", err, f.sets())
	}
	f.containers["bridge"] = tImage + "@" + live
	useAncestry(t, func(_, _, _ string, _ []string) error { return nil })
	if _, err := opImageDo(context.Background(), setImageLine(next), nil, nil, tImage+"@"+live); err != nil || len(f.sets()) != 1 {
		t.Fatalf("the image in the approved state: %v %v", err, f.sets())
	}
}

// The same for a tree: a move approved from another commit than the tree is
// at does not run.
func TestATreeMovesOnlyFromTheCommitItWasApprovedToMoveFrom(t *testing.T) {
	if os.Geteuid() != 0 {
		t.Skip("needs root for the tree's ownership checks")
	}
	setupBox(t)
	useFileRepos(t)
	origin, tree, c1, c2 := twoCommitRepo(t, func(string) {}, func(o string) {
		_ = os.WriteFile(filepath.Join(o, "a.txt"), []byte("two\n"), 0o644)
	})
	line := opVerbLine{Verb: "tree.checkout", Args: map[string]string{"path": tree, "repo": "file://" + origin, "commit": c2}}
	if _, err := opTreeDo(context.Background(), line, nil, strings.Repeat("9", 40)); err == nil || !strings.Contains(err.Error(), "something else moved it since") {
		t.Fatalf("a tree at another commit than approved: %v", err)
	}
	if head := gitT(t, tree, "rev-parse", "HEAD"); head != c1 {
		t.Fatalf("the tree moved: %s", head)
	}
	// Something else moves it while the fetch runs: read again right before
	// the move, and it is left where the other change put it.
	prev := opTreeBeforeMove
	var other string
	opTreeBeforeMove = func(path string) {
		gitT(t, path, "commit", "-q", "--allow-empty", "-m", "someone else")
		other = gitT(t, path, "rev-parse", "HEAD")
	}
	if _, err := opTreeDo(context.Background(), line, nil, c1); err == nil || !strings.Contains(err.Error(), "something else moved it since") {
		t.Fatalf("a tree moved while the fetch ran: %v", err)
	}
	opTreeBeforeMove = prev
	if head := gitT(t, tree, "rev-parse", "HEAD"); head != other {
		t.Fatalf("the other change was moved over: %s", head)
	}
	gitT(t, tree, "reset", "-q", "--keep", c1)
	if _, err := opTreeDo(context.Background(), line, nil, c1); err != nil {
		t.Fatalf("the tree at the approved commit: %v", err)
	}
}

// Two changes this bastion makes to one workload never interleave between a
// read and a write: the second waits for the first, then finds it moved.
func TestTwoChangesToOneTargetNeverInterleave(t *testing.T) {
	setupBox(t)
	live, a, b := "sha256:"+strings.Repeat("1", 64), "sha256:"+strings.Repeat("2", 64), "sha256:"+strings.Repeat("3", 64)
	issueReceipt(t, live, strings.Repeat("a", 40))
	issueReceipt(t, a, strings.Repeat("b", 40))
	issueReceipt(t, b, strings.Repeat("c", 40))
	if err := opRecordRoll(live, opRollRec{Image: tImage, Source: tRepo, Commit: strings.Repeat("a", 40)}); err != nil {
		t.Fatal(err)
	}
	useAncestry(t, func(_, _, _ string, _ []string) error { return nil })
	f := &fakeKube{containers: map[string]string{"bridge": tImage + "@" + live}}
	var mu sync.Mutex
	gets := 0
	paused, release := make(chan struct{}), make(chan struct{})
	prev := opKubectl
	opKubectl = func(ctx context.Context, args ...string) (string, error) {
		mu.Lock()
		out, err := f.run(ctx, args...)
		pause := false
		if len(args) >= 3 && args[2] == "get" {
			gets++
			pause = gets == 2 // the first roll's second read, one command before its write
		}
		mu.Unlock()
		if pause {
			close(paused)
			<-release
		}
		return out, err
	}
	t.Cleanup(func() { opKubectl = prev })
	first := make(chan error, 1)
	go func() {
		_, err := opImageDo(context.Background(), setImageLine(a), nil, nil, tImage+"@"+live)
		first <- err
	}()
	<-paused
	second := make(chan error, 1)
	go func() {
		_, err := opImageDo(context.Background(), setImageLine(b), nil, nil, tImage+"@"+live)
		second <- err
	}()
	time.Sleep(300 * time.Millisecond)
	mu.Lock()
	early := gets
	mu.Unlock()
	close(release)
	if err := <-first; err != nil {
		t.Fatalf("the first roll: %v", err)
	}
	err := <-second
	if early != 2 {
		t.Fatalf("the second roll read the workload while the first was between its read and its write (%d reads)", early)
	}
	if err == nil || !strings.Contains(err.Error(), "something else moved it since") || f.containers["bridge"] != tImage+"@"+a {
		t.Fatalf("the second roll after the first: %v, live %s", err, f.containers["bridge"])
	}
}

// An undo reads its target again right before it writes: a change made
// since its first read is left alone.
func TestAnUndoReadsItsTargetAgainBeforeItWrites(t *testing.T) {
	t.Run("an image", func(t *testing.T) {
		from, to := tImage+"@sha256:"+strings.Repeat("1", 64), tImage+"@sha256:"+strings.Repeat("2", 64)
		fix := tImage + "@sha256:" + strings.Repeat("3", 64)
		f := &fakeKube{containers: map[string]string{"bridge": to}}
		gets := 0
		prev := opKubectl
		opKubectl = func(ctx context.Context, args ...string) (string, error) {
			out, err := f.run(ctx, args...)
			if len(args) >= 3 && args[2] == "get" {
				if gets++; gets == 1 {
					f.containers["bridge"] = fix // rolled right after the undo's first read
				}
			}
			return out, err
		}
		t.Cleanup(func() { opKubectl = prev })
		if _, err := planUndoVerb(context.Background(), setImageLine("sha256:"+strings.Repeat("2", 64)), from, to); err == nil || !strings.Contains(err.Error(), "something else moved it since") {
			t.Fatalf("an undo over a roll made after its first read: %v", err)
		}
		if f.containers["bridge"] != fix || len(f.sets()) != 0 {
			t.Fatalf("the other roll was taken back: %s %v", f.containers["bridge"], f.sets())
		}
	})
	t.Run("a tree", func(t *testing.T) {
		if os.Geteuid() != 0 {
			t.Skip("needs root for the tree's ownership checks")
		}
		setupBox(t)
		useFileRepos(t)
		origin, tree, c1, c2 := twoCommitRepo(t, func(string) {}, func(o string) {
			_ = os.WriteFile(filepath.Join(o, "a.txt"), []byte("two\n"), 0o644)
		})
		line := opVerbLine{Verb: "tree.checkout", Args: map[string]string{"path": tree, "repo": "file://" + origin, "commit": c2}}
		if _, err := opTreeDo(context.Background(), line, nil, c1); err != nil {
			t.Fatal(err)
		}
		prev := opTreeBeforeMove
		t.Cleanup(func() { opTreeBeforeMove = prev })
		var other string
		opTreeBeforeMove = func(path string) {
			gitT(t, path, "commit", "-q", "--allow-empty", "-m", "someone else")
			other = gitT(t, path, "rev-parse", "HEAD")
		}
		if _, err := planUndoVerb(context.Background(), line, c1, c2); err == nil || !strings.Contains(err.Error(), "something else moved it since") {
			t.Fatalf("an undo over a commit made after its first read: %v", err)
		}
		if head := gitT(t, tree, "rev-parse", "HEAD"); head != other {
			t.Fatalf("the other commit was taken back: %s", head)
		}
	})
}

// A tree move and an undo wait for a change in flight on the same target.
func TestAChangeWaitsForOneInFlightOnTheSameTarget(t *testing.T) {
	waits := func(t *testing.T, line opVerbLine, run func() error, reached func() bool) {
		t.Helper()
		unlock := opLockTarget(line)
		done := make(chan error, 1)
		go func() { done <- run() }()
		time.Sleep(300 * time.Millisecond)
		if reached() {
			unlock()
			<-done
			t.Fatal("it read the target while another change held it")
		}
		unlock()
		select {
		case <-done:
		case <-time.After(30 * time.Second):
			t.Fatal("it never ran once the other change was done")
		}
	}
	t.Run("an image undo", func(t *testing.T) {
		line := setImageLine("sha256:" + strings.Repeat("2", 64))
		f := &fakeKube{containers: map[string]string{"bridge": tImage + "@sha256:" + strings.Repeat("2", 64)}}
		var mu sync.Mutex
		prev := opKubectl
		opKubectl = func(ctx context.Context, args ...string) (string, error) {
			mu.Lock()
			defer mu.Unlock()
			return f.run(ctx, args...)
		}
		t.Cleanup(func() { opKubectl = prev })
		waits(t, line, func() error {
			_, err := planUndoVerb(context.Background(), line, tImage+"@sha256:"+strings.Repeat("1", 64), tImage+"@sha256:"+strings.Repeat("2", 64))
			return err
		}, func() bool { mu.Lock(); defer mu.Unlock(); return len(f.calls) > 0 })
	})
	for _, undo := range []bool{false, true} {
		name := "a tree move"
		if undo {
			name = "a tree undo"
		}
		t.Run(name, func(t *testing.T) {
			if os.Geteuid() != 0 {
				t.Skip("needs root for the tree's ownership checks")
			}
			setupBox(t)
			useFileRepos(t)
			origin, tree, c1, c2 := twoCommitRepo(t, func(string) {}, func(o string) {
				_ = os.WriteFile(filepath.Join(o, "a.txt"), []byte("two\n"), 0o644)
			})
			line := opVerbLine{Verb: "tree.checkout", Args: map[string]string{"path": tree, "repo": "file://" + origin, "commit": c2}}
			if undo {
				if _, err := opTreeDo(context.Background(), line, nil, c1); err != nil {
					t.Fatal(err)
				}
			}
			var mu sync.Mutex
			hit := false
			prev := opTreeBeforeMove
			t.Cleanup(func() { opTreeBeforeMove = prev })
			opTreeBeforeMove = func(string) { mu.Lock(); hit = true; mu.Unlock() }
			waits(t, line, func() error {
				if undo {
					_, err := planUndoVerb(context.Background(), line, c1, c2)
					return err
				}
				_, err := opTreeDo(context.Background(), line, nil, c1)
				return err
			}, func() bool { mu.Lock(); defer mu.Unlock(); return hit })
		})
	}
}