bastion 0.53.2
This commit is contained in:
Privatae
@@ -0,0 +1,854 @@
|
||||
//go:build linux
|
||||
|
||||
package main
|
||||
|
||||
// ops_plan_linux.go — a machine running its lines of a signed ship plan
|
||||
// (ops_plan.go), and undoing one.
|
||||
//
|
||||
// op_plan_line <plan_b64> <n> [receipts_b64] [sealed credential ...]
|
||||
// op_plan_undo <plan_b64> <n> <failed_receipt_b64>
|
||||
//
|
||||
// A line runs as an action op's apply (ops_verbs_linux.go) with the plan as
|
||||
// its approval: no change list is bound (the digest a build produces cannot
|
||||
// be known when the person approves), and the verbs' own rules still hold —
|
||||
// a tree only forward, an image only from a receipt of this plan and only
|
||||
// forward from what this machine rolled, unless the line adopts the live
|
||||
// image. Every run, success or not, ends in a line receipt signed with this
|
||||
// machine's key: a failed one is what a machine needs to undo an earlier line.
|
||||
//
|
||||
// A line's attempts run one at a time, each only after the one before it
|
||||
// failed, and never after one passed: the machine keeps one record per line
|
||||
// (planRunRecord), written before an attempt starts and before each of its
|
||||
// verbs runs, and that record — not the pod — says which attempt is next.
|
||||
//
|
||||
// An undo takes a line back to the state THIS machine recorded before the
|
||||
// line's FIRST attempt, and nothing else: the tree to its old commit, the
|
||||
// image to its old reference, each unit the line restarted restarted again.
|
||||
// It needs a failed check of a LATER line of the same plan on that line's
|
||||
// last allowed attempt, or the failure of this line's own latest attempt when
|
||||
// none of its attempts passed — a receipt signed by the key the plan names
|
||||
// for that line's machine — and it is spent once.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/websocket"
|
||||
)
|
||||
|
||||
const planRunsDirName = "plan_runs"
|
||||
|
||||
// A line's state on this machine. Only a failed line runs again; passed and
|
||||
// undone are final.
|
||||
const (
|
||||
planStateRunning = "running"
|
||||
planStatePassed = "passed"
|
||||
planStateFailed = "failed"
|
||||
planStateUndone = "undone"
|
||||
)
|
||||
|
||||
// planKeepAfterExpiry: how long a line's record and receipts outlive their
|
||||
// plan, for op_plan_status (an expired plan runs and undoes nothing).
|
||||
const planKeepAfterExpiry = 7 * 24 * time.Hour
|
||||
|
||||
// planRunRecord is what this machine keeps of a line: which attempt is the
|
||||
// latest and how it ended, and every verb any attempt ran, for its undo.
|
||||
type planRunRecord struct {
|
||||
Plan string `json:"plan"`
|
||||
N int `json:"n"`
|
||||
At string `json:"at"`
|
||||
Expires string `json:"expires"`
|
||||
Attempt int `json:"attempt"` // the latest attempt started here
|
||||
State string `json:"state"`
|
||||
// The bastion process running the attempt (planProcAt): an attempt whose
|
||||
// process is gone was stopped by a restart, and is settled as failed.
|
||||
Proc string `json:"proc,omitempty"`
|
||||
// The receipt an attempt stopped by a restart is settled with.
|
||||
Receipt lineReceipt `json:"receipt"`
|
||||
// Every verb any attempt ran, in the order first run, each with the state
|
||||
// it was in before the line's first attempt touched it and the state the
|
||||
// latest attempt was to leave it in.
|
||||
Verbs []opVerbLine `json:"verbs"`
|
||||
Froms []string `json:"froms"`
|
||||
Tos []string `json:"tos"`
|
||||
}
|
||||
|
||||
// planLineMu orders the reads and writes of the line records.
|
||||
var planLineMu sync.Mutex
|
||||
|
||||
// planProc names this bastion process: the boot, the pid, and its start
|
||||
// time, so a pid reused later, or after a reboot, is another name. The pid is
|
||||
// the one /proc shows (read from /proc/self), so the name and the check that
|
||||
// the process is alive read the same /proc even in another pid namespace.
|
||||
var planProc, _ = planProcAt("self")
|
||||
|
||||
// planProcRoot is /proc (a test points it elsewhere).
|
||||
var planProcRoot = "/proc"
|
||||
|
||||
// planProcAt names a process as planProc does. Only a process that is not
|
||||
// there returns an error errors.Is(os.ErrNotExist); any other failure to read
|
||||
// is not an answer.
|
||||
func planProcAt(which string) (string, error) {
|
||||
boot, err := os.ReadFile(filepath.Join(planProcRoot, "sys/kernel/random/boot_id"))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the boot id does not read: %v", err)
|
||||
}
|
||||
stat, err := os.ReadFile(filepath.Join(planProcRoot, which, "stat"))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
// The pid, then the command in parentheses (which may hold anything),
|
||||
// then the fields after it: the start time is field 22, the 20th after.
|
||||
s := string(stat)
|
||||
i := strings.LastIndexByte(s, ')')
|
||||
j := strings.IndexByte(s, ' ')
|
||||
f := []string{}
|
||||
if i >= 0 {
|
||||
f = strings.Fields(s[i+1:])
|
||||
}
|
||||
if i < 0 || j < 0 || j > i || len(f) < 20 {
|
||||
return "", errors.New("the process's stat does not read")
|
||||
}
|
||||
return strings.TrimSpace(string(boot)) + ":" + s[:j] + ":" + f[19], nil
|
||||
}
|
||||
|
||||
// planProcGone: the process an attempt was running in is known to be gone —
|
||||
// not there, or another process under its pid. A /proc that cannot be read
|
||||
// is not known, and never settles an attempt.
|
||||
func planProcGone(id string) bool {
|
||||
parts := strings.Split(id, ":")
|
||||
if len(parts) != 3 {
|
||||
return true // not a name planProcAt makes: no process runs under it
|
||||
}
|
||||
if _, err := strconv.Atoi(parts[1]); err != nil {
|
||||
return true
|
||||
}
|
||||
now, err := planProcAt(parts[1])
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return true
|
||||
}
|
||||
return err == nil && now != id
|
||||
}
|
||||
|
||||
func planRunPath(nonce string, n int) (string, error) {
|
||||
d, err := configDir()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
dir := filepath.Join(d, planRunsDirName)
|
||||
if err := os.MkdirAll(dir, 0o700); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !printableASCII(nonce) || strings.ContainsAny(nonce, "/\\.") {
|
||||
return "", errors.New("a plan nonce that cannot name a file")
|
||||
}
|
||||
return filepath.Join(dir, nonce+"-"+strconv.Itoa(n)+".json"), nil
|
||||
}
|
||||
|
||||
// planDecodeArgs reads the plan envelope and the line number.
|
||||
func planDecodeArgs(args []string) (*shipPlan, string, int, error) {
|
||||
raw, err := base64.StdEncoding.DecodeString(strings.TrimSpace(args[0]))
|
||||
if err != nil {
|
||||
return nil, "", 0, errors.New("the plan does not decode")
|
||||
}
|
||||
var env shipPlanEnvelope
|
||||
if err := json.Unmarshal(raw, &env); err != nil {
|
||||
return nil, "", 0, errors.New("the plan is not an envelope")
|
||||
}
|
||||
p, sha, err := planOpen(env)
|
||||
if err != nil {
|
||||
return nil, "", 0, err
|
||||
}
|
||||
n, err := strconv.Atoi(strings.TrimSpace(args[1]))
|
||||
if err != nil {
|
||||
return nil, "", 0, errors.New("the line is not a number")
|
||||
}
|
||||
return p, sha, n, nil
|
||||
}
|
||||
|
||||
func planDecodeReceipts(b64 string) ([]opSignedReceipt, error) {
|
||||
if strings.TrimSpace(b64) == "" {
|
||||
return nil, nil
|
||||
}
|
||||
raw, err := base64.StdEncoding.DecodeString(strings.TrimSpace(b64))
|
||||
if err != nil {
|
||||
return nil, errors.New("the receipts do not decode")
|
||||
}
|
||||
var out []opSignedReceipt
|
||||
if err := json.Unmarshal(raw, &out); err != nil {
|
||||
return nil, errors.New("the receipts are not a list")
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func handlePlanLine(ws *websocket.Conn, dir string, args []string) {
|
||||
if dir == "" || len(args) < 2 {
|
||||
opFail(ws, "op_plan_line: need <plan_b64> <n> [receipts_b64] [sealed credential ...]")
|
||||
return
|
||||
}
|
||||
recB64 := ""
|
||||
if len(args) > 2 {
|
||||
recB64 = args[2]
|
||||
}
|
||||
var creds []string
|
||||
if len(args) > 3 {
|
||||
creds = args[3:]
|
||||
}
|
||||
out, err := planRunLine(dir, args[0], args[1], recB64, creds)
|
||||
if err != nil {
|
||||
opFail(ws, "op_plan_line refused: "+err.Error())
|
||||
return
|
||||
}
|
||||
b, _ := json.Marshal(out)
|
||||
sendMgmt(ws, 'O', b)
|
||||
sendMgmtExit(ws, 0)
|
||||
}
|
||||
|
||||
// planRunLine checks a line and runs it. A refusal before the run returns an
|
||||
// error (nothing ran, nothing is spent unless the spend itself was reached);
|
||||
// a run that ran returns its signed receipt, successful or not.
|
||||
func planRunLine(dir, planB64, nArg, recB64 string, credBlobs []string) (map[string]any, error) {
|
||||
p, sha, n, err := planDecodeArgs([]string{planB64, nArg})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
l, err := planLineFor(p, n)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
receipts, err := planDecodeReceipts(recB64)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
params, steps, err := planInputs(p, sha, l, receipts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m, err := opLoad(dir, l.OpSHA256)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("this machine does not hold the op %s@%s (op_sync it first): %v", l.Op, truncate(l.OpSHA256, 12), err)
|
||||
}
|
||||
if err := planLineMatchesOp(l, m, params); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := opOpenCreds(credBlobs, m, p.Nonce, false); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if ok, why := opActionAvailable(); !ok {
|
||||
return nil, errors.New("this machine cannot run an action op: " + why)
|
||||
}
|
||||
if err := planRecordVersion(p.Pipeline, p.Version); err != nil {
|
||||
return nil, fmt.Errorf("could not record the plan's version: %v", err)
|
||||
}
|
||||
hn, _ := os.Hostname()
|
||||
lr := lineReceipt{Plan: p.Nonce, PlanSHA256: sha, Pipeline: p.Pipeline, Commit: p.Commit, N: n, Step: l.Step,
|
||||
Machine: hn, OpSHA256: l.OpSHA256, ParamsSHA256: planParamsSHA(params)}
|
||||
// One attempt at a time, the next only after the last failed, and spent
|
||||
// before it runs, as an approval is.
|
||||
attempt, err := planBeginAttempt(p, l, lr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
lr.Attempt = attempt
|
||||
id, _ := newID()
|
||||
t := &opTerms{Kind: opKindRun, Op: m.Name, Version: m.Version, SHA256: l.OpSHA256, Privilege: m.Privilege,
|
||||
Params: params, Nonce: p.Nonce, Uses: 1}
|
||||
pc := &opPlanCtx{Commit: p.Commit, AdoptLive: l.AdoptLive, Steps: steps,
|
||||
beforeVerb: func(v opVerbLine, from, to string) error { return planRecordVerb(p.Nonce, n, attempt, v, from, to) }}
|
||||
rep, runErr := opRunAction(opStoreDir(dir, l.OpSHA256), m, t, opModeApply, id, credBlobs, pc)
|
||||
lr.At, lr.OK = time.Now().UTC().Format(time.RFC3339), runErr == nil
|
||||
if runErr != nil {
|
||||
lr.Error = truncate(runErr.Error(), 500)
|
||||
}
|
||||
if rep != nil {
|
||||
lr.VerifyError, lr.Verbs, lr.Checks = rep.VerifyError, rep.Verbs, rep.Checks
|
||||
for _, s := range rep.Receipts {
|
||||
if r, err := opVerifyReceipt(s); err == nil {
|
||||
lr.Images = append(lr.Images, opResultImage{Image: r.Image, Digest: r.Digest, Source: r.Source, Commit: r.Commit})
|
||||
}
|
||||
}
|
||||
}
|
||||
signed, err := lineReceiptSign(lr)
|
||||
if err != nil {
|
||||
// Ended as failed with no receipt: nothing can retry or undo it on
|
||||
// the strength of a receipt that does not exist.
|
||||
_ = planEndAttempt(p.Nonce, n, attempt, false)
|
||||
return nil, fmt.Errorf("the line ran and its receipt could not be signed: %v", err)
|
||||
}
|
||||
// The receipt is kept before the attempt is ended, so a record that says
|
||||
// an attempt ended always has its receipt beside it. A record that could
|
||||
// not be ended stays running: nothing runs or undoes the line after it
|
||||
// until this bastion restarts and settles it from the kept receipt.
|
||||
keepErr := planKeepReceipt(p.Nonce, n, signed)
|
||||
endErr := planEndAttempt(p.Nonce, n, attempt, !lr.failed())
|
||||
out := map[string]any{"receipt": signed, "ok": lr.OK, "error": lr.Error, "verify_error": lr.VerifyError}
|
||||
if keepErr != nil || endErr != nil {
|
||||
out["keep_error"] = fmt.Sprint(errors.Join(keepErr, endErr))
|
||||
}
|
||||
if rep != nil {
|
||||
out["report"] = rep
|
||||
}
|
||||
dcnAudit(dir, "op_plan_line", 0, p.Pipeline+"#"+strconv.Itoa(n)+" "+l.Op, fmt.Sprintf("ok=%v attempt=%d plan=%s", lr.OK, attempt, truncate(p.Nonce, 12)))
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func planParamsSHA(params string) string {
|
||||
sum := sha256.Sum256([]byte(params))
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// ── the line record ─────────────────────────────────────────────────────────
|
||||
|
||||
// planLoadRun reads a line's record: nil when the line never started here.
|
||||
func planLoadRun(nonce string, n int) (*planRunRecord, error) {
|
||||
path, err := planRunPath(nonce, n)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw, err := os.ReadFile(path)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not read this machine's record of line %d: %v", n, err)
|
||||
}
|
||||
var rec planRunRecord
|
||||
if err := json.Unmarshal(raw, &rec); err != nil || rec.Plan != nonce || rec.N != n {
|
||||
return nil, fmt.Errorf("this machine's record of line %d does not read", n)
|
||||
}
|
||||
return &rec, nil
|
||||
}
|
||||
|
||||
func planSaveRun(rec *planRunRecord) error {
|
||||
path, err := planRunPath(rec.Plan, rec.N)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rec.At = time.Now().UTC().Format(time.RFC3339)
|
||||
raw, _ := json.MarshalIndent(rec, "", " ")
|
||||
return atomicWrite(path, raw)
|
||||
}
|
||||
|
||||
// planBeginAttempt starts the line's next attempt: the first, or the one
|
||||
// after an attempt that FAILED here. It refuses while an attempt is running,
|
||||
// once one passed, and once the line was undone, and records the attempt as
|
||||
// running before anything of it runs; a record it cannot write refuses.
|
||||
func planBeginAttempt(p *shipPlan, l *shipPlanLine, tmpl lineReceipt) (int, error) {
|
||||
planLineMu.Lock()
|
||||
defer planLineMu.Unlock()
|
||||
if planProc == "" {
|
||||
return 0, errors.New("this machine cannot name its own process, so it cannot keep a line's attempts apart")
|
||||
}
|
||||
rec, err := planLoadRun(p.Nonce, l.N)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
next := 1
|
||||
if rec != nil {
|
||||
if rec.State == planStateRunning && planProcGone(rec.Proc) {
|
||||
if err := planSettleLocked(rec); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
switch rec.State {
|
||||
case planStateRunning:
|
||||
return 0, fmt.Errorf("attempt %d of line %d is running here; the next waits for its receipt", rec.Attempt, l.N)
|
||||
case planStatePassed:
|
||||
return 0, fmt.Errorf("line %d of this plan already succeeded here; it is not run again", l.N)
|
||||
case planStateUndone:
|
||||
return 0, fmt.Errorf("line %d of this plan was undone here; it is not run again", l.N)
|
||||
case planStateFailed:
|
||||
next = rec.Attempt + 1
|
||||
default:
|
||||
return 0, fmt.Errorf("this machine's record of line %d does not read", l.N)
|
||||
}
|
||||
} else {
|
||||
rec = &planRunRecord{Plan: p.Nonce, N: l.N, Expires: p.Expires}
|
||||
}
|
||||
k, err := planSpendAttempt(p, l)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if k != next {
|
||||
return 0, fmt.Errorf("line %d: its record here says attempt %d is next and its spent attempts say %d; refused rather than run out of order", l.N, next, k)
|
||||
}
|
||||
tmpl.Attempt = k
|
||||
rec.Attempt, rec.State, rec.Proc, rec.Receipt = k, planStateRunning, planProc, tmpl
|
||||
if err := planSaveRun(rec); err != nil {
|
||||
return 0, fmt.Errorf("could not record attempt %d of line %d before it runs (%v); refused rather than run unrecorded", k, l.N, err)
|
||||
}
|
||||
return k, nil
|
||||
}
|
||||
|
||||
// planRecordVerb adds a verb the running attempt is about to run to the
|
||||
// line's record. A verb an earlier attempt already ran keeps the state it
|
||||
// started in before the first attempt, so an undo goes back to before the
|
||||
// line, not to where a failed attempt left it.
|
||||
func planRecordVerb(nonce string, n, attempt int, v opVerbLine, from, to string) error {
|
||||
planLineMu.Lock()
|
||||
defer planLineMu.Unlock()
|
||||
rec, err := planLoadRun(nonce, n)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if rec == nil || rec.Attempt != attempt || rec.State != planStateRunning || rec.Proc != planProc {
|
||||
return fmt.Errorf("the record of line %d is not this attempt's", n)
|
||||
}
|
||||
for i, had := range rec.Verbs {
|
||||
if had.Verb == v.Verb && had.target() == v.target() {
|
||||
// Where an earlier attempt left it, or where it was before: a
|
||||
// target in neither state was moved by something else between
|
||||
// the attempts, and an undo to before the first attempt would
|
||||
// take that change back too. Refused before it runs.
|
||||
if from != rec.Froms[i] && from != rec.Tos[i] {
|
||||
return fmt.Errorf("it is at %s, which is neither where it was before this line's first attempt nor where that attempt was taking it: something else moved it since, and it is left alone", truncate(from, 90))
|
||||
}
|
||||
rec.Verbs[i], rec.Tos[i] = v, to
|
||||
return planSaveRun(rec)
|
||||
}
|
||||
}
|
||||
rec.Verbs, rec.Froms, rec.Tos = append(rec.Verbs, v), append(rec.Froms, from), append(rec.Tos, to)
|
||||
return planSaveRun(rec)
|
||||
}
|
||||
|
||||
// planEndAttempt records how the running attempt ended.
|
||||
func planEndAttempt(nonce string, n, attempt int, passed bool) error {
|
||||
planLineMu.Lock()
|
||||
defer planLineMu.Unlock()
|
||||
rec, err := planLoadRun(nonce, n)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if rec == nil || rec.Attempt != attempt || rec.State != planStateRunning || rec.Proc != planProc {
|
||||
return fmt.Errorf("the record of line %d is not this attempt's", n)
|
||||
}
|
||||
rec.State, rec.Proc = planStateFailed, ""
|
||||
if passed {
|
||||
rec.State = planStatePassed
|
||||
}
|
||||
return planSaveRun(rec)
|
||||
}
|
||||
|
||||
// planSettleLocked ends an attempt whose bastion process is gone (the op
|
||||
// unit went with it, opStartupSweep). If its receipt was kept, that is how it
|
||||
// ended; if not, it failed, and a receipt saying so is signed and kept, so
|
||||
// the pod can read it (op_plan_status) and retry or undo on it.
|
||||
func planSettleLocked(rec *planRunRecord) error {
|
||||
if r := planKeptReceipt(rec.Plan, rec.N, rec.Attempt); r != nil {
|
||||
rec.State = planStateFailed
|
||||
if !r.failed() {
|
||||
rec.State = planStatePassed
|
||||
}
|
||||
} else {
|
||||
lr := rec.Receipt
|
||||
lr.Attempt, lr.OK, lr.At = rec.Attempt, false, time.Now().UTC().Format(time.RFC3339)
|
||||
lr.Error = fmt.Sprintf("attempt %d stopped when the bastion running it did", rec.Attempt)
|
||||
signed, err := lineReceiptSign(lr)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not sign the receipt of line %d's stopped attempt: %v", rec.N, err)
|
||||
}
|
||||
if err := planKeepReceipt(rec.Plan, rec.N, signed); err != nil {
|
||||
return fmt.Errorf("could not keep the receipt of line %d's stopped attempt: %v", rec.N, err)
|
||||
}
|
||||
rec.State = planStateFailed
|
||||
}
|
||||
rec.Proc = ""
|
||||
return planSaveRun(rec)
|
||||
}
|
||||
|
||||
// planSettleStopped runs at startup (opStartupSweep, after the op units are
|
||||
// gone): every attempt the last process died in is settled, and a line whose
|
||||
// plan expired more than planKeepAfterExpiry ago is let go with its receipts.
|
||||
func planSettleStopped() {
|
||||
d, err := configDir()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
dir := filepath.Join(d, planRunsDirName)
|
||||
entries, _ := os.ReadDir(dir)
|
||||
planLineMu.Lock()
|
||||
defer planLineMu.Unlock()
|
||||
for _, e := range entries {
|
||||
name := e.Name()
|
||||
if !strings.HasSuffix(name, ".json") || strings.HasSuffix(name, ".receipts.json") {
|
||||
continue
|
||||
}
|
||||
raw, err := os.ReadFile(filepath.Join(dir, name))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
var rec planRunRecord
|
||||
if json.Unmarshal(raw, &rec) != nil {
|
||||
continue
|
||||
}
|
||||
if exp, err := time.Parse(time.RFC3339, rec.Expires); err == nil && time.Since(exp) > planKeepAfterExpiry {
|
||||
_ = os.Remove(filepath.Join(dir, name))
|
||||
_ = os.Remove(filepath.Join(dir, strings.TrimSuffix(name, ".json")+".receipts.json"))
|
||||
continue
|
||||
}
|
||||
if rec.State == planStateRunning && planProcGone(rec.Proc) {
|
||||
if err := planSettleLocked(&rec); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "privatae-bastion: ship plan %s line %d: %v\n", truncate(rec.Plan, 12), rec.N, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func handlePlanUndo(ws *websocket.Conn, dir string, args []string) {
|
||||
if dir == "" || len(args) < 3 {
|
||||
opFail(ws, "op_plan_undo: need <plan_b64> <n> <failed_receipt_b64>")
|
||||
return
|
||||
}
|
||||
out, err := planUndoLine(args[0], args[1], args[2])
|
||||
if err != nil {
|
||||
opFail(ws, "op_plan_undo refused: "+err.Error())
|
||||
return
|
||||
}
|
||||
b, _ := json.Marshal(out)
|
||||
sendMgmt(ws, 'O', b)
|
||||
sendMgmtExit(ws, 0)
|
||||
}
|
||||
|
||||
// planUndoLine takes line n back to what this machine recorded before it ran.
|
||||
func planUndoLine(planB64, nArg, failedB64 string) (map[string]any, error) {
|
||||
p, sha, n, err := planDecodeArgs([]string{planB64, nArg})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
l, err := planLineFor(p, n)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !l.Undo {
|
||||
return nil, fmt.Errorf("line %d is not one the plan lets be undone", n)
|
||||
}
|
||||
raw, err := base64.StdEncoding.DecodeString(strings.TrimSpace(failedB64))
|
||||
if err != nil {
|
||||
return nil, errors.New("the failed receipt does not decode")
|
||||
}
|
||||
var s opSignedReceipt
|
||||
if err := json.Unmarshal(raw, &s); err != nil {
|
||||
return nil, errors.New("the failed receipt is not a receipt")
|
||||
}
|
||||
fr, err := lineReceiptVerify(s, p, sha)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// What may start an undo: a later line whose CHECK failed (its apply ran,
|
||||
// its verify or log check did not pass — the machines' own verdict, not
|
||||
// something a pod can cause by handing a bad credential) on the last
|
||||
// attempt the plan allows it, so no retry of it can still pass; or the
|
||||
// line's own failure, which takes back only what that line itself did.
|
||||
later := fr.N > n && fr.OK && fr.VerifyError != ""
|
||||
own := fr.N == n && fr.failed()
|
||||
if fr.Undo || !(later || own) {
|
||||
return nil, fmt.Errorf("an undo of line %d needs a signed receipt of a LATER line of this plan whose check failed, or of line %d's own failure", n, n)
|
||||
}
|
||||
if later {
|
||||
if last := planLineAttempts(p, fr.N); fr.Attempt != last {
|
||||
return nil, fmt.Errorf("line %d's check failed on attempt %d of the %d the plan allows it; only its last attempt's failure undoes an earlier line, since a retry may still pass", fr.N, fr.Attempt, last)
|
||||
}
|
||||
}
|
||||
planLineMu.Lock()
|
||||
rec, err := planLoadRun(p.Nonce, n)
|
||||
if err == nil && rec == nil {
|
||||
err = fmt.Errorf("this machine has no record of running line %d, so there is nothing of it to undo", n)
|
||||
}
|
||||
if err == nil && rec.State == planStateRunning && planProcGone(rec.Proc) {
|
||||
err = planSettleLocked(rec)
|
||||
}
|
||||
if err == nil {
|
||||
err = planUndoAllowed(rec, fr, later)
|
||||
}
|
||||
if err == nil {
|
||||
err = planSpend(p, strconv.Itoa(n)+":undo")
|
||||
}
|
||||
if err == nil {
|
||||
// Undone before the undo runs, so no attempt of the line starts
|
||||
// under it; an undo that cannot be recorded does not run.
|
||||
prev := rec.State
|
||||
rec.State = planStateUndone
|
||||
if err = planSaveRun(rec); err != nil {
|
||||
rec.State = prev
|
||||
err = fmt.Errorf("could not record the undo of line %d before it runs (%v); refused", n, err)
|
||||
}
|
||||
}
|
||||
planLineMu.Unlock()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
hn, _ := os.Hostname()
|
||||
ur := lineReceipt{Plan: p.Nonce, PlanSHA256: sha, Pipeline: p.Pipeline, Commit: p.Commit, N: n, Step: l.Step,
|
||||
Machine: hn, OpSHA256: l.OpSHA256, At: time.Now().UTC().Format(time.RFC3339), Undo: true, OK: true}
|
||||
ctx := context.Background()
|
||||
// The state first, newest first (the tree, the image back where they
|
||||
// were), and only then each unit the line restarted, in its order, so it
|
||||
// starts again on what was put back — never on the failed code.
|
||||
order := []int{}
|
||||
for i := len(rec.Verbs) - 1; i >= 0; i-- {
|
||||
if !strings.HasPrefix(rec.Verbs[i].Verb, "unit.") {
|
||||
order = append(order, i)
|
||||
}
|
||||
}
|
||||
for i := range rec.Verbs {
|
||||
if strings.HasPrefix(rec.Verbs[i].Verb, "unit.") {
|
||||
order = append(order, i)
|
||||
}
|
||||
}
|
||||
for _, i := range order {
|
||||
v := rec.Verbs[i]
|
||||
note, err := planUndoVerb(ctx, v, rec.Froms[i], rec.Tos[i])
|
||||
ur.Verbs = append(ur.Verbs, opVerbResult{Verb: v.Verb, Target: v.target(), OK: err == nil, Note: note})
|
||||
if err != nil {
|
||||
ur.OK, ur.Error = false, truncate(v.Verb+" "+v.target()+": "+err.Error(), 500)
|
||||
break
|
||||
}
|
||||
}
|
||||
signed, err := lineReceiptSign(ur)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the undo ran and its receipt could not be signed: %v", err)
|
||||
}
|
||||
out := map[string]any{"receipt": signed, "ok": ur.OK, "error": ur.Error}
|
||||
if err := planKeepReceipt(p.Nonce, n, signed); err != nil {
|
||||
out["keep_error"] = err.Error()
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// planLineAttempts is how many times the plan lets line n run.
|
||||
func planLineAttempts(p *shipPlan, n int) int {
|
||||
for i := range p.Lines {
|
||||
if p.Lines[i].N == n && p.Lines[i].Attempts > 1 {
|
||||
return p.Lines[i].Attempts
|
||||
}
|
||||
}
|
||||
return 1
|
||||
}
|
||||
|
||||
// planUndoAllowed checks the receipt against what this machine recorded of
|
||||
// the line: a later line's failure undoes a line that passed here; the
|
||||
// line's own failure must be its LATEST attempt's, and none may have passed.
|
||||
func planUndoAllowed(rec *planRunRecord, fr *lineReceipt, later bool) error {
|
||||
switch rec.State {
|
||||
case planStateRunning:
|
||||
return fmt.Errorf("attempt %d of line %d is running here; an undo waits for its receipt", rec.Attempt, rec.N)
|
||||
case planStateUndone:
|
||||
return fmt.Errorf("line %d of this plan was undone here already", rec.N)
|
||||
}
|
||||
if later {
|
||||
if rec.State != planStatePassed {
|
||||
return fmt.Errorf("line %d did not pass here, so no later line ran on it", rec.N)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if rec.State == planStatePassed {
|
||||
return fmt.Errorf("an attempt of line %d passed here; an earlier attempt's failure does not undo it", rec.N)
|
||||
}
|
||||
if fr.Attempt != rec.Attempt {
|
||||
return fmt.Errorf("that receipt is of attempt %d of line %d, and its latest attempt here is %d: only the latest attempt's failure undoes the line", fr.Attempt, rec.N, rec.Attempt)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// planUndoVerb takes one verb back from `to` to `from`, and only from `to`:
|
||||
// a tree or image that moved since the line ran is someone else's change, and
|
||||
// is left alone.
|
||||
func planUndoVerb(ctx context.Context, v opVerbLine, from, to string) (string, error) {
|
||||
switch v.Verb {
|
||||
case "tree.checkout":
|
||||
defer opLockTarget(v)()
|
||||
path := v.Args["path"]
|
||||
head, branch, err := opTreeState(v)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if head == from {
|
||||
return "already at " + truncate(from, 12), nil
|
||||
}
|
||||
if head != to {
|
||||
return "", fmt.Errorf("the tree is at %s, not the %s the line left it at: it moved since, and is left alone", truncate(head, 12), truncate(to, 12))
|
||||
}
|
||||
if err := opTreeWriteDirs(path, head, from); err != nil {
|
||||
return "", err
|
||||
}
|
||||
// Never over a file the machine keeps: a path the move back writes
|
||||
// that is there now, untracked and ignored, stops it (reset --keep,
|
||||
// unlike merge, has no --no-overwrite-ignore).
|
||||
// The names are matched as they are, never as patterns.
|
||||
names, err := opTreeGitStdout(path, "diff", "--name-only", "--no-renames", "-z", head, from)
|
||||
if err != nil {
|
||||
return "", errors.New("could not read which files moving back writes: " + err.Error())
|
||||
}
|
||||
var paths []string
|
||||
for _, n := range strings.Split(names, "\x00") {
|
||||
if n != "" {
|
||||
paths = append(paths, n)
|
||||
}
|
||||
}
|
||||
if len(paths) > 0 {
|
||||
kept, err := opTreeGitStdout(path, append([]string{"--literal-pathspecs", "ls-files", "-z", "--others", "--ignored", "--exclude-standard", "--"}, paths...)...)
|
||||
if err != nil {
|
||||
return "", errors.New("could not read which files the tree keeps: " + err.Error())
|
||||
}
|
||||
if strings.Trim(kept, "\x00") != "" {
|
||||
return "", fmt.Errorf("moving back would overwrite %s, a file this machine keeps and git ignores", strings.SplitN(strings.Trim(kept, "\x00"), "\x00", 2)[0])
|
||||
}
|
||||
}
|
||||
// Read again right before the move back: reset --keep moves the
|
||||
// branch from wherever it is, and a commit something else made
|
||||
// since the first read is left alone.
|
||||
opTreeBeforeMove(path)
|
||||
if now, err := opTreeGit(path, nil, "rev-parse", "HEAD"); err != nil {
|
||||
return "", errors.New("could not read the tree's commit: " + truncate(now, 300))
|
||||
} else if err := opMovedSince(now, head); err != nil {
|
||||
return "", err
|
||||
}
|
||||
var out string
|
||||
if branch != "" {
|
||||
// Back on the branch it was on, keeping any local work: reset --keep
|
||||
// refuses when a local change is in the way.
|
||||
out, err = opTreeGit(path, nil, "reset", "--keep", "--quiet", from)
|
||||
} else {
|
||||
out, err = opTreeGit(path, nil, "checkout", "--no-overwrite-ignore", "--quiet", "--detach", from)
|
||||
}
|
||||
if err != nil {
|
||||
return "", errors.New("could not move the tree back: " + truncate(out, 300))
|
||||
}
|
||||
return truncate(to, 12) + " → " + truncate(from, 12) + " (undone)", nil
|
||||
case "k8s.set_image":
|
||||
defer opLockTarget(v)()
|
||||
a := v.Args
|
||||
live, err := opLiveImage(ctx, v)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if live == from {
|
||||
return "already " + truncate(from, 90), nil
|
||||
}
|
||||
if live != to {
|
||||
return "", fmt.Errorf("the container runs %s, not the %s the line set: it moved since, and is left alone", truncate(live, 90), truncate(to, 90))
|
||||
}
|
||||
// Read again right before the write, as a roll does.
|
||||
if now, err := opLiveImage(ctx, v); err != nil {
|
||||
return "", err
|
||||
} else if err := opMovedSince(now, live); err != nil {
|
||||
return "", err
|
||||
}
|
||||
wl := a["kind"] + "/" + a["name"]
|
||||
if out, err := opKubectl(ctx, "-n", a["namespace"], "set", "image", wl, a["container"]+"="+from); err != nil {
|
||||
return "", errors.New("set image failed: " + truncate(out, 300))
|
||||
}
|
||||
if out, err := opKubectl(ctx, "-n", a["namespace"], "rollout", "status", wl, "--timeout=5m"); err != nil {
|
||||
return "", errors.New("the rollout did not finish: " + truncate(out, 300))
|
||||
}
|
||||
return truncate(to, 60) + " → " + truncate(from, 60) + " (undone), rolled out", nil
|
||||
case "unit.reload", "unit.restart":
|
||||
// The unit is started again on what the other verbs just put back.
|
||||
return opUnitDo(ctx, v)
|
||||
}
|
||||
return "", fmt.Errorf("%s cannot be undone", v.Verb)
|
||||
}
|
||||
|
||||
// ── what a line left: its receipts, for a pod that lost the answer ─────────
|
||||
|
||||
func planReceiptsPath(nonce string, n int) (string, error) {
|
||||
p, err := planRunPath(nonce, n)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return strings.TrimSuffix(p, ".json") + ".receipts.json", nil
|
||||
}
|
||||
|
||||
// planKeepReceipt keeps every receipt a line (and its undo) signed, in order.
|
||||
func planKeepReceipt(nonce string, n int, s opSignedReceipt) error {
|
||||
path, err := planReceiptsPath(nonce, n)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
planMu.Lock()
|
||||
defer planMu.Unlock()
|
||||
var all []opSignedReceipt
|
||||
raw, err := os.ReadFile(path)
|
||||
if err == nil {
|
||||
if err := json.Unmarshal(raw, &all); err != nil {
|
||||
return fmt.Errorf("the kept receipts of line %d do not read", n)
|
||||
}
|
||||
} else if !errors.Is(err, os.ErrNotExist) {
|
||||
return err
|
||||
}
|
||||
all = append(all, s)
|
||||
raw, _ = json.MarshalIndent(all, "", " ")
|
||||
return atomicWrite(path, raw)
|
||||
}
|
||||
|
||||
// op_plan_status <plan nonce> <n>: the receipts line n of that plan signed
|
||||
// here, in order — a read, for a pod whose answer to op_plan_line was lost
|
||||
// (a restart mid-run). Nothing is spent by asking.
|
||||
func handlePlanStatus(ws *websocket.Conn, dir string, args []string) {
|
||||
if dir == "" || len(args) < 2 {
|
||||
opFail(ws, "op_plan_status: need <plan nonce> <n>")
|
||||
return
|
||||
}
|
||||
n, err := strconv.Atoi(strings.TrimSpace(args[1]))
|
||||
if err != nil {
|
||||
opFail(ws, "op_plan_status: the line is not a number")
|
||||
return
|
||||
}
|
||||
path, err := planReceiptsPath(strings.TrimSpace(args[0]), n)
|
||||
if err != nil {
|
||||
opFail(ws, "op_plan_status: "+err.Error())
|
||||
return
|
||||
}
|
||||
var all []opSignedReceipt
|
||||
planMu.Lock()
|
||||
if raw, err := os.ReadFile(path); err == nil {
|
||||
_ = json.Unmarshal(raw, &all)
|
||||
}
|
||||
planMu.Unlock()
|
||||
if all == nil {
|
||||
all = []opSignedReceipt{}
|
||||
}
|
||||
b, _ := json.Marshal(map[string]any{"receipts": all})
|
||||
sendMgmt(ws, 'O', b)
|
||||
sendMgmtExit(ws, 0)
|
||||
}
|
||||
|
||||
// planKeptReceipt is the kept receipt of attempt k of line n, or nil.
|
||||
func planKeptReceipt(nonce string, n, k int) *lineReceipt {
|
||||
path, err := planReceiptsPath(nonce, n)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
planMu.Lock()
|
||||
raw, err := os.ReadFile(path)
|
||||
planMu.Unlock()
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
var all []opSignedReceipt
|
||||
if json.Unmarshal(raw, &all) != nil {
|
||||
return nil
|
||||
}
|
||||
for _, sr := range all {
|
||||
body, err := base64.StdEncoding.DecodeString(sr.Payload)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
var r lineReceipt
|
||||
if json.Unmarshal(body, &r) == nil && r.Plan == nonce && r.N == n && !r.Undo && r.Attempt == k {
|
||||
return &r
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user