bastion 0.53.2

client-go/mgmt_audit.go

@@ -0,0 +1,462 @@
+package main
+
+// Accounting on the command lane.
+//
+// Every other wall on this box already leaves a record of what it decided:
+// dcn.go writes the pairing's audit.jsonl for a store op, session.go writes a
+// row per terminal, pairing_ops.go writes pairing_audit.jsonl for a membership
+// change. The MANAGEMENT lane — the one every other op actually arrives on —
+// wrote nothing. A host could ask this machine for an op it was told it does
+// not hold, be refused correctly, and the only trace was a stderr frame that
+// died with the websocket. The refusal worked and left no evidence it happened,
+// which is the same failure shape directive_refusal.go was built to fix, one
+// layer down: the gate refuses loudly and then nobody can show that it did.
+//
+// So: one JSON line per management job, allowed or refused, in
+// <configDir>/mgmt_audit.jsonl. Box-level and not per-pairing, for the same
+// reason pairing_audit.jsonl is — the interesting question is "what has been
+// ASKED of this machine", across every host paired to it, and a per-pairing
+// file can only ever answer it one pairing at a time.
+//
+//	{"ts":"<rfc3339 utc>","self":"acme.host.privatae.ai","op":"uptime",
+//	 "family":"observe","verdict":"allowed","why":"","exit":0,
+//	 "grant_sig":"<b64>","binary_sha256":"<hex>"}
+//
+// `self` is the PAIRING ID — the host name the pairing is filed under. Never a
+// key, never a token: this file is read by a human on the user's own machine
+// and fetched over the tunnel by the anchor, and neither of those is a reason
+// to put a secret in it. `grant_sig` is the SIGNATURE of the grant that was
+// consulted, which is the same public value the enforcing report already ships
+// (enforcing.go:22-24) and is what lets the host tell "refused under the grant
+// I issued" from "refused under a stale copy".
+//
+// Every line is written BEFORE the terminal frame leaves this box, and a
+// refusal's line before any frame at all (refuseMgmt, mgmt.go). A refusal is
+// the interesting half of this file, and a crash between the decision and the
+// write would lose exactly the line somebody would later go looking for.
+//
+// Three things live here: the writer, the heartbeat summary the host reads
+// without pulling anything (mgmtAuditReport), and the read-only fetch for when
+// it wants the lines themselves (handleAuditFetch).
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sync"
+	"time"
+	"unicode/utf8"
+
+	"github.com/gorilla/websocket"
+)
+
+const (
+	mgmtAuditFile = "mgmt_audit.jsonl"
+
+	// auditAllowed / auditRefused are the only two verdicts. "allowed" means the
+	// WALL let the op through — not that it succeeded. A handler that ran and
+	// failed is allowed with a non-zero exit, and the distinction matters: one is
+	// a permission event the host should react to, the other is a bad argument.
+	auditAllowed = "allowed"
+	auditRefused = "refused"
+
+	// mgmtAuditFetchOp is the op name, held as a constant because three places
+	// need to agree on it: handlerOps, the anchor-only gate in mgmt.go, and
+	// opFamily's classification of it.
+	mgmtAuditFetchOp = "audit_fetch"
+
+	// mgmtAuditFetchMaxBytes caps ONE fetch, and it is a ceiling the caller can
+	// only ask below, never above — the same shape as backupfetch.go's
+	// max_bytes. 1 MiB is on the order of 4-5k lines of this file, which is far
+	// more than anyone reads at once and still small enough to cross the
+	// management frame without the host having to stream it.
+	mgmtAuditFetchMaxBytes = 1 << 20
+
+	// mgmtAuditLineMax is the longest single line the summary reader will take.
+	// These records are machine-written and a few hundred bytes; anything longer
+	// is not one of ours. A line over this is SKIPPED and the scan continues —
+	// see mgmtAuditReport for why that distinction is the whole point.
+	mgmtAuditLineMax = 64 * 1024
+
+	// mgmtAuditOpMax / mgmtAuditWhyMax bound the two caller-influenced fields
+	// BEFORE they are marshalled.
+	//
+	// op is job.Op straight off the wire and the refusal strings embed it again,
+	// so without a bound here one job writes a record of whatever size the
+	// sender chose. Sized from the real values, not guessed: the longest op name
+	// in handlerOps is 22 bytes ("pairing_anchor_consent"), and the longest
+	// refusal this lane produces is 293 bytes (unanchoredNetworkedRefusal with
+	// the longest op interpolated). 128 and 512 clear both several times over,
+	// so nothing legitimate is ever truncated, and a marker says so when
+	// something is.
+	mgmtAuditOpMax  = 128
+	mgmtAuditWhyMax = 512
+
+	// mgmtAuditMaxBytes is when the file rolls to mgmt_audit.jsonl.1, one
+	// generation kept. At roughly 250 bytes a record that is ~33k ops of
+	// history, and it bounds three things at once: the disk this takes on the
+	// user's own machine (2x this, worst case), the work the heartbeat summary
+	// does (it reads the current file only, every beat), and how much one
+	// pairing can make either of those grow by driving traffic at the lane.
+	mgmtAuditMaxBytes = 8 << 20
+)
+
+// mgmtAuditPath is <configDir>/mgmt_audit.jsonl. configDir (config.go:137)
+// creates the directory as a side effect, which is what every other writer on
+// this path already relies on.
+func mgmtAuditPath() (string, error) {
+	d, err := configDir()
+	if err != nil {
+		return "", err
+	}
+	return filepath.Join(d, mgmtAuditFile), nil
+}
+
+// mgmtAudit appends one record. Best-effort, exactly like pairingAudit
+// (pairing_ops.go:32-50) and dcnAudit: a decision stands whether or not the
+// line lands, and a full disk must not turn into a lane that stops answering.
+// That is a deliberate trade and not an oversight — the alternative is a box
+// that refuses work because it cannot write about the work.
+//
+// args and approval (writes.go): the sha256 of the args array the job carried —
+// the same digest the approval bound — and how a gated write was approved
+// ("device" | "console"), so a line can be matched to the approval that let it
+// run. Never the args themselves: the observe shelf records those, bounded, for
+// the three ops that take a value (machinestore.go), and an audit line that
+// carried a ctx_put body would be the wrong file holding user data.
+func mgmtAudit(self, op, family, verdict, why string, exit int, args []string, approval string) {
+	p, err := mgmtAuditPath()
+	if err != nil {
+		return
+	}
+	line := map[string]any{
+		"ts":            time.Now().UTC().Format(time.RFC3339),
+		"self":          self,
+		"op":            auditClamp(op, mgmtAuditOpMax),
+		"family":        family,
+		"verdict":       verdict,
+		"why":           auditClamp(why, mgmtAuditWhyMax),
+		"exit":          exit,
+		"grant_sig":     pairingGrantSig(self),
+		"binary_sha256": selfDigest(),
+		"args_sha256":   argsDigest(args),
+	}
+	if approval != "" {
+		line["approval"] = approval
+	}
+	rec, _ := json.Marshal(line)
+	mgmtAuditMu.Lock()
+	defer mgmtAuditMu.Unlock()
+
+	fh, err := os.OpenFile(p, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o600)
+	if err != nil {
+		return
+	}
+	_, _ = fh.Write(append(rec, '\n'))
+	// CLOSED BEFORE THE ROLL, NOT DEFERRED PAST IT — see mgmtAuditRoll.
+	_ = fh.Close()
+	mgmtAuditRoll(p)
+}
+
+// mgmtAuditMu serialises the append-and-roll above.
+//
+// The append does not need it — O_APPEND is a single atomic write on both
+// families (POSIX for one; FILE_APPEND_DATA for the other) — but the ROLL does.
+// Two management jobs landing together can each stat an over-cap file, and the
+// second rename then moves the fresh, nearly-empty live file over the
+// generation the first one just saved: 8 MiB of history destroyed to preserve a
+// few hundred bytes. The lane is one websocket per pairing and a box holds
+// several, so concurrent jobs are the normal case, not the rare one.
+//
+// Taken AFTER pairingGrantSig/selfDigest deliberately: those read pairing files
+// under saveMu (config.go), and a lock taken across them would be a second
+// ordering between the two mutexes for no benefit. The cost is that record
+// order under contention is the order writers reach this line rather than the
+// order their decisions were made, which was already true.
+var mgmtAuditMu sync.Mutex
+
+// auditClamp bounds one caller-influenced field and SAYS when it did.
+//
+// A silent truncation in an audit record is its own small lie — a reader cannot
+// tell a short op from a clipped one — so the marker carries the original
+// length. Cut on a rune boundary: the bytes arrive from the network and may be
+// any encoding, and json.Marshal would otherwise turn a split rune into a
+// replacement character that looks like content rather than damage.
+func auditClamp(v string, max int) string {
+	if len(v) <= max {
+		return v
+	}
+	cut := max
+	for cut > 0 && !utf8.RuneStart(v[cut]) {
+		cut--
+	}
+	return v[:cut] + fmt.Sprintf("...[truncated, %d bytes]", len(v))
+}
+
+// mgmtAuditRoll moves the file aside once it passes mgmtAuditMaxBytes, keeping
+// exactly one previous generation. Best-effort like everything else here.
+//
+// Rolling rather than trimming in place because a record is only useful whole:
+// chopping the front of a JSONL file leaves a partial first line, and every
+// reader of it then has to know to discard one. Renaming leaves both files
+// valid. One generation because the point is a bound, and the heartbeat summary
+// reads the current file only — history past the roll is for a human with the
+// machine in front of them, not for the wire.
+//
+// TAKES THE PATH, NOT THE OPEN HANDLE, AND THAT IS THE WHOLE FIX ON WINDOWS.
+// The first cut of this was called as mgmtAuditRoll(p, fh) from inside mgmtAudit
+// while `defer fh.Close()` still held the file. On unix that renames fine — the
+// directory entry moves and the open handle keeps writing to the same inode —
+// so every test here passed. On Windows it cannot: os.Rename is MoveFileEx with
+// MOVEFILE_REPLACE_EXISTING (internal/syscall/windows/syscall_windows.go:366),
+// which needs DELETE access on the source, and os.OpenFile's share mode is
+// FILE_SHARE_READ|FILE_SHARE_WRITE with no FILE_SHARE_DELETE (the `sharemode :=`
+// line in syscall/syscall_windows.go's Open) — so the handle this function was
+// handed is exactly what refuses the rename, with a sharing violation.
+//
+// The failure was silent and permanent. The error is discarded by design
+// (best-effort, like every writer on this path), so past 8 MiB the file simply
+// kept growing at ~250 bytes a job, on the USER'S OWN MACHINE — and
+// mgmtAuditReport re-reads the whole of it on every heartbeat, per pairing. The
+// cap that is supposed to bound the disk, the summary's work, and how much one
+// pairing can grow either by driving traffic at the lane was bounding none of
+// them. mgmt_audit_test.go is //go:build !windows (it borrows setupBox), so no
+// test in that file can ever see this; mgmt_audit_roll_test.go carries no tag.
+//
+// Because MoveFileEx is REPLACE_EXISTING, renaming over an existing .1 is
+// correct on both families and needs no unlink first.
+func mgmtAuditRoll(p string) {
+	st, err := os.Stat(p)
+	if err != nil || st.Size() < mgmtAuditMaxBytes {
+		return
+	}
+	_ = os.Rename(p, p+".1")
+}
+
+// pairingGrantSig is the signature of the pairing grant that was consulted for
+// this pairing, or "" when none was.
+//
+// Empty for the ANCHOR pairing, and that emptiness is meaningful rather than
+// missing data: the anchor is unconstrained by construction (capsAllow,
+// anchor.go:542-544), so no grant is read and there is no signature to name.
+// Empty too when the pairing holds no grant that still verifies — pairingCaps
+// (anchor.go:381-391) re-verifies on every call and returns ok=false for a
+// missing, revoked, tampered or wrong-subject grant, and in every one of those
+// cases nothing was consulted, so reporting a signature read off the file would
+// be claiming an authority that was never established. Same order as
+// pairingEnforcing (anchor.go:702-712): verify first, read the signature second.
+func pairingGrantSig(self string) string {
+	if self == "" {
+		return ""
+	}
+	if a := currentAnchor(); a == nil || a.AnchorHost == self {
+		return ""
+	}
+	if _, ok := pairingCaps(self); !ok {
+		return ""
+	}
+	g, err := loadPairingGrant(self)
+	if err != nil {
+		return ""
+	}
+	return g.Signature
+}
+
+type mgmtAuditRecord struct {
+	TS      string `json:"ts"`
+	Op      string `json:"op"`
+	Verdict string `json:"verdict"`
+	Why     string `json:"why"`
+}
+
+// mgmtAuditReport is the enforcing-report fragment: how many lines the file
+// holds, how many of them are refusals inside the last hour, and the last
+// refusal itself.
+//
+// The point of the summary is that the host learns a machine is refusing its
+// own host's requests WITHOUT pulling the file — the heartbeat already rides
+// every beat and is replaced each time (directive_refusal.go:22-25), so a count
+// here is live by construction. Pulling the file is the second step, taken only
+// once the count says there is something in it worth reading.
+//
+// AN OVER-LONG LINE IS SKIPPED, NOT FATAL, and that is the whole reason this
+// does not use bufio.Scanner. Scanner stops at the first token past its buffer
+// and returns "token too long" — measured: five lines whose third is 70 KiB
+// count as TWO, with the rest of the file never read. On a signal whose job is
+// to say "this machine is refusing things", that turns one over-long record
+// into a permanent freeze of the count, with no error on the wire and a block
+// that still looks confident. Every input here arrives from a pairing, so a
+// reader that can be stopped by its input is a reader that can be switched off
+// by the party it is watching. bufio.Reader.ReadLine drops the fragment and
+// carries on, and `unreadable` says how many it dropped.
+//
+// nil when the file does not exist: a box that has answered nothing says
+// nothing, rather than reporting a confident zero that reads the same as a box
+// whose audit was deleted.
+func mgmtAuditReport() map[string]any {
+	p, err := mgmtAuditPath()
+	if err != nil {
+		return nil
+	}
+	fh, err := os.Open(p)
+	if err != nil {
+		return nil
+	}
+	defer fh.Close()
+
+	cutoff := time.Now().UTC().Add(-time.Hour)
+	lines, refusedLastHour, unreadable := 0, 0, 0
+	var last *mgmtAuditRecord
+
+	rd := bufio.NewReaderSize(fh, mgmtAuditLineMax)
+	for {
+		line, isPrefix, err := rd.ReadLine()
+		if err != nil {
+			break
+		}
+		if isPrefix {
+			// Longer than the buffer. Drain its remaining fragments so the next
+			// read starts on a real record boundary, count it as a line that is
+			// there but unreadable, and keep going.
+			for isPrefix && err == nil {
+				_, isPrefix, err = rd.ReadLine()
+			}
+			lines++
+			unreadable++
+			if err != nil {
+				break
+			}
+			continue
+		}
+		if len(line) == 0 {
+			continue
+		}
+		lines++
+		var r mgmtAuditRecord
+		if err := json.Unmarshal(line, &r); err != nil {
+			unreadable++
+			continue
+		}
+		if r.Verdict != auditRefused {
+			continue
+		}
+		rec := r
+		last = &rec
+		// An unparseable ts is not counted rather than counted as recent: this
+		// number drives a posture drop on the host, and a clock or a format we
+		// cannot read is not evidence that something was refused in the last hour.
+		if ts, err := time.Parse(time.RFC3339, r.TS); err == nil && ts.After(cutoff) {
+			refusedLastHour++
+		}
+	}
+
+	out := map[string]any{"lines": lines, "refused_last_hour": refusedLastHour}
+	if last != nil {
+		out["last_refusal"] = map[string]any{"ts": last.TS, "op": last.Op, "why": last.Why}
+	}
+	// Only when there are any, so a clean file never carries a zero that invites
+	// someone to go looking for damage that is not there.
+	if unreadable > 0 {
+		out["unreadable"] = unreadable
+	}
+	return out
+}
+
+type auditFetchReq struct {
+	// MaxBytes is the caller's proposed ceiling. The smaller of it and
+	// mgmtAuditFetchMaxBytes wins — a host asking for more gets this box's
+	// number, the same rule contextextract.go states for its own pull.
+	MaxBytes int64 `json:"max_bytes"`
+}
+
+// handleAuditFetch returns the TAIL of the audit file, whole lines only.
+//
+// The tail and not the head: this file only ever grows at the end, so the lines
+// anybody wants are the recent ones, and a head-first read would hand back the
+// oldest kilobyte of a machine that has been running for a year. Whole lines
+// only because the consumer parses JSONL, and a fetch that starts mid-record
+// makes its first line garbage.
+//
+// Read-only. There is no op here that truncates or rotates this file: the
+// record of what was asked of a machine is not something a host gets to edit
+// over the tunnel, and the same is true of every other audit file on this box.
+func handleAuditFetch(ws *websocket.Conn, _ string, args []string) {
+	max := int64(mgmtAuditFetchMaxBytes)
+	if len(args) > 0 && args[0] != "" {
+		var req auditFetchReq
+		if err := json.Unmarshal([]byte(args[0]), &req); err != nil {
+			sendMgmt(ws, 'R', []byte("audit_fetch: argument is not JSON: "+err.Error()+"\n"))
+			sendMgmtExit(ws, 1)
+			return
+		}
+		if req.MaxBytes > 0 && req.MaxBytes < max {
+			max = req.MaxBytes
+		}
+	}
+	p, err := mgmtAuditPath()
+	if err != nil {
+		sendMgmt(ws, 'R', []byte("audit_fetch: "+err.Error()+"\n"))
+		sendMgmtExit(ws, 1)
+		return
+	}
+	fh, err := os.Open(p)
+	if err != nil {
+		if os.IsNotExist(err) {
+			// Nothing has been asked of this box yet. Empty, not an error — the
+			// same call on a quiet machine and on a busy one must differ in its
+			// CONTENT, not in whether it worked.
+			auditFetchReply(ws, map[string]any{"bytes": 0, "truncated": false, "text": ""})
+			return
+		}
+		sendMgmt(ws, 'R', []byte("audit_fetch: "+err.Error()+"\n"))
+		sendMgmtExit(ws, 1)
+		return
+	}
+	defer fh.Close()
+	st, err := fh.Stat()
+	if err != nil {
+		sendMgmt(ws, 'R', []byte("audit_fetch: "+err.Error()+"\n"))
+		sendMgmtExit(ws, 1)
+		return
+	}
+	size := st.Size()
+	off, truncated := int64(0), false
+	if size > max {
+		off, truncated = size-max, true
+	}
+	buf := make([]byte, size-off)
+	if _, err := fh.ReadAt(buf, off); err != nil {
+		sendMgmt(ws, 'R', []byte("audit_fetch: "+err.Error()+"\n"))
+		sendMgmtExit(ws, 1)
+		return
+	}
+	if truncated {
+		// Drop the partial first record. If the window happens to hold no newline
+		// at all there is no whole line in it, and the honest answer is nothing
+		// rather than a fragment the caller will fail to parse.
+		if i := bytes.IndexByte(buf, '\n'); i >= 0 {
+			buf = buf[i+1:]
+		} else {
+			buf = nil
+		}
+	}
+	auditFetchReply(ws, map[string]any{
+		"bytes":     size,
+		"truncated": truncated,
+		"text":      string(buf),
+	})
+}
+
+func auditFetchReply(ws *websocket.Conn, v map[string]any) {
+	b, err := json.Marshal(v)
+	if err != nil {
+		sendMgmt(ws, 'R', []byte(fmt.Sprintf("audit_fetch: result marshal failed: %v\n", err)))
+		sendMgmtExit(ws, 1)
+		return
+	}
+	sendMgmt(ws, 'O', b)
+	sendMgmtExit(ws, 0)
+}