privatae-bastion/client-go/observe_ops_linux.go

//go:build linux

package main

// Linux drivers for observe:*. Each reads a file under /proc, /run, /var or
// runs ONE fixed argv through observeExec — the same read-only forms a
// read-only sudoers policy pins (nft list …, ss -tlnp, dmesg -T, ufw status, …).
// Nothing in this file touches a job argument. Every op's execs share one
// budget (opBudget); a field whose exec ran out of it is marked "timeout"
// and the op still answers.

import (
	"context"
	"os"
	"runtime"
	"strings"
	"syscall"
	"time"
)

const observeHostSupported = true

// textOf runs a fixed argv and returns its stdout and whether the op budget
// ran out during it. stderr and a non-zero exit are not errors here: nft
// with no such table exits 1 with nothing on stdout, and "" is what the
// parser turns into "absent".
func textOf(ctx context.Context, name string, args ...string) (string, bool) {
	out, _ := observeExec(ctx, name, args...)
	return string(out), ctx.Err() != nil
}

// readHostPosture is the host's probe, run natively. Each field's rule and
// vocabulary are documented on hostPosture (observe_ops.go).
func readHostPosture() hostPosture {
	ctx, cancel := opBudget()
	defer cancel()
	p := hostPosture{Ufw: "absent", Pubwall: "absent", Ssh22: "unknown", F2b: "absent",
		Input: "absent", Sec: -1, Load: -1, Cpus: runtime.NumCPU(), Reboot: "no", DiskPct: -1,
		Jrn: -1, Puv: "absent"}
	timedOut := func(field string) { p.Timeouts = append(p.Timeouts, field) }

	haveUfw := observeHave("ufw")
	ufwStatus := ""
	if haveUfw {
		var late bool
		ufwStatus, late = textOf(ctx, "ufw", "status")
		p.Ufw = parseUfwStatus(ufwStatus)
		if late {
			p.Ufw = "timeout"
			timedOut("ufw")
		}
	}
	nftTable := ""
	if observeHave("nft") {
		chain, late := textOf(ctx, "nft", "list", "chain", "inet", "pubwall", "input")
		p.Pubwall = parsePubwall(chain)
		if late {
			p.Pubwall = "timeout"
			timedOut("pubwall")
		}
		nftTable, late = textOf(ctx, "nft", "list", "table", "inet", "pubwall")
		if late {
			timedOut("ssh22")
		}
	}
	p.Ssh22 = parseSSH22(nftTable, ufwStatus, haveUfw)
	if ctx.Err() != nil {
		p.Ssh22 = "timeout"
	}
	if observeHave("systemctl") {
		// is-active exits non-zero for anything but "active" and still prints
		// the word; Output() returns both, so read the text regardless.
		out, late := textOf(ctx, "systemctl", "is-active", "fail2ban")
		switch {
		case late:
			p.F2b = "timeout"
			timedOut("f2b")
		case strings.TrimSpace(out) != "":
			p.F2b = strings.TrimSpace(out)
		default:
			p.F2b = "unknown"
		}
	}
	if observeHave("iptables") {
		out, late := textOf(ctx, "iptables", "-S")
		p.Input = parseIptablesInputPolicy(out)
		if late {
			p.Input = "timeout"
			timedOut("input")
		}
	}
	if observeHave("apt-get") {
		// The error is read for THIS field: an apt that failed or printed
		// nothing must count -1, never 0 (parseAptSecurityCount says why).
		out, err := observeExec(ctx, "apt-get", "-s", "upgrade")
		switch {
		case ctx.Err() != nil:
			timedOut("sec")
		case err != nil && len(out) == 0:
			p.Sec = -1
		default:
			p.Sec = parseAptSecurityCount(string(out))
		}
	}
	if b, err := os.ReadFile("/proc/loadavg"); err == nil {
		p.Load = parseLoadavg(string(b))
	}
	if _, err := os.Stat("/var/run/reboot-required"); err == nil {
		p.Reboot = "yes"
	}
	p.DiskPct = statfsPct("/")
	if observeHave("journalctl") {
		out, late := textOf(ctx, "journalctl", "-n1", "-o", "short-unix", "--no-pager")
		if late {
			timedOut("jrn")
		} else {
			p.Jrn = parseJournalAge(out, time.Now())
		}
	}
	if b, err := os.ReadFile("/run/privatae/post-upgrade-verify.status"); err == nil {
		if f := strings.Fields(string(b)); len(f) > 0 {
			p.Puv = f[0]
		}
	}
	return p
}

// statfsPct is the percent `df -P` prints for a mount: used over
// (used + available to unprivileged users), rounded UP — df's own arithmetic,
// so this field agrees with the df column an operator compares it to.
func statfsPct(path string) int {
	var st syscall.Statfs_t
	if err := syscall.Statfs(path, &st); err != nil {
		return -1
	}
	used := st.Blocks - st.Bfree
	total := used + st.Bavail
	if total == 0 {
		return -1
	}
	return int((used*100 + total - 1) / total)
}

type socketsReport struct {
	Supported bool   `json:"supported"`
	Reason    string `json:"reason,omitempty"`
	Partial   string `json:"partial,omitempty"` // which half ran out of budget
	// Failed is a half that RAN and exited non-zero. Separate from Partial
	// because a timeout and a failure are different operator problems, and
	// separate from Reason because the binary was there and did run.
	Failed  string         `json:"failed,omitempty"`
	Sockets []listenSocket `json:"sockets"`
}

// A FAILED `ss` IS NOT AN EMPTY SOCKET TABLE, and textOf cannot tell the
// caller which it got. It is `out, _ := observeExec(...)` on purpose — nft
// exits 1 with empty stdout for "no such table" and "" is the right answer
// there — but on this path an empty table becomes the DEFINITION OF NORMAL the
// next accounting is taken from, and then every listener on the box reports
// gone or new depending on which side was blind. Measured:
// an `ss` that exits non-zero produced 54 of 54 listeners reported gone, with
// no note. exec.Cmd.Output also returns partial stdout WITH the error, so a
// truncated run reads as a complete short table unless the error is checked.
//
// So this path checks it, and only this path: textOf keeps its contract for
// every other op that legitimately reads "" as absent.
func readSockets() socketsReport {
	if !observeHave("ss") {
		return socketsReport{Reason: "ss not installed", Sockets: []listenSocket{}}
	}
	ctx, cancel := opBudget()
	defer cancel()
	r := socketsReport{Supported: true}
	tcp, tcpErr := observeExec(ctx, "ss", "-tlnpH")
	r.Sockets = parseSS("tcp", string(tcp))
	if ctx.Err() != nil {
		r.Partial = "tcp: timeout"
		return r
	}
	if tcpErr != nil {
		r.Failed = "tcp: " + tcpErr.Error()
		return r
	}
	udp, udpErr := observeExec(ctx, "ss", "-ulnpH")
	r.Sockets = append(r.Sockets, parseSS("udp", string(udp))...)
	if ctx.Err() != nil {
		r.Partial = "udp: timeout"
	} else if udpErr != nil {
		r.Failed = "udp: " + udpErr.Error()
	}
	return r
}

type firewallReport struct {
	Nft          string `json:"nft"`
	NftTruncated bool   `json:"nft_truncated"`
	Ufw          string `json:"ufw"`
	UfwTruncated bool   `json:"ufw_truncated"`
	Partial      string `json:"partial,omitempty"`
}

func readFirewall() firewallReport {
	ctx, cancel := opBudget()
	defer cancel()
	r := firewallReport{Nft: "absent", Ufw: "absent"}
	if observeHave("nft") {
		out, late := textOf(ctx, "nft", "list", "ruleset")
		r.Nft, r.NftTruncated = capBytes(out, observeTextCap)
		if late {
			r.Nft, r.Partial = "timeout", "nft: timeout"
			return r
		}
	}
	if observeHave("ufw") {
		out, late := textOf(ctx, "ufw", "status", "verbose")
		r.Ufw, r.UfwTruncated = capBytes(out, observeTextCap)
		if late {
			r.Ufw, r.Partial = "timeout", "ufw: timeout"
		}
	}
	return r
}

type unitsReport struct {
	Supported bool         `json:"supported"`
	Reason    string       `json:"reason,omitempty"`
	Partial   string       `json:"partial,omitempty"`
	Failed    []failedUnit `json:"failed"`
	Timers    []timerUnit  `json:"timers"`
}

func readUnits() unitsReport {
	if !observeHave("systemctl") {
		return unitsReport{Reason: "systemctl not installed", Failed: []failedUnit{}, Timers: []timerUnit{}}
	}
	ctx, cancel := opBudget()
	defer cancel()
	r := unitsReport{Supported: true, Timers: []timerUnit{}}
	failed, late := textOf(ctx, "systemctl", "list-units", "--failed", "--no-legend", "--plain")
	r.Failed = parseFailedUnits(failed)
	if late {
		r.Partial = "failed: timeout"
		return r
	}
	timers, late := textOf(ctx, "systemctl", "list-timers", "--all", "--no-legend", "--plain")
	r.Timers = parseTimers(timers)
	if late {
		r.Partial = "timers: timeout"
	}
	return r
}

type dirUsage struct {
	Path   string `json:"path"`
	Exists bool   `json:"exists"`
	// The filesystem the path lives on and how full THAT is — the question
	// "is the disk under kubelet filling" — read by statfs in microseconds.
	// du over /var/lib/rancher (container images, tens of GB) can run past the
	// op budget, so the directory's own byte count is not attempted here.
	FsPct int `json:"fs_pct"`
}

type diskReport struct {
	Filesystems []diskFS   `json:"filesystems"`
	K8sDirs     []dirUsage `json:"k8s_dirs"`
	Partial     string     `json:"partial,omitempty"`
}

func readDisk() diskReport {
	ctx, cancel := opBudget()
	defer cancel()
	r := diskReport{Filesystems: []diskFS{}, K8sDirs: []dirUsage{}}
	if observeHave("df") {
		out, late := textOf(ctx, "df", "-P")
		r.Filesystems = parseDF(out)
		if late {
			r.Partial = "df: timeout"
		}
	}
	for _, d := range []string{"/var/lib/rancher", "/var/lib/kubelet"} {
		if st, err := os.Stat(d); err == nil && st.IsDir() {
			r.K8sDirs = append(r.K8sDirs, dirUsage{Path: d, Exists: true, FsPct: statfsPct(d)})
		}
	}
	return r
}

type kernelReport struct {
	Supported bool   `json:"supported"`
	Reason    string `json:"reason,omitempty"`
	Lines     int    `json:"lines"`
	Truncated bool   `json:"truncated"`
	Dmesg     string `json:"dmesg"`
}

func readKernel() kernelReport {
	if !observeHave("dmesg") {
		return kernelReport{Reason: "dmesg not installed"}
	}
	ctx, cancel := opBudget()
	defer cancel()
	out, err := observeExec(ctx, "dmesg", "-T")
	if ctx.Err() != nil {
		return kernelReport{Reason: "dmesg: timeout"}
	}
	if err != nil && len(out) == 0 {
		return kernelReport{Reason: "dmesg: " + err.Error()}
	}
	text, cut := capLines(string(out), 200, observeTextCap)
	n := 0
	if text != "" {
		n = strings.Count(text, "\n") + 1
	}
	return kernelReport{Supported: true, Lines: n, Truncated: cut, Dmesg: text}
}

type k8sReport struct {
	Supported bool      `json:"supported"`
	Reason    string    `json:"reason,omitempty"`
	Nodes     []k8sNode `json:"nodes,omitempty"`
	Pods      *k8sPods  `json:"pods,omitempty"`
}

func readK8s() k8sReport {
	if _, err := os.Stat(observeKubeconfig); err != nil {
		return k8sReport{Reason: "no read-only kubeconfig at " + observeKubeconfig + " (control planes only)"}
	}
	if !observeHave("kubectl") {
		return k8sReport{Reason: "kubectl not installed"}
	}
	ctx, cancel := opBudget()
	defer cancel()
	rawNodes, err := observeExec(ctx, "kubectl", "--kubeconfig", observeKubeconfig, "get", "nodes", "-o", "json")
	if ctx.Err() != nil {
		return k8sReport{Reason: "kubectl get nodes: timeout"}
	}
	if err != nil {
		return k8sReport{Reason: "kubectl get nodes: " + err.Error()}
	}
	nodes, err := parseK8sNodes(rawNodes)
	if err != nil {
		return k8sReport{Reason: "nodes json: " + err.Error()}
	}
	rawPods, err := observeExec(ctx, "kubectl", "--kubeconfig", observeKubeconfig, "get", "pods", "-A", "-o", "json")
	if ctx.Err() != nil {
		return k8sReport{Reason: "kubectl get pods: timeout", Nodes: nodes}
	}
	if err != nil {
		return k8sReport{Reason: "kubectl get pods: " + err.Error(), Nodes: nodes}
	}
	pods, err := parseK8sPods(rawPods)
	if err != nil {
		return k8sReport{Reason: "pods json: " + err.Error(), Nodes: nodes}
	}
	return k8sReport{Supported: true, Nodes: nodes, Pods: &pods}
}