bastion 0.53.2
This commit is contained in:
@@ -0,0 +1,968 @@
|
||||
package main
|
||||
|
||||
// observe:* — read-only root ops on the networked bastion.
|
||||
//
|
||||
// The bastion runs as root (deploy/install.sh has no User=), so anything it
|
||||
// execs runs as root.
|
||||
// The discipline is the one a read-only sudoers policy lives by: every op is
|
||||
// a FIXED argv, read-only forms only, and no byte of a job's args ever reaches a shell or an
|
||||
// argv position. The single exception is observe:net, whose one argument is
|
||||
// a host[:port] that passes a strict charset check and is then handed to
|
||||
// Go's net package — never to a program.
|
||||
//
|
||||
// Every op answers typed JSON on stdout and exit 0. A box that cannot answer
|
||||
// (a tool missing, a platform without the probes) says so in the JSON —
|
||||
// "absent"/"unknown"/-1 per field, or {"unsupported": platform} for the whole
|
||||
// op — and never invents a value, because these fields feed findings and a
|
||||
// guessed "drop" hides an open firewall.
|
||||
//
|
||||
// Layout: this file is platform-neutral — the handler table entries, the
|
||||
// parsers (pure functions over captured text, unit-tested with fixtures),
|
||||
// the observe:net probe (pure Go), and the posture cache the heartbeat reads.
|
||||
// observe_ops_linux.go holds the drivers that run the fixed argvs;
|
||||
// observe_ops_other.go answers unsupported everywhere else.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/netip"
|
||||
"os"
|
||||
"os/exec"
|
||||
"regexp"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/websocket"
|
||||
)
|
||||
|
||||
// observeOpBudget bounds a whole op: every exec it runs shares ONE deadline.
|
||||
// The host waits 30 s for a dispatch and the lane's own timeout is 25 s;
|
||||
// seven execs each allowed 20 s would be 140 s, past the 180 s after which
|
||||
// the host marks this box offline, so a per-exec bound is the wrong shape. With one
|
||||
// budget a hung tool costs the fields after it — marked "timeout" — and the
|
||||
// op still answers inside the wait. A variable only so a test can shrink it.
|
||||
var observeOpBudget = 20 * time.Second
|
||||
|
||||
// observeTextCap is the byte cap on the free-text ops (observe:firewall,
|
||||
// observe:kernel). A typical ruleset is a few KB; 200 KB is room for a
|
||||
// pathological one without letting a job body grow without bound.
|
||||
const observeTextCap = 200 * 1024
|
||||
|
||||
// observeExec is the ONE seam through which every observe driver runs a
|
||||
// program. It is a variable so the test can replace it with a recorder and
|
||||
// prove that no job argument ever appears in an argv (TestObserveArgsNeverReachArgv).
|
||||
var observeExec = func(ctx context.Context, name string, args ...string) ([]byte, error) {
|
||||
cmd := exec.CommandContext(ctx, name, args...)
|
||||
hideConsole(cmd) // console_windows.go
|
||||
// A grandchild that inherited stdout (apt's helpers, a kubectl plugin)
|
||||
// would otherwise keep Output() blocked after the deadline killed the
|
||||
// child; WaitDelay closes the pipe 1 s after the kill.
|
||||
cmd.WaitDelay = time.Second
|
||||
return cmd.Output()
|
||||
}
|
||||
|
||||
// opBudget is the context every op runs its execs under.
|
||||
func opBudget() (context.Context, context.CancelFunc) {
|
||||
return context.WithTimeout(context.Background(), observeOpBudget)
|
||||
}
|
||||
|
||||
// observeHave reports whether a program resolves on PATH. The drivers ask
|
||||
// before running so a missing tool reads "absent" (a posture in its own
|
||||
// right: an edge box may have no ufw binary at all) rather than an exec error.
|
||||
var observeHave = func(name string) bool {
|
||||
_, err := exec.LookPath(name)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// ── observe:host ─────────────────────────────────────────────────────────────
|
||||
|
||||
// hostPosture reproduces the host's own probe field for field, same names and
|
||||
// same value vocabulary, so the finding rules already written against that
|
||||
// table read this unchanged.
|
||||
// - ufw: status word from `ufw status` line 1 ("active"/"inactive");
|
||||
// "absent" = no binary; "unknown" = binary answered nothing.
|
||||
// - pubwall: `<policy>[+tdrop]` of the `inet pubwall input` chain, or
|
||||
// "absent". "drop+tdrop" is the hardened shape; "accept+tdrop"
|
||||
// fails OPEN if the last rule is lost.
|
||||
// - ssh22: "scoped"/"unscoped"/"unknown" — whether tcp/22 is source-
|
||||
// restricted; the WORST match wins (one bare accept = unscoped).
|
||||
// - f2b: `systemctl is-active fail2ban` verbatim; "absent" = no systemctl.
|
||||
// - input: iptables INPUT policy; "absent" = no iptables; "unknown" = no
|
||||
// policy line.
|
||||
// - sec: pending security packages (^Inst lines only — counting the
|
||||
// Conf lines too doubles it); -1 = no apt-get.
|
||||
// - load: 1-minute load; cpus: core count so load is judged per core.
|
||||
// - reboot: "yes"/"no" for /var/run/reboot-required.
|
||||
// - diskpct: root filesystem use percent as df -P reports it; -1 unreadable.
|
||||
// - jrn: seconds since the last journal line; -1 = could not read one.
|
||||
// - puv: first word of /run/privatae/post-upgrade-verify.status, or "absent".
|
||||
type hostPosture struct {
|
||||
Ufw string `json:"ufw"`
|
||||
Pubwall string `json:"pubwall"`
|
||||
Ssh22 string `json:"ssh22"`
|
||||
F2b string `json:"f2b"`
|
||||
Input string `json:"input"`
|
||||
Sec int `json:"sec"`
|
||||
Load float64 `json:"load"`
|
||||
Cpus int `json:"cpus"`
|
||||
Reboot string `json:"reboot"`
|
||||
DiskPct int `json:"diskpct"`
|
||||
Jrn int64 `json:"jrn"`
|
||||
Puv string `json:"puv"`
|
||||
// Fields whose probe did not finish inside the op budget. String fields
|
||||
// also read "timeout"; sec and jrn read -1, so this list is the only
|
||||
// place that tells "timed out" from "could not read".
|
||||
Timeouts []string `json:"timeouts,omitempty"`
|
||||
}
|
||||
|
||||
// heartbeatPosture is the compact subset the heartbeat carries: the fields a
|
||||
// finding rule reads at a glance. input and puv stay on observe:host.
|
||||
type heartbeatPosture struct {
|
||||
Ufw string `json:"ufw"`
|
||||
Pubwall string `json:"pubwall"`
|
||||
Ssh22 string `json:"ssh22"`
|
||||
F2b string `json:"f2b"`
|
||||
Sec int `json:"sec"`
|
||||
Load float64 `json:"load"`
|
||||
Cpus int `json:"cpus"`
|
||||
Reboot string `json:"reboot"`
|
||||
DiskPct int `json:"diskpct"`
|
||||
Jrn int64 `json:"jrn"`
|
||||
}
|
||||
|
||||
func (p hostPosture) compact() heartbeatPosture {
|
||||
return heartbeatPosture{Ufw: p.Ufw, Pubwall: p.Pubwall, Ssh22: p.Ssh22, F2b: p.F2b,
|
||||
Sec: p.Sec, Load: p.Load, Cpus: p.Cpus, Reboot: p.Reboot, DiskPct: p.DiskPct, Jrn: p.Jrn}
|
||||
}
|
||||
|
||||
// parseUfwStatus reads the first line of `ufw status` ("Status: active").
|
||||
func parseUfwStatus(out string) string {
|
||||
line, _, _ := strings.Cut(out, "\n")
|
||||
f := strings.Fields(line)
|
||||
if len(f) < 2 {
|
||||
return "unknown"
|
||||
}
|
||||
return f[1]
|
||||
}
|
||||
|
||||
var nftPolicyRe = regexp.MustCompile(`hook input.*policy ([a-z]+)`)
|
||||
var nftTerminalDropRe = regexp.MustCompile(`(?m)^\s+counter packets [0-9]+ bytes [0-9]+ drop`)
|
||||
|
||||
// parsePubwall classifies the text of `nft list chain inet pubwall input`.
|
||||
// It reads ONE chain, not the table: an unconditional drop in forward/output
|
||||
// would otherwise count as this chain's terminal drop. Empty text (no table,
|
||||
// nft missing, chain renamed) is "absent" — the direction that raises a
|
||||
// finding rather than hiding one.
|
||||
func parsePubwall(chain string) string {
|
||||
m := nftPolicyRe.FindStringSubmatch(chain)
|
||||
if m == nil {
|
||||
return "absent"
|
||||
}
|
||||
if nftTerminalDropRe.MatchString(chain) {
|
||||
return m[1] + "+tdrop"
|
||||
}
|
||||
return m[1]
|
||||
}
|
||||
|
||||
var nftSSHAcceptRe = regexp.MustCompile(`dport (22|\{ ?22)`)
|
||||
|
||||
// parseSSH22 answers whether sshd is source-restricted, from the pubwall
|
||||
// TABLE text first (every non-ufw host) and the ufw rule list second. Worst
|
||||
// match wins: a chain with one scoped accept and one bare one is "unscoped",
|
||||
// because additive hand edits are how these rulesets actually change and a
|
||||
// debug leftover below a scoped rule is exactly the case that must not read
|
||||
// as protected. haveUfw says whether the second source was even consulted.
|
||||
func parseSSH22(nftTable, ufwStatus string, haveUfw bool) string {
|
||||
found := false
|
||||
for _, l := range strings.Split(nftTable, "\n") {
|
||||
if !nftSSHAcceptRe.MatchString(l) || !strings.Contains(l, "accept") {
|
||||
continue
|
||||
}
|
||||
found = true
|
||||
if !strings.Contains(l, "saddr") {
|
||||
return "unscoped"
|
||||
}
|
||||
}
|
||||
if found {
|
||||
return "scoped"
|
||||
}
|
||||
if haveUfw {
|
||||
for _, l := range strings.Split(ufwStatus, "\n") {
|
||||
if !strings.HasPrefix(l, "22/tcp") || !strings.Contains(l, "ALLOW") {
|
||||
continue
|
||||
}
|
||||
found = true
|
||||
if strings.Contains(l, "Anywhere") {
|
||||
return "unscoped"
|
||||
}
|
||||
}
|
||||
if found {
|
||||
return "scoped"
|
||||
}
|
||||
}
|
||||
return "unknown"
|
||||
}
|
||||
|
||||
// parseIptablesInputPolicy reads the `-P INPUT <policy>` line of `iptables -S`.
|
||||
func parseIptablesInputPolicy(out string) string {
|
||||
for _, l := range strings.Split(out, "\n") {
|
||||
f := strings.Fields(l)
|
||||
if len(f) >= 3 && f[0] == "-P" && f[1] == "INPUT" {
|
||||
return f[2]
|
||||
}
|
||||
}
|
||||
return "unknown"
|
||||
}
|
||||
|
||||
// parseAptSecurityCount counts PACKAGES with a pending security upgrade in
|
||||
// `apt-get -s upgrade` output. Anchored on ^Inst: the simulation prints an
|
||||
// Inst and a Conf line per package and both name the pocket, so counting
|
||||
// every line that says "security" reported exactly double (measured: 40
|
||||
// lines for 20 packages, the same 2:1 on every host checked). Empty output
|
||||
// is -1, "could not read", never 0: a 0 here
|
||||
// clears a finding, and an apt that printed nothing has not said there is
|
||||
// nothing pending.
|
||||
func parseAptSecurityCount(out string) int {
|
||||
if strings.TrimSpace(out) == "" {
|
||||
return -1
|
||||
}
|
||||
n := 0
|
||||
for _, l := range strings.Split(out, "\n") {
|
||||
if strings.HasPrefix(l, "Inst") && (strings.Contains(l, "security") || strings.Contains(l, "Security")) {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// parseLoadavg reads the 1-minute figure from /proc/loadavg text.
|
||||
func parseLoadavg(out string) float64 {
|
||||
f := strings.Fields(out)
|
||||
if len(f) == 0 {
|
||||
return -1
|
||||
}
|
||||
v, err := strconv.ParseFloat(f[0], 64)
|
||||
if err != nil {
|
||||
return -1
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
// parseJournalAge turns one `journalctl -n1 -o short-unix` line into seconds
|
||||
// since it was written, against now. -1 when there is no timestamp to read —
|
||||
// journald has died silently on a production box, which then ran 24 h with
|
||||
// no logs; that is the state this field exists to catch.
|
||||
func parseJournalAge(out string, now time.Time) int64 {
|
||||
lines := strings.Split(strings.TrimRight(out, "\n"), "\n")
|
||||
f := strings.Fields(lines[len(lines)-1])
|
||||
if len(f) == 0 {
|
||||
return -1
|
||||
}
|
||||
stamp, _, _ := strings.Cut(f[0], ".")
|
||||
ts, err := strconv.ParseInt(stamp, 10, 64)
|
||||
if err != nil || ts <= 0 {
|
||||
return -1
|
||||
}
|
||||
return now.Unix() - ts
|
||||
}
|
||||
|
||||
// ── observe:sockets ──────────────────────────────────────────────────────────
|
||||
|
||||
type listenSocket struct {
|
||||
Proto string `json:"proto"`
|
||||
Local string `json:"local"`
|
||||
Port int `json:"port"`
|
||||
Process string `json:"process,omitempty"`
|
||||
PID int `json:"pid,omitempty"`
|
||||
}
|
||||
|
||||
var ssUsersRe = regexp.MustCompile(`\("([^"]*)",pid=([0-9]+)`)
|
||||
|
||||
// parseSS parses `ss -tlnpH` / `ss -ulnpH` (no header; -H). Columns are
|
||||
// State Recv-Q Send-Q Local:Port Peer:Port [Process]; the local address is
|
||||
// split at its LAST colon so "[::]:22" and "10.99.0.1:443" both read right.
|
||||
// Only the first owning process is kept (sshd's socket also lists systemd,
|
||||
// pid 1, which is the socket activator, not the listener that matters).
|
||||
func parseSS(proto, out string) []listenSocket {
|
||||
socks := []listenSocket{}
|
||||
for _, l := range strings.Split(out, "\n") {
|
||||
f := strings.Fields(l)
|
||||
if len(f) < 5 {
|
||||
continue
|
||||
}
|
||||
local := f[3]
|
||||
i := strings.LastIndex(local, ":")
|
||||
if i < 0 {
|
||||
continue
|
||||
}
|
||||
port, err := strconv.Atoi(local[i+1:])
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
s := listenSocket{Proto: proto, Local: local[:i], Port: port}
|
||||
if m := ssUsersRe.FindStringSubmatch(l); m != nil {
|
||||
s.Process = m[1]
|
||||
s.PID, _ = strconv.Atoi(m[2])
|
||||
}
|
||||
socks = append(socks, s)
|
||||
}
|
||||
return socks
|
||||
}
|
||||
|
||||
// ── observe:units ────────────────────────────────────────────────────────────
|
||||
|
||||
type failedUnit struct {
|
||||
Unit string `json:"unit"`
|
||||
Load string `json:"load"`
|
||||
Active string `json:"active"`
|
||||
Sub string `json:"sub"`
|
||||
Description string `json:"description"`
|
||||
}
|
||||
|
||||
type timerUnit struct {
|
||||
Next string `json:"next"`
|
||||
Left string `json:"left"`
|
||||
Last string `json:"last"`
|
||||
Passed string `json:"passed"`
|
||||
Unit string `json:"unit"`
|
||||
Activates string `json:"activates"`
|
||||
}
|
||||
|
||||
// parseFailedUnits parses `systemctl list-units --failed --no-legend --plain`:
|
||||
// UNIT LOAD ACTIVE SUB DESCRIPTION..., description free text.
|
||||
func parseFailedUnits(out string) []failedUnit {
|
||||
units := []failedUnit{}
|
||||
for _, l := range strings.Split(out, "\n") {
|
||||
f := strings.Fields(l)
|
||||
if len(f) < 4 {
|
||||
continue
|
||||
}
|
||||
units = append(units, failedUnit{Unit: f[0], Load: f[1], Active: f[2], Sub: f[3],
|
||||
Description: strings.Join(f[4:], " ")})
|
||||
}
|
||||
return units
|
||||
}
|
||||
|
||||
var weekdays = map[string]bool{"Mon": true, "Tue": true, "Wed": true, "Thu": true, "Fri": true, "Sat": true, "Sun": true}
|
||||
|
||||
// timerStampLen says how many tokens the timestamp at f[i] occupies: 1 for
|
||||
// "-"/"n/a", 4 for "<weekday> <yyyy-mm-dd> <hh:mm:ss> UTC", 0 if f[i] is not a stamp.
|
||||
func timerStampLen(f []string, i int) int {
|
||||
if i >= len(f) {
|
||||
return 0
|
||||
}
|
||||
if f[i] == "-" || f[i] == "n/a" {
|
||||
return 1
|
||||
}
|
||||
if weekdays[f[i]] && i+3 < len(f) && len(f[i+1]) == 10 && f[i+1][4] == '-' {
|
||||
return 4
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// parseTimers parses `systemctl list-timers --all --no-legend --plain`. The
|
||||
// LEFT and PASSED columns are free text of varying token count ("14s",
|
||||
// "4min 39s ago"), so the line is read from both ends: UNIT and ACTIVATES are
|
||||
// the last two tokens (unit names carry no spaces), NEXT and LAST are either
|
||||
// "-" or a four-token timestamp, and whatever lies between is LEFT / PASSED.
|
||||
func parseTimers(out string) []timerUnit {
|
||||
timers := []timerUnit{}
|
||||
for _, l := range strings.Split(out, "\n") {
|
||||
f := strings.Fields(l)
|
||||
if len(f) < 4 {
|
||||
continue
|
||||
}
|
||||
t := timerUnit{Unit: f[len(f)-2], Activates: f[len(f)-1]}
|
||||
f = f[:len(f)-2]
|
||||
n := timerStampLen(f, 0)
|
||||
if n == 0 {
|
||||
continue
|
||||
}
|
||||
t.Next = strings.Join(f[:n], " ")
|
||||
rest := f[n:]
|
||||
// LEFT is at least one token ("14s", "-", "1 day 2h left") and runs
|
||||
// until the next timestamp starts — so a "-" in first position is
|
||||
// LEFT, not LAST.
|
||||
j := 1
|
||||
for j < len(rest) && timerStampLen(rest, j) == 0 {
|
||||
j++
|
||||
}
|
||||
t.Left = strings.Join(rest[:j], " ")
|
||||
if m := timerStampLen(rest, j); m > 0 {
|
||||
t.Last = strings.Join(rest[j:j+m], " ")
|
||||
t.Passed = strings.Join(rest[j+m:], " ")
|
||||
}
|
||||
timers = append(timers, t)
|
||||
}
|
||||
return timers
|
||||
}
|
||||
|
||||
// ── observe:disk ─────────────────────────────────────────────────────────────
|
||||
|
||||
type diskFS struct {
|
||||
Filesystem string `json:"filesystem"`
|
||||
SizeKB int64 `json:"size_kb"`
|
||||
UsedKB int64 `json:"used_kb"`
|
||||
AvailKB int64 `json:"avail_kb"`
|
||||
Pct int `json:"pct"`
|
||||
Mount string `json:"mount"`
|
||||
}
|
||||
|
||||
// parseDF parses `df -P` (POSIX format: one line per filesystem, header
|
||||
// first). Returns the root filesystem's percent too, for the posture field.
|
||||
func parseDF(out string) []diskFS {
|
||||
fss := []diskFS{}
|
||||
for i, l := range strings.Split(out, "\n") {
|
||||
f := strings.Fields(l)
|
||||
if i == 0 || len(f) < 6 {
|
||||
continue
|
||||
}
|
||||
size, e1 := strconv.ParseInt(f[1], 10, 64)
|
||||
used, e2 := strconv.ParseInt(f[2], 10, 64)
|
||||
avail, e3 := strconv.ParseInt(f[3], 10, 64)
|
||||
pct, e4 := strconv.Atoi(strings.TrimSuffix(f[4], "%"))
|
||||
if e1 != nil || e2 != nil || e3 != nil || e4 != nil {
|
||||
continue
|
||||
}
|
||||
fss = append(fss, diskFS{Filesystem: f[0], SizeKB: size, UsedKB: used, AvailKB: avail, Pct: pct,
|
||||
Mount: strings.Join(f[5:], " ")})
|
||||
}
|
||||
return fss
|
||||
}
|
||||
|
||||
func rootDiskPct(fss []diskFS) int {
|
||||
for _, d := range fss {
|
||||
if d.Mount == "/" {
|
||||
return d.Pct
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
// ── observe:k8s ──────────────────────────────────────────────────────────────
|
||||
|
||||
// observeKubeconfig is placed by the control-plane installer as a READ-ONLY
|
||||
// kubeconfig (a ServiceAccount bound to `view`, nothing more). This code never
|
||||
// creates it: a bastion on a worker or a home box has no file and observe:k8s
|
||||
// says so. Where it exists the op is read-only by the credential, not by
|
||||
// argument discipline alone.
|
||||
const observeKubeconfig = "/etc/privatae-bastion/observe.kubeconfig"
|
||||
|
||||
type k8sNode struct {
|
||||
Name string `json:"name"`
|
||||
Ready bool `json:"ready"`
|
||||
Version string `json:"version"`
|
||||
}
|
||||
|
||||
type k8sPods struct {
|
||||
Running int `json:"running"`
|
||||
Pending int `json:"pending"`
|
||||
Failed int `json:"failed"`
|
||||
Crashloop []string `json:"crashloop"`
|
||||
}
|
||||
|
||||
// parseK8sNodes reduces `kubectl get nodes -o json` to name/ready/version.
|
||||
func parseK8sNodes(raw []byte) ([]k8sNode, error) {
|
||||
var list struct {
|
||||
Items []struct {
|
||||
Metadata struct {
|
||||
Name string `json:"name"`
|
||||
} `json:"metadata"`
|
||||
Status struct {
|
||||
Conditions []struct {
|
||||
Type string `json:"type"`
|
||||
Status string `json:"status"`
|
||||
} `json:"conditions"`
|
||||
NodeInfo struct {
|
||||
KubeletVersion string `json:"kubeletVersion"`
|
||||
} `json:"nodeInfo"`
|
||||
} `json:"status"`
|
||||
} `json:"items"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &list); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
nodes := []k8sNode{}
|
||||
for _, it := range list.Items {
|
||||
n := k8sNode{Name: it.Metadata.Name, Version: it.Status.NodeInfo.KubeletVersion}
|
||||
for _, c := range it.Status.Conditions {
|
||||
if c.Type == "Ready" && c.Status == "True" {
|
||||
n.Ready = true
|
||||
}
|
||||
}
|
||||
nodes = append(nodes, n)
|
||||
}
|
||||
return nodes, nil
|
||||
}
|
||||
|
||||
// parseK8sPods reduces `kubectl get pods -A -o json` to phase counts and the
|
||||
// ns/name of every pod with a container in CrashLoopBackOff.
|
||||
func parseK8sPods(raw []byte) (k8sPods, error) {
|
||||
var list struct {
|
||||
Items []struct {
|
||||
Metadata struct {
|
||||
Name string `json:"name"`
|
||||
Namespace string `json:"namespace"`
|
||||
} `json:"metadata"`
|
||||
Status struct {
|
||||
Phase string `json:"phase"`
|
||||
ContainerStatuses []struct {
|
||||
State struct {
|
||||
Waiting *struct {
|
||||
Reason string `json:"reason"`
|
||||
} `json:"waiting"`
|
||||
} `json:"state"`
|
||||
} `json:"containerStatuses"`
|
||||
} `json:"status"`
|
||||
} `json:"items"`
|
||||
}
|
||||
out := k8sPods{Crashloop: []string{}}
|
||||
if err := json.Unmarshal(raw, &list); err != nil {
|
||||
return out, err
|
||||
}
|
||||
for _, it := range list.Items {
|
||||
switch it.Status.Phase {
|
||||
case "Running":
|
||||
out.Running++
|
||||
case "Pending":
|
||||
out.Pending++
|
||||
case "Failed":
|
||||
out.Failed++
|
||||
}
|
||||
for _, cs := range it.Status.ContainerStatuses {
|
||||
if cs.State.Waiting != nil && cs.State.Waiting.Reason == "CrashLoopBackOff" {
|
||||
out.Crashloop = append(out.Crashloop, it.Metadata.Namespace+"/"+it.Metadata.Name)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ── observe:net ──────────────────────────────────────────────────────────────
|
||||
|
||||
// netTargetRe is the whole grammar of the one caller-supplied value in this
|
||||
// family: letters, digits, dot, dash, colon, and brackets for a v6 literal.
|
||||
// No slash (a path), no space, no shell character of any kind — and the value
|
||||
// still never reaches a program; it goes to net.Dial.
|
||||
var netTargetRe = regexp.MustCompile(`^[A-Za-z0-9.\-:\[\]]{1,253}$`)
|
||||
|
||||
// parseNetTarget validates and splits host[:port]. Port defaults to 443, the
|
||||
// port a public front door answers on. A v6 literal must be bracketed
|
||||
// when it carries a port ("[::1]:443"); a bare "::1" is accepted as a host.
|
||||
func parseNetTarget(target string) (host string, port int, err error) {
|
||||
if !netTargetRe.MatchString(target) {
|
||||
return "", 0, errors.New("target must be host[:port] using letters, digits, '.', '-', ':' and [] only")
|
||||
}
|
||||
port = 443
|
||||
host = target
|
||||
switch {
|
||||
case strings.HasPrefix(target, "["):
|
||||
end := strings.Index(target, "]")
|
||||
if end < 0 {
|
||||
return "", 0, errors.New("unclosed '[' in target")
|
||||
}
|
||||
host = target[1:end]
|
||||
rest := target[end+1:]
|
||||
if rest != "" {
|
||||
if !strings.HasPrefix(rest, ":") {
|
||||
return "", 0, errors.New("expected ':port' after ']'")
|
||||
}
|
||||
p, e := strconv.Atoi(rest[1:])
|
||||
if e != nil || p < 1 || p > 65535 {
|
||||
return "", 0, errors.New("bad port")
|
||||
}
|
||||
port = p
|
||||
}
|
||||
if net.ParseIP(host) == nil {
|
||||
return "", 0, errors.New("bracketed target must be an IPv6 literal")
|
||||
}
|
||||
case strings.Count(target, ":") == 1:
|
||||
h, p, _ := strings.Cut(target, ":")
|
||||
host = h
|
||||
n, e := strconv.Atoi(p)
|
||||
if e != nil || n < 1 || n > 65535 {
|
||||
return "", 0, errors.New("bad port")
|
||||
}
|
||||
port = n
|
||||
case strings.Count(target, ":") > 1:
|
||||
if net.ParseIP(target) == nil {
|
||||
return "", 0, errors.New("multiple ':' but not an IPv6 literal")
|
||||
}
|
||||
}
|
||||
if host == "" || strings.HasPrefix(host, "-") || strings.HasPrefix(host, ".") || strings.Contains(host, "..") {
|
||||
return "", 0, errors.New("bad host")
|
||||
}
|
||||
for _, ch := range host {
|
||||
if ch == '[' || ch == ']' {
|
||||
return "", 0, errors.New("bad host")
|
||||
}
|
||||
}
|
||||
return host, port, nil
|
||||
}
|
||||
|
||||
// nonPublicRanges are the addresses observe:net refuses: the unspecified
|
||||
// net, loopback, RFC1918, CGNAT (100.64/10), link-local, ULA. The purpose of
|
||||
// this op is an external vantage on a NAMED PUBLIC door; a probe of a
|
||||
// mesh or LAN address from this box is a different tool, so this is the
|
||||
// default, not an option.
|
||||
var nonPublicRanges = func() []netip.Prefix {
|
||||
var out []netip.Prefix
|
||||
for _, s := range []string{"0.0.0.0/8", "10.0.0.0/8", "100.64.0.0/10", "127.0.0.0/8", "169.254.0.0/16",
|
||||
"172.16.0.0/12", "192.168.0.0/16", "::/128", "::1/128", "fc00::/7", "fe80::/10"} {
|
||||
out = append(out, netip.MustParsePrefix(s))
|
||||
}
|
||||
return out
|
||||
}()
|
||||
|
||||
func isPublicAddr(a netip.Addr) bool {
|
||||
a = a.Unmap()
|
||||
for _, p := range nonPublicRanges {
|
||||
if p.Contains(a) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
const errNetNotPublic = "observe:net probes public doors only"
|
||||
|
||||
// netResolve is the resolver seam (tests answer without DNS).
|
||||
var netResolve = func(ctx context.Context, host string) ([]netip.Addr, error) {
|
||||
return net.DefaultResolver.LookupNetIP(ctx, "ip", host)
|
||||
}
|
||||
|
||||
// publicTarget resolves host and returns its addresses, or an error when ANY
|
||||
// of them is non-public: one private record among public ones is enough to
|
||||
// turn the probe inward, so all must pass.
|
||||
func publicTarget(ctx context.Context, host string) ([]string, error) {
|
||||
var addrs []netip.Addr
|
||||
if a, err := netip.ParseAddr(host); err == nil {
|
||||
addrs = []netip.Addr{a}
|
||||
} else {
|
||||
var err error
|
||||
if addrs, err = netResolve(ctx, host); err != nil {
|
||||
return nil, errors.New("resolve: " + err.Error())
|
||||
}
|
||||
if len(addrs) == 0 {
|
||||
return nil, errors.New("resolve: no addresses")
|
||||
}
|
||||
}
|
||||
out := make([]string, 0, len(addrs))
|
||||
for _, a := range addrs {
|
||||
if !isPublicAddr(a) {
|
||||
return nil, errors.New(errNetNotPublic)
|
||||
}
|
||||
out = append(out, a.Unmap().String())
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
type netProbe struct {
|
||||
Target string `json:"target"`
|
||||
// ValidatedTarget is the normalized host:port the probe actually dialed
|
||||
// and Resolved the addresses that passed the public check, so the pod
|
||||
// can audit what was reached, not just what was asked.
|
||||
ValidatedTarget string `json:"validated_target"`
|
||||
Resolved []string `json:"resolved"`
|
||||
Host string `json:"host"`
|
||||
Port int `json:"port"`
|
||||
TCP bool `json:"tcp"`
|
||||
TCPMs int64 `json:"tcp_ms"`
|
||||
TCPError string `json:"tcp_error,omitempty"`
|
||||
TLS *bool `json:"tls,omitempty"`
|
||||
TLSDaysLeft *int `json:"tls_days_left,omitempty"`
|
||||
TLSSubject string `json:"tls_subject,omitempty"`
|
||||
TLSVerified *bool `json:"tls_verified,omitempty"`
|
||||
TLSError string `json:"tls_error,omitempty"`
|
||||
HTTPStatus *int `json:"http_status,omitempty"`
|
||||
HTTPError string `json:"http_error,omitempty"`
|
||||
ProbedFrom string `json:"probed_from"`
|
||||
ProbedAt string `json:"probed_at"`
|
||||
Unsupported *string `json:"unsupported,omitempty"`
|
||||
}
|
||||
|
||||
const netProbeTimeout = 5 * time.Second
|
||||
|
||||
// probeNet is the external vantage: tcp connect, then TLS days-left when the
|
||||
// port is 443 or the target is a name, then GET / on 443/80. Pure Go; nothing
|
||||
// is exec'd. The TLS handshake skips verification so an expired or self-signed
|
||||
// leaf still yields its dates — that is the fact an operator wants — and
|
||||
// reports verification separately against the system roots.
|
||||
func probeNet(target string) (netProbe, error) {
|
||||
host, port, err := parseNetTarget(target)
|
||||
if err != nil {
|
||||
return netProbe{}, err
|
||||
}
|
||||
addr := net.JoinHostPort(host, strconv.Itoa(port))
|
||||
rctx, rcancel := context.WithTimeout(context.Background(), netProbeTimeout)
|
||||
defer rcancel()
|
||||
resolved, err := publicTarget(rctx, host)
|
||||
if err != nil {
|
||||
return netProbe{}, err
|
||||
}
|
||||
p := netProbe{Target: target, ValidatedTarget: addr, Resolved: resolved, Host: host, Port: port,
|
||||
ProbedAt: time.Now().UTC().Format(time.RFC3339)}
|
||||
if hn, e := os.Hostname(); e == nil {
|
||||
p.ProbedFrom = hn
|
||||
}
|
||||
t0 := time.Now()
|
||||
conn, err := net.DialTimeout("tcp", addr, netProbeTimeout)
|
||||
p.TCPMs = time.Since(t0).Milliseconds()
|
||||
if err != nil {
|
||||
p.TCPError = err.Error()
|
||||
return p, nil
|
||||
}
|
||||
p.TCP = true
|
||||
_ = conn.Close()
|
||||
|
||||
if port == 443 || net.ParseIP(host) == nil {
|
||||
ok := false
|
||||
p.TLS = &ok
|
||||
d := &net.Dialer{Timeout: netProbeTimeout}
|
||||
tc, err := tls.DialWithDialer(d, "tcp", addr, &tls.Config{ServerName: host, InsecureSkipVerify: true}) //nolint:gosec // dates first, verification reported below
|
||||
if err != nil {
|
||||
p.TLSError = err.Error()
|
||||
} else {
|
||||
ok = true
|
||||
_ = tc.SetDeadline(time.Now().Add(netProbeTimeout))
|
||||
certs := tc.ConnectionState().PeerCertificates
|
||||
if len(certs) > 0 {
|
||||
leaf := certs[0]
|
||||
days := int(time.Until(leaf.NotAfter).Hours() / 24)
|
||||
p.TLSDaysLeft = &days
|
||||
p.TLSSubject = leaf.Subject.CommonName
|
||||
verified := verifyChain(certs, host) == nil
|
||||
p.TLSVerified = &verified
|
||||
}
|
||||
_ = tc.Close()
|
||||
}
|
||||
}
|
||||
|
||||
if port == 443 || port == 80 {
|
||||
scheme := "http"
|
||||
if port == 443 {
|
||||
scheme = "https"
|
||||
}
|
||||
client := &http.Client{
|
||||
Timeout: netProbeTimeout,
|
||||
Transport: &http.Transport{
|
||||
TLSClientConfig: &tls.Config{ServerName: host, InsecureSkipVerify: true}, //nolint:gosec // status of the door, not trust in it
|
||||
DisableKeepAlives: true,
|
||||
},
|
||||
CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
|
||||
}
|
||||
resp, err := client.Get(scheme + "://" + addr + "/")
|
||||
if err != nil {
|
||||
p.HTTPError = err.Error()
|
||||
} else {
|
||||
resp.Body.Close()
|
||||
code := resp.StatusCode
|
||||
p.HTTPStatus = &code
|
||||
}
|
||||
}
|
||||
return p, nil
|
||||
}
|
||||
|
||||
func verifyChain(certs []*x509.Certificate, host string) error {
|
||||
inter := x509.NewCertPool()
|
||||
for _, c := range certs[1:] {
|
||||
inter.AddCert(c)
|
||||
}
|
||||
_, err := certs[0].Verify(x509.VerifyOptions{DNSName: host, Intermediates: inter})
|
||||
return err
|
||||
}
|
||||
|
||||
// ── heartbeat posture cache ──────────────────────────────────────────────────
|
||||
|
||||
// posturePayload is what buildHeartbeatBody folds in: the LAST value the
|
||||
// background refresh produced, never a fresh read. The heartbeat ticks every
|
||||
// 60 s (run.go) and a late beat is what flips this box stale on the host
|
||||
// (180 s without a beat) and can trigger quarantine, so a beat may not wait on
|
||||
// a probe at all — one op budget (20 s) of hung apt would already be a third
|
||||
// of the window. Measured on a real box: `apt-get -s upgrade`
|
||||
// costs 0.70 s CPU, the other nine probes 45 ms together; a refresh every
|
||||
// 5 minutes is 12 apt simulations an hour instead of 60. The first beat after
|
||||
// start may carry no posture; off Linux there is never one (no key).
|
||||
const postureRefreshEvery = 5 * time.Minute
|
||||
|
||||
var (
|
||||
postureMu sync.Mutex
|
||||
postureLast *heartbeatPosture
|
||||
)
|
||||
|
||||
func posturePayload() *heartbeatPosture {
|
||||
postureMu.Lock()
|
||||
defer postureMu.Unlock()
|
||||
return postureLast
|
||||
}
|
||||
|
||||
// refreshPosture reads the posture once and publishes it.
|
||||
func refreshPosture() {
|
||||
if !observeHostSupported {
|
||||
return
|
||||
}
|
||||
p := readHostPosture().compact()
|
||||
postureMu.Lock()
|
||||
postureLast = &p
|
||||
postureMu.Unlock()
|
||||
}
|
||||
|
||||
// runPostureRefresh is started by the heartbeat goroutine (run.go) and lives
|
||||
// as long as it does: one read now, then one per window.
|
||||
func runPostureRefresh(ctx context.Context) {
|
||||
if !observeHostSupported {
|
||||
return
|
||||
}
|
||||
refreshPosture()
|
||||
t := time.NewTicker(postureRefreshEvery)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
refreshPosture()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── the handlers ─────────────────────────────────────────────────────────────
|
||||
|
||||
// writeObserve answers with the JSON on stdout and exit 0 — an unsupported box
|
||||
// or a missing tool is a true answer, not a refusal (126 stays "not permitted").
|
||||
func writeObserve(ws *websocket.Conn, v any) {
|
||||
b, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
sendMgmt(ws, 'R', []byte("report: "+err.Error()+"\n"))
|
||||
sendMgmtExit(ws, 1)
|
||||
return
|
||||
}
|
||||
// The machine's own record of what it answered (machinestore.go, the
|
||||
// observe shelf): the same bytes that go down the lane, filed under the op
|
||||
// and the host that asked. Read off this connection's account rather than
|
||||
// threaded through eleven handlers. BEFORE the reply, so a slow disk
|
||||
// cannot leave a result sent-but-unrecorded; a failed write is counted,
|
||||
// never allowed to fail the reply.
|
||||
if a := mgmtAccountOf(ws); a != nil {
|
||||
storeRecordObserve(a.op, a.self, a.args, b, time.Now())
|
||||
}
|
||||
observeSendFrame(ws, 'O', append(b, '\n'))
|
||||
sendMgmtExit(ws, 0)
|
||||
}
|
||||
|
||||
// observeSendFrame is the seam the ordering test uses: it swaps this for a
|
||||
// sender that checks the record is already on disk at the moment the reply
|
||||
// goes. The property the comment above calls load-bearing had no test behind
|
||||
// it — moving the record after the reply passed the entire suite.
|
||||
var observeSendFrame = sendMgmt
|
||||
|
||||
func observeUnsupported() map[string]string { return map[string]string{"unsupported": runtime.GOOS} }
|
||||
|
||||
// Every handler below ignores args (there are none) except observe:net, which
|
||||
// reads exactly one and validates it before use. That property is what
|
||||
// TestObserveArgsNeverReachArgv checks through the observeExec seam.
|
||||
|
||||
func handleObserveHost(ws *websocket.Conn, _ string, _ []string) {
|
||||
if !observeHostSupported {
|
||||
writeObserve(ws, observeUnsupported())
|
||||
return
|
||||
}
|
||||
writeObserve(ws, readHostPosture())
|
||||
}
|
||||
|
||||
func handleObserveSockets(ws *websocket.Conn, _ string, _ []string) {
|
||||
if !observeHostSupported {
|
||||
writeObserve(ws, observeUnsupported())
|
||||
return
|
||||
}
|
||||
writeObserve(ws, readSockets())
|
||||
}
|
||||
|
||||
func handleObserveFirewall(ws *websocket.Conn, _ string, _ []string) {
|
||||
if !observeHostSupported {
|
||||
writeObserve(ws, observeUnsupported())
|
||||
return
|
||||
}
|
||||
writeObserve(ws, readFirewall())
|
||||
}
|
||||
|
||||
func handleObserveUnits(ws *websocket.Conn, _ string, _ []string) {
|
||||
if !observeHostSupported {
|
||||
writeObserve(ws, observeUnsupported())
|
||||
return
|
||||
}
|
||||
writeObserve(ws, readUnits())
|
||||
}
|
||||
|
||||
func handleObserveDisk(ws *websocket.Conn, _ string, _ []string) {
|
||||
if !observeHostSupported {
|
||||
writeObserve(ws, observeUnsupported())
|
||||
return
|
||||
}
|
||||
writeObserve(ws, readDisk())
|
||||
}
|
||||
|
||||
func handleObserveKernel(ws *websocket.Conn, _ string, _ []string) {
|
||||
if !observeHostSupported {
|
||||
writeObserve(ws, observeUnsupported())
|
||||
return
|
||||
}
|
||||
writeObserve(ws, readKernel())
|
||||
}
|
||||
|
||||
func handleObserveK8s(ws *websocket.Conn, _ string, _ []string) {
|
||||
if !observeHostSupported {
|
||||
writeObserve(ws, observeUnsupported())
|
||||
return
|
||||
}
|
||||
writeObserve(ws, readK8s())
|
||||
}
|
||||
|
||||
// observe:net works on every platform: it is pure Go.
|
||||
func handleObserveNet(ws *websocket.Conn, _ string, args []string) {
|
||||
if len(args) != 1 {
|
||||
sendMgmt(ws, 'R', []byte("observe:net takes exactly one argument: host[:port]\n"))
|
||||
sendMgmtExit(ws, 2)
|
||||
return
|
||||
}
|
||||
p, err := probeNet(args[0])
|
||||
if err != nil {
|
||||
sendMgmt(ws, 'R', []byte("refused: "+err.Error()+"\n"))
|
||||
sendMgmtExit(ws, 2)
|
||||
return
|
||||
}
|
||||
writeObserve(ws, p)
|
||||
}
|
||||
|
||||
// capLines keeps the LAST n lines and at most capBytes of text, and says
|
||||
// whether anything was cut.
|
||||
func capLines(s string, n, capBytes int) (string, bool) {
|
||||
cut := false
|
||||
lines := strings.Split(strings.TrimRight(s, "\n"), "\n")
|
||||
if len(lines) > n {
|
||||
lines = lines[len(lines)-n:]
|
||||
cut = true
|
||||
}
|
||||
out := strings.Join(lines, "\n")
|
||||
if len(out) > capBytes {
|
||||
out = out[len(out)-capBytes:]
|
||||
cut = true
|
||||
}
|
||||
return out, cut
|
||||
}
|
||||
|
||||
func capBytes(s string, capBytes int) (string, bool) {
|
||||
if len(s) > capBytes {
|
||||
return s[:capBytes], true
|
||||
}
|
||||
return s, false
|
||||
}
|
||||
Reference in New Issue
Block a user