Files
Privatae 01889e2a48
ci / personal-data (push) Successful in 4s
ci / test (push) Successful in 4m5s
ci / fuzz (push) Successful in 4m10s
bastion 0.58.0
2026-10-29 15:54:51 +00:00

1867 lines
70 KiB
Go

package main
// machinestore_test.go — the machine store's journal.
//
// EVERY TEST HERE DRIVES storeRecordPass AND READS THE FILE BACK. None of them
// asserts a value this test supplied one seam above the line under test — a
// failure earlier rounds kept producing. The fixtures build a deltaReport the
// way deltaPass would; what is checked is what landed on disk.
import (
"context"
"encoding/json"
"fmt"
"github.com/gorilla/websocket"
"os"
"path/filepath"
"regexp"
"strconv"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
)
// storeReport builds a pass that DID compare, over a COMPLETE scan, holding the
// findings given. Complete matters: a partial scan closes nothing (the platform
// saw only recognised programs), so a helper that left it false would make
// every close-path test silently vacuous.
func storeReport(baselineDigest string, findings ...deltaEvent) deltaReport {
return deltaReport{
Baseline: deltaBaselineOK, BaselineDigest: baselineDigest,
BaselineTakenAt: "2026-09-19T00:00:00Z", Coverage: "all-processes",
Complete: true, Findings: findings,
}
}
func storeFinding(fp, level string) deltaEvent {
return deltaEvent{
Kind: deltaProcNew, Level: level, Tier: "new", FP: fp,
Entry: baselineEntry{
FP: fp, Kind: "process", Exe: "/usr/local/bin/" + fp,
ExeDigest: "sha256:" + fp, User: "root", ArgvShape: fp + " --serve",
},
FirstSeen: "2026-09-19T01:00:00Z",
}
}
// storeReadJournal returns the journal lines for a day, in file order.
func storeReadJournal(t *testing.T, day time.Time) []storeLine {
t.Helper()
p, err := storeDayPath(storeFindings, day)
if err != nil {
t.Fatal(err)
}
b, err := os.ReadFile(p)
if err != nil {
if os.IsNotExist(err) {
return nil
}
t.Fatal(err)
}
var out []storeLine
for _, ln := range strings.Split(strings.TrimSpace(string(b)), "\n") {
if ln == "" {
continue
}
var l storeLine
if err := json.Unmarshal([]byte(ln), &l); err != nil {
t.Fatalf("a journal line is not valid JSON: %v\n%s", err, ln)
}
out = append(out, l)
}
return out
}
// THE CENTRAL CLAIM OF THE FILE. A finding is pinned until a human resolves it,
// and the pass runs every 60 seconds — so the difference between a journal and
// a clock is that the second pass writes nothing. If this ever regresses, one
// unresolved finding becomes 1,440 lines a day.
func TestAPinnedFindingIsWrittenOnceNotOncePerPass(t *testing.T) {
isolateConfigDir(t)
r := storeReport("base-a", storeFinding("aaa", deltaHigh))
at := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
n, err := storeRecordPass(r, at)
if err != nil || n != 1 {
t.Fatalf("the first sighting must write exactly one line: n=%d err=%v", n, err)
}
for i := 0; i < 20; i++ {
n, err := storeRecordPass(r, at.Add(time.Duration(i+1)*time.Minute))
if err != nil {
t.Fatal(err)
}
if n != 0 {
t.Fatalf("pass %d re-wrote a finding nothing about which had changed (%d lines)", i+2, n)
}
}
lines := storeReadJournal(t, at)
if len(lines) != 1 {
t.Fatalf("21 passes over one unchanged finding wrote %d lines, want 1", len(lines))
}
if lines[0].Transition != "opened" {
t.Fatalf("transition = %q, want opened", lines[0].Transition)
}
if lines[0].Event == nil || lines[0].Event.FP != "aaa" {
t.Fatalf("the line must carry the whole event: %+v", lines[0].Event)
}
if lines[0].Baseline != "base-a" {
t.Fatalf("baseline = %q — a line that cannot name its accounting is not evidence", lines[0].Baseline)
}
}
// A notice that becomes a critical is not the same finding, and a journal that
// records only `opened` would never say it moved.
func TestAFindingThatChangesLevelIsWrittenAgain(t *testing.T) {
isolateConfigDir(t)
at := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
if _, err := storeRecordPass(storeReport("base-a", storeFinding("aaa", deltaNotice)), at); err != nil {
t.Fatal(err)
}
n, err := storeRecordPass(storeReport("base-a", storeFinding("aaa", deltaCritical)), at.Add(time.Minute))
if err != nil {
t.Fatal(err)
}
if n != 1 {
t.Fatalf("a level change wrote %d lines, want 1", n)
}
lines := storeReadJournal(t, at)
if len(lines) != 2 || lines[1].Transition != "changed" {
t.Fatalf("want opened then changed, got %s", storeTransitions(lines))
}
if lines[1].Event.Level != deltaCritical {
t.Fatalf("the changed line carries level %q, want critical", lines[1].Event.Level)
}
}
// PIDs churn on every service restart. If they were material, a machine that
// restarts nginx nightly would write a `changed` line every night about a
// finding that did not change.
func TestARestartAloneIsNotAChange(t *testing.T) {
isolateConfigDir(t)
at := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
first := storeFinding("aaa", deltaHigh)
first.PIDs = []int{101, 102}
second := storeFinding("aaa", deltaHigh)
second.PIDs = []int{9001, 9002}
if _, err := storeRecordPass(storeReport("base-a", first), at); err != nil {
t.Fatal(err)
}
n, err := storeRecordPass(storeReport("base-a", second), at.Add(time.Minute))
if err != nil {
t.Fatal(err)
}
if n != 0 {
t.Fatalf("new pids alone wrote %d lines — pids must not be material", n)
}
}
func TestAFindingThatStopsBeingReportedIsClosed(t *testing.T) {
isolateConfigDir(t)
at := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
if _, err := storeRecordPass(storeReport("base-a",
storeFinding("aaa", deltaHigh), storeFinding("bbb", deltaWarn)), at); err != nil {
t.Fatal(err)
}
if _, err := storeRecordPass(storeReport("base-a", storeFinding("aaa", deltaHigh)), at.Add(time.Minute)); err != nil {
t.Fatal(err)
}
lines := storeReadJournal(t, at)
if len(lines) != 3 {
t.Fatalf("want 2 opened + 1 closed, got %s", storeTransitions(lines))
}
last := lines[2]
if last.Transition != "closed" || last.Event.FP != "bbb" {
t.Fatalf("the third line must close bbb: %+v", last)
}
// The host must never read `closed` as `resolved` — oversight.py:39-44 makes
// the same distinction and this is the half that has to say so out loud.
if !strings.Contains(last.Note, "not the same as resolved") {
t.Fatalf("a closed line must say what it does not mean: %q", last.Note)
}
// And it must not re-close on the next pass.
n, err := storeRecordPass(storeReport("base-a", storeFinding("aaa", deltaHigh)), at.Add(2*time.Minute))
if err != nil || n != 0 {
t.Fatalf("bbb was closed twice: n=%d err=%v", n, err)
}
}
// THE DANGEROUS ONE. "I could not look" must never be written as "there is
// nothing there". A pass with no authority, or no accounting, closes nothing —
// otherwise deleting the accounting produces a journal showing every finding on
// the box resolving cleanly, which is the most reassuring possible lie.
func TestAPassThatCouldNotCompareWritesNothingAndClosesNothing(t *testing.T) {
isolateConfigDir(t)
at := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
if _, err := storeRecordPass(storeReport("base-a", storeFinding("aaa", deltaCritical)), at); err != nil {
t.Fatal(err)
}
for _, state := range []string{deltaBaselineNone, deltaBaselineNoAuth, deltaBaselineStale} {
blind := deltaReport{Baseline: state}
n, err := storeRecordPass(blind, at.Add(time.Minute))
if err != nil {
t.Fatal(err)
}
if n != 0 {
t.Fatalf("a %q pass wrote %d lines", state, n)
}
lines := storeReadJournal(t, at)
if len(lines) != 1 {
t.Fatalf("a %q pass changed the journal: %s", state, storeTransitions(lines))
}
}
// And the finding is still open afterwards, so a real pass does not re-open it.
n, err := storeRecordPass(storeReport("base-a", storeFinding("aaa", deltaCritical)), at.Add(time.Hour))
if err != nil || n != 0 {
t.Fatalf("a blind pass lost the open set: n=%d err=%v", n, err)
}
}
// An `edited` accounting still compares — it is a warning, not a blindness —
// so its findings must be journalled.
func TestAnEditedAccountingStillJournals(t *testing.T) {
isolateConfigDir(t)
at := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
r := storeReport("base-a", storeFinding("aaa", deltaHigh))
r.Baseline = deltaBaselineEdit
n, err := storeRecordPass(r, at)
if err != nil || n != 1 {
t.Fatalf("an edited accounting must still journal: n=%d err=%v", n, err)
}
}
// A retake redefines normal. Findings measured against the old accounting are
// not comparable to the new one (deltastate.go:36-38), so the open set resets —
// and the journal shows the re-open rather than going quiet about it.
func TestARetakeReopensAgainstTheNewAccounting(t *testing.T) {
isolateConfigDir(t)
at := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
if _, err := storeRecordPass(storeReport("base-a", storeFinding("aaa", deltaHigh)), at); err != nil {
t.Fatal(err)
}
// THIS TEST PREVIOUSLY PINNED A BUG. It asserted exactly 2 lines, which is
// what you get when a retake silently drops the old open set — the finding
// against base-a left the journal with no line at all, and the file's claim
// that replaying reconstructs the open set was false. The retake must CLOSE
// what it stops tracking and then re-open what is still there.
n, err := storeRecordPass(storeReport("base-b", storeFinding("aaa", deltaHigh)), at.Add(time.Minute))
if err != nil {
t.Fatal(err)
}
if n != 2 {
t.Fatalf("a retake wrote %d lines, want 2 (close against the old accounting, open against the new)", n)
}
lines := storeReadJournal(t, at)
if len(lines) != 3 {
t.Fatalf("want opened closed opened, got %s", storeTransitions(lines))
}
if lines[1].Transition != "closed" || lines[1].Event.FP != "aaa" {
t.Fatalf("the retake must close the finding it made incomparable: %+v", lines[1])
}
if !strings.Contains(lines[1].Note, "retaken") {
t.Fatalf("the closing line must say why: %q", lines[1].Note)
}
if lines[2].Transition != "opened" || lines[2].Baseline != "base-b" {
t.Fatalf("the re-opened line must stand against base-b: %+v", lines[2])
}
}
// A finding that opens while the day-file is SEALED must not be recorded as
// open — otherwise the journal never mentions it again, on any pass or any
// later day, and its eventual `closed` refers to a fingerprint nothing opened.
// This was measured in review as a permanent loss.
func TestAFindingRefusedByTheCapIsNotRecordedAsOpen(t *testing.T) {
isolateConfigDir(t)
day1 := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
storeFillToCap(t, day1)
// Day 1: the finding is refused, so nothing is written for it.
if _, err := storeRecordPass(storeReport("base-a", storeFinding("crit", deltaCritical)), day1); err != nil {
t.Fatal(err)
}
for _, l := range storeReadJournal(t, day1) {
if l.Event != nil && l.Event.FP == "crit" {
t.Fatal("the fixture did not actually seal the day")
}
}
// Day 2: a fresh, unsealed file. The finding must appear as OPENED here.
day2 := day1.Add(24 * time.Hour)
n, err := storeRecordPass(storeReport("base-a", storeFinding("crit", deltaCritical)), day2)
if err != nil {
t.Fatal(err)
}
if n == 0 {
t.Fatal("a finding refused by a sealed day was lost for good — it never re-opened")
}
lines := storeReadJournal(t, day2)
found := false
for _, l := range lines {
if l.Transition == "opened" && l.Event != nil && l.Event.FP == "crit" {
found = true
}
}
if !found {
t.Fatalf("day 2 must open the finding the sealed day refused, got %s", storeTransitions(lines))
}
}
// The notice is written ONCE and the day is then sealed. Measured in review at
// four notices in one file with ordinary lines between them, while the notice
// itself said the file "stops here".
func TestACappedDaySealsAndNothingFollowsTheNotice(t *testing.T) {
isolateConfigDir(t)
at := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
storeFillToCap(t, at)
// Many further passes, each offering new findings of varying size.
for i := 0; i < 6; i++ {
if _, err := storeRecordPass(storeReport("base-a",
storeFinding(fmt.Sprintf("f%d", i), deltaHigh)), at.Add(time.Duration(i)*time.Minute)); err != nil {
t.Fatal(err)
}
}
lines := storeReadJournal(t, at)
notices := 0
for i, l := range lines {
if l.Transition == storeCapped {
notices++
if i != len(lines)-1 {
t.Fatalf("a line follows the capped notice at %d of %d: %s",
i, len(lines), storeTransitions(lines[i:]))
}
}
}
if notices != 1 {
t.Fatalf("want exactly one capped notice, got %d", notices)
}
p, err := storeDayPath(storeFindings, at)
if err != nil {
t.Fatal(err)
}
if _, err := os.Stat(p + storeSealSuffix); err != nil {
t.Fatal("the capped day was not sealed")
}
fi, err := os.Stat(p)
if err != nil {
t.Fatal(err)
}
if fi.Size() > storeDayCap {
t.Fatalf("the file crossed its ceiling: %d > %d", fi.Size(), storeDayCap)
}
}
// A PARTIAL SCAN CLOSES NOTHING. On a platform with no unfiltered process walk
// only recognised programs were compared, so an entry missing from the pass may
// never have been looked at — closing it writes "this went away" from a pass
// that did not check.
func TestAPartialScanClosesNothing(t *testing.T) {
isolateConfigDir(t)
at := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
full := storeReport("base-a", storeFinding("aaa", deltaHigh), storeFinding("bbb", deltaHigh))
full.Complete = true
if _, err := storeRecordPass(full, at); err != nil {
t.Fatal(err)
}
partial := storeReport("base-a", storeFinding("aaa", deltaHigh))
partial.Complete = false
n, err := storeRecordPass(partial, at.Add(time.Minute))
if err != nil {
t.Fatal(err)
}
if n != 0 {
t.Fatalf("a partial scan closed %d finding(s)", n)
}
// A COMPLETE pass that still omits bbb does close it.
again := storeReport("base-a", storeFinding("aaa", deltaHigh))
again.Complete = true
if _, err := storeRecordPass(again, at.Add(2*time.Minute)); err != nil {
t.Fatal(err)
}
lines := storeReadJournal(t, at)
if lines[len(lines)-1].Transition != "closed" {
t.Fatalf("a complete pass must still close: %s", storeTransitions(lines))
}
}
// A CONSISTENTLY FILTERED PLATFORM STILL CLOSES. baseline_all_other.go returns
// false unconditionally on darwin and windows, so Complete is permanently false
// there — gating closes on it alone meant no finding ever closed on a Mac or a
// Windows box and the open set grew for the life of the machine. Those
// platforms are not blind, they are filtered on BOTH sides: the accounting was
// taken through the same recognised-only walk, so the scopes agree and an
// absence really is an absence.
func TestAConsistentlyFilteredPlatformStillCloses(t *testing.T) {
isolateConfigDir(t)
at := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
filtered := func(findings ...deltaEvent) deltaReport {
r := storeReport("base-a", findings...)
r.Complete = false // the non-Linux walk
r.Coverage = "processes=recognised-only,schedulers=recognised-only" // and so was the accounting
return r
}
if _, err := storeRecordPass(filtered(
storeFinding("aaa", deltaHigh), storeFinding("bbb", deltaHigh)), at); err != nil {
t.Fatal(err)
}
if _, err := storeRecordPass(filtered(storeFinding("aaa", deltaHigh)), at.Add(time.Minute)); err != nil {
t.Fatal(err)
}
lines := storeReadJournal(t, at)
if lines[len(lines)-1].Transition != "closed" {
t.Fatalf("a filtered platform never closed anything: %s", storeTransitions(lines))
}
// And the genuine MISMATCH still closes nothing: an all-processes accounting
// compared against a partial walk may simply not have looked.
isolateConfigDir(t)
mismatch := func(findings ...deltaEvent) deltaReport {
r := storeReport("base-a", findings...)
r.Complete = false
r.Coverage = "processes=all,schedulers=all"
return r
}
if _, err := storeRecordPass(mismatch(
storeFinding("aaa", deltaHigh), storeFinding("bbb", deltaHigh)), at); err != nil {
t.Fatal(err)
}
n, err := storeRecordPass(mismatch(storeFinding("aaa", deltaHigh)), at.Add(time.Minute))
if err != nil {
t.Fatal(err)
}
if n != 0 {
t.Fatalf("a narrower pass than the accounting closed %d finding(s)", n)
}
}
// Each day-file is self-contained: a finding still open at a rollover is named
// in ONE snapshot line, so a replay whose window starts inside this file still
// knows what was open.
func TestANewDayFileOpensWithOneSnapshotLine(t *testing.T) {
isolateConfigDir(t)
day1 := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
r := storeReport("base-a", storeFinding("aaa", deltaHigh), storeFinding("bbb", deltaHigh))
if _, err := storeRecordPass(r, day1); err != nil {
t.Fatal(err)
}
day2 := day1.Add(24 * time.Hour)
if _, err := storeRecordPass(r, day2); err != nil {
t.Fatal(err)
}
lines := storeReadJournal(t, day2)
if len(lines) != 1 || lines[0].Transition != storeDayOpen {
t.Fatalf("day 2 must open with ONE snapshot line, got %s", storeTransitions(lines))
}
if lines[0].OpenCount != 2 {
t.Fatalf("snapshot says %d open, want 2", lines[0].OpenCount)
}
if strings.Join(lines[0].OpenFPs, ",") != "aaa,bbb" {
t.Fatalf("snapshot names %v", lines[0].OpenFPs)
}
// Not repeated within the day.
if _, err := storeRecordPass(r, day2.Add(time.Minute)); err != nil {
t.Fatal(err)
}
if got := len(storeReadJournal(t, day2)); got != 1 {
t.Fatalf("the snapshot repeated within one day: %d lines", got)
}
}
// THE LOCKUP THIS REPLACED. The first version wrote one line PER open finding,
// ahead of the day's real work — measured in review as a permanent fixed point
// where the shelf wrote nothing but its own memory and a new critical was never
// journalled at all. A large open set must still leave room for the day's
// findings.
func TestALargeOpenSetDoesNotStarveTheDay(t *testing.T) {
isolateConfigDir(t)
day1 := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
var many []deltaEvent
for i := 0; i < 4000; i++ {
many = append(many, storeFinding(fmt.Sprintf("f%04d", i), deltaWarn))
}
if _, err := storeRecordPass(storeReport("base-a", many...), day1); err != nil {
t.Fatal(err)
}
day2 := day1.Add(24 * time.Hour)
// The same 4,000 still open, PLUS a new critical.
withNew := append(append([]deltaEvent(nil), many...), storeFinding("crit", deltaCritical))
if _, err := storeRecordPass(storeReport("base-a", withNew...), day2); err != nil {
t.Fatal(err)
}
lines := storeReadJournal(t, day2)
var snapshots, opened int
sawCritical := false
for _, l := range lines {
switch l.Transition {
case storeDayOpen:
snapshots++
case "opened":
opened++
if l.Event != nil && l.Event.FP == "crit" {
sawCritical = true
}
}
}
if snapshots != 1 {
t.Fatalf("want exactly one snapshot line, got %d", snapshots)
}
if !sawCritical {
t.Fatalf("a new critical was starved by the open set: %d lines, %d opened",
len(lines), opened)
}
}
// The snapshot is BOUNDED, and says so when it cuts the list — otherwise a
// truncated list reads as the whole open set.
func TestTheSnapshotIsBoundedAndSaysSoWhenItCuts(t *testing.T) {
isolateConfigDir(t)
day1 := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
var many []deltaEvent
for i := 0; i < storeSnapshotMaxFPs+50; i++ {
many = append(many, storeFinding(fmt.Sprintf("f%04d", i), deltaWarn))
}
if _, err := storeRecordPass(storeReport("base-a", many...), day1); err != nil {
t.Fatal(err)
}
day2 := day1.Add(24 * time.Hour)
if _, err := storeRecordPass(storeReport("base-a", many...), day2); err != nil {
t.Fatal(err)
}
snap := storeReadJournal(t, day2)[0]
if len(snap.OpenFPs) != storeSnapshotMaxFPs {
t.Fatalf("listed %d fingerprints, want the bound %d", len(snap.OpenFPs), storeSnapshotMaxFPs)
}
if snap.OpenCount != storeSnapshotMaxFPs+50 {
t.Fatalf("count is %d — the count must be the WHOLE set, not the listed part",
snap.OpenCount)
}
if !strings.Contains(snap.Note, "counted, not listed") {
t.Fatalf("a cut list must say it was cut: %q", snap.Note)
}
}
// A box that was off for a week still opens its next day-file with a snapshot.
// Nothing covered the multi-day gap, so a mutation narrowing the test to
// "yesterday only" survived.
func TestAMultiDayGapStillWritesTheSnapshot(t *testing.T) {
isolateConfigDir(t)
day1 := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
r := storeReport("base-a", storeFinding("aaa", deltaHigh))
if _, err := storeRecordPass(r, day1); err != nil {
t.Fatal(err)
}
later := day1.Add(7 * 24 * time.Hour)
if _, err := storeRecordPass(r, later); err != nil {
t.Fatal(err)
}
lines := storeReadJournal(t, later)
if len(lines) != 1 || lines[0].Transition != storeDayOpen {
t.Fatalf("a box back after a week must still open with a snapshot, got %s",
storeTransitions(lines))
}
}
// A REFUSED CLOSE LEAVES THE FINDING OPEN. The close did not happen, so
// forgetting it loses the close for ever — the retake path did exactly that and
// dropped a finding's only record permanently.
func TestARefusedCloseKeepsTheFindingOpen(t *testing.T) {
isolateConfigDir(t)
day1 := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
if _, err := storeRecordPass(storeReport("base-a",
storeFinding("aaa", deltaHigh), storeFinding("bbb", deltaHigh)), day1); err != nil {
t.Fatal(err)
}
storeSealDay(t, day1)
// bbb goes away while the day is sealed: the close is refused.
if _, err := storeRecordPass(storeReport("base-a", storeFinding("aaa", deltaHigh)), day1); err != nil {
t.Fatal(err)
}
// The next day must carry the close, not have forgotten it.
day2 := day1.Add(24 * time.Hour)
if _, err := storeRecordPass(storeReport("base-a", storeFinding("aaa", deltaHigh)), day2); err != nil {
t.Fatal(err)
}
var closed bool
for _, l := range storeReadJournal(t, day2) {
if l.Transition == "closed" && l.Event != nil && l.Event.FP == "bbb" {
closed = true
}
}
if !closed {
t.Fatalf("a close refused by a sealed day was lost: %s",
storeTransitions(storeReadJournal(t, day2)))
}
}
// A RETAKE ON A SEALED DAY MUST RETRY. If the remembered baseline advances
// while its closes were refused, the retake branch is never re-entered and
// those findings' only line anywhere is their original `opened`.
func TestARetakeRefusedByASealedDayIsRetriedTheNextDay(t *testing.T) {
isolateConfigDir(t)
day1 := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
if _, err := storeRecordPass(storeReport("base-a",
storeFinding("aaa", deltaHigh), storeFinding("bbb", deltaHigh)), day1); err != nil {
t.Fatal(err)
}
storeSealDay(t, day1)
// The retake lands on a sealed day: every close is refused.
if _, err := storeRecordPass(storeReport("base-b", storeFinding("aaa", deltaHigh)), day1); err != nil {
t.Fatal(err)
}
day2 := day1.Add(24 * time.Hour)
if _, err := storeRecordPass(storeReport("base-b", storeFinding("aaa", deltaHigh)), day2); err != nil {
t.Fatal(err)
}
seen := map[string]bool{}
for _, l := range storeReadJournal(t, day2) {
if l.Transition == "closed" && l.Event != nil {
seen[l.Event.FP] = true
}
}
if !seen["aaa"] || !seen["bbb"] {
t.Fatalf("the retake's closes were not retried after the seal: %s",
storeTransitions(storeReadJournal(t, day2)))
}
}
// storeSealDay fills a day-file to its ceiling so every later line is refused.
func storeSealDay(t *testing.T, day time.Time) {
t.Helper()
p, err := storeDayPath(storeFindings, day)
if err != nil {
t.Fatal(err)
}
cur, err := os.ReadFile(p)
if err != nil && !os.IsNotExist(err) {
t.Fatal(err)
}
filler := append(bytesOfLine(t), '\n')
buf := append([]byte(nil), cur...)
for len(buf)+len(filler) < storeDayCap-storeNoticeReserve {
buf = append(buf, filler...)
}
if err := os.WriteFile(p, buf, 0o600); err != nil {
t.Fatal(err)
}
if err := storeWriteLine(p+storeSealSuffix, nil); err != nil {
t.Fatal(err)
}
}
// A `changed` that displaces a DIFFERENT baseline entry is a real change, even
// when every other material field is equal.
func TestDisplacingADifferentEntryIsAChange(t *testing.T) {
isolateConfigDir(t)
at := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
mk := func(wasFP string) deltaEvent {
ev := storeFinding("aaa", deltaHigh)
ev.Kind, ev.Reason = deltaProcChanged, deltaReasonExeDigest
ev.Was = &baselineEntry{FP: wasFP, Kind: "process"}
return ev
}
if _, err := storeRecordPass(storeReport("base-a", mk("old-x")), at); err != nil {
t.Fatal(err)
}
n, err := storeRecordPass(storeReport("base-a", mk("old-y")), at.Add(time.Minute))
if err != nil {
t.Fatal(err)
}
if n != 1 {
t.Fatalf("displacing a different entry wrote %d lines, want 1", n)
}
}
// storeFillToCap seals a day-file by filling it to its append ceiling.
func storeFillToCap(t *testing.T, day time.Time) {
t.Helper()
p, err := storeDayPath(storeFindings, day)
if err != nil {
t.Fatal(err)
}
filler := append(bytesOfLine(t), '\n')
var buf []byte
for len(buf)+len(filler) < storeDayCap-storeNoticeReserve {
buf = append(buf, filler...)
}
if err := os.WriteFile(p, buf, 0o600); err != nil {
t.Fatal(err)
}
// Trip the cap so the notice is written and the seal laid down.
if _, err := storeRecordPass(storeReport("base-a", storeFinding("sealer", deltaWarn)), day); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(p + storeSealSuffix); err != nil {
t.Fatal("storeFillToCap did not seal the day")
}
}
// The day-file rolls on UTC, not local time — a host reading a fleet across
// timezones cannot know each box's offset.
func TestTheJournalRollsOnUTCMidnight(t *testing.T) {
isolateConfigDir(t)
// THE TWO INSTANTS MUST DISAGREE ABOUT WHICH DAY IT IS, or this test cannot
// fail. 08:00 on the 20th in a zone ten hours ahead is 22:00 on the 19th in
// UTC: local time names a second day-file, UTC names one. The first version
// of this test used 23:30 in the same zone, which is 13:30 UTC — still the
// 19th either way, so it passed with the UTC conversion deleted.
ahead := time.FixedZone("ahead", 10*3600)
first := time.Date(2026, 9, 19, 13, 0, 0, 0, time.UTC)
second := time.Date(2026, 9, 20, 8, 0, 0, 0, ahead)
if second.Format("2006-01-02") == second.UTC().Format("2006-01-02") {
t.Fatal("the fixture's two clocks agree on the day, so this test cannot fail")
}
if _, err := storeRecordPass(storeReport("base-a", storeFinding("aaa", deltaHigh)), first); err != nil {
t.Fatal(err)
}
if _, err := storeRecordPass(storeReport("base-a",
storeFinding("aaa", deltaHigh), storeFinding("bbb", deltaHigh)), second); err != nil {
t.Fatal(err)
}
if got := len(storeReadJournal(t, first)); got != 2 {
t.Fatalf("both lines belong to the same UTC day: got %d", got)
}
}
// The cap writes its notice ONCE and then goes silent — and the notice is in
// band, so a host replaying the day sees the truncation instead of inferring it
// from a gap.
func TestACappedDayFileSaysSoOnceAndThenStops(t *testing.T) {
isolateConfigDir(t)
at := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
p, err := storeDayPath(storeFindings, at)
if err != nil {
t.Fatal(err)
}
// Fill to just under the append ceiling with well-formed lines, so the NEXT
// line is the one that trips the cap. The append ceiling is the cap less
// the reserve the notice is written out of.
filler := append(bytesOfLine(t), '\n')
var buf []byte
for len(buf)+len(filler) < storeDayCap-storeNoticeReserve {
buf = append(buf, filler...)
}
if err := os.WriteFile(p, buf, 0o600); err != nil {
t.Fatal(err)
}
if _, err := storeRecordPass(storeReport("base-a", storeFinding("aaa", deltaHigh)), at); err != nil {
t.Fatal(err)
}
lines := storeReadJournal(t, at)
last := lines[len(lines)-1]
if last.Transition != storeCapped {
t.Fatalf("the capping line must be the notice, got %q", last.Transition)
}
if !strings.Contains(last.Note, "ceiling") {
t.Fatalf("the notice must say what happened: %q", last.Note)
}
before, err := os.Stat(p)
if err != nil {
t.Fatal(err)
}
// A further pass with a NEW finding must add nothing at all.
if _, err := storeRecordPass(storeReport("base-a",
storeFinding("aaa", deltaHigh), storeFinding("ccc", deltaHigh)), at.Add(time.Minute)); err != nil {
t.Fatal(err)
}
after, err := os.Stat(p)
if err != nil {
t.Fatal(err)
}
if after.Size() != before.Size() {
t.Fatalf("a capped file grew by %d bytes", after.Size()-before.Size())
}
if after.Size() > storeDayCap {
t.Fatalf("the file crossed its own ceiling: %d > %d", after.Size(), storeDayCap)
}
}
func bytesOfLine(t *testing.T) []byte {
t.Helper()
// Computed, not a literal. TestNoFixtureHardcodesAMomentInTime bans a
// literal `At:` because stepup age-checks that field name; this At is a
// journal write-stamp and is never age-checked, but the filler's timestamp
// means nothing to this test either way, so the fixture moves rather than
// the guard. Widening a scan to admit one benign case is how the next real
// one gets through.
b, err := json.Marshal(storeLine{
At: stamp(time.Now()), Transition: "opened", Machine: "box",
Version: "test", Baseline: "base-a", Event: &deltaEvent{FP: "filler"},
})
if err != nil {
t.Fatal(err)
}
return b
}
// Retention reads the NAME, not the mtime. A restored-from-backup file has a
// fresh mtime and is still a month-old record; a sweep keyed on mtime would
// keep exactly the wrong ones and delete the rest.
func TestTheSweepDeletesByTheDayInTheNameNotTheMtime(t *testing.T) {
isolateConfigDir(t)
now := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
shelf, err := storeShelf(storeFindings)
if err != nil {
t.Fatal(err)
}
old := filepath.Join(shelf, "2026-07-01.jsonl") // 80 days back
oldSeal := old + storeSealSuffix // its seal marker
keep := filepath.Join(shelf, "2026-09-18.jsonl") // yesterday
keepSeal := keep + storeSealSuffix
odd := filepath.Join(shelf, "notes-for-me.jsonl") // not ours
// A DATE-SHAPED NAME WITH NO SUFFIX THIS STORE WRITES. Trimming the
// extension optionally meant this parsed as a day and was deleted; nothing
// else writes into this 0700 dir today, but a sweep whose blast radius is
// wider than its comment says is a sweep waiting to surprise somebody.
bare := filepath.Join(shelf, "2026-07-01")
for _, p := range []string{old, oldSeal, keep, keepSeal, odd, bare} {
if err := os.WriteFile(p, []byte("{}\n"), 0o600); err != nil {
t.Fatal(err)
}
// Every file gets a mtime of RIGHT NOW, so an mtime-based sweep would
// delete none of them and this test would fail.
if err := os.Chtimes(p, now, now); err != nil {
t.Fatal(err)
}
}
removed, err := storeSweep(now)
if err != nil {
t.Fatal(err)
}
// The day-file AND its seal marker: a sweep blind to the marker leaves a
// permanent .capped file behind for every day that ever hit its ceiling.
if removed != 2 {
t.Fatalf("swept %d files, want 2 (the day-file and its seal)", removed)
}
if _, err := os.Stat(oldSeal); !os.IsNotExist(err) {
t.Fatal("the old day's seal marker survived the sweep")
}
if _, err := os.Stat(old); !os.IsNotExist(err) {
t.Fatal("the 80-day-old day-file survived the sweep")
}
for _, p := range []string{keep, keepSeal, odd, bare} {
if _, err := os.Stat(p); err != nil {
t.Fatalf("%s was removed and should not have been", filepath.Base(p))
}
}
}
// The store root lives INSIDE the state dir, so a read that escapes it lands on
// the bastion's own state rather than beside the pairings and the TOTP secret.
func TestTheStoreRootIsNestedInsideTheStateDir(t *testing.T) {
cd := isolateConfigDir(t)
root, err := storeDir()
if err != nil {
t.Fatal(err)
}
if filepath.Dir(root) != cd {
t.Fatalf("store root %q is not directly inside the state dir %q", root, cd)
}
fi, err := os.Stat(root)
if err != nil {
t.Fatal(err)
}
if fi.Mode().Perm() != 0o700 {
t.Fatalf("store root is %v, want 0700", fi.Mode().Perm())
}
}
func storeTransitions(lines []storeLine) string {
var out []string
for _, l := range lines {
out = append(out, l.Transition)
}
return "[" + strings.Join(out, " ") + "]"
}
// ── the alerts shelf ────────────────────────────────────────────────────────
func storeReadAlerts(t *testing.T, day time.Time) []storeAlertLine {
t.Helper()
p, err := storeDayPath(storeAlerts, day)
if err != nil {
t.Fatal(err)
}
b, err := os.ReadFile(p)
if err != nil {
if os.IsNotExist(err) {
return nil
}
t.Fatal(err)
}
var out []storeAlertLine
for _, ln := range strings.Split(strings.TrimSpace(string(b)), "\n") {
if ln == "" {
continue
}
var l storeAlertLine
if err := json.Unmarshal([]byte(ln), &l); err != nil {
t.Fatalf("alert line is not valid JSON: %v\n%s", err, ln)
}
out = append(out, l)
}
return out
}
// A refresh runs every few minutes and republishes the same firing set. Like
// the findings journal, only the edges are written.
func TestAnAlertIsJournalledOnceUntilItClears(t *testing.T) {
isolateConfigDir(t)
at := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
firing := &alertsSummary{Firing: 2, Critical: 1, Warning: 1,
Fingerprints: []string{"fp-a", "fp-b"}}
n, err := storeRecordAlerts(firing, at)
if err != nil || n != 2 {
t.Fatalf("two firing alerts must write two lines: n=%d err=%v", n, err)
}
for i := 0; i < 5; i++ {
if n, err := storeRecordAlerts(firing, at.Add(time.Duration(i+1)*time.Minute)); err != nil || n != 0 {
t.Fatalf("an unchanged alert set re-wrote %d lines (err=%v)", n, err)
}
}
// fp-b clears.
cleared := &alertsSummary{Firing: 1, Critical: 1, Fingerprints: []string{"fp-a"}}
if n, err := storeRecordAlerts(cleared, at.Add(time.Hour)); err != nil || n != 1 {
t.Fatalf("a cleared alert must write one line: n=%d err=%v", n, err)
}
lines := storeReadAlerts(t, at)
if len(lines) != 3 {
t.Fatalf("want firing firing cleared, got %d lines", len(lines))
}
if lines[2].Transition != "cleared" || lines[2].FP != "fp-b" {
t.Fatalf("the third line must clear fp-b: %+v", lines[2])
}
// EVERY count, not just firing. A reader joining the journal mid-day takes
// its totals from the line; asserting one of the three left the other two
// untested, which a mutation dropping them proved by surviving.
if lines[0].Firing != 2 || lines[0].Critical != 1 || lines[0].Warning != 1 {
t.Fatalf("the opening line must carry all three counts as of itself: %+v", lines[0])
}
if lines[2].Firing != 1 || lines[2].Critical != 1 || lines[2].Warning != 0 {
t.Fatalf("the clearing line must carry the counts as of itself: %+v", lines[2])
}
}
// NIL IS NOT EMPTY. A box with no Alertmanager configured, or one that has not
// refreshed yet, is not watching anything — it must not be read as watching
// nothing, and it must not clear alerts it can no longer see.
func TestAnUnconfiguredAlertSourceClearsNothing(t *testing.T) {
isolateConfigDir(t)
at := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
if _, err := storeRecordAlerts(&alertsSummary{Firing: 1, Critical: 1,
Fingerprints: []string{"fp-a"}}, at); err != nil {
t.Fatal(err)
}
n, err := storeRecordAlerts(nil, at.Add(time.Minute))
if err != nil || n != 0 {
t.Fatalf("a nil summary wrote %d lines (err=%v)", n, err)
}
if got := len(storeReadAlerts(t, at)); got != 1 {
t.Fatalf("a nil summary changed the journal: %d lines", got)
}
// And fp-a is still open, so a real summary does not re-open it.
if n, err := storeRecordAlerts(&alertsSummary{Firing: 1, Critical: 1,
Fingerprints: []string{"fp-a"}}, at.Add(time.Hour)); err != nil || n != 0 {
t.Fatalf("the nil summary lost the open set: n=%d err=%v", n, err)
}
}
// The summary names at most twenty fingerprints. Past that the set is a sample,
// so the journal says so rather than letting a truncated list read as the whole
// one — otherwise alert 21 clearing is indistinguishable from it never firing.
func TestATruncatedAlertSummarySaysSoInBand(t *testing.T) {
isolateConfigDir(t)
at := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
if _, err := storeRecordAlerts(&alertsSummary{Firing: 57, Critical: 3,
Fingerprints: []string{"fp-a", "fp-b"}}, at); err != nil {
t.Fatal(err)
}
lines := storeReadAlerts(t, at)
last := lines[len(lines)-1]
if last.Transition != "truncated" {
t.Fatalf("a truncated summary must end in a truncated line, got %q", last.Transition)
}
if !strings.Contains(last.Note, "57") || !strings.Contains(last.Note, "2") {
t.Fatalf("the note must name both numbers: %q", last.Note)
}
}
// ── the sweep's cadence ─────────────────────────────────────────────────────
// Keyed on the DATE, not an interval. An interval measures uptime, so a laptop
// awake two hours a day would sweep every twelve days and its store would grow
// past the retention it advertises.
func TestTheSweepRunsOncePerDayAndNotOncePerUptimeInterval(t *testing.T) {
isolateConfigDir(t)
shelf, err := storeShelf(storeFindings)
if err != nil {
t.Fatal(err)
}
write := func(name string) string {
p := filepath.Join(shelf, name)
if err := os.WriteFile(p, []byte("{}\n"), 0o600); err != nil {
t.Fatal(err)
}
return p
}
old := write("2026-07-01.jsonl")
day := time.Date(2026, 9, 19, 0, 30, 0, 0, time.UTC)
storeSweepIfDue(day)
if _, err := os.Stat(old); !os.IsNotExist(err) {
t.Fatal("the first call of the day did not sweep")
}
// A second old file appearing later the SAME day is not swept again.
old2 := write("2026-07-02.jsonl")
storeSweepIfDue(day.Add(23 * time.Hour))
if _, err := os.Stat(old2); err != nil {
t.Fatal("the sweep ran twice in one UTC day")
}
// The next UTC day sweeps again.
storeSweepIfDue(day.Add(25 * time.Hour))
if _, err := os.Stat(old2); !os.IsNotExist(err) {
t.Fatal("the sweep did not run on the next day")
}
}
// The truncation notice must not become a clock. Measured in review: written
// unconditionally, five refreshes of an unchanged 57-alert set produced
// 21, 1, 1, 1, 1 lines — and refreshAlerts runs every 60s PER PAIRING.
func TestTheTruncationNoticeIsNotRepeatedOnEveryRefresh(t *testing.T) {
isolateConfigDir(t)
at := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
big := &alertsSummary{Firing: 57, Critical: 3, Fingerprints: []string{"fp-a", "fp-b"}}
first, err := storeRecordAlerts(big, at)
if err != nil || first == 0 {
t.Fatalf("the first refresh must write: n=%d err=%v", first, err)
}
for i := 0; i < 5; i++ {
n, err := storeRecordAlerts(big, at.Add(time.Duration(i+1)*time.Minute))
if err != nil {
t.Fatal(err)
}
if n != 0 {
t.Fatalf("refresh %d re-wrote %d line(s) for an unchanged alert set", i+2, n)
}
}
notices := 0
for _, l := range storeReadAlerts(t, at) {
if l.Transition == "truncated" {
notices++
}
}
if notices != 1 {
t.Fatalf("want one truncation notice across six refreshes, got %d", notices)
}
// A CHANGE in the truncation is still reported — otherwise the count going
// from 57 to 400 would be invisible.
n, err := storeRecordAlerts(&alertsSummary{Firing: 400, Critical: 9,
Fingerprints: []string{"fp-a", "fp-b"}}, at.Add(time.Hour))
if err != nil || n != 1 {
t.Fatalf("a changed truncation must be reported: n=%d err=%v", n, err)
}
// AND THE LISTED HALF COUNTS TOO. Keying the dedup on `firing` alone misses
// 400/2 -> 400/1: the same number of alerts firing, but the journal now
// names fewer of them, so its blind spot GREW and the reader must be told.
if _, err := storeRecordAlerts(&alertsSummary{Firing: 400, Critical: 9,
Fingerprints: []string{"fp-a"}}, at.Add(2*time.Hour)); err != nil {
t.Fatal(err)
}
var sawShrink bool
for _, l := range storeReadAlerts(t, at) {
if l.Transition == "truncated" && strings.Contains(l.Note, "names 1 of them") {
sawShrink = true
}
}
if !sawShrink {
t.Fatal("the listed count shrank from 2 to 1 and no notice said so")
}
}
// A single line larger than the ceiling must not create a file already past it.
// os.Stat returns ErrNotExist for a missing file, and an earlier version
// skipped the whole cap check in that branch.
func TestAnOversizedFirstLineCannotCreateAnAlreadyOverflowingFile(t *testing.T) {
isolateConfigDir(t)
at := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
huge := storeFinding("huge", deltaCritical)
// One evidence line past the ceiling on its own.
huge.Evidence = []evidence{{Signal: "fixture", Tier: tierVerified,
Value: strings.Repeat("x", storeDayCap+1)}}
p, err := storeDayPath(storeFindings, at)
if err != nil {
t.Fatal(err)
}
if _, err := os.Stat(p); !os.IsNotExist(err) {
t.Fatal("the day-file must not exist yet, or this test proves nothing")
}
if _, err := storeRecordPass(storeReport("base-a", huge), at); err != nil {
t.Fatal(err)
}
fi, err := os.Stat(p)
if err != nil {
t.Fatal(err)
}
if fi.Size() > storeDayCap {
t.Fatalf("an oversized first line crossed the ceiling: %d > %d", fi.Size(), storeDayCap)
}
lines := storeReadJournal(t, at)
if len(lines) != 1 || lines[0].Transition != storeCapped {
t.Fatalf("want a lone capped notice, got %s", storeTransitions(lines))
}
}
// refreshAlerts had no in-flight guard while refreshDelta did, and run.go
// starts ONE refresher PER PAIRING — so a box serving several hosts runs
// several in phase. Without the guard a slow goroutine holding an older summary
// can take storeMu after a newer one and journal an alert as `firing` that the
// newer pass just journalled as `cleared`: the order of the record inverts.
//
// Driven through the observeHTTPGet seam the suite already uses, so this
// measures the guard rather than asserting it.
func TestOnlyOneAlertsRefreshRunsAtATime(t *testing.T) {
isolateConfigDir(t)
defer withEndpointsFile(t, `{"alertmanager":"http://127.0.0.1:9093"}`)()
release := make(chan struct{})
var inFlight, calls int32
origGet := observeHTTPGet
observeHTTPGet = func(_ context.Context, _ string) ([]byte, int, error) {
atomic.AddInt32(&calls, 1)
if atomic.AddInt32(&inFlight, 1) > 1 {
t.Error("two alerts refreshes were in flight at once")
}
<-release
atomic.AddInt32(&inFlight, -1)
return []byte(`{"status":"success","data":[]}`), 200, nil
}
defer func() { observeHTTPGet = origGet }()
var wg sync.WaitGroup
wg.Add(1)
go func() { defer wg.Done(); refreshAlerts(context.Background()) }()
// Wait until the first refresh is genuinely inside the fetch, or this races
// to a pass rather than measuring anything.
deadline := time.Now().Add(5 * time.Second)
for atomic.LoadInt32(&calls) == 0 {
if time.Now().After(deadline) {
close(release)
wg.Wait()
t.Fatal("the first refresh never reached the fetch seam")
}
time.Sleep(time.Millisecond)
}
// The second must return AT ONCE rather than queueing behind the first.
done := make(chan struct{})
go func() { refreshAlerts(context.Background()); close(done) }()
select {
case <-done:
case <-time.After(2 * time.Second):
close(release)
wg.Wait()
t.Fatal("a concurrent refresh blocked instead of being refused by the in-flight guard")
}
if got := atomic.LoadInt32(&calls); got != 1 {
t.Fatalf("the guarded refresh still fetched: %d calls, want 1", got)
}
close(release)
wg.Wait()
}
// The beat reports where the machine keeps its own record, and reporting it
// must not CREATE the directory: the beat fires every sixty seconds, and a
// path is a claim about where the record is, not an instruction to make one.
func TestTheBeatReportsTheStoreRootWithoutCreatingIt(t *testing.T) {
isolateConfigDir(t)
raw := buildHeartbeatBody("", "h", "linux", "test", "pk", "fp", "100.64.0.9", "ep", false, nil, nil, nil)
var body map[string]any
if err := json.Unmarshal([]byte(raw), &body); err != nil {
t.Fatal(err)
}
got, _ := body["machine_record_root"].(string)
want, err := storeRootPath()
if err != nil {
t.Fatal(err)
}
if got != want {
t.Fatalf("beat reports %q, want %q", got, want)
}
if !filepath.IsAbs(got) {
t.Fatalf("the reported root is not absolute: %q", got)
}
if _, err := os.Stat(got); !os.IsNotExist(err) {
t.Fatalf("building a beat created the store directory %q", got)
}
}
// ── round 2 verification: the last three ────────────────────────────────────
// A retake that coincides with a rollover must produce a day-file that replays
// on its own: the snapshot runs BEFORE the retake-close. Measured before the
// fix: `[closed aaa, closed bbb, opened aaa]` and no day_open line — two closes
// for fingerprints the file never introduced.
func TestARetakeOnARolloverDayStillOpensWithTheSnapshot(t *testing.T) {
isolateConfigDir(t)
day1 := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
if _, err := storeRecordPass(storeReport("base-a",
storeFinding("aaa", deltaHigh), storeFinding("bbb", deltaHigh)), day1); err != nil {
t.Fatal(err)
}
day2 := day1.Add(24 * time.Hour)
if _, err := storeRecordPass(storeReport("base-b", storeFinding("aaa", deltaHigh)), day2); err != nil {
t.Fatal(err)
}
lines := storeReadJournal(t, day2)
got := storeTransitions(lines)
if got != "[day_open closed closed opened]" {
t.Fatalf("day 2 must replay alone: want [day_open closed closed opened], got %s", got)
}
if lines[0].OpenCount != 2 || strings.Join(lines[0].OpenFPs, ",") != "aaa,bbb" {
t.Fatalf("the snapshot must name what the closes are about: %+v", lines[0])
}
}
// A refused `changed` stays open under its OLD digest, so the next unsealed
// day writes `changed` — not a second `opened` for a finding already announced.
func TestARefusedChangeIsWrittenAsAChangeNotASecondOpen(t *testing.T) {
isolateConfigDir(t)
day1 := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
if _, err := storeRecordPass(storeReport("base-a", storeFinding("aaa", deltaNotice)), day1); err != nil {
t.Fatal(err)
}
storeSealDay(t, day1)
// Escalates while the day is sealed: the `changed` line is refused.
if _, err := storeRecordPass(storeReport("base-a", storeFinding("aaa", deltaCritical)), day1); err != nil {
t.Fatal(err)
}
day2 := day1.Add(24 * time.Hour)
if _, err := storeRecordPass(storeReport("base-a", storeFinding("aaa", deltaCritical)), day2); err != nil {
t.Fatal(err)
}
var seen []string
for _, l := range storeReadJournal(t, day2) {
if l.Event != nil && l.Event.FP == "aaa" {
seen = append(seen, l.Transition)
}
}
if strings.Join(seen, " ") != "changed" {
t.Fatalf("the escalation must land as one `changed` on day 2, got %v", seen)
}
}
// The alerts truncation state must not advance on a REFUSED line — a sealed
// alerts day recorded the notice as delivered, and the next day never wrote
// it. The fix was in round 1; nothing exercised the sealed-day path.
func TestATruncationNoticeRefusedByASealedDayIsWrittenTheNextDay(t *testing.T) {
isolateConfigDir(t)
day1 := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
// Seal the ALERTS shelf for day 1.
p, err := storeDayPath(storeAlerts, day1)
if err != nil {
t.Fatal(err)
}
filler := append(bytesOfLine(t), '\n')
var buf []byte
for len(buf)+len(filler) < storeDayCap-storeNoticeReserve {
buf = append(buf, filler...)
}
if err := os.WriteFile(p, buf, 0o600); err != nil {
t.Fatal(err)
}
if err := storeWriteLine(p+storeSealSuffix, nil); err != nil {
t.Fatal(err)
}
big := &alertsSummary{Firing: 57, Critical: 3, Fingerprints: []string{"fp-a"}}
if _, err := storeRecordAlerts(big, day1); err != nil {
t.Fatal(err)
}
day2 := day1.Add(24 * time.Hour)
if _, err := storeRecordAlerts(big, day2); err != nil {
t.Fatal(err)
}
var notice bool
for _, l := range storeReadAlerts(t, day2) {
if l.Transition == "truncated" {
notice = true
}
}
if !notice {
t.Fatal("the truncation notice refused on the sealed day was never written")
}
}
// A v1 state file is MIGRATED, not discarded: a finding open under v1 that is
// gone after the upgrade gets its `closed` line.
func TestAV1StateFileIsCarriedAcrossTheVersionBump(t *testing.T) {
isolateConfigDir(t)
sp, err := storeStatePath()
if err != nil {
t.Fatal(err)
}
if err := os.MkdirAll(filepath.Dir(sp), 0o700); err != nil {
t.Fatal(err)
}
v1 := `{"version":"privatae-store-state-v1","baseline":"base-a","open":{"aaa":"x","zzz":"y"}}`
if err := os.WriteFile(sp, []byte(v1), 0o600); err != nil {
t.Fatal(err)
}
at := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
// Only aaa is still present; zzz went away during the upgrade window.
if _, err := storeRecordPass(storeReport("base-a", storeFinding("aaa", deltaHigh)), at); err != nil {
t.Fatal(err)
}
var closedZ bool
for _, l := range storeReadJournal(t, at) {
if l.Transition == "closed" && l.Event != nil && l.Event.FP == "zzz" {
closedZ = true
}
if l.Transition == "opened" && l.Event != nil && l.Event.FP == "aaa" {
t.Fatal("a finding open under v1 was re-opened after the upgrade")
}
}
if !closedZ {
t.Fatal("a finding open under v1 that went away got no closed line after the upgrade")
}
}
// ── the observe shelf ───────────────────────────────────────────────────────
func storeReadObserve(t *testing.T, op string, day time.Time) []storeObserveLine {
t.Helper()
p, err := storeDayPath(filepath.Join(storeObserve, storeSafeOpName(op)), day)
if err != nil {
t.Fatal(err)
}
b, err := os.ReadFile(p)
if err != nil {
if os.IsNotExist(err) {
return nil
}
t.Fatal(err)
}
var out []storeObserveLine
for _, ln := range strings.Split(strings.TrimSpace(string(b)), "\n") {
if ln == "" {
continue
}
var l storeObserveLine
if err := json.Unmarshal([]byte(ln), &l); err != nil {
t.Fatalf("observe line is not valid JSON: %v\n%s", err, ln)
}
out = append(out, l)
}
return out
}
// END TO END THROUGH THE REAL PATH: the management dispatcher, this
// connection's account, writeObserve, the store. Not the writer called by
// hand — the round-1 review removed a writer nothing called, and the proof
// that this one is called is the dispatcher calling it.
func TestAnObserveResultIsRecordedOnTheMachine(t *testing.T) {
isolateConfigDir(t)
storeObserveDroppedReset()
out, errOut, code := driveObserve(t, "observe:host", nil)
if code != 0 {
t.Fatalf("observe:host failed: %d %s", code, errOut)
}
lines := storeReadObserve(t, "observe:host", time.Now())
if len(lines) != 1 {
t.Fatalf("want one recorded result, got %d", len(lines))
}
l := lines[0]
if l.Op != "observe:host" || l.Truncated || len(l.Result) == 0 {
t.Fatalf("recorded line is not the result: %+v", l)
}
// The SAME bytes the host received — a record of a different answer than
// the one sent is not a record.
var sent, kept any
if err := json.Unmarshal([]byte(strings.TrimSpace(out)), &sent); err != nil {
t.Fatalf("the reply is not JSON: %v", err)
}
if err := json.Unmarshal(l.Result, &kept); err != nil {
t.Fatal(err)
}
if fmt.Sprint(sent) != fmt.Sprint(kept) {
t.Fatal("the recorded result differs from the reply the host received")
}
if storeObserveDroppedSnapshot() != nil {
t.Fatal("a successful record counted as dropped")
}
}
func TestTheRecordNamesWhoAskedAndWhat(t *testing.T) {
isolateConfigDir(t)
now := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
if !storeRecordObserve("observe:net", "member.host.privatae.ai", []string{"10.0.0.5:443"},
[]byte(`{"tcp":"open"}`), now) {
t.Fatal("a small result was not recorded")
}
l := storeReadObserve(t, "observe:net", now)[0]
if l.AskedBy != "member.host.privatae.ai" || strings.Join(l.Args, ",") != "10.0.0.5:443" {
t.Fatalf("who/what missing: %+v", l)
}
if string(l.Result) != `{"tcp":"open"}` || l.Bytes != len(`{"tcp":"open"}`) {
t.Fatalf("result not kept whole: %+v", l)
}
}
// A result past the line bound keeps its first 4 KiB and says how much was
// cut — the shape of what was seen, never nothing, and never a day-file eaten
// by one chatty op.
func TestAnOversizedResultIsKeptAsAHeadWithItsSize(t *testing.T) {
isolateConfigDir(t)
now := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
big := []byte(`{"units":"` + strings.Repeat("u", storeObserveLineMax+5000) + `"}`)
if !storeRecordObserve("observe:units", "h", nil, big, now) {
t.Fatal("an oversized result was refused outright")
}
l := storeReadObserve(t, "observe:units", now)[0]
if !l.Truncated || len(l.Result) != 0 || len(l.Head) != storeObserveHeadKeep || l.Bytes != len(big) {
t.Fatalf("truncation shape wrong: truncated=%v result=%d head=%d bytes=%d",
l.Truncated, len(l.Result), len(l.Head), l.Bytes)
}
p, _ := storeDayPath(filepath.Join(storeObserve, "observe-units"), now)
fi, _ := os.Stat(p)
if fi.Size() > int64(storeObserveLineMax) {
t.Fatalf("the line on disk is %d bytes, over the %d bound", fi.Size(), storeObserveLineMax)
}
}
// Only observe results are recorded here, and an op name cannot choose where
// on disk its record lands.
func TestOnlyObserveOpsAreRecordedAndTheNameCannotEscape(t *testing.T) {
isolateConfigDir(t)
now := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
for _, op := range []string{"ctx_get", "dcn_list", "", "uptime"} {
if storeRecordObserve(op, "h", nil, []byte(`{}`), now) {
t.Fatalf("%q was recorded on the observe shelf", op)
}
}
if !storeRecordObserve("observe:../../etc", "h", nil, []byte(`{}`), now) {
t.Fatal("a hostile-looking observe name was refused rather than sanitised")
}
root, _ := storeDir()
p := filepath.Join(root, storeObserve, "observe-etc", "2026-09-19.jsonl")
if _, err := os.Stat(p); err != nil {
t.Fatalf("the sanitised record is not where it must be: %v", err)
}
}
// A result the record could not keep is COUNTED, and the count rides the beat
// — a record with holes must never read as a quiet machine.
func TestADroppedResultIsCountedOnTheBeatByShelf(t *testing.T) {
cd := isolateConfigDir(t)
storeObserveDroppedReset()
// A regular FILE where the store root must be a directory: every write fails.
if err := os.MkdirAll(cd, 0o700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(cd, storeDirName), []byte("in the way"), 0o600); err != nil {
t.Fatal(err)
}
if storeRecordObserve("observe:host", "h", nil, []byte(`{}`), time.Now()) {
t.Fatal("a write into a file-as-directory succeeded")
}
storeRecordObserve("observe:sockets", "h", nil, []byte(`{}`), time.Now())
storeRecordObserve("observe:sockets", "h", nil, []byte(`{}`), time.Now())
snap := storeObserveDroppedSnapshot()
if snap["observe-host"] != 1 || snap["observe-sockets"] != 2 {
t.Fatalf("dropped counts by shelf are wrong: %v", snap)
}
raw := buildHeartbeatBody("", "h", "linux", "test", "pk", "fp", "100.64.0.9", "ep", false, nil, nil, nil)
var body map[string]any
if err := json.Unmarshal([]byte(raw), &body); err != nil {
t.Fatal(err)
}
d, _ := body["machine_record_dropped"].(map[string]any)
if d["observe-host"] != 1.0 || d["observe-sockets"] != 2.0 {
t.Fatalf("the beat does not name the shelves with holes: %v", body["machine_record_dropped"])
}
// And an EMPTY OBJECT — never absent — when nothing was dropped and the
// pairing may observe. Absent has to mean exactly one thing to the host:
// this pairing is not entitled to say. (The first version omitted the key
// at zero, and the host could not tell "whole" from "no longer allowed".)
// A FRESH map: Unmarshal into a populated map merges keys.
storeObserveDroppedReset()
raw = buildHeartbeatBody("", "h", "linux", "test", "pk", "fp", "100.64.0.9", "ep", false, nil, nil, nil)
body = map[string]any{}
_ = json.Unmarshal([]byte(raw), &body)
d2, present := body["machine_record_dropped"].(map[string]any)
if !present || len(d2) != 0 {
t.Fatalf("a whole record must travel as an empty object, got %v (present=%v)", body["machine_record_dropped"], present)
}
}
// The sweep reaches the nested observe shelves.
func TestTheSweepReachesTheObserveShelves(t *testing.T) {
isolateConfigDir(t)
now := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
dir, err := storeShelf(filepath.Join(storeObserve, "observe-net"))
if err != nil {
t.Fatal(err)
}
old := filepath.Join(dir, "2026-07-01.jsonl")
keep := filepath.Join(dir, "2026-09-18.jsonl")
for _, p := range []string{old, keep} {
if err := os.WriteFile(p, []byte("{}\n"), 0o600); err != nil {
t.Fatal(err)
}
}
if _, err := storeSweep(now); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(old); !os.IsNotExist(err) {
t.Fatal("an 80-day-old observe day-file survived the sweep")
}
if _, err := os.Stat(keep); err != nil {
t.Fatal("yesterday's observe day-file was swept")
}
}
// THE ARGS, END TO END. The direct test above hands args to the writer itself,
// so a dispatcher that dropped them survived it. observe:metrics takes a name
// from a fixed table and, with no endpoints file, answers "unconfigured" at
// exit 0 through writeObserve — the one arg-taking op that needs no network.
func TestTheArgsTheHostPassedAreRecordedEndToEnd(t *testing.T) {
isolateConfigDir(t)
defer withNoEndpointsFile(t)()
_, errOut, code := driveObserve(t, "observe:metrics", []string{"up"})
if code != 0 {
t.Fatalf("observe:metrics up failed: %d %s", code, errOut)
}
lines := storeReadObserve(t, "observe:metrics", time.Now())
if len(lines) != 1 {
t.Fatalf("want one recorded result, got %d", len(lines))
}
if strings.Join(lines[0].Args, ",") != "up" {
t.Fatalf("the args the host passed were not recorded: %v", lines[0].Args)
}
}
// ── the review round ────────────────────────────────────────────────────────
// THE ATTACK, verbatim: a host with `observe` sends an op that IGNORES its
// arguments a 5 MiB argument. The op answers normally. Before the fix the
// record got one 5 MiB line and the shelf was sealed for the day — every later
// result for that op refused, the record of what that host saw erased by that
// host. Now: no args recorded (the op reads none), the line is bounded, the
// shelf stays open, the next result lands.
func TestAHostCannotSealTheObserveShelfWithAHugeArgument(t *testing.T) {
isolateConfigDir(t)
storeObserveDroppedReset()
huge := strings.Repeat("x", 5<<20)
_, errOut, code := driveObserve(t, "observe:host", []string{huge})
if code != 0 {
t.Fatalf("observe:host with a huge (ignored) arg failed: %d %s", code, errOut)
}
lines := storeReadObserve(t, "observe:host", time.Now())
if len(lines) != 1 {
t.Fatalf("want one line, got %d", len(lines))
}
if len(lines[0].Args) != 0 {
t.Fatalf("an op that ignores its args had %d recorded", len(lines[0].Args))
}
p, _ := storeDayPath(filepath.Join(storeObserve, "observe-host"), time.Now())
fi, _ := os.Stat(p)
if fi.Size() > storeObserveLineMax {
t.Fatalf("the line is %d bytes, over the %d bound", fi.Size(), storeObserveLineMax)
}
if _, err := os.Stat(p + storeSealSuffix); err == nil {
t.Fatal("one frame sealed the shelf for the day")
}
// The next genuine result still lands.
if _, _, code := driveObserve(t, "observe:host", nil); code != 0 {
t.Fatal(code)
}
if got := len(storeReadObserve(t, "observe:host", time.Now())); got != 2 {
t.Fatalf("the shelf was not open to the next result: %d lines", got)
}
if storeObserveDroppedSnapshot() != nil {
t.Fatalf("something was dropped: %v", storeObserveDroppedSnapshot())
}
}
// An op that READS its args keeps them — bounded, and no more than it reads.
func TestArgsAreKeptOnlyForOpsThatReadThemAndAreBounded(t *testing.T) {
isolateConfigDir(t)
now := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
long := strings.Repeat("a", storeObserveArgMax+50)
if !storeRecordObserve("observe:logs", "h", []string{"kernel", long, "third-arg-nobody-reads"},
[]byte(`{}`), now) {
t.Fatal("not recorded")
}
l := storeReadObserve(t, "observe:logs", now)[0]
if len(l.Args) != 2 {
t.Fatalf("observe:logs reads at most 2 args; %d recorded", len(l.Args))
}
if l.Args[0] != "kernel" || !strings.HasSuffix(l.Args[1], "…(cut)") || len(l.Args[1]) > storeObserveArgMax+len("…(cut)") {
t.Fatalf("args not bounded the way the comment says: %q", l.Args)
}
}
// THE LINE BOUND, at the boundary. No result of any size may produce a line
// over storeObserveLineMax; a result small enough to fit whole is kept whole.
func TestNoResultOfAnySizeProducesALineOverTheBound(t *testing.T) {
isolateConfigDir(t)
now := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
shelf := filepath.Join(storeObserve, "observe-units")
p, _ := storeDayPath(shelf, now)
var last int64
for size := 1; size <= 40000; size += 997 {
res := []byte(`{"u":"` + strings.Repeat("v", size) + `"}`)
if !storeRecordObserve("observe:units", "h", nil, res, now) {
t.Fatalf("size %d refused", size)
}
fi, _ := os.Stat(p)
if fi.Size()-last > storeObserveLineMax {
t.Fatalf("result of %d bytes produced a %d-byte line", len(res), fi.Size()-last)
}
last = fi.Size()
}
lines := storeReadObserve(t, "observe:units", now)
// A result well under the bound is whole; one at the bound is not.
if lines[0].Truncated {
t.Fatal("a tiny result was truncated")
}
big := []byte(`{"u":"` + strings.Repeat("v", storeObserveLineMax) + `"}`)
storeRecordObserve("observe:units", "h", nil, big, now)
lines = storeReadObserve(t, "observe:units", now)
if l := lines[len(lines)-1]; !l.Truncated || l.Bytes != len(big) {
t.Fatalf("a result at the bound must be truncated with its size kept: %+v", l)
}
}
func TestTheOpNameSanitiserHandlesEveryAllowedSeparator(t *testing.T) {
for in, want := range map[string]string{
"observe:net": "observe-net",
"observe:k8s_nodes-x": "observe-k8s-nodes-x",
"observe:K8S-Nodes": "observe-k8s-nodes", // lowercased: the host's [a-z0-9-] rule is a fact
// BOTH ENDS of the uppercase range. A boundary slip ('A'..'Y', or
// 'B'..'Z') silently DELETES a letter rather than producing an
// unreadable name, so the host files the holes under a wrong shelf —
// and no registered op has an uppercase letter, so the registry walk
// can never reach this branch. Only a table row can.
"observe:AZ": "observe-az",
"observe:Za": "observe-za",
"observe::x::": "observe--x", // separators kept, ends trimmed
"observe:../etc": "observe-etc",
":::": "",
} {
if got := storeSafeOpName(in); got != want {
t.Fatalf("%q -> %q, want %q", in, got, want)
}
}
}
// THE ORDERING. The record is on disk BEFORE the reply goes — the property the
// comment in writeObserve calls load-bearing, which had no test: moving the
// record after the reply passed the whole suite. The seam swaps the sender
// for one that checks the day-file exists at the moment the reply is sent.
func TestTheRecordIsOnDiskBeforeTheReplyIsSent(t *testing.T) {
isolateConfigDir(t)
orig := observeSendFrame
var existedAtSend bool
observeSendFrame = func(ws *websocket.Conn, kind byte, b []byte) {
if kind == 'O' {
p, _ := storeDayPath(filepath.Join(storeObserve, "observe-host"), time.Now())
_, err := os.Stat(p)
existedAtSend = err == nil
}
orig(ws, kind, b)
}
defer func() { observeSendFrame = orig }()
if _, errOut, code := driveObserve(t, "observe:host", nil); code != 0 {
t.Fatalf("%d %s", code, errOut)
}
if !existedAtSend {
t.Fatal("the reply was sent before the record was on disk")
}
}
// The recorded-args table is pinned against the handlers themselves.
func TestObserveArgsRecordedMatchesTheHandlers(t *testing.T) {
for op := range observeArgOps {
if observeArgsRecorded[op] == 0 {
t.Fatalf("%s reads a caller value but records no args", op)
}
}
for op := range observeArgsRecorded {
if !observeArgOps[op] {
t.Fatalf("%s records args but the handler ignores them", op)
}
}
// Arities, from the handlers' own checks (observe_ops.go handleObserveNet
// len(args) != 1; observe_pull.go handleObserveMetrics len(args) != 1;
// handleObserveLogs len(args) > 2; observe_reads.go folders, journal and
// cert len(args) > 2, logins len(args) > 1).
if observeArgsRecorded["observe:net"] != 1 || observeArgsRecorded["observe:metrics"] != 1 ||
observeArgsRecorded["observe:logs"] != 2 || observeArgsRecorded["observe:folders"] != 2 ||
observeArgsRecorded["observe:journal"] != 2 || observeArgsRecorded["observe:logins"] != 1 ||
observeArgsRecorded["observe:cert"] != 2 {
t.Fatalf("arities drifted from the handlers: %v", observeArgsRecorded)
}
}
// ── round 2 ─────────────────────────────────────────────────────────────────
// THE WINDOW THE LOOP EXISTS FOR, byte by byte: results under the bound whose
// envelope pushes the line over. The size sweep above steps by 997 and jumps
// clean over it, and the at-the-bound case takes the shortcut path — so the
// loop's own truncation branch was exercised by nothing, and removing
// `line.Result = nil` there passed the suite while dropping 134 sizes.
func TestEveryResultJustUnderTheBoundIsRecordedAndNoLineExceedsIt(t *testing.T) {
isolateConfigDir(t)
storeObserveDroppedReset()
// ONE DAY-FILE PER SIZE. Four hundred max-size lines in one file is over
// the day cap, and the first version of this test read the seal — doing
// its job — as the loop dropping results.
start := time.Date(2026, 1, 1, 10, 0, 0, 0, time.UTC)
var maxLine int64
for i, size := 0, storeObserveLineMax-400; size <= storeObserveLineMax+8; i, size = i+1, size+1 {
day := start.AddDate(0, 0, i)
res := []byte(`{"u":"` + strings.Repeat("v", size-7) + `"}`)
if !storeRecordObserve("observe:units", "h", nil, res, day) {
t.Fatalf("result of %d bytes was DROPPED — the loop bottomed out instead of truncating", len(res))
}
p, _ := storeDayPath(filepath.Join(storeObserve, "observe-units"), day)
fi, err := os.Stat(p)
if err != nil {
t.Fatal(err)
}
if fi.Size() > storeObserveLineMax {
t.Fatalf("result of %d bytes produced a %d-byte line", len(res), fi.Size())
}
if fi.Size() > maxLine {
maxLine = fi.Size()
}
}
if storeObserveDroppedSnapshot() != nil {
t.Fatalf("drops: %v", storeObserveDroppedSnapshot())
}
// THE NEWLINE IS PINNED AT THE BYTE: a max-size line measures the bound
// EXACTLY, newline included. Counting the line without its newline let one
// byte over through; the comment claims exactness, so a test holds it.
if maxLine != storeObserveLineMax {
t.Fatalf("largest line is %d bytes, want exactly %d", maxLine, storeObserveLineMax)
}
}
// The refuse branch: an envelope alone over the bound degrades to a tiny head
// and then refuses — never seals, never writes over the bound. Unreachable in
// production (asked_by is a host name, ≤253 bytes by validHostName), so this is
// the only place the branch runs.
func TestAnImpossibleEnvelopeDegradesThenRefusesWithoutSealing(t *testing.T) {
isolateConfigDir(t)
storeObserveDroppedReset()
now := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
res := []byte(`{"tcp":"open"}`)
if !storeRecordObserve("observe:net", strings.Repeat("h", 32600), []string{"x"}, res, now) {
t.Fatal("an envelope with room for a 4-byte head was refused")
}
if storeRecordObserve("observe:net", strings.Repeat("h", 32700), []string{"x"}, res, now) {
t.Fatal("an envelope over the bound on its own was written")
}
p, _ := storeDayPath(filepath.Join(storeObserve, "observe-net"), now)
if _, err := os.Stat(p + storeSealSuffix); err == nil {
t.Fatal("the refuse branch sealed the day")
}
if storeObserveDroppedSnapshot()["observe-net"] != 1 {
t.Fatalf("the refusal was not counted: %v", storeObserveDroppedSnapshot())
}
}
// THE ARITIES ARE MEASURED, NOT WRITTEN DOWN. For each op that records args,
// its recorded count must equal the largest argument count the handler
// ACCEPTS, driven through the real dispatcher: n args answers, n+1 refuses.
// A handler that grew an argument would make the shelf silently record one
// too few. observe:net needs the network to answer, so only its refusal side
// is driven; its accepted count is pinned by observe_ops_test's own probe.
func TestRecordedArgCountsMatchWhatTheHandlersAccept(t *testing.T) {
isolateConfigDir(t)
defer withNoEndpointsFile(t)()
// Each op's first argument must be one it accepts, or the refusal is about
// the NAME rather than the count and proves nothing about arity.
// And every ACCEPTED position must hold a value the handler accepts —
// observe:logs's second argument is a minutes window, and "x" there is
// refused for its shape, which is not an arity refusal either.
//
// The four in observe_reads.go answer through their drivers on Linux, so
// what they read is put in front of them here and nothing on the box that
// runs the suite is: one unit this machine "has", one TLS listener ss
// "reports", an empty journal, and a folder of the test's own.
cert := observeTestCert(t, "box-a.example", []string{"box-a.example"}, nil, time.Now().Add(24*time.Hour))
port := strconv.Itoa(observeTLSListener(t, cert))
origExec, origHave, origAccess := observeExec, observeHave, observeJournalAccess
observeExec = func(_ context.Context, name string, args ...string) ([]byte, error) {
switch name + " " + args[0] {
case "systemctl list-units":
return []byte("example.service loaded active running Example\n"), nil
case "ss -tlnpH":
return []byte("LISTEN 0 4096 127.0.0.1:" + port + " 0.0.0.0:*\n"), nil
}
return nil, nil
}
observeHave = func(string) bool { return true }
observeJournalAccess = func() string { return "" }
defer func() { observeExec, observeHave, observeJournalAccess = origExec, origHave, origAccess }()
folder := "/"
if observeHostSupported {
folder = t.TempDir()
}
accepted := map[string][]string{
"observe:net": {"203.0.113.9:443"},
"observe:metrics": {"up"},
"observe:logs": {"kernel", "5"},
"observe:folders": {folder, "1"},
"observe:journal": {"example", "5"},
"observe:logins": {"24"},
"observe:cert": {port, "box-a.example"},
}
for op, n := range observeArgsRecorded {
if len(accepted[op]) != n {
t.Fatalf("%s: the fixture has %d accepted args, the table says %d", op, len(accepted[op]), n)
}
tooMany := append(append([]string(nil), accepted[op]...), "x")
if _, _, code := driveObserve(t, op, tooMany); code == 0 {
t.Fatalf("%s accepted %d args but records %d — the table is short", op, n+1, n)
}
if op == "observe:net" {
continue
}
if _, errOut, code := driveObserve(t, op, tooMany[:n]); code != 0 {
t.Fatalf("%s refused %d args (%s) but records %d — the table is long", op, n, errOut, n)
}
}
}
// THE CROSS-REPO INVARIANT, ENFORCED. The host rejects a holes map with any
// shelf name outside ^[a-z0-9][a-z0-9-]{0,63}$ as unreadable — and an
// unreadable beat freezes that machine's stored holes under the host's
// COALESCE. So "every shelf name this box emits matches that rule" is a hard
// dependency, and an example table of six names cannot hold it: registering
// an op whose sanitised name is over 64 characters, or teaching the
// sanitiser a new character, would leave the table green and silently
// blank the host's reading of this box. This walks EVERY registered op.
func TestEveryRegisteredOpSanitisesToTheHostsShelfRule(t *testing.T) {
rule := regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,63}$`)
n := 0
for op := range handlerOps {
if !strings.HasPrefix(op, "observe:") {
continue
}
n++
got := storeSafeOpName(op)
if !rule.MatchString(got) {
t.Fatalf("%s sanitises to %q, which the host rejects as unreadable", op, got)
}
}
if n < 15 {
t.Fatalf("only %d observe ops walked — the registry moved and this test is not covering it", n)
}
}