1867 lines
70 KiB
Go
1867 lines
70 KiB
Go
package main
|
|
|
|
// machinestore_test.go — the machine store's journal.
|
|
//
|
|
// EVERY TEST HERE DRIVES storeRecordPass AND READS THE FILE BACK. None of them
|
|
// asserts a value this test supplied one seam above the line under test — a
|
|
// failure earlier rounds kept producing. The fixtures build a deltaReport the
|
|
// way deltaPass would; what is checked is what landed on disk.
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"github.com/gorilla/websocket"
|
|
"os"
|
|
"path/filepath"
|
|
"regexp"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// storeReport builds a pass that DID compare, over a COMPLETE scan, holding the
|
|
// findings given. Complete matters: a partial scan closes nothing (the platform
|
|
// saw only recognised programs), so a helper that left it false would make
|
|
// every close-path test silently vacuous.
|
|
func storeReport(baselineDigest string, findings ...deltaEvent) deltaReport {
|
|
return deltaReport{
|
|
Baseline: deltaBaselineOK, BaselineDigest: baselineDigest,
|
|
BaselineTakenAt: "2026-09-19T00:00:00Z", Coverage: "all-processes",
|
|
Complete: true, Findings: findings,
|
|
}
|
|
}
|
|
|
|
func storeFinding(fp, level string) deltaEvent {
|
|
return deltaEvent{
|
|
Kind: deltaProcNew, Level: level, Tier: "new", FP: fp,
|
|
Entry: baselineEntry{
|
|
FP: fp, Kind: "process", Exe: "/usr/local/bin/" + fp,
|
|
ExeDigest: "sha256:" + fp, User: "root", ArgvShape: fp + " --serve",
|
|
},
|
|
FirstSeen: "2026-09-19T01:00:00Z",
|
|
}
|
|
}
|
|
|
|
// storeReadJournal returns the journal lines for a day, in file order.
|
|
func storeReadJournal(t *testing.T, day time.Time) []storeLine {
|
|
t.Helper()
|
|
p, err := storeDayPath(storeFindings, day)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
b, err := os.ReadFile(p)
|
|
if err != nil {
|
|
if os.IsNotExist(err) {
|
|
return nil
|
|
}
|
|
t.Fatal(err)
|
|
}
|
|
var out []storeLine
|
|
for _, ln := range strings.Split(strings.TrimSpace(string(b)), "\n") {
|
|
if ln == "" {
|
|
continue
|
|
}
|
|
var l storeLine
|
|
if err := json.Unmarshal([]byte(ln), &l); err != nil {
|
|
t.Fatalf("a journal line is not valid JSON: %v\n%s", err, ln)
|
|
}
|
|
out = append(out, l)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// THE CENTRAL CLAIM OF THE FILE. A finding is pinned until a human resolves it,
|
|
// and the pass runs every 60 seconds — so the difference between a journal and
|
|
// a clock is that the second pass writes nothing. If this ever regresses, one
|
|
// unresolved finding becomes 1,440 lines a day.
|
|
func TestAPinnedFindingIsWrittenOnceNotOncePerPass(t *testing.T) {
|
|
isolateConfigDir(t)
|
|
r := storeReport("base-a", storeFinding("aaa", deltaHigh))
|
|
at := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
|
|
|
|
n, err := storeRecordPass(r, at)
|
|
if err != nil || n != 1 {
|
|
t.Fatalf("the first sighting must write exactly one line: n=%d err=%v", n, err)
|
|
}
|
|
for i := 0; i < 20; i++ {
|
|
n, err := storeRecordPass(r, at.Add(time.Duration(i+1)*time.Minute))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if n != 0 {
|
|
t.Fatalf("pass %d re-wrote a finding nothing about which had changed (%d lines)", i+2, n)
|
|
}
|
|
}
|
|
lines := storeReadJournal(t, at)
|
|
if len(lines) != 1 {
|
|
t.Fatalf("21 passes over one unchanged finding wrote %d lines, want 1", len(lines))
|
|
}
|
|
if lines[0].Transition != "opened" {
|
|
t.Fatalf("transition = %q, want opened", lines[0].Transition)
|
|
}
|
|
if lines[0].Event == nil || lines[0].Event.FP != "aaa" {
|
|
t.Fatalf("the line must carry the whole event: %+v", lines[0].Event)
|
|
}
|
|
if lines[0].Baseline != "base-a" {
|
|
t.Fatalf("baseline = %q — a line that cannot name its accounting is not evidence", lines[0].Baseline)
|
|
}
|
|
}
|
|
|
|
// A notice that becomes a critical is not the same finding, and a journal that
|
|
// records only `opened` would never say it moved.
|
|
func TestAFindingThatChangesLevelIsWrittenAgain(t *testing.T) {
|
|
isolateConfigDir(t)
|
|
at := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
|
|
if _, err := storeRecordPass(storeReport("base-a", storeFinding("aaa", deltaNotice)), at); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
n, err := storeRecordPass(storeReport("base-a", storeFinding("aaa", deltaCritical)), at.Add(time.Minute))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if n != 1 {
|
|
t.Fatalf("a level change wrote %d lines, want 1", n)
|
|
}
|
|
lines := storeReadJournal(t, at)
|
|
if len(lines) != 2 || lines[1].Transition != "changed" {
|
|
t.Fatalf("want opened then changed, got %s", storeTransitions(lines))
|
|
}
|
|
if lines[1].Event.Level != deltaCritical {
|
|
t.Fatalf("the changed line carries level %q, want critical", lines[1].Event.Level)
|
|
}
|
|
}
|
|
|
|
// PIDs churn on every service restart. If they were material, a machine that
|
|
// restarts nginx nightly would write a `changed` line every night about a
|
|
// finding that did not change.
|
|
func TestARestartAloneIsNotAChange(t *testing.T) {
|
|
isolateConfigDir(t)
|
|
at := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
|
|
first := storeFinding("aaa", deltaHigh)
|
|
first.PIDs = []int{101, 102}
|
|
second := storeFinding("aaa", deltaHigh)
|
|
second.PIDs = []int{9001, 9002}
|
|
|
|
if _, err := storeRecordPass(storeReport("base-a", first), at); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
n, err := storeRecordPass(storeReport("base-a", second), at.Add(time.Minute))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if n != 0 {
|
|
t.Fatalf("new pids alone wrote %d lines — pids must not be material", n)
|
|
}
|
|
}
|
|
|
|
func TestAFindingThatStopsBeingReportedIsClosed(t *testing.T) {
|
|
isolateConfigDir(t)
|
|
at := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
|
|
if _, err := storeRecordPass(storeReport("base-a",
|
|
storeFinding("aaa", deltaHigh), storeFinding("bbb", deltaWarn)), at); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := storeRecordPass(storeReport("base-a", storeFinding("aaa", deltaHigh)), at.Add(time.Minute)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
lines := storeReadJournal(t, at)
|
|
if len(lines) != 3 {
|
|
t.Fatalf("want 2 opened + 1 closed, got %s", storeTransitions(lines))
|
|
}
|
|
last := lines[2]
|
|
if last.Transition != "closed" || last.Event.FP != "bbb" {
|
|
t.Fatalf("the third line must close bbb: %+v", last)
|
|
}
|
|
// The host must never read `closed` as `resolved` — oversight.py:39-44 makes
|
|
// the same distinction and this is the half that has to say so out loud.
|
|
if !strings.Contains(last.Note, "not the same as resolved") {
|
|
t.Fatalf("a closed line must say what it does not mean: %q", last.Note)
|
|
}
|
|
// And it must not re-close on the next pass.
|
|
n, err := storeRecordPass(storeReport("base-a", storeFinding("aaa", deltaHigh)), at.Add(2*time.Minute))
|
|
if err != nil || n != 0 {
|
|
t.Fatalf("bbb was closed twice: n=%d err=%v", n, err)
|
|
}
|
|
}
|
|
|
|
// THE DANGEROUS ONE. "I could not look" must never be written as "there is
|
|
// nothing there". A pass with no authority, or no accounting, closes nothing —
|
|
// otherwise deleting the accounting produces a journal showing every finding on
|
|
// the box resolving cleanly, which is the most reassuring possible lie.
|
|
func TestAPassThatCouldNotCompareWritesNothingAndClosesNothing(t *testing.T) {
|
|
isolateConfigDir(t)
|
|
at := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
|
|
if _, err := storeRecordPass(storeReport("base-a", storeFinding("aaa", deltaCritical)), at); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, state := range []string{deltaBaselineNone, deltaBaselineNoAuth, deltaBaselineStale} {
|
|
blind := deltaReport{Baseline: state}
|
|
n, err := storeRecordPass(blind, at.Add(time.Minute))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if n != 0 {
|
|
t.Fatalf("a %q pass wrote %d lines", state, n)
|
|
}
|
|
lines := storeReadJournal(t, at)
|
|
if len(lines) != 1 {
|
|
t.Fatalf("a %q pass changed the journal: %s", state, storeTransitions(lines))
|
|
}
|
|
}
|
|
// And the finding is still open afterwards, so a real pass does not re-open it.
|
|
n, err := storeRecordPass(storeReport("base-a", storeFinding("aaa", deltaCritical)), at.Add(time.Hour))
|
|
if err != nil || n != 0 {
|
|
t.Fatalf("a blind pass lost the open set: n=%d err=%v", n, err)
|
|
}
|
|
}
|
|
|
|
// An `edited` accounting still compares — it is a warning, not a blindness —
|
|
// so its findings must be journalled.
|
|
func TestAnEditedAccountingStillJournals(t *testing.T) {
|
|
isolateConfigDir(t)
|
|
at := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
|
|
r := storeReport("base-a", storeFinding("aaa", deltaHigh))
|
|
r.Baseline = deltaBaselineEdit
|
|
n, err := storeRecordPass(r, at)
|
|
if err != nil || n != 1 {
|
|
t.Fatalf("an edited accounting must still journal: n=%d err=%v", n, err)
|
|
}
|
|
}
|
|
|
|
// A retake redefines normal. Findings measured against the old accounting are
|
|
// not comparable to the new one (deltastate.go:36-38), so the open set resets —
|
|
// and the journal shows the re-open rather than going quiet about it.
|
|
func TestARetakeReopensAgainstTheNewAccounting(t *testing.T) {
|
|
isolateConfigDir(t)
|
|
at := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
|
|
if _, err := storeRecordPass(storeReport("base-a", storeFinding("aaa", deltaHigh)), at); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// THIS TEST PREVIOUSLY PINNED A BUG. It asserted exactly 2 lines, which is
|
|
// what you get when a retake silently drops the old open set — the finding
|
|
// against base-a left the journal with no line at all, and the file's claim
|
|
// that replaying reconstructs the open set was false. The retake must CLOSE
|
|
// what it stops tracking and then re-open what is still there.
|
|
n, err := storeRecordPass(storeReport("base-b", storeFinding("aaa", deltaHigh)), at.Add(time.Minute))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if n != 2 {
|
|
t.Fatalf("a retake wrote %d lines, want 2 (close against the old accounting, open against the new)", n)
|
|
}
|
|
lines := storeReadJournal(t, at)
|
|
if len(lines) != 3 {
|
|
t.Fatalf("want opened closed opened, got %s", storeTransitions(lines))
|
|
}
|
|
if lines[1].Transition != "closed" || lines[1].Event.FP != "aaa" {
|
|
t.Fatalf("the retake must close the finding it made incomparable: %+v", lines[1])
|
|
}
|
|
if !strings.Contains(lines[1].Note, "retaken") {
|
|
t.Fatalf("the closing line must say why: %q", lines[1].Note)
|
|
}
|
|
if lines[2].Transition != "opened" || lines[2].Baseline != "base-b" {
|
|
t.Fatalf("the re-opened line must stand against base-b: %+v", lines[2])
|
|
}
|
|
}
|
|
|
|
// A finding that opens while the day-file is SEALED must not be recorded as
|
|
// open — otherwise the journal never mentions it again, on any pass or any
|
|
// later day, and its eventual `closed` refers to a fingerprint nothing opened.
|
|
// This was measured in review as a permanent loss.
|
|
func TestAFindingRefusedByTheCapIsNotRecordedAsOpen(t *testing.T) {
|
|
isolateConfigDir(t)
|
|
day1 := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
|
|
storeFillToCap(t, day1)
|
|
|
|
// Day 1: the finding is refused, so nothing is written for it.
|
|
if _, err := storeRecordPass(storeReport("base-a", storeFinding("crit", deltaCritical)), day1); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, l := range storeReadJournal(t, day1) {
|
|
if l.Event != nil && l.Event.FP == "crit" {
|
|
t.Fatal("the fixture did not actually seal the day")
|
|
}
|
|
}
|
|
// Day 2: a fresh, unsealed file. The finding must appear as OPENED here.
|
|
day2 := day1.Add(24 * time.Hour)
|
|
n, err := storeRecordPass(storeReport("base-a", storeFinding("crit", deltaCritical)), day2)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if n == 0 {
|
|
t.Fatal("a finding refused by a sealed day was lost for good — it never re-opened")
|
|
}
|
|
lines := storeReadJournal(t, day2)
|
|
found := false
|
|
for _, l := range lines {
|
|
if l.Transition == "opened" && l.Event != nil && l.Event.FP == "crit" {
|
|
found = true
|
|
}
|
|
}
|
|
if !found {
|
|
t.Fatalf("day 2 must open the finding the sealed day refused, got %s", storeTransitions(lines))
|
|
}
|
|
}
|
|
|
|
// The notice is written ONCE and the day is then sealed. Measured in review at
|
|
// four notices in one file with ordinary lines between them, while the notice
|
|
// itself said the file "stops here".
|
|
func TestACappedDaySealsAndNothingFollowsTheNotice(t *testing.T) {
|
|
isolateConfigDir(t)
|
|
at := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
|
|
storeFillToCap(t, at)
|
|
|
|
// Many further passes, each offering new findings of varying size.
|
|
for i := 0; i < 6; i++ {
|
|
if _, err := storeRecordPass(storeReport("base-a",
|
|
storeFinding(fmt.Sprintf("f%d", i), deltaHigh)), at.Add(time.Duration(i)*time.Minute)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
lines := storeReadJournal(t, at)
|
|
notices := 0
|
|
for i, l := range lines {
|
|
if l.Transition == storeCapped {
|
|
notices++
|
|
if i != len(lines)-1 {
|
|
t.Fatalf("a line follows the capped notice at %d of %d: %s",
|
|
i, len(lines), storeTransitions(lines[i:]))
|
|
}
|
|
}
|
|
}
|
|
if notices != 1 {
|
|
t.Fatalf("want exactly one capped notice, got %d", notices)
|
|
}
|
|
p, err := storeDayPath(storeFindings, at)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := os.Stat(p + storeSealSuffix); err != nil {
|
|
t.Fatal("the capped day was not sealed")
|
|
}
|
|
fi, err := os.Stat(p)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if fi.Size() > storeDayCap {
|
|
t.Fatalf("the file crossed its ceiling: %d > %d", fi.Size(), storeDayCap)
|
|
}
|
|
}
|
|
|
|
// A PARTIAL SCAN CLOSES NOTHING. On a platform with no unfiltered process walk
|
|
// only recognised programs were compared, so an entry missing from the pass may
|
|
// never have been looked at — closing it writes "this went away" from a pass
|
|
// that did not check.
|
|
func TestAPartialScanClosesNothing(t *testing.T) {
|
|
isolateConfigDir(t)
|
|
at := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
|
|
full := storeReport("base-a", storeFinding("aaa", deltaHigh), storeFinding("bbb", deltaHigh))
|
|
full.Complete = true
|
|
if _, err := storeRecordPass(full, at); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
partial := storeReport("base-a", storeFinding("aaa", deltaHigh))
|
|
partial.Complete = false
|
|
n, err := storeRecordPass(partial, at.Add(time.Minute))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if n != 0 {
|
|
t.Fatalf("a partial scan closed %d finding(s)", n)
|
|
}
|
|
// A COMPLETE pass that still omits bbb does close it.
|
|
again := storeReport("base-a", storeFinding("aaa", deltaHigh))
|
|
again.Complete = true
|
|
if _, err := storeRecordPass(again, at.Add(2*time.Minute)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
lines := storeReadJournal(t, at)
|
|
if lines[len(lines)-1].Transition != "closed" {
|
|
t.Fatalf("a complete pass must still close: %s", storeTransitions(lines))
|
|
}
|
|
}
|
|
|
|
// A CONSISTENTLY FILTERED PLATFORM STILL CLOSES. baseline_all_other.go returns
|
|
// false unconditionally on darwin and windows, so Complete is permanently false
|
|
// there — gating closes on it alone meant no finding ever closed on a Mac or a
|
|
// Windows box and the open set grew for the life of the machine. Those
|
|
// platforms are not blind, they are filtered on BOTH sides: the accounting was
|
|
// taken through the same recognised-only walk, so the scopes agree and an
|
|
// absence really is an absence.
|
|
func TestAConsistentlyFilteredPlatformStillCloses(t *testing.T) {
|
|
isolateConfigDir(t)
|
|
at := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
|
|
filtered := func(findings ...deltaEvent) deltaReport {
|
|
r := storeReport("base-a", findings...)
|
|
r.Complete = false // the non-Linux walk
|
|
r.Coverage = "processes=recognised-only,schedulers=recognised-only" // and so was the accounting
|
|
return r
|
|
}
|
|
if _, err := storeRecordPass(filtered(
|
|
storeFinding("aaa", deltaHigh), storeFinding("bbb", deltaHigh)), at); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := storeRecordPass(filtered(storeFinding("aaa", deltaHigh)), at.Add(time.Minute)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
lines := storeReadJournal(t, at)
|
|
if lines[len(lines)-1].Transition != "closed" {
|
|
t.Fatalf("a filtered platform never closed anything: %s", storeTransitions(lines))
|
|
}
|
|
|
|
// And the genuine MISMATCH still closes nothing: an all-processes accounting
|
|
// compared against a partial walk may simply not have looked.
|
|
isolateConfigDir(t)
|
|
mismatch := func(findings ...deltaEvent) deltaReport {
|
|
r := storeReport("base-a", findings...)
|
|
r.Complete = false
|
|
r.Coverage = "processes=all,schedulers=all"
|
|
return r
|
|
}
|
|
if _, err := storeRecordPass(mismatch(
|
|
storeFinding("aaa", deltaHigh), storeFinding("bbb", deltaHigh)), at); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
n, err := storeRecordPass(mismatch(storeFinding("aaa", deltaHigh)), at.Add(time.Minute))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if n != 0 {
|
|
t.Fatalf("a narrower pass than the accounting closed %d finding(s)", n)
|
|
}
|
|
}
|
|
|
|
// Each day-file is self-contained: a finding still open at a rollover is named
|
|
// in ONE snapshot line, so a replay whose window starts inside this file still
|
|
// knows what was open.
|
|
func TestANewDayFileOpensWithOneSnapshotLine(t *testing.T) {
|
|
isolateConfigDir(t)
|
|
day1 := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
|
|
r := storeReport("base-a", storeFinding("aaa", deltaHigh), storeFinding("bbb", deltaHigh))
|
|
if _, err := storeRecordPass(r, day1); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
day2 := day1.Add(24 * time.Hour)
|
|
if _, err := storeRecordPass(r, day2); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
lines := storeReadJournal(t, day2)
|
|
if len(lines) != 1 || lines[0].Transition != storeDayOpen {
|
|
t.Fatalf("day 2 must open with ONE snapshot line, got %s", storeTransitions(lines))
|
|
}
|
|
if lines[0].OpenCount != 2 {
|
|
t.Fatalf("snapshot says %d open, want 2", lines[0].OpenCount)
|
|
}
|
|
if strings.Join(lines[0].OpenFPs, ",") != "aaa,bbb" {
|
|
t.Fatalf("snapshot names %v", lines[0].OpenFPs)
|
|
}
|
|
// Not repeated within the day.
|
|
if _, err := storeRecordPass(r, day2.Add(time.Minute)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got := len(storeReadJournal(t, day2)); got != 1 {
|
|
t.Fatalf("the snapshot repeated within one day: %d lines", got)
|
|
}
|
|
}
|
|
|
|
// THE LOCKUP THIS REPLACED. The first version wrote one line PER open finding,
|
|
// ahead of the day's real work — measured in review as a permanent fixed point
|
|
// where the shelf wrote nothing but its own memory and a new critical was never
|
|
// journalled at all. A large open set must still leave room for the day's
|
|
// findings.
|
|
func TestALargeOpenSetDoesNotStarveTheDay(t *testing.T) {
|
|
isolateConfigDir(t)
|
|
day1 := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
|
|
var many []deltaEvent
|
|
for i := 0; i < 4000; i++ {
|
|
many = append(many, storeFinding(fmt.Sprintf("f%04d", i), deltaWarn))
|
|
}
|
|
if _, err := storeRecordPass(storeReport("base-a", many...), day1); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
day2 := day1.Add(24 * time.Hour)
|
|
// The same 4,000 still open, PLUS a new critical.
|
|
withNew := append(append([]deltaEvent(nil), many...), storeFinding("crit", deltaCritical))
|
|
if _, err := storeRecordPass(storeReport("base-a", withNew...), day2); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
lines := storeReadJournal(t, day2)
|
|
var snapshots, opened int
|
|
sawCritical := false
|
|
for _, l := range lines {
|
|
switch l.Transition {
|
|
case storeDayOpen:
|
|
snapshots++
|
|
case "opened":
|
|
opened++
|
|
if l.Event != nil && l.Event.FP == "crit" {
|
|
sawCritical = true
|
|
}
|
|
}
|
|
}
|
|
if snapshots != 1 {
|
|
t.Fatalf("want exactly one snapshot line, got %d", snapshots)
|
|
}
|
|
if !sawCritical {
|
|
t.Fatalf("a new critical was starved by the open set: %d lines, %d opened",
|
|
len(lines), opened)
|
|
}
|
|
}
|
|
|
|
// The snapshot is BOUNDED, and says so when it cuts the list — otherwise a
|
|
// truncated list reads as the whole open set.
|
|
func TestTheSnapshotIsBoundedAndSaysSoWhenItCuts(t *testing.T) {
|
|
isolateConfigDir(t)
|
|
day1 := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
|
|
var many []deltaEvent
|
|
for i := 0; i < storeSnapshotMaxFPs+50; i++ {
|
|
many = append(many, storeFinding(fmt.Sprintf("f%04d", i), deltaWarn))
|
|
}
|
|
if _, err := storeRecordPass(storeReport("base-a", many...), day1); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
day2 := day1.Add(24 * time.Hour)
|
|
if _, err := storeRecordPass(storeReport("base-a", many...), day2); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
snap := storeReadJournal(t, day2)[0]
|
|
if len(snap.OpenFPs) != storeSnapshotMaxFPs {
|
|
t.Fatalf("listed %d fingerprints, want the bound %d", len(snap.OpenFPs), storeSnapshotMaxFPs)
|
|
}
|
|
if snap.OpenCount != storeSnapshotMaxFPs+50 {
|
|
t.Fatalf("count is %d — the count must be the WHOLE set, not the listed part",
|
|
snap.OpenCount)
|
|
}
|
|
if !strings.Contains(snap.Note, "counted, not listed") {
|
|
t.Fatalf("a cut list must say it was cut: %q", snap.Note)
|
|
}
|
|
}
|
|
|
|
// A box that was off for a week still opens its next day-file with a snapshot.
|
|
// Nothing covered the multi-day gap, so a mutation narrowing the test to
|
|
// "yesterday only" survived.
|
|
func TestAMultiDayGapStillWritesTheSnapshot(t *testing.T) {
|
|
isolateConfigDir(t)
|
|
day1 := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
|
|
r := storeReport("base-a", storeFinding("aaa", deltaHigh))
|
|
if _, err := storeRecordPass(r, day1); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
later := day1.Add(7 * 24 * time.Hour)
|
|
if _, err := storeRecordPass(r, later); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
lines := storeReadJournal(t, later)
|
|
if len(lines) != 1 || lines[0].Transition != storeDayOpen {
|
|
t.Fatalf("a box back after a week must still open with a snapshot, got %s",
|
|
storeTransitions(lines))
|
|
}
|
|
}
|
|
|
|
// A REFUSED CLOSE LEAVES THE FINDING OPEN. The close did not happen, so
|
|
// forgetting it loses the close for ever — the retake path did exactly that and
|
|
// dropped a finding's only record permanently.
|
|
func TestARefusedCloseKeepsTheFindingOpen(t *testing.T) {
|
|
isolateConfigDir(t)
|
|
day1 := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
|
|
if _, err := storeRecordPass(storeReport("base-a",
|
|
storeFinding("aaa", deltaHigh), storeFinding("bbb", deltaHigh)), day1); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
storeSealDay(t, day1)
|
|
// bbb goes away while the day is sealed: the close is refused.
|
|
if _, err := storeRecordPass(storeReport("base-a", storeFinding("aaa", deltaHigh)), day1); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// The next day must carry the close, not have forgotten it.
|
|
day2 := day1.Add(24 * time.Hour)
|
|
if _, err := storeRecordPass(storeReport("base-a", storeFinding("aaa", deltaHigh)), day2); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var closed bool
|
|
for _, l := range storeReadJournal(t, day2) {
|
|
if l.Transition == "closed" && l.Event != nil && l.Event.FP == "bbb" {
|
|
closed = true
|
|
}
|
|
}
|
|
if !closed {
|
|
t.Fatalf("a close refused by a sealed day was lost: %s",
|
|
storeTransitions(storeReadJournal(t, day2)))
|
|
}
|
|
}
|
|
|
|
// A RETAKE ON A SEALED DAY MUST RETRY. If the remembered baseline advances
|
|
// while its closes were refused, the retake branch is never re-entered and
|
|
// those findings' only line anywhere is their original `opened`.
|
|
func TestARetakeRefusedByASealedDayIsRetriedTheNextDay(t *testing.T) {
|
|
isolateConfigDir(t)
|
|
day1 := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
|
|
if _, err := storeRecordPass(storeReport("base-a",
|
|
storeFinding("aaa", deltaHigh), storeFinding("bbb", deltaHigh)), day1); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
storeSealDay(t, day1)
|
|
// The retake lands on a sealed day: every close is refused.
|
|
if _, err := storeRecordPass(storeReport("base-b", storeFinding("aaa", deltaHigh)), day1); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
day2 := day1.Add(24 * time.Hour)
|
|
if _, err := storeRecordPass(storeReport("base-b", storeFinding("aaa", deltaHigh)), day2); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
seen := map[string]bool{}
|
|
for _, l := range storeReadJournal(t, day2) {
|
|
if l.Transition == "closed" && l.Event != nil {
|
|
seen[l.Event.FP] = true
|
|
}
|
|
}
|
|
if !seen["aaa"] || !seen["bbb"] {
|
|
t.Fatalf("the retake's closes were not retried after the seal: %s",
|
|
storeTransitions(storeReadJournal(t, day2)))
|
|
}
|
|
}
|
|
|
|
// storeSealDay fills a day-file to its ceiling so every later line is refused.
|
|
func storeSealDay(t *testing.T, day time.Time) {
|
|
t.Helper()
|
|
p, err := storeDayPath(storeFindings, day)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
cur, err := os.ReadFile(p)
|
|
if err != nil && !os.IsNotExist(err) {
|
|
t.Fatal(err)
|
|
}
|
|
filler := append(bytesOfLine(t), '\n')
|
|
buf := append([]byte(nil), cur...)
|
|
for len(buf)+len(filler) < storeDayCap-storeNoticeReserve {
|
|
buf = append(buf, filler...)
|
|
}
|
|
if err := os.WriteFile(p, buf, 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := storeWriteLine(p+storeSealSuffix, nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
// A `changed` that displaces a DIFFERENT baseline entry is a real change, even
|
|
// when every other material field is equal.
|
|
func TestDisplacingADifferentEntryIsAChange(t *testing.T) {
|
|
isolateConfigDir(t)
|
|
at := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
|
|
mk := func(wasFP string) deltaEvent {
|
|
ev := storeFinding("aaa", deltaHigh)
|
|
ev.Kind, ev.Reason = deltaProcChanged, deltaReasonExeDigest
|
|
ev.Was = &baselineEntry{FP: wasFP, Kind: "process"}
|
|
return ev
|
|
}
|
|
if _, err := storeRecordPass(storeReport("base-a", mk("old-x")), at); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
n, err := storeRecordPass(storeReport("base-a", mk("old-y")), at.Add(time.Minute))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if n != 1 {
|
|
t.Fatalf("displacing a different entry wrote %d lines, want 1", n)
|
|
}
|
|
}
|
|
|
|
// storeFillToCap seals a day-file by filling it to its append ceiling.
|
|
func storeFillToCap(t *testing.T, day time.Time) {
|
|
t.Helper()
|
|
p, err := storeDayPath(storeFindings, day)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
filler := append(bytesOfLine(t), '\n')
|
|
var buf []byte
|
|
for len(buf)+len(filler) < storeDayCap-storeNoticeReserve {
|
|
buf = append(buf, filler...)
|
|
}
|
|
if err := os.WriteFile(p, buf, 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// Trip the cap so the notice is written and the seal laid down.
|
|
if _, err := storeRecordPass(storeReport("base-a", storeFinding("sealer", deltaWarn)), day); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := os.Stat(p + storeSealSuffix); err != nil {
|
|
t.Fatal("storeFillToCap did not seal the day")
|
|
}
|
|
}
|
|
|
|
// The day-file rolls on UTC, not local time — a host reading a fleet across
|
|
// timezones cannot know each box's offset.
|
|
func TestTheJournalRollsOnUTCMidnight(t *testing.T) {
|
|
isolateConfigDir(t)
|
|
// THE TWO INSTANTS MUST DISAGREE ABOUT WHICH DAY IT IS, or this test cannot
|
|
// fail. 08:00 on the 20th in a zone ten hours ahead is 22:00 on the 19th in
|
|
// UTC: local time names a second day-file, UTC names one. The first version
|
|
// of this test used 23:30 in the same zone, which is 13:30 UTC — still the
|
|
// 19th either way, so it passed with the UTC conversion deleted.
|
|
ahead := time.FixedZone("ahead", 10*3600)
|
|
first := time.Date(2026, 9, 19, 13, 0, 0, 0, time.UTC)
|
|
second := time.Date(2026, 9, 20, 8, 0, 0, 0, ahead)
|
|
if second.Format("2006-01-02") == second.UTC().Format("2006-01-02") {
|
|
t.Fatal("the fixture's two clocks agree on the day, so this test cannot fail")
|
|
}
|
|
if _, err := storeRecordPass(storeReport("base-a", storeFinding("aaa", deltaHigh)), first); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := storeRecordPass(storeReport("base-a",
|
|
storeFinding("aaa", deltaHigh), storeFinding("bbb", deltaHigh)), second); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got := len(storeReadJournal(t, first)); got != 2 {
|
|
t.Fatalf("both lines belong to the same UTC day: got %d", got)
|
|
}
|
|
}
|
|
|
|
// The cap writes its notice ONCE and then goes silent — and the notice is in
|
|
// band, so a host replaying the day sees the truncation instead of inferring it
|
|
// from a gap.
|
|
func TestACappedDayFileSaysSoOnceAndThenStops(t *testing.T) {
|
|
isolateConfigDir(t)
|
|
at := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
|
|
p, err := storeDayPath(storeFindings, at)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// Fill to just under the append ceiling with well-formed lines, so the NEXT
|
|
// line is the one that trips the cap. The append ceiling is the cap less
|
|
// the reserve the notice is written out of.
|
|
filler := append(bytesOfLine(t), '\n')
|
|
var buf []byte
|
|
for len(buf)+len(filler) < storeDayCap-storeNoticeReserve {
|
|
buf = append(buf, filler...)
|
|
}
|
|
if err := os.WriteFile(p, buf, 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := storeRecordPass(storeReport("base-a", storeFinding("aaa", deltaHigh)), at); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
lines := storeReadJournal(t, at)
|
|
last := lines[len(lines)-1]
|
|
if last.Transition != storeCapped {
|
|
t.Fatalf("the capping line must be the notice, got %q", last.Transition)
|
|
}
|
|
if !strings.Contains(last.Note, "ceiling") {
|
|
t.Fatalf("the notice must say what happened: %q", last.Note)
|
|
}
|
|
before, err := os.Stat(p)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// A further pass with a NEW finding must add nothing at all.
|
|
if _, err := storeRecordPass(storeReport("base-a",
|
|
storeFinding("aaa", deltaHigh), storeFinding("ccc", deltaHigh)), at.Add(time.Minute)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
after, err := os.Stat(p)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if after.Size() != before.Size() {
|
|
t.Fatalf("a capped file grew by %d bytes", after.Size()-before.Size())
|
|
}
|
|
if after.Size() > storeDayCap {
|
|
t.Fatalf("the file crossed its own ceiling: %d > %d", after.Size(), storeDayCap)
|
|
}
|
|
}
|
|
|
|
func bytesOfLine(t *testing.T) []byte {
|
|
t.Helper()
|
|
// Computed, not a literal. TestNoFixtureHardcodesAMomentInTime bans a
|
|
// literal `At:` because stepup age-checks that field name; this At is a
|
|
// journal write-stamp and is never age-checked, but the filler's timestamp
|
|
// means nothing to this test either way, so the fixture moves rather than
|
|
// the guard. Widening a scan to admit one benign case is how the next real
|
|
// one gets through.
|
|
b, err := json.Marshal(storeLine{
|
|
At: stamp(time.Now()), Transition: "opened", Machine: "box",
|
|
Version: "test", Baseline: "base-a", Event: &deltaEvent{FP: "filler"},
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return b
|
|
}
|
|
|
|
// Retention reads the NAME, not the mtime. A restored-from-backup file has a
|
|
// fresh mtime and is still a month-old record; a sweep keyed on mtime would
|
|
// keep exactly the wrong ones and delete the rest.
|
|
func TestTheSweepDeletesByTheDayInTheNameNotTheMtime(t *testing.T) {
|
|
isolateConfigDir(t)
|
|
now := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
|
|
shelf, err := storeShelf(storeFindings)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
old := filepath.Join(shelf, "2026-07-01.jsonl") // 80 days back
|
|
oldSeal := old + storeSealSuffix // its seal marker
|
|
keep := filepath.Join(shelf, "2026-09-18.jsonl") // yesterday
|
|
keepSeal := keep + storeSealSuffix
|
|
odd := filepath.Join(shelf, "notes-for-me.jsonl") // not ours
|
|
// A DATE-SHAPED NAME WITH NO SUFFIX THIS STORE WRITES. Trimming the
|
|
// extension optionally meant this parsed as a day and was deleted; nothing
|
|
// else writes into this 0700 dir today, but a sweep whose blast radius is
|
|
// wider than its comment says is a sweep waiting to surprise somebody.
|
|
bare := filepath.Join(shelf, "2026-07-01")
|
|
for _, p := range []string{old, oldSeal, keep, keepSeal, odd, bare} {
|
|
if err := os.WriteFile(p, []byte("{}\n"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// Every file gets a mtime of RIGHT NOW, so an mtime-based sweep would
|
|
// delete none of them and this test would fail.
|
|
if err := os.Chtimes(p, now, now); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
removed, err := storeSweep(now)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// The day-file AND its seal marker: a sweep blind to the marker leaves a
|
|
// permanent .capped file behind for every day that ever hit its ceiling.
|
|
if removed != 2 {
|
|
t.Fatalf("swept %d files, want 2 (the day-file and its seal)", removed)
|
|
}
|
|
if _, err := os.Stat(oldSeal); !os.IsNotExist(err) {
|
|
t.Fatal("the old day's seal marker survived the sweep")
|
|
}
|
|
if _, err := os.Stat(old); !os.IsNotExist(err) {
|
|
t.Fatal("the 80-day-old day-file survived the sweep")
|
|
}
|
|
for _, p := range []string{keep, keepSeal, odd, bare} {
|
|
if _, err := os.Stat(p); err != nil {
|
|
t.Fatalf("%s was removed and should not have been", filepath.Base(p))
|
|
}
|
|
}
|
|
}
|
|
|
|
// The store root lives INSIDE the state dir, so a read that escapes it lands on
|
|
// the bastion's own state rather than beside the pairings and the TOTP secret.
|
|
func TestTheStoreRootIsNestedInsideTheStateDir(t *testing.T) {
|
|
cd := isolateConfigDir(t)
|
|
root, err := storeDir()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if filepath.Dir(root) != cd {
|
|
t.Fatalf("store root %q is not directly inside the state dir %q", root, cd)
|
|
}
|
|
fi, err := os.Stat(root)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if fi.Mode().Perm() != 0o700 {
|
|
t.Fatalf("store root is %v, want 0700", fi.Mode().Perm())
|
|
}
|
|
}
|
|
|
|
func storeTransitions(lines []storeLine) string {
|
|
var out []string
|
|
for _, l := range lines {
|
|
out = append(out, l.Transition)
|
|
}
|
|
return "[" + strings.Join(out, " ") + "]"
|
|
}
|
|
|
|
// ── the alerts shelf ────────────────────────────────────────────────────────
|
|
|
|
func storeReadAlerts(t *testing.T, day time.Time) []storeAlertLine {
|
|
t.Helper()
|
|
p, err := storeDayPath(storeAlerts, day)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
b, err := os.ReadFile(p)
|
|
if err != nil {
|
|
if os.IsNotExist(err) {
|
|
return nil
|
|
}
|
|
t.Fatal(err)
|
|
}
|
|
var out []storeAlertLine
|
|
for _, ln := range strings.Split(strings.TrimSpace(string(b)), "\n") {
|
|
if ln == "" {
|
|
continue
|
|
}
|
|
var l storeAlertLine
|
|
if err := json.Unmarshal([]byte(ln), &l); err != nil {
|
|
t.Fatalf("alert line is not valid JSON: %v\n%s", err, ln)
|
|
}
|
|
out = append(out, l)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// A refresh runs every few minutes and republishes the same firing set. Like
|
|
// the findings journal, only the edges are written.
|
|
func TestAnAlertIsJournalledOnceUntilItClears(t *testing.T) {
|
|
isolateConfigDir(t)
|
|
at := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
|
|
firing := &alertsSummary{Firing: 2, Critical: 1, Warning: 1,
|
|
Fingerprints: []string{"fp-a", "fp-b"}}
|
|
|
|
n, err := storeRecordAlerts(firing, at)
|
|
if err != nil || n != 2 {
|
|
t.Fatalf("two firing alerts must write two lines: n=%d err=%v", n, err)
|
|
}
|
|
for i := 0; i < 5; i++ {
|
|
if n, err := storeRecordAlerts(firing, at.Add(time.Duration(i+1)*time.Minute)); err != nil || n != 0 {
|
|
t.Fatalf("an unchanged alert set re-wrote %d lines (err=%v)", n, err)
|
|
}
|
|
}
|
|
// fp-b clears.
|
|
cleared := &alertsSummary{Firing: 1, Critical: 1, Fingerprints: []string{"fp-a"}}
|
|
if n, err := storeRecordAlerts(cleared, at.Add(time.Hour)); err != nil || n != 1 {
|
|
t.Fatalf("a cleared alert must write one line: n=%d err=%v", n, err)
|
|
}
|
|
lines := storeReadAlerts(t, at)
|
|
if len(lines) != 3 {
|
|
t.Fatalf("want firing firing cleared, got %d lines", len(lines))
|
|
}
|
|
if lines[2].Transition != "cleared" || lines[2].FP != "fp-b" {
|
|
t.Fatalf("the third line must clear fp-b: %+v", lines[2])
|
|
}
|
|
// EVERY count, not just firing. A reader joining the journal mid-day takes
|
|
// its totals from the line; asserting one of the three left the other two
|
|
// untested, which a mutation dropping them proved by surviving.
|
|
if lines[0].Firing != 2 || lines[0].Critical != 1 || lines[0].Warning != 1 {
|
|
t.Fatalf("the opening line must carry all three counts as of itself: %+v", lines[0])
|
|
}
|
|
if lines[2].Firing != 1 || lines[2].Critical != 1 || lines[2].Warning != 0 {
|
|
t.Fatalf("the clearing line must carry the counts as of itself: %+v", lines[2])
|
|
}
|
|
}
|
|
|
|
// NIL IS NOT EMPTY. A box with no Alertmanager configured, or one that has not
|
|
// refreshed yet, is not watching anything — it must not be read as watching
|
|
// nothing, and it must not clear alerts it can no longer see.
|
|
func TestAnUnconfiguredAlertSourceClearsNothing(t *testing.T) {
|
|
isolateConfigDir(t)
|
|
at := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
|
|
if _, err := storeRecordAlerts(&alertsSummary{Firing: 1, Critical: 1,
|
|
Fingerprints: []string{"fp-a"}}, at); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
n, err := storeRecordAlerts(nil, at.Add(time.Minute))
|
|
if err != nil || n != 0 {
|
|
t.Fatalf("a nil summary wrote %d lines (err=%v)", n, err)
|
|
}
|
|
if got := len(storeReadAlerts(t, at)); got != 1 {
|
|
t.Fatalf("a nil summary changed the journal: %d lines", got)
|
|
}
|
|
// And fp-a is still open, so a real summary does not re-open it.
|
|
if n, err := storeRecordAlerts(&alertsSummary{Firing: 1, Critical: 1,
|
|
Fingerprints: []string{"fp-a"}}, at.Add(time.Hour)); err != nil || n != 0 {
|
|
t.Fatalf("the nil summary lost the open set: n=%d err=%v", n, err)
|
|
}
|
|
}
|
|
|
|
// The summary names at most twenty fingerprints. Past that the set is a sample,
|
|
// so the journal says so rather than letting a truncated list read as the whole
|
|
// one — otherwise alert 21 clearing is indistinguishable from it never firing.
|
|
func TestATruncatedAlertSummarySaysSoInBand(t *testing.T) {
|
|
isolateConfigDir(t)
|
|
at := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
|
|
if _, err := storeRecordAlerts(&alertsSummary{Firing: 57, Critical: 3,
|
|
Fingerprints: []string{"fp-a", "fp-b"}}, at); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
lines := storeReadAlerts(t, at)
|
|
last := lines[len(lines)-1]
|
|
if last.Transition != "truncated" {
|
|
t.Fatalf("a truncated summary must end in a truncated line, got %q", last.Transition)
|
|
}
|
|
if !strings.Contains(last.Note, "57") || !strings.Contains(last.Note, "2") {
|
|
t.Fatalf("the note must name both numbers: %q", last.Note)
|
|
}
|
|
}
|
|
|
|
// ── the sweep's cadence ─────────────────────────────────────────────────────
|
|
|
|
// Keyed on the DATE, not an interval. An interval measures uptime, so a laptop
|
|
// awake two hours a day would sweep every twelve days and its store would grow
|
|
// past the retention it advertises.
|
|
func TestTheSweepRunsOncePerDayAndNotOncePerUptimeInterval(t *testing.T) {
|
|
isolateConfigDir(t)
|
|
shelf, err := storeShelf(storeFindings)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
write := func(name string) string {
|
|
p := filepath.Join(shelf, name)
|
|
if err := os.WriteFile(p, []byte("{}\n"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return p
|
|
}
|
|
old := write("2026-07-01.jsonl")
|
|
day := time.Date(2026, 9, 19, 0, 30, 0, 0, time.UTC)
|
|
|
|
storeSweepIfDue(day)
|
|
if _, err := os.Stat(old); !os.IsNotExist(err) {
|
|
t.Fatal("the first call of the day did not sweep")
|
|
}
|
|
// A second old file appearing later the SAME day is not swept again.
|
|
old2 := write("2026-07-02.jsonl")
|
|
storeSweepIfDue(day.Add(23 * time.Hour))
|
|
if _, err := os.Stat(old2); err != nil {
|
|
t.Fatal("the sweep ran twice in one UTC day")
|
|
}
|
|
// The next UTC day sweeps again.
|
|
storeSweepIfDue(day.Add(25 * time.Hour))
|
|
if _, err := os.Stat(old2); !os.IsNotExist(err) {
|
|
t.Fatal("the sweep did not run on the next day")
|
|
}
|
|
}
|
|
|
|
// The truncation notice must not become a clock. Measured in review: written
|
|
// unconditionally, five refreshes of an unchanged 57-alert set produced
|
|
// 21, 1, 1, 1, 1 lines — and refreshAlerts runs every 60s PER PAIRING.
|
|
func TestTheTruncationNoticeIsNotRepeatedOnEveryRefresh(t *testing.T) {
|
|
isolateConfigDir(t)
|
|
at := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
|
|
big := &alertsSummary{Firing: 57, Critical: 3, Fingerprints: []string{"fp-a", "fp-b"}}
|
|
|
|
first, err := storeRecordAlerts(big, at)
|
|
if err != nil || first == 0 {
|
|
t.Fatalf("the first refresh must write: n=%d err=%v", first, err)
|
|
}
|
|
for i := 0; i < 5; i++ {
|
|
n, err := storeRecordAlerts(big, at.Add(time.Duration(i+1)*time.Minute))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if n != 0 {
|
|
t.Fatalf("refresh %d re-wrote %d line(s) for an unchanged alert set", i+2, n)
|
|
}
|
|
}
|
|
notices := 0
|
|
for _, l := range storeReadAlerts(t, at) {
|
|
if l.Transition == "truncated" {
|
|
notices++
|
|
}
|
|
}
|
|
if notices != 1 {
|
|
t.Fatalf("want one truncation notice across six refreshes, got %d", notices)
|
|
}
|
|
// A CHANGE in the truncation is still reported — otherwise the count going
|
|
// from 57 to 400 would be invisible.
|
|
n, err := storeRecordAlerts(&alertsSummary{Firing: 400, Critical: 9,
|
|
Fingerprints: []string{"fp-a", "fp-b"}}, at.Add(time.Hour))
|
|
if err != nil || n != 1 {
|
|
t.Fatalf("a changed truncation must be reported: n=%d err=%v", n, err)
|
|
}
|
|
// AND THE LISTED HALF COUNTS TOO. Keying the dedup on `firing` alone misses
|
|
// 400/2 -> 400/1: the same number of alerts firing, but the journal now
|
|
// names fewer of them, so its blind spot GREW and the reader must be told.
|
|
if _, err := storeRecordAlerts(&alertsSummary{Firing: 400, Critical: 9,
|
|
Fingerprints: []string{"fp-a"}}, at.Add(2*time.Hour)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var sawShrink bool
|
|
for _, l := range storeReadAlerts(t, at) {
|
|
if l.Transition == "truncated" && strings.Contains(l.Note, "names 1 of them") {
|
|
sawShrink = true
|
|
}
|
|
}
|
|
if !sawShrink {
|
|
t.Fatal("the listed count shrank from 2 to 1 and no notice said so")
|
|
}
|
|
}
|
|
|
|
// A single line larger than the ceiling must not create a file already past it.
|
|
// os.Stat returns ErrNotExist for a missing file, and an earlier version
|
|
// skipped the whole cap check in that branch.
|
|
func TestAnOversizedFirstLineCannotCreateAnAlreadyOverflowingFile(t *testing.T) {
|
|
isolateConfigDir(t)
|
|
at := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
|
|
huge := storeFinding("huge", deltaCritical)
|
|
// One evidence line past the ceiling on its own.
|
|
huge.Evidence = []evidence{{Signal: "fixture", Tier: tierVerified,
|
|
Value: strings.Repeat("x", storeDayCap+1)}}
|
|
|
|
p, err := storeDayPath(storeFindings, at)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := os.Stat(p); !os.IsNotExist(err) {
|
|
t.Fatal("the day-file must not exist yet, or this test proves nothing")
|
|
}
|
|
if _, err := storeRecordPass(storeReport("base-a", huge), at); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
fi, err := os.Stat(p)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if fi.Size() > storeDayCap {
|
|
t.Fatalf("an oversized first line crossed the ceiling: %d > %d", fi.Size(), storeDayCap)
|
|
}
|
|
lines := storeReadJournal(t, at)
|
|
if len(lines) != 1 || lines[0].Transition != storeCapped {
|
|
t.Fatalf("want a lone capped notice, got %s", storeTransitions(lines))
|
|
}
|
|
}
|
|
|
|
// refreshAlerts had no in-flight guard while refreshDelta did, and run.go
|
|
// starts ONE refresher PER PAIRING — so a box serving several hosts runs
|
|
// several in phase. Without the guard a slow goroutine holding an older summary
|
|
// can take storeMu after a newer one and journal an alert as `firing` that the
|
|
// newer pass just journalled as `cleared`: the order of the record inverts.
|
|
//
|
|
// Driven through the observeHTTPGet seam the suite already uses, so this
|
|
// measures the guard rather than asserting it.
|
|
func TestOnlyOneAlertsRefreshRunsAtATime(t *testing.T) {
|
|
isolateConfigDir(t)
|
|
defer withEndpointsFile(t, `{"alertmanager":"http://127.0.0.1:9093"}`)()
|
|
|
|
release := make(chan struct{})
|
|
var inFlight, calls int32
|
|
origGet := observeHTTPGet
|
|
observeHTTPGet = func(_ context.Context, _ string) ([]byte, int, error) {
|
|
atomic.AddInt32(&calls, 1)
|
|
if atomic.AddInt32(&inFlight, 1) > 1 {
|
|
t.Error("two alerts refreshes were in flight at once")
|
|
}
|
|
<-release
|
|
atomic.AddInt32(&inFlight, -1)
|
|
return []byte(`{"status":"success","data":[]}`), 200, nil
|
|
}
|
|
defer func() { observeHTTPGet = origGet }()
|
|
|
|
var wg sync.WaitGroup
|
|
wg.Add(1)
|
|
go func() { defer wg.Done(); refreshAlerts(context.Background()) }()
|
|
|
|
// Wait until the first refresh is genuinely inside the fetch, or this races
|
|
// to a pass rather than measuring anything.
|
|
deadline := time.Now().Add(5 * time.Second)
|
|
for atomic.LoadInt32(&calls) == 0 {
|
|
if time.Now().After(deadline) {
|
|
close(release)
|
|
wg.Wait()
|
|
t.Fatal("the first refresh never reached the fetch seam")
|
|
}
|
|
time.Sleep(time.Millisecond)
|
|
}
|
|
|
|
// The second must return AT ONCE rather than queueing behind the first.
|
|
done := make(chan struct{})
|
|
go func() { refreshAlerts(context.Background()); close(done) }()
|
|
select {
|
|
case <-done:
|
|
case <-time.After(2 * time.Second):
|
|
close(release)
|
|
wg.Wait()
|
|
t.Fatal("a concurrent refresh blocked instead of being refused by the in-flight guard")
|
|
}
|
|
if got := atomic.LoadInt32(&calls); got != 1 {
|
|
t.Fatalf("the guarded refresh still fetched: %d calls, want 1", got)
|
|
}
|
|
close(release)
|
|
wg.Wait()
|
|
}
|
|
|
|
// The beat reports where the machine keeps its own record, and reporting it
|
|
// must not CREATE the directory: the beat fires every sixty seconds, and a
|
|
// path is a claim about where the record is, not an instruction to make one.
|
|
func TestTheBeatReportsTheStoreRootWithoutCreatingIt(t *testing.T) {
|
|
isolateConfigDir(t)
|
|
raw := buildHeartbeatBody("", "h", "linux", "test", "pk", "fp", "100.64.0.9", "ep", false, nil, nil, nil)
|
|
var body map[string]any
|
|
if err := json.Unmarshal([]byte(raw), &body); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
got, _ := body["machine_record_root"].(string)
|
|
want, err := storeRootPath()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got != want {
|
|
t.Fatalf("beat reports %q, want %q", got, want)
|
|
}
|
|
if !filepath.IsAbs(got) {
|
|
t.Fatalf("the reported root is not absolute: %q", got)
|
|
}
|
|
if _, err := os.Stat(got); !os.IsNotExist(err) {
|
|
t.Fatalf("building a beat created the store directory %q", got)
|
|
}
|
|
}
|
|
|
|
// ── round 2 verification: the last three ────────────────────────────────────
|
|
|
|
// A retake that coincides with a rollover must produce a day-file that replays
|
|
// on its own: the snapshot runs BEFORE the retake-close. Measured before the
|
|
// fix: `[closed aaa, closed bbb, opened aaa]` and no day_open line — two closes
|
|
// for fingerprints the file never introduced.
|
|
func TestARetakeOnARolloverDayStillOpensWithTheSnapshot(t *testing.T) {
|
|
isolateConfigDir(t)
|
|
day1 := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
|
|
if _, err := storeRecordPass(storeReport("base-a",
|
|
storeFinding("aaa", deltaHigh), storeFinding("bbb", deltaHigh)), day1); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
day2 := day1.Add(24 * time.Hour)
|
|
if _, err := storeRecordPass(storeReport("base-b", storeFinding("aaa", deltaHigh)), day2); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
lines := storeReadJournal(t, day2)
|
|
got := storeTransitions(lines)
|
|
if got != "[day_open closed closed opened]" {
|
|
t.Fatalf("day 2 must replay alone: want [day_open closed closed opened], got %s", got)
|
|
}
|
|
if lines[0].OpenCount != 2 || strings.Join(lines[0].OpenFPs, ",") != "aaa,bbb" {
|
|
t.Fatalf("the snapshot must name what the closes are about: %+v", lines[0])
|
|
}
|
|
}
|
|
|
|
// A refused `changed` stays open under its OLD digest, so the next unsealed
|
|
// day writes `changed` — not a second `opened` for a finding already announced.
|
|
func TestARefusedChangeIsWrittenAsAChangeNotASecondOpen(t *testing.T) {
|
|
isolateConfigDir(t)
|
|
day1 := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
|
|
if _, err := storeRecordPass(storeReport("base-a", storeFinding("aaa", deltaNotice)), day1); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
storeSealDay(t, day1)
|
|
// Escalates while the day is sealed: the `changed` line is refused.
|
|
if _, err := storeRecordPass(storeReport("base-a", storeFinding("aaa", deltaCritical)), day1); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
day2 := day1.Add(24 * time.Hour)
|
|
if _, err := storeRecordPass(storeReport("base-a", storeFinding("aaa", deltaCritical)), day2); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var seen []string
|
|
for _, l := range storeReadJournal(t, day2) {
|
|
if l.Event != nil && l.Event.FP == "aaa" {
|
|
seen = append(seen, l.Transition)
|
|
}
|
|
}
|
|
if strings.Join(seen, " ") != "changed" {
|
|
t.Fatalf("the escalation must land as one `changed` on day 2, got %v", seen)
|
|
}
|
|
}
|
|
|
|
// The alerts truncation state must not advance on a REFUSED line — a sealed
|
|
// alerts day recorded the notice as delivered, and the next day never wrote
|
|
// it. The fix was in round 1; nothing exercised the sealed-day path.
|
|
func TestATruncationNoticeRefusedByASealedDayIsWrittenTheNextDay(t *testing.T) {
|
|
isolateConfigDir(t)
|
|
day1 := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
|
|
// Seal the ALERTS shelf for day 1.
|
|
p, err := storeDayPath(storeAlerts, day1)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
filler := append(bytesOfLine(t), '\n')
|
|
var buf []byte
|
|
for len(buf)+len(filler) < storeDayCap-storeNoticeReserve {
|
|
buf = append(buf, filler...)
|
|
}
|
|
if err := os.WriteFile(p, buf, 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := storeWriteLine(p+storeSealSuffix, nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
big := &alertsSummary{Firing: 57, Critical: 3, Fingerprints: []string{"fp-a"}}
|
|
if _, err := storeRecordAlerts(big, day1); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
day2 := day1.Add(24 * time.Hour)
|
|
if _, err := storeRecordAlerts(big, day2); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var notice bool
|
|
for _, l := range storeReadAlerts(t, day2) {
|
|
if l.Transition == "truncated" {
|
|
notice = true
|
|
}
|
|
}
|
|
if !notice {
|
|
t.Fatal("the truncation notice refused on the sealed day was never written")
|
|
}
|
|
}
|
|
|
|
// A v1 state file is MIGRATED, not discarded: a finding open under v1 that is
|
|
// gone after the upgrade gets its `closed` line.
|
|
func TestAV1StateFileIsCarriedAcrossTheVersionBump(t *testing.T) {
|
|
isolateConfigDir(t)
|
|
sp, err := storeStatePath()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.MkdirAll(filepath.Dir(sp), 0o700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
v1 := `{"version":"privatae-store-state-v1","baseline":"base-a","open":{"aaa":"x","zzz":"y"}}`
|
|
if err := os.WriteFile(sp, []byte(v1), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
at := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
|
|
// Only aaa is still present; zzz went away during the upgrade window.
|
|
if _, err := storeRecordPass(storeReport("base-a", storeFinding("aaa", deltaHigh)), at); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var closedZ bool
|
|
for _, l := range storeReadJournal(t, at) {
|
|
if l.Transition == "closed" && l.Event != nil && l.Event.FP == "zzz" {
|
|
closedZ = true
|
|
}
|
|
if l.Transition == "opened" && l.Event != nil && l.Event.FP == "aaa" {
|
|
t.Fatal("a finding open under v1 was re-opened after the upgrade")
|
|
}
|
|
}
|
|
if !closedZ {
|
|
t.Fatal("a finding open under v1 that went away got no closed line after the upgrade")
|
|
}
|
|
}
|
|
|
|
// ── the observe shelf ───────────────────────────────────────────────────────
|
|
|
|
func storeReadObserve(t *testing.T, op string, day time.Time) []storeObserveLine {
|
|
t.Helper()
|
|
p, err := storeDayPath(filepath.Join(storeObserve, storeSafeOpName(op)), day)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
b, err := os.ReadFile(p)
|
|
if err != nil {
|
|
if os.IsNotExist(err) {
|
|
return nil
|
|
}
|
|
t.Fatal(err)
|
|
}
|
|
var out []storeObserveLine
|
|
for _, ln := range strings.Split(strings.TrimSpace(string(b)), "\n") {
|
|
if ln == "" {
|
|
continue
|
|
}
|
|
var l storeObserveLine
|
|
if err := json.Unmarshal([]byte(ln), &l); err != nil {
|
|
t.Fatalf("observe line is not valid JSON: %v\n%s", err, ln)
|
|
}
|
|
out = append(out, l)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// END TO END THROUGH THE REAL PATH: the management dispatcher, this
|
|
// connection's account, writeObserve, the store. Not the writer called by
|
|
// hand — the round-1 review removed a writer nothing called, and the proof
|
|
// that this one is called is the dispatcher calling it.
|
|
func TestAnObserveResultIsRecordedOnTheMachine(t *testing.T) {
|
|
isolateConfigDir(t)
|
|
storeObserveDroppedReset()
|
|
out, errOut, code := driveObserve(t, "observe:host", nil)
|
|
if code != 0 {
|
|
t.Fatalf("observe:host failed: %d %s", code, errOut)
|
|
}
|
|
lines := storeReadObserve(t, "observe:host", time.Now())
|
|
if len(lines) != 1 {
|
|
t.Fatalf("want one recorded result, got %d", len(lines))
|
|
}
|
|
l := lines[0]
|
|
if l.Op != "observe:host" || l.Truncated || len(l.Result) == 0 {
|
|
t.Fatalf("recorded line is not the result: %+v", l)
|
|
}
|
|
// The SAME bytes the host received — a record of a different answer than
|
|
// the one sent is not a record.
|
|
var sent, kept any
|
|
if err := json.Unmarshal([]byte(strings.TrimSpace(out)), &sent); err != nil {
|
|
t.Fatalf("the reply is not JSON: %v", err)
|
|
}
|
|
if err := json.Unmarshal(l.Result, &kept); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if fmt.Sprint(sent) != fmt.Sprint(kept) {
|
|
t.Fatal("the recorded result differs from the reply the host received")
|
|
}
|
|
if storeObserveDroppedSnapshot() != nil {
|
|
t.Fatal("a successful record counted as dropped")
|
|
}
|
|
}
|
|
|
|
func TestTheRecordNamesWhoAskedAndWhat(t *testing.T) {
|
|
isolateConfigDir(t)
|
|
now := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
|
|
if !storeRecordObserve("observe:net", "member.host.privatae.ai", []string{"10.0.0.5:443"},
|
|
[]byte(`{"tcp":"open"}`), now) {
|
|
t.Fatal("a small result was not recorded")
|
|
}
|
|
l := storeReadObserve(t, "observe:net", now)[0]
|
|
if l.AskedBy != "member.host.privatae.ai" || strings.Join(l.Args, ",") != "10.0.0.5:443" {
|
|
t.Fatalf("who/what missing: %+v", l)
|
|
}
|
|
if string(l.Result) != `{"tcp":"open"}` || l.Bytes != len(`{"tcp":"open"}`) {
|
|
t.Fatalf("result not kept whole: %+v", l)
|
|
}
|
|
}
|
|
|
|
// A result past the line bound keeps its first 4 KiB and says how much was
|
|
// cut — the shape of what was seen, never nothing, and never a day-file eaten
|
|
// by one chatty op.
|
|
func TestAnOversizedResultIsKeptAsAHeadWithItsSize(t *testing.T) {
|
|
isolateConfigDir(t)
|
|
now := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
|
|
big := []byte(`{"units":"` + strings.Repeat("u", storeObserveLineMax+5000) + `"}`)
|
|
if !storeRecordObserve("observe:units", "h", nil, big, now) {
|
|
t.Fatal("an oversized result was refused outright")
|
|
}
|
|
l := storeReadObserve(t, "observe:units", now)[0]
|
|
if !l.Truncated || len(l.Result) != 0 || len(l.Head) != storeObserveHeadKeep || l.Bytes != len(big) {
|
|
t.Fatalf("truncation shape wrong: truncated=%v result=%d head=%d bytes=%d",
|
|
l.Truncated, len(l.Result), len(l.Head), l.Bytes)
|
|
}
|
|
p, _ := storeDayPath(filepath.Join(storeObserve, "observe-units"), now)
|
|
fi, _ := os.Stat(p)
|
|
if fi.Size() > int64(storeObserveLineMax) {
|
|
t.Fatalf("the line on disk is %d bytes, over the %d bound", fi.Size(), storeObserveLineMax)
|
|
}
|
|
}
|
|
|
|
// Only observe results are recorded here, and an op name cannot choose where
|
|
// on disk its record lands.
|
|
func TestOnlyObserveOpsAreRecordedAndTheNameCannotEscape(t *testing.T) {
|
|
isolateConfigDir(t)
|
|
now := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
|
|
for _, op := range []string{"ctx_get", "dcn_list", "", "uptime"} {
|
|
if storeRecordObserve(op, "h", nil, []byte(`{}`), now) {
|
|
t.Fatalf("%q was recorded on the observe shelf", op)
|
|
}
|
|
}
|
|
if !storeRecordObserve("observe:../../etc", "h", nil, []byte(`{}`), now) {
|
|
t.Fatal("a hostile-looking observe name was refused rather than sanitised")
|
|
}
|
|
root, _ := storeDir()
|
|
p := filepath.Join(root, storeObserve, "observe-etc", "2026-09-19.jsonl")
|
|
if _, err := os.Stat(p); err != nil {
|
|
t.Fatalf("the sanitised record is not where it must be: %v", err)
|
|
}
|
|
}
|
|
|
|
// A result the record could not keep is COUNTED, and the count rides the beat
|
|
// — a record with holes must never read as a quiet machine.
|
|
func TestADroppedResultIsCountedOnTheBeatByShelf(t *testing.T) {
|
|
cd := isolateConfigDir(t)
|
|
storeObserveDroppedReset()
|
|
// A regular FILE where the store root must be a directory: every write fails.
|
|
if err := os.MkdirAll(cd, 0o700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(filepath.Join(cd, storeDirName), []byte("in the way"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if storeRecordObserve("observe:host", "h", nil, []byte(`{}`), time.Now()) {
|
|
t.Fatal("a write into a file-as-directory succeeded")
|
|
}
|
|
storeRecordObserve("observe:sockets", "h", nil, []byte(`{}`), time.Now())
|
|
storeRecordObserve("observe:sockets", "h", nil, []byte(`{}`), time.Now())
|
|
snap := storeObserveDroppedSnapshot()
|
|
if snap["observe-host"] != 1 || snap["observe-sockets"] != 2 {
|
|
t.Fatalf("dropped counts by shelf are wrong: %v", snap)
|
|
}
|
|
raw := buildHeartbeatBody("", "h", "linux", "test", "pk", "fp", "100.64.0.9", "ep", false, nil, nil, nil)
|
|
var body map[string]any
|
|
if err := json.Unmarshal([]byte(raw), &body); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
d, _ := body["machine_record_dropped"].(map[string]any)
|
|
if d["observe-host"] != 1.0 || d["observe-sockets"] != 2.0 {
|
|
t.Fatalf("the beat does not name the shelves with holes: %v", body["machine_record_dropped"])
|
|
}
|
|
// And an EMPTY OBJECT — never absent — when nothing was dropped and the
|
|
// pairing may observe. Absent has to mean exactly one thing to the host:
|
|
// this pairing is not entitled to say. (The first version omitted the key
|
|
// at zero, and the host could not tell "whole" from "no longer allowed".)
|
|
// A FRESH map: Unmarshal into a populated map merges keys.
|
|
storeObserveDroppedReset()
|
|
raw = buildHeartbeatBody("", "h", "linux", "test", "pk", "fp", "100.64.0.9", "ep", false, nil, nil, nil)
|
|
body = map[string]any{}
|
|
_ = json.Unmarshal([]byte(raw), &body)
|
|
d2, present := body["machine_record_dropped"].(map[string]any)
|
|
if !present || len(d2) != 0 {
|
|
t.Fatalf("a whole record must travel as an empty object, got %v (present=%v)", body["machine_record_dropped"], present)
|
|
}
|
|
}
|
|
|
|
// The sweep reaches the nested observe shelves.
|
|
func TestTheSweepReachesTheObserveShelves(t *testing.T) {
|
|
isolateConfigDir(t)
|
|
now := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
|
|
dir, err := storeShelf(filepath.Join(storeObserve, "observe-net"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
old := filepath.Join(dir, "2026-07-01.jsonl")
|
|
keep := filepath.Join(dir, "2026-09-18.jsonl")
|
|
for _, p := range []string{old, keep} {
|
|
if err := os.WriteFile(p, []byte("{}\n"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if _, err := storeSweep(now); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := os.Stat(old); !os.IsNotExist(err) {
|
|
t.Fatal("an 80-day-old observe day-file survived the sweep")
|
|
}
|
|
if _, err := os.Stat(keep); err != nil {
|
|
t.Fatal("yesterday's observe day-file was swept")
|
|
}
|
|
}
|
|
|
|
// THE ARGS, END TO END. The direct test above hands args to the writer itself,
|
|
// so a dispatcher that dropped them survived it. observe:metrics takes a name
|
|
// from a fixed table and, with no endpoints file, answers "unconfigured" at
|
|
// exit 0 through writeObserve — the one arg-taking op that needs no network.
|
|
func TestTheArgsTheHostPassedAreRecordedEndToEnd(t *testing.T) {
|
|
isolateConfigDir(t)
|
|
defer withNoEndpointsFile(t)()
|
|
_, errOut, code := driveObserve(t, "observe:metrics", []string{"up"})
|
|
if code != 0 {
|
|
t.Fatalf("observe:metrics up failed: %d %s", code, errOut)
|
|
}
|
|
lines := storeReadObserve(t, "observe:metrics", time.Now())
|
|
if len(lines) != 1 {
|
|
t.Fatalf("want one recorded result, got %d", len(lines))
|
|
}
|
|
if strings.Join(lines[0].Args, ",") != "up" {
|
|
t.Fatalf("the args the host passed were not recorded: %v", lines[0].Args)
|
|
}
|
|
}
|
|
|
|
// ── the review round ────────────────────────────────────────────────────────
|
|
|
|
// THE ATTACK, verbatim: a host with `observe` sends an op that IGNORES its
|
|
// arguments a 5 MiB argument. The op answers normally. Before the fix the
|
|
// record got one 5 MiB line and the shelf was sealed for the day — every later
|
|
// result for that op refused, the record of what that host saw erased by that
|
|
// host. Now: no args recorded (the op reads none), the line is bounded, the
|
|
// shelf stays open, the next result lands.
|
|
func TestAHostCannotSealTheObserveShelfWithAHugeArgument(t *testing.T) {
|
|
isolateConfigDir(t)
|
|
storeObserveDroppedReset()
|
|
huge := strings.Repeat("x", 5<<20)
|
|
_, errOut, code := driveObserve(t, "observe:host", []string{huge})
|
|
if code != 0 {
|
|
t.Fatalf("observe:host with a huge (ignored) arg failed: %d %s", code, errOut)
|
|
}
|
|
lines := storeReadObserve(t, "observe:host", time.Now())
|
|
if len(lines) != 1 {
|
|
t.Fatalf("want one line, got %d", len(lines))
|
|
}
|
|
if len(lines[0].Args) != 0 {
|
|
t.Fatalf("an op that ignores its args had %d recorded", len(lines[0].Args))
|
|
}
|
|
p, _ := storeDayPath(filepath.Join(storeObserve, "observe-host"), time.Now())
|
|
fi, _ := os.Stat(p)
|
|
if fi.Size() > storeObserveLineMax {
|
|
t.Fatalf("the line is %d bytes, over the %d bound", fi.Size(), storeObserveLineMax)
|
|
}
|
|
if _, err := os.Stat(p + storeSealSuffix); err == nil {
|
|
t.Fatal("one frame sealed the shelf for the day")
|
|
}
|
|
// The next genuine result still lands.
|
|
if _, _, code := driveObserve(t, "observe:host", nil); code != 0 {
|
|
t.Fatal(code)
|
|
}
|
|
if got := len(storeReadObserve(t, "observe:host", time.Now())); got != 2 {
|
|
t.Fatalf("the shelf was not open to the next result: %d lines", got)
|
|
}
|
|
if storeObserveDroppedSnapshot() != nil {
|
|
t.Fatalf("something was dropped: %v", storeObserveDroppedSnapshot())
|
|
}
|
|
}
|
|
|
|
// An op that READS its args keeps them — bounded, and no more than it reads.
|
|
func TestArgsAreKeptOnlyForOpsThatReadThemAndAreBounded(t *testing.T) {
|
|
isolateConfigDir(t)
|
|
now := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
|
|
long := strings.Repeat("a", storeObserveArgMax+50)
|
|
if !storeRecordObserve("observe:logs", "h", []string{"kernel", long, "third-arg-nobody-reads"},
|
|
[]byte(`{}`), now) {
|
|
t.Fatal("not recorded")
|
|
}
|
|
l := storeReadObserve(t, "observe:logs", now)[0]
|
|
if len(l.Args) != 2 {
|
|
t.Fatalf("observe:logs reads at most 2 args; %d recorded", len(l.Args))
|
|
}
|
|
if l.Args[0] != "kernel" || !strings.HasSuffix(l.Args[1], "…(cut)") || len(l.Args[1]) > storeObserveArgMax+len("…(cut)") {
|
|
t.Fatalf("args not bounded the way the comment says: %q", l.Args)
|
|
}
|
|
}
|
|
|
|
// THE LINE BOUND, at the boundary. No result of any size may produce a line
|
|
// over storeObserveLineMax; a result small enough to fit whole is kept whole.
|
|
func TestNoResultOfAnySizeProducesALineOverTheBound(t *testing.T) {
|
|
isolateConfigDir(t)
|
|
now := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
|
|
shelf := filepath.Join(storeObserve, "observe-units")
|
|
p, _ := storeDayPath(shelf, now)
|
|
var last int64
|
|
for size := 1; size <= 40000; size += 997 {
|
|
res := []byte(`{"u":"` + strings.Repeat("v", size) + `"}`)
|
|
if !storeRecordObserve("observe:units", "h", nil, res, now) {
|
|
t.Fatalf("size %d refused", size)
|
|
}
|
|
fi, _ := os.Stat(p)
|
|
if fi.Size()-last > storeObserveLineMax {
|
|
t.Fatalf("result of %d bytes produced a %d-byte line", len(res), fi.Size()-last)
|
|
}
|
|
last = fi.Size()
|
|
}
|
|
lines := storeReadObserve(t, "observe:units", now)
|
|
// A result well under the bound is whole; one at the bound is not.
|
|
if lines[0].Truncated {
|
|
t.Fatal("a tiny result was truncated")
|
|
}
|
|
big := []byte(`{"u":"` + strings.Repeat("v", storeObserveLineMax) + `"}`)
|
|
storeRecordObserve("observe:units", "h", nil, big, now)
|
|
lines = storeReadObserve(t, "observe:units", now)
|
|
if l := lines[len(lines)-1]; !l.Truncated || l.Bytes != len(big) {
|
|
t.Fatalf("a result at the bound must be truncated with its size kept: %+v", l)
|
|
}
|
|
}
|
|
|
|
func TestTheOpNameSanitiserHandlesEveryAllowedSeparator(t *testing.T) {
|
|
for in, want := range map[string]string{
|
|
"observe:net": "observe-net",
|
|
"observe:k8s_nodes-x": "observe-k8s-nodes-x",
|
|
"observe:K8S-Nodes": "observe-k8s-nodes", // lowercased: the host's [a-z0-9-] rule is a fact
|
|
// BOTH ENDS of the uppercase range. A boundary slip ('A'..'Y', or
|
|
// 'B'..'Z') silently DELETES a letter rather than producing an
|
|
// unreadable name, so the host files the holes under a wrong shelf —
|
|
// and no registered op has an uppercase letter, so the registry walk
|
|
// can never reach this branch. Only a table row can.
|
|
"observe:AZ": "observe-az",
|
|
"observe:Za": "observe-za",
|
|
"observe::x::": "observe--x", // separators kept, ends trimmed
|
|
"observe:../etc": "observe-etc",
|
|
":::": "",
|
|
} {
|
|
if got := storeSafeOpName(in); got != want {
|
|
t.Fatalf("%q -> %q, want %q", in, got, want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// THE ORDERING. The record is on disk BEFORE the reply goes — the property the
|
|
// comment in writeObserve calls load-bearing, which had no test: moving the
|
|
// record after the reply passed the whole suite. The seam swaps the sender
|
|
// for one that checks the day-file exists at the moment the reply is sent.
|
|
func TestTheRecordIsOnDiskBeforeTheReplyIsSent(t *testing.T) {
|
|
isolateConfigDir(t)
|
|
orig := observeSendFrame
|
|
var existedAtSend bool
|
|
observeSendFrame = func(ws *websocket.Conn, kind byte, b []byte) {
|
|
if kind == 'O' {
|
|
p, _ := storeDayPath(filepath.Join(storeObserve, "observe-host"), time.Now())
|
|
_, err := os.Stat(p)
|
|
existedAtSend = err == nil
|
|
}
|
|
orig(ws, kind, b)
|
|
}
|
|
defer func() { observeSendFrame = orig }()
|
|
if _, errOut, code := driveObserve(t, "observe:host", nil); code != 0 {
|
|
t.Fatalf("%d %s", code, errOut)
|
|
}
|
|
if !existedAtSend {
|
|
t.Fatal("the reply was sent before the record was on disk")
|
|
}
|
|
}
|
|
|
|
// The recorded-args table is pinned against the handlers themselves.
|
|
func TestObserveArgsRecordedMatchesTheHandlers(t *testing.T) {
|
|
for op := range observeArgOps {
|
|
if observeArgsRecorded[op] == 0 {
|
|
t.Fatalf("%s reads a caller value but records no args", op)
|
|
}
|
|
}
|
|
for op := range observeArgsRecorded {
|
|
if !observeArgOps[op] {
|
|
t.Fatalf("%s records args but the handler ignores them", op)
|
|
}
|
|
}
|
|
// Arities, from the handlers' own checks (observe_ops.go handleObserveNet
|
|
// len(args) != 1; observe_pull.go handleObserveMetrics len(args) != 1;
|
|
// handleObserveLogs len(args) > 2; observe_reads.go folders, journal and
|
|
// cert len(args) > 2, logins len(args) > 1).
|
|
if observeArgsRecorded["observe:net"] != 1 || observeArgsRecorded["observe:metrics"] != 1 ||
|
|
observeArgsRecorded["observe:logs"] != 2 || observeArgsRecorded["observe:folders"] != 2 ||
|
|
observeArgsRecorded["observe:journal"] != 2 || observeArgsRecorded["observe:logins"] != 1 ||
|
|
observeArgsRecorded["observe:cert"] != 2 {
|
|
t.Fatalf("arities drifted from the handlers: %v", observeArgsRecorded)
|
|
}
|
|
}
|
|
|
|
// ── round 2 ─────────────────────────────────────────────────────────────────
|
|
|
|
// THE WINDOW THE LOOP EXISTS FOR, byte by byte: results under the bound whose
|
|
// envelope pushes the line over. The size sweep above steps by 997 and jumps
|
|
// clean over it, and the at-the-bound case takes the shortcut path — so the
|
|
// loop's own truncation branch was exercised by nothing, and removing
|
|
// `line.Result = nil` there passed the suite while dropping 134 sizes.
|
|
func TestEveryResultJustUnderTheBoundIsRecordedAndNoLineExceedsIt(t *testing.T) {
|
|
isolateConfigDir(t)
|
|
storeObserveDroppedReset()
|
|
// ONE DAY-FILE PER SIZE. Four hundred max-size lines in one file is over
|
|
// the day cap, and the first version of this test read the seal — doing
|
|
// its job — as the loop dropping results.
|
|
start := time.Date(2026, 1, 1, 10, 0, 0, 0, time.UTC)
|
|
var maxLine int64
|
|
for i, size := 0, storeObserveLineMax-400; size <= storeObserveLineMax+8; i, size = i+1, size+1 {
|
|
day := start.AddDate(0, 0, i)
|
|
res := []byte(`{"u":"` + strings.Repeat("v", size-7) + `"}`)
|
|
if !storeRecordObserve("observe:units", "h", nil, res, day) {
|
|
t.Fatalf("result of %d bytes was DROPPED — the loop bottomed out instead of truncating", len(res))
|
|
}
|
|
p, _ := storeDayPath(filepath.Join(storeObserve, "observe-units"), day)
|
|
fi, err := os.Stat(p)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if fi.Size() > storeObserveLineMax {
|
|
t.Fatalf("result of %d bytes produced a %d-byte line", len(res), fi.Size())
|
|
}
|
|
if fi.Size() > maxLine {
|
|
maxLine = fi.Size()
|
|
}
|
|
}
|
|
if storeObserveDroppedSnapshot() != nil {
|
|
t.Fatalf("drops: %v", storeObserveDroppedSnapshot())
|
|
}
|
|
// THE NEWLINE IS PINNED AT THE BYTE: a max-size line measures the bound
|
|
// EXACTLY, newline included. Counting the line without its newline let one
|
|
// byte over through; the comment claims exactness, so a test holds it.
|
|
if maxLine != storeObserveLineMax {
|
|
t.Fatalf("largest line is %d bytes, want exactly %d", maxLine, storeObserveLineMax)
|
|
}
|
|
}
|
|
|
|
// The refuse branch: an envelope alone over the bound degrades to a tiny head
|
|
// and then refuses — never seals, never writes over the bound. Unreachable in
|
|
// production (asked_by is a host name, ≤253 bytes by validHostName), so this is
|
|
// the only place the branch runs.
|
|
func TestAnImpossibleEnvelopeDegradesThenRefusesWithoutSealing(t *testing.T) {
|
|
isolateConfigDir(t)
|
|
storeObserveDroppedReset()
|
|
now := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
|
|
res := []byte(`{"tcp":"open"}`)
|
|
if !storeRecordObserve("observe:net", strings.Repeat("h", 32600), []string{"x"}, res, now) {
|
|
t.Fatal("an envelope with room for a 4-byte head was refused")
|
|
}
|
|
if storeRecordObserve("observe:net", strings.Repeat("h", 32700), []string{"x"}, res, now) {
|
|
t.Fatal("an envelope over the bound on its own was written")
|
|
}
|
|
p, _ := storeDayPath(filepath.Join(storeObserve, "observe-net"), now)
|
|
if _, err := os.Stat(p + storeSealSuffix); err == nil {
|
|
t.Fatal("the refuse branch sealed the day")
|
|
}
|
|
if storeObserveDroppedSnapshot()["observe-net"] != 1 {
|
|
t.Fatalf("the refusal was not counted: %v", storeObserveDroppedSnapshot())
|
|
}
|
|
}
|
|
|
|
// THE ARITIES ARE MEASURED, NOT WRITTEN DOWN. For each op that records args,
|
|
// its recorded count must equal the largest argument count the handler
|
|
// ACCEPTS, driven through the real dispatcher: n args answers, n+1 refuses.
|
|
// A handler that grew an argument would make the shelf silently record one
|
|
// too few. observe:net needs the network to answer, so only its refusal side
|
|
// is driven; its accepted count is pinned by observe_ops_test's own probe.
|
|
func TestRecordedArgCountsMatchWhatTheHandlersAccept(t *testing.T) {
|
|
isolateConfigDir(t)
|
|
defer withNoEndpointsFile(t)()
|
|
// Each op's first argument must be one it accepts, or the refusal is about
|
|
// the NAME rather than the count and proves nothing about arity.
|
|
// And every ACCEPTED position must hold a value the handler accepts —
|
|
// observe:logs's second argument is a minutes window, and "x" there is
|
|
// refused for its shape, which is not an arity refusal either.
|
|
//
|
|
// The four in observe_reads.go answer through their drivers on Linux, so
|
|
// what they read is put in front of them here and nothing on the box that
|
|
// runs the suite is: one unit this machine "has", one TLS listener ss
|
|
// "reports", an empty journal, and a folder of the test's own.
|
|
cert := observeTestCert(t, "box-a.example", []string{"box-a.example"}, nil, time.Now().Add(24*time.Hour))
|
|
port := strconv.Itoa(observeTLSListener(t, cert))
|
|
origExec, origHave, origAccess := observeExec, observeHave, observeJournalAccess
|
|
observeExec = func(_ context.Context, name string, args ...string) ([]byte, error) {
|
|
switch name + " " + args[0] {
|
|
case "systemctl list-units":
|
|
return []byte("example.service loaded active running Example\n"), nil
|
|
case "ss -tlnpH":
|
|
return []byte("LISTEN 0 4096 127.0.0.1:" + port + " 0.0.0.0:*\n"), nil
|
|
}
|
|
return nil, nil
|
|
}
|
|
observeHave = func(string) bool { return true }
|
|
observeJournalAccess = func() string { return "" }
|
|
defer func() { observeExec, observeHave, observeJournalAccess = origExec, origHave, origAccess }()
|
|
folder := "/"
|
|
if observeHostSupported {
|
|
folder = t.TempDir()
|
|
}
|
|
accepted := map[string][]string{
|
|
"observe:net": {"203.0.113.9:443"},
|
|
"observe:metrics": {"up"},
|
|
"observe:logs": {"kernel", "5"},
|
|
"observe:folders": {folder, "1"},
|
|
"observe:journal": {"example", "5"},
|
|
"observe:logins": {"24"},
|
|
"observe:cert": {port, "box-a.example"},
|
|
}
|
|
for op, n := range observeArgsRecorded {
|
|
if len(accepted[op]) != n {
|
|
t.Fatalf("%s: the fixture has %d accepted args, the table says %d", op, len(accepted[op]), n)
|
|
}
|
|
tooMany := append(append([]string(nil), accepted[op]...), "x")
|
|
if _, _, code := driveObserve(t, op, tooMany); code == 0 {
|
|
t.Fatalf("%s accepted %d args but records %d — the table is short", op, n+1, n)
|
|
}
|
|
if op == "observe:net" {
|
|
continue
|
|
}
|
|
if _, errOut, code := driveObserve(t, op, tooMany[:n]); code != 0 {
|
|
t.Fatalf("%s refused %d args (%s) but records %d — the table is long", op, n, errOut, n)
|
|
}
|
|
}
|
|
}
|
|
|
|
// THE CROSS-REPO INVARIANT, ENFORCED. The host rejects a holes map with any
|
|
// shelf name outside ^[a-z0-9][a-z0-9-]{0,63}$ as unreadable — and an
|
|
// unreadable beat freezes that machine's stored holes under the host's
|
|
// COALESCE. So "every shelf name this box emits matches that rule" is a hard
|
|
// dependency, and an example table of six names cannot hold it: registering
|
|
// an op whose sanitised name is over 64 characters, or teaching the
|
|
// sanitiser a new character, would leave the table green and silently
|
|
// blank the host's reading of this box. This walks EVERY registered op.
|
|
func TestEveryRegisteredOpSanitisesToTheHostsShelfRule(t *testing.T) {
|
|
rule := regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,63}$`)
|
|
n := 0
|
|
for op := range handlerOps {
|
|
if !strings.HasPrefix(op, "observe:") {
|
|
continue
|
|
}
|
|
n++
|
|
got := storeSafeOpName(op)
|
|
if !rule.MatchString(got) {
|
|
t.Fatalf("%s sanitises to %q, which the host rejects as unreadable", op, got)
|
|
}
|
|
}
|
|
if n < 15 {
|
|
t.Fatalf("only %d observe ops walked — the registry moved and this test is not covering it", n)
|
|
}
|
|
}
|