privatae-bastion/client-go/opmask_test.go

package main

// Gate: a host's backup folder on a mount that stopped answering cannot
// stop op runs. Its mask is left out for the run when a stat of it errors
// or does not return in time, and it stays refused by every text check.

import (
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"testing"
	"time"
)

func opMaskFixture(t *testing.T) (nas, conf string) {
	t.Helper()
	uiTestConfigDir(t)
	nas = filepath.Join(t.TempDir(), "nas")
	_ = os.MkdirAll(nas, 0o700)
	savePairingWithRoot(t, "nas.host.privatae.ai", nas)
	conf, _ = configDirPath()
	prevT, prevS, prevR, prevL := opMaskProbeTimeout, opMaskStat, opResolveFn, opMaskLog
	opMaskProbeTimeout = 100 * time.Millisecond
	var logged []string
	opMaskLog = func(format string, a ...any) { logged = append(logged, format) }
	t.Cleanup(func() {
		opMaskProbeTimeout, opMaskStat, opResolveFn, opMaskLog = prevT, prevS, prevR, prevL
		opProbeMu.Lock()
		opProbeLogged = map[string]bool{}
		opProbeMu.Unlock()
	})
	return nas, conf
}

func has(list []string, p string) bool {
	for _, x := range list {
		if x == p {
			return true
		}
	}
	return false
}

func stillRefused(t *testing.T, nas string) {
	t.Helper()
	offs, err := opOffLimits()
	if err != nil || !has(offs, nas) {
		t.Fatalf("the backup folder left the off-limits set: %v %v", err, offs)
	}
	if off, err := underOffLimits(filepath.Join(nas, fixtureData)); err != nil || off == "" {
		t.Fatalf("an op may now name the backup folder: %q %v", off, err)
	}
}

func TestOpMasksSkipABackupFolderWhoseStatErrors(t *testing.T) {
	nas, conf := opMaskFixture(t)
	masks, err := opMaskSet()
	if err != nil || !has(masks, nas) || !has(masks, conf) {
		t.Fatalf("a folder that answers is masked: %v %v", err, masks)
	}
	opMaskStat = func(p string) (os.FileInfo, error) {
		if p == nas {
			return nil, errors.New("stale file handle")
		}
		return os.Stat(p)
	}
	masks, err = opMaskSet()
	if err != nil {
		t.Fatalf("an erroring backup folder refused the run: %v", err)
	}
	if has(masks, nas) {
		t.Fatal("an erroring backup folder is still masked, so the run would fail on it")
	}
	if !has(masks, conf) {
		t.Fatal("the configuration folder lost its mask")
	}
	stillRefused(t, nas)
	// Missing is not an error: the mask step skips it itself.
	opMaskStat = func(p string) (os.FileInfo, error) { return nil, os.ErrNotExist }
	if masks, _ := opMaskSet(); !has(masks, nas) {
		t.Fatal("a missing backup folder was treated as unanswering")
	}
}

func TestOpMasksDoNotWaitOnABackupFolderThatHangs(t *testing.T) {
	nas, conf := opMaskFixture(t)
	gate := make(chan struct{})
	// Release the blocked probes and wait for them to finish before the
	// fixture puts the hooks back, so nothing still running reads them.
	t.Cleanup(func() {
		close(gate)
		deadline := time.Now().Add(5 * time.Second)
		for time.Now().Before(deadline) {
			opProbeMu.Lock()
			n := len(opProbeInFlight)
			opProbeMu.Unlock()
			if n == 0 {
				return
			}
			time.Sleep(5 * time.Millisecond)
		}
		t.Error("the released probes never finished")
	})
	calls := 0
	opMaskStat = func(p string) (os.FileInfo, error) {
		if p == nas {
			opProbeMu.Lock()
			calls++
			opProbeMu.Unlock()
			<-gate
		}
		return os.Stat(p)
	}
	opResolveFn = func(p string) string {
		if p == nas {
			<-gate
		}
		return resolveExisting(p)
	}
	for i := 0; i < 3; i++ {
		start := time.Now()
		masks, err := opMaskSet()
		if err != nil {
			t.Fatal(err)
		}
		if el := time.Since(start); el > 2*time.Second {
			t.Fatalf("building the masks waited %s on a folder that does not answer", el)
		}
		if has(masks, nas) || !has(masks, conf) {
			t.Fatalf("masks %v", masks)
		}
	}
	opProbeMu.Lock()
	n := calls
	opProbeMu.Unlock()
	if n != 1 {
		t.Fatalf("a hung stat was started %d times; later runs must not pile up behind it", n)
	}
	stillRefused(t, nas)
	// Said once, not per run.
	opProbeMu.Lock()
	logged := opProbeLogged[nas]
	opProbeMu.Unlock()
	if !logged {
		t.Fatal("the unanswering folder was not logged")
	}
}

func TestOpMaskNoteLogsOnce(t *testing.T) {
	opMaskFixture(t)
	var lines []string
	opMaskLog = func(format string, a ...any) { lines = append(lines, format) }
	for i := 0; i < 3; i++ {
		opMaskNote("/mnt/nas/x", errors.New("stale"))
	}
	if len(lines) != 1 || !strings.Contains(lines[0], "cannot be checked") {
		t.Fatalf("logged %d lines: %v", len(lines), lines)
	}
}

// hangFolder makes resolving (and stating) the folder p block until the
// test ends, as a network mount that stopped answering does. The blocked
// probes are released and waited for before the hooks go back.
func hangFolder(t *testing.T, p string) {
	t.Helper()
	gate := make(chan struct{})
	prevR, prevS := opResolveFn, opMaskStat
	opResolveFn = func(q string) string {
		if q == p {
			<-gate
		}
		return prevR(q)
	}
	opMaskStat = func(q string) (os.FileInfo, error) {
		if q == p {
			<-gate
		}
		return prevS(q)
	}
	t.Cleanup(func() {
		close(gate)
		deadline := time.Now().Add(5 * time.Second)
		for time.Now().Before(deadline) {
			opProbeMu.Lock()
			n := len(opProbeInFlight)
			opProbeMu.Unlock()
			if n == 0 {
				break
			}
			time.Sleep(5 * time.Millisecond)
		}
		opResolveFn, opMaskStat = prevR, prevS
	})
}

// One host's folder not answering blocks no other host's backup ops, and a
// host whose own folder does not answer is refused with a sentence.
func TestBackupRootChecksDoNotWaitOnAStaleFolder(t *testing.T) {
	opMaskFixture(t)
	const a, b = "a.host.privatae.ai", "b.host.privatae.ai"
	rootA := filepath.Join(t.TempDir(), "a")
	stale := filepath.Join(t.TempDir(), "stale")
	_ = os.MkdirAll(rootA, 0o700)
	_ = os.MkdirAll(stale, 0o700)
	savePairingWithRoot(t, a, rootA)
	savePairingWithRoot(t, b, stale)
	hangFolder(t, stale)
	for i := 0; i < 3; i++ {
		start := time.Now()
		if why := backupRootProblem(a, rootA); why != "" {
			t.Fatalf("another host's stale folder refused this one: %s", why)
		}
		if el := time.Since(start); el > 2*time.Second {
			t.Fatalf("the check waited %s on another host's stale folder", el)
		}
	}
	start := time.Now()
	why := backupRootProblem(b, stale)
	if why == "" || !strings.Contains(why, "does not answer") {
		t.Fatalf("a host whose own folder does not answer was not refused as such: %q", why)
	}
	if el := time.Since(start); el > 2*time.Second {
		t.Fatalf("refusing took %s", el)
	}
}

// While a folder does not answer, where it last resolved to still counts:
// no op may name the real place, and no host's folder may be set over it.
func TestAStaleBackupFolderKeepsItsLastKnownPlace(t *testing.T) {
	opMaskFixture(t)
	const a, b = "a.host.privatae.ai", "b.host.privatae.ai"
	real := filepath.Join(t.TempDir(), "nas-real")
	_ = os.MkdirAll(real, 0o700)
	link := filepath.Join(t.TempDir(), "nas-link")
	if err := os.Symlink(real, link); err != nil {
		t.Skip("no symlinks here")
	}
	savePairingWithRoot(t, b, link)
	// It answers once: its real place is learned.
	offs, err := opOffLimits()
	if err != nil || !has(offs, real) {
		t.Fatalf("the real place is not off limits while it answers: %v %v", err, offs)
	}
	hangFolder(t, link)
	offs, err = opOffLimits()
	if err != nil || !has(offs, real) || !has(offs, link) {
		t.Fatalf("the real place left the off-limits set during the outage: %v %v", err, offs)
	}
	if off, err := underOffLimits(filepath.Join(real, fixtureData)); err != nil || off == "" {
		t.Fatalf("an op may name the real place during the outage: %q %v", off, err)
	}
	// Another host set over the real place is refused during the outage too.
	rootA := filepath.Join(real, "a")
	_ = os.MkdirAll(rootA, 0o700)
	savePairingWithRoot(t, a, rootA)
	if why := backupRootProblem(a, rootA); why == "" || !strings.Contains(why, b) {
		t.Fatalf("a folder inside the stale host's real place was accepted: %q", why)
	}
	// And the last place survives a restart.
	lastResolvedMu.Lock()
	lastResolvedLoaded, lastResolvedMap = false, nil
	lastResolvedMu.Unlock()
	abs, _ := filepath.Abs(link)
	if got := lastResolved(abs); got != real {
		t.Fatalf("after a restart the last place is %q, not %q", got, real)
	}
}

// The record of where each backup folder last resolved is bounded: past
// lastResolvedMax folders, the one resolved longest ago goes. A folder still
// in use keeps its place (its time is refreshed at least daily, on disk
// too), and the record's first shape (folder -> place) still loads.
func TestLastResolvedIsBounded(t *testing.T) {
	uiTestConfigDir(t)
	reset := func() {
		lastResolvedMu.Lock()
		lastResolvedLoaded, lastResolvedMap = false, nil
		lastResolvedMu.Unlock()
	}
	reset()
	prevNow := lastResolvedNow
	clock := int64(1_000_000)
	lastResolvedNow = func() int64 { return clock }
	t.Cleanup(func() { lastResolvedNow = prevNow; reset() })

	noteResolved("/in-use", "/real/in-use")
	for i := 0; i < lastResolvedMax+50; i++ {
		clock += 3600
		noteResolved(fmt.Sprintf("/f%04d", i), fmt.Sprintf("/real/f%04d", i))
		if i%20 == 0 {
			noteResolved("/in-use", "/real/in-use") // resolved again, in use
		}
	}
	lastResolvedMu.Lock()
	n := len(lastResolvedMap)
	lastResolvedMu.Unlock()
	if n > lastResolvedMax {
		t.Fatalf("%d folders kept, bound %d", n, lastResolvedMax)
	}
	if lastResolved("/in-use") != "/real/in-use" {
		t.Fatal("a folder in use was dropped")
	}
	if lastResolved("/f0000") != "" || lastResolved(fmt.Sprintf("/f%04d", lastResolvedMax+49)) == "" {
		t.Fatal("the oldest folder was kept or the newest dropped")
	}
	// After a restart: the same, read from disk.
	reset()
	if lastResolved("/in-use") != "/real/in-use" || lastResolved("/f0000") != "" {
		t.Fatal("the bound or the order did not survive a restart")
	}
	// The first shape of the file loads.
	p, _ := lastResolvedPath()
	_ = os.WriteFile(p, []byte(`{"/old":"/real/old"}`), 0o600)
	reset()
	if lastResolved("/old") != "/real/old" {
		t.Fatal("the first shape of the record did not load")
	}
}