package main // Gate: a host's backup folder on a mount that stopped answering cannot // stop op runs. Its mask is left out for the run when a stat of it errors // or does not return in time, and it stays refused by every text check. import ( "errors" "fmt" "os" "path/filepath" "strings" "testing" "time" ) func opMaskFixture(t *testing.T) (nas, conf string) { t.Helper() uiTestConfigDir(t) nas = filepath.Join(t.TempDir(), "nas") _ = os.MkdirAll(nas, 0o700) savePairingWithRoot(t, "nas.host.privatae.ai", nas) conf, _ = configDirPath() prevT, prevS, prevR, prevL := opMaskProbeTimeout, opMaskStat, opResolveFn, opMaskLog opMaskProbeTimeout = 100 * time.Millisecond var logged []string opMaskLog = func(format string, a ...any) { logged = append(logged, format) } t.Cleanup(func() { opMaskProbeTimeout, opMaskStat, opResolveFn, opMaskLog = prevT, prevS, prevR, prevL opProbeMu.Lock() opProbeLogged = map[string]bool{} opProbeMu.Unlock() }) return nas, conf } func has(list []string, p string) bool { for _, x := range list { if x == p { return true } } return false } func stillRefused(t *testing.T, nas string) { t.Helper() offs, err := opOffLimits() if err != nil || !has(offs, nas) { t.Fatalf("the backup folder left the off-limits set: %v %v", err, offs) } if off, err := underOffLimits(filepath.Join(nas, fixtureData)); err != nil || off == "" { t.Fatalf("an op may now name the backup folder: %q %v", off, err) } } func TestOpMasksSkipABackupFolderWhoseStatErrors(t *testing.T) { nas, conf := opMaskFixture(t) masks, err := opMaskSet() if err != nil || !has(masks, nas) || !has(masks, conf) { t.Fatalf("a folder that answers is masked: %v %v", err, masks) } opMaskStat = func(p string) (os.FileInfo, error) { if p == nas { return nil, errors.New("stale file handle") } return os.Stat(p) } masks, err = opMaskSet() if err != nil { t.Fatalf("an erroring backup folder refused the run: %v", err) } if has(masks, nas) { t.Fatal("an erroring backup folder is still masked, so the run would fail on it") } if !has(masks, conf) { t.Fatal("the configuration folder lost its mask") } stillRefused(t, nas) // Missing is not an error: the mask step skips it itself. opMaskStat = func(p string) (os.FileInfo, error) { return nil, os.ErrNotExist } if masks, _ := opMaskSet(); !has(masks, nas) { t.Fatal("a missing backup folder was treated as unanswering") } } func TestOpMasksDoNotWaitOnABackupFolderThatHangs(t *testing.T) { nas, conf := opMaskFixture(t) gate := make(chan struct{}) // Release the blocked probes and wait for them to finish before the // fixture puts the hooks back, so nothing still running reads them. t.Cleanup(func() { close(gate) deadline := time.Now().Add(5 * time.Second) for time.Now().Before(deadline) { opProbeMu.Lock() n := len(opProbeInFlight) opProbeMu.Unlock() if n == 0 { return } time.Sleep(5 * time.Millisecond) } t.Error("the released probes never finished") }) calls := 0 opMaskStat = func(p string) (os.FileInfo, error) { if p == nas { opProbeMu.Lock() calls++ opProbeMu.Unlock() <-gate } return os.Stat(p) } opResolveFn = func(p string) string { if p == nas { <-gate } return resolveExisting(p) } for i := 0; i < 3; i++ { start := time.Now() masks, err := opMaskSet() if err != nil { t.Fatal(err) } if el := time.Since(start); el > 2*time.Second { t.Fatalf("building the masks waited %s on a folder that does not answer", el) } if has(masks, nas) || !has(masks, conf) { t.Fatalf("masks %v", masks) } } opProbeMu.Lock() n := calls opProbeMu.Unlock() if n != 1 { t.Fatalf("a hung stat was started %d times; later runs must not pile up behind it", n) } stillRefused(t, nas) // Said once, not per run. opProbeMu.Lock() logged := opProbeLogged[nas] opProbeMu.Unlock() if !logged { t.Fatal("the unanswering folder was not logged") } } func TestOpMaskNoteLogsOnce(t *testing.T) { opMaskFixture(t) var lines []string opMaskLog = func(format string, a ...any) { lines = append(lines, format) } for i := 0; i < 3; i++ { opMaskNote("/mnt/nas/x", errors.New("stale")) } if len(lines) != 1 || !strings.Contains(lines[0], "cannot be checked") { t.Fatalf("logged %d lines: %v", len(lines), lines) } } // hangFolder makes resolving (and stating) the folder p block until the // test ends, as a network mount that stopped answering does. The blocked // probes are released and waited for before the hooks go back. func hangFolder(t *testing.T, p string) { t.Helper() gate := make(chan struct{}) prevR, prevS := opResolveFn, opMaskStat opResolveFn = func(q string) string { if q == p { <-gate } return prevR(q) } opMaskStat = func(q string) (os.FileInfo, error) { if q == p { <-gate } return prevS(q) } t.Cleanup(func() { close(gate) deadline := time.Now().Add(5 * time.Second) for time.Now().Before(deadline) { opProbeMu.Lock() n := len(opProbeInFlight) opProbeMu.Unlock() if n == 0 { break } time.Sleep(5 * time.Millisecond) } opResolveFn, opMaskStat = prevR, prevS }) } // One host's folder not answering blocks no other host's backup ops, and a // host whose own folder does not answer is refused with a sentence. func TestBackupRootChecksDoNotWaitOnAStaleFolder(t *testing.T) { opMaskFixture(t) const a, b = "a.host.privatae.ai", "b.host.privatae.ai" rootA := filepath.Join(t.TempDir(), "a") stale := filepath.Join(t.TempDir(), "stale") _ = os.MkdirAll(rootA, 0o700) _ = os.MkdirAll(stale, 0o700) savePairingWithRoot(t, a, rootA) savePairingWithRoot(t, b, stale) hangFolder(t, stale) for i := 0; i < 3; i++ { start := time.Now() if why := backupRootProblem(a, rootA); why != "" { t.Fatalf("another host's stale folder refused this one: %s", why) } if el := time.Since(start); el > 2*time.Second { t.Fatalf("the check waited %s on another host's stale folder", el) } } start := time.Now() why := backupRootProblem(b, stale) if why == "" || !strings.Contains(why, "does not answer") { t.Fatalf("a host whose own folder does not answer was not refused as such: %q", why) } if el := time.Since(start); el > 2*time.Second { t.Fatalf("refusing took %s", el) } } // While a folder does not answer, where it last resolved to still counts: // no op may name the real place, and no host's folder may be set over it. func TestAStaleBackupFolderKeepsItsLastKnownPlace(t *testing.T) { opMaskFixture(t) const a, b = "a.host.privatae.ai", "b.host.privatae.ai" real := filepath.Join(t.TempDir(), "nas-real") _ = os.MkdirAll(real, 0o700) link := filepath.Join(t.TempDir(), "nas-link") if err := os.Symlink(real, link); err != nil { t.Skip("no symlinks here") } savePairingWithRoot(t, b, link) // It answers once: its real place is learned. offs, err := opOffLimits() if err != nil || !has(offs, real) { t.Fatalf("the real place is not off limits while it answers: %v %v", err, offs) } hangFolder(t, link) offs, err = opOffLimits() if err != nil || !has(offs, real) || !has(offs, link) { t.Fatalf("the real place left the off-limits set during the outage: %v %v", err, offs) } if off, err := underOffLimits(filepath.Join(real, fixtureData)); err != nil || off == "" { t.Fatalf("an op may name the real place during the outage: %q %v", off, err) } // Another host set over the real place is refused during the outage too. rootA := filepath.Join(real, "a") _ = os.MkdirAll(rootA, 0o700) savePairingWithRoot(t, a, rootA) if why := backupRootProblem(a, rootA); why == "" || !strings.Contains(why, b) { t.Fatalf("a folder inside the stale host's real place was accepted: %q", why) } // And the last place survives a restart. lastResolvedMu.Lock() lastResolvedLoaded, lastResolvedMap = false, nil lastResolvedMu.Unlock() abs, _ := filepath.Abs(link) if got := lastResolved(abs); got != real { t.Fatalf("after a restart the last place is %q, not %q", got, real) } } // The record of where each backup folder last resolved is bounded: past // lastResolvedMax folders, the one resolved longest ago goes. A folder still // in use keeps its place (its time is refreshed at least daily, on disk // too), and the record's first shape (folder -> place) still loads. func TestLastResolvedIsBounded(t *testing.T) { uiTestConfigDir(t) reset := func() { lastResolvedMu.Lock() lastResolvedLoaded, lastResolvedMap = false, nil lastResolvedMu.Unlock() } reset() prevNow := lastResolvedNow clock := int64(1_000_000) lastResolvedNow = func() int64 { return clock } t.Cleanup(func() { lastResolvedNow = prevNow; reset() }) noteResolved("/in-use", "/real/in-use") for i := 0; i < lastResolvedMax+50; i++ { clock += 3600 noteResolved(fmt.Sprintf("/f%04d", i), fmt.Sprintf("/real/f%04d", i)) if i%20 == 0 { noteResolved("/in-use", "/real/in-use") // resolved again, in use } } lastResolvedMu.Lock() n := len(lastResolvedMap) lastResolvedMu.Unlock() if n > lastResolvedMax { t.Fatalf("%d folders kept, bound %d", n, lastResolvedMax) } if lastResolved("/in-use") != "/real/in-use" { t.Fatal("a folder in use was dropped") } if lastResolved("/f0000") != "" || lastResolved(fmt.Sprintf("/f%04d", lastResolvedMax+49)) == "" { t.Fatal("the oldest folder was kept or the newest dropped") } // After a restart: the same, read from disk. reset() if lastResolved("/in-use") != "/real/in-use" || lastResolved("/f0000") != "" { t.Fatal("the bound or the order did not survive a restart") } // The first shape of the file loads. p, _ := lastResolvedPath() _ = os.WriteFile(p, []byte(`{"/old":"/real/old"}`), 0o600) reset() if lastResolved("/old") != "/real/old" { t.Fatal("the first shape of the record did not load") } }