Privatae
345 lines
12 KiB
Go
345 lines
12 KiB
Go
package main
|
|
|
|
// Gate: a push whose file stops answering leaves behind at most one stuck
|
|
// read per folder, and at most backupReadsMax on the whole bastion,
|
|
// however often a host starts a push there. A start in a folder whose read
|
|
// has not returned is refused with a sentence; when the read returns the
|
|
// folder is read again. A push that waits for a slot fails on that wait with
|
|
// a sentence naming what it waited on, never one blaming its own folder.
|
|
|
|
import (
|
|
"encoding/json"
|
|
"errors"
|
|
"os"
|
|
"path/filepath"
|
|
"runtime"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// stuckReads makes every read of a pushed file block until release, and
|
|
// counts the reads blocked at any moment: each one is a goroutine (and, on a
|
|
// real mount, an OS thread) that cannot be stopped.
|
|
type stuckReads struct {
|
|
gate chan struct{}
|
|
once sync.Once
|
|
blocked atomic.Int64
|
|
prev func(*os.File, []byte) (int, error)
|
|
}
|
|
|
|
// newStuckReads blocks the reads of files in dirs, or of every file when no
|
|
// folder is named; the others read as usual.
|
|
func newStuckReads(t *testing.T, dirs ...string) *stuckReads {
|
|
t.Helper()
|
|
s := &stuckReads{gate: make(chan struct{})}
|
|
dead := map[string]bool{}
|
|
for _, d := range dirs {
|
|
dead[d] = true
|
|
}
|
|
backupPushHookMu.Lock()
|
|
s.prev = backupPushRead
|
|
prev := s.prev
|
|
backupPushRead = func(f *os.File, p []byte) (int, error) {
|
|
if len(dead) > 0 && !dead[filepath.Dir(f.Name())] {
|
|
return prev(f, p)
|
|
}
|
|
s.blocked.Add(1)
|
|
defer s.blocked.Add(-1)
|
|
<-s.gate
|
|
return 0, errors.New("the mount went away")
|
|
}
|
|
backupPushHookMu.Unlock()
|
|
t.Cleanup(s.release)
|
|
return s
|
|
}
|
|
|
|
// release lets the blocked reads return, and the file answers again.
|
|
func (s *stuckReads) release() {
|
|
s.once.Do(func() {
|
|
backupPushHookMu.Lock()
|
|
backupPushRead = s.prev
|
|
backupPushHookMu.Unlock()
|
|
close(s.gate)
|
|
})
|
|
}
|
|
|
|
func shortPushStall(t *testing.T) {
|
|
t.Helper()
|
|
backupPushHookMu.Lock()
|
|
prev := backupPushFileStall
|
|
backupPushFileStall = 200 * time.Millisecond
|
|
backupPushHookMu.Unlock()
|
|
t.Cleanup(func() { backupPushHookMu.Lock(); backupPushFileStall = prev; backupPushHookMu.Unlock() })
|
|
}
|
|
|
|
// pushSlotsIdle waits for reads left blocked by an earlier test to return.
|
|
func pushSlotsIdle(t *testing.T) {
|
|
t.Helper()
|
|
eventually(t, "the push slots of an earlier test are freed", func() bool {
|
|
backupReadMu.Lock()
|
|
defer backupReadMu.Unlock()
|
|
return len(backupReads) == 0
|
|
})
|
|
}
|
|
|
|
func eventually(t *testing.T, what string, ok func() bool) {
|
|
t.Helper()
|
|
deadline := time.Now().Add(5 * time.Second)
|
|
for !ok() {
|
|
if time.Now().After(deadline) {
|
|
t.Fatalf("never: %s", what)
|
|
}
|
|
time.Sleep(10 * time.Millisecond)
|
|
}
|
|
}
|
|
|
|
// fdsOn counts this process's open descriptors on path, or -1 where the
|
|
// system does not list them.
|
|
func fdsOn(path string) int {
|
|
dir := "/proc/self/fd"
|
|
if _, err := os.Stat(dir); err != nil {
|
|
dir = "/dev/fd"
|
|
}
|
|
ents, err := os.ReadDir(dir)
|
|
if err != nil {
|
|
return -1
|
|
}
|
|
want, err := filepath.EvalSymlinks(path)
|
|
if err != nil {
|
|
return -1
|
|
}
|
|
n := 0
|
|
for _, e := range ents {
|
|
if l, err := os.Readlink(filepath.Join(dir, e.Name())); err == nil && l == want {
|
|
n++
|
|
}
|
|
}
|
|
return n
|
|
}
|
|
|
|
func startPush(t *testing.T, root, rel, sum string) (string, string, int) {
|
|
t.Helper()
|
|
return driveBackupOp(t, "backup_push", "", root,
|
|
backupPushReq{Op: "start", Path: rel, URL: "https://storage.example/put?sig=1", SHA256: sum, Headers: signedPut})
|
|
}
|
|
|
|
// pushStarted starts a push and returns its job id.
|
|
func pushStarted(t *testing.T, root, rel, sum string) string {
|
|
t.Helper()
|
|
out, errOut, code := startPush(t, root, rel, sum)
|
|
if code != 0 {
|
|
t.Fatalf("start refused: %s", errOut)
|
|
}
|
|
var st map[string]string
|
|
if err := json.Unmarshal([]byte(out), &st); err != nil || st["job_id"] == "" {
|
|
t.Fatalf("start reply: %s", out)
|
|
}
|
|
return st["job_id"]
|
|
}
|
|
|
|
func mustFailStuck(t *testing.T, root, id string) {
|
|
t.Helper()
|
|
if st := awaitPush(t, root, id); st["state"] != "failed" || !strings.Contains(st["error"], "stopped answering") {
|
|
t.Fatalf("a push whose file stopped answering reads %v", st)
|
|
}
|
|
}
|
|
|
|
// failures waits for each push to fail and counts the ones whose own read
|
|
// stopped answering and the ones that failed waiting, with a sentence that
|
|
// holds want, for a slot to read. Any other outcome fails the test.
|
|
func failures(t *testing.T, want string, pushes ...[2]string) (stuck, waited int) {
|
|
t.Helper()
|
|
for _, p := range pushes {
|
|
st := awaitPush(t, p[0], p[1])
|
|
switch {
|
|
case st["state"] == "failed" && strings.Contains(st["error"], "stopped answering"):
|
|
stuck++
|
|
case st["state"] == "failed" && strings.Contains(st["error"], "this push waited more than") &&
|
|
strings.Contains(st["error"], want):
|
|
waited++
|
|
default:
|
|
t.Fatalf("a push from a dead folder reads %v", st)
|
|
}
|
|
}
|
|
return stuck, waited
|
|
}
|
|
|
|
// capReads lowers backupReadsMax to n for the test.
|
|
func capReads(t *testing.T, n int) {
|
|
t.Helper()
|
|
backupReadMu.Lock()
|
|
prev := backupReadsMax
|
|
backupReadsMax = n
|
|
backupReadMu.Unlock()
|
|
t.Cleanup(func() { backupReadMu.Lock(); backupReadsMax = prev; backupReadMu.Unlock() })
|
|
}
|
|
|
|
func TestRepeatedPushesFromADeadFolderHoldOneRead(t *testing.T) {
|
|
stubPublicDNS(t)
|
|
shortPushStall(t)
|
|
pushSlotsIdle(t)
|
|
root := t.TempDir()
|
|
rel, sum := pushFixture(t, root, []byte(strings.Repeat("a backup ", 1000)))
|
|
abs := filepath.Join(root, fixtureData)
|
|
s := newStuckReads(t)
|
|
|
|
// Two starts back to back, before either is known to be stuck: one read
|
|
// blocks, the other push waits for the folder and fails at the same
|
|
// bound without a read of its own, saying the folder has not answered.
|
|
a := pushStarted(t, root, rel, sum)
|
|
b := pushStarted(t, root, rel, sum)
|
|
if stuck, waited := failures(t, "the folder of "+rel+" has not answered an earlier push",
|
|
[2]string{root, a}, [2]string{root, b}); stuck != 1 || waited != 1 {
|
|
t.Fatalf("%d pushes failed on their own read and %d waiting for the folder; want 1 and 1", stuck, waited)
|
|
}
|
|
eventually(t, "one read is left blocked and one descriptor open", func() bool {
|
|
f := fdsOn(abs)
|
|
return s.blocked.Load() == 1 && (f == -1 || f == 1)
|
|
})
|
|
time.Sleep(100 * time.Millisecond)
|
|
g0, f0 := runtime.NumGoroutine(), fdsOn(abs)
|
|
|
|
for i := 0; i < 10; i++ {
|
|
out, errOut, code := startPush(t, root, rel, sum)
|
|
if code == 0 {
|
|
t.Fatalf("start %d in a folder whose read has not returned was accepted: %s", i, out)
|
|
}
|
|
if !strings.Contains(errOut, "backup_push: the folder of "+rel+" has not answered an earlier push") ||
|
|
!strings.Contains(errOut, "nothing more is sent from that folder until it answers") {
|
|
t.Fatalf("start %d refused without the sentence: %s", i, errOut)
|
|
}
|
|
}
|
|
eventually(t, "the refused starts leave no goroutine behind", func() bool {
|
|
return runtime.NumGoroutine() <= g0+3
|
|
})
|
|
if n := s.blocked.Load(); n != 1 {
|
|
t.Fatalf("%d reads are blocked on the dead folder after repeated starts; want 1", n)
|
|
}
|
|
if f := fdsOn(abs); f != f0 {
|
|
t.Fatalf("%d descriptors are open on the file after repeated starts; was %d", f, f0)
|
|
}
|
|
}
|
|
|
|
func TestAPushFolderIsReadAgainWhenItsReadReturns(t *testing.T) {
|
|
stubPublicDNS(t)
|
|
shortPushStall(t)
|
|
pushSlotsIdle(t)
|
|
root := t.TempDir()
|
|
rel, sum := pushFixture(t, root, []byte(strings.Repeat("a backup ", 1000)))
|
|
s := newStuckReads(t)
|
|
mustFailStuck(t, root, pushStarted(t, root, rel, sum))
|
|
if _, errOut, code := startPush(t, root, rel, sum); code == 0 || !strings.Contains(errOut, "has not answered an earlier push") {
|
|
t.Fatalf("a start while the read is stuck: %d %s", code, errOut)
|
|
}
|
|
|
|
s.release()
|
|
eventually(t, "the returned read frees its folder", func() bool {
|
|
backupReadMu.Lock()
|
|
defer backupReadMu.Unlock()
|
|
return len(backupReads) == 0
|
|
})
|
|
// Read to the end this time: a digest that is not the file's fails on
|
|
// the comparison, which only a full read reaches.
|
|
id := pushStarted(t, root, rel, strings.Repeat("ab", 32))
|
|
if st := awaitPush(t, root, id); st["state"] != "failed" || !strings.Contains(st["error"], "sha256 differs") {
|
|
t.Fatalf("the folder was not read again once its read returned: %v", st)
|
|
}
|
|
}
|
|
|
|
func TestStuckPushReadsAreCappedPerBastion(t *testing.T) {
|
|
stubPublicDNS(t)
|
|
shortPushStall(t)
|
|
pushSlotsIdle(t)
|
|
capReads(t, 2)
|
|
|
|
type folder struct{ root, rel, sum string }
|
|
var fs []folder
|
|
for i := 0; i < 4; i++ {
|
|
root := t.TempDir()
|
|
rel, sum := pushFixture(t, root, []byte(strings.Repeat("a backup ", 1000)))
|
|
fs = append(fs, folder{root, rel, sum})
|
|
}
|
|
s := newStuckReads(t)
|
|
|
|
// Three dead folders started at once: two reads block, the third push
|
|
// waits for a slot and fails at the bound without a read of its own,
|
|
// saying the cap is held.
|
|
var ps [][2]string
|
|
for _, f := range fs[:3] {
|
|
ps = append(ps, [2]string{f.root, pushStarted(t, f.root, f.rel, f.sum)})
|
|
}
|
|
if stuck, waited := failures(t, "2 reads of backup folders have not returned", ps...); stuck != 2 || waited != 1 {
|
|
t.Fatalf("%d pushes failed on their own read and %d waiting for a slot; want 2 and 1", stuck, waited)
|
|
}
|
|
time.Sleep(100 * time.Millisecond)
|
|
if n := s.blocked.Load(); n != 2 {
|
|
t.Fatalf("%d reads are blocked; the cap is 2", n)
|
|
}
|
|
|
|
// A folder that answers is refused too while the cap is held.
|
|
f := fs[3]
|
|
_, errOut, code := startPush(t, f.root, f.rel, f.sum)
|
|
if code == 0 || !strings.Contains(errOut, "backup_push: 2 reads of backup folders have not returned") ||
|
|
!strings.Contains(errOut, "no push starts on this machine until one does") {
|
|
t.Fatalf("a start with the cap held: %d %s", code, errOut)
|
|
}
|
|
if n := s.blocked.Load(); n != 2 {
|
|
t.Fatalf("%d reads are blocked after a refused start; want 2", n)
|
|
}
|
|
|
|
s.release()
|
|
eventually(t, "the returned reads free their slots", func() bool {
|
|
backupReadMu.Lock()
|
|
defer backupReadMu.Unlock()
|
|
return len(backupReads) == 0
|
|
})
|
|
id := pushStarted(t, f.root, f.rel, strings.Repeat("ab", 32))
|
|
if st := awaitPush(t, f.root, id); st["state"] != "failed" || !strings.Contains(st["error"], "sha256 differs") {
|
|
t.Fatalf("a push after the reads returned: %v", st)
|
|
}
|
|
}
|
|
|
|
// A push already under way in a folder that answers, waiting for a slot while
|
|
// other folders' reads hold the cap, fails with the sentence about those
|
|
// folders, not one blaming its own: the wait is not a read of its file.
|
|
func TestAPushWaitingForASlotIsNotBlamedOnItsOwnFolder(t *testing.T) {
|
|
stubPublicDNS(t)
|
|
pushSlotsIdle(t)
|
|
// Long enough that the third start is taken before the two stuck reads
|
|
// are past the bound, even on a loaded machine.
|
|
backupPushHookMu.Lock()
|
|
prevStall := backupPushFileStall
|
|
backupPushFileStall = time.Second
|
|
backupPushHookMu.Unlock()
|
|
t.Cleanup(func() { backupPushHookMu.Lock(); backupPushFileStall = prevStall; backupPushHookMu.Unlock() })
|
|
capReads(t, 2)
|
|
|
|
dead1, dead2, healthy := t.TempDir(), t.TempDir(), t.TempDir()
|
|
body := []byte(strings.Repeat("a backup ", 1000))
|
|
rel, sum := pushFixture(t, dead1, body)
|
|
pushFixture(t, dead2, body)
|
|
pushFixture(t, healthy, body)
|
|
s := newStuckReads(t, dead1, dead2)
|
|
|
|
a := pushStarted(t, dead1, rel, sum)
|
|
b := pushStarted(t, dead2, rel, sum)
|
|
eventually(t, "both dead folders hold a read", func() bool { return s.blocked.Load() == 2 })
|
|
// Neither read is past the bound yet, so this start is taken; its first
|
|
// operation waits for a slot.
|
|
c := pushStarted(t, healthy, rel, sum)
|
|
mustFailStuck(t, dead1, a)
|
|
mustFailStuck(t, dead2, b)
|
|
st := awaitPush(t, healthy, c)
|
|
if st["state"] != "failed" || strings.Contains(st["error"], "stopped answering") ||
|
|
!strings.Contains(st["error"], "this push waited more than 1s for a turn to read its file: "+
|
|
"2 reads of backup folders have not returned") {
|
|
t.Fatalf("a push from a folder that answers, waiting for a slot, reads %v", st)
|
|
}
|
|
if n := s.blocked.Load(); n != 2 {
|
|
t.Fatalf("%d reads are blocked; the cap is 2", n)
|
|
}
|
|
}
|