bastion 0.53.2
This commit is contained in:
Privatae
@@ -0,0 +1,231 @@
|
||||
package main
|
||||
|
||||
// Gate: pushes and the background hashing keep their reads of backup folders
|
||||
// in one record. A push is not started in a folder whose hashing read has not
|
||||
// returned, the hashing does not read a folder whose push read has not
|
||||
// returned, and the reads of both count toward the one cap.
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// heldHashes makes the hashing job's read of any file in dirs block until
|
||||
// release, and counts the reads it starts in each folder.
|
||||
type heldHashes struct {
|
||||
mu sync.Mutex
|
||||
reads map[string]int
|
||||
gate chan struct{}
|
||||
once sync.Once
|
||||
}
|
||||
|
||||
func holdHashes(t *testing.T, dirs ...string) *heldHashes {
|
||||
t.Helper()
|
||||
h := &heldHashes{reads: map[string]int{}, gate: make(chan struct{})}
|
||||
dead := map[string]bool{}
|
||||
for _, d := range dirs {
|
||||
dead[d] = true
|
||||
}
|
||||
hashMu.Lock()
|
||||
prevBound := hashJobTimeout
|
||||
hashJobTimeout = 200 * time.Millisecond
|
||||
hashMu.Unlock()
|
||||
quiet := opMaskLog
|
||||
opMaskLog = func(string, ...any) {}
|
||||
prev := hashFileFn
|
||||
hashFileFn = func(p string) (string, int64, error) {
|
||||
d := filepath.Dir(p)
|
||||
h.mu.Lock()
|
||||
h.reads[d]++
|
||||
h.mu.Unlock()
|
||||
if dead[d] {
|
||||
<-h.gate
|
||||
}
|
||||
return hashFile(p)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
h.release()
|
||||
readsIdle(t)
|
||||
eventually(t, "the hashing job ends", func() bool {
|
||||
hashMu.Lock()
|
||||
defer hashMu.Unlock()
|
||||
return !hashRunning
|
||||
})
|
||||
hashFileFn = prev
|
||||
opMaskLog = quiet
|
||||
hashMu.Lock()
|
||||
hashJobTimeout = prevBound
|
||||
hashMu.Unlock()
|
||||
})
|
||||
return h
|
||||
}
|
||||
|
||||
func (h *heldHashes) release() { h.once.Do(func() { close(h.gate) }) }
|
||||
|
||||
func (h *heldHashes) readsIn(dir string) int {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
return h.reads[dir]
|
||||
}
|
||||
|
||||
// readsIdle waits until no read of a backup folder is in flight.
|
||||
func readsIdle(t *testing.T) {
|
||||
t.Helper()
|
||||
eventually(t, "every read of a backup folder returned", func() bool {
|
||||
backupReadMu.Lock()
|
||||
defer backupReadMu.Unlock()
|
||||
return len(backupReads) == 0
|
||||
})
|
||||
}
|
||||
|
||||
// stuckPast waits until dir has a read in flight past its bound.
|
||||
func stuckPast(t *testing.T, dir string) {
|
||||
t.Helper()
|
||||
eventually(t, "a read of "+dir+" is past its bound", func() bool {
|
||||
_, stuck := folderStuck(dir, time.Now(), pushStallBound(), hashBound())
|
||||
return stuck
|
||||
})
|
||||
}
|
||||
|
||||
func TestAPushIsNotStartedInAFolderWhoseHashingReadIsStuck(t *testing.T) {
|
||||
stubPublicDNS(t)
|
||||
shortPushStall(t)
|
||||
readsIdle(t)
|
||||
root := t.TempDir()
|
||||
rel, sum := pushFixture(t, root, []byte(strings.Repeat("a backup ", 1000)))
|
||||
abs := filepath.Join(root, fixtureData)
|
||||
h := holdHashes(t, root)
|
||||
s := newStuckReads(t, root)
|
||||
|
||||
requestHash(abs)
|
||||
eventually(t, "the hashing job reads the file", func() bool { return h.readsIn(root) == 1 })
|
||||
stuckPast(t, root)
|
||||
|
||||
for i := 0; i < 5; i++ {
|
||||
_, errOut, code := startPush(t, root, rel, sum)
|
||||
if code == 0 || !strings.Contains(errOut,
|
||||
"backup_push: the folder of "+rel+" has not answered the reading of a backup for its digest") {
|
||||
t.Fatalf("start %d in a folder whose hashing read is stuck: %d %s", i, code, errOut)
|
||||
}
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
if n := s.blocked.Load(); n != 0 {
|
||||
t.Fatalf("%d push reads started in a folder whose hashing read is stuck", n)
|
||||
}
|
||||
|
||||
// When the hashing read returns, its digest is kept and a push starts.
|
||||
s.release()
|
||||
h.release()
|
||||
readsIdle(t)
|
||||
awaitHash(t, abs, sum, "the stuck hashing read's digest was not kept when it returned")
|
||||
id := pushStarted(t, root, rel, strings.Repeat("ab", 32))
|
||||
if st := awaitPush(t, root, id); st["state"] != "failed" || !strings.Contains(st["error"], "sha256 differs") {
|
||||
t.Fatalf("a push once the hashing read returned: %v", st)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheHashingDoesNotReadAFolderWhosePushReadIsStuck(t *testing.T) {
|
||||
stubPublicDNS(t)
|
||||
shortPushStall(t)
|
||||
readsIdle(t)
|
||||
dead, live := t.TempDir(), t.TempDir()
|
||||
rel, sum := pushFixture(t, dead, []byte(strings.Repeat("a backup ", 1000)))
|
||||
other := filepath.Join(dead, "20260101T000000Z.tar.zst.age")
|
||||
liveFile := filepath.Join(live, fixtureData)
|
||||
for p, b := range map[string]string{other: "another backup", liveFile: "answers"} {
|
||||
if err := os.WriteFile(p, []byte(b), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
h := holdHashes(t)
|
||||
s := newStuckReads(t, dead)
|
||||
|
||||
mustFailStuck(t, dead, pushStarted(t, dead, rel, sum))
|
||||
stuckPast(t, dead)
|
||||
|
||||
// Asked for a digest in the dead folder, the hashing reads nothing there;
|
||||
// a folder that answers is hashed as usual.
|
||||
requestHash(other)
|
||||
requestHash(liveFile)
|
||||
awaitHash(t, liveFile, sha([]byte("answers")), "the hashing job did not read a folder that answers")
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
if n := h.readsIn(dead); n != 0 {
|
||||
t.Fatalf("the hashing read the folder whose push read is stuck %d times", n)
|
||||
}
|
||||
if n := s.blocked.Load(); n != 1 {
|
||||
t.Fatalf("%d reads are blocked on the dead folder; want the push's 1", n)
|
||||
}
|
||||
|
||||
// When the push read returns, the folder is hashed again.
|
||||
s.release()
|
||||
readsIdle(t)
|
||||
requestHash(other)
|
||||
awaitHash(t, other, sha([]byte("another backup")), "the folder was not hashed once its push read returned")
|
||||
}
|
||||
|
||||
func TestStuckHashingAndPushReadsShareTheCap(t *testing.T) {
|
||||
stubPublicDNS(t)
|
||||
shortPushStall(t)
|
||||
readsIdle(t)
|
||||
capReads(t, 2)
|
||||
hashDead, pushDead, live := t.TempDir(), t.TempDir(), t.TempDir()
|
||||
body := []byte(strings.Repeat("a backup ", 1000))
|
||||
rel, sum := pushFixture(t, hashDead, body)
|
||||
pushFixture(t, pushDead, body)
|
||||
pushFixture(t, live, body)
|
||||
liveOther := filepath.Join(live, "20260101T000000Z.tar.zst.age")
|
||||
if err := os.WriteFile(liveOther, []byte("answers"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h := holdHashes(t, hashDead)
|
||||
s := newStuckReads(t, pushDead)
|
||||
|
||||
// One folder's hashing read and another's push read stick: two reads,
|
||||
// the cap.
|
||||
requestHash(filepath.Join(hashDead, fixtureData))
|
||||
eventually(t, "the hashing job reads the dead folder", func() bool { return h.readsIn(hashDead) == 1 })
|
||||
mustFailStuck(t, pushDead, pushStarted(t, pushDead, rel, sum))
|
||||
stuckPast(t, hashDead)
|
||||
stuckPast(t, pushDead)
|
||||
|
||||
// A folder that answers gets neither a push nor a hashing read.
|
||||
for i := 0; i < 3; i++ {
|
||||
_, errOut, code := startPush(t, live, rel, sum)
|
||||
if code == 0 || !strings.Contains(errOut, "backup_push: 2 reads of backup folders have not returned") ||
|
||||
!strings.Contains(errOut, "no push starts on this machine until one does") {
|
||||
t.Fatalf("start %d with the cap held by a hashing read and a push read: %d %s", i, code, errOut)
|
||||
}
|
||||
}
|
||||
requestHash(liveOther)
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
if n := h.readsIn(live); n != 0 {
|
||||
t.Fatalf("the hashing read a folder %d times with the cap held", n)
|
||||
}
|
||||
if n, m := s.blocked.Load(), h.readsIn(hashDead); n != 1 || m != 1 {
|
||||
t.Fatalf("%d push reads and %d hashing reads were started on the dead folders; want 1 and 1", n, m)
|
||||
}
|
||||
backupReadMu.Lock()
|
||||
inFlight := readsInFlightLocked()
|
||||
backupReadMu.Unlock()
|
||||
if inFlight != 2 {
|
||||
t.Fatalf("%d reads are in flight; the cap is 2", inFlight)
|
||||
}
|
||||
|
||||
// Once one returns, the folder that answers is served.
|
||||
s.release()
|
||||
eventually(t, "the push read frees its slot", func() bool {
|
||||
backupReadMu.Lock()
|
||||
defer backupReadMu.Unlock()
|
||||
return readsInFlightLocked() == 1
|
||||
})
|
||||
requestHash(liveOther)
|
||||
awaitHash(t, liveOther, sha([]byte("answers")), "the hashing did not read a folder that answers once under the cap")
|
||||
id := pushStarted(t, live, rel, strings.Repeat("ab", 32))
|
||||
if st := awaitPush(t, live, id); st["state"] != "failed" || !strings.Contains(st["error"], "sha256 differs") {
|
||||
t.Fatalf("a push once under the cap: %v", st)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user