bastion 0.63.1

client-go/pairing_retry.go

@@ -0,0 +1,136 @@
+package main
+
+// pairing_retry.go — each pairing runs on its own.
+//
+// Until 0.63.0 one pairing's fatal error (a port it could not bind, a lane
+// that would not come up, a bad certificate in its file) went to cmdRun and
+// stopped the whole daemon, so every other Host on the machine went dark with
+// it, and on Windows the service manager restarted all of them into the same
+// failure every five seconds. Now that error stops that pairing alone. It is
+// reported (the log, the machine's events, `status`), and the supervisor
+// (run.go superviseOnce) starts it again by itself after a pause that doubles
+// from supervisePeriod up to retryMaxPause, the other pairings serving all
+// along. A run that stays up for retryResetAfter clears the record.
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+)
+
+const (
+	retryMaxPause   = 5 * time.Minute
+	retryResetAfter = 2 * time.Minute
+)
+
+// pairingFailure is one pairing's last fatal error, as status reads it.
+type pairingFailure struct {
+	Error   string `json:"error"`
+	At      string `json:"at"`
+	NextTry string `json:"next_try"`
+	Tries   int    `json:"tries"`
+}
+
+var pairingRetry = struct {
+	sync.Mutex
+	next  map[string]time.Time
+	tries map[string]int
+}{next: map[string]time.Time{}, tries: map[string]int{}}
+
+func pairingFailuresPath() (string, error) {
+	d, err := configDir()
+	if err != nil {
+		return "", err
+	}
+	return filepath.Join(d, "pairing_failures.json"), nil
+}
+
+// loadPairingFailures is the record status reads; empty when there is none.
+func loadPairingFailures() map[string]pairingFailure {
+	out := map[string]pairingFailure{}
+	p, err := pairingFailuresPath()
+	if err != nil {
+		return out
+	}
+	if raw, err := os.ReadFile(p); err == nil {
+		_ = json.Unmarshal(raw, &out)
+	}
+	return out
+}
+
+func savePairingFailuresLocked(m map[string]pairingFailure) {
+	p, err := pairingFailuresPath()
+	if err != nil {
+		return
+	}
+	if len(m) == 0 {
+		_ = os.Remove(p)
+		return
+	}
+	if b, err := json.MarshalIndent(m, "", "  "); err == nil {
+		_ = atomicWrite(p, b)
+	}
+}
+
+// retryPause is the pause before try n+1 after n failures in a row.
+func retryPause(n int) time.Duration {
+	d := supervisePeriod
+	for i := 1; i < n && d < retryMaxPause; i++ {
+		d *= 2
+	}
+	return min(d, retryMaxPause)
+}
+
+// pairingFailed records host's fatal error and when it is tried again, and
+// says so. ranFor is how long the run had been up: one that served for
+// retryResetAfter starts the count again.
+func pairingFailed(host string, err error, ranFor time.Duration) {
+	pairingRetry.Lock()
+	defer pairingRetry.Unlock()
+	if ranFor >= retryResetAfter {
+		pairingRetry.tries[host] = 0
+	}
+	pairingRetry.tries[host]++
+	n := pairingRetry.tries[host]
+	now := time.Now()
+	next := now.Add(retryPause(n))
+	pairingRetry.next[host] = next
+	m := loadPairingFailures()
+	m[host] = pairingFailure{Error: err.Error(), At: now.UTC().Format(time.RFC3339),
+		NextTry: next.UTC().Format(time.RFC3339), Tries: n}
+	savePairingFailuresLocked(m)
+	why := strings.TrimRight(strings.TrimSpace(err.Error()), ".")
+	line := fmt.Sprintf("stopped: %s. The other Hosts go on being served; it is tried again by itself at %s.",
+		why, next.Local().Format("15:04:05"))
+	fmt.Printf("[%s] %s\n", host, line)
+	recordLocalEvent("pairing", host+" "+line)
+}
+
+// pairingRetryDue: host has no failure pending a pause, or its pause is over.
+func pairingRetryDue(host string) bool {
+	pairingRetry.Lock()
+	defer pairingRetry.Unlock()
+	next, ok := pairingRetry.next[host]
+	return !ok || !time.Now().Before(next)
+}
+
+// pairingRecovered clears host's record: its run has stayed up, or it ended
+// cleanly.
+func pairingRecovered(host string) {
+	pairingRetry.Lock()
+	defer pairingRetry.Unlock()
+	if _, ok := pairingRetry.next[host]; !ok {
+		return
+	}
+	delete(pairingRetry.next, host)
+	delete(pairingRetry.tries, host)
+	m := loadPairingFailures()
+	if _, ok := m[host]; ok {
+		delete(m, host)
+		savePairingFailuresLocked(m)
+	}
+}