Privatae
137 lines
3.8 KiB
Go
137 lines
3.8 KiB
Go
package main
|
|
|
|
// pairing_retry.go — each pairing runs on its own.
|
|
//
|
|
// Until 0.63.0 one pairing's fatal error (a port it could not bind, a lane
|
|
// that would not come up, a bad certificate in its file) went to cmdRun and
|
|
// stopped the whole daemon, so every other Host on the machine went dark with
|
|
// it, and on Windows the service manager restarted all of them into the same
|
|
// failure every five seconds. Now that error stops that pairing alone. It is
|
|
// reported (the log, the machine's events, `status`), and the supervisor
|
|
// (run.go superviseOnce) starts it again by itself after a pause that doubles
|
|
// from supervisePeriod up to retryMaxPause, the other pairings serving all
|
|
// along. A run that stays up for retryResetAfter clears the record.
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
)
|
|
|
|
const (
|
|
retryMaxPause = 5 * time.Minute
|
|
retryResetAfter = 2 * time.Minute
|
|
)
|
|
|
|
// pairingFailure is one pairing's last fatal error, as status reads it.
|
|
type pairingFailure struct {
|
|
Error string `json:"error"`
|
|
At string `json:"at"`
|
|
NextTry string `json:"next_try"`
|
|
Tries int `json:"tries"`
|
|
}
|
|
|
|
var pairingRetry = struct {
|
|
sync.Mutex
|
|
next map[string]time.Time
|
|
tries map[string]int
|
|
}{next: map[string]time.Time{}, tries: map[string]int{}}
|
|
|
|
func pairingFailuresPath() (string, error) {
|
|
d, err := configDir()
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return filepath.Join(d, "pairing_failures.json"), nil
|
|
}
|
|
|
|
// loadPairingFailures is the record status reads; empty when there is none.
|
|
func loadPairingFailures() map[string]pairingFailure {
|
|
out := map[string]pairingFailure{}
|
|
p, err := pairingFailuresPath()
|
|
if err != nil {
|
|
return out
|
|
}
|
|
if raw, err := os.ReadFile(p); err == nil {
|
|
_ = json.Unmarshal(raw, &out)
|
|
}
|
|
return out
|
|
}
|
|
|
|
func savePairingFailuresLocked(m map[string]pairingFailure) {
|
|
p, err := pairingFailuresPath()
|
|
if err != nil {
|
|
return
|
|
}
|
|
if len(m) == 0 {
|
|
_ = os.Remove(p)
|
|
return
|
|
}
|
|
if b, err := json.MarshalIndent(m, "", " "); err == nil {
|
|
_ = atomicWrite(p, b)
|
|
}
|
|
}
|
|
|
|
// retryPause is the pause before try n+1 after n failures in a row.
|
|
func retryPause(n int) time.Duration {
|
|
d := supervisePeriod
|
|
for i := 1; i < n && d < retryMaxPause; i++ {
|
|
d *= 2
|
|
}
|
|
return min(d, retryMaxPause)
|
|
}
|
|
|
|
// pairingFailed records host's fatal error and when it is tried again, and
|
|
// says so. ranFor is how long the run had been up: one that served for
|
|
// retryResetAfter starts the count again.
|
|
func pairingFailed(host string, err error, ranFor time.Duration) {
|
|
pairingRetry.Lock()
|
|
defer pairingRetry.Unlock()
|
|
if ranFor >= retryResetAfter {
|
|
pairingRetry.tries[host] = 0
|
|
}
|
|
pairingRetry.tries[host]++
|
|
n := pairingRetry.tries[host]
|
|
now := time.Now()
|
|
next := now.Add(retryPause(n))
|
|
pairingRetry.next[host] = next
|
|
m := loadPairingFailures()
|
|
m[host] = pairingFailure{Error: err.Error(), At: now.UTC().Format(time.RFC3339),
|
|
NextTry: next.UTC().Format(time.RFC3339), Tries: n}
|
|
savePairingFailuresLocked(m)
|
|
why := strings.TrimRight(strings.TrimSpace(err.Error()), ".")
|
|
line := fmt.Sprintf("stopped: %s. The other Hosts go on being served; it is tried again by itself at %s.",
|
|
why, next.Local().Format("15:04:05"))
|
|
fmt.Printf("[%s] %s\n", host, line)
|
|
recordLocalEvent("pairing", host+" "+line)
|
|
}
|
|
|
|
// pairingRetryDue: host has no failure pending a pause, or its pause is over.
|
|
func pairingRetryDue(host string) bool {
|
|
pairingRetry.Lock()
|
|
defer pairingRetry.Unlock()
|
|
next, ok := pairingRetry.next[host]
|
|
return !ok || !time.Now().Before(next)
|
|
}
|
|
|
|
// pairingRecovered clears host's record: its run has stayed up, or it ended
|
|
// cleanly.
|
|
func pairingRecovered(host string) {
|
|
pairingRetry.Lock()
|
|
defer pairingRetry.Unlock()
|
|
if _, ok := pairingRetry.next[host]; !ok {
|
|
return
|
|
}
|
|
delete(pairingRetry.next, host)
|
|
delete(pairingRetry.tries, host)
|
|
m := loadPairingFailures()
|
|
if _, ok := m[host]; ok {
|
|
delete(m, host)
|
|
savePairingFailuresLocked(m)
|
|
}
|
|
}
|