Privatae
75 lines
3.3 KiB
Go
75 lines
3.3 KiB
Go
package main
|
|
|
|
import (
|
|
"time"
|
|
|
|
"github.com/gorilla/websocket"
|
|
)
|
|
|
|
// The worker lanes (egress / mgmt / harness) are reverse-relay WebSockets the bastion
|
|
// opens TO the host and then PARKS on — blocked in ReadMessage until the host dispatches
|
|
// a job. The transport is the WireGuard tunnel through the host's pod-Hub. When that Hub's
|
|
// userspace netstack vanishes (a pod roll or Hub sidecar restart) the TCP session is
|
|
// severed with NO FIN/RST reaching us — so a parked ReadMessage blocks FOREVER and the
|
|
// lane stays dark until the process is restarted by hand. (The heartbeat and the
|
|
// per-request lanes self-heal because they dial per request; only these long-parked
|
|
// workers are affected.)
|
|
//
|
|
// armWSKeepalive makes that silently-dead tunnel observable. The bastion pings the host
|
|
// on an interval and holds a read deadline that every pong (or server-initiated ping)
|
|
// pushes forward. Lose the peer -> pongs stop -> the read deadline fires -> the parked
|
|
// ReadMessage returns an error -> the worker loop's reconnect kicks in. On a healthy lane
|
|
// the deadline never trips: the host's WebSocket layer answers our pings with pongs at the
|
|
// protocol level (below the app handler), so an idle-but-alive worker is refreshed every
|
|
// interval.
|
|
const (
|
|
wsPingInterval = 20 * time.Second
|
|
wsReadTimeout = 70 * time.Second // > 3 ping intervals: tolerate a couple of dropped pongs
|
|
wsWriteWait = 10 * time.Second
|
|
)
|
|
|
|
// armWSKeepalive installs the read deadline + pong/ping handlers and starts a background
|
|
// pinger for one worker connection. It MUST be called from the worker's own goroutine
|
|
// BEFORE that goroutine starts reading: SetReadDeadline / SetPongHandler / SetPingHandler
|
|
// are read-side state, and the returned stop() is the only other toucher of the ping
|
|
// goroutine. WriteControl (the ping) is safe to call concurrently with the worker's
|
|
// WriteMessage sends, per gorilla's concurrency contract. Call the returned stop() once
|
|
// the job finishes (normally) so the pinger exits; it is safe to call exactly once.
|
|
func armWSKeepalive(ws *websocket.Conn) (stop func()) {
|
|
return armWSKeepaliveEvery(ws, wsPingInterval, wsReadTimeout)
|
|
}
|
|
|
|
// armWSKeepaliveEvery is armWSKeepalive with injectable timings (tests use short ones).
|
|
func armWSKeepaliveEvery(ws *websocket.Conn, interval, timeout time.Duration) (stop func()) {
|
|
extend := func() { _ = ws.SetReadDeadline(time.Now().Add(timeout)) }
|
|
extend()
|
|
ws.SetPongHandler(func(string) error { extend(); return nil })
|
|
// A server-initiated ping is also proof of life: keep gorilla's default pong reply
|
|
// AND push the deadline (the host's server pings by default — belt and suspenders).
|
|
ws.SetPingHandler(func(appData string) error {
|
|
extend()
|
|
_ = ws.WriteControl(websocket.PongMessage, []byte(appData), time.Now().Add(wsWriteWait))
|
|
return nil
|
|
})
|
|
done := make(chan struct{})
|
|
go func() {
|
|
t := time.NewTicker(interval)
|
|
defer t.Stop()
|
|
for {
|
|
select {
|
|
case <-done:
|
|
return
|
|
case <-t.C:
|
|
if err := ws.WriteControl(websocket.PingMessage, nil, time.Now().Add(wsWriteWait)); err != nil {
|
|
// Write failed => the peer is gone and the buffer won't drain.
|
|
// Force-close so the parked reader unblocks immediately instead of
|
|
// waiting out the full read deadline.
|
|
_ = ws.Close()
|
|
return
|
|
}
|
|
}
|
|
}
|
|
}()
|
|
return func() { close(done) }
|
|
}
|