bastion 0.56.0
This commit is contained in:
+64
-6
@@ -512,6 +512,30 @@ func resolveLLMIPs() map[string]string {
|
||||
// to the anchor host, because approving is the anchor's job. "" = no pairing
|
||||
// identity (tests, tools): mode, anchor and unapproved attempts still travel.
|
||||
func buildHeartbeatBody(self, hostname, platform, version, wgPub, fingerprint, tunnelIP, endpoint string,
|
||||
savePending bool, mirror *mirrorReport, enforcing map[string]any, peers []peerAnnounce) string {
|
||||
return buildHeartbeatBodyOn(beatLanes{}, self, hostname, platform, version, wgPub, fingerprint, tunnelIP, endpoint,
|
||||
savePending, mirror, enforcing, peers)
|
||||
}
|
||||
|
||||
// beatLanes is which of a pairing's lanes run, so its beat advertises those
|
||||
// and no others. The zero value is a tunnel pairing: the forwards listen on
|
||||
// the tunnel's stack and the service slots are reachable there.
|
||||
type beatLanes struct {
|
||||
// local: a Host with no platform (run.go runUnixPairing). The forwards and
|
||||
// the service slots are absent; the model servers ride the serve lane
|
||||
// (servelane.go), and the beat says which in lane_serves.
|
||||
local bool
|
||||
// noCoders: a tcp pairing, whose coder runs have no control plane to use
|
||||
// (pair_tcp.go coderTransportRefusal), so it is never offered one.
|
||||
noCoders bool
|
||||
}
|
||||
|
||||
func beatLanesFor(cfg *Config) beatLanes {
|
||||
return beatLanes{local: cfg.localTransport(), noCoders: cfg.tcpTransport()}
|
||||
}
|
||||
|
||||
// buildHeartbeatBodyOn is buildHeartbeatBody for a pairing whose lanes are lanes.
|
||||
func buildHeartbeatBodyOn(lanes beatLanes, self, hostname, platform, version, wgPub, fingerprint, tunnelIP, endpoint string,
|
||||
savePending bool, mirror *mirrorReport, enforcing map[string]any, peers []peerAnnounce) string {
|
||||
caps := []string{}
|
||||
// WHY SOME CAPABILITIES ARE WITHHELD HERE AND OTHERS ARE NOT — the asymmetry
|
||||
@@ -549,7 +573,17 @@ func buildHeartbeatBody(self, hostname, platform, version, wgPub, fingerprint, t
|
||||
// gets exactly what it got before: scope.filterModels is the identity then.
|
||||
scope := pairingModelScope(self)
|
||||
reachable, models := false, []string(nil)
|
||||
if inference {
|
||||
// A local pairing has no forward to advertise: its model servers ride the
|
||||
// serve lane, and each lane below is advertised only when lane_serves
|
||||
// carries its service (the target set and this machine's, the server
|
||||
// answering, the family held). Its probes are of the address the lane
|
||||
// dials, never the tunnel's guesses.
|
||||
var served laneServeReport
|
||||
if lanes.local {
|
||||
served = laneServes(self)
|
||||
reachable = containsStr(served.served, "ollama")
|
||||
models = served.models
|
||||
} else if inference {
|
||||
reachable, models = ollamaModels(tunnelIP)
|
||||
models = scope.filterModels(models)
|
||||
// Reachable but nothing shared: advertising `embed` would put this box
|
||||
@@ -572,7 +606,15 @@ func buildHeartbeatBody(self, hostname, platform, version, wgPub, fingerprint, t
|
||||
// box running both.
|
||||
llmModels := []string{}
|
||||
llmPort := llmForwardPort
|
||||
if !inference {
|
||||
if lanes.local {
|
||||
// No port listens for it here: the Host reaches the server through the
|
||||
// serve lane, so llm_port is the "no port" value.
|
||||
llmPort = 0
|
||||
if containsStr(served.served, "llm") {
|
||||
caps = append(caps, "llm-openai")
|
||||
llmModels = served.llmModels
|
||||
}
|
||||
} else if !inference {
|
||||
// The host stores 0 as NULL; with llm-openai withheld and
|
||||
// llm_models empty nothing on the host resolves a port for this row.
|
||||
llmPort = 0
|
||||
@@ -598,7 +640,9 @@ func buildHeartbeatBody(self, hostname, platform, version, wgPub, fingerprint, t
|
||||
// probe: an up forwarder with a dead upstream is a distinct state the
|
||||
// host wants before committing a fail-closed write path to it.
|
||||
piiAdvertised, piiOK := false, false
|
||||
if inference && scope.allowsPII() {
|
||||
if lanes.local {
|
||||
piiAdvertised, piiOK = containsStr(served.served, "pii"), served.piiOK
|
||||
} else if inference && scope.allowsPII() {
|
||||
piiAdvertised, piiOK = piiProbe()
|
||||
}
|
||||
if piiAdvertised {
|
||||
@@ -638,7 +682,7 @@ func buildHeartbeatBody(self, hostname, platform, version, wgPub, fingerprint, t
|
||||
// "build a server on this machine" on it, and an offer the box would refuse
|
||||
// is a control that fails after the user chose it. The slot list rides with
|
||||
// it, withheld (absent, not empty) from a pairing outside the family.
|
||||
servicesOn := servicesCapable(self)
|
||||
servicesOn := !lanes.local && servicesCapable(self)
|
||||
if servicesOn {
|
||||
caps = append(caps, serviceCapability)
|
||||
}
|
||||
@@ -646,7 +690,10 @@ func buildHeartbeatBody(self, hostname, platform, version, wgPub, fingerprint, t
|
||||
// only when this machine would accept a run-up from this pairing — Linux,
|
||||
// root, coders on, a runtime, the family, and (anchored) a standing coder
|
||||
// grant. Absent, never empty, otherwise.
|
||||
coders := coderHeartbeat(self)
|
||||
var coders map[string]any
|
||||
if !lanes.noCoders {
|
||||
coders = coderHeartbeat(self)
|
||||
}
|
||||
if coders != nil {
|
||||
caps = append(caps, coderCapability)
|
||||
}
|
||||
@@ -774,6 +821,13 @@ func buildHeartbeatBody(self, hostname, platform, version, wgPub, fingerprint, t
|
||||
// absence as a client too old to say.
|
||||
"endpoint_save_pending": savePending,
|
||||
}
|
||||
// What this pairing serves over the serve lane (servelane.go), on a local
|
||||
// pairing only: ["ollama","llm","pii"] or fewer, [] when nothing is. Absent
|
||||
// on a tunnel pairing, which has the forwards instead, and from a bastion
|
||||
// that predates the lane.
|
||||
if lanes.local {
|
||||
body["lane_serves"] = served.served
|
||||
}
|
||||
// The service slots this pairing may reach, with the tunnel port for each
|
||||
// (service.go). Absent when the capability is withheld, never empty-for-no.
|
||||
if servicesOn {
|
||||
@@ -1004,7 +1058,11 @@ func buildHeartbeatBody(self, hostname, platform, version, wgPub, fingerprint, t
|
||||
}
|
||||
}
|
||||
if piiAdvertised {
|
||||
body["pii_port"] = piiForwardPort
|
||||
// pii_port is the tunnel's port; a local pairing has none and reaches
|
||||
// the sterilizer through the serve lane.
|
||||
if !lanes.local {
|
||||
body["pii_port"] = piiForwardPort
|
||||
}
|
||||
body["pii_ok"] = piiOK
|
||||
}
|
||||
if faAdvertised {
|
||||
|
||||
+63
-32
@@ -23,6 +23,7 @@ package main
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"io"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -150,17 +151,32 @@ func coderVettedAddrs(ctx context.Context, hostport string, allowLAN []string) (
|
||||
}
|
||||
|
||||
func coderRefuse(c net.Conn, why string) {
|
||||
_, _ = c.Write([]byte("HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\nConnection: close\r\nX-Privatae-Refused: " +
|
||||
strings.NewReplacer("\r", " ", "\n", " ").Replace(why) + "\r\n\r\n"))
|
||||
writeEgressRefusal(c, "403 Forbidden", why, why)
|
||||
}
|
||||
|
||||
// handleCoderEgressConn serves one connection from a run: HTTP CONNECT or an
|
||||
// absolute-form plain HTTP request, the same parsing as handleEgressConn.
|
||||
// Destinations are never logged; refusals are counted on the run.
|
||||
// coderEgressMaxHeadBytes bounds a request's line and headers together.
|
||||
const coderEgressMaxHeadBytes = 64 << 10
|
||||
|
||||
// coderEgressMaxConns bounds the connections one run's proxy carries at once.
|
||||
// A whole Chromium profile opens at most 256 sockets at a time (its socket
|
||||
// pool limit), and package managers far fewer, so four times the browser's
|
||||
// figure never refuses real work; past it a connection is answered 503 and
|
||||
// closed, and each one held costs this machine at most its head and two copy
|
||||
// buffers.
|
||||
const coderEgressMaxConns = 1024
|
||||
|
||||
func handleCoderEgressConn(client net.Conn, allowLAN []string, refused *int64) {
|
||||
defer client.Close()
|
||||
_ = client.SetReadDeadline(time.Now().Add(30 * time.Second))
|
||||
br := bufio.NewReader(client)
|
||||
// The request line and headers are read under one bound, the head limit
|
||||
// below has always named: the run's own processes send them, and a line
|
||||
// that never ends must not grow in this machine's memory before that
|
||||
// limit is looked at.
|
||||
headLim := &io.LimitedReader{R: client, N: coderEgressMaxHeadBytes}
|
||||
br := bufio.NewReader(headLim)
|
||||
line, err := br.ReadString('\n')
|
||||
if err != nil {
|
||||
return
|
||||
@@ -172,7 +188,6 @@ func handleCoderEgressConn(client net.Conn, allowLAN []string, refused *int64) {
|
||||
method, target := parts[0], parts[1]
|
||||
var head strings.Builder
|
||||
head.WriteString(line)
|
||||
hostHdr := ""
|
||||
for {
|
||||
h, err := br.ReadString('\n')
|
||||
if err != nil {
|
||||
@@ -182,38 +197,41 @@ func handleCoderEgressConn(client net.Conn, allowLAN []string, refused *int64) {
|
||||
if h == "\r\n" || h == "\n" {
|
||||
break
|
||||
}
|
||||
if head.Len() > 64<<10 {
|
||||
if head.Len() > coderEgressMaxHeadBytes {
|
||||
return
|
||||
}
|
||||
if len(h) >= 5 && strings.EqualFold(h[:5], "host:") {
|
||||
hostHdr = strings.TrimSpace(h[5:])
|
||||
}
|
||||
}
|
||||
_ = client.SetReadDeadline(time.Time{})
|
||||
// The head is in; what follows is the stream itself. What the reader
|
||||
// already buffered past the head stays first in line.
|
||||
headLim.N = 1<<63 - 1
|
||||
ctx, cancel := context.WithTimeout(context.Background(), egressDialTimeout)
|
||||
defer cancel()
|
||||
|
||||
// Read the way the bastion's own egress lane reads it (egress_http.go): a
|
||||
// CONNECT target through parseEgressAuthority, and plain HTTP as ONE
|
||||
// request, rebuilt and sent with `Connection: close`, so the run's later
|
||||
// requests on a keep-alive connection never reach the destination unvetted.
|
||||
connect := strings.EqualFold(method, "CONNECT")
|
||||
hp := ""
|
||||
var req egressRequest
|
||||
if connect {
|
||||
hp = ensurePort(target, "443")
|
||||
} else {
|
||||
host := hostHdr
|
||||
if i := strings.Index(target, "://"); i >= 0 {
|
||||
rest := target[i+3:]
|
||||
if s := strings.IndexAny(rest, "/?#"); s >= 0 {
|
||||
host = rest[:s]
|
||||
} else {
|
||||
host = rest
|
||||
}
|
||||
if at := strings.LastIndexByte(host, '@'); at >= 0 {
|
||||
host = host[at+1:]
|
||||
}
|
||||
}
|
||||
if host == "" {
|
||||
host, port, ok := parseEgressAuthority(target, 443)
|
||||
if !ok {
|
||||
atomic.AddInt64(refused, 1)
|
||||
writeEgressRefusal(client, "400 Bad Request", egressNoHost, egressNoHost)
|
||||
return
|
||||
}
|
||||
hp = ensurePort(host, "80")
|
||||
hp = egressHostPort(host, port)
|
||||
} else {
|
||||
r, bad := parseEgressRequest(head.String())
|
||||
if bad != nil {
|
||||
atomic.AddInt64(refused, 1)
|
||||
writeEgressRefusal(client, bad.status, bad.body, bad.body)
|
||||
return
|
||||
}
|
||||
req = r
|
||||
hp = egressHostPort(req.host, req.port)
|
||||
}
|
||||
dialAddrs, why := coderVettedAddrs(ctx, hp, allowLAN)
|
||||
if why != "" {
|
||||
@@ -223,15 +241,14 @@ func handleCoderEgressConn(client net.Conn, allowLAN []string, refused *int64) {
|
||||
}
|
||||
upstream, err := dialEach(ctx, coderDialOut, dialAddrs)
|
||||
if err != nil {
|
||||
_, _ = client.Write([]byte("HTTP/1.1 502 Bad Gateway\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"))
|
||||
writeEgressRefusal(client, "502 Bad Gateway", egressUnreachable, "")
|
||||
return
|
||||
}
|
||||
if connect {
|
||||
if _, err := client.Write([]byte("HTTP/1.1 200 Connection established\r\n\r\n")); err != nil {
|
||||
_ = upstream.Close()
|
||||
return
|
||||
}
|
||||
} else if _, err := upstream.Write([]byte(head.String())); err != nil {
|
||||
if !connect {
|
||||
oneEgressExchange(br, client, upstream, req)
|
||||
return
|
||||
}
|
||||
if _, err := client.Write([]byte("HTTP/1.1 200 Connection established\r\n\r\n")); err != nil {
|
||||
_ = upstream.Close()
|
||||
return
|
||||
}
|
||||
@@ -277,6 +294,7 @@ func coderListenEgress(runID, dir string, uid int) error {
|
||||
}
|
||||
run := &coderEgressRun{ln: ln}
|
||||
coderEgress[runID] = run
|
||||
slots := make(chan struct{}, coderEgressMaxConns)
|
||||
go func() {
|
||||
for {
|
||||
c, err := ln.Accept()
|
||||
@@ -285,7 +303,20 @@ func coderListenEgress(runID, dir string, uid int) error {
|
||||
}
|
||||
// allow_lan is read per connection: the owner's edit applies to
|
||||
// running runs at once.
|
||||
go handleCoderEgressConn(c, loadCoderConfig().AllowLAN, &run.refused)
|
||||
select {
|
||||
case slots <- struct{}{}:
|
||||
go func() {
|
||||
defer func() { <-slots }()
|
||||
handleCoderEgressConn(c, loadCoderConfig().AllowLAN, &run.refused)
|
||||
}()
|
||||
default:
|
||||
go func() {
|
||||
defer c.Close()
|
||||
_ = c.SetWriteDeadline(time.Now().Add(time.Second))
|
||||
why := "this run already has " + strconv.Itoa(coderEgressMaxConns) + " connections open"
|
||||
writeEgressRefusal(c, "503 Service Unavailable", why, why)
|
||||
}()
|
||||
}
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
|
||||
@@ -474,8 +474,14 @@ func TestCoderEgressProxyOverTheSocket(t *testing.T) {
|
||||
} {
|
||||
head, c := ask(req)
|
||||
c.Close()
|
||||
if !strings.HasPrefix(head, "HTTP/1.1 403") || !strings.Contains(head, "X-Privatae-Refused:") {
|
||||
t.Fatalf("must be refused with 403 + X-Privatae-Refused: %q -> %q", req, head)
|
||||
want := "HTTP/1.1 403"
|
||||
if strings.Contains(req, "user@") {
|
||||
// Userinfo is refused outright, before any lookup, as the Host's
|
||||
// egress gate refuses it (parseEgressAuthority); still counted.
|
||||
want = "HTTP/1.1 400"
|
||||
}
|
||||
if !strings.HasPrefix(head, want) || !strings.Contains(head, "X-Privatae-Refused:") {
|
||||
t.Fatalf("must be refused with %s + X-Privatae-Refused: %q -> %q", want, req, head)
|
||||
}
|
||||
}
|
||||
head, c := ask("CONNECT public.example:443 HTTP/1.1\r\n\r\n")
|
||||
|
||||
+82
-10
@@ -43,9 +43,11 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -126,18 +128,81 @@ func startContextSock(done <-chan struct{}) {
|
||||
_ = ln.Close()
|
||||
_ = os.Remove(p)
|
||||
}()
|
||||
go func() {
|
||||
for {
|
||||
c, err := ln.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
go contextSockServe(c)
|
||||
}
|
||||
}()
|
||||
go contextSockAccept(ln)
|
||||
uilogf("context socket listening on %s\n", p)
|
||||
}
|
||||
|
||||
// The socket is 0666 and the daemon runs as root (or SYSTEM), so every user on
|
||||
// the machine can make it read. These bound what they can make it hold: an ask
|
||||
// is a folder path and a display name, far under contextAskMaxBytes, and no
|
||||
// more than contextSockMaxInFlight asks are read at once. Past that a
|
||||
// connection is told the daemon is busy and closed; the asker tries again.
|
||||
//
|
||||
// One person may hold at most contextSockMaxPerUser of those slots, keyed on
|
||||
// the identity the kernel gives (never on anything the asker says): without
|
||||
// it, one user opening contextSockMaxInFlight idle connections and dialling
|
||||
// again as each deadline ran out kept every other user's asks "busy" for as
|
||||
// long as they liked. A person adds a folder at a time; four at once is far
|
||||
// past that.
|
||||
const (
|
||||
contextAskMaxBytes = 64 << 10
|
||||
contextSockMaxInFlight = 32
|
||||
)
|
||||
|
||||
// contextSockMaxPerUser is a var only so the test of the overall cap can lift
|
||||
// it (every test connection is the same user).
|
||||
var contextSockMaxPerUser = 4
|
||||
|
||||
var contextSockUsers = struct {
|
||||
sync.Mutex
|
||||
held map[string]int
|
||||
}{held: map[string]int{}}
|
||||
|
||||
// contextSockTakeUser takes one of user's slots, or reports there is none.
|
||||
func contextSockTakeUser(user string) bool {
|
||||
contextSockUsers.Lock()
|
||||
defer contextSockUsers.Unlock()
|
||||
if contextSockUsers.held[user] >= contextSockMaxPerUser {
|
||||
return false
|
||||
}
|
||||
contextSockUsers.held[user]++
|
||||
return true
|
||||
}
|
||||
|
||||
func contextSockReleaseUser(user string) {
|
||||
contextSockUsers.Lock()
|
||||
defer contextSockUsers.Unlock()
|
||||
if contextSockUsers.held[user]--; contextSockUsers.held[user] <= 0 {
|
||||
delete(contextSockUsers.held, user)
|
||||
}
|
||||
}
|
||||
|
||||
// contextSockAccept serves ln until it is closed, at most
|
||||
// contextSockMaxInFlight connections at a time.
|
||||
func contextSockAccept(ln net.Listener) {
|
||||
slots := make(chan struct{}, contextSockMaxInFlight)
|
||||
for {
|
||||
c, err := ln.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
select {
|
||||
case slots <- struct{}{}:
|
||||
go func() {
|
||||
defer func() { <-slots }()
|
||||
contextSockServe(c)
|
||||
}()
|
||||
default:
|
||||
go func() {
|
||||
defer c.Close()
|
||||
_ = c.SetDeadline(time.Now().Add(time.Second))
|
||||
_ = json.NewEncoder(c).Encode(contextAskReply{Why: "the bastion is busy " +
|
||||
"answering other asks on this machine; try again in a moment"})
|
||||
}()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func contextSockServe(c net.Conn) {
|
||||
defer c.Close()
|
||||
_ = c.SetDeadline(time.Now().Add(30 * time.Second))
|
||||
@@ -154,8 +219,15 @@ func contextSockServe(c net.Conn) {
|
||||
return
|
||||
}
|
||||
|
||||
if !contextSockTakeUser(user) {
|
||||
_ = json.NewEncoder(c).Encode(contextAskReply{User: user, Why: "the bastion is busy " +
|
||||
"answering other asks of yours; try again in a moment"})
|
||||
return
|
||||
}
|
||||
defer contextSockReleaseUser(user)
|
||||
|
||||
var req contextAskRequest
|
||||
if err := json.NewDecoder(c).Decode(&req); err != nil {
|
||||
if err := json.NewDecoder(io.LimitReader(c, contextAskMaxBytes)).Decode(&req); err != nil {
|
||||
_ = json.NewEncoder(c).Encode(contextAskReply{User: user, Why: "the request could not be read"})
|
||||
return
|
||||
}
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
package main
|
||||
|
||||
// The context socket is 0666: every user on the machine may connect, and the
|
||||
// daemon behind it runs as root (or SYSTEM). So what one connection can make
|
||||
// the daemon hold must be bounded by the daemon, not by the asker.
|
||||
|
||||
import (
|
||||
"net"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// An ask is a folder path and a display name. A client that streams a request
|
||||
// with no end gets the daemon to read only a bounded amount of it, then a
|
||||
// reply and a close; before, the decoder kept every byte until the 30 s
|
||||
// deadline, and nothing capped how many such connections were open.
|
||||
func TestAContextAskWithNoEndIsReadOnlyToABound(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("no unix-socket peer credentials on windows; contextsock_windows.go says why")
|
||||
}
|
||||
sock := filepath.Join(t.TempDir(), "s.sock")
|
||||
ln, err := net.Listen("unix", sock)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer ln.Close()
|
||||
go func() {
|
||||
c, err := ln.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
contextSockServe(c)
|
||||
}()
|
||||
|
||||
c, err := net.Dial("unix", sock)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer c.Close()
|
||||
const total = 16 << 20
|
||||
chunk := []byte(strings.Repeat("a", 64<<10))
|
||||
sent := 0
|
||||
if _, err := c.Write([]byte(`{"root":"`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for sent < total {
|
||||
_ = c.SetWriteDeadline(time.Now().Add(2 * time.Second))
|
||||
n, err := c.Write(chunk)
|
||||
sent += n
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
// The kernel's socket buffers hold a few hundred KiB the daemon never
|
||||
// read; anything near the whole stream means the daemon read it all.
|
||||
if sent >= 4<<20 {
|
||||
t.Fatalf("the daemon accepted %d MiB of one ask with no end; it must stop at a bound", sent>>20)
|
||||
}
|
||||
}
|
||||
|
||||
// Asks in flight are capped: a connection past the cap is closed at once, so
|
||||
// many idle or endless connections cannot multiply the bound above.
|
||||
func TestContextAsksInFlightAreCapped(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("no unix-socket peer credentials on windows; contextsock_windows.go says why")
|
||||
}
|
||||
sock := filepath.Join(t.TempDir(), "s.sock")
|
||||
ln, err := net.Listen("unix", sock)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer ln.Close()
|
||||
// Every connection here is the same user; lift the per-user share so the
|
||||
// overall cap is what this test meets (TestOneUserCannotHoldEveryContextSockSlot
|
||||
// holds the share).
|
||||
oldShare := contextSockMaxPerUser
|
||||
contextSockMaxPerUser = contextSockMaxInFlight
|
||||
t.Cleanup(func() { contextSockMaxPerUser = oldShare })
|
||||
go contextSockAccept(ln)
|
||||
|
||||
var held []net.Conn
|
||||
defer func() {
|
||||
for _, h := range held {
|
||||
_ = h.Close()
|
||||
}
|
||||
}()
|
||||
// Each of these sends nothing and holds its slot until its deadline.
|
||||
for i := 0; i < contextSockMaxInFlight; i++ {
|
||||
h, err := net.Dial("unix", sock)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
held = append(held, h)
|
||||
}
|
||||
// Give the daemon's accept loop time to take every held connection.
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
|
||||
extra, err := net.Dial("unix", sock)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer extra.Close()
|
||||
_ = extra.SetReadDeadline(time.Now().Add(2 * time.Second))
|
||||
buf := make([]byte, 512)
|
||||
n, rerr := extra.Read(buf)
|
||||
if ne, ok := rerr.(net.Error); ok && ne.Timeout() {
|
||||
t.Fatalf("a connection past the cap of %d was left open; it must be refused", contextSockMaxInFlight)
|
||||
}
|
||||
if n > 0 && !strings.Contains(string(buf[:n]), "busy") {
|
||||
t.Fatalf("a connection past the cap got %q; it must be told the daemon is busy", buf[:n])
|
||||
}
|
||||
|
||||
// Once a held connection goes, a new ask gets a slot again.
|
||||
_ = held[0].Close()
|
||||
held = held[1:]
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
again, err := net.Dial("unix", sock)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer again.Close()
|
||||
_ = again.SetReadDeadline(time.Now().Add(500 * time.Millisecond))
|
||||
n, rerr = again.Read(buf)
|
||||
if ne, ok := rerr.(net.Error); !ok || !ne.Timeout() {
|
||||
t.Fatalf("after a slot was freed, a new ask was not served (read %q, %v)", buf[:n], rerr)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"net"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// One person cannot take every slot: past their own share they are told the
|
||||
// daemon is busy at once, while slots remain for everyone else. Before, one
|
||||
// user's idle connections, re-dialled as each deadline ran out, held all of
|
||||
// them and every other user's ask was refused.
|
||||
func TestOneUserCannotHoldEveryContextSockSlot(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("no unix-socket peer credentials on windows; contextsock_windows.go says why")
|
||||
}
|
||||
const share = 4 // contextSockMaxPerUser
|
||||
// A test before this one may still be releasing its slots.
|
||||
if n := contextSockSlotsHeld(t, 0); n != 0 {
|
||||
t.Fatalf("%d slots still held from an earlier test", n)
|
||||
}
|
||||
sock := filepath.Join(t.TempDir(), "s.sock")
|
||||
ln, err := net.Listen("unix", sock)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer ln.Close()
|
||||
go contextSockAccept(ln)
|
||||
|
||||
var held []net.Conn
|
||||
defer func() {
|
||||
for _, h := range held {
|
||||
_ = h.Close()
|
||||
}
|
||||
}()
|
||||
for i := 0; i < share; i++ {
|
||||
h, err := net.Dial("unix", sock)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
held = append(held, h)
|
||||
}
|
||||
// Wait until the daemon has given those four their slots (a fixed sleep
|
||||
// lost this race on a loaded machine and handed the 5th one first).
|
||||
if n := contextSockSlotsHeld(t, share); n != share {
|
||||
t.Fatalf("the daemon gave %d slots to %d idle connections of one user", n, share)
|
||||
}
|
||||
|
||||
extra, err := net.Dial("unix", sock)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer extra.Close()
|
||||
_ = extra.SetReadDeadline(time.Now().Add(2 * time.Second))
|
||||
buf := make([]byte, 512)
|
||||
n, rerr := extra.Read(buf)
|
||||
if ne, ok := rerr.(net.Error); ok && ne.Timeout() {
|
||||
t.Fatalf("one user held %d idle connections and a %dth was still given a slot: "+
|
||||
"one user can take all %d", share, share+1, contextSockMaxInFlight)
|
||||
}
|
||||
if !strings.Contains(string(buf[:n]), "busy") {
|
||||
t.Fatalf("past one user's share the ask got %q; it must be told the daemon is busy", buf[:n])
|
||||
}
|
||||
}
|
||||
|
||||
// contextSockSlotsHeld waits up to 10 s for the per-user slots held across
|
||||
// all users to reach want, and returns the count it last saw.
|
||||
func contextSockSlotsHeld(t *testing.T, want int) int {
|
||||
t.Helper()
|
||||
n := -1
|
||||
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(10 * time.Millisecond) {
|
||||
contextSockUsers.Lock()
|
||||
n = 0
|
||||
for _, k := range contextSockUsers.held {
|
||||
n += k
|
||||
}
|
||||
contextSockUsers.Unlock()
|
||||
if n == want {
|
||||
break
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
+68
-7
@@ -26,6 +26,7 @@ import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
@@ -74,6 +75,10 @@ type dcnGrantRec struct {
|
||||
// the signed bytes (a public key; the terms stay exactly what was
|
||||
// signed). The managed local agent seals the store's catalog to it.
|
||||
SealRecipient string `json:"seal_recipient,omitempty"`
|
||||
// RootReal is where the granted folder led when this box installed the
|
||||
// grant: its real path, pinned here (dcn_store.go), outside the signed
|
||||
// bytes. An op whose root leads anywhere else is refused.
|
||||
RootReal string `json:"root_real,omitempty"`
|
||||
}
|
||||
|
||||
// dcnTerms is parsed FROM the verified canonical bytes — never from anything else.
|
||||
@@ -524,6 +529,17 @@ func handleDcnGrantSync(ws *websocket.Conn, dir string, args []string) {
|
||||
dcnRefuse(ws, dir, "grant_sync", payload.StoreID, "", "grant does not verify: "+err.Error())
|
||||
return
|
||||
}
|
||||
// The folder this grant names is pinned as it resolves now; a re-sync or a
|
||||
// renewal of the same root keeps the pin it already has (dcn_store.go).
|
||||
if prev, held := f.Stores[fmt.Sprint(payload.StoreID)]; held {
|
||||
prevRoot := ""
|
||||
if pt, perr := dcnVerifyRec(f.VerifyPubkey, prev); perr == nil {
|
||||
prevRoot = pt.Root
|
||||
}
|
||||
rec.RootReal = dcnPinFor(&prev, prevRoot, terms.Root)
|
||||
} else {
|
||||
rec.RootReal = dcnPinFor(nil, "", terms.Root)
|
||||
}
|
||||
if hn, _ := os.Hostname(); hn != "" && terms.Server != "" && terms.Server != hn {
|
||||
dcnRefuse(ws, dir, "grant_sync", payload.StoreID, "",
|
||||
fmt.Sprintf("grant is for server '%s', this box is '%s'", terms.Server, hn))
|
||||
@@ -658,15 +674,26 @@ func handleDcnList(ws *websocket.Conn, dir string, args []string) {
|
||||
if terms == nil {
|
||||
return
|
||||
}
|
||||
base := terms.Root
|
||||
base := "."
|
||||
if a.Prefix != "" {
|
||||
p, ok := dcnResolvePath(terms.Root, a.Prefix)
|
||||
if !ok {
|
||||
dcnRefuse(ws, dir, "list", storeID, a.Prefix, "prefix escapes the store root")
|
||||
return
|
||||
}
|
||||
base = p
|
||||
rel, err := dcnRel(terms.Root, p)
|
||||
if err != nil {
|
||||
dcnRefuse(ws, dir, "list", storeID, a.Prefix, "prefix escapes the store root")
|
||||
return
|
||||
}
|
||||
base = filepath.ToSlash(rel)
|
||||
}
|
||||
r, err := dcnOpenStore(dir, storeID, terms)
|
||||
if err != nil {
|
||||
dcnRefuse(ws, dir, "list", storeID, a.Prefix, err.Error())
|
||||
return
|
||||
}
|
||||
defer r.Close()
|
||||
limit := a.Limit
|
||||
if limit <= 0 || limit > dcnMaxListEntries {
|
||||
limit = dcnMaxListEntries
|
||||
@@ -678,12 +705,14 @@ func handleDcnList(ws *websocket.Conn, dir string, args []string) {
|
||||
}
|
||||
var files []entry
|
||||
truncated := false
|
||||
_ = filepath.WalkDir(base, func(p string, d os.DirEntry, err error) error {
|
||||
// Walked through the store's handle: a link that leads out of it is not
|
||||
// followed, whatever the names do during the walk.
|
||||
_ = fs.WalkDir(r.FS(), base, func(p string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return nil // unreadable subtree: skip, never abort the listing
|
||||
}
|
||||
rel, rerr := filepath.Rel(terms.Root, p)
|
||||
if rerr != nil || rel == "." {
|
||||
rel := filepath.FromSlash(p)
|
||||
if rel == "." {
|
||||
return nil
|
||||
}
|
||||
if dcnExcluded(terms.Excludes, rel) {
|
||||
@@ -727,8 +756,23 @@ func handleDcnStat(ws *websocket.Conn, dir string, args []string) {
|
||||
dcnRefuse(ws, dir, "stat", storeID, a.Path, "path escapes the store root")
|
||||
return
|
||||
}
|
||||
info, err := os.Stat(abs)
|
||||
rel, err := dcnRel(terms.Root, abs)
|
||||
if err != nil {
|
||||
dcnRefuse(ws, dir, "stat", storeID, a.Path, err.Error())
|
||||
return
|
||||
}
|
||||
r, err := dcnOpenStore(dir, storeID, terms)
|
||||
if err != nil {
|
||||
dcnRefuse(ws, dir, "stat", storeID, a.Path, err.Error())
|
||||
return
|
||||
}
|
||||
defer r.Close()
|
||||
info, err := r.Stat(rel)
|
||||
if err != nil {
|
||||
if dcnEscaped(err) {
|
||||
dcnRefuse(ws, dir, "stat", storeID, a.Path, "path escapes the store root")
|
||||
return
|
||||
}
|
||||
if os.IsNotExist(err) {
|
||||
dcnResult(ws, dir, "stat", storeID, a.Path, map[string]any{
|
||||
"state": "gone", "path": a.Path})
|
||||
@@ -771,8 +815,25 @@ func handleDcnReadRange(ws *websocket.Conn, dir string, args []string) {
|
||||
dcnRefuse(ws, dir, "read_range", storeID, a.Path, "path escapes the store root")
|
||||
return
|
||||
}
|
||||
fh, err := os.Open(abs)
|
||||
rel, err := dcnRel(terms.Root, abs)
|
||||
if err != nil {
|
||||
dcnRefuse(ws, dir, "read_range", storeID, a.Path, err.Error())
|
||||
return
|
||||
}
|
||||
r, err := dcnOpenStore(dir, storeID, terms)
|
||||
if err != nil {
|
||||
dcnRefuse(ws, dir, "read_range", storeID, a.Path, err.Error())
|
||||
return
|
||||
}
|
||||
defer r.Close()
|
||||
// Opened through the store's handle: a name swapped for a link to a file
|
||||
// outside the store after the check above is refused, never followed.
|
||||
fh, err := r.Open(rel)
|
||||
if err != nil {
|
||||
if dcnEscaped(err) {
|
||||
dcnRefuse(ws, dir, "read_range", storeID, a.Path, "path escapes the store root")
|
||||
return
|
||||
}
|
||||
if os.IsNotExist(err) {
|
||||
dcnResult(ws, dir, "read_range", storeID, a.Path, map[string]any{
|
||||
"state": "gone", "path": a.Path})
|
||||
|
||||
+57
-21
@@ -191,10 +191,28 @@ func dcnReportPath(root string, planID int64) string {
|
||||
return p
|
||||
}
|
||||
|
||||
// dcnApplyPlan runs the plan: check, rename in order (each re-checked just
|
||||
// before it runs; rolling back on the first failure), verify, report. Every
|
||||
// path it touches has passed dcnCheckPlan.
|
||||
// dcnApplyPlan is dcnApplyPlanIn on root opened where it leads now (the op
|
||||
// handler opens the PINNED folder instead, dcn_store.go).
|
||||
func dcnApplyPlan(root string, excludes []string, plan *dcnPlanTerms) map[string]any {
|
||||
real, err := dcnRealRoot(root)
|
||||
if err != nil {
|
||||
return map[string]any{"state": "refused", "mode": "apply", "plan_id": plan.PlanID,
|
||||
"reason": "the store's folder cannot be opened: " + err.Error()}
|
||||
}
|
||||
r, err := dcnOpenRootAt(real)
|
||||
if err != nil {
|
||||
return map[string]any{"state": "refused", "mode": "apply", "plan_id": plan.PlanID, "reason": err.Error()}
|
||||
}
|
||||
defer r.Close()
|
||||
return dcnApplyPlanIn(r, root, excludes, plan)
|
||||
}
|
||||
|
||||
// dcnApplyPlanIn runs the plan: check, rename in order (each re-checked just
|
||||
// before it runs; rolling back on the first failure), verify, report. Every
|
||||
// path it touches has passed dcnCheckPlan, and every change is made through
|
||||
// r, the store's handle, so a folder that becomes a link to somewhere else
|
||||
// between the check and the rename is refused rather than followed.
|
||||
func dcnApplyPlanIn(r *os.Root, root string, excludes []string, plan *dcnPlanTerms) map[string]any {
|
||||
problems := dcnCheckPlan(root, excludes, plan.Moves)
|
||||
if len(problems) > 0 {
|
||||
return map[string]any{"state": "refused", "mode": "apply", "plan_id": plan.PlanID,
|
||||
@@ -207,11 +225,11 @@ func dcnApplyPlan(root string, excludes []string, plan *dcnPlanTerms) map[string
|
||||
// and what is NOT (so the user knows exactly what is still displaced).
|
||||
rollback := func() (back, stuck []string) {
|
||||
for i := len(applied) - 1; i >= 0; i-- {
|
||||
if _, err := os.Lstat(applied[i].src); err == nil {
|
||||
if _, err := r.Lstat(applied[i].src); err == nil {
|
||||
stuck = append(stuck, applied[i].from) // something now sits at the source; never replace it
|
||||
continue
|
||||
}
|
||||
if err := os.Rename(applied[i].dst, applied[i].src); err == nil {
|
||||
if err := r.Rename(applied[i].dst, applied[i].src); err == nil {
|
||||
back = append(back, applied[i].from)
|
||||
} else {
|
||||
stuck = append(stuck, applied[i].from)
|
||||
@@ -230,10 +248,17 @@ func dcnApplyPlan(root string, excludes []string, plan *dcnPlanTerms) map[string
|
||||
return rep
|
||||
}
|
||||
for i, m := range plan.Moves {
|
||||
src, _ := dcnResolvePath(root, m.From)
|
||||
dst, _ := cleanRel(root, m.To)
|
||||
if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
|
||||
return fail(i, m, "could not create the target folder: "+err.Error())
|
||||
srcAbs, _ := dcnResolvePath(root, m.From)
|
||||
dstAbs, _ := cleanRel(root, m.To)
|
||||
src, serr := dcnRel(root, srcAbs)
|
||||
dst, derr := dcnRel(root, dstAbs)
|
||||
if serr != nil || derr != nil {
|
||||
return fail(i, m, "path escapes the store root")
|
||||
}
|
||||
if d := filepath.Dir(dst); d != "." {
|
||||
if err := r.MkdirAll(d, 0o755); err != nil {
|
||||
return fail(i, m, "could not create the target folder: "+err.Error())
|
||||
}
|
||||
}
|
||||
// Re-check the instant before the rename: the world may have changed since
|
||||
// the pre-check (a file appeared at the target; a parent became a symlink).
|
||||
@@ -241,13 +266,12 @@ func dcnApplyPlan(root string, excludes []string, plan *dcnPlanTerms) map[string
|
||||
if _, ok := dcnResolvePath(root, m.To); !ok {
|
||||
return fail(i, m, "target's parent no longer resolves inside the store root")
|
||||
}
|
||||
if _, err := os.Lstat(dst); err == nil {
|
||||
if _, err := r.Lstat(dst); err == nil {
|
||||
return fail(i, m, "target appeared since the check (a plan never overwrites)")
|
||||
}
|
||||
if err := os.Rename(src, dst); err != nil {
|
||||
if err := r.Rename(src, dst); err != nil {
|
||||
why := err.Error()
|
||||
var le *os.LinkError
|
||||
if errors.As(err, &le) && errors.Is(le.Err, syscall.EXDEV) {
|
||||
if errors.Is(err, syscall.EXDEV) {
|
||||
why = "target is on a different filesystem than the source (a plan only renames; it never copies and removes)"
|
||||
}
|
||||
return fail(i, m, why)
|
||||
@@ -257,14 +281,16 @@ func dcnApplyPlan(root string, excludes []string, plan *dcnPlanTerms) map[string
|
||||
// Verify pass: every target present, every source gone, declared sizes hold.
|
||||
var mismatches []dcnPlanProblem
|
||||
for i, m := range plan.Moves {
|
||||
src, _ := dcnResolvePath(root, m.From)
|
||||
dst, _ := cleanRel(root, m.To)
|
||||
info, err := os.Lstat(dst)
|
||||
srcAbs, _ := dcnResolvePath(root, m.From)
|
||||
dstAbs, _ := cleanRel(root, m.To)
|
||||
src, _ := dcnRel(root, srcAbs)
|
||||
dst, _ := dcnRel(root, dstAbs)
|
||||
info, err := r.Lstat(dst)
|
||||
if err != nil {
|
||||
mismatches = append(mismatches, dcnPlanProblem{i, m.From, m.To, "target missing after move"})
|
||||
continue
|
||||
}
|
||||
if _, err := os.Lstat(src); err == nil {
|
||||
if _, err := r.Lstat(src); err == nil {
|
||||
mismatches = append(mismatches, dcnPlanProblem{i, m.From, m.To, "source still present after move"})
|
||||
}
|
||||
if m.SizeBytes != nil && !info.IsDir() && info.Size() != *m.SizeBytes {
|
||||
@@ -372,9 +398,19 @@ func handleDcnPlanApply(ws *websocket.Conn, dir string, args []string) {
|
||||
// a retry (a lost reply, a host re-run) gets the earlier report back BEFORE
|
||||
// any gate can turn it into a refusal. (The signed expiry is the guard that
|
||||
// does not depend on a file the user can remove.)
|
||||
store, err := dcnOpenStore(dir, storeID, terms)
|
||||
if err != nil {
|
||||
dcnRefuse(ws, dir, op, storeID, planKey, err.Error())
|
||||
return
|
||||
}
|
||||
defer store.Close()
|
||||
repPath := dcnReportPath(terms.Root, plan.PlanID)
|
||||
repRel := ""
|
||||
if repPath != "" {
|
||||
if prev, err := os.ReadFile(repPath); err == nil {
|
||||
repRel, _ = dcnRel(terms.Root, repPath)
|
||||
}
|
||||
if repRel != "" {
|
||||
if prev, err := store.ReadFile(repRel); err == nil {
|
||||
var old map[string]any
|
||||
if json.Unmarshal(prev, &old) == nil && old["state"] == "ok" {
|
||||
old["note"] = "already applied on this machine; this is the earlier report"
|
||||
@@ -398,13 +434,13 @@ func handleDcnPlanApply(ws *websocket.Conn, dir string, args []string) {
|
||||
return
|
||||
}
|
||||
}
|
||||
rep := dcnApplyPlan(terms.Root, terms.Excludes, plan)
|
||||
rep := dcnApplyPlanIn(store, terms.Root, terms.Excludes, plan)
|
||||
if rep["state"] == "ok" && plan.Approval != "" {
|
||||
markApprovalUsed(dir, plan.Approval) // single use: the files have moved
|
||||
}
|
||||
if repPath != "" {
|
||||
if repRel != "" {
|
||||
if raw, err := json.MarshalIndent(rep, "", " "); err == nil {
|
||||
_ = atomicWrite(repPath, raw) // the machine's own record; best-effort
|
||||
_ = dcnRootAtomicWrite(store, repRel, raw) // the machine's own record; best-effort
|
||||
}
|
||||
}
|
||||
dcnResult(ws, dir, op, storeID, planKey, rep)
|
||||
|
||||
@@ -0,0 +1,322 @@
|
||||
package main
|
||||
|
||||
// A machine store is the folder the person granted, for as long as the grant
|
||||
// lives. The bastion often runs as root and the folder belongs to someone who
|
||||
// is not, so whoever can write in it, or above it, must not be able to point
|
||||
// a read or a move somewhere else: by turning the granted folder into a link
|
||||
// after the grant (no race needed), or by swapping a file for a link between
|
||||
// the check and the open.
|
||||
|
||||
import (
|
||||
"crypto/ed25519"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/websocket"
|
||||
)
|
||||
|
||||
// dcnRunOp runs one DCN handler on the far side of a real WebSocket and
|
||||
// returns what it sent: the O payload, the R text and the exit code.
|
||||
func dcnRunOp(t *testing.T, h func(ws *websocket.Conn)) (out, errText string, exit int) {
|
||||
t.Helper()
|
||||
up := websocket.Upgrader{}
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
ws, err := up.Upgrade(w, r, nil)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer ws.Close()
|
||||
h(ws)
|
||||
}))
|
||||
defer srv.Close()
|
||||
ws, _, err := websocket.DefaultDialer.Dial("ws"+strings.TrimPrefix(srv.URL, "http"), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer ws.Close()
|
||||
_ = ws.SetReadDeadline(time.Now().Add(10 * time.Second))
|
||||
exit = -1
|
||||
for {
|
||||
_, data, err := ws.ReadMessage()
|
||||
if err != nil || len(data) == 0 {
|
||||
return
|
||||
}
|
||||
switch data[0] {
|
||||
case 'O':
|
||||
out += string(data[1:])
|
||||
case 'R':
|
||||
errText += string(data[1:])
|
||||
case 'X':
|
||||
exit, _ = strconv.Atoi(string(data[1:]))
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type dcnStoreFixture struct {
|
||||
dir string // the pairing's dcn dir
|
||||
base string
|
||||
store string // the granted folder
|
||||
other string // a folder the grant never named
|
||||
priv ed25519.PrivateKey
|
||||
pub ed25519.PublicKey
|
||||
}
|
||||
|
||||
func newDcnStoreFixture(t *testing.T) *dcnStoreFixture {
|
||||
t.Helper()
|
||||
fx := &dcnStoreFixture{dir: t.TempDir(), base: t.TempDir()}
|
||||
fx.store = filepath.Join(fx.base, "store")
|
||||
fx.other = filepath.Join(fx.base, "other")
|
||||
for _, d := range []string{fx.store, fx.other, filepath.Join(fx.store, "docs")} {
|
||||
if err := os.MkdirAll(d, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(fx.store, "a.txt"), []byte("MINE"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(fx.other, "a.txt"), []byte("OTHER"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fx.pub, fx.priv, _ = ed25519.GenerateKey(nil)
|
||||
return fx
|
||||
}
|
||||
|
||||
// sync installs a read grant for the store through the real grant_sync op.
|
||||
func (fx *dcnStoreFixture) sync(t *testing.T, mask ...string) {
|
||||
t.Helper()
|
||||
if len(mask) == 0 {
|
||||
mask = []string{"read"}
|
||||
}
|
||||
rec := signedRec(t, fx.priv, map[string]any{"kind": "dcn_store_grant", "class": "machine",
|
||||
"root": fx.store, "mask": mask,
|
||||
"expires": time.Now().Add(24 * time.Hour).UTC().Format(time.RFC3339)})
|
||||
payload, _ := json.Marshal(map[string]any{"store_id": 1, "terms_canonical": rec.TermsCanonical,
|
||||
"signature": rec.Signature, "verify_pubkey": base64.StdEncoding.EncodeToString(fx.pub)})
|
||||
_, why, exit := dcnRunOp(t, func(ws *websocket.Conn) {
|
||||
handleDcnGrantSync(ws, fx.dir, []string{base64.StdEncoding.EncodeToString(payload)})
|
||||
})
|
||||
if exit != 0 {
|
||||
t.Fatalf("grant_sync: exit %d: %s", exit, why)
|
||||
}
|
||||
}
|
||||
|
||||
func (fx *dcnStoreFixture) op(t *testing.T, h func(*websocket.Conn, string, []string), args map[string]any) (map[string]any, string, int) {
|
||||
t.Helper()
|
||||
raw, _ := json.Marshal(args)
|
||||
out, why, exit := dcnRunOp(t, func(ws *websocket.Conn) {
|
||||
h(ws, fx.dir, []string{"1", string(raw)})
|
||||
})
|
||||
var obj map[string]any
|
||||
_ = json.Unmarshal([]byte(out), &obj)
|
||||
return obj, why, exit
|
||||
}
|
||||
|
||||
func (fx *dcnStoreFixture) read(t *testing.T, path string) (string, string, int) {
|
||||
t.Helper()
|
||||
obj, why, exit := fx.op(t, handleDcnReadRange, map[string]any{"path": path})
|
||||
b, _ := base64.StdEncoding.DecodeString(func() string { s, _ := obj["content_b64"].(string); return s }())
|
||||
return string(b), why, exit
|
||||
}
|
||||
|
||||
// repoint turns the granted folder into a link to the other one.
|
||||
func (fx *dcnStoreFixture) repoint(t *testing.T) {
|
||||
t.Helper()
|
||||
if err := os.Rename(fx.store, fx.store+".was"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Symlink(fx.other, fx.store); err != nil {
|
||||
t.Skipf("symlinks unavailable: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDcnStoreRepointedAfterTheGrantIsRefused(t *testing.T) {
|
||||
fx := newDcnStoreFixture(t)
|
||||
fx.sync(t)
|
||||
if got, why, exit := fx.read(t, "a.txt"); exit != 0 || got != "MINE" {
|
||||
t.Fatalf("the granted file must read: exit %d %q %s", exit, got, why)
|
||||
}
|
||||
fx.repoint(t)
|
||||
if got, _, exit := fx.read(t, "a.txt"); exit == 0 || got == "OTHER" {
|
||||
t.Fatalf("after the granted folder became a link elsewhere, read_range returned %q (exit %d); it must refuse", got, exit)
|
||||
}
|
||||
if obj, _, exit := fx.op(t, handleDcnStat, map[string]any{"path": "a.txt"}); exit == 0 {
|
||||
t.Fatalf("stat through the repointed folder must refuse, got %v", obj)
|
||||
}
|
||||
if obj, _, exit := fx.op(t, handleDcnList, map[string]any{}); exit == 0 {
|
||||
t.Fatalf("list through the repointed folder must refuse, got %v", obj)
|
||||
}
|
||||
}
|
||||
|
||||
// A folder inside the store that becomes a link to somewhere else after the
|
||||
// grant is the same escape one level down.
|
||||
func TestDcnSubfolderRepointedIsRefused(t *testing.T) {
|
||||
fx := newDcnStoreFixture(t)
|
||||
fx.sync(t)
|
||||
if err := os.RemoveAll(filepath.Join(fx.store, "docs")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Symlink(fx.other, filepath.Join(fx.store, "docs")); err != nil {
|
||||
t.Skipf("symlinks unavailable: %v", err)
|
||||
}
|
||||
if got, _, exit := fx.read(t, "docs/a.txt"); exit == 0 || got == "OTHER" {
|
||||
t.Fatalf("a link inside the store to a folder outside it read %q (exit %d)", got, exit)
|
||||
}
|
||||
}
|
||||
|
||||
// The race: a file swapped for a link to a file outside the store between the
|
||||
// check and the open. Run for long enough that the old check-then-open loses.
|
||||
func TestDcnReadRangeSwapRaceNeverLeavesTheStore(t *testing.T) {
|
||||
fx := newDcnStoreFixture(t)
|
||||
fx.sync(t)
|
||||
target := filepath.Join(fx.store, "a.txt")
|
||||
realTmp := filepath.Join(fx.store, ".real")
|
||||
linkTmp := filepath.Join(fx.store, ".link")
|
||||
var stop atomic.Bool
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
for !stop.Load() {
|
||||
_ = os.Symlink(filepath.Join(fx.other, "a.txt"), linkTmp)
|
||||
_ = os.Rename(linkTmp, target)
|
||||
_ = os.WriteFile(realTmp, []byte("MINE"), 0o644)
|
||||
_ = os.Rename(realTmp, target)
|
||||
}
|
||||
}()
|
||||
deadline := time.Now().Add(3 * time.Second)
|
||||
n := 0
|
||||
for time.Now().Before(deadline) {
|
||||
n++
|
||||
if got, _, _ := fx.read(t, "a.txt"); got == "OTHER" {
|
||||
stop.Store(true)
|
||||
<-done
|
||||
t.Fatalf("read %d returned the file outside the store through a swapped link", n)
|
||||
}
|
||||
}
|
||||
stop.Store(true)
|
||||
<-done
|
||||
}
|
||||
|
||||
// A move plan applies through the store too: a target folder that leads
|
||||
// outside the store for an instant between the re-check and the rename must
|
||||
// not receive the file.
|
||||
func TestDcnPlanMoveRaceNeverLeavesTheStore(t *testing.T) {
|
||||
fx := newDcnStoreFixture(t)
|
||||
if err := os.Remove(filepath.Join(fx.other, "a.txt")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
docs := filepath.Join(fx.store, "docs")
|
||||
var stop atomic.Bool
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
for !stop.Load() {
|
||||
_ = os.Rename(docs, docs+".real")
|
||||
_ = os.Symlink(fx.other, docs+".link")
|
||||
_ = os.Rename(docs+".link", docs)
|
||||
_ = os.Rename(docs, docs+".link")
|
||||
_ = os.Remove(docs + ".link")
|
||||
_ = os.Rename(docs+".real", docs)
|
||||
}
|
||||
}()
|
||||
defer func() { stop.Store(true); <-done }()
|
||||
deadline := time.Now().Add(3 * time.Second)
|
||||
for i := 0; time.Now().Before(deadline); i++ {
|
||||
_ = os.WriteFile(filepath.Join(fx.store, "a.txt"), []byte("MINE"), 0o644)
|
||||
plan := &dcnPlanTerms{PlanID: int64(i), StoreID: 1, Root: fx.store,
|
||||
Moves: []dcnMove{{From: "a.txt", To: "docs/a.txt"}}}
|
||||
_ = dcnApplyPlan(fx.store, nil, plan)
|
||||
if _, err := os.Lstat(filepath.Join(fx.other, "a.txt")); err == nil {
|
||||
t.Fatalf("move %d landed outside the store, through a folder that was a link for an instant", i)
|
||||
}
|
||||
_ = os.Remove(filepath.Join(fx.store, "docs.real", "a.txt"))
|
||||
_ = os.Remove(filepath.Join(docs, "a.txt"))
|
||||
}
|
||||
}
|
||||
|
||||
// The plan report is written beside its name and renamed into place. Whoever
|
||||
// can write in the store could plant a link at that side name; the write must
|
||||
// never follow it, or the root bastion truncates whatever file the link names
|
||||
// (an excluded .ssh/authorized_keys included) and fills it with the report.
|
||||
func TestDcnReportWriteNeverFollowsAPlantedSideName(t *testing.T) {
|
||||
store := t.TempDir()
|
||||
victim := filepath.Join(store, ".ssh", "authorized_keys")
|
||||
if err := os.MkdirAll(filepath.Dir(victim), 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(victim, []byte("ssh-ed25519 AAAAexample user@example\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
plans := filepath.Join(store, ".agent", "plans")
|
||||
if err := os.MkdirAll(plans, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rel := filepath.Join(".agent", "plans", "7.json")
|
||||
// Every side name a writer could guess: this process's pid, and a spread
|
||||
// of others, all pointing at the victim.
|
||||
for _, pid := range []int{os.Getpid(), 1, 2, 100, 1000} {
|
||||
side := filepath.Join(store, rel+".tmp-"+strconv.Itoa(pid))
|
||||
if err := os.Symlink(filepath.Join("..", "..", ".ssh", "authorized_keys"), side); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
r, err := os.OpenRoot(store)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer r.Close()
|
||||
if err := dcnRootAtomicWrite(r, rel, []byte(`{"state":"ok"}`)); err != nil {
|
||||
t.Fatalf("report write: %v", err)
|
||||
}
|
||||
if got, _ := os.ReadFile(victim); string(got) != "ssh-ed25519 AAAAexample user@example\n" {
|
||||
t.Fatalf("the report write followed a planted link: the victim now holds %q", got)
|
||||
}
|
||||
if got, err := os.ReadFile(filepath.Join(store, rel)); err != nil || string(got) != `{"state":"ok"}` {
|
||||
t.Fatalf("report not in place: %q %v", got, err)
|
||||
}
|
||||
if fi, err := os.Lstat(filepath.Join(store, rel)); err != nil || fi.Mode()&os.ModeSymlink != 0 || fi.Mode().Perm() != 0o600 {
|
||||
t.Fatalf("report is not a 0600 regular file: %v %v", fi, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A grant installed before pins existed has no root_real. It still works,
|
||||
// pins where its folder leads at first use, keeps that pin, and a repoint
|
||||
// after that is refused like any other.
|
||||
func TestDcnGrantFromBeforePinsPinsAtFirstUse(t *testing.T) {
|
||||
fx := newDcnStoreFixture(t)
|
||||
fx.sync(t)
|
||||
f, err := dcnLoadFile(fx.dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec := f.Stores["1"]
|
||||
rec.RootReal = ""
|
||||
f.Stores["1"] = rec
|
||||
if err := dcnSaveFile(fx.dir, f); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, why, exit := fx.read(t, "a.txt"); exit != 0 || got != "MINE" {
|
||||
t.Fatalf("a grant from before pins no longer reads: exit %d %q %s", exit, got, why)
|
||||
}
|
||||
f, err = dcnLoadFile(fx.dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want, _ := filepath.EvalSymlinks(fx.store)
|
||||
if got := f.Stores["1"].RootReal; got != want {
|
||||
t.Fatalf("first use pinned %q, want %q", got, want)
|
||||
}
|
||||
fx.repoint(t)
|
||||
if got, _, exit := fx.read(t, "a.txt"); exit == 0 || got == "OTHER" {
|
||||
t.Fatalf("after the first-use pin a repointed folder was read: exit %d %q", exit, got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,199 @@
|
||||
package main
|
||||
|
||||
// dcn_store.go — every byte a machine-store op touches is reached through ONE
|
||||
// handle on the folder the person granted.
|
||||
//
|
||||
// WHY. The bastion is often root (a networked install, the macOS and Windows
|
||||
// system service), and the granted folder is often someone else's. Checking a
|
||||
// path and then opening it by name leaves two ways out of the store for
|
||||
// whoever can write in it or above it:
|
||||
//
|
||||
// - the granted folder itself turned into a link to another folder after the
|
||||
// grant. The old check resolved the root again on every op and judged the
|
||||
// path against wherever it led NOW, so no race was needed;
|
||||
// - a file or folder inside the store swapped for a link between the check
|
||||
// and the open (or the rename, for a move plan).
|
||||
//
|
||||
// So: the folder's real path is PINNED at grant sync (rec.RootReal, this box's
|
||||
// own field, outside the signed bytes), an op whose root now leads elsewhere
|
||||
// is refused, and the op opens the pinned folder as an os.Root, which never
|
||||
// follows a link out of it, whatever happens to the names afterwards. That
|
||||
// handle is then checked to be the folder at the pinned path reached with no
|
||||
// link at all (dcnNoLinkStat: an open that refuses a link in every component
|
||||
// on Linux and macOS), so a swap of a folder above the store between the two
|
||||
// opens is refused too.
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
// errDcnRepointed is the refusal for a granted folder that leads elsewhere now.
|
||||
var errDcnRepointed = errors.New("the granted folder now leads somewhere other than the folder that was granted; " +
|
||||
"a new grant for it is the repair")
|
||||
|
||||
// dcnRealRoot is where the grant's root leads now: its links resolved.
|
||||
func dcnRealRoot(root string) (string, error) {
|
||||
real, err := filepath.EvalSymlinks(filepath.Clean(root))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !filepath.IsAbs(real) {
|
||||
return "", errors.New("the granted folder is not an absolute path")
|
||||
}
|
||||
return filepath.Clean(real), nil
|
||||
}
|
||||
|
||||
// dcnPinFor is the real root a sync records: the one already pinned for the
|
||||
// same root while the store is held (a re-sync, a renewed grant), else where
|
||||
// the root leads now ("" when it cannot be resolved yet; pinned at first use).
|
||||
func dcnPinFor(prev *dcnGrantRec, prevRoot, root string) string {
|
||||
if prev != nil && prev.RootReal != "" && filepath.Clean(prevRoot) == filepath.Clean(root) {
|
||||
return prev.RootReal
|
||||
}
|
||||
real, err := dcnRealRoot(root)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return real
|
||||
}
|
||||
|
||||
// dcnStorePin is the pinned real root of a held store, pinning it now (and
|
||||
// saving it) for a grant synced before pins existed.
|
||||
func dcnStorePin(dir string, storeID int64, root string) (string, error) {
|
||||
grantsMu.Lock()
|
||||
defer grantsMu.Unlock()
|
||||
f, err := dcnLoadFile(dir)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
key := strconv.FormatInt(storeID, 10)
|
||||
rec, ok := f.Stores[key]
|
||||
if !ok {
|
||||
return "", errors.New("no grant for this store on this box")
|
||||
}
|
||||
if rec.RootReal != "" {
|
||||
return rec.RootReal, nil
|
||||
}
|
||||
real, err := dcnRealRoot(root)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
rec.RootReal = real
|
||||
f.Stores[key] = rec
|
||||
if err := dcnSaveFile(dir, f); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return real, nil
|
||||
}
|
||||
|
||||
// dcnOpenStore opens the store a verified grant names, as the folder that was
|
||||
// granted, or refuses.
|
||||
func dcnOpenStore(dir string, storeID int64, terms *dcnTerms) (*os.Root, error) {
|
||||
pin, err := dcnStorePin(dir, storeID, terms.Root)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the granted folder cannot be opened: %w", err)
|
||||
}
|
||||
real, err := dcnRealRoot(terms.Root)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the granted folder cannot be opened: %w", err)
|
||||
}
|
||||
if real != pin {
|
||||
return nil, errDcnRepointed
|
||||
}
|
||||
return dcnOpenRootAt(pin)
|
||||
}
|
||||
|
||||
// dcnOpenRootAt opens real (a resolved, absolute path) as an os.Root and
|
||||
// checks that the handle is the folder reached at real with no link in any
|
||||
// component, so nothing above the folder was swapped while it was opened.
|
||||
func dcnOpenRootAt(real string) (*os.Root, error) {
|
||||
r, err := os.OpenRoot(real)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the granted folder cannot be opened: %w", err)
|
||||
}
|
||||
rfi, err := r.Stat(".")
|
||||
if err != nil {
|
||||
_ = r.Close()
|
||||
return nil, fmt.Errorf("the granted folder cannot be opened: %w", err)
|
||||
}
|
||||
nfi, err := dcnNoLinkStat(real)
|
||||
if err != nil || !nfi.IsDir() || !os.SameFile(rfi, nfi) {
|
||||
_ = r.Close()
|
||||
return nil, errDcnRepointed
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// dcnRel is a path the sandbox accepted (cleanRel/dcnResolvePath's joined
|
||||
// form) as a name inside the root handle.
|
||||
func dcnRel(root, joined string) (string, error) {
|
||||
rel, err := filepath.Rel(filepath.Clean(root), joined)
|
||||
if err != nil || rel == ".." || filepath.IsAbs(rel) || len(rel) >= 3 && rel[:3] == ".."+string(os.PathSeparator) {
|
||||
return "", errors.New("path escapes the store root")
|
||||
}
|
||||
return rel, nil
|
||||
}
|
||||
|
||||
// dcnEscaped: the root handle refused a name because it leads out of the store.
|
||||
func dcnEscaped(err error) bool {
|
||||
var pe *os.PathError
|
||||
return errors.As(err, &pe) && pe.Err != nil && pe.Err.Error() == "path escapes from parent"
|
||||
}
|
||||
|
||||
// dcnRootAtomicWrite is atomicWrite inside the store's handle: folders 0700,
|
||||
// the file 0600, written beside its name and renamed into place.
|
||||
func dcnRootAtomicWrite(r *os.Root, rel string, data []byte) error {
|
||||
if d := filepath.Dir(rel); d != "." {
|
||||
if err := r.MkdirAll(d, 0o700); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// The side name is unguessable and opened O_EXCL, as atomicWrite's
|
||||
// CreateTemp is: whoever can write in the store could otherwise plant a
|
||||
// link at a guessable name, and an open that follows it would truncate
|
||||
// and fill whatever file in the store the link names. O_EXCL refuses any
|
||||
// existing name, a link included, without following it.
|
||||
var f *os.File
|
||||
var tmp string
|
||||
for try := 0; ; try++ {
|
||||
var nb [12]byte
|
||||
if _, err := rand.Read(nb[:]); err != nil {
|
||||
return err
|
||||
}
|
||||
tmp = filepath.Join(filepath.Dir(rel), ".dcn-"+hex.EncodeToString(nb[:])+".tmp")
|
||||
var err error
|
||||
f, err = r.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
if !errors.Is(err, fs.ErrExist) || try >= 8 {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if _, err := f.Write(data); err != nil {
|
||||
_ = f.Close()
|
||||
_ = r.Remove(tmp)
|
||||
return err
|
||||
}
|
||||
if err := f.Sync(); err != nil {
|
||||
_ = f.Close()
|
||||
_ = r.Remove(tmp)
|
||||
return err
|
||||
}
|
||||
if err := f.Close(); err != nil {
|
||||
_ = r.Remove(tmp)
|
||||
return err
|
||||
}
|
||||
if err := r.Rename(tmp, rel); err != nil {
|
||||
_ = r.Remove(tmp)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
//go:build darwin
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
// dcnNoLinkStat describes the directory at p opened with O_NOFOLLOW_ANY, which
|
||||
// fails on a link in any component (macOS 11 and later).
|
||||
func dcnNoLinkStat(p string) (os.FileInfo, error) {
|
||||
fd, err := unix.Open(p, unix.O_RDONLY|unix.O_DIRECTORY|unix.O_NOFOLLOW_ANY|unix.O_CLOEXEC, 0)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
f := os.NewFile(uintptr(fd), p)
|
||||
defer f.Close()
|
||||
return f.Stat()
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
//go:build linux
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
// dcnNoLinkStat describes the directory at p (absolute, clean) reached one
|
||||
// component at a time with O_NOFOLLOW, so a link anywhere in the path fails
|
||||
// instead of being followed. O_PATH: a component the bastion may traverse but
|
||||
// not list (a home at 0711) still opens.
|
||||
func dcnNoLinkStat(p string) (os.FileInfo, error) {
|
||||
if !strings.HasPrefix(p, "/") {
|
||||
return nil, errors.New("not an absolute path")
|
||||
}
|
||||
fd, err := unix.Open("/", unix.O_PATH|unix.O_DIRECTORY|unix.O_CLOEXEC, 0)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, c := range strings.Split(p, "/") {
|
||||
if c == "" {
|
||||
continue
|
||||
}
|
||||
if c == "." || c == ".." {
|
||||
_ = unix.Close(fd)
|
||||
return nil, errors.New("not a clean path")
|
||||
}
|
||||
next, err := unix.Openat(fd, c, unix.O_PATH|unix.O_NOFOLLOW|unix.O_DIRECTORY|unix.O_CLOEXEC, 0)
|
||||
_ = unix.Close(fd)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var st unix.Stat_t
|
||||
if err := unix.Fstat(next, &st); err != nil || st.Mode&unix.S_IFMT != unix.S_IFDIR {
|
||||
_ = unix.Close(next)
|
||||
return nil, errors.New("a component is not a directory")
|
||||
}
|
||||
fd = next
|
||||
}
|
||||
f := os.NewFile(uintptr(fd), p)
|
||||
defer f.Close()
|
||||
return f.Stat()
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
//go:build !linux && !darwin
|
||||
|
||||
package main
|
||||
|
||||
import "os"
|
||||
|
||||
// dcnNoLinkStat on a platform with no open that refuses a link in every
|
||||
// component (Windows): the final component is judged without following it.
|
||||
// The store is still reached only through its os.Root and only while its real
|
||||
// path matches the pin; what this leaves is a swap of a folder above the
|
||||
// store, by someone who can rename it, timed between the two opens.
|
||||
func dcnNoLinkStat(p string) (os.FileInfo, error) {
|
||||
return os.Lstat(p)
|
||||
}
|
||||
+79
-46
@@ -6,6 +6,7 @@ import (
|
||||
"errors"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
@@ -24,94 +25,110 @@ type egressDialer func(ctx context.Context, network, addr string) (net.Conn, err
|
||||
// egress is refused (403, before any dial) unless its pairing grant carries
|
||||
// the "egress" cap: this lane makes requests leave THIS machine's connection
|
||||
// under its address, and nothing in the mgmt op table gates it.
|
||||
func handleEgressConn(client net.Conn, dialOut egressDialer, self string) {
|
||||
//
|
||||
// It reports whether it took the job: true once the request passed every check
|
||||
// and was dialled out, whatever the destination then answered; false when the
|
||||
// worker closed before a request, or the request was malformed or refused. The
|
||||
// worker pool (runEgressWorkers) parks again at once after a job it took, and
|
||||
// waits after one it did not.
|
||||
func handleEgressConn(client net.Conn, dialOut egressDialer, self string) bool {
|
||||
defer client.Close()
|
||||
br := bufio.NewReader(client)
|
||||
// The request line and headers are read under a bound, the one Go's own HTTP
|
||||
// server uses (http.DefaultMaxHeaderBytes, 1 MiB): the host sends them, and a
|
||||
// head that never ends must not grow in this machine's memory.
|
||||
headLim := &io.LimitedReader{R: client, N: egressMaxHeadBytes}
|
||||
br := bufio.NewReader(headLim)
|
||||
line, err := br.ReadString('\n')
|
||||
if err != nil {
|
||||
return
|
||||
return false
|
||||
}
|
||||
parts := strings.Fields(strings.TrimSpace(line))
|
||||
if len(parts) < 2 {
|
||||
return
|
||||
return false
|
||||
}
|
||||
method, target := parts[0], parts[1]
|
||||
|
||||
// Buffer the head so a plain-HTTP request can be replayed verbatim; capture Host.
|
||||
// Buffer the head: a plain-HTTP request is parsed from it and rebuilt
|
||||
// (parseEgressRequest), never replayed verbatim.
|
||||
var head strings.Builder
|
||||
head.WriteString(line)
|
||||
hostHdr := ""
|
||||
for {
|
||||
h, err := br.ReadString('\n')
|
||||
if err != nil {
|
||||
return
|
||||
return false
|
||||
}
|
||||
head.WriteString(h)
|
||||
if h == "\r\n" || h == "\n" {
|
||||
break
|
||||
}
|
||||
if len(h) >= 5 && strings.EqualFold(h[:5], "host:") {
|
||||
hostHdr = strings.TrimSpace(h[5:])
|
||||
}
|
||||
}
|
||||
|
||||
// The head is in; what follows is the stream itself, unbounded. What the
|
||||
// reader already buffered past the head stays first in line.
|
||||
headLim.N = 1<<63 - 1
|
||||
|
||||
if why := capsAllow(self, "egress"); why != "" {
|
||||
_, _ = client.Write([]byte("HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\nX-Privatae-Refused: " + why + "\r\n\r\n"))
|
||||
return
|
||||
writeEgressRefusal(client, "403 Forbidden", why, why)
|
||||
return false
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), egressDialTimeout)
|
||||
defer cancel()
|
||||
|
||||
if strings.EqualFold(method, "CONNECT") {
|
||||
dialAddrs, why := egressVettedAddrs(ctx, ensurePort(target, "443"))
|
||||
// The target is read the way the Host's egress gate reads it
|
||||
// (parseEgressAuthority): userinfo, an unbracketed IPv6 literal or a
|
||||
// malformed port is refused, never guessed at.
|
||||
host, port, ok := parseEgressAuthority(target, 443)
|
||||
if !ok {
|
||||
writeEgressRefusal(client, "400 Bad Request", egressNoHost, egressNoHost)
|
||||
return false
|
||||
}
|
||||
dialAddrs, why := egressVettedAddrs(ctx, egressHostPort(host, port))
|
||||
if why != "" {
|
||||
_, _ = client.Write([]byte("HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\nX-Privatae-Refused: " + why + "\r\n\r\n"))
|
||||
return
|
||||
writeEgressRefusal(client, "403 Forbidden", why, why)
|
||||
return false
|
||||
}
|
||||
upstream, err := dialEach(ctx, dialOut, dialAddrs)
|
||||
if err != nil {
|
||||
_, _ = client.Write([]byte("HTTP/1.1 502 Bad Gateway\r\nContent-Length: 0\r\n\r\n"))
|
||||
return
|
||||
writeEgressRefusal(client, "502 Bad Gateway", egressUnreachable, "")
|
||||
return true
|
||||
}
|
||||
if _, err := client.Write([]byte("HTTP/1.1 200 Connection established\r\n\r\n")); err != nil {
|
||||
_ = upstream.Close()
|
||||
return
|
||||
return true
|
||||
}
|
||||
splice(br, client, upstream)
|
||||
return
|
||||
return true
|
||||
}
|
||||
|
||||
// Plain HTTP, absolute-form: "GET http://host/path HTTP/1.1".
|
||||
host := hostHdr
|
||||
if i := strings.Index(target, "://"); i >= 0 {
|
||||
rest := target[i+3:]
|
||||
if s := strings.IndexByte(rest, '/'); s >= 0 {
|
||||
host = rest[:s]
|
||||
} else {
|
||||
host = rest
|
||||
}
|
||||
// Plain HTTP: ONE request per connection, parsed and rebuilt so the
|
||||
// destination reads the host, port and body the lane vetted, sent with
|
||||
// `Connection: close` (egress_http.go). Anything the Host sends after that
|
||||
// request's body closes the connection unforwarded: a keep-alive connection
|
||||
// can no longer carry later requests past the vetting.
|
||||
req, refused := parseEgressRequest(head.String())
|
||||
if refused != nil {
|
||||
writeEgressRefusal(client, refused.status, refused.body, refused.body)
|
||||
return false
|
||||
}
|
||||
if host == "" {
|
||||
return
|
||||
}
|
||||
dialAddrs, why := egressVettedAddrs(ctx, ensurePort(host, "80"))
|
||||
dialAddrs, why := egressVettedAddrs(ctx, egressHostPort(req.host, req.port))
|
||||
if why != "" {
|
||||
_, _ = client.Write([]byte("HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\nX-Privatae-Refused: " + why + "\r\n\r\n"))
|
||||
return
|
||||
writeEgressRefusal(client, "403 Forbidden", why, why)
|
||||
return false
|
||||
}
|
||||
upstream, err := dialEach(ctx, dialOut, dialAddrs)
|
||||
if err != nil {
|
||||
_, _ = client.Write([]byte("HTTP/1.1 502 Bad Gateway\r\nContent-Length: 0\r\n\r\n"))
|
||||
return
|
||||
writeEgressRefusal(client, "502 Bad Gateway", egressUnreachable, "")
|
||||
return true
|
||||
}
|
||||
if _, err := upstream.Write([]byte(head.String())); err != nil {
|
||||
_ = upstream.Close()
|
||||
return
|
||||
}
|
||||
splice(br, client, upstream)
|
||||
oneEgressExchange(br, client, upstream, req)
|
||||
return true
|
||||
}
|
||||
|
||||
// egressMaxHeadBytes bounds an egress request's line and headers together.
|
||||
const egressMaxHeadBytes = http.DefaultMaxHeaderBytes
|
||||
|
||||
// ensurePort returns host:port, adding defPort when target has no explicit port.
|
||||
func ensurePort(target, defPort string) string {
|
||||
target = strings.TrimSpace(target)
|
||||
@@ -157,11 +174,14 @@ func splice(clientReader io.Reader, client, upstream net.Conn) {
|
||||
// included), and any address the machine takes as its own (ownaddr.go), such
|
||||
// as a public one on its NIC or a range a `local` route delivers here.
|
||||
//
|
||||
// What differs is the network around it. On an ANCHORED box the lane reaches
|
||||
// the public internet and nothing else: private, CGNAT and reserved addresses
|
||||
// (the LAN, the mesh) are refused too. On a LOCAL box the LAN stays reachable:
|
||||
// the host there is the one the person paired, and their own network is
|
||||
// theirs to reach.
|
||||
// The network around it is refused too, on every box: the lane reaches the
|
||||
// public internet and nothing else, so private, CGNAT and reserved addresses
|
||||
// (the LAN, the mesh) are refused, by address and by any address a name
|
||||
// resolves to. A LOCAL box used to keep its LAN reachable here; it no longer
|
||||
// does (2026-10-26): the Host's egress gate holds every dial it makes to the
|
||||
// same rule, and a person's own machines
|
||||
// are reached through their approved lanes, never through egress. Only the
|
||||
// wording differs between the two kinds of box.
|
||||
//
|
||||
// Resolution failures refuse too: a name this box cannot resolve is not one
|
||||
// it can vouch for.
|
||||
@@ -245,6 +265,19 @@ func egressVettedAddrs(ctx context.Context, hostport string) ([]string, string)
|
||||
}
|
||||
return nil, "refused: egress to " + host + " — that is this machine; this lane never reaches this machine"
|
||||
}
|
||||
// The network around this machine, on a local box as on an anchored one
|
||||
// (above): any one address of the name that is not the public internet
|
||||
// refuses it. Checked after this machine's own addresses, so those keep
|
||||
// saying "that is this machine".
|
||||
if !anchored {
|
||||
for _, ip := range addrs {
|
||||
if internalIP(ip) {
|
||||
return nil, "refused: egress to " + host + " (" + ip.String() + ") — this lane reaches the " +
|
||||
"public internet only, never this machine's network or the mesh; reach your own " +
|
||||
"machines through their approved lanes"
|
||||
}
|
||||
}
|
||||
}
|
||||
out := make([]string, 0, len(addrs))
|
||||
for _, ip := range addrs {
|
||||
out = append(out, net.JoinHostPort(ip.String(), port))
|
||||
|
||||
@@ -0,0 +1,420 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"errors"
|
||||
"io"
|
||||
"net"
|
||||
"net/http/httputil"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// ONE plain-HTTP request per egress connection, read the way the Host's egress
|
||||
// gate reads it (its strict request parse and its authority parse: the same
|
||||
// refusals, the same normalised host, the same rebuilt head).
|
||||
//
|
||||
// Everything the lane writes to a destination is what it checked, and nothing
|
||||
// more:
|
||||
// - the head is parsed strictly and REBUILT, so the destination reads the same
|
||||
// host, port and body length the lane vetted: never a second reading of the
|
||||
// Host's bytes (a Host header naming another site, a query or userinfo glued
|
||||
// to the host, a padded Transfer-Encoding);
|
||||
// - the request goes out with `Connection: close`, its body is framed by
|
||||
// Content-Length or chunked encoding, and a byte the Host sends after the
|
||||
// body closes the connection with nothing of it forwarded. Before this the
|
||||
// lane wrote the first head and then spliced the whole connection, so every
|
||||
// later request on a keep-alive connection reached the destination unread.
|
||||
//
|
||||
// An honest client sends its next request only after the response, which the
|
||||
// destination closes behind `Connection: close`, so it opens a new connection
|
||||
// and meets the vetting again.
|
||||
|
||||
// egressHostChars are the characters a host name may have here (lower-cased).
|
||||
const egressHostChars = "abcdefghijklmnopqrstuvwxyz0123456789-._"
|
||||
|
||||
// egressTokenChars are RFC 9110 token characters: methods and header names.
|
||||
const egressTokenChars = "!#$%&'*+-.^_`|~0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
|
||||
|
||||
// egressHopByHop are headers that belong to one hop. The lane speaks for itself
|
||||
// on the next one, and a proxy credential never goes on to the destination.
|
||||
var egressHopByHop = map[string]bool{
|
||||
"connection": true, "proxy-connection": true, "keep-alive": true, "upgrade": true,
|
||||
"te": true, "trailer": true, "proxy-authorization": true, "proxy-authenticate": true,
|
||||
}
|
||||
|
||||
// egressTrailerLimit bounds the trailer section of a chunked body, which is
|
||||
// read to its end and dropped (the gate's limit).
|
||||
const egressTrailerLimit = 8192
|
||||
|
||||
// egressRefusal is a request the lane refuses before it dials anything.
|
||||
type egressRefusal struct{ status, body string }
|
||||
|
||||
func egressBad(body string) *egressRefusal {
|
||||
return &egressRefusal{"400 Bad Request", body}
|
||||
}
|
||||
|
||||
const egressNoHost = "The bastion could not read where this request is going (no host in the " +
|
||||
"request line or in a Host header), so it sent nothing."
|
||||
|
||||
// egressUnreachable is the body of a 502: the destination passed the vetting,
|
||||
// and no address of it answered from this machine.
|
||||
const egressUnreachable = "The bastion could not reach that destination from this machine's own " +
|
||||
"connection, so nothing was sent."
|
||||
|
||||
// egressHostPort is host and port as the vetting takes them.
|
||||
func egressHostPort(host string, port int) string {
|
||||
return net.JoinHostPort(host, strconv.Itoa(port))
|
||||
}
|
||||
|
||||
// writeEgressRefusal answers the Host with a status and a short plain-text
|
||||
// body saying why, and closes the exchange. why, when set, also goes in
|
||||
// X-Privatae-Refused, the header the Host already reads.
|
||||
func writeEgressRefusal(w io.Writer, status, body, why string) {
|
||||
body += "\n"
|
||||
var b strings.Builder
|
||||
b.WriteString("HTTP/1.1 " + status + "\r\n")
|
||||
b.WriteString("Content-Type: text/plain; charset=utf-8\r\n")
|
||||
b.WriteString("Content-Length: " + strconv.Itoa(len(body)) + "\r\n")
|
||||
b.WriteString("Connection: close\r\n")
|
||||
if why != "" {
|
||||
b.WriteString("X-Privatae-Refused: " + strings.NewReplacer("\r", " ", "\n", " ").Replace(why) + "\r\n")
|
||||
}
|
||||
b.WriteString("\r\n")
|
||||
b.WriteString(body)
|
||||
_, _ = io.WriteString(w, b.String())
|
||||
}
|
||||
|
||||
// parseEgressAuthority: (host, port, true) from "host", "host:port", "[v6]" or
|
||||
// "[v6]:port", or false when the lane can't read it exactly as the destination
|
||||
// would. Refused rather than guessed: userinfo ("a@b"), an unbracketed IPv6
|
||||
// literal, characters no host name has, a non-numeric or out-of-range port.
|
||||
// The host is lower-cased and loses one trailing dot.
|
||||
func parseEgressAuthority(s string, defaultPort int) (string, int, bool) {
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" || strings.Contains(s, "@") {
|
||||
return "", 0, false
|
||||
}
|
||||
var host string
|
||||
port := defaultPort
|
||||
digits := func(p string) bool {
|
||||
if len(p) == 0 || len(p) > 5 {
|
||||
return false
|
||||
}
|
||||
for _, c := range p {
|
||||
if c < '0' || c > '9' {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
if strings.HasPrefix(s, "[") {
|
||||
end := strings.IndexByte(s, ']')
|
||||
if end < 0 {
|
||||
return "", 0, false
|
||||
}
|
||||
// An IPv6 literal, IPv4-mapped forms included (the vetting judges the
|
||||
// address they map to); a bare IPv4 address in brackets or a zone is
|
||||
// not one.
|
||||
inner := s[1:end]
|
||||
ip := net.ParseIP(inner)
|
||||
if ip == nil || !strings.Contains(inner, ":") || strings.Contains(inner, "%") {
|
||||
return "", 0, false
|
||||
}
|
||||
host = ip.String()
|
||||
rest := s[end+1:]
|
||||
switch {
|
||||
case rest == "":
|
||||
case strings.HasPrefix(rest, ":") && digits(rest[1:]):
|
||||
port, _ = strconv.Atoi(rest[1:])
|
||||
default:
|
||||
return "", 0, false
|
||||
}
|
||||
} else {
|
||||
if strings.Count(s, ":") > 1 {
|
||||
return "", 0, false
|
||||
}
|
||||
h, p, hasPort := strings.Cut(s, ":")
|
||||
if hasPort {
|
||||
if !digits(p) {
|
||||
return "", 0, false
|
||||
}
|
||||
port, _ = strconv.Atoi(p)
|
||||
}
|
||||
h = strings.ToLower(h)
|
||||
h = strings.TrimSuffix(h, ".")
|
||||
if h == "" || strings.HasPrefix(h, ".") || strings.Contains(h, "..") {
|
||||
return "", 0, false
|
||||
}
|
||||
for _, c := range h {
|
||||
if !strings.ContainsRune(egressHostChars, c) {
|
||||
return "", 0, false
|
||||
}
|
||||
}
|
||||
host = h
|
||||
}
|
||||
if port <= 0 || port >= 65536 {
|
||||
return "", 0, false
|
||||
}
|
||||
return host, port, true
|
||||
}
|
||||
|
||||
// fmtEgressAuthority writes host[:port] back, the port left out when it is the
|
||||
// scheme's default.
|
||||
func fmtEgressAuthority(host string, port, defaultPort int) string {
|
||||
h := host
|
||||
if strings.Contains(host, ":") {
|
||||
h = "[" + host + "]"
|
||||
}
|
||||
if port == defaultPort {
|
||||
return h
|
||||
}
|
||||
return h + ":" + strconv.Itoa(port)
|
||||
}
|
||||
|
||||
// egressRequest is the one plain-HTTP request the lane forwards.
|
||||
type egressRequest struct {
|
||||
host string
|
||||
port int
|
||||
head []byte // rebuilt, CRLF, ends at its blank line
|
||||
chunked bool
|
||||
length int64
|
||||
}
|
||||
|
||||
func egressTokenOK(s string) bool {
|
||||
if s == "" {
|
||||
return false
|
||||
}
|
||||
for _, c := range s {
|
||||
if !strings.ContainsRune(egressTokenChars, c) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// parseEgressRequest reads the one request in head (the request line and
|
||||
// headers through their blank line, as handleEgressConn read them) and
|
||||
// rebuilds it, or refuses it. CONNECT never comes here.
|
||||
func parseEgressRequest(head string) (egressRequest, *egressRefusal) {
|
||||
lines := strings.Split(head, "\n")
|
||||
// The read stops at the first blank line, so the head ends "\n", blank, "\n":
|
||||
// drop the empty tail after the last "\n" and the blank line itself.
|
||||
if len(lines) < 3 {
|
||||
return egressRequest{}, egressBad("The request ended before its headers did, so the bastion sent nothing.")
|
||||
}
|
||||
lines = lines[:len(lines)-2]
|
||||
for i, ln := range lines {
|
||||
lines[i] = strings.TrimSuffix(ln, "\r")
|
||||
if strings.ContainsAny(lines[i], "\r\x00") {
|
||||
return egressRequest{}, egressBad("The request's headers carry a stray carriage return or NUL, so the bastion sent nothing.")
|
||||
}
|
||||
}
|
||||
parts := strings.Split(lines[0], " ")
|
||||
if len(parts) != 3 || parts[0] == "" || parts[1] == "" || parts[2] == "" || !egressTokenOK(parts[0]) {
|
||||
return egressRequest{}, egressBad("The request line is not 'METHOD target HTTP/1.x', so the bastion sent nothing.")
|
||||
}
|
||||
method, rawTarget, version := strings.ToUpper(parts[0]), parts[1], parts[2]
|
||||
if version != "HTTP/1.1" && version != "HTTP/1.0" {
|
||||
return egressRequest{}, egressBad("The bastion forwards HTTP/1.0 and HTTP/1.1 only, so it sent nothing.")
|
||||
}
|
||||
type header struct{ name, value string }
|
||||
var headers []header
|
||||
for _, ln := range lines[1:] {
|
||||
if strings.HasPrefix(ln, " ") || strings.HasPrefix(ln, "\t") {
|
||||
return egressRequest{}, egressBad("A header is folded onto a second line (obsolete line folding), so the bastion sent nothing.")
|
||||
}
|
||||
name, value, ok := strings.Cut(ln, ":")
|
||||
if !ok || !egressTokenOK(name) {
|
||||
return egressRequest{}, egressBad("A header line is not 'Name: value' (no space may come before the colon), so the bastion sent nothing.")
|
||||
}
|
||||
headers = append(headers, header{name, strings.Trim(value, " \t")})
|
||||
}
|
||||
values := func(n string) []string {
|
||||
var out []string
|
||||
for _, h := range headers {
|
||||
if strings.EqualFold(h.name, n) {
|
||||
out = append(out, h.value)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
hosts := values("host")
|
||||
if len(hosts) > 1 {
|
||||
return egressRequest{}, egressBad("The request carries more than one Host header, so the bastion sent nothing.")
|
||||
}
|
||||
var host, target, authority string
|
||||
var port int
|
||||
if scheme, after, abs := strings.Cut(rawTarget, "://"); abs {
|
||||
if strings.ToLower(scheme) != "http" {
|
||||
// https:// in absolute form asks the lane to make the TLS connection,
|
||||
// which it does not do: it used to send the request in plain text to
|
||||
// port 80. Every client sends https as a CONNECT.
|
||||
return egressRequest{}, egressBad("The bastion forwards plain http:// requests only; send https as a CONNECT. Nothing was sent.")
|
||||
}
|
||||
cut := len(after)
|
||||
for _, sep := range []string{"/", "?", "#"} {
|
||||
if i := strings.Index(after, sep); i >= 0 && i < cut {
|
||||
cut = i
|
||||
}
|
||||
}
|
||||
var ok bool
|
||||
if host, port, ok = parseEgressAuthority(after[:cut], 80); !ok {
|
||||
return egressRequest{}, egressBad(egressNoHost)
|
||||
}
|
||||
path, _, _ := strings.Cut(after[cut:], "#")
|
||||
if !strings.HasPrefix(path, "/") {
|
||||
path = "/" + path
|
||||
}
|
||||
authority = fmtEgressAuthority(host, port, 80)
|
||||
target = "http://" + authority + path
|
||||
} else {
|
||||
if !strings.HasPrefix(rawTarget, "/") && rawTarget != "*" {
|
||||
return egressRequest{}, egressBad(egressNoHost)
|
||||
}
|
||||
hostHdr := ""
|
||||
if len(hosts) == 1 {
|
||||
hostHdr = hosts[0]
|
||||
}
|
||||
var ok bool
|
||||
if host, port, ok = parseEgressAuthority(hostHdr, 80); !ok {
|
||||
return egressRequest{}, egressBad(egressNoHost)
|
||||
}
|
||||
authority = fmtEgressAuthority(host, port, 80)
|
||||
target = rawTarget
|
||||
}
|
||||
|
||||
// Body framing (RFC 9112 section 6). Every reading two parsers could disagree
|
||||
// on is refused: both length headers, an unknown or doubled coding, a coding
|
||||
// on HTTP/1.0, Content-Length values that differ.
|
||||
req := egressRequest{host: host, port: port}
|
||||
te, cl := values("transfer-encoding"), values("content-length")
|
||||
if len(te) > 0 {
|
||||
var codings []string
|
||||
for _, v := range te {
|
||||
for _, c := range strings.Split(v, ",") {
|
||||
codings = append(codings, strings.ToLower(strings.TrimSpace(c)))
|
||||
}
|
||||
}
|
||||
if len(cl) > 0 || version == "HTTP/1.0" || len(codings) != 1 || codings[0] != "chunked" {
|
||||
return egressRequest{}, egressBad("The request's body length is ambiguous (Transfer-Encoding with " +
|
||||
"Content-Length, on HTTP/1.0, or a coding other than chunked), so the bastion sent nothing.")
|
||||
}
|
||||
req.chunked = true
|
||||
} else if len(cl) > 0 {
|
||||
seen := map[string]bool{}
|
||||
for _, v := range cl {
|
||||
for _, c := range strings.Split(v, ",") {
|
||||
seen[strings.TrimSpace(c)] = true
|
||||
}
|
||||
}
|
||||
var one string
|
||||
for k := range seen {
|
||||
one = k
|
||||
}
|
||||
n, err := strconv.ParseInt(one, 10, 64)
|
||||
if len(seen) != 1 || one == "" || len(one) > 18 || err != nil || n < 0 || strings.TrimLeft(one, "0123456789") != "" {
|
||||
return egressRequest{}, egressBad("The request's Content-Length is not one number, so the bastion sent nothing.")
|
||||
}
|
||||
req.length = n
|
||||
}
|
||||
|
||||
named := map[string]bool{}
|
||||
for _, v := range values("connection") {
|
||||
for _, t := range strings.Split(v, ",") {
|
||||
named[strings.ToLower(strings.TrimSpace(t))] = true
|
||||
}
|
||||
}
|
||||
var b strings.Builder
|
||||
b.WriteString(method + " " + target + " " + version + "\r\n")
|
||||
b.WriteString("Host: " + authority + "\r\n")
|
||||
for _, h := range headers {
|
||||
lk := strings.ToLower(h.name)
|
||||
if egressHopByHop[lk] || named[lk] || lk == "host" || lk == "content-length" || lk == "transfer-encoding" {
|
||||
continue
|
||||
}
|
||||
b.WriteString(h.name + ": " + h.value + "\r\n")
|
||||
}
|
||||
if req.chunked {
|
||||
b.WriteString("Transfer-Encoding: chunked\r\n")
|
||||
} else if len(cl) > 0 {
|
||||
b.WriteString("Content-Length: " + strconv.FormatInt(req.length, 10) + "\r\n")
|
||||
}
|
||||
b.WriteString("Connection: close\r\n\r\n")
|
||||
req.head = []byte(b.String())
|
||||
return req, nil
|
||||
}
|
||||
|
||||
// forwardEgressBody copies the one request body from src to dst, framed as the
|
||||
// rebuilt head says. A chunked body goes out re-encoded (plain sizes, no
|
||||
// extensions, trailers read and dropped), so the destination parses exactly the
|
||||
// body this parser found.
|
||||
func forwardEgressBody(dst io.Writer, src *bufio.Reader, req egressRequest) error {
|
||||
if !req.chunked {
|
||||
if req.length == 0 {
|
||||
return nil
|
||||
}
|
||||
_, err := io.CopyN(dst, src, req.length)
|
||||
return err
|
||||
}
|
||||
cw := httputil.NewChunkedWriter(dst)
|
||||
if _, err := io.Copy(cw, httputil.NewChunkedReader(src)); err != nil {
|
||||
return err
|
||||
}
|
||||
// The trailer section, read to its blank line within a bound, and dropped.
|
||||
read := 0
|
||||
for {
|
||||
line, err := src.ReadSlice('\n')
|
||||
read += len(line)
|
||||
if err != nil || read > egressTrailerLimit {
|
||||
return errors.New("chunked body: bad trailer section")
|
||||
}
|
||||
if string(line) == "\r\n" || string(line) == "\n" {
|
||||
break
|
||||
}
|
||||
}
|
||||
if err := cw.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := io.WriteString(dst, "\r\n")
|
||||
return err
|
||||
}
|
||||
|
||||
// oneEgressExchange writes the one rebuilt request and its body to upstream,
|
||||
// and carries the destination's answer back until the destination closes.
|
||||
// Nothing the Host sends after the body is forwarded: its first byte closes
|
||||
// both sides.
|
||||
func oneEgressExchange(clientReader *bufio.Reader, client, upstream net.Conn, req egressRequest) {
|
||||
answered := make(chan struct{})
|
||||
go func() {
|
||||
_, _ = io.Copy(client, upstream)
|
||||
close(answered)
|
||||
_ = client.Close()
|
||||
}()
|
||||
defer func() {
|
||||
_ = upstream.Close()
|
||||
<-answered
|
||||
}()
|
||||
if _, err := upstream.Write(req.head); err != nil {
|
||||
return
|
||||
}
|
||||
if err := forwardEgressBody(upstream, clientReader, req); err != nil {
|
||||
return
|
||||
}
|
||||
// The request is complete. A further byte is a second request on this
|
||||
// connection: never forwarded, and the connection closes. End of stream
|
||||
// from the Host is passed on, as the splice did.
|
||||
go func() {
|
||||
var one [1]byte
|
||||
if n, _ := clientReader.Read(one[:]); n > 0 {
|
||||
_ = upstream.Close()
|
||||
_ = client.Close()
|
||||
return
|
||||
}
|
||||
if cw, ok := upstream.(interface{ CloseWrite() error }); ok {
|
||||
_ = cw.CloseWrite()
|
||||
}
|
||||
}()
|
||||
<-answered
|
||||
}
|
||||
@@ -7,8 +7,10 @@ package main
|
||||
|
||||
// The egress lane on a local (unanchored) box. It refuses this machine and
|
||||
// its link-local addresses, so a host cannot reach this machine's services
|
||||
// (ollama above all) around the switches that govern them, and it keeps the
|
||||
// person's LAN. An anchored box is unchanged: the public internet only.
|
||||
// (ollama above all) around the switches that govern them, and, since
|
||||
// 2026-10-26, the network around it as well: the LAN and the mesh
|
||||
// (egress_network_test.go). An anchored box is unchanged: the public internet
|
||||
// only.
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
@@ -57,13 +59,14 @@ func localEgressFixture(t *testing.T) {
|
||||
"self.example.test": {"192.168.50.7"},
|
||||
"nas.example.test": {"192.168.50.30"},
|
||||
"dual.example.test": {"2001:db8::8", "198.51.100.8"},
|
||||
"site.example.test": {"198.51.100.9"},
|
||||
})
|
||||
}
|
||||
|
||||
// MUTATION: restore the early return for an unanchored box in egressVettedAddrs.
|
||||
// RESULT: red — 127.0.0.1:11434 is handed to the dial, and a name is dialled
|
||||
// as a name instead of the address that was vetted.
|
||||
func TestLocalEgressRefusesThisMachineAndKeepsTheLAN(t *testing.T) {
|
||||
func TestLocalEgressRefusesThisMachineAndTheLAN(t *testing.T) {
|
||||
localEgressFixture(t)
|
||||
ctx := context.Background()
|
||||
for _, hp := range []string{
|
||||
@@ -75,24 +78,22 @@ func TestLocalEgressRefusesThisMachineAndKeepsTheLAN(t *testing.T) {
|
||||
"203.0.113.5:80", // a range a `local` route delivers here
|
||||
"ollama.example.test:11434", "mixed.example.test:443", "self.example.test:11434",
|
||||
"unknown.example.test:443",
|
||||
// the network around this machine (egress_network_test.go has the rest)
|
||||
"192.168.50.20:8080", "10.1.2.3:443", "172.16.4.4:22", "[fd12:3456::9]:80", "100.64.0.9:80",
|
||||
"nas.example.test:443",
|
||||
} {
|
||||
if addr, why := egressVettedAddr(ctx, hp); why == "" || addr != "" {
|
||||
t.Errorf("%s must be refused on a local box, got dial %q", hp, addr)
|
||||
}
|
||||
}
|
||||
for hp, want := range map[string][]string{
|
||||
"192.168.50.20:8080": {"192.168.50.20:8080"},
|
||||
"10.1.2.3:443": {"10.1.2.3:443"},
|
||||
"172.16.4.4:22": {"172.16.4.4:22"},
|
||||
"[fd12:3456::9]:80": {"[fd12:3456::9]:80"},
|
||||
"100.64.0.9:80": {"100.64.0.9:80"},
|
||||
"198.51.100.8:443": {"198.51.100.8:443"},
|
||||
"nas.example.test:443": {"192.168.50.30:443"},
|
||||
"site.example.test:443": {"198.51.100.9:443"},
|
||||
"dual.example.test:443": {"[2001:db8::8]:443", "198.51.100.8:443"},
|
||||
} {
|
||||
got, why := egressVettedAddrs(ctx, hp)
|
||||
if why != "" || !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("%s is the LAN or the internet, so a local box dials %v: got %v %q", hp, want, got, why)
|
||||
t.Errorf("%s is the internet, so a local box dials %v: got %v %q", hp, want, got, why)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -114,8 +115,8 @@ func TestAnchoredEgressStillRefusesTheLAN(t *testing.T) {
|
||||
}
|
||||
|
||||
// Through the lane itself, with a real dialer: a request for this machine is
|
||||
// answered 403 and never reaches the listener; a LAN request is dialled at its
|
||||
// vetted address; a dual-stack name falls back to its second address.
|
||||
// answered 403 and never reaches the listener; a public name is dialled at
|
||||
// its vetted address; a dual-stack name falls back to its second address.
|
||||
func TestLocalEgressLaneNeverDialsThisMachine(t *testing.T) {
|
||||
localEgressFixture(t)
|
||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
@@ -164,11 +165,11 @@ func TestLocalEgressLaneNeverDialsThisMachine(t *testing.T) {
|
||||
asked = append(asked, addr)
|
||||
return nil, errors.New("stop here")
|
||||
}
|
||||
if line := ask(recording, "CONNECT nas.example.test:443 HTTP/1.1\r\n\r\n"); !strings.HasPrefix(line, "HTTP/1.1 502") {
|
||||
t.Fatalf("a LAN destination is dialled (and this dial fails): %q", line)
|
||||
if line := ask(recording, "CONNECT site.example.test:443 HTTP/1.1\r\n\r\n"); !strings.HasPrefix(line, "HTTP/1.1 502") {
|
||||
t.Fatalf("a public destination is dialled (and this dial fails): %q", line)
|
||||
}
|
||||
if !reflect.DeepEqual(asked, []string{"192.168.50.30:443"}) {
|
||||
t.Fatalf("the vetted LAN address, never the name: %v", asked)
|
||||
if !reflect.DeepEqual(asked, []string{"198.51.100.9:443"}) {
|
||||
t.Fatalf("the vetted address, never the name: %v", asked)
|
||||
}
|
||||
|
||||
asked = nil
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
//go:build !windows
|
||||
|
||||
// Tagged !windows: it borrows localEgressFixture and answerEgressNames
|
||||
// (egress_local_test.go).
|
||||
|
||||
package main
|
||||
|
||||
// The egress lane on a local (unanchored) box refuses the network around the
|
||||
// machine as well as the machine itself: the LAN, the mesh and every other
|
||||
// address that is not the public internet, by address and by any address a
|
||||
// name resolves to. The Host's egress gate holds every dial to the same rule;
|
||||
// a person's own machines are reached through their approved lanes.
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// MUTATION: drop the local-box internalIP loop at the end of egressVettedAddrs.
|
||||
// RESULT: red — 192.168.50.20, nas.example.test and the rest are handed to the
|
||||
// dial, and the lane dials 192.168.50.30.
|
||||
func TestLocalEgressRefusesTheLANAndTheMesh(t *testing.T) {
|
||||
localEgressFixture(t)
|
||||
answerEgressNames(t, map[string][]string{
|
||||
"nas.example.test": {"192.168.50.30"},
|
||||
"half.example.test": {"198.51.100.8", "10.1.2.3"},
|
||||
"mesh.example.test": {"fd12:3456::9"},
|
||||
"nat.example.test": {"64:ff9b::a01:203"}, // NAT64 for 10.1.2.3
|
||||
"pub.example.test": {"198.51.100.8"},
|
||||
})
|
||||
ctx := context.Background()
|
||||
for _, hp := range []string{
|
||||
"192.168.50.20:8080", "10.1.2.3:443", "172.16.4.4:22", "[fd12:3456::9]:80", "100.64.0.9:80",
|
||||
"198.18.0.1:80", "[64:ff9b::a01:203]:443",
|
||||
"nas.example.test:443", "half.example.test:443", "mesh.example.test:80", "nat.example.test:443",
|
||||
} {
|
||||
if addrs, why := egressVettedAddrs(ctx, hp); why == "" || addrs != nil || !strings.Contains(why, "approved lanes") {
|
||||
t.Errorf("%s is not the public internet and must be refused on a local box, got dial %v %q", hp, addrs, why)
|
||||
}
|
||||
}
|
||||
if got, why := egressVettedAddrs(ctx, "pub.example.test:443"); why != "" || !reflect.DeepEqual(got, []string{"198.51.100.8:443"}) {
|
||||
t.Fatalf("the public internet passes at its vetted address: %v %q", got, why)
|
||||
}
|
||||
|
||||
// Through the lane itself: 403, and nothing is dialled.
|
||||
var asked []string
|
||||
recording := func(_ context.Context, _, addr string) (net.Conn, error) {
|
||||
asked = append(asked, addr)
|
||||
return nil, errors.New("stop here")
|
||||
}
|
||||
for _, req := range []string{
|
||||
"CONNECT nas.example.test:443 HTTP/1.1\r\n\r\n",
|
||||
"GET http://10.1.2.3/ HTTP/1.1\r\nHost: 10.1.2.3\r\n\r\n",
|
||||
"CONNECT half.example.test:443 HTTP/1.1\r\n\r\n",
|
||||
} {
|
||||
srv, cli := net.Pipe()
|
||||
go handleEgressConn(srv, recording, "home.host.example")
|
||||
_ = cli.SetDeadline(time.Now().Add(5 * time.Second))
|
||||
_, _ = io.WriteString(cli, req)
|
||||
line, _ := bufio.NewReader(cli).ReadString('\n')
|
||||
cli.Close()
|
||||
if !strings.HasPrefix(line, "HTTP/1.1 403") {
|
||||
t.Errorf("a local box must refuse %q, answered %q", req, line)
|
||||
}
|
||||
}
|
||||
if len(asked) != 0 {
|
||||
t.Fatalf("nothing on the network around this machine is dialled: %v", asked)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,393 @@
|
||||
//go:build !windows
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The bastion's egress lanes carry ONE plain-HTTP request per connection, read
|
||||
// the way the Host's egress gate reads it, and every refusal says
|
||||
// why in its body.
|
||||
//
|
||||
// MUTATIONS, each red:
|
||||
// - handleEgressConn writes head.String() to upstream and splices, as before:
|
||||
// TestEgressPlainHTTPCarriesOneRequestPerConnection, ...SendsTheHostItChecked;
|
||||
// - oneEgressExchange's watcher forwards instead of closing: ...OneRequest...;
|
||||
// - parseEgressAuthority accepts "@" or cuts the authority at "/" only:
|
||||
// TestEgressPlainHTTPReadsTheHostAsTheGateDoes;
|
||||
// - writeEgressRefusal writes Content-Length: 0: TestEgressRefusalsCarryABody.
|
||||
|
||||
// recordingOrigin stands in for a destination that keeps a connection open
|
||||
// after answering (it ignores Connection: close, as a careless server may),
|
||||
// and records every byte one connection carried to it.
|
||||
type recordingOrigin struct {
|
||||
ln net.Listener
|
||||
conns chan string
|
||||
// silent: never answer, so the lane never holds an unread answer when it
|
||||
// closes (a close with unread data is a reset, which can cost the origin
|
||||
// bytes it has not read yet).
|
||||
silent bool
|
||||
}
|
||||
|
||||
func newSilentOrigin(t *testing.T) *recordingOrigin {
|
||||
o := newRecordingOrigin(t)
|
||||
o.silent = true
|
||||
return o
|
||||
}
|
||||
|
||||
func newRecordingOrigin(t *testing.T) *recordingOrigin {
|
||||
t.Helper()
|
||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
o := &recordingOrigin{ln: ln, conns: make(chan string, 16)}
|
||||
t.Cleanup(func() { _ = ln.Close() })
|
||||
go func() {
|
||||
for {
|
||||
c, err := ln.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
go o.serve(c)
|
||||
}
|
||||
}()
|
||||
return o
|
||||
}
|
||||
|
||||
func (o *recordingOrigin) serve(c net.Conn) {
|
||||
defer c.Close()
|
||||
var mu sync.Mutex
|
||||
var got bytes.Buffer
|
||||
answered := false
|
||||
buf := make([]byte, 4096)
|
||||
for {
|
||||
_ = c.SetReadDeadline(time.Now().Add(800 * time.Millisecond))
|
||||
n, err := c.Read(buf)
|
||||
mu.Lock()
|
||||
got.Write(buf[:n])
|
||||
if !o.silent && !answered && bytes.Contains(got.Bytes(), []byte("\r\n\r\n")) {
|
||||
answered = true
|
||||
_, _ = io.WriteString(c, "HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")
|
||||
}
|
||||
mu.Unlock()
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
o.conns <- got.String()
|
||||
}
|
||||
|
||||
// next is everything the origin's next connection carried, once it ended.
|
||||
func (o *recordingOrigin) next(t *testing.T) string {
|
||||
t.Helper()
|
||||
s, ok := o.nextWithin(8 * time.Second)
|
||||
if !ok {
|
||||
t.Fatal("the destination never saw its connection end")
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func (o *recordingOrigin) nextWithin(d time.Duration) (string, bool) {
|
||||
select {
|
||||
case s := <-o.conns:
|
||||
return s, true
|
||||
case <-time.After(d):
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
|
||||
func (o *recordingOrigin) addr() string { return o.ln.Addr().String() }
|
||||
|
||||
// A keep-alive connection carries one request to the destination, never two:
|
||||
// neither a request sent after the first answer (a client reusing the
|
||||
// connection) nor one pipelined behind the first.
|
||||
func TestEgressPlainHTTPCarriesOneRequestPerConnection(t *testing.T) {
|
||||
first := "GET http://198.51.100.30/first HTTP/1.1\r\nHost: 198.51.100.30\r\n\r\n"
|
||||
second := "GET http://198.51.100.30/second HTTP/1.1\r\nHost: 198.51.100.30\r\n\r\n"
|
||||
|
||||
t.Run("reused after the answer", func(t *testing.T) {
|
||||
o := newRecordingOrigin(t)
|
||||
var asked []string
|
||||
cli, cleanup := dialHandler(t, redirectDial(o.addr(), &asked))
|
||||
defer cleanup()
|
||||
_, _ = io.WriteString(cli, first)
|
||||
br := bufio.NewReader(cli)
|
||||
resp, err := http.ReadResponse(br, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("the first request got no answer: %v", err)
|
||||
}
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
if string(body) != "ok" {
|
||||
t.Fatalf("the first answer was %q", body)
|
||||
}
|
||||
_, _ = io.WriteString(cli, second)
|
||||
got := o.next(t)
|
||||
if strings.Contains(got, "/second") {
|
||||
t.Fatalf("a second request on one connection reached the destination unvetted:\n%s", got)
|
||||
}
|
||||
if !strings.Contains(got, "\r\nConnection: close\r\n") {
|
||||
t.Fatalf("the request went out without Connection: close:\n%s", got)
|
||||
}
|
||||
_ = cli.SetReadDeadline(time.Now().Add(3 * time.Second))
|
||||
if rest, _ := io.ReadAll(br); len(rest) != 0 {
|
||||
t.Fatalf("the lane answered the second request: %q", rest)
|
||||
}
|
||||
if len(asked) != 1 {
|
||||
t.Fatalf("one connection, one dial: %v", asked)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("pipelined", func(t *testing.T) {
|
||||
o := newRecordingOrigin(t)
|
||||
var asked []string
|
||||
cli, cleanup := dialHandler(t, redirectDial(o.addr(), &asked))
|
||||
defer cleanup()
|
||||
_, _ = io.WriteString(cli, first+second)
|
||||
got := o.next(t)
|
||||
if !strings.Contains(got, "GET http://198.51.100.30/first HTTP/1.1\r\n") {
|
||||
t.Fatalf("the first request did not reach the destination:\n%s", got)
|
||||
}
|
||||
if strings.Contains(got, "/second") {
|
||||
t.Fatalf("a pipelined second request reached the destination unvetted:\n%s", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// The destination reads the host the lane vetted, in the one Host header; hop
|
||||
// headers, those Connection names and a proxy credential stay on this hop.
|
||||
func TestEgressPlainHTTPSendsTheHostItChecked(t *testing.T) {
|
||||
o := newRecordingOrigin(t)
|
||||
var asked []string
|
||||
cli, cleanup := dialHandler(t, redirectDial(o.addr(), &asked))
|
||||
defer cleanup()
|
||||
_, _ = io.WriteString(cli, "GET http://198.51.100.31/p?q=1 HTTP/1.1\r\n"+
|
||||
"Host: inside.example.test\r\nProxy-Authorization: Basic eDp5\r\n"+
|
||||
"Connection: keep-alive, X-Hop\r\nX-Hop: 1\r\nX-Keep: 2\r\n\r\n")
|
||||
got := o.next(t)
|
||||
head, _, _ := strings.Cut(got, "\r\n\r\n")
|
||||
want := "GET http://198.51.100.31/p?q=1 HTTP/1.1\r\nHost: 198.51.100.31\r\nX-Keep: 2\r\nConnection: close"
|
||||
if head != want {
|
||||
t.Fatalf("the destination got\n%q\nwant\n%q", head, want)
|
||||
}
|
||||
if len(asked) != 1 || asked[0] != "198.51.100.31:80" {
|
||||
t.Fatalf("the lane must dial the address it vetted: %v", asked)
|
||||
}
|
||||
}
|
||||
|
||||
// stubEgressNames answers names without DNS and records each name exactly as
|
||||
// the lane asked for it.
|
||||
func stubEgressNames(t *testing.T, m map[string]string) *[]string {
|
||||
t.Helper()
|
||||
var mu sync.Mutex
|
||||
var asked []string
|
||||
old := egressResolve
|
||||
t.Cleanup(func() { egressResolve = old })
|
||||
egressResolve = func(_ context.Context, host string) ([]net.IPAddr, error) {
|
||||
mu.Lock()
|
||||
asked = append(asked, host)
|
||||
mu.Unlock()
|
||||
ip, ok := m[host]
|
||||
if !ok {
|
||||
return nil, errors.New("no such host")
|
||||
}
|
||||
return []net.IPAddr{{IP: net.ParseIP(ip)}}, nil
|
||||
}
|
||||
return &asked
|
||||
}
|
||||
|
||||
// The host is read as the gate reads it: the authority ends at the first "/",
|
||||
// "?" or "#", is lower-cased and loses one trailing dot; userinfo, an
|
||||
// unbracketed IPv6 literal, a bad port, https in absolute form, two Host
|
||||
// headers and an ambiguous body length are refused with 400 and a body, before
|
||||
// anything is resolved or dialled.
|
||||
func TestEgressPlainHTTPReadsTheHostAsTheGateDoes(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
req, wantName, wantLine string
|
||||
}{
|
||||
{"GET http://public.example.test?@10.0.0.1/ HTTP/1.1\r\n\r\n", "public.example.test",
|
||||
"GET http://public.example.test/?@10.0.0.1/ HTTP/1.1"},
|
||||
{"GET http://public.example.test#@10.0.0.1/ HTTP/1.1\r\n\r\n", "public.example.test",
|
||||
"GET http://public.example.test/ HTTP/1.1"},
|
||||
{"GET http://Public.Example.Test./x HTTP/1.1\r\n\r\n", "public.example.test",
|
||||
"GET http://public.example.test/x HTTP/1.1"},
|
||||
{"GET /x HTTP/1.1\r\nHost: Public.Example.Test:8080\r\n\r\n", "public.example.test",
|
||||
"GET /x HTTP/1.1"},
|
||||
} {
|
||||
o := newRecordingOrigin(t)
|
||||
names := stubEgressNames(t, map[string]string{"public.example.test": "198.51.100.40"})
|
||||
var asked []string
|
||||
cli, cleanup := dialHandler(t, redirectDial(o.addr(), &asked))
|
||||
_, _ = io.WriteString(cli, c.req)
|
||||
got, _ := o.nextWithin(4 * time.Second)
|
||||
cleanup()
|
||||
if len(*names) != 1 || (*names)[0] != c.wantName {
|
||||
t.Errorf("%q: the lane resolved %q; want only %q", c.req, *names, c.wantName)
|
||||
}
|
||||
if line, _, _ := strings.Cut(got, "\r\n"); line != c.wantLine {
|
||||
t.Errorf("%q: the destination got the request line %q; want %q", c.req, line, c.wantLine)
|
||||
}
|
||||
}
|
||||
|
||||
for _, req := range []string{
|
||||
"GET http://user:pw@public.example.test/ HTTP/1.1\r\n\r\n",
|
||||
"GET http://198.51.100.41:80@public.example.test/ HTTP/1.1\r\n\r\n",
|
||||
"GET http://2001:db8::1/ HTTP/1.1\r\n\r\n",
|
||||
"GET http://public.example.test:80x/ HTTP/1.1\r\n\r\n",
|
||||
"GET https://public.example.test/ HTTP/1.1\r\n\r\n",
|
||||
"GET / HTTP/1.1\r\nHost: public.example.test\r\nHost: 198.51.100.41\r\n\r\n",
|
||||
"GET / HTTP/1.1\r\n\r\n",
|
||||
"POST http://public.example.test/ HTTP/1.1\r\nContent-Length: 3\r\nTransfer-Encoding: chunked\r\n\r\n",
|
||||
"POST http://public.example.test/ HTTP/1.1\r\nContent-Length: 3\r\nContent-Length: 4\r\n\r\n",
|
||||
"POST http://public.example.test/ HTTP/1.1\r\nTransfer-Encoding: gzip, chunked\r\n\r\n",
|
||||
"GET http://public.example.test/ HTTP/1.1\r\n X-Folded: 1\r\n\r\n",
|
||||
"CONNECT user@public.example.test:443 HTTP/1.1\r\n\r\n",
|
||||
"CONNECT 2001:db8::1:443 HTTP/1.1\r\n\r\n",
|
||||
"CONNECT [198.51.100.41]:443 HTTP/1.1\r\n\r\n",
|
||||
} {
|
||||
names := stubEgressNames(t, map[string]string{"public.example.test": "198.51.100.40"})
|
||||
var asked []string
|
||||
cli, cleanup := dialHandler(t, redirectDial("127.0.0.1:9", &asked))
|
||||
_, _ = io.WriteString(cli, req)
|
||||
resp, err := http.ReadResponse(bufio.NewReader(cli), nil)
|
||||
if err != nil {
|
||||
t.Errorf("%q: no answer: %v", req, err)
|
||||
cleanup()
|
||||
continue
|
||||
}
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
cleanup()
|
||||
if resp.StatusCode != 400 || len(body) == 0 || int64(len(body)) != resp.ContentLength {
|
||||
t.Errorf("%q: answered %d with body %q; want 400 with a body saying why", req, resp.StatusCode, body)
|
||||
}
|
||||
if len(*names) != 0 || len(asked) != 0 {
|
||||
t.Errorf("%q: refused requests resolve and dial nothing; resolved %q, dialled %q", req, *names, asked)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A request body goes out framed as its head says, chunked bodies re-encoded,
|
||||
// and nothing after it.
|
||||
func TestEgressPlainHTTPBodyIsFramed(t *testing.T) {
|
||||
for _, c := range []struct{ req, wantBody string }{
|
||||
{"POST http://198.51.100.42/u HTTP/1.1\r\nHost: 198.51.100.42\r\nContent-Length: 5\r\n\r\nhello" +
|
||||
"GET http://198.51.100.42/smuggled HTTP/1.1\r\n\r\n", "hello"},
|
||||
{"POST http://198.51.100.42/u HTTP/1.1\r\nHost: 198.51.100.42\r\nTransfer-Encoding: chunked\r\n\r\n" +
|
||||
"5;ext=1\r\nhello\r\n0\r\nX-Trailer: 1\r\n\r\nGET /smuggled HTTP/1.1\r\n\r\n", "5\r\nhello\r\n0\r\n\r\n"},
|
||||
} {
|
||||
o := newSilentOrigin(t)
|
||||
var asked []string
|
||||
cli, cleanup := dialHandler(t, redirectDial(o.addr(), &asked))
|
||||
_, _ = io.WriteString(cli, c.req)
|
||||
got := o.next(t)
|
||||
cleanup()
|
||||
_, body, _ := strings.Cut(got, "\r\n\r\n")
|
||||
if body != c.wantBody {
|
||||
t.Errorf("the destination got the body %q; want %q", body, c.wantBody)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A refusal says why in its body, not only in a header: a 403 from the vetting
|
||||
// and a 502 from a dial that found no answer, on both request forms, and the
|
||||
// coder-run proxy's refusals too.
|
||||
func TestEgressRefusalsCarryABody(t *testing.T) {
|
||||
failDial := func(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||
return nil, fmt.Errorf("no route")
|
||||
}
|
||||
okDial := func(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||
return nil, fmt.Errorf("must not dial %s", addr)
|
||||
}
|
||||
check := func(name string, resp *http.Response, err error, want int) {
|
||||
t.Helper()
|
||||
if err != nil {
|
||||
t.Errorf("%s: no answer: %v", name, err)
|
||||
return
|
||||
}
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
if resp.StatusCode != want || len(body) < 10 || int64(len(body)) != resp.ContentLength {
|
||||
t.Errorf("%s: answered %d with body %q (Content-Length %d); want %d with a short body saying why",
|
||||
name, resp.StatusCode, body, resp.ContentLength, want)
|
||||
}
|
||||
}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
dial egressDialer
|
||||
req string
|
||||
want int
|
||||
}{
|
||||
{"plain HTTP to this machine", okDial, "GET http://127.0.0.1:9/ HTTP/1.1\r\n\r\n", 403},
|
||||
{"CONNECT to this machine", okDial, "CONNECT 127.0.0.1:9 HTTP/1.1\r\n\r\n", 403},
|
||||
{"plain HTTP, no answer", failDial, "GET http://198.51.100.43/ HTTP/1.1\r\n\r\n", 502},
|
||||
{"CONNECT, no answer", failDial, "CONNECT 198.51.100.43:443 HTTP/1.1\r\n\r\n", 502},
|
||||
} {
|
||||
cli, cleanup := dialHandler(t, c.dial)
|
||||
_, _ = io.WriteString(cli, c.req)
|
||||
resp, err := http.ReadResponse(bufio.NewReader(cli), nil)
|
||||
check(c.name, resp, err, c.want)
|
||||
cleanup()
|
||||
}
|
||||
|
||||
oldDial := coderDialOut
|
||||
t.Cleanup(func() { coderDialOut = oldDial })
|
||||
coderDialOut = failDial
|
||||
for _, c := range []struct {
|
||||
name, req string
|
||||
want int
|
||||
}{
|
||||
{"coder run to a private address", "GET http://10.0.0.1/ HTTP/1.1\r\nHost: 10.0.0.1\r\n\r\n", 403},
|
||||
{"coder run, no answer", "GET http://93.184.216.34/ HTTP/1.1\r\nHost: 93.184.216.34\r\n\r\n", 502},
|
||||
} {
|
||||
boxEnd, runEnd := net.Pipe()
|
||||
var refused int64
|
||||
go handleCoderEgressConn(boxEnd, nil, &refused)
|
||||
_ = runEnd.SetDeadline(time.Now().Add(5 * time.Second))
|
||||
_, _ = io.WriteString(runEnd, c.req)
|
||||
resp, err := http.ReadResponse(bufio.NewReader(runEnd), nil)
|
||||
check(c.name, resp, err, c.want)
|
||||
_ = runEnd.Close()
|
||||
}
|
||||
}
|
||||
|
||||
// The coder-run proxy carries one request per connection too: the run's own
|
||||
// processes are the sender there.
|
||||
func TestCoderEgressCarriesOneRequestPerConnection(t *testing.T) {
|
||||
o := newRecordingOrigin(t)
|
||||
oldDial := coderDialOut
|
||||
t.Cleanup(func() { coderDialOut = oldDial })
|
||||
coderDialOut = func(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||
return (&net.Dialer{}).DialContext(ctx, network, o.addr())
|
||||
}
|
||||
boxEnd, runEnd := net.Pipe()
|
||||
var refused int64
|
||||
go handleCoderEgressConn(boxEnd, nil, &refused)
|
||||
_ = runEnd.SetDeadline(time.Now().Add(8 * time.Second))
|
||||
_, _ = io.WriteString(runEnd, "GET http://93.184.216.34/first HTTP/1.1\r\nHost: 93.184.216.34\r\n\r\n")
|
||||
br := bufio.NewReader(runEnd)
|
||||
resp, err := http.ReadResponse(br, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("the first request got no answer: %v", err)
|
||||
}
|
||||
_, _ = io.ReadAll(resp.Body)
|
||||
_, _ = io.WriteString(runEnd, "GET http://93.184.216.34/second HTTP/1.1\r\nHost: inside.example.test\r\n\r\n")
|
||||
got := o.next(t)
|
||||
_ = runEnd.Close()
|
||||
if strings.Contains(got, "/second") || strings.Contains(got, "inside.example.test") {
|
||||
t.Fatalf("a second request on one connection reached the destination unvetted:\n%s", got)
|
||||
}
|
||||
if !strings.Contains(got, "\r\nConnection: close\r\n") {
|
||||
t.Fatalf("the request went out without Connection: close:\n%s", got)
|
||||
}
|
||||
}
|
||||
+49
-14
@@ -14,35 +14,60 @@ import (
|
||||
|
||||
const egressWorkerCount = 4
|
||||
|
||||
// egressRefusedWait is how long a worker waits before it parks again after a
|
||||
// job it did not take (malformed, refused) or after its WebSocket closed while
|
||||
// parked. A job it took parks the worker again at once, so the person's own
|
||||
// traffic is never slowed by it; a Host that answers every worker with a job
|
||||
// this machine refuses costs it at most egressWorkerCount dials, handshakes
|
||||
// and refusals a second, not as many as the Host can send. The serve lane's
|
||||
// workers wait the same (serveRefusedWait).
|
||||
var egressRefusedWait = time.Second
|
||||
|
||||
// wsConn adapts a worker WebSocket into a net.Conn so the existing handleEgressConn drives
|
||||
// it unchanged. The host frames the proxied client stream as binary messages: 'D'+payload =
|
||||
// data, 'E' = end-of-stream. Reads surface the host's bytes; writes go back as 'D' frames;
|
||||
// Close/CloseWrite send 'E'.
|
||||
//
|
||||
// A 'D' frame is STREAMED into the caller's buffer, never read whole: the host picks a
|
||||
// frame's length, and one frame of any length must cost this machine no more memory
|
||||
// than the read it is asked for.
|
||||
type wsConn struct {
|
||||
ws *websocket.Conn
|
||||
rbuf []byte
|
||||
wmu sync.Mutex
|
||||
ws *websocket.Conn
|
||||
cur io.Reader // the rest of the 'D' frame being read, or nil
|
||||
wmu sync.Mutex
|
||||
}
|
||||
|
||||
func (c *wsConn) Read(p []byte) (int, error) {
|
||||
for len(c.rbuf) == 0 {
|
||||
mt, data, err := c.ws.ReadMessage()
|
||||
for {
|
||||
if c.cur != nil {
|
||||
n, err := c.cur.Read(p)
|
||||
if err == io.EOF {
|
||||
c.cur = nil
|
||||
err = nil
|
||||
}
|
||||
if n > 0 || err != nil {
|
||||
return n, err
|
||||
}
|
||||
continue
|
||||
}
|
||||
mt, r, err := c.ws.NextReader()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if mt != websocket.BinaryMessage || len(data) == 0 {
|
||||
continue
|
||||
if mt != websocket.BinaryMessage {
|
||||
continue // NextReader discards what is left of it
|
||||
}
|
||||
switch data[0] {
|
||||
var tag [1]byte
|
||||
if _, err := io.ReadFull(r, tag[:]); err != nil {
|
||||
continue // an empty frame
|
||||
}
|
||||
switch tag[0] {
|
||||
case 'D':
|
||||
c.rbuf = data[1:]
|
||||
c.cur = r
|
||||
case 'E':
|
||||
return 0, io.EOF
|
||||
}
|
||||
}
|
||||
n := copy(p, c.rbuf)
|
||||
c.rbuf = c.rbuf[n:]
|
||||
return n, nil
|
||||
}
|
||||
|
||||
func (c *wsConn) Write(p []byte) (int, error) {
|
||||
@@ -80,19 +105,25 @@ func (c *wsConn) SetWriteDeadline(t time.Time) error { return c.ws.SetWriteDeadl
|
||||
// bastion's tunnel IP) and serves one egress job per worker, reconnecting after each. Each
|
||||
// worker authenticates with the bastion's bearer, then blocks until the host dispatches a
|
||||
// request; handleEgressConn then dials it out the USER's connection (dialOut) and relays.
|
||||
// A worker reconnects at once after a job it took, and after egressRefusedWait after one
|
||||
// it refused or a close while parked. The func returned waits until every worker has
|
||||
// exited, which is once pctx is done and each one's job has ended.
|
||||
//
|
||||
// pctx is the pairing's context (run.go): when the anchor revokes a pairing the
|
||||
// workers must exit, not re-dial a closed tunnel every 3s for the process's life.
|
||||
func runEgressWorkers(pctx context.Context, self string, wsURL string, tlsCfg *tls.Config,
|
||||
dialHost func(ctx context.Context) (net.Conn, error), bearer string,
|
||||
dialOut egressDialer, n int) {
|
||||
dialOut egressDialer, n int) (wait func()) {
|
||||
dialer := &websocket.Dialer{
|
||||
NetDialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { return dialHost(ctx) },
|
||||
TLSClientConfig: tlsCfg,
|
||||
HandshakeTimeout: 20 * time.Second,
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < n; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for pctx.Err() == nil {
|
||||
ctx, cancel := context.WithTimeout(pctx, 25*time.Second)
|
||||
ws, _, err := dialer.DialContext(ctx, wsURL, http.Header{})
|
||||
@@ -108,11 +139,15 @@ func runEgressWorkers(pctx context.Context, self string, wsURL string, tlsCfg *t
|
||||
}
|
||||
stopKA := armWSKeepalive(ws) // ping/read-deadline: a dead tunnel unblocks the park
|
||||
unpark := context.AfterFunc(pctx, func() { _ = ws.Close() })
|
||||
handleEgressConn(&wsConn{ws: ws}, dialOut, self) // blocks until a job, serves it, then returns
|
||||
took := handleEgressConn(&wsConn{ws: ws}, dialOut, self) // blocks until a job, serves it, then returns
|
||||
unpark()
|
||||
stopKA()
|
||||
_ = ws.Close()
|
||||
if !took {
|
||||
sleepCtx(pctx, egressRefusedWait)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
return wg.Wait
|
||||
}
|
||||
|
||||
@@ -0,0 +1,166 @@
|
||||
//go:build !windows
|
||||
|
||||
// The egress worker pool (egress_relay.go: runEgressWorkers) against a
|
||||
// stand-in Host's /bastion-egress: how fast workers park again after a job
|
||||
// they took and after one they refused. Uses localEgressFixture
|
||||
// (egress_local_test.go), so it carries the same build tag.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/websocket"
|
||||
)
|
||||
|
||||
// egressHost is a Host's /bastion-egress as the pool meets it: every worker
|
||||
// that parks is counted and handed the job job(n) (n is its place in line),
|
||||
// and what the bastion answered is read to its end.
|
||||
type egressHost struct {
|
||||
parked atomic.Int32
|
||||
ok atomic.Int32 // answers that began "HTTP/1.1 200"
|
||||
}
|
||||
|
||||
func startEgressHost(t *testing.T, job func(n int32) string) (*egressHost, string, func(context.Context) (net.Conn, error)) {
|
||||
t.Helper()
|
||||
h := &egressHost{}
|
||||
up := websocket.Upgrader{}
|
||||
srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
c, err := up.Upgrade(w, r, nil)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer c.Close()
|
||||
if _, _, err := c.ReadMessage(); err != nil { // BEARER
|
||||
return
|
||||
}
|
||||
n := h.parked.Add(1)
|
||||
if err := c.WriteMessage(websocket.BinaryMessage, append([]byte{'D'}, job(n)...)); err != nil {
|
||||
return
|
||||
}
|
||||
_ = c.SetReadDeadline(time.Now().Add(5 * time.Second))
|
||||
var answer strings.Builder
|
||||
for {
|
||||
_, m, err := c.ReadMessage()
|
||||
if err != nil || len(m) == 0 || m[0] == 'E' {
|
||||
break
|
||||
}
|
||||
answer.Write(m[1:])
|
||||
}
|
||||
if strings.HasPrefix(answer.String(), "HTTP/1.1 200") {
|
||||
h.ok.Add(1)
|
||||
}
|
||||
}))
|
||||
srv.Config.ErrorLog = log.New(io.Discard, "", 0)
|
||||
srv.Start()
|
||||
t.Cleanup(srv.Close)
|
||||
addr := srv.Listener.Addr().String()
|
||||
dial := func(ctx context.Context) (net.Conn, error) {
|
||||
return (&net.Dialer{}).DialContext(ctx, "tcp", addr)
|
||||
}
|
||||
return h, "ws://" + addr + "/bastion-egress", dial
|
||||
}
|
||||
|
||||
// runEgressPool runs egressWorkerCount workers for the test's life, and at its
|
||||
// end stops them and waits until each has exited: a worker still vetting a job
|
||||
// would otherwise read the fixture's stubs while the test's cleanup puts them
|
||||
// back.
|
||||
func runEgressPool(t *testing.T, wsURL string, dialHost func(context.Context) (net.Conn, error), dialOut egressDialer) {
|
||||
t.Helper()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
wait := runEgressWorkers(ctx, "home.host.example", wsURL, nil, dialHost, "test-bearer", dialOut, egressWorkerCount)
|
||||
t.Cleanup(func() {
|
||||
cancel()
|
||||
done := make(chan struct{})
|
||||
go func() { wait(); close(done) }()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Error("the egress workers did not exit 10 s after the pool stopped")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// SECURITY (this machine's resources): a Host that answers every parked egress
|
||||
// worker with a job this machine refuses (a request for this machine, a line
|
||||
// that is no request) cannot make it dial, handshake and refuse in a loop as
|
||||
// fast as the Host answers. Each worker that refused waits before it parks
|
||||
// again, so the pool costs at most egressWorkerCount dials a second.
|
||||
//
|
||||
// MUTATION: in runEgressWorkers, park again at once whatever handleEgressConn
|
||||
// reports (as the pool did).
|
||||
// RESULT: red — hundreds of workers park in the 2 s window.
|
||||
func TestARefusingHostCannotSpinTheEgressWorkers(t *testing.T) {
|
||||
localEgressFixture(t)
|
||||
var dialled atomic.Int32
|
||||
dialOut := func(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||
dialled.Add(1)
|
||||
return nil, io.EOF
|
||||
}
|
||||
h, wsURL, dialHost := startEgressHost(t, func(n int32) string {
|
||||
if n%2 == 0 {
|
||||
return "NOT-A-REQUEST\r\n"
|
||||
}
|
||||
return "CONNECT localhost:443 HTTP/1.1\r\nHost: localhost:443\r\n\r\n"
|
||||
})
|
||||
runEgressPool(t, wsURL, dialHost, dialOut)
|
||||
time.Sleep(2 * time.Second)
|
||||
// egressWorkerCount park at the start, then each worker once a second:
|
||||
// about 12 in 2 s. A pool that parks again at once makes hundreds.
|
||||
n := h.parked.Load()
|
||||
t.Logf("%d workers parked in 2 s", n)
|
||||
if limit := int32(egressWorkerCount * 4); n > limit {
|
||||
t.Fatalf("%d egress workers parked in 2 s on a Host that refuses every job (at most %d): the pool re-dials in a loop", n, limit)
|
||||
}
|
||||
if n < egressWorkerCount {
|
||||
t.Fatalf("only %d workers parked: the pool did not run, so the count proves nothing", n)
|
||||
}
|
||||
if d := dialled.Load(); d != 0 {
|
||||
t.Fatalf("a refused job was dialled out %d times", d)
|
||||
}
|
||||
}
|
||||
|
||||
// The person's own traffic is not slowed: a worker that took a job parks again
|
||||
// at once, so 40 jobs through 4 workers finish in a fraction of the time one
|
||||
// wait per job would take (40 / 4 x egressRefusedWait = 10 s).
|
||||
//
|
||||
// MUTATION: in runEgressWorkers, wait egressRefusedWait after every job.
|
||||
// RESULT: red — 40 jobs need about 10 s.
|
||||
func TestEgressWorkersParkAgainAtOnceAfterAJobTheyTook(t *testing.T) {
|
||||
localEgressFixture(t)
|
||||
dialOut := func(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||
if addr != "198.51.100.9:443" {
|
||||
t.Errorf("dialled %q, not the address the request named", addr)
|
||||
}
|
||||
ours, theirs := net.Pipe()
|
||||
_ = theirs.Close() // the destination accepts and ends at once
|
||||
return ours, nil
|
||||
}
|
||||
h, wsURL, dialHost := startEgressHost(t, func(int32) string {
|
||||
return "CONNECT 198.51.100.9:443 HTTP/1.1\r\nHost: 198.51.100.9:443\r\n\r\n"
|
||||
})
|
||||
start := time.Now()
|
||||
runEgressPool(t, wsURL, dialHost, dialOut)
|
||||
const jobs = 40
|
||||
deadline := start.Add(10 * time.Second)
|
||||
for h.ok.Load() < jobs && time.Now().Before(deadline) {
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
took := time.Since(start)
|
||||
t.Logf("%d accepted jobs answered in %v", h.ok.Load(), took)
|
||||
if got := h.ok.Load(); got < jobs {
|
||||
t.Fatalf("only %d of %d accepted jobs were answered 200 in %v", got, jobs, took)
|
||||
}
|
||||
if limit := 3 * egressRefusedWait; took > limit {
|
||||
t.Fatalf("%d accepted jobs took %v (over %v): a worker that took a job waited before it parked again", jobs, took, limit)
|
||||
}
|
||||
}
|
||||
+51
-12
@@ -252,16 +252,39 @@ func harnessGate(self, dcnDir string, spec harnessSpec) string {
|
||||
// `BEARER <bearer> lane=session`); a harness worker refuses it so a shell can never hold
|
||||
// an agent worker for hours.
|
||||
func handleHarnessConn(ws *websocket.Conn, self, dcnDir string) {
|
||||
mt, data, err := ws.ReadMessage()
|
||||
if err != nil || mt != websocket.BinaryMessage || len(data) == 0 {
|
||||
// The spec is read up to harnessSpecMaxBytes (lane_bounds.go) and refused
|
||||
// past it; a frame of any other tag is not read at all.
|
||||
tag, body, tooBig, err := readLaneJob(ws, func(t byte) int64 {
|
||||
if t == 'H' {
|
||||
return harnessSpecMaxBytes
|
||||
}
|
||||
return 0
|
||||
})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if data[0] == 'S' {
|
||||
if tag == 'S' {
|
||||
sendHarness(ws, 'E', []byte("this is a harness worker; sessions go to a worker that said lane=session\n"))
|
||||
sendHarnessExit(ws, 1)
|
||||
return
|
||||
}
|
||||
if data[0] != 'H' {
|
||||
if tag != 'H' {
|
||||
return
|
||||
}
|
||||
if tooBig {
|
||||
// Gated like a spec that does not parse (below): a pairing the caps
|
||||
// refuse learns nothing more from a frame that was too large.
|
||||
if why := capsAllow(self, "harness:run"); why != "" {
|
||||
b, _ := json.Marshal(map[string]any{"ok": false, "error": why})
|
||||
sendHarness(ws, 'J', b)
|
||||
sendHarnessExit(ws, 126)
|
||||
return
|
||||
}
|
||||
b, _ := json.Marshal(map[string]any{"ok": false, "error": fmt.Sprintf(
|
||||
"the harness spec is more than the %d MiB a job on this lane may be; nothing of it was read as a job",
|
||||
harnessSpecMaxBytes>>20)})
|
||||
sendHarness(ws, 'J', b)
|
||||
sendHarnessExit(ws, 1)
|
||||
return
|
||||
}
|
||||
// The gate depends on WHAT is asked: a service-slot job (service.go) is the
|
||||
@@ -270,7 +293,7 @@ func handleHarnessConn(ws *websocket.Conn, self, dcnDir string) {
|
||||
// before its parse error is reported, exactly as it always was, so an
|
||||
// unauthorized pairing learns nothing from a malformed frame.
|
||||
var spec harnessSpec
|
||||
perr := json.Unmarshal(data[1:], &spec)
|
||||
perr := json.Unmarshal(body, &spec)
|
||||
gateOp := "harness:run"
|
||||
if perr == nil && spec.Mode == "service" {
|
||||
gateOp = serviceGateOp(spec.ServiceOp)
|
||||
@@ -332,12 +355,10 @@ func handleHarnessConn(ws *websocket.Conn, self, dcnDir string) {
|
||||
go func() { b, c := runHarness(spec); resCh <- harnessRes{b, c} }()
|
||||
closed := make(chan struct{})
|
||||
go func() {
|
||||
for {
|
||||
if _, _, err := ws.ReadMessage(); err != nil { // host close or drop
|
||||
close(closed)
|
||||
return
|
||||
}
|
||||
}
|
||||
// Nothing the host sends during a run is read as anything: each frame
|
||||
// is dropped as it arrives (lane_bounds.go), never held whole.
|
||||
_ = drainLane(ws) // returns on a host close or drop
|
||||
close(closed)
|
||||
}()
|
||||
ticker := time.NewTicker(8 * time.Second)
|
||||
defer ticker.Stop()
|
||||
@@ -951,7 +972,10 @@ func driveHarnessAgent(rt, dir string, spec harnessSpec, maxMin int) (agentRes m
|
||||
return nil, fmt.Errorf("harness /run: %v", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
rb, _ := io.ReadAll(resp.Body)
|
||||
rb, err := readHarnessRunReply(resp.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var res map[string]interface{}
|
||||
if err := json.Unmarshal(rb, &res); err != nil {
|
||||
return nil, fmt.Errorf("harness /run bad json (HTTP %d): %s", resp.StatusCode, truncate(string(rb), 200))
|
||||
@@ -959,6 +983,21 @@ func driveHarnessAgent(rt, dir string, spec harnessSpec, maxMin int) (agentRes m
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// readHarnessRunReply reads the agent container's answer to /run. The
|
||||
// container writes it, and it goes back to the host in one lane message, so it
|
||||
// is read to the most that message carries (uvicornWSMaxSize) and no further:
|
||||
// a reply that never ends must not grow in this machine's memory.
|
||||
func readHarnessRunReply(body io.Reader) ([]byte, error) {
|
||||
rb, err := io.ReadAll(io.LimitReader(body, uvicornWSMaxSize+1))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("harness /run: reading the reply: %v", err)
|
||||
}
|
||||
if len(rb) > uvicornWSMaxSize {
|
||||
return nil, fmt.Errorf("harness /run: the reply is more than the %d MiB the lane to the host carries", uvicornWSMaxSize>>20)
|
||||
}
|
||||
return rb, nil
|
||||
}
|
||||
|
||||
// harnessAgentUID is the user the harness image runs its agent as, the owner
|
||||
// of its home /home/privatae, where the credential folder is mounted.
|
||||
const harnessAgentUID = 10005
|
||||
|
||||
@@ -1010,15 +1010,15 @@ func TestAnchoredEgressRefusesLoopbackAndPrivateDestinations(t *testing.T) {
|
||||
if addr, why := egressVettedAddr(ctx, "localhost:80"); why == "" || addr != "" {
|
||||
t.Fatalf("a refused name yields no address: %q %q", addr, why)
|
||||
}
|
||||
// A local box refuses this machine too, and keeps its LAN
|
||||
// (egress_local_test.go has the rest).
|
||||
// A local box refuses this machine too, and the network around it
|
||||
// (egress_local_test.go and egress_network_test.go have the rest).
|
||||
t.Setenv("XDG_CONFIG_HOME", t.TempDir())
|
||||
stubOwnAddrs(t, loopbackOnly, localRange())
|
||||
if why := egressInternalRefusal(ctx, "127.0.0.1:11434"); why == "" {
|
||||
t.Fatal("a local box must refuse its own ollama on the egress lane")
|
||||
}
|
||||
if addr, why := egressVettedAddr(ctx, "192.168.1.1:80"); why != "" || addr != "192.168.1.1:80" {
|
||||
t.Fatalf("a local box keeps its LAN: %q %q", addr, why)
|
||||
if addr, why := egressVettedAddr(ctx, "192.168.1.1:80"); why == "" || addr != "" {
|
||||
t.Fatalf("a local box refuses its LAN on the egress lane: %q %q", addr, why)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -133,18 +133,39 @@ func inferenceFilterNeeded(self string, lane inferenceLane, scope modelScope) bo
|
||||
//
|
||||
// With nothing to enforce it is the original raw splice, byte for byte.
|
||||
func proxyInference(in net.Conn, upstream string, lane inferenceLane, self string) {
|
||||
proxyInferenceVia(in, dialUpstream(upstream), lane, self, spliceConnVia)
|
||||
}
|
||||
|
||||
// upstreamDial opens the connection to the service behind a forward.
|
||||
type upstreamDial func() (net.Conn, error)
|
||||
|
||||
// dialUpstream is the forwards' dial: TCP to the address, 5 s.
|
||||
func dialUpstream(upstream string) upstreamDial {
|
||||
return func() (net.Conn, error) { return net.DialTimeout("tcp", upstream, 5*time.Second) }
|
||||
}
|
||||
|
||||
// proxyInferenceVia is proxyInference with the dial, and the pass-through for
|
||||
// a connection with nothing to enforce, in the caller's hands. The serve lane
|
||||
// (servelane.go) uses it so its connections take the same decision and the
|
||||
// same filter as the tunnel's, while the dial goes to the address it vetted
|
||||
// and is watched by its idle bound.
|
||||
func proxyInferenceVia(in net.Conn, dial upstreamDial, lane inferenceLane, self string, pass func(net.Conn, upstreamDial)) {
|
||||
if inferenceFilterNeeded(self, lane, pairingModelScope(self)) {
|
||||
serveInferenceFiltered(in, upstream, lane, self)
|
||||
serveInferenceFilteredVia(in, dial, lane, self)
|
||||
return
|
||||
}
|
||||
spliceConn(in, upstream)
|
||||
pass(in, dial)
|
||||
}
|
||||
|
||||
// spliceConn is what the forwards did before this file existed, unchanged and
|
||||
// in one place now that three call sites share it.
|
||||
func spliceConn(in net.Conn, upstream string) {
|
||||
spliceConnVia(in, dialUpstream(upstream))
|
||||
}
|
||||
|
||||
func spliceConnVia(in net.Conn, dial upstreamDial) {
|
||||
defer in.Close()
|
||||
out, err := net.DialTimeout("tcp", upstream, 5*time.Second)
|
||||
out, err := dial()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
@@ -163,8 +184,12 @@ func spliceConn(in net.Conn, upstream string) {
|
||||
// grant on every request below, so there is nowhere for a caller — including a
|
||||
// test — to hand it one the grant does not say.
|
||||
func serveInferenceFiltered(in net.Conn, upstream string, lane inferenceLane, self string) {
|
||||
serveInferenceFilteredVia(in, dialUpstream(upstream), lane, self)
|
||||
}
|
||||
|
||||
func serveInferenceFilteredVia(in net.Conn, dial upstreamDial, lane inferenceLane, self string) {
|
||||
defer in.Close()
|
||||
out, err := net.DialTimeout("tcp", upstream, 5*time.Second)
|
||||
out, err := dial()
|
||||
if err != nil {
|
||||
writeFilterError(in, nil, http.StatusBadGateway, "the service behind this lane is not answering")
|
||||
return
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
package main
|
||||
|
||||
// What a paired host sends down a worker lane is read at a size this machine
|
||||
// chose, never at one the host chose: the host picks a frame's length, and a
|
||||
// frame of any length must not be held whole in this machine's memory. The
|
||||
// egress lane (egress_relay.go) and the context socket (contextsock.go) follow
|
||||
// the same rule.
|
||||
//
|
||||
// - A job's first frame (the H spec on a harness worker, the S spec on a
|
||||
// session worker) is read up to its lane's bound and refused past it, by
|
||||
// name, before any of it is parsed.
|
||||
// - What a lane carries after that is streamed: terminal input into the pty
|
||||
// (session.go), the replica's bytes into its socket (mirror.go). Frames that
|
||||
// nothing reads (a harness run and a root command, which read nothing after
|
||||
// their spec) are dropped as they arrive.
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
|
||||
"github.com/gorilla/websocket"
|
||||
)
|
||||
|
||||
const (
|
||||
// harnessSpecMaxBytes bounds the H frame. Its bulk is the worktree tar
|
||||
// (worktree_b64). The largest tree this lane hands back is
|
||||
// maxReturnedTreeBytes (9 MiB, so 12 MiB in base64) inside one message of
|
||||
// uvicornWSMaxSize (16 MiB), the most the host takes in (harness.go). A host
|
||||
// that sends that tree into its next run needs those 12 MiB, plus the
|
||||
// prompt and the run's credentials. Twice the lane's message size holds
|
||||
// that with 20 MiB to spare.
|
||||
harnessSpecMaxBytes = 2 * uvicornWSMaxSize
|
||||
|
||||
// sessionSpecMaxBytes bounds the S frame: a signed grant (op ids and the
|
||||
// person's own words), ids, a public key, and for session:root:exec the
|
||||
// command, itself held to rootExecCommandMax (64 KiB), which is at most
|
||||
// 384 KiB even when JSON escapes every byte as \u00XX. 1 MiB is the bound
|
||||
// Go's own HTTP server puts on a request head.
|
||||
sessionSpecMaxBytes = http.DefaultMaxHeaderBytes
|
||||
|
||||
// sessionInputMaxBytes bounds one frame on an open terminal. The host relays
|
||||
// what the person's browser sent it, one message each, and its own server
|
||||
// takes a message up to uvicornWSMaxSize; the tag is one byte more. Input is
|
||||
// streamed into the pty, so this caps a frame's length, not memory.
|
||||
sessionInputMaxBytes = uvicornWSMaxSize + 1
|
||||
|
||||
// sessionResizeMaxBytes bounds the body of an R frame, {"cols":N,"rows":N}.
|
||||
sessionResizeMaxBytes = 4 << 10
|
||||
|
||||
// heartbeatReplyMaxBytes bounds the host's answer to a heartbeat. It
|
||||
// carries directives and the lists of grants this box should hold, each
|
||||
// entry an id and a signature of about 130 bytes, so 4 MiB is tens of
|
||||
// thousands of grants. Nothing in it is bulk: an update names a version,
|
||||
// and the binary comes from the release manifest, not from this reply.
|
||||
heartbeatReplyMaxBytes = 4 << 20
|
||||
)
|
||||
|
||||
// errLaneNotJob is a first frame that is not a binary frame with a tag.
|
||||
var errLaneNotJob = errors.New("the first frame is not a job")
|
||||
|
||||
// readLaneJob reads a worker's first frame: its tag, and its body up to
|
||||
// maxFor(tag) bytes. A body past its bound is never kept: tooBig reports it,
|
||||
// and body is nil. A tag the caller does not take should get a bound of 0;
|
||||
// its body is then not read at all.
|
||||
func readLaneJob(ws *websocket.Conn, maxFor func(tag byte) int64) (tag byte, body []byte, tooBig bool, err error) {
|
||||
mt, r, err := ws.NextReader()
|
||||
if err != nil {
|
||||
return 0, nil, false, err
|
||||
}
|
||||
if mt != websocket.BinaryMessage {
|
||||
return 0, nil, false, errLaneNotJob
|
||||
}
|
||||
var t [1]byte
|
||||
if _, err := io.ReadFull(r, t[:]); err != nil {
|
||||
return 0, nil, false, errLaneNotJob // an empty frame
|
||||
}
|
||||
max := maxFor(t[0])
|
||||
if max <= 0 {
|
||||
return t[0], nil, false, nil
|
||||
}
|
||||
body, err = io.ReadAll(io.LimitReader(r, max+1))
|
||||
if err != nil {
|
||||
return t[0], nil, false, err
|
||||
}
|
||||
if int64(len(body)) > max {
|
||||
return t[0], nil, true, nil
|
||||
}
|
||||
return t[0], body, false, nil
|
||||
}
|
||||
|
||||
// drainLane reads and drops every frame until the socket fails, holding none
|
||||
// of them, and returns that failure. A lane that reads nothing after its job
|
||||
// still has to read, to answer pings and to notice the host going away.
|
||||
func drainLane(ws *websocket.Conn) error {
|
||||
for {
|
||||
_, r, err := ws.NextReader()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := io.Copy(io.Discard, r); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// readHeartbeatReply reads the host's answer to a heartbeat up to
|
||||
// heartbeatReplyMaxBytes. ok is false for a reply past the bound or one that
|
||||
// failed to arrive: such a beat is treated as unanswered, so none of its
|
||||
// directives runs and it does not count as the host answering.
|
||||
func readHeartbeatReply(resp *http.Response) (body []byte, ok bool) {
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, heartbeatReplyMaxBytes+1))
|
||||
if err != nil {
|
||||
return nil, false
|
||||
}
|
||||
if len(body) > heartbeatReplyMaxBytes {
|
||||
fmt.Printf("heartbeat: the host's reply is more than %d MiB; ignored\n", heartbeatReplyMaxBytes>>20)
|
||||
return nil, false
|
||||
}
|
||||
return body, true
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
package main
|
||||
|
||||
// The helpers lane_bounds.go and harness.go added for the reads that were
|
||||
// inline before (the heartbeat reply in run.go, the agent's /run reply in
|
||||
// driveHarnessAgent, the drains in harness.go and rootexec.go). They have no
|
||||
// counterpart on main to run against.
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/websocket"
|
||||
)
|
||||
|
||||
// boxLaneHost stands up the HOST side of one worker socket (host runs on the
|
||||
// upgraded server end) and returns the box side, dialled.
|
||||
func boxLaneHost(t *testing.T, host func(c *websocket.Conn)) *websocket.Conn {
|
||||
t.Helper()
|
||||
up := websocket.Upgrader{}
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
c, err := up.Upgrade(w, r, nil)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer c.Close()
|
||||
host(c)
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
ws, _, err := websocket.DefaultDialer.Dial("ws"+strings.TrimPrefix(srv.URL, "http"), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = ws.Close() })
|
||||
return ws
|
||||
}
|
||||
|
||||
// endless is a body that never ends; served counts what was taken from it.
|
||||
type endless struct{ served *atomic.Int64 }
|
||||
|
||||
func (e endless) Read(p []byte) (int, error) {
|
||||
for i := range p {
|
||||
p[i] = 'z'
|
||||
}
|
||||
e.served.Add(int64(len(p)))
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
func TestHeartbeatReplyIsReadOnlyToItsBound(t *testing.T) {
|
||||
var served atomic.Int64
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(200)
|
||||
_, _ = io.Copy(w, io.LimitReader(endless{&served}, 64<<20))
|
||||
}))
|
||||
defer srv.Close()
|
||||
resp, err := http.Get(srv.URL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, ok := readHeartbeatReply(resp)
|
||||
resp.Body.Close()
|
||||
if ok || body != nil {
|
||||
t.Fatalf("a 64 MiB heartbeat reply was taken (%d bytes)", len(body))
|
||||
}
|
||||
|
||||
small := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = w.Write([]byte(`{"ok":true,"dcn_grants":{"stores":[]}}`))
|
||||
}))
|
||||
defer small.Close()
|
||||
resp, err = http.Get(small.URL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, ok = readHeartbeatReply(resp)
|
||||
resp.Body.Close()
|
||||
if !ok || !strings.Contains(string(body), "dcn_grants") {
|
||||
t.Fatalf("an ordinary reply was not taken: ok=%v %q", ok, body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHarnessRunReplyIsReadOnlyToItsBound(t *testing.T) {
|
||||
var served atomic.Int64
|
||||
if _, err := readHarnessRunReply(endless{&served}); err == nil || !strings.Contains(err.Error(), "more than the 16 MiB") {
|
||||
t.Fatalf("an endless /run reply: %v", err)
|
||||
}
|
||||
if n := served.Load(); n > 17<<20 {
|
||||
t.Fatalf("an endless /run reply was read to %d MiB", n>>20)
|
||||
}
|
||||
big := append([]byte(`{"ok":true,"pad":"`), bytes.Repeat([]byte{'a'}, 15<<20)...)
|
||||
big = append(big, `"}`...)
|
||||
if rb, err := readHarnessRunReply(bytes.NewReader(big)); err != nil || len(rb) != len(big) {
|
||||
t.Fatalf("a 15 MiB reply the lane carries: %d bytes, %v", len(rb), err)
|
||||
}
|
||||
}
|
||||
|
||||
// A drained lane holds no frame whole, and ends when the host goes away.
|
||||
func TestDrainLaneHoldsNoFrame(t *testing.T) {
|
||||
const frame = 64 << 20
|
||||
payload := bytes.Repeat([]byte{'d'}, frame)
|
||||
start := make(chan struct{})
|
||||
box := boxLaneHost(t, func(c *websocket.Conn) {
|
||||
<-start
|
||||
_ = c.WriteMessage(websocket.BinaryMessage, payload)
|
||||
})
|
||||
runtime.GC()
|
||||
var before, after runtime.MemStats
|
||||
runtime.ReadMemStats(&before)
|
||||
close(start)
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- drainLane(box) }()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(20 * time.Second):
|
||||
t.Fatal("drainLane did not return after the host went away")
|
||||
}
|
||||
runtime.ReadMemStats(&after)
|
||||
if grew := after.TotalAlloc - before.TotalAlloc; grew > 8<<20 {
|
||||
t.Fatalf("draining a %d MiB frame allocated %d MiB", frame>>20, grew>>20)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,481 @@
|
||||
//go:build !windows
|
||||
|
||||
package main
|
||||
|
||||
// What a paired host sends down the harness, session, root-exec and mirror
|
||||
// lanes, and what a coder run sends its egress proxy, is read at the size this
|
||||
// machine chose (lane_bounds.go). Every test here drives only code that main
|
||||
// already has, so each one also runs against main, where it fails.
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/websocket"
|
||||
)
|
||||
|
||||
// hostSendsAJob is a host that writes one job frame while it reads the box's
|
||||
// replies, and reports whether its write completed and what came back.
|
||||
type hostResult struct {
|
||||
werr error
|
||||
frames map[byte][]byte
|
||||
}
|
||||
|
||||
func hostSendsAJob(frame []byte, out chan<- hostResult) func(c *websocket.Conn) {
|
||||
return hostSendsAJobAs(func(c *websocket.Conn) error {
|
||||
return c.WriteMessage(websocket.BinaryMessage, frame)
|
||||
}, out)
|
||||
}
|
||||
|
||||
// writeInPieces writes msg as ONE message carried by many small frames: the
|
||||
// writer flushes a continuation frame each time its 4 KiB buffer fills
|
||||
// (gorilla's default write buffer), so a bound that counted a frame instead
|
||||
// of the message would let all of it through.
|
||||
func writeInPieces(msg []byte) func(c *websocket.Conn) error {
|
||||
return func(c *websocket.Conn) error {
|
||||
w, err := c.NextWriter(websocket.BinaryMessage)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for off := 0; off < len(msg); off += 1 << 10 {
|
||||
if _, err := w.Write(msg[off:min(off+1<<10, len(msg))]); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return w.Close()
|
||||
}
|
||||
}
|
||||
|
||||
func hostSendsAJobAs(write func(c *websocket.Conn) error, out chan<- hostResult) func(c *websocket.Conn) {
|
||||
return func(c *websocket.Conn) {
|
||||
got := map[byte][]byte{}
|
||||
readDone := make(chan struct{})
|
||||
go func() {
|
||||
defer close(readDone)
|
||||
_ = c.SetReadDeadline(time.Now().Add(20 * time.Second))
|
||||
for {
|
||||
_, d, err := c.ReadMessage()
|
||||
if err != nil || len(d) == 0 {
|
||||
return
|
||||
}
|
||||
got[d[0]] = append(got[d[0]], d[1:]...)
|
||||
if d[0] == 'X' {
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
werr := write(c)
|
||||
<-readDone
|
||||
out <- hostResult{werr, got}
|
||||
}
|
||||
}
|
||||
|
||||
// jsonFrame is tag + a JSON object whose field `pad` carries n bytes of 'a'.
|
||||
func jsonFrame(tag byte, head string, n int) []byte {
|
||||
b := make([]byte, 0, len(head)+n+16)
|
||||
b = append(b, tag)
|
||||
b = append(b, head...)
|
||||
b = append(b, `,"pad":"`...)
|
||||
b = append(b, bytes.Repeat([]byte{'a'}, n)...)
|
||||
b = append(b, `"}`...)
|
||||
return b
|
||||
}
|
||||
|
||||
// A harness spec far past the bound is not read to its end: the box answers
|
||||
// that it is too large, names the bound, and closes, so the host's write of
|
||||
// the rest fails. On main the box read all of it into memory, then ran it.
|
||||
func TestHarnessWorkerStopsReadingASpecPastItsBound(t *testing.T) {
|
||||
const frame = 96 << 20
|
||||
out := make(chan hostResult, 1)
|
||||
box := boxLaneHost(t, hostSendsAJob(jsonFrame('H', `{"harness_id":"big","mode":"echo","prompt":"ping"`, frame), out))
|
||||
handleHarnessConn(box, "", "")
|
||||
_ = box.Close()
|
||||
r := <-out
|
||||
if r.werr == nil {
|
||||
t.Fatalf("the box read the whole %d MiB spec (it replied %.200q); it must stop at its bound", frame>>20, r.frames['J'])
|
||||
}
|
||||
if !strings.Contains(string(r.frames['J']), "more than the 32 MiB") || string(r.frames['X']) != "1" {
|
||||
t.Fatalf("want a refusal naming the 32 MiB bound and exit 1: J=%.200q X=%q", r.frames['J'], r.frames['X'])
|
||||
}
|
||||
}
|
||||
|
||||
// The bound counts the whole message, not one frame of it: a spec past the
|
||||
// bound, sent as one message in thousands of 4 KiB frames, is refused the same
|
||||
// way and not read to its end. On main the box read all of it.
|
||||
func TestHarnessWorkerBoundCountsEveryFrameOfTheSpec(t *testing.T) {
|
||||
const frame = 40 << 20
|
||||
out := make(chan hostResult, 1)
|
||||
spec := jsonFrame('H', `{"harness_id":"pieces","mode":"echo","prompt":"ping"`, frame)
|
||||
box := boxLaneHost(t, hostSendsAJobAs(writeInPieces(spec), out))
|
||||
handleHarnessConn(box, "", "")
|
||||
_ = box.Close()
|
||||
r := <-out
|
||||
if r.werr == nil {
|
||||
t.Fatalf("the box read the whole %d MiB spec sent in small frames (it replied %.200q); it must stop at its bound", frame>>20, r.frames['J'])
|
||||
}
|
||||
if !strings.Contains(string(r.frames['J']), "more than the 32 MiB") || string(r.frames['X']) != "1" {
|
||||
t.Fatalf("want a refusal naming the 32 MiB bound and exit 1: J=%.200q X=%q", r.frames['J'], r.frames['X'])
|
||||
}
|
||||
}
|
||||
|
||||
// The same for a session spec sent in small frames.
|
||||
func TestSessionWorkerBoundCountsEveryFrameOfTheSpec(t *testing.T) {
|
||||
env, _, _ := sessionTestEnv(t, []string{sessionOpOpen}, nil)
|
||||
withSink(t)
|
||||
const frame = 64 << 20
|
||||
out := make(chan hostResult, 1)
|
||||
spec := jsonFrame('S', `{"op":"session:open","session_id":"sess-pieces"`, frame)
|
||||
box := boxLaneHost(t, hostSendsAJobAs(writeInPieces(spec), out))
|
||||
handleSessionWorkerConn(box, "", env)
|
||||
_ = box.Close()
|
||||
r := <-out
|
||||
if r.werr == nil {
|
||||
t.Fatalf("the box read the whole %d MiB session spec sent in small frames (E=%q X=%q); it must stop at its bound",
|
||||
frame>>20, r.frames['E'], r.frames['X'])
|
||||
}
|
||||
if !strings.Contains(string(r.frames['E']), "more than the 1024 KiB") || string(r.frames['X']) != "1" {
|
||||
t.Fatalf("want a refusal naming the bound and exit 1: E=%q X=%q", r.frames['E'], r.frames['X'])
|
||||
}
|
||||
}
|
||||
|
||||
// A spec the lane can carry still runs: a tree as large as the most this lane
|
||||
// hands back (12 MiB in base64), with room on top, past the 16 MiB a host
|
||||
// takes in.
|
||||
func TestHarnessWorkerTakesALargeSpecTheLaneCarries(t *testing.T) {
|
||||
out := make(chan hostResult, 1)
|
||||
box := boxLaneHost(t, hostSendsAJob(jsonFrame('H', `{"harness_id":"tree","mode":"echo","prompt":"ping","worktree_b64":"dGFy"`, 20<<20), out))
|
||||
handleHarnessConn(box, "", "")
|
||||
r := <-out
|
||||
if r.werr != nil {
|
||||
t.Fatalf("the host could not send a 20 MiB spec: %v", r.werr)
|
||||
}
|
||||
var res map[string]any
|
||||
if err := json.Unmarshal(r.frames['J'], &res); err != nil || res["ok"] != true || res["echo"] != "ping" {
|
||||
t.Fatalf("a 20 MiB spec did not run: J=%.300q X=%q", r.frames['J'], r.frames['X'])
|
||||
}
|
||||
}
|
||||
|
||||
// The same on a session worker: an S frame far past its 1 MiB bound is
|
||||
// refused by name and not read to its end.
|
||||
func TestSessionWorkerStopsReadingASpecPastItsBound(t *testing.T) {
|
||||
env, _, _ := sessionTestEnv(t, []string{sessionOpOpen}, nil)
|
||||
withSink(t) // a box that wrongly took the spec opens no real shell
|
||||
const frame = 64 << 20
|
||||
out := make(chan hostResult, 1)
|
||||
box := boxLaneHost(t, hostSendsAJob(jsonFrame('S', `{"op":"session:open","session_id":"sess-big"`, frame), out))
|
||||
handleSessionWorkerConn(box, "", env)
|
||||
_ = box.Close()
|
||||
r := <-out
|
||||
if r.werr == nil {
|
||||
t.Fatalf("the box read the whole %d MiB session spec (E=%q X=%q); it must stop at its bound",
|
||||
frame>>20, r.frames['E'], r.frames['X'])
|
||||
}
|
||||
if !strings.Contains(string(r.frames['E']), "more than the 1024 KiB") || string(r.frames['X']) != "1" {
|
||||
t.Fatalf("want a refusal naming the bound and exit 1: E=%q X=%q", r.frames['E'], r.frames['X'])
|
||||
}
|
||||
}
|
||||
|
||||
// A session spec near its bound (a grant, and room the size of a root command
|
||||
// escaped six times over) still opens a terminal.
|
||||
func TestSessionWorkerOpensASpecNearItsBound(t *testing.T) {
|
||||
env, terms, sig := sessionTestEnv(t, []string{sessionOpOpen}, nil)
|
||||
withFake(t)
|
||||
// Here the test is the host: it takes the server end, and the box runs on
|
||||
// the dialled end.
|
||||
hostEnd := make(chan *websocket.Conn, 1)
|
||||
hold := make(chan struct{})
|
||||
t.Cleanup(func() { close(hold) })
|
||||
boxEnd := boxLaneHost(t, func(c *websocket.Conn) { hostEnd <- c; <-hold })
|
||||
b := <-hostEnd
|
||||
head := fmt.Sprintf(`{"op":"session:open","session_id":"sess-near","grant_terms_b64":%q,"grant_sig_b64":%q`, terms, sig)
|
||||
done := make(chan struct{})
|
||||
go func() { defer close(done); handleSessionWorkerConn(boxEnd, "", env) }()
|
||||
if err := b.WriteMessage(websocket.BinaryMessage, jsonFrame('S', head, 900<<10)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := b.WriteMessage(websocket.BinaryMessage, []byte("Ihello\n")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := readUntil(t, b, 'O', 5*time.Second)
|
||||
if string(got['O']) != "hello\n" {
|
||||
t.Fatalf("a 900 KiB spec did not open a terminal: %q", got)
|
||||
}
|
||||
_ = b.Close()
|
||||
<-done
|
||||
}
|
||||
|
||||
// sinkProc is a terminal that counts what is typed into it and prints nothing.
|
||||
type sinkProc struct {
|
||||
n atomic.Int64
|
||||
killed chan struct{}
|
||||
once sync.Once
|
||||
}
|
||||
|
||||
func (p *sinkProc) Read(b []byte) (int, error) { <-p.killed; return 0, io.EOF }
|
||||
func (p *sinkProc) Write(b []byte) (int, error) { p.n.Add(int64(len(b))); return len(b), nil }
|
||||
func (p *sinkProc) Resize(c, r int) error { return nil }
|
||||
func (p *sinkProc) Wait() int { <-p.killed; return 0 }
|
||||
func (p *sinkProc) Kill() { p.once.Do(func() { close(p.killed) }) }
|
||||
func (p *sinkProc) User() string { return "sink" }
|
||||
func (p *sinkProc) Shell() string { return "/bin/sink" }
|
||||
|
||||
func withSink(t *testing.T) *sinkProc {
|
||||
t.Helper()
|
||||
sp := &sinkProc{killed: make(chan struct{})}
|
||||
old := sessionSpawn
|
||||
sessionSpawn = func(op, shell string, cols, rows int) (sessionProc, error) { return sp, nil }
|
||||
t.Cleanup(func() { sessionSpawn = old; sp.Kill() })
|
||||
return sp
|
||||
}
|
||||
|
||||
func waitFor(d time.Duration, cond func() bool) bool {
|
||||
end := time.Now().Add(d)
|
||||
for time.Now().Before(end) {
|
||||
if cond() {
|
||||
return true
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
return cond()
|
||||
}
|
||||
|
||||
// Terminal input is streamed into the pty: a 16 MiB paste, the most a host
|
||||
// relays in one frame, all arrives, and the box never holds it whole.
|
||||
func TestSessionInputIsStreamedIntoTheTerminal(t *testing.T) {
|
||||
env, terms, sig := sessionTestEnv(t, []string{sessionOpOpen}, nil)
|
||||
sp := withSink(t)
|
||||
ws := sessionDial(t, env)
|
||||
sendSpec(t, ws, sessionSpec{Op: sessionOpOpen, GrantB64: terms, SigB64: sig, SessionID: "sess-paste"})
|
||||
const paste = 16 << 20
|
||||
frame := append([]byte{'I'}, bytes.Repeat([]byte{'x'}, paste)...)
|
||||
runtime.GC()
|
||||
var before, after runtime.MemStats
|
||||
runtime.ReadMemStats(&before)
|
||||
if err := ws.WriteMessage(websocket.BinaryMessage, frame); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !waitFor(20*time.Second, func() bool { return sp.n.Load() >= paste }) {
|
||||
t.Fatalf("the terminal got %d of %d bytes", sp.n.Load(), paste)
|
||||
}
|
||||
runtime.ReadMemStats(&after)
|
||||
if grew := after.TotalAlloc - before.TotalAlloc; grew > 8<<20 {
|
||||
t.Fatalf("typing a %d MiB paste allocated %d MiB: the frame was held whole", paste>>20, grew>>20)
|
||||
}
|
||||
}
|
||||
|
||||
// One frame of input past the most a host relays ends the session, and the
|
||||
// terminal never receives all of it.
|
||||
func TestSessionInputFramePastItsBoundEndsTheSession(t *testing.T) {
|
||||
env, terms, sig := sessionTestEnv(t, []string{sessionOpOpen}, nil)
|
||||
sp := withSink(t)
|
||||
ws := sessionDial(t, env)
|
||||
sendSpec(t, ws, sessionSpec{Op: sessionOpOpen, GrantB64: terms, SigB64: sig, SessionID: "sess-huge"})
|
||||
const frame = 17 << 20
|
||||
go func() {
|
||||
_ = ws.WriteMessage(websocket.BinaryMessage, append([]byte{'I'}, bytes.Repeat([]byte{'x'}, frame)...))
|
||||
}()
|
||||
ended := func() bool {
|
||||
select {
|
||||
case <-sp.killed:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
if !waitFor(20*time.Second, ended) {
|
||||
t.Fatalf("a %d MiB input frame did not end the session; the terminal got %d bytes", frame>>20, sp.n.Load())
|
||||
}
|
||||
if n := sp.n.Load(); n >= frame {
|
||||
t.Fatalf("the terminal received all %d bytes of a frame past the bound", n)
|
||||
}
|
||||
}
|
||||
|
||||
// The mirror forwarder streams a host frame into the replica's socket: reading
|
||||
// 4 KiB of a 64 MiB frame does not need the frame held whole.
|
||||
func TestMirrorForwarderStreamsAHostFrame(t *testing.T) {
|
||||
const frame = 64 << 20
|
||||
payload := bytes.Repeat([]byte{'m'}, frame)
|
||||
start := make(chan struct{})
|
||||
up := websocket.Upgrader{}
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
c, err := up.Upgrade(w, r, nil)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer c.Close()
|
||||
_, _, _ = c.ReadMessage() // the BEARER hello
|
||||
<-start
|
||||
_ = c.WriteMessage(websocket.BinaryMessage, payload)
|
||||
time.Sleep(3 * time.Second)
|
||||
}))
|
||||
defer srv.Close()
|
||||
addr := srv.Listener.Addr().String()
|
||||
dialer := &websocket.Dialer{NetDialContext: func(ctx context.Context, n, a string) (net.Conn, error) {
|
||||
return (&net.Dialer{}).DialContext(ctx, "tcp", addr)
|
||||
}}
|
||||
boxEnd, replica := net.Pipe()
|
||||
defer replica.Close()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go pipeMirrorConn(ctx, boxEnd, dialer, "ws://"+addr+"/", "b")
|
||||
buf := make([]byte, 4096)
|
||||
runtime.GC()
|
||||
var before, after runtime.MemStats
|
||||
runtime.ReadMemStats(&before)
|
||||
close(start)
|
||||
_ = replica.SetReadDeadline(time.Now().Add(20 * time.Second))
|
||||
if _, err := io.ReadFull(replica, buf); err != nil || buf[0] != 'm' {
|
||||
t.Fatalf("the replica read: %v", err)
|
||||
}
|
||||
runtime.ReadMemStats(&after)
|
||||
if grew := after.TotalAlloc - before.TotalAlloc; grew > 8<<20 {
|
||||
t.Fatalf("passing 4 KiB of a %d MiB host frame to the replica allocated %d MiB: the frame was held whole",
|
||||
frame>>20, grew>>20)
|
||||
}
|
||||
}
|
||||
|
||||
// The forwarder carries at most eight connections at once. It listens on a
|
||||
// fixed port (5433), so the test needs that port free: run it in its own
|
||||
// network namespace on a machine where something already holds it.
|
||||
func TestMirrorForwarderCapsConnectionsInFlight(t *testing.T) {
|
||||
probe, err := net.Listen("tcp", "127.0.0.1:5433")
|
||||
if err != nil {
|
||||
t.Skipf("127.0.0.1:5433 is in use here (%v); run this test in its own network namespace", err)
|
||||
}
|
||||
_ = probe.Close()
|
||||
var live, most atomic.Int64
|
||||
release := make(chan struct{})
|
||||
up := websocket.Upgrader{}
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
c, err := up.Upgrade(w, r, nil)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer c.Close()
|
||||
n := live.Add(1)
|
||||
for {
|
||||
m := most.Load()
|
||||
if n <= m || most.CompareAndSwap(m, n) {
|
||||
break
|
||||
}
|
||||
}
|
||||
<-release
|
||||
live.Add(-1)
|
||||
}))
|
||||
defer srv.Close()
|
||||
defer close(release)
|
||||
addr := srv.Listener.Addr().String()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go serveMirrorForward(ctx, "127.0.0.1", "127.0.0.0/8", "ws://"+addr+"/", nil,
|
||||
func(ctx context.Context) (net.Conn, error) { return (&net.Dialer{}).DialContext(ctx, "tcp", addr) }, "b")
|
||||
var conns []net.Conn
|
||||
defer func() {
|
||||
for _, c := range conns {
|
||||
_ = c.Close()
|
||||
}
|
||||
}()
|
||||
if !waitFor(5*time.Second, func() bool {
|
||||
c, err := net.Dial("tcp", "127.0.0.1:5433")
|
||||
if err == nil {
|
||||
conns = append(conns, c)
|
||||
}
|
||||
return err == nil
|
||||
}) {
|
||||
t.Fatal("the forwarder never listened")
|
||||
}
|
||||
for i := 1; i < 12; i++ {
|
||||
c, err := net.Dial("tcp", "127.0.0.1:5433")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
conns = append(conns, c)
|
||||
}
|
||||
waitFor(3*time.Second, func() bool { return most.Load() >= 12 })
|
||||
if m := most.Load(); m != 8 {
|
||||
t.Fatalf("12 local connections made %d WebSockets to the host at once; want 8", m)
|
||||
}
|
||||
}
|
||||
|
||||
// A coder run's egress proxy reads a request head under a bound: a request
|
||||
// line that never ends is not read to its end.
|
||||
func TestCoderEgressHeadIsBounded(t *testing.T) {
|
||||
boxEnd, runEnd := net.Pipe()
|
||||
var refused int64
|
||||
done := make(chan struct{})
|
||||
go func() { defer close(done); handleCoderEgressConn(boxEnd, nil, &refused) }()
|
||||
chunk := bytes.Repeat([]byte{'a'}, 64<<10)
|
||||
_, _ = runEnd.Write([]byte("GET http://example.invalid/"))
|
||||
sent := 0
|
||||
var werr error
|
||||
for sent < 8<<20 {
|
||||
_ = runEnd.SetWriteDeadline(time.Now().Add(5 * time.Second))
|
||||
var n int
|
||||
n, werr = runEnd.Write(chunk)
|
||||
sent += n
|
||||
if werr != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
_ = runEnd.Close()
|
||||
<-done
|
||||
if werr == nil {
|
||||
t.Fatalf("the proxy read %d MiB of a request line with no end; it must stop at a bound", sent>>20)
|
||||
}
|
||||
}
|
||||
|
||||
// A coder run's proxy carries at most 1024 connections at once; the next is
|
||||
// answered 503 at once, never left waiting.
|
||||
func TestCoderEgressCapsConnectionsInFlight(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
run := "lane-bounds-test"
|
||||
if err := coderListenEgress(run, dir, os.Getuid()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer coderCloseEgress(run)
|
||||
sock := filepath.Join(dir, coderEgressSock)
|
||||
var conns []net.Conn
|
||||
defer func() {
|
||||
for _, c := range conns {
|
||||
_ = c.Close()
|
||||
}
|
||||
}()
|
||||
for i := 0; i < 1024; i++ {
|
||||
c, err := net.Dial("unix", sock)
|
||||
if err != nil {
|
||||
if errors.Is(err, syscall.EMFILE) || errors.Is(err, syscall.ENFILE) {
|
||||
t.Skipf("this process may not open %d sockets: %v", 2*1024, err)
|
||||
}
|
||||
t.Fatal(err)
|
||||
}
|
||||
conns = append(conns, c)
|
||||
}
|
||||
// Every one of them is idle in its head, so the proxy holds them all.
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
extra, err := net.Dial("unix", sock)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
conns = append(conns, extra)
|
||||
_ = extra.SetReadDeadline(time.Now().Add(3 * time.Second))
|
||||
line, _ := io.ReadAll(io.LimitReader(extra, 256))
|
||||
if !strings.HasPrefix(string(line), "HTTP/1.1 503") {
|
||||
t.Fatalf("connection 1025 of one run got %q; want a 503 at once", line)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,197 @@
|
||||
package main
|
||||
|
||||
// What the host sends down a lane is read at the size the bastion asks for,
|
||||
// never at the size the host chooses: one frame of any length must not be
|
||||
// held whole in this machine's memory, and an egress request head that never
|
||||
// ends must not be either.
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/websocket"
|
||||
)
|
||||
|
||||
func TestEgressLaneDoesNotHoldAWholeHostFrame(t *testing.T) {
|
||||
const frame = 64 << 20
|
||||
payload := make([]byte, frame+1)
|
||||
payload[0] = 'D'
|
||||
for i := 1; i < len(payload); i++ {
|
||||
payload[i] = 'a'
|
||||
}
|
||||
start := make(chan struct{})
|
||||
up := websocket.Upgrader{}
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
ws, err := up.Upgrade(w, r, nil)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer ws.Close()
|
||||
<-start
|
||||
_ = ws.WriteMessage(websocket.BinaryMessage, payload)
|
||||
time.Sleep(2 * time.Second)
|
||||
}))
|
||||
defer srv.Close()
|
||||
ws, _, err := websocket.DefaultDialer.Dial("ws"+strings.TrimPrefix(srv.URL, "http"), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer ws.Close()
|
||||
c := &wsConn{ws: ws}
|
||||
buf := make([]byte, 4096)
|
||||
runtime.GC()
|
||||
var before, after runtime.MemStats
|
||||
runtime.ReadMemStats(&before)
|
||||
close(start)
|
||||
n, err := c.Read(buf)
|
||||
runtime.ReadMemStats(&after)
|
||||
if err != nil || n == 0 || buf[0] != 'a' {
|
||||
t.Fatalf("read: n=%d err=%v", n, err)
|
||||
}
|
||||
if grew := after.TotalAlloc - before.TotalAlloc; grew > 8<<20 {
|
||||
t.Fatalf("reading 4 KiB of a %d MiB host frame allocated %d MiB: the frame was held whole", frame>>20, grew>>20)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEgressRequestHeadIsBounded(t *testing.T) {
|
||||
bastionEnd, hostEnd := net.Pipe()
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
handleEgressConn(bastionEnd, func(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||
return nil, errors.New("no dial in this test")
|
||||
}, "test")
|
||||
}()
|
||||
chunk := []byte(strings.Repeat("a", 64<<10))
|
||||
_, _ = hostEnd.Write([]byte("GET http://example.invalid/"))
|
||||
sent := 0
|
||||
var werr error
|
||||
for sent < 8<<20 {
|
||||
_ = hostEnd.SetWriteDeadline(time.Now().Add(5 * time.Second))
|
||||
var n int
|
||||
n, werr = hostEnd.Write(chunk)
|
||||
sent += n
|
||||
if werr != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
_ = hostEnd.Close()
|
||||
<-done
|
||||
if werr == nil {
|
||||
t.Fatalf("the bastion read %d MiB of a request line with no end; it must stop at a bound", sent>>20)
|
||||
}
|
||||
}
|
||||
|
||||
// Streaming a frame must not change what the reader sees: every byte of every
|
||||
// 'D' frame in order, with empty frames, text frames and unknown tags skipped,
|
||||
// a frame boundary never visible, and 'E' as the end, read through a buffer
|
||||
// far smaller than the frames.
|
||||
func TestEgressLaneFramesReadInOrderThroughASmallBuffer(t *testing.T) {
|
||||
var want []byte
|
||||
frames := [][]byte{{'D'}, {}, {'X', 'z', 'z'}}
|
||||
for i, size := range []int{1, 7, 4096, 70000, 3} {
|
||||
f := []byte{'D'}
|
||||
for j := 0; j < size; j++ {
|
||||
f = append(f, byte('a'+(i+j)%26))
|
||||
}
|
||||
want = append(want, f[1:]...)
|
||||
frames = append(frames, f, []byte{'D'})
|
||||
}
|
||||
up := websocket.Upgrader{}
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
ws, err := up.Upgrade(w, r, nil)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer ws.Close()
|
||||
_ = ws.WriteMessage(websocket.TextMessage, []byte("Dnot data"))
|
||||
for _, f := range frames {
|
||||
_ = ws.WriteMessage(websocket.BinaryMessage, f)
|
||||
}
|
||||
_ = ws.WriteMessage(websocket.BinaryMessage, []byte{'E'})
|
||||
time.Sleep(time.Second)
|
||||
}))
|
||||
defer srv.Close()
|
||||
ws, _, err := websocket.DefaultDialer.Dial("ws"+strings.TrimPrefix(srv.URL, "http"), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer ws.Close()
|
||||
c := &wsConn{ws: ws}
|
||||
var got []byte
|
||||
buf := make([]byte, 13)
|
||||
for {
|
||||
n, err := c.Read(buf)
|
||||
if n == 0 && err == nil {
|
||||
t.Fatal("Read returned 0, nil")
|
||||
}
|
||||
got = append(got, buf[:n]...)
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("read: %v", err)
|
||||
}
|
||||
}
|
||||
if !bytes.Equal(got, want) {
|
||||
t.Fatalf("read %d bytes, want %d; equal prefix %d", len(got), len(want), commonPrefix(got, want))
|
||||
}
|
||||
}
|
||||
|
||||
func commonPrefix(a, b []byte) int {
|
||||
i := 0
|
||||
for i < len(a) && i < len(b) && a[i] == b[i] {
|
||||
i++
|
||||
}
|
||||
return i
|
||||
}
|
||||
|
||||
// A large upload through the lane still arrives whole: the head bound applies
|
||||
// to the head alone, and the bytes the reader buffered past it go upstream
|
||||
// first, in order.
|
||||
func TestEgressLargeUploadPassesTheHeadBound(t *testing.T) {
|
||||
bastionEnd, hostEnd := net.Pipe()
|
||||
upBastion, upFar := net.Pipe()
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
handleEgressConn(bastionEnd, func(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||
return upBastion, nil
|
||||
}, "")
|
||||
}()
|
||||
body := make([]byte, 3*egressMaxHeadBytes)
|
||||
for i := range body {
|
||||
body[i] = byte(i % 251)
|
||||
}
|
||||
head := "POST http://198.51.100.20/upload HTTP/1.1\r\nHost: 198.51.100.20\r\nContent-Length: " +
|
||||
strconv.Itoa(len(body)) + "\r\n\r\n"
|
||||
// Head and body in ONE write, so the reader holds body bytes when the head ends.
|
||||
go func() {
|
||||
_, _ = hostEnd.Write(append([]byte(head), body...))
|
||||
}()
|
||||
// The lane rebuilds a plain-HTTP head (egress_http.go): this request comes
|
||||
// out the same, with `Connection: close` added as its last header.
|
||||
want := head[:len(head)-len("\r\n")] + "Connection: close\r\n\r\n"
|
||||
got := make([]byte, len(want)+len(body))
|
||||
_ = upFar.SetReadDeadline(time.Now().Add(20 * time.Second))
|
||||
if _, err := io.ReadFull(upFar, got); err != nil {
|
||||
t.Fatalf("upstream got %v", err)
|
||||
}
|
||||
if string(got[:len(want)]) != want || !bytes.Equal(got[len(want):], body) {
|
||||
t.Fatalf("upstream did not receive the head (%q) and the %d-byte body intact",
|
||||
got[:len(want)], len(body))
|
||||
}
|
||||
_ = hostEnd.Close()
|
||||
_ = upFar.Close()
|
||||
<-done
|
||||
}
|
||||
@@ -493,7 +493,7 @@ func startLaneHostOn(t *testing.T, raw net.Listener, cert tls.Certificate, clien
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, p := range []string{"/bastion-harness", "/bastion-mgmt", "/bastion-egress"} {
|
||||
for _, p := range []string{"/bastion-harness", "/bastion-mgmt", "/bastion-egress", "/bastion-serve"} {
|
||||
mux.HandleFunc(p, ws)
|
||||
}
|
||||
srv := &http.Server{Handler: mux, ErrorLog: log.New(io.Discard, "", 0)}
|
||||
@@ -622,7 +622,8 @@ func TestTheUnixLaneCarriesTheBearerAndTheSameProtocol(t *testing.T) {
|
||||
h.mu.Unlock()
|
||||
if maxPark != harnessWorkerCount || count["/bastion-harness"] < harnessWorkerCount ||
|
||||
count["/bastion-harness session"] != sessionWorkerCount ||
|
||||
count["/bastion-mgmt"] != mgmtWorkerCount || count["/bastion-egress"] != egressWorkerCount {
|
||||
count["/bastion-mgmt"] != mgmtWorkerCount || count["/bastion-egress"] != egressWorkerCount ||
|
||||
count["/bastion-serve"] != serveWorkerCount {
|
||||
t.Fatalf("the worker pools over the socket: %v, parked at once %d", count, maxPark)
|
||||
}
|
||||
select {
|
||||
|
||||
@@ -417,7 +417,8 @@ func TestTheTCPLaneCarriesTheCertTheBearerAndTheSameProtocol(t *testing.T) {
|
||||
h.mu.Unlock()
|
||||
if maxPark != harnessWorkerCount || count["/bastion-harness"] < harnessWorkerCount ||
|
||||
count["/bastion-harness session"] != sessionWorkerCount ||
|
||||
count["/bastion-mgmt"] != mgmtWorkerCount || count["/bastion-egress"] != egressWorkerCount {
|
||||
count["/bastion-mgmt"] != mgmtWorkerCount || count["/bastion-egress"] != egressWorkerCount ||
|
||||
count["/bastion-serve"] != serveWorkerCount {
|
||||
t.Fatalf("the worker pools over the port: %v, parked at once %d", count, maxPark)
|
||||
}
|
||||
for _, n := range sni {
|
||||
|
||||
+26
-5
@@ -491,6 +491,12 @@ func ensureMirrorNet(rt, name string) (string, string, error) {
|
||||
// container and a BASE_BACKUP of the user's memory. Nothing here is defence in
|
||||
// depth; this is the depth.
|
||||
//
|
||||
// mirrorForwardMaxConns bounds the connections the forwarder carries at once.
|
||||
// The replica holds at most two (`pg_basebackup -X stream` while it seeds; one
|
||||
// walreceiver after), so four times that leaves room for a diagnostic session
|
||||
// and a reconnect that overlaps the connection it replaces.
|
||||
const mirrorForwardMaxConns = 8
|
||||
|
||||
// One WebSocket per accepted connection, because `pg_basebackup -X stream`
|
||||
// holds two replication connections at once and multiplexing them would mean
|
||||
// inventing a framing layer for no gain.
|
||||
@@ -507,6 +513,11 @@ func serveMirrorForward(ctx context.Context, gw, subnet, wsURL string,
|
||||
allowed = n
|
||||
}
|
||||
}
|
||||
// Connections carried at once, across both listeners. Loopback is let in
|
||||
// for diagnostics, so any local user can connect, and each connection is a
|
||||
// WebSocket to the host and two pumps here; past the cap one is closed at
|
||||
// once and its client tries again.
|
||||
slots := make(chan struct{}, mirrorForwardMaxConns)
|
||||
for _, addr := range []string{
|
||||
net.JoinHostPort(gw, strconv.Itoa(mirrorForwardPort)),
|
||||
net.JoinHostPort("127.0.0.1", strconv.Itoa(mirrorForwardPort)),
|
||||
@@ -527,7 +538,15 @@ func serveMirrorForward(ctx context.Context, gw, subnet, wsURL string,
|
||||
_ = c.Close()
|
||||
continue
|
||||
}
|
||||
go pipeMirrorConn(ctx, c, dialer, wsURL, bearer)
|
||||
select {
|
||||
case slots <- struct{}{}:
|
||||
go func() {
|
||||
defer func() { <-slots }()
|
||||
pipeMirrorConn(ctx, c, dialer, wsURL, bearer)
|
||||
}()
|
||||
default:
|
||||
_ = c.Close()
|
||||
}
|
||||
}
|
||||
}(ln)
|
||||
}
|
||||
@@ -578,15 +597,17 @@ func pipeMirrorConn(ctx context.Context, local net.Conn, dialer *websocket.Diale
|
||||
done <- struct{}{}
|
||||
}()
|
||||
go func() {
|
||||
// Each host frame is streamed into the replica's socket as it arrives,
|
||||
// never held whole: the host picks a frame's length (lane_bounds.go).
|
||||
for {
|
||||
mt, data, err := ws.ReadMessage()
|
||||
mt, r, err := ws.NextReader()
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
if mt != websocket.BinaryMessage || len(data) == 0 {
|
||||
continue
|
||||
if mt != websocket.BinaryMessage {
|
||||
continue // NextReader drops what is left of it
|
||||
}
|
||||
if _, werr := local.Write(data); werr != nil {
|
||||
if _, err := io.Copy(local, r); err != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,7 +19,9 @@ import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
@@ -186,17 +188,29 @@ func TestFilterVerdictPolicy(t *testing.T) {
|
||||
|
||||
// fakeOllama answers the handful of routes the filter cares about and records
|
||||
// what actually reached it — the point of most of these assertions is that a
|
||||
// refused request never arrives at all.
|
||||
// refused request never arrives at all. Its handler runs on the server's
|
||||
// goroutines, beside the test's (and a pairing's beat probes it on its own),
|
||||
// so the record is read through requests(), under the lock.
|
||||
type fakeOllama struct {
|
||||
srv *httptest.Server
|
||||
mu sync.Mutex
|
||||
seen []string
|
||||
}
|
||||
|
||||
// requests is a copy of what reached the fake so far.
|
||||
func (f *fakeOllama) requests() []string {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return slices.Clone(f.seen)
|
||||
}
|
||||
|
||||
func newFakeOllama(t *testing.T) *fakeOllama {
|
||||
t.Helper()
|
||||
f := &fakeOllama{}
|
||||
f.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
f.mu.Lock()
|
||||
f.seen = append(f.seen, r.Method+" "+r.URL.Path)
|
||||
f.mu.Unlock()
|
||||
switch r.URL.Path {
|
||||
case "/api/tags":
|
||||
_, _ = w.Write([]byte(`{"models":[{"name":"qwen4-embed:latest","size":42},` +
|
||||
@@ -264,8 +278,8 @@ func TestFilteredForwardRefusesAnotherModelAndNeverForwardsIt(t *testing.T) {
|
||||
if b := bodyOf(t, resp); !strings.Contains(b, "llama3:70b") || !strings.Contains(b, "qwen4-embed") {
|
||||
t.Fatalf("the refusal must name what was asked for and what is shared: %s", b)
|
||||
}
|
||||
if len(up.seen) != 0 {
|
||||
t.Fatalf("a refused request reached the upstream: %v", up.seen)
|
||||
if len(up.requests()) != 0 {
|
||||
t.Fatalf("a refused request reached the upstream: %v", up.requests())
|
||||
}
|
||||
|
||||
resp = roundTrip(t, addr, laneOllama, tMember, "POST", "/api/embed",
|
||||
@@ -273,8 +287,8 @@ func TestFilteredForwardRefusesAnotherModelAndNeverForwardsIt(t *testing.T) {
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("the shared model must work: %d", resp.StatusCode)
|
||||
}
|
||||
if len(up.seen) != 1 || up.seen[0] != "POST /api/embed" {
|
||||
t.Fatalf("the allowed request must reach the upstream once: %v", up.seen)
|
||||
if len(up.requests()) != 1 || up.requests()[0] != "POST /api/embed" {
|
||||
t.Fatalf("the allowed request must reach the upstream once: %v", up.requests())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -295,8 +309,8 @@ func TestFilteredForwardRefusesTheStoreMutatingHalf(t *testing.T) {
|
||||
t.Fatalf("%s: status %d, want 403", p, resp.StatusCode)
|
||||
}
|
||||
}
|
||||
if len(up.seen) != 0 {
|
||||
t.Fatalf("a store-mutating request reached ollama: %v", up.seen)
|
||||
if len(up.requests()) != 0 {
|
||||
t.Fatalf("a store-mutating request reached ollama: %v", up.requests())
|
||||
}
|
||||
// ...and using a model is untouched by that refusal.
|
||||
if resp := roundTrip(t, addr, laneOllama, tMember, "POST", "/api/embed",
|
||||
@@ -344,8 +358,8 @@ func TestAnchoredBoxRefusesModelChangesThroughTheForwardForTheAnchor(t *testing.
|
||||
t.Fatalf("%s for the anchor: status %d, want 403", p, resp.StatusCode)
|
||||
}
|
||||
}
|
||||
if len(up.seen) != 0 {
|
||||
t.Fatalf("a store-mutating request from the anchor reached ollama: %v", up.seen)
|
||||
if len(up.requests()) != 0 {
|
||||
t.Fatalf("a store-mutating request from the anchor reached ollama: %v", up.requests())
|
||||
}
|
||||
if resp := roundTrip(t, addr, laneOllama, tAnchor, "POST", "/api/embed", `{"model":"anything"}`); resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("the anchor still USES models: %d", resp.StatusCode)
|
||||
@@ -581,8 +595,8 @@ func TestOversizedBodyIsRefusedNotForwarded(t *testing.T) {
|
||||
if resp.StatusCode != http.StatusForbidden {
|
||||
t.Fatalf("a request naming no model must be refused: %d", resp.StatusCode)
|
||||
}
|
||||
if len(up.seen) != 0 {
|
||||
t.Fatalf("it reached the upstream anyway: %v", up.seen)
|
||||
if len(up.requests()) != 0 {
|
||||
t.Fatalf("it reached the upstream anyway: %v", up.requests())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -612,8 +626,8 @@ func TestAScopedPairingCannotRenameAModelIntoItsScope(t *testing.T) {
|
||||
t.Fatalf("%s: the refusal must say why: %s", p, b)
|
||||
}
|
||||
}
|
||||
if len(up.seen) != 0 {
|
||||
t.Fatalf("a rename reached ollama: %v", up.seen)
|
||||
if len(up.requests()) != 0 {
|
||||
t.Fatalf("a rename reached ollama: %v", up.requests())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -803,8 +817,8 @@ func TestAnEndlessHeaderIsCutOff(t *testing.T) {
|
||||
t.Fatal("an endless header was read without limit")
|
||||
}
|
||||
client.Close()
|
||||
if len(up.seen) != 0 {
|
||||
t.Fatalf("it reached the upstream: %v", up.seen)
|
||||
if len(up.requests()) != 0 {
|
||||
t.Fatalf("it reached the upstream: %v", up.requests())
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+73
-31
@@ -419,38 +419,53 @@ func placedByThisHost(root, model string) bool {
|
||||
// ── talking to the model daemon ────────────────────────────────────────────
|
||||
|
||||
var (
|
||||
modelDaemonMu sync.Mutex
|
||||
modelDaemonUp bool
|
||||
modelDaemonAt time.Time
|
||||
modelDaemonMu sync.Mutex
|
||||
modelDaemonUp bool
|
||||
modelDaemonAt time.Time
|
||||
modelDaemonAddr string // the address the cached answer is for
|
||||
)
|
||||
|
||||
// modelDaemonReachable probes the SAME address the rest of this binary uses
|
||||
// (localOllamaAddr, run.go:714-719) so the box can never advertise management
|
||||
// of a daemon the forwards do not point at.
|
||||
func modelDaemonReachable() bool {
|
||||
func modelDaemonReachable() bool { return modelDaemonReachableAt(localOllamaAddr()) }
|
||||
|
||||
// modelDaemonReachableAt is modelDaemonReachable's probe of one address. The
|
||||
// cached answer is for the address it was asked of, so a pairing whose
|
||||
// address is the vetted one (modelDaemonTarget) never reads another's answer.
|
||||
func modelDaemonReachableAt(addr string) bool {
|
||||
modelDaemonMu.Lock()
|
||||
defer modelDaemonMu.Unlock()
|
||||
if !modelDaemonAt.IsZero() && time.Since(modelDaemonAt) < modelDaemonCacheFor {
|
||||
if !modelDaemonAt.IsZero() && modelDaemonAddr == addr && time.Since(modelDaemonAt) < modelDaemonCacheFor {
|
||||
return modelDaemonUp
|
||||
}
|
||||
cl := &http.Client{Timeout: 3 * time.Second}
|
||||
resp, err := cl.Get("http://" + localOllamaAddr() + "/api/tags")
|
||||
resp, err := cl.Get("http://" + addr + "/api/tags")
|
||||
up := false
|
||||
if err == nil {
|
||||
up = resp.StatusCode == http.StatusOK
|
||||
resp.Body.Close()
|
||||
}
|
||||
modelDaemonUp, modelDaemonAt = up, time.Now()
|
||||
modelDaemonUp, modelDaemonAt, modelDaemonAddr = up, time.Now(), addr
|
||||
return up
|
||||
}
|
||||
|
||||
// resetModelDaemonCache exists for the tests.
|
||||
func resetModelDaemonCache() {
|
||||
modelDaemonMu.Lock()
|
||||
modelDaemonUp, modelDaemonAt = false, time.Time{}
|
||||
modelDaemonUp, modelDaemonAt, modelDaemonAddr = false, time.Time{}, ""
|
||||
modelDaemonMu.Unlock()
|
||||
}
|
||||
|
||||
// modelSelf is the pairing a model:* job came from, or "" when the job was
|
||||
// driven without one (modelDaemonTarget then keeps OLLAMA_ADDR, as before).
|
||||
func modelSelf(ws *websocket.Conn) string {
|
||||
if a := mgmtAccountOf(ws); a != nil {
|
||||
return a.self
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// modelSwapCapable answers the heartbeat's question: could this box have a
|
||||
// model placed on it BY THIS PAIRING. Two conditions, and the caps one is
|
||||
// first so a pairing outside the family costs no probe — the rule the
|
||||
@@ -459,13 +474,19 @@ func modelSwapCapable(self string) bool {
|
||||
if capsAllow(self, modelGateOp) != "" {
|
||||
return false
|
||||
}
|
||||
return modelDaemonReachable()
|
||||
// A Host with no platform is offered management only of a model server on
|
||||
// this machine, and a refused one is not probed at all (modelDaemonTarget).
|
||||
addr, why := modelDaemonTarget(self)
|
||||
if why != "" {
|
||||
return false
|
||||
}
|
||||
return modelDaemonReachableAt(addr)
|
||||
}
|
||||
|
||||
// modelPostJSON posts one JSON body and returns (status, body). The body is
|
||||
// read whatever the status is: a streaming ollama reports a FAILED pull inside
|
||||
// a 200 (measured, see the header), so the caller needs both.
|
||||
func modelPostJSON(method, path string, payload any, timeout time.Duration) (int, []byte, error) {
|
||||
func modelPostJSON(addr, method, path string, payload any, timeout time.Duration) (int, []byte, error) {
|
||||
var rdr io.Reader
|
||||
if payload != nil {
|
||||
b, err := json.Marshal(payload)
|
||||
@@ -474,7 +495,7 @@ func modelPostJSON(method, path string, payload any, timeout time.Duration) (int
|
||||
}
|
||||
rdr = bytes.NewReader(b)
|
||||
}
|
||||
req, err := http.NewRequest(method, "http://"+localOllamaAddr()+path, rdr)
|
||||
req, err := http.NewRequest(method, "http://"+addr+path, rdr)
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
@@ -528,8 +549,8 @@ func modelCosine(a, b []float64) float64 {
|
||||
return dot / (math.Sqrt(na) * math.Sqrt(nb))
|
||||
}
|
||||
|
||||
func modelEmbedOnce(model, text string) ([]float64, error) {
|
||||
status, body, err := modelPostJSON(http.MethodPost, "/api/embed",
|
||||
func modelEmbedOnce(addr, model, text string) ([]float64, error) {
|
||||
status, body, err := modelPostJSON(addr, http.MethodPost, "/api/embed",
|
||||
map[string]any{"model": model, "input": []string{text}}, modelProbeTimeout)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -549,8 +570,8 @@ func modelEmbedOnce(model, text string) ([]float64, error) {
|
||||
return out.Embeddings[0], nil
|
||||
}
|
||||
|
||||
func modelGenerateOnce(model, prompt string) (string, error) {
|
||||
status, body, err := modelPostJSON(http.MethodPost, "/api/generate",
|
||||
func modelGenerateOnce(addr, model, prompt string) (string, error) {
|
||||
status, body, err := modelPostJSON(addr, http.MethodPost, "/api/generate",
|
||||
map[string]any{"model": model, "prompt": prompt, "stream": false, "think": false}, modelProbeTimeout)
|
||||
if err != nil {
|
||||
return "", err
|
||||
@@ -573,12 +594,12 @@ func modelGenerateOnce(model, prompt string) (string, error) {
|
||||
// probeEmbedModel proves the model EMBEDS: the same text twice must come back
|
||||
// as one answer, and a different text must move it. Returns the measured width
|
||||
// and "" , or 0 and the reason it is not usable.
|
||||
func probeEmbedModel(model string) (int, string) {
|
||||
v1, err := modelEmbedOnce(model, modelProbeTextA)
|
||||
func probeEmbedModel(addr, model string) (int, string) {
|
||||
v1, err := modelEmbedOnce(addr, model, modelProbeTextA)
|
||||
if err != nil {
|
||||
return 0, "it was downloaded but does not answer as an embedding model: " + err.Error()
|
||||
}
|
||||
v2, err := modelEmbedOnce(model, modelProbeTextA)
|
||||
v2, err := modelEmbedOnce(addr, model, modelProbeTextA)
|
||||
if err != nil {
|
||||
return 0, "it answered once and then failed: " + err.Error()
|
||||
}
|
||||
@@ -587,7 +608,7 @@ func probeEmbedModel(model string) (int, string) {
|
||||
return 0, fmt.Sprintf("it gave two different answers to the SAME text (similarity %.4f) — "+
|
||||
"nothing routed at it would be reproducible", same)
|
||||
}
|
||||
v3, err := modelEmbedOnce(model, modelProbeTextB)
|
||||
v3, err := modelEmbedOnce(addr, model, modelProbeTextB)
|
||||
if err != nil {
|
||||
return 0, "the control text failed: " + err.Error()
|
||||
}
|
||||
@@ -604,15 +625,15 @@ func probeEmbedModel(model string) (int, string) {
|
||||
|
||||
// probeGenerateModel proves the model GENERATES: a non-empty answer, and a
|
||||
// different prompt must change it (the same constant-answer trap).
|
||||
func probeGenerateModel(model string) string {
|
||||
a, err := modelGenerateOnce(model, modelProbePromptA)
|
||||
func probeGenerateModel(addr, model string) string {
|
||||
a, err := modelGenerateOnce(addr, model, modelProbePromptA)
|
||||
if err != nil {
|
||||
return "it was downloaded but does not answer as a text model: " + err.Error()
|
||||
}
|
||||
if strings.TrimSpace(a) == "" {
|
||||
return "it answered with nothing at all"
|
||||
}
|
||||
b, err := modelGenerateOnce(model, modelProbePromptB)
|
||||
b, err := modelGenerateOnce(addr, model, modelProbePromptB)
|
||||
if err != nil {
|
||||
return "it answered once and then failed: " + err.Error()
|
||||
}
|
||||
@@ -704,7 +725,12 @@ func modelReply(ws *websocket.Conn, v any) {
|
||||
// here — the two together are what makes an honest proposal possible: the host
|
||||
// can see a model is present without being able to claim it may remove it.
|
||||
func handleModelList(ws *websocket.Conn, root string, _ []string) {
|
||||
status, body, err := modelPostJSON(http.MethodGet, "/api/tags", nil, modelListTimeout)
|
||||
addr, why := modelDaemonTarget(modelSelf(ws))
|
||||
if why != "" {
|
||||
modelFail(ws, "model:list: "+why)
|
||||
return
|
||||
}
|
||||
status, body, err := modelPostJSON(addr, http.MethodGet, "/api/tags", nil, modelListTimeout)
|
||||
if err != nil {
|
||||
modelFail(ws, "model:list: no model daemon answering on this machine: "+err.Error())
|
||||
return
|
||||
@@ -974,12 +1000,12 @@ func modelPullRunningNote(j *modelPullJob) string {
|
||||
// runBackupFetch updates job.Got inside its read loop (backupfetch.go:309-311):
|
||||
// progress that only exists when the transfer ends is not progress. nil is
|
||||
// allowed — the direct-call tests pass it.
|
||||
func modelPullStreaming(model string, maxBytes int64, note func(want, got int64)) (int64, error) {
|
||||
func modelPullStreaming(addr, model string, maxBytes int64, note func(want, got int64)) (int64, error) {
|
||||
body, err := json.Marshal(map[string]any{"model": model, "stream": true})
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
httpReq, err := http.NewRequest(http.MethodPost, "http://"+localOllamaAddr()+"/api/pull",
|
||||
httpReq, err := http.NewRequest(http.MethodPost, "http://"+addr+"/api/pull",
|
||||
bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
@@ -1109,6 +1135,17 @@ func modelPullStart(ws *websocket.Conn, root string, req modelReq) {
|
||||
return
|
||||
}
|
||||
|
||||
// WHERE THE PULL GOES, decided once and held for the download and its
|
||||
// probe: a Host with no platform pulls only onto a model server on this
|
||||
// machine (modelDaemonTarget). Refused before the approval is read, so a
|
||||
// pull that cannot go here costs the person nothing. For such a pairing a
|
||||
// name in OLLAMA_ADDR is looked up here; nothing is dialled.
|
||||
addr, why := modelDaemonTarget(modelSelf(ws))
|
||||
if why != "" {
|
||||
modelRefuse(ws, req.Model, why)
|
||||
return
|
||||
}
|
||||
|
||||
// THE APPROVAL GATE. Nothing above this line has touched the network; nothing
|
||||
// below it runs until the user has said yes to THIS proposal on this machine.
|
||||
// The async split moved the download off this frame and deliberately did NOT
|
||||
@@ -1152,7 +1189,7 @@ func modelPullStart(ws *websocket.Conn, root string, req modelReq) {
|
||||
snap := *job
|
||||
modelPullJobsMu.Unlock()
|
||||
|
||||
go runModelPull(job, root, req)
|
||||
go runModelPull(job, root, req, addr)
|
||||
modelReply(ws, snap)
|
||||
}
|
||||
|
||||
@@ -1167,7 +1204,7 @@ func modelPullStart(ws *websocket.Conn, root string, req modelReq) {
|
||||
// bytes on this disk", which is true the moment the download succeeds. A model
|
||||
// that downloaded but failed its probe must stay droppable, or a bad pull would
|
||||
// leave dead weight on the user's machine that nothing is allowed to remove.
|
||||
func runModelPull(job *modelPullJob, root string, req modelReq) {
|
||||
func runModelPull(job *modelPullJob, root string, req modelReq, addr string) {
|
||||
set := func(f func(j *modelPullJob)) {
|
||||
modelPullJobsMu.Lock()
|
||||
f(job)
|
||||
@@ -1201,7 +1238,7 @@ func runModelPull(job *modelPullJob, root string, req modelReq) {
|
||||
// The 200-with-an-error-in-the-body case that stream:false was chosen for is
|
||||
// handled either way: modelBodyError reads EVERY line, so a failure on the
|
||||
// last line of a stream is caught exactly as before.
|
||||
total, err := modelPullStreaming(req.Model, req.MaxBytes,
|
||||
total, err := modelPullStreaming(addr, req.Model, req.MaxBytes,
|
||||
func(want, got int64) { set(func(j *modelPullJob) { j.Want, j.Got = want, got }) })
|
||||
if err != nil {
|
||||
finish("failed", func(j *modelPullJob) {
|
||||
@@ -1242,9 +1279,9 @@ func runModelPull(job *modelPullJob, root string, req modelReq) {
|
||||
var dim int
|
||||
var why string
|
||||
if req.Role == "embed" {
|
||||
dim, why = probeEmbedModel(req.Model)
|
||||
dim, why = probeEmbedModel(addr, req.Model)
|
||||
} else {
|
||||
why = probeGenerateModel(req.Model)
|
||||
why = probeGenerateModel(addr, req.Model)
|
||||
}
|
||||
if why != "" {
|
||||
finish("done", func(j *modelPullJob) {
|
||||
@@ -1284,7 +1321,12 @@ func handleModelDrop(ws *websocket.Conn, root string, args []string) {
|
||||
"not remove it. Only a model this host pulled here is its to drop.")
|
||||
return
|
||||
}
|
||||
status, body, err := modelPostJSON(http.MethodDelete, "/api/delete",
|
||||
addr, why := modelDaemonTarget(modelSelf(ws))
|
||||
if why != "" {
|
||||
modelFail(ws, "model:drop: "+why)
|
||||
return
|
||||
}
|
||||
status, body, err := modelPostJSON(addr, http.MethodDelete, "/api/delete",
|
||||
map[string]any{"model": req.Model}, modelListTimeout)
|
||||
if err != nil {
|
||||
modelFail(ws, "model:drop: the model daemon did not answer: "+err.Error())
|
||||
|
||||
@@ -286,7 +286,8 @@ func TestNetworksInsideThisMachineAreThisMachine(t *testing.T) {
|
||||
}
|
||||
|
||||
// The local egress lane on an unanchored box: an inside address, by address
|
||||
// and by name, is refused as this machine; the LAN passes.
|
||||
// and by name, is refused as this machine; the LAN is refused as the network
|
||||
// around it; the public internet passes.
|
||||
t.Setenv("XDG_CONFIG_HOME", t.TempDir())
|
||||
if currentAnchor() != nil || boxMode() != modeLocal {
|
||||
t.Fatal("the namespaced run must be a local, unanchored box")
|
||||
@@ -301,12 +302,16 @@ func TestNetworksInsideThisMachineAreThisMachine(t *testing.T) {
|
||||
t.Errorf("local egress: %s is on a network inside this machine and must be refused as this machine, got %v %q", hp, got, why)
|
||||
}
|
||||
}
|
||||
for hp, want := range map[string]string{"10.77.0.9:445": "10.77.0.9:445", "192.168.50.9:631": "192.168.50.9:631",
|
||||
"nas.example.test:445": "10.77.0.9:445", "198.51.100.9:443": "198.51.100.9:443",
|
||||
"[2001:db8:ff::9]:443": "[2001:db8:ff::9]:443", "192.168.60.9:445": "192.168.60.9:445",
|
||||
"10.88.0.9:80": "10.88.0.9:80", "10.89.0.9:80": "10.89.0.9:80", "[2001:db8:88::9]:80": "[2001:db8:88::9]:80"} {
|
||||
for _, hp := range []string{"10.77.0.9:445", "192.168.50.9:631", "nas.example.test:445", "192.168.60.9:445",
|
||||
"10.88.0.9:80", "10.89.0.9:80"} {
|
||||
if got, why := egressVettedAddrs(ctx, hp); !strings.Contains(why, "approved lanes") {
|
||||
t.Errorf("local egress: %s is the LAN and must be refused as the network around this machine, got %v %q", hp, got, why)
|
||||
}
|
||||
}
|
||||
for hp, want := range map[string]string{"198.51.100.9:443": "198.51.100.9:443",
|
||||
"[2001:db8:ff::9]:443": "[2001:db8:ff::9]:443", "[2001:db8:88::9]:80": "[2001:db8:88::9]:80"} {
|
||||
if got, why := egressVettedAddrs(ctx, hp); why != "" || len(got) != 1 || got[0] != want {
|
||||
t.Errorf("local egress: %s is the LAN and passes as %s: %v %q", hp, want, got, why)
|
||||
t.Errorf("local egress: %s leaves this machine for the internet and passes as %s: %v %q", hp, want, got, why)
|
||||
}
|
||||
}
|
||||
// A coder run's proxy refuses the public range on an inside network too
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"errors"
|
||||
"net"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
@@ -33,8 +34,13 @@ func stubOwnAddrs(t *testing.T, ifaces []string, route func(net.IP) (routeKind,
|
||||
}
|
||||
asked := &[]string{}
|
||||
if route != nil {
|
||||
// Locked: a lane's workers vet their jobs at once, each on its own
|
||||
// goroutine (egress_relay_test.go).
|
||||
var mu sync.Mutex
|
||||
ownRouteLookup = func(ip net.IP) (routeKind, error) {
|
||||
mu.Lock()
|
||||
*asked = append(*asked, ip.String())
|
||||
mu.Unlock()
|
||||
return route(ip)
|
||||
}
|
||||
} else {
|
||||
|
||||
+5
-1
@@ -29,7 +29,11 @@ const piiForwardPort = 18081
|
||||
// The bearer-authed surface is proxied opaquely — this client never
|
||||
// holds the PII_API_KEY.
|
||||
func piiProbe() (bool, bool) {
|
||||
a := piiTarget()
|
||||
return piiProbeAt(piiTarget())
|
||||
}
|
||||
|
||||
// piiProbeAt is piiProbe's probe of one address.
|
||||
func piiProbeAt(a string) (bool, bool) {
|
||||
if a == "" {
|
||||
return false, false
|
||||
}
|
||||
|
||||
@@ -686,11 +686,8 @@ func handleRootExec(ws *websocket.Conn, spec sessionSpec, self string, env sessi
|
||||
closed := make(chan struct{})
|
||||
go func() {
|
||||
defer close(closed)
|
||||
for {
|
||||
if _, _, err := ws.ReadMessage(); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
// Each frame is dropped as it arrives (lane_bounds.go), never held whole.
|
||||
_ = drainLane(ws)
|
||||
}()
|
||||
done := make(chan int, 1)
|
||||
go func() { done <- proc.Wait() }()
|
||||
|
||||
+36
-7
@@ -617,8 +617,11 @@ func runPairing(ctx context.Context, cfg *Config) error {
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Authorization", "Bearer "+cfg.Bearer)
|
||||
if resp, err := upstream.Do(req); err == nil {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
body, ok := readHeartbeatReply(resp)
|
||||
resp.Body.Close()
|
||||
if !ok {
|
||||
return // a reply past its bound is no answer (lane_bounds.go)
|
||||
}
|
||||
if resp.StatusCode < 300 {
|
||||
// last_ok per pairing (anchor.go): the anchor rotation's
|
||||
// silence path reads this — 14 days with no 2xx from the
|
||||
@@ -727,14 +730,19 @@ func unixLaneDialer(cfg *Config) (func(ctx context.Context) (net.Conn, error), *
|
||||
// parked harness workers, sessions, git, the heartbeat and what its reply
|
||||
// directs. Absent, because they listen on the tunnel's stack or move the
|
||||
// tunnel: the ollama, LLM and PII forwards, the service slots, and endpoint
|
||||
// rotation and failover. A unix pairing's coder run has its control plane on
|
||||
// a socket in RunsDir (coder_socket.go), not a tunnel port; a tcp pairing has
|
||||
// no coder runs.
|
||||
// rotation and failover. The model servers the forwards reach are served
|
||||
// over the serve lane instead (servelane.go), by service name. A unix
|
||||
// pairing's coder run has its control plane on a socket in RunsDir
|
||||
// (coder_socket.go), not a tunnel port; a tcp pairing has no coder runs. The
|
||||
// heartbeat advertises only what runs here (agents.go, beatLanes).
|
||||
func runUnixPairing(ctx context.Context, cfg *Config) error {
|
||||
logf := func(format string, a ...any) { fmt.Printf("["+cfg.HostName+"] "+format, a...) }
|
||||
if !cfg.active() {
|
||||
return errPairingNotActive
|
||||
}
|
||||
// Before any lane: model management for this pairing (modelDaemonTarget)
|
||||
// reaches only a model server on this machine, as the serve lane does.
|
||||
defer markLocalPairing(cfg.HostName)()
|
||||
dialHost, laneTLS, err := localLaneDialer(cfg)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -762,6 +770,13 @@ func runUnixPairing(ctx context.Context, cfg *Config) error {
|
||||
logf("egress workers up — outbound exits THIS machine's internet via %s on %s\n", wsURL, lane)
|
||||
runEgressWorkers(ctx, cfg.HostName, wsURL, laneTLS.Clone(), dialHost, cfg.Bearer, realDialer.DialContext, egressWorkerCount)
|
||||
}()
|
||||
// The serve lane: the model servers set on this machine, which the tunnel
|
||||
// forwards would carry, by service name and never by address.
|
||||
go func() {
|
||||
wsURL := fmt.Sprintf("wss://%s:8443/bastion-serve", cfg.HostName)
|
||||
logf("serve workers up — the model servers set on this machine, by name, via %s on %s\n", wsURL, lane)
|
||||
runServeWorkers(ctx, cfg.HostName, cfg.dcnDir(), wsURL, laneTLS.Clone(), dialHost, cfg.Bearer, serveWorkerCount)
|
||||
}()
|
||||
go func() {
|
||||
wsURL := fmt.Sprintf("wss://%s:8443/bastion-mgmt", cfg.HostName)
|
||||
if n := sweepStaleScratch(cfg.contextRoot()); n > 0 {
|
||||
@@ -841,8 +856,11 @@ func runUnixPairing(ctx context.Context, cfg *Config) error {
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
body, ok := readHeartbeatReply(resp)
|
||||
resp.Body.Close()
|
||||
if !ok {
|
||||
return // a reply past its bound is no answer (lane_bounds.go)
|
||||
}
|
||||
if resp.StatusCode < 300 {
|
||||
recordPairingOK(cfg.HostName)
|
||||
}
|
||||
@@ -890,7 +908,7 @@ func heartbeatPayload(cfg *Config, dev interface{ IpcGet() (string, error) }, ho
|
||||
for k, v := range pairingEnforcing(cfg.HostName) {
|
||||
enforcing[k] = v
|
||||
}
|
||||
return buildHeartbeatBody(cfg.HostName, hostname, runtime.GOOS, clientVersion, wgPub, cfg.Fingerprint, cfg.BastionTunnelIP, cfg.Endpoint,
|
||||
return buildHeartbeatBodyOn(beatLanesFor(cfg), cfg.HostName, hostname, runtime.GOOS, clientVersion, wgPub, cfg.Fingerprint, cfg.BastionTunnelIP, cfg.Endpoint,
|
||||
rotation.isUnsaved(cfg), currentMirrorReport(cfg.HostName), enforcing,
|
||||
// Read fresh on every beat: a host paired a minute ago must
|
||||
// appear as a peer on the NEXT beat, not after a restart.
|
||||
@@ -1124,6 +1142,13 @@ func ollamaModels(tunnelIP string) (bool, []string) {
|
||||
addrs = append(addrs, net.JoinHostPort(tunnelIP, "11434"))
|
||||
}
|
||||
addrs = append(addrs, "127.0.0.1:11434")
|
||||
return ollamaModelsAt(addrs)
|
||||
}
|
||||
|
||||
// ollamaModelsAt is ollamaModels' probe of the addresses given, in order. A
|
||||
// local pairing's beat asks it about the one address its serve lane dials
|
||||
// (servelane.go), so what the beat says is served is what the lane reaches.
|
||||
func ollamaModelsAt(addrs []string) (bool, []string) {
|
||||
cl := &http.Client{Timeout: 3 * time.Second}
|
||||
for _, a := range addrs {
|
||||
resp, err := cl.Get("http://" + a + "/api/tags")
|
||||
@@ -1172,7 +1197,11 @@ const llmForwardPort = 18080
|
||||
// and the operator setting the env IS the opt-in. Same 200-is-alive rule as
|
||||
// /api/tags: a live server with an unparseable list still counts.
|
||||
func openAICompatModels() (bool, []string) {
|
||||
a := llmTarget()
|
||||
return openAICompatModelsAt(llmTarget())
|
||||
}
|
||||
|
||||
// openAICompatModelsAt is openAICompatModels' probe of one address.
|
||||
func openAICompatModelsAt(a string) (bool, []string) {
|
||||
if a == "" {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,721 @@
|
||||
package main
|
||||
|
||||
// The serve lane (/bastion-serve): how a Host with no platform reaches the
|
||||
// model servers on this machine.
|
||||
//
|
||||
// Over the tunnel the Host dials a forward on the tunnel's stack
|
||||
// (serveOllamaForward and serveLLMForward in run.go, servePIIForward in
|
||||
// pii.go). A local pairing (a Host on this machine's socket, or in a VM on
|
||||
// it over TCP) has no tunnel, so those forwards are absent. This lane turns
|
||||
// the direction round, as egress does: this machine parks worker WebSockets
|
||||
// on the Host, and the Host hands each one connection.
|
||||
//
|
||||
// THE HOST NAMES A SERVICE, NEVER AN ADDRESS. Its first frame on a worker is
|
||||
// 'S' + {"service":"ollama"}. The address that service reaches is the one set
|
||||
// on this machine (OLLAMA_ADDR, PRIVATAE_LLM_ADDR, PRIVATAE_PII_ADDR), read by
|
||||
// the functions the tunnel forwards read, and it must be this machine: a
|
||||
// server on another computer on the LAN is refused. Nothing the Host sends
|
||||
// can name, change or widen a target. The bastion answers 'A' and carries
|
||||
// the connection, or 'E' + a reason and closes, having dialled nothing.
|
||||
//
|
||||
// After 'A' the worker is a wsConn ('D' + bytes, 'E' for the end, both ways)
|
||||
// handed to the forwards' own code: ollama and llm through proxyInference's
|
||||
// decision and filter (the model scope, /api/pull and /api/delete refused
|
||||
// without `models`, the list rewrite, its size bounds), pii through the PII
|
||||
// scope and a pass-through. The `inference` family is checked on every
|
||||
// connection, as inferenceRefuser does at a forward's accept.
|
||||
//
|
||||
// Bounds, per pairing: 16 streams in flight across the services, the 17th
|
||||
// answered 'E' busy; a stream with no byte either way for 10 minutes is
|
||||
// closed, on every service; a response's length is not capped, because a
|
||||
// long generation is the person's own traffic.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/websocket"
|
||||
)
|
||||
|
||||
const (
|
||||
// serveWorkerCount workers stay parked per pairing. A replacement is
|
||||
// dialled as soon as one takes a job, so a long stream never holds up the
|
||||
// next request.
|
||||
serveWorkerCount = 2
|
||||
// serveMaxInFlight is the streams one pairing may hold open here at once.
|
||||
// It protects this machine and must never be what slows the person down:
|
||||
// ollama runs one request per model at a time and queues 512 more, and the
|
||||
// Host's own client holds at most 10 connections per server, so a request
|
||||
// waits in the server's queue before it waits here.
|
||||
serveMaxInFlight = 16
|
||||
// serveHelloMax bounds the 'S' frame. A service name and its JSON fit in a
|
||||
// fraction of it; anything longer is not a request this lane reads.
|
||||
serveHelloMax = 1024
|
||||
// serveTargetLookup bounds resolving a target set by name (localhost:11434).
|
||||
serveTargetLookup = 2 * time.Second
|
||||
)
|
||||
|
||||
// serveIdleTimeout closes a stream that has carried no byte either way for
|
||||
// this long. The filter's own read bound (filteredIdleTimeout) covers a request
|
||||
// being read; this covers the rest: a server that answers headers and then
|
||||
// nothing, and a pass-through, which has no bound of its own. A var so a test
|
||||
// can shorten it.
|
||||
var serveIdleTimeout = filteredIdleTimeout
|
||||
|
||||
// serveService is one service the lane carries: its name on the wire, where
|
||||
// its target is read, which forward's code serves it.
|
||||
type serveService struct {
|
||||
name string
|
||||
envVar string // named in a refusal, so the person knows what to set
|
||||
target func() string
|
||||
filter bool // through proxyInference's decision and filter
|
||||
lane inferenceLane // the filter's lane, when filter
|
||||
}
|
||||
|
||||
// serveServices is every service the lane knows, in the order the beat lists
|
||||
// them. A name not in it is refused before anything is read or dialled.
|
||||
var serveServices = []serveService{
|
||||
{name: "ollama", envVar: "OLLAMA_ADDR", target: localOllamaAddr, filter: true, lane: laneOllama},
|
||||
{name: "llm", envVar: "PRIVATAE_LLM_ADDR", target: llmTarget, filter: true, lane: laneLLM},
|
||||
{name: "pii", envVar: "PRIVATAE_PII_ADDR", target: piiTarget},
|
||||
}
|
||||
|
||||
func serveServiceNamed(name string) (serveService, bool) {
|
||||
for _, s := range serveServices {
|
||||
if s.name == name {
|
||||
return s, true
|
||||
}
|
||||
}
|
||||
return serveService{}, false
|
||||
}
|
||||
|
||||
// serveLookupIP resolves a target set by name. A seam so a test can answer.
|
||||
var serveLookupIP = func(ctx context.Context, host string) ([]net.IPAddr, error) {
|
||||
return net.DefaultResolver.LookupIPAddr(ctx, host)
|
||||
}
|
||||
|
||||
// serveVetTarget checks the address set on this machine for a service and
|
||||
// returns the addresses to dial, or the reason it is refused. Every address
|
||||
// the target names must be this machine: loopback, an address on its
|
||||
// interfaces, or one the kernel delivers here (ownaddr.go), which takes in a
|
||||
// server in a container on a network made on this machine. A server on
|
||||
// another computer is refused: a device the bastion does not run on is the
|
||||
// device lane's problem, solved there with pinning. 0.0.0.0 and :: are listen
|
||||
// addresses, and dialling one reaches this machine, so each is dialled as
|
||||
// loopback of its family. The addresses dialled are the ones checked, never
|
||||
// the name again.
|
||||
//
|
||||
// Every address checked is returned, in the resolver's order, and the dial
|
||||
// tries them in turn (serveDial, servePick), as the OS dialler does with a
|
||||
// name: localhost answers ::1 first on most systems, and ollama listens on
|
||||
// 127.0.0.1 only by default, so dialling the first address alone fails where
|
||||
// a plain dial of the name reaches it.
|
||||
func serveVetTarget(target string) ([]string, string) {
|
||||
host, port, err := net.SplitHostPort(target)
|
||||
if err != nil {
|
||||
return nil, fmt.Sprintf("%q is not host:port", target)
|
||||
}
|
||||
if n, err := strconv.Atoi(port); err != nil || n < 1 || n > 65535 {
|
||||
return nil, fmt.Sprintf("%q has no port between 1 and 65535", target)
|
||||
}
|
||||
var ips []net.IP
|
||||
if ip := net.ParseIP(host); ip != nil {
|
||||
ips = []net.IP{ip}
|
||||
} else {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), serveTargetLookup)
|
||||
as, err := serveLookupIP(ctx, host)
|
||||
cancel()
|
||||
if err != nil || len(as) == 0 {
|
||||
return nil, fmt.Sprintf("%q could not be resolved on this machine", host)
|
||||
}
|
||||
for _, a := range as {
|
||||
ips = append(ips, a.IP)
|
||||
}
|
||||
}
|
||||
for i, ip := range ips {
|
||||
switch {
|
||||
case ip.IsUnspecified() && ip.To4() != nil:
|
||||
ips[i] = net.IPv4(127, 0, 0, 1)
|
||||
case ip.IsUnspecified():
|
||||
ips[i] = net.IPv6loopback
|
||||
}
|
||||
}
|
||||
for _, ip := range ips {
|
||||
if ip.IsLoopback() {
|
||||
continue
|
||||
}
|
||||
own, err := ownDestination([]net.IP{ip})
|
||||
if err != nil {
|
||||
return nil, fmt.Sprintf("this machine could not tell whether %s is its own address (%v)", ip, err)
|
||||
}
|
||||
if !own {
|
||||
return nil, fmt.Sprintf("%s is not this machine; the lane reaches only servers on the computer it runs on", ip)
|
||||
}
|
||||
}
|
||||
out := make([]string, 0, len(ips))
|
||||
for _, ip := range ips {
|
||||
a := net.JoinHostPort(ip.String(), port)
|
||||
if !slices.Contains(out, a) {
|
||||
out = append(out, a)
|
||||
}
|
||||
}
|
||||
return out, ""
|
||||
}
|
||||
|
||||
// serveDialTimeout bounds one dial of a service's addresses, all of them
|
||||
// together, as the forwards' dial (dialUpstream) bounds its one address.
|
||||
const serveDialTimeout = 5 * time.Second
|
||||
|
||||
// serveDial dials the checked addresses in turn (dialEach: the next starts
|
||||
// when one fails, or has not answered within the Happy Eyeballs delay) and
|
||||
// returns the first that answers, with the address it was dialled at. Only
|
||||
// these addresses are dialled.
|
||||
func serveDial(addrs []string, timeout time.Duration) (net.Conn, string, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
||||
defer cancel()
|
||||
var mu sync.Mutex
|
||||
at := map[net.Conn]string{}
|
||||
d := &net.Dialer{}
|
||||
c, err := dialEach(ctx, func(ctx context.Context, network, a string) (net.Conn, error) {
|
||||
c, err := d.DialContext(ctx, network, a)
|
||||
if err == nil {
|
||||
mu.Lock()
|
||||
at[c] = a
|
||||
mu.Unlock()
|
||||
}
|
||||
return c, err
|
||||
}, addrs)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return c, at[c], nil
|
||||
}
|
||||
|
||||
// servePick is the one address an HTTP client here is given for a target
|
||||
// whose check returned several (the beat's probes, model management): the
|
||||
// first that accepts a connection, tried in turn as serveDial does, or the
|
||||
// first checked when none does, so the request fails as it would have. One
|
||||
// address is returned as it is, with nothing dialled.
|
||||
func servePick(addrs []string) string {
|
||||
if len(addrs) == 1 {
|
||||
return addrs[0]
|
||||
}
|
||||
c, a, err := serveDial(addrs, serveTargetLookup)
|
||||
if err != nil {
|
||||
return addrs[0]
|
||||
}
|
||||
_ = c.Close()
|
||||
return a
|
||||
}
|
||||
|
||||
// serveTargetFor is the addresses a service dials here, or why it does not.
|
||||
// The beat and the lane both ask it, so what the beat says is served is what
|
||||
// the lane reaches (the rule env.go states for the forwards).
|
||||
func serveTargetFor(s serveService) ([]string, string) {
|
||||
t := s.target()
|
||||
if t == "" {
|
||||
return nil, "unavailable: no " + s.name + " server is set on this machine (" + s.envVar + ")"
|
||||
}
|
||||
addrs, why := serveVetTarget(t)
|
||||
if why != "" {
|
||||
return nil, "refused: the " + s.name + " server set on this machine (" + s.envVar + ") is refused: " + why
|
||||
}
|
||||
return addrs, ""
|
||||
}
|
||||
|
||||
// serveInFlight counts each pairing's open streams, keyed by host name, so a
|
||||
// pairing restarted while its streams still run counts them.
|
||||
var serveInFlight sync.Map // string -> *atomic.Int32
|
||||
|
||||
func serveAcquire(self string) bool {
|
||||
v, _ := serveInFlight.LoadOrStore(self, new(atomic.Int32))
|
||||
n := v.(*atomic.Int32)
|
||||
for {
|
||||
cur := n.Load()
|
||||
if cur >= serveMaxInFlight {
|
||||
return false
|
||||
}
|
||||
if n.CompareAndSwap(cur, cur+1) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func serveRelease(self string) {
|
||||
if v, ok := serveInFlight.Load(self); ok {
|
||||
v.(*atomic.Int32).Add(-1)
|
||||
}
|
||||
}
|
||||
|
||||
// serveHello is the 'S' frame's JSON. Fields this build does not know are
|
||||
// ignored, so a Host may add one without breaking an older bastion.
|
||||
type serveHello struct {
|
||||
Service string `json:"service"`
|
||||
}
|
||||
|
||||
// serveVerdict decides one job before anything is dialled: the service, the
|
||||
// family, the PII scope, the target. It returns the service and the addresses,
|
||||
// or the reason the job is refused.
|
||||
func serveVerdict(self, name string) (serveService, []string, string) {
|
||||
s, ok := serveServiceNamed(name)
|
||||
if !ok {
|
||||
return s, nil, "refused: " + strconv.Quote(truncate(name, 64)) + " is not a service this lane carries " +
|
||||
"(ollama, llm or pii). The lane takes a service's name and reaches the server set on this machine for it; " +
|
||||
"it never takes an address."
|
||||
}
|
||||
if why := capsAllow(self, "inference"); why != "" {
|
||||
return s, nil, "refused: " + why
|
||||
}
|
||||
if s.name == "pii" && !pairingModelScope(self).allowsPII() {
|
||||
return s, nil, "refused: this pairing is scoped to named models and the PII sterilizer was not one of them " +
|
||||
"(approve with 'pii' in the model list to share it)"
|
||||
}
|
||||
addrs, why := serveTargetFor(s)
|
||||
if why != "" {
|
||||
return s, nil, why
|
||||
}
|
||||
return s, addrs, ""
|
||||
}
|
||||
|
||||
// serveJob serves the one job a parked worker is given. taken is called once
|
||||
// the job is accepted (its stream counted, before 'A'), so the pool parks a
|
||||
// replacement while this one works. It reports whether a job was accepted. A
|
||||
// refused job (malformed, refused, busy) reports false, and its worker waits
|
||||
// before it parks again: the replacement is not dialled at once, so a Host
|
||||
// that answers every worker with a frame it knows is refused cannot make this
|
||||
// machine dial, handshake and write an audit line in a loop as fast as it
|
||||
// answers.
|
||||
func serveJob(ws *websocket.Conn, self, dcnDir string, taken func()) bool {
|
||||
mt, r, err := ws.NextReader()
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
refuse := func(service, why string) {
|
||||
_ = ws.WriteMessage(websocket.BinaryMessage, append([]byte{'E'}, why...))
|
||||
serveAudit(dcnDir, map[string]any{"service": service, "decision": "refused", "reason": why})
|
||||
}
|
||||
raw, err := io.ReadAll(io.LimitReader(r, serveHelloMax+1))
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
var hello serveHello
|
||||
if mt != websocket.BinaryMessage || len(raw) == 0 || raw[0] != 'S' || len(raw) > serveHelloMax ||
|
||||
json.Unmarshal(raw[1:], &hello) != nil {
|
||||
// Malformed: answered, not audited (as a malformed terminal request is not).
|
||||
_ = ws.WriteMessage(websocket.BinaryMessage, []byte("Erefused: the first frame on this lane is 'S' and "+
|
||||
"{\"service\":\"<name>\"}, at most 1024 bytes"))
|
||||
return false
|
||||
}
|
||||
s, addrs, why := serveVerdict(self, hello.Service)
|
||||
if why != "" {
|
||||
refuse(truncate(hello.Service, 64), why)
|
||||
return false
|
||||
}
|
||||
if !serveAcquire(self) {
|
||||
refuse(s.name, fmt.Sprintf("busy: this pairing already has %d streams open on this machine, "+
|
||||
"which is as many as it may hold; try again when one ends", serveMaxInFlight))
|
||||
return false
|
||||
}
|
||||
defer serveRelease(self)
|
||||
taken()
|
||||
if err := ws.WriteMessage(websocket.BinaryMessage, []byte{'A'}); err != nil {
|
||||
return true
|
||||
}
|
||||
w := newServeWatch(serveIdleTimeout)
|
||||
in := w.wrap(&wsConn{ws: ws})
|
||||
var reachedMu sync.Mutex
|
||||
reached := ""
|
||||
dial := func() (net.Conn, error) {
|
||||
out, a, err := serveDial(addrs, serveDialTimeout)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
reachedMu.Lock()
|
||||
reached = a
|
||||
reachedMu.Unlock()
|
||||
return w.wrap(out), nil
|
||||
}
|
||||
if s.filter {
|
||||
proxyInferenceVia(in, dial, s.lane, self, servePass)
|
||||
} else {
|
||||
servePass(in, dial)
|
||||
}
|
||||
w.stop()
|
||||
fromHost, toHost := w.counts()
|
||||
reachedMu.Lock()
|
||||
target := reached
|
||||
reachedMu.Unlock()
|
||||
if target == "" {
|
||||
target = strings.Join(addrs, " ") // none answered, or none was dialled
|
||||
}
|
||||
serveAudit(dcnDir, map[string]any{"service": s.name, "target": target, "decision": "served",
|
||||
"bytes_from_host": fromHost, "bytes_to_host": toHost, "idle_closed": w.idleClosed()})
|
||||
return true
|
||||
}
|
||||
|
||||
// servePass is the lane's pass-through: egress's splice, which keeps a
|
||||
// half-close (the Host's 'E' ends what it sends, and the answer still comes
|
||||
// back), where the tunnel's spliceConn ends both directions with the first.
|
||||
func servePass(in net.Conn, dial upstreamDial) {
|
||||
out, err := dial()
|
||||
if err != nil {
|
||||
_ = in.Close()
|
||||
return
|
||||
}
|
||||
splice(in, in, out)
|
||||
}
|
||||
|
||||
// serveWatch is the idle bound of one stream. Both ends are wrapped; any byte
|
||||
// read or written on either one counts, and once none has for the bound,
|
||||
// both are closed, which unblocks whichever copy or read is waiting.
|
||||
type serveWatch struct {
|
||||
bound time.Duration
|
||||
last atomic.Int64 // unix nanoseconds of the last byte
|
||||
fromHost atomic.Int64
|
||||
toHost atomic.Int64
|
||||
mu sync.Mutex
|
||||
conns []net.Conn
|
||||
host net.Conn
|
||||
done chan struct{}
|
||||
once sync.Once
|
||||
idled atomic.Bool
|
||||
}
|
||||
|
||||
func newServeWatch(bound time.Duration) *serveWatch {
|
||||
w := &serveWatch{bound: bound, done: make(chan struct{})}
|
||||
w.last.Store(time.Now().UnixNano())
|
||||
go w.run()
|
||||
return w
|
||||
}
|
||||
|
||||
func (w *serveWatch) run() {
|
||||
tick := w.bound / 4
|
||||
if tick < 10*time.Millisecond {
|
||||
tick = 10 * time.Millisecond
|
||||
}
|
||||
if tick > 5*time.Second {
|
||||
tick = 5 * time.Second
|
||||
}
|
||||
t := time.NewTicker(tick)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-w.done:
|
||||
return
|
||||
case <-t.C:
|
||||
if time.Since(time.Unix(0, w.last.Load())) >= w.bound {
|
||||
w.idled.Store(true)
|
||||
w.mu.Lock()
|
||||
for _, c := range w.conns {
|
||||
_ = c.Close()
|
||||
}
|
||||
w.mu.Unlock()
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (w *serveWatch) stop() { w.once.Do(func() { close(w.done) }) }
|
||||
|
||||
func (w *serveWatch) idleClosed() bool { return w.idled.Load() }
|
||||
|
||||
func (w *serveWatch) counts() (fromHost, toHost int64) { return w.fromHost.Load(), w.toHost.Load() }
|
||||
|
||||
// wrap watches c. The first conn wrapped is the Host's: what is read from it
|
||||
// came from the Host, what is written to it goes back.
|
||||
func (w *serveWatch) wrap(c net.Conn) net.Conn {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
if w.host == nil {
|
||||
w.host = c
|
||||
}
|
||||
w.conns = append(w.conns, c)
|
||||
return &serveWatched{Conn: c, w: w, host: w.host == c}
|
||||
}
|
||||
|
||||
type serveWatched struct {
|
||||
net.Conn
|
||||
w *serveWatch
|
||||
host bool
|
||||
}
|
||||
|
||||
func (c *serveWatched) Read(p []byte) (int, error) {
|
||||
n, err := c.Conn.Read(p)
|
||||
if n > 0 {
|
||||
c.w.last.Store(time.Now().UnixNano())
|
||||
if c.host {
|
||||
c.w.fromHost.Add(int64(n))
|
||||
}
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (c *serveWatched) Write(p []byte) (int, error) {
|
||||
n, err := c.Conn.Write(p)
|
||||
if n > 0 {
|
||||
c.w.last.Store(time.Now().UnixNano())
|
||||
if c.host {
|
||||
c.w.toHost.Add(int64(n))
|
||||
}
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
// CloseWrite passes a half-close through, so splice can end one direction.
|
||||
func (c *serveWatched) CloseWrite() error {
|
||||
if cw, ok := c.Conn.(interface{ CloseWrite() error }); ok {
|
||||
return cw.CloseWrite()
|
||||
}
|
||||
return c.Conn.Close()
|
||||
}
|
||||
|
||||
// serveAudit writes one line per job to the pairing's audit log, the file
|
||||
// dcnAudit writes, and to the system log.
|
||||
func serveAudit(dir string, rec map[string]any) {
|
||||
if dir == "" {
|
||||
return
|
||||
}
|
||||
rec["ts"] = time.Now().UTC().Format(time.RFC3339)
|
||||
rec["op"] = "serve"
|
||||
raw, _ := json.Marshal(rec)
|
||||
auditToSystemLog(auditFilePairingD, auditPairingOfDir(dir), raw)
|
||||
if err := os.MkdirAll(dir, 0o700); err != nil {
|
||||
return
|
||||
}
|
||||
fh, err := os.OpenFile(filepath.Join(dir, auditFilePairingD), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o600)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer fh.Close()
|
||||
_, _ = fh.Write(append(raw, '\n'))
|
||||
}
|
||||
|
||||
// serveRetryMax is the longest a worker waits before dialling again while the
|
||||
// Host answers the lane's upgrade and refuses it. A Host that predates the
|
||||
// lane does that (an unknown WebSocket route is a 404, or a 403 from a server
|
||||
// that closes it before accepting), and two workers dialling it every 3 s
|
||||
// for good would be work for nothing on the person's machine. A Host that
|
||||
// cannot be reached at all (restarting) is dialled every 3 s, as egress is.
|
||||
var serveRetryMax = time.Minute
|
||||
|
||||
// serveRefusedWait is how long a worker waits before it parks again after its
|
||||
// WebSocket closed while parked, or after its job was refused. With
|
||||
// serveWorkerCount workers, a Host that refuses or misuses every job costs
|
||||
// this machine at most that many dials a second.
|
||||
var serveRefusedWait = time.Second
|
||||
|
||||
// localPairings counts the runs of each pairing with a Host that has no
|
||||
// platform (runUnixPairing), by host name, while they run. A count, not a
|
||||
// flag, so a run that is still stopping when the same pairing starts again
|
||||
// cannot unmark the new one.
|
||||
var localPairings sync.Map // string -> *atomic.Int32
|
||||
|
||||
// markLocalPairing records that self runs as a pairing with a Host that has no
|
||||
// platform, until the returned func is called.
|
||||
func markLocalPairing(self string) func() {
|
||||
v, _ := localPairings.LoadOrStore(self, new(atomic.Int32))
|
||||
n := v.(*atomic.Int32)
|
||||
n.Add(1)
|
||||
var once sync.Once
|
||||
return func() { once.Do(func() { n.Add(-1) }) }
|
||||
}
|
||||
|
||||
// isLocalPairing reports whether self is running as a pairing with a Host that
|
||||
// has no platform.
|
||||
func isLocalPairing(self string) bool {
|
||||
v, ok := localPairings.Load(self)
|
||||
return ok && v.(*atomic.Int32).Load() > 0
|
||||
}
|
||||
|
||||
// modelDaemonTarget is the address model management (modelswap.go: the
|
||||
// model-swap probe in the beat, model:list, model:pull and its probes,
|
||||
// model:drop) dials for the pairing self, or the reason it dials nothing.
|
||||
// For a Host with no platform it is the serve lane's own ollama target, under
|
||||
// the same rule: OLLAMA_ADDR (else loopback) only when every address it names
|
||||
// is this machine, and the address dialled is one of those checked: the first
|
||||
// that answers, tried in turn (servePick). A bastion serves only the computer
|
||||
// it runs on, and managing the models on another computer is serving it. A
|
||||
// tunnel pairing keeps OLLAMA_ADDR wherever it points, as its forwards do.
|
||||
func modelDaemonTarget(self string) (string, string) {
|
||||
t := localOllamaAddr()
|
||||
if !isLocalPairing(self) {
|
||||
return t, ""
|
||||
}
|
||||
addrs, why := serveVetTarget(t)
|
||||
if why != "" {
|
||||
s, _ := serveServiceNamed("ollama")
|
||||
serveTargetNotice(self, s, t, why)
|
||||
return "", "refused: the model server set on this machine (OLLAMA_ADDR) is not used for this Host: " + why
|
||||
}
|
||||
return servePick(addrs), ""
|
||||
}
|
||||
|
||||
// serveTargetNoticed is the last refusal said at the machine for each pairing
|
||||
// and service, so the console carries it once, and again only when it changes.
|
||||
var serveTargetNoticed sync.Map // "<self>\x00<service>" -> string
|
||||
|
||||
// serveTargetNotice says at this machine, in plain words, that the server set
|
||||
// here for a service is not used for the Host with no platform, and why. A
|
||||
// refusal that only the Host hears leaves the person who set the address with
|
||||
// nothing on their own screen to explain why the model is missing.
|
||||
func serveTargetNotice(self string, s serveService, target, why string) {
|
||||
key := self + "\x00" + s.name
|
||||
// The beat and a model op can refuse the same target at once; only the one
|
||||
// that records this reason says it.
|
||||
for {
|
||||
prev, loaded := serveTargetNoticed.LoadOrStore(key, why)
|
||||
if !loaded {
|
||||
break
|
||||
}
|
||||
if prev.(string) == why {
|
||||
return
|
||||
}
|
||||
if serveTargetNoticed.CompareAndSwap(key, prev, why) {
|
||||
break
|
||||
}
|
||||
}
|
||||
fmt.Printf("[%s] %s=%s is not used for the Host on this computer: %s. Set %s to a server on this "+
|
||||
"computer, or unset it.\n", self, s.envVar, target, why, s.envVar)
|
||||
}
|
||||
|
||||
// serveTargetNoticeClear forgets what was said for a service that is fine
|
||||
// again, so a later refusal is said again.
|
||||
func serveTargetNoticeClear(self string, s serveService) {
|
||||
serveTargetNoticed.Delete(self + "\x00" + s.name)
|
||||
}
|
||||
|
||||
// runServeWorkers keeps n workers parked on the Host's /bastion-serve for the
|
||||
// pairing self, each serving one job (serveJob). A worker that takes a job
|
||||
// starts its replacement at once and ends with the job; one whose WebSocket
|
||||
// dies while parked dials again. pctx is the pairing's context: a revoked
|
||||
// pairing's workers exit.
|
||||
func runServeWorkers(pctx context.Context, self, dcnDir, wsURL string, tlsCfg *tls.Config,
|
||||
dialHost func(ctx context.Context) (net.Conn, error), bearer string, n int) {
|
||||
dialer := &websocket.Dialer{
|
||||
NetDialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { return dialHost(ctx) },
|
||||
TLSClientConfig: tlsCfg,
|
||||
HandshakeTimeout: 20 * time.Second,
|
||||
}
|
||||
var spawn func()
|
||||
spawn = func() {
|
||||
go func() {
|
||||
wait := 3 * time.Second
|
||||
for pctx.Err() == nil {
|
||||
ctx, cancel := context.WithTimeout(pctx, 25*time.Second)
|
||||
ws, resp, err := dialer.DialContext(ctx, wsURL, http.Header{})
|
||||
cancel()
|
||||
if err != nil {
|
||||
if resp != nil {
|
||||
sleepCtx(pctx, wait)
|
||||
wait = min(2*wait, serveRetryMax)
|
||||
} else {
|
||||
sleepCtx(pctx, 3*time.Second)
|
||||
}
|
||||
continue
|
||||
}
|
||||
wait = 3 * time.Second
|
||||
if err := ws.WriteMessage(websocket.TextMessage, []byte("BEARER "+bearer)); err != nil {
|
||||
_ = ws.Close()
|
||||
sleepCtx(pctx, time.Second)
|
||||
continue
|
||||
}
|
||||
stopKA := armWSKeepalive(ws)
|
||||
unpark := context.AfterFunc(pctx, func() { _ = ws.Close() })
|
||||
took := serveJob(ws, self, dcnDir, spawn)
|
||||
unpark()
|
||||
stopKA()
|
||||
_ = ws.Close()
|
||||
if took {
|
||||
return
|
||||
}
|
||||
// Closed while parked (a Host restart), or the job refused: not
|
||||
// straight back, so a Host that closes or misuses every worker is
|
||||
// not dialled in a loop.
|
||||
sleepCtx(pctx, serveRefusedWait)
|
||||
}
|
||||
}()
|
||||
}
|
||||
for i := 0; i < n; i++ {
|
||||
spawn()
|
||||
}
|
||||
}
|
||||
|
||||
// laneServeReport is what a local pairing's beat says of its serve lane.
|
||||
type laneServeReport struct {
|
||||
served []string // lane_serves
|
||||
models []string // ollama's, filtered to the scope
|
||||
llmModels []string // the llm server's, filtered to the scope
|
||||
piiOK bool
|
||||
}
|
||||
|
||||
// laneServes is what this pairing serves over the lane right now, for the
|
||||
// beat: each service whose target is set and is this machine, whose server
|
||||
// answers its probe, and whose family the pairing holds. A scoped pairing
|
||||
// with no model left in a server's list is not served that server, as the
|
||||
// tunnel's beat withholds a lane with nothing in it.
|
||||
func laneServes(self string) laneServeReport {
|
||||
rep := laneServeReport{served: []string{}, models: []string{}, llmModels: []string{}}
|
||||
if capsAllow(self, "inference") != "" {
|
||||
return rep
|
||||
}
|
||||
scope := pairingModelScope(self)
|
||||
for _, s := range serveServices {
|
||||
t := s.target()
|
||||
if t == "" {
|
||||
continue
|
||||
}
|
||||
addrs, why := serveVetTarget(t)
|
||||
if why != "" {
|
||||
serveTargetNotice(self, s, t, why)
|
||||
continue
|
||||
}
|
||||
serveTargetNoticeClear(self, s)
|
||||
addr := servePick(addrs)
|
||||
switch s.name {
|
||||
case "ollama":
|
||||
ok, models := ollamaModelsAt([]string{addr})
|
||||
models = scope.filterModels(models)
|
||||
if !ok || (scope.scoped && len(models) == 0) {
|
||||
continue
|
||||
}
|
||||
if models != nil {
|
||||
rep.models = models
|
||||
}
|
||||
case "llm":
|
||||
ok, models := openAICompatModelsAt(addr)
|
||||
models = scope.filterModels(models)
|
||||
if !ok || (scope.scoped && len(models) == 0) {
|
||||
continue
|
||||
}
|
||||
if models != nil {
|
||||
rep.llmModels = models
|
||||
}
|
||||
case "pii":
|
||||
if !scope.allowsPII() {
|
||||
continue
|
||||
}
|
||||
if _, healthy := piiProbeAt(addr); !healthy {
|
||||
continue
|
||||
}
|
||||
rep.piiOK = true
|
||||
}
|
||||
rep.served = append(rep.served, s.name)
|
||||
}
|
||||
return rep
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
//go:build linux
|
||||
|
||||
// The serve lane dials a target set by name at each address it checked, in
|
||||
// turn (servelane.go: serveVetTarget, serveDial, servePick). Uses the fixtures
|
||||
// of servelane_linux_test.go, modelscope_test.go and modelswap_test.go.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// A target set by name is dialled at every address it was checked for, in
|
||||
// turn, as a plain dial of the name is. OLLAMA_ADDR=localhost:<port>, a
|
||||
// resolver that answers ::1 first (as most do) and a server listening on
|
||||
// 127.0.0.1 only (ollama's default): a job on the lane, the beat and model
|
||||
// management each reach it. The resolver's two addresses are both this
|
||||
// machine's and are the only ones there are to dial.
|
||||
//
|
||||
// MUTATION: serveVetTarget returns only the first address it checked (as it
|
||||
// did), or serveJob dials only addrs[0].
|
||||
// RESULT: red — the job's request gets no answer (its one dial, to [::1], is
|
||||
// refused), the beat has no ollama in lane_serves, and model management is
|
||||
// handed [::1].
|
||||
func TestTheServeLaneDialsEachCheckedAddressInTurn(t *testing.T) {
|
||||
up := newFakeOllama(t)
|
||||
listening := up.srv.Listener.Addr().String()
|
||||
if !strings.HasPrefix(listening, "127.0.0.1:") {
|
||||
t.Fatalf("the stand-in listens on %s, not 127.0.0.1 only, so this case proves nothing", listening)
|
||||
}
|
||||
port := portOf(up.srv.Listener)
|
||||
if c, err := net.DialTimeout("tcp", net.JoinHostPort("::1", port), time.Second); err == nil {
|
||||
_ = c.Close()
|
||||
t.Fatalf("something answers on [::1]:%s, so the first address would not fail and this case proves nothing", port)
|
||||
}
|
||||
old := serveLookupIP
|
||||
t.Cleanup(func() { serveLookupIP = old })
|
||||
serveLookupIP = func(_ context.Context, host string) ([]net.IPAddr, error) {
|
||||
if host != "localhost" {
|
||||
return nil, fmt.Errorf("no such host")
|
||||
}
|
||||
return []net.IPAddr{{IP: net.IPv6loopback}, {IP: net.ParseIP("127.0.0.1")}}, nil
|
||||
}
|
||||
t.Setenv("OLLAMA_ADDR", "localhost:"+port)
|
||||
t.Setenv("PRIVATAE_LLM_ADDR", "")
|
||||
t.Setenv("PRIVATAE_PII_ADDR", "")
|
||||
resetModelDaemonCache()
|
||||
h, cfg := serveLaneBox(t, "dual.example.test")
|
||||
runInBackground(t, cfg)
|
||||
|
||||
if got := strsOf(h.beat(t)["lane_serves"]); !slices.Contains(got, "ollama") {
|
||||
t.Errorf("the beat does not say ollama is served: lane_serves %q", got)
|
||||
}
|
||||
|
||||
ws := h.take(t)
|
||||
if got := ask(t, ws, "ollama"); got != "A" {
|
||||
t.Fatalf("the lane must accept ollama: %q", got)
|
||||
}
|
||||
conn := laneConn(ws)
|
||||
req, _ := http.NewRequest(http.MethodGet, "http://serve/api/tags", nil)
|
||||
req.Close = true
|
||||
if err := req.Write(conn); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = conn.SetReadDeadline(time.Now().Add(10 * time.Second))
|
||||
resp, err := http.ReadResponse(bufio.NewReader(conn), req)
|
||||
if err != nil {
|
||||
t.Fatalf("no answer over the lane (only [::1] was dialled?): %v", err)
|
||||
}
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
if resp.StatusCode != http.StatusOK || !strings.Contains(string(b), "qwen4-embed") {
|
||||
t.Fatalf("over the lane: %d %s", resp.StatusCode, b)
|
||||
}
|
||||
|
||||
if addr, why := modelDaemonTarget(cfg.HostName); addr != listening || why != "" {
|
||||
t.Errorf("model management is handed %q (%q), want the address that answers, %s", addr, why, listening)
|
||||
}
|
||||
out, errOut, code, _ := driveModelOp(t, cfg.HostName, "model:list", nil, t.TempDir())
|
||||
if code != 0 || !strings.Contains(out, "qwen4-embed") {
|
||||
t.Errorf("model:list: exit %d, %q %q", code, out, errOut)
|
||||
}
|
||||
for _, s := range up.requests() {
|
||||
if !strings.HasPrefix(s, "GET ") {
|
||||
t.Errorf("the stand-in was asked %q", s)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// servePick returns the first checked address that accepts, and the first
|
||||
// checked when none does; it never returns an address it was not given.
|
||||
func TestServePickTakesTheFirstCheckedAddressThatAnswers(t *testing.T) {
|
||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer ln.Close()
|
||||
go func() {
|
||||
for {
|
||||
c, err := ln.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
_ = c.Close()
|
||||
}
|
||||
}()
|
||||
port := portOf(ln)
|
||||
v6 := net.JoinHostPort("::1", port)
|
||||
if c, err := net.DialTimeout("tcp", v6, time.Second); err == nil {
|
||||
_ = c.Close()
|
||||
t.Fatalf("something answers on %s, so this case proves nothing", v6)
|
||||
}
|
||||
if got := servePick([]string{v6, ln.Addr().String()}); got != ln.Addr().String() {
|
||||
t.Errorf("pick %q, want %s", got, ln.Addr())
|
||||
}
|
||||
_ = ln.Close()
|
||||
if got := servePick([]string{v6, "127.0.0.1:" + port}); got != v6 {
|
||||
t.Errorf("with nothing answering, pick %q, want the first checked, %s", got, v6)
|
||||
}
|
||||
if got := servePick([]string{"127.0.0.1:" + port}); got != "127.0.0.1:"+port {
|
||||
t.Errorf("one address: %q", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,992 @@
|
||||
//go:build linux
|
||||
|
||||
// The serve lane (servelane.go) over a tcp pairing: a Host in a VM on this
|
||||
// machine reaches the model servers set here by service name, through the
|
||||
// forwards' own filter, within the lane's bounds, and the pairing's beat says
|
||||
// what it serves.
|
||||
//
|
||||
// Uses the fixtures of local_host_linux_test.go and local_host_tcp_linux_test.go
|
||||
// (newTestCA, tcpPairingFor, laneListener, pairingBox...), so it carries the
|
||||
// same build tag.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/websocket"
|
||||
)
|
||||
|
||||
// ── fixtures ────────────────────────────────────────────────────────────────
|
||||
|
||||
// serveHost is a Host's lane port as the serve lane meets it: the lane
|
||||
// port's mutual TLS, every worker route kept parked, each beat recorded
|
||||
// whole, and each /bastion-serve worker handed to the test once it has sent
|
||||
// its hello.
|
||||
type serveHost struct {
|
||||
mu sync.Mutex
|
||||
beats []map[string]any
|
||||
hellos []string
|
||||
workers chan *websocket.Conn
|
||||
}
|
||||
|
||||
func startServeHost(t *testing.T, raw net.Listener, cert tls.Certificate, clientCA *testCA) *serveHost {
|
||||
t.Helper()
|
||||
h := &serveHost{workers: make(chan *websocket.Conn, 64)}
|
||||
pool := x509.NewCertPool()
|
||||
pool.AddCert(clientCA.cert)
|
||||
ln := tls.NewListener(raw, &tls.Config{Certificates: []tls.Certificate{cert}, ClientCAs: pool,
|
||||
ClientAuth: tls.RequireAndVerifyClientCert})
|
||||
up := websocket.Upgrader{}
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/api/network/bastions/heartbeat", func(w http.ResponseWriter, r *http.Request) {
|
||||
var m map[string]any
|
||||
_ = json.NewDecoder(r.Body).Decode(&m)
|
||||
h.mu.Lock()
|
||||
h.beats = append(h.beats, m)
|
||||
h.mu.Unlock()
|
||||
_, _ = w.Write([]byte("{}"))
|
||||
})
|
||||
mux.HandleFunc("/bastion-serve", func(w http.ResponseWriter, r *http.Request) {
|
||||
c, err := up.Upgrade(w, r, nil)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
_, hello, err := c.ReadMessage()
|
||||
if err != nil {
|
||||
c.Close()
|
||||
return
|
||||
}
|
||||
h.mu.Lock()
|
||||
h.hellos = append(h.hellos, string(hello))
|
||||
h.mu.Unlock()
|
||||
h.workers <- c // the test owns it from here
|
||||
})
|
||||
park := func(w http.ResponseWriter, r *http.Request) {
|
||||
c, err := up.Upgrade(w, r, nil)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer c.Close()
|
||||
for {
|
||||
if _, _, err := c.ReadMessage(); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, p := range []string{"/bastion-harness", "/bastion-mgmt", "/bastion-egress"} {
|
||||
mux.HandleFunc(p, park)
|
||||
}
|
||||
srv := &http.Server{Handler: mux, ErrorLog: log.New(io.Discard, "", 0)}
|
||||
go func() { _ = srv.Serve(ln) }()
|
||||
t.Cleanup(func() {
|
||||
_ = srv.Close()
|
||||
for {
|
||||
select {
|
||||
case c := <-h.workers:
|
||||
c.Close()
|
||||
default:
|
||||
return
|
||||
}
|
||||
}
|
||||
})
|
||||
return h
|
||||
}
|
||||
|
||||
// take is the next parked serve worker.
|
||||
func (h *serveHost) take(t *testing.T) *websocket.Conn {
|
||||
t.Helper()
|
||||
select {
|
||||
case c := <-h.workers:
|
||||
t.Cleanup(func() { c.Close() })
|
||||
return c
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("no /bastion-serve worker parked on the Host within 10 s")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ask sends the 'S' frame for service and returns the bastion's answer: "A",
|
||||
// or "E" and the reason.
|
||||
func ask(t *testing.T, c *websocket.Conn, service string) string {
|
||||
t.Helper()
|
||||
b, _ := json.Marshal(map[string]string{"service": service})
|
||||
return askRaw(t, c, websocket.BinaryMessage, append([]byte{'S'}, b...))
|
||||
}
|
||||
|
||||
func askRaw(t *testing.T, c *websocket.Conn, mt int, frame []byte) string {
|
||||
t.Helper()
|
||||
if err := c.WriteMessage(mt, frame); err != nil {
|
||||
t.Fatalf("send the first frame: %v", err)
|
||||
}
|
||||
_ = c.SetReadDeadline(time.Now().Add(10 * time.Second))
|
||||
_, m, err := c.ReadMessage()
|
||||
_ = c.SetReadDeadline(time.Time{})
|
||||
if err != nil {
|
||||
t.Fatalf("no answer to the first frame: %v", err)
|
||||
}
|
||||
return string(m)
|
||||
}
|
||||
|
||||
// beat is the first beat the Host received, waiting up to 10 s.
|
||||
func (h *serveHost) beat(t *testing.T) map[string]any {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(10 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
h.mu.Lock()
|
||||
if len(h.beats) > 0 {
|
||||
b := h.beats[0]
|
||||
h.mu.Unlock()
|
||||
return b
|
||||
}
|
||||
h.mu.Unlock()
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
t.Fatal("no heartbeat reached the Host within 10 s")
|
||||
return nil
|
||||
}
|
||||
|
||||
// runInBackground runs the pairing until the test ends.
|
||||
func runInBackground(t *testing.T, cfg *Config) {
|
||||
t.Helper()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- runPairing(ctx, cfg) }()
|
||||
t.Cleanup(func() {
|
||||
cancel()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Error("the pairing did not stop")
|
||||
}
|
||||
// A stopped (revoked) pairing's streams end with it: each one's worker
|
||||
// is closed. Waited for here too because a stream's last act is its
|
||||
// audit line, in the pairing's directory, which the test then removes.
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for {
|
||||
v, ok := serveInFlight.Load(cfg.HostName)
|
||||
if !ok || v.(*atomic.Int32).Load() == 0 {
|
||||
return
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Errorf("%d streams still open 5 s after the pairing stopped", v.(*atomic.Int32).Load())
|
||||
return
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// serveLaneBox is an unanchored local machine with a tcp pairing to a
|
||||
// stand-in Host on 127.0.0.1, running. host keeps each test's pairing (and its
|
||||
// stream count) its own.
|
||||
func serveLaneBox(t *testing.T, host string) (*serveHost, *Config) {
|
||||
t.Helper()
|
||||
pairingBox(t)
|
||||
laneTestProcess(t)
|
||||
ca := newTestCA(t, "home host CA")
|
||||
ln := laneListener(t)
|
||||
h := startServeHost(t, ln, ca.serverCert(t, tcpServerName), ca)
|
||||
cfg := tcpPairingFor(t, ca, host, ln.Addr().String())
|
||||
if err := cfg.save(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return h, cfg
|
||||
}
|
||||
|
||||
// laneConn is the Host's end of an accepted job: 'D' frames both ways, 'E'
|
||||
// for the end, as the Host's relay frames a spliced connection.
|
||||
func laneConn(c *websocket.Conn) net.Conn { return &wsConn{ws: c} }
|
||||
|
||||
// chatRequest writes one streaming chat request on conn and returns the reader
|
||||
// its response is read from.
|
||||
func chatRequest(t *testing.T, conn net.Conn, model string) (*http.Response, *http.Request) {
|
||||
t.Helper()
|
||||
body := fmt.Sprintf(`{"model":%q,"stream":true,"messages":[{"role":"user","content":"hello"}]}`, model)
|
||||
req, _ := http.NewRequest(http.MethodPost, "http://serve/v1/chat/completions", strings.NewReader(body))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
if err := req.Write(conn); err != nil {
|
||||
t.Fatalf("write the request: %v", err)
|
||||
}
|
||||
_ = conn.SetReadDeadline(time.Now().Add(15 * time.Second))
|
||||
resp, err := http.ReadResponse(bufio.NewReader(conn), req)
|
||||
if err != nil {
|
||||
t.Fatalf("read the response: %v", err)
|
||||
}
|
||||
return resp, req
|
||||
}
|
||||
|
||||
// ── 1. a chat streams ───────────────────────────────────────────────────────
|
||||
|
||||
// streamingServer is a stand-in OpenAI-compatible server. Its chat answer is
|
||||
// three chunks; after the first it waits until the test says the Host has
|
||||
// read it, and records whether it had to give up waiting.
|
||||
type streamingServer struct {
|
||||
srv *httptest.Server
|
||||
sawFirst chan struct{}
|
||||
gated atomic.Bool
|
||||
seen atomic.Int32
|
||||
}
|
||||
|
||||
func newStreamingServer(t *testing.T) *streamingServer {
|
||||
t.Helper()
|
||||
s := &streamingServer{sawFirst: make(chan struct{})}
|
||||
s.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
s.seen.Add(1)
|
||||
switch r.URL.Path {
|
||||
case "/v1/chat/completions":
|
||||
case "/api/tags":
|
||||
_, _ = w.Write([]byte(`{"models":[{"name":"test-model:latest"}]}`))
|
||||
return
|
||||
default:
|
||||
_, _ = w.Write([]byte(`{"ok":true}`))
|
||||
return
|
||||
}
|
||||
_, _ = io.Copy(io.Discard, r.Body)
|
||||
w.Header().Set("Content-Type", "text/event-stream")
|
||||
fl := w.(http.Flusher)
|
||||
_, _ = io.WriteString(w, "data: {\"chunk\":1}\n\n")
|
||||
fl.Flush()
|
||||
select {
|
||||
case <-s.sawFirst:
|
||||
s.gated.Store(true)
|
||||
case <-time.After(10 * time.Second):
|
||||
}
|
||||
_, _ = io.WriteString(w, "data: {\"chunk\":2}\n\n")
|
||||
fl.Flush()
|
||||
_, _ = io.WriteString(w, "data: {\"chunk\":3}\n\n")
|
||||
fl.Flush()
|
||||
_, _ = io.WriteString(w, "data: [DONE]\n\n")
|
||||
}))
|
||||
t.Cleanup(s.srv.Close)
|
||||
return s
|
||||
}
|
||||
|
||||
// SECURITY (and the lane's purpose): over a tcp pairing, the Host asks for a
|
||||
// service by name and a chat streams from the server set here. The Host reads
|
||||
// chunk 1 before the server writes chunk 3, so nothing in the path holds a
|
||||
// response back: not the lane's framing, not the pass-through an unscoped
|
||||
// pairing gets, not the filter a pairing without `models` gets.
|
||||
//
|
||||
// MUTATION: drop the serve workers from runUnixPairing.
|
||||
// RESULT: red — no /bastion-serve worker ever parks on the Host.
|
||||
func TestTheServeLaneStreamsAChatOverATCPPairing(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
name, service, env string
|
||||
modelsOff bool
|
||||
}{
|
||||
{"llm, passed through", "llm", "PRIVATAE_LLM_ADDR", false},
|
||||
{"ollama, through the filter", "ollama", "OLLAMA_ADDR", true},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
s := newStreamingServer(t)
|
||||
t.Setenv(c.env, s.srv.Listener.Addr().String())
|
||||
h, cfg := serveLaneBox(t, "stream-"+c.service+".example.test")
|
||||
if c.modelsOff {
|
||||
if err := localDenySet(cfg.HostName, "models", true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !inferenceFilterNeeded(cfg.HostName, laneOllama, pairingModelScope(cfg.HostName)) {
|
||||
t.Fatal("with models off the ollama lane must be filtered, or this case proves nothing")
|
||||
}
|
||||
}
|
||||
runInBackground(t, cfg)
|
||||
ws := h.take(t)
|
||||
if got := ask(t, ws, c.service); got != "A" {
|
||||
t.Fatalf("the lane must accept %s: %q", c.service, got)
|
||||
}
|
||||
conn := laneConn(ws)
|
||||
resp, _ := chatRequest(t, conn, "test-model")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("the chat answered %d", resp.StatusCode)
|
||||
}
|
||||
br := bufio.NewReader(resp.Body)
|
||||
var got []string
|
||||
first := false
|
||||
for {
|
||||
line, err := br.ReadString('\n')
|
||||
if strings.HasPrefix(line, "data: ") {
|
||||
got = append(got, strings.TrimSpace(line))
|
||||
if !first && strings.Contains(line, `"chunk":1`) {
|
||||
first = true
|
||||
close(s.sawFirst)
|
||||
}
|
||||
}
|
||||
if err != nil || strings.Contains(line, "[DONE]") {
|
||||
break
|
||||
}
|
||||
}
|
||||
if !first {
|
||||
t.Fatalf("the stream never carried chunk 1: %q", got)
|
||||
}
|
||||
if !s.gated.Load() {
|
||||
t.Fatal("the server wrote chunk 3 before the Host read chunk 1: the lane held the stream back")
|
||||
}
|
||||
want := []string{`data: {"chunk":1}`, `data: {"chunk":2}`, `data: {"chunk":3}`, `data: [DONE]`}
|
||||
if !slices.Equal(got, want) {
|
||||
t.Fatalf("the stream arrived as %q", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// ── 2. a service, never an address ──────────────────────────────────────────
|
||||
|
||||
// SECURITY: the Host names a service and the target is the one set on this
|
||||
// machine. A first frame naming an address, a name the lane does not carry, a
|
||||
// frame that is not 'S', a target set to a computer that is not this one, and
|
||||
// a service with no target each answer 'E' with a reason, and nothing is
|
||||
// dialled: a listener at the address named gets no connection.
|
||||
//
|
||||
// MUTATION: in serveVetTarget, skip the ownDestination check.
|
||||
// RESULT: red — ollama with OLLAMA_ADDR on another computer is accepted.
|
||||
func TestTheServeLaneTakesAServiceNameNeverAnAddress(t *testing.T) {
|
||||
named, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer named.Close()
|
||||
var reached atomic.Int32
|
||||
go func() {
|
||||
for {
|
||||
c, err := named.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
reached.Add(1)
|
||||
c.Close()
|
||||
}
|
||||
}()
|
||||
t.Setenv("OLLAMA_ADDR", "192.0.2.10:11434") // TEST-NET-1: never this machine
|
||||
t.Setenv("PRIVATAE_LLM_ADDR", "")
|
||||
t.Setenv("PRIVATAE_PII_ADDR", "198.51.100.7:"+portOf(named))
|
||||
h, cfg := serveLaneBox(t, "names.example.test")
|
||||
runInBackground(t, cfg)
|
||||
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
frame []byte
|
||||
mt int
|
||||
want []string
|
||||
}{
|
||||
{"an address for a service", sFrame(named.Addr().String()), websocket.BinaryMessage,
|
||||
[]string{"Erefused: ", "never takes an address"}},
|
||||
{"a name the lane does not carry", sFrame("nas"), websocket.BinaryMessage,
|
||||
[]string{"Erefused: ", `"nas" is not a service`}},
|
||||
{"a URL for a service", sFrame("http://" + named.Addr().String() + "/api/tags"), websocket.BinaryMessage,
|
||||
[]string{"Erefused: ", "is not a service"}},
|
||||
{"a frame that is not S", []byte("D" + "GET / HTTP/1.1\r\n\r\n"), websocket.BinaryMessage,
|
||||
[]string{"Erefused: ", "the first frame on this lane is 'S'"}},
|
||||
{"a text frame", []byte(`S{"service":"ollama"}`), websocket.TextMessage,
|
||||
[]string{"Erefused: ", "the first frame on this lane is 'S'"}},
|
||||
{"an S frame past its bound", append(sFrame("ollama"), make([]byte, serveHelloMax)...), websocket.BinaryMessage,
|
||||
[]string{"Erefused: ", "at most 1024 bytes"}},
|
||||
{"ollama set to another computer", sFrame("ollama"), websocket.BinaryMessage,
|
||||
[]string{"Erefused: ", "OLLAMA_ADDR", "192.0.2.10 is not this machine"}},
|
||||
{"pii set to another computer", sFrame("pii"), websocket.BinaryMessage,
|
||||
[]string{"Erefused: ", "PRIVATAE_PII_ADDR", "198.51.100.7 is not this machine"}},
|
||||
{"llm with nothing set", sFrame("llm"), websocket.BinaryMessage,
|
||||
[]string{"Eunavailable: ", "PRIVATAE_LLM_ADDR"}},
|
||||
} {
|
||||
ws := h.take(t)
|
||||
got := askRaw(t, ws, c.mt, c.frame)
|
||||
for _, w := range c.want {
|
||||
if !strings.Contains(got, w) {
|
||||
t.Errorf("%s: the answer %q does not carry %q", c.name, got, w)
|
||||
}
|
||||
}
|
||||
if !strings.HasPrefix(got, "E") {
|
||||
t.Errorf("%s: answered %q, not E", c.name, got)
|
||||
}
|
||||
// Refused means closed: nothing after the E.
|
||||
_ = ws.SetReadDeadline(time.Now().Add(3 * time.Second))
|
||||
if _, m, err := ws.ReadMessage(); err == nil {
|
||||
t.Errorf("%s: the worker carried on after its refusal: %q", c.name, m)
|
||||
}
|
||||
}
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
if n := reached.Load(); n != 0 {
|
||||
t.Fatalf("a refused job dialled the address it named (%d connections)", n)
|
||||
}
|
||||
}
|
||||
|
||||
func sFrame(service string) []byte {
|
||||
b, _ := json.Marshal(map[string]string{"service": service})
|
||||
return append([]byte{'S'}, b...)
|
||||
}
|
||||
|
||||
func portOf(ln net.Listener) string {
|
||||
_, p, _ := net.SplitHostPort(ln.Addr().String())
|
||||
return p
|
||||
}
|
||||
|
||||
// A target set by name is resolved and each address checked; the addresses
|
||||
// dialled are the ones checked, every one of them, in the resolver's order.
|
||||
// 0.0.0.0 is a listen address and dials as loopback (once, if loopback is
|
||||
// there already). This machine's own interface address passes.
|
||||
func TestServeVetTargetChecksEveryAddressItWouldDial(t *testing.T) {
|
||||
old := serveLookupIP
|
||||
t.Cleanup(func() { serveLookupIP = old })
|
||||
serveLookupIP = func(_ context.Context, host string) ([]net.IPAddr, error) {
|
||||
switch host {
|
||||
case "localhost":
|
||||
return []net.IPAddr{{IP: net.ParseIP("127.0.0.1")}, {IP: net.ParseIP("::1")}}, nil
|
||||
case "v6first.example.test":
|
||||
return []net.IPAddr{{IP: net.ParseIP("::1")}, {IP: net.ParseIP("127.0.0.1")}, {IP: net.ParseIP("0.0.0.0")}}, nil
|
||||
case "mixed.example.test":
|
||||
return []net.IPAddr{{IP: net.ParseIP("127.0.0.1")}, {IP: net.ParseIP("192.0.2.10")}}, nil
|
||||
}
|
||||
return nil, fmt.Errorf("no such host")
|
||||
}
|
||||
for target, want := range map[string][]string{
|
||||
"127.0.0.1:11434": {"127.0.0.1:11434"},
|
||||
"[::1]:8080": {"[::1]:8080"},
|
||||
"0.0.0.0:11434": {"127.0.0.1:11434"},
|
||||
"[::]:11434": {"[::1]:11434"},
|
||||
"localhost:11434": {"127.0.0.1:11434", "[::1]:11434"},
|
||||
"v6first.example.test:80": {"[::1]:80", "127.0.0.1:80"},
|
||||
"mixed.example.test:11434": nil,
|
||||
"nowhere.example.test:80": nil,
|
||||
"192.0.2.10:11434": nil,
|
||||
"198.51.100.9:80": nil,
|
||||
"127.0.0.1": nil,
|
||||
"127.0.0.1:0": nil,
|
||||
"127.0.0.1:99999": nil,
|
||||
} {
|
||||
got, why := serveVetTarget(target)
|
||||
if !slices.Equal(got, want) || (want == nil) != (why != "") {
|
||||
t.Errorf("%q: dial %q (%q), want %q", target, got, why, want)
|
||||
}
|
||||
}
|
||||
ips, err := ownInterfaceIPs()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, ip := range ips {
|
||||
if ip.IsLoopback() || ip.IsLinkLocalUnicast() {
|
||||
continue
|
||||
}
|
||||
a := net.JoinHostPort(ip.String(), "11434")
|
||||
if got, why := serveVetTarget(a); !slices.Equal(got, []string{a}) {
|
||||
t.Errorf("this machine's own address %s is refused: %q", a, why)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── 3. the forwards' walls, unchanged ───────────────────────────────────────
|
||||
|
||||
// SECURITY: the lane keeps every wall the tunnel forwards keep. With
|
||||
// `inference` switched off at this machine, a job is refused before any dial.
|
||||
// With `models` off, /api/pull and /api/delete are refused 403 and never reach
|
||||
// ollama, while a chat does. On an anchored machine they are refused to the
|
||||
// anchor itself. A pairing scoped to a model is refused another one with 403,
|
||||
// which never reaches the server, and is served its own.
|
||||
func TestTheServeLaneKeepsTheForwardsWalls(t *testing.T) {
|
||||
t.Run("inference off at the machine", func(t *testing.T) {
|
||||
// A raw listener, so a dial by the serve job is seen even though the
|
||||
// Host sends it nothing: a connection that does not open with a beat's
|
||||
// probe (GET /api/tags, which any pairing's beat in this process may
|
||||
// send to OLLAMA_ADDR) is the job's.
|
||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { ln.Close() })
|
||||
var jobDials atomic.Int32
|
||||
go func() {
|
||||
for {
|
||||
c, err := ln.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
go func() {
|
||||
defer c.Close()
|
||||
_ = c.SetReadDeadline(time.Now().Add(2 * time.Second))
|
||||
b := make([]byte, len("GET /api/tags "))
|
||||
if _, err := io.ReadFull(c, b); err != nil || string(b) != "GET /api/tags " {
|
||||
jobDials.Add(1)
|
||||
}
|
||||
}()
|
||||
}
|
||||
}()
|
||||
t.Setenv("OLLAMA_ADDR", ln.Addr().String())
|
||||
h, cfg := serveLaneBox(t, "walls-off.example.test")
|
||||
if err := localDenySet(cfg.HostName, "inference", true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runInBackground(t, cfg)
|
||||
got := ask(t, h.take(t), "ollama")
|
||||
if !strings.HasPrefix(got, "Erefused: ") {
|
||||
t.Fatalf("with inference off the job must be refused: %q", got)
|
||||
}
|
||||
time.Sleep(2500 * time.Millisecond)
|
||||
if n := jobDials.Load(); n != 0 {
|
||||
t.Fatalf("a refused job dialled ollama (%d connections)", n)
|
||||
}
|
||||
})
|
||||
pullAndDelete := func(t *testing.T, h *serveHost, up *fakeOllama) {
|
||||
t.Helper()
|
||||
for _, p := range []string{"/api/pull", "/api/delete"} {
|
||||
ws := h.take(t)
|
||||
if got := ask(t, ws, "ollama"); got != "A" {
|
||||
t.Fatalf("%s: the job must be accepted, and refused by the filter: %q", p, got)
|
||||
}
|
||||
conn := laneConn(ws)
|
||||
req, _ := http.NewRequest(http.MethodPost, "http://serve"+p, strings.NewReader(`{"model":"test-model"}`))
|
||||
if err := req.Write(conn); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = conn.SetReadDeadline(time.Now().Add(10 * time.Second))
|
||||
resp, err := http.ReadResponse(bufio.NewReader(conn), req)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", p, err)
|
||||
}
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
if resp.StatusCode != http.StatusForbidden || !strings.Contains(string(b), "models") {
|
||||
t.Fatalf("%s over the lane: %d %s", p, resp.StatusCode, b)
|
||||
}
|
||||
}
|
||||
for _, s := range up.requests() {
|
||||
if strings.Contains(s, "/api/pull") || strings.Contains(s, "/api/delete") {
|
||||
t.Fatalf("a refused store request reached ollama: %q", up.requests())
|
||||
}
|
||||
}
|
||||
}
|
||||
t.Run("models off at the machine", func(t *testing.T) {
|
||||
up := newFakeOllama(t)
|
||||
t.Setenv("OLLAMA_ADDR", up.srv.Listener.Addr().String())
|
||||
h, cfg := serveLaneBox(t, "walls-models.example.test")
|
||||
if err := localDenySet(cfg.HostName, "models", true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runInBackground(t, cfg)
|
||||
pullAndDelete(t, h, up)
|
||||
})
|
||||
t.Run("an anchored machine, as its anchor", func(t *testing.T) {
|
||||
up := newFakeOllama(t)
|
||||
t.Setenv("OLLAMA_ADDR", up.srv.Listener.Addr().String())
|
||||
h, cfg := serveLaneBox(t, "walls-anchor.example.test")
|
||||
p, _ := modePath()
|
||||
if err := os.WriteFile(p, []byte(`{"mode":"networked"}`), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := writeAnchor("Acme IT", cfg.HostName); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if currentAnchor() == nil || capsAllow(cfg.HostName, "inference") != "" {
|
||||
t.Fatal("the pairing must be this machine's anchor, holding inference, or this case proves nothing")
|
||||
}
|
||||
runInBackground(t, cfg)
|
||||
pullAndDelete(t, h, up)
|
||||
})
|
||||
t.Run("a pairing scoped to one model", func(t *testing.T) {
|
||||
priv := setupBox(t)
|
||||
withMarkerAt(t, filepath.Join(t.TempDir(), "machine.json"))
|
||||
laneTestProcess(t)
|
||||
up := newFakeOllama(t)
|
||||
t.Setenv("PRIVATAE_LLM_ADDR", up.srv.Listener.Addr().String())
|
||||
ca := newTestCA(t, "home host CA")
|
||||
ln := laneListener(t)
|
||||
h := startServeHost(t, ln, ca.serverCert(t, tcpServerName), ca)
|
||||
member := tcpPairingFor(t, ca, tMember, ln.Addr().String())
|
||||
member.State = pairingPending
|
||||
if err := member.save(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
syncGrant(t, priv, tMember, []string{"inference"}, []string{"qwen4-embed"})
|
||||
cfgs, err := loadPairings()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg := pairingFor(cfgs, tMember)
|
||||
if cfg == nil || !cfg.active() || !cfg.tcpTransport() {
|
||||
t.Fatalf("the member's tcp pairing must be active after the grant: %+v", cfg)
|
||||
}
|
||||
runInBackground(t, cfg)
|
||||
for model, want := range map[string]int{"llama3:70b": http.StatusForbidden, "qwen4-embed": http.StatusOK} {
|
||||
ws := h.take(t)
|
||||
if got := ask(t, ws, "llm"); got != "A" {
|
||||
t.Fatalf("the member holds inference, so the job is accepted: %q", got)
|
||||
}
|
||||
resp, _ := chatRequest(t, laneConn(ws), model)
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
if resp.StatusCode != want {
|
||||
t.Fatalf("%s over the scoped lane: %d %s", model, resp.StatusCode, b)
|
||||
}
|
||||
}
|
||||
chats := 0
|
||||
for _, s := range up.requests() {
|
||||
if s == "POST /v1/chat/completions" {
|
||||
chats++
|
||||
}
|
||||
}
|
||||
if chats != 1 {
|
||||
t.Fatalf("only the scoped model's request may reach the server: %q", up.requests())
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// ── 4. the bounds ───────────────────────────────────────────────────────────
|
||||
|
||||
// SECURITY: one pairing holds at most 16 streams open here. Sixteen run at
|
||||
// once; the 17th is answered 'E' busy with the reason and dials nothing; once
|
||||
// one ends, the next is accepted.
|
||||
func TestTheServeLaneHoldsSixteenStreamsAndRefusesTheSeventeenth(t *testing.T) {
|
||||
var held, opened atomic.Int32
|
||||
release := make(chan struct{})
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/v1/chat/completions" {
|
||||
_, _ = w.Write([]byte(`{}`)) // the beat's probe
|
||||
return
|
||||
}
|
||||
opened.Add(1)
|
||||
held.Add(1)
|
||||
defer held.Add(-1)
|
||||
w.Header().Set("Content-Type", "text/event-stream")
|
||||
_, _ = io.WriteString(w, "data: {\"chunk\":1}\n\n")
|
||||
w.(http.Flusher).Flush()
|
||||
select {
|
||||
case <-release:
|
||||
case <-r.Context().Done():
|
||||
}
|
||||
}))
|
||||
t.Cleanup(func() { close(release); srv.Close() })
|
||||
t.Setenv("PRIVATAE_LLM_ADDR", srv.Listener.Addr().String())
|
||||
h, cfg := serveLaneBox(t, "sixteen.example.test")
|
||||
runInBackground(t, cfg)
|
||||
|
||||
var streams []*websocket.Conn
|
||||
for i := 0; i < serveMaxInFlight; i++ {
|
||||
ws := h.take(t)
|
||||
if got := ask(t, ws, "llm"); got != "A" {
|
||||
t.Fatalf("stream %d of %d must be accepted: %q", i+1, serveMaxInFlight, got)
|
||||
}
|
||||
resp, _ := chatRequest(t, laneConn(ws), "test-model")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("stream %d: %d", i+1, resp.StatusCode)
|
||||
}
|
||||
streams = append(streams, ws)
|
||||
}
|
||||
if n := held.Load(); n != serveMaxInFlight {
|
||||
t.Fatalf("the server holds %d streams, want %d at once", n, serveMaxInFlight)
|
||||
}
|
||||
got := ask(t, h.take(t), "llm")
|
||||
if !strings.HasPrefix(got, "Ebusy: ") || !strings.Contains(got, "16 streams") {
|
||||
t.Fatalf("the 17th stream must be answered busy with the reason: %q", got)
|
||||
}
|
||||
if n := opened.Load(); n != serveMaxInFlight {
|
||||
t.Fatalf("the refused 17th reached the server (%d requests)", n)
|
||||
}
|
||||
// One ends; the next is accepted.
|
||||
streams[0].Close()
|
||||
deadline := time.Now().Add(10 * time.Second)
|
||||
for {
|
||||
ws := h.take(t)
|
||||
got := ask(t, ws, "llm")
|
||||
if got == "A" {
|
||||
break
|
||||
}
|
||||
if !strings.HasPrefix(got, "Ebusy: ") || time.Now().After(deadline) {
|
||||
t.Fatalf("after a stream ended the next must be accepted: %q", got)
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
// SECURITY: a stream with no byte either way for the idle bound is closed,
|
||||
// both ends, on a pass-through and through the filter: a server that answers
|
||||
// headers and then nothing does not hold the stream (or this machine's slot)
|
||||
// for ever. A stream that keeps sending, for several times the bound, is not
|
||||
// cut: the bound is on silence, never on a response's length.
|
||||
func TestTheServeLaneClosesAStreamIdleForTheBound(t *testing.T) {
|
||||
old := serveIdleTimeout
|
||||
serveIdleTimeout = 400 * time.Millisecond
|
||||
t.Cleanup(func() { serveIdleTimeout = old })
|
||||
|
||||
// silent answers headers and then holds the connection, recording when
|
||||
// this machine closes it.
|
||||
silentClosed := make(chan time.Time, 4)
|
||||
silent, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { silent.Close() })
|
||||
go func() {
|
||||
for {
|
||||
c, err := silent.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
go func(c net.Conn) {
|
||||
defer c.Close()
|
||||
br := bufio.NewReader(c)
|
||||
req, err := http.ReadRequest(br)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if req.URL.Path != "/v1/chat/completions" {
|
||||
// The beat's probes get an ordinary answer.
|
||||
_, _ = io.WriteString(c, "HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\n{}")
|
||||
return
|
||||
}
|
||||
_, _ = io.WriteString(c, "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nTransfer-Encoding: chunked\r\n\r\n")
|
||||
_, _ = io.Copy(io.Discard, br)
|
||||
silentClosed <- time.Now()
|
||||
}(c)
|
||||
}
|
||||
}()
|
||||
// steady sends a chunk every 100 ms for 1.6 s: four times the bound.
|
||||
steady := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/v1/chat/completions" {
|
||||
_, _ = w.Write([]byte(`{}`))
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "text/event-stream")
|
||||
for i := 1; i <= 16; i++ {
|
||||
fmt.Fprintf(w, "data: {\"chunk\":%d}\n\n", i)
|
||||
w.(http.Flusher).Flush()
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
_, _ = io.WriteString(w, "data: [DONE]\n\n")
|
||||
}))
|
||||
t.Cleanup(steady.Close)
|
||||
|
||||
for _, c := range []struct {
|
||||
name, service, env string
|
||||
modelsOff bool
|
||||
}{
|
||||
{"a pass-through", "llm", "PRIVATAE_LLM_ADDR", false},
|
||||
{"through the filter", "ollama", "OLLAMA_ADDR", true},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
t.Setenv(c.env, silent.Addr().String())
|
||||
h, cfg := serveLaneBox(t, "idle-"+c.service+".example.test")
|
||||
if c.modelsOff {
|
||||
if err := localDenySet(cfg.HostName, "models", true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
runInBackground(t, cfg)
|
||||
ws := h.take(t)
|
||||
if got := ask(t, ws, c.service); got != "A" {
|
||||
t.Fatalf("accept: %q", got)
|
||||
}
|
||||
conn := laneConn(ws)
|
||||
resp, _ := chatRequest(t, conn, "test-model")
|
||||
start := time.Now()
|
||||
_ = conn.SetReadDeadline(time.Now().Add(10 * time.Second))
|
||||
_, err := io.Copy(io.Discard, resp.Body)
|
||||
took := time.Since(start)
|
||||
if err == nil {
|
||||
t.Fatal("a silent stream must end in a close, not a complete answer")
|
||||
}
|
||||
if took < 300*time.Millisecond || took > 3*time.Second {
|
||||
t.Fatalf("the silent stream was closed after %v; the bound is %v", took, serveIdleTimeout)
|
||||
}
|
||||
select {
|
||||
case <-silentClosed:
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("the server's end of the silent stream was left open")
|
||||
}
|
||||
|
||||
t.Setenv(c.env, steady.Listener.Addr().String())
|
||||
ws = h.take(t)
|
||||
if got := ask(t, ws, c.service); got != "A" {
|
||||
t.Fatalf("accept: %q", got)
|
||||
}
|
||||
resp, _ = chatRequest(t, laneConn(ws), "test-model")
|
||||
b, err := io.ReadAll(resp.Body)
|
||||
if err != nil || !strings.Contains(string(b), `"chunk":16`) || !strings.Contains(string(b), "[DONE]") {
|
||||
t.Fatalf("a stream that keeps sending was cut: %v %q", err, b)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// ── 5. the beat says what is served ─────────────────────────────────────────
|
||||
|
||||
func capsOf(m map[string]any) []string {
|
||||
var out []string
|
||||
if l, ok := m["capabilities"].([]any); ok {
|
||||
for _, c := range l {
|
||||
out = append(out, fmt.Sprint(c))
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func strsOf(v any) []string {
|
||||
out := []string{}
|
||||
if l, ok := v.([]any); ok {
|
||||
for _, c := range l {
|
||||
out = append(out, fmt.Sprint(c))
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// SECURITY (an honest beat): a tcp pairing's beat lists in lane_serves what
|
||||
// its serve lane carries, and advertises a model lane only when its service
|
||||
// is there. With ollama answering here, llm unset and pii set to another
|
||||
// computer, the beat carries lane_serves ["ollama"] and `embed` with its
|
||||
// models, and no `llm-openai`, `pii`, pii_port, llm_port, `services` or
|
||||
// `coders`, none of which runs on a tcp pairing. With all three served, all
|
||||
// three are listed and advertised, pii_ok with them and pii_port still not. A
|
||||
// unix pairing's beat lists its lane too and keeps `coders` (its runs have a
|
||||
// socket); a tunnel pairing's beat has no lane_serves.
|
||||
//
|
||||
// MUTATION: have heartbeatPayload call buildHeartbeatBody (the tunnel's beat).
|
||||
// RESULT: red — no lane_serves, and pii and services advertised with nothing
|
||||
// serving them.
|
||||
func TestATCPPairingsBeatAdvertisesOnlyWhatItServes(t *testing.T) {
|
||||
up := newFakeOllama(t)
|
||||
t.Setenv("OLLAMA_ADDR", up.srv.Listener.Addr().String())
|
||||
t.Setenv("PRIVATAE_LLM_ADDR", "")
|
||||
t.Setenv("PRIVATAE_PII_ADDR", "192.0.2.10:9")
|
||||
stubServiceRuntime(t, "docker")
|
||||
stubCoders(t)
|
||||
h, cfg := serveLaneBox(t, "beat.example.test")
|
||||
coderOn(t, 2)
|
||||
if !servicesCapable(cfg.HostName) || coderHeartbeat(cfg.HostName) == nil {
|
||||
t.Fatal("the slots and coder runs must be available on this machine, or their absence proves nothing")
|
||||
}
|
||||
runInBackground(t, cfg)
|
||||
beat := h.beat(t)
|
||||
caps := capsOf(beat)
|
||||
if got := strsOf(beat["lane_serves"]); !slices.Equal(got, []string{"ollama"}) {
|
||||
t.Fatalf("lane_serves: %q (the beat: %v)", got, beat)
|
||||
}
|
||||
if !slices.Contains(caps, "embed") || !slices.Contains(strsOf(beat["models"]), "qwen4-embed:latest") {
|
||||
t.Fatalf("ollama is served, so embed and its models are advertised: %q %v", caps, beat["models"])
|
||||
}
|
||||
for _, c := range []string{"llm-openai", "pii", serviceCapability, coderCapability} {
|
||||
if slices.Contains(caps, c) {
|
||||
t.Errorf("a tcp pairing advertises %q, which nothing serves on it: %q", c, caps)
|
||||
}
|
||||
}
|
||||
for _, k := range []string{"pii_port", "pii_ok", "services", "coders"} {
|
||||
if _, ok := beat[k]; ok {
|
||||
t.Errorf("a tcp pairing's beat carries %q: %v", k, beat[k])
|
||||
}
|
||||
}
|
||||
if beat["llm_port"] != float64(0) {
|
||||
t.Errorf("no port listens for the llm server on a tcp pairing: llm_port %v", beat["llm_port"])
|
||||
}
|
||||
|
||||
// All three served.
|
||||
health := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = w.Write([]byte(`{"status":"ok"}`))
|
||||
}))
|
||||
t.Cleanup(health.Close)
|
||||
t.Setenv("PRIVATAE_LLM_ADDR", up.srv.Listener.Addr().String())
|
||||
t.Setenv("PRIVATAE_PII_ADDR", health.Listener.Addr().String())
|
||||
var all map[string]any
|
||||
if err := json.Unmarshal([]byte(heartbeatPayload(cfg, nil, "box", "pub")), &all); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := strsOf(all["lane_serves"]); !slices.Equal(got, []string{"ollama", "llm", "pii"}) {
|
||||
t.Fatalf("lane_serves with all three set: %q", got)
|
||||
}
|
||||
caps = capsOf(all)
|
||||
for _, c := range []string{"embed", "llm-openai", "pii"} {
|
||||
if !slices.Contains(caps, c) {
|
||||
t.Errorf("%q is served, so it is advertised: %q", c, caps)
|
||||
}
|
||||
}
|
||||
if all["pii_ok"] != true || all["pii_port"] != nil || !slices.Contains(strsOf(all["llm_models"]), "qwen4-embed") {
|
||||
t.Errorf("pii_ok %v, pii_port %v, llm_models %v", all["pii_ok"], all["pii_port"], all["llm_models"])
|
||||
}
|
||||
|
||||
// A unix pairing: its lane, and its coder runs (a socket each).
|
||||
unix := *cfg
|
||||
unix.Transport, unix.LaneAddr, unix.LaneSocket, unix.RunsDir = transportUnix, "", "/run/privatae-host/lane.sock", "/run/privatae-host/runs"
|
||||
var u map[string]any
|
||||
_ = json.Unmarshal([]byte(heartbeatPayload(&unix, nil, "box", "pub")), &u)
|
||||
if _, ok := u["lane_serves"]; !ok || slices.Contains(capsOf(u), serviceCapability) || !slices.Contains(capsOf(u), coderCapability) {
|
||||
t.Errorf("a unix pairing: lane_serves %v, caps %q", u["lane_serves"], capsOf(u))
|
||||
}
|
||||
// A tunnel pairing: the forwards, no lane.
|
||||
tunnel := *cfg
|
||||
tunnel.Transport, tunnel.LaneAddr = "", ""
|
||||
var tb map[string]any
|
||||
_ = json.Unmarshal([]byte(heartbeatPayload(&tunnel, nil, "box", "pub")), &tb)
|
||||
if _, ok := tb["lane_serves"]; ok || tb["pii_port"] != float64(piiForwardPort) {
|
||||
t.Errorf("a tunnel pairing: lane_serves %v, pii_port %v", tb["lane_serves"], tb["pii_port"])
|
||||
}
|
||||
}
|
||||
|
||||
// The lane writes one audit line per job: the service, the address dialled
|
||||
// and the bytes each way for a served one; the reason for a refused one.
|
||||
func TestTheServeLaneAuditsEachJob(t *testing.T) {
|
||||
up := newFakeOllama(t)
|
||||
t.Setenv("OLLAMA_ADDR", up.srv.Listener.Addr().String())
|
||||
h, cfg := serveLaneBox(t, "audit.example.test")
|
||||
runInBackground(t, cfg)
|
||||
ws := h.take(t)
|
||||
if got := ask(t, ws, "ollama"); got != "A" {
|
||||
t.Fatalf("accept: %q", got)
|
||||
}
|
||||
conn := laneConn(ws)
|
||||
req, _ := http.NewRequest(http.MethodGet, "http://serve/api/tags", nil)
|
||||
req.Close = true
|
||||
_ = req.Write(conn)
|
||||
resp, err := http.ReadResponse(bufio.NewReader(conn), req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, _ = io.ReadAll(resp.Body)
|
||||
ws.Close()
|
||||
if got := ask(t, h.take(t), "10.0.0.2:11434"); !strings.HasPrefix(got, "E") {
|
||||
t.Fatalf("refuse: %q", got)
|
||||
}
|
||||
path := filepath.Join(cfg.dcnDir(), "audit.jsonl")
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for {
|
||||
raw, _ := os.ReadFile(path)
|
||||
var served, refused map[string]any
|
||||
for _, line := range strings.Split(strings.TrimSpace(string(raw)), "\n") {
|
||||
var m map[string]any
|
||||
if json.Unmarshal([]byte(line), &m) != nil || m["op"] != "serve" {
|
||||
continue
|
||||
}
|
||||
switch m["decision"] {
|
||||
case "served":
|
||||
served = m
|
||||
case "refused":
|
||||
refused = m
|
||||
}
|
||||
}
|
||||
if served != nil && refused != nil {
|
||||
if served["service"] != "ollama" || served["target"] != up.srv.Listener.Addr().String() ||
|
||||
served["bytes_from_host"].(float64) <= 0 || served["bytes_to_host"].(float64) <= 0 {
|
||||
t.Fatalf("the served line: %v", served)
|
||||
}
|
||||
if !strings.Contains(fmt.Sprint(refused["reason"]), "never takes an address") {
|
||||
t.Fatalf("the refused line: %v", refused)
|
||||
}
|
||||
return
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("no served and refused lines in %s: %s", path, raw)
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,240 @@
|
||||
//go:build linux
|
||||
|
||||
// Model management for a Host with no platform (modelswap.go through
|
||||
// modelDaemonTarget): the model-swap probe, model:list, model:pull and
|
||||
// model:drop reach only a model server on this machine, as the serve lane
|
||||
// does, and the machine's console says why one is not used. Uses the
|
||||
// fixtures of servelane_linux_test.go and modelswap_test.go.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/websocket"
|
||||
)
|
||||
|
||||
// captureStdout collects what this process prints until the test ends, and
|
||||
// returns a func that reads it so far.
|
||||
func captureStdout(t *testing.T) func() string {
|
||||
t.Helper()
|
||||
r, w, err := os.Pipe()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
old := os.Stdout
|
||||
os.Stdout = w
|
||||
var mu sync.Mutex
|
||||
var buf bytes.Buffer
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
b := make([]byte, 4096)
|
||||
for {
|
||||
n, err := r.Read(b)
|
||||
if n > 0 {
|
||||
mu.Lock()
|
||||
buf.Write(b[:n])
|
||||
mu.Unlock()
|
||||
_, _ = old.Write(b[:n])
|
||||
}
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
t.Cleanup(func() {
|
||||
os.Stdout = old
|
||||
_ = w.Close()
|
||||
<-done
|
||||
_ = r.Close()
|
||||
})
|
||||
return func() string {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return buf.String()
|
||||
}
|
||||
}
|
||||
|
||||
// SECURITY: a tcp pairing (a Host with no platform) is not given management of
|
||||
// a model server on another computer. With OLLAMA_ADDR on TEST-NET, model:list,
|
||||
// an approved model:pull and model:drop of a model this Host placed are each
|
||||
// refused with the reason, before anything is dialled or downloaded, and the
|
||||
// machine's console says in plain words that OLLAMA_ADDR is not used and what
|
||||
// to set. With OLLAMA_ADDR on this machine the same pairing lists its models.
|
||||
//
|
||||
// MUTATION: modelDaemonTarget returns localOllamaAddr() for every pairing.
|
||||
// RESULT: red — model:list fails dialling 192.0.2.10, the approved pull
|
||||
// starts a download job (exit 0), and no line is printed at the machine.
|
||||
func TestModelManagementOnALocalPairingReachesOnlyThisMachine(t *testing.T) {
|
||||
console := captureStdout(t)
|
||||
t.Setenv("OLLAMA_ADDR", "192.0.2.10:11434") // TEST-NET-1: never this machine
|
||||
t.Setenv("PRIVATAE_LLM_ADDR", "")
|
||||
t.Setenv("PRIVATAE_PII_ADDR", "")
|
||||
resetModelDaemonCache()
|
||||
h, cfg := serveLaneBox(t, "models.example.test")
|
||||
runInBackground(t, cfg)
|
||||
beat := h.beat(t)
|
||||
for _, c := range capsOf(beat) {
|
||||
if c == modelSwapCapability {
|
||||
t.Fatalf("a Host with no platform is offered management of a server on another computer: %q", capsOf(beat))
|
||||
}
|
||||
}
|
||||
|
||||
root := t.TempDir()
|
||||
start := time.Now()
|
||||
_, errOut, code, _ := driveModelOp(t, cfg.HostName, "model:list", nil, root)
|
||||
if code == 0 || !strings.Contains(errOut, "OLLAMA_ADDR") || !strings.Contains(errOut, "192.0.2.10 is not this machine") {
|
||||
t.Fatalf("model:list: exit %d, %q; want a refusal naming OLLAMA_ADDR and why", code, errOut)
|
||||
}
|
||||
if d := time.Since(start); d > 2*time.Second {
|
||||
t.Errorf("model:list took %v: it waited on a dial instead of refusing first", d)
|
||||
}
|
||||
|
||||
enrollFactor(t, root)
|
||||
approval := proposeAndApprove(t, root, "nomic-embed-text", "embed", 1<<30, "")
|
||||
_, errOut, code, _ = driveModelOp(t, cfg.HostName, "model:pull",
|
||||
[]string{pullArg("nomic-embed-text", "embed", 1<<30, "", approval)}, root)
|
||||
if code != 126 || !strings.Contains(errOut, "was not downloaded") || !strings.Contains(errOut, "192.0.2.10 is not this machine") {
|
||||
t.Fatalf("model:pull: exit %d, %q; want 126 and a refusal saying why", code, errOut)
|
||||
}
|
||||
modelPullJobsMu.Lock()
|
||||
for id, j := range modelPullJobs {
|
||||
if j.Model == "nomic-embed-text" {
|
||||
t.Errorf("a pull job %s was started for a server on another computer", id)
|
||||
}
|
||||
}
|
||||
modelPullJobsMu.Unlock()
|
||||
|
||||
if err := recordPlaced(root, "placed-here", placedModel{PlacedAt: "now", Role: "embed"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, errOut, code, _ = driveModelOp(t, cfg.HostName, "model:drop", []string{`{"model":"placed-here"}`}, root)
|
||||
if code == 0 || !strings.Contains(errOut, "192.0.2.10 is not this machine") {
|
||||
t.Fatalf("model:drop: exit %d, %q; want a refusal saying why", code, errOut)
|
||||
}
|
||||
if !placedByThisHost(root, "placed-here") {
|
||||
t.Error("a refused drop changed the ledger")
|
||||
}
|
||||
|
||||
said := console()
|
||||
if !strings.Contains(said, "["+cfg.HostName+"] OLLAMA_ADDR=192.0.2.10:11434 is not used for the Host on this computer") ||
|
||||
!strings.Contains(said, "Set OLLAMA_ADDR to a server on this computer, or unset it.") {
|
||||
t.Errorf("the machine's console does not say why OLLAMA_ADDR is not used:\n%s", said)
|
||||
}
|
||||
// Counted for this pairing only: a pairing from an earlier test whose beat
|
||||
// is still ending can say it for itself.
|
||||
if n := strings.Count(said, "["+cfg.HostName+"] OLLAMA_ADDR=192.0.2.10:11434 is not used"); n != 1 {
|
||||
t.Errorf("the console said it %d times; once, until it changes", n)
|
||||
}
|
||||
|
||||
// The control: a model server on this machine is listed for the same pairing.
|
||||
newFakeDaemon(t, &fakeDaemon{tags: []string{"here-model"}})
|
||||
out, errOut, code, _ := driveModelOp(t, cfg.HostName, "model:list", nil, root)
|
||||
if code != 0 || !strings.Contains(out, "here-model") {
|
||||
t.Fatalf("model:list of a server on this machine: exit %d, %q %q", code, out, errOut)
|
||||
}
|
||||
}
|
||||
|
||||
// The beat and a model op can refuse the same target at the same moment. The
|
||||
// machine's console still says it once: the reason is recorded and said by one
|
||||
// caller, not checked by each and then said by all.
|
||||
//
|
||||
// MUTATION: in serveTargetNotice, Load the last reason, then Store and print
|
||||
// (as it did).
|
||||
// RESULT: red — more than one line for 32 callers released together (the
|
||||
// count varies from run to run).
|
||||
func TestTheConsoleSaysARefusedTargetOnceUnderConcurrentCallers(t *testing.T) {
|
||||
console := captureStdout(t)
|
||||
s, _ := serveServiceNamed("ollama")
|
||||
self := "notice-once.example.test"
|
||||
t.Cleanup(func() { serveTargetNoticeClear(self, s) })
|
||||
start := make(chan struct{})
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < 32; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
<-start
|
||||
serveTargetNotice(self, s, "192.0.2.10:11434", "192.0.2.10 is not this machine")
|
||||
}()
|
||||
}
|
||||
close(start)
|
||||
wg.Wait()
|
||||
serveTargetNotice(self, s, "192.0.2.10:11434", "another reason")
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for !strings.Contains(console(), "another reason") && time.Now().Before(deadline) {
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
said := console()
|
||||
if n := strings.Count(said, "["+self+"] OLLAMA_ADDR=192.0.2.10:11434 is not used"); n != 2 {
|
||||
t.Fatalf("the console said it %d times; once for the first reason and once when it changed:\n%s", n, said)
|
||||
}
|
||||
}
|
||||
|
||||
// A tunnel pairing keeps OLLAMA_ADDR wherever it points, as its forwards do:
|
||||
// the rule is for a Host with no platform.
|
||||
func TestModelDaemonTargetKeepsATunnelPairingsAddress(t *testing.T) {
|
||||
t.Setenv("OLLAMA_ADDR", "192.0.2.10:11434")
|
||||
if addr, why := modelDaemonTarget("tunnel.example.test"); addr != "192.0.2.10:11434" || why != "" {
|
||||
t.Fatalf("a tunnel pairing: %q %q", addr, why)
|
||||
}
|
||||
unmark := markLocalPairing("tunnel.example.test")
|
||||
if addr, why := modelDaemonTarget("tunnel.example.test"); addr != "" || !strings.Contains(why, "192.0.2.10 is not this machine") {
|
||||
t.Fatalf("the same name run as a local pairing: %q %q", addr, why)
|
||||
}
|
||||
unmark()
|
||||
unmark() // once only
|
||||
if isLocalPairing("tunnel.example.test") {
|
||||
t.Fatal("still marked local after its run ended")
|
||||
}
|
||||
}
|
||||
|
||||
// SECURITY (this machine's resources): a Host that answers every parked serve
|
||||
// worker with a job it knows is refused (a malformed frame, a name the lane
|
||||
// does not carry) cannot make this machine dial, handshake and audit in a
|
||||
// loop. Each refused worker waits before it parks again, so two workers cost
|
||||
// at most a few dials a second.
|
||||
//
|
||||
// MUTATION: spawn the replacement when the first frame arrives (taken() before
|
||||
// the hello is read), as the lane first did.
|
||||
// RESULT: red — hundreds of workers park in the 2 s window.
|
||||
func TestARefusingHostCannotSpinTheServeWorkers(t *testing.T) {
|
||||
t.Setenv("OLLAMA_ADDR", "")
|
||||
h, cfg := serveLaneBox(t, "spin.example.test")
|
||||
runInBackground(t, cfg)
|
||||
var answered atomic.Int32
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
var ws *websocket.Conn
|
||||
select {
|
||||
case ws = <-h.workers:
|
||||
case <-time.After(time.Until(deadline)):
|
||||
}
|
||||
if ws == nil {
|
||||
break
|
||||
}
|
||||
frame := sFrame("nas")
|
||||
if answered.Load()%2 == 1 {
|
||||
frame = []byte("Dnot a hello")
|
||||
}
|
||||
_ = ws.WriteMessage(websocket.BinaryMessage, frame)
|
||||
_ = ws.SetReadDeadline(time.Now().Add(2 * time.Second))
|
||||
_, _, _ = ws.ReadMessage()
|
||||
ws.Close()
|
||||
answered.Add(1)
|
||||
}
|
||||
// 2 parked at the start, then each worker once a second: about 6 in 2 s.
|
||||
if n := answered.Load(); n > 10 {
|
||||
t.Fatalf("%d serve workers parked in 2 s on a Host that refuses every job: the pool re-dials in a loop", n)
|
||||
}
|
||||
if answered.Load() < 2 {
|
||||
t.Fatalf("only %d workers parked: the pool did not run, so the count proves nothing", answered.Load())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,274 @@
|
||||
//go:build linux
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
const serveNetnsEnv = "PRIVATAE_TEST_SERVE_NETNS"
|
||||
|
||||
// otherComputer is a computer on this machine's LAN, at 10.0.0.2 in a network
|
||||
// namespace of its own, joined to this one by a veth that carries this
|
||||
// machine's default route (so 10.0.0.2 is the LAN, not a container here). It
|
||||
// listens on 80 and 11434 and counts what reaches it off the wire: IPv4
|
||||
// packets addressed to it, TCP SYNs to any address (IPv4 and IPv6), and
|
||||
// accepted connections.
|
||||
type otherComputer struct {
|
||||
toIt atomic.Int64
|
||||
syns atomic.Int64
|
||||
accepts atomic.Int64
|
||||
stop atomic.Bool
|
||||
}
|
||||
|
||||
// startOtherComputer makes the namespace on one OS thread locked for good
|
||||
// (the thread ends with its goroutine), so every socket it opens and every
|
||||
// ip(8) it runs are the other computer's, and this test's are not.
|
||||
func startOtherComputer(t *testing.T, ipBin string) *otherComputer {
|
||||
t.Helper()
|
||||
oc := &otherComputer{}
|
||||
t.Cleanup(func() { oc.stop.Store(true) })
|
||||
errc := make(chan error, 1)
|
||||
go func() {
|
||||
runtime.LockOSThread()
|
||||
// This test's namespace, held open before the thread leaves it: the
|
||||
// veth's other end goes there. Not /proc/<pid>/ns/net, which is the
|
||||
// main thread's, and the main thread may be this one.
|
||||
here, err := os.Open("/proc/thread-self/ns/net")
|
||||
if err != nil {
|
||||
errc <- err
|
||||
return
|
||||
}
|
||||
defer here.Close()
|
||||
if err := unix.Unshare(unix.CLONE_NEWNET); err != nil {
|
||||
errc <- fmt.Errorf("unshare: %w", err)
|
||||
return
|
||||
}
|
||||
for _, a := range [][]string{
|
||||
{"link", "add", "vB", "type", "veth", "peer", "name", "vA", "netns", "/proc/self/fd/3"},
|
||||
{"link", "set", "lo", "up"},
|
||||
{"addr", "add", "10.0.0.2/24", "dev", "vB"},
|
||||
{"link", "set", "vB", "up"},
|
||||
} {
|
||||
cmd := exec.Command(ipBin, a...)
|
||||
cmd.ExtraFiles = []*os.File{here}
|
||||
if out, err := cmd.CombinedOutput(); err != nil {
|
||||
errc <- fmt.Errorf("ip %s: %v %s", strings.Join(a, " "), err, out)
|
||||
return
|
||||
}
|
||||
}
|
||||
for _, port := range []string{"80", "11434"} {
|
||||
ln, err := net.Listen("tcp", "10.0.0.2:"+port)
|
||||
if err != nil {
|
||||
errc <- err
|
||||
return
|
||||
}
|
||||
go func() {
|
||||
for !oc.stop.Load() {
|
||||
c, err := ln.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
oc.accepts.Add(1)
|
||||
c.Close()
|
||||
}
|
||||
}()
|
||||
t.Cleanup(func() { ln.Close() })
|
||||
}
|
||||
ifi, err := net.InterfaceByName("vB")
|
||||
if err != nil {
|
||||
errc <- err
|
||||
return
|
||||
}
|
||||
proto := int(htons16(unix.ETH_P_ALL))
|
||||
fd, err := unix.Socket(unix.AF_PACKET, unix.SOCK_RAW, proto)
|
||||
if err != nil {
|
||||
errc <- fmt.Errorf("packet socket: %w", err)
|
||||
return
|
||||
}
|
||||
if err := unix.Bind(fd, &unix.SockaddrLinklayer{Protocol: htons16(unix.ETH_P_ALL), Ifindex: ifi.Index}); err != nil {
|
||||
unix.Close(fd)
|
||||
errc <- fmt.Errorf("bind the packet socket: %w", err)
|
||||
return
|
||||
}
|
||||
tv := unix.NsecToTimeval((200 * time.Millisecond).Nanoseconds())
|
||||
_ = unix.SetsockoptTimeval(fd, unix.SOL_SOCKET, unix.SO_RCVTIMEO, &tv)
|
||||
go oc.capture(fd)
|
||||
errc <- nil
|
||||
}()
|
||||
if err := <-errc; err != nil {
|
||||
t.Skipf("the other computer's namespace could not be made: %v", err)
|
||||
}
|
||||
return oc
|
||||
}
|
||||
|
||||
func htons16(v uint16) uint16 { return v<<8 | v>>8 }
|
||||
|
||||
// capture counts the frames arriving on vB (its own replies are left out).
|
||||
func (oc *otherComputer) capture(fd int) {
|
||||
defer unix.Close(fd)
|
||||
buf := make([]byte, 65536)
|
||||
for !oc.stop.Load() {
|
||||
n, from, err := unix.Recvfrom(fd, buf, 0)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if ll, ok := from.(*unix.SockaddrLinklayer); ok && ll.Pkttype == unix.PACKET_OUTGOING {
|
||||
continue
|
||||
}
|
||||
f := buf[:n]
|
||||
if len(f) < 14 {
|
||||
continue
|
||||
}
|
||||
switch uint16(f[12])<<8 | uint16(f[13]) {
|
||||
case 0x0800:
|
||||
if len(f) < 34 {
|
||||
continue
|
||||
}
|
||||
ihl := int(f[14]&0x0f) * 4
|
||||
if net.IP(f[30:34]).Equal(net.IPv4(10, 0, 0, 2)) {
|
||||
oc.toIt.Add(1)
|
||||
}
|
||||
if f[23] == 6 && len(f) >= 14+ihl+14 && isSYN(f[14+ihl+13]) {
|
||||
oc.syns.Add(1)
|
||||
}
|
||||
case 0x86dd:
|
||||
if len(f) >= 14+40+14 && f[20] == 6 && isSYN(f[14+40+13]) {
|
||||
oc.syns.Add(1)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func isSYN(flags byte) bool { return flags&0x02 != 0 && flags&0x10 == 0 }
|
||||
|
||||
// SECURITY, IN A REAL NETWORK NAMESPACE: a job that names an address instead
|
||||
// of a service, a name the lane does not carry, and each service set on this
|
||||
// machine to a computer on its LAN (OLLAMA_ADDR, PRIVATAE_LLM_ADDR and
|
||||
// PRIVATAE_PII_ADDR all at 10.0.0.2) each answer 'E', and the other computer
|
||||
// sees no packet addressed to it and no SYN at all, from the lane or from the
|
||||
// beat that probes what the lane serves, the beat's model-management probe, or
|
||||
// model:list. A direct dial from this machine
|
||||
// afterwards is seen, so the count is real. Runs the test binary again in a
|
||||
// private user and network namespace; skipped where one cannot be made.
|
||||
//
|
||||
// MUTATION: in serveVetTarget, skip the ownDestination check.
|
||||
// RESULT: red — the other computer sees the lane's SYN on 11434 and 80.
|
||||
// MUTATION: modelDaemonTarget returns localOllamaAddr() for every pairing.
|
||||
// RESULT: red — the beat's model-swap probe and model:list SYN 10.0.0.2:11434.
|
||||
func TestTheServeLaneSendsNoPacketForAnAddressInARealNamespace(t *testing.T) {
|
||||
if os.Getenv(serveNetnsEnv) != "1" {
|
||||
if _, err := exec.LookPath("ip"); err != nil {
|
||||
if _, err := os.Stat("/usr/sbin/ip"); err != nil {
|
||||
t.Skip("needs ip(8)")
|
||||
}
|
||||
}
|
||||
cmd := exec.Command(os.Args[0], "-test.run=^TestTheServeLaneSendsNoPacketForAnAddressInARealNamespace$",
|
||||
"-test.v", "-test.count=1")
|
||||
cmd.Env = append(os.Environ(), serveNetnsEnv+"=1")
|
||||
cmd.SysProcAttr = &syscall.SysProcAttr{
|
||||
Cloneflags: syscall.CLONE_NEWUSER | syscall.CLONE_NEWNET,
|
||||
UidMappings: []syscall.SysProcIDMap{{ContainerID: 0, HostID: os.Getuid(), Size: 1}},
|
||||
GidMappings: []syscall.SysProcIDMap{{ContainerID: 0, HostID: os.Getgid(), Size: 1}},
|
||||
}
|
||||
out, err := cmd.CombinedOutput()
|
||||
if cmd.ProcessState == nil {
|
||||
t.Skipf("no private user and network namespace here: %v", err)
|
||||
}
|
||||
t.Logf("in a private network namespace:\n%s", out)
|
||||
if strings.Contains(string(out), "--- SKIP") {
|
||||
t.Skip("the namespaced run could not set up its network")
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("the namespaced run failed: %v", err)
|
||||
}
|
||||
if !strings.Contains(string(out), "--- PASS: TestTheServeLaneSendsNoPacketForAnAddressInARealNamespace") {
|
||||
t.Fatal("the namespaced run did not run")
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
ipBin, err := exec.LookPath("ip")
|
||||
if err != nil {
|
||||
ipBin = "/usr/sbin/ip"
|
||||
}
|
||||
if out, err := exec.Command(ipBin, "link", "set", "lo", "up").CombinedOutput(); err != nil {
|
||||
t.Skipf("ip link set lo up: %v %s", err, out)
|
||||
}
|
||||
oc := startOtherComputer(t, ipBin)
|
||||
for _, a := range [][]string{
|
||||
{"addr", "add", "10.0.0.1/24", "dev", "vA"},
|
||||
{"link", "set", "vA", "up"},
|
||||
// This machine's way out: 10.0.0.2 is its LAN (and its router), not a
|
||||
// container on a veth of its own (ownaddr.go).
|
||||
{"route", "add", "default", "via", "10.0.0.2", "dev", "vA"},
|
||||
} {
|
||||
if out, err := exec.Command(ipBin, a...).CombinedOutput(); err != nil {
|
||||
t.Skipf("ip %s: %v %s", strings.Join(a, " "), err, out)
|
||||
}
|
||||
}
|
||||
t.Setenv("OLLAMA_ADDR", "10.0.0.2:11434")
|
||||
t.Setenv("PRIVATAE_LLM_ADDR", "10.0.0.2:80")
|
||||
t.Setenv("PRIVATAE_PII_ADDR", "10.0.0.2:80")
|
||||
h, cfg := serveLaneBox(t, "netns.example.test")
|
||||
// `models` stays on: the model-management probe in the beat and model:list
|
||||
// (modelswap.go, modelDaemonTarget) are held to this machine for a Host
|
||||
// with no platform too, so they are in the count.
|
||||
resetModelDaemonCache()
|
||||
runInBackground(t, cfg)
|
||||
beat := h.beat(t)
|
||||
if got := strsOf(beat["lane_serves"]); len(got) != 0 {
|
||||
t.Fatalf("nothing set here is this machine's, so nothing is served: %q", got)
|
||||
}
|
||||
for _, c := range capsOf(beat) {
|
||||
if c == modelSwapCapability {
|
||||
t.Fatalf("model management of the other computer is offered: %q", capsOf(beat))
|
||||
}
|
||||
}
|
||||
if _, errOut, code, _ := driveModelOp(t, cfg.HostName, "model:list", nil, ""); code == 0 ||
|
||||
!strings.Contains(errOut, "10.0.0.2 is not this machine") {
|
||||
t.Errorf("model:list of the other computer: exit %d, %q", code, errOut)
|
||||
}
|
||||
for service, want := range map[string]string{
|
||||
"10.0.0.2:80": "never takes an address",
|
||||
"10.0.0.2:11434": "never takes an address",
|
||||
"nas": "is not a service",
|
||||
"ollama": "10.0.0.2 is not this machine",
|
||||
"llm": "10.0.0.2 is not this machine",
|
||||
"pii": "10.0.0.2 is not this machine",
|
||||
} {
|
||||
got := ask(t, h.take(t), service)
|
||||
if !strings.HasPrefix(got, "Erefused: ") || !strings.Contains(got, want) {
|
||||
t.Errorf("%q: answered %q, want E carrying %q", service, got, want)
|
||||
}
|
||||
}
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
if a, s, c := oc.toIt.Load(), oc.syns.Load(), oc.accepts.Load(); a != 0 || s != 0 || c != 0 {
|
||||
t.Fatalf("the other computer saw %d packets addressed to it, %d SYNs and %d connections", a, s, c)
|
||||
}
|
||||
// The count is real: a dial from this machine is seen.
|
||||
c, err := net.DialTimeout("tcp", "10.0.0.2:80", 3*time.Second)
|
||||
if err != nil {
|
||||
t.Fatalf("the control dial to the other computer failed, so the count proves nothing: %v", err)
|
||||
}
|
||||
c.Close()
|
||||
deadline := time.Now().Add(3 * time.Second)
|
||||
for oc.toIt.Load() == 0 || oc.syns.Load() == 0 || oc.accepts.Load() == 0 {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("the control dial was not counted: %d packets, %d SYNs, %d connections",
|
||||
oc.toIt.Load(), oc.syns.Load(), oc.accepts.Load())
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
+31
-12
@@ -113,13 +113,26 @@ func runSessionWorkers(pctx context.Context, self string, wsURL string, tlsCfg *
|
||||
// session; an H frame is refused with a plain line (harness jobs belong on the
|
||||
// harness workers) so a host that routes wrongly learns it at once.
|
||||
func handleSessionWorkerConn(ws *websocket.Conn, self string, senv sessionEnv) {
|
||||
mt, data, err := ws.ReadMessage()
|
||||
if err != nil || mt != websocket.BinaryMessage || len(data) == 0 {
|
||||
// The spec is read up to sessionSpecMaxBytes (lane_bounds.go) and refused
|
||||
// past it; a frame of any other tag is not read at all.
|
||||
tag, body, tooBig, err := readLaneJob(ws, func(t byte) int64 {
|
||||
if t == 'S' {
|
||||
return sessionSpecMaxBytes
|
||||
}
|
||||
return 0
|
||||
})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
switch data[0] {
|
||||
switch tag {
|
||||
case 'S':
|
||||
handleSessionConn(ws, data, self, senv)
|
||||
if tooBig {
|
||||
sendSession(ws, 'E', []byte(fmt.Sprintf("the session spec is more than the %d KiB a session may ask with; nothing of it was read\n",
|
||||
sessionSpecMaxBytes>>10)))
|
||||
sendSessionExit(ws, 1)
|
||||
return
|
||||
}
|
||||
handleSessionConn(ws, append([]byte{'S'}, body...), self, senv)
|
||||
case 'H':
|
||||
sendSession(ws, 'E', []byte("this is a session worker (lane=session); harness jobs go to a harness worker\n"))
|
||||
sendSessionExit(ws, 1)
|
||||
@@ -573,23 +586,29 @@ func handleSessionConn(ws *websocket.Conn, first []byte, self string, env sessio
|
||||
closed := make(chan struct{})
|
||||
go func() {
|
||||
defer close(closed)
|
||||
// Each frame is streamed (lane_bounds.go): input goes into the pty as it
|
||||
// arrives, never held whole, and a frame is at most sessionInputMaxBytes.
|
||||
ws.SetReadLimit(sessionInputMaxBytes)
|
||||
for {
|
||||
mt, data, err := ws.ReadMessage()
|
||||
mt, r, err := ws.NextReader()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if mt != websocket.BinaryMessage || len(data) == 0 {
|
||||
continue
|
||||
if mt != websocket.BinaryMessage {
|
||||
continue // NextReader drops what is left of it
|
||||
}
|
||||
switch data[0] {
|
||||
var tag [1]byte
|
||||
if _, err := io.ReadFull(r, tag[:]); err != nil {
|
||||
continue // an empty frame
|
||||
}
|
||||
switch tag[0] {
|
||||
case 'I':
|
||||
lastInput.Store(time.Now().UnixNano())
|
||||
if len(data) > 1 {
|
||||
_, _ = proc.Write(data[1:])
|
||||
}
|
||||
_, _ = io.Copy(proc, r)
|
||||
case 'R':
|
||||
var rz sessionResize
|
||||
if json.Unmarshal(data[1:], &rz) == nil && rz.Cols > 0 && rz.Rows > 0 {
|
||||
b, _ := io.ReadAll(io.LimitReader(r, sessionResizeMaxBytes))
|
||||
if json.Unmarshal(b, &rz) == nil && rz.Cols > 0 && rz.Rows > 0 {
|
||||
_ = proc.Resize(rz.Cols, rz.Rows)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user