bastion 0.56.0
ci / personal-data (push) Successful in 10s
ci / test (push) Successful in 5m30s
ci / fuzz (push) Successful in 3m16s

This commit is contained in:
Privatae
2026-10-28 02:24:07 +00:00
parent 8822d5df85
commit 4012da0b75
53 changed files with 6430 additions and 270 deletions
+64 -6
View File
@@ -512,6 +512,30 @@ func resolveLLMIPs() map[string]string {
// to the anchor host, because approving is the anchor's job. "" = no pairing
// identity (tests, tools): mode, anchor and unapproved attempts still travel.
func buildHeartbeatBody(self, hostname, platform, version, wgPub, fingerprint, tunnelIP, endpoint string,
savePending bool, mirror *mirrorReport, enforcing map[string]any, peers []peerAnnounce) string {
return buildHeartbeatBodyOn(beatLanes{}, self, hostname, platform, version, wgPub, fingerprint, tunnelIP, endpoint,
savePending, mirror, enforcing, peers)
}
// beatLanes is which of a pairing's lanes run, so its beat advertises those
// and no others. The zero value is a tunnel pairing: the forwards listen on
// the tunnel's stack and the service slots are reachable there.
type beatLanes struct {
// local: a Host with no platform (run.go runUnixPairing). The forwards and
// the service slots are absent; the model servers ride the serve lane
// (servelane.go), and the beat says which in lane_serves.
local bool
// noCoders: a tcp pairing, whose coder runs have no control plane to use
// (pair_tcp.go coderTransportRefusal), so it is never offered one.
noCoders bool
}
func beatLanesFor(cfg *Config) beatLanes {
return beatLanes{local: cfg.localTransport(), noCoders: cfg.tcpTransport()}
}
// buildHeartbeatBodyOn is buildHeartbeatBody for a pairing whose lanes are lanes.
func buildHeartbeatBodyOn(lanes beatLanes, self, hostname, platform, version, wgPub, fingerprint, tunnelIP, endpoint string,
savePending bool, mirror *mirrorReport, enforcing map[string]any, peers []peerAnnounce) string {
caps := []string{}
// WHY SOME CAPABILITIES ARE WITHHELD HERE AND OTHERS ARE NOT — the asymmetry
@@ -549,7 +573,17 @@ func buildHeartbeatBody(self, hostname, platform, version, wgPub, fingerprint, t
// gets exactly what it got before: scope.filterModels is the identity then.
scope := pairingModelScope(self)
reachable, models := false, []string(nil)
if inference {
// A local pairing has no forward to advertise: its model servers ride the
// serve lane, and each lane below is advertised only when lane_serves
// carries its service (the target set and this machine's, the server
// answering, the family held). Its probes are of the address the lane
// dials, never the tunnel's guesses.
var served laneServeReport
if lanes.local {
served = laneServes(self)
reachable = containsStr(served.served, "ollama")
models = served.models
} else if inference {
reachable, models = ollamaModels(tunnelIP)
models = scope.filterModels(models)
// Reachable but nothing shared: advertising `embed` would put this box
@@ -572,7 +606,15 @@ func buildHeartbeatBody(self, hostname, platform, version, wgPub, fingerprint, t
// box running both.
llmModels := []string{}
llmPort := llmForwardPort
if !inference {
if lanes.local {
// No port listens for it here: the Host reaches the server through the
// serve lane, so llm_port is the "no port" value.
llmPort = 0
if containsStr(served.served, "llm") {
caps = append(caps, "llm-openai")
llmModels = served.llmModels
}
} else if !inference {
// The host stores 0 as NULL; with llm-openai withheld and
// llm_models empty nothing on the host resolves a port for this row.
llmPort = 0
@@ -598,7 +640,9 @@ func buildHeartbeatBody(self, hostname, platform, version, wgPub, fingerprint, t
// probe: an up forwarder with a dead upstream is a distinct state the
// host wants before committing a fail-closed write path to it.
piiAdvertised, piiOK := false, false
if inference && scope.allowsPII() {
if lanes.local {
piiAdvertised, piiOK = containsStr(served.served, "pii"), served.piiOK
} else if inference && scope.allowsPII() {
piiAdvertised, piiOK = piiProbe()
}
if piiAdvertised {
@@ -638,7 +682,7 @@ func buildHeartbeatBody(self, hostname, platform, version, wgPub, fingerprint, t
// "build a server on this machine" on it, and an offer the box would refuse
// is a control that fails after the user chose it. The slot list rides with
// it, withheld (absent, not empty) from a pairing outside the family.
servicesOn := servicesCapable(self)
servicesOn := !lanes.local && servicesCapable(self)
if servicesOn {
caps = append(caps, serviceCapability)
}
@@ -646,7 +690,10 @@ func buildHeartbeatBody(self, hostname, platform, version, wgPub, fingerprint, t
// only when this machine would accept a run-up from this pairing — Linux,
// root, coders on, a runtime, the family, and (anchored) a standing coder
// grant. Absent, never empty, otherwise.
coders := coderHeartbeat(self)
var coders map[string]any
if !lanes.noCoders {
coders = coderHeartbeat(self)
}
if coders != nil {
caps = append(caps, coderCapability)
}
@@ -774,6 +821,13 @@ func buildHeartbeatBody(self, hostname, platform, version, wgPub, fingerprint, t
// absence as a client too old to say.
"endpoint_save_pending": savePending,
}
// What this pairing serves over the serve lane (servelane.go), on a local
// pairing only: ["ollama","llm","pii"] or fewer, [] when nothing is. Absent
// on a tunnel pairing, which has the forwards instead, and from a bastion
// that predates the lane.
if lanes.local {
body["lane_serves"] = served.served
}
// The service slots this pairing may reach, with the tunnel port for each
// (service.go). Absent when the capability is withheld, never empty-for-no.
if servicesOn {
@@ -1004,7 +1058,11 @@ func buildHeartbeatBody(self, hostname, platform, version, wgPub, fingerprint, t
}
}
if piiAdvertised {
body["pii_port"] = piiForwardPort
// pii_port is the tunnel's port; a local pairing has none and reaches
// the sterilizer through the serve lane.
if !lanes.local {
body["pii_port"] = piiForwardPort
}
body["pii_ok"] = piiOK
}
if faAdvertised {
+63 -32
View File
@@ -23,6 +23,7 @@ package main
import (
"bufio"
"context"
"io"
"net"
"os"
"path/filepath"
@@ -150,17 +151,32 @@ func coderVettedAddrs(ctx context.Context, hostport string, allowLAN []string) (
}
func coderRefuse(c net.Conn, why string) {
_, _ = c.Write([]byte("HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\nConnection: close\r\nX-Privatae-Refused: " +
strings.NewReplacer("\r", " ", "\n", " ").Replace(why) + "\r\n\r\n"))
writeEgressRefusal(c, "403 Forbidden", why, why)
}
// handleCoderEgressConn serves one connection from a run: HTTP CONNECT or an
// absolute-form plain HTTP request, the same parsing as handleEgressConn.
// Destinations are never logged; refusals are counted on the run.
// coderEgressMaxHeadBytes bounds a request's line and headers together.
const coderEgressMaxHeadBytes = 64 << 10
// coderEgressMaxConns bounds the connections one run's proxy carries at once.
// A whole Chromium profile opens at most 256 sockets at a time (its socket
// pool limit), and package managers far fewer, so four times the browser's
// figure never refuses real work; past it a connection is answered 503 and
// closed, and each one held costs this machine at most its head and two copy
// buffers.
const coderEgressMaxConns = 1024
func handleCoderEgressConn(client net.Conn, allowLAN []string, refused *int64) {
defer client.Close()
_ = client.SetReadDeadline(time.Now().Add(30 * time.Second))
br := bufio.NewReader(client)
// The request line and headers are read under one bound, the head limit
// below has always named: the run's own processes send them, and a line
// that never ends must not grow in this machine's memory before that
// limit is looked at.
headLim := &io.LimitedReader{R: client, N: coderEgressMaxHeadBytes}
br := bufio.NewReader(headLim)
line, err := br.ReadString('\n')
if err != nil {
return
@@ -172,7 +188,6 @@ func handleCoderEgressConn(client net.Conn, allowLAN []string, refused *int64) {
method, target := parts[0], parts[1]
var head strings.Builder
head.WriteString(line)
hostHdr := ""
for {
h, err := br.ReadString('\n')
if err != nil {
@@ -182,38 +197,41 @@ func handleCoderEgressConn(client net.Conn, allowLAN []string, refused *int64) {
if h == "\r\n" || h == "\n" {
break
}
if head.Len() > 64<<10 {
if head.Len() > coderEgressMaxHeadBytes {
return
}
if len(h) >= 5 && strings.EqualFold(h[:5], "host:") {
hostHdr = strings.TrimSpace(h[5:])
}
}
_ = client.SetReadDeadline(time.Time{})
// The head is in; what follows is the stream itself. What the reader
// already buffered past the head stays first in line.
headLim.N = 1<<63 - 1
ctx, cancel := context.WithTimeout(context.Background(), egressDialTimeout)
defer cancel()
// Read the way the bastion's own egress lane reads it (egress_http.go): a
// CONNECT target through parseEgressAuthority, and plain HTTP as ONE
// request, rebuilt and sent with `Connection: close`, so the run's later
// requests on a keep-alive connection never reach the destination unvetted.
connect := strings.EqualFold(method, "CONNECT")
hp := ""
var req egressRequest
if connect {
hp = ensurePort(target, "443")
} else {
host := hostHdr
if i := strings.Index(target, "://"); i >= 0 {
rest := target[i+3:]
if s := strings.IndexAny(rest, "/?#"); s >= 0 {
host = rest[:s]
} else {
host = rest
}
if at := strings.LastIndexByte(host, '@'); at >= 0 {
host = host[at+1:]
}
}
if host == "" {
host, port, ok := parseEgressAuthority(target, 443)
if !ok {
atomic.AddInt64(refused, 1)
writeEgressRefusal(client, "400 Bad Request", egressNoHost, egressNoHost)
return
}
hp = ensurePort(host, "80")
hp = egressHostPort(host, port)
} else {
r, bad := parseEgressRequest(head.String())
if bad != nil {
atomic.AddInt64(refused, 1)
writeEgressRefusal(client, bad.status, bad.body, bad.body)
return
}
req = r
hp = egressHostPort(req.host, req.port)
}
dialAddrs, why := coderVettedAddrs(ctx, hp, allowLAN)
if why != "" {
@@ -223,15 +241,14 @@ func handleCoderEgressConn(client net.Conn, allowLAN []string, refused *int64) {
}
upstream, err := dialEach(ctx, coderDialOut, dialAddrs)
if err != nil {
_, _ = client.Write([]byte("HTTP/1.1 502 Bad Gateway\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"))
writeEgressRefusal(client, "502 Bad Gateway", egressUnreachable, "")
return
}
if connect {
if _, err := client.Write([]byte("HTTP/1.1 200 Connection established\r\n\r\n")); err != nil {
_ = upstream.Close()
return
}
} else if _, err := upstream.Write([]byte(head.String())); err != nil {
if !connect {
oneEgressExchange(br, client, upstream, req)
return
}
if _, err := client.Write([]byte("HTTP/1.1 200 Connection established\r\n\r\n")); err != nil {
_ = upstream.Close()
return
}
@@ -277,6 +294,7 @@ func coderListenEgress(runID, dir string, uid int) error {
}
run := &coderEgressRun{ln: ln}
coderEgress[runID] = run
slots := make(chan struct{}, coderEgressMaxConns)
go func() {
for {
c, err := ln.Accept()
@@ -285,7 +303,20 @@ func coderListenEgress(runID, dir string, uid int) error {
}
// allow_lan is read per connection: the owner's edit applies to
// running runs at once.
go handleCoderEgressConn(c, loadCoderConfig().AllowLAN, &run.refused)
select {
case slots <- struct{}{}:
go func() {
defer func() { <-slots }()
handleCoderEgressConn(c, loadCoderConfig().AllowLAN, &run.refused)
}()
default:
go func() {
defer c.Close()
_ = c.SetWriteDeadline(time.Now().Add(time.Second))
why := "this run already has " + strconv.Itoa(coderEgressMaxConns) + " connections open"
writeEgressRefusal(c, "503 Service Unavailable", why, why)
}()
}
}
}()
return nil
+8 -2
View File
@@ -474,8 +474,14 @@ func TestCoderEgressProxyOverTheSocket(t *testing.T) {
} {
head, c := ask(req)
c.Close()
if !strings.HasPrefix(head, "HTTP/1.1 403") || !strings.Contains(head, "X-Privatae-Refused:") {
t.Fatalf("must be refused with 403 + X-Privatae-Refused: %q -> %q", req, head)
want := "HTTP/1.1 403"
if strings.Contains(req, "user@") {
// Userinfo is refused outright, before any lookup, as the Host's
// egress gate refuses it (parseEgressAuthority); still counted.
want = "HTTP/1.1 400"
}
if !strings.HasPrefix(head, want) || !strings.Contains(head, "X-Privatae-Refused:") {
t.Fatalf("must be refused with %s + X-Privatae-Refused: %q -> %q", want, req, head)
}
}
head, c := ask("CONNECT public.example:443 HTTP/1.1\r\n\r\n")
+82 -10
View File
@@ -43,9 +43,11 @@ import (
"encoding/json"
"errors"
"fmt"
"io"
"net"
"os"
"path/filepath"
"sync"
"time"
)
@@ -126,18 +128,81 @@ func startContextSock(done <-chan struct{}) {
_ = ln.Close()
_ = os.Remove(p)
}()
go func() {
for {
c, err := ln.Accept()
if err != nil {
return
}
go contextSockServe(c)
}
}()
go contextSockAccept(ln)
uilogf("context socket listening on %s\n", p)
}
// The socket is 0666 and the daemon runs as root (or SYSTEM), so every user on
// the machine can make it read. These bound what they can make it hold: an ask
// is a folder path and a display name, far under contextAskMaxBytes, and no
// more than contextSockMaxInFlight asks are read at once. Past that a
// connection is told the daemon is busy and closed; the asker tries again.
//
// One person may hold at most contextSockMaxPerUser of those slots, keyed on
// the identity the kernel gives (never on anything the asker says): without
// it, one user opening contextSockMaxInFlight idle connections and dialling
// again as each deadline ran out kept every other user's asks "busy" for as
// long as they liked. A person adds a folder at a time; four at once is far
// past that.
const (
contextAskMaxBytes = 64 << 10
contextSockMaxInFlight = 32
)
// contextSockMaxPerUser is a var only so the test of the overall cap can lift
// it (every test connection is the same user).
var contextSockMaxPerUser = 4
var contextSockUsers = struct {
sync.Mutex
held map[string]int
}{held: map[string]int{}}
// contextSockTakeUser takes one of user's slots, or reports there is none.
func contextSockTakeUser(user string) bool {
contextSockUsers.Lock()
defer contextSockUsers.Unlock()
if contextSockUsers.held[user] >= contextSockMaxPerUser {
return false
}
contextSockUsers.held[user]++
return true
}
func contextSockReleaseUser(user string) {
contextSockUsers.Lock()
defer contextSockUsers.Unlock()
if contextSockUsers.held[user]--; contextSockUsers.held[user] <= 0 {
delete(contextSockUsers.held, user)
}
}
// contextSockAccept serves ln until it is closed, at most
// contextSockMaxInFlight connections at a time.
func contextSockAccept(ln net.Listener) {
slots := make(chan struct{}, contextSockMaxInFlight)
for {
c, err := ln.Accept()
if err != nil {
return
}
select {
case slots <- struct{}{}:
go func() {
defer func() { <-slots }()
contextSockServe(c)
}()
default:
go func() {
defer c.Close()
_ = c.SetDeadline(time.Now().Add(time.Second))
_ = json.NewEncoder(c).Encode(contextAskReply{Why: "the bastion is busy " +
"answering other asks on this machine; try again in a moment"})
}()
}
}
}
func contextSockServe(c net.Conn) {
defer c.Close()
_ = c.SetDeadline(time.Now().Add(30 * time.Second))
@@ -154,8 +219,15 @@ func contextSockServe(c net.Conn) {
return
}
if !contextSockTakeUser(user) {
_ = json.NewEncoder(c).Encode(contextAskReply{User: user, Why: "the bastion is busy " +
"answering other asks of yours; try again in a moment"})
return
}
defer contextSockReleaseUser(user)
var req contextAskRequest
if err := json.NewDecoder(c).Decode(&req); err != nil {
if err := json.NewDecoder(io.LimitReader(c, contextAskMaxBytes)).Decode(&req); err != nil {
_ = json.NewEncoder(c).Encode(contextAskReply{User: user, Why: "the request could not be read"})
return
}
+130
View File
@@ -0,0 +1,130 @@
package main
// The context socket is 0666: every user on the machine may connect, and the
// daemon behind it runs as root (or SYSTEM). So what one connection can make
// the daemon hold must be bounded by the daemon, not by the asker.
import (
"net"
"path/filepath"
"runtime"
"strings"
"testing"
"time"
)
// An ask is a folder path and a display name. A client that streams a request
// with no end gets the daemon to read only a bounded amount of it, then a
// reply and a close; before, the decoder kept every byte until the 30 s
// deadline, and nothing capped how many such connections were open.
func TestAContextAskWithNoEndIsReadOnlyToABound(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("no unix-socket peer credentials on windows; contextsock_windows.go says why")
}
sock := filepath.Join(t.TempDir(), "s.sock")
ln, err := net.Listen("unix", sock)
if err != nil {
t.Fatal(err)
}
defer ln.Close()
go func() {
c, err := ln.Accept()
if err != nil {
return
}
contextSockServe(c)
}()
c, err := net.Dial("unix", sock)
if err != nil {
t.Fatal(err)
}
defer c.Close()
const total = 16 << 20
chunk := []byte(strings.Repeat("a", 64<<10))
sent := 0
if _, err := c.Write([]byte(`{"root":"`)); err != nil {
t.Fatal(err)
}
for sent < total {
_ = c.SetWriteDeadline(time.Now().Add(2 * time.Second))
n, err := c.Write(chunk)
sent += n
if err != nil {
break
}
}
// The kernel's socket buffers hold a few hundred KiB the daemon never
// read; anything near the whole stream means the daemon read it all.
if sent >= 4<<20 {
t.Fatalf("the daemon accepted %d MiB of one ask with no end; it must stop at a bound", sent>>20)
}
}
// Asks in flight are capped: a connection past the cap is closed at once, so
// many idle or endless connections cannot multiply the bound above.
func TestContextAsksInFlightAreCapped(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("no unix-socket peer credentials on windows; contextsock_windows.go says why")
}
sock := filepath.Join(t.TempDir(), "s.sock")
ln, err := net.Listen("unix", sock)
if err != nil {
t.Fatal(err)
}
defer ln.Close()
// Every connection here is the same user; lift the per-user share so the
// overall cap is what this test meets (TestOneUserCannotHoldEveryContextSockSlot
// holds the share).
oldShare := contextSockMaxPerUser
contextSockMaxPerUser = contextSockMaxInFlight
t.Cleanup(func() { contextSockMaxPerUser = oldShare })
go contextSockAccept(ln)
var held []net.Conn
defer func() {
for _, h := range held {
_ = h.Close()
}
}()
// Each of these sends nothing and holds its slot until its deadline.
for i := 0; i < contextSockMaxInFlight; i++ {
h, err := net.Dial("unix", sock)
if err != nil {
t.Fatal(err)
}
held = append(held, h)
}
// Give the daemon's accept loop time to take every held connection.
time.Sleep(200 * time.Millisecond)
extra, err := net.Dial("unix", sock)
if err != nil {
t.Fatal(err)
}
defer extra.Close()
_ = extra.SetReadDeadline(time.Now().Add(2 * time.Second))
buf := make([]byte, 512)
n, rerr := extra.Read(buf)
if ne, ok := rerr.(net.Error); ok && ne.Timeout() {
t.Fatalf("a connection past the cap of %d was left open; it must be refused", contextSockMaxInFlight)
}
if n > 0 && !strings.Contains(string(buf[:n]), "busy") {
t.Fatalf("a connection past the cap got %q; it must be told the daemon is busy", buf[:n])
}
// Once a held connection goes, a new ask gets a slot again.
_ = held[0].Close()
held = held[1:]
time.Sleep(200 * time.Millisecond)
again, err := net.Dial("unix", sock)
if err != nil {
t.Fatal(err)
}
defer again.Close()
_ = again.SetReadDeadline(time.Now().Add(500 * time.Millisecond))
n, rerr = again.Read(buf)
if ne, ok := rerr.(net.Error); !ok || !ne.Timeout() {
t.Fatalf("after a slot was freed, a new ask was not served (read %q, %v)", buf[:n], rerr)
}
}
+86
View File
@@ -0,0 +1,86 @@
package main
import (
"net"
"path/filepath"
"runtime"
"strings"
"testing"
"time"
)
// One person cannot take every slot: past their own share they are told the
// daemon is busy at once, while slots remain for everyone else. Before, one
// user's idle connections, re-dialled as each deadline ran out, held all of
// them and every other user's ask was refused.
func TestOneUserCannotHoldEveryContextSockSlot(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("no unix-socket peer credentials on windows; contextsock_windows.go says why")
}
const share = 4 // contextSockMaxPerUser
// A test before this one may still be releasing its slots.
if n := contextSockSlotsHeld(t, 0); n != 0 {
t.Fatalf("%d slots still held from an earlier test", n)
}
sock := filepath.Join(t.TempDir(), "s.sock")
ln, err := net.Listen("unix", sock)
if err != nil {
t.Fatal(err)
}
defer ln.Close()
go contextSockAccept(ln)
var held []net.Conn
defer func() {
for _, h := range held {
_ = h.Close()
}
}()
for i := 0; i < share; i++ {
h, err := net.Dial("unix", sock)
if err != nil {
t.Fatal(err)
}
held = append(held, h)
}
// Wait until the daemon has given those four their slots (a fixed sleep
// lost this race on a loaded machine and handed the 5th one first).
if n := contextSockSlotsHeld(t, share); n != share {
t.Fatalf("the daemon gave %d slots to %d idle connections of one user", n, share)
}
extra, err := net.Dial("unix", sock)
if err != nil {
t.Fatal(err)
}
defer extra.Close()
_ = extra.SetReadDeadline(time.Now().Add(2 * time.Second))
buf := make([]byte, 512)
n, rerr := extra.Read(buf)
if ne, ok := rerr.(net.Error); ok && ne.Timeout() {
t.Fatalf("one user held %d idle connections and a %dth was still given a slot: "+
"one user can take all %d", share, share+1, contextSockMaxInFlight)
}
if !strings.Contains(string(buf[:n]), "busy") {
t.Fatalf("past one user's share the ask got %q; it must be told the daemon is busy", buf[:n])
}
}
// contextSockSlotsHeld waits up to 10 s for the per-user slots held across
// all users to reach want, and returns the count it last saw.
func contextSockSlotsHeld(t *testing.T, want int) int {
t.Helper()
n := -1
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(10 * time.Millisecond) {
contextSockUsers.Lock()
n = 0
for _, k := range contextSockUsers.held {
n += k
}
contextSockUsers.Unlock()
if n == want {
break
}
}
return n
}
+68 -7
View File
@@ -26,6 +26,7 @@ import (
"encoding/base64"
"encoding/json"
"fmt"
"io/fs"
"os"
"path/filepath"
"runtime"
@@ -74,6 +75,10 @@ type dcnGrantRec struct {
// the signed bytes (a public key; the terms stay exactly what was
// signed). The managed local agent seals the store's catalog to it.
SealRecipient string `json:"seal_recipient,omitempty"`
// RootReal is where the granted folder led when this box installed the
// grant: its real path, pinned here (dcn_store.go), outside the signed
// bytes. An op whose root leads anywhere else is refused.
RootReal string `json:"root_real,omitempty"`
}
// dcnTerms is parsed FROM the verified canonical bytes — never from anything else.
@@ -524,6 +529,17 @@ func handleDcnGrantSync(ws *websocket.Conn, dir string, args []string) {
dcnRefuse(ws, dir, "grant_sync", payload.StoreID, "", "grant does not verify: "+err.Error())
return
}
// The folder this grant names is pinned as it resolves now; a re-sync or a
// renewal of the same root keeps the pin it already has (dcn_store.go).
if prev, held := f.Stores[fmt.Sprint(payload.StoreID)]; held {
prevRoot := ""
if pt, perr := dcnVerifyRec(f.VerifyPubkey, prev); perr == nil {
prevRoot = pt.Root
}
rec.RootReal = dcnPinFor(&prev, prevRoot, terms.Root)
} else {
rec.RootReal = dcnPinFor(nil, "", terms.Root)
}
if hn, _ := os.Hostname(); hn != "" && terms.Server != "" && terms.Server != hn {
dcnRefuse(ws, dir, "grant_sync", payload.StoreID, "",
fmt.Sprintf("grant is for server '%s', this box is '%s'", terms.Server, hn))
@@ -658,15 +674,26 @@ func handleDcnList(ws *websocket.Conn, dir string, args []string) {
if terms == nil {
return
}
base := terms.Root
base := "."
if a.Prefix != "" {
p, ok := dcnResolvePath(terms.Root, a.Prefix)
if !ok {
dcnRefuse(ws, dir, "list", storeID, a.Prefix, "prefix escapes the store root")
return
}
base = p
rel, err := dcnRel(terms.Root, p)
if err != nil {
dcnRefuse(ws, dir, "list", storeID, a.Prefix, "prefix escapes the store root")
return
}
base = filepath.ToSlash(rel)
}
r, err := dcnOpenStore(dir, storeID, terms)
if err != nil {
dcnRefuse(ws, dir, "list", storeID, a.Prefix, err.Error())
return
}
defer r.Close()
limit := a.Limit
if limit <= 0 || limit > dcnMaxListEntries {
limit = dcnMaxListEntries
@@ -678,12 +705,14 @@ func handleDcnList(ws *websocket.Conn, dir string, args []string) {
}
var files []entry
truncated := false
_ = filepath.WalkDir(base, func(p string, d os.DirEntry, err error) error {
// Walked through the store's handle: a link that leads out of it is not
// followed, whatever the names do during the walk.
_ = fs.WalkDir(r.FS(), base, func(p string, d fs.DirEntry, err error) error {
if err != nil {
return nil // unreadable subtree: skip, never abort the listing
}
rel, rerr := filepath.Rel(terms.Root, p)
if rerr != nil || rel == "." {
rel := filepath.FromSlash(p)
if rel == "." {
return nil
}
if dcnExcluded(terms.Excludes, rel) {
@@ -727,8 +756,23 @@ func handleDcnStat(ws *websocket.Conn, dir string, args []string) {
dcnRefuse(ws, dir, "stat", storeID, a.Path, "path escapes the store root")
return
}
info, err := os.Stat(abs)
rel, err := dcnRel(terms.Root, abs)
if err != nil {
dcnRefuse(ws, dir, "stat", storeID, a.Path, err.Error())
return
}
r, err := dcnOpenStore(dir, storeID, terms)
if err != nil {
dcnRefuse(ws, dir, "stat", storeID, a.Path, err.Error())
return
}
defer r.Close()
info, err := r.Stat(rel)
if err != nil {
if dcnEscaped(err) {
dcnRefuse(ws, dir, "stat", storeID, a.Path, "path escapes the store root")
return
}
if os.IsNotExist(err) {
dcnResult(ws, dir, "stat", storeID, a.Path, map[string]any{
"state": "gone", "path": a.Path})
@@ -771,8 +815,25 @@ func handleDcnReadRange(ws *websocket.Conn, dir string, args []string) {
dcnRefuse(ws, dir, "read_range", storeID, a.Path, "path escapes the store root")
return
}
fh, err := os.Open(abs)
rel, err := dcnRel(terms.Root, abs)
if err != nil {
dcnRefuse(ws, dir, "read_range", storeID, a.Path, err.Error())
return
}
r, err := dcnOpenStore(dir, storeID, terms)
if err != nil {
dcnRefuse(ws, dir, "read_range", storeID, a.Path, err.Error())
return
}
defer r.Close()
// Opened through the store's handle: a name swapped for a link to a file
// outside the store after the check above is refused, never followed.
fh, err := r.Open(rel)
if err != nil {
if dcnEscaped(err) {
dcnRefuse(ws, dir, "read_range", storeID, a.Path, "path escapes the store root")
return
}
if os.IsNotExist(err) {
dcnResult(ws, dir, "read_range", storeID, a.Path, map[string]any{
"state": "gone", "path": a.Path})
+57 -21
View File
@@ -191,10 +191,28 @@ func dcnReportPath(root string, planID int64) string {
return p
}
// dcnApplyPlan runs the plan: check, rename in order (each re-checked just
// before it runs; rolling back on the first failure), verify, report. Every
// path it touches has passed dcnCheckPlan.
// dcnApplyPlan is dcnApplyPlanIn on root opened where it leads now (the op
// handler opens the PINNED folder instead, dcn_store.go).
func dcnApplyPlan(root string, excludes []string, plan *dcnPlanTerms) map[string]any {
real, err := dcnRealRoot(root)
if err != nil {
return map[string]any{"state": "refused", "mode": "apply", "plan_id": plan.PlanID,
"reason": "the store's folder cannot be opened: " + err.Error()}
}
r, err := dcnOpenRootAt(real)
if err != nil {
return map[string]any{"state": "refused", "mode": "apply", "plan_id": plan.PlanID, "reason": err.Error()}
}
defer r.Close()
return dcnApplyPlanIn(r, root, excludes, plan)
}
// dcnApplyPlanIn runs the plan: check, rename in order (each re-checked just
// before it runs; rolling back on the first failure), verify, report. Every
// path it touches has passed dcnCheckPlan, and every change is made through
// r, the store's handle, so a folder that becomes a link to somewhere else
// between the check and the rename is refused rather than followed.
func dcnApplyPlanIn(r *os.Root, root string, excludes []string, plan *dcnPlanTerms) map[string]any {
problems := dcnCheckPlan(root, excludes, plan.Moves)
if len(problems) > 0 {
return map[string]any{"state": "refused", "mode": "apply", "plan_id": plan.PlanID,
@@ -207,11 +225,11 @@ func dcnApplyPlan(root string, excludes []string, plan *dcnPlanTerms) map[string
// and what is NOT (so the user knows exactly what is still displaced).
rollback := func() (back, stuck []string) {
for i := len(applied) - 1; i >= 0; i-- {
if _, err := os.Lstat(applied[i].src); err == nil {
if _, err := r.Lstat(applied[i].src); err == nil {
stuck = append(stuck, applied[i].from) // something now sits at the source; never replace it
continue
}
if err := os.Rename(applied[i].dst, applied[i].src); err == nil {
if err := r.Rename(applied[i].dst, applied[i].src); err == nil {
back = append(back, applied[i].from)
} else {
stuck = append(stuck, applied[i].from)
@@ -230,10 +248,17 @@ func dcnApplyPlan(root string, excludes []string, plan *dcnPlanTerms) map[string
return rep
}
for i, m := range plan.Moves {
src, _ := dcnResolvePath(root, m.From)
dst, _ := cleanRel(root, m.To)
if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
return fail(i, m, "could not create the target folder: "+err.Error())
srcAbs, _ := dcnResolvePath(root, m.From)
dstAbs, _ := cleanRel(root, m.To)
src, serr := dcnRel(root, srcAbs)
dst, derr := dcnRel(root, dstAbs)
if serr != nil || derr != nil {
return fail(i, m, "path escapes the store root")
}
if d := filepath.Dir(dst); d != "." {
if err := r.MkdirAll(d, 0o755); err != nil {
return fail(i, m, "could not create the target folder: "+err.Error())
}
}
// Re-check the instant before the rename: the world may have changed since
// the pre-check (a file appeared at the target; a parent became a symlink).
@@ -241,13 +266,12 @@ func dcnApplyPlan(root string, excludes []string, plan *dcnPlanTerms) map[string
if _, ok := dcnResolvePath(root, m.To); !ok {
return fail(i, m, "target's parent no longer resolves inside the store root")
}
if _, err := os.Lstat(dst); err == nil {
if _, err := r.Lstat(dst); err == nil {
return fail(i, m, "target appeared since the check (a plan never overwrites)")
}
if err := os.Rename(src, dst); err != nil {
if err := r.Rename(src, dst); err != nil {
why := err.Error()
var le *os.LinkError
if errors.As(err, &le) && errors.Is(le.Err, syscall.EXDEV) {
if errors.Is(err, syscall.EXDEV) {
why = "target is on a different filesystem than the source (a plan only renames; it never copies and removes)"
}
return fail(i, m, why)
@@ -257,14 +281,16 @@ func dcnApplyPlan(root string, excludes []string, plan *dcnPlanTerms) map[string
// Verify pass: every target present, every source gone, declared sizes hold.
var mismatches []dcnPlanProblem
for i, m := range plan.Moves {
src, _ := dcnResolvePath(root, m.From)
dst, _ := cleanRel(root, m.To)
info, err := os.Lstat(dst)
srcAbs, _ := dcnResolvePath(root, m.From)
dstAbs, _ := cleanRel(root, m.To)
src, _ := dcnRel(root, srcAbs)
dst, _ := dcnRel(root, dstAbs)
info, err := r.Lstat(dst)
if err != nil {
mismatches = append(mismatches, dcnPlanProblem{i, m.From, m.To, "target missing after move"})
continue
}
if _, err := os.Lstat(src); err == nil {
if _, err := r.Lstat(src); err == nil {
mismatches = append(mismatches, dcnPlanProblem{i, m.From, m.To, "source still present after move"})
}
if m.SizeBytes != nil && !info.IsDir() && info.Size() != *m.SizeBytes {
@@ -372,9 +398,19 @@ func handleDcnPlanApply(ws *websocket.Conn, dir string, args []string) {
// a retry (a lost reply, a host re-run) gets the earlier report back BEFORE
// any gate can turn it into a refusal. (The signed expiry is the guard that
// does not depend on a file the user can remove.)
store, err := dcnOpenStore(dir, storeID, terms)
if err != nil {
dcnRefuse(ws, dir, op, storeID, planKey, err.Error())
return
}
defer store.Close()
repPath := dcnReportPath(terms.Root, plan.PlanID)
repRel := ""
if repPath != "" {
if prev, err := os.ReadFile(repPath); err == nil {
repRel, _ = dcnRel(terms.Root, repPath)
}
if repRel != "" {
if prev, err := store.ReadFile(repRel); err == nil {
var old map[string]any
if json.Unmarshal(prev, &old) == nil && old["state"] == "ok" {
old["note"] = "already applied on this machine; this is the earlier report"
@@ -398,13 +434,13 @@ func handleDcnPlanApply(ws *websocket.Conn, dir string, args []string) {
return
}
}
rep := dcnApplyPlan(terms.Root, terms.Excludes, plan)
rep := dcnApplyPlanIn(store, terms.Root, terms.Excludes, plan)
if rep["state"] == "ok" && plan.Approval != "" {
markApprovalUsed(dir, plan.Approval) // single use: the files have moved
}
if repPath != "" {
if repRel != "" {
if raw, err := json.MarshalIndent(rep, "", " "); err == nil {
_ = atomicWrite(repPath, raw) // the machine's own record; best-effort
_ = dcnRootAtomicWrite(store, repRel, raw) // the machine's own record; best-effort
}
}
dcnResult(ws, dir, op, storeID, planKey, rep)
+322
View File
@@ -0,0 +1,322 @@
package main
// A machine store is the folder the person granted, for as long as the grant
// lives. The bastion often runs as root and the folder belongs to someone who
// is not, so whoever can write in it, or above it, must not be able to point
// a read or a move somewhere else: by turning the granted folder into a link
// after the grant (no race needed), or by swapping a file for a link between
// the check and the open.
import (
"crypto/ed25519"
"encoding/base64"
"encoding/json"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strconv"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/gorilla/websocket"
)
// dcnRunOp runs one DCN handler on the far side of a real WebSocket and
// returns what it sent: the O payload, the R text and the exit code.
func dcnRunOp(t *testing.T, h func(ws *websocket.Conn)) (out, errText string, exit int) {
t.Helper()
up := websocket.Upgrader{}
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
ws, err := up.Upgrade(w, r, nil)
if err != nil {
return
}
defer ws.Close()
h(ws)
}))
defer srv.Close()
ws, _, err := websocket.DefaultDialer.Dial("ws"+strings.TrimPrefix(srv.URL, "http"), nil)
if err != nil {
t.Fatal(err)
}
defer ws.Close()
_ = ws.SetReadDeadline(time.Now().Add(10 * time.Second))
exit = -1
for {
_, data, err := ws.ReadMessage()
if err != nil || len(data) == 0 {
return
}
switch data[0] {
case 'O':
out += string(data[1:])
case 'R':
errText += string(data[1:])
case 'X':
exit, _ = strconv.Atoi(string(data[1:]))
return
}
}
}
type dcnStoreFixture struct {
dir string // the pairing's dcn dir
base string
store string // the granted folder
other string // a folder the grant never named
priv ed25519.PrivateKey
pub ed25519.PublicKey
}
func newDcnStoreFixture(t *testing.T) *dcnStoreFixture {
t.Helper()
fx := &dcnStoreFixture{dir: t.TempDir(), base: t.TempDir()}
fx.store = filepath.Join(fx.base, "store")
fx.other = filepath.Join(fx.base, "other")
for _, d := range []string{fx.store, fx.other, filepath.Join(fx.store, "docs")} {
if err := os.MkdirAll(d, 0o755); err != nil {
t.Fatal(err)
}
}
if err := os.WriteFile(filepath.Join(fx.store, "a.txt"), []byte("MINE"), 0o644); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(fx.other, "a.txt"), []byte("OTHER"), 0o600); err != nil {
t.Fatal(err)
}
fx.pub, fx.priv, _ = ed25519.GenerateKey(nil)
return fx
}
// sync installs a read grant for the store through the real grant_sync op.
func (fx *dcnStoreFixture) sync(t *testing.T, mask ...string) {
t.Helper()
if len(mask) == 0 {
mask = []string{"read"}
}
rec := signedRec(t, fx.priv, map[string]any{"kind": "dcn_store_grant", "class": "machine",
"root": fx.store, "mask": mask,
"expires": time.Now().Add(24 * time.Hour).UTC().Format(time.RFC3339)})
payload, _ := json.Marshal(map[string]any{"store_id": 1, "terms_canonical": rec.TermsCanonical,
"signature": rec.Signature, "verify_pubkey": base64.StdEncoding.EncodeToString(fx.pub)})
_, why, exit := dcnRunOp(t, func(ws *websocket.Conn) {
handleDcnGrantSync(ws, fx.dir, []string{base64.StdEncoding.EncodeToString(payload)})
})
if exit != 0 {
t.Fatalf("grant_sync: exit %d: %s", exit, why)
}
}
func (fx *dcnStoreFixture) op(t *testing.T, h func(*websocket.Conn, string, []string), args map[string]any) (map[string]any, string, int) {
t.Helper()
raw, _ := json.Marshal(args)
out, why, exit := dcnRunOp(t, func(ws *websocket.Conn) {
h(ws, fx.dir, []string{"1", string(raw)})
})
var obj map[string]any
_ = json.Unmarshal([]byte(out), &obj)
return obj, why, exit
}
func (fx *dcnStoreFixture) read(t *testing.T, path string) (string, string, int) {
t.Helper()
obj, why, exit := fx.op(t, handleDcnReadRange, map[string]any{"path": path})
b, _ := base64.StdEncoding.DecodeString(func() string { s, _ := obj["content_b64"].(string); return s }())
return string(b), why, exit
}
// repoint turns the granted folder into a link to the other one.
func (fx *dcnStoreFixture) repoint(t *testing.T) {
t.Helper()
if err := os.Rename(fx.store, fx.store+".was"); err != nil {
t.Fatal(err)
}
if err := os.Symlink(fx.other, fx.store); err != nil {
t.Skipf("symlinks unavailable: %v", err)
}
}
func TestDcnStoreRepointedAfterTheGrantIsRefused(t *testing.T) {
fx := newDcnStoreFixture(t)
fx.sync(t)
if got, why, exit := fx.read(t, "a.txt"); exit != 0 || got != "MINE" {
t.Fatalf("the granted file must read: exit %d %q %s", exit, got, why)
}
fx.repoint(t)
if got, _, exit := fx.read(t, "a.txt"); exit == 0 || got == "OTHER" {
t.Fatalf("after the granted folder became a link elsewhere, read_range returned %q (exit %d); it must refuse", got, exit)
}
if obj, _, exit := fx.op(t, handleDcnStat, map[string]any{"path": "a.txt"}); exit == 0 {
t.Fatalf("stat through the repointed folder must refuse, got %v", obj)
}
if obj, _, exit := fx.op(t, handleDcnList, map[string]any{}); exit == 0 {
t.Fatalf("list through the repointed folder must refuse, got %v", obj)
}
}
// A folder inside the store that becomes a link to somewhere else after the
// grant is the same escape one level down.
func TestDcnSubfolderRepointedIsRefused(t *testing.T) {
fx := newDcnStoreFixture(t)
fx.sync(t)
if err := os.RemoveAll(filepath.Join(fx.store, "docs")); err != nil {
t.Fatal(err)
}
if err := os.Symlink(fx.other, filepath.Join(fx.store, "docs")); err != nil {
t.Skipf("symlinks unavailable: %v", err)
}
if got, _, exit := fx.read(t, "docs/a.txt"); exit == 0 || got == "OTHER" {
t.Fatalf("a link inside the store to a folder outside it read %q (exit %d)", got, exit)
}
}
// The race: a file swapped for a link to a file outside the store between the
// check and the open. Run for long enough that the old check-then-open loses.
func TestDcnReadRangeSwapRaceNeverLeavesTheStore(t *testing.T) {
fx := newDcnStoreFixture(t)
fx.sync(t)
target := filepath.Join(fx.store, "a.txt")
realTmp := filepath.Join(fx.store, ".real")
linkTmp := filepath.Join(fx.store, ".link")
var stop atomic.Bool
done := make(chan struct{})
go func() {
defer close(done)
for !stop.Load() {
_ = os.Symlink(filepath.Join(fx.other, "a.txt"), linkTmp)
_ = os.Rename(linkTmp, target)
_ = os.WriteFile(realTmp, []byte("MINE"), 0o644)
_ = os.Rename(realTmp, target)
}
}()
deadline := time.Now().Add(3 * time.Second)
n := 0
for time.Now().Before(deadline) {
n++
if got, _, _ := fx.read(t, "a.txt"); got == "OTHER" {
stop.Store(true)
<-done
t.Fatalf("read %d returned the file outside the store through a swapped link", n)
}
}
stop.Store(true)
<-done
}
// A move plan applies through the store too: a target folder that leads
// outside the store for an instant between the re-check and the rename must
// not receive the file.
func TestDcnPlanMoveRaceNeverLeavesTheStore(t *testing.T) {
fx := newDcnStoreFixture(t)
if err := os.Remove(filepath.Join(fx.other, "a.txt")); err != nil {
t.Fatal(err)
}
docs := filepath.Join(fx.store, "docs")
var stop atomic.Bool
done := make(chan struct{})
go func() {
defer close(done)
for !stop.Load() {
_ = os.Rename(docs, docs+".real")
_ = os.Symlink(fx.other, docs+".link")
_ = os.Rename(docs+".link", docs)
_ = os.Rename(docs, docs+".link")
_ = os.Remove(docs + ".link")
_ = os.Rename(docs+".real", docs)
}
}()
defer func() { stop.Store(true); <-done }()
deadline := time.Now().Add(3 * time.Second)
for i := 0; time.Now().Before(deadline); i++ {
_ = os.WriteFile(filepath.Join(fx.store, "a.txt"), []byte("MINE"), 0o644)
plan := &dcnPlanTerms{PlanID: int64(i), StoreID: 1, Root: fx.store,
Moves: []dcnMove{{From: "a.txt", To: "docs/a.txt"}}}
_ = dcnApplyPlan(fx.store, nil, plan)
if _, err := os.Lstat(filepath.Join(fx.other, "a.txt")); err == nil {
t.Fatalf("move %d landed outside the store, through a folder that was a link for an instant", i)
}
_ = os.Remove(filepath.Join(fx.store, "docs.real", "a.txt"))
_ = os.Remove(filepath.Join(docs, "a.txt"))
}
}
// The plan report is written beside its name and renamed into place. Whoever
// can write in the store could plant a link at that side name; the write must
// never follow it, or the root bastion truncates whatever file the link names
// (an excluded .ssh/authorized_keys included) and fills it with the report.
func TestDcnReportWriteNeverFollowsAPlantedSideName(t *testing.T) {
store := t.TempDir()
victim := filepath.Join(store, ".ssh", "authorized_keys")
if err := os.MkdirAll(filepath.Dir(victim), 0o700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(victim, []byte("ssh-ed25519 AAAAexample user@example\n"), 0o600); err != nil {
t.Fatal(err)
}
plans := filepath.Join(store, ".agent", "plans")
if err := os.MkdirAll(plans, 0o700); err != nil {
t.Fatal(err)
}
rel := filepath.Join(".agent", "plans", "7.json")
// Every side name a writer could guess: this process's pid, and a spread
// of others, all pointing at the victim.
for _, pid := range []int{os.Getpid(), 1, 2, 100, 1000} {
side := filepath.Join(store, rel+".tmp-"+strconv.Itoa(pid))
if err := os.Symlink(filepath.Join("..", "..", ".ssh", "authorized_keys"), side); err != nil {
t.Fatal(err)
}
}
r, err := os.OpenRoot(store)
if err != nil {
t.Fatal(err)
}
defer r.Close()
if err := dcnRootAtomicWrite(r, rel, []byte(`{"state":"ok"}`)); err != nil {
t.Fatalf("report write: %v", err)
}
if got, _ := os.ReadFile(victim); string(got) != "ssh-ed25519 AAAAexample user@example\n" {
t.Fatalf("the report write followed a planted link: the victim now holds %q", got)
}
if got, err := os.ReadFile(filepath.Join(store, rel)); err != nil || string(got) != `{"state":"ok"}` {
t.Fatalf("report not in place: %q %v", got, err)
}
if fi, err := os.Lstat(filepath.Join(store, rel)); err != nil || fi.Mode()&os.ModeSymlink != 0 || fi.Mode().Perm() != 0o600 {
t.Fatalf("report is not a 0600 regular file: %v %v", fi, err)
}
}
// A grant installed before pins existed has no root_real. It still works,
// pins where its folder leads at first use, keeps that pin, and a repoint
// after that is refused like any other.
func TestDcnGrantFromBeforePinsPinsAtFirstUse(t *testing.T) {
fx := newDcnStoreFixture(t)
fx.sync(t)
f, err := dcnLoadFile(fx.dir)
if err != nil {
t.Fatal(err)
}
rec := f.Stores["1"]
rec.RootReal = ""
f.Stores["1"] = rec
if err := dcnSaveFile(fx.dir, f); err != nil {
t.Fatal(err)
}
if got, why, exit := fx.read(t, "a.txt"); exit != 0 || got != "MINE" {
t.Fatalf("a grant from before pins no longer reads: exit %d %q %s", exit, got, why)
}
f, err = dcnLoadFile(fx.dir)
if err != nil {
t.Fatal(err)
}
want, _ := filepath.EvalSymlinks(fx.store)
if got := f.Stores["1"].RootReal; got != want {
t.Fatalf("first use pinned %q, want %q", got, want)
}
fx.repoint(t)
if got, _, exit := fx.read(t, "a.txt"); exit == 0 || got == "OTHER" {
t.Fatalf("after the first-use pin a repointed folder was read: exit %d %q", exit, got)
}
}
+199
View File
@@ -0,0 +1,199 @@
package main
// dcn_store.go — every byte a machine-store op touches is reached through ONE
// handle on the folder the person granted.
//
// WHY. The bastion is often root (a networked install, the macOS and Windows
// system service), and the granted folder is often someone else's. Checking a
// path and then opening it by name leaves two ways out of the store for
// whoever can write in it or above it:
//
// - the granted folder itself turned into a link to another folder after the
// grant. The old check resolved the root again on every op and judged the
// path against wherever it led NOW, so no race was needed;
// - a file or folder inside the store swapped for a link between the check
// and the open (or the rename, for a move plan).
//
// So: the folder's real path is PINNED at grant sync (rec.RootReal, this box's
// own field, outside the signed bytes), an op whose root now leads elsewhere
// is refused, and the op opens the pinned folder as an os.Root, which never
// follows a link out of it, whatever happens to the names afterwards. That
// handle is then checked to be the folder at the pinned path reached with no
// link at all (dcnNoLinkStat: an open that refuses a link in every component
// on Linux and macOS), so a swap of a folder above the store between the two
// opens is refused too.
import (
"crypto/rand"
"encoding/hex"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"strconv"
)
// errDcnRepointed is the refusal for a granted folder that leads elsewhere now.
var errDcnRepointed = errors.New("the granted folder now leads somewhere other than the folder that was granted; " +
"a new grant for it is the repair")
// dcnRealRoot is where the grant's root leads now: its links resolved.
func dcnRealRoot(root string) (string, error) {
real, err := filepath.EvalSymlinks(filepath.Clean(root))
if err != nil {
return "", err
}
if !filepath.IsAbs(real) {
return "", errors.New("the granted folder is not an absolute path")
}
return filepath.Clean(real), nil
}
// dcnPinFor is the real root a sync records: the one already pinned for the
// same root while the store is held (a re-sync, a renewed grant), else where
// the root leads now ("" when it cannot be resolved yet; pinned at first use).
func dcnPinFor(prev *dcnGrantRec, prevRoot, root string) string {
if prev != nil && prev.RootReal != "" && filepath.Clean(prevRoot) == filepath.Clean(root) {
return prev.RootReal
}
real, err := dcnRealRoot(root)
if err != nil {
return ""
}
return real
}
// dcnStorePin is the pinned real root of a held store, pinning it now (and
// saving it) for a grant synced before pins existed.
func dcnStorePin(dir string, storeID int64, root string) (string, error) {
grantsMu.Lock()
defer grantsMu.Unlock()
f, err := dcnLoadFile(dir)
if err != nil {
return "", err
}
key := strconv.FormatInt(storeID, 10)
rec, ok := f.Stores[key]
if !ok {
return "", errors.New("no grant for this store on this box")
}
if rec.RootReal != "" {
return rec.RootReal, nil
}
real, err := dcnRealRoot(root)
if err != nil {
return "", err
}
rec.RootReal = real
f.Stores[key] = rec
if err := dcnSaveFile(dir, f); err != nil {
return "", err
}
return real, nil
}
// dcnOpenStore opens the store a verified grant names, as the folder that was
// granted, or refuses.
func dcnOpenStore(dir string, storeID int64, terms *dcnTerms) (*os.Root, error) {
pin, err := dcnStorePin(dir, storeID, terms.Root)
if err != nil {
return nil, fmt.Errorf("the granted folder cannot be opened: %w", err)
}
real, err := dcnRealRoot(terms.Root)
if err != nil {
return nil, fmt.Errorf("the granted folder cannot be opened: %w", err)
}
if real != pin {
return nil, errDcnRepointed
}
return dcnOpenRootAt(pin)
}
// dcnOpenRootAt opens real (a resolved, absolute path) as an os.Root and
// checks that the handle is the folder reached at real with no link in any
// component, so nothing above the folder was swapped while it was opened.
func dcnOpenRootAt(real string) (*os.Root, error) {
r, err := os.OpenRoot(real)
if err != nil {
return nil, fmt.Errorf("the granted folder cannot be opened: %w", err)
}
rfi, err := r.Stat(".")
if err != nil {
_ = r.Close()
return nil, fmt.Errorf("the granted folder cannot be opened: %w", err)
}
nfi, err := dcnNoLinkStat(real)
if err != nil || !nfi.IsDir() || !os.SameFile(rfi, nfi) {
_ = r.Close()
return nil, errDcnRepointed
}
return r, nil
}
// dcnRel is a path the sandbox accepted (cleanRel/dcnResolvePath's joined
// form) as a name inside the root handle.
func dcnRel(root, joined string) (string, error) {
rel, err := filepath.Rel(filepath.Clean(root), joined)
if err != nil || rel == ".." || filepath.IsAbs(rel) || len(rel) >= 3 && rel[:3] == ".."+string(os.PathSeparator) {
return "", errors.New("path escapes the store root")
}
return rel, nil
}
// dcnEscaped: the root handle refused a name because it leads out of the store.
func dcnEscaped(err error) bool {
var pe *os.PathError
return errors.As(err, &pe) && pe.Err != nil && pe.Err.Error() == "path escapes from parent"
}
// dcnRootAtomicWrite is atomicWrite inside the store's handle: folders 0700,
// the file 0600, written beside its name and renamed into place.
func dcnRootAtomicWrite(r *os.Root, rel string, data []byte) error {
if d := filepath.Dir(rel); d != "." {
if err := r.MkdirAll(d, 0o700); err != nil {
return err
}
}
// The side name is unguessable and opened O_EXCL, as atomicWrite's
// CreateTemp is: whoever can write in the store could otherwise plant a
// link at a guessable name, and an open that follows it would truncate
// and fill whatever file in the store the link names. O_EXCL refuses any
// existing name, a link included, without following it.
var f *os.File
var tmp string
for try := 0; ; try++ {
var nb [12]byte
if _, err := rand.Read(nb[:]); err != nil {
return err
}
tmp = filepath.Join(filepath.Dir(rel), ".dcn-"+hex.EncodeToString(nb[:])+".tmp")
var err error
f, err = r.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
if err == nil {
break
}
if !errors.Is(err, fs.ErrExist) || try >= 8 {
return err
}
}
if _, err := f.Write(data); err != nil {
_ = f.Close()
_ = r.Remove(tmp)
return err
}
if err := f.Sync(); err != nil {
_ = f.Close()
_ = r.Remove(tmp)
return err
}
if err := f.Close(); err != nil {
_ = r.Remove(tmp)
return err
}
if err := r.Rename(tmp, rel); err != nil {
_ = r.Remove(tmp)
return err
}
return nil
}
+21
View File
@@ -0,0 +1,21 @@
//go:build darwin
package main
import (
"os"
"golang.org/x/sys/unix"
)
// dcnNoLinkStat describes the directory at p opened with O_NOFOLLOW_ANY, which
// fails on a link in any component (macOS 11 and later).
func dcnNoLinkStat(p string) (os.FileInfo, error) {
fd, err := unix.Open(p, unix.O_RDONLY|unix.O_DIRECTORY|unix.O_NOFOLLOW_ANY|unix.O_CLOEXEC, 0)
if err != nil {
return nil, err
}
f := os.NewFile(uintptr(fd), p)
defer f.Close()
return f.Stat()
}
+48
View File
@@ -0,0 +1,48 @@
//go:build linux
package main
import (
"errors"
"os"
"strings"
"golang.org/x/sys/unix"
)
// dcnNoLinkStat describes the directory at p (absolute, clean) reached one
// component at a time with O_NOFOLLOW, so a link anywhere in the path fails
// instead of being followed. O_PATH: a component the bastion may traverse but
// not list (a home at 0711) still opens.
func dcnNoLinkStat(p string) (os.FileInfo, error) {
if !strings.HasPrefix(p, "/") {
return nil, errors.New("not an absolute path")
}
fd, err := unix.Open("/", unix.O_PATH|unix.O_DIRECTORY|unix.O_CLOEXEC, 0)
if err != nil {
return nil, err
}
for _, c := range strings.Split(p, "/") {
if c == "" {
continue
}
if c == "." || c == ".." {
_ = unix.Close(fd)
return nil, errors.New("not a clean path")
}
next, err := unix.Openat(fd, c, unix.O_PATH|unix.O_NOFOLLOW|unix.O_DIRECTORY|unix.O_CLOEXEC, 0)
_ = unix.Close(fd)
if err != nil {
return nil, err
}
var st unix.Stat_t
if err := unix.Fstat(next, &st); err != nil || st.Mode&unix.S_IFMT != unix.S_IFDIR {
_ = unix.Close(next)
return nil, errors.New("a component is not a directory")
}
fd = next
}
f := os.NewFile(uintptr(fd), p)
defer f.Close()
return f.Stat()
}
+14
View File
@@ -0,0 +1,14 @@
//go:build !linux && !darwin
package main
import "os"
// dcnNoLinkStat on a platform with no open that refuses a link in every
// component (Windows): the final component is judged without following it.
// The store is still reached only through its os.Root and only while its real
// path matches the pin; what this leaves is a swap of a folder above the
// store, by someone who can rename it, timed between the two opens.
func dcnNoLinkStat(p string) (os.FileInfo, error) {
return os.Lstat(p)
}
+79 -46
View File
@@ -6,6 +6,7 @@ import (
"errors"
"io"
"net"
"net/http"
"strings"
"time"
)
@@ -24,94 +25,110 @@ type egressDialer func(ctx context.Context, network, addr string) (net.Conn, err
// egress is refused (403, before any dial) unless its pairing grant carries
// the "egress" cap: this lane makes requests leave THIS machine's connection
// under its address, and nothing in the mgmt op table gates it.
func handleEgressConn(client net.Conn, dialOut egressDialer, self string) {
//
// It reports whether it took the job: true once the request passed every check
// and was dialled out, whatever the destination then answered; false when the
// worker closed before a request, or the request was malformed or refused. The
// worker pool (runEgressWorkers) parks again at once after a job it took, and
// waits after one it did not.
func handleEgressConn(client net.Conn, dialOut egressDialer, self string) bool {
defer client.Close()
br := bufio.NewReader(client)
// The request line and headers are read under a bound, the one Go's own HTTP
// server uses (http.DefaultMaxHeaderBytes, 1 MiB): the host sends them, and a
// head that never ends must not grow in this machine's memory.
headLim := &io.LimitedReader{R: client, N: egressMaxHeadBytes}
br := bufio.NewReader(headLim)
line, err := br.ReadString('\n')
if err != nil {
return
return false
}
parts := strings.Fields(strings.TrimSpace(line))
if len(parts) < 2 {
return
return false
}
method, target := parts[0], parts[1]
// Buffer the head so a plain-HTTP request can be replayed verbatim; capture Host.
// Buffer the head: a plain-HTTP request is parsed from it and rebuilt
// (parseEgressRequest), never replayed verbatim.
var head strings.Builder
head.WriteString(line)
hostHdr := ""
for {
h, err := br.ReadString('\n')
if err != nil {
return
return false
}
head.WriteString(h)
if h == "\r\n" || h == "\n" {
break
}
if len(h) >= 5 && strings.EqualFold(h[:5], "host:") {
hostHdr = strings.TrimSpace(h[5:])
}
}
// The head is in; what follows is the stream itself, unbounded. What the
// reader already buffered past the head stays first in line.
headLim.N = 1<<63 - 1
if why := capsAllow(self, "egress"); why != "" {
_, _ = client.Write([]byte("HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\nX-Privatae-Refused: " + why + "\r\n\r\n"))
return
writeEgressRefusal(client, "403 Forbidden", why, why)
return false
}
ctx, cancel := context.WithTimeout(context.Background(), egressDialTimeout)
defer cancel()
if strings.EqualFold(method, "CONNECT") {
dialAddrs, why := egressVettedAddrs(ctx, ensurePort(target, "443"))
// The target is read the way the Host's egress gate reads it
// (parseEgressAuthority): userinfo, an unbracketed IPv6 literal or a
// malformed port is refused, never guessed at.
host, port, ok := parseEgressAuthority(target, 443)
if !ok {
writeEgressRefusal(client, "400 Bad Request", egressNoHost, egressNoHost)
return false
}
dialAddrs, why := egressVettedAddrs(ctx, egressHostPort(host, port))
if why != "" {
_, _ = client.Write([]byte("HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\nX-Privatae-Refused: " + why + "\r\n\r\n"))
return
writeEgressRefusal(client, "403 Forbidden", why, why)
return false
}
upstream, err := dialEach(ctx, dialOut, dialAddrs)
if err != nil {
_, _ = client.Write([]byte("HTTP/1.1 502 Bad Gateway\r\nContent-Length: 0\r\n\r\n"))
return
writeEgressRefusal(client, "502 Bad Gateway", egressUnreachable, "")
return true
}
if _, err := client.Write([]byte("HTTP/1.1 200 Connection established\r\n\r\n")); err != nil {
_ = upstream.Close()
return
return true
}
splice(br, client, upstream)
return
return true
}
// Plain HTTP, absolute-form: "GET http://host/path HTTP/1.1".
host := hostHdr
if i := strings.Index(target, "://"); i >= 0 {
rest := target[i+3:]
if s := strings.IndexByte(rest, '/'); s >= 0 {
host = rest[:s]
} else {
host = rest
}
// Plain HTTP: ONE request per connection, parsed and rebuilt so the
// destination reads the host, port and body the lane vetted, sent with
// `Connection: close` (egress_http.go). Anything the Host sends after that
// request's body closes the connection unforwarded: a keep-alive connection
// can no longer carry later requests past the vetting.
req, refused := parseEgressRequest(head.String())
if refused != nil {
writeEgressRefusal(client, refused.status, refused.body, refused.body)
return false
}
if host == "" {
return
}
dialAddrs, why := egressVettedAddrs(ctx, ensurePort(host, "80"))
dialAddrs, why := egressVettedAddrs(ctx, egressHostPort(req.host, req.port))
if why != "" {
_, _ = client.Write([]byte("HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\nX-Privatae-Refused: " + why + "\r\n\r\n"))
return
writeEgressRefusal(client, "403 Forbidden", why, why)
return false
}
upstream, err := dialEach(ctx, dialOut, dialAddrs)
if err != nil {
_, _ = client.Write([]byte("HTTP/1.1 502 Bad Gateway\r\nContent-Length: 0\r\n\r\n"))
return
writeEgressRefusal(client, "502 Bad Gateway", egressUnreachable, "")
return true
}
if _, err := upstream.Write([]byte(head.String())); err != nil {
_ = upstream.Close()
return
}
splice(br, client, upstream)
oneEgressExchange(br, client, upstream, req)
return true
}
// egressMaxHeadBytes bounds an egress request's line and headers together.
const egressMaxHeadBytes = http.DefaultMaxHeaderBytes
// ensurePort returns host:port, adding defPort when target has no explicit port.
func ensurePort(target, defPort string) string {
target = strings.TrimSpace(target)
@@ -157,11 +174,14 @@ func splice(clientReader io.Reader, client, upstream net.Conn) {
// included), and any address the machine takes as its own (ownaddr.go), such
// as a public one on its NIC or a range a `local` route delivers here.
//
// What differs is the network around it. On an ANCHORED box the lane reaches
// the public internet and nothing else: private, CGNAT and reserved addresses
// (the LAN, the mesh) are refused too. On a LOCAL box the LAN stays reachable:
// the host there is the one the person paired, and their own network is
// theirs to reach.
// The network around it is refused too, on every box: the lane reaches the
// public internet and nothing else, so private, CGNAT and reserved addresses
// (the LAN, the mesh) are refused, by address and by any address a name
// resolves to. A LOCAL box used to keep its LAN reachable here; it no longer
// does (2026-10-26): the Host's egress gate holds every dial it makes to the
// same rule, and a person's own machines
// are reached through their approved lanes, never through egress. Only the
// wording differs between the two kinds of box.
//
// Resolution failures refuse too: a name this box cannot resolve is not one
// it can vouch for.
@@ -245,6 +265,19 @@ func egressVettedAddrs(ctx context.Context, hostport string) ([]string, string)
}
return nil, "refused: egress to " + host + " — that is this machine; this lane never reaches this machine"
}
// The network around this machine, on a local box as on an anchored one
// (above): any one address of the name that is not the public internet
// refuses it. Checked after this machine's own addresses, so those keep
// saying "that is this machine".
if !anchored {
for _, ip := range addrs {
if internalIP(ip) {
return nil, "refused: egress to " + host + " (" + ip.String() + ") — this lane reaches the " +
"public internet only, never this machine's network or the mesh; reach your own " +
"machines through their approved lanes"
}
}
}
out := make([]string, 0, len(addrs))
for _, ip := range addrs {
out = append(out, net.JoinHostPort(ip.String(), port))
+420
View File
@@ -0,0 +1,420 @@
package main
import (
"bufio"
"errors"
"io"
"net"
"net/http/httputil"
"strconv"
"strings"
)
// ONE plain-HTTP request per egress connection, read the way the Host's egress
// gate reads it (its strict request parse and its authority parse: the same
// refusals, the same normalised host, the same rebuilt head).
//
// Everything the lane writes to a destination is what it checked, and nothing
// more:
// - the head is parsed strictly and REBUILT, so the destination reads the same
// host, port and body length the lane vetted: never a second reading of the
// Host's bytes (a Host header naming another site, a query or userinfo glued
// to the host, a padded Transfer-Encoding);
// - the request goes out with `Connection: close`, its body is framed by
// Content-Length or chunked encoding, and a byte the Host sends after the
// body closes the connection with nothing of it forwarded. Before this the
// lane wrote the first head and then spliced the whole connection, so every
// later request on a keep-alive connection reached the destination unread.
//
// An honest client sends its next request only after the response, which the
// destination closes behind `Connection: close`, so it opens a new connection
// and meets the vetting again.
// egressHostChars are the characters a host name may have here (lower-cased).
const egressHostChars = "abcdefghijklmnopqrstuvwxyz0123456789-._"
// egressTokenChars are RFC 9110 token characters: methods and header names.
const egressTokenChars = "!#$%&'*+-.^_`|~0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
// egressHopByHop are headers that belong to one hop. The lane speaks for itself
// on the next one, and a proxy credential never goes on to the destination.
var egressHopByHop = map[string]bool{
"connection": true, "proxy-connection": true, "keep-alive": true, "upgrade": true,
"te": true, "trailer": true, "proxy-authorization": true, "proxy-authenticate": true,
}
// egressTrailerLimit bounds the trailer section of a chunked body, which is
// read to its end and dropped (the gate's limit).
const egressTrailerLimit = 8192
// egressRefusal is a request the lane refuses before it dials anything.
type egressRefusal struct{ status, body string }
func egressBad(body string) *egressRefusal {
return &egressRefusal{"400 Bad Request", body}
}
const egressNoHost = "The bastion could not read where this request is going (no host in the " +
"request line or in a Host header), so it sent nothing."
// egressUnreachable is the body of a 502: the destination passed the vetting,
// and no address of it answered from this machine.
const egressUnreachable = "The bastion could not reach that destination from this machine's own " +
"connection, so nothing was sent."
// egressHostPort is host and port as the vetting takes them.
func egressHostPort(host string, port int) string {
return net.JoinHostPort(host, strconv.Itoa(port))
}
// writeEgressRefusal answers the Host with a status and a short plain-text
// body saying why, and closes the exchange. why, when set, also goes in
// X-Privatae-Refused, the header the Host already reads.
func writeEgressRefusal(w io.Writer, status, body, why string) {
body += "\n"
var b strings.Builder
b.WriteString("HTTP/1.1 " + status + "\r\n")
b.WriteString("Content-Type: text/plain; charset=utf-8\r\n")
b.WriteString("Content-Length: " + strconv.Itoa(len(body)) + "\r\n")
b.WriteString("Connection: close\r\n")
if why != "" {
b.WriteString("X-Privatae-Refused: " + strings.NewReplacer("\r", " ", "\n", " ").Replace(why) + "\r\n")
}
b.WriteString("\r\n")
b.WriteString(body)
_, _ = io.WriteString(w, b.String())
}
// parseEgressAuthority: (host, port, true) from "host", "host:port", "[v6]" or
// "[v6]:port", or false when the lane can't read it exactly as the destination
// would. Refused rather than guessed: userinfo ("a@b"), an unbracketed IPv6
// literal, characters no host name has, a non-numeric or out-of-range port.
// The host is lower-cased and loses one trailing dot.
func parseEgressAuthority(s string, defaultPort int) (string, int, bool) {
s = strings.TrimSpace(s)
if s == "" || strings.Contains(s, "@") {
return "", 0, false
}
var host string
port := defaultPort
digits := func(p string) bool {
if len(p) == 0 || len(p) > 5 {
return false
}
for _, c := range p {
if c < '0' || c > '9' {
return false
}
}
return true
}
if strings.HasPrefix(s, "[") {
end := strings.IndexByte(s, ']')
if end < 0 {
return "", 0, false
}
// An IPv6 literal, IPv4-mapped forms included (the vetting judges the
// address they map to); a bare IPv4 address in brackets or a zone is
// not one.
inner := s[1:end]
ip := net.ParseIP(inner)
if ip == nil || !strings.Contains(inner, ":") || strings.Contains(inner, "%") {
return "", 0, false
}
host = ip.String()
rest := s[end+1:]
switch {
case rest == "":
case strings.HasPrefix(rest, ":") && digits(rest[1:]):
port, _ = strconv.Atoi(rest[1:])
default:
return "", 0, false
}
} else {
if strings.Count(s, ":") > 1 {
return "", 0, false
}
h, p, hasPort := strings.Cut(s, ":")
if hasPort {
if !digits(p) {
return "", 0, false
}
port, _ = strconv.Atoi(p)
}
h = strings.ToLower(h)
h = strings.TrimSuffix(h, ".")
if h == "" || strings.HasPrefix(h, ".") || strings.Contains(h, "..") {
return "", 0, false
}
for _, c := range h {
if !strings.ContainsRune(egressHostChars, c) {
return "", 0, false
}
}
host = h
}
if port <= 0 || port >= 65536 {
return "", 0, false
}
return host, port, true
}
// fmtEgressAuthority writes host[:port] back, the port left out when it is the
// scheme's default.
func fmtEgressAuthority(host string, port, defaultPort int) string {
h := host
if strings.Contains(host, ":") {
h = "[" + host + "]"
}
if port == defaultPort {
return h
}
return h + ":" + strconv.Itoa(port)
}
// egressRequest is the one plain-HTTP request the lane forwards.
type egressRequest struct {
host string
port int
head []byte // rebuilt, CRLF, ends at its blank line
chunked bool
length int64
}
func egressTokenOK(s string) bool {
if s == "" {
return false
}
for _, c := range s {
if !strings.ContainsRune(egressTokenChars, c) {
return false
}
}
return true
}
// parseEgressRequest reads the one request in head (the request line and
// headers through their blank line, as handleEgressConn read them) and
// rebuilds it, or refuses it. CONNECT never comes here.
func parseEgressRequest(head string) (egressRequest, *egressRefusal) {
lines := strings.Split(head, "\n")
// The read stops at the first blank line, so the head ends "\n", blank, "\n":
// drop the empty tail after the last "\n" and the blank line itself.
if len(lines) < 3 {
return egressRequest{}, egressBad("The request ended before its headers did, so the bastion sent nothing.")
}
lines = lines[:len(lines)-2]
for i, ln := range lines {
lines[i] = strings.TrimSuffix(ln, "\r")
if strings.ContainsAny(lines[i], "\r\x00") {
return egressRequest{}, egressBad("The request's headers carry a stray carriage return or NUL, so the bastion sent nothing.")
}
}
parts := strings.Split(lines[0], " ")
if len(parts) != 3 || parts[0] == "" || parts[1] == "" || parts[2] == "" || !egressTokenOK(parts[0]) {
return egressRequest{}, egressBad("The request line is not 'METHOD target HTTP/1.x', so the bastion sent nothing.")
}
method, rawTarget, version := strings.ToUpper(parts[0]), parts[1], parts[2]
if version != "HTTP/1.1" && version != "HTTP/1.0" {
return egressRequest{}, egressBad("The bastion forwards HTTP/1.0 and HTTP/1.1 only, so it sent nothing.")
}
type header struct{ name, value string }
var headers []header
for _, ln := range lines[1:] {
if strings.HasPrefix(ln, " ") || strings.HasPrefix(ln, "\t") {
return egressRequest{}, egressBad("A header is folded onto a second line (obsolete line folding), so the bastion sent nothing.")
}
name, value, ok := strings.Cut(ln, ":")
if !ok || !egressTokenOK(name) {
return egressRequest{}, egressBad("A header line is not 'Name: value' (no space may come before the colon), so the bastion sent nothing.")
}
headers = append(headers, header{name, strings.Trim(value, " \t")})
}
values := func(n string) []string {
var out []string
for _, h := range headers {
if strings.EqualFold(h.name, n) {
out = append(out, h.value)
}
}
return out
}
hosts := values("host")
if len(hosts) > 1 {
return egressRequest{}, egressBad("The request carries more than one Host header, so the bastion sent nothing.")
}
var host, target, authority string
var port int
if scheme, after, abs := strings.Cut(rawTarget, "://"); abs {
if strings.ToLower(scheme) != "http" {
// https:// in absolute form asks the lane to make the TLS connection,
// which it does not do: it used to send the request in plain text to
// port 80. Every client sends https as a CONNECT.
return egressRequest{}, egressBad("The bastion forwards plain http:// requests only; send https as a CONNECT. Nothing was sent.")
}
cut := len(after)
for _, sep := range []string{"/", "?", "#"} {
if i := strings.Index(after, sep); i >= 0 && i < cut {
cut = i
}
}
var ok bool
if host, port, ok = parseEgressAuthority(after[:cut], 80); !ok {
return egressRequest{}, egressBad(egressNoHost)
}
path, _, _ := strings.Cut(after[cut:], "#")
if !strings.HasPrefix(path, "/") {
path = "/" + path
}
authority = fmtEgressAuthority(host, port, 80)
target = "http://" + authority + path
} else {
if !strings.HasPrefix(rawTarget, "/") && rawTarget != "*" {
return egressRequest{}, egressBad(egressNoHost)
}
hostHdr := ""
if len(hosts) == 1 {
hostHdr = hosts[0]
}
var ok bool
if host, port, ok = parseEgressAuthority(hostHdr, 80); !ok {
return egressRequest{}, egressBad(egressNoHost)
}
authority = fmtEgressAuthority(host, port, 80)
target = rawTarget
}
// Body framing (RFC 9112 section 6). Every reading two parsers could disagree
// on is refused: both length headers, an unknown or doubled coding, a coding
// on HTTP/1.0, Content-Length values that differ.
req := egressRequest{host: host, port: port}
te, cl := values("transfer-encoding"), values("content-length")
if len(te) > 0 {
var codings []string
for _, v := range te {
for _, c := range strings.Split(v, ",") {
codings = append(codings, strings.ToLower(strings.TrimSpace(c)))
}
}
if len(cl) > 0 || version == "HTTP/1.0" || len(codings) != 1 || codings[0] != "chunked" {
return egressRequest{}, egressBad("The request's body length is ambiguous (Transfer-Encoding with " +
"Content-Length, on HTTP/1.0, or a coding other than chunked), so the bastion sent nothing.")
}
req.chunked = true
} else if len(cl) > 0 {
seen := map[string]bool{}
for _, v := range cl {
for _, c := range strings.Split(v, ",") {
seen[strings.TrimSpace(c)] = true
}
}
var one string
for k := range seen {
one = k
}
n, err := strconv.ParseInt(one, 10, 64)
if len(seen) != 1 || one == "" || len(one) > 18 || err != nil || n < 0 || strings.TrimLeft(one, "0123456789") != "" {
return egressRequest{}, egressBad("The request's Content-Length is not one number, so the bastion sent nothing.")
}
req.length = n
}
named := map[string]bool{}
for _, v := range values("connection") {
for _, t := range strings.Split(v, ",") {
named[strings.ToLower(strings.TrimSpace(t))] = true
}
}
var b strings.Builder
b.WriteString(method + " " + target + " " + version + "\r\n")
b.WriteString("Host: " + authority + "\r\n")
for _, h := range headers {
lk := strings.ToLower(h.name)
if egressHopByHop[lk] || named[lk] || lk == "host" || lk == "content-length" || lk == "transfer-encoding" {
continue
}
b.WriteString(h.name + ": " + h.value + "\r\n")
}
if req.chunked {
b.WriteString("Transfer-Encoding: chunked\r\n")
} else if len(cl) > 0 {
b.WriteString("Content-Length: " + strconv.FormatInt(req.length, 10) + "\r\n")
}
b.WriteString("Connection: close\r\n\r\n")
req.head = []byte(b.String())
return req, nil
}
// forwardEgressBody copies the one request body from src to dst, framed as the
// rebuilt head says. A chunked body goes out re-encoded (plain sizes, no
// extensions, trailers read and dropped), so the destination parses exactly the
// body this parser found.
func forwardEgressBody(dst io.Writer, src *bufio.Reader, req egressRequest) error {
if !req.chunked {
if req.length == 0 {
return nil
}
_, err := io.CopyN(dst, src, req.length)
return err
}
cw := httputil.NewChunkedWriter(dst)
if _, err := io.Copy(cw, httputil.NewChunkedReader(src)); err != nil {
return err
}
// The trailer section, read to its blank line within a bound, and dropped.
read := 0
for {
line, err := src.ReadSlice('\n')
read += len(line)
if err != nil || read > egressTrailerLimit {
return errors.New("chunked body: bad trailer section")
}
if string(line) == "\r\n" || string(line) == "\n" {
break
}
}
if err := cw.Close(); err != nil {
return err
}
_, err := io.WriteString(dst, "\r\n")
return err
}
// oneEgressExchange writes the one rebuilt request and its body to upstream,
// and carries the destination's answer back until the destination closes.
// Nothing the Host sends after the body is forwarded: its first byte closes
// both sides.
func oneEgressExchange(clientReader *bufio.Reader, client, upstream net.Conn, req egressRequest) {
answered := make(chan struct{})
go func() {
_, _ = io.Copy(client, upstream)
close(answered)
_ = client.Close()
}()
defer func() {
_ = upstream.Close()
<-answered
}()
if _, err := upstream.Write(req.head); err != nil {
return
}
if err := forwardEgressBody(upstream, clientReader, req); err != nil {
return
}
// The request is complete. A further byte is a second request on this
// connection: never forwarded, and the connection closes. End of stream
// from the Host is passed on, as the splice did.
go func() {
var one [1]byte
if n, _ := clientReader.Read(one[:]); n > 0 {
_ = upstream.Close()
_ = client.Close()
return
}
if cw, ok := upstream.(interface{ CloseWrite() error }); ok {
_ = cw.CloseWrite()
}
}()
<-answered
}
+17 -16
View File
@@ -7,8 +7,10 @@ package main
// The egress lane on a local (unanchored) box. It refuses this machine and
// its link-local addresses, so a host cannot reach this machine's services
// (ollama above all) around the switches that govern them, and it keeps the
// person's LAN. An anchored box is unchanged: the public internet only.
// (ollama above all) around the switches that govern them, and, since
// 2026-10-26, the network around it as well: the LAN and the mesh
// (egress_network_test.go). An anchored box is unchanged: the public internet
// only.
import (
"bufio"
@@ -57,13 +59,14 @@ func localEgressFixture(t *testing.T) {
"self.example.test": {"192.168.50.7"},
"nas.example.test": {"192.168.50.30"},
"dual.example.test": {"2001:db8::8", "198.51.100.8"},
"site.example.test": {"198.51.100.9"},
})
}
// MUTATION: restore the early return for an unanchored box in egressVettedAddrs.
// RESULT: red — 127.0.0.1:11434 is handed to the dial, and a name is dialled
// as a name instead of the address that was vetted.
func TestLocalEgressRefusesThisMachineAndKeepsTheLAN(t *testing.T) {
func TestLocalEgressRefusesThisMachineAndTheLAN(t *testing.T) {
localEgressFixture(t)
ctx := context.Background()
for _, hp := range []string{
@@ -75,24 +78,22 @@ func TestLocalEgressRefusesThisMachineAndKeepsTheLAN(t *testing.T) {
"203.0.113.5:80", // a range a `local` route delivers here
"ollama.example.test:11434", "mixed.example.test:443", "self.example.test:11434",
"unknown.example.test:443",
// the network around this machine (egress_network_test.go has the rest)
"192.168.50.20:8080", "10.1.2.3:443", "172.16.4.4:22", "[fd12:3456::9]:80", "100.64.0.9:80",
"nas.example.test:443",
} {
if addr, why := egressVettedAddr(ctx, hp); why == "" || addr != "" {
t.Errorf("%s must be refused on a local box, got dial %q", hp, addr)
}
}
for hp, want := range map[string][]string{
"192.168.50.20:8080": {"192.168.50.20:8080"},
"10.1.2.3:443": {"10.1.2.3:443"},
"172.16.4.4:22": {"172.16.4.4:22"},
"[fd12:3456::9]:80": {"[fd12:3456::9]:80"},
"100.64.0.9:80": {"100.64.0.9:80"},
"198.51.100.8:443": {"198.51.100.8:443"},
"nas.example.test:443": {"192.168.50.30:443"},
"site.example.test:443": {"198.51.100.9:443"},
"dual.example.test:443": {"[2001:db8::8]:443", "198.51.100.8:443"},
} {
got, why := egressVettedAddrs(ctx, hp)
if why != "" || !reflect.DeepEqual(got, want) {
t.Errorf("%s is the LAN or the internet, so a local box dials %v: got %v %q", hp, want, got, why)
t.Errorf("%s is the internet, so a local box dials %v: got %v %q", hp, want, got, why)
}
}
}
@@ -114,8 +115,8 @@ func TestAnchoredEgressStillRefusesTheLAN(t *testing.T) {
}
// Through the lane itself, with a real dialer: a request for this machine is
// answered 403 and never reaches the listener; a LAN request is dialled at its
// vetted address; a dual-stack name falls back to its second address.
// answered 403 and never reaches the listener; a public name is dialled at
// its vetted address; a dual-stack name falls back to its second address.
func TestLocalEgressLaneNeverDialsThisMachine(t *testing.T) {
localEgressFixture(t)
ln, err := net.Listen("tcp", "127.0.0.1:0")
@@ -164,11 +165,11 @@ func TestLocalEgressLaneNeverDialsThisMachine(t *testing.T) {
asked = append(asked, addr)
return nil, errors.New("stop here")
}
if line := ask(recording, "CONNECT nas.example.test:443 HTTP/1.1\r\n\r\n"); !strings.HasPrefix(line, "HTTP/1.1 502") {
t.Fatalf("a LAN destination is dialled (and this dial fails): %q", line)
if line := ask(recording, "CONNECT site.example.test:443 HTTP/1.1\r\n\r\n"); !strings.HasPrefix(line, "HTTP/1.1 502") {
t.Fatalf("a public destination is dialled (and this dial fails): %q", line)
}
if !reflect.DeepEqual(asked, []string{"192.168.50.30:443"}) {
t.Fatalf("the vetted LAN address, never the name: %v", asked)
if !reflect.DeepEqual(asked, []string{"198.51.100.9:443"}) {
t.Fatalf("the vetted address, never the name: %v", asked)
}
asked = nil
+76
View File
@@ -0,0 +1,76 @@
//go:build !windows
// Tagged !windows: it borrows localEgressFixture and answerEgressNames
// (egress_local_test.go).
package main
// The egress lane on a local (unanchored) box refuses the network around the
// machine as well as the machine itself: the LAN, the mesh and every other
// address that is not the public internet, by address and by any address a
// name resolves to. The Host's egress gate holds every dial to the same rule;
// a person's own machines are reached through their approved lanes.
import (
"bufio"
"context"
"errors"
"io"
"net"
"reflect"
"strings"
"testing"
"time"
)
// MUTATION: drop the local-box internalIP loop at the end of egressVettedAddrs.
// RESULT: red — 192.168.50.20, nas.example.test and the rest are handed to the
// dial, and the lane dials 192.168.50.30.
func TestLocalEgressRefusesTheLANAndTheMesh(t *testing.T) {
localEgressFixture(t)
answerEgressNames(t, map[string][]string{
"nas.example.test": {"192.168.50.30"},
"half.example.test": {"198.51.100.8", "10.1.2.3"},
"mesh.example.test": {"fd12:3456::9"},
"nat.example.test": {"64:ff9b::a01:203"}, // NAT64 for 10.1.2.3
"pub.example.test": {"198.51.100.8"},
})
ctx := context.Background()
for _, hp := range []string{
"192.168.50.20:8080", "10.1.2.3:443", "172.16.4.4:22", "[fd12:3456::9]:80", "100.64.0.9:80",
"198.18.0.1:80", "[64:ff9b::a01:203]:443",
"nas.example.test:443", "half.example.test:443", "mesh.example.test:80", "nat.example.test:443",
} {
if addrs, why := egressVettedAddrs(ctx, hp); why == "" || addrs != nil || !strings.Contains(why, "approved lanes") {
t.Errorf("%s is not the public internet and must be refused on a local box, got dial %v %q", hp, addrs, why)
}
}
if got, why := egressVettedAddrs(ctx, "pub.example.test:443"); why != "" || !reflect.DeepEqual(got, []string{"198.51.100.8:443"}) {
t.Fatalf("the public internet passes at its vetted address: %v %q", got, why)
}
// Through the lane itself: 403, and nothing is dialled.
var asked []string
recording := func(_ context.Context, _, addr string) (net.Conn, error) {
asked = append(asked, addr)
return nil, errors.New("stop here")
}
for _, req := range []string{
"CONNECT nas.example.test:443 HTTP/1.1\r\n\r\n",
"GET http://10.1.2.3/ HTTP/1.1\r\nHost: 10.1.2.3\r\n\r\n",
"CONNECT half.example.test:443 HTTP/1.1\r\n\r\n",
} {
srv, cli := net.Pipe()
go handleEgressConn(srv, recording, "home.host.example")
_ = cli.SetDeadline(time.Now().Add(5 * time.Second))
_, _ = io.WriteString(cli, req)
line, _ := bufio.NewReader(cli).ReadString('\n')
cli.Close()
if !strings.HasPrefix(line, "HTTP/1.1 403") {
t.Errorf("a local box must refuse %q, answered %q", req, line)
}
}
if len(asked) != 0 {
t.Fatalf("nothing on the network around this machine is dialled: %v", asked)
}
}
+393
View File
@@ -0,0 +1,393 @@
//go:build !windows
package main
import (
"bufio"
"bytes"
"context"
"errors"
"fmt"
"io"
"net"
"net/http"
"strings"
"sync"
"testing"
"time"
)
// The bastion's egress lanes carry ONE plain-HTTP request per connection, read
// the way the Host's egress gate reads it, and every refusal says
// why in its body.
//
// MUTATIONS, each red:
// - handleEgressConn writes head.String() to upstream and splices, as before:
// TestEgressPlainHTTPCarriesOneRequestPerConnection, ...SendsTheHostItChecked;
// - oneEgressExchange's watcher forwards instead of closing: ...OneRequest...;
// - parseEgressAuthority accepts "@" or cuts the authority at "/" only:
// TestEgressPlainHTTPReadsTheHostAsTheGateDoes;
// - writeEgressRefusal writes Content-Length: 0: TestEgressRefusalsCarryABody.
// recordingOrigin stands in for a destination that keeps a connection open
// after answering (it ignores Connection: close, as a careless server may),
// and records every byte one connection carried to it.
type recordingOrigin struct {
ln net.Listener
conns chan string
// silent: never answer, so the lane never holds an unread answer when it
// closes (a close with unread data is a reset, which can cost the origin
// bytes it has not read yet).
silent bool
}
func newSilentOrigin(t *testing.T) *recordingOrigin {
o := newRecordingOrigin(t)
o.silent = true
return o
}
func newRecordingOrigin(t *testing.T) *recordingOrigin {
t.Helper()
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
o := &recordingOrigin{ln: ln, conns: make(chan string, 16)}
t.Cleanup(func() { _ = ln.Close() })
go func() {
for {
c, err := ln.Accept()
if err != nil {
return
}
go o.serve(c)
}
}()
return o
}
func (o *recordingOrigin) serve(c net.Conn) {
defer c.Close()
var mu sync.Mutex
var got bytes.Buffer
answered := false
buf := make([]byte, 4096)
for {
_ = c.SetReadDeadline(time.Now().Add(800 * time.Millisecond))
n, err := c.Read(buf)
mu.Lock()
got.Write(buf[:n])
if !o.silent && !answered && bytes.Contains(got.Bytes(), []byte("\r\n\r\n")) {
answered = true
_, _ = io.WriteString(c, "HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")
}
mu.Unlock()
if err != nil {
break
}
}
o.conns <- got.String()
}
// next is everything the origin's next connection carried, once it ended.
func (o *recordingOrigin) next(t *testing.T) string {
t.Helper()
s, ok := o.nextWithin(8 * time.Second)
if !ok {
t.Fatal("the destination never saw its connection end")
}
return s
}
func (o *recordingOrigin) nextWithin(d time.Duration) (string, bool) {
select {
case s := <-o.conns:
return s, true
case <-time.After(d):
return "", false
}
}
func (o *recordingOrigin) addr() string { return o.ln.Addr().String() }
// A keep-alive connection carries one request to the destination, never two:
// neither a request sent after the first answer (a client reusing the
// connection) nor one pipelined behind the first.
func TestEgressPlainHTTPCarriesOneRequestPerConnection(t *testing.T) {
first := "GET http://198.51.100.30/first HTTP/1.1\r\nHost: 198.51.100.30\r\n\r\n"
second := "GET http://198.51.100.30/second HTTP/1.1\r\nHost: 198.51.100.30\r\n\r\n"
t.Run("reused after the answer", func(t *testing.T) {
o := newRecordingOrigin(t)
var asked []string
cli, cleanup := dialHandler(t, redirectDial(o.addr(), &asked))
defer cleanup()
_, _ = io.WriteString(cli, first)
br := bufio.NewReader(cli)
resp, err := http.ReadResponse(br, nil)
if err != nil {
t.Fatalf("the first request got no answer: %v", err)
}
body, _ := io.ReadAll(resp.Body)
if string(body) != "ok" {
t.Fatalf("the first answer was %q", body)
}
_, _ = io.WriteString(cli, second)
got := o.next(t)
if strings.Contains(got, "/second") {
t.Fatalf("a second request on one connection reached the destination unvetted:\n%s", got)
}
if !strings.Contains(got, "\r\nConnection: close\r\n") {
t.Fatalf("the request went out without Connection: close:\n%s", got)
}
_ = cli.SetReadDeadline(time.Now().Add(3 * time.Second))
if rest, _ := io.ReadAll(br); len(rest) != 0 {
t.Fatalf("the lane answered the second request: %q", rest)
}
if len(asked) != 1 {
t.Fatalf("one connection, one dial: %v", asked)
}
})
t.Run("pipelined", func(t *testing.T) {
o := newRecordingOrigin(t)
var asked []string
cli, cleanup := dialHandler(t, redirectDial(o.addr(), &asked))
defer cleanup()
_, _ = io.WriteString(cli, first+second)
got := o.next(t)
if !strings.Contains(got, "GET http://198.51.100.30/first HTTP/1.1\r\n") {
t.Fatalf("the first request did not reach the destination:\n%s", got)
}
if strings.Contains(got, "/second") {
t.Fatalf("a pipelined second request reached the destination unvetted:\n%s", got)
}
})
}
// The destination reads the host the lane vetted, in the one Host header; hop
// headers, those Connection names and a proxy credential stay on this hop.
func TestEgressPlainHTTPSendsTheHostItChecked(t *testing.T) {
o := newRecordingOrigin(t)
var asked []string
cli, cleanup := dialHandler(t, redirectDial(o.addr(), &asked))
defer cleanup()
_, _ = io.WriteString(cli, "GET http://198.51.100.31/p?q=1 HTTP/1.1\r\n"+
"Host: inside.example.test\r\nProxy-Authorization: Basic eDp5\r\n"+
"Connection: keep-alive, X-Hop\r\nX-Hop: 1\r\nX-Keep: 2\r\n\r\n")
got := o.next(t)
head, _, _ := strings.Cut(got, "\r\n\r\n")
want := "GET http://198.51.100.31/p?q=1 HTTP/1.1\r\nHost: 198.51.100.31\r\nX-Keep: 2\r\nConnection: close"
if head != want {
t.Fatalf("the destination got\n%q\nwant\n%q", head, want)
}
if len(asked) != 1 || asked[0] != "198.51.100.31:80" {
t.Fatalf("the lane must dial the address it vetted: %v", asked)
}
}
// stubEgressNames answers names without DNS and records each name exactly as
// the lane asked for it.
func stubEgressNames(t *testing.T, m map[string]string) *[]string {
t.Helper()
var mu sync.Mutex
var asked []string
old := egressResolve
t.Cleanup(func() { egressResolve = old })
egressResolve = func(_ context.Context, host string) ([]net.IPAddr, error) {
mu.Lock()
asked = append(asked, host)
mu.Unlock()
ip, ok := m[host]
if !ok {
return nil, errors.New("no such host")
}
return []net.IPAddr{{IP: net.ParseIP(ip)}}, nil
}
return &asked
}
// The host is read as the gate reads it: the authority ends at the first "/",
// "?" or "#", is lower-cased and loses one trailing dot; userinfo, an
// unbracketed IPv6 literal, a bad port, https in absolute form, two Host
// headers and an ambiguous body length are refused with 400 and a body, before
// anything is resolved or dialled.
func TestEgressPlainHTTPReadsTheHostAsTheGateDoes(t *testing.T) {
for _, c := range []struct {
req, wantName, wantLine string
}{
{"GET http://public.example.test?@10.0.0.1/ HTTP/1.1\r\n\r\n", "public.example.test",
"GET http://public.example.test/?@10.0.0.1/ HTTP/1.1"},
{"GET http://public.example.test#@10.0.0.1/ HTTP/1.1\r\n\r\n", "public.example.test",
"GET http://public.example.test/ HTTP/1.1"},
{"GET http://Public.Example.Test./x HTTP/1.1\r\n\r\n", "public.example.test",
"GET http://public.example.test/x HTTP/1.1"},
{"GET /x HTTP/1.1\r\nHost: Public.Example.Test:8080\r\n\r\n", "public.example.test",
"GET /x HTTP/1.1"},
} {
o := newRecordingOrigin(t)
names := stubEgressNames(t, map[string]string{"public.example.test": "198.51.100.40"})
var asked []string
cli, cleanup := dialHandler(t, redirectDial(o.addr(), &asked))
_, _ = io.WriteString(cli, c.req)
got, _ := o.nextWithin(4 * time.Second)
cleanup()
if len(*names) != 1 || (*names)[0] != c.wantName {
t.Errorf("%q: the lane resolved %q; want only %q", c.req, *names, c.wantName)
}
if line, _, _ := strings.Cut(got, "\r\n"); line != c.wantLine {
t.Errorf("%q: the destination got the request line %q; want %q", c.req, line, c.wantLine)
}
}
for _, req := range []string{
"GET http://user:pw@public.example.test/ HTTP/1.1\r\n\r\n",
"GET http://198.51.100.41:80@public.example.test/ HTTP/1.1\r\n\r\n",
"GET http://2001:db8::1/ HTTP/1.1\r\n\r\n",
"GET http://public.example.test:80x/ HTTP/1.1\r\n\r\n",
"GET https://public.example.test/ HTTP/1.1\r\n\r\n",
"GET / HTTP/1.1\r\nHost: public.example.test\r\nHost: 198.51.100.41\r\n\r\n",
"GET / HTTP/1.1\r\n\r\n",
"POST http://public.example.test/ HTTP/1.1\r\nContent-Length: 3\r\nTransfer-Encoding: chunked\r\n\r\n",
"POST http://public.example.test/ HTTP/1.1\r\nContent-Length: 3\r\nContent-Length: 4\r\n\r\n",
"POST http://public.example.test/ HTTP/1.1\r\nTransfer-Encoding: gzip, chunked\r\n\r\n",
"GET http://public.example.test/ HTTP/1.1\r\n X-Folded: 1\r\n\r\n",
"CONNECT user@public.example.test:443 HTTP/1.1\r\n\r\n",
"CONNECT 2001:db8::1:443 HTTP/1.1\r\n\r\n",
"CONNECT [198.51.100.41]:443 HTTP/1.1\r\n\r\n",
} {
names := stubEgressNames(t, map[string]string{"public.example.test": "198.51.100.40"})
var asked []string
cli, cleanup := dialHandler(t, redirectDial("127.0.0.1:9", &asked))
_, _ = io.WriteString(cli, req)
resp, err := http.ReadResponse(bufio.NewReader(cli), nil)
if err != nil {
t.Errorf("%q: no answer: %v", req, err)
cleanup()
continue
}
body, _ := io.ReadAll(resp.Body)
cleanup()
if resp.StatusCode != 400 || len(body) == 0 || int64(len(body)) != resp.ContentLength {
t.Errorf("%q: answered %d with body %q; want 400 with a body saying why", req, resp.StatusCode, body)
}
if len(*names) != 0 || len(asked) != 0 {
t.Errorf("%q: refused requests resolve and dial nothing; resolved %q, dialled %q", req, *names, asked)
}
}
}
// A request body goes out framed as its head says, chunked bodies re-encoded,
// and nothing after it.
func TestEgressPlainHTTPBodyIsFramed(t *testing.T) {
for _, c := range []struct{ req, wantBody string }{
{"POST http://198.51.100.42/u HTTP/1.1\r\nHost: 198.51.100.42\r\nContent-Length: 5\r\n\r\nhello" +
"GET http://198.51.100.42/smuggled HTTP/1.1\r\n\r\n", "hello"},
{"POST http://198.51.100.42/u HTTP/1.1\r\nHost: 198.51.100.42\r\nTransfer-Encoding: chunked\r\n\r\n" +
"5;ext=1\r\nhello\r\n0\r\nX-Trailer: 1\r\n\r\nGET /smuggled HTTP/1.1\r\n\r\n", "5\r\nhello\r\n0\r\n\r\n"},
} {
o := newSilentOrigin(t)
var asked []string
cli, cleanup := dialHandler(t, redirectDial(o.addr(), &asked))
_, _ = io.WriteString(cli, c.req)
got := o.next(t)
cleanup()
_, body, _ := strings.Cut(got, "\r\n\r\n")
if body != c.wantBody {
t.Errorf("the destination got the body %q; want %q", body, c.wantBody)
}
}
}
// A refusal says why in its body, not only in a header: a 403 from the vetting
// and a 502 from a dial that found no answer, on both request forms, and the
// coder-run proxy's refusals too.
func TestEgressRefusalsCarryABody(t *testing.T) {
failDial := func(ctx context.Context, network, addr string) (net.Conn, error) {
return nil, fmt.Errorf("no route")
}
okDial := func(ctx context.Context, network, addr string) (net.Conn, error) {
return nil, fmt.Errorf("must not dial %s", addr)
}
check := func(name string, resp *http.Response, err error, want int) {
t.Helper()
if err != nil {
t.Errorf("%s: no answer: %v", name, err)
return
}
body, _ := io.ReadAll(resp.Body)
if resp.StatusCode != want || len(body) < 10 || int64(len(body)) != resp.ContentLength {
t.Errorf("%s: answered %d with body %q (Content-Length %d); want %d with a short body saying why",
name, resp.StatusCode, body, resp.ContentLength, want)
}
}
for _, c := range []struct {
name string
dial egressDialer
req string
want int
}{
{"plain HTTP to this machine", okDial, "GET http://127.0.0.1:9/ HTTP/1.1\r\n\r\n", 403},
{"CONNECT to this machine", okDial, "CONNECT 127.0.0.1:9 HTTP/1.1\r\n\r\n", 403},
{"plain HTTP, no answer", failDial, "GET http://198.51.100.43/ HTTP/1.1\r\n\r\n", 502},
{"CONNECT, no answer", failDial, "CONNECT 198.51.100.43:443 HTTP/1.1\r\n\r\n", 502},
} {
cli, cleanup := dialHandler(t, c.dial)
_, _ = io.WriteString(cli, c.req)
resp, err := http.ReadResponse(bufio.NewReader(cli), nil)
check(c.name, resp, err, c.want)
cleanup()
}
oldDial := coderDialOut
t.Cleanup(func() { coderDialOut = oldDial })
coderDialOut = failDial
for _, c := range []struct {
name, req string
want int
}{
{"coder run to a private address", "GET http://10.0.0.1/ HTTP/1.1\r\nHost: 10.0.0.1\r\n\r\n", 403},
{"coder run, no answer", "GET http://93.184.216.34/ HTTP/1.1\r\nHost: 93.184.216.34\r\n\r\n", 502},
} {
boxEnd, runEnd := net.Pipe()
var refused int64
go handleCoderEgressConn(boxEnd, nil, &refused)
_ = runEnd.SetDeadline(time.Now().Add(5 * time.Second))
_, _ = io.WriteString(runEnd, c.req)
resp, err := http.ReadResponse(bufio.NewReader(runEnd), nil)
check(c.name, resp, err, c.want)
_ = runEnd.Close()
}
}
// The coder-run proxy carries one request per connection too: the run's own
// processes are the sender there.
func TestCoderEgressCarriesOneRequestPerConnection(t *testing.T) {
o := newRecordingOrigin(t)
oldDial := coderDialOut
t.Cleanup(func() { coderDialOut = oldDial })
coderDialOut = func(ctx context.Context, network, addr string) (net.Conn, error) {
return (&net.Dialer{}).DialContext(ctx, network, o.addr())
}
boxEnd, runEnd := net.Pipe()
var refused int64
go handleCoderEgressConn(boxEnd, nil, &refused)
_ = runEnd.SetDeadline(time.Now().Add(8 * time.Second))
_, _ = io.WriteString(runEnd, "GET http://93.184.216.34/first HTTP/1.1\r\nHost: 93.184.216.34\r\n\r\n")
br := bufio.NewReader(runEnd)
resp, err := http.ReadResponse(br, nil)
if err != nil {
t.Fatalf("the first request got no answer: %v", err)
}
_, _ = io.ReadAll(resp.Body)
_, _ = io.WriteString(runEnd, "GET http://93.184.216.34/second HTTP/1.1\r\nHost: inside.example.test\r\n\r\n")
got := o.next(t)
_ = runEnd.Close()
if strings.Contains(got, "/second") || strings.Contains(got, "inside.example.test") {
t.Fatalf("a second request on one connection reached the destination unvetted:\n%s", got)
}
if !strings.Contains(got, "\r\nConnection: close\r\n") {
t.Fatalf("the request went out without Connection: close:\n%s", got)
}
}
+49 -14
View File
@@ -14,35 +14,60 @@ import (
const egressWorkerCount = 4
// egressRefusedWait is how long a worker waits before it parks again after a
// job it did not take (malformed, refused) or after its WebSocket closed while
// parked. A job it took parks the worker again at once, so the person's own
// traffic is never slowed by it; a Host that answers every worker with a job
// this machine refuses costs it at most egressWorkerCount dials, handshakes
// and refusals a second, not as many as the Host can send. The serve lane's
// workers wait the same (serveRefusedWait).
var egressRefusedWait = time.Second
// wsConn adapts a worker WebSocket into a net.Conn so the existing handleEgressConn drives
// it unchanged. The host frames the proxied client stream as binary messages: 'D'+payload =
// data, 'E' = end-of-stream. Reads surface the host's bytes; writes go back as 'D' frames;
// Close/CloseWrite send 'E'.
//
// A 'D' frame is STREAMED into the caller's buffer, never read whole: the host picks a
// frame's length, and one frame of any length must cost this machine no more memory
// than the read it is asked for.
type wsConn struct {
ws *websocket.Conn
rbuf []byte
wmu sync.Mutex
ws *websocket.Conn
cur io.Reader // the rest of the 'D' frame being read, or nil
wmu sync.Mutex
}
func (c *wsConn) Read(p []byte) (int, error) {
for len(c.rbuf) == 0 {
mt, data, err := c.ws.ReadMessage()
for {
if c.cur != nil {
n, err := c.cur.Read(p)
if err == io.EOF {
c.cur = nil
err = nil
}
if n > 0 || err != nil {
return n, err
}
continue
}
mt, r, err := c.ws.NextReader()
if err != nil {
return 0, err
}
if mt != websocket.BinaryMessage || len(data) == 0 {
continue
if mt != websocket.BinaryMessage {
continue // NextReader discards what is left of it
}
switch data[0] {
var tag [1]byte
if _, err := io.ReadFull(r, tag[:]); err != nil {
continue // an empty frame
}
switch tag[0] {
case 'D':
c.rbuf = data[1:]
c.cur = r
case 'E':
return 0, io.EOF
}
}
n := copy(p, c.rbuf)
c.rbuf = c.rbuf[n:]
return n, nil
}
func (c *wsConn) Write(p []byte) (int, error) {
@@ -80,19 +105,25 @@ func (c *wsConn) SetWriteDeadline(t time.Time) error { return c.ws.SetWriteDeadl
// bastion's tunnel IP) and serves one egress job per worker, reconnecting after each. Each
// worker authenticates with the bastion's bearer, then blocks until the host dispatches a
// request; handleEgressConn then dials it out the USER's connection (dialOut) and relays.
// A worker reconnects at once after a job it took, and after egressRefusedWait after one
// it refused or a close while parked. The func returned waits until every worker has
// exited, which is once pctx is done and each one's job has ended.
//
// pctx is the pairing's context (run.go): when the anchor revokes a pairing the
// workers must exit, not re-dial a closed tunnel every 3s for the process's life.
func runEgressWorkers(pctx context.Context, self string, wsURL string, tlsCfg *tls.Config,
dialHost func(ctx context.Context) (net.Conn, error), bearer string,
dialOut egressDialer, n int) {
dialOut egressDialer, n int) (wait func()) {
dialer := &websocket.Dialer{
NetDialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { return dialHost(ctx) },
TLSClientConfig: tlsCfg,
HandshakeTimeout: 20 * time.Second,
}
var wg sync.WaitGroup
for i := 0; i < n; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for pctx.Err() == nil {
ctx, cancel := context.WithTimeout(pctx, 25*time.Second)
ws, _, err := dialer.DialContext(ctx, wsURL, http.Header{})
@@ -108,11 +139,15 @@ func runEgressWorkers(pctx context.Context, self string, wsURL string, tlsCfg *t
}
stopKA := armWSKeepalive(ws) // ping/read-deadline: a dead tunnel unblocks the park
unpark := context.AfterFunc(pctx, func() { _ = ws.Close() })
handleEgressConn(&wsConn{ws: ws}, dialOut, self) // blocks until a job, serves it, then returns
took := handleEgressConn(&wsConn{ws: ws}, dialOut, self) // blocks until a job, serves it, then returns
unpark()
stopKA()
_ = ws.Close()
if !took {
sleepCtx(pctx, egressRefusedWait)
}
}
}()
}
return wg.Wait
}
+166
View File
@@ -0,0 +1,166 @@
//go:build !windows
// The egress worker pool (egress_relay.go: runEgressWorkers) against a
// stand-in Host's /bastion-egress: how fast workers park again after a job
// they took and after one they refused. Uses localEgressFixture
// (egress_local_test.go), so it carries the same build tag.
package main
import (
"context"
"io"
"log"
"net"
"net/http"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/gorilla/websocket"
)
// egressHost is a Host's /bastion-egress as the pool meets it: every worker
// that parks is counted and handed the job job(n) (n is its place in line),
// and what the bastion answered is read to its end.
type egressHost struct {
parked atomic.Int32
ok atomic.Int32 // answers that began "HTTP/1.1 200"
}
func startEgressHost(t *testing.T, job func(n int32) string) (*egressHost, string, func(context.Context) (net.Conn, error)) {
t.Helper()
h := &egressHost{}
up := websocket.Upgrader{}
srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
c, err := up.Upgrade(w, r, nil)
if err != nil {
return
}
defer c.Close()
if _, _, err := c.ReadMessage(); err != nil { // BEARER
return
}
n := h.parked.Add(1)
if err := c.WriteMessage(websocket.BinaryMessage, append([]byte{'D'}, job(n)...)); err != nil {
return
}
_ = c.SetReadDeadline(time.Now().Add(5 * time.Second))
var answer strings.Builder
for {
_, m, err := c.ReadMessage()
if err != nil || len(m) == 0 || m[0] == 'E' {
break
}
answer.Write(m[1:])
}
if strings.HasPrefix(answer.String(), "HTTP/1.1 200") {
h.ok.Add(1)
}
}))
srv.Config.ErrorLog = log.New(io.Discard, "", 0)
srv.Start()
t.Cleanup(srv.Close)
addr := srv.Listener.Addr().String()
dial := func(ctx context.Context) (net.Conn, error) {
return (&net.Dialer{}).DialContext(ctx, "tcp", addr)
}
return h, "ws://" + addr + "/bastion-egress", dial
}
// runEgressPool runs egressWorkerCount workers for the test's life, and at its
// end stops them and waits until each has exited: a worker still vetting a job
// would otherwise read the fixture's stubs while the test's cleanup puts them
// back.
func runEgressPool(t *testing.T, wsURL string, dialHost func(context.Context) (net.Conn, error), dialOut egressDialer) {
t.Helper()
ctx, cancel := context.WithCancel(context.Background())
wait := runEgressWorkers(ctx, "home.host.example", wsURL, nil, dialHost, "test-bearer", dialOut, egressWorkerCount)
t.Cleanup(func() {
cancel()
done := make(chan struct{})
go func() { wait(); close(done) }()
select {
case <-done:
case <-time.After(10 * time.Second):
t.Error("the egress workers did not exit 10 s after the pool stopped")
}
})
}
// SECURITY (this machine's resources): a Host that answers every parked egress
// worker with a job this machine refuses (a request for this machine, a line
// that is no request) cannot make it dial, handshake and refuse in a loop as
// fast as the Host answers. Each worker that refused waits before it parks
// again, so the pool costs at most egressWorkerCount dials a second.
//
// MUTATION: in runEgressWorkers, park again at once whatever handleEgressConn
// reports (as the pool did).
// RESULT: red — hundreds of workers park in the 2 s window.
func TestARefusingHostCannotSpinTheEgressWorkers(t *testing.T) {
localEgressFixture(t)
var dialled atomic.Int32
dialOut := func(ctx context.Context, network, addr string) (net.Conn, error) {
dialled.Add(1)
return nil, io.EOF
}
h, wsURL, dialHost := startEgressHost(t, func(n int32) string {
if n%2 == 0 {
return "NOT-A-REQUEST\r\n"
}
return "CONNECT localhost:443 HTTP/1.1\r\nHost: localhost:443\r\n\r\n"
})
runEgressPool(t, wsURL, dialHost, dialOut)
time.Sleep(2 * time.Second)
// egressWorkerCount park at the start, then each worker once a second:
// about 12 in 2 s. A pool that parks again at once makes hundreds.
n := h.parked.Load()
t.Logf("%d workers parked in 2 s", n)
if limit := int32(egressWorkerCount * 4); n > limit {
t.Fatalf("%d egress workers parked in 2 s on a Host that refuses every job (at most %d): the pool re-dials in a loop", n, limit)
}
if n < egressWorkerCount {
t.Fatalf("only %d workers parked: the pool did not run, so the count proves nothing", n)
}
if d := dialled.Load(); d != 0 {
t.Fatalf("a refused job was dialled out %d times", d)
}
}
// The person's own traffic is not slowed: a worker that took a job parks again
// at once, so 40 jobs through 4 workers finish in a fraction of the time one
// wait per job would take (40 / 4 x egressRefusedWait = 10 s).
//
// MUTATION: in runEgressWorkers, wait egressRefusedWait after every job.
// RESULT: red — 40 jobs need about 10 s.
func TestEgressWorkersParkAgainAtOnceAfterAJobTheyTook(t *testing.T) {
localEgressFixture(t)
dialOut := func(ctx context.Context, network, addr string) (net.Conn, error) {
if addr != "198.51.100.9:443" {
t.Errorf("dialled %q, not the address the request named", addr)
}
ours, theirs := net.Pipe()
_ = theirs.Close() // the destination accepts and ends at once
return ours, nil
}
h, wsURL, dialHost := startEgressHost(t, func(int32) string {
return "CONNECT 198.51.100.9:443 HTTP/1.1\r\nHost: 198.51.100.9:443\r\n\r\n"
})
start := time.Now()
runEgressPool(t, wsURL, dialHost, dialOut)
const jobs = 40
deadline := start.Add(10 * time.Second)
for h.ok.Load() < jobs && time.Now().Before(deadline) {
time.Sleep(10 * time.Millisecond)
}
took := time.Since(start)
t.Logf("%d accepted jobs answered in %v", h.ok.Load(), took)
if got := h.ok.Load(); got < jobs {
t.Fatalf("only %d of %d accepted jobs were answered 200 in %v", got, jobs, took)
}
if limit := 3 * egressRefusedWait; took > limit {
t.Fatalf("%d accepted jobs took %v (over %v): a worker that took a job waited before it parked again", jobs, took, limit)
}
}
+51 -12
View File
@@ -252,16 +252,39 @@ func harnessGate(self, dcnDir string, spec harnessSpec) string {
// `BEARER <bearer> lane=session`); a harness worker refuses it so a shell can never hold
// an agent worker for hours.
func handleHarnessConn(ws *websocket.Conn, self, dcnDir string) {
mt, data, err := ws.ReadMessage()
if err != nil || mt != websocket.BinaryMessage || len(data) == 0 {
// The spec is read up to harnessSpecMaxBytes (lane_bounds.go) and refused
// past it; a frame of any other tag is not read at all.
tag, body, tooBig, err := readLaneJob(ws, func(t byte) int64 {
if t == 'H' {
return harnessSpecMaxBytes
}
return 0
})
if err != nil {
return
}
if data[0] == 'S' {
if tag == 'S' {
sendHarness(ws, 'E', []byte("this is a harness worker; sessions go to a worker that said lane=session\n"))
sendHarnessExit(ws, 1)
return
}
if data[0] != 'H' {
if tag != 'H' {
return
}
if tooBig {
// Gated like a spec that does not parse (below): a pairing the caps
// refuse learns nothing more from a frame that was too large.
if why := capsAllow(self, "harness:run"); why != "" {
b, _ := json.Marshal(map[string]any{"ok": false, "error": why})
sendHarness(ws, 'J', b)
sendHarnessExit(ws, 126)
return
}
b, _ := json.Marshal(map[string]any{"ok": false, "error": fmt.Sprintf(
"the harness spec is more than the %d MiB a job on this lane may be; nothing of it was read as a job",
harnessSpecMaxBytes>>20)})
sendHarness(ws, 'J', b)
sendHarnessExit(ws, 1)
return
}
// The gate depends on WHAT is asked: a service-slot job (service.go) is the
@@ -270,7 +293,7 @@ func handleHarnessConn(ws *websocket.Conn, self, dcnDir string) {
// before its parse error is reported, exactly as it always was, so an
// unauthorized pairing learns nothing from a malformed frame.
var spec harnessSpec
perr := json.Unmarshal(data[1:], &spec)
perr := json.Unmarshal(body, &spec)
gateOp := "harness:run"
if perr == nil && spec.Mode == "service" {
gateOp = serviceGateOp(spec.ServiceOp)
@@ -332,12 +355,10 @@ func handleHarnessConn(ws *websocket.Conn, self, dcnDir string) {
go func() { b, c := runHarness(spec); resCh <- harnessRes{b, c} }()
closed := make(chan struct{})
go func() {
for {
if _, _, err := ws.ReadMessage(); err != nil { // host close or drop
close(closed)
return
}
}
// Nothing the host sends during a run is read as anything: each frame
// is dropped as it arrives (lane_bounds.go), never held whole.
_ = drainLane(ws) // returns on a host close or drop
close(closed)
}()
ticker := time.NewTicker(8 * time.Second)
defer ticker.Stop()
@@ -951,7 +972,10 @@ func driveHarnessAgent(rt, dir string, spec harnessSpec, maxMin int) (agentRes m
return nil, fmt.Errorf("harness /run: %v", err)
}
defer resp.Body.Close()
rb, _ := io.ReadAll(resp.Body)
rb, err := readHarnessRunReply(resp.Body)
if err != nil {
return nil, err
}
var res map[string]interface{}
if err := json.Unmarshal(rb, &res); err != nil {
return nil, fmt.Errorf("harness /run bad json (HTTP %d): %s", resp.StatusCode, truncate(string(rb), 200))
@@ -959,6 +983,21 @@ func driveHarnessAgent(rt, dir string, spec harnessSpec, maxMin int) (agentRes m
return res, nil
}
// readHarnessRunReply reads the agent container's answer to /run. The
// container writes it, and it goes back to the host in one lane message, so it
// is read to the most that message carries (uvicornWSMaxSize) and no further:
// a reply that never ends must not grow in this machine's memory.
func readHarnessRunReply(body io.Reader) ([]byte, error) {
rb, err := io.ReadAll(io.LimitReader(body, uvicornWSMaxSize+1))
if err != nil {
return nil, fmt.Errorf("harness /run: reading the reply: %v", err)
}
if len(rb) > uvicornWSMaxSize {
return nil, fmt.Errorf("harness /run: the reply is more than the %d MiB the lane to the host carries", uvicornWSMaxSize>>20)
}
return rb, nil
}
// harnessAgentUID is the user the harness image runs its agent as, the owner
// of its home /home/privatae, where the credential folder is mounted.
const harnessAgentUID = 10005
+4 -4
View File
@@ -1010,15 +1010,15 @@ func TestAnchoredEgressRefusesLoopbackAndPrivateDestinations(t *testing.T) {
if addr, why := egressVettedAddr(ctx, "localhost:80"); why == "" || addr != "" {
t.Fatalf("a refused name yields no address: %q %q", addr, why)
}
// A local box refuses this machine too, and keeps its LAN
// (egress_local_test.go has the rest).
// A local box refuses this machine too, and the network around it
// (egress_local_test.go and egress_network_test.go have the rest).
t.Setenv("XDG_CONFIG_HOME", t.TempDir())
stubOwnAddrs(t, loopbackOnly, localRange())
if why := egressInternalRefusal(ctx, "127.0.0.1:11434"); why == "" {
t.Fatal("a local box must refuse its own ollama on the egress lane")
}
if addr, why := egressVettedAddr(ctx, "192.168.1.1:80"); why != "" || addr != "192.168.1.1:80" {
t.Fatalf("a local box keeps its LAN: %q %q", addr, why)
if addr, why := egressVettedAddr(ctx, "192.168.1.1:80"); why == "" || addr != "" {
t.Fatalf("a local box refuses its LAN on the egress lane: %q %q", addr, why)
}
}
+29 -4
View File
@@ -133,18 +133,39 @@ func inferenceFilterNeeded(self string, lane inferenceLane, scope modelScope) bo
//
// With nothing to enforce it is the original raw splice, byte for byte.
func proxyInference(in net.Conn, upstream string, lane inferenceLane, self string) {
proxyInferenceVia(in, dialUpstream(upstream), lane, self, spliceConnVia)
}
// upstreamDial opens the connection to the service behind a forward.
type upstreamDial func() (net.Conn, error)
// dialUpstream is the forwards' dial: TCP to the address, 5 s.
func dialUpstream(upstream string) upstreamDial {
return func() (net.Conn, error) { return net.DialTimeout("tcp", upstream, 5*time.Second) }
}
// proxyInferenceVia is proxyInference with the dial, and the pass-through for
// a connection with nothing to enforce, in the caller's hands. The serve lane
// (servelane.go) uses it so its connections take the same decision and the
// same filter as the tunnel's, while the dial goes to the address it vetted
// and is watched by its idle bound.
func proxyInferenceVia(in net.Conn, dial upstreamDial, lane inferenceLane, self string, pass func(net.Conn, upstreamDial)) {
if inferenceFilterNeeded(self, lane, pairingModelScope(self)) {
serveInferenceFiltered(in, upstream, lane, self)
serveInferenceFilteredVia(in, dial, lane, self)
return
}
spliceConn(in, upstream)
pass(in, dial)
}
// spliceConn is what the forwards did before this file existed, unchanged and
// in one place now that three call sites share it.
func spliceConn(in net.Conn, upstream string) {
spliceConnVia(in, dialUpstream(upstream))
}
func spliceConnVia(in net.Conn, dial upstreamDial) {
defer in.Close()
out, err := net.DialTimeout("tcp", upstream, 5*time.Second)
out, err := dial()
if err != nil {
return
}
@@ -163,8 +184,12 @@ func spliceConn(in net.Conn, upstream string) {
// grant on every request below, so there is nowhere for a caller — including a
// test — to hand it one the grant does not say.
func serveInferenceFiltered(in net.Conn, upstream string, lane inferenceLane, self string) {
serveInferenceFilteredVia(in, dialUpstream(upstream), lane, self)
}
func serveInferenceFilteredVia(in net.Conn, dial upstreamDial, lane inferenceLane, self string) {
defer in.Close()
out, err := net.DialTimeout("tcp", upstream, 5*time.Second)
out, err := dial()
if err != nil {
writeFilterError(in, nil, http.StatusBadGateway, "the service behind this lane is not answering")
return
+122
View File
@@ -0,0 +1,122 @@
package main
// What a paired host sends down a worker lane is read at a size this machine
// chose, never at one the host chose: the host picks a frame's length, and a
// frame of any length must not be held whole in this machine's memory. The
// egress lane (egress_relay.go) and the context socket (contextsock.go) follow
// the same rule.
//
// - A job's first frame (the H spec on a harness worker, the S spec on a
// session worker) is read up to its lane's bound and refused past it, by
// name, before any of it is parsed.
// - What a lane carries after that is streamed: terminal input into the pty
// (session.go), the replica's bytes into its socket (mirror.go). Frames that
// nothing reads (a harness run and a root command, which read nothing after
// their spec) are dropped as they arrive.
import (
"errors"
"fmt"
"io"
"net/http"
"github.com/gorilla/websocket"
)
const (
// harnessSpecMaxBytes bounds the H frame. Its bulk is the worktree tar
// (worktree_b64). The largest tree this lane hands back is
// maxReturnedTreeBytes (9 MiB, so 12 MiB in base64) inside one message of
// uvicornWSMaxSize (16 MiB), the most the host takes in (harness.go). A host
// that sends that tree into its next run needs those 12 MiB, plus the
// prompt and the run's credentials. Twice the lane's message size holds
// that with 20 MiB to spare.
harnessSpecMaxBytes = 2 * uvicornWSMaxSize
// sessionSpecMaxBytes bounds the S frame: a signed grant (op ids and the
// person's own words), ids, a public key, and for session:root:exec the
// command, itself held to rootExecCommandMax (64 KiB), which is at most
// 384 KiB even when JSON escapes every byte as \u00XX. 1 MiB is the bound
// Go's own HTTP server puts on a request head.
sessionSpecMaxBytes = http.DefaultMaxHeaderBytes
// sessionInputMaxBytes bounds one frame on an open terminal. The host relays
// what the person's browser sent it, one message each, and its own server
// takes a message up to uvicornWSMaxSize; the tag is one byte more. Input is
// streamed into the pty, so this caps a frame's length, not memory.
sessionInputMaxBytes = uvicornWSMaxSize + 1
// sessionResizeMaxBytes bounds the body of an R frame, {"cols":N,"rows":N}.
sessionResizeMaxBytes = 4 << 10
// heartbeatReplyMaxBytes bounds the host's answer to a heartbeat. It
// carries directives and the lists of grants this box should hold, each
// entry an id and a signature of about 130 bytes, so 4 MiB is tens of
// thousands of grants. Nothing in it is bulk: an update names a version,
// and the binary comes from the release manifest, not from this reply.
heartbeatReplyMaxBytes = 4 << 20
)
// errLaneNotJob is a first frame that is not a binary frame with a tag.
var errLaneNotJob = errors.New("the first frame is not a job")
// readLaneJob reads a worker's first frame: its tag, and its body up to
// maxFor(tag) bytes. A body past its bound is never kept: tooBig reports it,
// and body is nil. A tag the caller does not take should get a bound of 0;
// its body is then not read at all.
func readLaneJob(ws *websocket.Conn, maxFor func(tag byte) int64) (tag byte, body []byte, tooBig bool, err error) {
mt, r, err := ws.NextReader()
if err != nil {
return 0, nil, false, err
}
if mt != websocket.BinaryMessage {
return 0, nil, false, errLaneNotJob
}
var t [1]byte
if _, err := io.ReadFull(r, t[:]); err != nil {
return 0, nil, false, errLaneNotJob // an empty frame
}
max := maxFor(t[0])
if max <= 0 {
return t[0], nil, false, nil
}
body, err = io.ReadAll(io.LimitReader(r, max+1))
if err != nil {
return t[0], nil, false, err
}
if int64(len(body)) > max {
return t[0], nil, true, nil
}
return t[0], body, false, nil
}
// drainLane reads and drops every frame until the socket fails, holding none
// of them, and returns that failure. A lane that reads nothing after its job
// still has to read, to answer pings and to notice the host going away.
func drainLane(ws *websocket.Conn) error {
for {
_, r, err := ws.NextReader()
if err != nil {
return err
}
if _, err := io.Copy(io.Discard, r); err != nil {
return err
}
}
}
// readHeartbeatReply reads the host's answer to a heartbeat up to
// heartbeatReplyMaxBytes. ok is false for a reply past the bound or one that
// failed to arrive: such a beat is treated as unanswered, so none of its
// directives runs and it does not count as the host answering.
func readHeartbeatReply(resp *http.Response) (body []byte, ok bool) {
body, err := io.ReadAll(io.LimitReader(resp.Body, heartbeatReplyMaxBytes+1))
if err != nil {
return nil, false
}
if len(body) > heartbeatReplyMaxBytes {
fmt.Printf("heartbeat: the host's reply is more than %d MiB; ignored\n", heartbeatReplyMaxBytes>>20)
return nil, false
}
return body, true
}
+126
View File
@@ -0,0 +1,126 @@
package main
// The helpers lane_bounds.go and harness.go added for the reads that were
// inline before (the heartbeat reply in run.go, the agent's /run reply in
// driveHarnessAgent, the drains in harness.go and rootexec.go). They have no
// counterpart on main to run against.
import (
"bytes"
"io"
"net/http"
"net/http/httptest"
"runtime"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/gorilla/websocket"
)
// boxLaneHost stands up the HOST side of one worker socket (host runs on the
// upgraded server end) and returns the box side, dialled.
func boxLaneHost(t *testing.T, host func(c *websocket.Conn)) *websocket.Conn {
t.Helper()
up := websocket.Upgrader{}
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
c, err := up.Upgrade(w, r, nil)
if err != nil {
return
}
defer c.Close()
host(c)
}))
t.Cleanup(srv.Close)
ws, _, err := websocket.DefaultDialer.Dial("ws"+strings.TrimPrefix(srv.URL, "http"), nil)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = ws.Close() })
return ws
}
// endless is a body that never ends; served counts what was taken from it.
type endless struct{ served *atomic.Int64 }
func (e endless) Read(p []byte) (int, error) {
for i := range p {
p[i] = 'z'
}
e.served.Add(int64(len(p)))
return len(p), nil
}
func TestHeartbeatReplyIsReadOnlyToItsBound(t *testing.T) {
var served atomic.Int64
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(200)
_, _ = io.Copy(w, io.LimitReader(endless{&served}, 64<<20))
}))
defer srv.Close()
resp, err := http.Get(srv.URL)
if err != nil {
t.Fatal(err)
}
body, ok := readHeartbeatReply(resp)
resp.Body.Close()
if ok || body != nil {
t.Fatalf("a 64 MiB heartbeat reply was taken (%d bytes)", len(body))
}
small := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte(`{"ok":true,"dcn_grants":{"stores":[]}}`))
}))
defer small.Close()
resp, err = http.Get(small.URL)
if err != nil {
t.Fatal(err)
}
body, ok = readHeartbeatReply(resp)
resp.Body.Close()
if !ok || !strings.Contains(string(body), "dcn_grants") {
t.Fatalf("an ordinary reply was not taken: ok=%v %q", ok, body)
}
}
func TestHarnessRunReplyIsReadOnlyToItsBound(t *testing.T) {
var served atomic.Int64
if _, err := readHarnessRunReply(endless{&served}); err == nil || !strings.Contains(err.Error(), "more than the 16 MiB") {
t.Fatalf("an endless /run reply: %v", err)
}
if n := served.Load(); n > 17<<20 {
t.Fatalf("an endless /run reply was read to %d MiB", n>>20)
}
big := append([]byte(`{"ok":true,"pad":"`), bytes.Repeat([]byte{'a'}, 15<<20)...)
big = append(big, `"}`...)
if rb, err := readHarnessRunReply(bytes.NewReader(big)); err != nil || len(rb) != len(big) {
t.Fatalf("a 15 MiB reply the lane carries: %d bytes, %v", len(rb), err)
}
}
// A drained lane holds no frame whole, and ends when the host goes away.
func TestDrainLaneHoldsNoFrame(t *testing.T) {
const frame = 64 << 20
payload := bytes.Repeat([]byte{'d'}, frame)
start := make(chan struct{})
box := boxLaneHost(t, func(c *websocket.Conn) {
<-start
_ = c.WriteMessage(websocket.BinaryMessage, payload)
})
runtime.GC()
var before, after runtime.MemStats
runtime.ReadMemStats(&before)
close(start)
done := make(chan error, 1)
go func() { done <- drainLane(box) }()
select {
case <-done:
case <-time.After(20 * time.Second):
t.Fatal("drainLane did not return after the host went away")
}
runtime.ReadMemStats(&after)
if grew := after.TotalAlloc - before.TotalAlloc; grew > 8<<20 {
t.Fatalf("draining a %d MiB frame allocated %d MiB", frame>>20, grew>>20)
}
}
+481
View File
@@ -0,0 +1,481 @@
//go:build !windows
package main
// What a paired host sends down the harness, session, root-exec and mirror
// lanes, and what a coder run sends its egress proxy, is read at the size this
// machine chose (lane_bounds.go). Every test here drives only code that main
// already has, so each one also runs against main, where it fails.
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"runtime"
"strings"
"sync"
"sync/atomic"
"syscall"
"testing"
"time"
"github.com/gorilla/websocket"
)
// hostSendsAJob is a host that writes one job frame while it reads the box's
// replies, and reports whether its write completed and what came back.
type hostResult struct {
werr error
frames map[byte][]byte
}
func hostSendsAJob(frame []byte, out chan<- hostResult) func(c *websocket.Conn) {
return hostSendsAJobAs(func(c *websocket.Conn) error {
return c.WriteMessage(websocket.BinaryMessage, frame)
}, out)
}
// writeInPieces writes msg as ONE message carried by many small frames: the
// writer flushes a continuation frame each time its 4 KiB buffer fills
// (gorilla's default write buffer), so a bound that counted a frame instead
// of the message would let all of it through.
func writeInPieces(msg []byte) func(c *websocket.Conn) error {
return func(c *websocket.Conn) error {
w, err := c.NextWriter(websocket.BinaryMessage)
if err != nil {
return err
}
for off := 0; off < len(msg); off += 1 << 10 {
if _, err := w.Write(msg[off:min(off+1<<10, len(msg))]); err != nil {
return err
}
}
return w.Close()
}
}
func hostSendsAJobAs(write func(c *websocket.Conn) error, out chan<- hostResult) func(c *websocket.Conn) {
return func(c *websocket.Conn) {
got := map[byte][]byte{}
readDone := make(chan struct{})
go func() {
defer close(readDone)
_ = c.SetReadDeadline(time.Now().Add(20 * time.Second))
for {
_, d, err := c.ReadMessage()
if err != nil || len(d) == 0 {
return
}
got[d[0]] = append(got[d[0]], d[1:]...)
if d[0] == 'X' {
return
}
}
}()
werr := write(c)
<-readDone
out <- hostResult{werr, got}
}
}
// jsonFrame is tag + a JSON object whose field `pad` carries n bytes of 'a'.
func jsonFrame(tag byte, head string, n int) []byte {
b := make([]byte, 0, len(head)+n+16)
b = append(b, tag)
b = append(b, head...)
b = append(b, `,"pad":"`...)
b = append(b, bytes.Repeat([]byte{'a'}, n)...)
b = append(b, `"}`...)
return b
}
// A harness spec far past the bound is not read to its end: the box answers
// that it is too large, names the bound, and closes, so the host's write of
// the rest fails. On main the box read all of it into memory, then ran it.
func TestHarnessWorkerStopsReadingASpecPastItsBound(t *testing.T) {
const frame = 96 << 20
out := make(chan hostResult, 1)
box := boxLaneHost(t, hostSendsAJob(jsonFrame('H', `{"harness_id":"big","mode":"echo","prompt":"ping"`, frame), out))
handleHarnessConn(box, "", "")
_ = box.Close()
r := <-out
if r.werr == nil {
t.Fatalf("the box read the whole %d MiB spec (it replied %.200q); it must stop at its bound", frame>>20, r.frames['J'])
}
if !strings.Contains(string(r.frames['J']), "more than the 32 MiB") || string(r.frames['X']) != "1" {
t.Fatalf("want a refusal naming the 32 MiB bound and exit 1: J=%.200q X=%q", r.frames['J'], r.frames['X'])
}
}
// The bound counts the whole message, not one frame of it: a spec past the
// bound, sent as one message in thousands of 4 KiB frames, is refused the same
// way and not read to its end. On main the box read all of it.
func TestHarnessWorkerBoundCountsEveryFrameOfTheSpec(t *testing.T) {
const frame = 40 << 20
out := make(chan hostResult, 1)
spec := jsonFrame('H', `{"harness_id":"pieces","mode":"echo","prompt":"ping"`, frame)
box := boxLaneHost(t, hostSendsAJobAs(writeInPieces(spec), out))
handleHarnessConn(box, "", "")
_ = box.Close()
r := <-out
if r.werr == nil {
t.Fatalf("the box read the whole %d MiB spec sent in small frames (it replied %.200q); it must stop at its bound", frame>>20, r.frames['J'])
}
if !strings.Contains(string(r.frames['J']), "more than the 32 MiB") || string(r.frames['X']) != "1" {
t.Fatalf("want a refusal naming the 32 MiB bound and exit 1: J=%.200q X=%q", r.frames['J'], r.frames['X'])
}
}
// The same for a session spec sent in small frames.
func TestSessionWorkerBoundCountsEveryFrameOfTheSpec(t *testing.T) {
env, _, _ := sessionTestEnv(t, []string{sessionOpOpen}, nil)
withSink(t)
const frame = 64 << 20
out := make(chan hostResult, 1)
spec := jsonFrame('S', `{"op":"session:open","session_id":"sess-pieces"`, frame)
box := boxLaneHost(t, hostSendsAJobAs(writeInPieces(spec), out))
handleSessionWorkerConn(box, "", env)
_ = box.Close()
r := <-out
if r.werr == nil {
t.Fatalf("the box read the whole %d MiB session spec sent in small frames (E=%q X=%q); it must stop at its bound",
frame>>20, r.frames['E'], r.frames['X'])
}
if !strings.Contains(string(r.frames['E']), "more than the 1024 KiB") || string(r.frames['X']) != "1" {
t.Fatalf("want a refusal naming the bound and exit 1: E=%q X=%q", r.frames['E'], r.frames['X'])
}
}
// A spec the lane can carry still runs: a tree as large as the most this lane
// hands back (12 MiB in base64), with room on top, past the 16 MiB a host
// takes in.
func TestHarnessWorkerTakesALargeSpecTheLaneCarries(t *testing.T) {
out := make(chan hostResult, 1)
box := boxLaneHost(t, hostSendsAJob(jsonFrame('H', `{"harness_id":"tree","mode":"echo","prompt":"ping","worktree_b64":"dGFy"`, 20<<20), out))
handleHarnessConn(box, "", "")
r := <-out
if r.werr != nil {
t.Fatalf("the host could not send a 20 MiB spec: %v", r.werr)
}
var res map[string]any
if err := json.Unmarshal(r.frames['J'], &res); err != nil || res["ok"] != true || res["echo"] != "ping" {
t.Fatalf("a 20 MiB spec did not run: J=%.300q X=%q", r.frames['J'], r.frames['X'])
}
}
// The same on a session worker: an S frame far past its 1 MiB bound is
// refused by name and not read to its end.
func TestSessionWorkerStopsReadingASpecPastItsBound(t *testing.T) {
env, _, _ := sessionTestEnv(t, []string{sessionOpOpen}, nil)
withSink(t) // a box that wrongly took the spec opens no real shell
const frame = 64 << 20
out := make(chan hostResult, 1)
box := boxLaneHost(t, hostSendsAJob(jsonFrame('S', `{"op":"session:open","session_id":"sess-big"`, frame), out))
handleSessionWorkerConn(box, "", env)
_ = box.Close()
r := <-out
if r.werr == nil {
t.Fatalf("the box read the whole %d MiB session spec (E=%q X=%q); it must stop at its bound",
frame>>20, r.frames['E'], r.frames['X'])
}
if !strings.Contains(string(r.frames['E']), "more than the 1024 KiB") || string(r.frames['X']) != "1" {
t.Fatalf("want a refusal naming the bound and exit 1: E=%q X=%q", r.frames['E'], r.frames['X'])
}
}
// A session spec near its bound (a grant, and room the size of a root command
// escaped six times over) still opens a terminal.
func TestSessionWorkerOpensASpecNearItsBound(t *testing.T) {
env, terms, sig := sessionTestEnv(t, []string{sessionOpOpen}, nil)
withFake(t)
// Here the test is the host: it takes the server end, and the box runs on
// the dialled end.
hostEnd := make(chan *websocket.Conn, 1)
hold := make(chan struct{})
t.Cleanup(func() { close(hold) })
boxEnd := boxLaneHost(t, func(c *websocket.Conn) { hostEnd <- c; <-hold })
b := <-hostEnd
head := fmt.Sprintf(`{"op":"session:open","session_id":"sess-near","grant_terms_b64":%q,"grant_sig_b64":%q`, terms, sig)
done := make(chan struct{})
go func() { defer close(done); handleSessionWorkerConn(boxEnd, "", env) }()
if err := b.WriteMessage(websocket.BinaryMessage, jsonFrame('S', head, 900<<10)); err != nil {
t.Fatal(err)
}
if err := b.WriteMessage(websocket.BinaryMessage, []byte("Ihello\n")); err != nil {
t.Fatal(err)
}
got := readUntil(t, b, 'O', 5*time.Second)
if string(got['O']) != "hello\n" {
t.Fatalf("a 900 KiB spec did not open a terminal: %q", got)
}
_ = b.Close()
<-done
}
// sinkProc is a terminal that counts what is typed into it and prints nothing.
type sinkProc struct {
n atomic.Int64
killed chan struct{}
once sync.Once
}
func (p *sinkProc) Read(b []byte) (int, error) { <-p.killed; return 0, io.EOF }
func (p *sinkProc) Write(b []byte) (int, error) { p.n.Add(int64(len(b))); return len(b), nil }
func (p *sinkProc) Resize(c, r int) error { return nil }
func (p *sinkProc) Wait() int { <-p.killed; return 0 }
func (p *sinkProc) Kill() { p.once.Do(func() { close(p.killed) }) }
func (p *sinkProc) User() string { return "sink" }
func (p *sinkProc) Shell() string { return "/bin/sink" }
func withSink(t *testing.T) *sinkProc {
t.Helper()
sp := &sinkProc{killed: make(chan struct{})}
old := sessionSpawn
sessionSpawn = func(op, shell string, cols, rows int) (sessionProc, error) { return sp, nil }
t.Cleanup(func() { sessionSpawn = old; sp.Kill() })
return sp
}
func waitFor(d time.Duration, cond func() bool) bool {
end := time.Now().Add(d)
for time.Now().Before(end) {
if cond() {
return true
}
time.Sleep(5 * time.Millisecond)
}
return cond()
}
// Terminal input is streamed into the pty: a 16 MiB paste, the most a host
// relays in one frame, all arrives, and the box never holds it whole.
func TestSessionInputIsStreamedIntoTheTerminal(t *testing.T) {
env, terms, sig := sessionTestEnv(t, []string{sessionOpOpen}, nil)
sp := withSink(t)
ws := sessionDial(t, env)
sendSpec(t, ws, sessionSpec{Op: sessionOpOpen, GrantB64: terms, SigB64: sig, SessionID: "sess-paste"})
const paste = 16 << 20
frame := append([]byte{'I'}, bytes.Repeat([]byte{'x'}, paste)...)
runtime.GC()
var before, after runtime.MemStats
runtime.ReadMemStats(&before)
if err := ws.WriteMessage(websocket.BinaryMessage, frame); err != nil {
t.Fatal(err)
}
if !waitFor(20*time.Second, func() bool { return sp.n.Load() >= paste }) {
t.Fatalf("the terminal got %d of %d bytes", sp.n.Load(), paste)
}
runtime.ReadMemStats(&after)
if grew := after.TotalAlloc - before.TotalAlloc; grew > 8<<20 {
t.Fatalf("typing a %d MiB paste allocated %d MiB: the frame was held whole", paste>>20, grew>>20)
}
}
// One frame of input past the most a host relays ends the session, and the
// terminal never receives all of it.
func TestSessionInputFramePastItsBoundEndsTheSession(t *testing.T) {
env, terms, sig := sessionTestEnv(t, []string{sessionOpOpen}, nil)
sp := withSink(t)
ws := sessionDial(t, env)
sendSpec(t, ws, sessionSpec{Op: sessionOpOpen, GrantB64: terms, SigB64: sig, SessionID: "sess-huge"})
const frame = 17 << 20
go func() {
_ = ws.WriteMessage(websocket.BinaryMessage, append([]byte{'I'}, bytes.Repeat([]byte{'x'}, frame)...))
}()
ended := func() bool {
select {
case <-sp.killed:
return true
default:
return false
}
}
if !waitFor(20*time.Second, ended) {
t.Fatalf("a %d MiB input frame did not end the session; the terminal got %d bytes", frame>>20, sp.n.Load())
}
if n := sp.n.Load(); n >= frame {
t.Fatalf("the terminal received all %d bytes of a frame past the bound", n)
}
}
// The mirror forwarder streams a host frame into the replica's socket: reading
// 4 KiB of a 64 MiB frame does not need the frame held whole.
func TestMirrorForwarderStreamsAHostFrame(t *testing.T) {
const frame = 64 << 20
payload := bytes.Repeat([]byte{'m'}, frame)
start := make(chan struct{})
up := websocket.Upgrader{}
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
c, err := up.Upgrade(w, r, nil)
if err != nil {
return
}
defer c.Close()
_, _, _ = c.ReadMessage() // the BEARER hello
<-start
_ = c.WriteMessage(websocket.BinaryMessage, payload)
time.Sleep(3 * time.Second)
}))
defer srv.Close()
addr := srv.Listener.Addr().String()
dialer := &websocket.Dialer{NetDialContext: func(ctx context.Context, n, a string) (net.Conn, error) {
return (&net.Dialer{}).DialContext(ctx, "tcp", addr)
}}
boxEnd, replica := net.Pipe()
defer replica.Close()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
go pipeMirrorConn(ctx, boxEnd, dialer, "ws://"+addr+"/", "b")
buf := make([]byte, 4096)
runtime.GC()
var before, after runtime.MemStats
runtime.ReadMemStats(&before)
close(start)
_ = replica.SetReadDeadline(time.Now().Add(20 * time.Second))
if _, err := io.ReadFull(replica, buf); err != nil || buf[0] != 'm' {
t.Fatalf("the replica read: %v", err)
}
runtime.ReadMemStats(&after)
if grew := after.TotalAlloc - before.TotalAlloc; grew > 8<<20 {
t.Fatalf("passing 4 KiB of a %d MiB host frame to the replica allocated %d MiB: the frame was held whole",
frame>>20, grew>>20)
}
}
// The forwarder carries at most eight connections at once. It listens on a
// fixed port (5433), so the test needs that port free: run it in its own
// network namespace on a machine where something already holds it.
func TestMirrorForwarderCapsConnectionsInFlight(t *testing.T) {
probe, err := net.Listen("tcp", "127.0.0.1:5433")
if err != nil {
t.Skipf("127.0.0.1:5433 is in use here (%v); run this test in its own network namespace", err)
}
_ = probe.Close()
var live, most atomic.Int64
release := make(chan struct{})
up := websocket.Upgrader{}
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
c, err := up.Upgrade(w, r, nil)
if err != nil {
return
}
defer c.Close()
n := live.Add(1)
for {
m := most.Load()
if n <= m || most.CompareAndSwap(m, n) {
break
}
}
<-release
live.Add(-1)
}))
defer srv.Close()
defer close(release)
addr := srv.Listener.Addr().String()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
go serveMirrorForward(ctx, "127.0.0.1", "127.0.0.0/8", "ws://"+addr+"/", nil,
func(ctx context.Context) (net.Conn, error) { return (&net.Dialer{}).DialContext(ctx, "tcp", addr) }, "b")
var conns []net.Conn
defer func() {
for _, c := range conns {
_ = c.Close()
}
}()
if !waitFor(5*time.Second, func() bool {
c, err := net.Dial("tcp", "127.0.0.1:5433")
if err == nil {
conns = append(conns, c)
}
return err == nil
}) {
t.Fatal("the forwarder never listened")
}
for i := 1; i < 12; i++ {
c, err := net.Dial("tcp", "127.0.0.1:5433")
if err != nil {
t.Fatal(err)
}
conns = append(conns, c)
}
waitFor(3*time.Second, func() bool { return most.Load() >= 12 })
if m := most.Load(); m != 8 {
t.Fatalf("12 local connections made %d WebSockets to the host at once; want 8", m)
}
}
// A coder run's egress proxy reads a request head under a bound: a request
// line that never ends is not read to its end.
func TestCoderEgressHeadIsBounded(t *testing.T) {
boxEnd, runEnd := net.Pipe()
var refused int64
done := make(chan struct{})
go func() { defer close(done); handleCoderEgressConn(boxEnd, nil, &refused) }()
chunk := bytes.Repeat([]byte{'a'}, 64<<10)
_, _ = runEnd.Write([]byte("GET http://example.invalid/"))
sent := 0
var werr error
for sent < 8<<20 {
_ = runEnd.SetWriteDeadline(time.Now().Add(5 * time.Second))
var n int
n, werr = runEnd.Write(chunk)
sent += n
if werr != nil {
break
}
}
_ = runEnd.Close()
<-done
if werr == nil {
t.Fatalf("the proxy read %d MiB of a request line with no end; it must stop at a bound", sent>>20)
}
}
// A coder run's proxy carries at most 1024 connections at once; the next is
// answered 503 at once, never left waiting.
func TestCoderEgressCapsConnectionsInFlight(t *testing.T) {
dir := t.TempDir()
run := "lane-bounds-test"
if err := coderListenEgress(run, dir, os.Getuid()); err != nil {
t.Fatal(err)
}
defer coderCloseEgress(run)
sock := filepath.Join(dir, coderEgressSock)
var conns []net.Conn
defer func() {
for _, c := range conns {
_ = c.Close()
}
}()
for i := 0; i < 1024; i++ {
c, err := net.Dial("unix", sock)
if err != nil {
if errors.Is(err, syscall.EMFILE) || errors.Is(err, syscall.ENFILE) {
t.Skipf("this process may not open %d sockets: %v", 2*1024, err)
}
t.Fatal(err)
}
conns = append(conns, c)
}
// Every one of them is idle in its head, so the proxy holds them all.
time.Sleep(200 * time.Millisecond)
extra, err := net.Dial("unix", sock)
if err != nil {
t.Fatal(err)
}
conns = append(conns, extra)
_ = extra.SetReadDeadline(time.Now().Add(3 * time.Second))
line, _ := io.ReadAll(io.LimitReader(extra, 256))
if !strings.HasPrefix(string(line), "HTTP/1.1 503") {
t.Fatalf("connection 1025 of one run got %q; want a 503 at once", line)
}
}
+197
View File
@@ -0,0 +1,197 @@
package main
// What the host sends down a lane is read at the size the bastion asks for,
// never at the size the host chooses: one frame of any length must not be
// held whole in this machine's memory, and an egress request head that never
// ends must not be either.
import (
"bytes"
"context"
"errors"
"io"
"net"
"net/http"
"net/http/httptest"
"runtime"
"strconv"
"strings"
"testing"
"time"
"github.com/gorilla/websocket"
)
func TestEgressLaneDoesNotHoldAWholeHostFrame(t *testing.T) {
const frame = 64 << 20
payload := make([]byte, frame+1)
payload[0] = 'D'
for i := 1; i < len(payload); i++ {
payload[i] = 'a'
}
start := make(chan struct{})
up := websocket.Upgrader{}
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
ws, err := up.Upgrade(w, r, nil)
if err != nil {
return
}
defer ws.Close()
<-start
_ = ws.WriteMessage(websocket.BinaryMessage, payload)
time.Sleep(2 * time.Second)
}))
defer srv.Close()
ws, _, err := websocket.DefaultDialer.Dial("ws"+strings.TrimPrefix(srv.URL, "http"), nil)
if err != nil {
t.Fatal(err)
}
defer ws.Close()
c := &wsConn{ws: ws}
buf := make([]byte, 4096)
runtime.GC()
var before, after runtime.MemStats
runtime.ReadMemStats(&before)
close(start)
n, err := c.Read(buf)
runtime.ReadMemStats(&after)
if err != nil || n == 0 || buf[0] != 'a' {
t.Fatalf("read: n=%d err=%v", n, err)
}
if grew := after.TotalAlloc - before.TotalAlloc; grew > 8<<20 {
t.Fatalf("reading 4 KiB of a %d MiB host frame allocated %d MiB: the frame was held whole", frame>>20, grew>>20)
}
}
func TestEgressRequestHeadIsBounded(t *testing.T) {
bastionEnd, hostEnd := net.Pipe()
done := make(chan struct{})
go func() {
defer close(done)
handleEgressConn(bastionEnd, func(ctx context.Context, network, addr string) (net.Conn, error) {
return nil, errors.New("no dial in this test")
}, "test")
}()
chunk := []byte(strings.Repeat("a", 64<<10))
_, _ = hostEnd.Write([]byte("GET http://example.invalid/"))
sent := 0
var werr error
for sent < 8<<20 {
_ = hostEnd.SetWriteDeadline(time.Now().Add(5 * time.Second))
var n int
n, werr = hostEnd.Write(chunk)
sent += n
if werr != nil {
break
}
}
_ = hostEnd.Close()
<-done
if werr == nil {
t.Fatalf("the bastion read %d MiB of a request line with no end; it must stop at a bound", sent>>20)
}
}
// Streaming a frame must not change what the reader sees: every byte of every
// 'D' frame in order, with empty frames, text frames and unknown tags skipped,
// a frame boundary never visible, and 'E' as the end, read through a buffer
// far smaller than the frames.
func TestEgressLaneFramesReadInOrderThroughASmallBuffer(t *testing.T) {
var want []byte
frames := [][]byte{{'D'}, {}, {'X', 'z', 'z'}}
for i, size := range []int{1, 7, 4096, 70000, 3} {
f := []byte{'D'}
for j := 0; j < size; j++ {
f = append(f, byte('a'+(i+j)%26))
}
want = append(want, f[1:]...)
frames = append(frames, f, []byte{'D'})
}
up := websocket.Upgrader{}
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
ws, err := up.Upgrade(w, r, nil)
if err != nil {
return
}
defer ws.Close()
_ = ws.WriteMessage(websocket.TextMessage, []byte("Dnot data"))
for _, f := range frames {
_ = ws.WriteMessage(websocket.BinaryMessage, f)
}
_ = ws.WriteMessage(websocket.BinaryMessage, []byte{'E'})
time.Sleep(time.Second)
}))
defer srv.Close()
ws, _, err := websocket.DefaultDialer.Dial("ws"+strings.TrimPrefix(srv.URL, "http"), nil)
if err != nil {
t.Fatal(err)
}
defer ws.Close()
c := &wsConn{ws: ws}
var got []byte
buf := make([]byte, 13)
for {
n, err := c.Read(buf)
if n == 0 && err == nil {
t.Fatal("Read returned 0, nil")
}
got = append(got, buf[:n]...)
if err == io.EOF {
break
}
if err != nil {
t.Fatalf("read: %v", err)
}
}
if !bytes.Equal(got, want) {
t.Fatalf("read %d bytes, want %d; equal prefix %d", len(got), len(want), commonPrefix(got, want))
}
}
func commonPrefix(a, b []byte) int {
i := 0
for i < len(a) && i < len(b) && a[i] == b[i] {
i++
}
return i
}
// A large upload through the lane still arrives whole: the head bound applies
// to the head alone, and the bytes the reader buffered past it go upstream
// first, in order.
func TestEgressLargeUploadPassesTheHeadBound(t *testing.T) {
bastionEnd, hostEnd := net.Pipe()
upBastion, upFar := net.Pipe()
done := make(chan struct{})
go func() {
defer close(done)
handleEgressConn(bastionEnd, func(ctx context.Context, network, addr string) (net.Conn, error) {
return upBastion, nil
}, "")
}()
body := make([]byte, 3*egressMaxHeadBytes)
for i := range body {
body[i] = byte(i % 251)
}
head := "POST http://198.51.100.20/upload HTTP/1.1\r\nHost: 198.51.100.20\r\nContent-Length: " +
strconv.Itoa(len(body)) + "\r\n\r\n"
// Head and body in ONE write, so the reader holds body bytes when the head ends.
go func() {
_, _ = hostEnd.Write(append([]byte(head), body...))
}()
// The lane rebuilds a plain-HTTP head (egress_http.go): this request comes
// out the same, with `Connection: close` added as its last header.
want := head[:len(head)-len("\r\n")] + "Connection: close\r\n\r\n"
got := make([]byte, len(want)+len(body))
_ = upFar.SetReadDeadline(time.Now().Add(20 * time.Second))
if _, err := io.ReadFull(upFar, got); err != nil {
t.Fatalf("upstream got %v", err)
}
if string(got[:len(want)]) != want || !bytes.Equal(got[len(want):], body) {
t.Fatalf("upstream did not receive the head (%q) and the %d-byte body intact",
got[:len(want)], len(body))
}
_ = hostEnd.Close()
_ = upFar.Close()
<-done
}
+3 -2
View File
@@ -493,7 +493,7 @@ func startLaneHostOn(t *testing.T, raw net.Listener, cert tls.Certificate, clien
}
}
}
for _, p := range []string{"/bastion-harness", "/bastion-mgmt", "/bastion-egress"} {
for _, p := range []string{"/bastion-harness", "/bastion-mgmt", "/bastion-egress", "/bastion-serve"} {
mux.HandleFunc(p, ws)
}
srv := &http.Server{Handler: mux, ErrorLog: log.New(io.Discard, "", 0)}
@@ -622,7 +622,8 @@ func TestTheUnixLaneCarriesTheBearerAndTheSameProtocol(t *testing.T) {
h.mu.Unlock()
if maxPark != harnessWorkerCount || count["/bastion-harness"] < harnessWorkerCount ||
count["/bastion-harness session"] != sessionWorkerCount ||
count["/bastion-mgmt"] != mgmtWorkerCount || count["/bastion-egress"] != egressWorkerCount {
count["/bastion-mgmt"] != mgmtWorkerCount || count["/bastion-egress"] != egressWorkerCount ||
count["/bastion-serve"] != serveWorkerCount {
t.Fatalf("the worker pools over the socket: %v, parked at once %d", count, maxPark)
}
select {
+2 -1
View File
@@ -417,7 +417,8 @@ func TestTheTCPLaneCarriesTheCertTheBearerAndTheSameProtocol(t *testing.T) {
h.mu.Unlock()
if maxPark != harnessWorkerCount || count["/bastion-harness"] < harnessWorkerCount ||
count["/bastion-harness session"] != sessionWorkerCount ||
count["/bastion-mgmt"] != mgmtWorkerCount || count["/bastion-egress"] != egressWorkerCount {
count["/bastion-mgmt"] != mgmtWorkerCount || count["/bastion-egress"] != egressWorkerCount ||
count["/bastion-serve"] != serveWorkerCount {
t.Fatalf("the worker pools over the port: %v, parked at once %d", count, maxPark)
}
for _, n := range sni {
+26 -5
View File
@@ -491,6 +491,12 @@ func ensureMirrorNet(rt, name string) (string, string, error) {
// container and a BASE_BACKUP of the user's memory. Nothing here is defence in
// depth; this is the depth.
//
// mirrorForwardMaxConns bounds the connections the forwarder carries at once.
// The replica holds at most two (`pg_basebackup -X stream` while it seeds; one
// walreceiver after), so four times that leaves room for a diagnostic session
// and a reconnect that overlaps the connection it replaces.
const mirrorForwardMaxConns = 8
// One WebSocket per accepted connection, because `pg_basebackup -X stream`
// holds two replication connections at once and multiplexing them would mean
// inventing a framing layer for no gain.
@@ -507,6 +513,11 @@ func serveMirrorForward(ctx context.Context, gw, subnet, wsURL string,
allowed = n
}
}
// Connections carried at once, across both listeners. Loopback is let in
// for diagnostics, so any local user can connect, and each connection is a
// WebSocket to the host and two pumps here; past the cap one is closed at
// once and its client tries again.
slots := make(chan struct{}, mirrorForwardMaxConns)
for _, addr := range []string{
net.JoinHostPort(gw, strconv.Itoa(mirrorForwardPort)),
net.JoinHostPort("127.0.0.1", strconv.Itoa(mirrorForwardPort)),
@@ -527,7 +538,15 @@ func serveMirrorForward(ctx context.Context, gw, subnet, wsURL string,
_ = c.Close()
continue
}
go pipeMirrorConn(ctx, c, dialer, wsURL, bearer)
select {
case slots <- struct{}{}:
go func() {
defer func() { <-slots }()
pipeMirrorConn(ctx, c, dialer, wsURL, bearer)
}()
default:
_ = c.Close()
}
}
}(ln)
}
@@ -578,15 +597,17 @@ func pipeMirrorConn(ctx context.Context, local net.Conn, dialer *websocket.Diale
done <- struct{}{}
}()
go func() {
// Each host frame is streamed into the replica's socket as it arrives,
// never held whole: the host picks a frame's length (lane_bounds.go).
for {
mt, data, err := ws.ReadMessage()
mt, r, err := ws.NextReader()
if err != nil {
break
}
if mt != websocket.BinaryMessage || len(data) == 0 {
continue
if mt != websocket.BinaryMessage {
continue // NextReader drops what is left of it
}
if _, werr := local.Write(data); werr != nil {
if _, err := io.Copy(local, r); err != nil {
break
}
}
+29 -15
View File
@@ -19,7 +19,9 @@ import (
"net/http"
"net/http/httptest"
"os"
"slices"
"strings"
"sync"
"testing"
"time"
)
@@ -186,17 +188,29 @@ func TestFilterVerdictPolicy(t *testing.T) {
// fakeOllama answers the handful of routes the filter cares about and records
// what actually reached it — the point of most of these assertions is that a
// refused request never arrives at all.
// refused request never arrives at all. Its handler runs on the server's
// goroutines, beside the test's (and a pairing's beat probes it on its own),
// so the record is read through requests(), under the lock.
type fakeOllama struct {
srv *httptest.Server
mu sync.Mutex
seen []string
}
// requests is a copy of what reached the fake so far.
func (f *fakeOllama) requests() []string {
f.mu.Lock()
defer f.mu.Unlock()
return slices.Clone(f.seen)
}
func newFakeOllama(t *testing.T) *fakeOllama {
t.Helper()
f := &fakeOllama{}
f.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
f.mu.Lock()
f.seen = append(f.seen, r.Method+" "+r.URL.Path)
f.mu.Unlock()
switch r.URL.Path {
case "/api/tags":
_, _ = w.Write([]byte(`{"models":[{"name":"qwen4-embed:latest","size":42},` +
@@ -264,8 +278,8 @@ func TestFilteredForwardRefusesAnotherModelAndNeverForwardsIt(t *testing.T) {
if b := bodyOf(t, resp); !strings.Contains(b, "llama3:70b") || !strings.Contains(b, "qwen4-embed") {
t.Fatalf("the refusal must name what was asked for and what is shared: %s", b)
}
if len(up.seen) != 0 {
t.Fatalf("a refused request reached the upstream: %v", up.seen)
if len(up.requests()) != 0 {
t.Fatalf("a refused request reached the upstream: %v", up.requests())
}
resp = roundTrip(t, addr, laneOllama, tMember, "POST", "/api/embed",
@@ -273,8 +287,8 @@ func TestFilteredForwardRefusesAnotherModelAndNeverForwardsIt(t *testing.T) {
if resp.StatusCode != http.StatusOK {
t.Fatalf("the shared model must work: %d", resp.StatusCode)
}
if len(up.seen) != 1 || up.seen[0] != "POST /api/embed" {
t.Fatalf("the allowed request must reach the upstream once: %v", up.seen)
if len(up.requests()) != 1 || up.requests()[0] != "POST /api/embed" {
t.Fatalf("the allowed request must reach the upstream once: %v", up.requests())
}
}
@@ -295,8 +309,8 @@ func TestFilteredForwardRefusesTheStoreMutatingHalf(t *testing.T) {
t.Fatalf("%s: status %d, want 403", p, resp.StatusCode)
}
}
if len(up.seen) != 0 {
t.Fatalf("a store-mutating request reached ollama: %v", up.seen)
if len(up.requests()) != 0 {
t.Fatalf("a store-mutating request reached ollama: %v", up.requests())
}
// ...and using a model is untouched by that refusal.
if resp := roundTrip(t, addr, laneOllama, tMember, "POST", "/api/embed",
@@ -344,8 +358,8 @@ func TestAnchoredBoxRefusesModelChangesThroughTheForwardForTheAnchor(t *testing.
t.Fatalf("%s for the anchor: status %d, want 403", p, resp.StatusCode)
}
}
if len(up.seen) != 0 {
t.Fatalf("a store-mutating request from the anchor reached ollama: %v", up.seen)
if len(up.requests()) != 0 {
t.Fatalf("a store-mutating request from the anchor reached ollama: %v", up.requests())
}
if resp := roundTrip(t, addr, laneOllama, tAnchor, "POST", "/api/embed", `{"model":"anything"}`); resp.StatusCode != http.StatusOK {
t.Fatalf("the anchor still USES models: %d", resp.StatusCode)
@@ -581,8 +595,8 @@ func TestOversizedBodyIsRefusedNotForwarded(t *testing.T) {
if resp.StatusCode != http.StatusForbidden {
t.Fatalf("a request naming no model must be refused: %d", resp.StatusCode)
}
if len(up.seen) != 0 {
t.Fatalf("it reached the upstream anyway: %v", up.seen)
if len(up.requests()) != 0 {
t.Fatalf("it reached the upstream anyway: %v", up.requests())
}
}
@@ -612,8 +626,8 @@ func TestAScopedPairingCannotRenameAModelIntoItsScope(t *testing.T) {
t.Fatalf("%s: the refusal must say why: %s", p, b)
}
}
if len(up.seen) != 0 {
t.Fatalf("a rename reached ollama: %v", up.seen)
if len(up.requests()) != 0 {
t.Fatalf("a rename reached ollama: %v", up.requests())
}
}
@@ -803,8 +817,8 @@ func TestAnEndlessHeaderIsCutOff(t *testing.T) {
t.Fatal("an endless header was read without limit")
}
client.Close()
if len(up.seen) != 0 {
t.Fatalf("it reached the upstream: %v", up.seen)
if len(up.requests()) != 0 {
t.Fatalf("it reached the upstream: %v", up.requests())
}
}
+73 -31
View File
@@ -419,38 +419,53 @@ func placedByThisHost(root, model string) bool {
// ── talking to the model daemon ────────────────────────────────────────────
var (
modelDaemonMu sync.Mutex
modelDaemonUp bool
modelDaemonAt time.Time
modelDaemonMu sync.Mutex
modelDaemonUp bool
modelDaemonAt time.Time
modelDaemonAddr string // the address the cached answer is for
)
// modelDaemonReachable probes the SAME address the rest of this binary uses
// (localOllamaAddr, run.go:714-719) so the box can never advertise management
// of a daemon the forwards do not point at.
func modelDaemonReachable() bool {
func modelDaemonReachable() bool { return modelDaemonReachableAt(localOllamaAddr()) }
// modelDaemonReachableAt is modelDaemonReachable's probe of one address. The
// cached answer is for the address it was asked of, so a pairing whose
// address is the vetted one (modelDaemonTarget) never reads another's answer.
func modelDaemonReachableAt(addr string) bool {
modelDaemonMu.Lock()
defer modelDaemonMu.Unlock()
if !modelDaemonAt.IsZero() && time.Since(modelDaemonAt) < modelDaemonCacheFor {
if !modelDaemonAt.IsZero() && modelDaemonAddr == addr && time.Since(modelDaemonAt) < modelDaemonCacheFor {
return modelDaemonUp
}
cl := &http.Client{Timeout: 3 * time.Second}
resp, err := cl.Get("http://" + localOllamaAddr() + "/api/tags")
resp, err := cl.Get("http://" + addr + "/api/tags")
up := false
if err == nil {
up = resp.StatusCode == http.StatusOK
resp.Body.Close()
}
modelDaemonUp, modelDaemonAt = up, time.Now()
modelDaemonUp, modelDaemonAt, modelDaemonAddr = up, time.Now(), addr
return up
}
// resetModelDaemonCache exists for the tests.
func resetModelDaemonCache() {
modelDaemonMu.Lock()
modelDaemonUp, modelDaemonAt = false, time.Time{}
modelDaemonUp, modelDaemonAt, modelDaemonAddr = false, time.Time{}, ""
modelDaemonMu.Unlock()
}
// modelSelf is the pairing a model:* job came from, or "" when the job was
// driven without one (modelDaemonTarget then keeps OLLAMA_ADDR, as before).
func modelSelf(ws *websocket.Conn) string {
if a := mgmtAccountOf(ws); a != nil {
return a.self
}
return ""
}
// modelSwapCapable answers the heartbeat's question: could this box have a
// model placed on it BY THIS PAIRING. Two conditions, and the caps one is
// first so a pairing outside the family costs no probe — the rule the
@@ -459,13 +474,19 @@ func modelSwapCapable(self string) bool {
if capsAllow(self, modelGateOp) != "" {
return false
}
return modelDaemonReachable()
// A Host with no platform is offered management only of a model server on
// this machine, and a refused one is not probed at all (modelDaemonTarget).
addr, why := modelDaemonTarget(self)
if why != "" {
return false
}
return modelDaemonReachableAt(addr)
}
// modelPostJSON posts one JSON body and returns (status, body). The body is
// read whatever the status is: a streaming ollama reports a FAILED pull inside
// a 200 (measured, see the header), so the caller needs both.
func modelPostJSON(method, path string, payload any, timeout time.Duration) (int, []byte, error) {
func modelPostJSON(addr, method, path string, payload any, timeout time.Duration) (int, []byte, error) {
var rdr io.Reader
if payload != nil {
b, err := json.Marshal(payload)
@@ -474,7 +495,7 @@ func modelPostJSON(method, path string, payload any, timeout time.Duration) (int
}
rdr = bytes.NewReader(b)
}
req, err := http.NewRequest(method, "http://"+localOllamaAddr()+path, rdr)
req, err := http.NewRequest(method, "http://"+addr+path, rdr)
if err != nil {
return 0, nil, err
}
@@ -528,8 +549,8 @@ func modelCosine(a, b []float64) float64 {
return dot / (math.Sqrt(na) * math.Sqrt(nb))
}
func modelEmbedOnce(model, text string) ([]float64, error) {
status, body, err := modelPostJSON(http.MethodPost, "/api/embed",
func modelEmbedOnce(addr, model, text string) ([]float64, error) {
status, body, err := modelPostJSON(addr, http.MethodPost, "/api/embed",
map[string]any{"model": model, "input": []string{text}}, modelProbeTimeout)
if err != nil {
return nil, err
@@ -549,8 +570,8 @@ func modelEmbedOnce(model, text string) ([]float64, error) {
return out.Embeddings[0], nil
}
func modelGenerateOnce(model, prompt string) (string, error) {
status, body, err := modelPostJSON(http.MethodPost, "/api/generate",
func modelGenerateOnce(addr, model, prompt string) (string, error) {
status, body, err := modelPostJSON(addr, http.MethodPost, "/api/generate",
map[string]any{"model": model, "prompt": prompt, "stream": false, "think": false}, modelProbeTimeout)
if err != nil {
return "", err
@@ -573,12 +594,12 @@ func modelGenerateOnce(model, prompt string) (string, error) {
// probeEmbedModel proves the model EMBEDS: the same text twice must come back
// as one answer, and a different text must move it. Returns the measured width
// and "" , or 0 and the reason it is not usable.
func probeEmbedModel(model string) (int, string) {
v1, err := modelEmbedOnce(model, modelProbeTextA)
func probeEmbedModel(addr, model string) (int, string) {
v1, err := modelEmbedOnce(addr, model, modelProbeTextA)
if err != nil {
return 0, "it was downloaded but does not answer as an embedding model: " + err.Error()
}
v2, err := modelEmbedOnce(model, modelProbeTextA)
v2, err := modelEmbedOnce(addr, model, modelProbeTextA)
if err != nil {
return 0, "it answered once and then failed: " + err.Error()
}
@@ -587,7 +608,7 @@ func probeEmbedModel(model string) (int, string) {
return 0, fmt.Sprintf("it gave two different answers to the SAME text (similarity %.4f) — "+
"nothing routed at it would be reproducible", same)
}
v3, err := modelEmbedOnce(model, modelProbeTextB)
v3, err := modelEmbedOnce(addr, model, modelProbeTextB)
if err != nil {
return 0, "the control text failed: " + err.Error()
}
@@ -604,15 +625,15 @@ func probeEmbedModel(model string) (int, string) {
// probeGenerateModel proves the model GENERATES: a non-empty answer, and a
// different prompt must change it (the same constant-answer trap).
func probeGenerateModel(model string) string {
a, err := modelGenerateOnce(model, modelProbePromptA)
func probeGenerateModel(addr, model string) string {
a, err := modelGenerateOnce(addr, model, modelProbePromptA)
if err != nil {
return "it was downloaded but does not answer as a text model: " + err.Error()
}
if strings.TrimSpace(a) == "" {
return "it answered with nothing at all"
}
b, err := modelGenerateOnce(model, modelProbePromptB)
b, err := modelGenerateOnce(addr, model, modelProbePromptB)
if err != nil {
return "it answered once and then failed: " + err.Error()
}
@@ -704,7 +725,12 @@ func modelReply(ws *websocket.Conn, v any) {
// here — the two together are what makes an honest proposal possible: the host
// can see a model is present without being able to claim it may remove it.
func handleModelList(ws *websocket.Conn, root string, _ []string) {
status, body, err := modelPostJSON(http.MethodGet, "/api/tags", nil, modelListTimeout)
addr, why := modelDaemonTarget(modelSelf(ws))
if why != "" {
modelFail(ws, "model:list: "+why)
return
}
status, body, err := modelPostJSON(addr, http.MethodGet, "/api/tags", nil, modelListTimeout)
if err != nil {
modelFail(ws, "model:list: no model daemon answering on this machine: "+err.Error())
return
@@ -974,12 +1000,12 @@ func modelPullRunningNote(j *modelPullJob) string {
// runBackupFetch updates job.Got inside its read loop (backupfetch.go:309-311):
// progress that only exists when the transfer ends is not progress. nil is
// allowed — the direct-call tests pass it.
func modelPullStreaming(model string, maxBytes int64, note func(want, got int64)) (int64, error) {
func modelPullStreaming(addr, model string, maxBytes int64, note func(want, got int64)) (int64, error) {
body, err := json.Marshal(map[string]any{"model": model, "stream": true})
if err != nil {
return 0, err
}
httpReq, err := http.NewRequest(http.MethodPost, "http://"+localOllamaAddr()+"/api/pull",
httpReq, err := http.NewRequest(http.MethodPost, "http://"+addr+"/api/pull",
bytes.NewReader(body))
if err != nil {
return 0, err
@@ -1109,6 +1135,17 @@ func modelPullStart(ws *websocket.Conn, root string, req modelReq) {
return
}
// WHERE THE PULL GOES, decided once and held for the download and its
// probe: a Host with no platform pulls only onto a model server on this
// machine (modelDaemonTarget). Refused before the approval is read, so a
// pull that cannot go here costs the person nothing. For such a pairing a
// name in OLLAMA_ADDR is looked up here; nothing is dialled.
addr, why := modelDaemonTarget(modelSelf(ws))
if why != "" {
modelRefuse(ws, req.Model, why)
return
}
// THE APPROVAL GATE. Nothing above this line has touched the network; nothing
// below it runs until the user has said yes to THIS proposal on this machine.
// The async split moved the download off this frame and deliberately did NOT
@@ -1152,7 +1189,7 @@ func modelPullStart(ws *websocket.Conn, root string, req modelReq) {
snap := *job
modelPullJobsMu.Unlock()
go runModelPull(job, root, req)
go runModelPull(job, root, req, addr)
modelReply(ws, snap)
}
@@ -1167,7 +1204,7 @@ func modelPullStart(ws *websocket.Conn, root string, req modelReq) {
// bytes on this disk", which is true the moment the download succeeds. A model
// that downloaded but failed its probe must stay droppable, or a bad pull would
// leave dead weight on the user's machine that nothing is allowed to remove.
func runModelPull(job *modelPullJob, root string, req modelReq) {
func runModelPull(job *modelPullJob, root string, req modelReq, addr string) {
set := func(f func(j *modelPullJob)) {
modelPullJobsMu.Lock()
f(job)
@@ -1201,7 +1238,7 @@ func runModelPull(job *modelPullJob, root string, req modelReq) {
// The 200-with-an-error-in-the-body case that stream:false was chosen for is
// handled either way: modelBodyError reads EVERY line, so a failure on the
// last line of a stream is caught exactly as before.
total, err := modelPullStreaming(req.Model, req.MaxBytes,
total, err := modelPullStreaming(addr, req.Model, req.MaxBytes,
func(want, got int64) { set(func(j *modelPullJob) { j.Want, j.Got = want, got }) })
if err != nil {
finish("failed", func(j *modelPullJob) {
@@ -1242,9 +1279,9 @@ func runModelPull(job *modelPullJob, root string, req modelReq) {
var dim int
var why string
if req.Role == "embed" {
dim, why = probeEmbedModel(req.Model)
dim, why = probeEmbedModel(addr, req.Model)
} else {
why = probeGenerateModel(req.Model)
why = probeGenerateModel(addr, req.Model)
}
if why != "" {
finish("done", func(j *modelPullJob) {
@@ -1284,7 +1321,12 @@ func handleModelDrop(ws *websocket.Conn, root string, args []string) {
"not remove it. Only a model this host pulled here is its to drop.")
return
}
status, body, err := modelPostJSON(http.MethodDelete, "/api/delete",
addr, why := modelDaemonTarget(modelSelf(ws))
if why != "" {
modelFail(ws, "model:drop: "+why)
return
}
status, body, err := modelPostJSON(addr, http.MethodDelete, "/api/delete",
map[string]any{"model": req.Model}, modelListTimeout)
if err != nil {
modelFail(ws, "model:drop: the model daemon did not answer: "+err.Error())
+11 -6
View File
@@ -286,7 +286,8 @@ func TestNetworksInsideThisMachineAreThisMachine(t *testing.T) {
}
// The local egress lane on an unanchored box: an inside address, by address
// and by name, is refused as this machine; the LAN passes.
// and by name, is refused as this machine; the LAN is refused as the network
// around it; the public internet passes.
t.Setenv("XDG_CONFIG_HOME", t.TempDir())
if currentAnchor() != nil || boxMode() != modeLocal {
t.Fatal("the namespaced run must be a local, unanchored box")
@@ -301,12 +302,16 @@ func TestNetworksInsideThisMachineAreThisMachine(t *testing.T) {
t.Errorf("local egress: %s is on a network inside this machine and must be refused as this machine, got %v %q", hp, got, why)
}
}
for hp, want := range map[string]string{"10.77.0.9:445": "10.77.0.9:445", "192.168.50.9:631": "192.168.50.9:631",
"nas.example.test:445": "10.77.0.9:445", "198.51.100.9:443": "198.51.100.9:443",
"[2001:db8:ff::9]:443": "[2001:db8:ff::9]:443", "192.168.60.9:445": "192.168.60.9:445",
"10.88.0.9:80": "10.88.0.9:80", "10.89.0.9:80": "10.89.0.9:80", "[2001:db8:88::9]:80": "[2001:db8:88::9]:80"} {
for _, hp := range []string{"10.77.0.9:445", "192.168.50.9:631", "nas.example.test:445", "192.168.60.9:445",
"10.88.0.9:80", "10.89.0.9:80"} {
if got, why := egressVettedAddrs(ctx, hp); !strings.Contains(why, "approved lanes") {
t.Errorf("local egress: %s is the LAN and must be refused as the network around this machine, got %v %q", hp, got, why)
}
}
for hp, want := range map[string]string{"198.51.100.9:443": "198.51.100.9:443",
"[2001:db8:ff::9]:443": "[2001:db8:ff::9]:443", "[2001:db8:88::9]:80": "[2001:db8:88::9]:80"} {
if got, why := egressVettedAddrs(ctx, hp); why != "" || len(got) != 1 || got[0] != want {
t.Errorf("local egress: %s is the LAN and passes as %s: %v %q", hp, want, got, why)
t.Errorf("local egress: %s leaves this machine for the internet and passes as %s: %v %q", hp, want, got, why)
}
}
// A coder run's proxy refuses the public range on an inside network too
+6
View File
@@ -9,6 +9,7 @@ import (
"errors"
"net"
"strings"
"sync"
"testing"
)
@@ -33,8 +34,13 @@ func stubOwnAddrs(t *testing.T, ifaces []string, route func(net.IP) (routeKind,
}
asked := &[]string{}
if route != nil {
// Locked: a lane's workers vet their jobs at once, each on its own
// goroutine (egress_relay_test.go).
var mu sync.Mutex
ownRouteLookup = func(ip net.IP) (routeKind, error) {
mu.Lock()
*asked = append(*asked, ip.String())
mu.Unlock()
return route(ip)
}
} else {
+5 -1
View File
@@ -29,7 +29,11 @@ const piiForwardPort = 18081
// The bearer-authed surface is proxied opaquely — this client never
// holds the PII_API_KEY.
func piiProbe() (bool, bool) {
a := piiTarget()
return piiProbeAt(piiTarget())
}
// piiProbeAt is piiProbe's probe of one address.
func piiProbeAt(a string) (bool, bool) {
if a == "" {
return false, false
}
+2 -5
View File
@@ -686,11 +686,8 @@ func handleRootExec(ws *websocket.Conn, spec sessionSpec, self string, env sessi
closed := make(chan struct{})
go func() {
defer close(closed)
for {
if _, _, err := ws.ReadMessage(); err != nil {
return
}
}
// Each frame is dropped as it arrives (lane_bounds.go), never held whole.
_ = drainLane(ws)
}()
done := make(chan int, 1)
go func() { done <- proc.Wait() }()
+36 -7
View File
@@ -617,8 +617,11 @@ func runPairing(ctx context.Context, cfg *Config) error {
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Authorization", "Bearer "+cfg.Bearer)
if resp, err := upstream.Do(req); err == nil {
body, _ := io.ReadAll(resp.Body)
body, ok := readHeartbeatReply(resp)
resp.Body.Close()
if !ok {
return // a reply past its bound is no answer (lane_bounds.go)
}
if resp.StatusCode < 300 {
// last_ok per pairing (anchor.go): the anchor rotation's
// silence path reads this — 14 days with no 2xx from the
@@ -727,14 +730,19 @@ func unixLaneDialer(cfg *Config) (func(ctx context.Context) (net.Conn, error), *
// parked harness workers, sessions, git, the heartbeat and what its reply
// directs. Absent, because they listen on the tunnel's stack or move the
// tunnel: the ollama, LLM and PII forwards, the service slots, and endpoint
// rotation and failover. A unix pairing's coder run has its control plane on
// a socket in RunsDir (coder_socket.go), not a tunnel port; a tcp pairing has
// no coder runs.
// rotation and failover. The model servers the forwards reach are served
// over the serve lane instead (servelane.go), by service name. A unix
// pairing's coder run has its control plane on a socket in RunsDir
// (coder_socket.go), not a tunnel port; a tcp pairing has no coder runs. The
// heartbeat advertises only what runs here (agents.go, beatLanes).
func runUnixPairing(ctx context.Context, cfg *Config) error {
logf := func(format string, a ...any) { fmt.Printf("["+cfg.HostName+"] "+format, a...) }
if !cfg.active() {
return errPairingNotActive
}
// Before any lane: model management for this pairing (modelDaemonTarget)
// reaches only a model server on this machine, as the serve lane does.
defer markLocalPairing(cfg.HostName)()
dialHost, laneTLS, err := localLaneDialer(cfg)
if err != nil {
return err
@@ -762,6 +770,13 @@ func runUnixPairing(ctx context.Context, cfg *Config) error {
logf("egress workers up — outbound exits THIS machine's internet via %s on %s\n", wsURL, lane)
runEgressWorkers(ctx, cfg.HostName, wsURL, laneTLS.Clone(), dialHost, cfg.Bearer, realDialer.DialContext, egressWorkerCount)
}()
// The serve lane: the model servers set on this machine, which the tunnel
// forwards would carry, by service name and never by address.
go func() {
wsURL := fmt.Sprintf("wss://%s:8443/bastion-serve", cfg.HostName)
logf("serve workers up — the model servers set on this machine, by name, via %s on %s\n", wsURL, lane)
runServeWorkers(ctx, cfg.HostName, cfg.dcnDir(), wsURL, laneTLS.Clone(), dialHost, cfg.Bearer, serveWorkerCount)
}()
go func() {
wsURL := fmt.Sprintf("wss://%s:8443/bastion-mgmt", cfg.HostName)
if n := sweepStaleScratch(cfg.contextRoot()); n > 0 {
@@ -841,8 +856,11 @@ func runUnixPairing(ctx context.Context, cfg *Config) error {
if err != nil {
return
}
body, _ := io.ReadAll(resp.Body)
body, ok := readHeartbeatReply(resp)
resp.Body.Close()
if !ok {
return // a reply past its bound is no answer (lane_bounds.go)
}
if resp.StatusCode < 300 {
recordPairingOK(cfg.HostName)
}
@@ -890,7 +908,7 @@ func heartbeatPayload(cfg *Config, dev interface{ IpcGet() (string, error) }, ho
for k, v := range pairingEnforcing(cfg.HostName) {
enforcing[k] = v
}
return buildHeartbeatBody(cfg.HostName, hostname, runtime.GOOS, clientVersion, wgPub, cfg.Fingerprint, cfg.BastionTunnelIP, cfg.Endpoint,
return buildHeartbeatBodyOn(beatLanesFor(cfg), cfg.HostName, hostname, runtime.GOOS, clientVersion, wgPub, cfg.Fingerprint, cfg.BastionTunnelIP, cfg.Endpoint,
rotation.isUnsaved(cfg), currentMirrorReport(cfg.HostName), enforcing,
// Read fresh on every beat: a host paired a minute ago must
// appear as a peer on the NEXT beat, not after a restart.
@@ -1124,6 +1142,13 @@ func ollamaModels(tunnelIP string) (bool, []string) {
addrs = append(addrs, net.JoinHostPort(tunnelIP, "11434"))
}
addrs = append(addrs, "127.0.0.1:11434")
return ollamaModelsAt(addrs)
}
// ollamaModelsAt is ollamaModels' probe of the addresses given, in order. A
// local pairing's beat asks it about the one address its serve lane dials
// (servelane.go), so what the beat says is served is what the lane reaches.
func ollamaModelsAt(addrs []string) (bool, []string) {
cl := &http.Client{Timeout: 3 * time.Second}
for _, a := range addrs {
resp, err := cl.Get("http://" + a + "/api/tags")
@@ -1172,7 +1197,11 @@ const llmForwardPort = 18080
// and the operator setting the env IS the opt-in. Same 200-is-alive rule as
// /api/tags: a live server with an unparseable list still counts.
func openAICompatModels() (bool, []string) {
a := llmTarget()
return openAICompatModelsAt(llmTarget())
}
// openAICompatModelsAt is openAICompatModels' probe of one address.
func openAICompatModelsAt(a string) (bool, []string) {
if a == "" {
return false, nil
}
+721
View File
@@ -0,0 +1,721 @@
package main
// The serve lane (/bastion-serve): how a Host with no platform reaches the
// model servers on this machine.
//
// Over the tunnel the Host dials a forward on the tunnel's stack
// (serveOllamaForward and serveLLMForward in run.go, servePIIForward in
// pii.go). A local pairing (a Host on this machine's socket, or in a VM on
// it over TCP) has no tunnel, so those forwards are absent. This lane turns
// the direction round, as egress does: this machine parks worker WebSockets
// on the Host, and the Host hands each one connection.
//
// THE HOST NAMES A SERVICE, NEVER AN ADDRESS. Its first frame on a worker is
// 'S' + {"service":"ollama"}. The address that service reaches is the one set
// on this machine (OLLAMA_ADDR, PRIVATAE_LLM_ADDR, PRIVATAE_PII_ADDR), read by
// the functions the tunnel forwards read, and it must be this machine: a
// server on another computer on the LAN is refused. Nothing the Host sends
// can name, change or widen a target. The bastion answers 'A' and carries
// the connection, or 'E' + a reason and closes, having dialled nothing.
//
// After 'A' the worker is a wsConn ('D' + bytes, 'E' for the end, both ways)
// handed to the forwards' own code: ollama and llm through proxyInference's
// decision and filter (the model scope, /api/pull and /api/delete refused
// without `models`, the list rewrite, its size bounds), pii through the PII
// scope and a pass-through. The `inference` family is checked on every
// connection, as inferenceRefuser does at a forward's accept.
//
// Bounds, per pairing: 16 streams in flight across the services, the 17th
// answered 'E' busy; a stream with no byte either way for 10 minutes is
// closed, on every service; a response's length is not capped, because a
// long generation is the person's own traffic.
import (
"context"
"crypto/tls"
"encoding/json"
"fmt"
"io"
"net"
"net/http"
"os"
"path/filepath"
"slices"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/gorilla/websocket"
)
const (
// serveWorkerCount workers stay parked per pairing. A replacement is
// dialled as soon as one takes a job, so a long stream never holds up the
// next request.
serveWorkerCount = 2
// serveMaxInFlight is the streams one pairing may hold open here at once.
// It protects this machine and must never be what slows the person down:
// ollama runs one request per model at a time and queues 512 more, and the
// Host's own client holds at most 10 connections per server, so a request
// waits in the server's queue before it waits here.
serveMaxInFlight = 16
// serveHelloMax bounds the 'S' frame. A service name and its JSON fit in a
// fraction of it; anything longer is not a request this lane reads.
serveHelloMax = 1024
// serveTargetLookup bounds resolving a target set by name (localhost:11434).
serveTargetLookup = 2 * time.Second
)
// serveIdleTimeout closes a stream that has carried no byte either way for
// this long. The filter's own read bound (filteredIdleTimeout) covers a request
// being read; this covers the rest: a server that answers headers and then
// nothing, and a pass-through, which has no bound of its own. A var so a test
// can shorten it.
var serveIdleTimeout = filteredIdleTimeout
// serveService is one service the lane carries: its name on the wire, where
// its target is read, which forward's code serves it.
type serveService struct {
name string
envVar string // named in a refusal, so the person knows what to set
target func() string
filter bool // through proxyInference's decision and filter
lane inferenceLane // the filter's lane, when filter
}
// serveServices is every service the lane knows, in the order the beat lists
// them. A name not in it is refused before anything is read or dialled.
var serveServices = []serveService{
{name: "ollama", envVar: "OLLAMA_ADDR", target: localOllamaAddr, filter: true, lane: laneOllama},
{name: "llm", envVar: "PRIVATAE_LLM_ADDR", target: llmTarget, filter: true, lane: laneLLM},
{name: "pii", envVar: "PRIVATAE_PII_ADDR", target: piiTarget},
}
func serveServiceNamed(name string) (serveService, bool) {
for _, s := range serveServices {
if s.name == name {
return s, true
}
}
return serveService{}, false
}
// serveLookupIP resolves a target set by name. A seam so a test can answer.
var serveLookupIP = func(ctx context.Context, host string) ([]net.IPAddr, error) {
return net.DefaultResolver.LookupIPAddr(ctx, host)
}
// serveVetTarget checks the address set on this machine for a service and
// returns the addresses to dial, or the reason it is refused. Every address
// the target names must be this machine: loopback, an address on its
// interfaces, or one the kernel delivers here (ownaddr.go), which takes in a
// server in a container on a network made on this machine. A server on
// another computer is refused: a device the bastion does not run on is the
// device lane's problem, solved there with pinning. 0.0.0.0 and :: are listen
// addresses, and dialling one reaches this machine, so each is dialled as
// loopback of its family. The addresses dialled are the ones checked, never
// the name again.
//
// Every address checked is returned, in the resolver's order, and the dial
// tries them in turn (serveDial, servePick), as the OS dialler does with a
// name: localhost answers ::1 first on most systems, and ollama listens on
// 127.0.0.1 only by default, so dialling the first address alone fails where
// a plain dial of the name reaches it.
func serveVetTarget(target string) ([]string, string) {
host, port, err := net.SplitHostPort(target)
if err != nil {
return nil, fmt.Sprintf("%q is not host:port", target)
}
if n, err := strconv.Atoi(port); err != nil || n < 1 || n > 65535 {
return nil, fmt.Sprintf("%q has no port between 1 and 65535", target)
}
var ips []net.IP
if ip := net.ParseIP(host); ip != nil {
ips = []net.IP{ip}
} else {
ctx, cancel := context.WithTimeout(context.Background(), serveTargetLookup)
as, err := serveLookupIP(ctx, host)
cancel()
if err != nil || len(as) == 0 {
return nil, fmt.Sprintf("%q could not be resolved on this machine", host)
}
for _, a := range as {
ips = append(ips, a.IP)
}
}
for i, ip := range ips {
switch {
case ip.IsUnspecified() && ip.To4() != nil:
ips[i] = net.IPv4(127, 0, 0, 1)
case ip.IsUnspecified():
ips[i] = net.IPv6loopback
}
}
for _, ip := range ips {
if ip.IsLoopback() {
continue
}
own, err := ownDestination([]net.IP{ip})
if err != nil {
return nil, fmt.Sprintf("this machine could not tell whether %s is its own address (%v)", ip, err)
}
if !own {
return nil, fmt.Sprintf("%s is not this machine; the lane reaches only servers on the computer it runs on", ip)
}
}
out := make([]string, 0, len(ips))
for _, ip := range ips {
a := net.JoinHostPort(ip.String(), port)
if !slices.Contains(out, a) {
out = append(out, a)
}
}
return out, ""
}
// serveDialTimeout bounds one dial of a service's addresses, all of them
// together, as the forwards' dial (dialUpstream) bounds its one address.
const serveDialTimeout = 5 * time.Second
// serveDial dials the checked addresses in turn (dialEach: the next starts
// when one fails, or has not answered within the Happy Eyeballs delay) and
// returns the first that answers, with the address it was dialled at. Only
// these addresses are dialled.
func serveDial(addrs []string, timeout time.Duration) (net.Conn, string, error) {
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
var mu sync.Mutex
at := map[net.Conn]string{}
d := &net.Dialer{}
c, err := dialEach(ctx, func(ctx context.Context, network, a string) (net.Conn, error) {
c, err := d.DialContext(ctx, network, a)
if err == nil {
mu.Lock()
at[c] = a
mu.Unlock()
}
return c, err
}, addrs)
if err != nil {
return nil, "", err
}
mu.Lock()
defer mu.Unlock()
return c, at[c], nil
}
// servePick is the one address an HTTP client here is given for a target
// whose check returned several (the beat's probes, model management): the
// first that accepts a connection, tried in turn as serveDial does, or the
// first checked when none does, so the request fails as it would have. One
// address is returned as it is, with nothing dialled.
func servePick(addrs []string) string {
if len(addrs) == 1 {
return addrs[0]
}
c, a, err := serveDial(addrs, serveTargetLookup)
if err != nil {
return addrs[0]
}
_ = c.Close()
return a
}
// serveTargetFor is the addresses a service dials here, or why it does not.
// The beat and the lane both ask it, so what the beat says is served is what
// the lane reaches (the rule env.go states for the forwards).
func serveTargetFor(s serveService) ([]string, string) {
t := s.target()
if t == "" {
return nil, "unavailable: no " + s.name + " server is set on this machine (" + s.envVar + ")"
}
addrs, why := serveVetTarget(t)
if why != "" {
return nil, "refused: the " + s.name + " server set on this machine (" + s.envVar + ") is refused: " + why
}
return addrs, ""
}
// serveInFlight counts each pairing's open streams, keyed by host name, so a
// pairing restarted while its streams still run counts them.
var serveInFlight sync.Map // string -> *atomic.Int32
func serveAcquire(self string) bool {
v, _ := serveInFlight.LoadOrStore(self, new(atomic.Int32))
n := v.(*atomic.Int32)
for {
cur := n.Load()
if cur >= serveMaxInFlight {
return false
}
if n.CompareAndSwap(cur, cur+1) {
return true
}
}
}
func serveRelease(self string) {
if v, ok := serveInFlight.Load(self); ok {
v.(*atomic.Int32).Add(-1)
}
}
// serveHello is the 'S' frame's JSON. Fields this build does not know are
// ignored, so a Host may add one without breaking an older bastion.
type serveHello struct {
Service string `json:"service"`
}
// serveVerdict decides one job before anything is dialled: the service, the
// family, the PII scope, the target. It returns the service and the addresses,
// or the reason the job is refused.
func serveVerdict(self, name string) (serveService, []string, string) {
s, ok := serveServiceNamed(name)
if !ok {
return s, nil, "refused: " + strconv.Quote(truncate(name, 64)) + " is not a service this lane carries " +
"(ollama, llm or pii). The lane takes a service's name and reaches the server set on this machine for it; " +
"it never takes an address."
}
if why := capsAllow(self, "inference"); why != "" {
return s, nil, "refused: " + why
}
if s.name == "pii" && !pairingModelScope(self).allowsPII() {
return s, nil, "refused: this pairing is scoped to named models and the PII sterilizer was not one of them " +
"(approve with 'pii' in the model list to share it)"
}
addrs, why := serveTargetFor(s)
if why != "" {
return s, nil, why
}
return s, addrs, ""
}
// serveJob serves the one job a parked worker is given. taken is called once
// the job is accepted (its stream counted, before 'A'), so the pool parks a
// replacement while this one works. It reports whether a job was accepted. A
// refused job (malformed, refused, busy) reports false, and its worker waits
// before it parks again: the replacement is not dialled at once, so a Host
// that answers every worker with a frame it knows is refused cannot make this
// machine dial, handshake and write an audit line in a loop as fast as it
// answers.
func serveJob(ws *websocket.Conn, self, dcnDir string, taken func()) bool {
mt, r, err := ws.NextReader()
if err != nil {
return false
}
refuse := func(service, why string) {
_ = ws.WriteMessage(websocket.BinaryMessage, append([]byte{'E'}, why...))
serveAudit(dcnDir, map[string]any{"service": service, "decision": "refused", "reason": why})
}
raw, err := io.ReadAll(io.LimitReader(r, serveHelloMax+1))
if err != nil {
return false
}
var hello serveHello
if mt != websocket.BinaryMessage || len(raw) == 0 || raw[0] != 'S' || len(raw) > serveHelloMax ||
json.Unmarshal(raw[1:], &hello) != nil {
// Malformed: answered, not audited (as a malformed terminal request is not).
_ = ws.WriteMessage(websocket.BinaryMessage, []byte("Erefused: the first frame on this lane is 'S' and "+
"{\"service\":\"<name>\"}, at most 1024 bytes"))
return false
}
s, addrs, why := serveVerdict(self, hello.Service)
if why != "" {
refuse(truncate(hello.Service, 64), why)
return false
}
if !serveAcquire(self) {
refuse(s.name, fmt.Sprintf("busy: this pairing already has %d streams open on this machine, "+
"which is as many as it may hold; try again when one ends", serveMaxInFlight))
return false
}
defer serveRelease(self)
taken()
if err := ws.WriteMessage(websocket.BinaryMessage, []byte{'A'}); err != nil {
return true
}
w := newServeWatch(serveIdleTimeout)
in := w.wrap(&wsConn{ws: ws})
var reachedMu sync.Mutex
reached := ""
dial := func() (net.Conn, error) {
out, a, err := serveDial(addrs, serveDialTimeout)
if err != nil {
return nil, err
}
reachedMu.Lock()
reached = a
reachedMu.Unlock()
return w.wrap(out), nil
}
if s.filter {
proxyInferenceVia(in, dial, s.lane, self, servePass)
} else {
servePass(in, dial)
}
w.stop()
fromHost, toHost := w.counts()
reachedMu.Lock()
target := reached
reachedMu.Unlock()
if target == "" {
target = strings.Join(addrs, " ") // none answered, or none was dialled
}
serveAudit(dcnDir, map[string]any{"service": s.name, "target": target, "decision": "served",
"bytes_from_host": fromHost, "bytes_to_host": toHost, "idle_closed": w.idleClosed()})
return true
}
// servePass is the lane's pass-through: egress's splice, which keeps a
// half-close (the Host's 'E' ends what it sends, and the answer still comes
// back), where the tunnel's spliceConn ends both directions with the first.
func servePass(in net.Conn, dial upstreamDial) {
out, err := dial()
if err != nil {
_ = in.Close()
return
}
splice(in, in, out)
}
// serveWatch is the idle bound of one stream. Both ends are wrapped; any byte
// read or written on either one counts, and once none has for the bound,
// both are closed, which unblocks whichever copy or read is waiting.
type serveWatch struct {
bound time.Duration
last atomic.Int64 // unix nanoseconds of the last byte
fromHost atomic.Int64
toHost atomic.Int64
mu sync.Mutex
conns []net.Conn
host net.Conn
done chan struct{}
once sync.Once
idled atomic.Bool
}
func newServeWatch(bound time.Duration) *serveWatch {
w := &serveWatch{bound: bound, done: make(chan struct{})}
w.last.Store(time.Now().UnixNano())
go w.run()
return w
}
func (w *serveWatch) run() {
tick := w.bound / 4
if tick < 10*time.Millisecond {
tick = 10 * time.Millisecond
}
if tick > 5*time.Second {
tick = 5 * time.Second
}
t := time.NewTicker(tick)
defer t.Stop()
for {
select {
case <-w.done:
return
case <-t.C:
if time.Since(time.Unix(0, w.last.Load())) >= w.bound {
w.idled.Store(true)
w.mu.Lock()
for _, c := range w.conns {
_ = c.Close()
}
w.mu.Unlock()
return
}
}
}
}
func (w *serveWatch) stop() { w.once.Do(func() { close(w.done) }) }
func (w *serveWatch) idleClosed() bool { return w.idled.Load() }
func (w *serveWatch) counts() (fromHost, toHost int64) { return w.fromHost.Load(), w.toHost.Load() }
// wrap watches c. The first conn wrapped is the Host's: what is read from it
// came from the Host, what is written to it goes back.
func (w *serveWatch) wrap(c net.Conn) net.Conn {
w.mu.Lock()
defer w.mu.Unlock()
if w.host == nil {
w.host = c
}
w.conns = append(w.conns, c)
return &serveWatched{Conn: c, w: w, host: w.host == c}
}
type serveWatched struct {
net.Conn
w *serveWatch
host bool
}
func (c *serveWatched) Read(p []byte) (int, error) {
n, err := c.Conn.Read(p)
if n > 0 {
c.w.last.Store(time.Now().UnixNano())
if c.host {
c.w.fromHost.Add(int64(n))
}
}
return n, err
}
func (c *serveWatched) Write(p []byte) (int, error) {
n, err := c.Conn.Write(p)
if n > 0 {
c.w.last.Store(time.Now().UnixNano())
if c.host {
c.w.toHost.Add(int64(n))
}
}
return n, err
}
// CloseWrite passes a half-close through, so splice can end one direction.
func (c *serveWatched) CloseWrite() error {
if cw, ok := c.Conn.(interface{ CloseWrite() error }); ok {
return cw.CloseWrite()
}
return c.Conn.Close()
}
// serveAudit writes one line per job to the pairing's audit log, the file
// dcnAudit writes, and to the system log.
func serveAudit(dir string, rec map[string]any) {
if dir == "" {
return
}
rec["ts"] = time.Now().UTC().Format(time.RFC3339)
rec["op"] = "serve"
raw, _ := json.Marshal(rec)
auditToSystemLog(auditFilePairingD, auditPairingOfDir(dir), raw)
if err := os.MkdirAll(dir, 0o700); err != nil {
return
}
fh, err := os.OpenFile(filepath.Join(dir, auditFilePairingD), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o600)
if err != nil {
return
}
defer fh.Close()
_, _ = fh.Write(append(raw, '\n'))
}
// serveRetryMax is the longest a worker waits before dialling again while the
// Host answers the lane's upgrade and refuses it. A Host that predates the
// lane does that (an unknown WebSocket route is a 404, or a 403 from a server
// that closes it before accepting), and two workers dialling it every 3 s
// for good would be work for nothing on the person's machine. A Host that
// cannot be reached at all (restarting) is dialled every 3 s, as egress is.
var serveRetryMax = time.Minute
// serveRefusedWait is how long a worker waits before it parks again after its
// WebSocket closed while parked, or after its job was refused. With
// serveWorkerCount workers, a Host that refuses or misuses every job costs
// this machine at most that many dials a second.
var serveRefusedWait = time.Second
// localPairings counts the runs of each pairing with a Host that has no
// platform (runUnixPairing), by host name, while they run. A count, not a
// flag, so a run that is still stopping when the same pairing starts again
// cannot unmark the new one.
var localPairings sync.Map // string -> *atomic.Int32
// markLocalPairing records that self runs as a pairing with a Host that has no
// platform, until the returned func is called.
func markLocalPairing(self string) func() {
v, _ := localPairings.LoadOrStore(self, new(atomic.Int32))
n := v.(*atomic.Int32)
n.Add(1)
var once sync.Once
return func() { once.Do(func() { n.Add(-1) }) }
}
// isLocalPairing reports whether self is running as a pairing with a Host that
// has no platform.
func isLocalPairing(self string) bool {
v, ok := localPairings.Load(self)
return ok && v.(*atomic.Int32).Load() > 0
}
// modelDaemonTarget is the address model management (modelswap.go: the
// model-swap probe in the beat, model:list, model:pull and its probes,
// model:drop) dials for the pairing self, or the reason it dials nothing.
// For a Host with no platform it is the serve lane's own ollama target, under
// the same rule: OLLAMA_ADDR (else loopback) only when every address it names
// is this machine, and the address dialled is one of those checked: the first
// that answers, tried in turn (servePick). A bastion serves only the computer
// it runs on, and managing the models on another computer is serving it. A
// tunnel pairing keeps OLLAMA_ADDR wherever it points, as its forwards do.
func modelDaemonTarget(self string) (string, string) {
t := localOllamaAddr()
if !isLocalPairing(self) {
return t, ""
}
addrs, why := serveVetTarget(t)
if why != "" {
s, _ := serveServiceNamed("ollama")
serveTargetNotice(self, s, t, why)
return "", "refused: the model server set on this machine (OLLAMA_ADDR) is not used for this Host: " + why
}
return servePick(addrs), ""
}
// serveTargetNoticed is the last refusal said at the machine for each pairing
// and service, so the console carries it once, and again only when it changes.
var serveTargetNoticed sync.Map // "<self>\x00<service>" -> string
// serveTargetNotice says at this machine, in plain words, that the server set
// here for a service is not used for the Host with no platform, and why. A
// refusal that only the Host hears leaves the person who set the address with
// nothing on their own screen to explain why the model is missing.
func serveTargetNotice(self string, s serveService, target, why string) {
key := self + "\x00" + s.name
// The beat and a model op can refuse the same target at once; only the one
// that records this reason says it.
for {
prev, loaded := serveTargetNoticed.LoadOrStore(key, why)
if !loaded {
break
}
if prev.(string) == why {
return
}
if serveTargetNoticed.CompareAndSwap(key, prev, why) {
break
}
}
fmt.Printf("[%s] %s=%s is not used for the Host on this computer: %s. Set %s to a server on this "+
"computer, or unset it.\n", self, s.envVar, target, why, s.envVar)
}
// serveTargetNoticeClear forgets what was said for a service that is fine
// again, so a later refusal is said again.
func serveTargetNoticeClear(self string, s serveService) {
serveTargetNoticed.Delete(self + "\x00" + s.name)
}
// runServeWorkers keeps n workers parked on the Host's /bastion-serve for the
// pairing self, each serving one job (serveJob). A worker that takes a job
// starts its replacement at once and ends with the job; one whose WebSocket
// dies while parked dials again. pctx is the pairing's context: a revoked
// pairing's workers exit.
func runServeWorkers(pctx context.Context, self, dcnDir, wsURL string, tlsCfg *tls.Config,
dialHost func(ctx context.Context) (net.Conn, error), bearer string, n int) {
dialer := &websocket.Dialer{
NetDialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { return dialHost(ctx) },
TLSClientConfig: tlsCfg,
HandshakeTimeout: 20 * time.Second,
}
var spawn func()
spawn = func() {
go func() {
wait := 3 * time.Second
for pctx.Err() == nil {
ctx, cancel := context.WithTimeout(pctx, 25*time.Second)
ws, resp, err := dialer.DialContext(ctx, wsURL, http.Header{})
cancel()
if err != nil {
if resp != nil {
sleepCtx(pctx, wait)
wait = min(2*wait, serveRetryMax)
} else {
sleepCtx(pctx, 3*time.Second)
}
continue
}
wait = 3 * time.Second
if err := ws.WriteMessage(websocket.TextMessage, []byte("BEARER "+bearer)); err != nil {
_ = ws.Close()
sleepCtx(pctx, time.Second)
continue
}
stopKA := armWSKeepalive(ws)
unpark := context.AfterFunc(pctx, func() { _ = ws.Close() })
took := serveJob(ws, self, dcnDir, spawn)
unpark()
stopKA()
_ = ws.Close()
if took {
return
}
// Closed while parked (a Host restart), or the job refused: not
// straight back, so a Host that closes or misuses every worker is
// not dialled in a loop.
sleepCtx(pctx, serveRefusedWait)
}
}()
}
for i := 0; i < n; i++ {
spawn()
}
}
// laneServeReport is what a local pairing's beat says of its serve lane.
type laneServeReport struct {
served []string // lane_serves
models []string // ollama's, filtered to the scope
llmModels []string // the llm server's, filtered to the scope
piiOK bool
}
// laneServes is what this pairing serves over the lane right now, for the
// beat: each service whose target is set and is this machine, whose server
// answers its probe, and whose family the pairing holds. A scoped pairing
// with no model left in a server's list is not served that server, as the
// tunnel's beat withholds a lane with nothing in it.
func laneServes(self string) laneServeReport {
rep := laneServeReport{served: []string{}, models: []string{}, llmModels: []string{}}
if capsAllow(self, "inference") != "" {
return rep
}
scope := pairingModelScope(self)
for _, s := range serveServices {
t := s.target()
if t == "" {
continue
}
addrs, why := serveVetTarget(t)
if why != "" {
serveTargetNotice(self, s, t, why)
continue
}
serveTargetNoticeClear(self, s)
addr := servePick(addrs)
switch s.name {
case "ollama":
ok, models := ollamaModelsAt([]string{addr})
models = scope.filterModels(models)
if !ok || (scope.scoped && len(models) == 0) {
continue
}
if models != nil {
rep.models = models
}
case "llm":
ok, models := openAICompatModelsAt(addr)
models = scope.filterModels(models)
if !ok || (scope.scoped && len(models) == 0) {
continue
}
if models != nil {
rep.llmModels = models
}
case "pii":
if !scope.allowsPII() {
continue
}
if _, healthy := piiProbeAt(addr); !healthy {
continue
}
rep.piiOK = true
}
rep.served = append(rep.served, s.name)
}
return rep
}
+131
View File
@@ -0,0 +1,131 @@
//go:build linux
// The serve lane dials a target set by name at each address it checked, in
// turn (servelane.go: serveVetTarget, serveDial, servePick). Uses the fixtures
// of servelane_linux_test.go, modelscope_test.go and modelswap_test.go.
package main
import (
"bufio"
"context"
"fmt"
"io"
"net"
"net/http"
"slices"
"strings"
"testing"
"time"
)
// A target set by name is dialled at every address it was checked for, in
// turn, as a plain dial of the name is. OLLAMA_ADDR=localhost:<port>, a
// resolver that answers ::1 first (as most do) and a server listening on
// 127.0.0.1 only (ollama's default): a job on the lane, the beat and model
// management each reach it. The resolver's two addresses are both this
// machine's and are the only ones there are to dial.
//
// MUTATION: serveVetTarget returns only the first address it checked (as it
// did), or serveJob dials only addrs[0].
// RESULT: red — the job's request gets no answer (its one dial, to [::1], is
// refused), the beat has no ollama in lane_serves, and model management is
// handed [::1].
func TestTheServeLaneDialsEachCheckedAddressInTurn(t *testing.T) {
up := newFakeOllama(t)
listening := up.srv.Listener.Addr().String()
if !strings.HasPrefix(listening, "127.0.0.1:") {
t.Fatalf("the stand-in listens on %s, not 127.0.0.1 only, so this case proves nothing", listening)
}
port := portOf(up.srv.Listener)
if c, err := net.DialTimeout("tcp", net.JoinHostPort("::1", port), time.Second); err == nil {
_ = c.Close()
t.Fatalf("something answers on [::1]:%s, so the first address would not fail and this case proves nothing", port)
}
old := serveLookupIP
t.Cleanup(func() { serveLookupIP = old })
serveLookupIP = func(_ context.Context, host string) ([]net.IPAddr, error) {
if host != "localhost" {
return nil, fmt.Errorf("no such host")
}
return []net.IPAddr{{IP: net.IPv6loopback}, {IP: net.ParseIP("127.0.0.1")}}, nil
}
t.Setenv("OLLAMA_ADDR", "localhost:"+port)
t.Setenv("PRIVATAE_LLM_ADDR", "")
t.Setenv("PRIVATAE_PII_ADDR", "")
resetModelDaemonCache()
h, cfg := serveLaneBox(t, "dual.example.test")
runInBackground(t, cfg)
if got := strsOf(h.beat(t)["lane_serves"]); !slices.Contains(got, "ollama") {
t.Errorf("the beat does not say ollama is served: lane_serves %q", got)
}
ws := h.take(t)
if got := ask(t, ws, "ollama"); got != "A" {
t.Fatalf("the lane must accept ollama: %q", got)
}
conn := laneConn(ws)
req, _ := http.NewRequest(http.MethodGet, "http://serve/api/tags", nil)
req.Close = true
if err := req.Write(conn); err != nil {
t.Fatal(err)
}
_ = conn.SetReadDeadline(time.Now().Add(10 * time.Second))
resp, err := http.ReadResponse(bufio.NewReader(conn), req)
if err != nil {
t.Fatalf("no answer over the lane (only [::1] was dialled?): %v", err)
}
b, _ := io.ReadAll(resp.Body)
if resp.StatusCode != http.StatusOK || !strings.Contains(string(b), "qwen4-embed") {
t.Fatalf("over the lane: %d %s", resp.StatusCode, b)
}
if addr, why := modelDaemonTarget(cfg.HostName); addr != listening || why != "" {
t.Errorf("model management is handed %q (%q), want the address that answers, %s", addr, why, listening)
}
out, errOut, code, _ := driveModelOp(t, cfg.HostName, "model:list", nil, t.TempDir())
if code != 0 || !strings.Contains(out, "qwen4-embed") {
t.Errorf("model:list: exit %d, %q %q", code, out, errOut)
}
for _, s := range up.requests() {
if !strings.HasPrefix(s, "GET ") {
t.Errorf("the stand-in was asked %q", s)
}
}
}
// servePick returns the first checked address that accepts, and the first
// checked when none does; it never returns an address it was not given.
func TestServePickTakesTheFirstCheckedAddressThatAnswers(t *testing.T) {
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
defer ln.Close()
go func() {
for {
c, err := ln.Accept()
if err != nil {
return
}
_ = c.Close()
}
}()
port := portOf(ln)
v6 := net.JoinHostPort("::1", port)
if c, err := net.DialTimeout("tcp", v6, time.Second); err == nil {
_ = c.Close()
t.Fatalf("something answers on %s, so this case proves nothing", v6)
}
if got := servePick([]string{v6, ln.Addr().String()}); got != ln.Addr().String() {
t.Errorf("pick %q, want %s", got, ln.Addr())
}
_ = ln.Close()
if got := servePick([]string{v6, "127.0.0.1:" + port}); got != v6 {
t.Errorf("with nothing answering, pick %q, want the first checked, %s", got, v6)
}
if got := servePick([]string{"127.0.0.1:" + port}); got != "127.0.0.1:"+port {
t.Errorf("one address: %q", got)
}
}
+992
View File
@@ -0,0 +1,992 @@
//go:build linux
// The serve lane (servelane.go) over a tcp pairing: a Host in a VM on this
// machine reaches the model servers set here by service name, through the
// forwards' own filter, within the lane's bounds, and the pairing's beat says
// what it serves.
//
// Uses the fixtures of local_host_linux_test.go and local_host_tcp_linux_test.go
// (newTestCA, tcpPairingFor, laneListener, pairingBox...), so it carries the
// same build tag.
package main
import (
"bufio"
"context"
"crypto/tls"
"crypto/x509"
"encoding/json"
"fmt"
"io"
"log"
"net"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"slices"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/gorilla/websocket"
)
// ── fixtures ────────────────────────────────────────────────────────────────
// serveHost is a Host's lane port as the serve lane meets it: the lane
// port's mutual TLS, every worker route kept parked, each beat recorded
// whole, and each /bastion-serve worker handed to the test once it has sent
// its hello.
type serveHost struct {
mu sync.Mutex
beats []map[string]any
hellos []string
workers chan *websocket.Conn
}
func startServeHost(t *testing.T, raw net.Listener, cert tls.Certificate, clientCA *testCA) *serveHost {
t.Helper()
h := &serveHost{workers: make(chan *websocket.Conn, 64)}
pool := x509.NewCertPool()
pool.AddCert(clientCA.cert)
ln := tls.NewListener(raw, &tls.Config{Certificates: []tls.Certificate{cert}, ClientCAs: pool,
ClientAuth: tls.RequireAndVerifyClientCert})
up := websocket.Upgrader{}
mux := http.NewServeMux()
mux.HandleFunc("/api/network/bastions/heartbeat", func(w http.ResponseWriter, r *http.Request) {
var m map[string]any
_ = json.NewDecoder(r.Body).Decode(&m)
h.mu.Lock()
h.beats = append(h.beats, m)
h.mu.Unlock()
_, _ = w.Write([]byte("{}"))
})
mux.HandleFunc("/bastion-serve", func(w http.ResponseWriter, r *http.Request) {
c, err := up.Upgrade(w, r, nil)
if err != nil {
return
}
_, hello, err := c.ReadMessage()
if err != nil {
c.Close()
return
}
h.mu.Lock()
h.hellos = append(h.hellos, string(hello))
h.mu.Unlock()
h.workers <- c // the test owns it from here
})
park := func(w http.ResponseWriter, r *http.Request) {
c, err := up.Upgrade(w, r, nil)
if err != nil {
return
}
defer c.Close()
for {
if _, _, err := c.ReadMessage(); err != nil {
return
}
}
}
for _, p := range []string{"/bastion-harness", "/bastion-mgmt", "/bastion-egress"} {
mux.HandleFunc(p, park)
}
srv := &http.Server{Handler: mux, ErrorLog: log.New(io.Discard, "", 0)}
go func() { _ = srv.Serve(ln) }()
t.Cleanup(func() {
_ = srv.Close()
for {
select {
case c := <-h.workers:
c.Close()
default:
return
}
}
})
return h
}
// take is the next parked serve worker.
func (h *serveHost) take(t *testing.T) *websocket.Conn {
t.Helper()
select {
case c := <-h.workers:
t.Cleanup(func() { c.Close() })
return c
case <-time.After(10 * time.Second):
t.Fatal("no /bastion-serve worker parked on the Host within 10 s")
}
return nil
}
// ask sends the 'S' frame for service and returns the bastion's answer: "A",
// or "E" and the reason.
func ask(t *testing.T, c *websocket.Conn, service string) string {
t.Helper()
b, _ := json.Marshal(map[string]string{"service": service})
return askRaw(t, c, websocket.BinaryMessage, append([]byte{'S'}, b...))
}
func askRaw(t *testing.T, c *websocket.Conn, mt int, frame []byte) string {
t.Helper()
if err := c.WriteMessage(mt, frame); err != nil {
t.Fatalf("send the first frame: %v", err)
}
_ = c.SetReadDeadline(time.Now().Add(10 * time.Second))
_, m, err := c.ReadMessage()
_ = c.SetReadDeadline(time.Time{})
if err != nil {
t.Fatalf("no answer to the first frame: %v", err)
}
return string(m)
}
// beat is the first beat the Host received, waiting up to 10 s.
func (h *serveHost) beat(t *testing.T) map[string]any {
t.Helper()
deadline := time.Now().Add(10 * time.Second)
for time.Now().Before(deadline) {
h.mu.Lock()
if len(h.beats) > 0 {
b := h.beats[0]
h.mu.Unlock()
return b
}
h.mu.Unlock()
time.Sleep(20 * time.Millisecond)
}
t.Fatal("no heartbeat reached the Host within 10 s")
return nil
}
// runInBackground runs the pairing until the test ends.
func runInBackground(t *testing.T, cfg *Config) {
t.Helper()
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- runPairing(ctx, cfg) }()
t.Cleanup(func() {
cancel()
select {
case <-done:
case <-time.After(5 * time.Second):
t.Error("the pairing did not stop")
}
// A stopped (revoked) pairing's streams end with it: each one's worker
// is closed. Waited for here too because a stream's last act is its
// audit line, in the pairing's directory, which the test then removes.
deadline := time.Now().Add(5 * time.Second)
for {
v, ok := serveInFlight.Load(cfg.HostName)
if !ok || v.(*atomic.Int32).Load() == 0 {
return
}
if time.Now().After(deadline) {
t.Errorf("%d streams still open 5 s after the pairing stopped", v.(*atomic.Int32).Load())
return
}
time.Sleep(10 * time.Millisecond)
}
})
}
// serveLaneBox is an unanchored local machine with a tcp pairing to a
// stand-in Host on 127.0.0.1, running. host keeps each test's pairing (and its
// stream count) its own.
func serveLaneBox(t *testing.T, host string) (*serveHost, *Config) {
t.Helper()
pairingBox(t)
laneTestProcess(t)
ca := newTestCA(t, "home host CA")
ln := laneListener(t)
h := startServeHost(t, ln, ca.serverCert(t, tcpServerName), ca)
cfg := tcpPairingFor(t, ca, host, ln.Addr().String())
if err := cfg.save(); err != nil {
t.Fatal(err)
}
return h, cfg
}
// laneConn is the Host's end of an accepted job: 'D' frames both ways, 'E'
// for the end, as the Host's relay frames a spliced connection.
func laneConn(c *websocket.Conn) net.Conn { return &wsConn{ws: c} }
// chatRequest writes one streaming chat request on conn and returns the reader
// its response is read from.
func chatRequest(t *testing.T, conn net.Conn, model string) (*http.Response, *http.Request) {
t.Helper()
body := fmt.Sprintf(`{"model":%q,"stream":true,"messages":[{"role":"user","content":"hello"}]}`, model)
req, _ := http.NewRequest(http.MethodPost, "http://serve/v1/chat/completions", strings.NewReader(body))
req.Header.Set("Content-Type", "application/json")
if err := req.Write(conn); err != nil {
t.Fatalf("write the request: %v", err)
}
_ = conn.SetReadDeadline(time.Now().Add(15 * time.Second))
resp, err := http.ReadResponse(bufio.NewReader(conn), req)
if err != nil {
t.Fatalf("read the response: %v", err)
}
return resp, req
}
// ── 1. a chat streams ───────────────────────────────────────────────────────
// streamingServer is a stand-in OpenAI-compatible server. Its chat answer is
// three chunks; after the first it waits until the test says the Host has
// read it, and records whether it had to give up waiting.
type streamingServer struct {
srv *httptest.Server
sawFirst chan struct{}
gated atomic.Bool
seen atomic.Int32
}
func newStreamingServer(t *testing.T) *streamingServer {
t.Helper()
s := &streamingServer{sawFirst: make(chan struct{})}
s.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
s.seen.Add(1)
switch r.URL.Path {
case "/v1/chat/completions":
case "/api/tags":
_, _ = w.Write([]byte(`{"models":[{"name":"test-model:latest"}]}`))
return
default:
_, _ = w.Write([]byte(`{"ok":true}`))
return
}
_, _ = io.Copy(io.Discard, r.Body)
w.Header().Set("Content-Type", "text/event-stream")
fl := w.(http.Flusher)
_, _ = io.WriteString(w, "data: {\"chunk\":1}\n\n")
fl.Flush()
select {
case <-s.sawFirst:
s.gated.Store(true)
case <-time.After(10 * time.Second):
}
_, _ = io.WriteString(w, "data: {\"chunk\":2}\n\n")
fl.Flush()
_, _ = io.WriteString(w, "data: {\"chunk\":3}\n\n")
fl.Flush()
_, _ = io.WriteString(w, "data: [DONE]\n\n")
}))
t.Cleanup(s.srv.Close)
return s
}
// SECURITY (and the lane's purpose): over a tcp pairing, the Host asks for a
// service by name and a chat streams from the server set here. The Host reads
// chunk 1 before the server writes chunk 3, so nothing in the path holds a
// response back: not the lane's framing, not the pass-through an unscoped
// pairing gets, not the filter a pairing without `models` gets.
//
// MUTATION: drop the serve workers from runUnixPairing.
// RESULT: red — no /bastion-serve worker ever parks on the Host.
func TestTheServeLaneStreamsAChatOverATCPPairing(t *testing.T) {
for _, c := range []struct {
name, service, env string
modelsOff bool
}{
{"llm, passed through", "llm", "PRIVATAE_LLM_ADDR", false},
{"ollama, through the filter", "ollama", "OLLAMA_ADDR", true},
} {
t.Run(c.name, func(t *testing.T) {
s := newStreamingServer(t)
t.Setenv(c.env, s.srv.Listener.Addr().String())
h, cfg := serveLaneBox(t, "stream-"+c.service+".example.test")
if c.modelsOff {
if err := localDenySet(cfg.HostName, "models", true); err != nil {
t.Fatal(err)
}
if !inferenceFilterNeeded(cfg.HostName, laneOllama, pairingModelScope(cfg.HostName)) {
t.Fatal("with models off the ollama lane must be filtered, or this case proves nothing")
}
}
runInBackground(t, cfg)
ws := h.take(t)
if got := ask(t, ws, c.service); got != "A" {
t.Fatalf("the lane must accept %s: %q", c.service, got)
}
conn := laneConn(ws)
resp, _ := chatRequest(t, conn, "test-model")
if resp.StatusCode != http.StatusOK {
t.Fatalf("the chat answered %d", resp.StatusCode)
}
br := bufio.NewReader(resp.Body)
var got []string
first := false
for {
line, err := br.ReadString('\n')
if strings.HasPrefix(line, "data: ") {
got = append(got, strings.TrimSpace(line))
if !first && strings.Contains(line, `"chunk":1`) {
first = true
close(s.sawFirst)
}
}
if err != nil || strings.Contains(line, "[DONE]") {
break
}
}
if !first {
t.Fatalf("the stream never carried chunk 1: %q", got)
}
if !s.gated.Load() {
t.Fatal("the server wrote chunk 3 before the Host read chunk 1: the lane held the stream back")
}
want := []string{`data: {"chunk":1}`, `data: {"chunk":2}`, `data: {"chunk":3}`, `data: [DONE]`}
if !slices.Equal(got, want) {
t.Fatalf("the stream arrived as %q", got)
}
})
}
}
// ── 2. a service, never an address ──────────────────────────────────────────
// SECURITY: the Host names a service and the target is the one set on this
// machine. A first frame naming an address, a name the lane does not carry, a
// frame that is not 'S', a target set to a computer that is not this one, and
// a service with no target each answer 'E' with a reason, and nothing is
// dialled: a listener at the address named gets no connection.
//
// MUTATION: in serveVetTarget, skip the ownDestination check.
// RESULT: red — ollama with OLLAMA_ADDR on another computer is accepted.
func TestTheServeLaneTakesAServiceNameNeverAnAddress(t *testing.T) {
named, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
defer named.Close()
var reached atomic.Int32
go func() {
for {
c, err := named.Accept()
if err != nil {
return
}
reached.Add(1)
c.Close()
}
}()
t.Setenv("OLLAMA_ADDR", "192.0.2.10:11434") // TEST-NET-1: never this machine
t.Setenv("PRIVATAE_LLM_ADDR", "")
t.Setenv("PRIVATAE_PII_ADDR", "198.51.100.7:"+portOf(named))
h, cfg := serveLaneBox(t, "names.example.test")
runInBackground(t, cfg)
for _, c := range []struct {
name string
frame []byte
mt int
want []string
}{
{"an address for a service", sFrame(named.Addr().String()), websocket.BinaryMessage,
[]string{"Erefused: ", "never takes an address"}},
{"a name the lane does not carry", sFrame("nas"), websocket.BinaryMessage,
[]string{"Erefused: ", `"nas" is not a service`}},
{"a URL for a service", sFrame("http://" + named.Addr().String() + "/api/tags"), websocket.BinaryMessage,
[]string{"Erefused: ", "is not a service"}},
{"a frame that is not S", []byte("D" + "GET / HTTP/1.1\r\n\r\n"), websocket.BinaryMessage,
[]string{"Erefused: ", "the first frame on this lane is 'S'"}},
{"a text frame", []byte(`S{"service":"ollama"}`), websocket.TextMessage,
[]string{"Erefused: ", "the first frame on this lane is 'S'"}},
{"an S frame past its bound", append(sFrame("ollama"), make([]byte, serveHelloMax)...), websocket.BinaryMessage,
[]string{"Erefused: ", "at most 1024 bytes"}},
{"ollama set to another computer", sFrame("ollama"), websocket.BinaryMessage,
[]string{"Erefused: ", "OLLAMA_ADDR", "192.0.2.10 is not this machine"}},
{"pii set to another computer", sFrame("pii"), websocket.BinaryMessage,
[]string{"Erefused: ", "PRIVATAE_PII_ADDR", "198.51.100.7 is not this machine"}},
{"llm with nothing set", sFrame("llm"), websocket.BinaryMessage,
[]string{"Eunavailable: ", "PRIVATAE_LLM_ADDR"}},
} {
ws := h.take(t)
got := askRaw(t, ws, c.mt, c.frame)
for _, w := range c.want {
if !strings.Contains(got, w) {
t.Errorf("%s: the answer %q does not carry %q", c.name, got, w)
}
}
if !strings.HasPrefix(got, "E") {
t.Errorf("%s: answered %q, not E", c.name, got)
}
// Refused means closed: nothing after the E.
_ = ws.SetReadDeadline(time.Now().Add(3 * time.Second))
if _, m, err := ws.ReadMessage(); err == nil {
t.Errorf("%s: the worker carried on after its refusal: %q", c.name, m)
}
}
time.Sleep(200 * time.Millisecond)
if n := reached.Load(); n != 0 {
t.Fatalf("a refused job dialled the address it named (%d connections)", n)
}
}
func sFrame(service string) []byte {
b, _ := json.Marshal(map[string]string{"service": service})
return append([]byte{'S'}, b...)
}
func portOf(ln net.Listener) string {
_, p, _ := net.SplitHostPort(ln.Addr().String())
return p
}
// A target set by name is resolved and each address checked; the addresses
// dialled are the ones checked, every one of them, in the resolver's order.
// 0.0.0.0 is a listen address and dials as loopback (once, if loopback is
// there already). This machine's own interface address passes.
func TestServeVetTargetChecksEveryAddressItWouldDial(t *testing.T) {
old := serveLookupIP
t.Cleanup(func() { serveLookupIP = old })
serveLookupIP = func(_ context.Context, host string) ([]net.IPAddr, error) {
switch host {
case "localhost":
return []net.IPAddr{{IP: net.ParseIP("127.0.0.1")}, {IP: net.ParseIP("::1")}}, nil
case "v6first.example.test":
return []net.IPAddr{{IP: net.ParseIP("::1")}, {IP: net.ParseIP("127.0.0.1")}, {IP: net.ParseIP("0.0.0.0")}}, nil
case "mixed.example.test":
return []net.IPAddr{{IP: net.ParseIP("127.0.0.1")}, {IP: net.ParseIP("192.0.2.10")}}, nil
}
return nil, fmt.Errorf("no such host")
}
for target, want := range map[string][]string{
"127.0.0.1:11434": {"127.0.0.1:11434"},
"[::1]:8080": {"[::1]:8080"},
"0.0.0.0:11434": {"127.0.0.1:11434"},
"[::]:11434": {"[::1]:11434"},
"localhost:11434": {"127.0.0.1:11434", "[::1]:11434"},
"v6first.example.test:80": {"[::1]:80", "127.0.0.1:80"},
"mixed.example.test:11434": nil,
"nowhere.example.test:80": nil,
"192.0.2.10:11434": nil,
"198.51.100.9:80": nil,
"127.0.0.1": nil,
"127.0.0.1:0": nil,
"127.0.0.1:99999": nil,
} {
got, why := serveVetTarget(target)
if !slices.Equal(got, want) || (want == nil) != (why != "") {
t.Errorf("%q: dial %q (%q), want %q", target, got, why, want)
}
}
ips, err := ownInterfaceIPs()
if err != nil {
t.Fatal(err)
}
for _, ip := range ips {
if ip.IsLoopback() || ip.IsLinkLocalUnicast() {
continue
}
a := net.JoinHostPort(ip.String(), "11434")
if got, why := serveVetTarget(a); !slices.Equal(got, []string{a}) {
t.Errorf("this machine's own address %s is refused: %q", a, why)
}
}
}
// ── 3. the forwards' walls, unchanged ───────────────────────────────────────
// SECURITY: the lane keeps every wall the tunnel forwards keep. With
// `inference` switched off at this machine, a job is refused before any dial.
// With `models` off, /api/pull and /api/delete are refused 403 and never reach
// ollama, while a chat does. On an anchored machine they are refused to the
// anchor itself. A pairing scoped to a model is refused another one with 403,
// which never reaches the server, and is served its own.
func TestTheServeLaneKeepsTheForwardsWalls(t *testing.T) {
t.Run("inference off at the machine", func(t *testing.T) {
// A raw listener, so a dial by the serve job is seen even though the
// Host sends it nothing: a connection that does not open with a beat's
// probe (GET /api/tags, which any pairing's beat in this process may
// send to OLLAMA_ADDR) is the job's.
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { ln.Close() })
var jobDials atomic.Int32
go func() {
for {
c, err := ln.Accept()
if err != nil {
return
}
go func() {
defer c.Close()
_ = c.SetReadDeadline(time.Now().Add(2 * time.Second))
b := make([]byte, len("GET /api/tags "))
if _, err := io.ReadFull(c, b); err != nil || string(b) != "GET /api/tags " {
jobDials.Add(1)
}
}()
}
}()
t.Setenv("OLLAMA_ADDR", ln.Addr().String())
h, cfg := serveLaneBox(t, "walls-off.example.test")
if err := localDenySet(cfg.HostName, "inference", true); err != nil {
t.Fatal(err)
}
runInBackground(t, cfg)
got := ask(t, h.take(t), "ollama")
if !strings.HasPrefix(got, "Erefused: ") {
t.Fatalf("with inference off the job must be refused: %q", got)
}
time.Sleep(2500 * time.Millisecond)
if n := jobDials.Load(); n != 0 {
t.Fatalf("a refused job dialled ollama (%d connections)", n)
}
})
pullAndDelete := func(t *testing.T, h *serveHost, up *fakeOllama) {
t.Helper()
for _, p := range []string{"/api/pull", "/api/delete"} {
ws := h.take(t)
if got := ask(t, ws, "ollama"); got != "A" {
t.Fatalf("%s: the job must be accepted, and refused by the filter: %q", p, got)
}
conn := laneConn(ws)
req, _ := http.NewRequest(http.MethodPost, "http://serve"+p, strings.NewReader(`{"model":"test-model"}`))
if err := req.Write(conn); err != nil {
t.Fatal(err)
}
_ = conn.SetReadDeadline(time.Now().Add(10 * time.Second))
resp, err := http.ReadResponse(bufio.NewReader(conn), req)
if err != nil {
t.Fatalf("%s: %v", p, err)
}
b, _ := io.ReadAll(resp.Body)
if resp.StatusCode != http.StatusForbidden || !strings.Contains(string(b), "models") {
t.Fatalf("%s over the lane: %d %s", p, resp.StatusCode, b)
}
}
for _, s := range up.requests() {
if strings.Contains(s, "/api/pull") || strings.Contains(s, "/api/delete") {
t.Fatalf("a refused store request reached ollama: %q", up.requests())
}
}
}
t.Run("models off at the machine", func(t *testing.T) {
up := newFakeOllama(t)
t.Setenv("OLLAMA_ADDR", up.srv.Listener.Addr().String())
h, cfg := serveLaneBox(t, "walls-models.example.test")
if err := localDenySet(cfg.HostName, "models", true); err != nil {
t.Fatal(err)
}
runInBackground(t, cfg)
pullAndDelete(t, h, up)
})
t.Run("an anchored machine, as its anchor", func(t *testing.T) {
up := newFakeOllama(t)
t.Setenv("OLLAMA_ADDR", up.srv.Listener.Addr().String())
h, cfg := serveLaneBox(t, "walls-anchor.example.test")
p, _ := modePath()
if err := os.WriteFile(p, []byte(`{"mode":"networked"}`), 0o600); err != nil {
t.Fatal(err)
}
if err := writeAnchor("Acme IT", cfg.HostName); err != nil {
t.Fatal(err)
}
if currentAnchor() == nil || capsAllow(cfg.HostName, "inference") != "" {
t.Fatal("the pairing must be this machine's anchor, holding inference, or this case proves nothing")
}
runInBackground(t, cfg)
pullAndDelete(t, h, up)
})
t.Run("a pairing scoped to one model", func(t *testing.T) {
priv := setupBox(t)
withMarkerAt(t, filepath.Join(t.TempDir(), "machine.json"))
laneTestProcess(t)
up := newFakeOllama(t)
t.Setenv("PRIVATAE_LLM_ADDR", up.srv.Listener.Addr().String())
ca := newTestCA(t, "home host CA")
ln := laneListener(t)
h := startServeHost(t, ln, ca.serverCert(t, tcpServerName), ca)
member := tcpPairingFor(t, ca, tMember, ln.Addr().String())
member.State = pairingPending
if err := member.save(); err != nil {
t.Fatal(err)
}
syncGrant(t, priv, tMember, []string{"inference"}, []string{"qwen4-embed"})
cfgs, err := loadPairings()
if err != nil {
t.Fatal(err)
}
cfg := pairingFor(cfgs, tMember)
if cfg == nil || !cfg.active() || !cfg.tcpTransport() {
t.Fatalf("the member's tcp pairing must be active after the grant: %+v", cfg)
}
runInBackground(t, cfg)
for model, want := range map[string]int{"llama3:70b": http.StatusForbidden, "qwen4-embed": http.StatusOK} {
ws := h.take(t)
if got := ask(t, ws, "llm"); got != "A" {
t.Fatalf("the member holds inference, so the job is accepted: %q", got)
}
resp, _ := chatRequest(t, laneConn(ws), model)
b, _ := io.ReadAll(resp.Body)
if resp.StatusCode != want {
t.Fatalf("%s over the scoped lane: %d %s", model, resp.StatusCode, b)
}
}
chats := 0
for _, s := range up.requests() {
if s == "POST /v1/chat/completions" {
chats++
}
}
if chats != 1 {
t.Fatalf("only the scoped model's request may reach the server: %q", up.requests())
}
})
}
// ── 4. the bounds ───────────────────────────────────────────────────────────
// SECURITY: one pairing holds at most 16 streams open here. Sixteen run at
// once; the 17th is answered 'E' busy with the reason and dials nothing; once
// one ends, the next is accepted.
func TestTheServeLaneHoldsSixteenStreamsAndRefusesTheSeventeenth(t *testing.T) {
var held, opened atomic.Int32
release := make(chan struct{})
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/v1/chat/completions" {
_, _ = w.Write([]byte(`{}`)) // the beat's probe
return
}
opened.Add(1)
held.Add(1)
defer held.Add(-1)
w.Header().Set("Content-Type", "text/event-stream")
_, _ = io.WriteString(w, "data: {\"chunk\":1}\n\n")
w.(http.Flusher).Flush()
select {
case <-release:
case <-r.Context().Done():
}
}))
t.Cleanup(func() { close(release); srv.Close() })
t.Setenv("PRIVATAE_LLM_ADDR", srv.Listener.Addr().String())
h, cfg := serveLaneBox(t, "sixteen.example.test")
runInBackground(t, cfg)
var streams []*websocket.Conn
for i := 0; i < serveMaxInFlight; i++ {
ws := h.take(t)
if got := ask(t, ws, "llm"); got != "A" {
t.Fatalf("stream %d of %d must be accepted: %q", i+1, serveMaxInFlight, got)
}
resp, _ := chatRequest(t, laneConn(ws), "test-model")
if resp.StatusCode != http.StatusOK {
t.Fatalf("stream %d: %d", i+1, resp.StatusCode)
}
streams = append(streams, ws)
}
if n := held.Load(); n != serveMaxInFlight {
t.Fatalf("the server holds %d streams, want %d at once", n, serveMaxInFlight)
}
got := ask(t, h.take(t), "llm")
if !strings.HasPrefix(got, "Ebusy: ") || !strings.Contains(got, "16 streams") {
t.Fatalf("the 17th stream must be answered busy with the reason: %q", got)
}
if n := opened.Load(); n != serveMaxInFlight {
t.Fatalf("the refused 17th reached the server (%d requests)", n)
}
// One ends; the next is accepted.
streams[0].Close()
deadline := time.Now().Add(10 * time.Second)
for {
ws := h.take(t)
got := ask(t, ws, "llm")
if got == "A" {
break
}
if !strings.HasPrefix(got, "Ebusy: ") || time.Now().After(deadline) {
t.Fatalf("after a stream ended the next must be accepted: %q", got)
}
time.Sleep(50 * time.Millisecond)
}
}
// SECURITY: a stream with no byte either way for the idle bound is closed,
// both ends, on a pass-through and through the filter: a server that answers
// headers and then nothing does not hold the stream (or this machine's slot)
// for ever. A stream that keeps sending, for several times the bound, is not
// cut: the bound is on silence, never on a response's length.
func TestTheServeLaneClosesAStreamIdleForTheBound(t *testing.T) {
old := serveIdleTimeout
serveIdleTimeout = 400 * time.Millisecond
t.Cleanup(func() { serveIdleTimeout = old })
// silent answers headers and then holds the connection, recording when
// this machine closes it.
silentClosed := make(chan time.Time, 4)
silent, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { silent.Close() })
go func() {
for {
c, err := silent.Accept()
if err != nil {
return
}
go func(c net.Conn) {
defer c.Close()
br := bufio.NewReader(c)
req, err := http.ReadRequest(br)
if err != nil {
return
}
if req.URL.Path != "/v1/chat/completions" {
// The beat's probes get an ordinary answer.
_, _ = io.WriteString(c, "HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\n{}")
return
}
_, _ = io.WriteString(c, "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nTransfer-Encoding: chunked\r\n\r\n")
_, _ = io.Copy(io.Discard, br)
silentClosed <- time.Now()
}(c)
}
}()
// steady sends a chunk every 100 ms for 1.6 s: four times the bound.
steady := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/v1/chat/completions" {
_, _ = w.Write([]byte(`{}`))
return
}
w.Header().Set("Content-Type", "text/event-stream")
for i := 1; i <= 16; i++ {
fmt.Fprintf(w, "data: {\"chunk\":%d}\n\n", i)
w.(http.Flusher).Flush()
time.Sleep(100 * time.Millisecond)
}
_, _ = io.WriteString(w, "data: [DONE]\n\n")
}))
t.Cleanup(steady.Close)
for _, c := range []struct {
name, service, env string
modelsOff bool
}{
{"a pass-through", "llm", "PRIVATAE_LLM_ADDR", false},
{"through the filter", "ollama", "OLLAMA_ADDR", true},
} {
t.Run(c.name, func(t *testing.T) {
t.Setenv(c.env, silent.Addr().String())
h, cfg := serveLaneBox(t, "idle-"+c.service+".example.test")
if c.modelsOff {
if err := localDenySet(cfg.HostName, "models", true); err != nil {
t.Fatal(err)
}
}
runInBackground(t, cfg)
ws := h.take(t)
if got := ask(t, ws, c.service); got != "A" {
t.Fatalf("accept: %q", got)
}
conn := laneConn(ws)
resp, _ := chatRequest(t, conn, "test-model")
start := time.Now()
_ = conn.SetReadDeadline(time.Now().Add(10 * time.Second))
_, err := io.Copy(io.Discard, resp.Body)
took := time.Since(start)
if err == nil {
t.Fatal("a silent stream must end in a close, not a complete answer")
}
if took < 300*time.Millisecond || took > 3*time.Second {
t.Fatalf("the silent stream was closed after %v; the bound is %v", took, serveIdleTimeout)
}
select {
case <-silentClosed:
case <-time.After(3 * time.Second):
t.Fatal("the server's end of the silent stream was left open")
}
t.Setenv(c.env, steady.Listener.Addr().String())
ws = h.take(t)
if got := ask(t, ws, c.service); got != "A" {
t.Fatalf("accept: %q", got)
}
resp, _ = chatRequest(t, laneConn(ws), "test-model")
b, err := io.ReadAll(resp.Body)
if err != nil || !strings.Contains(string(b), `"chunk":16`) || !strings.Contains(string(b), "[DONE]") {
t.Fatalf("a stream that keeps sending was cut: %v %q", err, b)
}
})
}
}
// ── 5. the beat says what is served ─────────────────────────────────────────
func capsOf(m map[string]any) []string {
var out []string
if l, ok := m["capabilities"].([]any); ok {
for _, c := range l {
out = append(out, fmt.Sprint(c))
}
}
return out
}
func strsOf(v any) []string {
out := []string{}
if l, ok := v.([]any); ok {
for _, c := range l {
out = append(out, fmt.Sprint(c))
}
}
return out
}
// SECURITY (an honest beat): a tcp pairing's beat lists in lane_serves what
// its serve lane carries, and advertises a model lane only when its service
// is there. With ollama answering here, llm unset and pii set to another
// computer, the beat carries lane_serves ["ollama"] and `embed` with its
// models, and no `llm-openai`, `pii`, pii_port, llm_port, `services` or
// `coders`, none of which runs on a tcp pairing. With all three served, all
// three are listed and advertised, pii_ok with them and pii_port still not. A
// unix pairing's beat lists its lane too and keeps `coders` (its runs have a
// socket); a tunnel pairing's beat has no lane_serves.
//
// MUTATION: have heartbeatPayload call buildHeartbeatBody (the tunnel's beat).
// RESULT: red — no lane_serves, and pii and services advertised with nothing
// serving them.
func TestATCPPairingsBeatAdvertisesOnlyWhatItServes(t *testing.T) {
up := newFakeOllama(t)
t.Setenv("OLLAMA_ADDR", up.srv.Listener.Addr().String())
t.Setenv("PRIVATAE_LLM_ADDR", "")
t.Setenv("PRIVATAE_PII_ADDR", "192.0.2.10:9")
stubServiceRuntime(t, "docker")
stubCoders(t)
h, cfg := serveLaneBox(t, "beat.example.test")
coderOn(t, 2)
if !servicesCapable(cfg.HostName) || coderHeartbeat(cfg.HostName) == nil {
t.Fatal("the slots and coder runs must be available on this machine, or their absence proves nothing")
}
runInBackground(t, cfg)
beat := h.beat(t)
caps := capsOf(beat)
if got := strsOf(beat["lane_serves"]); !slices.Equal(got, []string{"ollama"}) {
t.Fatalf("lane_serves: %q (the beat: %v)", got, beat)
}
if !slices.Contains(caps, "embed") || !slices.Contains(strsOf(beat["models"]), "qwen4-embed:latest") {
t.Fatalf("ollama is served, so embed and its models are advertised: %q %v", caps, beat["models"])
}
for _, c := range []string{"llm-openai", "pii", serviceCapability, coderCapability} {
if slices.Contains(caps, c) {
t.Errorf("a tcp pairing advertises %q, which nothing serves on it: %q", c, caps)
}
}
for _, k := range []string{"pii_port", "pii_ok", "services", "coders"} {
if _, ok := beat[k]; ok {
t.Errorf("a tcp pairing's beat carries %q: %v", k, beat[k])
}
}
if beat["llm_port"] != float64(0) {
t.Errorf("no port listens for the llm server on a tcp pairing: llm_port %v", beat["llm_port"])
}
// All three served.
health := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte(`{"status":"ok"}`))
}))
t.Cleanup(health.Close)
t.Setenv("PRIVATAE_LLM_ADDR", up.srv.Listener.Addr().String())
t.Setenv("PRIVATAE_PII_ADDR", health.Listener.Addr().String())
var all map[string]any
if err := json.Unmarshal([]byte(heartbeatPayload(cfg, nil, "box", "pub")), &all); err != nil {
t.Fatal(err)
}
if got := strsOf(all["lane_serves"]); !slices.Equal(got, []string{"ollama", "llm", "pii"}) {
t.Fatalf("lane_serves with all three set: %q", got)
}
caps = capsOf(all)
for _, c := range []string{"embed", "llm-openai", "pii"} {
if !slices.Contains(caps, c) {
t.Errorf("%q is served, so it is advertised: %q", c, caps)
}
}
if all["pii_ok"] != true || all["pii_port"] != nil || !slices.Contains(strsOf(all["llm_models"]), "qwen4-embed") {
t.Errorf("pii_ok %v, pii_port %v, llm_models %v", all["pii_ok"], all["pii_port"], all["llm_models"])
}
// A unix pairing: its lane, and its coder runs (a socket each).
unix := *cfg
unix.Transport, unix.LaneAddr, unix.LaneSocket, unix.RunsDir = transportUnix, "", "/run/privatae-host/lane.sock", "/run/privatae-host/runs"
var u map[string]any
_ = json.Unmarshal([]byte(heartbeatPayload(&unix, nil, "box", "pub")), &u)
if _, ok := u["lane_serves"]; !ok || slices.Contains(capsOf(u), serviceCapability) || !slices.Contains(capsOf(u), coderCapability) {
t.Errorf("a unix pairing: lane_serves %v, caps %q", u["lane_serves"], capsOf(u))
}
// A tunnel pairing: the forwards, no lane.
tunnel := *cfg
tunnel.Transport, tunnel.LaneAddr = "", ""
var tb map[string]any
_ = json.Unmarshal([]byte(heartbeatPayload(&tunnel, nil, "box", "pub")), &tb)
if _, ok := tb["lane_serves"]; ok || tb["pii_port"] != float64(piiForwardPort) {
t.Errorf("a tunnel pairing: lane_serves %v, pii_port %v", tb["lane_serves"], tb["pii_port"])
}
}
// The lane writes one audit line per job: the service, the address dialled
// and the bytes each way for a served one; the reason for a refused one.
func TestTheServeLaneAuditsEachJob(t *testing.T) {
up := newFakeOllama(t)
t.Setenv("OLLAMA_ADDR", up.srv.Listener.Addr().String())
h, cfg := serveLaneBox(t, "audit.example.test")
runInBackground(t, cfg)
ws := h.take(t)
if got := ask(t, ws, "ollama"); got != "A" {
t.Fatalf("accept: %q", got)
}
conn := laneConn(ws)
req, _ := http.NewRequest(http.MethodGet, "http://serve/api/tags", nil)
req.Close = true
_ = req.Write(conn)
resp, err := http.ReadResponse(bufio.NewReader(conn), req)
if err != nil {
t.Fatal(err)
}
_, _ = io.ReadAll(resp.Body)
ws.Close()
if got := ask(t, h.take(t), "10.0.0.2:11434"); !strings.HasPrefix(got, "E") {
t.Fatalf("refuse: %q", got)
}
path := filepath.Join(cfg.dcnDir(), "audit.jsonl")
deadline := time.Now().Add(5 * time.Second)
for {
raw, _ := os.ReadFile(path)
var served, refused map[string]any
for _, line := range strings.Split(strings.TrimSpace(string(raw)), "\n") {
var m map[string]any
if json.Unmarshal([]byte(line), &m) != nil || m["op"] != "serve" {
continue
}
switch m["decision"] {
case "served":
served = m
case "refused":
refused = m
}
}
if served != nil && refused != nil {
if served["service"] != "ollama" || served["target"] != up.srv.Listener.Addr().String() ||
served["bytes_from_host"].(float64) <= 0 || served["bytes_to_host"].(float64) <= 0 {
t.Fatalf("the served line: %v", served)
}
if !strings.Contains(fmt.Sprint(refused["reason"]), "never takes an address") {
t.Fatalf("the refused line: %v", refused)
}
return
}
if time.Now().After(deadline) {
t.Fatalf("no served and refused lines in %s: %s", path, raw)
}
time.Sleep(50 * time.Millisecond)
}
}
+240
View File
@@ -0,0 +1,240 @@
//go:build linux
// Model management for a Host with no platform (modelswap.go through
// modelDaemonTarget): the model-swap probe, model:list, model:pull and
// model:drop reach only a model server on this machine, as the serve lane
// does, and the machine's console says why one is not used. Uses the
// fixtures of servelane_linux_test.go and modelswap_test.go.
package main
import (
"bytes"
"os"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/gorilla/websocket"
)
// captureStdout collects what this process prints until the test ends, and
// returns a func that reads it so far.
func captureStdout(t *testing.T) func() string {
t.Helper()
r, w, err := os.Pipe()
if err != nil {
t.Fatal(err)
}
old := os.Stdout
os.Stdout = w
var mu sync.Mutex
var buf bytes.Buffer
done := make(chan struct{})
go func() {
defer close(done)
b := make([]byte, 4096)
for {
n, err := r.Read(b)
if n > 0 {
mu.Lock()
buf.Write(b[:n])
mu.Unlock()
_, _ = old.Write(b[:n])
}
if err != nil {
return
}
}
}()
t.Cleanup(func() {
os.Stdout = old
_ = w.Close()
<-done
_ = r.Close()
})
return func() string {
mu.Lock()
defer mu.Unlock()
return buf.String()
}
}
// SECURITY: a tcp pairing (a Host with no platform) is not given management of
// a model server on another computer. With OLLAMA_ADDR on TEST-NET, model:list,
// an approved model:pull and model:drop of a model this Host placed are each
// refused with the reason, before anything is dialled or downloaded, and the
// machine's console says in plain words that OLLAMA_ADDR is not used and what
// to set. With OLLAMA_ADDR on this machine the same pairing lists its models.
//
// MUTATION: modelDaemonTarget returns localOllamaAddr() for every pairing.
// RESULT: red — model:list fails dialling 192.0.2.10, the approved pull
// starts a download job (exit 0), and no line is printed at the machine.
func TestModelManagementOnALocalPairingReachesOnlyThisMachine(t *testing.T) {
console := captureStdout(t)
t.Setenv("OLLAMA_ADDR", "192.0.2.10:11434") // TEST-NET-1: never this machine
t.Setenv("PRIVATAE_LLM_ADDR", "")
t.Setenv("PRIVATAE_PII_ADDR", "")
resetModelDaemonCache()
h, cfg := serveLaneBox(t, "models.example.test")
runInBackground(t, cfg)
beat := h.beat(t)
for _, c := range capsOf(beat) {
if c == modelSwapCapability {
t.Fatalf("a Host with no platform is offered management of a server on another computer: %q", capsOf(beat))
}
}
root := t.TempDir()
start := time.Now()
_, errOut, code, _ := driveModelOp(t, cfg.HostName, "model:list", nil, root)
if code == 0 || !strings.Contains(errOut, "OLLAMA_ADDR") || !strings.Contains(errOut, "192.0.2.10 is not this machine") {
t.Fatalf("model:list: exit %d, %q; want a refusal naming OLLAMA_ADDR and why", code, errOut)
}
if d := time.Since(start); d > 2*time.Second {
t.Errorf("model:list took %v: it waited on a dial instead of refusing first", d)
}
enrollFactor(t, root)
approval := proposeAndApprove(t, root, "nomic-embed-text", "embed", 1<<30, "")
_, errOut, code, _ = driveModelOp(t, cfg.HostName, "model:pull",
[]string{pullArg("nomic-embed-text", "embed", 1<<30, "", approval)}, root)
if code != 126 || !strings.Contains(errOut, "was not downloaded") || !strings.Contains(errOut, "192.0.2.10 is not this machine") {
t.Fatalf("model:pull: exit %d, %q; want 126 and a refusal saying why", code, errOut)
}
modelPullJobsMu.Lock()
for id, j := range modelPullJobs {
if j.Model == "nomic-embed-text" {
t.Errorf("a pull job %s was started for a server on another computer", id)
}
}
modelPullJobsMu.Unlock()
if err := recordPlaced(root, "placed-here", placedModel{PlacedAt: "now", Role: "embed"}); err != nil {
t.Fatal(err)
}
_, errOut, code, _ = driveModelOp(t, cfg.HostName, "model:drop", []string{`{"model":"placed-here"}`}, root)
if code == 0 || !strings.Contains(errOut, "192.0.2.10 is not this machine") {
t.Fatalf("model:drop: exit %d, %q; want a refusal saying why", code, errOut)
}
if !placedByThisHost(root, "placed-here") {
t.Error("a refused drop changed the ledger")
}
said := console()
if !strings.Contains(said, "["+cfg.HostName+"] OLLAMA_ADDR=192.0.2.10:11434 is not used for the Host on this computer") ||
!strings.Contains(said, "Set OLLAMA_ADDR to a server on this computer, or unset it.") {
t.Errorf("the machine's console does not say why OLLAMA_ADDR is not used:\n%s", said)
}
// Counted for this pairing only: a pairing from an earlier test whose beat
// is still ending can say it for itself.
if n := strings.Count(said, "["+cfg.HostName+"] OLLAMA_ADDR=192.0.2.10:11434 is not used"); n != 1 {
t.Errorf("the console said it %d times; once, until it changes", n)
}
// The control: a model server on this machine is listed for the same pairing.
newFakeDaemon(t, &fakeDaemon{tags: []string{"here-model"}})
out, errOut, code, _ := driveModelOp(t, cfg.HostName, "model:list", nil, root)
if code != 0 || !strings.Contains(out, "here-model") {
t.Fatalf("model:list of a server on this machine: exit %d, %q %q", code, out, errOut)
}
}
// The beat and a model op can refuse the same target at the same moment. The
// machine's console still says it once: the reason is recorded and said by one
// caller, not checked by each and then said by all.
//
// MUTATION: in serveTargetNotice, Load the last reason, then Store and print
// (as it did).
// RESULT: red — more than one line for 32 callers released together (the
// count varies from run to run).
func TestTheConsoleSaysARefusedTargetOnceUnderConcurrentCallers(t *testing.T) {
console := captureStdout(t)
s, _ := serveServiceNamed("ollama")
self := "notice-once.example.test"
t.Cleanup(func() { serveTargetNoticeClear(self, s) })
start := make(chan struct{})
var wg sync.WaitGroup
for i := 0; i < 32; i++ {
wg.Add(1)
go func() {
defer wg.Done()
<-start
serveTargetNotice(self, s, "192.0.2.10:11434", "192.0.2.10 is not this machine")
}()
}
close(start)
wg.Wait()
serveTargetNotice(self, s, "192.0.2.10:11434", "another reason")
deadline := time.Now().Add(2 * time.Second)
for !strings.Contains(console(), "another reason") && time.Now().Before(deadline) {
time.Sleep(10 * time.Millisecond)
}
said := console()
if n := strings.Count(said, "["+self+"] OLLAMA_ADDR=192.0.2.10:11434 is not used"); n != 2 {
t.Fatalf("the console said it %d times; once for the first reason and once when it changed:\n%s", n, said)
}
}
// A tunnel pairing keeps OLLAMA_ADDR wherever it points, as its forwards do:
// the rule is for a Host with no platform.
func TestModelDaemonTargetKeepsATunnelPairingsAddress(t *testing.T) {
t.Setenv("OLLAMA_ADDR", "192.0.2.10:11434")
if addr, why := modelDaemonTarget("tunnel.example.test"); addr != "192.0.2.10:11434" || why != "" {
t.Fatalf("a tunnel pairing: %q %q", addr, why)
}
unmark := markLocalPairing("tunnel.example.test")
if addr, why := modelDaemonTarget("tunnel.example.test"); addr != "" || !strings.Contains(why, "192.0.2.10 is not this machine") {
t.Fatalf("the same name run as a local pairing: %q %q", addr, why)
}
unmark()
unmark() // once only
if isLocalPairing("tunnel.example.test") {
t.Fatal("still marked local after its run ended")
}
}
// SECURITY (this machine's resources): a Host that answers every parked serve
// worker with a job it knows is refused (a malformed frame, a name the lane
// does not carry) cannot make this machine dial, handshake and audit in a
// loop. Each refused worker waits before it parks again, so two workers cost
// at most a few dials a second.
//
// MUTATION: spawn the replacement when the first frame arrives (taken() before
// the hello is read), as the lane first did.
// RESULT: red — hundreds of workers park in the 2 s window.
func TestARefusingHostCannotSpinTheServeWorkers(t *testing.T) {
t.Setenv("OLLAMA_ADDR", "")
h, cfg := serveLaneBox(t, "spin.example.test")
runInBackground(t, cfg)
var answered atomic.Int32
deadline := time.Now().Add(2 * time.Second)
for time.Now().Before(deadline) {
var ws *websocket.Conn
select {
case ws = <-h.workers:
case <-time.After(time.Until(deadline)):
}
if ws == nil {
break
}
frame := sFrame("nas")
if answered.Load()%2 == 1 {
frame = []byte("Dnot a hello")
}
_ = ws.WriteMessage(websocket.BinaryMessage, frame)
_ = ws.SetReadDeadline(time.Now().Add(2 * time.Second))
_, _, _ = ws.ReadMessage()
ws.Close()
answered.Add(1)
}
// 2 parked at the start, then each worker once a second: about 6 in 2 s.
if n := answered.Load(); n > 10 {
t.Fatalf("%d serve workers parked in 2 s on a Host that refuses every job: the pool re-dials in a loop", n)
}
if answered.Load() < 2 {
t.Fatalf("only %d workers parked: the pool did not run, so the count proves nothing", answered.Load())
}
}
+274
View File
@@ -0,0 +1,274 @@
//go:build linux
package main
import (
"fmt"
"net"
"os"
"os/exec"
"runtime"
"strings"
"sync/atomic"
"syscall"
"testing"
"time"
"golang.org/x/sys/unix"
)
const serveNetnsEnv = "PRIVATAE_TEST_SERVE_NETNS"
// otherComputer is a computer on this machine's LAN, at 10.0.0.2 in a network
// namespace of its own, joined to this one by a veth that carries this
// machine's default route (so 10.0.0.2 is the LAN, not a container here). It
// listens on 80 and 11434 and counts what reaches it off the wire: IPv4
// packets addressed to it, TCP SYNs to any address (IPv4 and IPv6), and
// accepted connections.
type otherComputer struct {
toIt atomic.Int64
syns atomic.Int64
accepts atomic.Int64
stop atomic.Bool
}
// startOtherComputer makes the namespace on one OS thread locked for good
// (the thread ends with its goroutine), so every socket it opens and every
// ip(8) it runs are the other computer's, and this test's are not.
func startOtherComputer(t *testing.T, ipBin string) *otherComputer {
t.Helper()
oc := &otherComputer{}
t.Cleanup(func() { oc.stop.Store(true) })
errc := make(chan error, 1)
go func() {
runtime.LockOSThread()
// This test's namespace, held open before the thread leaves it: the
// veth's other end goes there. Not /proc/<pid>/ns/net, which is the
// main thread's, and the main thread may be this one.
here, err := os.Open("/proc/thread-self/ns/net")
if err != nil {
errc <- err
return
}
defer here.Close()
if err := unix.Unshare(unix.CLONE_NEWNET); err != nil {
errc <- fmt.Errorf("unshare: %w", err)
return
}
for _, a := range [][]string{
{"link", "add", "vB", "type", "veth", "peer", "name", "vA", "netns", "/proc/self/fd/3"},
{"link", "set", "lo", "up"},
{"addr", "add", "10.0.0.2/24", "dev", "vB"},
{"link", "set", "vB", "up"},
} {
cmd := exec.Command(ipBin, a...)
cmd.ExtraFiles = []*os.File{here}
if out, err := cmd.CombinedOutput(); err != nil {
errc <- fmt.Errorf("ip %s: %v %s", strings.Join(a, " "), err, out)
return
}
}
for _, port := range []string{"80", "11434"} {
ln, err := net.Listen("tcp", "10.0.0.2:"+port)
if err != nil {
errc <- err
return
}
go func() {
for !oc.stop.Load() {
c, err := ln.Accept()
if err != nil {
return
}
oc.accepts.Add(1)
c.Close()
}
}()
t.Cleanup(func() { ln.Close() })
}
ifi, err := net.InterfaceByName("vB")
if err != nil {
errc <- err
return
}
proto := int(htons16(unix.ETH_P_ALL))
fd, err := unix.Socket(unix.AF_PACKET, unix.SOCK_RAW, proto)
if err != nil {
errc <- fmt.Errorf("packet socket: %w", err)
return
}
if err := unix.Bind(fd, &unix.SockaddrLinklayer{Protocol: htons16(unix.ETH_P_ALL), Ifindex: ifi.Index}); err != nil {
unix.Close(fd)
errc <- fmt.Errorf("bind the packet socket: %w", err)
return
}
tv := unix.NsecToTimeval((200 * time.Millisecond).Nanoseconds())
_ = unix.SetsockoptTimeval(fd, unix.SOL_SOCKET, unix.SO_RCVTIMEO, &tv)
go oc.capture(fd)
errc <- nil
}()
if err := <-errc; err != nil {
t.Skipf("the other computer's namespace could not be made: %v", err)
}
return oc
}
func htons16(v uint16) uint16 { return v<<8 | v>>8 }
// capture counts the frames arriving on vB (its own replies are left out).
func (oc *otherComputer) capture(fd int) {
defer unix.Close(fd)
buf := make([]byte, 65536)
for !oc.stop.Load() {
n, from, err := unix.Recvfrom(fd, buf, 0)
if err != nil {
continue
}
if ll, ok := from.(*unix.SockaddrLinklayer); ok && ll.Pkttype == unix.PACKET_OUTGOING {
continue
}
f := buf[:n]
if len(f) < 14 {
continue
}
switch uint16(f[12])<<8 | uint16(f[13]) {
case 0x0800:
if len(f) < 34 {
continue
}
ihl := int(f[14]&0x0f) * 4
if net.IP(f[30:34]).Equal(net.IPv4(10, 0, 0, 2)) {
oc.toIt.Add(1)
}
if f[23] == 6 && len(f) >= 14+ihl+14 && isSYN(f[14+ihl+13]) {
oc.syns.Add(1)
}
case 0x86dd:
if len(f) >= 14+40+14 && f[20] == 6 && isSYN(f[14+40+13]) {
oc.syns.Add(1)
}
}
}
}
func isSYN(flags byte) bool { return flags&0x02 != 0 && flags&0x10 == 0 }
// SECURITY, IN A REAL NETWORK NAMESPACE: a job that names an address instead
// of a service, a name the lane does not carry, and each service set on this
// machine to a computer on its LAN (OLLAMA_ADDR, PRIVATAE_LLM_ADDR and
// PRIVATAE_PII_ADDR all at 10.0.0.2) each answer 'E', and the other computer
// sees no packet addressed to it and no SYN at all, from the lane or from the
// beat that probes what the lane serves, the beat's model-management probe, or
// model:list. A direct dial from this machine
// afterwards is seen, so the count is real. Runs the test binary again in a
// private user and network namespace; skipped where one cannot be made.
//
// MUTATION: in serveVetTarget, skip the ownDestination check.
// RESULT: red — the other computer sees the lane's SYN on 11434 and 80.
// MUTATION: modelDaemonTarget returns localOllamaAddr() for every pairing.
// RESULT: red — the beat's model-swap probe and model:list SYN 10.0.0.2:11434.
func TestTheServeLaneSendsNoPacketForAnAddressInARealNamespace(t *testing.T) {
if os.Getenv(serveNetnsEnv) != "1" {
if _, err := exec.LookPath("ip"); err != nil {
if _, err := os.Stat("/usr/sbin/ip"); err != nil {
t.Skip("needs ip(8)")
}
}
cmd := exec.Command(os.Args[0], "-test.run=^TestTheServeLaneSendsNoPacketForAnAddressInARealNamespace$",
"-test.v", "-test.count=1")
cmd.Env = append(os.Environ(), serveNetnsEnv+"=1")
cmd.SysProcAttr = &syscall.SysProcAttr{
Cloneflags: syscall.CLONE_NEWUSER | syscall.CLONE_NEWNET,
UidMappings: []syscall.SysProcIDMap{{ContainerID: 0, HostID: os.Getuid(), Size: 1}},
GidMappings: []syscall.SysProcIDMap{{ContainerID: 0, HostID: os.Getgid(), Size: 1}},
}
out, err := cmd.CombinedOutput()
if cmd.ProcessState == nil {
t.Skipf("no private user and network namespace here: %v", err)
}
t.Logf("in a private network namespace:\n%s", out)
if strings.Contains(string(out), "--- SKIP") {
t.Skip("the namespaced run could not set up its network")
}
if err != nil {
t.Fatalf("the namespaced run failed: %v", err)
}
if !strings.Contains(string(out), "--- PASS: TestTheServeLaneSendsNoPacketForAnAddressInARealNamespace") {
t.Fatal("the namespaced run did not run")
}
return
}
ipBin, err := exec.LookPath("ip")
if err != nil {
ipBin = "/usr/sbin/ip"
}
if out, err := exec.Command(ipBin, "link", "set", "lo", "up").CombinedOutput(); err != nil {
t.Skipf("ip link set lo up: %v %s", err, out)
}
oc := startOtherComputer(t, ipBin)
for _, a := range [][]string{
{"addr", "add", "10.0.0.1/24", "dev", "vA"},
{"link", "set", "vA", "up"},
// This machine's way out: 10.0.0.2 is its LAN (and its router), not a
// container on a veth of its own (ownaddr.go).
{"route", "add", "default", "via", "10.0.0.2", "dev", "vA"},
} {
if out, err := exec.Command(ipBin, a...).CombinedOutput(); err != nil {
t.Skipf("ip %s: %v %s", strings.Join(a, " "), err, out)
}
}
t.Setenv("OLLAMA_ADDR", "10.0.0.2:11434")
t.Setenv("PRIVATAE_LLM_ADDR", "10.0.0.2:80")
t.Setenv("PRIVATAE_PII_ADDR", "10.0.0.2:80")
h, cfg := serveLaneBox(t, "netns.example.test")
// `models` stays on: the model-management probe in the beat and model:list
// (modelswap.go, modelDaemonTarget) are held to this machine for a Host
// with no platform too, so they are in the count.
resetModelDaemonCache()
runInBackground(t, cfg)
beat := h.beat(t)
if got := strsOf(beat["lane_serves"]); len(got) != 0 {
t.Fatalf("nothing set here is this machine's, so nothing is served: %q", got)
}
for _, c := range capsOf(beat) {
if c == modelSwapCapability {
t.Fatalf("model management of the other computer is offered: %q", capsOf(beat))
}
}
if _, errOut, code, _ := driveModelOp(t, cfg.HostName, "model:list", nil, ""); code == 0 ||
!strings.Contains(errOut, "10.0.0.2 is not this machine") {
t.Errorf("model:list of the other computer: exit %d, %q", code, errOut)
}
for service, want := range map[string]string{
"10.0.0.2:80": "never takes an address",
"10.0.0.2:11434": "never takes an address",
"nas": "is not a service",
"ollama": "10.0.0.2 is not this machine",
"llm": "10.0.0.2 is not this machine",
"pii": "10.0.0.2 is not this machine",
} {
got := ask(t, h.take(t), service)
if !strings.HasPrefix(got, "Erefused: ") || !strings.Contains(got, want) {
t.Errorf("%q: answered %q, want E carrying %q", service, got, want)
}
}
time.Sleep(500 * time.Millisecond)
if a, s, c := oc.toIt.Load(), oc.syns.Load(), oc.accepts.Load(); a != 0 || s != 0 || c != 0 {
t.Fatalf("the other computer saw %d packets addressed to it, %d SYNs and %d connections", a, s, c)
}
// The count is real: a dial from this machine is seen.
c, err := net.DialTimeout("tcp", "10.0.0.2:80", 3*time.Second)
if err != nil {
t.Fatalf("the control dial to the other computer failed, so the count proves nothing: %v", err)
}
c.Close()
deadline := time.Now().Add(3 * time.Second)
for oc.toIt.Load() == 0 || oc.syns.Load() == 0 || oc.accepts.Load() == 0 {
if time.Now().After(deadline) {
t.Fatalf("the control dial was not counted: %d packets, %d SYNs, %d connections",
oc.toIt.Load(), oc.syns.Load(), oc.accepts.Load())
}
time.Sleep(20 * time.Millisecond)
}
}
+31 -12
View File
@@ -113,13 +113,26 @@ func runSessionWorkers(pctx context.Context, self string, wsURL string, tlsCfg *
// session; an H frame is refused with a plain line (harness jobs belong on the
// harness workers) so a host that routes wrongly learns it at once.
func handleSessionWorkerConn(ws *websocket.Conn, self string, senv sessionEnv) {
mt, data, err := ws.ReadMessage()
if err != nil || mt != websocket.BinaryMessage || len(data) == 0 {
// The spec is read up to sessionSpecMaxBytes (lane_bounds.go) and refused
// past it; a frame of any other tag is not read at all.
tag, body, tooBig, err := readLaneJob(ws, func(t byte) int64 {
if t == 'S' {
return sessionSpecMaxBytes
}
return 0
})
if err != nil {
return
}
switch data[0] {
switch tag {
case 'S':
handleSessionConn(ws, data, self, senv)
if tooBig {
sendSession(ws, 'E', []byte(fmt.Sprintf("the session spec is more than the %d KiB a session may ask with; nothing of it was read\n",
sessionSpecMaxBytes>>10)))
sendSessionExit(ws, 1)
return
}
handleSessionConn(ws, append([]byte{'S'}, body...), self, senv)
case 'H':
sendSession(ws, 'E', []byte("this is a session worker (lane=session); harness jobs go to a harness worker\n"))
sendSessionExit(ws, 1)
@@ -573,23 +586,29 @@ func handleSessionConn(ws *websocket.Conn, first []byte, self string, env sessio
closed := make(chan struct{})
go func() {
defer close(closed)
// Each frame is streamed (lane_bounds.go): input goes into the pty as it
// arrives, never held whole, and a frame is at most sessionInputMaxBytes.
ws.SetReadLimit(sessionInputMaxBytes)
for {
mt, data, err := ws.ReadMessage()
mt, r, err := ws.NextReader()
if err != nil {
return
}
if mt != websocket.BinaryMessage || len(data) == 0 {
continue
if mt != websocket.BinaryMessage {
continue // NextReader drops what is left of it
}
switch data[0] {
var tag [1]byte
if _, err := io.ReadFull(r, tag[:]); err != nil {
continue // an empty frame
}
switch tag[0] {
case 'I':
lastInput.Store(time.Now().UnixNano())
if len(data) > 1 {
_, _ = proc.Write(data[1:])
}
_, _ = io.Copy(proc, r)
case 'R':
var rz sessionResize
if json.Unmarshal(data[1:], &rz) == nil && rz.Cols > 0 && rz.Rows > 0 {
b, _ := io.ReadAll(io.LimitReader(r, sessionResizeMaxBytes))
if json.Unmarshal(b, &rz) == nil && rz.Cols > 0 && rz.Rows > 0 {
_ = proc.Resize(rz.Cols, rz.Rows)
}
}